bingo: strip photo metadata, identify to Wikimedia
Photos are served byte-for-byte, and the Commons shot of Medvedkovo carries GPS coordinates in its EXIF, while the card is meant to stay city-level. Drop Exif/XMP/IPTC/comment segments from JPEG, text/eXIf chunks from PNG and EXIF/XMP chunks from WebP on store; pixel data is left alone and malformed files are rejected. upload.wikimedia.org answered 429 to the generic agent; its policy asks for a tool name and contact, so send one. Add a "re-import all" admin action so photos stored before this change get re-fetched and stripped. Assisted-by: Claude:opus-5
This commit is contained in:
parent
7820f138f6
commit
903dc17a75
GPG key ID:
4F8851660FA4121B
4 changed files with 105 additions and 4 deletions
66
lib/bingo.rb
66
lib/bingo.rb
|
|
@ -135,18 +135,82 @@ module Bingo
|
|||
bytes = bytes.b
|
||||
raise Error, 'image too large (max 8 MB)' if bytes.bytesize > MAX_BYTES
|
||||
ext = sniff(bytes) or raise Error, 'not a JPEG, PNG or WebP image'
|
||||
bytes = strip_metadata(bytes, ext)
|
||||
key = "#{PREFIX}#{slug}-#{SecureRandom.hex(4)}.#{ext}"
|
||||
S3.put(key, bytes, content_type: TYPES[ext])
|
||||
key
|
||||
end
|
||||
|
||||
# Photos are served byte-for-byte, so drop EXIF/XMP/IPTC/text metadata:
|
||||
# phone shots carry GPS coordinates, and the card stays city-level.
|
||||
# Pixel data is untouched; malformed input is rejected rather than
|
||||
# passed through.
|
||||
def strip_metadata(bytes, ext)
|
||||
case ext
|
||||
when 'jpg' then strip_jpeg(bytes)
|
||||
when 'png' then strip_png(bytes)
|
||||
when 'webp' then strip_webp(bytes)
|
||||
end
|
||||
end
|
||||
|
||||
JPEG_DROP = [0xE1, *0xE3..0xEF, 0xFE].freeze # APP1 (Exif/XMP), APP3-15 (IPTC...), COM
|
||||
|
||||
def strip_jpeg(b)
|
||||
out = +"\xFF\xD8".b
|
||||
i = 2
|
||||
while i < b.bytesize
|
||||
raise Error, 'corrupt JPEG' unless b.getbyte(i) == 0xFF
|
||||
marker = b.getbyte(i + 1)
|
||||
if marker == 0xFF then i += 1; next end # fill byte
|
||||
if marker == 0xDA || marker == 0xD9 # start of scan: rest is image data
|
||||
return out << b.byteslice(i..)
|
||||
end
|
||||
len = b.byteslice(i + 2, 2)&.unpack1('n') or raise Error, 'corrupt JPEG'
|
||||
out << b.byteslice(i, len + 2) unless JPEG_DROP.include?(marker)
|
||||
i += len + 2
|
||||
end
|
||||
raise Error, 'corrupt JPEG'
|
||||
end
|
||||
|
||||
PNG_DROP = %w[eXIf tEXt iTXt zTXt tIME].freeze
|
||||
|
||||
def strip_png(b)
|
||||
out = b.byteslice(0, 8)
|
||||
i = 8
|
||||
while i + 8 <= b.bytesize
|
||||
len, type = b.byteslice(i, 8).unpack('Na4')
|
||||
out << b.byteslice(i, len + 12) unless PNG_DROP.include?(type)
|
||||
i += len + 12
|
||||
return out if type == 'IEND'
|
||||
end
|
||||
raise Error, 'corrupt PNG'
|
||||
end
|
||||
|
||||
def strip_webp(b)
|
||||
body = +''.b
|
||||
i = 12
|
||||
while i + 8 <= b.bytesize
|
||||
type, len = b.byteslice(i, 8).unpack('a4V')
|
||||
chunk = b.byteslice(i, 8 + len + (len & 1))
|
||||
if type == 'VP8X'
|
||||
chunk = chunk.dup
|
||||
chunk.setbyte(8, chunk.getbyte(8) & ~0x0C) # clear EXIF + XMP flags
|
||||
end
|
||||
body << chunk unless %w[EXIF XMP\ ].include?(type)
|
||||
i += 8 + len + (len & 1)
|
||||
end
|
||||
raise Error, 'corrupt WebP' if body.empty?
|
||||
'RIFF'.b + [body.bytesize + 4].pack('V') + 'WEBP'.b + body
|
||||
end
|
||||
|
||||
# Admin-only "import from URL". https only, a few redirects, size-capped.
|
||||
def fetch(url, redirects: 3)
|
||||
uri = URI(url)
|
||||
raise Error, 'https URL required' unless uri.is_a?(URI::HTTPS)
|
||||
Net::HTTP.start(uri.host, uri.port, use_ssl: true, open_timeout: 5, read_timeout: 15) do |http|
|
||||
req = Net::HTTP::Get.new(uri)
|
||||
req['User-Agent'] = 'Mozilla/5.0 (bingo photo import)'
|
||||
# Wikimedia rate-limits generic agents; its policy wants name + contact.
|
||||
req['User-Agent'] = 'asxpio-bingo/1.0 (https://asxp.io; ie@asxp.io)'
|
||||
http.request(req) do |res|
|
||||
case res
|
||||
when Net::HTTPRedirection
|
||||
|
|
|
|||
|
|
@ -87,10 +87,13 @@ module BingoAdmin
|
|||
redirect "/admin/bingo/dc/#{dc.id}"
|
||||
end
|
||||
|
||||
# Bulk: import every missing photo from its photo_source_url.
|
||||
# Bulk: import every missing photo from its photo_source_url, or with
|
||||
# all=1 re-import every photo that has a source URL.
|
||||
app.post '/admin/bingo/photos/import' do
|
||||
ok, failed = 0, []
|
||||
Bingo::Datacenter.where(photo_key: nil).exclude(photo_source_url: nil).each do |dc|
|
||||
scope = Bingo::Datacenter.exclude(photo_source_url: nil)
|
||||
scope = scope.where(photo_key: nil) unless params[:all] == '1'
|
||||
scope.each do |dc|
|
||||
dc.update(photo_key: Bingo::Photo.store(dc.slug, Bingo::Photo.fetch(dc.photo_source_url)))
|
||||
ok += 1
|
||||
rescue Bingo::Photo::Error => e
|
||||
|
|
|
|||
|
|
@ -3,7 +3,8 @@ require_relative 'test_helper'
|
|||
class BingoTest < Minitest::Test
|
||||
include Rack::Test::Methods
|
||||
|
||||
JPEG = "\xFF\xD8\xFF\xE0fakejpeg".b
|
||||
# Minimal well-formed JPEG: SOI, start-of-scan, data, EOI.
|
||||
JPEG = ("\xFF\xD8\xFF\xDA".b + [8].pack('n') + "\1\2\3\4\5\6data\xFF\xD9".b).freeze
|
||||
|
||||
def app
|
||||
@app || BingoWeb
|
||||
|
|
@ -84,6 +85,34 @@ class BingoTest < Minitest::Test
|
|||
S3.instance_variable_set(:@client, nil)
|
||||
end
|
||||
|
||||
def test_strips_jpeg_exif
|
||||
exif = "\xFF\xE1".b + [17].pack('n') + "Exif\0\0GPSDATA!!".b
|
||||
jfif = "\xFF\xE0".b + [16].pack('n') + "JFIF\0".b + ("\0" * 9).b
|
||||
scan = "\xFF\xDA".b + [8].pack('n') + "\1\2\3\4\5\6pixels\xFF\xD9".b
|
||||
out = Bingo::Photo.strip_metadata("\xFF\xD8".b + jfif + exif + scan, 'jpg')
|
||||
refute_includes out, 'GPSDATA'
|
||||
assert_equal "\xFF\xD8".b + jfif + scan, out
|
||||
end
|
||||
|
||||
def test_strips_png_text_chunks
|
||||
chunk = ->(type, data) { [data.bytesize].pack('N') + type + data + "\0\0\0\0".b }
|
||||
sig = "\x89PNG\r\n\x1A\n".b
|
||||
ihdr = chunk.('IHDR', "\0" * 13)
|
||||
out = Bingo::Photo.strip_metadata(sig + ihdr + chunk.('eXIf', 'GPS') + chunk.('tEXt', 'k\0v') + chunk.('IEND', ''), 'png')
|
||||
assert_equal sig + ihdr + chunk.('IEND', ''), out
|
||||
end
|
||||
|
||||
def test_strips_webp_exif
|
||||
vp8x = 'VP8X'.b + [10].pack('V') + [0x08 | 0x10].pack('C') + ("\0" * 9).b # EXIF + alpha flags
|
||||
img = 'VP8L'.b + [5].pack('V') + "abcde\0".b
|
||||
exif = 'EXIF'.b + [3].pack('V') + "GPS\0".b
|
||||
body = vp8x + img + exif
|
||||
out = Bingo::Photo.strip_metadata('RIFF'.b + [body.bytesize + 4].pack('V') + 'WEBPh'.b[0, 4] + body, 'webp')
|
||||
refute_includes out, 'GPS'
|
||||
assert_equal 0x10, out.getbyte(20) # alpha kept, EXIF flag cleared
|
||||
assert_equal out.bytesize - 8, out.byteslice(4, 4).unpack1('V')
|
||||
end
|
||||
|
||||
def test_photo_rejects_non_images
|
||||
assert_raises(Bingo::Photo::Error) { Bingo::Photo.store('dc-1', '<html>') }
|
||||
end
|
||||
|
|
|
|||
|
|
@ -71,6 +71,11 @@
|
|||
<input type="hidden" name="authenticity_token" value="<%= csrf_token %>" />
|
||||
<button type="submit" class="link-button">↻ Import all missing photos from their source URLs</button>
|
||||
</form>
|
||||
<form method="post" action="/admin/bingo/photos/import" onsubmit="return confirm('Re-download every photo that has a source URL? Manually uploaded photos with a source URL get replaced too.')">
|
||||
<input type="hidden" name="authenticity_token" value="<%= csrf_token %>" />
|
||||
<input type="hidden" name="all" value="1" />
|
||||
<button type="submit" class="link-button">↻ Re-import all photos from their source URLs</button>
|
||||
</form>
|
||||
<table class="invoice-list">
|
||||
<thead><tr><th>#</th><th>Photo</th><th>Name</th><th>Operator</th><th class="num">MW</th><th>Status</th><th class="num">Incidents</th><th></th></tr></thead>
|
||||
<tbody>
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue