diff --git a/Gemfile b/Gemfile index 56e293d2a..325deb032 100644 --- a/Gemfile +++ b/Gemfile @@ -141,6 +141,5 @@ gem 'prawn' gem 'prawn-svg', '~> 0.35' gem 'carrierwave-aws', '~> 1.6' -gem 'sitemap_generator', '~> 7.1' gem 'solid_cache', '~> 1.0' diff --git a/Gemfile.lock b/Gemfile.lock index 6037f1260..06d8cb708 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -551,8 +551,6 @@ GEM activemodel (>= 7.0) simplecov (1.2.0) simplecov-lcov (0.9.0) - sitemap_generator (7.1.1) - builder (~> 3.0) slop (3.6.0) snaky_hash (2.0.6) hashie (>= 0.1.0, < 6) @@ -710,7 +708,6 @@ DEPENDENCIES simple_form simplecov simplecov-lcov - sitemap_generator (~> 7.1) solid_cache (~> 1.0) sprockets-rails stimulus-rails @@ -930,7 +927,6 @@ CHECKSUMS simple_form (5.4.1) sha256=58c3d229034c7e5545035c3271b6f030ef730c340b9d7d8eb730e0a385b20808 simplecov (1.2.0) sha256=ea6acd05eece5a41990e2a5171c57d15700d329326c7666c85ee8c6a0dd0977e simplecov-lcov (0.9.0) sha256=7a77a31e200a595ed4b0249493056efd0c920601f53d2ef135ca34ee796346cd - sitemap_generator (7.1.1) sha256=ab2a133d512a7b33ed713a27a9977f61d8379c9943e3b79900243fc1c4f7f81d slop (3.6.0) sha256=76ccab03be66bfcab4838cdc07cab019cd3e192a3538266246749e79e4788803 snaky_hash (2.0.6) sha256=3663cae48cdef582b517025cf8a39d8789996eaf0b4ed89e2f0624836505654a solid_cache (1.0.10) sha256=bc05a2fb3ac78a6f43cbb5946679cf9db67dd30d22939ededc385cb93e120d41 diff --git a/app/controllers/sitemaps_controller.rb b/app/controllers/sitemaps_controller.rb new file mode 100644 index 000000000..6e4a80176 --- /dev/null +++ b/app/controllers/sitemaps_controller.rb @@ -0,0 +1,8 @@ +class SitemapsController < ApplicationController + def show + # Crawler-facing endpoint; let CDNs absorb repeat fetches. The fragment + # caches still bound the DB cost of a cache-miss render. + expires_in 1.hour, public: true + render xml: render_to_string(formats: [:xml]) + end +end diff --git a/app/helpers/sitemaps_helper.rb b/app/helpers/sitemaps_helper.rb new file mode 100644 index 000000000..4fb921f81 --- /dev/null +++ b/app/helpers/sitemaps_helper.rb @@ -0,0 +1,17 @@ +module SitemapsHelper + def sitemap_static_urls + [root_url, code_of_conduct_url, coaches_url, teaching_guide_url, faq_url, + attendance_policy_url, student_guide_url, privacy_policy_url, cookie_policy_url, + breach_code_of_conduct_url, volunteer_url, fundraise_url, donate_url, + codebar_stories_podcast_url] + end + + def sitemap_record_sections + [ + { name: 'chapters', records: Chapter.active, url: ->(chapter) { chapter_url(chapter.slug) } }, + { name: 'workshops', records: Workshop.all, url: ->(workshop) { workshop_url(workshop) } }, + { name: 'events', records: Event.all, url: ->(event) { event_url(event) } }, + { name: 'meetings', records: Meeting.all, url: ->(meeting) { meeting_url(meeting) } } + ] + end +end diff --git a/app/views/sitemaps/show.xml.builder b/app/views/sitemaps/show.xml.builder new file mode 100644 index 000000000..f2e0a060f --- /dev/null +++ b/app/views/sitemaps/show.xml.builder @@ -0,0 +1,21 @@ +xml.instruct! + +# `maximum(:updated_at).to_f` — a raw Time in a cache key is stringified with +# second precision, so changes made within the same second would be missed. +# expires_in makes sections stale after record deletions self-heal. +xml.urlset('xmlns' => 'http://www.sitemaps.org/schemas/sitemap/0.9') do + cache 'sitemap/static', expires_in: 1.week do + sitemap_static_urls.each { |url| xml.url { xml.loc(url) } } + end + + sitemap_record_sections.each do |section| + cache ['sitemap', section[:name], section[:records].maximum(:updated_at).to_f], expires_in: 1.day do + section[:records].find_each do |record| + xml.url do + xml.loc(section[:url].call(record)) + xml.lastmod(record.updated_at.utc.iso8601) + end + end + end + end +end diff --git a/config/nginx.conf.erb b/config/nginx.conf.erb index ab2382539..7877c9b96 100644 --- a/config/nginx.conf.erb +++ b/config/nginx.conf.erb @@ -48,6 +48,15 @@ http { set $plausible_script_url https://plausible.io/js/pa-PFruVsE_br97UUCRXE_6f.js; set $plausible_event_url https://plausible.io/api/event; + # Gzip: the Heroku router does not compress, and Cloudflare only re-compresses + # when the origin has not already. Compress proxied responses here. + gzip on; + gzip_vary on; + gzip_proxied any; + gzip_comp_level 5; + gzip_min_length 1024; + gzip_types application/xml application/json text/plain text/css application/javascript text/javascript; + # Plausible: Proxy script.js (cached) location = /js/script.js { proxy_cache plausible_cache; diff --git a/config/routes.rb b/config/routes.rb index 6c524fa25..8fbc52ea5 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -192,6 +192,12 @@ post "check-in/w/:code" => "check_ins#create" get "check-in/w/:code/confirm" => "check_ins#confirm", as: :check_in_w_confirm + # Consumers pinned to the old static path keep working while crawlers + # migrate via robots.txt (e.g. an old Search Console submission). + get 'sitemap.xml.gz', to: redirect('/sitemap.xml', status: 301) + + get 'sitemap.xml', to: 'sitemaps#show', as: :sitemap + get 'cookie-policy' => 'pages#show', id: 'cookie-policy' get 'privacy-policy' => 'pages#show', id: 'privacy-policy' get 'breach-code-of-conduct' => 'pages#show', id: 'breach-code-of-conduct' diff --git a/config/sitemap.rb b/config/sitemap.rb deleted file mode 100644 index 6b744bd2b..000000000 --- a/config/sitemap.rb +++ /dev/null @@ -1,32 +0,0 @@ -# This file is generated with `rake sitemap:install` -# To update `public/sitemap.xml.gz` run `rake sitemap:refresh`. -# -# See https://github.com/kjvarga/sitemap_generator?tab=readme-ov-file#rake-tasks - -# Set the host name for URL creation -SitemapGenerator::Sitemap.default_host = 'https://codebar.io' - -SitemapGenerator::Sitemap.create do - # Put links creation logic here. - # - # The root path '/' and sitemap index file are added automatically for you. - # Links are added to the Sitemap in the order they are specified. - # - # Usage: add(path, options={}) - # (default options are used if you don't specify) - # - # Defaults: :priority => 0.5, :changefreq => 'weekly', - # :lastmod => Time.now, :host => default_host - # - # Examples: - # - # Add '/articles' - # - # add articles_path, :priority => 0.7, :changefreq => 'daily' - # - # Add all articles: - # - # Article.find_each do |article| - # add article_path(article), :lastmod => article.updated_at - # end -end diff --git a/db/migrate/20260916120000_add_updated_at_indexes_for_sitemap_cache_keys.rb b/db/migrate/20260916120000_add_updated_at_indexes_for_sitemap_cache_keys.rb new file mode 100644 index 000000000..047d3f140 --- /dev/null +++ b/db/migrate/20260916120000_add_updated_at_indexes_for_sitemap_cache_keys.rb @@ -0,0 +1,11 @@ +class AddUpdatedAtIndexesForSitemapCacheKeys < ActiveRecord::Migration[8.1] + disable_ddl_transaction! + + def change + # if_not_exists keeps a re-run after a partial concurrent-index failure + # (an INVALID or half-built index) from failing the next deploy. + add_index :workshops, :updated_at, algorithm: :concurrently, if_not_exists: true + add_index :events, :updated_at, algorithm: :concurrently, if_not_exists: true + add_index :meetings, :updated_at, algorithm: :concurrently, if_not_exists: true + end +end diff --git a/public/robots.txt b/public/robots.txt index 7da454548..fc588ec65 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,2 +1,2 @@ # See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file -Sitemap: https://codebar.io/sitemap.xml.gz +Sitemap: https://codebar.io/sitemap.xml diff --git a/public/sitemap.xml.gz b/public/sitemap.xml.gz deleted file mode 100644 index 0a516f7eb..000000000 Binary files a/public/sitemap.xml.gz and /dev/null differ diff --git a/spec/requests/sitemap_spec.rb b/spec/requests/sitemap_spec.rb new file mode 100644 index 000000000..fff3e0a6c --- /dev/null +++ b/spec/requests/sitemap_spec.rb @@ -0,0 +1,98 @@ +require 'rails_helper' + +RSpec.describe 'Sitemap' do + let!(:chapter) { Fabricate(:chapter) } + let!(:inactive_chapter) { Fabricate(:chapter, active: false) } + let!(:workshop) { Fabricate(:workshop_no_sponsor, chapter:) } + let!(:event) { Fabricate(:event) } + let!(:meeting) { Fabricate(:meeting) } + + it 'serves the sitemap as XML' do + get '/sitemap.xml' + + expect(response).to have_http_status(:ok) + expect(response.media_type).to eq('application/xml') + end + + it 'lists chapters, workshops, events, meetings and static pages' do + get '/sitemap.xml' + + body = response.body + expect(body).to include(root_url) + expect(body).to include(chapter_url(chapter.slug)) + expect(body).to include(workshop_url(workshop)) + expect(body).to include(event_url(event)) + expect(body).to include(meeting_url(meeting)) + expect(body).to include(code_of_conduct_url) + expect(body).to include(faq_url) + expect(body).to include(privacy_policy_url) + end + + it 'excludes inactive chapters' do + get '/sitemap.xml' + + expect(response.body).not_to include(chapter_url(inactive_chapter.slug)) + end + + it 'includes lastmod for records' do + get '/sitemap.xml' + + expect(response.body).to include("#{workshop.reload.updated_at.utc.iso8601}") + expect(response.headers['Cache-Control']).to include('public') + expect(response.headers['Cache-Control']).to include('max-age=3600') + end + + it 'includes new records once their section cache key changes' do + with_fragment_caching do + get '/sitemap.xml' + expect(response.body).to include(workshop_url(workshop)) + + new_workshop = Fabricate(:workshop_no_sponsor, chapter:) + get '/sitemap.xml' + + expect(response.body).to include(workshop_url(new_workshop)) + end + end + + it 'stops listing a deleted record once the section cache expires' do + Fabricate(:workshop_no_sponsor, chapter:) + workshop.update_columns(updated_at: 1.hour.ago) + with_fragment_caching do + get '/sitemap.xml' + expect(response.body).to include(workshop_url(workshop)) + + workshop.destroy + + # The stale fragment is still served before expiry — proves caching is + # actually engaged, so the post-expiry assertion is meaningful. + get '/sitemap.xml' + expect(response.body).to include(workshop_url(workshop)) + + travel 2.days do + get '/sitemap.xml' + end + + expect(response.body).not_to include(workshop_url(workshop)) + end + end + + it 'redirects the old sitemap.xml.gz path to the new endpoint' do + get '/sitemap.xml.gz' + + expect(response).to redirect_to('/sitemap.xml') + expect(response).to have_http_status(:moved_permanently) + end + + def with_fragment_caching + old_cache = Rails.cache + old_perform_caching = ActionController::Base.perform_caching + Rails.cache = ActiveSupport::Cache::MemoryStore.new + ActionController::Base.cache_store = Rails.cache + ActionController::Base.perform_caching = true + yield + ensure + Rails.cache = old_cache + ActionController::Base.cache_store = old_cache + ActionController::Base.perform_caching = old_perform_caching + end +end