Add SELF_DESTRUCT env variable to process self-destructions in the background (#26439)

app/controllers/application_controller.rb

@@ -12,6 +12,7 @@ class ApplicationController < ActionController::Base
   include DomainControlHelper
   include DatabaseHelper
   include AuthorizedFetchHelper
+  include SelfDestructHelper
 
   helper_method :current_account
   helper_method :current_session
@@ -39,6 +40,8 @@ class ApplicationController < ActionController::Base
     service_unavailable
   end
 
+  before_action :check_self_destruct!
+
   before_action :store_referrer, except: :raise_not_found, if: :devise_controller?
   before_action :require_functional!, if: :user_signed_in?
 
@@ -170,6 +173,15 @@ class ApplicationController < ActionController::Base
     end
   end
 
+  def check_self_destruct!
+    return unless self_destruct?
+
+    respond_to do |format|
+      format.any  { render 'errors/self_destruct', layout: 'auth', status: 410, formats: [:html] }
+      format.json { render json: { error: Rack::Utils::HTTP_STATUS_CODES[410] }, status: code }
+    end
+  end
+
   def set_cache_control_defaults
     response.cache_control.replace(private: true, no_store: true)
   end

app/controllers/auth/challenges_controller.rb

@@ -7,6 +7,7 @@ class Auth::ChallengesController < ApplicationController
 
   before_action :authenticate_user!
 
+  skip_before_action :check_self_destruct!
   skip_before_action :require_functional!
 
   def create

app/controllers/auth/confirmations_controller.rb

@@ -12,6 +12,7 @@ class Auth::ConfirmationsController < Devise::ConfirmationsController
   before_action :extend_csp_for_captcha!, only: [:show, :confirm_captcha]
   before_action :require_captcha_if_needed!, only: [:show]
 
+  skip_before_action :check_self_destruct!
   skip_before_action :require_functional!
 
   def show

app/controllers/auth/omniauth_callbacks_controller.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 class Auth::OmniauthCallbacksController < Devise::OmniauthCallbacksController
+  skip_before_action :check_self_destruct!
   skip_before_action :verify_authenticity_token
 
   def self.provides_callback_for(provider)

app/controllers/auth/passwords_controller.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 class Auth::PasswordsController < Devise::PasswordsController
+  skip_before_action :check_self_destruct!
   before_action :check_validity_of_reset_password_token, only: :edit
   before_action :set_body_classes
 

app/controllers/auth/registrations_controller.rb

@@ -17,6 +17,7 @@ class Auth::RegistrationsController < Devise::RegistrationsController
   before_action :require_rules_acceptance!, only: :new
   before_action :set_registration_form_time, only: :new
 
+  skip_before_action :check_self_destruct!, only: [:edit, :update]
   skip_before_action :require_functional!, only: [:edit, :update]
 
   def new

app/controllers/auth/sessions_controller.rb

@@ -3,6 +3,7 @@
 class Auth::SessionsController < Devise::SessionsController
   layout 'auth'
 
+  skip_before_action :check_self_destruct!
   skip_before_action :require_no_authentication, only: [:create]
   skip_before_action :require_functional!
   skip_before_action :update_user_sign_in

app/controllers/backups_controller.rb

@@ -3,6 +3,7 @@
 class BackupsController < ApplicationController
   include RoutingHelper
 
+  skip_before_action :check_self_destruct!
   skip_before_action :require_functional!
 
   before_action :authenticate_user!

app/controllers/concerns/export_controller_concern.rb

@@ -7,6 +7,7 @@ module ExportControllerConcern
     before_action :authenticate_user!
     before_action :load_export
 
+    skip_before_action :check_self_destruct!
     skip_before_action :require_functional!
   end
 

app/controllers/settings/exports_controller.rb

@@ -5,6 +5,7 @@ class Settings::ExportsController < Settings::BaseController
   include Redisable
   include Lockable
 
+  skip_before_action :check_self_destruct!
   skip_before_action :require_functional!
 
   def show

app/controllers/settings/login_activities_controller.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
 class Settings::LoginActivitiesController < Settings::BaseController
+  skip_before_action :check_self_destruct!
+  skip_before_action :require_functional!
+
   def index
     @login_activities = LoginActivity.where(user: current_user).order(id: :desc).page(params[:page])
   end

app/controllers/settings/two_factor_authentication/webauthn_credentials_controller.rb

@@ -3,6 +3,7 @@
 module Settings
   module TwoFactorAuthentication
     class WebauthnCredentialsController < BaseController
+      skip_before_action :check_self_destruct!
       skip_before_action :require_functional!
 
       before_action :require_otp_enabled

app/controllers/settings/two_factor_authentication_methods_controller.rb

@@ -4,6 +4,7 @@ module Settings
   class TwoFactorAuthenticationMethodsController < BaseController
     include ChallengableConcern
 
+    skip_before_action :check_self_destruct!
     skip_before_action :require_functional!
 
     before_action :require_challenge!, only: :disable

app/helpers/self_destruct_helper.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module SelfDestructHelper
+  def self.self_destruct?
+    value = ENV.fetch('SELF_DESTRUCT', nil)
+    value.present? && Rails.application.message_verifier('self-destruct').verify(value) == ENV['LOCAL_DOMAIN']
+  rescue ActiveSupport::MessageVerifier::InvalidSignature
+    false
+  end
+
+  def self_destruct?
+    SelfDestructHelper.self_destruct?
+  end
+end

app/views/auth/registrations/edit.html.haml

@@ -1,7 +1,11 @@
 - content_for :page_title do
   = t('settings.account_settings')
 
-= render partial: 'status', locals: { user: @user, strikes: @strikes }
+- if self_destruct?
+  .flash-message.warning
+    = t('auth.status.self_destruct', domain: ENV['LOCAL_DOMAIN'])
+- else
+  = render partial: 'status', locals: { user: @user, strikes: @strikes }
 
 %h3= t('auth.security')
 
@@ -32,7 +36,7 @@
 
 = render partial: 'sessions', object: @sessions
 
-- unless current_account.suspended?
+- unless current_account.suspended? || self_destruct?
   %hr.spacer/
 
   %h3= t('auth.migrate_account')

app/views/errors/self_destruct.html.haml

@@ -0,0 +1,20 @@
+- content_for :page_title do
+  = t('self_destruct.title')
+
+.simple_form
+  %h1.title= t('self_destruct.title')
+  %p.lead= t('self_destruct.lead_html', domain: ENV['LOCAL_DOMAIN'])
+
+.form-footer
+  %ul.no-list
+    - if user_signed_in?
+      %li= link_to t('settings.account_settings'), edit_user_registration_path
+    - else
+      - if controller_name != 'sessions'
+        %li= link_to_login t('auth.login')
+
+      - if controller_name != 'passwords' && controller_name != 'registrations'
+        %li= link_to t('auth.forgot_password'), new_user_password_path
+
+    - if user_signed_in?
+      %li= link_to t('auth.logout'), destroy_user_session_path, data: { method: :delete }

app/workers/scheduler/self_destruct_scheduler.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+class Scheduler::SelfDestructScheduler
+  include Sidekiq::Worker
+  include SelfDestructHelper
+
+  MAX_ENQUEUED = 10_000
+  MAX_REDIS_MEM_USAGE = 0.5
+  MAX_ACCOUNT_DELETIONS_PER_JOB = 50
+
+  sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i
+
+  def perform
+    return unless self_destruct?
+    return if sidekiq_overwhelmed?
+
+    delete_accounts!
+  end
+
+  private
+
+  def sidekiq_overwhelmed?
+    redis_mem_info = Sidekiq.redis_info
+
+    Sidekiq::Stats.new.enqueued > MAX_ENQUEUED || redis_mem_info['used_memory'].to_f > redis_mem_info['total_system_memory'].to_f * MAX_REDIS_MEM_USAGE
+  end
+
+  def delete_accounts!
+    # We currently do not distinguish between deleted accounts and suspended
+    # accounts, and we do not want to remove the records in this scheduler, as
+    # we still rely on it for account delivery and don't want to perform
+    # needless work when the database can be outright dropped after the
+    # self-destruct.
+    # Deleted accounts are suspended accounts that do not have a pending
+    # deletion request.
+
+    # This targets accounts that have not been deleted nor marked for deletion yet
+    Account.local.without_suspended.reorder(id: :asc).take(MAX_ACCOUNT_DELETIONS_PER_JOB).each do |account|
+      delete_account!(account)
+    end
+
+    return if sidekiq_overwhelmed?
+
+    # This targets accounts that have been marked for deletion but have not been
+    # deleted yet
+    Account.local.suspended.joins(:deletion_request).take(MAX_ACCOUNT_DELETIONS_PER_JOB).each do |account|
+      delete_account!(account)
+      account.deletion_request&.destroy
+    end
+  end
+
+  def inboxes
+    @inboxes ||= Account.inboxes
+  end
+
+  def delete_account!(account)
+    payload = ActiveModelSerializers::SerializableResource.new(
+      account,
+      serializer: ActivityPub::DeleteActorSerializer,
+      adapter: ActivityPub::Adapter
+    ).as_json
+
+    json = Oj.dump(ActivityPub::LinkedDataSignature.new(payload).sign!(account))
+
+    ActivityPub::DeliveryWorker.push_bulk(inboxes, limit: 1_000) do |inbox_url|
+      [json, account.id, inbox_url]
+    end
+
+    # Do not call `Account#suspend!` because we don't want to issue a deletion request
+    account.update!(suspended_at: Time.now.utc, suspension_origin: :local)
+  end
+end

config/initializers/sidekiq.rb

@@ -17,6 +17,18 @@ Sidekiq.configure_server do |config|
     chain.add SidekiqUniqueJobs::Middleware::Client
   end
 
+  config.on(:startup) do
+    if SelfDestructHelper.self_destruct?
+      Sidekiq.schedule = {
+        'self_destruct_scheduler' => {
+          'interval' => ['1m'],
+          'class' => 'Scheduler::SelfDestructScheduler',
+          'queue' => 'scheduler',
+        },
+      }
+    end
+  end
+
   SidekiqUniqueJobs::Server.configure(config)
 end
 

config/locales/en.yml

@@ -1102,6 +1102,7 @@ en:
       functional: Your account is fully operational.
       pending: Your application is pending review by our staff. This may take some time. You will receive an e-mail if your application is approved.
       redirecting_to: Your account is inactive because it is currently redirecting to %{acct}.
+      self_destruct: As %{domain} is closing down, you will only get limited access to your account.
       view_strikes: View past strikes against your account
     too_fast: Form submitted too fast, try again.
     use_security_key: Use security key
@@ -1572,6 +1573,9 @@ en:
     over_daily_limit: You have exceeded the limit of %{limit} scheduled posts for today
     over_total_limit: You have exceeded the limit of %{limit} scheduled posts
     too_soon: The scheduled date must be in the future
+  self_destruct:
+    lead_html: Unfortunately, <strong>%{domain}</strong> is permanently closing down. If you had an account there, you will not be able to continue using it, but you can still request a backup of your data.
+    title: This server is closing down
   sessions:
     activity: Last activity
     browser: Browser

config/navigation.rb

@@ -1,44 +1,46 @@
 # frozen_string_literal: true
 
 SimpleNavigation::Configuration.run do |navigation|
+  self_destruct = SelfDestructHelper.self_destruct?
+
   navigation.items do |n|
     n.item :web, safe_join([fa_icon('chevron-left fw'), t('settings.back')]), root_path
 
     n.item :software_updates, safe_join([fa_icon('exclamation-circle fw'), t('admin.critical_update_pending')]), admin_software_updates_path, if: -> { ENV['UPDATE_CHECK_URL'] != '' && current_user.can?(:view_devops) && SoftwareUpdate.urgent_pending? }, html: { class: 'warning' }
 
-    n.item :profile, safe_join([fa_icon('user fw'), t('settings.profile')]), settings_profile_path, if: -> { current_user.functional? }, highlights_on: %r{/settings/profile|/settings/featured_tags|/settings/verification|/settings/privacy}
+    n.item :profile, safe_join([fa_icon('user fw'), t('settings.profile')]), settings_profile_path, if: -> { current_user.functional? && !self_destruct }, highlights_on: %r{/settings/profile|/settings/featured_tags|/settings/verification|/settings/privacy}
 
-    n.item :preferences, safe_join([fa_icon('cog fw'), t('settings.preferences')]), settings_preferences_path, if: -> { current_user.functional? } do |s|
+    n.item :preferences, safe_join([fa_icon('cog fw'), t('settings.preferences')]), settings_preferences_path, if: -> { current_user.functional? && !self_destruct } do |s|
       s.item :appearance, safe_join([fa_icon('desktop fw'), t('settings.appearance')]), settings_preferences_appearance_path
       s.item :notifications, safe_join([fa_icon('bell fw'), t('settings.notifications')]), settings_preferences_notifications_path
       s.item :other, safe_join([fa_icon('cog fw'), t('preferences.other')]), settings_preferences_other_path
     end
 
-    n.item :relationships, safe_join([fa_icon('users fw'), t('settings.relationships')]), relationships_path, if: -> { current_user.functional? }
-    n.item :filters, safe_join([fa_icon('filter fw'), t('filters.index.title')]), filters_path, highlights_on: %r{/filters}, if: -> { current_user.functional? }
-    n.item :statuses_cleanup, safe_join([fa_icon('history fw'), t('settings.statuses_cleanup')]), statuses_cleanup_path, if: -> { current_user.functional_or_moved? }
+    n.item :relationships, safe_join([fa_icon('users fw'), t('settings.relationships')]), relationships_path, if: -> { current_user.functional? && !self_destruct }
+    n.item :filters, safe_join([fa_icon('filter fw'), t('filters.index.title')]), filters_path, highlights_on: %r{/filters}, if: -> { current_user.functional? && !self_destruct }
+    n.item :statuses_cleanup, safe_join([fa_icon('history fw'), t('settings.statuses_cleanup')]), statuses_cleanup_path, if: -> { current_user.functional_or_moved? && !self_destruct }
 
     n.item :security, safe_join([fa_icon('lock fw'), t('settings.account')]), edit_user_registration_path do |s|
       s.item :password, safe_join([fa_icon('lock fw'), t('settings.account_settings')]), edit_user_registration_path, highlights_on: %r{/auth/edit|/settings/delete|/settings/migration|/settings/aliases|/settings/login_activities|^/disputes}
       s.item :two_factor_authentication, safe_join([fa_icon('mobile fw'), t('settings.two_factor_authentication')]), settings_two_factor_authentication_methods_path, highlights_on: %r{/settings/two_factor_authentication|/settings/otp_authentication|/settings/security_keys}
-      s.item :authorized_apps, safe_join([fa_icon('list fw'), t('settings.authorized_apps')]), oauth_authorized_applications_path
+      s.item :authorized_apps, safe_join([fa_icon('list fw'), t('settings.authorized_apps')]), oauth_authorized_applications_path, if: -> { !self_destruct }
     end
 
     n.item :data, safe_join([fa_icon('cloud-download fw'), t('settings.import_and_export')]), settings_export_path do |s|
-      s.item :import, safe_join([fa_icon('cloud-upload fw'), t('settings.import')]), settings_imports_path, if: -> { current_user.functional? }
+      s.item :import, safe_join([fa_icon('cloud-upload fw'), t('settings.import')]), settings_imports_path, if: -> { current_user.functional? && !self_destruct }
       s.item :export, safe_join([fa_icon('cloud-download fw'), t('settings.export')]), settings_export_path
     end
 
-    n.item :invites, safe_join([fa_icon('user-plus fw'), t('invites.title')]), invites_path, if: -> { current_user.can?(:invite_users) && current_user.functional? }
-    n.item :development, safe_join([fa_icon('code fw'), t('settings.development')]), settings_applications_path, if: -> { current_user.functional? }
+    n.item :invites, safe_join([fa_icon('user-plus fw'), t('invites.title')]), invites_path, if: -> { current_user.can?(:invite_users) && current_user.functional? && !self_destruct }
+    n.item :development, safe_join([fa_icon('code fw'), t('settings.development')]), settings_applications_path, if: -> { current_user.functional? && !self_destruct }
 
-    n.item :trends, safe_join([fa_icon('fire fw'), t('admin.trends.title')]), admin_trends_statuses_path, if: -> { current_user.can?(:manage_taxonomies) } do |s|
+    n.item :trends, safe_join([fa_icon('fire fw'), t('admin.trends.title')]), admin_trends_statuses_path, if: -> { current_user.can?(:manage_taxonomies) && !self_destruct } do |s|
       s.item :statuses, safe_join([fa_icon('comments-o fw'), t('admin.trends.statuses.title')]), admin_trends_statuses_path, highlights_on: %r{/admin/trends/statuses}
       s.item :tags, safe_join([fa_icon('hashtag fw'), t('admin.trends.tags.title')]), admin_trends_tags_path, highlights_on: %r{/admin/tags|/admin/trends/tags}
       s.item :links, safe_join([fa_icon('newspaper-o fw'), t('admin.trends.links.title')]), admin_trends_links_path, highlights_on: %r{/admin/trends/links}
     end
 
-    n.item :moderation, safe_join([fa_icon('gavel fw'), t('moderation.title')]), nil, if: -> { current_user.can?(:manage_reports, :view_audit_log, :manage_users, :manage_invites, :manage_taxonomies, :manage_federation, :manage_blocks) } do |s|
+    n.item :moderation, safe_join([fa_icon('gavel fw'), t('moderation.title')]), nil, if: -> { current_user.can?(:manage_reports, :view_audit_log, :manage_users, :manage_invites, :manage_taxonomies, :manage_federation, :manage_blocks) && !self_destruct } do |s|
       s.item :reports, safe_join([fa_icon('flag fw'), t('admin.reports.title')]), admin_reports_path, highlights_on: %r{/admin/reports}, if: -> { current_user.can?(:manage_reports) }
       s.item :accounts, safe_join([fa_icon('users fw'), t('admin.accounts.title')]), admin_accounts_path(origin: 'local'), highlights_on: %r{/admin/accounts|/admin/pending_accounts|/admin/disputes|/admin/users}, if: -> { current_user.can?(:manage_users) }
       s.item :invites, safe_join([fa_icon('user-plus fw'), t('admin.invites.title')]), admin_invites_path, if: -> { current_user.can?(:manage_invites) }
@@ -49,7 +51,7 @@ SimpleNavigation::Configuration.run do |navigation|
       s.item :action_logs, safe_join([fa_icon('bars fw'), t('admin.action_logs.title')]), admin_action_logs_path, if: -> { current_user.can?(:view_audit_log) }
     end
 
-    n.item :admin, safe_join([fa_icon('cogs fw'), t('admin.title')]), nil, if: -> { current_user.can?(:view_dashboard, :manage_settings, :manage_rules, :manage_announcements, :manage_custom_emojis, :manage_webhooks, :manage_federation) } do |s|
+    n.item :admin, safe_join([fa_icon('cogs fw'), t('admin.title')]), nil, if: -> { current_user.can?(:view_dashboard, :manage_settings, :manage_rules, :manage_announcements, :manage_custom_emojis, :manage_webhooks, :manage_federation) && !self_destruct } do |s|
       s.item :dashboard, safe_join([fa_icon('tachometer fw'), t('admin.dashboard.title')]), admin_dashboard_path, if: -> { current_user.can?(:view_dashboard) }
       s.item :settings, safe_join([fa_icon('cogs fw'), t('admin.settings.title')]), admin_settings_path, if: -> { current_user.can?(:manage_settings) }, highlights_on: %r{/admin/settings}
       s.item :rules, safe_join([fa_icon('gavel fw'), t('admin.rules.title')]), admin_rules_path, highlights_on: %r{/admin/rules}, if: -> { current_user.can?(:manage_rules) }

config/sidekiq.yml

@@ -7,6 +7,7 @@
   - [mailers, 2]
   - [pull]
   - [scheduler]
+
 :scheduler:
   :listened_queues_only: true
   :schedule:

lib/mastodon/cli/main.rb

@@ -65,7 +65,6 @@ module Mastodon::CLI
     desc 'maintenance SUBCOMMAND ...ARGS', 'Various maintenance utilities'
     subcommand 'maintenance', Maintenance
 
-    option :dry_run, type: :boolean
     desc 'self-destruct', 'Erase the server from the federation'
     long_desc <<~LONG_DESC
       Erase the server from the federation by broadcasting account delete
@@ -92,55 +91,37 @@ module Mastodon::CLI
 
       prompt = TTY::Prompt.new
 
-      exit(1) unless prompt.ask('Type in the domain of the server to confirm:', required: true) == Rails.configuration.x.local_domain
+      if SelfDestructHelper.self_destruct?
+        prompt.ok('Self-destruct mode is already enabled for this Mastodon server')
 
-      unless dry_run?
-        prompt.warn('This operation WILL NOT be reversible. It can also take a long time.')
-        prompt.warn('While the data won\'t be erased locally, the server will be in a BROKEN STATE afterwards.')
-        prompt.warn('A running Sidekiq process is required. Do not shut it down until queues clear.')
+        pending_accounts = Account.local.without_suspended.count + Account.local.suspended.joins(:deletion_request).count
+        sidekiq_stats = Sidekiq::Stats.new
 
-        exit(1) if prompt.no?('Are you sure you want to proceed?')
-      end
-
-      inboxes   = Account.inboxes
-      processed = 0
-
-      Setting.registrations_mode = 'none' unless dry_run?
-
-      if inboxes.empty?
-        Account.local.without_suspended.in_batches.update_all(suspended_at: Time.now.utc, suspension_origin: :local) unless dry_run?
-        prompt.ok('It seems like your server has not federated with anything')
-        prompt.ok('You can shut it down and delete it any time')
-        return
-      end
-
-      prompt.warn('Do NOT interrupt this process...')
-
-      delete_account = lambda do |account|
-        payload = ActiveModelSerializers::SerializableResource.new(
-          account,
-          serializer: ActivityPub::DeleteActorSerializer,
-          adapter: ActivityPub::Adapter
-        ).as_json
-
-        json = Oj.dump(ActivityPub::LinkedDataSignature.new(payload).sign!(account))
-
-        unless dry_run?
-          ActivityPub::DeliveryWorker.push_bulk(inboxes, limit: 1_000) do |inbox_url|
-            [json, account.id, inbox_url]
-          end
-
-          account.suspend!(block_email: false)
+        if pending_accounts.positive?
+          prompt.warn("#{pending_accounts} accounts are still pending deletion.")
+        elsif sidekiq_stats.enqueued.positive?
+          prompt.warn('Deletion notices are still being processed')
+        elsif sidekiq_stats.retry_size.positive?
+          prompt.warn('At least one delivery attempt for each deletion notice has been made, but some have failed and are scheduled for retry')
+        else
+          prompt.ok('Every deletion notice has been sent! You can safely delete all data and decomission your servers!')
         end
 
-        processed += 1
+        exit(0)
       end
 
-      Account.local.without_suspended.find_each { |account| delete_account.call(account) }
-      Account.local.suspended.joins(:deletion_request).find_each { |account| delete_account.call(account) }
+      exit(1) unless prompt.ask('Type in the domain of the server to confirm:', required: true) == Rails.configuration.x.local_domain
 
-      prompt.ok("Queued #{inboxes.size * processed} items into Sidekiq for #{processed} accounts#{dry_run_mode_suffix}")
-      prompt.ok('Wait until Sidekiq processes all items, then you can shut everything down and delete the data')
+      prompt.warn('This operation WILL NOT be reversible.')
+      prompt.warn('While the data won\'t be erased locally, the server will be in a BROKEN STATE afterwards.')
+      prompt.warn('The deletion process itself may take a long time, and will be handled by Sidekiq, so do not shut it down until it has finished (you will be able to re-run this command to see the state of the self-destruct process).')
+
+      exit(1) if prompt.no?('Are you sure you want to proceed?')
+
+      self_destruct_value = Rails.application.message_verifier('self-destruct').generate(Rails.configuration.x.local_domain)
+      prompt.ok('To switch Mastodon to self-destruct mode, add the following variable to your evironment (e.g. by adding a line to your `.env.production`) and restart all Mastodon processes:')
+      prompt.ok("  SELF_DESTRUCT=#{self_destruct_value}")
+      prompt.ok("\nYou can re-run this command to see the state of the self-destruct process.")
     rescue TTY::Reader::InputInterrupt
       exit(1)
     end