Disaster Recovery Planning

What Is Disaster Recovery?

Recovery Point Objective (RPO)

-- Check the last backup time and calculate data loss window
SELECT
  backup_id,
  backup_type,
  started_at,
  finished_at,
  EXTRACT(EPOCH FROM (NOW() - finished_at)) / 3600 AS hours_since_backup
FROM backup_log
WHERE status = 'SUCCESS'
ORDER BY finished_at DESC
LIMIT 5;

Recovery Time Objective (RTO)

-- Track restore durations to validate RTO compliance
SELECT
  restore_id,
  triggered_at,
  completed_at,
  EXTRACT(EPOCH FROM (completed_at - triggered_at)) / 60 AS restore_minutes,
  CASE
    WHEN EXTRACT(EPOCH FROM (completed_at - triggered_at)) / 3600 <= 4
    THEN 'WITHIN RTO'
    ELSE 'RTO BREACHED'
  END AS rto_status
FROM restore_log
ORDER BY triggered_at DESC;

RPO vs RTO — The Key Difference

-- Store RPO and RTO targets per database in a DR configuration table
CREATE TABLE dr_config (
  db_name       VARCHAR(100) PRIMARY KEY,
  rpo_minutes   INT NOT NULL,
  rto_minutes   INT NOT NULL,
  tier          VARCHAR(20) CHECK (tier IN ('CRITICAL', 'HIGH', 'MEDIUM', 'LOW')),
  updated_at    TIMESTAMP DEFAULT NOW()
);

INSERT INTO dr_config (db_name, rpo_minutes, rto_minutes, tier) VALUES
  ('orders_db',    15,   60, 'CRITICAL'),
  ('analytics_db', 120, 240, 'MEDIUM'),
  ('archive_db',   480, 480, 'LOW');

Types of Backups

-- Log each backup with its type for audit and recovery planning
CREATE TABLE backup_log (
  backup_id   SERIAL PRIMARY KEY,
  db_name     VARCHAR(100) NOT NULL,
  backup_type VARCHAR(20) CHECK (backup_type IN ('FULL', 'DIFFERENTIAL', 'INCREMENTAL')),
  started_at  TIMESTAMP NOT NULL,
  finished_at TIMESTAMP,
  size_mb     NUMERIC(12, 2),
  status      VARCHAR(20) DEFAULT 'IN_PROGRESS'
);

INSERT INTO backup_log (db_name, backup_type, started_at, finished_at, size_mb, status) VALUES
  ('orders_db', 'FULL',        '2024-06-01 01:00:00', '2024-06-01 02:15:00', 45200, 'SUCCESS'),
  ('orders_db', 'INCREMENTAL', '2024-06-02 01:00:00', '2024-06-02 01:08:00',   320, 'SUCCESS'),
  ('orders_db', 'INCREMENTAL', '2024-06-03 01:00:00', '2024-06-03 01:07:00',   290, 'SUCCESS');

Point-in-Time Recovery (PITR)

-- Record WAL archive events for PITR tracking
CREATE TABLE wal_archive_log (
  segment_name  VARCHAR(200) PRIMARY KEY,
  archived_at   TIMESTAMP DEFAULT NOW(),
  size_bytes    BIGINT,
  storage_path  TEXT
);

-- Find all WAL segments archived within a recovery window
SELECT
  segment_name,
  archived_at,
  ROUND(size_bytes / 1024.0 / 1024.0, 2) AS size_mb
FROM wal_archive_log
WHERE archived_at BETWEEN '2024-06-03 09:00:00' AND '2024-06-03 11:00:00'
ORDER BY archived_at;

Standby Databases and Replication

-- Monitor replication lag on a PostgreSQL primary
SELECT
  client_addr,
  application_name,
  state,
  sent_lsn,
  replay_lsn,
  (sent_lsn - replay_lsn) AS lag_bytes,
  EXTRACT(EPOCH FROM (NOW() - reply_time)) AS seconds_since_reply
FROM pg_stat_replication
ORDER BY lag_bytes DESC;

Runbooks: Documenting Recovery Procedures

-- Store runbook metadata in the database for audit tracking
CREATE TABLE runbook (
  runbook_id   SERIAL PRIMARY KEY,
  title        VARCHAR(200) NOT NULL,
  scenario     VARCHAR(100),
  version      VARCHAR(20) DEFAULT '1.0',
  last_tested  DATE,
  owner        VARCHAR(100),
  doc_url      TEXT
);

INSERT INTO runbook (title, scenario, version, last_tested, owner, doc_url) VALUES
  ('Full Database Restore from S3', 'total_loss',     '2.1', '2024-05-15', 'dba_team', 'https://wiki.internal/dr/full-restore'),
  ('Failover to Hot Standby',       'primary_down',   '1.4', '2024-04-20', 'dba_team', 'https://wiki.internal/dr/failover'),
  ('PITR to Specific Timestamp',    'data_corruption','1.2', '2024-03-10', 'dba_team', 'https://wiki.internal/dr/pitr');

Runbook Execution Logging

-- Log each runbook execution for RTO validation and audit
CREATE TABLE runbook_execution (
  execution_id  SERIAL PRIMARY KEY,
  runbook_id    INT REFERENCES runbook(runbook_id),
  triggered_by  VARCHAR(100),
  is_drill      BOOLEAN DEFAULT FALSE,
  started_at    TIMESTAMP NOT NULL,
  completed_at  TIMESTAMP,
  outcome       VARCHAR(20) CHECK (outcome IN ('SUCCESS', 'PARTIAL', 'FAILED'))
);

-- Report average restore time per runbook
SELECT
  r.title,
  COUNT(*) AS executions,
  ROUND(AVG(EXTRACT(EPOCH FROM (e.completed_at - e.started_at)) / 60), 1) AS avg_minutes,
  MAX(EXTRACT(EPOCH FROM (e.completed_at - e.started_at)) / 60) AS max_minutes
FROM runbook_execution e
JOIN runbook r ON r.runbook_id = e.runbook_id
WHERE e.outcome = 'SUCCESS'
GROUP BY r.title;

DR Testing: Regular Drills

-- Find runbooks that have not been drilled in over 90 days
SELECT
  r.runbook_id,
  r.title,
  r.scenario,
  MAX(e.completed_at) AS last_drill,
  CURRENT_DATE - MAX(e.completed_at::DATE) AS days_since_drill
FROM runbook r
LEFT JOIN runbook_execution e
  ON e.runbook_id = r.runbook_id
  AND e.is_drill = TRUE
  AND e.outcome = 'SUCCESS'
GROUP BY r.runbook_id, r.title, r.scenario
HAVING MAX(e.completed_at) IS NULL
    OR CURRENT_DATE - MAX(e.completed_at::DATE) > 90
ORDER BY days_since_drill DESC NULLS FIRST;

Backup Retention Policies

-- Identify backups that are outside their retention window and ready to purge
SELECT
  backup_id,
  db_name,
  backup_type,
  finished_at,
  CURRENT_DATE - finished_at::DATE AS age_days,
  CASE backup_type
    WHEN 'INCREMENTAL' THEN 7
    WHEN 'DIFFERENTIAL' THEN 28
    WHEN 'FULL'         THEN 365
  END AS retention_days,
  CASE
    WHEN (CURRENT_DATE - finished_at::DATE) >
         CASE backup_type
           WHEN 'INCREMENTAL' THEN 7
           WHEN 'DIFFERENTIAL' THEN 28
           WHEN 'FULL'         THEN 365
         END
    THEN 'PURGE'
    ELSE 'KEEP'
  END AS action
FROM backup_log
WHERE status = 'SUCCESS'
ORDER BY finished_at;

Quick Check

Recap: Disaster Recovery Planning