Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .env.development
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# Active Record encryption keys for local development.
#
# These are throwaway, non-production keys, committed on purpose so that a fresh clone boots
# without any setup. Production and staging read the same variables from the real environment.
AR_ENCRYPTION_PRIMARY_KEY=WSUDLoAAbKSnldNQe4YXVoC5WTdBiiFt
AR_ENCRYPTION_KEY_DERIVATION_SALT=6dlX6mQqooFztaeIOxOBHJ9SmynoWaXC
7 changes: 7 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,10 @@ RECAPTCHA_PRIVATE_KEY=6LeIxAcTAAAAAGG-vFI1TnRWxMZNFuojJ4WifJWe
# [OPTIONAL] - Used to fetch copies of production DB
AZURE_STORAGE_ACCOUNT_NAME=
AZURE_STORAGE_ACCESS_KEY=

# [PRODUCTION/STAGING ONLY] - Active Record encryption keys, used to encrypt PII at rest.
# Locally these come from the committed .env.development and .env.test, so you can leave them blank.
# Generate a pair with: bin/rails db:encryption:init
# Losing these keys means the encrypted data cannot be recovered.
AR_ENCRYPTION_PRIMARY_KEY=
AR_ENCRYPTION_KEY_DERIVATION_SALT=
6 changes: 6 additions & 0 deletions .env.test
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# Active Record encryption keys for the test suite.
#
# These are throwaway, non-production keys, committed on purpose: CI has no .env file, and dotenv
# does not load .env.development in the test environment.
AR_ENCRYPTION_PRIMARY_KEY=3WdNnb4TBItgQxzJd5CUdk2g2v03UvzZ
AR_ENCRYPTION_KEY_DERIVATION_SALT=SggkroBqG4UVZINKNAaURapc6PREICJ5
8 changes: 8 additions & 0 deletions app/controllers/partners/children_controller.rb
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,13 @@ def index

@children = @filterrific.find

# date_of_birth is encrypted => can't ORDER BY it in SQL. Sort the decrypted value in Ruby
# (safe: this index isn't paginated, whole set is already loaded).
if params[:sort] == "date_of_birth"
@children = @children.sort_by { |c| [c.date_of_birth || Date.new(9999, 12, 31), c.last_name.to_s, c.id] }
@children = @children.reverse if sort_direction == "desc"
end

respond_to do |format|
format.js
format.html
Expand Down Expand Up @@ -91,6 +98,7 @@ def child_params
end

def sort_order
return "last_name, id" if params[:sort] == "date_of_birth" # encrypted; sorted in Ruby, not SQL
sort_column + ' ' + sort_direction
end

Expand Down
5 changes: 4 additions & 1 deletion app/models/partners/authorized_family_member.rb
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
#
# id :bigint not null, primary key
# comments :text
# date_of_birth :date
# date_of_birth :text
# first_name :string
# gender :string
# last_name :string
Expand All @@ -18,6 +18,9 @@ class AuthorizedFamilyMember < Base
belongs_to :family
has_many :child_item_requests, dependent: :nullify

attribute :date_of_birth, :date
encrypts :date_of_birth

def display_name
"#{first_name} #{last_name}"
end
Expand Down
5 changes: 4 additions & 1 deletion app/models/partners/child.rb
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
# archived :boolean
# child_lives_with :jsonb
# comments :text
# date_of_birth :date
# date_of_birth :text
# first_name :string
# gender :string
# health_insurance :jsonb
Expand All @@ -27,6 +27,9 @@ class Child < Base
has_many :child_item_requests, dependent: :destroy
has_and_belongs_to_many :requested_items, class_name: 'Item'

attribute :date_of_birth, :date
encrypts :date_of_birth

include Filterable
include Exportable

Expand Down
1 change: 1 addition & 0 deletions app/models/partners/family.rb
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
module Partners
class Family < Base
has_paper_trail
encrypts :guardian_phone
belongs_to :partner, class_name: '::Partner'
has_many :children, dependent: :destroy
has_many :authorized_family_members, dependent: :destroy
Expand Down
7 changes: 7 additions & 0 deletions app/models/partners/profile.rb
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,13 @@ module Partners
class Profile < Base
has_paper_trail
self.table_name = "partner_profiles"

# Contact details of named individuals at the agency, so PII by the same standard as the
# product drive participants. None of these are queried by value, so non-deterministic.
encrypts :executive_director_email, :executive_director_phone,
:primary_contact_email, :primary_contact_phone, :primary_contact_mobile,
:pick_up_email, :pick_up_phone

belongs_to :partner
has_one :organization, through: :partner, class_name: "::Organization"

Expand Down
1 change: 1 addition & 0 deletions app/models/product_drive_participant.rb
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

class ProductDriveParticipant < ApplicationRecord
has_paper_trail
encrypts :phone, :email
include Provideable
include Geocodable

Expand Down
13 changes: 13 additions & 0 deletions config/initializers/active_record_encryption.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Keys come from the environment everywhere, so there is no env-specific branch here. Development
# and test read them from the committed .env.development and .env.test, which dotenv loads before
# initializers run. Production and staging get them from the real environment, and ENV.fetch has no
# default, so a missing key fails the boot loudly instead of silently writing data that cannot be
# read back later.
ActiveRecord::Encryption.configure(
primary_key: ENV.fetch("AR_ENCRYPTION_PRIMARY_KEY"),
key_derivation_salt: ENV.fetch("AR_ENCRYPTION_KEY_DERIVATION_SALT"),

# Bridge for migrating existing plaintext rows: lets reads of not-yet-backfilled
# columns work instead of raising. Flip to false (follow-up) after prod backfill.
support_unencrypted_data: true
)
21 changes: 21 additions & 0 deletions db/migrate/20260703191550_encrypt_dates_of_birth.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
class EncryptDatesOfBirth < ActiveRecord::Migration[8.0]
def up
# date -> text: encrypted values are string blobs and won't fit a date column.
#
# `using` pins the serialization format. Without it Postgres falls back to the date output
# function, which follows the server's DateStyle setting: under `SQL, MDY` the same date is
# written as "03/04/1990", which Ruby then reads back as April 3rd. This rewrite is in place
# and irreversible, so the format cannot be left to a server setting.
safety_assured do
change_column :children, :date_of_birth, :text, using: "to_char(date_of_birth, 'YYYY-MM-DD')"
change_column :authorized_family_members, :date_of_birth, :text, using: "to_char(date_of_birth, 'YYYY-MM-DD')"
end
end

def down
safety_assured do
change_column :children, :date_of_birth, :date, using: "date_of_birth::date"
change_column :authorized_family_members, :date_of_birth, :date, using: "date_of_birth::date"
end
end
end
46 changes: 46 additions & 0 deletions db/migrate/20260703191552_backfill_encrypted_pii.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
class BackfillEncryptedPii < ActiveRecord::Migration[8.0]
# `encrypts` only encrypts new writes, so existing rows stay plaintext until re-saved.
# This re-encrypts them in place. `record.encrypt` uses update_columns (no validations,
# callbacks, timestamps or paper_trail), and relies on
# config.active_record.encryption.support_unencrypted_data = true to read the plaintext.
#
# No wrapping transaction: each row commits on its own, so a big table doesn't hold one long
# transaction/lock, and a run that fails halfway keeps the rows it already encrypted.
disable_ddl_transaction!

MODELS = [
Partners::Child,
Partners::AuthorizedFamilyMember,
Partners::Family,
Partners::Profile,
ProductDriveParticipant
].freeze

def up
MODELS.each do |model|
model.find_in_batches do |batch|

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we find only non-encrypted rows to handle the case where a run might fail?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You're right, and my "idempotent" comment was wrong: idempotent in the value, not in the work. record.encrypt writes unconditionally (it just calls update_columns), and our encryption is non-deterministic, so re-encrypting an already-encrypted row rewrites it with new ciphertext. Same value, wasted write. A failed run redid the whole table.

There's no way to filter this in SQL. Ciphertext is opaque to Postgres, and Rails has no scope for it (extend_queries only helps deterministic attributes, and ours are all non-deterministic). Matching the payload prefix with NOT LIKE '{"p":%' would work, but it hardcodes Rails' internal JSON format into a migration.

So I used the public API instead, skipping the row before the write:

batch.reject { |record| encrypted?(record) }.each(&:encrypt)

encrypted? uses record.encrypted_attribute?, and treats a blank value as done. We still read every row, which is one sequential scan, but we only write the ones that need it. That's the expensive part (WAL, replication), so a re-run after a failure only does what's left.

Covered in spec/migrations/backfill_encrypted_pii_spec.rb, including already-encrypted rows staying byte for byte unchanged.

batch.reject { |record| encrypted?(record) }.each(&:encrypt)
sleep(0.1) # throttle prod DB load (~40-50k rows); matches BackfillItemReportingCategoryField
end
end
end

def down
raise ActiveRecord::IrreversibleMigration
end

private

# `encrypt` writes unconditionally, and the encryption is non-deterministic, so re-encrypting an
# already-encrypted row rewrites it with fresh ciphertext: same value, wasted write. Skipping
# those rows is what lets a run that failed halfway be re-run and only do what is left.
#
# Ciphertext is opaque to Postgres, so there is no scope for this: the check is per record.
# A blank value counts as done, since there is nothing to encrypt.
def encrypted?(record)
record.class.encrypted_attributes.all? do |attribute|
value = record.read_attribute_before_type_cast(attribute)
value.blank? || record.encrypted_attribute?(attribute)
end
end
end
6 changes: 3 additions & 3 deletions db/schema.rb
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.

ActiveRecord::Schema[8.0].define(version: 2026_03_13_201123) do
ActiveRecord::Schema[8.0].define(version: 2026_07_03_191552) do
# These are extensions that must be enabled in order to support this database
enable_extension "pg_catalog.plpgsql"

Expand Down Expand Up @@ -112,7 +112,7 @@
create_table "authorized_family_members", force: :cascade do |t|
t.string "first_name"
t.string "last_name"
t.date "date_of_birth"
t.text "date_of_birth"
t.string "gender"
t.text "comments"
t.bigint "family_id"
Expand Down Expand Up @@ -173,7 +173,7 @@
create_table "children", force: :cascade do |t|
t.string "first_name"
t.string "last_name"
t.date "date_of_birth"
t.text "date_of_birth"
t.string "gender"
t.jsonb "child_lives_with"
t.jsonb "race"
Expand Down
2 changes: 1 addition & 1 deletion spec/factories/partners/children.rb
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
# archived :boolean
# child_lives_with :jsonb
# comments :text
# date_of_birth :date
# date_of_birth :text
# first_name :string
# gender :string
# health_insurance :jsonb
Expand Down
73 changes: 73 additions & 0 deletions spec/migrations/backfill_encrypted_pii_spec.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
require "rails_helper"
require Rails.root.join("db/migrate/20260703191552_backfill_encrypted_pii")

RSpec.describe BackfillEncryptedPii, type: :migration do
let(:connection) { ActiveRecord::Base.connection }
let(:migration) { described_class.new }

let(:family) { create(:partners_family) }
let(:child) { create(:partners_child, family: family) }
let(:participant) { create(:product_drive_participant) }
let(:profile) { create(:partner_profile) }

# Rows as production holds them: written before `encrypts` existed, so still plaintext.
# They have to be seeded with raw SQL, since the models would encrypt them on the way in.
def write_plaintext(record, values)
assignments = values.map { |column, value| "#{column} = #{connection.quote(value)}" }.join(", ")
connection.execute("UPDATE #{record.class.table_name} SET #{assignments} WHERE id = #{record.id}")
record.reload
end

def stored(record, column)
connection.select_value("SELECT #{column} FROM #{record.class.table_name} WHERE id = #{record.id}")
end

it "encrypts the rows that are still plaintext, keeping their values" do
write_plaintext(child, date_of_birth: "1990-03-04")
write_plaintext(family, guardian_phone: "555-1234")
write_plaintext(participant, phone: "555-9876", email: "donor@example.com")
write_plaintext(profile, primary_contact_email: "contact@example.com", pick_up_phone: "555-4444")

migration.up

expect(stored(child, :date_of_birth)).to start_with('{"p":')
expect(stored(family, :guardian_phone)).to start_with('{"p":')
expect(stored(participant, :email)).to start_with('{"p":')
expect(stored(profile, :primary_contact_email)).to start_with('{"p":')

expect(child.reload.date_of_birth).to eq(Date.new(1990, 3, 4))
expect(family.reload.guardian_phone).to eq("555-1234")
expect(participant.reload.phone).to eq("555-9876")
expect(participant.reload.email).to eq("donor@example.com")
expect(profile.reload.primary_contact_email).to eq("contact@example.com")
expect(profile.pick_up_phone).to eq("555-4444")
end

it "leaves rows that are already encrypted untouched" do
# The factories write through the models, so these rows are encrypted already. Encryption is
# non-deterministic: if the migration re-encrypted them, the ciphertext would come out different.
ciphertext = stored(child, :date_of_birth)

expect { migration.up }.not_to change { stored(child, :date_of_birth) }.from(ciphertext)
end

it "picks up where a failed run stopped" do
encrypted_child = child
plaintext_child = create(:partners_child, family: family)
write_plaintext(plaintext_child, date_of_birth: "1985-12-31")
ciphertext = stored(encrypted_child, :date_of_birth)

migration.up

expect(stored(encrypted_child, :date_of_birth)).to eq(ciphertext)
expect(plaintext_child.reload.date_of_birth).to eq(Date.new(1985, 12, 31))
end

it "leaves a missing value null" do
write_plaintext(child, date_of_birth: nil)

migration.up

expect(stored(child, :date_of_birth)).to be_nil
end
end
Loading
Loading