Add CI that checks that old schemas are upgradable and give the same final schema.

[reivilibre]

We have a couple of sad problems with our database schema:

• Some old versions can't be upgraded to new versions currently, as we broke the deltas
• Some old versions, when upgraded, are different (e.g. missing indexes) to a fresh schema.

Both these issues are footgunsome.

Related: #20084

Follows: #20027 (loosely)

---

1. make_full_schema: Allow testing upgrade from a specified full schema

2. Add CI check that upgrades old schema and checks its identical

[github-actions]

Schema Diff

Please check that this looks as expected!

common (sqlite)

Unchanged

common (postgres)

Unchanged

main (sqlite)

--- /tmp/schema_diff_m2szd55p/before/main/full_schemas/9999/full.sql.sqlite	2001-05-25 12:42:42.000000000 +0000
+++ /tmp/schema_diff_m2szd55p/after/main/full_schemas/9999/full.sql.sqlite	2001-05-25 12:42:42.000000000 +0000
@@ -1,15 +1,14 @@
 CREATE TABLE application_services_txns( as_id TEXT NOT NULL, txn_id INTEGER NOT NULL, event_ids TEXT NOT NULL, UNIQUE(as_id, txn_id) );
 CREATE INDEX application_services_txns_id ON application_services_txns ( as_id );
 CREATE TABLE users( name TEXT, password_hash TEXT, creation_ts BIGINT, admin SMALLINT DEFAULT 0 NOT NULL, upgrade_ts BIGINT, is_guest SMALLINT DEFAULT 0 NOT NULL, appservice_id TEXT, consent_version TEXT, consent_server_notice_sent TEXT, user_type TEXT DEFAULT NULL, deactivated SMALLINT DEFAULT 0 NOT NULL, shadow_banned BOOLEAN, consent_ts bigint, approved BOOLEAN, locked BOOLEAN DEFAULT FALSE NOT NULL, suspended BOOLEAN DEFAULT FALSE NOT NULL, UNIQUE(name) );
 CREATE TABLE user_ips ( user_id TEXT NOT NULL, access_token TEXT NOT NULL, device_id TEXT, ip TEXT NOT NULL, user_agent TEXT NOT NULL, last_seen BIGINT NOT NULL );
-CREATE TABLE profiles( user_id TEXT NOT NULL, displayname TEXT, avatar_url TEXT, full_user_id TEXT, fields JSONB, UNIQUE(user_id) );
 CREATE TABLE received_transactions( transaction_id TEXT, origin TEXT, ts BIGINT, response_code INTEGER, response_json bytea, has_been_referenced smallint default 0, UNIQUE (transaction_id, origin) );
 CREATE TABLE destinations( destination TEXT PRIMARY KEY, retry_last_ts BIGINT, retry_interval INTEGER , failure_ts BIGINT, last_successful_stream_ordering BIGINT);
 CREATE TABLE events( stream_ordering INTEGER PRIMARY KEY, topological_ordering BIGINT NOT NULL, event_id TEXT NOT NULL, type TEXT NOT NULL, room_id TEXT NOT NULL, content TEXT, unrecognized_keys TEXT, processed BOOL NOT NULL, outlier BOOL NOT NULL, depth BIGINT DEFAULT 0 NOT NULL, origin_server_ts BIGINT, received_ts BIGINT, sender TEXT, contains_url BOOLEAN, instance_name TEXT, state_key TEXT DEFAULT NULL, rejection_reason TEXT DEFAULT NULL, UNIQUE (event_id) );
 CREATE INDEX events_order_room ON events ( room_id, topological_ordering, stream_ordering );
 CREATE TABLE event_json( event_id TEXT NOT NULL, room_id TEXT NOT NULL, internal_metadata TEXT NOT NULL, json TEXT NOT NULL, format_version INTEGER, UNIQUE (event_id) );
 CREATE TABLE state_events( event_id TEXT NOT NULL, room_id TEXT NOT NULL, type TEXT NOT NULL, state_key TEXT NOT NULL, prev_state TEXT, UNIQUE (event_id) );
 CREATE TABLE current_state_events( event_id TEXT NOT NULL, room_id TEXT NOT NULL, type TEXT NOT NULL, state_key TEXT NOT NULL, membership TEXT, event_stream_ordering BIGINT REFERENCES events(stream_ordering), UNIQUE (event_id), UNIQUE (room_id, type, state_key) );
 CREATE TABLE room_memberships( event_id TEXT NOT NULL, user_id TEXT NOT NULL, sender TEXT NOT NULL, room_id TEXT NOT NULL, membership TEXT NOT NULL, forgotten INTEGER DEFAULT 0, display_name TEXT, avatar_url TEXT, event_stream_ordering BIGINT REFERENCES events(stream_ordering), participant BOOLEAN DEFAULT FALSE, UNIQUE (event_id) );
 CREATE INDEX room_memberships_room_id ON room_memberships (room_id);
 CREATE INDEX room_memberships_user_id ON room_memberships (user_id);
@@ -261,22 +260,20 @@
     canonical_alias TEXT,
     join_rules TEXT,
     history_visibility TEXT,
     encryption TEXT,
     avatar TEXT,
     guest_access TEXT,
     is_federatable BOOLEAN,
     topic TEXT
 , room_type TEXT);
 CREATE UNIQUE INDEX room_stats_state_room ON room_stats_state(room_id);
-CREATE TABLE IF NOT EXISTS "user_filters" ( user_id TEXT NOT NULL, filter_id BIGINT NOT NULL, filter_json BYTEA NOT NULL , full_user_id TEXT);
-CREATE UNIQUE INDEX user_filters_unique ON "user_filters" (user_id, filter_id);
 CREATE TABLE user_external_ids (
     auth_provider TEXT NOT NULL,
     external_id TEXT NOT NULL,
     user_id TEXT NOT NULL,
     UNIQUE (auth_provider, external_id)
 );
 CREATE INDEX users_in_public_rooms_r_idx ON users_in_public_rooms(room_id);
 CREATE TABLE device_lists_remote_resync (
     user_id TEXT NOT NULL,
     added_ts BIGINT NOT NULL
@@ -786,23 +783,20 @@
 CREATE INDEX event_push_actions_u_highlight ON event_push_actions (user_id, stream_ordering);
 CREATE UNIQUE INDEX event_push_summary_unique_index2 ON event_push_summary (user_id, room_id, thread_id) ;
 CREATE TABLE IF NOT EXISTS "event_forward_extremities"(
         event_id TEXT NOT NULL,
         room_id TEXT NOT NULL,
         UNIQUE (event_id, room_id),
         CONSTRAINT event_forward_extremities_event_id FOREIGN KEY (event_id) REFERENCES events (event_id) DEFERRABLE INITIALLY DEFERRED
     );
 CREATE INDEX ev_extrem_room ON event_forward_extremities(room_id);
 CREATE INDEX ev_extrem_id ON event_forward_extremities(event_id);
-CREATE UNIQUE INDEX user_filters_full_user_id_unique ON
-        user_filters (full_user_id, filter_id)
-        ;
 CREATE TABLE worker_read_write_locks_mode (
     lock_name TEXT NOT NULL,
     lock_key TEXT NOT NULL,
     -- Whether this lock is in read (false) or write (true) mode
     write_lock BOOLEAN NOT NULL,
     -- A token that has currently acquired the lock. We need this so that we can
     -- add a foreign constraint from this table to `worker_read_write_locks`.
     token TEXT NOT NULL,
     -- Add a foreign key constraint to ensure that if a lock is in
     -- `worker_read_write_locks_mode` then there must be a corresponding row in
@@ -1195,27 +1189,43 @@
 
   -- The full user ID of the local user that should receive the profile update.
   user_id TEXT NOT NULL,
 
   -- Unix timestamp (milliseconds). Used to determine when to prune rows (to prevent the table
   -- from growing indefinitely).
   inserted_ts BIGINT NOT NULL,
 
   PRIMARY KEY (user_id, stream_id)
 );
+CREATE TABLE IF NOT EXISTS "profiles" (
+    full_user_id text NOT NULL,
+    user_id text,
+    displayname text,
+    avatar_url text,
+    fields JSONB,
+    UNIQUE (full_user_id),
+    UNIQUE (user_id)
+);
+CREATE TABLE IF NOT EXISTS "user_filters" (
+    full_user_id text NOT NULL,
+    user_id text NOT NULL,
+    filter_id bigint NOT NULL,
+    filter_json bytea NOT NULL
+);
+CREATE UNIQUE INDEX profiles_full_user_id_key ON profiles (full_user_id);
+CREATE UNIQUE INDEX user_filters_unique ON user_filters (user_id, filter_id);
+CREATE UNIQUE INDEX full_users_unique_idx ON user_filters (full_user_id, filter_id);
 CREATE UNIQUE INDEX receipts_graph_unique_index ON receipts_graph (room_id, receipt_type, user_id) WHERE thread_id IS NULL;
 CREATE UNIQUE INDEX receipts_linearized_unique_index ON receipts_linearized (room_id, receipt_type, user_id) WHERE thread_id IS NULL;
 CREATE INDEX device_lists_changes_in_room_by_room_idx ON device_lists_changes_in_room (room_id, stream_id) ;
 CREATE INDEX events_jump_to_date_idx ON events (room_id, origin_server_ts) WHERE NOT outlier;
 CREATE INDEX room_membership_user_room_idx ON room_memberships (user_id, room_id) ;
-CREATE UNIQUE INDEX profiles_full_user_id_key ON profiles (full_user_id) ;
-CREATE UNIQUE INDEX full_users_unique_idx ON user_filters (full_user_id, filter_id) ;
 CREATE INDEX event_push_actions_stream_highlight_index ON event_push_actions (highlight, stream_ordering) WHERE highlight=0;
 CREATE INDEX current_state_events_stream_ordering_idx ON current_state_events (event_stream_ordering) ;
 CREATE INDEX local_current_membership_stream_ordering_idx ON local_current_membership (event_stream_ordering) ;
 CREATE INDEX room_memberships_stream_ordering_idx ON room_memberships (event_stream_ordering) ;
 CREATE INDEX e2e_room_keys_room_id ON e2e_room_keys (room_id) ;
 CREATE INDEX room_account_data_room_id ON room_account_data (room_id) ;
 CREATE INDEX event_push_summary_index_room_id ON event_push_summary (room_id) ;
 CREATE INDEX event_auth_chain_links_origin_index ON event_auth_chain_links (origin_chain_id, origin_sequence_number) ;
 CREATE INDEX access_tokens_refresh_token_id_idx ON access_tokens (refresh_token_id) ;
 CREATE INDEX receipts_linearized_event_id ON receipts_linearized (room_id, event_id) ;

main (postgres)

Unchanged

state (sqlite)

Unchanged

state (postgres)

Unchanged