From 2d04dd1b6c13859df986191d471353c0e8eb8dcd Mon Sep 17 00:00:00 2001 From: mkhara Date: Fri, 25 Sep 2026 16:21:03 -0700 Subject: [PATCH] CASSANALYTICS-202: Opening one keyspace instance rebuilds every keyspace instance in the Cassandra 6.0 bridge AbstractSchemaBuilder created a missing keyspace instance by calling SchemaUpdater.openKeyspaceInstances(), which diffs the current cluster metadata against DistributedSchema.empty(). Every keyspace in the process therefore looks newly created and all of them are rebuilt, so each keyspace that already has an instance constructs its KeyspaceMetrics again, and every metric it re-registers makes Dropwizard's MetricRegistry.register throw IllegalArgumentException. Filling in those stack traces is the real cost, and it scales with the number of keyspaces the process has seen. This does not affect production, where a bulk job uses a single keyspace and builds its schema once. It is visible in tests: DataTypeTests opens a fresh keyspace per generated case, so the 6.0 core test job slowed down through the class until the CI step exceeded its deadline part way through. Two thread dumps taken fifteen seconds apart were both in Throwable.fillInStackTrace under MetricRegistry.register, and jstat ruled out memory pressure with zero full collections and seven seconds of GC across twelve minutes. openKeyspaceInstances() is replaced by openKeyspaceInstance(String), which diffs against the current keyspaces minus the one being opened, leaving exactly one keyspace in Keyspaces.diff().created so a single instance is built and no metric is registered twice. That also keeps the keyspace out of diff().altered, which is the property removeTables relies on to avoid Keyspace.dropCf initializing CompactionManager in client mode, so the CDC caller passes keyspaceMetadata.name and the blunt method is removed altogether. DataTypeTests on 6.0 goes from not completing to about ten minutes. --- CHANGES.txt | 1 + .../cassandra/bridge/SchemaUpdater.java | 31 ++++++++++--------- .../spark/reader/AbstractSchemaBuilder.java | 6 ++-- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 5e6d18b99..7f6914309 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,5 +1,6 @@ 0.5.0 ----- + * Opening one keyspace instance rebuilds every keyspace instance in the Cassandra 6.0 bridge (CASSANALYTICS-202) * CDC logs NPE for deleted column values (CASSANALYTICS-178) * Add Cassandra 6.0 support (CASSANALYTICS-37) * Eliminate redundant filesystem lookups in SSTable direct streaming (CASSANALYTICS-104) diff --git a/cassandra-six-zero-types/src/main/java/org/apache/cassandra/bridge/SchemaUpdater.java b/cassandra-six-zero-types/src/main/java/org/apache/cassandra/bridge/SchemaUpdater.java index 5802d4de3..e034f8264 100644 --- a/cassandra-six-zero-types/src/main/java/org/apache/cassandra/bridge/SchemaUpdater.java +++ b/cassandra-six-zero-types/src/main/java/org/apache/cassandra/bridge/SchemaUpdater.java @@ -60,16 +60,19 @@ public static ClusterMetadata submit(SchemaProvider schema, SchemaTransformation } /** - * Creates a keyspace instance for every keyspace of the current cluster metadata. Use this only when a - * keyspace has metadata but no instance, which happens when code outside this class commits it; Cassandra - * 6.0's {@code Keyspace.openWithoutSSTables} only reads the instance and no longer creates it. + * Creates the instance for a single keyspace, leaving every other keyspace's instance untouched. Needed when a + * keyspace has metadata but no instance. Cassandra 6.0's {@code Keyspace.openWithoutSSTables} only reads the + * instance rather than creating it. * - *

Passing an empty previous schema replaces every existing instance, dropping the column family stores - * it held, so prefer {@link #submit}, which keeps the instances of each commit. + *

Targeting one keyspace avoids rebuilding all of them, which re-registers metrics and can throw + * stack-trace-filling exceptions. A bulk job builds schema once per keyspace, so this only shows up in tests that + * open many distinct keyspaces in one JVM. */ - public static void openKeyspaceInstances() + public static void openKeyspaceInstance(String keyspaceName) { - ClusterMetadata.current().schema.initializeKeyspaceInstances(DistributedSchema.empty(), false); + DistributedSchema current = ClusterMetadata.current().schema; + DistributedSchema before = new DistributedSchema(current.getKeyspaces().without(keyspaceName)); + current.initializeKeyspaceInstances(before, false); } public static void load(SchemaProvider schema, KeyspaceMetadata keyspaceMetadata) @@ -90,18 +93,16 @@ public static void load(SchemaProvider schema, Types userTypes) /** * Replaces the metadata of an existing keyspace with metadata that holds fewer tables. * - *

Cassandra 4.0's {@code Schema.load} added or reloaded, whereas {@link SchemaTransformations#addKeyspace} - * only adds and otherwise throws {@code AlreadyExistsException}, so a caller that means to replace needs a - * transformation of its own. {@link #submit} is also the wrong follow-up here: it reports the table in - * {@code Keyspaces.diff().altered}, which makes {@code DistributedSchema.initializeKeyspaceInstances} call - * {@code Keyspace.dropCf} and so initialize {@code CompactionManager}, which throws in client mode where - * concurrent_compactors is zero. Rebuild the instances from the committed metadata instead, which leaves the - * removed table without a column family store and touches no compaction machinery. + *

{@link SchemaTransformations#addKeyspace} only adds, throwing {@code AlreadyExistsException} otherwise, so + * replacing needs its own transformation. {@link #submit} is also wrong here: it reports the table as altered, + * which calls {@code Keyspace.dropCf} and initializes {@code CompactionManager}, throwing in client mode where + * concurrent_compactors is zero. {@link #openKeyspaceInstance} reports the keyspace as created instead, leaving + * the removed table without a column family store and touching no compaction machinery. */ public static void removeTables(SchemaProvider schema, KeyspaceMetadata keyspaceMetadata) { schema.submit(replace(keyspaceMetadata)); - openKeyspaceInstances(); + openKeyspaceInstance(keyspaceMetadata.name); } public static void updateTable(SchemaProvider schema, KeyspaceMetadata keyspaceMetadata, TableMetadata tableMetadata) diff --git a/cassandra-six-zero-types/src/main/java/org/apache/cassandra/spark/reader/AbstractSchemaBuilder.java b/cassandra-six-zero-types/src/main/java/org/apache/cassandra/spark/reader/AbstractSchemaBuilder.java index aa1c15e8d..a8157e0c7 100644 --- a/cassandra-six-zero-types/src/main/java/org/apache/cassandra/spark/reader/AbstractSchemaBuilder.java +++ b/cassandra-six-zero-types/src/main/java/org/apache/cassandra/spark/reader/AbstractSchemaBuilder.java @@ -365,7 +365,7 @@ private static void setupKeyspace(SchemaProvider schema, keyspaceName, replicationFactor.getReplicationStrategy().name(), partitioner); // Create keyspace instance and also initCf (cfs) for the table. Cassandra 6.0 holds the keyspace // instances in the cluster metadata, which Keyspace.openWithoutSSTables only reads. - SchemaUpdater.openKeyspaceInstances(); + SchemaUpdater.openKeyspaceInstance(keyspaceName); } } @@ -417,8 +417,8 @@ private static void setupTableAndUdt(SchemaProvider schema, else { // The keyspace has not yet opened, create/open keyspace instance and also initCf (cfs) for the - // table. See SchemaUpdater.openKeyspaceInstances. - SchemaUpdater.openKeyspaceInstances(); + // table. See SchemaUpdater.openKeyspaceInstance. + SchemaUpdater.openKeyspaceInstance(keyspaceName); } }