fix: make iterate_properties query scale linearly

src/plexosdb/db.py

@@ -3401,7 +3401,7 @@ def iterate_properties(
         if scenario_category:
             scenario_categories = normalize_names(scenario_category)
             placeholders = ", ".join("?" for _ in scenario_categories)
-            conditions.append(f"scenario.category_name IN ({placeholders})")
+            conditions.append(f"scenario_category.name IN ({placeholders})")
             query_params.extend(scenario_categories)
 
         where_clause = "WHERE " + " AND ".join(conditions) if conditions else ""

src/plexosdb/schema.sql

@@ -422,6 +422,8 @@ CREATE TABLE `t_data`
                FOREIGN KEY (`property_id`) REFERENCES `t_property`(`property_id`)
 );
 
+CREATE INDEX `IX_t_data_membership` ON `t_data` (`membership_id`);
+
 CREATE TABLE `t_date_from`
 (
     `data_id` INT NOT NULL,

src/plexosdb/sql/object_properties.sql

@@ -1,34 +1,5 @@
 -- Query to get all the objects and nested objects
 -- We do not include scenario tags on nested objects.
-WITH scenario_cte AS (
-SELECT
-	object.name,
-	object.object_id,
-	tag.data_id,
-	mem.collection_id,
-	cat.name AS category_name
-FROM
-	t_membership AS mem
-LEFT JOIN t_tag AS tag ON
-	tag.object_id = mem.child_object_id
-LEFT JOIN t_object AS object ON
-	mem.child_object_id = object.object_id
-LEFT JOIN t_category AS cat ON
-	object.category_id = cat.category_id
-LEFT JOIN t_class AS class ON
-	class.class_id = object.class_id
-WHERE
-	mem.collection_id in
-        (
-	SELECT
-		collection_id
-	FROM
-		t_collection
-	LEFT JOIN t_class ON
-		t_class.class_id = t_collection.parent_class_id
-        )
-    and class.name = 'Scenario'
- )
 SELECT
 	parent_class.name AS parent_class,
 	child_class.name AS child_class,
@@ -88,9 +59,32 @@ LEFT JOIN t_object AS tag_object ON
 	tag_object.object_id = tag.object_id
 LEFT JOIN t_class AS tag_class ON
 	tag_class.class_id = tag_object.class_id
-LEFT JOIN scenario_cte AS scenario ON
-	scenario.data_id = data.data_id
-	and tag.data_id = scenario.data_id
+-- Join scenario tags through the t_tag primary key so each data row is an index lookup.
+LEFT JOIN t_tag AS scenario_tag ON
+	scenario_tag.data_id = data.data_id
+	AND tag.data_id = scenario_tag.data_id
+	AND scenario_tag.object_id IN (
+	SELECT
+		scenario_object.object_id
+	FROM
+		t_object AS scenario_object
+	JOIN t_class AS scenario_class ON
+		scenario_class.class_id = scenario_object.class_id
+	WHERE
+		scenario_class.name = 'Scenario'
+		AND EXISTS (
+		SELECT
+			1
+		FROM
+			t_membership AS mem
+		WHERE
+			mem.child_object_id = scenario_object.object_id
+		)
+	)
+LEFT JOIN t_object AS scenario ON
+	scenario.object_id = scenario_tag.object_id
+LEFT JOIN t_category AS scenario_category ON
+	scenario.category_id = scenario_category.category_id
 ${where_clause}
 GROUP BY
 	membership.membership_id,

tests/test_plexosdb_iterate_properties.py

@@ -263,3 +263,18 @@ def test_get_object_properties_filters_by_scenario_and_category(
 
     assert len(props) == 1
     assert props[0]["scenario_name"] == "ScenarioC"
+
+
+def test_iterate_properties_query_plan_has_no_per_row_scans(db_with_topology: PlexosDB) -> None:
+    """Data and scenario lookups must use indexes; a full scan per membership is quadratic."""
+    import re
+    from string import Template
+
+    from plexosdb.db import PROPERTY_QUERY
+
+    _setup_scenario_tagged_generator_properties(db_with_topology)
+    query = Template(PROPERTY_QUERY).safe_substitute(where_clause="")
+    plan = [row[3] for row in db_with_topology._db.connection.execute(f"EXPLAIN QUERY PLAN {query}")]
+
+    assert any(step.startswith("SEARCH data USING INDEX") for step in plan), plan
+    assert not any(re.match(r"SCAN (data|scenario)\b", step) for step in plan), plan