From e18655ebef3b8f057165540ba971dc12f33e076f Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Fri, 2 Oct 2026 19:15:07 -0700 Subject: [PATCH 001/134] fix(post-processing): write the size filter's last batch even when the scan skips its final detection The small size filter wrote its gathered rows at the end of the loop body, after the checks that skip a detection with no box or no image size. A skipped final detection therefore jumped past the write, and every classification flagged since the previous batch was lost with no error. The write now lives in one flush() helper called ahead of the skip checks at each batch boundary and once more after the loop, so the last batch is always written. The batch size, progress reports and stage metrics are unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01L52AN9tabp76yjhjyCZkSJ --- ami/ml/post_processing/small_size_filter.py | 83 ++++++++++--------- .../tests/test_small_size_filter.py | 57 +++++++++++++ 2 files changed, 102 insertions(+), 38 deletions(-) create mode 100644 ami/ml/post_processing/tests/test_small_size_filter.py diff --git a/ami/ml/post_processing/small_size_filter.py b/ami/ml/post_processing/small_size_filter.py index f0b8289eb..51d7ace98 100644 --- a/ami/ml/post_processing/small_size_filter.py +++ b/ami/ml/post_processing/small_size_filter.py @@ -91,7 +91,51 @@ def run(self) -> None: modified_occurrences = 0 checked = 0 + def flush(i: int) -> None: + """Write the rows gathered so far and report progress as of detection ``i``.""" + nonlocal checked, modified_detections, modified_occurrences + checked = i + modified_detections += len(detections_to_update) + + self.logger.info(f"Creating {len(classifications_to_create)} new classifications") + Classification.objects.bulk_create(classifications_to_create) + classifications_to_create.clear() + + self.logger.info(f"Marking {len(detections_to_update)} detections as {not_identifiable_taxon.name}") + for det in detections_to_update: + det.updated_at = timezone.now() + Detection.objects.bulk_update(detections_to_update, ["updated_at"]) + detections_to_update.clear() + + self.logger.info(f"Updating {len(occcurrences_to_update)} occurrences") + for occ in occcurrences_to_update: + # Count an occurrence only when flagging its detection actually + # changes the determination. Re-saving recomputes it in place, so + # an occurrence pinned to a human identification keeps its taxon + # and must not inflate the metric. + prev_determination_id = occ.determination_id + occ.save(update_determination=True) + if occ.pk is not None and occ.determination_id != prev_determination_id: + updated_occurrence_ids.add(occ.pk) + modified_occurrences = len(updated_occurrence_ids) + occcurrences_to_update.clear() + + progress = i / total if total > 0 else 1.0 + self.update_progress(progress) + self.report_stage_metrics( + { + "detections_checked": checked, + "detections_flagged": modified_detections, + "occurrences_updated": modified_occurrences, + } + ) + + i = 0 for i, det in enumerate(detections.iterator(), start=1): + # Write each full batch of 100 before the next begins. The flush sits ahead of the + # skip checks below so a skipped detection never swallows the batch before it. + if i > 1 and (i - 1) % 100 == 0: + flush(i - 1) bbox = det.get_bbox() if not bbox: self.logger.debug(f"Detection {det.pk}: no bbox, skipping") @@ -134,43 +178,6 @@ def run(self) -> None: occcurrences_to_update.add(det.occurrence) self.logger.debug(f"Marking detection {det.pk} as {not_identifiable_taxon.name}") - # Update progress every 100 detections - if i % 100 == 0 or i == total: - checked = i - modified_detections += len(detections_to_update) - - # with transaction.atomic(): - self.logger.info(f"Creating {len(classifications_to_create)} new classifications") - Classification.objects.bulk_create(classifications_to_create) - classifications_to_create.clear() - - self.logger.info(f"Marking {len(detections_to_update)} detections as {not_identifiable_taxon.name}") - for det in detections_to_update: - det.updated_at = timezone.now() - Detection.objects.bulk_update(detections_to_update, ["updated_at"]) - detections_to_update.clear() - - self.logger.info(f"Updating {len(occcurrences_to_update)} occurrences") - for occ in occcurrences_to_update: - # Count an occurrence only when flagging its detection actually - # changes the determination. Re-saving recomputes it in place, so - # an occurrence pinned to a human identification keeps its taxon - # and must not inflate the metric. - prev_determination_id = occ.determination_id - occ.save(update_determination=True) - if occ.pk is not None and occ.determination_id != prev_determination_id: - updated_occurrence_ids.add(occ.pk) - modified_occurrences = len(updated_occurrence_ids) - occcurrences_to_update.clear() - - progress = i / total if total > 0 else 1.0 - self.update_progress(progress) - self.report_stage_metrics( - { - "detections_checked": checked, - "detections_flagged": modified_detections, - "occurrences_updated": modified_occurrences, - } - ) + flush(i) self.logger.info(f"=== Completed {self.name}: {modified_detections} of {total} detections modified ===") diff --git a/ami/ml/post_processing/tests/test_small_size_filter.py b/ami/ml/post_processing/tests/test_small_size_filter.py new file mode 100644 index 000000000..7d96c7af5 --- /dev/null +++ b/ami/ml/post_processing/tests/test_small_size_filter.py @@ -0,0 +1,57 @@ +"""What the small size filter writes: a "Not identifiable" classification per flagged detection, in batches.""" + +import logging + +from django.test import TestCase + +from ami.main.models import Detection, Occurrence, SourceImage, Taxon +from ami.ml.post_processing.small_size_filter import SmallSizeFilterTask +from ami.tests.fixtures.main import create_captures, create_taxa, setup_test_project + +logger = logging.getLogger(__name__) + + +class SizeFilterTestCase(TestCase): + def setUp(self) -> None: + self.project, self.deployment = setup_test_project(reuse=False) + create_taxa(project=self.project) + create_captures(deployment=self.deployment, num_nights=1, images_per_night=3, interval_minutes=1) + self.captures = list(SourceImage.objects.filter(deployment=self.deployment).order_by("timestamp")) + SourceImage.objects.filter(pk__in=[c.pk for c in self.captures]).update(width=1000, height=1000) + self.event = self.captures[0].event + self.taxon = Taxon.objects.filter(projects=self.project).order_by("pk").first() + + def _singleton(self, capture: SourceImage, bbox: list[int]) -> Occurrence: + """One detection in an occurrence of its own, classified as the fixture taxon.""" + occurrence = Occurrence.objects.create(event=self.event, deployment=self.deployment, project=self.project) + detection = Detection.objects.create( + source_image=capture, bbox=bbox, timestamp=capture.timestamp, occurrence=occurrence + ) + detection.classifications.create(taxon=self.taxon, score=0.9, timestamp=capture.timestamp) + occurrence.save() + return occurrence + + def run_filter(self, occurrence: Occurrence, **kwargs) -> None: + SmallSizeFilterTask(logger=logger, occurrence_id=occurrence.pk, size_threshold=0.01, **kwargs).run() + + +class SizeFilterBatchTestCase(SizeFilterTestCase): + def test_a_skipped_last_detection_still_saves_the_batch_before_it(self): + """A detection the scan skips (no box) must not swallow the flagged ones gathered before it.""" + occurrence = self._singleton(self.captures[0], [0, 0, 10, 10]) + # Inserted last, so the scan reaches it last; it has no box and is skipped. + Detection.objects.create( + source_image=self.captures[1], bbox=None, timestamp=self.captures[1].timestamp, occurrence=occurrence + ) + + self.run_filter(occurrence) + + occurrence.refresh_from_db() + self.assertEqual(occurrence.determination.name, "Not identifiable") + + def test_a_detection_above_the_threshold_is_left_alone(self): + occurrence = self._singleton(self.captures[0], [0, 0, 500, 500]) + self.run_filter(occurrence) + occurrence.refresh_from_db() + self.assertEqual(occurrence.determination, self.taxon) + self.assertEqual(occurrence.detections.get().classifications.count(), 1) From 514a37343ae5244d7ab7cdfa52fa784e8709fba6 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Fri, 2 Oct 2026 19:20:19 -0700 Subject: [PATCH 002/134] feat(results): store what post-processing runs decide and what people comment on an occurrence Class masking and the size filter change an occurrence's classifications and determination, but nothing records that a run happened, with which settings, or what it changed. Comments from reviewers have no home at all. This adds the two tables the occurrence history reads. AlgorithmResult is one row per run and occurrence: the algorithm, the job, a kind (class masking or size filter) and a data column validated against the schema registered for that kind. A new result for the same occurrence, algorithm and kind becomes the current one and the earlier ones stay as history. The determination never reads this table; a run changes a taxon only through the classifications it creates. The project is copied from the occurrence, or from its station when the occurrence has none, so per-project queries need no join. A result for an occurrence with neither is skipped with a warning rather than failing a run that has already changed data. ValidationReview is one row per thing a person said about an occurrence. Only comments are written for now; the aspect and verdict choices are there for tracking's grouping review to extend. A review can point at the result it answers and can be withdrawn without being deleted. Part of #1431 and #1457. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01L52AN9tabp76yjhjyCZkSJ --- ...lgorithm_results_and_validation_reviews.py | 157 ++++++++++++++ ami/main/models.py | 203 ++++++++++++++++++ ami/main/schemas.py | 66 ++++++ 3 files changed, 426 insertions(+) create mode 100644 ami/main/migrations/0096_algorithm_results_and_validation_reviews.py create mode 100644 ami/main/schemas.py diff --git a/ami/main/migrations/0096_algorithm_results_and_validation_reviews.py b/ami/main/migrations/0096_algorithm_results_and_validation_reviews.py new file mode 100644 index 000000000..cfb2b9afc --- /dev/null +++ b/ami/main/migrations/0096_algorithm_results_and_validation_reviews.py @@ -0,0 +1,157 @@ +# Generated by Django 4.2.10 on 2026-10-02 22:19 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + dependencies = [ + ("jobs", "0023_alter_job_job_type_key"), + ("ml", "0028_normalize_empty_endpoint_url_to_null"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("main", "0095_grant_sync_deployment_to_mldatamanager"), + ] + + operations = [ + migrations.CreateModel( + name="AlgorithmResult", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "kind", + models.CharField( + choices=[("class_masking", "Class Masking"), ("size_filter", "Size Filter")], max_length=32 + ), + ), + ("data", models.JSONField(blank=True, default=dict)), + ("is_current", models.BooleanField(default=True)), + ("timestamp", models.DateTimeField(default=django.utils.timezone.now)), + ( + "algorithm", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="algorithm_results", + to="ml.algorithm", + ), + ), + ( + "job", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="algorithm_results", + to="jobs.job", + ), + ), + ( + "occurrence", + models.ForeignKey( + db_index=False, + on_delete=django.db.models.deletion.CASCADE, + related_name="algorithm_results", + to="main.occurrence", + ), + ), + ( + "project", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="algorithm_results", + to="main.project", + ), + ), + ], + ), + migrations.CreateModel( + name="ValidationReview", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ("aspect", models.CharField(choices=[("comment", "Comment")], max_length=32)), + ( + "verdict", + models.CharField( + blank=True, + choices=[("confirmed", "Confirmed"), ("rejected", "Rejected")], + max_length=16, + null=True, + ), + ), + ("payload", models.JSONField(blank=True, default=dict)), + ("comment", models.TextField(blank=True, default="")), + ("withdrawn", models.BooleanField(default=False)), + ("timestamp", models.DateTimeField(default=django.utils.timezone.now)), + ( + "occurrence", + models.ForeignKey( + db_index=False, + on_delete=django.db.models.deletion.CASCADE, + related_name="validation_reviews", + to="main.occurrence", + ), + ), + ( + "project", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="validation_reviews", + to="main.project", + ), + ), + ( + "reviewed_result", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="reviews", + to="main.algorithmresult", + ), + ), + ( + "user", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="validation_reviews", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "indexes": [models.Index(fields=["occurrence", "-timestamp"], name="validation_review_occ_time")], + }, + ), + migrations.AddConstraint( + model_name="validationreview", + constraint=models.CheckConstraint( + check=models.Q( + models.Q( + ("aspect", "comment"), ("verdict__isnull", True), models.Q(("comment", ""), _negated=True) + ), + models.Q(models.Q(("aspect", "comment"), _negated=True), ("verdict__isnull", False)), + _connector="OR", + ), + name="validation_review_verdict_by_aspect", + ), + ), + migrations.AddIndex( + model_name="algorithmresult", + index=models.Index(fields=["occurrence", "-timestamp"], name="algorithm_result_occ_time"), + ), + migrations.AddConstraint( + model_name="algorithmresult", + constraint=models.UniqueConstraint( + condition=models.Q(("is_current", True)), + fields=("occurrence", "algorithm", "kind"), + name="algorithm_result_current_occurrence", + ), + ), + ] diff --git a/ami/main/models.py b/ami/main/models.py index a4c0c82f1..77a0aa5b8 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -42,6 +42,7 @@ build_taxa_recursive_filter_q, ) from ami.main.models_future.projects import ProjectSettingsMixin +from ami.main.schemas import validate_result_data, validate_review_payload from ami.ml.schemas import BoundingBox from ami.users.models import User from ami.utils.media import calculate_file_checksum, extract_timestamp, fetch_image_content @@ -3855,6 +3856,208 @@ class Meta: ] +def occurrence_project_ids(occurrences: typing.Iterable["Occurrence"]) -> dict[int, int | None]: + """The project each occurrence belongs to: its own, or its station's when it has none. + + One query for every occurrence that lacks a project of its own; none when all have one. + The value is None for an occurrence with neither, which callers skip with a warning. + """ + occurrences = list(occurrences) + projects = {occurrence.pk: occurrence.project_id for occurrence in occurrences} + deployment_ids = { + occurrence.deployment_id + for occurrence in occurrences + if occurrence.project_id is None and occurrence.deployment_id is not None + } + if deployment_ids: + station_projects = dict(Deployment.objects.filter(pk__in=deployment_ids).values_list("pk", "project_id")) + for occurrence in occurrences: + if occurrence.project_id is None and occurrence.deployment_id is not None: + projects[occurrence.pk] = station_projects.get(occurrence.deployment_id) + return projects + + +class AlgorithmResultQuerySet(BaseQuerySet): + def for_occurrence(self, occurrence): + return self.filter(occurrence=occurrence) + + def current(self): + return self.filter(is_current=True) + + def bulk_create(self, objs, *args, **kwargs): + objs = list(objs) + for obj in objs: + obj.data = validate_result_data(obj.kind, obj.data) + return super().bulk_create(objs, *args, **kwargs) + + def record(self, **fields) -> "AlgorithmResult": + """Insert one result as the current one for its occurrence, algorithm and kind.""" + results = self.record_many([self.model(**fields)]) + if not results: + raise ValueError("The occurrence has no project, directly or through its station.") + return results[0] + + def record_many(self, results: typing.Iterable["AlgorithmResult"]) -> list["AlgorithmResult"]: + """Insert results as current, keeping each occurrence's earlier result of the same algorithm and kind. + + Rows are only inserted, never rewritten, apart from clearing ``is_current`` on the result + each new one replaces. A result whose occurrence has no project, directly or through + its station, is skipped with a warning rather than failing a run that has already + changed data. The returned list holds the results that were written, in order. + """ + results = list(results) + if not results: + return [] + projects = occurrence_project_ids(result.occurrence for result in results if result.project_id is None) + kept: list[AlgorithmResult] = [] + groups: dict[tuple[int, str], set[int]] = collections.defaultdict(set) + for result in results: + if result.project_id is None: + result.project_id = projects.get(result.occurrence_id) + if result.project_id is None: + logger.warning( + f"Not recording the {result.kind} result for Occurrence #{result.occurrence_id}: " + "it has no project, directly or through its station." + ) + continue + key = (result.algorithm_id, result.kind) + if result.occurrence_id in groups[key]: + raise ValueError( + f"Two {result.kind} results for Occurrence #{result.occurrence_id} " + "from one algorithm in one write." + ) + groups[key].add(result.occurrence_id) + result.is_current = True + kept.append(result) + if not kept: + return [] + with transaction.atomic(): + for (algorithm_id, kind), occurrence_ids in groups.items(): + self.filter( + occurrence_id__in=occurrence_ids, algorithm_id=algorithm_id, kind=kind, is_current=True + ).update(is_current=False) + return self.bulk_create(kept) + + +@final +class AlgorithmResult(BaseModel): + """What a post-processing run decided about one occurrence: a record of the run, never an input to it. + + The determination never reads this table. A run that changes an occurrence's taxon does so + through the ``Classification`` rows it creates, and those rows point back here through + ``Classification.algorithm_result`` so the history can show a run with what it changed. + ``data`` holds the details, validated against the schema for ``kind`` (ami/main/schemas.py). + A new result for the same occurrence, algorithm and kind becomes the current one and the + earlier ones stay as history. Write through ``AlgorithmResult.objects.record`` or + ``record_many``. See #1431. + """ + + class Kind(models.TextChoices): + CLASS_MASKING = "class_masking" + SIZE_FILTER = "size_filter" + + # Copied from the occurrence (or its station) when the result is written, so per-project + # queries and permission checks need no join. + project = models.ForeignKey("main.Project", on_delete=models.CASCADE, related_name="algorithm_results") + # Indexed together with the timestamp, below. + occurrence = models.ForeignKey( + Occurrence, on_delete=models.CASCADE, related_name="algorithm_results", db_index=False + ) + algorithm = models.ForeignKey("ml.Algorithm", on_delete=models.CASCADE, related_name="algorithm_results") + # The run that wrote the result. Deleting the job keeps the result, with no run to link to. + job = models.ForeignKey( + "jobs.Job", on_delete=models.SET_NULL, null=True, blank=True, related_name="algorithm_results" + ) + kind = models.CharField(max_length=32, choices=Kind.choices) + data = models.JSONField(default=dict, blank=True) + is_current = models.BooleanField(default=True) + timestamp = models.DateTimeField(default=timezone.now) + + objects = AlgorithmResultQuerySet.as_manager() + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["occurrence", "algorithm", "kind"], + condition=Q(is_current=True), + name="algorithm_result_current_occurrence", + ), + ] + indexes = [ + models.Index(fields=["occurrence", "-timestamp"], name="algorithm_result_occ_time"), + ] + + def __str__(self) -> str: + return f"#{self.pk} {self.kind} for Occurrence #{self.occurrence_id} from Algorithm #{self.algorithm_id}" + + def save(self, *args, **kwargs): + self.data = validate_result_data(self.kind, self.data) + if self.project_id is None: + self.project_id = occurrence_project_ids([self.occurrence])[self.occurrence_id] + if self.project_id is None: + raise ValueError("The occurrence has no project, directly or through its station.") + super().save(*args, **kwargs) + + +@final +class ValidationReview(BaseModel): + """What a person said about one occurrence: for now a comment; tracking adds its grouping verdict. + + ``aspect`` names what was reviewed and ``verdict`` the person's call on it; a comment has + no verdict. ``reviewed_result`` is the algorithm result the review answers, when there is + one. ``withdrawn`` marks a review taken back without deleting it. See #1431. + """ + + class Aspect(models.TextChoices): + COMMENT = "comment" + + class Verdict(models.TextChoices): + CONFIRMED = "confirmed" + REJECTED = "rejected" + + project = models.ForeignKey("main.Project", on_delete=models.CASCADE, related_name="validation_reviews") + # Indexed together with the timestamp, below. + occurrence = models.ForeignKey( + Occurrence, on_delete=models.CASCADE, related_name="validation_reviews", db_index=False + ) + user = models.ForeignKey( + settings.AUTH_USER_MODEL, on_delete=models.SET_NULL, null=True, blank=True, related_name="validation_reviews" + ) + aspect = models.CharField(max_length=32, choices=Aspect.choices) + verdict = models.CharField(max_length=16, choices=Verdict.choices, null=True, blank=True) + reviewed_result = models.ForeignKey( + AlgorithmResult, on_delete=models.SET_NULL, null=True, blank=True, related_name="reviews" + ) + payload = models.JSONField(default=dict, blank=True) + comment = models.TextField(blank=True, default="") + withdrawn = models.BooleanField(default=False) + timestamp = models.DateTimeField(default=timezone.now) + + class Meta: + constraints = [ + # A comment carries text and no verdict; every other aspect needs a verdict. + models.CheckConstraint( + check=(Q(aspect="comment", verdict__isnull=True) & ~Q(comment="")) + | (~Q(aspect="comment") & Q(verdict__isnull=False)), + name="validation_review_verdict_by_aspect", + ), + ] + indexes = [ + models.Index(fields=["occurrence", "-timestamp"], name="validation_review_occ_time"), + ] + + def __str__(self) -> str: + return f"#{self.pk} {self.aspect} {self.verdict or ''} by User #{self.user_id}".strip() + + def save(self, *args, **kwargs): + self.payload = validate_review_payload(self.aspect, self.payload) + if self.project_id is None: + self.project_id = occurrence_project_ids([self.occurrence])[self.occurrence_id] + if self.project_id is None: + raise ValueError("The occurrence has no project, directly or through its station.") + super().save(*args, **kwargs) + + def update_occurrence_determination( occurrence: Occurrence, current_determination: typing.Optional["Taxon"] = None, save=True ) -> bool: diff --git a/ami/main/schemas.py b/ami/main/schemas.py new file mode 100644 index 000000000..676a26bbb --- /dev/null +++ b/ami/main/schemas.py @@ -0,0 +1,66 @@ +"""Shapes of the JSON on algorithm results and validation reviews, one per kind or aspect. + +``AlgorithmResult.data`` and ``ValidationReview.payload`` are JSON columns, so these schemas keep +each kind's shape fixed: every write validates against the schema registered for it, and a kind +or aspect with no schema cannot be written. +""" + +import pydantic + + +class OutputPayload(pydantic.BaseModel): + class Config: + extra = "forbid" + + +class DeterminationChangeData(OutputPayload): + # The occurrence's determination before and after the run, as taxon ids. Equal when the + # run changed predictions without moving the determination. + taxon_before_id: int | None = None + taxon_after_id: int | None = None + + +class ClassMaskingResultData(DeterminationChangeData): + taxa_list_id: int + source_algorithm_id: int + detection_ids: list[int] + + +class SizeFilterResultData(DeterminationChangeData): + size_threshold: float + detection_ids: list[int] + + +class CommentReviewPayload(OutputPayload): + """A comment's text is in ``ValidationReview.comment``; the payload stays empty.""" + + +ALGORITHM_RESULT_DATA_SCHEMAS: dict[str, type[OutputPayload]] = { + "class_masking": ClassMaskingResultData, + "size_filter": SizeFilterResultData, +} + +VALIDATION_REVIEW_PAYLOAD_SCHEMAS: dict[str, type[OutputPayload]] = { + "comment": CommentReviewPayload, +} + + +def _validate(schemas: dict[str, type[OutputPayload]], name: str, value: dict | OutputPayload) -> dict: + schema = schemas.get(name) + if schema is None: + raise ValueError(f"No payload schema is registered for {name!r}.") + if isinstance(value, OutputPayload) and not isinstance(value, schema): + raise ValueError(f"{type(value).__name__} is not the payload schema for {name!r}.") + data = value.dict() if isinstance(value, OutputPayload) else value + # Round-trip through JSON so datetimes are stored as ISO strings. + return pydantic.parse_raw_as(dict, schema.parse_obj(data).json()) + + +def validate_result_data(kind: str, data: dict | OutputPayload | None) -> dict: + """``data`` as JSON-ready values, or a ValueError when it does not fit the kind's schema.""" + return _validate(ALGORITHM_RESULT_DATA_SCHEMAS, kind, data or {}) + + +def validate_review_payload(aspect: str, payload: dict | OutputPayload | None) -> dict: + """``payload`` as JSON-ready values, or a ValueError when it does not fit the aspect's schema.""" + return _validate(VALIDATION_REVIEW_PAYLOAD_SCHEMAS, aspect, payload or {}) From 11f3a41ac1bc1967bdad3e1995a873f98dfea3a9 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Fri, 2 Oct 2026 19:21:25 -0700 Subject: [PATCH 003/134] feat(jobs): record which job wrote each detection and classification, and the settings it ran with Rows written by a pipeline or a post-processing run now name that run: Detection.job and Classification.job are nullable links the pipeline's save_results, class masking and the size filter fill. Rows that already existed keep the job that first wrote them. Classification.algorithm_result is the post-processing result a row was created for, so the occurrence history can show a run with the classifications it produced; the writers fill it in the next commit. All three links clear themselves when the job or result is deleted, so no job becomes undeletable. The job id is exposed read-only on the detection and classification endpoints, which is what #1156 asked for. A post-processing job also stores its settings as the task's schema validated them, defaults included, so the job records exactly what the run used. The three columns have their own migration, separate from the one that creates the result tables, so the lock on the large detection and classification tables is held only for these quick operations. Closes #1156. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01L52AN9tabp76yjhjyCZkSJ --- ami/jobs/models.py | 7 ++++ ...put_jobs_and_results_on_classifications.py | 26 ++++++++++++++ ami/main/models.py | 14 +++++++- .../tests/test_small_size_filter.py | 34 +++++++++++++++++++ 4 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 ami/main/migrations/0097_output_jobs_and_results_on_classifications.py diff --git a/ami/jobs/models.py b/ami/jobs/models.py index ff65f31f2..cc422a7ed 100644 --- a/ami/jobs/models.py +++ b/ami/jobs/models.py @@ -1,4 +1,5 @@ import datetime +import json import logging import random import time @@ -913,6 +914,12 @@ def run(cls, job: "Job"): raise ValueError(f"Unknown post-processing task '{task_key}'") task = task_cls(job=job, **config) + # Store the config as the task's schema validated it, defaults included, so the job + # records exactly what this run used. The occurrence history shows it. + validated_config = json.loads(task.config.json()) + if validated_config != config: + job.params = {**params, "config": validated_config} + job.save(update_fields=["params", "updated_at"]) task.run() job.progress.update_stage(cls.key, status=JobState.SUCCESS, progress=1) job.finished_at = datetime.datetime.now() diff --git a/ami/main/migrations/0097_output_jobs_and_results_on_classifications.py b/ami/main/migrations/0097_output_jobs_and_results_on_classifications.py new file mode 100644 index 000000000..fa8270eba --- /dev/null +++ b/ami/main/migrations/0097_output_jobs_and_results_on_classifications.py @@ -0,0 +1,26 @@ +# Generated by Django 4.2.10 on 2026-10-02 22:20 +"""The link from a classification to the algorithm result it was created for.""" + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("jobs", "0023_alter_job_job_type_key"), + ("main", "0096_algorithm_results_and_validation_reviews"), + ] + + operations = [ + migrations.AddField( + model_name="classification", + name="algorithm_result", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="classifications", + to="main.algorithmresult", + ), + ), + ] diff --git a/ami/main/models.py b/ami/main/models.py index 77a0aa5b8..b88cc6dd5 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -2964,7 +2964,12 @@ class ClassificationManager(models.Manager.from_queryset(ClassificationQuerySet) @final class Classification(BaseModel): - """The output of a classifier""" + """A taxon classification of one detection: the taxon a classifier or a post-processing run assigned it. + + ``job`` records the run that wrote it, when there was one. ``algorithm_result`` is the + post-processing result the row was created for, so the occurrence history can show a run + with the classifications it produced; classifications from a pipeline leave it empty. + """ project_accessor = "detection__source_image__project" detection = models.ForeignKey( @@ -3006,6 +3011,13 @@ class Classification(BaseModel): related_name="classifications", db_index=False, ) + algorithm_result = models.ForeignKey( + "main.AlgorithmResult", + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="classifications", + ) applied_to = models.ForeignKey( "self", on_delete=models.SET_NULL, diff --git a/ami/ml/post_processing/tests/test_small_size_filter.py b/ami/ml/post_processing/tests/test_small_size_filter.py index 7d96c7af5..8e1472cd3 100644 --- a/ami/ml/post_processing/tests/test_small_size_filter.py +++ b/ami/ml/post_processing/tests/test_small_size_filter.py @@ -4,6 +4,7 @@ from django.test import TestCase +from ami.jobs.models import Job, PostProcessingJob from ami.main.models import Detection, Occurrence, SourceImage, Taxon from ami.ml.post_processing.small_size_filter import SmallSizeFilterTask from ami.tests.fixtures.main import create_captures, create_taxa, setup_test_project @@ -55,3 +56,36 @@ def test_a_detection_above_the_threshold_is_left_alone(self): occurrence.refresh_from_db() self.assertEqual(occurrence.determination, self.taxon) self.assertEqual(occurrence.detections.get().classifications.count(), 1) + + +class SizeFilterJobTestCase(SizeFilterTestCase): + def _job(self, occurrence: Occurrence, **config) -> Job: + return Job.objects.create( + project=self.project, + name="Size filter", + job_type_key="post_processing", + params={"task": "small_size_filter", "config": {"occurrence_id": occurrence.pk, **config}}, + ) + + def test_the_job_stores_the_settings_the_run_used_with_their_defaults(self): + occurrence = self._singleton(self.captures[0], [0, 0, 10, 10]) + job = self._job(occurrence) + + PostProcessingJob.run(job) + + job.refresh_from_db() + self.assertEqual( + job.params["config"], + {"occurrence_id": occurrence.pk, "source_image_collection_id": None, "size_threshold": 0.0008}, + ) + self.assertEqual(job.status, "SUCCESS") + + def test_a_run_records_its_job_on_each_classification_it_writes(self): + occurrence = self._singleton(self.captures[0], [0, 0, 10, 10]) + job = self._job(occurrence, size_threshold=0.01) + + PostProcessingJob.run(job) + + classification = occurrence.detections.get().classifications.get(taxon__name="Not identifiable") + self.assertEqual(classification.job_id, job.pk) + self.assertEqual(job.classifications.get().pk, classification.pk) From 04becb41df634afc27cf11444b25eb6e095cce6b Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Fri, 2 Oct 2026 19:34:18 -0700 Subject: [PATCH 004/134] feat(post-processing): record what class masking and the size filter change, with the batch that changes it Each occurrence a run re-scores or flags now gets one algorithm result: the settings the run used, the detections it changed, and the determination before and after. The result is written inside the same transaction as the batch that changes the occurrence's classifications and determination, so a batch either lands with its results or not at all. A run that fails mid-way can no longer leave changed data behind with no record of the run. The result is created by the first batch that touches an occurrence, before that batch inserts its new classifications, so each classification the run creates points at the result through Classification.algorithm_result. Later batches extend the same result with the detections they changed, so an occurrence that spans batches still has exactly one current result. An occurrence with no project, directly or through its station, is re-scored as before but gets no result, with a warning, rather than failing the run after its data has changed. Part of #1431 and #1457. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01L52AN9tabp76yjhjyCZkSJ --- ami/main/models.py | 7 ++ ami/ml/post_processing/class_masking.py | 17 +++- ami/ml/post_processing/results.py | 90 +++++++++++++++++++ ami/ml/post_processing/small_size_filter.py | 65 +++++++++----- .../tests/test_class_masking.py | 89 ++++++++++++++++++ .../tests/test_small_size_filter.py | 59 +++++++++++- 6 files changed, 303 insertions(+), 24 deletions(-) create mode 100644 ami/ml/post_processing/results.py diff --git a/ami/main/models.py b/ami/main/models.py index b88cc6dd5..8b3ee2304 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -3902,6 +3902,13 @@ def bulk_create(self, objs, *args, **kwargs): obj.data = validate_result_data(obj.kind, obj.data) return super().bulk_create(objs, *args, **kwargs) + def bulk_update(self, objs, fields, *args, **kwargs): + objs = list(objs) + if "data" in fields: + for obj in objs: + obj.data = validate_result_data(obj.kind, obj.data) + return super().bulk_update(objs, fields, *args, **kwargs) + def record(self, **fields) -> "AlgorithmResult": """Insert one result as the current one for its occurrence, algorithm and kind.""" results = self.record_many([self.model(**fields)]) diff --git a/ami/ml/post_processing/class_masking.py b/ami/ml/post_processing/class_masking.py index 18b22c26d..e69e55740 100644 --- a/ami/ml/post_processing/class_masking.py +++ b/ami/ml/post_processing/class_masking.py @@ -8,9 +8,10 @@ from django.db.models import QuerySet from django.utils import timezone -from ami.main.models import Classification, Occurrence, SourceImageCollection, TaxaList +from ami.main.models import AlgorithmResult, Classification, Occurrence, SourceImageCollection, TaxaList from ami.ml.models.algorithm import Algorithm, AlgorithmTaskType from ami.ml.post_processing.base import BasePostProcessingTask +from ami.ml.post_processing.results import BatchResults if typing.TYPE_CHECKING: from ami.jobs.models import Job @@ -81,6 +82,10 @@ def make_classifications_filtered_by_taxa_list( New classifications record ``job`` as the run that wrote them. + Every occurrence with a re-scored classification gets one algorithm result, written in + the same transaction as the batch that changes it, and its new classifications point at + that result (see ``BatchResults``). + Returns final counters (checked / masked / occurrences updated) for stage metrics. """ taxa_in_list = set(taxa_list.taxa.all()) @@ -123,6 +128,13 @@ def make_classifications_filtered_by_taxa_list( timestamp = timezone.now() masked_count = 0 + results = BatchResults( + kind=AlgorithmResult.Kind.CLASS_MASKING, + algorithm=new_algorithm, + job=job, + data={"taxa_list_id": taxa_list.pk, "source_algorithm_id": algorithm.pk}, + timestamp=timestamp, + ) # Sizing the scope and expanding the category map both take time on a large # classifier, so report before touching a row. @@ -198,6 +210,7 @@ def make_classifications_filtered_by_taxa_list( detection = classification.detection if detection is not None and detection.occurrence is not None: occurrences_to_update.add(detection.occurrence) + results.note(detection.occurrence, detection.pk) # Flush every batch_size items and at the final item. The flush fires even # when nothing was accumulated so the job health-check sees a heartbeat during @@ -206,6 +219,7 @@ def make_classifications_filtered_by_taxa_list( with transaction.atomic(): if classifications_to_demote: Classification.objects.bulk_update(classifications_to_demote, ["terminal", "updated_at"]) + results.start_batch(occurrences_to_update, classifications_to_add) if classifications_to_add: Classification.objects.bulk_create(classifications_to_add) # Count an occurrence only when saving its new terminal classification @@ -217,6 +231,7 @@ def make_classifications_filtered_by_taxa_list( occurrence.save(update_determination=True) if occurrence.pk is not None and occurrence.determination_id != prev: changed_occurrence_ids.add(occurrence.pk) + results.finish_batch(occurrences_to_update) classifications_to_demote.clear() classifications_to_add.clear() diff --git a/ami/ml/post_processing/results.py b/ami/ml/post_processing/results.py new file mode 100644 index 000000000..9e87b9be9 --- /dev/null +++ b/ami/ml/post_processing/results.py @@ -0,0 +1,90 @@ +"""One algorithm result per occurrence a post-processing run touches, written inside each batch. + +A run that re-scores or flags detections changes classifications and determinations in batches, +each in its own transaction. ``BatchResults`` writes the result rows in those same transactions, +so a batch either lands with its results or not at all. Every occurrence the run touches gets +exactly one current result: it is created by the first batch that touches the occurrence, before +the classifications that batch inserts so they can point at it, and later batches extend it with +the detections they changed and the determination after their saves. +""" + +from __future__ import annotations + +import collections +import datetime +import typing + +from ami.main.models import AlgorithmResult, Classification, Occurrence + +if typing.TYPE_CHECKING: + from ami.jobs.models import Job + from ami.ml.models import Algorithm + + +class BatchResults: + def __init__( + self, + *, + kind: str, + algorithm: Algorithm, + job: Job | None, + data: dict, + timestamp: datetime.datetime | None = None, + ): + """``data`` holds the fields every result of the run shares, such as the settings it ran with.""" + self.kind = kind + self.algorithm = algorithm + self.job = job + self.data = data + self.timestamp = timestamp + self.results: dict[int, AlgorithmResult] = {} + # Occurrences with no project, directly or through their station; tried once, then left alone. + self.skipped: set[int] = set() + self.changed_detections: dict[int, set[int]] = collections.defaultdict(set) + + def note(self, occurrence: Occurrence, detection_id: int) -> None: + """Record that the run changed one of the occurrence's detections, ahead of the batch write.""" + self.changed_detections[occurrence.pk].add(detection_id) + + def start_batch(self, occurrences: typing.Iterable[Occurrence], classifications: typing.Iterable[Classification]): + """Create the results for occurrences the run has not seen and point the batch's new classifications at theirs. + + Call inside the batch transaction, before the classifications are inserted and before the + occurrences are saved, so a result's ``taxon_before_id`` is the determination the run found. + """ + new = [o for o in occurrences if o.pk not in self.results and o.pk not in self.skipped] + written = AlgorithmResult.objects.record_many( + AlgorithmResult( + occurrence=occurrence, + algorithm=self.algorithm, + job=self.job, + kind=self.kind, + data={ + **self.data, + "detection_ids": [], + "taxon_before_id": occurrence.determination_id, + "taxon_after_id": None, + }, + **({"timestamp": self.timestamp} if self.timestamp else {}), + ) + for occurrence in new + ) + for result in written: + self.results[result.occurrence_id] = result + self.skipped.update(o.pk for o in new if o.pk not in self.results) + for classification in classifications: + classification.algorithm_result = self.results.get(classification.detection.occurrence_id) + + def finish_batch(self, occurrences: typing.Iterable[Occurrence]) -> None: + """Record each occurrence's determination after the batch's saves and the detections changed so far.""" + touched = [] + for occurrence in occurrences: + result = self.results.get(occurrence.pk) + if result is None: + continue + changed = self.changed_detections.pop(occurrence.pk, set()) + result.data["detection_ids"] = sorted(set(result.data["detection_ids"]) | changed) + result.data["taxon_after_id"] = occurrence.determination_id + touched.append(result) + if touched: + AlgorithmResult.objects.bulk_update(touched, ["data"]) diff --git a/ami/ml/post_processing/small_size_filter.py b/ami/ml/post_processing/small_size_filter.py index 51d7ace98..6ccbbee23 100644 --- a/ami/ml/post_processing/small_size_filter.py +++ b/ami/ml/post_processing/small_size_filter.py @@ -1,9 +1,19 @@ import pydantic +from django.db import transaction from django.db.models import QuerySet from django.utils import timezone -from ami.main.models import Classification, Detection, Occurrence, SourceImageCollection, Taxon, TaxonRank +from ami.main.models import ( + AlgorithmResult, + Classification, + Detection, + Occurrence, + SourceImageCollection, + Taxon, + TaxonRank, +) from ami.ml.post_processing.base import BasePostProcessingTask +from ami.ml.post_processing.results import BatchResults from ami.ml.schemas import BoundingBox @@ -90,33 +100,43 @@ def run(self) -> None: updated_occurrence_ids: set[int] = set() modified_occurrences = 0 checked = 0 + # One algorithm result per flagged occurrence, written with the batch that flags it. + results = BatchResults( + kind=AlgorithmResult.Kind.SIZE_FILTER, + algorithm=self.algorithm, + job=self.job, + data={"size_threshold": threshold}, + ) def flush(i: int) -> None: - """Write the rows gathered so far and report progress as of detection ``i``.""" + """Write the rows gathered so far, in one transaction, and report progress as of detection ``i``.""" nonlocal checked, modified_detections, modified_occurrences checked = i modified_detections += len(detections_to_update) - self.logger.info(f"Creating {len(classifications_to_create)} new classifications") - Classification.objects.bulk_create(classifications_to_create) - classifications_to_create.clear() - - self.logger.info(f"Marking {len(detections_to_update)} detections as {not_identifiable_taxon.name}") - for det in detections_to_update: - det.updated_at = timezone.now() - Detection.objects.bulk_update(detections_to_update, ["updated_at"]) - detections_to_update.clear() - - self.logger.info(f"Updating {len(occcurrences_to_update)} occurrences") - for occ in occcurrences_to_update: - # Count an occurrence only when flagging its detection actually - # changes the determination. Re-saving recomputes it in place, so - # an occurrence pinned to a human identification keeps its taxon - # and must not inflate the metric. - prev_determination_id = occ.determination_id - occ.save(update_determination=True) - if occ.pk is not None and occ.determination_id != prev_determination_id: - updated_occurrence_ids.add(occ.pk) + with transaction.atomic(): + results.start_batch(occcurrences_to_update, classifications_to_create) + self.logger.info(f"Creating {len(classifications_to_create)} new classifications") + Classification.objects.bulk_create(classifications_to_create) + classifications_to_create.clear() + + self.logger.info(f"Marking {len(detections_to_update)} detections as {not_identifiable_taxon.name}") + for det in detections_to_update: + det.updated_at = timezone.now() + Detection.objects.bulk_update(detections_to_update, ["updated_at"]) + detections_to_update.clear() + + self.logger.info(f"Updating {len(occcurrences_to_update)} occurrences") + for occ in occcurrences_to_update: + # Count an occurrence only when flagging its detection actually + # changes the determination. Re-saving recomputes it in place, so + # an occurrence pinned to a human identification keeps its taxon + # and must not inflate the metric. + prev_determination_id = occ.determination_id + occ.save(update_determination=True) + if occ.pk is not None and occ.determination_id != prev_determination_id: + updated_occurrence_ids.add(occ.pk) + results.finish_batch(occcurrences_to_update) modified_occurrences = len(updated_occurrence_ids) occcurrences_to_update.clear() @@ -176,6 +196,7 @@ def flush(i: int) -> None: detections_to_update.add(det) if det.occurrence is not None: occcurrences_to_update.add(det.occurrence) + results.note(det.occurrence, det.pk) self.logger.debug(f"Marking detection {det.pk} as {not_identifiable_taxon.name}") flush(i) diff --git a/ami/ml/post_processing/tests/test_class_masking.py b/ami/ml/post_processing/tests/test_class_masking.py index 541a70195..bf8839a86 100644 --- a/ami/ml/post_processing/tests/test_class_masking.py +++ b/ami/ml/post_processing/tests/test_class_masking.py @@ -16,6 +16,7 @@ from django.test import TestCase from ami.main.models import ( + AlgorithmResult, Classification, Detection, Occurrence, @@ -383,6 +384,94 @@ def test_task_run_occurrence_scope(self): self.assertIsNotNone(new_clf) self.assertEqual(new_clf.taxon, self.species_taxa[0]) + # ----- algorithm results ----------------------------------------------- + + def _masking_fixture(self, name: str = "History list") -> tuple[TaxaList, Detection, Occurrence]: + """One occurrence whose top class (index 2) the taxa list excludes, so masking re-scores it.""" + logits = [2.0, 1.0, 5.0] + taxa_list = TaxaList.objects.create(name=name) + taxa_list.taxa.set(self.species_taxa[:2]) + det, occ = self._detection_with_occurrence() + self._create_classification_with_logits(det, self.species_taxa[2], _softmax(logits), logits) + occ.save() + return taxa_list, det, occ + + def test_a_rescored_occurrence_gets_one_result_linked_to_the_classification_it_created(self): + taxa_list, det, occ = self._masking_fixture() + + # The second run finds nothing left to mask, so the first run's result stays the only one. + for _ in range(2): + ClassMaskingTask(occurrence_id=occ.pk, taxa_list_id=taxa_list.pk, algorithm_id=self.algorithm.pk).run() + + result = AlgorithmResult.objects.get(occurrence=occ, kind="class_masking") + self.assertTrue(result.is_current) + self.assertEqual(result.project_id, self.project.pk) + self.assertTrue(result.algorithm.key.startswith(f"{self.algorithm.key}_filtered_by_taxa_list_{taxa_list.pk}")) + self.assertEqual( + result.data, + { + "taxa_list_id": taxa_list.pk, + "source_algorithm_id": self.algorithm.pk, + "detection_ids": [det.pk], + "taxon_before_id": self.species_taxa[2].pk, + "taxon_after_id": self.species_taxa[0].pk, + }, + ) + masked = result.classifications.get() + self.assertEqual((masked.taxon, masked.terminal), (self.species_taxa[0], True)) + self.assertEqual(masked.applied_to.taxon, self.species_taxa[2]) + + def test_an_occurrence_masking_does_not_change_gets_no_result(self): + logits = [2.0, 1.0, 5.0] + taxa_list = TaxaList.objects.create(name="Keeps everything") + taxa_list.taxa.set(self.species_taxa) + det, occ = self._detection_with_occurrence() + self._create_classification_with_logits(det, self.species_taxa[2], _softmax(logits), logits) + + ClassMaskingTask(occurrence_id=occ.pk, taxa_list_id=taxa_list.pk, algorithm_id=self.algorithm.pk).run() + + self.assertFalse(AlgorithmResult.objects.filter(occurrence=occ).exists()) + + def test_an_occurrence_spanning_batches_gets_one_result_covering_every_batch(self): + taxa_list, det1, occ = self._masking_fixture() + det2 = Detection.objects.create(source_image=self.collection.images.first(), bbox=[0, 0, 100, 100]) + occ.detections.add(det2) + logits = [2.0, 1.0, 5.0] + self._create_classification_with_logits(det2, self.species_taxa[2], _softmax(logits), logits) + new_algorithm = Algorithm.objects.create( + name="masked_span", + key="masked_span_test", + task_type=AlgorithmTaskType.CLASSIFICATION.value, + category_map=self.algorithm.category_map, + ) + + make_classifications_filtered_by_taxa_list( + classifications=Classification.objects.filter(detection__in=[det1, det2]), + taxa_list=taxa_list, + algorithm=self.algorithm, + new_algorithm=new_algorithm, + batch_size=1, + ) + + result = AlgorithmResult.objects.get(occurrence=occ) + self.assertEqual(result.data["detection_ids"], sorted([det1.pk, det2.pk])) + self.assertEqual(result.data["taxon_after_id"], self.species_taxa[0].pk) + self.assertEqual(result.classifications.count(), 2) + + def test_an_occurrence_without_a_project_is_still_rescored_but_gets_no_result(self): + """A missing project must not fail a run after it has changed data; the result is skipped with a warning.""" + taxa_list, det, occ = self._masking_fixture() + Occurrence.objects.filter(pk=occ.pk).update(project=None, deployment=None) + + with self.assertLogs("ami.main.models", level="WARNING") as logs: + ClassMaskingTask(occurrence_id=occ.pk, taxa_list_id=taxa_list.pk, algorithm_id=self.algorithm.pk).run() + + self.assertIn(f"Occurrence #{occ.pk}", logs.output[0]) + occ.refresh_from_db() + self.assertEqual(occ.determination, self.species_taxa[0]) + self.assertFalse(AlgorithmResult.objects.filter(occurrence=occ).exists()) + self.assertIsNone(Classification.objects.get(applied_to__detection=det).algorithm_result) + # ----- batched commit + heartbeat ------------------------------------- def test_batched_commit_flushes_multiple_times(self): diff --git a/ami/ml/post_processing/tests/test_small_size_filter.py b/ami/ml/post_processing/tests/test_small_size_filter.py index 8e1472cd3..544ddb5b6 100644 --- a/ami/ml/post_processing/tests/test_small_size_filter.py +++ b/ami/ml/post_processing/tests/test_small_size_filter.py @@ -1,11 +1,12 @@ """What the small size filter writes: a "Not identifiable" classification per flagged detection, in batches.""" import logging +from unittest import mock from django.test import TestCase from ami.jobs.models import Job, PostProcessingJob -from ami.main.models import Detection, Occurrence, SourceImage, Taxon +from ami.main.models import AlgorithmResult, Detection, Occurrence, SourceImage, Taxon from ami.ml.post_processing.small_size_filter import SmallSizeFilterTask from ami.tests.fixtures.main import create_captures, create_taxa, setup_test_project @@ -89,3 +90,59 @@ def test_a_run_records_its_job_on_each_classification_it_writes(self): classification = occurrence.detections.get().classifications.get(taxon__name="Not identifiable") self.assertEqual(classification.job_id, job.pk) self.assertEqual(job.classifications.get().pk, classification.pk) + self.assertEqual(job.algorithm_results.get().pk, classification.algorithm_result_id) + + +class SizeFilterResultsTestCase(SizeFilterTestCase): + """The size filter leaves one algorithm result per occurrence it flags, written with the batch that flags it.""" + + def records(self): + return AlgorithmResult.objects.filter(kind=AlgorithmResult.Kind.SIZE_FILTER).order_by("pk") + + def test_a_flagged_occurrence_gets_one_result_linked_to_the_classifications_it_created(self): + occurrence = self._singleton(self.captures[0], [0, 0, 10, 10]) + Detection.objects.create( + source_image=self.captures[1], + bbox=[0, 0, 12, 12], + timestamp=self.captures[1].timestamp, + occurrence=occurrence, + ) + + self.run_filter(occurrence) + + result = self.records().get() + detection_ids = sorted(occurrence.detections.values_list("pk", flat=True)) + self.assertEqual((result.occurrence_id, result.project_id), (occurrence.pk, self.project.pk)) + self.assertEqual(result.algorithm.key, "small_size_filter") + self.assertEqual( + result.data, + { + "size_threshold": 0.01, + "detection_ids": detection_ids, + "taxon_before_id": self.taxon.pk, + "taxon_after_id": Taxon.objects.get(name="Not identifiable").pk, + }, + ) + self.assertEqual(sorted(result.classifications.values_list("detection_id", flat=True)), detection_ids) + + def test_a_second_run_replaces_the_current_result_and_keeps_the_first(self): + occurrence = self._singleton(self.captures[0], [0, 0, 10, 10]) + for _ in range(2): + self.run_filter(occurrence) + first, second = self.records() + self.assertEqual((first.is_current, second.is_current), (False, True)) + + def test_an_occurrence_with_nothing_flagged_gets_no_result(self): + occurrence = self._singleton(self.captures[0], [0, 0, 500, 500]) + self.run_filter(occurrence) + self.assertFalse(self.records().exists()) + + def test_a_batch_that_fails_leaves_neither_its_classifications_nor_its_result(self): + occurrence = self._singleton(self.captures[0], [0, 0, 10, 10]) + + with mock.patch.object(Occurrence, "save", side_effect=RuntimeError("determination update failed")): + with self.assertRaises(RuntimeError): + self.run_filter(occurrence) + + self.assertFalse(self.records().exists()) + self.assertFalse(occurrence.detections.get().classifications.filter(taxon__name="Not identifiable").exists()) From 781279aecaf28da6eb7837082652ce62d9ecb835 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Fri, 2 Oct 2026 19:34:58 -0700 Subject: [PATCH 005/134] feat(occurrences): show an occurrence's history in one list and let identifiers leave comments GET /occurrences/{id}/history/ returns everything that happened to an occurrence, newest first: post-processing results, comments, identifications and predictions. A result comes with the classifications its run created, matched through Classification.algorithm_result, or through the run's job for rows written before that link existed. Those classifications are shown inside the result rather than as predictions of their own, so a run and what it changed read as one entry. A prediction that a run demoted names the result that superseded it. The query count is fixed, and the score and logit arrays are never loaded. The endpoint is visible to whoever can open the occurrence, and it opens an occurrence the project's default filters hide from lists, since the session view links to those too. POST /occurrences/{id}/reviews/ leaves a comment, for whoever may identify on the project. Comments are the only review written so far. Part of #1431, #1433 and #1457. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01L52AN9tabp76yjhjyCZkSJ --- ami/main/api/serializers.py | 86 +++++ ami/main/api/views.py | 45 +++ ami/main/models.py | 6 + ami/main/models_future/history.py | 203 +++++++++++ ami/main/test_occurrence_history.py | 499 ++++++++++++++++++++++++++++ 5 files changed, 839 insertions(+) create mode 100644 ami/main/models_future/history.py create mode 100644 ami/main/test_occurrence_history.py diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index 1ca588980..445d7d753 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -2065,3 +2065,89 @@ class ModelAgreementSerializer(serializers.Serializer): required=False, help_text="agreed_coarser_rank_count / comparable_count. Null when no threshold supplied.", ) + + +class HistoryUserSerializer(serializers.Serializer): + """A person in an occurrence's history: name and picture only, never an email address.""" + + id = serializers.IntegerField() + name = serializers.CharField() + image = serializers.ImageField(allow_null=True) + + +class HistoryAlgorithmSerializer(serializers.Serializer): + id = serializers.IntegerField() + name = serializers.CharField() + key = serializers.CharField() + + +class HistoryJobSerializer(serializers.Serializer): + id = serializers.IntegerField() + name = serializers.CharField() + config = serializers.SerializerMethodField( + help_text=( + "The settings a post-processing job ran with, as its task validated them. " + "Null for other jobs, and for a post-processing job whose stored config is not an object." + ) + ) + + @extend_schema_field(serializers.JSONField(allow_null=True)) + def get_config(self, job) -> dict | None: + config = (job.params or {}).get("config") + return config if isinstance(config, dict) else None + + +class HistoryTaxonSerializer(serializers.Serializer): + id = serializers.IntegerField() + name = serializers.CharField() + rank = serializers.CharField() + + +class HistoryClassificationSerializer(serializers.Serializer): + """A classification a post-processing run created, shown inside the run's result.""" + + id = serializers.IntegerField() + taxon = HistoryTaxonSerializer() + score = serializers.FloatField(allow_null=True) + terminal = serializers.BooleanField() + detection_id = serializers.IntegerField() + applied_to_id = serializers.IntegerField( + allow_null=True, help_text="The classification this one re-scored and demoted, when there is one." + ) + + +class OccurrenceHistoryEntrySerializer(serializers.Serializer): + """One entry of an occurrence's history, newest first. ``type`` says which table it came from.""" + + type = serializers.ChoiceField(choices=["algorithm_result", "review", "identification", "prediction"]) + id = serializers.IntegerField(help_text="Primary key of the row in the table ``type`` names.") + timestamp = serializers.DateTimeField() + subtype = serializers.CharField( + allow_null=True, + help_text="An algorithm result's kind (class_masking, size_filter) or a review's aspect (comment).", + ) + user = HistoryUserSerializer(allow_null=True) + algorithm = HistoryAlgorithmSerializer(allow_null=True) + job = HistoryJobSerializer(allow_null=True) + taxon = HistoryTaxonSerializer( + allow_null=True, help_text="The identified or predicted taxon, or the determination after a result." + ) + taxon_before = HistoryTaxonSerializer(allow_null=True, help_text="The determination before a result.") + score = serializers.FloatField(allow_null=True, help_text="A prediction's score.") + payload = serializers.JSONField(help_text="Details that depend on the type and subtype.") + classifications = HistoryClassificationSerializer( + many=True, help_text="For a result: the classifications its run created, best score first." + ) + verdict = serializers.CharField(allow_null=True, help_text="A review's verdict: confirmed or rejected.") + comment = serializers.CharField(allow_blank=True) + withdrawn = serializers.BooleanField() + is_current = serializers.BooleanField( + allow_null=True, help_text="For a result: whether it is the latest of its kind, not replaced by a later run." + ) + + +class OccurrenceReviewCreateSerializer(serializers.Serializer): + """A review to add to an occurrence. Only comments are written here.""" + + aspect = serializers.ChoiceField(choices=["comment"], default="comment") + comment = serializers.CharField(max_length=10000, trim_whitespace=True) diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 99709eb29..35dabee04 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -34,6 +34,7 @@ from ami.base.views import ProjectMixin from ami.main.api.schemas import limit_doc_param, project_id_doc_param from ami.main.api.serializers import TagSerializer +from ami.main.models_future.history import add_comment, occurrence_timeline, review_entry from ami.main.models_future.identifications import create_identifications_batch, resolve_occurrences from ami.main.models_future.occurrence import model_agreement_for_project, top_identifiers_for_project from ami.ml.models.algorithm import Algorithm @@ -82,7 +83,9 @@ ExampleOccurrenceSerializer, IdentificationSerializer, ModelAgreementSerializer, + OccurrenceHistoryEntrySerializer, OccurrenceListSerializer, + OccurrenceReviewCreateSerializer, OccurrenceSerializer, PageListSerializer, PageSerializer, @@ -1536,11 +1539,23 @@ def get_serializer_class(self): else: return OccurrenceSerializer + # Actions that read or add to one occurrence's history. They build their own queries and + # never serialize the occurrence, and the default filters, which shape lists, do not hide it. + HISTORY_ACTIONS = ("history", "reviews") + + def get_permissions(self): + # Commenting is open to whoever may identify (see Occurrence.check_custom_permission). + if self.action == "reviews": + return [ObjectPermission()] + return super().get_permissions() + def get_queryset(self) -> QuerySet["Occurrence"]: project = self.get_active_project() qs = super().get_queryset().valid() # type: ignore if project: qs = qs.filter(project=project) + if self.action in self.HISTORY_ACTIONS: + return qs qs = qs.select_related( "determination", "deployment", @@ -1589,6 +1604,36 @@ def get_queryset(self) -> QuerySet["Occurrence"]: def list(self, request, *args, **kwargs): return super().list(request, *args, **kwargs) + @extend_schema(parameters=[project_id_doc_param], responses=OccurrenceHistoryEntrySerializer(many=True)) + @action(detail=True, methods=["get"], name="history", pagination_class=None) + def history(self, request: Request, pk=None) -> Response: + """Everything that happened to this occurrence, newest first. + + Merges post-processing results with the classifications they created, people's + comments, identifications and predictions into one list. Visible to whoever can + open the occurrence itself. + """ + occurrence = self.get_object() + entries = occurrence_timeline(occurrence) + return Response(OccurrenceHistoryEntrySerializer(entries, many=True, context={"request": request}).data) + + @extend_schema( + parameters=[project_id_doc_param], + request=OccurrenceReviewCreateSerializer, + responses={201: OccurrenceHistoryEntrySerializer}, + ) + @action(detail=True, methods=["post"], name="reviews") + def reviews(self, request: Request, pk=None) -> Response: + """Leave a comment on this occurrence. It shows in the occurrence's history.""" + occurrence = self.get_object() + body = OccurrenceReviewCreateSerializer(data=request.data) + body.is_valid(raise_exception=True) + review = add_comment(occurrence, request.user, body.validated_data["comment"]) + return Response( + OccurrenceHistoryEntrySerializer(review_entry(review), context={"request": request}).data, + status=status.HTTP_201_CREATED, + ) + @extend_schema(parameters=[project_id_doc_param], responses=AlgorithmSerializer(many=True)) @action(detail=False, methods=["get"], name="algorithms") def algorithms(self, request: Request) -> Response: diff --git a/ami/main/models.py b/ami/main/models.py index 8b3ee2304..05d48841f 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -3840,6 +3840,12 @@ def save(self, update_determination=True, *args, **kwargs): else: self.save(update_determination=False) + def check_custom_permission(self, user, action: str) -> bool: + # Leaving a comment (the reviews action) takes the rights to identify the occurrence. + if action == "reviews": + return user.has_perm(Project.Permissions.CREATE_IDENTIFICATION, self.get_project()) + return super().check_custom_permission(user, action) + class Meta: ordering = ["-determination_score"] indexes = [ diff --git a/ami/main/models_future/history.py b/ami/main/models_future/history.py new file mode 100644 index 000000000..18780b553 --- /dev/null +++ b/ami/main/models_future/history.py @@ -0,0 +1,203 @@ +"""An occurrence's history: what algorithms and people did to it, newest first. + +What a post-processing run decided about an occurrence is an ``AlgorithmResult`` and what a +person said about it is a ``ValidationReview``. Identifications and predictions keep their own +tables, and ``occurrence_timeline`` merges all four into one list for the history endpoint. +""" + +from __future__ import annotations + +import dataclasses +import datetime +import typing + +from django.db.models import Q + +from ami.main.models import AlgorithmResult, Classification, Identification, Occurrence, Taxon, User, ValidationReview + +if typing.TYPE_CHECKING: + from ami.jobs.models import Job + from ami.ml.models import Algorithm + +COMMENT = ValidationReview.Aspect.COMMENT + + +def add_comment(occurrence: Occurrence, user: User, comment: str) -> ValidationReview: + """Leave a note on an occurrence: a review with ``aspect = comment`` and no verdict.""" + return ValidationReview.objects.create(occurrence=occurrence, user=user, aspect=COMMENT, comment=comment) + + +@dataclasses.dataclass +class TimelineEntry: + """One entry of the merged history, in the shape ``OccurrenceHistoryEntrySerializer`` reads.""" + + type: str + id: int + timestamp: datetime.datetime + subtype: str | None = None + user: User | None = None + algorithm: Algorithm | None = None + job: Job | None = None + taxon: Taxon | None = None + taxon_before: Taxon | None = None + score: float | None = None + payload: dict = dataclasses.field(default_factory=dict) + verdict: str | None = None + comment: str = "" + withdrawn: bool = False + is_current: bool | None = None + # For a result: the classifications the run created, best score first. + classifications: list[Classification] = dataclasses.field(default_factory=list) + + +def review_entry(review: ValidationReview) -> TimelineEntry: + return TimelineEntry( + type="review", + id=review.pk, + timestamp=review.timestamp, + subtype=review.aspect, + user=review.user, + payload=review.payload, + verdict=review.verdict, + comment=review.comment, + withdrawn=review.withdrawn, + ) + + +def occurrence_timeline(occurrence: Occurrence) -> list[TimelineEntry]: + """Results, reviews, identifications and predictions of one occurrence, merged newest first. + + A result comes with the classifications its run created, so the history shows the run and + what it changed as one entry; those classifications are not listed again as predictions. + Every other algorithm contributes one prediction, its best. A prediction a run demoted + names the result that superseded it. The query count does not grow with the entries. + """ + results = list( + AlgorithmResult.objects.filter(occurrence=occurrence) + .select_related("algorithm", "job") + .order_by("-timestamp", "-pk") + ) + taxon_ids = {result.data.get(key) for result in results for key in ("taxon_before_id", "taxon_after_id")} - {None} + taxa = {taxon.pk: taxon for taxon in Taxon.objects.filter(pk__in=taxon_ids)} if taxon_ids else {} + created_by_result = _classifications_created_by(results, occurrence) + created_ids = {c.pk for created in created_by_result.values() for c in created} + superseded_by = { + c.applied_to_id: result_id + for result_id, created in created_by_result.items() + for c in created + if c.applied_to_id is not None + } + + entries = [ + TimelineEntry( + type="algorithm_result", + id=result.pk, + timestamp=result.timestamp, + subtype=result.kind, + algorithm=result.algorithm, + job=result.job, + taxon=taxa.get(result.data.get("taxon_after_id")), + taxon_before=taxa.get(result.data.get("taxon_before_id")), + payload=result.data, + is_current=result.is_current, + classifications=created_by_result.get(result.pk, []), + ) + for result in results + ] + + reviews = ValidationReview.objects.filter(occurrence=occurrence).select_related("user") + entries.extend(review_entry(review) for review in reviews) + + identifications = Identification.objects.filter(occurrence=occurrence).select_related("user", "taxon") + entries.extend( + TimelineEntry( + type="identification", + id=identification.pk, + timestamp=identification.created_at, + user=identification.user, + taxon=identification.taxon, + comment=identification.comment or "", + withdrawn=identification.withdrawn, + payload={ + "comment": identification.comment, + "withdrawn": identification.withdrawn, + "agreed_with_identification_id": identification.agreed_with_identification_id, + "agreed_with_prediction_id": identification.agreed_with_prediction_id, + }, + ) + for identification in identifications + ) + + entries.extend( + TimelineEntry( + type="prediction", + id=prediction.pk, + timestamp=prediction.created_at, + algorithm=prediction.algorithm, + taxon=prediction.taxon, + score=prediction.score, + payload={ + "detection_id": prediction.detection_id, + "terminal": prediction.terminal, + "applied_to_id": prediction.applied_to_id, + "superseded_by_result_id": superseded_by.get(prediction.pk), + }, + ) + for prediction in _one_prediction_per_algorithm(occurrence) + if prediction.pk not in created_ids + ) + + entries.sort(key=lambda entry: (entry.timestamp, entry.id), reverse=True) + return entries + + +def _classifications_created_by( + results: list[AlgorithmResult], occurrence: Occurrence +) -> dict[int, list[Classification]]: + """The classifications each result's run created on the occurrence, keyed by result id. + + A classification names its result directly through ``algorithm_result``. One written by a + run before that link existed is matched through its job instead; when that job left more + than one result on the occurrence, the latest takes it. + """ + if not results: + return {} + result_by_job = {} + for result in reversed(results): + if result.job_id is not None: + result_by_job[result.job_id] = result.pk + created = ( + Classification.objects.filter(detection__occurrence=occurrence) + .filter( + Q(algorithm_result_id__in=[result.pk for result in results]) + | Q(algorithm_result__isnull=True, job_id__in=list(result_by_job)) + ) + .select_related("taxon", "algorithm") + # The score arrays can hold tens of thousands of values per row and are not shown here. + .defer("scores", "logits") + .order_by("-score", "-pk") + ) + by_result: dict[int, list[Classification]] = {} + for classification in created: + result_id = classification.algorithm_result_id or result_by_job[classification.job_id] + by_result.setdefault(result_id, []).append(classification) + return by_result + + +def _one_prediction_per_algorithm(occurrence: Occurrence) -> list[Classification]: + """The best prediction of each algorithm: highest score, then terminal, then latest. + + ``Occurrence.predictions()`` keeps every classification tied for an algorithm's top + score, which would show the same prediction once per detection. + """ + + def rank(prediction: Classification) -> tuple: + score = prediction.score if prediction.score is not None else float("-inf") + return (score, prediction.terminal, prediction.created_at, prediction.pk) + + best: dict[int | None, Classification] = {} + for prediction in occurrence.predictions().defer("scores", "logits"): + current = best.get(prediction.algorithm_id) + if current is None or rank(prediction) > rank(current): + best[prediction.algorithm_id] = prediction + return list(best.values()) diff --git a/ami/main/test_occurrence_history.py b/ami/main/test_occurrence_history.py new file mode 100644 index 000000000..3f24284b1 --- /dev/null +++ b/ami/main/test_occurrence_history.py @@ -0,0 +1,499 @@ +"""Algorithm results, validation reviews, and the occurrence history endpoint that reads them.""" + +import contextlib +import datetime + +from cachalot.api import cachalot_disabled +from django.db import IntegrityError, connection, transaction +from django.test import TestCase +from django.test.utils import CaptureQueriesContext +from rest_framework.test import APITestCase + +from ami.jobs.models import Job +from ami.main.models import ( + AlgorithmResult, + Classification, + Detection, + Identification, + Occurrence, + Project, + SourceImage, + Taxon, + ValidationReview, +) +from ami.ml.models import Algorithm +from ami.tests.fixtures.main import create_captures, create_taxa, setup_test_project +from ami.users.models import User +from ami.users.roles import BasicMember, Identifier, ProjectManager + +# Measured: two savepoints, the project, the occurrence, then the results with their algorithm and +# job, their taxa, the classifications they created, reviews, identifications and predictions. +HISTORY_QUERIES = 10 + +SIZE_FILTER = AlgorithmResult.Kind.SIZE_FILTER +CLASS_MASKING = AlgorithmResult.Kind.CLASS_MASKING + + +@contextlib.contextmanager +def no_query_cache(): + """``cachalot_disabled`` that restores the cache even when the block raises, so one failure stays one.""" + context = cachalot_disabled() + context.__enter__() + try: + yield + finally: + context.__exit__(None, None, None) + + +class AlgorithmResultTestCase(TestCase): + """A result's data fits its kind, its project is its occurrence's, and a new result replaces the current one.""" + + def setUp(self) -> None: + self.project, self.deployment = setup_test_project(reuse=False) + self.occurrence = Occurrence.objects.create(project=self.project, deployment=self.deployment) + self.algorithm = Algorithm.objects.create(name="Size filter", key="size-filter-result-test") + + def _size_filter(self, data: dict | None, **fields) -> AlgorithmResult: + fields.setdefault("occurrence", self.occurrence) + return AlgorithmResult(algorithm=self.algorithm, kind=SIZE_FILTER, data=data, **fields) + + def test_data_that_does_not_fit_its_kind_is_refused_on_every_write_path(self): + for label, data in ( + ("missing field", {"size_threshold": 0.001}), + ("wrong type", {"size_threshold": "small", "detection_ids": []}), + ("unknown field", {"size_threshold": 0.001, "detection_ids": [], "note": "x"}), + ): + with self.subTest(label): + with self.assertRaises(ValueError): + self._size_filter(data).save() + with self.assertRaises(ValueError): + AlgorithmResult.objects.record_many([self._size_filter(data)]) + with self.assertRaises(ValueError): + AlgorithmResult.objects.record( + occurrence=self.occurrence, algorithm=self.algorithm, kind="unregistered", data={} + ) + self.assertFalse(AlgorithmResult.objects.exists()) + + result = AlgorithmResult.objects.record( + occurrence=self.occurrence, + algorithm=self.algorithm, + kind=SIZE_FILTER, + data={"size_threshold": 0.01, "detection_ids": []}, + ) + result.data["size_threshold"] = "small" + with self.assertRaises(ValueError): + AlgorithmResult.objects.bulk_update([result], ["data"]) + + def test_a_new_result_becomes_current_and_the_earlier_one_stays_as_history(self): + first = AlgorithmResult.objects.record( + occurrence=self.occurrence, + algorithm=self.algorithm, + kind=SIZE_FILTER, + data={"size_threshold": 0.01, "detection_ids": [1]}, + ) + other_occurrence = Occurrence.objects.create(project=self.project, deployment=self.deployment) + second, untouched = AlgorithmResult.objects.record_many( + [ + self._size_filter({"size_threshold": 0.02, "detection_ids": [1]}), + self._size_filter({"size_threshold": 0.02, "detection_ids": [2]}, occurrence=other_occurrence), + ] + ) + first.refresh_from_db() + self.assertFalse(first.is_current) + self.assertTrue(second.is_current and untouched.is_current) + self.assertEqual(AlgorithmResult.objects.for_occurrence(self.occurrence).count(), 2) + self.assertEqual(AlgorithmResult.objects.for_occurrence(self.occurrence).current().get().pk, second.pk) + + def test_the_database_holds_one_current_result_per_occurrence_algorithm_and_kind(self): + self._size_filter({"size_threshold": 0.01, "detection_ids": []}).save() + with transaction.atomic(), self.assertRaises(IntegrityError): + self._size_filter({"size_threshold": 0.02, "detection_ids": []}).save() + + def test_project_comes_from_the_occurrence_or_its_station_and_a_result_with_neither_is_skipped(self): + other = Project.objects.create(name="Another project") + Occurrence.objects.filter(pk=self.occurrence.pk).update(project=None) + self.occurrence.refresh_from_db() + result = self._size_filter({"size_threshold": 0.01, "detection_ids": []}) + result.save() + self.assertEqual(result.project_id, self.project.pk) + + Occurrence.objects.filter(pk=self.occurrence.pk).update(project=other) + self.occurrence.refresh_from_db() + self.assertEqual( + AlgorithmResult.objects.record_many([self._size_filter({"size_threshold": 0.02, "detection_ids": []})])[ + 0 + ].project_id, + other.pk, + ) + + Occurrence.objects.filter(pk=self.occurrence.pk).update(project=None, deployment=None) + self.occurrence.refresh_from_db() + with self.assertLogs("ami.main.models", level="WARNING"): + written = AlgorithmResult.objects.record_many( + [self._size_filter({"size_threshold": 0.03, "detection_ids": []})] + ) + self.assertEqual(written, []) + with self.assertRaises(ValueError): + self._size_filter({"size_threshold": 0.03, "detection_ids": []}).save() + self.assertEqual(AlgorithmResult.objects.count(), 2) + + +class ValidationReviewTestCase(TestCase): + def setUp(self) -> None: + self.project, self.deployment = setup_test_project(reuse=False) + self.occurrence = Occurrence.objects.create(project=self.project, deployment=self.deployment) + self.user = User.objects.create_user(email="review-model@insectai.org") # type: ignore + + def test_a_comment_has_text_and_no_verdict(self): + for label, fields in ( + ("comment with a verdict", {"comment": "x", "verdict": "confirmed"}), + ("empty comment", {"comment": ""}), + ): + with self.subTest(label), transaction.atomic(), self.assertRaises(IntegrityError): + ValidationReview.objects.create(occurrence=self.occurrence, user=self.user, aspect="comment", **fields) + with self.assertRaises(ValueError): + ValidationReview.objects.create( + occurrence=self.occurrence, user=self.user, aspect="comment", comment="x", payload={"extra": 1} + ) + + def test_a_review_takes_its_occurrences_project(self): + Occurrence.objects.filter(pk=self.occurrence.pk).update(project=None) + self.occurrence.refresh_from_db() + review = ValidationReview.objects.create( + occurrence=self.occurrence, user=self.user, aspect="comment", comment="x" + ) + self.assertEqual(review.project_id, self.project.pk) + + +class OccurrenceFixtureTestCase(APITestCase): + """One occurrence of four classified detections, with a project manager and a basic member.""" + + def setUp(self) -> None: + self.project, self.deployment = setup_test_project(reuse=False) + create_taxa(project=self.project) + create_captures(deployment=self.deployment, num_nights=1, images_per_night=4, interval_minutes=1) + captures = list(SourceImage.objects.filter(deployment=self.deployment).order_by("timestamp")) + self.taxon = Taxon.objects.filter(projects=self.project).order_by("pk").first() + + self.manager = User.objects.create_user(email="history-manager@insectai.org") # type: ignore + self.reader = User.objects.create_user(email="history-reader@insectai.org") # type: ignore + ProjectManager.assign_user(self.manager, self.project) + BasicMember.assign_user(self.reader, self.project) + + self.occurrence = Occurrence.objects.create( + event=captures[0].event, deployment=self.deployment, project=self.project + ) + self.detections = [] + for capture in captures[:4]: + detection = Detection.objects.create( + source_image=capture, timestamp=capture.timestamp, bbox=[10, 10, 40, 40], occurrence=self.occurrence + ) + detection.classifications.create(taxon=self.taxon, score=0.9, timestamp=capture.timestamp) + self.detections.append(detection) + self.occurrence.save() + return super().setUp() + + +class OccurrenceHistoryEndpointTestCase(OccurrenceFixtureTestCase): + """GET /occurrences/{id}/history/ merges results, reviews, identifications and predictions, newest first.""" + + def setUp(self) -> None: + super().setUp() + self.size_filter = Algorithm.objects.create(name="Small size filter", key="size-filter-history-test") + self.job = Job.objects.create( + project=self.project, + name="Size filter run", + job_type_key="post_processing", + params={"task": "small_size_filter", "config": {"size_threshold": 0.01}}, + ) + self.other_taxon = Taxon.objects.filter(projects=self.project).exclude(pk=self.taxon.pk).first() + self.superuser = User.objects.create_superuser(email="history-super@insectai.org") # type: ignore + self.outsider = User.objects.create_user(email="history-outsider@insectai.org") # type: ignore + + def url(self, occurrence: Occurrence | None = None) -> str: + return f"/api/v2/occurrences/{(occurrence or self.occurrence).pk}/history/?project_id={self.project.pk}" + + def get(self, user: User | None = None): + self.client.force_authenticate(user=user or self.reader) + response = self.client.get(self.url()) + self.assertEqual(response.status_code, 200, response.data) + return response.data + + def _add_history(self, start: datetime.datetime, rounds: int = 1) -> None: + """Per round: a size filter result with the classification it created, an identification and a comment.""" + for i in range(rounds): + at = start + datetime.timedelta(hours=4 * i) + result = AlgorithmResult.objects.record( + occurrence=self.occurrence, + algorithm=self.size_filter, + job=self.job, + kind=SIZE_FILTER, + data={ + "size_threshold": 0.01, + "detection_ids": [self.detections[0].pk], + "taxon_before_id": self.other_taxon.pk, + "taxon_after_id": self.taxon.pk, + }, + timestamp=at, + ) + Classification.objects.create( + detection=self.detections[0], + taxon=self.taxon, + score=0.1, + algorithm=self.size_filter, + timestamp=at, + job=self.job, + algorithm_result=result, + ) + identification = Identification.objects.create( + occurrence=self.occurrence, user=self.reader, taxon=self.other_taxon, comment=f"round {i}" + ) + Identification.objects.filter(pk=identification.pk).update(created_at=at + datetime.timedelta(hours=1)) + ValidationReview.objects.create( + occurrence=self.occurrence, + user=self.manager, + aspect="comment", + comment=f"note {i}", + timestamp=at + datetime.timedelta(hours=2), + ) + + def test_entries_are_merged_newest_first_and_a_result_carries_the_classifications_it_created(self): + # Predictions are stamped with created_at (now), so the history rows sit in the past. + self._add_history(datetime.datetime.now() - datetime.timedelta(days=2)) + data = self.get() + + types = [entry["type"] for entry in data] + self.assertEqual(types[-3:], ["review", "identification", "algorithm_result"]) + # The fixture's detections share one tied top prediction, shown once; the result's + # classification is shown inside the result, not as a prediction of its own. + self.assertEqual(types[:-3], ["prediction"]) + self.assertIsNone(data[0]["algorithm"]) + timestamps = [entry["timestamp"] for entry in data] + self.assertEqual(timestamps, sorted(timestamps, reverse=True)) + + review, identification, result = data[-3:] + self.assertEqual(set(review["user"]), {"id", "name", "image"}) + self.assertEqual(review["user"]["id"], self.manager.pk) + self.assertEqual((review["subtype"], review["comment"], review["verdict"]), ("comment", "note 0", None)) + self.assertEqual(identification["user"]["id"], self.reader.pk) + self.assertEqual(identification["taxon"]["id"], self.other_taxon.pk) + self.assertEqual(identification["payload"]["comment"], "round 0") + self.assertEqual(result["subtype"], "size_filter") + self.assertTrue(result["is_current"]) + self.assertEqual(result["algorithm"]["key"], self.size_filter.key) + self.assertEqual(result["job"], {"id": self.job.pk, "name": self.job.name, "config": {"size_threshold": 0.01}}) + self.assertEqual(result["taxon"]["id"], self.taxon.pk) + self.assertEqual(result["taxon_before"]["id"], self.other_taxon.pk) + created = Classification.objects.get(algorithm=self.size_filter) + self.assertEqual( + result["classifications"], + [ + { + "id": created.pk, + "taxon": {"id": self.taxon.pk, "name": self.taxon.name, "rank": self.taxon.rank}, + "score": 0.1, + "terminal": True, + "detection_id": self.detections[0].pk, + "applied_to_id": None, + } + ], + ) + self.assertNotIn("email", str(data)) + + def test_a_demoted_prediction_names_the_result_that_superseded_it(self): + """Class masking demotes the original classification and links its replacement to the result.""" + classifier = Algorithm.objects.create(name="Classifier", key="classifier-history-test") + masker = Algorithm.objects.create(name="Masked classifier", key="masker-history-test") + original = Classification.objects.get(detection=self.detections[1]) + Classification.objects.filter(pk=original.pk).update(terminal=False, algorithm=classifier) + result = AlgorithmResult.objects.record( + occurrence=self.occurrence, + algorithm=masker, + job=self.job, + kind=CLASS_MASKING, + data={ + "taxa_list_id": 1, + "source_algorithm_id": 1, + "detection_ids": [self.detections[1].pk], + "taxon_before_id": self.taxon.pk, + "taxon_after_id": self.other_taxon.pk, + }, + ) + masked = Classification.objects.create( + detection=self.detections[1], + taxon=self.other_taxon, + score=0.95, + algorithm=masker, + timestamp=datetime.datetime.now(), + applied_to=original, + job=self.job, + algorithm_result=result, + ) + data = self.get() + + result_entry = next(entry for entry in data if entry["type"] == "algorithm_result") + self.assertEqual([c["id"] for c in result_entry["classifications"]], [masked.pk]) + # The masked classification shows inside the result, not as a prediction; the original + # still has its card, marked as superseded by the run. + predictions = { + entry["algorithm"]["key"] if entry["algorithm"] else None: entry + for entry in data + if entry["type"] == "prediction" + } + self.assertEqual(set(predictions), {None, classifier.key}) + demoted = predictions[classifier.key] + self.assertEqual(demoted["id"], original.pk) + self.assertEqual(demoted["payload"]["superseded_by_result_id"], result.pk) + self.assertFalse(demoted["payload"]["terminal"]) + self.assertIsNone(predictions[None]["payload"]["superseded_by_result_id"]) + + def test_a_classification_without_a_result_link_is_matched_to_its_jobs_result(self): + result = AlgorithmResult.objects.record( + occurrence=self.occurrence, + algorithm=self.size_filter, + job=self.job, + kind=SIZE_FILTER, + data={"size_threshold": 0.01, "detection_ids": [self.detections[0].pk]}, + ) + created = Classification.objects.create( + detection=self.detections[0], + taxon=self.taxon, + score=0.1, + algorithm=self.size_filter, + timestamp=datetime.datetime.now(), + job=self.job, + ) + data = self.get() + + result_entry = next(entry for entry in data if entry["type"] == "algorithm_result") + self.assertEqual(result_entry["id"], result.pk) + self.assertEqual([c["id"] for c in result_entry["classifications"]], [created.pk]) + predictions = [e for e in data if e["type"] == "prediction"] + self.assertEqual([e["algorithm"] for e in predictions], [None]) + self.assertNotEqual(predictions[0]["id"], created.pk) + + def test_each_algorithm_shows_one_prediction_preferring_terminal_then_latest(self): + """Tied top scores would otherwise show the same prediction once per detection.""" + classifier = Algorithm.objects.create(name="Tied classifier", key="tied-classifier-history-test") + other = Algorithm.objects.create(name="Second tied classifier", key="second-tied-classifier-history-test") + now = datetime.datetime.now() + + def classify(detection, algorithm, terminal, created_at): + classification = Classification.objects.create( + detection=detection, taxon=self.taxon, score=0.8, algorithm=algorithm, terminal=terminal, timestamp=now + ) + Classification.objects.filter(pk=classification.pk).update(created_at=created_at) + return classification + + classify(self.detections[0], classifier, False, now) + terminal = classify(self.detections[1], classifier, True, now - datetime.timedelta(hours=1)) + classify(self.detections[2], other, True, now - datetime.timedelta(hours=1)) + latest = classify(self.detections[3], other, True, now) + + predictions = [entry for entry in self.get() if entry["type"] == "prediction"] + by_algorithm = { + entry["algorithm"]["key"] if entry["algorithm"] else None: entry["id"] for entry in predictions + } + self.assertEqual(len(predictions), 3) + self.assertEqual(by_algorithm[classifier.key], terminal.pk) + self.assertEqual(by_algorithm[other.key], latest.pk) + self.assertIn(None, by_algorithm) + + def test_query_count_does_not_grow_with_the_entries_and_no_score_arrays_are_loaded(self): + self.client.force_authenticate(user=self.reader) + now = datetime.datetime.now() + for rounds, total in ((1, 1), (2, 3)): + self._add_history(now - datetime.timedelta(days=3 * total), rounds=rounds) + with self.subTest(rounds=total), no_query_cache(): + with CaptureQueriesContext(connection) as queries, self.assertNumQueries(HISTORY_QUERIES): + response = self.client.get(self.url()) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 3 * total + 1) + self.assertFalse( + [q["sql"] for q in queries.captured_queries if '"main_classification"."scores"' in q["sql"]] + ) + + def test_visible_to_whoever_can_open_the_occurrence(self): + """Member, non-member, anonymous and superuser, on a public and on a draft project.""" + expected = { + False: {"member": 200, "non-member": 200, "anonymous": 200, "superuser": 200}, + True: {"member": 200, "non-member": 404, "anonymous": 404, "superuser": 200}, + } + users = {"member": self.reader, "non-member": self.outsider, "anonymous": None, "superuser": self.superuser} + for draft, codes in expected.items(): + self.project.draft = draft + self.project.save() + for label, user in users.items(): + with self.subTest(draft=draft, user=label): + self.client.force_authenticate(user=user) + detail = self.client.get(f"/api/v2/occurrences/{self.occurrence.pk}/?project_id={self.project.pk}") + history = self.client.get(self.url()) + self.assertEqual(history.status_code, codes[label]) + self.assertEqual(history.status_code, detail.status_code) + + def test_an_occurrence_the_default_filters_hide_still_has_a_history(self): + """The session view lists occurrences with the default filters off, so their history must open too.""" + self.project.default_filters_score_threshold = 0.95 + self.project.save() + self.client.force_authenticate(user=self.reader) + detail = self.client.get(f"/api/v2/occurrences/{self.occurrence.pk}/?project_id={self.project.pk}") + self.assertEqual(detail.status_code, 404) + self.assertEqual(self.client.get(self.url()).status_code, 200) + + +class OccurrenceCommentEndpointTestCase(OccurrenceFixtureTestCase): + """POST /occurrences/{id}/reviews/ leaves a comment, for whoever may identify on the project.""" + + def setUp(self) -> None: + super().setUp() + self.identifier = User.objects.create_user(email="comment-identifier@insectai.org") # type: ignore + Identifier.assign_user(self.identifier, self.project) + self.superuser = User.objects.create_superuser(email="comment-super@insectai.org") # type: ignore + self.outsider = User.objects.create_user(email="comment-outsider@insectai.org") # type: ignore + + def comment(self, user: User | None, body: dict | None = None): + self.client.force_authenticate(user=user) + return self.client.post( + f"/api/v2/occurrences/{self.occurrence.pk}/reviews/?project_id={self.project.pk}", + body if body is not None else {"comment": " The second detection is a different moth. "}, + format="json", + ) + + def test_a_comment_is_stored_as_a_review_and_returned_as_a_history_entry(self): + response = self.comment(self.identifier) + self.assertEqual(response.status_code, 201, response.data) + review = ValidationReview.objects.get() + self.assertEqual((review.aspect, review.verdict, review.user), ("comment", None, self.identifier)) + self.assertEqual(review.comment, "The second detection is a different moth.") + self.assertEqual(review.project_id, self.project.pk) + self.assertEqual((response.data["type"], response.data["id"]), ("review", review.pk)) + self.assertNotIn("email", str(response.data)) + + def test_an_empty_comment_or_another_aspect_is_refused(self): + for body in ({"comment": " "}, {"aspect": "grouping", "comment": "x"}): + with self.subTest(body=body): + self.assertEqual(self.comment(self.identifier, body).status_code, 400) + self.assertFalse(ValidationReview.objects.exists()) + + def test_permission_matrix(self): + """Whoever may identify may comment; a basic member, a non-member and anonymous may not.""" + expected = { + "identifier": 201, + "project manager": 201, + "superuser": 201, + "basic member": 403, + "non-member": 403, + "anonymous": 401, + } + users = { + "identifier": self.identifier, + "project manager": self.manager, + "superuser": self.superuser, + "basic member": self.reader, + "non-member": self.outsider, + "anonymous": None, + } + for label, user in users.items(): + with self.subTest(user=label): + self.assertEqual(self.comment(user).status_code, expected[label]) + self.assertEqual(ValidationReview.objects.count(), 3) From 804fe3a1331ee849341cfb6610241207816f3c27 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Fri, 2 Oct 2026 19:35:00 -0700 Subject: [PATCH 006/134] feat(ui): show post-processing results and comments in the occurrence's identification timeline The Identification tab of an occurrence now reads the history endpoint and shows one list, newest first: a card for each post-processing result (class masking or the size filter), a card for each comment, and the identification and prediction cards as before. Until the history loads, or if it cannot be loaded, the identifications and predictions the occurrence already carries stand in, so the tab never goes blank. A result card shows the prediction the run made, which can be agreed with, then the determination before and after, the species list or size threshold the run used, how many detections it changed, and a link to the job with the settings it ran with. A prediction a run demoted keeps its own card, labelled as superseded by that run, so a reader can tell why it no longer counts. Part of #1433. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01L52AN9tabp76yjhjyCZkSJ --- .../hooks/occurrences/useOccurrenceHistory.ts | 33 ++ .../models/occurrence-history.test.ts | 355 ++++++++++++++++ .../models/occurrence-history.ts | 394 ++++++++++++++++++ .../identifications/identification-card.tsx | 26 +- .../identification-card/algorithm-result.tsx | 203 +++++++++ .../identification-card/comment-review.tsx | 44 ++ .../identification-card/history-stats.tsx | 30 ++ .../machine-prediction.tsx | 8 +- .../occurrence-timeline.tsx | 114 +++++ .../occurrence-details/occurrence-details.tsx | 33 +- ui/src/utils/language.ts | 39 ++ ui/src/utils/user/getUserLabel.test.ts | 30 ++ ui/src/utils/user/getUserLabel.ts | 24 ++ 13 files changed, 1301 insertions(+), 32 deletions(-) create mode 100644 ui/src/data-services/hooks/occurrences/useOccurrenceHistory.ts create mode 100644 ui/src/data-services/models/occurrence-history.test.ts create mode 100644 ui/src/data-services/models/occurrence-history.ts create mode 100644 ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx create mode 100644 ui/src/pages/occurrence-details/identification-card/comment-review.tsx create mode 100644 ui/src/pages/occurrence-details/identification-card/history-stats.tsx create mode 100644 ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx create mode 100644 ui/src/utils/user/getUserLabel.test.ts create mode 100644 ui/src/utils/user/getUserLabel.ts diff --git a/ui/src/data-services/hooks/occurrences/useOccurrenceHistory.ts b/ui/src/data-services/hooks/occurrences/useOccurrenceHistory.ts new file mode 100644 index 000000000..9e3a2cdc7 --- /dev/null +++ b/ui/src/data-services/hooks/occurrences/useOccurrenceHistory.ts @@ -0,0 +1,33 @@ +import { API_ROUTES, API_URL } from 'data-services/constants' +import { ServerOccurrenceHistoryEntry } from 'data-services/models/occurrence-history' +import { useAuthorizedQuery } from '../auth/useAuthorizedQuery' + +// Under the occurrences prefix, so every mutation that invalidates occurrences refreshes it. +export const getOccurrenceHistoryQueryKey = (occurrenceId: string) => [ + API_ROUTES.OCCURRENCES, + occurrenceId, + 'history', +] + +export const useOccurrenceHistory = ({ + occurrenceId, + projectId, +}: { + occurrenceId: string + projectId?: string +}): { + entries?: ServerOccurrenceHistoryEntry[] + isLoading: boolean + error?: unknown +} => { + const params = new URLSearchParams(projectId ? { project_id: projectId } : {}) + const { data, isLoading, error } = useAuthorizedQuery< + ServerOccurrenceHistoryEntry[] + >({ + enabled: !!occurrenceId, + queryKey: getOccurrenceHistoryQueryKey(occurrenceId), + url: `${API_URL}/${API_ROUTES.OCCURRENCES}/${occurrenceId}/history/?${params}`, + }) + + return { entries: data, isLoading, error } +} diff --git a/ui/src/data-services/models/occurrence-history.test.ts b/ui/src/data-services/models/occurrence-history.test.ts new file mode 100644 index 000000000..8a367debe --- /dev/null +++ b/ui/src/data-services/models/occurrence-history.test.ts @@ -0,0 +1,355 @@ +import { getOccurrenceHistoryQueryKey } from 'data-services/hooks/occurrences/useOccurrenceHistory' +import { API_ROUTES } from 'data-services/constants' +import { Algorithm } from './algorithm' +import { HumanIdentification, MachinePrediction } from './occurrence-details' +import { + getFallbackTimelineItems, + getJobConfigSummary, + getResultPrediction, + getTimelineItems, + ServerOccurrenceHistoryEntry, +} from './occurrence-history' +import { Taxon } from './taxa' +import { UserPermission } from 'utils/user/types' + +const NOCTUA = { id: 3, name: 'Noctua pronuba', rank: 'SPECIES' } +const XESTIA = { id: 4, name: 'Xestia c-nigrum', rank: 'SPECIES' } + +const base = { + algorithm: null, + classifications: [], + comment: '', + is_current: null, + job: null, + score: null, + subtype: null, + taxon: null, + taxon_before: null, + timestamp: '2026-04-29T22:00:00', + user: null, + verdict: null, + withdrawn: false, +} + +const identificationEntry = (id: number): ServerOccurrenceHistoryEntry => ({ + ...base, + id, + payload: { + agreed_with_identification_id: null, + agreed_with_prediction_id: null, + comment: 'Looks right', + withdrawn: false, + }, + taxon: NOCTUA, + type: 'identification', + user: { id: 9, image: null, name: '' }, +}) + +const predictionEntry = ( + id: number, + taxon: typeof NOCTUA | null = XESTIA, + supersededByResultId: number | null = null +): ServerOccurrenceHistoryEntry => ({ + ...base, + algorithm: { id: 7, key: 'classifier', name: 'Classifier' }, + id, + payload: { + applied_to_id: null, + detection_id: 1, + superseded_by_result_id: supersededByResultId, + terminal: true, + }, + score: 0.8, + taxon, + type: 'prediction', +}) + +const comment = (id: number): ServerOccurrenceHistoryEntry => ({ + ...base, + comment: 'The second detection is another moth', + id, + payload: {}, + subtype: 'comment', + type: 'review', +}) + +const classMasking: ServerOccurrenceHistoryEntry = { + ...base, + algorithm: { id: 12, key: 'mask', name: 'Masked classifier' }, + classifications: [ + { + applied_to_id: 6, + detection_id: 1, + id: 20, + score: 0.7, + taxon: NOCTUA, + terminal: true, + }, + { + applied_to_id: 7, + detection_id: 2, + id: 21, + score: 0.4, + taxon: XESTIA, + terminal: true, + }, + ], + id: 5, + is_current: true, + payload: { + detection_ids: [1, 2], + source_algorithm_id: 7, + taxa_list_id: 2, + taxon_after_id: 3, + taxon_before_id: 4, + }, + subtype: 'class_masking', + taxon: NOCTUA, + taxon_before: XESTIA, + type: 'algorithm_result', +} + +const ownIdentification: HumanIdentification = { + comment: '', + createdAt: '2026-04-29T22:00:00', + id: '1', + taxon: new Taxon({ ...NOCTUA, id: `${NOCTUA.id}`, cover_image_url: null }), + user: { name: 'Someone' }, + userPermissions: [UserPermission.Delete], +} + +const ownPrediction = ( + id: string, + algorithmId: number, + taxon = NOCTUA, + createdAt = '2026-04-29T21:00:00' +): MachinePrediction => ({ + algorithm: new Algorithm({ id: algorithmId }), + createdAt, + id, + score: 0.9, + taxon: new Taxon({ ...taxon, id: `${taxon.id}`, cover_image_url: null }), + terminal: true, + userPermissions: [UserPermission.Update], +}) + +describe('getTimelineItems', () => { + test('maps each entry type to its card, keeping the server order', () => { + const items = getTimelineItems({ + entries: [ + comment(9), + classMasking, + identificationEntry(2), + predictionEntry(6), + ], + identifications: [], + predictions: [], + }) + + expect(items.map((item) => item.type)).toEqual([ + 'review', + 'algorithm_result', + 'identification', + 'prediction', + ]) + }) + + test('leaves out a review aspect it has no card for', () => { + const items = getTimelineItems({ + entries: [{ ...comment(8), subtype: 'grouping' } as never], + identifications: [], + predictions: [], + }) + + expect(items).toEqual([]) + }) + + test("reuses the occurrence's own records, which carry the viewer's permissions", () => { + const prediction = ownPrediction('6', 7) + const items = getTimelineItems({ + entries: [identificationEntry(1), predictionEntry(6)], + identifications: [ownIdentification], + predictions: [prediction], + }) + + expect(items[0]).toMatchObject({ identification: ownIdentification }) + expect(items[1]).toMatchObject({ prediction }) + }) + + test('builds a read-only card when the occurrence lacks the record', () => { + const [item] = getTimelineItems({ + determinationTaxonId: '3', + entries: [identificationEntry(2)], + identifications: [], + predictions: [], + }) + + expect(item.type === 'identification' && item.identification).toMatchObject( + { + applied: true, + comment: 'Looks right', + user: { id: '9', name: 'Unnamed user' }, + userPermissions: [], + } + ) + }) + + test('names the result that superseded a demoted prediction', () => { + const items = getTimelineItems({ + entries: [classMasking, predictionEntry(6, XESTIA, 5), predictionEntry(8)], + identifications: [], + predictions: [], + }) + + expect(items[1]).toMatchObject({ supersededBy: classMasking }) + expect(items[2]).not.toHaveProperty('supersededBy', expect.anything()) + }) + + test('drops unknown subtypes and predictions without a taxon', () => { + const unknown = { + ...classMasking, + subtype: 'something_new', + } as unknown as ServerOccurrenceHistoryEntry + + expect( + getTimelineItems({ + entries: [unknown, predictionEntry(6, null)], + identifications: [], + predictions: [], + }) + ).toEqual([]) + }) +}) + +describe('cards built from the history', () => { + test('are left out when a prediction names no algorithm or an identification no taxon', () => { + expect( + getTimelineItems({ + entries: [ + { ...predictionEntry(6), algorithm: null }, + { ...identificationEntry(2), taxon: null }, + ], + identifications: [], + predictions: [], + }) + ).toEqual([]) + }) +}) + +describe('getFallbackTimelineItems', () => { + test('merges identifications and predictions newest first', () => { + const items = getFallbackTimelineItems({ + identifications: [ownIdentification], + predictions: [ + ownPrediction('6', 7, NOCTUA, '2026-04-29T23:00:00'), + ownPrediction('7', 12, NOCTUA, '2026-04-29T20:00:00'), + ], + }) + + expect(items.map((item) => item.id)).toEqual([ + 'prediction-6', + 'identification-1', + 'prediction-7', + ]) + }) + + test('breaks timestamp ties on id, newest first, as the server does', () => { + const items = getFallbackTimelineItems({ + identifications: [ownIdentification], + predictions: [ownPrediction('6', 7, NOCTUA, ownIdentification.createdAt)], + }) + + expect(items.map((item) => item.id)).toEqual([ + 'prediction-6', + 'identification-1', + ]) + }) +}) + +describe('fallback predictions', () => { + const tied = (id: string, terminal: boolean, createdAt: string) => ({ + ...ownPrediction(id, 7, NOCTUA, createdAt), + terminal, + }) + + test('keep one per algorithm when detections tie for its top score', () => { + const items = getFallbackTimelineItems({ + identifications: [], + predictions: [ + tied('6', true, '2026-04-29T21:00:00'), + tied('7', true, '2026-04-29T22:00:00'), + ownPrediction('8', 12), + ], + }) + + expect(items.map((item) => item.id)).toEqual([ + 'prediction-7', + 'prediction-8', + ]) + }) + + test('prefer a terminal prediction over a later intermediate one', () => { + const items = getFallbackTimelineItems({ + identifications: [], + predictions: [ + tied('6', true, '2026-04-29T21:00:00'), + tied('7', false, '2026-04-29T22:00:00'), + ], + }) + + expect(items.map((item) => item.id)).toEqual(['prediction-6']) + }) +}) + +describe('getResultPrediction', () => { + test('is the best classification the run created, marked applied when it is the determination', () => { + expect(getResultPrediction(classMasking as never, '3')).toMatchObject({ + applied: true, + count: 2, + id: '20', + score: 0.7, + taxon: { id: '3', name: 'Noctua pronuba' }, + }) + expect(getResultPrediction(classMasking as never, '4')).toMatchObject({ + applied: false, + }) + }) + + test('is undefined for a run that created no classification', () => { + expect( + getResultPrediction( + { ...classMasking, classifications: [] } as never, + '3' + ) + ).toBeUndefined() + }) +}) + +describe('getOccurrenceHistoryQueryKey', () => { + test('sits under the occurrences key that mutations invalidate', () => { + expect(getOccurrenceHistoryQueryKey('100')[0]).toBe(API_ROUTES.OCCURRENCES) + }) +}) + +describe('getJobConfigSummary', () => { + test('lists the settings a job ran with, leaving out unset ones', () => { + expect( + getJobConfigSummary({ + config: { + occurrence_id: 4, + size_threshold: 0.01, + source_image_collection_id: null, + }, + id: 1, + name: 'Size filter', + }) + ).toBe('occurrence_id: 4, size_threshold: 0.01') + }) + + test('is empty for a job without settings', () => { + expect(getJobConfigSummary({ config: null, id: 1, name: 'Pipeline' })).toBe( + '' + ) + expect(getJobConfigSummary(null)).toBe('') + }) +}) diff --git a/ui/src/data-services/models/occurrence-history.ts b/ui/src/data-services/models/occurrence-history.ts new file mode 100644 index 000000000..70f02a500 --- /dev/null +++ b/ui/src/data-services/models/occurrence-history.ts @@ -0,0 +1,394 @@ +import { getUserLabel } from 'utils/user/getUserLabel' +import { Algorithm } from './algorithm' +import { HumanIdentification, MachinePrediction } from './occurrence-details' +import { Taxon } from './taxa' + +export interface ServerHistoryTaxon { + id: number + name: string + rank: string +} + +export interface ServerHistoryUser { + id: number + image: string | null + name: string +} + +export interface ServerHistoryAlgorithm { + id: number + key: string + name: string +} + +export interface ServerHistoryJob { + /** The settings a post-processing job ran with; null for other jobs. */ + config: Record | null + id: number + name: string +} + +/** A classification a post-processing run created, listed inside the run's result. */ +export interface ServerHistoryClassification { + applied_to_id: number | null + detection_id: number + id: number + score: number | null + taxon: ServerHistoryTaxon + terminal: boolean +} + +interface ServerDeterminationChangePayload { + taxon_after_id: number | null + taxon_before_id: number | null +} + +export interface ServerClassMaskingPayload + extends ServerDeterminationChangePayload { + detection_ids: number[] + source_algorithm_id: number + taxa_list_id: number +} + +export interface ServerSizeFilterPayload + extends ServerDeterminationChangePayload { + detection_ids: number[] + size_threshold: number +} + +export interface ServerIdentificationPayload { + agreed_with_identification_id: number | null + agreed_with_prediction_id: number | null + comment: string + withdrawn: boolean +} + +export interface ServerPredictionPayload { + applied_to_id: number | null + detection_id: number | null + /** The result of the run that re-scored and demoted this prediction, when one did. */ + superseded_by_result_id: number | null + terminal: boolean | null +} + +export type ReviewVerdict = 'confirmed' | 'rejected' + +interface ServerHistoryEntryBase { + algorithm: ServerHistoryAlgorithm | null + /** For a result: the classifications its run created, best score first. */ + classifications: ServerHistoryClassification[] + comment: string + id: number + /** For a result: whether it is the latest of its kind, not replaced by a later run. */ + is_current: boolean | null + job: ServerHistoryJob | null + payload: Payload + score: number | null + subtype: Subtype + taxon: ServerHistoryTaxon | null + taxon_before: ServerHistoryTaxon | null + timestamp: string + type: Type + user: ServerHistoryUser | null + verdict: ReviewVerdict | null + withdrawn: boolean +} + +export type ClassMaskingResultEntry = ServerHistoryEntryBase< + 'algorithm_result', + 'class_masking', + ServerClassMaskingPayload +> +export type SizeFilterResultEntry = ServerHistoryEntryBase< + 'algorithm_result', + 'size_filter', + ServerSizeFilterPayload +> +export type AlgorithmResultEntry = + | ClassMaskingResultEntry + | SizeFilterResultEntry +export type CommentReviewEntry = ServerHistoryEntryBase< + 'review', + 'comment', + Record +> +export type ReviewEntry = CommentReviewEntry +export type IdentificationEntry = ServerHistoryEntryBase< + 'identification', + null, + ServerIdentificationPayload +> +export type PredictionEntry = ServerHistoryEntryBase< + 'prediction', + null, + ServerPredictionPayload +> + +export type ServerOccurrenceHistoryEntry = + | AlgorithmResultEntry + | ReviewEntry + | IdentificationEntry + | PredictionEntry + +export type TimelineItem = + | { + type: 'identification' + id: string + identification: HumanIdentification + } + | { + type: 'prediction' + id: string + prediction: MachinePrediction + /** The result of the run that demoted this prediction, when one did. */ + supersededBy?: AlgorithmResultEntry + } + | { type: 'algorithm_result'; id: string; entry: AlgorithmResultEntry } + | { type: 'review'; id: string; entry: ReviewEntry } + +const ALGORITHM_RESULT_SUBTYPES: string[] = ['class_masking', 'size_filter'] +const REVIEW_SUBTYPES: string[] = ['comment'] + +export const convertHistoryTaxon = (taxon: ServerHistoryTaxon) => + new Taxon({ ...taxon, id: `${taxon.id}`, cover_image_url: null }) + +const toIdentification = ( + entry: IdentificationEntry & { taxon: ServerHistoryTaxon }, + determinationTaxonId?: string +): HumanIdentification => { + const taxon = convertHistoryTaxon(entry.taxon) + + return { + applied: taxon.id === determinationTaxonId, + comment: entry.payload.comment, + createdAt: entry.timestamp, + id: `${entry.id}`, + overridden: entry.payload.withdrawn, + taxon, + user: entry.user + ? { + id: `${entry.user.id}`, + image: entry.user.image ?? undefined, + name: getUserLabel(entry.user), + } + : { name: getUserLabel(entry.user) }, + userPermissions: [], + } +} + +const toPrediction = ( + entry: PredictionEntry & { + algorithm: ServerHistoryAlgorithm + taxon: ServerHistoryTaxon + }, + determinationTaxonId?: string +): MachinePrediction => { + const taxon = convertHistoryTaxon(entry.taxon) + + return { + algorithm: new Algorithm(entry.algorithm), + applied: taxon.id === determinationTaxonId, + createdAt: entry.timestamp, + id: `${entry.id}`, + overridden: taxon.id !== determinationTaxonId, + score: entry.score ?? 0, + taxon, + terminal: !!entry.payload.terminal, + userPermissions: [], + } +} + +const isAlgorithmResult = ( + entry: ServerOccurrenceHistoryEntry +): entry is AlgorithmResultEntry => + entry.type === 'algorithm_result' && + ALGORITHM_RESULT_SUBTYPES.includes(entry.subtype) + +/** + * The history as cards to render, newest first. Identifications and predictions reuse the + * occurrence's own records when it has them, since only those carry the viewer's permissions. + */ +export const getTimelineItems = ({ + determinationTaxonId, + entries, + identifications, + predictions, +}: { + determinationTaxonId?: string + entries: ServerOccurrenceHistoryEntry[] + identifications: HumanIdentification[] + predictions: MachinePrediction[] +}): TimelineItem[] => { + const results = entries.filter(isAlgorithmResult) + + return entries.flatMap((entry): TimelineItem[] => { + const id = `${entry.type}-${entry.id}` + + switch (entry.type) { + case 'identification': { + const { taxon } = entry + const identification = + identifications.find((i) => i.id === `${entry.id}`) ?? + (taxon + ? toIdentification({ ...entry, taxon }, determinationTaxonId) + : undefined) + + return identification + ? [{ type: 'identification', id, identification }] + : [] + } + case 'prediction': { + const { algorithm, taxon } = entry + const prediction = + predictions.find((p) => p.id === `${entry.id}`) ?? + (algorithm && taxon + ? toPrediction({ ...entry, algorithm, taxon }, determinationTaxonId) + : undefined) + const supersededBy = results.find( + (result) => result.id === entry.payload.superseded_by_result_id + ) + + return prediction + ? [{ type: 'prediction', id, prediction, supersededBy }] + : [] + } + case 'algorithm_result': + return isAlgorithmResult(entry) + ? [{ type: 'algorithm_result', id, entry }] + : [] + case 'review': + return REVIEW_SUBTYPES.includes(entry.subtype) + ? [{ type: 'review', id, entry }] + : [] + default: + return [] + } + }) +} + +/** + * The best prediction of each algorithm: highest score, then terminal, then latest, then id. + * The occurrence lists every detection tied for an algorithm's top score, and the history keeps one. + */ +const onePredictionPerAlgorithm = (predictions: MachinePrediction[]) => { + const rank = (p: MachinePrediction) => [ + p.score ?? -Infinity, + p.terminal ? 1 : 0, + new Date(p.createdAt).getTime() || 0, + Number(p.id) || 0, + ] + const isBetter = (a: MachinePrediction, b: MachinePrediction) => { + const [rankA, rankB] = [rank(a), rank(b)] + const index = rankA.findIndex((value, i) => value !== rankB[i]) + + return index !== -1 && rankA[index] > rankB[index] + } + const best = new Map() + predictions.forEach((prediction) => { + const key = `${prediction.algorithm?.id}` + const current = best.get(key) + if (!current || isBetter(prediction, current)) { + best.set(key, prediction) + } + }) + + return [...best.values()] +} + +/** + * Identifications and predictions merged newest first, for while the history loads or when it is + * unavailable. Predictions and tie breaks follow the server, so cards keep their place once it arrives. + */ +export const getFallbackTimelineItems = ({ + identifications, + predictions, +}: { + identifications: HumanIdentification[] + predictions: MachinePrediction[] +}): TimelineItem[] => + [ + ...identifications.map( + (identification): TimelineItem => ({ + type: 'identification', + id: `identification-${identification.id}`, + identification, + }) + ), + ...onePredictionPerAlgorithm(predictions).map( + (prediction): TimelineItem => ({ + type: 'prediction', + id: `prediction-${prediction.id}`, + prediction, + }) + ), + ] + .map((item) => ({ + item, + time: new Date(getCreatedAt(item)).getTime(), + id: Number(getSourceId(item)), + })) + .sort((a, b) => b.time - a.time || b.id - a.id) + .map(({ item }) => item) + +const getSourceId = (item: TimelineItem) => { + switch (item.type) { + case 'identification': + return item.identification.id + case 'prediction': + return item.prediction.id + default: + return item.entry.id + } +} + +const getCreatedAt = (item: TimelineItem) => { + switch (item.type) { + case 'identification': + return item.identification.createdAt + case 'prediction': + return item.prediction.createdAt + default: + return item.entry.timestamp + } +} + +export interface ResultPrediction { + applied: boolean + /** How many classifications the run created on the occurrence; the card shows the best. */ + count: number + id: string + score: number + taxon: Taxon +} + +/** The prediction a result stands for: the best classification its run created, if any. */ +export const getResultPrediction = ( + entry: AlgorithmResultEntry, + determinationTaxonId?: string +): ResultPrediction | undefined => { + const [best] = entry.classifications + if (!best) { + return undefined + } + const taxon = convertHistoryTaxon(best.taxon) + + return { + applied: taxon.id === determinationTaxonId, + count: entry.classifications.length, + id: `${best.id}`, + score: best.score ?? 0, + taxon, + } +} + +/** A job's settings as "name: value" pairs, leaving out unset ones, for a one-line summary. */ +export const getJobConfigSummary = (job: ServerHistoryJob | null) => + Object.entries(job?.config ?? {}) + .filter(([, value]) => value !== null && value !== undefined) + .map( + ([name, value]) => + `${name}: ${ + typeof value === 'object' ? JSON.stringify(value) : `${value}` + }` + ) + .join(', ') diff --git a/ui/src/nova-ui-kit/components/identifications/identification-card.tsx b/ui/src/nova-ui-kit/components/identifications/identification-card.tsx index 294a6921d..5666b87f2 100644 --- a/ui/src/nova-ui-kit/components/identifications/identification-card.tsx +++ b/ui/src/nova-ui-kit/components/identifications/identification-card.tsx @@ -12,6 +12,8 @@ interface IdentificationCard { open?: boolean subTitle?: string title: string + /** Rendered inline after the title, e.g. a badge qualifying it. */ + titleAddon?: ReactNode onTitleClick?: () => void } @@ -24,6 +26,7 @@ export const IdentificationCard = ({ open, subTitle, title, + titleAddon, onTitleClick, }: IdentificationCard) => (
@@ -33,16 +36,19 @@ export const IdentificationCard = ({ {avatar}
- - {title} - +
+ + {title} + + {titleAddon} +
{subTitle?.length ? ( {subTitle} diff --git a/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx b/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx new file mode 100644 index 000000000..9644add9c --- /dev/null +++ b/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx @@ -0,0 +1,203 @@ +import { TaxonDetails } from 'components/taxon-details/taxon-details' +import { + AlgorithmResultEntry, + getJobConfigSummary, + getResultPrediction, + ServerHistoryTaxon, +} from 'data-services/models/occurrence-history' +import { OccurrenceDetails as Occurrence } from 'data-services/models/occurrence-details' +import { FilterIcon, RulerIcon } from 'lucide-react' +import { + BasicTooltip, + IdentificationCard, + IdentificationDetails, + IdentificationScore, +} from 'nova-ui-kit' +import { Link, useParams } from 'react-router-dom' +import { APP_ROUTES } from 'utils/constants' +import { getFormatedDateTimeString } from 'utils/date/getFormatedDateTimeString/getFormatedDateTimeString' +import { getAppRoute } from 'utils/getAppRoute' +import { STRING, translate } from 'utils/language' +import { UserInfo, UserPermission } from 'utils/user/types' +import { Agree } from '../agree/agree' +import { + HistoryStat, + HistoryStats, + HistoryTime, + HistoryTypeBadge, +} from './history-stats' + +const SUBTYPES = { + class_masking: { icon: FilterIcon, label: STRING.HISTORY_CLASS_MASKING }, + size_filter: { icon: RulerIcon, label: STRING.HISTORY_SIZE_FILTER }, +} + +/** What to call a result's kind, e.g. "Class masking", for the card and for predictions it superseded. */ +export const getResultKindLabel = (entry: AlgorithmResultEntry) => + translate(SUBTYPES[entry.subtype].label) + +/** The determination row, left out when there was no determination before or after. */ +const getDeterminationStats = ( + before: ServerHistoryTaxon | null, + after: ServerHistoryTaxon | null +): HistoryStat[] => { + if (!before && !after) { + return [] + } + + const notAvailable = translate(STRING.VALUE_NOT_AVAILABLE) + const value = + before?.id === after?.id + ? translate(STRING.HISTORY_DETERMINATION_UNCHANGED, { + name: after?.name ?? notAvailable, + }) + : translate(STRING.HISTORY_DETERMINATION_CHANGED, { + after: after?.name ?? notAvailable, + before: before?.name ?? notAvailable, + }) + + return [{ label: translate(STRING.HISTORY_DETERMINATION), value }] +} + +export const AlgorithmResult = ({ + currentUser, + entry, + occurrence, +}: { + currentUser?: UserInfo + entry: AlgorithmResultEntry + occurrence: Occurrence +}) => { + const { projectId } = useParams() + const { icon: Icon } = SUBTYPES[entry.subtype] + const prediction = getResultPrediction( + entry, + occurrence.determinationTaxon?.id + ) + const showAgree = occurrence.userPermissions.includes(UserPermission.Update) + + const stats: HistoryStat[] = getDeterminationStats( + entry.taxon_before, + entry.taxon + ) + switch (entry.subtype) { + case 'class_masking': + stats.push({ + label: translate(STRING.HISTORY_SPECIES_LIST), + value: ( + + {translate(STRING.HISTORY_SPECIES_LIST_ID, { + id: `${entry.payload.taxa_list_id}`, + })} + + ), + }) + break + case 'size_filter': + stats.push({ + label: translate(STRING.HISTORY_SIZE_THRESHOLD), + value: entry.payload.size_threshold, + }) + break + } + stats.push({ + label: translate(STRING.HISTORY_DETECTIONS_AFFECTED), + value: entry.payload.detection_ids.length, + }) + if (entry.job) { + const configSummary = getJobConfigSummary(entry.job) + stats.push({ + label: translate(STRING.FIELD_LABEL_JOB), + value: ( + + {entry.job.name} + + ), + }) + if (configSummary) { + stats.push({ + label: translate(STRING.HISTORY_JOB_SETTINGS), + value: {configSummary}, + }) + } + } + + return ( +
+ + } + subTitle={entry.algorithm ? getResultKindLabel(entry) : undefined} + title={entry.algorithm?.name ?? getResultKindLabel(entry)} + titleAddon={ + + } + > + {/* The prediction the run made: its best new classification, which can be agreed with. */} + {prediction ? ( + +
+
+ +
+ +
+ + + + +
+ {showAgree && ( + + )} +
+ + ) : null} + + +
+ ) +} diff --git a/ui/src/pages/occurrence-details/identification-card/comment-review.tsx b/ui/src/pages/occurrence-details/identification-card/comment-review.tsx new file mode 100644 index 000000000..13165bf08 --- /dev/null +++ b/ui/src/pages/occurrence-details/identification-card/comment-review.tsx @@ -0,0 +1,44 @@ +import { CommentReviewEntry } from 'data-services/models/occurrence-history' +import { UserIcon } from 'lucide-react' +import { IdentificationCard } from 'nova-ui-kit' +import { getFormatedDateTimeString } from 'utils/date/getFormatedDateTimeString/getFormatedDateTimeString' +import { STRING, translate } from 'utils/language' +import { getUserLabel } from 'utils/user/getUserLabel' +import { UserInfo } from 'utils/user/types' +import { HistoryTime, HistoryTypeBadge } from './history-stats' + +export const CommentReview = ({ + currentUser, + entry, +}: { + currentUser?: UserInfo + entry: CommentReviewEntry +}) => ( +
+ + + ) : ( + + ) + } + subTitle={ + entry.withdrawn ? translate(STRING.HISTORY_WITHDRAWN) : undefined + } + title={translate(STRING.HISTORY_COMMENT_BY, { + name: getUserLabel(entry.user, currentUser), + })} + titleAddon={ + + } + > +

+ {entry.comment} +

+ +
+) diff --git a/ui/src/pages/occurrence-details/identification-card/history-stats.tsx b/ui/src/pages/occurrence-details/identification-card/history-stats.tsx new file mode 100644 index 000000000..bdbdb7a82 --- /dev/null +++ b/ui/src/pages/occurrence-details/identification-card/history-stats.tsx @@ -0,0 +1,30 @@ +import { Fragment, ReactNode } from 'react' + +export interface HistoryStat { + label: string + value: ReactNode +} + +export const HistoryStats = ({ stats }: { stats: HistoryStat[] }) => + stats.length ? ( +
+ {stats.map(({ label, value }) => ( + + {label} + {value} + + ))} +
+ ) : null + +export const HistoryTime = ({ label }: { label: string }) => ( + + {label} + +) + +export const HistoryTypeBadge = ({ label }: { label: string }) => ( + + {label} + +) diff --git a/ui/src/pages/occurrence-details/identification-card/machine-prediction.tsx b/ui/src/pages/occurrence-details/identification-card/machine-prediction.tsx index 5150c078a..6c5b6da83 100644 --- a/ui/src/pages/occurrence-details/identification-card/machine-prediction.tsx +++ b/ui/src/pages/occurrence-details/identification-card/machine-prediction.tsx @@ -28,10 +28,13 @@ export const MachinePrediction = ({ currentUser, identification, occurrence, + subTitle, }: { currentUser?: UserInfo identification: Identification occurrence: Occurrence + /** Replaces the terminal/intermediate label, e.g. to say which run superseded the prediction. */ + subTitle?: string }) => { const [open, setOpen] = useState(false) const { classification, error, isLoading } = useClassificationDetails( @@ -62,9 +65,10 @@ export const MachinePrediction = ({ onOpenChange={setOpen} open={open} subTitle={ - identification.terminal + subTitle ?? + (identification.terminal ? translate(STRING.TERMINAL_CLASSIFICATION) - : translate(STRING.INTERMEDIATE_CLASSIFICATION) + : translate(STRING.INTERMEDIATE_CLASSIFICATION)) } title={ identification.algorithm?.name ?? translate(STRING.MACHINE_SUGGESTION) diff --git a/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx b/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx new file mode 100644 index 000000000..b1e6baae2 --- /dev/null +++ b/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx @@ -0,0 +1,114 @@ +import { OccurrenceDetails as Occurrence } from 'data-services/models/occurrence-details' +import { + getFallbackTimelineItems, + getTimelineItems, + ServerOccurrenceHistoryEntry, +} from 'data-services/models/occurrence-history' +import { Loader2Icon } from 'lucide-react' +import { useMemo } from 'react' +import { STRING, translate } from 'utils/language' +import { UserInfo } from 'utils/user/types' +import { AlgorithmResult, getResultKindLabel } from './algorithm-result' +import { CommentReview } from './comment-review' +import { HumanIdentification } from './human-identification' +import { MachinePrediction } from './machine-prediction' + +export const OccurrenceTimeline = ({ + currentUser, + entries, + error, + isLoading, + occurrence, +}: { + currentUser?: UserInfo + entries?: ServerOccurrenceHistoryEntry[] + error?: unknown + isLoading: boolean + occurrence: Occurrence +}) => { + const items = useMemo( + () => + entries + ? getTimelineItems({ + determinationTaxonId: occurrence.determinationTaxon?.id, + entries, + identifications: occurrence.humanIdentifications, + predictions: occurrence.machinePredictions, + }) + : getFallbackTimelineItems({ + identifications: occurrence.humanIdentifications, + predictions: occurrence.machinePredictions, + }), + [entries, occurrence] + ) + + return ( + <> + {/* The fallback cards stand in while the history loads, so the spinner only fills an empty list. */} + {isLoading && !items.length ? ( +
+ +
+ ) : null} + {error ? ( +

+ {translate(STRING.HISTORY_LOAD_ERROR)} +

+ ) : null} + {!isLoading && !items.length ? ( +

+ {translate(STRING.HISTORY_EMPTY)} +

+ ) : null} + {items.map((item) => { + switch (item.type) { + case 'identification': + return ( + + ) + case 'prediction': + return ( + + ) + case 'algorithm_result': + return ( + + ) + case 'review': + return ( + + ) + } + })} + + ) +} diff --git a/ui/src/pages/occurrence-details/occurrence-details.tsx b/ui/src/pages/occurrence-details/occurrence-details.tsx index 5bc502d89..383739fa2 100644 --- a/ui/src/pages/occurrence-details/occurrence-details.tsx +++ b/ui/src/pages/occurrence-details/occurrence-details.tsx @@ -3,6 +3,7 @@ import { BlueprintItem, } from 'components/blueprint-collection/blueprint-collection' import { TaxonDetails } from 'components/taxon-details/taxon-details' +import { useOccurrenceHistory } from 'data-services/hooks/occurrences/useOccurrenceHistory' import { OccurrenceDetails as Occurrence } from 'data-services/models/occurrence-details' import { SearchIcon } from 'lucide-react' import { @@ -26,8 +27,7 @@ import { useUser } from 'utils/user/userContext' import { useUserInfo } from 'utils/user/userInfoContext' import { Agree } from './agree/agree' import { IdQuickActions } from './id-quick-actions/id-quick-actions' -import { HumanIdentification } from './identification-card/human-identification' -import { MachinePrediction } from './identification-card/machine-prediction' +import { OccurrenceTimeline } from './identification-card/occurrence-timeline' import styles from './occurrence-details.module.scss' import { StatusLabel } from './status-label/status-label' import { SuggestId } from './suggest-id/suggest-id' @@ -57,6 +57,10 @@ export const OccurrenceDetails = ({ const navigate = useNavigate() const location = useLocation() const [suggestIdOpen, setSuggestIdOpen] = useState(false) + const history = useOccurrenceHistory({ + occurrenceId: occurrence.id, + projectId, + }) const canUpdate = occurrence.userPermissions.includes(UserPermission.Update) const blueprintItems = useMemo( @@ -256,24 +260,13 @@ export const OccurrenceDetails = ({ )} - {occurrence.humanIdentifications.map((i) => ( - - ))} - - {occurrence.machinePredictions.map((p) => ( - - ))} +
diff --git a/ui/src/utils/language.ts b/ui/src/utils/language.ts index 38e20a5e4..75bcbddd3 100644 --- a/ui/src/utils/language.ts +++ b/ui/src/utils/language.ts @@ -319,6 +319,24 @@ export enum STRING { EXTERNAL_RESOURCES, FORGOT_PASSWORD_DETAILS, FORGOT_PASSWORD, + HISTORY_ALGORITHM_RESULT, + HISTORY_CLASS_MASKING, + HISTORY_COMMENT, + HISTORY_COMMENT_BY, + HISTORY_DETECTIONS_AFFECTED, + HISTORY_DETERMINATION, + HISTORY_DETERMINATION_CHANGED, + HISTORY_DETERMINATION_UNCHANGED, + HISTORY_EMPTY, + HISTORY_JOB_SETTINGS, + HISTORY_LOAD_ERROR, + HISTORY_PREDICTION, + HISTORY_SIZE_FILTER, + HISTORY_SIZE_THRESHOLD, + HISTORY_SPECIES_LIST, + HISTORY_SPECIES_LIST_ID, + HISTORY_SUPERSEDED_BY, + HISTORY_WITHDRAWN, ID_APPLIED, INFO, INTERMEDIATE_CLASSIFICATION, @@ -370,6 +388,7 @@ export enum STRING { TIP, UNKNOWN_ERROR, UNKNOWN, + UNNAMED_USER, UPDATING_DATA, UPLOAD_CAPTURES, USER_INFO, @@ -755,6 +774,25 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.EXTERNAL_RESOURCES]: 'External resources', [STRING.FORGOT_PASSWORD_DETAILS]: `No worries, we'll send you reset instructions.`, [STRING.FORGOT_PASSWORD]: 'Forgot password?', + [STRING.HISTORY_ALGORITHM_RESULT]: 'Algorithm result', + [STRING.HISTORY_CLASS_MASKING]: 'Class masking', + [STRING.HISTORY_COMMENT]: 'Comment', + [STRING.HISTORY_COMMENT_BY]: 'Comment by {{name}}', + [STRING.HISTORY_DETECTIONS_AFFECTED]: 'Detections changed', + [STRING.HISTORY_DETERMINATION]: 'Determination', + [STRING.HISTORY_DETERMINATION_CHANGED]: '{{before}} → {{after}}', + [STRING.HISTORY_DETERMINATION_UNCHANGED]: '{{name}} (unchanged)', + [STRING.HISTORY_EMPTY]: 'Nothing has happened to this occurrence yet.', + [STRING.HISTORY_JOB_SETTINGS]: 'Settings', + [STRING.HISTORY_LOAD_ERROR]: + 'Could not load the full history. Showing identifications and predictions only.', + [STRING.HISTORY_PREDICTION]: 'Prediction', + [STRING.HISTORY_SIZE_FILTER]: 'Size filter', + [STRING.HISTORY_SIZE_THRESHOLD]: 'Size threshold', + [STRING.HISTORY_SPECIES_LIST]: 'Species list', + [STRING.HISTORY_SPECIES_LIST_ID]: 'Species list #{{id}}', + [STRING.HISTORY_SUPERSEDED_BY]: 'Superseded by {{name}}', + [STRING.HISTORY_WITHDRAWN]: 'Withdrawn', [STRING.ID_APPLIED]: 'ID applied', [STRING.INFO]: 'Info', [STRING.INTERMEDIATE_CLASSIFICATION]: 'Intermediate classification', @@ -805,6 +843,7 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.TIP]: 'Tip', [STRING.UNKNOWN_ERROR]: 'Unknown error', [STRING.UNKNOWN]: 'Unknown', + [STRING.UNNAMED_USER]: 'Unnamed user', [STRING.UPDATING_DATA]: 'Updating data', [STRING.UPLOAD_CAPTURES]: 'Upload captures', [STRING.USER_INFO]: 'User info', diff --git a/ui/src/utils/user/getUserLabel.test.ts b/ui/src/utils/user/getUserLabel.test.ts new file mode 100644 index 000000000..0e6c6225d --- /dev/null +++ b/ui/src/utils/user/getUserLabel.test.ts @@ -0,0 +1,30 @@ +import { getUserLabel } from './getUserLabel' + +describe('getUserLabel', () => { + test('names the current user "You" when they have no display name', () => { + expect(getUserLabel({ id: 4, name: '' }, { id: '4', name: '' })).toBe('You') + expect( + getUserLabel( + { id: 4, name: 'Unnamed user' }, + { id: '4', name: undefined } + ) + ).toBe('You') + }) + + test('uses the display name of a user who has one, including the current user', () => { + expect(getUserLabel({ id: 4, name: 'Ada' }, { id: '4', name: 'Ada' })).toBe( + 'Ada' + ) + expect(getUserLabel({ id: 5, name: 'Ada' }, { id: '4' })).toBe('Ada') + }) + + test('labels another user without a display name as unnamed, not anonymous', () => { + expect(getUserLabel({ id: 5, name: '' }, { id: '4' })).toBe('Unnamed user') + expect(getUserLabel({ id: 5, name: null })).toBe('Unnamed user') + }) + + test('keeps "Anonymous user" for actions with no user', () => { + expect(getUserLabel(undefined, { id: '4' })).toBe('Anonymous user') + expect(getUserLabel(null)).toBe('Anonymous user') + }) +}) diff --git a/ui/src/utils/user/getUserLabel.ts b/ui/src/utils/user/getUserLabel.ts new file mode 100644 index 000000000..ed1ac3196 --- /dev/null +++ b/ui/src/utils/user/getUserLabel.ts @@ -0,0 +1,24 @@ +import { STRING, translate } from 'utils/language' + +/** + * The name to show for the author of an action. "Anonymous" is kept for actions with no user, so + * a signed-in user who has not set a display name reads as "You" to themselves and as unnamed to + * others. The viewer's own name is read from their profile, since `user.name` may already be a label. + */ +export const getUserLabel = ( + user?: { id?: string | number; name?: string | null } | null, + currentUser?: { id: string; name?: string } +) => { + if (!user) { + return translate(STRING.ANONYMOUS_USER) + } + + const isCurrentUser = + !!currentUser && user.id !== undefined && `${user.id}` === currentUser.id + + if (isCurrentUser && !currentUser.name?.length) { + return translate(STRING.YOU) + } + + return user.name?.length ? user.name : translate(STRING.UNNAMED_USER) +} From fe0f428c9bb5bb997092ab1bd8fa2aa150e6e640 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Fri, 2 Oct 2026 20:01:25 -0700 Subject: [PATCH 007/134] feat(occurrences): name the species list and classifier on a result, and mark predictions a run outranked A class masking entry in the occurrence history now carries the species list and the source classifier by name, so the page can say what the run used instead of showing ids. A prediction is marked as superseded not only when a run re-scored it, as class masking does, but also when a run's classification on the same detection outranks it as a terminal prediction, as the size filter does. The label a reader sees is then the same for both methods. Part of #1433. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01L52AN9tabp76yjhjyCZkSJ --- ami/main/api/serializers.py | 9 ++++++ ami/main/models_future/history.py | 43 +++++++++++++++++++++++++---- ami/main/test_occurrence_history.py | 38 ++++++++++++++----------- 3 files changed, 68 insertions(+), 22 deletions(-) diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index 445d7d753..4f116209d 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -2103,6 +2103,11 @@ class HistoryTaxonSerializer(serializers.Serializer): rank = serializers.CharField() +class HistoryTaxaListSerializer(serializers.Serializer): + id = serializers.IntegerField() + name = serializers.CharField() + + class HistoryClassificationSerializer(serializers.Serializer): """A classification a post-processing run created, shown inside the run's result.""" @@ -2138,6 +2143,10 @@ class OccurrenceHistoryEntrySerializer(serializers.Serializer): classifications = HistoryClassificationSerializer( many=True, help_text="For a result: the classifications its run created, best score first." ) + taxa_list = HistoryTaxaListSerializer(allow_null=True, help_text="For class masking: the species list used.") + source_algorithm = HistoryAlgorithmSerializer( + allow_null=True, help_text="For class masking: the classifier whose predictions were re-scored." + ) verdict = serializers.CharField(allow_null=True, help_text="A review's verdict: confirmed or rejected.") comment = serializers.CharField(allow_blank=True) withdrawn = serializers.BooleanField() diff --git a/ami/main/models_future/history.py b/ami/main/models_future/history.py index 18780b553..5fd8e89c9 100644 --- a/ami/main/models_future/history.py +++ b/ami/main/models_future/history.py @@ -13,11 +13,20 @@ from django.db.models import Q -from ami.main.models import AlgorithmResult, Classification, Identification, Occurrence, Taxon, User, ValidationReview +from ami.main.models import ( + AlgorithmResult, + Classification, + Identification, + Occurrence, + TaxaList, + Taxon, + User, + ValidationReview, +) +from ami.ml.models.algorithm import Algorithm if typing.TYPE_CHECKING: from ami.jobs.models import Job - from ami.ml.models import Algorithm COMMENT = ValidationReview.Aspect.COMMENT @@ -48,6 +57,9 @@ class TimelineEntry: is_current: bool | None = None # For a result: the classifications the run created, best score first. classifications: list[Classification] = dataclasses.field(default_factory=list) + # For a class masking result: the species list and the classifier it re-scored. + taxa_list: TaxaList | None = None + source_algorithm: Algorithm | None = None def review_entry(review: ValidationReview) -> TimelineEntry: @@ -69,8 +81,10 @@ def occurrence_timeline(occurrence: Occurrence) -> list[TimelineEntry]: A result comes with the classifications its run created, so the history shows the run and what it changed as one entry; those classifications are not listed again as predictions. - Every other algorithm contributes one prediction, its best. A prediction a run demoted - names the result that superseded it. The query count does not grow with the entries. + Every other algorithm contributes one prediction, its best. A prediction names the result + that superseded it when a result's classification re-scored it, or when a result's + classification on the same detection outranks it as a terminal prediction. The query count + does not grow with the entries. """ results = list( AlgorithmResult.objects.filter(occurrence=occurrence) @@ -79,14 +93,24 @@ def occurrence_timeline(occurrence: Occurrence) -> list[TimelineEntry]: ) taxon_ids = {result.data.get(key) for result in results for key in ("taxon_before_id", "taxon_after_id")} - {None} taxa = {taxon.pk: taxon for taxon in Taxon.objects.filter(pk__in=taxon_ids)} if taxon_ids else {} + taxa_lists = _by_id(TaxaList, {result.data.get("taxa_list_id") for result in results}) + source_algorithms = _by_id(Algorithm, {result.data.get("source_algorithm_id") for result in results}) created_by_result = _classifications_created_by(results, occurrence) created_ids = {c.pk for created in created_by_result.values() for c in created} - superseded_by = { + rescored_by = { c.applied_to_id: result_id for result_id, created in created_by_result.items() for c in created if c.applied_to_id is not None } + outranked_by = {c.detection_id: result_id for result_id, created in created_by_result.items() for c in created} + + def superseded_by(prediction: Classification) -> int | None: + if prediction.pk in rescored_by: + return rescored_by[prediction.pk] + if prediction.terminal: + return outranked_by.get(prediction.detection_id) + return None entries = [ TimelineEntry( @@ -101,6 +125,8 @@ def occurrence_timeline(occurrence: Occurrence) -> list[TimelineEntry]: payload=result.data, is_current=result.is_current, classifications=created_by_result.get(result.pk, []), + taxa_list=taxa_lists.get(result.data.get("taxa_list_id")), + source_algorithm=source_algorithms.get(result.data.get("source_algorithm_id")), ) for result in results ] @@ -140,7 +166,7 @@ def occurrence_timeline(occurrence: Occurrence) -> list[TimelineEntry]: "detection_id": prediction.detection_id, "terminal": prediction.terminal, "applied_to_id": prediction.applied_to_id, - "superseded_by_result_id": superseded_by.get(prediction.pk), + "superseded_by_result_id": superseded_by(prediction), }, ) for prediction in _one_prediction_per_algorithm(occurrence) @@ -151,6 +177,11 @@ def occurrence_timeline(occurrence: Occurrence) -> list[TimelineEntry]: return entries +def _by_id(model, ids: set) -> dict: + ids.discard(None) + return {row.pk: row for row in model.objects.filter(pk__in=ids)} if ids else {} + + def _classifications_created_by( results: list[AlgorithmResult], occurrence: Occurrence ) -> dict[int, list[Classification]]: diff --git a/ami/main/test_occurrence_history.py b/ami/main/test_occurrence_history.py index 3f24284b1..c9595c054 100644 --- a/ami/main/test_occurrence_history.py +++ b/ami/main/test_occurrence_history.py @@ -18,6 +18,7 @@ Occurrence, Project, SourceImage, + TaxaList, Taxon, ValidationReview, ) @@ -27,7 +28,9 @@ from ami.users.roles import BasicMember, Identifier, ProjectManager # Measured: two savepoints, the project, the occurrence, then the results with their algorithm and -# job, their taxa, the classifications they created, reviews, identifications and predictions. +# job, their taxa, the classifications they created, reviews, identifications and predictions. A +# class masking result adds one query each for the species lists and source algorithms, whatever +# the number of entries. HISTORY_QUERIES = 10 SIZE_FILTER = AlgorithmResult.Kind.SIZE_FILTER @@ -304,6 +307,7 @@ def test_a_demoted_prediction_names_the_result_that_superseded_it(self): """Class masking demotes the original classification and links its replacement to the result.""" classifier = Algorithm.objects.create(name="Classifier", key="classifier-history-test") masker = Algorithm.objects.create(name="Masked classifier", key="masker-history-test") + taxa_list = TaxaList.objects.create(name="Kept species") original = Classification.objects.get(detection=self.detections[1]) Classification.objects.filter(pk=original.pk).update(terminal=False, algorithm=classifier) result = AlgorithmResult.objects.record( @@ -312,8 +316,8 @@ def test_a_demoted_prediction_names_the_result_that_superseded_it(self): job=self.job, kind=CLASS_MASKING, data={ - "taxa_list_id": 1, - "source_algorithm_id": 1, + "taxa_list_id": taxa_list.pk, + "source_algorithm_id": classifier.pk, "detection_ids": [self.detections[1].pk], "taxon_before_id": self.taxon.pk, "taxon_after_id": self.other_taxon.pk, @@ -346,31 +350,33 @@ def test_a_demoted_prediction_names_the_result_that_superseded_it(self): self.assertEqual(demoted["payload"]["superseded_by_result_id"], result.pk) self.assertFalse(demoted["payload"]["terminal"]) self.assertIsNone(predictions[None]["payload"]["superseded_by_result_id"]) + self.assertEqual(result_entry["taxa_list"], {"id": taxa_list.pk, "name": "Kept species"}) + self.assertEqual(result_entry["source_algorithm"]["key"], classifier.key) - def test_a_classification_without_a_result_link_is_matched_to_its_jobs_result(self): + def test_a_terminal_prediction_outranked_on_its_detection_is_superseded(self): + """The size filter does not demote the prediction it outranks, so the link is the shared detection.""" + # The fixture's tied predictions resolve to the latest one, on the last detection. + outranked = self.detections[3].classifications.get() result = AlgorithmResult.objects.record( occurrence=self.occurrence, algorithm=self.size_filter, job=self.job, kind=SIZE_FILTER, - data={"size_threshold": 0.01, "detection_ids": [self.detections[0].pk]}, + data={"size_threshold": 0.01, "detection_ids": [self.detections[3].pk]}, ) - created = Classification.objects.create( - detection=self.detections[0], - taxon=self.taxon, - score=0.1, + Classification.objects.create( + detection=self.detections[3], + taxon=self.other_taxon, + score=1.0, algorithm=self.size_filter, timestamp=datetime.datetime.now(), - job=self.job, + algorithm_result=result, ) data = self.get() - result_entry = next(entry for entry in data if entry["type"] == "algorithm_result") - self.assertEqual(result_entry["id"], result.pk) - self.assertEqual([c["id"] for c in result_entry["classifications"]], [created.pk]) - predictions = [e for e in data if e["type"] == "prediction"] - self.assertEqual([e["algorithm"] for e in predictions], [None]) - self.assertNotEqual(predictions[0]["id"], created.pk) + prediction = next(entry for entry in data if entry["type"] == "prediction") + self.assertEqual(prediction["id"], outranked.pk) + self.assertEqual(prediction["payload"]["superseded_by_result_id"], result.pk) def test_each_algorithm_shows_one_prediction_preferring_terminal_then_latest(self): """Tied top scores would otherwise show the same prediction once per detection.""" From ddb0c20ae0fddddf9719a0a824063be0b25007ca Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Fri, 2 Oct 2026 20:01:27 -0700 Subject: [PATCH 008/134] feat(ui): make result cards readable: method as title, species list by name, plain setting labels, one author rule A result card is now titled by its method ("Class masking", "Size filter") with the classifier and species list named underneath, instead of the long generated algorithm name. The species list row links by name. The remaining job settings are rows with plain labels (capture set, occurrence, re-weighted scores) rather than a line of raw keys, and settings the card already shows, such as the species list, the threshold and the source algorithm, are not repeated. A prediction a run outranked reads "Superseded by size filter", matching the class masking label. The author of a comment follows the same rule as the identification cards: their display name, else "Anonymous user". Part of #1433. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01L52AN9tabp76yjhjyCZkSJ --- .../models/occurrence-history.test.ts | 31 ++++++--- .../models/occurrence-history.ts | 38 ++++++++--- .../identification-card/algorithm-result.tsx | 64 +++++++++++++++---- .../identification-card/comment-review.tsx | 11 +--- .../occurrence-timeline.tsx | 8 +-- ui/src/utils/language.ts | 16 +++-- ui/src/utils/user/getUserLabel.test.ts | 27 ++------ ui/src/utils/user/getUserLabel.ts | 24 ++----- 8 files changed, 127 insertions(+), 92 deletions(-) diff --git a/ui/src/data-services/models/occurrence-history.test.ts b/ui/src/data-services/models/occurrence-history.test.ts index 8a367debe..0442d0619 100644 --- a/ui/src/data-services/models/occurrence-history.test.ts +++ b/ui/src/data-services/models/occurrence-history.test.ts @@ -4,7 +4,7 @@ import { Algorithm } from './algorithm' import { HumanIdentification, MachinePrediction } from './occurrence-details' import { getFallbackTimelineItems, - getJobConfigSummary, + getJobSettings, getResultPrediction, getTimelineItems, ServerOccurrenceHistoryEntry, @@ -22,7 +22,9 @@ const base = { is_current: null, job: null, score: null, + source_algorithm: null, subtype: null, + taxa_list: null, taxon: null, taxon_before: null, timestamp: '2026-04-29T22:00:00', @@ -188,7 +190,7 @@ describe('getTimelineItems', () => { { applied: true, comment: 'Looks right', - user: { id: '9', name: 'Unnamed user' }, + user: { id: '9', name: 'Anonymous user' }, userPermissions: [], } ) @@ -196,7 +198,11 @@ describe('getTimelineItems', () => { test('names the result that superseded a demoted prediction', () => { const items = getTimelineItems({ - entries: [classMasking, predictionEntry(6, XESTIA, 5), predictionEntry(8)], + entries: [ + classMasking, + predictionEntry(6, XESTIA, 5), + predictionEntry(8), + ], identifications: [], predictions: [], }) @@ -331,25 +337,30 @@ describe('getOccurrenceHistoryQueryKey', () => { }) }) -describe('getJobConfigSummary', () => { - test('lists the settings a job ran with, leaving out unset ones', () => { +describe('getJobSettings', () => { + test('lists the settings a job ran with, leaving out unset ones and those shown elsewhere', () => { expect( - getJobConfigSummary({ + getJobSettings({ config: { occurrence_id: 4, + reweight: true, size_threshold: 0.01, source_image_collection_id: null, + taxa_list_id: 2, }, id: 1, name: 'Size filter', }) - ).toBe('occurrence_id: 4, size_threshold: 0.01') + ).toEqual([ + { key: 'occurrence_id', value: 4 }, + { key: 'reweight', value: true }, + ]) }) test('is empty for a job without settings', () => { - expect(getJobConfigSummary({ config: null, id: 1, name: 'Pipeline' })).toBe( - '' + expect(getJobSettings({ config: null, id: 1, name: 'Pipeline' })).toEqual( + [] ) - expect(getJobConfigSummary(null)).toBe('') + expect(getJobSettings(null)).toEqual([]) }) }) diff --git a/ui/src/data-services/models/occurrence-history.ts b/ui/src/data-services/models/occurrence-history.ts index 70f02a500..3834bff0b 100644 --- a/ui/src/data-services/models/occurrence-history.ts +++ b/ui/src/data-services/models/occurrence-history.ts @@ -21,6 +21,11 @@ export interface ServerHistoryAlgorithm { name: string } +export interface ServerHistoryTaxaList { + id: number + name: string +} + export interface ServerHistoryJob { /** The settings a post-processing job ran with; null for other jobs. */ config: Record | null @@ -84,7 +89,11 @@ interface ServerHistoryEntryBase { job: ServerHistoryJob | null payload: Payload score: number | null + /** For a class masking result: the classifier whose predictions the run re-scored. */ + source_algorithm: ServerHistoryAlgorithm | null subtype: Subtype + /** For a class masking result: the species list the run kept. */ + taxa_list: ServerHistoryTaxaList | null taxon: ServerHistoryTaxon | null taxon_before: ServerHistoryTaxon | null timestamp: string @@ -381,14 +390,25 @@ export const getResultPrediction = ( } } -/** A job's settings as "name: value" pairs, leaving out unset ones, for a one-line summary. */ -export const getJobConfigSummary = (job: ServerHistoryJob | null) => +export interface JobSetting { + key: string + value: unknown +} + +/** Settings the result card already shows in a row of their own. */ +const SETTINGS_SHOWN_ELSEWHERE = [ + 'algorithm_id', + 'size_threshold', + 'taxa_list_id', +] + +/** A job's remaining settings, leaving out unset ones and those the card shows elsewhere. */ +export const getJobSettings = (job: ServerHistoryJob | null): JobSetting[] => Object.entries(job?.config ?? {}) - .filter(([, value]) => value !== null && value !== undefined) - .map( - ([name, value]) => - `${name}: ${ - typeof value === 'object' ? JSON.stringify(value) : `${value}` - }` + .filter( + ([key, value]) => + value !== null && + value !== undefined && + !SETTINGS_SHOWN_ELSEWHERE.includes(key) ) - .join(', ') + .map(([key, value]) => ({ key, value })) diff --git a/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx b/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx index 9644add9c..0c793da91 100644 --- a/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx +++ b/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx @@ -1,7 +1,7 @@ import { TaxonDetails } from 'components/taxon-details/taxon-details' import { AlgorithmResultEntry, - getJobConfigSummary, + getJobSettings, getResultPrediction, ServerHistoryTaxon, } from 'data-services/models/occurrence-history' @@ -32,10 +32,47 @@ const SUBTYPES = { size_filter: { icon: RulerIcon, label: STRING.HISTORY_SIZE_FILTER }, } +/** Labels for the job settings shown as rows; a setting not listed here shows its raw key. */ +const SETTING_LABELS: Partial> = { + occurrence_id: STRING.HISTORY_SETTING_OCCURRENCE, + reweight: STRING.HISTORY_SETTING_REWEIGHT, + source_image_collection_id: STRING.HISTORY_SETTING_CAPTURE_SET, +} + /** What to call a result's kind, e.g. "Class masking", for the card and for predictions it superseded. */ export const getResultKindLabel = (entry: AlgorithmResultEntry) => translate(SUBTYPES[entry.subtype].label) +const formatSettingValue = (key: string, value: unknown) => { + if (typeof value === 'boolean') { + return translate(value ? STRING.YES : STRING.NO) + } + if (key.endsWith('_id')) { + return `#${value}` + } + + return typeof value === 'object' ? JSON.stringify(value) : `${value}` +} + +/** The card's subtitle: for class masking the classifier and species list, otherwise the algorithm's name. */ +const getSubTitle = (entry: AlgorithmResultEntry) => { + if (entry.subtype === 'class_masking') { + return translate(STRING.HISTORY_MASKING_SUBTITLE, { + algorithm: + entry.source_algorithm?.name ?? + entry.algorithm?.name ?? + translate(STRING.VALUE_NOT_AVAILABLE), + list: + entry.taxa_list?.name ?? + translate(STRING.HISTORY_SPECIES_LIST_ID, { + id: `${entry.payload.taxa_list_id}`, + }), + }) + } + + return entry.algorithm?.name +} + /** The determination row, left out when there was no determination before or after. */ const getDeterminationStats = ( before: ServerHistoryTaxon | null, @@ -92,9 +129,10 @@ export const AlgorithmResult = ({ taxaListId: `${entry.payload.taxa_list_id}`, })} > - {translate(STRING.HISTORY_SPECIES_LIST_ID, { - id: `${entry.payload.taxa_list_id}`, - })} + {entry.taxa_list?.name ?? + translate(STRING.HISTORY_SPECIES_LIST_ID, { + id: `${entry.payload.taxa_list_id}`, + })} ), }) @@ -110,8 +148,14 @@ export const AlgorithmResult = ({ label: translate(STRING.HISTORY_DETECTIONS_AFFECTED), value: entry.payload.detection_ids.length, }) + getJobSettings(entry.job).forEach(({ key, value }) => { + const label = SETTING_LABELS[key] + stats.push({ + label: label !== undefined ? translate(label) : key, + value: formatSettingValue(key, value), + }) + }) if (entry.job) { - const configSummary = getJobConfigSummary(entry.job) stats.push({ label: translate(STRING.FIELD_LABEL_JOB), value: ( @@ -126,12 +170,6 @@ export const AlgorithmResult = ({ ), }) - if (configSummary) { - stats.push({ - label: translate(STRING.HISTORY_JOB_SETTINGS), - value: {configSummary}, - }) - } } return ( @@ -141,8 +179,8 @@ export const AlgorithmResult = ({ /> } - subTitle={entry.algorithm ? getResultKindLabel(entry) : undefined} - title={entry.algorithm?.name ?? getResultKindLabel(entry)} + subTitle={getSubTitle(entry)} + title={getResultKindLabel(entry)} titleAddon={ ( +export const CommentReview = ({ entry }: { entry: CommentReviewEntry }) => (
diff --git a/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx b/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx index b1e6baae2..1aa889950 100644 --- a/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx +++ b/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx @@ -100,13 +100,7 @@ export const OccurrenceTimeline = ({ /> ) case 'review': - return ( - - ) + return } })} diff --git a/ui/src/utils/language.ts b/ui/src/utils/language.ts index 75bcbddd3..c77302cf8 100644 --- a/ui/src/utils/language.ts +++ b/ui/src/utils/language.ts @@ -226,6 +226,7 @@ export enum STRING { MESSAGE_VALUE_MISSING, /* NAV_ITEM */ + NO, NAV_ITEM_ABOUT, NAV_ITEM_ALGORITHMS, NAV_ITEM_CAPTURE_SETS, @@ -328,9 +329,12 @@ export enum STRING { HISTORY_DETERMINATION_CHANGED, HISTORY_DETERMINATION_UNCHANGED, HISTORY_EMPTY, - HISTORY_JOB_SETTINGS, + HISTORY_MASKING_SUBTITLE, HISTORY_LOAD_ERROR, HISTORY_PREDICTION, + HISTORY_SETTING_CAPTURE_SET, + HISTORY_SETTING_OCCURRENCE, + HISTORY_SETTING_REWEIGHT, HISTORY_SIZE_FILTER, HISTORY_SIZE_THRESHOLD, HISTORY_SPECIES_LIST, @@ -388,7 +392,6 @@ export enum STRING { TIP, UNKNOWN_ERROR, UNKNOWN, - UNNAMED_USER, UPDATING_DATA, UPLOAD_CAPTURES, USER_INFO, @@ -399,6 +402,7 @@ export enum STRING { VIEW_IN_SESSION, VIEW_OCCURRENCES, VIEW_SETTINGS, + YES, YOU, } @@ -661,6 +665,7 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.MESSAGE_VALUE_MISSING]: 'Please provide a value', /* NAV_ITEM */ + [STRING.NO]: 'No', [STRING.NAV_ITEM_ABOUT]: 'About', [STRING.NAV_ITEM_ALGORITHMS]: 'Algorithms', [STRING.NAV_ITEM_CAPTURE_SETS]: 'Capture sets', @@ -783,10 +788,13 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.HISTORY_DETERMINATION_CHANGED]: '{{before}} → {{after}}', [STRING.HISTORY_DETERMINATION_UNCHANGED]: '{{name}} (unchanged)', [STRING.HISTORY_EMPTY]: 'Nothing has happened to this occurrence yet.', - [STRING.HISTORY_JOB_SETTINGS]: 'Settings', + [STRING.HISTORY_MASKING_SUBTITLE]: '{{algorithm}}, filtered by {{list}}', [STRING.HISTORY_LOAD_ERROR]: 'Could not load the full history. Showing identifications and predictions only.', [STRING.HISTORY_PREDICTION]: 'Prediction', + [STRING.HISTORY_SETTING_CAPTURE_SET]: 'Capture set', + [STRING.HISTORY_SETTING_OCCURRENCE]: 'Occurrence', + [STRING.HISTORY_SETTING_REWEIGHT]: 'Re-weighted scores', [STRING.HISTORY_SIZE_FILTER]: 'Size filter', [STRING.HISTORY_SIZE_THRESHOLD]: 'Size threshold', [STRING.HISTORY_SPECIES_LIST]: 'Species list', @@ -843,7 +851,6 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.TIP]: 'Tip', [STRING.UNKNOWN_ERROR]: 'Unknown error', [STRING.UNKNOWN]: 'Unknown', - [STRING.UNNAMED_USER]: 'Unnamed user', [STRING.UPDATING_DATA]: 'Updating data', [STRING.UPLOAD_CAPTURES]: 'Upload captures', [STRING.USER_INFO]: 'User info', @@ -854,6 +861,7 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.VIEW_IN_SESSION]: 'View in session', [STRING.VIEW_OCCURRENCES]: 'View occurrences', [STRING.VIEW_SETTINGS]: 'View settings', + [STRING.YES]: 'Yes', [STRING.YOU]: 'You', } diff --git a/ui/src/utils/user/getUserLabel.test.ts b/ui/src/utils/user/getUserLabel.test.ts index 0e6c6225d..1dbbb5624 100644 --- a/ui/src/utils/user/getUserLabel.test.ts +++ b/ui/src/utils/user/getUserLabel.test.ts @@ -1,30 +1,17 @@ import { getUserLabel } from './getUserLabel' describe('getUserLabel', () => { - test('names the current user "You" when they have no display name', () => { - expect(getUserLabel({ id: 4, name: '' }, { id: '4', name: '' })).toBe('You') - expect( - getUserLabel( - { id: 4, name: 'Unnamed user' }, - { id: '4', name: undefined } - ) - ).toBe('You') + test('uses the display name of a user who has one', () => { + expect(getUserLabel({ name: 'Ada' })).toBe('Ada') }) - test('uses the display name of a user who has one, including the current user', () => { - expect(getUserLabel({ id: 4, name: 'Ada' }, { id: '4', name: 'Ada' })).toBe( - 'Ada' - ) - expect(getUserLabel({ id: 5, name: 'Ada' }, { id: '4' })).toBe('Ada') + test('reads "Anonymous user" for a user without a display name, as the identification cards do', () => { + expect(getUserLabel({ name: '' })).toBe('Anonymous user') + expect(getUserLabel({ name: null })).toBe('Anonymous user') }) - test('labels another user without a display name as unnamed, not anonymous', () => { - expect(getUserLabel({ id: 5, name: '' }, { id: '4' })).toBe('Unnamed user') - expect(getUserLabel({ id: 5, name: null })).toBe('Unnamed user') - }) - - test('keeps "Anonymous user" for actions with no user', () => { - expect(getUserLabel(undefined, { id: '4' })).toBe('Anonymous user') + test('reads "Anonymous user" for an action with no user', () => { + expect(getUserLabel(undefined)).toBe('Anonymous user') expect(getUserLabel(null)).toBe('Anonymous user') }) }) diff --git a/ui/src/utils/user/getUserLabel.ts b/ui/src/utils/user/getUserLabel.ts index ed1ac3196..f82ecaf95 100644 --- a/ui/src/utils/user/getUserLabel.ts +++ b/ui/src/utils/user/getUserLabel.ts @@ -1,24 +1,8 @@ import { STRING, translate } from 'utils/language' /** - * The name to show for the author of an action. "Anonymous" is kept for actions with no user, so - * a signed-in user who has not set a display name reads as "You" to themselves and as unnamed to - * others. The viewer's own name is read from their profile, since `user.name` may already be a label. + * The name to show for the author of an action: their display name, else "Anonymous user". The + * same rule the identification cards use, so one person reads the same on every card. */ -export const getUserLabel = ( - user?: { id?: string | number; name?: string | null } | null, - currentUser?: { id: string; name?: string } -) => { - if (!user) { - return translate(STRING.ANONYMOUS_USER) - } - - const isCurrentUser = - !!currentUser && user.id !== undefined && `${user.id}` === currentUser.id - - if (isCurrentUser && !currentUser.name?.length) { - return translate(STRING.YOU) - } - - return user.name?.length ? user.name : translate(STRING.UNNAMED_USER) -} +export const getUserLabel = (user?: { name?: string | null } | null) => + user?.name?.length ? user.name : translate(STRING.ANONYMOUS_USER) From 22e249fc51c142da4fd419bfa42b66ad54a5c6b6 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Fri, 2 Oct 2026 20:40:31 -0700 Subject: [PATCH 009/134] feat(results): record each run's own figures per kind, with a sortable value, and drop the review model A result's data is now a pydantic model per kind that records only what the run alone knew, plus a free-form extra object for whatever else a method or processing service returns. Class masking records the share of the source classifier's probability outside the species list, the original top prediction and its score, and where the new winner ranked before masking, taken from the occurrence's winning detection. The size filter records the detection's size as a fraction of the image, taken from the smallest flagged detection. Both keep the determination before and after. Settings, changed detections and new taxa are no longer copied in; they live on the job, on the linked classifications and on those classifications' taxa. The headline figure of each kind is repeated in a nullable value column with a partial index on project, kind and value for current rows, so lists can filter and sort on it. The result table migration is rewritten accordingly; the branch is unmerged. ValidationReview, the comment endpoint and their tests are removed: nothing produces a review yet. They return when tracking's grouping confirmation or a comment box needs them. Part of #1431 and #1457. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01L52AN9tabp76yjhjyCZkSJ --- ami/main/api/serializers.py | 26 ++- ami/main/api/views.py | 37 +--- ami/main/migrations/0096_algorithm_results.py | 89 ++++++++ ...lgorithm_results_and_validation_reviews.py | 157 --------------- ...put_jobs_and_results_on_classifications.py | 2 +- ami/main/models.py | 84 ++------ ami/main/models_future/history.py | 79 +++----- ami/main/schemas.py | 85 ++++---- ami/main/test_occurrence_history.py | 190 +++++------------- ami/ml/post_processing/class_masking.py | 18 +- ami/ml/post_processing/results.py | 43 ++-- ami/ml/post_processing/small_size_filter.py | 5 +- .../tests/test_class_masking.py | 22 +- .../tests/test_small_size_filter.py | 10 +- 14 files changed, 312 insertions(+), 535 deletions(-) create mode 100644 ami/main/migrations/0096_algorithm_results.py delete mode 100644 ami/main/migrations/0096_algorithm_results_and_validation_reviews.py diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index 4f116209d..e653a3210 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -2124,12 +2124,11 @@ class HistoryClassificationSerializer(serializers.Serializer): class OccurrenceHistoryEntrySerializer(serializers.Serializer): """One entry of an occurrence's history, newest first. ``type`` says which table it came from.""" - type = serializers.ChoiceField(choices=["algorithm_result", "review", "identification", "prediction"]) + type = serializers.ChoiceField(choices=["algorithm_result", "identification", "prediction"]) id = serializers.IntegerField(help_text="Primary key of the row in the table ``type`` names.") timestamp = serializers.DateTimeField() subtype = serializers.CharField( - allow_null=True, - help_text="An algorithm result's kind (class_masking, size_filter) or a review's aspect (comment).", + allow_null=True, help_text="An algorithm result's kind (class_masking, size_filter)." ) user = HistoryUserSerializer(allow_null=True) algorithm = HistoryAlgorithmSerializer(allow_null=True) @@ -2138,25 +2137,24 @@ class OccurrenceHistoryEntrySerializer(serializers.Serializer): allow_null=True, help_text="The identified or predicted taxon, or the determination after a result." ) taxon_before = HistoryTaxonSerializer(allow_null=True, help_text="The determination before a result.") - score = serializers.FloatField(allow_null=True, help_text="A prediction's score.") + score = serializers.FloatField( + allow_null=True, help_text="A prediction's score, or a result's headline value (see its kind)." + ) payload = serializers.JSONField(help_text="Details that depend on the type and subtype.") classifications = HistoryClassificationSerializer( many=True, help_text="For a result: the classifications its run created, best score first." ) - taxa_list = HistoryTaxaListSerializer(allow_null=True, help_text="For class masking: the species list used.") + original_taxon = HistoryTaxonSerializer( + allow_null=True, help_text="For class masking: the source classifier's top taxon before masking." + ) + taxa_list = HistoryTaxaListSerializer( + allow_null=True, help_text="For class masking: the species list used, from the job's settings." + ) source_algorithm = HistoryAlgorithmSerializer( - allow_null=True, help_text="For class masking: the classifier whose predictions were re-scored." + allow_null=True, help_text="For class masking: the classifier re-scored, from the job's settings." ) - verdict = serializers.CharField(allow_null=True, help_text="A review's verdict: confirmed or rejected.") comment = serializers.CharField(allow_blank=True) withdrawn = serializers.BooleanField() is_current = serializers.BooleanField( allow_null=True, help_text="For a result: whether it is the latest of its kind, not replaced by a later run." ) - - -class OccurrenceReviewCreateSerializer(serializers.Serializer): - """A review to add to an occurrence. Only comments are written here.""" - - aspect = serializers.ChoiceField(choices=["comment"], default="comment") - comment = serializers.CharField(max_length=10000, trim_whitespace=True) diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 35dabee04..7a5f080f7 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -34,7 +34,7 @@ from ami.base.views import ProjectMixin from ami.main.api.schemas import limit_doc_param, project_id_doc_param from ami.main.api.serializers import TagSerializer -from ami.main.models_future.history import add_comment, occurrence_timeline, review_entry +from ami.main.models_future.history import occurrence_timeline from ami.main.models_future.identifications import create_identifications_batch, resolve_occurrences from ami.main.models_future.occurrence import model_agreement_for_project, top_identifiers_for_project from ami.ml.models.algorithm import Algorithm @@ -85,7 +85,6 @@ ModelAgreementSerializer, OccurrenceHistoryEntrySerializer, OccurrenceListSerializer, - OccurrenceReviewCreateSerializer, OccurrenceSerializer, PageListSerializer, PageSerializer, @@ -1539,15 +1538,9 @@ def get_serializer_class(self): else: return OccurrenceSerializer - # Actions that read or add to one occurrence's history. They build their own queries and - # never serialize the occurrence, and the default filters, which shape lists, do not hide it. - HISTORY_ACTIONS = ("history", "reviews") - - def get_permissions(self): - # Commenting is open to whoever may identify (see Occurrence.check_custom_permission). - if self.action == "reviews": - return [ObjectPermission()] - return super().get_permissions() + # Actions that read one occurrence's history. They build their own queries and never + # serialize the occurrence, and the default filters, which shape lists, do not hide it. + HISTORY_ACTIONS = ("history",) def get_queryset(self) -> QuerySet["Occurrence"]: project = self.get_active_project() @@ -1609,31 +1602,13 @@ def list(self, request, *args, **kwargs): def history(self, request: Request, pk=None) -> Response: """Everything that happened to this occurrence, newest first. - Merges post-processing results with the classifications they created, people's - comments, identifications and predictions into one list. Visible to whoever can - open the occurrence itself. + Merges post-processing results with the classifications they created, identifications + and predictions into one list. Visible to whoever can open the occurrence itself. """ occurrence = self.get_object() entries = occurrence_timeline(occurrence) return Response(OccurrenceHistoryEntrySerializer(entries, many=True, context={"request": request}).data) - @extend_schema( - parameters=[project_id_doc_param], - request=OccurrenceReviewCreateSerializer, - responses={201: OccurrenceHistoryEntrySerializer}, - ) - @action(detail=True, methods=["post"], name="reviews") - def reviews(self, request: Request, pk=None) -> Response: - """Leave a comment on this occurrence. It shows in the occurrence's history.""" - occurrence = self.get_object() - body = OccurrenceReviewCreateSerializer(data=request.data) - body.is_valid(raise_exception=True) - review = add_comment(occurrence, request.user, body.validated_data["comment"]) - return Response( - OccurrenceHistoryEntrySerializer(review_entry(review), context={"request": request}).data, - status=status.HTTP_201_CREATED, - ) - @extend_schema(parameters=[project_id_doc_param], responses=AlgorithmSerializer(many=True)) @action(detail=False, methods=["get"], name="algorithms") def algorithms(self, request: Request) -> Response: diff --git a/ami/main/migrations/0096_algorithm_results.py b/ami/main/migrations/0096_algorithm_results.py new file mode 100644 index 000000000..402b1b6c4 --- /dev/null +++ b/ami/main/migrations/0096_algorithm_results.py @@ -0,0 +1,89 @@ +# Generated by Django 4.2.10 on 2026-10-02 23:31 + +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + dependencies = [ + ("jobs", "0023_alter_job_job_type_key"), + ("ml", "0028_normalize_empty_endpoint_url_to_null"), + ("main", "0095_grant_sync_deployment_to_mldatamanager"), + ] + + operations = [ + migrations.CreateModel( + name="AlgorithmResult", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "kind", + models.CharField( + choices=[("class_masking", "Class Masking"), ("size_filter", "Size Filter")], max_length=32 + ), + ), + ("value", models.FloatField(blank=True, null=True)), + ("data", models.JSONField(blank=True, default=dict)), + ("is_current", models.BooleanField(default=True)), + ("timestamp", models.DateTimeField(default=django.utils.timezone.now)), + ( + "algorithm", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="algorithm_results", + to="ml.algorithm", + ), + ), + ( + "job", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="algorithm_results", + to="jobs.job", + ), + ), + ( + "occurrence", + models.ForeignKey( + db_index=False, + on_delete=django.db.models.deletion.CASCADE, + related_name="algorithm_results", + to="main.occurrence", + ), + ), + ( + "project", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="algorithm_results", + to="main.project", + ), + ), + ], + ), + migrations.AddIndex( + model_name="algorithmresult", + index=models.Index(fields=["occurrence", "-timestamp"], name="algorithm_result_occ_time"), + ), + migrations.AddIndex( + model_name="algorithmresult", + index=models.Index( + condition=models.Q(("is_current", True), ("value__isnull", False)), + fields=["project", "kind", "value"], + name="algorithm_result_current_value", + ), + ), + migrations.AddConstraint( + model_name="algorithmresult", + constraint=models.UniqueConstraint( + condition=models.Q(("is_current", True)), + fields=("occurrence", "algorithm", "kind"), + name="algorithm_result_current_occurrence", + ), + ), + ] diff --git a/ami/main/migrations/0096_algorithm_results_and_validation_reviews.py b/ami/main/migrations/0096_algorithm_results_and_validation_reviews.py deleted file mode 100644 index cfb2b9afc..000000000 --- a/ami/main/migrations/0096_algorithm_results_and_validation_reviews.py +++ /dev/null @@ -1,157 +0,0 @@ -# Generated by Django 4.2.10 on 2026-10-02 22:19 - -from django.conf import settings -from django.db import migrations, models -import django.db.models.deletion -import django.utils.timezone - - -class Migration(migrations.Migration): - dependencies = [ - ("jobs", "0023_alter_job_job_type_key"), - ("ml", "0028_normalize_empty_endpoint_url_to_null"), - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ("main", "0095_grant_sync_deployment_to_mldatamanager"), - ] - - operations = [ - migrations.CreateModel( - name="AlgorithmResult", - fields=[ - ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), - ("created_at", models.DateTimeField(auto_now_add=True)), - ("updated_at", models.DateTimeField(auto_now=True)), - ( - "kind", - models.CharField( - choices=[("class_masking", "Class Masking"), ("size_filter", "Size Filter")], max_length=32 - ), - ), - ("data", models.JSONField(blank=True, default=dict)), - ("is_current", models.BooleanField(default=True)), - ("timestamp", models.DateTimeField(default=django.utils.timezone.now)), - ( - "algorithm", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name="algorithm_results", - to="ml.algorithm", - ), - ), - ( - "job", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - related_name="algorithm_results", - to="jobs.job", - ), - ), - ( - "occurrence", - models.ForeignKey( - db_index=False, - on_delete=django.db.models.deletion.CASCADE, - related_name="algorithm_results", - to="main.occurrence", - ), - ), - ( - "project", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name="algorithm_results", - to="main.project", - ), - ), - ], - ), - migrations.CreateModel( - name="ValidationReview", - fields=[ - ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), - ("created_at", models.DateTimeField(auto_now_add=True)), - ("updated_at", models.DateTimeField(auto_now=True)), - ("aspect", models.CharField(choices=[("comment", "Comment")], max_length=32)), - ( - "verdict", - models.CharField( - blank=True, - choices=[("confirmed", "Confirmed"), ("rejected", "Rejected")], - max_length=16, - null=True, - ), - ), - ("payload", models.JSONField(blank=True, default=dict)), - ("comment", models.TextField(blank=True, default="")), - ("withdrawn", models.BooleanField(default=False)), - ("timestamp", models.DateTimeField(default=django.utils.timezone.now)), - ( - "occurrence", - models.ForeignKey( - db_index=False, - on_delete=django.db.models.deletion.CASCADE, - related_name="validation_reviews", - to="main.occurrence", - ), - ), - ( - "project", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name="validation_reviews", - to="main.project", - ), - ), - ( - "reviewed_result", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - related_name="reviews", - to="main.algorithmresult", - ), - ), - ( - "user", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - related_name="validation_reviews", - to=settings.AUTH_USER_MODEL, - ), - ), - ], - options={ - "indexes": [models.Index(fields=["occurrence", "-timestamp"], name="validation_review_occ_time")], - }, - ), - migrations.AddConstraint( - model_name="validationreview", - constraint=models.CheckConstraint( - check=models.Q( - models.Q( - ("aspect", "comment"), ("verdict__isnull", True), models.Q(("comment", ""), _negated=True) - ), - models.Q(models.Q(("aspect", "comment"), _negated=True), ("verdict__isnull", False)), - _connector="OR", - ), - name="validation_review_verdict_by_aspect", - ), - ), - migrations.AddIndex( - model_name="algorithmresult", - index=models.Index(fields=["occurrence", "-timestamp"], name="algorithm_result_occ_time"), - ), - migrations.AddConstraint( - model_name="algorithmresult", - constraint=models.UniqueConstraint( - condition=models.Q(("is_current", True)), - fields=("occurrence", "algorithm", "kind"), - name="algorithm_result_current_occurrence", - ), - ), - ] diff --git a/ami/main/migrations/0097_output_jobs_and_results_on_classifications.py b/ami/main/migrations/0097_output_jobs_and_results_on_classifications.py index fa8270eba..4231e8867 100644 --- a/ami/main/migrations/0097_output_jobs_and_results_on_classifications.py +++ b/ami/main/migrations/0097_output_jobs_and_results_on_classifications.py @@ -8,7 +8,7 @@ class Migration(migrations.Migration): dependencies = [ ("jobs", "0023_alter_job_job_type_key"), - ("main", "0096_algorithm_results_and_validation_reviews"), + ("main", "0096_algorithm_results"), ] operations = [ diff --git a/ami/main/models.py b/ami/main/models.py index 05d48841f..4e7e86224 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -42,7 +42,7 @@ build_taxa_recursive_filter_q, ) from ami.main.models_future.projects import ProjectSettingsMixin -from ami.main.schemas import validate_result_data, validate_review_payload +from ami.main.schemas import validate_result_data from ami.ml.schemas import BoundingBox from ami.users.models import User from ami.utils.media import calculate_file_checksum, extract_timestamp, fetch_image_content @@ -3840,12 +3840,6 @@ def save(self, update_determination=True, *args, **kwargs): else: self.save(update_determination=False) - def check_custom_permission(self, user, action: str) -> bool: - # Leaving a comment (the reviews action) takes the rights to identify the occurrence. - if action == "reviews": - return user.has_perm(Project.Permissions.CREATE_IDENTIFICATION, self.get_project()) - return super().check_custom_permission(user, action) - class Meta: ordering = ["-determination_score"] indexes = [ @@ -3971,10 +3965,11 @@ class AlgorithmResult(BaseModel): The determination never reads this table. A run that changes an occurrence's taxon does so through the ``Classification`` rows it creates, and those rows point back here through ``Classification.algorithm_result`` so the history can show a run with what it changed. - ``data`` holds the details, validated against the schema for ``kind`` (ami/main/schemas.py). - A new result for the same occurrence, algorithm and kind becomes the current one and the - earlier ones stay as history. Write through ``AlgorithmResult.objects.record`` or - ``record_many``. See #1431. + ``data`` holds the run's own figures, validated against the model for ``kind`` + (ami/main/schemas.py), and ``value`` repeats the one figure lists filter and sort on. A new + result for the same occurrence, algorithm and kind becomes the current one and the earlier + ones stay as history. Write through ``AlgorithmResult.objects.record`` or ``record_many``. + Tracking and rank roll-ups are the next kinds expected. See #1431. """ class Kind(models.TextChoices): @@ -3994,6 +3989,9 @@ class Kind(models.TextChoices): "jobs.Job", on_delete=models.SET_NULL, null=True, blank=True, related_name="algorithm_results" ) kind = models.CharField(max_length=32, choices=Kind.choices) + # The kind's headline figure, for filtering and sorting: the excluded probability for class + # masking, the relative size for the size filter. + value = models.FloatField(null=True, blank=True) data = models.JSONField(default=dict, blank=True) is_current = models.BooleanField(default=True) timestamp = models.DateTimeField(default=timezone.now) @@ -4010,6 +4008,11 @@ class Meta: ] indexes = [ models.Index(fields=["occurrence", "-timestamp"], name="algorithm_result_occ_time"), + models.Index( + fields=["project", "kind", "value"], + condition=Q(is_current=True, value__isnull=False), + name="algorithm_result_current_value", + ), ] def __str__(self) -> str: @@ -4024,65 +4027,6 @@ def save(self, *args, **kwargs): super().save(*args, **kwargs) -@final -class ValidationReview(BaseModel): - """What a person said about one occurrence: for now a comment; tracking adds its grouping verdict. - - ``aspect`` names what was reviewed and ``verdict`` the person's call on it; a comment has - no verdict. ``reviewed_result`` is the algorithm result the review answers, when there is - one. ``withdrawn`` marks a review taken back without deleting it. See #1431. - """ - - class Aspect(models.TextChoices): - COMMENT = "comment" - - class Verdict(models.TextChoices): - CONFIRMED = "confirmed" - REJECTED = "rejected" - - project = models.ForeignKey("main.Project", on_delete=models.CASCADE, related_name="validation_reviews") - # Indexed together with the timestamp, below. - occurrence = models.ForeignKey( - Occurrence, on_delete=models.CASCADE, related_name="validation_reviews", db_index=False - ) - user = models.ForeignKey( - settings.AUTH_USER_MODEL, on_delete=models.SET_NULL, null=True, blank=True, related_name="validation_reviews" - ) - aspect = models.CharField(max_length=32, choices=Aspect.choices) - verdict = models.CharField(max_length=16, choices=Verdict.choices, null=True, blank=True) - reviewed_result = models.ForeignKey( - AlgorithmResult, on_delete=models.SET_NULL, null=True, blank=True, related_name="reviews" - ) - payload = models.JSONField(default=dict, blank=True) - comment = models.TextField(blank=True, default="") - withdrawn = models.BooleanField(default=False) - timestamp = models.DateTimeField(default=timezone.now) - - class Meta: - constraints = [ - # A comment carries text and no verdict; every other aspect needs a verdict. - models.CheckConstraint( - check=(Q(aspect="comment", verdict__isnull=True) & ~Q(comment="")) - | (~Q(aspect="comment") & Q(verdict__isnull=False)), - name="validation_review_verdict_by_aspect", - ), - ] - indexes = [ - models.Index(fields=["occurrence", "-timestamp"], name="validation_review_occ_time"), - ] - - def __str__(self) -> str: - return f"#{self.pk} {self.aspect} {self.verdict or ''} by User #{self.user_id}".strip() - - def save(self, *args, **kwargs): - self.payload = validate_review_payload(self.aspect, self.payload) - if self.project_id is None: - self.project_id = occurrence_project_ids([self.occurrence])[self.occurrence_id] - if self.project_id is None: - raise ValueError("The occurrence has no project, directly or through its station.") - super().save(*args, **kwargs) - - def update_occurrence_determination( occurrence: Occurrence, current_determination: typing.Optional["Taxon"] = None, save=True ) -> bool: diff --git a/ami/main/models_future/history.py b/ami/main/models_future/history.py index 5fd8e89c9..27d5ca2c4 100644 --- a/ami/main/models_future/history.py +++ b/ami/main/models_future/history.py @@ -1,8 +1,8 @@ """An occurrence's history: what algorithms and people did to it, newest first. -What a post-processing run decided about an occurrence is an ``AlgorithmResult`` and what a -person said about it is a ``ValidationReview``. Identifications and predictions keep their own -tables, and ``occurrence_timeline`` merges all four into one list for the history endpoint. +What a post-processing run decided about an occurrence is an ``AlgorithmResult``. Identifications +and predictions keep their own tables, and ``occurrence_timeline`` merges all three into one list +for the history endpoint. """ from __future__ import annotations @@ -13,28 +13,12 @@ from django.db.models import Q -from ami.main.models import ( - AlgorithmResult, - Classification, - Identification, - Occurrence, - TaxaList, - Taxon, - User, - ValidationReview, -) +from ami.main.models import AlgorithmResult, Classification, Identification, Occurrence, TaxaList, Taxon, User from ami.ml.models.algorithm import Algorithm if typing.TYPE_CHECKING: from ami.jobs.models import Job -COMMENT = ValidationReview.Aspect.COMMENT - - -def add_comment(occurrence: Occurrence, user: User, comment: str) -> ValidationReview: - """Leave a note on an occurrence: a review with ``aspect = comment`` and no verdict.""" - return ValidationReview.objects.create(occurrence=occurrence, user=user, aspect=COMMENT, comment=comment) - @dataclasses.dataclass class TimelineEntry: @@ -51,33 +35,20 @@ class TimelineEntry: taxon_before: Taxon | None = None score: float | None = None payload: dict = dataclasses.field(default_factory=dict) - verdict: str | None = None comment: str = "" withdrawn: bool = False is_current: bool | None = None # For a result: the classifications the run created, best score first. classifications: list[Classification] = dataclasses.field(default_factory=list) - # For a class masking result: the species list and the classifier it re-scored. + # For a class masking result: the source classifier's top taxon before masking, the species + # list and the classifier, the latter two from the job's settings. + original_taxon: Taxon | None = None taxa_list: TaxaList | None = None source_algorithm: Algorithm | None = None -def review_entry(review: ValidationReview) -> TimelineEntry: - return TimelineEntry( - type="review", - id=review.pk, - timestamp=review.timestamp, - subtype=review.aspect, - user=review.user, - payload=review.payload, - verdict=review.verdict, - comment=review.comment, - withdrawn=review.withdrawn, - ) - - def occurrence_timeline(occurrence: Occurrence) -> list[TimelineEntry]: - """Results, reviews, identifications and predictions of one occurrence, merged newest first. + """Results, identifications and predictions of one occurrence, merged newest first. A result comes with the classifications its run created, so the history shows the run and what it changed as one entry; those classifications are not listed again as predictions. @@ -91,10 +62,17 @@ def occurrence_timeline(occurrence: Occurrence) -> list[TimelineEntry]: .select_related("algorithm", "job") .order_by("-timestamp", "-pk") ) - taxon_ids = {result.data.get(key) for result in results for key in ("taxon_before_id", "taxon_after_id")} - {None} - taxa = {taxon.pk: taxon for taxon in Taxon.objects.filter(pk__in=taxon_ids)} if taxon_ids else {} - taxa_lists = _by_id(TaxaList, {result.data.get("taxa_list_id") for result in results}) - source_algorithms = _by_id(Algorithm, {result.data.get("source_algorithm_id") for result in results}) + settings = {result.pk: _job_settings(result.job) for result in results} + taxa = _by_id( + Taxon, + { + result.data.get(key) + for result in results + for key in ("determination_before_id", "determination_after_id", "original_taxon_id") + }, + ) + taxa_lists = _by_id(TaxaList, {config.get("taxa_list_id") for config in settings.values()}) + source_algorithms = _by_id(Algorithm, {config.get("algorithm_id") for config in settings.values()}) created_by_result = _classifications_created_by(results, occurrence) created_ids = {c.pk for created in created_by_result.values() for c in created} rescored_by = { @@ -120,20 +98,19 @@ def superseded_by(prediction: Classification) -> int | None: subtype=result.kind, algorithm=result.algorithm, job=result.job, - taxon=taxa.get(result.data.get("taxon_after_id")), - taxon_before=taxa.get(result.data.get("taxon_before_id")), + taxon=taxa.get(result.data.get("determination_after_id")), + taxon_before=taxa.get(result.data.get("determination_before_id")), + score=result.value, payload=result.data, is_current=result.is_current, classifications=created_by_result.get(result.pk, []), - taxa_list=taxa_lists.get(result.data.get("taxa_list_id")), - source_algorithm=source_algorithms.get(result.data.get("source_algorithm_id")), + original_taxon=taxa.get(result.data.get("original_taxon_id")), + taxa_list=taxa_lists.get(settings[result.pk].get("taxa_list_id")), + source_algorithm=source_algorithms.get(settings[result.pk].get("algorithm_id")), ) for result in results ] - reviews = ValidationReview.objects.filter(occurrence=occurrence).select_related("user") - entries.extend(review_entry(review) for review in reviews) - identifications = Identification.objects.filter(occurrence=occurrence).select_related("user", "taxon") entries.extend( TimelineEntry( @@ -177,6 +154,12 @@ def superseded_by(prediction: Classification) -> int | None: return entries +def _job_settings(job: Job | None) -> dict: + """The settings a post-processing job ran with, or an empty dict for any other job.""" + config = (job.params or {}).get("config") if job is not None else None + return config if isinstance(config, dict) else {} + + def _by_id(model, ids: set) -> dict: ids.discard(None) return {row.pk: row for row in model.objects.filter(pk__in=ids)} if ids else {} diff --git a/ami/main/schemas.py b/ami/main/schemas.py index 676a26bbb..e6d72ae6c 100644 --- a/ami/main/schemas.py +++ b/ami/main/schemas.py @@ -1,66 +1,73 @@ -"""Shapes of the JSON on algorithm results and validation reviews, one per kind or aspect. +"""The shape of ``AlgorithmResult.data``, one pydantic model per result kind. -``AlgorithmResult.data`` and ``ValidationReview.payload`` are JSON columns, so these schemas keep -each kind's shape fixed: every write validates against the schema registered for it, and a kind -or aspect with no schema cannot be written. +Each model records only what the run alone knows: figures computed while it ran and the +occurrence's determination before and after, which cannot be derived later. Anything that lives +elsewhere is left out: the settings are on the job, the classifications a run created point at +the result, and the new taxon is on those classifications. ``extra`` takes whatever else a +method or processing service returns, kept as JSON and not interpreted. + +Adding a kind means adding a model here and registering it in ``ALGORITHM_RESULT_DATA_SCHEMAS``; +tracking (merges, statistics) and rank roll-ups (``rank_rollup``) are the next ones expected. """ +import json +from typing import Any + import pydantic -class OutputPayload(pydantic.BaseModel): +class ResultData(pydantic.BaseModel): + # Whatever else the method returned, stored as JSON and never read by Antenna. + extra: dict[str, Any] = {} + + @pydantic.validator("extra") + def _extra_is_json(cls, value: dict[str, Any]) -> dict[str, Any]: + return json.loads(json.dumps(value)) + class Config: extra = "forbid" -class DeterminationChangeData(OutputPayload): +class DeterminationSnapshot(ResultData): # The occurrence's determination before and after the run, as taxon ids. Equal when the # run changed predictions without moving the determination. - taxon_before_id: int | None = None - taxon_after_id: int | None = None + determination_before_id: int | None = None + determination_after_id: int | None = None -class ClassMaskingResultData(DeterminationChangeData): - taxa_list_id: int - source_algorithm_id: int - detection_ids: list[int] +class ClassMaskingResultData(DeterminationSnapshot): + """Figures from the occurrence's winning detection: the one whose masked classification scores highest.""" + # The share of the source classifier's probability on classes outside the species list: + # one minus the kept mass of the unmasked softmax. + excluded_probability: float + # The source classifier's top prediction before masking. + original_taxon_id: int | None = None + original_score: float | None = None + # Where the class that wins after masking ranked before it; 1 means it was already the top. + new_winner_original_rank: int | None = None -class SizeFilterResultData(DeterminationChangeData): - size_threshold: float - detection_ids: list[int] +class SizeFilterResultData(DeterminationSnapshot): + """Figures from the occurrence's smallest filtered detection.""" -class CommentReviewPayload(OutputPayload): - """A comment's text is in ``ValidationReview.comment``; the payload stays empty.""" + # The detection's box area as a fraction of its image. + relative_size: float -ALGORITHM_RESULT_DATA_SCHEMAS: dict[str, type[OutputPayload]] = { +ALGORITHM_RESULT_DATA_SCHEMAS: dict[str, type[ResultData]] = { "class_masking": ClassMaskingResultData, "size_filter": SizeFilterResultData, } -VALIDATION_REVIEW_PAYLOAD_SCHEMAS: dict[str, type[OutputPayload]] = { - "comment": CommentReviewPayload, -} - -def _validate(schemas: dict[str, type[OutputPayload]], name: str, value: dict | OutputPayload) -> dict: - schema = schemas.get(name) +def validate_result_data(kind: str, data: dict | ResultData | None) -> dict: + """``data`` as JSON-ready values, or a ValueError when it does not fit the kind's model.""" + schema = ALGORITHM_RESULT_DATA_SCHEMAS.get(kind) if schema is None: - raise ValueError(f"No payload schema is registered for {name!r}.") - if isinstance(value, OutputPayload) and not isinstance(value, schema): - raise ValueError(f"{type(value).__name__} is not the payload schema for {name!r}.") - data = value.dict() if isinstance(value, OutputPayload) else value + raise ValueError(f"No result data model is registered for {kind!r}.") + if isinstance(data, ResultData) and not isinstance(data, schema): + raise ValueError(f"{type(data).__name__} is not the result data model for {kind!r}.") + values = data.dict() if isinstance(data, ResultData) else (data or {}) # Round-trip through JSON so datetimes are stored as ISO strings. - return pydantic.parse_raw_as(dict, schema.parse_obj(data).json()) - - -def validate_result_data(kind: str, data: dict | OutputPayload | None) -> dict: - """``data`` as JSON-ready values, or a ValueError when it does not fit the kind's schema.""" - return _validate(ALGORITHM_RESULT_DATA_SCHEMAS, kind, data or {}) - - -def validate_review_payload(aspect: str, payload: dict | OutputPayload | None) -> dict: - """``payload`` as JSON-ready values, or a ValueError when it does not fit the aspect's schema.""" - return _validate(VALIDATION_REVIEW_PAYLOAD_SCHEMAS, aspect, payload or {}) + return pydantic.parse_raw_as(dict, schema.parse_obj(values).json()) diff --git a/ami/main/test_occurrence_history.py b/ami/main/test_occurrence_history.py index c9595c054..5c94a152c 100644 --- a/ami/main/test_occurrence_history.py +++ b/ami/main/test_occurrence_history.py @@ -1,4 +1,4 @@ -"""Algorithm results, validation reviews, and the occurrence history endpoint that reads them.""" +"""Algorithm results and the occurrence history endpoint that reads them.""" import contextlib import datetime @@ -20,18 +20,17 @@ SourceImage, TaxaList, Taxon, - ValidationReview, ) from ami.ml.models import Algorithm from ami.tests.fixtures.main import create_captures, create_taxa, setup_test_project from ami.users.models import User -from ami.users.roles import BasicMember, Identifier, ProjectManager +from ami.users.roles import BasicMember, ProjectManager # Measured: two savepoints, the project, the occurrence, then the results with their algorithm and -# job, their taxa, the classifications they created, reviews, identifications and predictions. A -# class masking result adds one query each for the species lists and source algorithms, whatever +# job, their taxa, the classifications they created, identifications and predictions. A class +# masking job's settings add one query each for the species lists and source algorithms, whatever # the number of entries. -HISTORY_QUERIES = 10 +HISTORY_QUERIES = 9 SIZE_FILTER = AlgorithmResult.Kind.SIZE_FILTER CLASS_MASKING = AlgorithmResult.Kind.CLASS_MASKING @@ -62,9 +61,10 @@ def _size_filter(self, data: dict | None, **fields) -> AlgorithmResult: def test_data_that_does_not_fit_its_kind_is_refused_on_every_write_path(self): for label, data in ( - ("missing field", {"size_threshold": 0.001}), - ("wrong type", {"size_threshold": "small", "detection_ids": []}), - ("unknown field", {"size_threshold": 0.001, "detection_ids": [], "note": "x"}), + ("missing field", {}), + ("wrong type", {"relative_size": "small"}), + ("unknown field", {"relative_size": 0.001, "note": "x"}), + ("extra that is not JSON", {"relative_size": 0.001, "extra": {"when": object()}}), ): with self.subTest(label): with self.assertRaises(ValueError): @@ -81,9 +81,12 @@ def test_data_that_does_not_fit_its_kind_is_refused_on_every_write_path(self): occurrence=self.occurrence, algorithm=self.algorithm, kind=SIZE_FILTER, - data={"size_threshold": 0.01, "detection_ids": []}, + value=0.001, + data={"relative_size": 0.001, "extra": {"service": "size-filter 1.2"}}, ) - result.data["size_threshold"] = "small" + self.assertEqual(result.value, 0.001) + self.assertEqual(result.data["extra"], {"service": "size-filter 1.2"}) + result.data["relative_size"] = "small" with self.assertRaises(ValueError): AlgorithmResult.objects.bulk_update([result], ["data"]) @@ -92,13 +95,13 @@ def test_a_new_result_becomes_current_and_the_earlier_one_stays_as_history(self) occurrence=self.occurrence, algorithm=self.algorithm, kind=SIZE_FILTER, - data={"size_threshold": 0.01, "detection_ids": [1]}, + data={"relative_size": 0.01}, ) other_occurrence = Occurrence.objects.create(project=self.project, deployment=self.deployment) second, untouched = AlgorithmResult.objects.record_many( [ - self._size_filter({"size_threshold": 0.02, "detection_ids": [1]}), - self._size_filter({"size_threshold": 0.02, "detection_ids": [2]}, occurrence=other_occurrence), + self._size_filter({"relative_size": 0.02}), + self._size_filter({"relative_size": 0.02}, occurrence=other_occurrence), ] ) first.refresh_from_db() @@ -108,66 +111,35 @@ def test_a_new_result_becomes_current_and_the_earlier_one_stays_as_history(self) self.assertEqual(AlgorithmResult.objects.for_occurrence(self.occurrence).current().get().pk, second.pk) def test_the_database_holds_one_current_result_per_occurrence_algorithm_and_kind(self): - self._size_filter({"size_threshold": 0.01, "detection_ids": []}).save() + self._size_filter({"relative_size": 0.01}).save() with transaction.atomic(), self.assertRaises(IntegrityError): - self._size_filter({"size_threshold": 0.02, "detection_ids": []}).save() + self._size_filter({"relative_size": 0.02}).save() def test_project_comes_from_the_occurrence_or_its_station_and_a_result_with_neither_is_skipped(self): other = Project.objects.create(name="Another project") Occurrence.objects.filter(pk=self.occurrence.pk).update(project=None) self.occurrence.refresh_from_db() - result = self._size_filter({"size_threshold": 0.01, "detection_ids": []}) + result = self._size_filter({"relative_size": 0.01}) result.save() self.assertEqual(result.project_id, self.project.pk) Occurrence.objects.filter(pk=self.occurrence.pk).update(project=other) self.occurrence.refresh_from_db() self.assertEqual( - AlgorithmResult.objects.record_many([self._size_filter({"size_threshold": 0.02, "detection_ids": []})])[ - 0 - ].project_id, + AlgorithmResult.objects.record_many([self._size_filter({"relative_size": 0.02})])[0].project_id, other.pk, ) Occurrence.objects.filter(pk=self.occurrence.pk).update(project=None, deployment=None) self.occurrence.refresh_from_db() with self.assertLogs("ami.main.models", level="WARNING"): - written = AlgorithmResult.objects.record_many( - [self._size_filter({"size_threshold": 0.03, "detection_ids": []})] - ) + written = AlgorithmResult.objects.record_many([self._size_filter({"relative_size": 0.03})]) self.assertEqual(written, []) with self.assertRaises(ValueError): - self._size_filter({"size_threshold": 0.03, "detection_ids": []}).save() + self._size_filter({"relative_size": 0.03}).save() self.assertEqual(AlgorithmResult.objects.count(), 2) -class ValidationReviewTestCase(TestCase): - def setUp(self) -> None: - self.project, self.deployment = setup_test_project(reuse=False) - self.occurrence = Occurrence.objects.create(project=self.project, deployment=self.deployment) - self.user = User.objects.create_user(email="review-model@insectai.org") # type: ignore - - def test_a_comment_has_text_and_no_verdict(self): - for label, fields in ( - ("comment with a verdict", {"comment": "x", "verdict": "confirmed"}), - ("empty comment", {"comment": ""}), - ): - with self.subTest(label), transaction.atomic(), self.assertRaises(IntegrityError): - ValidationReview.objects.create(occurrence=self.occurrence, user=self.user, aspect="comment", **fields) - with self.assertRaises(ValueError): - ValidationReview.objects.create( - occurrence=self.occurrence, user=self.user, aspect="comment", comment="x", payload={"extra": 1} - ) - - def test_a_review_takes_its_occurrences_project(self): - Occurrence.objects.filter(pk=self.occurrence.pk).update(project=None) - self.occurrence.refresh_from_db() - review = ValidationReview.objects.create( - occurrence=self.occurrence, user=self.user, aspect="comment", comment="x" - ) - self.assertEqual(review.project_id, self.project.pk) - - class OccurrenceFixtureTestCase(APITestCase): """One occurrence of four classified detections, with a project manager and a basic member.""" @@ -223,7 +195,7 @@ def get(self, user: User | None = None): return response.data def _add_history(self, start: datetime.datetime, rounds: int = 1) -> None: - """Per round: a size filter result with the classification it created, an identification and a comment.""" + """Per round: a size filter result with the classification it created, and an identification.""" for i in range(rounds): at = start + datetime.timedelta(hours=4 * i) result = AlgorithmResult.objects.record( @@ -231,11 +203,11 @@ def _add_history(self, start: datetime.datetime, rounds: int = 1) -> None: algorithm=self.size_filter, job=self.job, kind=SIZE_FILTER, + value=0.001, data={ - "size_threshold": 0.01, - "detection_ids": [self.detections[0].pk], - "taxon_before_id": self.other_taxon.pk, - "taxon_after_id": self.taxon.pk, + "relative_size": 0.001, + "determination_before_id": self.other_taxon.pk, + "determination_after_id": self.taxon.pk, }, timestamp=at, ) @@ -252,13 +224,6 @@ def _add_history(self, start: datetime.datetime, rounds: int = 1) -> None: occurrence=self.occurrence, user=self.reader, taxon=self.other_taxon, comment=f"round {i}" ) Identification.objects.filter(pk=identification.pk).update(created_at=at + datetime.timedelta(hours=1)) - ValidationReview.objects.create( - occurrence=self.occurrence, - user=self.manager, - aspect="comment", - comment=f"note {i}", - timestamp=at + datetime.timedelta(hours=2), - ) def test_entries_are_merged_newest_first_and_a_result_carries_the_classifications_it_created(self): # Predictions are stamped with created_at (now), so the history rows sit in the past. @@ -266,18 +231,16 @@ def test_entries_are_merged_newest_first_and_a_result_carries_the_classification data = self.get() types = [entry["type"] for entry in data] - self.assertEqual(types[-3:], ["review", "identification", "algorithm_result"]) + self.assertEqual(types[-2:], ["identification", "algorithm_result"]) # The fixture's detections share one tied top prediction, shown once; the result's # classification is shown inside the result, not as a prediction of its own. - self.assertEqual(types[:-3], ["prediction"]) + self.assertEqual(types[:-2], ["prediction"]) self.assertIsNone(data[0]["algorithm"]) timestamps = [entry["timestamp"] for entry in data] self.assertEqual(timestamps, sorted(timestamps, reverse=True)) - review, identification, result = data[-3:] - self.assertEqual(set(review["user"]), {"id", "name", "image"}) - self.assertEqual(review["user"]["id"], self.manager.pk) - self.assertEqual((review["subtype"], review["comment"], review["verdict"]), ("comment", "note 0", None)) + identification, result = data[-2:] + self.assertEqual(set(identification["user"]), {"id", "name", "image"}) self.assertEqual(identification["user"]["id"], self.reader.pk) self.assertEqual(identification["taxon"]["id"], self.other_taxon.pk) self.assertEqual(identification["payload"]["comment"], "round 0") @@ -287,6 +250,9 @@ def test_entries_are_merged_newest_first_and_a_result_carries_the_classification self.assertEqual(result["job"], {"id": self.job.pk, "name": self.job.name, "config": {"size_threshold": 0.01}}) self.assertEqual(result["taxon"]["id"], self.taxon.pk) self.assertEqual(result["taxon_before"]["id"], self.other_taxon.pk) + self.assertEqual(result["score"], 0.001) + self.assertEqual(result["payload"]["relative_size"], 0.001) + self.assertIsNone(result["original_taxon"]) created = Classification.objects.get(algorithm=self.size_filter) self.assertEqual( result["classifications"], @@ -308,19 +274,27 @@ def test_a_demoted_prediction_names_the_result_that_superseded_it(self): classifier = Algorithm.objects.create(name="Classifier", key="classifier-history-test") masker = Algorithm.objects.create(name="Masked classifier", key="masker-history-test") taxa_list = TaxaList.objects.create(name="Kept species") + job = Job.objects.create( + project=self.project, + name="Masking run", + job_type_key="post_processing", + params={"task": "class_masking", "config": {"taxa_list_id": taxa_list.pk, "algorithm_id": classifier.pk}}, + ) original = Classification.objects.get(detection=self.detections[1]) Classification.objects.filter(pk=original.pk).update(terminal=False, algorithm=classifier) result = AlgorithmResult.objects.record( occurrence=self.occurrence, algorithm=masker, - job=self.job, + job=job, kind=CLASS_MASKING, + value=0.4, data={ - "taxa_list_id": taxa_list.pk, - "source_algorithm_id": classifier.pk, - "detection_ids": [self.detections[1].pk], - "taxon_before_id": self.taxon.pk, - "taxon_after_id": self.other_taxon.pk, + "excluded_probability": 0.4, + "original_taxon_id": self.taxon.pk, + "original_score": 0.9, + "new_winner_original_rank": 2, + "determination_before_id": self.taxon.pk, + "determination_after_id": self.other_taxon.pk, }, ) masked = Classification.objects.create( @@ -330,7 +304,7 @@ def test_a_demoted_prediction_names_the_result_that_superseded_it(self): algorithm=masker, timestamp=datetime.datetime.now(), applied_to=original, - job=self.job, + job=job, algorithm_result=result, ) data = self.get() @@ -352,6 +326,8 @@ def test_a_demoted_prediction_names_the_result_that_superseded_it(self): self.assertIsNone(predictions[None]["payload"]["superseded_by_result_id"]) self.assertEqual(result_entry["taxa_list"], {"id": taxa_list.pk, "name": "Kept species"}) self.assertEqual(result_entry["source_algorithm"]["key"], classifier.key) + self.assertEqual(result_entry["original_taxon"]["id"], self.taxon.pk) + self.assertEqual(result_entry["score"], 0.4) def test_a_terminal_prediction_outranked_on_its_detection_is_superseded(self): """The size filter does not demote the prediction it outranks, so the link is the shared detection.""" @@ -362,7 +338,7 @@ def test_a_terminal_prediction_outranked_on_its_detection_is_superseded(self): algorithm=self.size_filter, job=self.job, kind=SIZE_FILTER, - data={"size_threshold": 0.01, "detection_ids": [self.detections[3].pk]}, + data={"relative_size": 0.001}, ) Classification.objects.create( detection=self.detections[3], @@ -414,7 +390,7 @@ def test_query_count_does_not_grow_with_the_entries_and_no_score_arrays_are_load with CaptureQueriesContext(connection) as queries, self.assertNumQueries(HISTORY_QUERIES): response = self.client.get(self.url()) self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.data), 3 * total + 1) + self.assertEqual(len(response.data), 2 * total + 1) self.assertFalse( [q["sql"] for q in queries.captured_queries if '"main_classification"."scores"' in q["sql"]] ) @@ -445,61 +421,3 @@ def test_an_occurrence_the_default_filters_hide_still_has_a_history(self): detail = self.client.get(f"/api/v2/occurrences/{self.occurrence.pk}/?project_id={self.project.pk}") self.assertEqual(detail.status_code, 404) self.assertEqual(self.client.get(self.url()).status_code, 200) - - -class OccurrenceCommentEndpointTestCase(OccurrenceFixtureTestCase): - """POST /occurrences/{id}/reviews/ leaves a comment, for whoever may identify on the project.""" - - def setUp(self) -> None: - super().setUp() - self.identifier = User.objects.create_user(email="comment-identifier@insectai.org") # type: ignore - Identifier.assign_user(self.identifier, self.project) - self.superuser = User.objects.create_superuser(email="comment-super@insectai.org") # type: ignore - self.outsider = User.objects.create_user(email="comment-outsider@insectai.org") # type: ignore - - def comment(self, user: User | None, body: dict | None = None): - self.client.force_authenticate(user=user) - return self.client.post( - f"/api/v2/occurrences/{self.occurrence.pk}/reviews/?project_id={self.project.pk}", - body if body is not None else {"comment": " The second detection is a different moth. "}, - format="json", - ) - - def test_a_comment_is_stored_as_a_review_and_returned_as_a_history_entry(self): - response = self.comment(self.identifier) - self.assertEqual(response.status_code, 201, response.data) - review = ValidationReview.objects.get() - self.assertEqual((review.aspect, review.verdict, review.user), ("comment", None, self.identifier)) - self.assertEqual(review.comment, "The second detection is a different moth.") - self.assertEqual(review.project_id, self.project.pk) - self.assertEqual((response.data["type"], response.data["id"]), ("review", review.pk)) - self.assertNotIn("email", str(response.data)) - - def test_an_empty_comment_or_another_aspect_is_refused(self): - for body in ({"comment": " "}, {"aspect": "grouping", "comment": "x"}): - with self.subTest(body=body): - self.assertEqual(self.comment(self.identifier, body).status_code, 400) - self.assertFalse(ValidationReview.objects.exists()) - - def test_permission_matrix(self): - """Whoever may identify may comment; a basic member, a non-member and anonymous may not.""" - expected = { - "identifier": 201, - "project manager": 201, - "superuser": 201, - "basic member": 403, - "non-member": 403, - "anonymous": 401, - } - users = { - "identifier": self.identifier, - "project manager": self.manager, - "superuser": self.superuser, - "basic member": self.reader, - "non-member": self.outsider, - "anonymous": None, - } - for label, user in users.items(): - with self.subTest(user=label): - self.assertEqual(self.comment(user).status_code, expected[label]) - self.assertEqual(ValidationReview.objects.count(), 3) diff --git a/ami/ml/post_processing/class_masking.py b/ami/ml/post_processing/class_masking.py index e69e55740..37bed8b8f 100644 --- a/ami/ml/post_processing/class_masking.py +++ b/ami/ml/post_processing/class_masking.py @@ -84,7 +84,8 @@ def make_classifications_filtered_by_taxa_list( Every occurrence with a re-scored classification gets one algorithm result, written in the same transaction as the batch that changes it, and its new classifications point at - that result (see ``BatchResults``). + that result (see ``BatchResults``). The result records the probability the list excluded + and the source's original top prediction for the occurrence's winning detection. Returns final counters (checked / masked / occurrences updated) for stage metrics. """ @@ -132,7 +133,7 @@ def make_classifications_filtered_by_taxa_list( kind=AlgorithmResult.Kind.CLASS_MASKING, algorithm=new_algorithm, job=job, - data={"taxa_list_id": taxa_list.pk, "source_algorithm_id": algorithm.pk}, + value_field="excluded_probability", timestamp=timestamp, ) @@ -210,7 +211,18 @@ def make_classifications_filtered_by_taxa_list( detection = classification.detection if detection is not None and detection.occurrence is not None: occurrences_to_update.add(detection.occurrence) - results.note(detection.occurrence, detection.pk) + # The occurrence's result carries the figures of its winning detection: + # the one whose masked classification scores highest (see BatchResults). + results.note( + detection.occurrence, + { + "excluded_probability": float(1.0 - kept_sum), + "original_taxon_id": classification.taxon_id, + "original_score": classification.score, + "new_winner_original_rank": int((full_softmax > full_softmax[top_index]).sum()) + 1, + }, + rank=score, + ) # Flush every batch_size items and at the final item. The flush fires even # when nothing was accumulated so the job health-check sees a heartbeat during diff --git a/ami/ml/post_processing/results.py b/ami/ml/post_processing/results.py index 9e87b9be9..7bdc3090a 100644 --- a/ami/ml/post_processing/results.py +++ b/ami/ml/post_processing/results.py @@ -4,13 +4,13 @@ each in its own transaction. ``BatchResults`` writes the result rows in those same transactions, so a batch either lands with its results or not at all. Every occurrence the run touches gets exactly one current result: it is created by the first batch that touches the occurrence, before -the classifications that batch inserts so they can point at it, and later batches extend it with -the detections they changed and the determination after their saves. +the classifications that batch inserts so they can point at it, and later batches update it with +the figures of the detection that represents the occurrence and the determination after their +saves. """ from __future__ import annotations -import collections import datetime import typing @@ -28,29 +28,32 @@ def __init__( kind: str, algorithm: Algorithm, job: Job | None, - data: dict, + value_field: str, timestamp: datetime.datetime | None = None, ): - """``data`` holds the fields every result of the run shares, such as the settings it ran with.""" + """``value_field`` names the figure in the result's data that is also stored in ``value``.""" self.kind = kind self.algorithm = algorithm self.job = job - self.data = data + self.value_field = value_field self.timestamp = timestamp self.results: dict[int, AlgorithmResult] = {} # Occurrences with no project, directly or through their station; tried once, then left alone. self.skipped: set[int] = set() - self.changed_detections: dict[int, set[int]] = collections.defaultdict(set) + # Per occurrence, the figures of the detection that represents it so far, with their rank. + self.figures: dict[int, tuple[float, dict]] = {} - def note(self, occurrence: Occurrence, detection_id: int) -> None: - """Record that the run changed one of the occurrence's detections, ahead of the batch write.""" - self.changed_detections[occurrence.pk].add(detection_id) + def note(self, occurrence: Occurrence, figures: dict, rank: float) -> None: + """Offer one changed detection's figures for the occurrence; the highest ``rank`` wins the result.""" + current = self.figures.get(occurrence.pk) + if current is None or rank > current[0]: + self.figures[occurrence.pk] = (rank, figures) def start_batch(self, occurrences: typing.Iterable[Occurrence], classifications: typing.Iterable[Classification]): """Create the results for occurrences the run has not seen and point the batch's new classifications at theirs. Call inside the batch transaction, before the classifications are inserted and before the - occurrences are saved, so a result's ``taxon_before_id`` is the determination the run found. + occurrences are saved, so a result's ``determination_before_id`` is what the run found. """ new = [o for o in occurrences if o.pk not in self.results and o.pk not in self.skipped] written = AlgorithmResult.objects.record_many( @@ -59,12 +62,8 @@ def start_batch(self, occurrences: typing.Iterable[Occurrence], classifications: algorithm=self.algorithm, job=self.job, kind=self.kind, - data={ - **self.data, - "detection_ids": [], - "taxon_before_id": occurrence.determination_id, - "taxon_after_id": None, - }, + value=self.figures[occurrence.pk][1][self.value_field], + data={**self.figures[occurrence.pk][1], "determination_before_id": occurrence.determination_id}, **({"timestamp": self.timestamp} if self.timestamp else {}), ) for occurrence in new @@ -76,15 +75,15 @@ def start_batch(self, occurrences: typing.Iterable[Occurrence], classifications: classification.algorithm_result = self.results.get(classification.detection.occurrence_id) def finish_batch(self, occurrences: typing.Iterable[Occurrence]) -> None: - """Record each occurrence's determination after the batch's saves and the detections changed so far.""" + """Record each occurrence's determination after the batch's saves and its best figures so far.""" touched = [] for occurrence in occurrences: result = self.results.get(occurrence.pk) if result is None: continue - changed = self.changed_detections.pop(occurrence.pk, set()) - result.data["detection_ids"] = sorted(set(result.data["detection_ids"]) | changed) - result.data["taxon_after_id"] = occurrence.determination_id + figures = self.figures[occurrence.pk][1] + result.data = {**result.data, **figures, "determination_after_id": occurrence.determination_id} + result.value = figures[self.value_field] touched.append(result) if touched: - AlgorithmResult.objects.bulk_update(touched, ["data"]) + AlgorithmResult.objects.bulk_update(touched, ["data", "value"]) diff --git a/ami/ml/post_processing/small_size_filter.py b/ami/ml/post_processing/small_size_filter.py index 6ccbbee23..f05e0d5a3 100644 --- a/ami/ml/post_processing/small_size_filter.py +++ b/ami/ml/post_processing/small_size_filter.py @@ -105,7 +105,7 @@ def run(self) -> None: kind=AlgorithmResult.Kind.SIZE_FILTER, algorithm=self.algorithm, job=self.job, - data={"size_threshold": threshold}, + value_field="relative_size", ) def flush(i: int) -> None: @@ -196,7 +196,8 @@ def flush(i: int) -> None: detections_to_update.add(det) if det.occurrence is not None: occcurrences_to_update.add(det.occurrence) - results.note(det.occurrence, det.pk) + # The occurrence's result carries the size of its smallest flagged detection. + results.note(det.occurrence, {"relative_size": rel_area}, rank=-rel_area) self.logger.debug(f"Marking detection {det.pk} as {not_identifiable_taxon.name}") flush(i) diff --git a/ami/ml/post_processing/tests/test_class_masking.py b/ami/ml/post_processing/tests/test_class_masking.py index bf8839a86..cbc2a3add 100644 --- a/ami/ml/post_processing/tests/test_class_masking.py +++ b/ami/ml/post_processing/tests/test_class_masking.py @@ -407,14 +407,19 @@ def test_a_rescored_occurrence_gets_one_result_linked_to_the_classification_it_c self.assertTrue(result.is_current) self.assertEqual(result.project_id, self.project.pk) self.assertTrue(result.algorithm.key.startswith(f"{self.algorithm.key}_filtered_by_taxa_list_{taxa_list.pk}")) + # Index 2 held e^5 / (e^2 + e^1 + e^5) of the probability and was excluded; index 0 was second. + scores = _softmax([2.0, 1.0, 5.0]) + self.assertAlmostEqual(result.value, scores[2]) + self.assertAlmostEqual(result.data.pop("excluded_probability"), scores[2]) + self.assertAlmostEqual(result.data.pop("original_score"), scores[2]) self.assertEqual( result.data, { - "taxa_list_id": taxa_list.pk, - "source_algorithm_id": self.algorithm.pk, - "detection_ids": [det.pk], - "taxon_before_id": self.species_taxa[2].pk, - "taxon_after_id": self.species_taxa[0].pk, + "original_taxon_id": self.species_taxa[2].pk, + "new_winner_original_rank": 2, + "determination_before_id": self.species_taxa[2].pk, + "determination_after_id": self.species_taxa[0].pk, + "extra": {}, }, ) masked = result.classifications.get() @@ -454,9 +459,10 @@ def test_an_occurrence_spanning_batches_gets_one_result_covering_every_batch(sel ) result = AlgorithmResult.objects.get(occurrence=occ) - self.assertEqual(result.data["detection_ids"], sorted([det1.pk, det2.pk])) - self.assertEqual(result.data["taxon_after_id"], self.species_taxa[0].pk) - self.assertEqual(result.classifications.count(), 2) + self.assertEqual(result.data["determination_after_id"], self.species_taxa[0].pk) + self.assertEqual( + sorted(result.classifications.values_list("detection_id", flat=True)), sorted([det1.pk, det2.pk]) + ) def test_an_occurrence_without_a_project_is_still_rescored_but_gets_no_result(self): """A missing project must not fail a run after it has changed data; the result is skipped with a warning.""" diff --git a/ami/ml/post_processing/tests/test_small_size_filter.py b/ami/ml/post_processing/tests/test_small_size_filter.py index 544ddb5b6..f19957c1e 100644 --- a/ami/ml/post_processing/tests/test_small_size_filter.py +++ b/ami/ml/post_processing/tests/test_small_size_filter.py @@ -114,13 +114,15 @@ def test_a_flagged_occurrence_gets_one_result_linked_to_the_classifications_it_c detection_ids = sorted(occurrence.detections.values_list("pk", flat=True)) self.assertEqual((result.occurrence_id, result.project_id), (occurrence.pk, self.project.pk)) self.assertEqual(result.algorithm.key, "small_size_filter") + # The smallest flagged detection, 10 by 10 pixels on a 1000 by 1000 image, represents the occurrence. + self.assertAlmostEqual(result.value, 0.0001) + self.assertAlmostEqual(result.data.pop("relative_size"), 0.0001) self.assertEqual( result.data, { - "size_threshold": 0.01, - "detection_ids": detection_ids, - "taxon_before_id": self.taxon.pk, - "taxon_after_id": Taxon.objects.get(name="Not identifiable").pk, + "determination_before_id": self.taxon.pk, + "determination_after_id": Taxon.objects.get(name="Not identifiable").pk, + "extra": {}, }, ) self.assertEqual(sorted(result.classifications.values_list("detection_id", flat=True)), detection_ids) From d5bb406e0f08d9df2545ef9210daa3dc7a9e2e0c Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Fri, 2 Oct 2026 20:40:34 -0700 Subject: [PATCH 010/134] feat(ui): show a run's own figures on its result card and drop the comment card A class masking card now reads "Probability outside the list" and "Original top prediction" with its score, and a size filter card reads the detection's size as a share of the image next to the threshold the job ran with. The species list is still named, from the job's settings, and the determination before and after stays. The comment card is gone with the review model. Part of #1433. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01L52AN9tabp76yjhjyCZkSJ --- .../models/occurrence-history.test.ts | 43 +++-------- .../models/occurrence-history.ts | 38 ++++------ .../identification-card/algorithm-result.tsx | 75 +++++++++++++------ .../identification-card/comment-review.tsx | 37 --------- .../occurrence-timeline.tsx | 3 - ui/src/utils/language.ts | 19 +++-- 6 files changed, 91 insertions(+), 124 deletions(-) delete mode 100644 ui/src/pages/occurrence-details/identification-card/comment-review.tsx diff --git a/ui/src/data-services/models/occurrence-history.test.ts b/ui/src/data-services/models/occurrence-history.test.ts index 0442d0619..70686b064 100644 --- a/ui/src/data-services/models/occurrence-history.test.ts +++ b/ui/src/data-services/models/occurrence-history.test.ts @@ -21,6 +21,7 @@ const base = { comment: '', is_current: null, job: null, + original_taxon: null, score: null, source_algorithm: null, subtype: null, @@ -29,7 +30,6 @@ const base = { taxon_before: null, timestamp: '2026-04-29T22:00:00', user: null, - verdict: null, withdrawn: false, } @@ -66,15 +66,6 @@ const predictionEntry = ( type: 'prediction', }) -const comment = (id: number): ServerOccurrenceHistoryEntry => ({ - ...base, - comment: 'The second detection is another moth', - id, - payload: {}, - subtype: 'comment', - type: 'review', -}) - const classMasking: ServerOccurrenceHistoryEntry = { ...base, algorithm: { id: 12, key: 'mask', name: 'Masked classifier' }, @@ -98,13 +89,17 @@ const classMasking: ServerOccurrenceHistoryEntry = { ], id: 5, is_current: true, + original_taxon: XESTIA, payload: { - detection_ids: [1, 2], - source_algorithm_id: 7, - taxa_list_id: 2, - taxon_after_id: 3, - taxon_before_id: 4, + determination_after_id: 3, + determination_before_id: 4, + excluded_probability: 0.38, + extra: {}, + new_winner_original_rank: 2, + original_score: 0.83, + original_taxon_id: 4, }, + score: 0.38, subtype: 'class_masking', taxon: NOCTUA, taxon_before: XESTIA, @@ -138,34 +133,18 @@ const ownPrediction = ( describe('getTimelineItems', () => { test('maps each entry type to its card, keeping the server order', () => { const items = getTimelineItems({ - entries: [ - comment(9), - classMasking, - identificationEntry(2), - predictionEntry(6), - ], + entries: [classMasking, identificationEntry(2), predictionEntry(6)], identifications: [], predictions: [], }) expect(items.map((item) => item.type)).toEqual([ - 'review', 'algorithm_result', 'identification', 'prediction', ]) }) - test('leaves out a review aspect it has no card for', () => { - const items = getTimelineItems({ - entries: [{ ...comment(8), subtype: 'grouping' } as never], - identifications: [], - predictions: [], - }) - - expect(items).toEqual([]) - }) - test("reuses the occurrence's own records, which carry the viewer's permissions", () => { const prediction = ownPrediction('6', 7) const items = getTimelineItems({ diff --git a/ui/src/data-services/models/occurrence-history.ts b/ui/src/data-services/models/occurrence-history.ts index 3834bff0b..20075ecd2 100644 --- a/ui/src/data-services/models/occurrence-history.ts +++ b/ui/src/data-services/models/occurrence-history.ts @@ -44,21 +44,26 @@ export interface ServerHistoryClassification { } interface ServerDeterminationChangePayload { - taxon_after_id: number | null - taxon_before_id: number | null + determination_after_id: number | null + determination_before_id: number | null + /** Whatever else the method returned; stored as JSON and not interpreted. */ + extra: Record } export interface ServerClassMaskingPayload extends ServerDeterminationChangePayload { - detection_ids: number[] - source_algorithm_id: number - taxa_list_id: number + /** The share of the source classifier's probability on classes outside the species list. */ + excluded_probability: number + /** Where the class that wins after masking ranked before it; 1 means it was already the top. */ + new_winner_original_rank: number | null + original_score: number | null + original_taxon_id: number | null } export interface ServerSizeFilterPayload extends ServerDeterminationChangePayload { - detection_ids: number[] - size_threshold: number + /** The filtered detection's box area as a fraction of its image. */ + relative_size: number } export interface ServerIdentificationPayload { @@ -76,8 +81,6 @@ export interface ServerPredictionPayload { terminal: boolean | null } -export type ReviewVerdict = 'confirmed' | 'rejected' - interface ServerHistoryEntryBase { algorithm: ServerHistoryAlgorithm | null /** For a result: the classifications its run created, best score first. */ @@ -87,7 +90,10 @@ interface ServerHistoryEntryBase { /** For a result: whether it is the latest of its kind, not replaced by a later run. */ is_current: boolean | null job: ServerHistoryJob | null + /** For a class masking result: the source classifier's top taxon before masking. */ + original_taxon: ServerHistoryTaxon | null payload: Payload + /** A prediction's score, or a result's headline value. */ score: number | null /** For a class masking result: the classifier whose predictions the run re-scored. */ source_algorithm: ServerHistoryAlgorithm | null @@ -99,7 +105,6 @@ interface ServerHistoryEntryBase { timestamp: string type: Type user: ServerHistoryUser | null - verdict: ReviewVerdict | null withdrawn: boolean } @@ -116,12 +121,6 @@ export type SizeFilterResultEntry = ServerHistoryEntryBase< export type AlgorithmResultEntry = | ClassMaskingResultEntry | SizeFilterResultEntry -export type CommentReviewEntry = ServerHistoryEntryBase< - 'review', - 'comment', - Record -> -export type ReviewEntry = CommentReviewEntry export type IdentificationEntry = ServerHistoryEntryBase< 'identification', null, @@ -135,7 +134,6 @@ export type PredictionEntry = ServerHistoryEntryBase< export type ServerOccurrenceHistoryEntry = | AlgorithmResultEntry - | ReviewEntry | IdentificationEntry | PredictionEntry @@ -153,10 +151,8 @@ export type TimelineItem = supersededBy?: AlgorithmResultEntry } | { type: 'algorithm_result'; id: string; entry: AlgorithmResultEntry } - | { type: 'review'; id: string; entry: ReviewEntry } const ALGORITHM_RESULT_SUBTYPES: string[] = ['class_masking', 'size_filter'] -const REVIEW_SUBTYPES: string[] = ['comment'] export const convertHistoryTaxon = (taxon: ServerHistoryTaxon) => new Taxon({ ...taxon, id: `${taxon.id}`, cover_image_url: null }) @@ -265,10 +261,6 @@ export const getTimelineItems = ({ return isAlgorithmResult(entry) ? [{ type: 'algorithm_result', id, entry }] : [] - case 'review': - return REVIEW_SUBTYPES.includes(entry.subtype) - ? [{ type: 'review', id, entry }] - : [] default: return [] } diff --git a/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx b/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx index 0c793da91..f901d936e 100644 --- a/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx +++ b/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx @@ -65,7 +65,7 @@ const getSubTitle = (entry: AlgorithmResultEntry) => { list: entry.taxa_list?.name ?? translate(STRING.HISTORY_SPECIES_LIST_ID, { - id: `${entry.payload.taxa_list_id}`, + id: `${entry.job?.config?.taxa_list_id}`, }), }) } @@ -73,6 +73,10 @@ const getSubTitle = (entry: AlgorithmResultEntry) => { return entry.algorithm?.name } +/** A fraction as a percentage with at most one decimal, e.g. 0.0315 reads "3.2%". */ +const formatPercent = (fraction: number) => + `${Math.round(fraction * 1000) / 10}%` + /** The determination row, left out when there was no determination before or after. */ const getDeterminationStats = ( before: ServerHistoryTaxon | null, @@ -118,35 +122,64 @@ export const AlgorithmResult = ({ entry.taxon ) switch (entry.subtype) { - case 'class_masking': - stats.push({ - label: translate(STRING.HISTORY_SPECIES_LIST), - value: ( - - {entry.taxa_list?.name ?? - translate(STRING.HISTORY_SPECIES_LIST_ID, { - id: `${entry.payload.taxa_list_id}`, + case 'class_masking': { + const taxaListId = entry.taxa_list?.id ?? entry.job?.config?.taxa_list_id + if (taxaListId !== undefined) { + stats.push({ + label: translate(STRING.HISTORY_SPECIES_LIST), + value: ( + - ), + > + {entry.taxa_list?.name ?? + translate(STRING.HISTORY_SPECIES_LIST_ID, { + id: `${taxaListId}`, + })} + + ), + }) + } + stats.push({ + label: translate(STRING.HISTORY_EXCLUDED_PROBABILITY), + value: formatPercent(entry.payload.excluded_probability), }) + if (entry.original_taxon) { + stats.push({ + label: translate(STRING.HISTORY_ORIGINAL_PREDICTION), + value: + entry.payload.original_score !== null + ? `${ + entry.original_taxon.name + } (${entry.payload.original_score.toFixed(2)})` + : entry.original_taxon.name, + }) + } break - case 'size_filter': + } + case 'size_filter': { + const threshold = entry.job?.config?.size_threshold stats.push({ - label: translate(STRING.HISTORY_SIZE_THRESHOLD), - value: entry.payload.size_threshold, + label: translate(STRING.HISTORY_DETECTION_SIZE), + value: + typeof threshold === 'number' + ? translate(STRING.HISTORY_DETECTION_SIZE_WITH_THRESHOLD, { + size: formatPercent(entry.payload.relative_size), + threshold: formatPercent(threshold), + }) + : translate(STRING.HISTORY_DETECTION_SIZE_VALUE, { + size: formatPercent(entry.payload.relative_size), + }), }) break + } } stats.push({ label: translate(STRING.HISTORY_DETECTIONS_AFFECTED), - value: entry.payload.detection_ids.length, + value: entry.classifications.length, }) getJobSettings(entry.job).forEach(({ key, value }) => { const label = SETTING_LABELS[key] diff --git a/ui/src/pages/occurrence-details/identification-card/comment-review.tsx b/ui/src/pages/occurrence-details/identification-card/comment-review.tsx deleted file mode 100644 index 7d7773eb2..000000000 --- a/ui/src/pages/occurrence-details/identification-card/comment-review.tsx +++ /dev/null @@ -1,37 +0,0 @@ -import { CommentReviewEntry } from 'data-services/models/occurrence-history' -import { UserIcon } from 'lucide-react' -import { IdentificationCard } from 'nova-ui-kit' -import { getFormatedDateTimeString } from 'utils/date/getFormatedDateTimeString/getFormatedDateTimeString' -import { STRING, translate } from 'utils/language' -import { getUserLabel } from 'utils/user/getUserLabel' -import { HistoryTime, HistoryTypeBadge } from './history-stats' - -export const CommentReview = ({ entry }: { entry: CommentReviewEntry }) => ( -
- - - ) : ( - - ) - } - subTitle={ - entry.withdrawn ? translate(STRING.HISTORY_WITHDRAWN) : undefined - } - title={translate(STRING.HISTORY_COMMENT_BY, { - name: getUserLabel(entry.user), - })} - titleAddon={ - - } - > -

- {entry.comment} -

- -
-) diff --git a/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx b/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx index 1aa889950..0eb5bde1d 100644 --- a/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx +++ b/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx @@ -9,7 +9,6 @@ import { useMemo } from 'react' import { STRING, translate } from 'utils/language' import { UserInfo } from 'utils/user/types' import { AlgorithmResult, getResultKindLabel } from './algorithm-result' -import { CommentReview } from './comment-review' import { HumanIdentification } from './human-identification' import { MachinePrediction } from './machine-prediction' @@ -99,8 +98,6 @@ export const OccurrenceTimeline = ({ occurrence={occurrence} /> ) - case 'review': - return } })} diff --git a/ui/src/utils/language.ts b/ui/src/utils/language.ts index c77302cf8..81a1e489c 100644 --- a/ui/src/utils/language.ts +++ b/ui/src/utils/language.ts @@ -322,25 +322,26 @@ export enum STRING { FORGOT_PASSWORD, HISTORY_ALGORITHM_RESULT, HISTORY_CLASS_MASKING, - HISTORY_COMMENT, - HISTORY_COMMENT_BY, + HISTORY_DETECTION_SIZE, + HISTORY_DETECTION_SIZE_VALUE, + HISTORY_DETECTION_SIZE_WITH_THRESHOLD, HISTORY_DETECTIONS_AFFECTED, HISTORY_DETERMINATION, HISTORY_DETERMINATION_CHANGED, HISTORY_DETERMINATION_UNCHANGED, HISTORY_EMPTY, + HISTORY_EXCLUDED_PROBABILITY, HISTORY_MASKING_SUBTITLE, + HISTORY_ORIGINAL_PREDICTION, HISTORY_LOAD_ERROR, HISTORY_PREDICTION, HISTORY_SETTING_CAPTURE_SET, HISTORY_SETTING_OCCURRENCE, HISTORY_SETTING_REWEIGHT, HISTORY_SIZE_FILTER, - HISTORY_SIZE_THRESHOLD, HISTORY_SPECIES_LIST, HISTORY_SPECIES_LIST_ID, HISTORY_SUPERSEDED_BY, - HISTORY_WITHDRAWN, ID_APPLIED, INFO, INTERMEDIATE_CLASSIFICATION, @@ -781,14 +782,18 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.FORGOT_PASSWORD]: 'Forgot password?', [STRING.HISTORY_ALGORITHM_RESULT]: 'Algorithm result', [STRING.HISTORY_CLASS_MASKING]: 'Class masking', - [STRING.HISTORY_COMMENT]: 'Comment', - [STRING.HISTORY_COMMENT_BY]: 'Comment by {{name}}', + [STRING.HISTORY_DETECTION_SIZE]: 'Detection size', + [STRING.HISTORY_DETECTION_SIZE_VALUE]: '{{size}} of the image', + [STRING.HISTORY_DETECTION_SIZE_WITH_THRESHOLD]: + '{{size}} of the image (threshold {{threshold}})', [STRING.HISTORY_DETECTIONS_AFFECTED]: 'Detections changed', [STRING.HISTORY_DETERMINATION]: 'Determination', [STRING.HISTORY_DETERMINATION_CHANGED]: '{{before}} → {{after}}', [STRING.HISTORY_DETERMINATION_UNCHANGED]: '{{name}} (unchanged)', [STRING.HISTORY_EMPTY]: 'Nothing has happened to this occurrence yet.', + [STRING.HISTORY_EXCLUDED_PROBABILITY]: 'Probability outside the list', [STRING.HISTORY_MASKING_SUBTITLE]: '{{algorithm}}, filtered by {{list}}', + [STRING.HISTORY_ORIGINAL_PREDICTION]: 'Original top prediction', [STRING.HISTORY_LOAD_ERROR]: 'Could not load the full history. Showing identifications and predictions only.', [STRING.HISTORY_PREDICTION]: 'Prediction', @@ -796,11 +801,9 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.HISTORY_SETTING_OCCURRENCE]: 'Occurrence', [STRING.HISTORY_SETTING_REWEIGHT]: 'Re-weighted scores', [STRING.HISTORY_SIZE_FILTER]: 'Size filter', - [STRING.HISTORY_SIZE_THRESHOLD]: 'Size threshold', [STRING.HISTORY_SPECIES_LIST]: 'Species list', [STRING.HISTORY_SPECIES_LIST_ID]: 'Species list #{{id}}', [STRING.HISTORY_SUPERSEDED_BY]: 'Superseded by {{name}}', - [STRING.HISTORY_WITHDRAWN]: 'Withdrawn', [STRING.ID_APPLIED]: 'ID applied', [STRING.INFO]: 'Info', [STRING.INTERMEDIATE_CLASSIFICATION]: 'Intermediate classification', From e923e4929523d0634c44f5470913afcbcb313a1e Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Fri, 2 Oct 2026 20:56:58 -0700 Subject: [PATCH 011/134] docs(results): state that extra is never read for logic and name the masking figure plainly The result model and the per-kind data models now say the rule out loud: the extra object a method or processing service returns is stored, shown and exported only, and a value a feature needs becomes a typed field. The class masking figure is labelled as the share of probability outside the list, on the card and in the code, and the comments say it measures what the list removed rather than being an out-of-distribution score. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01L52AN9tabp76yjhjyCZkSJ --- ami/main/models.py | 14 ++++++++------ ami/main/schemas.py | 11 +++++++---- ui/src/data-services/models/occurrence-history.ts | 2 +- ui/src/utils/language.ts | 3 ++- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/ami/main/models.py b/ami/main/models.py index 4e7e86224..082b75c0d 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -3966,10 +3966,12 @@ class AlgorithmResult(BaseModel): through the ``Classification`` rows it creates, and those rows point back here through ``Classification.algorithm_result`` so the history can show a run with what it changed. ``data`` holds the run's own figures, validated against the model for ``kind`` - (ami/main/schemas.py), and ``value`` repeats the one figure lists filter and sort on. A new - result for the same occurrence, algorithm and kind becomes the current one and the earlier - ones stay as history. Write through ``AlgorithmResult.objects.record`` or ``record_many``. - Tracking and rank roll-ups are the next kinds expected. See #1431. + (ami/main/schemas.py), and ``value`` repeats the one figure lists filter and sort on. The + ``extra`` object inside ``data`` is stored, shown and exported only; nothing reads it for + logic, and a value a feature needs becomes a typed field. A new result for the same + occurrence, algorithm and kind becomes the current one and the earlier ones stay as + history. Write through ``AlgorithmResult.objects.record`` or ``record_many``. Tracking and + rank roll-ups are the next kinds expected. See #1431. """ class Kind(models.TextChoices): @@ -3989,8 +3991,8 @@ class Kind(models.TextChoices): "jobs.Job", on_delete=models.SET_NULL, null=True, blank=True, related_name="algorithm_results" ) kind = models.CharField(max_length=32, choices=Kind.choices) - # The kind's headline figure, for filtering and sorting: the excluded probability for class - # masking, the relative size for the size filter. + # The kind's headline figure, for filtering and sorting: the share of probability outside + # the list for class masking, the relative size for the size filter. value = models.FloatField(null=True, blank=True) data = models.JSONField(default=dict, blank=True) is_current = models.BooleanField(default=True) diff --git a/ami/main/schemas.py b/ami/main/schemas.py index e6d72ae6c..433cace36 100644 --- a/ami/main/schemas.py +++ b/ami/main/schemas.py @@ -4,7 +4,8 @@ occurrence's determination before and after, which cannot be derived later. Anything that lives elsewhere is left out: the settings are on the job, the classifications a run created point at the result, and the new taxon is on those classifications. ``extra`` takes whatever else a -method or processing service returns, kept as JSON and not interpreted. +method or processing service returns. Nothing reads ``extra`` for logic: it is stored, shown and +exported only. When a feature needs a value from it, that value becomes a typed field. Adding a kind means adding a model here and registering it in ``ALGORITHM_RESULT_DATA_SCHEMAS``; tracking (merges, statistics) and rank roll-ups (``rank_rollup``) are the next ones expected. @@ -17,7 +18,8 @@ class ResultData(pydantic.BaseModel): - # Whatever else the method returned, stored as JSON and never read by Antenna. + # Whatever else the method returned. Stored, shown and exported only, never read for logic; + # a value a feature needs is promoted to a typed field on the kind's model. extra: dict[str, Any] = {} @pydantic.validator("extra") @@ -38,8 +40,9 @@ class DeterminationSnapshot(ResultData): class ClassMaskingResultData(DeterminationSnapshot): """Figures from the occurrence's winning detection: the one whose masked classification scores highest.""" - # The share of the source classifier's probability on classes outside the species list: - # one minus the kept mass of the unmasked softmax. + # The share of the source classifier's probability outside the species list: one minus the + # kept mass of the unmasked softmax. A measure of what the list removed, not an + # out-of-distribution score. excluded_probability: float # The source classifier's top prediction before masking. original_taxon_id: int | None = None diff --git a/ui/src/data-services/models/occurrence-history.ts b/ui/src/data-services/models/occurrence-history.ts index 20075ecd2..35f72262f 100644 --- a/ui/src/data-services/models/occurrence-history.ts +++ b/ui/src/data-services/models/occurrence-history.ts @@ -52,7 +52,7 @@ interface ServerDeterminationChangePayload { export interface ServerClassMaskingPayload extends ServerDeterminationChangePayload { - /** The share of the source classifier's probability on classes outside the species list. */ + /** The share of the source classifier's probability outside the species list. */ excluded_probability: number /** Where the class that wins after masking ranked before it; 1 means it was already the top. */ new_winner_original_rank: number | null diff --git a/ui/src/utils/language.ts b/ui/src/utils/language.ts index 81a1e489c..9f6b74a73 100644 --- a/ui/src/utils/language.ts +++ b/ui/src/utils/language.ts @@ -791,7 +791,8 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.HISTORY_DETERMINATION_CHANGED]: '{{before}} → {{after}}', [STRING.HISTORY_DETERMINATION_UNCHANGED]: '{{name}} (unchanged)', [STRING.HISTORY_EMPTY]: 'Nothing has happened to this occurrence yet.', - [STRING.HISTORY_EXCLUDED_PROBABILITY]: 'Probability outside the list', + [STRING.HISTORY_EXCLUDED_PROBABILITY]: + 'Share of probability outside the list', [STRING.HISTORY_MASKING_SUBTITLE]: '{{algorithm}}, filtered by {{list}}', [STRING.HISTORY_ORIGINAL_PREDICTION]: 'Original top prediction', [STRING.HISTORY_LOAD_ERROR]: From c855924543602673570740d5e36267cb0c10cd8d Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Fri, 2 Oct 2026 20:57:56 -0700 Subject: [PATCH 012/134] fix(ui): keep the masking card label as "Probability outside the list" The PR screenshots show this label; the longer wording added no meaning. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01APWsYj7AAYL5aiq7T3HUwq --- ui/src/utils/language.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ui/src/utils/language.ts b/ui/src/utils/language.ts index 9f6b74a73..81a1e489c 100644 --- a/ui/src/utils/language.ts +++ b/ui/src/utils/language.ts @@ -791,8 +791,7 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.HISTORY_DETERMINATION_CHANGED]: '{{before}} → {{after}}', [STRING.HISTORY_DETERMINATION_UNCHANGED]: '{{name}} (unchanged)', [STRING.HISTORY_EMPTY]: 'Nothing has happened to this occurrence yet.', - [STRING.HISTORY_EXCLUDED_PROBABILITY]: - 'Share of probability outside the list', + [STRING.HISTORY_EXCLUDED_PROBABILITY]: 'Probability outside the list', [STRING.HISTORY_MASKING_SUBTITLE]: '{{algorithm}}, filtered by {{list}}', [STRING.HISTORY_ORIGINAL_PREDICTION]: 'Original top prediction', [STRING.HISTORY_LOAD_ERROR]: From e169a3d4ba93faf586c11afd412349aa11e2fd8d Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Sun, 4 Oct 2026 08:59:17 -0700 Subject: [PATCH 013/134] fix(history): only mark a prediction superseded by the run that replaced it A class masking classification names the one it re-scored through applied_to, so another classifier's prediction on the same detection is no longer labelled as superseded by class masking. The shared-detection rule stays for the size filter, which adds a classification without re-scoring one. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01APWsYj7AAYL5aiq7T3HUwq --- ami/main/models_future/history.py | 9 ++++++++- ami/main/test_occurrence_history.py | 14 +++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/ami/main/models_future/history.py b/ami/main/models_future/history.py index 27d5ca2c4..88be2790c 100644 --- a/ami/main/models_future/history.py +++ b/ami/main/models_future/history.py @@ -81,7 +81,14 @@ def occurrence_timeline(occurrence: Occurrence) -> list[TimelineEntry]: for c in created if c.applied_to_id is not None } - outranked_by = {c.detection_id: result_id for result_id, created in created_by_result.items() for c in created} + # Only a classification that re-scored nothing (the size filter's) supersedes by detection; a + # re-scored one names its original through applied_to, so other classifiers stay unmarked. + outranked_by = { + c.detection_id: result_id + for result_id, created in created_by_result.items() + for c in created + if c.applied_to_id is None + } def superseded_by(prediction: Classification) -> int | None: if prediction.pk in rescored_by: diff --git a/ami/main/test_occurrence_history.py b/ami/main/test_occurrence_history.py index 5c94a152c..a3ccd902a 100644 --- a/ami/main/test_occurrence_history.py +++ b/ami/main/test_occurrence_history.py @@ -282,6 +282,16 @@ def test_a_demoted_prediction_names_the_result_that_superseded_it(self): ) original = Classification.objects.get(detection=self.detections[1]) Classification.objects.filter(pk=original.pk).update(terminal=False, algorithm=classifier) + # A second classifier on the same detection that the run did not re-score. + bystander_algorithm = Algorithm.objects.create(name="Second classifier", key="bystander-history-test") + bystander = Classification.objects.create( + detection=self.detections[1], + taxon=self.taxon, + score=0.99, + terminal=True, + algorithm=bystander_algorithm, + timestamp=datetime.datetime.now(), + ) result = AlgorithmResult.objects.record( occurrence=self.occurrence, algorithm=masker, @@ -318,7 +328,9 @@ def test_a_demoted_prediction_names_the_result_that_superseded_it(self): for entry in data if entry["type"] == "prediction" } - self.assertEqual(set(predictions), {None, classifier.key}) + self.assertEqual(set(predictions), {None, classifier.key, bystander_algorithm.key}) + self.assertEqual(predictions[bystander_algorithm.key]["id"], bystander.pk) + self.assertIsNone(predictions[bystander_algorithm.key]["payload"]["superseded_by_result_id"]) demoted = predictions[classifier.key] self.assertEqual(demoted["id"], original.pk) self.assertEqual(demoted["payload"]["superseded_by_result_id"], result.pk) From ab25209bd5f8063203a781829e16e99f5c694606 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Sun, 4 Oct 2026 08:59:18 -0700 Subject: [PATCH 014/134] fix(ui): stop showing "Species list #undefined" and an empty history after a load error - A class masking card whose job no longer names a list says the list is not available. - When the history request fails, the timeline shows the error without also claiming nothing happened. - An identification's comment is read from the normalised field, never null. - Drop an unused string. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01APWsYj7AAYL5aiq7T3HUwq --- .../models/occurrence-history.test.ts | 1 + ui/src/data-services/models/occurrence-history.ts | 2 +- .../identification-card/algorithm-result.tsx | 15 ++++++++++----- .../identification-card/occurrence-timeline.tsx | 2 +- ui/src/utils/language.ts | 2 -- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/ui/src/data-services/models/occurrence-history.test.ts b/ui/src/data-services/models/occurrence-history.test.ts index 70686b064..3969951b8 100644 --- a/ui/src/data-services/models/occurrence-history.test.ts +++ b/ui/src/data-services/models/occurrence-history.test.ts @@ -35,6 +35,7 @@ const base = { const identificationEntry = (id: number): ServerOccurrenceHistoryEntry => ({ ...base, + comment: 'Looks right', id, payload: { agreed_with_identification_id: null, diff --git a/ui/src/data-services/models/occurrence-history.ts b/ui/src/data-services/models/occurrence-history.ts index 35f72262f..9c1517d70 100644 --- a/ui/src/data-services/models/occurrence-history.ts +++ b/ui/src/data-services/models/occurrence-history.ts @@ -165,7 +165,7 @@ const toIdentification = ( return { applied: taxon.id === determinationTaxonId, - comment: entry.payload.comment, + comment: entry.comment, createdAt: entry.timestamp, id: `${entry.id}`, overridden: entry.payload.withdrawn, diff --git a/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx b/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx index f901d936e..5b60ad55a 100644 --- a/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx +++ b/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx @@ -54,6 +54,15 @@ const formatSettingValue = (key: string, value: unknown) => { return typeof value === 'object' ? JSON.stringify(value) : `${value}` } +/** The list's id when only the job's settings name it, e.g. after the list was deleted. */ +const getSpeciesListFallback = (entry: AlgorithmResultEntry) => { + const id = entry.job?.config?.taxa_list_id + + return id !== undefined && id !== null + ? translate(STRING.HISTORY_SPECIES_LIST_ID, { id: `${id}` }) + : translate(STRING.VALUE_NOT_AVAILABLE) +} + /** The card's subtitle: for class masking the classifier and species list, otherwise the algorithm's name. */ const getSubTitle = (entry: AlgorithmResultEntry) => { if (entry.subtype === 'class_masking') { @@ -62,11 +71,7 @@ const getSubTitle = (entry: AlgorithmResultEntry) => { entry.source_algorithm?.name ?? entry.algorithm?.name ?? translate(STRING.VALUE_NOT_AVAILABLE), - list: - entry.taxa_list?.name ?? - translate(STRING.HISTORY_SPECIES_LIST_ID, { - id: `${entry.job?.config?.taxa_list_id}`, - }), + list: entry.taxa_list?.name ?? getSpeciesListFallback(entry), }) } diff --git a/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx b/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx index 0eb5bde1d..aa72d85e9 100644 --- a/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx +++ b/ui/src/pages/occurrence-details/identification-card/occurrence-timeline.tsx @@ -54,7 +54,7 @@ export const OccurrenceTimeline = ({ {translate(STRING.HISTORY_LOAD_ERROR)}

) : null} - {!isLoading && !items.length ? ( + {!isLoading && !error && !items.length ? (

{translate(STRING.HISTORY_EMPTY)}

diff --git a/ui/src/utils/language.ts b/ui/src/utils/language.ts index 81a1e489c..b637a079a 100644 --- a/ui/src/utils/language.ts +++ b/ui/src/utils/language.ts @@ -334,7 +334,6 @@ export enum STRING { HISTORY_MASKING_SUBTITLE, HISTORY_ORIGINAL_PREDICTION, HISTORY_LOAD_ERROR, - HISTORY_PREDICTION, HISTORY_SETTING_CAPTURE_SET, HISTORY_SETTING_OCCURRENCE, HISTORY_SETTING_REWEIGHT, @@ -796,7 +795,6 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.HISTORY_ORIGINAL_PREDICTION]: 'Original top prediction', [STRING.HISTORY_LOAD_ERROR]: 'Could not load the full history. Showing identifications and predictions only.', - [STRING.HISTORY_PREDICTION]: 'Prediction', [STRING.HISTORY_SETTING_CAPTURE_SET]: 'Capture set', [STRING.HISTORY_SETTING_OCCURRENCE]: 'Occurrence', [STRING.HISTORY_SETTING_REWEIGHT]: 'Re-weighted scores', From 800062ff5fe86aa97d24abb12a601cc8de7e3e5d Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Sun, 4 Oct 2026 09:36:36 -0700 Subject: [PATCH 015/134] feat(results): add result kinds without a migration and move results when occurrences merge - AlgorithmResult.kind has no choices; the registry of data models is the list of kinds, and every write path already rejects a kind without one. Choices were never enforced by PostgreSQL (the column is a plain varchar either way). - AlgorithmResult.objects.move_to_occurrence moves the results of merged occurrences to the kept one, keeping one current result per algorithm and kind, for tracking and later manual merges. - record_many locks the affected occurrences in a stable order, so two runs writing results for one occurrence wait for each other. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01APWsYj7AAYL5aiq7T3HUwq --- ami/main/migrations/0096_algorithm_results.py | 7 +--- ami/main/models.py | 38 ++++++++++++++++++- ami/main/test_occurrence_history.py | 24 ++++++++++++ 3 files changed, 62 insertions(+), 7 deletions(-) diff --git a/ami/main/migrations/0096_algorithm_results.py b/ami/main/migrations/0096_algorithm_results.py index 402b1b6c4..fc5fbb0a4 100644 --- a/ami/main/migrations/0096_algorithm_results.py +++ b/ami/main/migrations/0096_algorithm_results.py @@ -19,12 +19,7 @@ class Migration(migrations.Migration): ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), ("created_at", models.DateTimeField(auto_now_add=True)), ("updated_at", models.DateTimeField(auto_now=True)), - ( - "kind", - models.CharField( - choices=[("class_masking", "Class Masking"), ("size_filter", "Size Filter")], max_length=32 - ), - ), + ("kind", models.CharField(max_length=32)), ("value", models.FloatField(blank=True, null=True)), ("data", models.JSONField(blank=True, default=dict)), ("is_current", models.BooleanField(default=True)), diff --git a/ami/main/models.py b/ami/main/models.py index 082b75c0d..cd715a5e8 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -3951,12 +3951,46 @@ def record_many(self, results: typing.Iterable["AlgorithmResult"]) -> list["Algo if not kept: return [] with transaction.atomic(): + # Lock the occurrences in a stable order, so two runs writing results for the same + # occurrence wait for each other instead of both inserting a current row. + list( + Occurrence.objects.select_for_update() + .filter(pk__in={result.occurrence_id for result in kept}) + .order_by("pk") + .values_list("pk", flat=True) + ) for (algorithm_id, kind), occurrence_ids in groups.items(): self.filter( occurrence_id__in=occurrence_ids, algorithm_id=algorithm_id, kind=kind, is_current=True ).update(is_current=False) return self.bulk_create(kept) + def move_to_occurrence(self, kept: "Occurrence", absorbed_ids: typing.Iterable[int]) -> int: + """Move the results of occurrences merged into ``kept``, so their history follows the merge. + + Each algorithm and kind keeps one current result on ``kept``: its own if it has one, + otherwise the latest moved one. The other moved results stay as history. Returns the + number of results moved. + """ + absorbed_ids = set(absorbed_ids) - {kept.pk} + if not absorbed_ids: + return 0 + with transaction.atomic(): + current_keys = set(self.filter(occurrence=kept, is_current=True).values_list("algorithm_id", "kind")) + demote: list[int] = [] + for pk, algorithm_id, kind in ( + self.filter(occurrence_id__in=absorbed_ids, is_current=True) + .order_by("-timestamp", "-pk") + .values_list("pk", "algorithm_id", "kind") + ): + if (algorithm_id, kind) in current_keys: + demote.append(pk) + else: + current_keys.add((algorithm_id, kind)) + if demote: + self.filter(pk__in=demote).update(is_current=False) + return self.filter(occurrence_id__in=absorbed_ids).update(occurrence=kept) + @final class AlgorithmResult(BaseModel): @@ -3990,7 +4024,9 @@ class Kind(models.TextChoices): job = models.ForeignKey( "jobs.Job", on_delete=models.SET_NULL, null=True, blank=True, related_name="algorithm_results" ) - kind = models.CharField(max_length=32, choices=Kind.choices) + # No ``choices``: the registry of data models in ami/main/schemas.py is the list of kinds, + # and every write path rejects a kind without one, so a new kind needs no migration. + kind = models.CharField(max_length=32) # The kind's headline figure, for filtering and sorting: the share of probability outside # the list for class masking, the relative size for the size filter. value = models.FloatField(null=True, blank=True) diff --git a/ami/main/test_occurrence_history.py b/ami/main/test_occurrence_history.py index a3ccd902a..8d68fd820 100644 --- a/ami/main/test_occurrence_history.py +++ b/ami/main/test_occurrence_history.py @@ -110,6 +110,30 @@ def test_a_new_result_becomes_current_and_the_earlier_one_stays_as_history(self) self.assertEqual(AlgorithmResult.objects.for_occurrence(self.occurrence).count(), 2) self.assertEqual(AlgorithmResult.objects.for_occurrence(self.occurrence).current().get().pk, second.pk) + def test_merging_moves_results_and_keeps_one_current_per_algorithm_and_kind(self): + """The kept occurrence's current result wins a collision; otherwise the latest moved one stays current.""" + other_algorithm = Algorithm.objects.create(name="Second size filter", key="size-filter-merge-test") + absorbed = [Occurrence.objects.create(project=self.project, deployment=self.deployment) for _ in range(3)] + + def record(occurrence, algorithm, size): + return AlgorithmResult.objects.record( + occurrence=occurrence, algorithm=algorithm, kind=SIZE_FILTER, data={"relative_size": size} + ) + + kept_current = record(self.occurrence, self.algorithm, 0.01) + history = record(absorbed[0], self.algorithm, 0.02) + colliding = record(absorbed[0], self.algorithm, 0.03) + older = record(absorbed[1], other_algorithm, 0.04) + latest = record(absorbed[2], other_algorithm, 0.05) + + moved = AlgorithmResult.objects.move_to_occurrence(self.occurrence, [o.pk for o in absorbed]) + + self.assertEqual(moved, 4) + results = {r.pk: r for r in AlgorithmResult.objects.filter(occurrence=self.occurrence)} + self.assertEqual(set(results), {kept_current.pk, history.pk, colliding.pk, older.pk, latest.pk}) + current = {pk for pk, r in results.items() if r.is_current} + self.assertEqual(current, {kept_current.pk, latest.pk}) + def test_the_database_holds_one_current_result_per_occurrence_algorithm_and_kind(self): self._size_filter({"relative_size": 0.01}).save() with transaction.atomic(), self.assertRaises(IntegrityError): From b35ae46b420060713ae8c32508d0d832f48583f2 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Sun, 4 Oct 2026 09:36:39 -0700 Subject: [PATCH 016/134] fix(history): survive a deleted taxon or species list and count changed detections - A classification whose taxon was deleted no longer breaks the result card. - A deleted species list shows its id as text instead of a link that 404s. - "Detections changed" counts distinct detections, not classifications. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01APWsYj7AAYL5aiq7T3HUwq --- ami/main/api/serializers.py | 2 +- .../models/occurrence-history.ts | 9 ++++++--- .../identification-card/algorithm-result.tsx | 20 +++++++++++-------- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index e653a3210..90b3e6ade 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -2112,7 +2112,7 @@ class HistoryClassificationSerializer(serializers.Serializer): """A classification a post-processing run created, shown inside the run's result.""" id = serializers.IntegerField() - taxon = HistoryTaxonSerializer() + taxon = HistoryTaxonSerializer(allow_null=True) score = serializers.FloatField(allow_null=True) terminal = serializers.BooleanField() detection_id = serializers.IntegerField() diff --git a/ui/src/data-services/models/occurrence-history.ts b/ui/src/data-services/models/occurrence-history.ts index 9c1517d70..598ac7432 100644 --- a/ui/src/data-services/models/occurrence-history.ts +++ b/ui/src/data-services/models/occurrence-history.ts @@ -39,7 +39,8 @@ export interface ServerHistoryClassification { detection_id: number id: number score: number | null - taxon: ServerHistoryTaxon + /** Null when the taxon was deleted. */ + taxon: ServerHistoryTaxon | null terminal: boolean } @@ -367,8 +368,10 @@ export const getResultPrediction = ( entry: AlgorithmResultEntry, determinationTaxonId?: string ): ResultPrediction | undefined => { - const [best] = entry.classifications - if (!best) { + const best = entry.classifications.find( + (classification) => classification.taxon !== null + ) + if (!best?.taxon) { return undefined } const taxon = convertHistoryTaxon(best.taxon) diff --git a/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx b/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx index 5b60ad55a..a98f39a98 100644 --- a/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx +++ b/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx @@ -128,8 +128,8 @@ export const AlgorithmResult = ({ ) switch (entry.subtype) { case 'class_masking': { - const taxaListId = entry.taxa_list?.id ?? entry.job?.config?.taxa_list_id - if (taxaListId !== undefined) { + // Link only a list that still exists; a deleted one shows its id as plain text. + if (entry.taxa_list) { stats.push({ label: translate(STRING.HISTORY_SPECIES_LIST), value: ( @@ -137,16 +137,20 @@ export const AlgorithmResult = ({ className="underline underline-offset-4" to={APP_ROUTES.TAXA_LIST_DETAILS({ projectId: projectId as string, - taxaListId: `${taxaListId}`, + taxaListId: `${entry.taxa_list.id}`, })} > - {entry.taxa_list?.name ?? - translate(STRING.HISTORY_SPECIES_LIST_ID, { - id: `${taxaListId}`, - })} + {entry.taxa_list.name} ), }) + } else if (entry.job?.config?.taxa_list_id != null) { + stats.push({ + label: translate(STRING.HISTORY_SPECIES_LIST), + value: translate(STRING.HISTORY_SPECIES_LIST_ID, { + id: `${entry.job.config.taxa_list_id}`, + }), + }) } stats.push({ label: translate(STRING.HISTORY_EXCLUDED_PROBABILITY), @@ -184,7 +188,7 @@ export const AlgorithmResult = ({ } stats.push({ label: translate(STRING.HISTORY_DETECTIONS_AFFECTED), - value: entry.classifications.length, + value: new Set(entry.classifications.map((c) => c.detection_id)).size, }) getJobSettings(entry.job).forEach(({ key, value }) => { const label = SETTING_LABELS[key] From 59ed47ca0a7f854c48a7e109a52fb2a7298e26bc Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 5 Oct 2026 16:27:57 -0700 Subject: [PATCH 017/134] chore(migrations): number the result migrations after the job-provenance migrations Rebased onto #1471 (head 797a6495), which takes main/0096 and main/0097 for the job columns and their indexes. The result migrations now follow them. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01APWsYj7AAYL5aiq7T3HUwq --- .../{0096_algorithm_results.py => 0098_algorithm_results.py} | 2 +- ...assifications.py => 0099_classification_algorithm_result.py} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename ami/main/migrations/{0096_algorithm_results.py => 0098_algorithm_results.py} (97%) rename ami/main/migrations/{0097_output_jobs_and_results_on_classifications.py => 0099_classification_algorithm_result.py} (94%) diff --git a/ami/main/migrations/0096_algorithm_results.py b/ami/main/migrations/0098_algorithm_results.py similarity index 97% rename from ami/main/migrations/0096_algorithm_results.py rename to ami/main/migrations/0098_algorithm_results.py index fc5fbb0a4..10dd4113a 100644 --- a/ami/main/migrations/0096_algorithm_results.py +++ b/ami/main/migrations/0098_algorithm_results.py @@ -9,7 +9,7 @@ class Migration(migrations.Migration): dependencies = [ ("jobs", "0023_alter_job_job_type_key"), ("ml", "0028_normalize_empty_endpoint_url_to_null"), - ("main", "0095_grant_sync_deployment_to_mldatamanager"), + ("main", "0097_detection_and_classification_job_indexes"), ] operations = [ diff --git a/ami/main/migrations/0097_output_jobs_and_results_on_classifications.py b/ami/main/migrations/0099_classification_algorithm_result.py similarity index 94% rename from ami/main/migrations/0097_output_jobs_and_results_on_classifications.py rename to ami/main/migrations/0099_classification_algorithm_result.py index 4231e8867..597a00ac3 100644 --- a/ami/main/migrations/0097_output_jobs_and_results_on_classifications.py +++ b/ami/main/migrations/0099_classification_algorithm_result.py @@ -8,7 +8,7 @@ class Migration(migrations.Migration): dependencies = [ ("jobs", "0023_alter_job_job_type_key"), - ("main", "0096_algorithm_results"), + ("main", "0098_algorithm_results"), ] operations = [ From 08067917a5740a2c9eeeca74a8e3c12b0e46180a Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 5 Oct 2026 16:33:30 -0700 Subject: [PATCH 018/134] refactor(results): move algorithm results into the ml app and take the project from the occurrence only AlgorithmResult now lives in ami/ml/models/algorithm_result.py (table ml_algorithmresult), the kind data models and their registry in ami/ml/results/schemas.py (AlgorithmResultData), and the batch writer in ami/ml/results/writer.py (AlgorithmResultWriter). The results schemas module is the only one that calls pydantic's API for results and imports nothing from Django, so a pydantic upgrade touches one file. A result takes its project from the occurrence. An occurrence without a project is a data problem tracked on #1188: record_many skips its result with a warning so the run's other occurrences still get theirs, and save() raises. Class masking no longer stores the original taxon and score: they are the classification the masked one replaced (applied_to), which the history reads directly. Classification.algorithm_result is indexed by a partial index built concurrently in its own migration, so adding the column holds the table lock only briefly. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01APWsYj7AAYL5aiq7T3HUwq --- ...> 0098_classification_algorithm_result.py} | 12 +- ...9_classification_algorithm_result_index.py | 36 +++ ami/main/models.py | 209 +----------------- ami/main/models_future/history.py | 4 +- ami/main/schemas.py | 76 ------- ami/main/test_occurrence_history.py | 54 ++--- .../migrations/0029_algorithm_result.py} | 26 +-- ami/ml/models/__init__.py | 2 + ami/ml/models/algorithm_result.py | 190 ++++++++++++++++ ami/ml/post_processing/class_masking.py | 13 +- ami/ml/post_processing/small_size_filter.py | 15 +- .../tests/test_class_masking.py | 12 +- .../tests/test_small_size_filter.py | 3 +- ami/ml/results/__init__.py | 0 ami/ml/results/schemas.py | 88 ++++++++ .../results.py => results/writer.py} | 9 +- 16 files changed, 389 insertions(+), 360 deletions(-) rename ami/main/migrations/{0099_classification_algorithm_result.py => 0098_classification_algorithm_result.py} (61%) create mode 100644 ami/main/migrations/0099_classification_algorithm_result_index.py delete mode 100644 ami/main/schemas.py rename ami/{main/migrations/0098_algorithm_results.py => ml/migrations/0029_algorithm_result.py} (83%) create mode 100644 ami/ml/models/algorithm_result.py create mode 100644 ami/ml/results/__init__.py create mode 100644 ami/ml/results/schemas.py rename ami/ml/{post_processing/results.py => results/writer.py} (92%) diff --git a/ami/main/migrations/0099_classification_algorithm_result.py b/ami/main/migrations/0098_classification_algorithm_result.py similarity index 61% rename from ami/main/migrations/0099_classification_algorithm_result.py rename to ami/main/migrations/0098_classification_algorithm_result.py index 597a00ac3..59238b19f 100644 --- a/ami/main/migrations/0099_classification_algorithm_result.py +++ b/ami/main/migrations/0098_classification_algorithm_result.py @@ -1,5 +1,6 @@ -# Generated by Django 4.2.10 on 2026-10-02 22:20 -"""The link from a classification to the algorithm result it was created for.""" +# Generated by Django 4.2.10 on 2026-10-05 19:29 +"""The link from a classification to the algorithm result it was created for. Its index is built +concurrently in 0099, so this ALTER TABLE holds its lock only for adding a nullable column.""" from django.db import migrations, models import django.db.models.deletion @@ -7,8 +8,8 @@ class Migration(migrations.Migration): dependencies = [ - ("jobs", "0023_alter_job_job_type_key"), - ("main", "0098_algorithm_results"), + ("ml", "0029_algorithm_result"), + ("main", "0097_detection_and_classification_job_indexes"), ] operations = [ @@ -17,10 +18,11 @@ class Migration(migrations.Migration): name="algorithm_result", field=models.ForeignKey( blank=True, + db_index=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="classifications", - to="main.algorithmresult", + to="ml.algorithmresult", ), ), ] diff --git a/ami/main/migrations/0099_classification_algorithm_result_index.py b/ami/main/migrations/0099_classification_algorithm_result_index.py new file mode 100644 index 000000000..83d081317 --- /dev/null +++ b/ami/main/migrations/0099_classification_algorithm_result_index.py @@ -0,0 +1,36 @@ +from django.contrib.postgres.operations import AddIndexConcurrently +from django.db import migrations, models + + +class Migration(migrations.Migration): + """Index the classification's link to its algorithm result, which deleting a result looks up. + + Partial, because pipeline classifications never have a result, so the index stays small. + Built CONCURRENTLY on this large table, which needs a non-atomic migration; see 0093 for + why the statement timeout is cleared and restored around the build. + """ + + atomic = False + + dependencies = [ + ("main", "0098_classification_algorithm_result"), + ] + + operations = [ + migrations.RunSQL( + sql="SET statement_timeout = 0;", + reverse_sql=migrations.RunSQL.noop, + ), + AddIndexConcurrently( + model_name="classification", + index=models.Index( + condition=models.Q(algorithm_result__isnull=False), + fields=["algorithm_result"], + name="cls_algorithm_result_idx", + ), + ), + migrations.RunSQL( + sql="RESET statement_timeout;", + reverse_sql=migrations.RunSQL.noop, + ), + ] diff --git a/ami/main/models.py b/ami/main/models.py index cd715a5e8..5da65ecdc 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -42,7 +42,6 @@ build_taxa_recursive_filter_q, ) from ami.main.models_future.projects import ProjectSettingsMixin -from ami.main.schemas import validate_result_data from ami.ml.schemas import BoundingBox from ami.users.models import User from ami.utils.media import calculate_file_checksum, extract_timestamp, fetch_image_content @@ -3011,12 +3010,14 @@ class Classification(BaseModel): related_name="classifications", db_index=False, ) + # Indexed by the partial index in Meta, not by the foreign key's own index. algorithm_result = models.ForeignKey( - "main.AlgorithmResult", + "ml.AlgorithmResult", on_delete=models.SET_NULL, null=True, blank=True, related_name="classifications", + db_index=False, ) applied_to = models.ForeignKey( "self", @@ -3043,6 +3044,13 @@ class Meta: models.Index( fields=["job", "detection"], name="cls_job_detection_idx", condition=models.Q(job__isnull=False) ), + # Deleting a result looks up its classifications. Partial, because pipeline + # classifications never have a result. + models.Index( + fields=["algorithm_result"], + name="cls_algorithm_result_idx", + condition=models.Q(algorithm_result__isnull=False), + ), ] def __str__(self) -> str: @@ -3868,203 +3876,6 @@ class Meta: ] -def occurrence_project_ids(occurrences: typing.Iterable["Occurrence"]) -> dict[int, int | None]: - """The project each occurrence belongs to: its own, or its station's when it has none. - - One query for every occurrence that lacks a project of its own; none when all have one. - The value is None for an occurrence with neither, which callers skip with a warning. - """ - occurrences = list(occurrences) - projects = {occurrence.pk: occurrence.project_id for occurrence in occurrences} - deployment_ids = { - occurrence.deployment_id - for occurrence in occurrences - if occurrence.project_id is None and occurrence.deployment_id is not None - } - if deployment_ids: - station_projects = dict(Deployment.objects.filter(pk__in=deployment_ids).values_list("pk", "project_id")) - for occurrence in occurrences: - if occurrence.project_id is None and occurrence.deployment_id is not None: - projects[occurrence.pk] = station_projects.get(occurrence.deployment_id) - return projects - - -class AlgorithmResultQuerySet(BaseQuerySet): - def for_occurrence(self, occurrence): - return self.filter(occurrence=occurrence) - - def current(self): - return self.filter(is_current=True) - - def bulk_create(self, objs, *args, **kwargs): - objs = list(objs) - for obj in objs: - obj.data = validate_result_data(obj.kind, obj.data) - return super().bulk_create(objs, *args, **kwargs) - - def bulk_update(self, objs, fields, *args, **kwargs): - objs = list(objs) - if "data" in fields: - for obj in objs: - obj.data = validate_result_data(obj.kind, obj.data) - return super().bulk_update(objs, fields, *args, **kwargs) - - def record(self, **fields) -> "AlgorithmResult": - """Insert one result as the current one for its occurrence, algorithm and kind.""" - results = self.record_many([self.model(**fields)]) - if not results: - raise ValueError("The occurrence has no project, directly or through its station.") - return results[0] - - def record_many(self, results: typing.Iterable["AlgorithmResult"]) -> list["AlgorithmResult"]: - """Insert results as current, keeping each occurrence's earlier result of the same algorithm and kind. - - Rows are only inserted, never rewritten, apart from clearing ``is_current`` on the result - each new one replaces. A result whose occurrence has no project, directly or through - its station, is skipped with a warning rather than failing a run that has already - changed data. The returned list holds the results that were written, in order. - """ - results = list(results) - if not results: - return [] - projects = occurrence_project_ids(result.occurrence for result in results if result.project_id is None) - kept: list[AlgorithmResult] = [] - groups: dict[tuple[int, str], set[int]] = collections.defaultdict(set) - for result in results: - if result.project_id is None: - result.project_id = projects.get(result.occurrence_id) - if result.project_id is None: - logger.warning( - f"Not recording the {result.kind} result for Occurrence #{result.occurrence_id}: " - "it has no project, directly or through its station." - ) - continue - key = (result.algorithm_id, result.kind) - if result.occurrence_id in groups[key]: - raise ValueError( - f"Two {result.kind} results for Occurrence #{result.occurrence_id} " - "from one algorithm in one write." - ) - groups[key].add(result.occurrence_id) - result.is_current = True - kept.append(result) - if not kept: - return [] - with transaction.atomic(): - # Lock the occurrences in a stable order, so two runs writing results for the same - # occurrence wait for each other instead of both inserting a current row. - list( - Occurrence.objects.select_for_update() - .filter(pk__in={result.occurrence_id for result in kept}) - .order_by("pk") - .values_list("pk", flat=True) - ) - for (algorithm_id, kind), occurrence_ids in groups.items(): - self.filter( - occurrence_id__in=occurrence_ids, algorithm_id=algorithm_id, kind=kind, is_current=True - ).update(is_current=False) - return self.bulk_create(kept) - - def move_to_occurrence(self, kept: "Occurrence", absorbed_ids: typing.Iterable[int]) -> int: - """Move the results of occurrences merged into ``kept``, so their history follows the merge. - - Each algorithm and kind keeps one current result on ``kept``: its own if it has one, - otherwise the latest moved one. The other moved results stay as history. Returns the - number of results moved. - """ - absorbed_ids = set(absorbed_ids) - {kept.pk} - if not absorbed_ids: - return 0 - with transaction.atomic(): - current_keys = set(self.filter(occurrence=kept, is_current=True).values_list("algorithm_id", "kind")) - demote: list[int] = [] - for pk, algorithm_id, kind in ( - self.filter(occurrence_id__in=absorbed_ids, is_current=True) - .order_by("-timestamp", "-pk") - .values_list("pk", "algorithm_id", "kind") - ): - if (algorithm_id, kind) in current_keys: - demote.append(pk) - else: - current_keys.add((algorithm_id, kind)) - if demote: - self.filter(pk__in=demote).update(is_current=False) - return self.filter(occurrence_id__in=absorbed_ids).update(occurrence=kept) - - -@final -class AlgorithmResult(BaseModel): - """What a post-processing run decided about one occurrence: a record of the run, never an input to it. - - The determination never reads this table. A run that changes an occurrence's taxon does so - through the ``Classification`` rows it creates, and those rows point back here through - ``Classification.algorithm_result`` so the history can show a run with what it changed. - ``data`` holds the run's own figures, validated against the model for ``kind`` - (ami/main/schemas.py), and ``value`` repeats the one figure lists filter and sort on. The - ``extra`` object inside ``data`` is stored, shown and exported only; nothing reads it for - logic, and a value a feature needs becomes a typed field. A new result for the same - occurrence, algorithm and kind becomes the current one and the earlier ones stay as - history. Write through ``AlgorithmResult.objects.record`` or ``record_many``. Tracking and - rank roll-ups are the next kinds expected. See #1431. - """ - - class Kind(models.TextChoices): - CLASS_MASKING = "class_masking" - SIZE_FILTER = "size_filter" - - # Copied from the occurrence (or its station) when the result is written, so per-project - # queries and permission checks need no join. - project = models.ForeignKey("main.Project", on_delete=models.CASCADE, related_name="algorithm_results") - # Indexed together with the timestamp, below. - occurrence = models.ForeignKey( - Occurrence, on_delete=models.CASCADE, related_name="algorithm_results", db_index=False - ) - algorithm = models.ForeignKey("ml.Algorithm", on_delete=models.CASCADE, related_name="algorithm_results") - # The run that wrote the result. Deleting the job keeps the result, with no run to link to. - job = models.ForeignKey( - "jobs.Job", on_delete=models.SET_NULL, null=True, blank=True, related_name="algorithm_results" - ) - # No ``choices``: the registry of data models in ami/main/schemas.py is the list of kinds, - # and every write path rejects a kind without one, so a new kind needs no migration. - kind = models.CharField(max_length=32) - # The kind's headline figure, for filtering and sorting: the share of probability outside - # the list for class masking, the relative size for the size filter. - value = models.FloatField(null=True, blank=True) - data = models.JSONField(default=dict, blank=True) - is_current = models.BooleanField(default=True) - timestamp = models.DateTimeField(default=timezone.now) - - objects = AlgorithmResultQuerySet.as_manager() - - class Meta: - constraints = [ - models.UniqueConstraint( - fields=["occurrence", "algorithm", "kind"], - condition=Q(is_current=True), - name="algorithm_result_current_occurrence", - ), - ] - indexes = [ - models.Index(fields=["occurrence", "-timestamp"], name="algorithm_result_occ_time"), - models.Index( - fields=["project", "kind", "value"], - condition=Q(is_current=True, value__isnull=False), - name="algorithm_result_current_value", - ), - ] - - def __str__(self) -> str: - return f"#{self.pk} {self.kind} for Occurrence #{self.occurrence_id} from Algorithm #{self.algorithm_id}" - - def save(self, *args, **kwargs): - self.data = validate_result_data(self.kind, self.data) - if self.project_id is None: - self.project_id = occurrence_project_ids([self.occurrence])[self.occurrence_id] - if self.project_id is None: - raise ValueError("The occurrence has no project, directly or through its station.") - super().save(*args, **kwargs) - - def update_occurrence_determination( occurrence: Occurrence, current_determination: typing.Optional["Taxon"] = None, save=True ) -> bool: diff --git a/ami/main/models_future/history.py b/ami/main/models_future/history.py index 88be2790c..a853c3bee 100644 --- a/ami/main/models_future/history.py +++ b/ami/main/models_future/history.py @@ -13,8 +13,8 @@ from django.db.models import Q -from ami.main.models import AlgorithmResult, Classification, Identification, Occurrence, TaxaList, Taxon, User -from ami.ml.models.algorithm import Algorithm +from ami.main.models import Classification, Identification, Occurrence, TaxaList, Taxon, User +from ami.ml.models import Algorithm, AlgorithmResult if typing.TYPE_CHECKING: from ami.jobs.models import Job diff --git a/ami/main/schemas.py b/ami/main/schemas.py deleted file mode 100644 index 433cace36..000000000 --- a/ami/main/schemas.py +++ /dev/null @@ -1,76 +0,0 @@ -"""The shape of ``AlgorithmResult.data``, one pydantic model per result kind. - -Each model records only what the run alone knows: figures computed while it ran and the -occurrence's determination before and after, which cannot be derived later. Anything that lives -elsewhere is left out: the settings are on the job, the classifications a run created point at -the result, and the new taxon is on those classifications. ``extra`` takes whatever else a -method or processing service returns. Nothing reads ``extra`` for logic: it is stored, shown and -exported only. When a feature needs a value from it, that value becomes a typed field. - -Adding a kind means adding a model here and registering it in ``ALGORITHM_RESULT_DATA_SCHEMAS``; -tracking (merges, statistics) and rank roll-ups (``rank_rollup``) are the next ones expected. -""" - -import json -from typing import Any - -import pydantic - - -class ResultData(pydantic.BaseModel): - # Whatever else the method returned. Stored, shown and exported only, never read for logic; - # a value a feature needs is promoted to a typed field on the kind's model. - extra: dict[str, Any] = {} - - @pydantic.validator("extra") - def _extra_is_json(cls, value: dict[str, Any]) -> dict[str, Any]: - return json.loads(json.dumps(value)) - - class Config: - extra = "forbid" - - -class DeterminationSnapshot(ResultData): - # The occurrence's determination before and after the run, as taxon ids. Equal when the - # run changed predictions without moving the determination. - determination_before_id: int | None = None - determination_after_id: int | None = None - - -class ClassMaskingResultData(DeterminationSnapshot): - """Figures from the occurrence's winning detection: the one whose masked classification scores highest.""" - - # The share of the source classifier's probability outside the species list: one minus the - # kept mass of the unmasked softmax. A measure of what the list removed, not an - # out-of-distribution score. - excluded_probability: float - # The source classifier's top prediction before masking. - original_taxon_id: int | None = None - original_score: float | None = None - # Where the class that wins after masking ranked before it; 1 means it was already the top. - new_winner_original_rank: int | None = None - - -class SizeFilterResultData(DeterminationSnapshot): - """Figures from the occurrence's smallest filtered detection.""" - - # The detection's box area as a fraction of its image. - relative_size: float - - -ALGORITHM_RESULT_DATA_SCHEMAS: dict[str, type[ResultData]] = { - "class_masking": ClassMaskingResultData, - "size_filter": SizeFilterResultData, -} - - -def validate_result_data(kind: str, data: dict | ResultData | None) -> dict: - """``data`` as JSON-ready values, or a ValueError when it does not fit the kind's model.""" - schema = ALGORITHM_RESULT_DATA_SCHEMAS.get(kind) - if schema is None: - raise ValueError(f"No result data model is registered for {kind!r}.") - if isinstance(data, ResultData) and not isinstance(data, schema): - raise ValueError(f"{type(data).__name__} is not the result data model for {kind!r}.") - values = data.dict() if isinstance(data, ResultData) else (data or {}) - # Round-trip through JSON so datetimes are stored as ISO strings. - return pydantic.parse_raw_as(dict, schema.parse_obj(values).json()) diff --git a/ami/main/test_occurrence_history.py b/ami/main/test_occurrence_history.py index 8d68fd820..5e0180ebf 100644 --- a/ami/main/test_occurrence_history.py +++ b/ami/main/test_occurrence_history.py @@ -10,18 +10,8 @@ from rest_framework.test import APITestCase from ami.jobs.models import Job -from ami.main.models import ( - AlgorithmResult, - Classification, - Detection, - Identification, - Occurrence, - Project, - SourceImage, - TaxaList, - Taxon, -) -from ami.ml.models import Algorithm +from ami.main.models import Classification, Detection, Identification, Occurrence, SourceImage, TaxaList, Taxon +from ami.ml.models import Algorithm, AlgorithmResult from ami.tests.fixtures.main import create_captures, create_taxa, setup_test_project from ami.users.models import User from ami.users.roles import BasicMember, ProjectManager @@ -107,8 +97,8 @@ def test_a_new_result_becomes_current_and_the_earlier_one_stays_as_history(self) first.refresh_from_db() self.assertFalse(first.is_current) self.assertTrue(second.is_current and untouched.is_current) - self.assertEqual(AlgorithmResult.objects.for_occurrence(self.occurrence).count(), 2) - self.assertEqual(AlgorithmResult.objects.for_occurrence(self.occurrence).current().get().pk, second.pk) + self.assertEqual(self.occurrence.algorithm_results.count(), 2) + self.assertEqual(self.occurrence.algorithm_results.current().get().pk, second.pk) def test_merging_moves_results_and_keeps_one_current_per_algorithm_and_kind(self): """The kept occurrence's current result wins a collision; otherwise the latest moved one stays current.""" @@ -139,29 +129,26 @@ def test_the_database_holds_one_current_result_per_occurrence_algorithm_and_kind with transaction.atomic(), self.assertRaises(IntegrityError): self._size_filter({"relative_size": 0.02}).save() - def test_project_comes_from_the_occurrence_or_its_station_and_a_result_with_neither_is_skipped(self): - other = Project.objects.create(name="Another project") - Occurrence.objects.filter(pk=self.occurrence.pk).update(project=None) - self.occurrence.refresh_from_db() + def test_project_comes_from_the_occurrence_and_a_result_without_one_is_skipped(self): result = self._size_filter({"relative_size": 0.01}) result.save() self.assertEqual(result.project_id, self.project.pk) - Occurrence.objects.filter(pk=self.occurrence.pk).update(project=other) - self.occurrence.refresh_from_db() - self.assertEqual( - AlgorithmResult.objects.record_many([self._size_filter({"relative_size": 0.02})])[0].project_id, - other.pk, - ) - - Occurrence.objects.filter(pk=self.occurrence.pk).update(project=None, deployment=None) - self.occurrence.refresh_from_db() - with self.assertLogs("ami.main.models", level="WARNING"): - written = AlgorithmResult.objects.record_many([self._size_filter({"relative_size": 0.03})]) - self.assertEqual(written, []) + orphan = Occurrence.objects.create(project=None, deployment=self.deployment) + with self.assertLogs("ami.ml.models.algorithm_result", level="WARNING"): + written = AlgorithmResult.objects.record_many( + [ + self._size_filter({"relative_size": 0.02}), + self._size_filter({"relative_size": 0.03}, occurrence=orphan), + ] + ) + # The occurrence with a project still gets its result; the other is skipped, not fatal. + self.assertEqual([r.occurrence_id for r in written], [self.occurrence.pk]) with self.assertRaises(ValueError): - self._size_filter({"relative_size": 0.03}).save() - self.assertEqual(AlgorithmResult.objects.count(), 2) + self._size_filter({"relative_size": 0.03}, occurrence=orphan).save() + + def test_the_model_lives_in_the_ml_app(self): + self.assertEqual(AlgorithmResult._meta.app_label, "ml") class OccurrenceFixtureTestCase(APITestCase): @@ -324,8 +311,6 @@ def test_a_demoted_prediction_names_the_result_that_superseded_it(self): value=0.4, data={ "excluded_probability": 0.4, - "original_taxon_id": self.taxon.pk, - "original_score": 0.9, "new_winner_original_rank": 2, "determination_before_id": self.taxon.pk, "determination_after_id": self.other_taxon.pk, @@ -362,7 +347,6 @@ def test_a_demoted_prediction_names_the_result_that_superseded_it(self): self.assertIsNone(predictions[None]["payload"]["superseded_by_result_id"]) self.assertEqual(result_entry["taxa_list"], {"id": taxa_list.pk, "name": "Kept species"}) self.assertEqual(result_entry["source_algorithm"]["key"], classifier.key) - self.assertEqual(result_entry["original_taxon"]["id"], self.taxon.pk) self.assertEqual(result_entry["score"], 0.4) def test_a_terminal_prediction_outranked_on_its_detection_is_superseded(self): diff --git a/ami/main/migrations/0098_algorithm_results.py b/ami/ml/migrations/0029_algorithm_result.py similarity index 83% rename from ami/main/migrations/0098_algorithm_results.py rename to ami/ml/migrations/0029_algorithm_result.py index 10dd4113a..afe64177a 100644 --- a/ami/main/migrations/0098_algorithm_results.py +++ b/ami/ml/migrations/0029_algorithm_result.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.10 on 2026-10-02 23:31 +# Generated by Django 4.2.10 on 2026-10-05 19:29 from django.db import migrations, models import django.db.models.deletion @@ -8,8 +8,8 @@ class Migration(migrations.Migration): dependencies = [ ("jobs", "0023_alter_job_job_type_key"), - ("ml", "0028_normalize_empty_endpoint_url_to_null"), ("main", "0097_detection_and_classification_job_indexes"), + ("ml", "0028_normalize_empty_endpoint_url_to_null"), ] operations = [ @@ -60,18 +60,16 @@ class Migration(migrations.Migration): ), ), ], - ), - migrations.AddIndex( - model_name="algorithmresult", - index=models.Index(fields=["occurrence", "-timestamp"], name="algorithm_result_occ_time"), - ), - migrations.AddIndex( - model_name="algorithmresult", - index=models.Index( - condition=models.Q(("is_current", True), ("value__isnull", False)), - fields=["project", "kind", "value"], - name="algorithm_result_current_value", - ), + options={ + "indexes": [ + models.Index(fields=["occurrence", "-timestamp"], name="algorithm_result_occ_time"), + models.Index( + condition=models.Q(("is_current", True), ("value__isnull", False)), + fields=["project", "kind", "value"], + name="algorithm_result_current_value", + ), + ], + }, ), migrations.AddConstraint( model_name="algorithmresult", diff --git a/ami/ml/models/__init__.py b/ami/ml/models/__init__.py index 5000c7f53..07eefda82 100644 --- a/ami/ml/models/__init__.py +++ b/ami/ml/models/__init__.py @@ -1,4 +1,5 @@ from ami.ml.models.algorithm import Algorithm, AlgorithmCategoryMap +from ami.ml.models.algorithm_result import AlgorithmResult from ami.ml.models.pipeline import Pipeline from ami.ml.models.processing_service import ProcessingService from ami.ml.models.project_pipeline_config import ProjectPipelineConfig @@ -6,6 +7,7 @@ __all__ = [ "Algorithm", "AlgorithmCategoryMap", + "AlgorithmResult", "Pipeline", "ProcessingService", "ProjectPipelineConfig", diff --git a/ami/ml/models/algorithm_result.py b/ami/ml/models/algorithm_result.py new file mode 100644 index 000000000..b25599bd8 --- /dev/null +++ b/ami/ml/models/algorithm_result.py @@ -0,0 +1,190 @@ +import collections +import logging +import typing + +from django.apps import apps +from django.db import models, transaction +from django.db.models import Q +from django.utils import timezone + +from ami.base.models import BaseModel, BaseQuerySet +from ami.ml.results.schemas import validate_result_data + +if typing.TYPE_CHECKING: + from ami.main.models import Occurrence + +logger = logging.getLogger(__name__) + + +class AlgorithmResultQuerySet(BaseQuerySet): + def current(self): + return self.filter(is_current=True) + + def bulk_create(self, objs, *args, **kwargs): + objs = list(objs) + for obj in objs: + obj.data = validate_result_data(obj.kind, obj.data) + return super().bulk_create(objs, *args, **kwargs) + + def bulk_update(self, objs, fields, *args, **kwargs): + objs = list(objs) + if "data" in fields: + for obj in objs: + obj.data = validate_result_data(obj.kind, obj.data) + return super().bulk_update(objs, fields, *args, **kwargs) + + def record(self, **fields) -> "AlgorithmResult": + """Insert one result as the current one for its occurrence, algorithm and kind.""" + results = self.record_many([self.model(**fields)]) + if not results: + raise ValueError("The occurrence has no project.") + return results[0] + + def record_many(self, results: typing.Iterable["AlgorithmResult"]) -> list["AlgorithmResult"]: + """Insert results as current, keeping each occurrence's earlier result of the same algorithm and kind. + + Rows are only inserted, never rewritten, apart from clearing ``is_current`` on the result + each new one replaces. A result whose occurrence has no project is skipped with a warning + rather than failing a run that has already changed data. The returned list holds the + results that were written, in order. + """ + results = list(results) + if not results: + return [] + kept: list[AlgorithmResult] = [] + groups: dict[tuple[int, str], set[int]] = collections.defaultdict(set) + for result in results: + if result.project_id is None: + result.project_id = result.occurrence.project_id + if result.project_id is None: + # Occurrences without a project are a data problem tracked on #1188; skipping keeps the run going. + logger.warning( + f"Not recording the {result.kind} result for Occurrence #{result.occurrence_id}: " + "it has no project." + ) + continue + key = (result.algorithm_id, result.kind) + if result.occurrence_id in groups[key]: + raise ValueError( + f"Two {result.kind} results for Occurrence #{result.occurrence_id} " + "from one algorithm in one write." + ) + groups[key].add(result.occurrence_id) + result.is_current = True + kept.append(result) + if not kept: + return [] + with transaction.atomic(): + # Lock the occurrences in a stable order, so two runs writing results for the same + # occurrence wait for each other instead of both inserting a current row. + list( + apps.get_model("main", "Occurrence") + .objects.select_for_update() + .filter(pk__in={result.occurrence_id for result in kept}) + .order_by("pk") + .values_list("pk", flat=True) + ) + for (algorithm_id, kind), occurrence_ids in groups.items(): + self.filter( + occurrence_id__in=occurrence_ids, algorithm_id=algorithm_id, kind=kind, is_current=True + ).update(is_current=False) + return self.bulk_create(kept) + + def move_to_occurrence(self, kept: "Occurrence", absorbed_ids: typing.Iterable[int]) -> int: + """Move the results of occurrences merged into ``kept``, so their history follows the merge. + + Each algorithm and kind keeps one current result on ``kept``: its own if it has one, + otherwise the latest moved one. The other moved results stay as history. Returns the + number of results moved. + """ + absorbed_ids = set(absorbed_ids) - {kept.pk} + if not absorbed_ids: + return 0 + with transaction.atomic(): + current_keys = set(self.filter(occurrence=kept, is_current=True).values_list("algorithm_id", "kind")) + demote: list[int] = [] + for pk, algorithm_id, kind in ( + self.filter(occurrence_id__in=absorbed_ids, is_current=True) + .order_by("-timestamp", "-pk") + .values_list("pk", "algorithm_id", "kind") + ): + if (algorithm_id, kind) in current_keys: + demote.append(pk) + else: + current_keys.add((algorithm_id, kind)) + if demote: + self.filter(pk__in=demote).update(is_current=False) + return self.filter(occurrence_id__in=absorbed_ids).update(occurrence=kept) + + +@typing.final +class AlgorithmResult(BaseModel): + """What a post-processing run decided about one occurrence: a record of the run, never an input to it. + + The determination never reads this table. A run that changes an occurrence's taxon does so + through the ``Classification`` rows it creates, and those rows point back here through + ``Classification.algorithm_result`` so the history can show a run with what it changed. + ``data`` holds the run's own figures, validated against the model for ``kind`` + (ami/ml/results/schemas.py), and ``value`` repeats the one figure lists filter and sort on. The + ``extra`` object inside ``data`` is stored, shown and exported only; nothing reads it for + logic, and a value a feature needs becomes a typed field. A new result for the same + occurrence, algorithm and kind becomes the current one and the earlier ones stay as + history. Write through ``AlgorithmResult.objects.record`` or ``record_many``. Tracking and + rank roll-ups are the next kinds expected. See #1431. + """ + + class Kind(models.TextChoices): + CLASS_MASKING = "class_masking" + SIZE_FILTER = "size_filter" + + # Copied from the occurrence when the result is written, so per-project + # queries and permission checks need no join. + project = models.ForeignKey("main.Project", on_delete=models.CASCADE, related_name="algorithm_results") + # Indexed together with the timestamp, below. + occurrence = models.ForeignKey( + "main.Occurrence", on_delete=models.CASCADE, related_name="algorithm_results", db_index=False + ) + algorithm = models.ForeignKey("ml.Algorithm", on_delete=models.CASCADE, related_name="algorithm_results") + # The run that wrote the result. Deleting the job keeps the result, with no run to link to. + job = models.ForeignKey( + "jobs.Job", on_delete=models.SET_NULL, null=True, blank=True, related_name="algorithm_results" + ) + # No ``choices``: the registry of data models in ami/ml/results/schemas.py is the list of kinds, + # and every write path rejects a kind without one, so a new kind needs no migration. + kind = models.CharField(max_length=32) + # The kind's headline figure, for filtering and sorting: the share of probability outside + # the list for class masking, the relative size for the size filter. + value = models.FloatField(null=True, blank=True) + data = models.JSONField(default=dict, blank=True) + is_current = models.BooleanField(default=True) + timestamp = models.DateTimeField(default=timezone.now) + + objects = AlgorithmResultQuerySet.as_manager() + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["occurrence", "algorithm", "kind"], + condition=Q(is_current=True), + name="algorithm_result_current_occurrence", + ), + ] + indexes = [ + models.Index(fields=["occurrence", "-timestamp"], name="algorithm_result_occ_time"), + models.Index( + fields=["project", "kind", "value"], + condition=Q(is_current=True, value__isnull=False), + name="algorithm_result_current_value", + ), + ] + + def __str__(self) -> str: + return f"#{self.pk} {self.kind} for Occurrence #{self.occurrence_id} from Algorithm #{self.algorithm_id}" + + def save(self, *args, **kwargs): + self.data = validate_result_data(self.kind, self.data) + if self.project_id is None: + self.project_id = self.occurrence.project_id + if self.project_id is None: + raise ValueError("The occurrence has no project.") + super().save(*args, **kwargs) diff --git a/ami/ml/post_processing/class_masking.py b/ami/ml/post_processing/class_masking.py index 37bed8b8f..94a72186e 100644 --- a/ami/ml/post_processing/class_masking.py +++ b/ami/ml/post_processing/class_masking.py @@ -8,10 +8,11 @@ from django.db.models import QuerySet from django.utils import timezone -from ami.main.models import AlgorithmResult, Classification, Occurrence, SourceImageCollection, TaxaList +from ami.main.models import Classification, Occurrence, SourceImageCollection, TaxaList from ami.ml.models.algorithm import Algorithm, AlgorithmTaskType +from ami.ml.models.algorithm_result import AlgorithmResult from ami.ml.post_processing.base import BasePostProcessingTask -from ami.ml.post_processing.results import BatchResults +from ami.ml.results.writer import AlgorithmResultWriter if typing.TYPE_CHECKING: from ami.jobs.models import Job @@ -84,7 +85,7 @@ def make_classifications_filtered_by_taxa_list( Every occurrence with a re-scored classification gets one algorithm result, written in the same transaction as the batch that changes it, and its new classifications point at - that result (see ``BatchResults``). The result records the probability the list excluded + that result (see ``AlgorithmResultWriter``). The result records the probability the list excluded and the source's original top prediction for the occurrence's winning detection. Returns final counters (checked / masked / occurrences updated) for stage metrics. @@ -129,7 +130,7 @@ def make_classifications_filtered_by_taxa_list( timestamp = timezone.now() masked_count = 0 - results = BatchResults( + results = AlgorithmResultWriter( kind=AlgorithmResult.Kind.CLASS_MASKING, algorithm=new_algorithm, job=job, @@ -212,13 +213,11 @@ def make_classifications_filtered_by_taxa_list( if detection is not None and detection.occurrence is not None: occurrences_to_update.add(detection.occurrence) # The occurrence's result carries the figures of its winning detection: - # the one whose masked classification scores highest (see BatchResults). + # the one whose masked classification scores highest (see AlgorithmResultWriter). results.note( detection.occurrence, { "excluded_probability": float(1.0 - kept_sum), - "original_taxon_id": classification.taxon_id, - "original_score": classification.score, "new_winner_original_rank": int((full_softmax > full_softmax[top_index]).sum()) + 1, }, rank=score, diff --git a/ami/ml/post_processing/small_size_filter.py b/ami/ml/post_processing/small_size_filter.py index f05e0d5a3..29ea7b89f 100644 --- a/ami/ml/post_processing/small_size_filter.py +++ b/ami/ml/post_processing/small_size_filter.py @@ -3,17 +3,10 @@ from django.db.models import QuerySet from django.utils import timezone -from ami.main.models import ( - AlgorithmResult, - Classification, - Detection, - Occurrence, - SourceImageCollection, - Taxon, - TaxonRank, -) +from ami.main.models import Classification, Detection, Occurrence, SourceImageCollection, Taxon, TaxonRank +from ami.ml.models.algorithm_result import AlgorithmResult from ami.ml.post_processing.base import BasePostProcessingTask -from ami.ml.post_processing.results import BatchResults +from ami.ml.results.writer import AlgorithmResultWriter from ami.ml.schemas import BoundingBox @@ -101,7 +94,7 @@ def run(self) -> None: modified_occurrences = 0 checked = 0 # One algorithm result per flagged occurrence, written with the batch that flags it. - results = BatchResults( + results = AlgorithmResultWriter( kind=AlgorithmResult.Kind.SIZE_FILTER, algorithm=self.algorithm, job=self.job, diff --git a/ami/ml/post_processing/tests/test_class_masking.py b/ami/ml/post_processing/tests/test_class_masking.py index cbc2a3add..7e0f3064e 100644 --- a/ami/ml/post_processing/tests/test_class_masking.py +++ b/ami/ml/post_processing/tests/test_class_masking.py @@ -16,7 +16,6 @@ from django.test import TestCase from ami.main.models import ( - AlgorithmResult, Classification, Detection, Occurrence, @@ -27,7 +26,7 @@ TaxonRank, group_images_into_events, ) -from ami.ml.models import Algorithm, AlgorithmCategoryMap +from ami.ml.models import Algorithm, AlgorithmCategoryMap, AlgorithmResult from ami.ml.models.algorithm import AlgorithmTaskType from ami.ml.post_processing.class_masking import ClassMaskingTask, make_classifications_filtered_by_taxa_list from ami.tests.fixtures.main import create_taxa, setup_test_project @@ -411,11 +410,9 @@ def test_a_rescored_occurrence_gets_one_result_linked_to_the_classification_it_c scores = _softmax([2.0, 1.0, 5.0]) self.assertAlmostEqual(result.value, scores[2]) self.assertAlmostEqual(result.data.pop("excluded_probability"), scores[2]) - self.assertAlmostEqual(result.data.pop("original_score"), scores[2]) self.assertEqual( result.data, { - "original_taxon_id": self.species_taxa[2].pk, "new_winner_original_rank": 2, "determination_before_id": self.species_taxa[2].pk, "determination_after_id": self.species_taxa[0].pk, @@ -424,6 +421,9 @@ def test_a_rescored_occurrence_gets_one_result_linked_to_the_classification_it_c ) masked = result.classifications.get() self.assertEqual((masked.taxon, masked.terminal), (self.species_taxa[0], True)) + # The top prediction before masking is the classification the masked one replaced. + self.assertEqual(masked.applied_to.taxon, self.species_taxa[2]) + self.assertAlmostEqual(masked.applied_to.score, scores[2]) self.assertEqual(masked.applied_to.taxon, self.species_taxa[2]) def test_an_occurrence_masking_does_not_change_gets_no_result(self): @@ -467,9 +467,9 @@ def test_an_occurrence_spanning_batches_gets_one_result_covering_every_batch(sel def test_an_occurrence_without_a_project_is_still_rescored_but_gets_no_result(self): """A missing project must not fail a run after it has changed data; the result is skipped with a warning.""" taxa_list, det, occ = self._masking_fixture() - Occurrence.objects.filter(pk=occ.pk).update(project=None, deployment=None) + Occurrence.objects.filter(pk=occ.pk).update(project=None) - with self.assertLogs("ami.main.models", level="WARNING") as logs: + with self.assertLogs("ami.ml.models.algorithm_result", level="WARNING") as logs: ClassMaskingTask(occurrence_id=occ.pk, taxa_list_id=taxa_list.pk, algorithm_id=self.algorithm.pk).run() self.assertIn(f"Occurrence #{occ.pk}", logs.output[0]) diff --git a/ami/ml/post_processing/tests/test_small_size_filter.py b/ami/ml/post_processing/tests/test_small_size_filter.py index f19957c1e..5e324eb79 100644 --- a/ami/ml/post_processing/tests/test_small_size_filter.py +++ b/ami/ml/post_processing/tests/test_small_size_filter.py @@ -6,7 +6,8 @@ from django.test import TestCase from ami.jobs.models import Job, PostProcessingJob -from ami.main.models import AlgorithmResult, Detection, Occurrence, SourceImage, Taxon +from ami.main.models import Detection, Occurrence, SourceImage, Taxon +from ami.ml.models import AlgorithmResult from ami.ml.post_processing.small_size_filter import SmallSizeFilterTask from ami.tests.fixtures.main import create_captures, create_taxa, setup_test_project diff --git a/ami/ml/results/__init__.py b/ami/ml/results/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ami/ml/results/schemas.py b/ami/ml/results/schemas.py new file mode 100644 index 000000000..e08ebe3f5 --- /dev/null +++ b/ami/ml/results/schemas.py @@ -0,0 +1,88 @@ +"""What each kind of algorithm result records: one pydantic model per kind, and the registry of kinds. + +A kind's model holds only what its run alone knows; settings live on the job, and the new taxon on +the classifications the run created. ``extra`` takes whatever else a method returns and is never read +for logic. This is the only module that calls pydantic's API for results, and it imports nothing from +Django, so models, writers and serializers can all import it. Adding a kind = a model + a registry entry. +""" + +import json +from typing import Any + +import pydantic + + +class AlgorithmResultData(pydantic.BaseModel): + # Stored, shown and exported only, never read for logic; a value a feature needs becomes a typed field. + extra: dict[str, Any] = {} + + @pydantic.validator("extra") + def _extra_is_json(cls, value: dict[str, Any]) -> dict[str, Any]: + return json.loads(json.dumps(value)) + + class Config: + extra = "forbid" + + +class DeterminationSnapshot(AlgorithmResultData): + # The occurrence's determination before and after the run, as taxon ids; equal when it did not move. + determination_before_id: int | None = None + determination_after_id: int | None = None + + +class ClassMaskingResultData(DeterminationSnapshot): + """Figures from the occurrence's winning detection: the one whose masked classification scores highest.""" + + # One minus the kept mass of the unmasked softmax: what the list removed, not an out-of-distribution score. + excluded_probability: float + # Where the class that wins after masking ranked before it; 1 means it was already the top. + new_winner_original_rank: int | None = None + + +class SizeFilterResultData(DeterminationSnapshot): + """Figures from the occurrence's smallest filtered detection.""" + + # The detection's box area as a fraction of its image. + relative_size: float + + +ALGORITHM_RESULT_DATA_SCHEMAS: dict[str, type[AlgorithmResultData]] = { + "class_masking": ClassMaskingResultData, + "size_filter": SizeFilterResultData, +} + + +def result_kinds() -> list[str]: + return list(ALGORITHM_RESULT_DATA_SCHEMAS) + + +def _schema_for(kind: str) -> type[AlgorithmResultData]: + schema = ALGORITHM_RESULT_DATA_SCHEMAS.get(kind) + if schema is None: + raise ValueError(f"No result data model is registered for {kind!r}.") + return schema + + +def validate_result_data(kind: str, data: dict | AlgorithmResultData | None) -> dict: + """``data`` as JSON-ready values, or a ValueError when it does not fit the kind's model.""" + schema = _schema_for(kind) + if isinstance(data, AlgorithmResultData) and not isinstance(data, schema): + raise ValueError(f"{type(data).__name__} is not the result data model for {kind!r}.") + values = data.dict() if isinstance(data, AlgorithmResultData) else (data or {}) + # Round-trip through JSON so datetimes are stored as ISO strings. + return json.loads(schema.parse_obj(values).json()) + + +def reference_fields(kind: str) -> dict[str, str]: + """The kind's data fields that hold another record's id, mapped to the reference type.""" + schema = _schema_for(kind) + return { + name: field.field_info.extra["reference"] + for name, field in schema.__fields__.items() + if "reference" in field.field_info.extra + } + + +def data_json_schema(kind: str) -> dict: + """The kind's data model as a JSON schema, for its OpenAPI component.""" + return _schema_for(kind).schema() diff --git a/ami/ml/post_processing/results.py b/ami/ml/results/writer.py similarity index 92% rename from ami/ml/post_processing/results.py rename to ami/ml/results/writer.py index 7bdc3090a..bb04aff63 100644 --- a/ami/ml/post_processing/results.py +++ b/ami/ml/results/writer.py @@ -1,7 +1,7 @@ """One algorithm result per occurrence a post-processing run touches, written inside each batch. A run that re-scores or flags detections changes classifications and determinations in batches, -each in its own transaction. ``BatchResults`` writes the result rows in those same transactions, +each in its own transaction. ``AlgorithmResultWriter`` writes the result rows in those same transactions, so a batch either lands with its results or not at all. Every occurrence the run touches gets exactly one current result: it is created by the first batch that touches the occurrence, before the classifications that batch inserts so they can point at it, and later batches update it with @@ -14,14 +14,15 @@ import datetime import typing -from ami.main.models import AlgorithmResult, Classification, Occurrence +from ami.ml.models import AlgorithmResult if typing.TYPE_CHECKING: from ami.jobs.models import Job + from ami.main.models import Classification, Occurrence from ami.ml.models import Algorithm -class BatchResults: +class AlgorithmResultWriter: def __init__( self, *, @@ -38,7 +39,7 @@ def __init__( self.value_field = value_field self.timestamp = timestamp self.results: dict[int, AlgorithmResult] = {} - # Occurrences with no project, directly or through their station; tried once, then left alone. + # Occurrences with no project; tried once, then left alone. self.skipped: set[int] = set() # Per occurrence, the figures of the detection that represents it so far, with their rank. self.figures: dict[int, tuple[float, dict]] = {} From bb174e24f4bdf302d529957d9637d8c6e5bb2655 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 5 Oct 2026 16:35:28 -0700 Subject: [PATCH 019/134] feat(jobs): list occurrences a job only wrote algorithm results for under its job filter A post-processing run that records a result without creating a detection or a classification, such as tracking, matched nothing under ?job=, so the "View occurrences" link on its job page opened an empty list. The filter gains a third EXISTS branch on the occurrence's algorithm results, backed by a (job, occurrence) index on the new results table, which replaces the job foreign key's own index. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01APWsYj7AAYL5aiq7T3HUwq --- ami/main/models.py | 10 +++++---- ami/main/tests.py | 24 ++++++++++++++++------ ami/ml/migrations/0029_algorithm_result.py | 2 ++ ami/ml/models/algorithm_result.py | 5 ++++- 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/ami/main/models.py b/ami/main/models.py index 5da65ecdc..a1330e88a 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -3438,15 +3438,17 @@ def not_processed_by_algorithm(self, algorithm_ids) -> "OccurrenceQuerySet": def created_or_updated_by_job(self, job_id: int) -> "OccurrenceQuerySet": """Occurrences created or updated by the given job. - An occurrence matches when the job created one of its detections, or a classification - on one of them, so several jobs can match the same occurrence. + An occurrence matches when the job created one of its detections, a classification + on one of them, or an algorithm result for it, so several jobs can match the same occurrence. - Identifications are not matched: people make them, not jobs. Two EXISTS subqueries - return each occurrence once, where a join would return one row per matching result. + Identifications are not matched: people make them, not jobs. EXISTS subqueries return + each occurrence once, where a join would return one row per matching result. """ + AlgorithmResult = apps.get_model("ml", "AlgorithmResult") return self.filter( Exists(Detection.objects.filter(occurrence=OuterRef("pk"), job_id=job_id)) | Exists(Classification.objects.filter(detection__occurrence=OuterRef("pk"), job_id=job_id)) + | Exists(AlgorithmResult.objects.filter(occurrence=OuterRef("pk"), job_id=job_id)) ) def with_timestamps(self): diff --git a/ami/main/tests.py b/ami/main/tests.py index 27785fe7f..b5d68c733 100644 --- a/ami/main/tests.py +++ b/ami/main/tests.py @@ -42,6 +42,7 @@ TaxonRank, group_images_into_events, ) +from ami.ml.models.algorithm_result import AlgorithmResult from ami.ml.models.pipeline import Pipeline from ami.ml.models.processing_service import ProcessingService from ami.ml.models.project_pipeline_config import ProjectPipelineConfig @@ -7244,8 +7245,8 @@ def test_exclude_is_the_complement_of_include(self): class TestOccurrenceJobFilter(APITestCase): """ - Covers the ``?job=`` occurrence filter: occurrences with a detection or a - classification written by the given job. Each occurrence must appear once, however + Covers the ``?job=`` occurrence filter: occurrences with a detection, a classification + or an algorithm result written by the given job. Each occurrence must appear once, however many of its rows the job wrote, and a malformed id must be a 400, not a 500. """ @@ -7281,6 +7282,15 @@ def setUp(self): # Only the other job, and no job at all, must not match. self.occ_other = self._make_occurrence([(self.other_job, [self.other_job])]) self.occ_none = self._make_occurrence([(None, [None])]) + # The job wrote only an algorithm result, as a post-processing run that changes no prediction does. + self.occ_result = self._make_occurrence([(None, [None])]) + AlgorithmResult.objects.record( + occurrence=self.occ_result, + algorithm=self.algorithm, + job=self.job, + kind=AlgorithmResult.Kind.SIZE_FILTER, + data={"relative_size": 0.001}, + ) def _make_occurrence(self, detections) -> Occurrence: """``detections`` is a list of (detection job, [classification jobs]).""" @@ -7316,8 +7326,10 @@ def test_returns_each_occurrence_the_job_wrote_once(self): self.assertEqual(response.status_code, 200) ids = [row["id"] for row in response.json()["results"]] self.assertEqual(len(ids), len(set(ids))) - self.assertEqual(set(ids), {self.occ_detected.pk, self.occ_classified.pk, self.occ_multi.pk}) - self.assertEqual(response.json()["count"], 3) + self.assertEqual( + set(ids), {self.occ_detected.pk, self.occ_classified.pk, self.occ_multi.pk, self.occ_result.pk} + ) + self.assertEqual(response.json()["count"], 4) def test_other_job_matches_only_its_own_occurrences(self): response = self._list(self.other_job.pk) @@ -7349,8 +7361,8 @@ def filtered(): finally: # cachalot_disabled() does not restore itself when the block raises. disabled.__exit__(None, None, None) - self.assertEqual(ids, {self.occ_detected.pk, self.occ_classified.pk, self.occ_multi.pk}) - self.assertEqual(count, 3) + self.assertEqual(ids, {self.occ_detected.pk, self.occ_classified.pk, self.occ_multi.pk, self.occ_result.pk}) + self.assertEqual(count, 4) class TestCleanupNullOnlyOccurrencesCommand(TestCase): diff --git a/ami/ml/migrations/0029_algorithm_result.py b/ami/ml/migrations/0029_algorithm_result.py index afe64177a..c35257979 100644 --- a/ami/ml/migrations/0029_algorithm_result.py +++ b/ami/ml/migrations/0029_algorithm_result.py @@ -36,6 +36,7 @@ class Migration(migrations.Migration): "job", models.ForeignKey( blank=True, + db_index=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="algorithm_results", @@ -68,6 +69,7 @@ class Migration(migrations.Migration): fields=["project", "kind", "value"], name="algorithm_result_current_value", ), + models.Index(fields=["job", "occurrence"], name="algorithm_result_job_occ"), ], }, ), diff --git a/ami/ml/models/algorithm_result.py b/ami/ml/models/algorithm_result.py index b25599bd8..1a16d1398 100644 --- a/ami/ml/models/algorithm_result.py +++ b/ami/ml/models/algorithm_result.py @@ -146,8 +146,9 @@ class Kind(models.TextChoices): ) algorithm = models.ForeignKey("ml.Algorithm", on_delete=models.CASCADE, related_name="algorithm_results") # The run that wrote the result. Deleting the job keeps the result, with no run to link to. + # Indexed together with the occurrence, below. job = models.ForeignKey( - "jobs.Job", on_delete=models.SET_NULL, null=True, blank=True, related_name="algorithm_results" + "jobs.Job", on_delete=models.SET_NULL, null=True, blank=True, related_name="algorithm_results", db_index=False ) # No ``choices``: the registry of data models in ami/ml/results/schemas.py is the list of kinds, # and every write path rejects a kind without one, so a new kind needs no migration. @@ -176,6 +177,8 @@ class Meta: condition=Q(is_current=True, value__isnull=False), name="algorithm_result_current_value", ), + # Leads with the job, for the ?job= occurrence filter and for deleting a job. + models.Index(fields=["job", "occurrence"], name="algorithm_result_job_occ"), ] def __str__(self) -> str: From f38679d52662c783f80795e19b7ed71f7332f74d Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 5 Oct 2026 16:35:59 -0700 Subject: [PATCH 020/134] feat(results): let a result kind declare which data fields reference other records schemas.reference("") marks a data field as holding another record's id, so the occurrence history can resolve it to a named link. Each kind's data model is also published as a closed JSON schema for its OpenAPI component. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01APWsYj7AAYL5aiq7T3HUwq --- ami/ml/results/schemas.py | 5 +++++ ami/ml/results/tests.py | 26 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 ami/ml/results/tests.py diff --git a/ami/ml/results/schemas.py b/ami/ml/results/schemas.py index e08ebe3f5..9f0eb04bd 100644 --- a/ami/ml/results/schemas.py +++ b/ami/ml/results/schemas.py @@ -12,6 +12,11 @@ import pydantic +def reference(ref_type: str, default: Any = None) -> Any: + """Declare a data field that holds another record's id; the history resolves it to ``{type, id, name}``.""" + return pydantic.Field(default, reference=ref_type) + + class AlgorithmResultData(pydantic.BaseModel): # Stored, shown and exported only, never read for logic; a value a feature needs becomes a typed field. extra: dict[str, Any] = {} diff --git a/ami/ml/results/tests.py b/ami/ml/results/tests.py new file mode 100644 index 000000000..21e8c91b6 --- /dev/null +++ b/ami/ml/results/tests.py @@ -0,0 +1,26 @@ +from django.test import SimpleTestCase + +from ami.ml.results import schemas + + +class ResultKindRegistryTest(SimpleTestCase): + """Each registered kind publishes its data schema and declares which fields reference other records.""" + + def test_every_kind_has_a_closed_json_schema(self): + for kind in schemas.result_kinds(): + with self.subTest(kind): + schema = schemas.data_json_schema(kind) + self.assertEqual(schema["type"], "object") + self.assertFalse(schema.get("additionalProperties", True)) + self.assertIn("extra", schema["properties"]) + + def test_reference_fields_come_from_field_metadata(self): + class Probe(schemas.AlgorithmResultData): + merged_into_id: int | None = schemas.reference("occurrence") + + schemas.ALGORITHM_RESULT_DATA_SCHEMAS["probe"] = Probe + try: + self.assertEqual(schemas.reference_fields("probe"), {"merged_into_id": "occurrence"}) + self.assertEqual(schemas.reference_fields("class_masking"), {}) + finally: + del schemas.ALGORITHM_RESULT_DATA_SCHEMAS["probe"] From 29aaa4944ddb21251a0dbb4dfd56e9ce0697f57f Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 5 Oct 2026 16:36:47 -0700 Subject: [PATCH 021/134] feat(history): resolve ids shown in an occurrence's history to named references A reference is {type, id, name}, read with one query per type; name is None when the row no longer exists, so the UI can show the id as plain text instead of a broken link. Job settings that name another record (capture set, species list, algorithm, occurrence) are listed by key. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01APWsYj7AAYL5aiq7T3HUwq --- ami/main/models_future/references.py | 69 ++++++++++++++++++++++++++++ ami/main/test_occurrence_history.py | 27 +++++++++++ 2 files changed, 96 insertions(+) create mode 100644 ami/main/models_future/references.py diff --git a/ami/main/models_future/references.py b/ami/main/models_future/references.py new file mode 100644 index 000000000..d88f88cd5 --- /dev/null +++ b/ami/main/models_future/references.py @@ -0,0 +1,69 @@ +"""Ids the occurrence history shows, resolved to ``{type, id, name}`` so the UI can link them with one table. + +``name`` is None when the row no longer exists; the UI then shows the id as text. Resolution reads names +only, one query per type, and is used for ids stored on the occurrence's own results and jobs. +""" + +import collections +import dataclasses +import typing + +from django.apps import apps + + +@dataclasses.dataclass(frozen=True) +class Ref: + type: str + id: int + name: str | None + + +# Reference type -> (model, attribute holding its display name; None shows "#"). +REFERENCE_TYPES: dict[str, tuple[str, str | None]] = { + "algorithm": ("ml.Algorithm", "name"), + "capture_set": ("main.SourceImageCollection", "name"), + "job": ("jobs.Job", "name"), + "occurrence": ("main.Occurrence", None), + "taxa_list": ("main.TaxaList", "name"), + "taxon": ("main.Taxon", "name"), +} + +# Job settings (post-processing configs) that hold another record's id. +JOB_SETTING_REFERENCES: dict[str, str] = { + "source_image_collection_id": "capture_set", + "taxa_list_id": "taxa_list", + "algorithm_id": "algorithm", + "occurrence_id": "occurrence", +} + + +def job_setting_references(config: typing.Any) -> list[tuple[str, str, int]]: + """``(setting key, reference type, id)`` for each job setting that names another record.""" + if not isinstance(config, dict): + return [] + return [ + (key, ref_type, config[key]) + for key, ref_type in JOB_SETTING_REFERENCES.items() + if isinstance(config.get(key), int) and not isinstance(config.get(key), bool) + ] + + +def resolve_references(wanted: typing.Iterable[tuple[str, int]]) -> dict[tuple[str, int], Ref]: + """Each ``(type, id)`` as a ``Ref``, reading names with one query per type. + + An unknown type raises KeyError: a result kind declared a reference type nobody mapped. + """ + ids_by_type: dict[str, set[int]] = collections.defaultdict(set) + for ref_type, ref_id in wanted: + ids_by_type[ref_type].add(ref_id) + resolved: dict[tuple[str, int], Ref] = {} + for ref_type, ids in ids_by_type.items(): + model_label, name_attr = REFERENCE_TYPES[ref_type] + model = apps.get_model(model_label) + if name_attr is None: + names = {pk: f"#{pk}" for pk in model.objects.filter(pk__in=ids).values_list("pk", flat=True)} + else: + names = dict(model.objects.filter(pk__in=ids).values_list("pk", name_attr)) + for ref_id in ids: + resolved[(ref_type, ref_id)] = Ref(ref_type, ref_id, names.get(ref_id)) + return resolved diff --git a/ami/main/test_occurrence_history.py b/ami/main/test_occurrence_history.py index 5e0180ebf..dccf7f82e 100644 --- a/ami/main/test_occurrence_history.py +++ b/ami/main/test_occurrence_history.py @@ -11,6 +11,7 @@ from ami.jobs.models import Job from ami.main.models import Classification, Detection, Identification, Occurrence, SourceImage, TaxaList, Taxon +from ami.main.models_future.references import Ref, job_setting_references, resolve_references from ami.ml.models import Algorithm, AlgorithmResult from ami.tests.fixtures.main import create_captures, create_taxa, setup_test_project from ami.users.models import User @@ -151,6 +152,32 @@ def test_the_model_lives_in_the_ml_app(self): self.assertEqual(AlgorithmResult._meta.app_label, "ml") +class ReferenceTestCase(TestCase): + """Ids shown in the history become {type, id, name}; a deleted row keeps its id with no name.""" + + def setUp(self): + self.project, self.deployment = setup_test_project(reuse=False) + + def test_resolves_each_type_in_one_query_and_marks_missing_rows(self): + taxa_list = TaxaList.objects.create(name="Kept species") + algorithm = Algorithm.objects.create(name="Classifier", key="ref-test-classifier") + wanted = [("taxa_list", taxa_list.pk), ("algorithm", algorithm.pk), ("taxa_list", 999999)] + with no_query_cache(), self.assertNumQueries(2): + refs = resolve_references(wanted) + self.assertEqual(refs[("taxa_list", taxa_list.pk)], Ref("taxa_list", taxa_list.pk, "Kept species")) + self.assertEqual(refs[("algorithm", algorithm.pk)].name, "Classifier") + self.assertEqual(refs[("taxa_list", 999999)], Ref("taxa_list", 999999, None)) + + def test_job_settings_yield_only_known_integer_references(self): + config = {"taxa_list_id": 3, "source_image_collection_id": 5, "size_threshold": 0.01, "algorithm_id": "x"} + self.assertEqual( + sorted(job_setting_references(config)), + [("source_image_collection_id", "capture_set", 5), ("taxa_list_id", "taxa_list", 3)], + ) + self.assertEqual(job_setting_references(None), []) + self.assertEqual(job_setting_references(["not", "a", "dict"]), []) # type: ignore[arg-type] + + class OccurrenceFixtureTestCase(APITestCase): """One occurrence of four classified detections, with a project manager and a basic member.""" From fadf0abbfd474656d2bce53b9f052eaedef20048 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 5 Oct 2026 16:38:56 -0700 Subject: [PATCH 022/134] feat(history): organise history entries in sections, show what a run replaced, and link classifications to results only directly OccurrenceTimelineEntry groups its fields by entry type: every entry, algorithm results (kind, data, data references, determination before and after, created classifications), and identifications and predictions (details). Each classification a run created comes with the classification it replaced, read in one separate query so no score arrays load; a deleted original shows nothing as replaced. Job settings and kind data fields that name another record are resolved to references in one query per type. A classification belongs to a result only through Classification.algorithm_result; the fallback that matched by job is removed. Predictions carry the job that wrote them. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01APWsYj7AAYL5aiq7T3HUwq --- ami/main/models_future/history.py | 179 ++++++++++++++++------------ ami/main/test_occurrence_history.py | 92 ++++++++++++++ 2 files changed, 196 insertions(+), 75 deletions(-) diff --git a/ami/main/models_future/history.py b/ami/main/models_future/history.py index a853c3bee..18e40b2bc 100644 --- a/ami/main/models_future/history.py +++ b/ami/main/models_future/history.py @@ -1,8 +1,9 @@ """An occurrence's history: what algorithms and people did to it, newest first. -What a post-processing run decided about an occurrence is an ``AlgorithmResult``. Identifications -and predictions keep their own tables, and ``occurrence_timeline`` merges all three into one list -for the history endpoint. +``occurrence_timeline`` merges an occurrence's algorithm results, identifications and predictions +into one list of ``OccurrenceTimelineEntry`` for the history endpoint. What a post-processing run +decided about an occurrence is an ``AlgorithmResult``; identifications and predictions keep their +own tables. """ from __future__ import annotations @@ -11,83 +12,84 @@ import datetime import typing -from django.db.models import Q - -from ami.main.models import Classification, Identification, Occurrence, TaxaList, Taxon, User +from ami.main.models import Classification, Identification, Occurrence, Taxon, User +from ami.main.models_future.references import Ref, job_setting_references, resolve_references from ami.ml.models import Algorithm, AlgorithmResult +from ami.ml.results.schemas import reference_fields if typing.TYPE_CHECKING: from ami.jobs.models import Job @dataclasses.dataclass -class TimelineEntry: - """One entry of the merged history, in the shape ``OccurrenceHistoryEntrySerializer`` reads.""" +class CreatedClassification: + """A classification a run created, with the one it replaced (through applied_to), if that still exists.""" + + classification: Classification + replaced: Classification | None + + +@dataclasses.dataclass +class OccurrenceTimelineEntry: + """One entry of an occurrence's merged history. Sections below say which fields each type fills.""" - type: str + # --- Every entry + type: str # "algorithm_result" | "identification" | "prediction" id: int timestamp: datetime.datetime - subtype: str | None = None - user: User | None = None - algorithm: Algorithm | None = None - job: Job | None = None - taxon: Taxon | None = None - taxon_before: Taxon | None = None - score: float | None = None - payload: dict = dataclasses.field(default_factory=dict) - comment: str = "" - withdrawn: bool = False + user: User | None = None # identification + algorithm: Algorithm | None = None # result, prediction + job: Job | None = None # result, prediction + job_references: dict[str, Ref] = dataclasses.field(default_factory=dict) # setting key -> Ref + taxon: Taxon | None = None # identification, prediction + score: float | None = None # prediction's score; result's value + # --- Algorithm result + kind: str | None = None + data: dict = dataclasses.field(default_factory=dict) # validated per kind + data_references: dict[str, Ref] = dataclasses.field(default_factory=dict) # data field -> Ref + determination_before: Taxon | None = None + determination_after: Taxon | None = None + classifications: list[CreatedClassification] = dataclasses.field(default_factory=list) is_current: bool | None = None - # For a result: the classifications the run created, best score first. - classifications: list[Classification] = dataclasses.field(default_factory=list) - # For a class masking result: the source classifier's top taxon before masking, the species - # list and the classifier, the latter two from the job's settings. - original_taxon: Taxon | None = None - taxa_list: TaxaList | None = None - source_algorithm: Algorithm | None = None + # --- Identification and prediction + details: dict = dataclasses.field(default_factory=dict) -def occurrence_timeline(occurrence: Occurrence) -> list[TimelineEntry]: +def occurrence_timeline(occurrence: Occurrence) -> list[OccurrenceTimelineEntry]: """Results, identifications and predictions of one occurrence, merged newest first. A result comes with the classifications its run created, so the history shows the run and what it changed as one entry; those classifications are not listed again as predictions. Every other algorithm contributes one prediction, its best. A prediction names the result that superseded it when a result's classification re-scored it, or when a result's - classification on the same detection outranks it as a terminal prediction. The query count - does not grow with the entries. + classification on the same detection outranks it as a terminal prediction. Ids the entries + show are resolved to references with one query per type. The query count does not grow + with the entries. """ results = list( AlgorithmResult.objects.filter(occurrence=occurrence) .select_related("algorithm", "job") .order_by("-timestamp", "-pk") ) - settings = {result.pk: _job_settings(result.job) for result in results} taxa = _by_id( Taxon, - { - result.data.get(key) - for result in results - for key in ("determination_before_id", "determination_after_id", "original_taxon_id") - }, + {result.data.get(key) for result in results for key in ("determination_before_id", "determination_after_id")}, ) - taxa_lists = _by_id(TaxaList, {config.get("taxa_list_id") for config in settings.values()}) - source_algorithms = _by_id(Algorithm, {config.get("algorithm_id") for config in settings.values()}) created_by_result = _classifications_created_by(results, occurrence) - created_ids = {c.pk for created in created_by_result.values() for c in created} + created_ids = {c.classification.pk for created in created_by_result.values() for c in created} rescored_by = { - c.applied_to_id: result_id + c.classification.applied_to_id: result_id for result_id, created in created_by_result.items() for c in created - if c.applied_to_id is not None + if c.classification.applied_to_id is not None } # Only a classification that re-scored nothing (the size filter's) supersedes by detection; a # re-scored one names its original through applied_to, so other classifiers stay unmarked. outranked_by = { - c.detection_id: result_id + c.classification.detection_id: result_id for result_id, created in created_by_result.items() for c in created - if c.applied_to_id is None + if c.classification.applied_to_id is None } def superseded_by(prediction: Classification) -> int | None: @@ -98,38 +100,33 @@ def superseded_by(prediction: Classification) -> int | None: return None entries = [ - TimelineEntry( + OccurrenceTimelineEntry( type="algorithm_result", id=result.pk, timestamp=result.timestamp, - subtype=result.kind, algorithm=result.algorithm, job=result.job, - taxon=taxa.get(result.data.get("determination_after_id")), - taxon_before=taxa.get(result.data.get("determination_before_id")), score=result.value, - payload=result.data, - is_current=result.is_current, + kind=result.kind, + data=result.data, + determination_before=taxa.get(result.data.get("determination_before_id")), + determination_after=taxa.get(result.data.get("determination_after_id")), classifications=created_by_result.get(result.pk, []), - original_taxon=taxa.get(result.data.get("original_taxon_id")), - taxa_list=taxa_lists.get(settings[result.pk].get("taxa_list_id")), - source_algorithm=source_algorithms.get(settings[result.pk].get("algorithm_id")), + is_current=result.is_current, ) for result in results ] identifications = Identification.objects.filter(occurrence=occurrence).select_related("user", "taxon") entries.extend( - TimelineEntry( + OccurrenceTimelineEntry( type="identification", id=identification.pk, timestamp=identification.created_at, user=identification.user, taxon=identification.taxon, - comment=identification.comment or "", - withdrawn=identification.withdrawn, - payload={ - "comment": identification.comment, + details={ + "comment": identification.comment or "", "withdrawn": identification.withdrawn, "agreed_with_identification_id": identification.agreed_with_identification_id, "agreed_with_prediction_id": identification.agreed_with_prediction_id, @@ -139,14 +136,15 @@ def superseded_by(prediction: Classification) -> int | None: ) entries.extend( - TimelineEntry( + OccurrenceTimelineEntry( type="prediction", id=prediction.pk, timestamp=prediction.created_at, algorithm=prediction.algorithm, + job=prediction.job, taxon=prediction.taxon, score=prediction.score, - payload={ + details={ "detection_id": prediction.detection_id, "terminal": prediction.terminal, "applied_to_id": prediction.applied_to_id, @@ -157,10 +155,34 @@ def superseded_by(prediction: Classification) -> int | None: if prediction.pk not in created_ids ) + _resolve_entry_references(entries) entries.sort(key=lambda entry: (entry.timestamp, entry.id), reverse=True) return entries +def _resolve_entry_references(entries: list[OccurrenceTimelineEntry]) -> None: + """Fill each entry's job and data references, resolving every id the entries name at once.""" + job_refs = { + id(entry): job_setting_references(_job_settings(entry.job)) for entry in entries if entry.job is not None + } + data_refs = { + id(entry): [ + (field, ref_type, entry.data[field]) + for field, ref_type in reference_fields(entry.kind).items() + if isinstance(entry.data.get(field), int) + ] + for entry in entries + if entry.type == "algorithm_result" + } + wanted = [(ref_type, ref_id) for refs in (*job_refs.values(), *data_refs.values()) for _, ref_type, ref_id in refs] + if not wanted: + return + resolved = resolve_references(wanted) + for entry in entries: + entry.job_references = {key: resolved[(t, i)] for key, t, i in job_refs.get(id(entry), [])} + entry.data_references = {field: resolved[(t, i)] for field, t, i in data_refs.get(id(entry), [])} + + def _job_settings(job: Job | None) -> dict: """The settings a post-processing job ran with, or an empty dict for any other job.""" config = (job.params or {}).get("config") if job is not None else None @@ -174,34 +196,41 @@ def _by_id(model, ids: set) -> dict: def _classifications_created_by( results: list[AlgorithmResult], occurrence: Occurrence -) -> dict[int, list[Classification]]: +) -> dict[int, list[CreatedClassification]]: """The classifications each result's run created on the occurrence, keyed by result id. - A classification names its result directly through ``algorithm_result``. One written by a - run before that link existed is matched through its job instead; when that job left more - than one result on the occurrence, the latest takes it. + A classification names its result through ``algorithm_result``. Each comes with the + classification it replaced, read in one more query. """ if not results: return {} - result_by_job = {} - for result in reversed(results): - if result.job_id is not None: - result_by_job[result.job_id] = result.pk - created = ( - Classification.objects.filter(detection__occurrence=occurrence) - .filter( - Q(algorithm_result_id__in=[result.pk for result in results]) - | Q(algorithm_result__isnull=True, job_id__in=list(result_by_job)) + created = list( + Classification.objects.filter( + detection__occurrence=occurrence, algorithm_result_id__in=[result.pk for result in results] ) .select_related("taxon", "algorithm") # The score arrays can hold tens of thousands of values per row and are not shown here. .defer("scores", "logits") .order_by("-score", "-pk") ) - by_result: dict[int, list[Classification]] = {} + # A separate query, not select_related("applied_to"): .defer() does not reach a self-join, + # so the replaced rows' score arrays would load. + replaced_ids = {c.applied_to_id for c in created if c.applied_to_id is not None} + replaced = ( + { + c.pk: c + for c in Classification.objects.filter(pk__in=replaced_ids) + .select_related("taxon") + .only("pk", "score", "taxon", "taxon__name", "taxon__rank") + } + if replaced_ids + else {} + ) + by_result: dict[int, list[CreatedClassification]] = {} for classification in created: - result_id = classification.algorithm_result_id or result_by_job[classification.job_id] - by_result.setdefault(result_id, []).append(classification) + by_result.setdefault(classification.algorithm_result_id, []).append( + CreatedClassification(classification, replaced.get(classification.applied_to_id)) + ) return by_result @@ -217,7 +246,7 @@ def rank(prediction: Classification) -> tuple: return (score, prediction.terminal, prediction.created_at, prediction.pk) best: dict[int | None, Classification] = {} - for prediction in occurrence.predictions().defer("scores", "logits"): + for prediction in occurrence.predictions().select_related("job").defer("scores", "logits"): current = best.get(prediction.algorithm_id) if current is None or rank(prediction) > rank(current): best[prediction.algorithm_id] = prediction diff --git a/ami/main/test_occurrence_history.py b/ami/main/test_occurrence_history.py index dccf7f82e..3270febda 100644 --- a/ami/main/test_occurrence_history.py +++ b/ami/main/test_occurrence_history.py @@ -11,6 +11,7 @@ from ami.jobs.models import Job from ami.main.models import Classification, Detection, Identification, Occurrence, SourceImage, TaxaList, Taxon +from ami.main.models_future.history import occurrence_timeline from ami.main.models_future.references import Ref, job_setting_references, resolve_references from ami.ml.models import Algorithm, AlgorithmResult from ami.tests.fixtures.main import create_captures, create_taxa, setup_test_project @@ -468,3 +469,94 @@ def test_an_occurrence_the_default_filters_hide_still_has_a_history(self): detail = self.client.get(f"/api/v2/occurrences/{self.occurrence.pk}/?project_id={self.project.pk}") self.assertEqual(detail.status_code, 404) self.assertEqual(self.client.get(self.url()).status_code, 200) + + def test_a_result_lists_what_each_created_classification_replaced(self): + classifier = Algorithm.objects.create(name="Classifier", key="replaced-test-classifier") + masker = Algorithm.objects.create(name="Masked", key="replaced-test-masker") + original = Classification.objects.get(detection=self.detections[1]) + Classification.objects.filter(pk=original.pk).update(terminal=False, algorithm=classifier) + result = AlgorithmResult.objects.record( + occurrence=self.occurrence, + algorithm=masker, + kind=CLASS_MASKING, + value=0.4, + data={"excluded_probability": 0.4}, + ) + masked = Classification.objects.create( + detection=self.detections[1], + taxon=self.other_taxon, + score=0.9, + algorithm=masker, + timestamp=datetime.datetime.now(), + applied_to=original, + algorithm_result=result, + ) + # The original was deleted (applied_to is SET_NULL until #1472), so nothing is shown as replaced. + orphan = Classification.objects.create( + detection=self.detections[2], + taxon=self.other_taxon, + score=0.8, + algorithm=masker, + timestamp=datetime.datetime.now(), + applied_to=None, + algorithm_result=result, + ) + entry = next(e for e in occurrence_timeline(self.occurrence) if e.type == "algorithm_result") + by_id = {c.classification.pk: c for c in entry.classifications} + self.assertEqual(by_id[masked.pk].replaced.pk, original.pk) + self.assertEqual(by_id[masked.pk].replaced.taxon_id, original.taxon_id) + self.assertIsNone(by_id[orphan.pk].replaced) + + def test_classifications_link_to_a_result_only_through_algorithm_result(self): + """A classification sharing the result's job but not linked to it stays an ordinary prediction.""" + AlgorithmResult.objects.record( + occurrence=self.occurrence, + algorithm=self.size_filter, + job=self.job, + kind=SIZE_FILTER, + data={"relative_size": 0.001}, + ) + unlinked = Classification.objects.create( + detection=self.detections[0], + taxon=self.taxon, + score=0.99, + algorithm=self.size_filter, + timestamp=datetime.datetime.now(), + job=self.job, + ) + entries = occurrence_timeline(self.occurrence) + result_entry = next(e for e in entries if e.type == "algorithm_result") + self.assertEqual(result_entry.classifications, []) + self.assertIn(unlinked.pk, [e.id for e in entries if e.type == "prediction"]) + + def test_predictions_carry_the_job_that_wrote_them(self): + Classification.objects.filter(detection__occurrence=self.occurrence).update(job=self.job) + predictions = [e for e in occurrence_timeline(self.occurrence) if e.type == "prediction"] + self.assertTrue(predictions) + self.assertTrue(all(e.job == self.job for e in predictions)) + + def test_job_settings_are_resolved_to_references(self): + taxa_list = TaxaList.objects.create(name="Kept species") + self.job.params = {"config": {"taxa_list_id": taxa_list.pk, "size_threshold": 0.01}} + self.job.save() + AlgorithmResult.objects.record( + occurrence=self.occurrence, + algorithm=self.size_filter, + job=self.job, + kind=SIZE_FILTER, + data={"relative_size": 0.001}, + ) + entry = next(e for e in occurrence_timeline(self.occurrence) if e.type == "algorithm_result") + self.assertEqual(entry.job_references, {"taxa_list_id": Ref("taxa_list", taxa_list.pk, "Kept species")}) + + def test_a_result_without_a_job_has_no_settings_or_references(self): + AlgorithmResult.objects.record( + occurrence=self.occurrence, + algorithm=self.size_filter, + job=None, + kind=SIZE_FILTER, + data={"relative_size": 0.001}, + ) + entry = next(e for e in occurrence_timeline(self.occurrence) if e.type == "algorithm_result") + self.assertIsNone(entry.job) + self.assertEqual(entry.job_references, {}) From 053afe26114343e0742b74c28da58e9649ae6624 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 5 Oct 2026 16:44:19 -0700 Subject: [PATCH 023/134] feat(history): publish each result kind as a typed OpenAPI component and return references with the history The history endpoint's schema is a plain oneOf of IdentificationEntry, PredictionEntry and one ResultEntry per registered result kind, whose literal type and kind fields let a client generated from the schema narrow data to the kind's model. The components are built from the kind registry, so adding a kind needs no serializer change; the single-value enums are named in SPECTACULAR_SETTINGS["ENUM_NAME_OVERRIDES"] so the schema keeps readable names. Each entry serializes in sections: identification and prediction details, a result's kind, data, data references, determination before and after, and its created classifications with the classification each replaced. A job carries its settings and the records they name as {type, id, name} references. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01APWsYj7AAYL5aiq7T3HUwq --- ami/main/api/serializers.py | 160 ++++++++++++++++++++-------- ami/main/api/views.py | 7 +- ami/main/test_occurrence_history.py | 87 +++++++++++---- ami/ml/results/schemas.py | 5 + config/settings/base.py | 11 ++ 5 files changed, 205 insertions(+), 65 deletions(-) diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index 90b3e6ade..820c99a57 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -2,7 +2,7 @@ import datetime from django.db.models import QuerySet -from drf_spectacular.utils import extend_schema_field +from drf_spectacular.utils import PolymorphicProxySerializer, extend_schema_field from guardian.shortcuts import get_perms from rest_framework import serializers from rest_framework.request import Request @@ -14,6 +14,7 @@ from ami.jobs.models import Job from ami.main.models import Tag from ami.ml.models import Algorithm, Pipeline +from ami.ml.results.schemas import data_json_schema, result_kind_title, result_kinds from ami.ml.serializers import AlgorithmSerializer, PipelineNestedSerializer from ami.users.models import User from ami.users.roles import ProjectManager @@ -2081,7 +2082,23 @@ class HistoryAlgorithmSerializer(serializers.Serializer): key = serializers.CharField() +class HistoryTaxonSerializer(serializers.Serializer): + id = serializers.IntegerField() + name = serializers.CharField() + rank = serializers.CharField() + + +class RefSerializer(serializers.Serializer): + """Another record the history mentions. ``name`` is null when it no longer exists.""" + + type = serializers.CharField(help_text="What kind of record: job, taxa_list, capture_set, algorithm, ...") + id = serializers.IntegerField() + name = serializers.CharField(allow_null=True) + + class HistoryJobSerializer(serializers.Serializer): + """A job in the history. ``references`` is filled per entry by ``HistoryEntryBaseSerializer``.""" + id = serializers.IntegerField() name = serializers.CharField() config = serializers.SerializerMethodField( @@ -2090,6 +2107,9 @@ class HistoryJobSerializer(serializers.Serializer): "Null for other jobs, and for a post-processing job whose stored config is not an object." ) ) + references = serializers.DictField( + child=RefSerializer(), read_only=True, help_text="Settings that name another record, by setting key." + ) @extend_schema_field(serializers.JSONField(allow_null=True)) def get_config(self, job) -> dict | None: @@ -2097,64 +2117,120 @@ def get_config(self, job) -> dict | None: return config if isinstance(config, dict) else None -class HistoryTaxonSerializer(serializers.Serializer): +class ReplacedClassificationSerializer(serializers.Serializer): id = serializers.IntegerField() - name = serializers.CharField() - rank = serializers.CharField() - + taxon = HistoryTaxonSerializer(allow_null=True) + score = serializers.FloatField(allow_null=True) -class HistoryTaxaListSerializer(serializers.Serializer): - id = serializers.IntegerField() - name = serializers.CharField() +class CreatedClassificationSerializer(serializers.Serializer): + """A classification a run created, and the one it replaced (null when none, or when it was deleted).""" -class HistoryClassificationSerializer(serializers.Serializer): - """A classification a post-processing run created, shown inside the run's result.""" + id = serializers.IntegerField(source="classification.id") + taxon = HistoryTaxonSerializer(source="classification.taxon", allow_null=True) + score = serializers.FloatField(source="classification.score", allow_null=True) + terminal = serializers.BooleanField(source="classification.terminal") + detection_id = serializers.IntegerField(source="classification.detection_id") + replaced = ReplacedClassificationSerializer(allow_null=True) - id = serializers.IntegerField() - taxon = HistoryTaxonSerializer(allow_null=True) - score = serializers.FloatField(allow_null=True) - terminal = serializers.BooleanField() - detection_id = serializers.IntegerField() - applied_to_id = serializers.IntegerField( - allow_null=True, help_text="The classification this one re-scored and demoted, when there is one." - ) +class HistoryEntryBaseSerializer(serializers.Serializer): + """The fields every entry of an occurrence's history has.""" -class OccurrenceHistoryEntrySerializer(serializers.Serializer): - """One entry of an occurrence's history, newest first. ``type`` says which table it came from.""" - - type = serializers.ChoiceField(choices=["algorithm_result", "identification", "prediction"]) id = serializers.IntegerField(help_text="Primary key of the row in the table ``type`` names.") timestamp = serializers.DateTimeField() - subtype = serializers.CharField( - allow_null=True, help_text="An algorithm result's kind (class_masking, size_filter)." - ) user = HistoryUserSerializer(allow_null=True) algorithm = HistoryAlgorithmSerializer(allow_null=True) - job = HistoryJobSerializer(allow_null=True) - taxon = HistoryTaxonSerializer( - allow_null=True, help_text="The identified or predicted taxon, or the determination after a result." - ) - taxon_before = HistoryTaxonSerializer(allow_null=True, help_text="The determination before a result.") + job = serializers.SerializerMethodField() + taxon = HistoryTaxonSerializer(allow_null=True, help_text="The identified or predicted taxon.") score = serializers.FloatField( allow_null=True, help_text="A prediction's score, or a result's headline value (see its kind)." ) - payload = serializers.JSONField(help_text="Details that depend on the type and subtype.") - classifications = HistoryClassificationSerializer( - many=True, help_text="For a result: the classifications its run created, best score first." + + @extend_schema_field(HistoryJobSerializer(allow_null=True)) + def get_job(self, entry) -> dict | None: + if entry.job is None: + return None + return { + **HistoryJobSerializer(entry.job).data, + "references": {key: RefSerializer(ref).data for key, ref in entry.job_references.items()}, + } + + +class IdentificationDetailsSerializer(serializers.Serializer): + comment = serializers.CharField(allow_blank=True) + withdrawn = serializers.BooleanField() + agreed_with_identification_id = serializers.IntegerField(allow_null=True) + agreed_with_prediction_id = serializers.IntegerField(allow_null=True) + + +class PredictionDetailsSerializer(serializers.Serializer): + detection_id = serializers.IntegerField() + terminal = serializers.BooleanField() + applied_to_id = serializers.IntegerField( + allow_null=True, help_text="The classification this one re-scored and demoted, when there is one." ) - original_taxon = HistoryTaxonSerializer( - allow_null=True, help_text="For class masking: the source classifier's top taxon before masking." + superseded_by_result_id = serializers.IntegerField( + allow_null=True, help_text="The algorithm result whose classification replaced this one." ) - taxa_list = HistoryTaxaListSerializer( - allow_null=True, help_text="For class masking: the species list used, from the job's settings." + + +class IdentificationEntrySerializer(HistoryEntryBaseSerializer): + type = serializers.ChoiceField(choices=["identification"]) + details = IdentificationDetailsSerializer() + + +class PredictionEntrySerializer(HistoryEntryBaseSerializer): + type = serializers.ChoiceField(choices=["prediction"]) + details = PredictionDetailsSerializer() + + +class AlgorithmResultEntrySerializer(HistoryEntryBaseSerializer): + """What a post-processing run decided about the occurrence, with the classifications it created.""" + + type = serializers.ChoiceField(choices=["algorithm_result"]) + kind = serializers.CharField() + data = serializers.JSONField(help_text="The kind's figures, validated against its data model.") + data_references = serializers.DictField( + child=RefSerializer(), help_text="Data fields that name another record, by field." ) - source_algorithm = HistoryAlgorithmSerializer( - allow_null=True, help_text="For class masking: the classifier re-scored, from the job's settings." + determination_before = HistoryTaxonSerializer(allow_null=True) + determination_after = HistoryTaxonSerializer(allow_null=True) + classifications = CreatedClassificationSerializer( + many=True, help_text="The classifications the run created, best score first." ) - comment = serializers.CharField(allow_blank=True) - withdrawn = serializers.BooleanField() is_current = serializers.BooleanField( - allow_null=True, help_text="For a result: whether it is the latest of its kind, not replaced by a later run." + help_text="Whether it is the latest of its kind, not replaced by a later run." ) + + +def _result_entry_component(kind: str) -> type[serializers.Serializer]: + """An OpenAPI-only serializer for one kind: ``kind`` is a literal and ``data`` is the kind's schema.""" + name = result_kind_title(kind) + data_field = extend_schema_field(data_json_schema(kind))(type(f"{name}DataField", (serializers.JSONField,), {})) + return type( + f"{name}ResultEntrySerializer", + (AlgorithmResultEntrySerializer,), + {"kind": serializers.ChoiceField(choices=[kind]), "data": data_field()}, + ) + + +# The history endpoint's response: a plain oneOf with literal ``type`` and ``kind`` fields, one +# component per result kind, so a client generated from the schema can narrow ``data`` by kind. +OCCURRENCE_HISTORY_ENTRY_SCHEMA = PolymorphicProxySerializer( + component_name="OccurrenceHistoryEntry", + serializers=[IdentificationEntrySerializer, PredictionEntrySerializer] + + [_result_entry_component(kind) for kind in result_kinds()], + resource_type_field_name=None, + many=True, +) + +HISTORY_ENTRY_SERIALIZERS = { + "identification": IdentificationEntrySerializer, + "prediction": PredictionEntrySerializer, + "algorithm_result": AlgorithmResultEntrySerializer, +} + + +def serialize_history(entries, context) -> list[dict]: + return [HISTORY_ENTRY_SERIALIZERS[entry.type](entry, context=context).data for entry in entries] diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 7a5f080f7..8e5ad5bb7 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -67,6 +67,7 @@ verified_taxon_counts, ) from .serializers import ( + OCCURRENCE_HISTORY_ENTRY_SCHEMA, BulkIdentificationRequestSerializer, BulkIdentificationResponseSerializer, ClassificationListSerializer, @@ -83,7 +84,6 @@ ExampleOccurrenceSerializer, IdentificationSerializer, ModelAgreementSerializer, - OccurrenceHistoryEntrySerializer, OccurrenceListSerializer, OccurrenceSerializer, PageListSerializer, @@ -105,6 +105,7 @@ TaxonSearchResultSerializer, TaxonSerializer, TopIdentifiersResponseSerializer, + serialize_history, ) logger = logging.getLogger(__name__) @@ -1597,7 +1598,7 @@ def get_queryset(self) -> QuerySet["Occurrence"]: def list(self, request, *args, **kwargs): return super().list(request, *args, **kwargs) - @extend_schema(parameters=[project_id_doc_param], responses=OccurrenceHistoryEntrySerializer(many=True)) + @extend_schema(parameters=[project_id_doc_param], responses=OCCURRENCE_HISTORY_ENTRY_SCHEMA) @action(detail=True, methods=["get"], name="history", pagination_class=None) def history(self, request: Request, pk=None) -> Response: """Everything that happened to this occurrence, newest first. @@ -1607,7 +1608,7 @@ def history(self, request: Request, pk=None) -> Response: """ occurrence = self.get_object() entries = occurrence_timeline(occurrence) - return Response(OccurrenceHistoryEntrySerializer(entries, many=True, context={"request": request}).data) + return Response(serialize_history(entries, {"request": request})) @extend_schema(parameters=[project_id_doc_param], responses=AlgorithmSerializer(many=True)) @action(detail=False, methods=["get"], name="algorithms") diff --git a/ami/main/test_occurrence_history.py b/ami/main/test_occurrence_history.py index 3270febda..1f6f58c55 100644 --- a/ami/main/test_occurrence_history.py +++ b/ami/main/test_occurrence_history.py @@ -18,11 +18,10 @@ from ami.users.models import User from ami.users.roles import BasicMember, ProjectManager -# Measured: two savepoints, the project, the occurrence, then the results with their algorithm and -# job, their taxa, the classifications they created, identifications and predictions. A class -# masking job's settings add one query each for the species lists and source algorithms, whatever -# the number of entries. -HISTORY_QUERIES = 9 +# Measured: two savepoints, the project, the occurrence, the results with their algorithm and job, +# their taxa, the classifications they created, the classifications those replaced, identifications, +# predictions with their jobs, and one query per type of record the entries name (here species lists). +HISTORY_QUERIES = 11 SIZE_FILTER = AlgorithmResult.Kind.SIZE_FILTER CLASS_MASKING = AlgorithmResult.Kind.CLASS_MASKING @@ -179,6 +178,36 @@ def test_job_settings_yield_only_known_integer_references(self): self.assertEqual(job_setting_references(["not", "a", "dict"]), []) # type: ignore[arg-type] +class OccurrenceHistorySchemaTestCase(TestCase): + """The OpenAPI schema publishes one history-entry component per result kind, typed by its data model.""" + + def test_every_result_kind_has_a_typed_component(self): + from drf_spectacular.generators import SchemaGenerator + + from ami.ml.results.schemas import result_kinds + + components = SchemaGenerator(api_version="api").get_schema(request=None, public=True)["components"]["schemas"] + + def resolve(schema: dict) -> dict: + # drf-spectacular moves every enum into its own named component. + return components[schema["$ref"].rsplit("/", 1)[-1]] if "$ref" in schema else schema + + entry_names = [ref["$ref"].rsplit("/", 1)[-1] for ref in components["OccurrenceHistoryEntry"]["oneOf"]] + for kind in result_kinds(): + name = "".join(part.title() for part in kind.split("_")) + "ResultEntry" + with self.subTest(kind): + self.assertIn(name, entry_names) + props = components[name]["properties"] + self.assertEqual(resolve(props["kind"])["enum"], [kind]) + self.assertEqual(resolve(props["type"])["enum"], ["algorithm_result"]) + self.assertIn("extra", props["data"]["properties"]) + self.assertIn("IdentificationEntry", entry_names) + self.assertIn("PredictionEntry", entry_names) + self.assertIn( + "excluded_probability", components["ClassMaskingResultEntry"]["properties"]["data"]["properties"] + ) + + class OccurrenceFixtureTestCase(APITestCase): """One occurrence of four classified detections, with a project manager and a basic member.""" @@ -282,16 +311,20 @@ def test_entries_are_merged_newest_first_and_a_result_carries_the_classification self.assertEqual(set(identification["user"]), {"id", "name", "image"}) self.assertEqual(identification["user"]["id"], self.reader.pk) self.assertEqual(identification["taxon"]["id"], self.other_taxon.pk) - self.assertEqual(identification["payload"]["comment"], "round 0") - self.assertEqual(result["subtype"], "size_filter") + self.assertEqual(identification["details"]["comment"], "round 0") + self.assertEqual(result["kind"], "size_filter") self.assertTrue(result["is_current"]) self.assertEqual(result["algorithm"]["key"], self.size_filter.key) - self.assertEqual(result["job"], {"id": self.job.pk, "name": self.job.name, "config": {"size_threshold": 0.01}}) - self.assertEqual(result["taxon"]["id"], self.taxon.pk) - self.assertEqual(result["taxon_before"]["id"], self.other_taxon.pk) + self.assertEqual( + result["job"], + {"id": self.job.pk, "name": self.job.name, "config": {"size_threshold": 0.01}, "references": {}}, + ) + self.assertIsNone(result["taxon"]) + self.assertEqual(result["determination_after"]["id"], self.taxon.pk) + self.assertEqual(result["determination_before"]["id"], self.other_taxon.pk) self.assertEqual(result["score"], 0.001) - self.assertEqual(result["payload"]["relative_size"], 0.001) - self.assertIsNone(result["original_taxon"]) + self.assertEqual(result["data"]["relative_size"], 0.001) + self.assertEqual(result["data_references"], {}) created = Classification.objects.get(algorithm=self.size_filter) self.assertEqual( result["classifications"], @@ -302,7 +335,7 @@ def test_entries_are_merged_newest_first_and_a_result_carries_the_classification "score": 0.1, "terminal": True, "detection_id": self.detections[0].pk, - "applied_to_id": None, + "replaced": None, } ], ) @@ -367,14 +400,22 @@ def test_a_demoted_prediction_names_the_result_that_superseded_it(self): } self.assertEqual(set(predictions), {None, classifier.key, bystander_algorithm.key}) self.assertEqual(predictions[bystander_algorithm.key]["id"], bystander.pk) - self.assertIsNone(predictions[bystander_algorithm.key]["payload"]["superseded_by_result_id"]) + self.assertIsNone(predictions[bystander_algorithm.key]["details"]["superseded_by_result_id"]) demoted = predictions[classifier.key] self.assertEqual(demoted["id"], original.pk) - self.assertEqual(demoted["payload"]["superseded_by_result_id"], result.pk) - self.assertFalse(demoted["payload"]["terminal"]) - self.assertIsNone(predictions[None]["payload"]["superseded_by_result_id"]) - self.assertEqual(result_entry["taxa_list"], {"id": taxa_list.pk, "name": "Kept species"}) - self.assertEqual(result_entry["source_algorithm"]["key"], classifier.key) + self.assertEqual(demoted["details"]["superseded_by_result_id"], result.pk) + self.assertFalse(demoted["details"]["terminal"]) + self.assertIsNone(predictions[None]["details"]["superseded_by_result_id"]) + self.assertEqual( + result_entry["job"]["references"], + { + "taxa_list_id": {"type": "taxa_list", "id": taxa_list.pk, "name": "Kept species"}, + "algorithm_id": {"type": "algorithm", "id": classifier.pk, "name": classifier.name}, + }, + ) + # The top prediction before masking is the classification the masked one replaced. + self.assertEqual(result_entry["classifications"][0]["replaced"]["id"], original.pk) + self.assertEqual(result_entry["classifications"][0]["replaced"]["taxon"]["id"], self.taxon.pk) self.assertEqual(result_entry["score"], 0.4) def test_a_terminal_prediction_outranked_on_its_detection_is_superseded(self): @@ -400,7 +441,7 @@ def test_a_terminal_prediction_outranked_on_its_detection_is_superseded(self): prediction = next(entry for entry in data if entry["type"] == "prediction") self.assertEqual(prediction["id"], outranked.pk) - self.assertEqual(prediction["payload"]["superseded_by_result_id"], result.pk) + self.assertEqual(prediction["details"]["superseded_by_result_id"], result.pk) def test_each_algorithm_shows_one_prediction_preferring_terminal_then_latest(self): """Tied top scores would otherwise show the same prediction once per detection.""" @@ -431,9 +472,15 @@ def classify(detection, algorithm, terminal, created_at): def test_query_count_does_not_grow_with_the_entries_and_no_score_arrays_are_loaded(self): self.client.force_authenticate(user=self.reader) + # A setting that names a record, and created classifications that replaced one, so the + # reference and replaced-classification queries are counted too. + self.job.params = {"config": {"size_threshold": 0.01, "taxa_list_id": TaxaList.objects.create(name="L").pk}} + self.job.save() + original = Classification.objects.get(detection=self.detections[0]) now = datetime.datetime.now() for rounds, total in ((1, 1), (2, 3)): self._add_history(now - datetime.timedelta(days=3 * total), rounds=rounds) + Classification.objects.filter(algorithm=self.size_filter).update(applied_to=original) with self.subTest(rounds=total), no_query_cache(): with CaptureQueriesContext(connection) as queries, self.assertNumQueries(HISTORY_QUERIES): response = self.client.get(self.url()) diff --git a/ami/ml/results/schemas.py b/ami/ml/results/schemas.py index 9f0eb04bd..270d562f5 100644 --- a/ami/ml/results/schemas.py +++ b/ami/ml/results/schemas.py @@ -61,6 +61,11 @@ def result_kinds() -> list[str]: return list(ALGORITHM_RESULT_DATA_SCHEMAS) +def result_kind_title(kind: str) -> str: + """The kind in CamelCase, for OpenAPI component names: ``class_masking`` -> ``ClassMasking``.""" + return "".join(part.title() for part in kind.split("_")) + + def _schema_for(kind: str) -> type[AlgorithmResultData]: schema = ALGORITHM_RESULT_DATA_SCHEMAS.get(kind) if schema is None: diff --git a/config/settings/base.py b/config/settings/base.py index 48580fed6..b4570c5b9 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -9,6 +9,9 @@ import django_stubs_ext import environ +# A leaf module (pydantic only, no Django imports), so settings can read the result kinds. +from ami.ml.results.schemas import result_kind_title, result_kinds + # Monkeypatching Django, so stubs will work for all generics, # see: https://github.com/typeddjango/django-stubs django_stubs_ext.monkeypatch() @@ -554,6 +557,14 @@ def _celery_result_backend_url(redis_url): "VERSION": "1.0.0", # "SERVE_PERMISSIONS": ["rest_framework.permissions.IsAdminUser"], "SERVE_PERMISSIONS": ["rest_framework.permissions.AllowAny"], + # Names for the single-value enums that tag each occurrence history entry (see + # OCCURRENCE_HISTORY_ENTRY_SCHEMA); without them drf-spectacular falls back to hashed names. + "ENUM_NAME_OVERRIDES": { + "AlgorithmResultEntryTypeEnum": ["algorithm_result"], + "IdentificationEntryTypeEnum": ["identification"], + "PredictionEntryTypeEnum": ["prediction"], + **{f"{result_kind_title(kind)}ResultKindEnum": [kind] for kind in result_kinds()}, + }, } # Your stuff... # ------------------------------------------------------------------------------ From 0710826c7b2b8decb6a6f8c2ce2ee218d8f01f8e Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 5 Oct 2026 16:49:25 -0700 Subject: [PATCH 024/134] feat(ui): link every record the history names through one table, and read the sectioned history A reference ({type, id, name}) becomes a link through linkFor, one table keyed by type; a deleted record (name null) or a type with no page shows as plain text. The result card uses it for every job setting that names a record (species list, classifier, capture set, occurrence) and for the job, prediction cards gain a job row, and the class masking card shows the original top prediction from the classification the run replaced. The history types mirror the OccurrenceHistoryEntry components in the OpenAPI schema: results carry kind, data, data references and determination before and after; identifications and predictions carry details. A result of a kind the UI has no card for is skipped. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01APWsYj7AAYL5aiq7T3HUwq --- .../models/occurrence-history.test.ts | 87 +++++++---- .../models/occurrence-history.ts | 139 ++++++++++-------- .../identification-card/algorithm-result.tsx | 114 ++++++-------- .../identification-card/history-stats.tsx | 25 ++++ .../machine-prediction.tsx | 17 +++ .../occurrence-timeline.tsx | 1 + ui/src/utils/language.ts | 6 +- ui/src/utils/references.test.ts | 39 +++++ ui/src/utils/references.ts | 31 ++++ 9 files changed, 294 insertions(+), 165 deletions(-) create mode 100644 ui/src/utils/references.test.ts create mode 100644 ui/src/utils/references.ts diff --git a/ui/src/data-services/models/occurrence-history.test.ts b/ui/src/data-services/models/occurrence-history.test.ts index 3969951b8..de008a69c 100644 --- a/ui/src/data-services/models/occurrence-history.test.ts +++ b/ui/src/data-services/models/occurrence-history.test.ts @@ -17,27 +17,24 @@ const XESTIA = { id: 4, name: 'Xestia c-nigrum', rank: 'SPECIES' } const base = { algorithm: null, - classifications: [], - comment: '', - is_current: null, job: null, - original_taxon: null, score: null, - source_algorithm: null, - subtype: null, - taxa_list: null, taxon: null, - taxon_before: null, timestamp: '2026-04-29T22:00:00', user: null, - withdrawn: false, +} + +const JOB = { + config: { taxa_list_id: 2 }, + id: 30, + name: 'Masking run', + references: { taxa_list_id: { type: 'taxa_list', id: 2, name: 'Kept' } }, } const identificationEntry = (id: number): ServerOccurrenceHistoryEntry => ({ ...base, - comment: 'Looks right', id, - payload: { + details: { agreed_with_identification_id: null, agreed_with_prediction_id: null, comment: 'Looks right', @@ -51,12 +48,14 @@ const identificationEntry = (id: number): ServerOccurrenceHistoryEntry => ({ const predictionEntry = ( id: number, taxon: typeof NOCTUA | null = XESTIA, - supersededByResultId: number | null = null + supersededByResultId: number | null = null, + job: typeof JOB | null = null ): ServerOccurrenceHistoryEntry => ({ ...base, algorithm: { id: 7, key: 'classifier', name: 'Classifier' }, id, - payload: { + job, + details: { applied_to_id: null, detection_id: 1, superseded_by_result_id: supersededByResultId, @@ -72,38 +71,37 @@ const classMasking: ServerOccurrenceHistoryEntry = { algorithm: { id: 12, key: 'mask', name: 'Masked classifier' }, classifications: [ { - applied_to_id: 6, detection_id: 1, id: 20, + replaced: { id: 6, score: 0.83, taxon: XESTIA }, score: 0.7, taxon: NOCTUA, terminal: true, }, { - applied_to_id: 7, detection_id: 2, id: 21, + replaced: null, score: 0.4, taxon: XESTIA, terminal: true, }, ], - id: 5, - is_current: true, - original_taxon: XESTIA, - payload: { + data: { determination_after_id: 3, determination_before_id: 4, excluded_probability: 0.38, extra: {}, new_winner_original_rank: 2, - original_score: 0.83, - original_taxon_id: 4, }, + data_references: {}, + determination_after: NOCTUA, + determination_before: XESTIA, + id: 5, + is_current: true, + job: JOB, + kind: 'class_masking', score: 0.38, - subtype: 'class_masking', - taxon: NOCTUA, - taxon_before: XESTIA, type: 'algorithm_result', } @@ -191,10 +189,24 @@ describe('getTimelineItems', () => { expect(items[2]).not.toHaveProperty('supersededBy', expect.anything()) }) - test('drops unknown subtypes and predictions without a taxon', () => { + test('carries the job that wrote a prediction as a reference', () => { + const items = getTimelineItems({ + entries: [predictionEntry(6, XESTIA, null, JOB), predictionEntry(8)], + identifications: [], + predictions: [], + }) + + expect(items[0]).toMatchObject({ + job: { type: 'job', id: 30, name: 'Masking run' }, + }) + expect(items[1]).not.toHaveProperty('job', expect.anything()) + }) + + test('drops results of a kind it has no card for, and predictions without a taxon', () => { + // A kind the server added before the UI has a card for it, e.g. tracking. const unknown = { ...classMasking, - subtype: 'something_new', + kind: 'tracking', } as unknown as ServerOccurrenceHistoryEntry expect( @@ -318,7 +330,8 @@ describe('getOccurrenceHistoryQueryKey', () => { }) describe('getJobSettings', () => { - test('lists the settings a job ran with, leaving out unset ones and those shown elsewhere', () => { + test('lists the settings a job ran with and the records they name, leaving out unset ones', () => { + const deletedList = { type: 'taxa_list', id: 2, name: null } expect( getJobSettings({ config: { @@ -330,17 +343,27 @@ describe('getJobSettings', () => { }, id: 1, name: 'Size filter', + references: { + occurrence_id: { type: 'occurrence', id: 4, name: '#4' }, + taxa_list_id: deletedList, + }, }) ).toEqual([ - { key: 'occurrence_id', value: 4 }, - { key: 'reweight', value: true }, + { + key: 'occurrence_id', + value: 4, + ref: { type: 'occurrence', id: 4, name: '#4' }, + }, + { key: 'reweight', value: true, ref: undefined }, + // A deleted list keeps its reference with no name, so the card shows its id as text. + { key: 'taxa_list_id', value: 2, ref: deletedList }, ]) }) test('is empty for a job without settings', () => { - expect(getJobSettings({ config: null, id: 1, name: 'Pipeline' })).toEqual( - [] - ) + expect( + getJobSettings({ config: null, id: 1, name: 'Pipeline', references: {} }) + ).toEqual([]) expect(getJobSettings(null)).toEqual([]) }) }) diff --git a/ui/src/data-services/models/occurrence-history.ts b/ui/src/data-services/models/occurrence-history.ts index 598ac7432..bcf378d96 100644 --- a/ui/src/data-services/models/occurrence-history.ts +++ b/ui/src/data-services/models/occurrence-history.ts @@ -1,8 +1,14 @@ +import { Ref } from 'utils/references' import { getUserLabel } from 'utils/user/getUserLabel' import { Algorithm } from './algorithm' import { HumanIdentification, MachinePrediction } from './occurrence-details' import { Taxon } from './taxa' +/* + * The server types below mirror the OccurrenceHistoryEntry components in the OpenAPI schema; + * replace them with generated types when the UI adopts OpenAPI generation. + */ + export interface ServerHistoryTaxon { id: number name: string @@ -21,117 +27,116 @@ export interface ServerHistoryAlgorithm { name: string } -export interface ServerHistoryTaxaList { - id: number - name: string -} - export interface ServerHistoryJob { /** The settings a post-processing job ran with; null for other jobs. */ config: Record | null id: number name: string + /** Settings that name another record, by setting key. */ + references: Record +} + +/** The classification a run's classification replaced, when there was one and it still exists. */ +export interface ServerReplacedClassification { + id: number + score: number | null + taxon: ServerHistoryTaxon | null } /** A classification a post-processing run created, listed inside the run's result. */ -export interface ServerHistoryClassification { - applied_to_id: number | null +export interface ServerCreatedClassification { detection_id: number id: number + replaced: ServerReplacedClassification | null score: number | null /** Null when the taxon was deleted. */ taxon: ServerHistoryTaxon | null terminal: boolean } -interface ServerDeterminationChangePayload { +interface ServerDeterminationSnapshot { determination_after_id: number | null determination_before_id: number | null /** Whatever else the method returned; stored as JSON and not interpreted. */ extra: Record } -export interface ServerClassMaskingPayload - extends ServerDeterminationChangePayload { +export interface ClassMaskingResultData extends ServerDeterminationSnapshot { /** The share of the source classifier's probability outside the species list. */ excluded_probability: number /** Where the class that wins after masking ranked before it; 1 means it was already the top. */ new_winner_original_rank: number | null - original_score: number | null - original_taxon_id: number | null } -export interface ServerSizeFilterPayload - extends ServerDeterminationChangePayload { +export interface SizeFilterResultData extends ServerDeterminationSnapshot { /** The filtered detection's box area as a fraction of its image. */ relative_size: number } -export interface ServerIdentificationPayload { +export interface ServerIdentificationDetails { agreed_with_identification_id: number | null agreed_with_prediction_id: number | null comment: string withdrawn: boolean } -export interface ServerPredictionPayload { +export interface ServerPredictionDetails { applied_to_id: number | null - detection_id: number | null + detection_id: number /** The result of the run that re-scored and demoted this prediction, when one did. */ superseded_by_result_id: number | null - terminal: boolean | null + terminal: boolean } -interface ServerHistoryEntryBase { +/** The fields every entry has. */ +interface ServerHistoryEntryBase { algorithm: ServerHistoryAlgorithm | null - /** For a result: the classifications its run created, best score first. */ - classifications: ServerHistoryClassification[] - comment: string id: number - /** For a result: whether it is the latest of its kind, not replaced by a later run. */ - is_current: boolean | null job: ServerHistoryJob | null - /** For a class masking result: the source classifier's top taxon before masking. */ - original_taxon: ServerHistoryTaxon | null - payload: Payload /** A prediction's score, or a result's headline value. */ score: number | null - /** For a class masking result: the classifier whose predictions the run re-scored. */ - source_algorithm: ServerHistoryAlgorithm | null - subtype: Subtype - /** For a class masking result: the species list the run kept. */ - taxa_list: ServerHistoryTaxaList | null + /** The identified or predicted taxon; null for a result. */ taxon: ServerHistoryTaxon | null - taxon_before: ServerHistoryTaxon | null timestamp: string - type: Type user: ServerHistoryUser | null - withdrawn: boolean } -export type ClassMaskingResultEntry = ServerHistoryEntryBase< - 'algorithm_result', +interface ServerResultEntry + extends ServerHistoryEntryBase { + /** The classifications the run created, best score first. */ + classifications: ServerCreatedClassification[] + data: Data + /** Data fields that name another record, by field. */ + data_references: Record + determination_after: ServerHistoryTaxon | null + determination_before: ServerHistoryTaxon | null + /** Whether it is the latest of its kind, not replaced by a later run. */ + is_current: boolean + kind: Kind + type: 'algorithm_result' +} + +export type ClassMaskingResultEntry = ServerResultEntry< 'class_masking', - ServerClassMaskingPayload + ClassMaskingResultData > -export type SizeFilterResultEntry = ServerHistoryEntryBase< - 'algorithm_result', +export type SizeFilterResultEntry = ServerResultEntry< 'size_filter', - ServerSizeFilterPayload + SizeFilterResultData > export type AlgorithmResultEntry = | ClassMaskingResultEntry | SizeFilterResultEntry -export type IdentificationEntry = ServerHistoryEntryBase< - 'identification', - null, - ServerIdentificationPayload -> -export type PredictionEntry = ServerHistoryEntryBase< - 'prediction', - null, - ServerPredictionPayload -> + +export interface IdentificationEntry extends ServerHistoryEntryBase { + details: ServerIdentificationDetails + type: 'identification' +} + +export interface PredictionEntry extends ServerHistoryEntryBase { + details: ServerPredictionDetails + type: 'prediction' +} export type ServerOccurrenceHistoryEntry = | AlgorithmResultEntry @@ -148,12 +153,15 @@ export type TimelineItem = type: 'prediction' id: string prediction: MachinePrediction + /** The job that wrote the prediction, when the history names one. */ + job?: Ref /** The result of the run that demoted this prediction, when one did. */ supersededBy?: AlgorithmResultEntry } | { type: 'algorithm_result'; id: string; entry: AlgorithmResultEntry } -const ALGORITHM_RESULT_SUBTYPES: string[] = ['class_masking', 'size_filter'] +/** The result kinds this UI has a card for; results of any other kind are skipped. */ +const ALGORITHM_RESULT_KINDS: string[] = ['class_masking', 'size_filter'] export const convertHistoryTaxon = (taxon: ServerHistoryTaxon) => new Taxon({ ...taxon, id: `${taxon.id}`, cover_image_url: null }) @@ -166,10 +174,10 @@ const toIdentification = ( return { applied: taxon.id === determinationTaxonId, - comment: entry.comment, + comment: entry.details.comment, createdAt: entry.timestamp, id: `${entry.id}`, - overridden: entry.payload.withdrawn, + overridden: entry.details.withdrawn, taxon, user: entry.user ? { @@ -199,7 +207,7 @@ const toPrediction = ( overridden: taxon.id !== determinationTaxonId, score: entry.score ?? 0, taxon, - terminal: !!entry.payload.terminal, + terminal: entry.details.terminal, userPermissions: [], } } @@ -208,7 +216,7 @@ const isAlgorithmResult = ( entry: ServerOccurrenceHistoryEntry ): entry is AlgorithmResultEntry => entry.type === 'algorithm_result' && - ALGORITHM_RESULT_SUBTYPES.includes(entry.subtype) + ALGORITHM_RESULT_KINDS.includes(entry.kind) /** * The history as cards to render, newest first. Identifications and predictions reuse the @@ -251,11 +259,14 @@ export const getTimelineItems = ({ ? toPrediction({ ...entry, algorithm, taxon }, determinationTaxonId) : undefined) const supersededBy = results.find( - (result) => result.id === entry.payload.superseded_by_result_id + (result) => result.id === entry.details.superseded_by_result_id ) + const job: Ref | undefined = entry.job + ? { type: 'job', id: entry.job.id, name: entry.job.name } + : undefined return prediction - ? [{ type: 'prediction', id, prediction, supersededBy }] + ? [{ type: 'prediction', id, prediction, job, supersededBy }] : [] } case 'algorithm_result': @@ -388,16 +399,14 @@ export const getResultPrediction = ( export interface JobSetting { key: string value: unknown + /** The record the setting names, when it names one. */ + ref?: Ref } /** Settings the result card already shows in a row of their own. */ -const SETTINGS_SHOWN_ELSEWHERE = [ - 'algorithm_id', - 'size_threshold', - 'taxa_list_id', -] +const SETTINGS_SHOWN_ELSEWHERE = ['size_threshold'] -/** A job's remaining settings, leaving out unset ones and those the card shows elsewhere. */ +/** A job's settings with the records they name, leaving out unset ones and those shown elsewhere. */ export const getJobSettings = (job: ServerHistoryJob | null): JobSetting[] => Object.entries(job?.config ?? {}) .filter( @@ -406,4 +415,4 @@ export const getJobSettings = (job: ServerHistoryJob | null): JobSetting[] => value !== undefined && !SETTINGS_SHOWN_ELSEWHERE.includes(key) ) - .map(([key, value]) => ({ key, value })) + .map(([key, value]) => ({ key, value, ref: job?.references[key] })) diff --git a/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx b/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx index a98f39a98..221682916 100644 --- a/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx +++ b/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx @@ -18,6 +18,7 @@ import { APP_ROUTES } from 'utils/constants' import { getFormatedDateTimeString } from 'utils/date/getFormatedDateTimeString/getFormatedDateTimeString' import { getAppRoute } from 'utils/getAppRoute' import { STRING, translate } from 'utils/language' +import { Ref } from 'utils/references' import { UserInfo, UserPermission } from 'utils/user/types' import { Agree } from '../agree/agree' import { @@ -25,53 +26,57 @@ import { HistoryStats, HistoryTime, HistoryTypeBadge, + RefValue, } from './history-stats' -const SUBTYPES = { +const KINDS = { class_masking: { icon: FilterIcon, label: STRING.HISTORY_CLASS_MASKING }, size_filter: { icon: RulerIcon, label: STRING.HISTORY_SIZE_FILTER }, } /** Labels for the job settings shown as rows; a setting not listed here shows its raw key. */ const SETTING_LABELS: Partial> = { + algorithm_id: STRING.HISTORY_SETTING_CLASSIFIER, occurrence_id: STRING.HISTORY_SETTING_OCCURRENCE, reweight: STRING.HISTORY_SETTING_REWEIGHT, source_image_collection_id: STRING.HISTORY_SETTING_CAPTURE_SET, + taxa_list_id: STRING.HISTORY_SPECIES_LIST, } /** What to call a result's kind, e.g. "Class masking", for the card and for predictions it superseded. */ export const getResultKindLabel = (entry: AlgorithmResultEntry) => - translate(SUBTYPES[entry.subtype].label) + translate(KINDS[entry.kind].label) -const formatSettingValue = (key: string, value: unknown) => { +const formatSettingValue = (value: unknown) => { if (typeof value === 'boolean') { return translate(value ? STRING.YES : STRING.NO) } - if (key.endsWith('_id')) { - return `#${value}` - } return typeof value === 'object' ? JSON.stringify(value) : `${value}` } -/** The list's id when only the job's settings name it, e.g. after the list was deleted. */ -const getSpeciesListFallback = (entry: AlgorithmResultEntry) => { - const id = entry.job?.config?.taxa_list_id +/** A reference's name, or its id when the record was deleted. */ +const getRefLabel = (reference?: Ref) => { + if (!reference) { + return translate(STRING.VALUE_NOT_AVAILABLE) + } - return id !== undefined && id !== null - ? translate(STRING.HISTORY_SPECIES_LIST_ID, { id: `${id}` }) - : translate(STRING.VALUE_NOT_AVAILABLE) + return ( + reference.name ?? + translate(STRING.HISTORY_RECORD_ID, { id: `${reference.id}` }) + ) } /** The card's subtitle: for class masking the classifier and species list, otherwise the algorithm's name. */ const getSubTitle = (entry: AlgorithmResultEntry) => { - if (entry.subtype === 'class_masking') { + if (entry.kind === 'class_masking') { + const references = entry.job?.references ?? {} + return translate(STRING.HISTORY_MASKING_SUBTITLE, { - algorithm: - entry.source_algorithm?.name ?? - entry.algorithm?.name ?? - translate(STRING.VALUE_NOT_AVAILABLE), - list: entry.taxa_list?.name ?? getSpeciesListFallback(entry), + algorithm: references.algorithm_id + ? getRefLabel(references.algorithm_id) + : entry.algorithm?.name ?? translate(STRING.VALUE_NOT_AVAILABLE), + list: getRefLabel(references.taxa_list_id), }) } @@ -115,7 +120,7 @@ export const AlgorithmResult = ({ occurrence: Occurrence }) => { const { projectId } = useParams() - const { icon: Icon } = SUBTYPES[entry.subtype] + const { icon: Icon } = KINDS[entry.kind] const prediction = getResultPrediction( entry, occurrence.determinationTaxon?.id @@ -123,48 +128,26 @@ export const AlgorithmResult = ({ const showAgree = occurrence.userPermissions.includes(UserPermission.Update) const stats: HistoryStat[] = getDeterminationStats( - entry.taxon_before, - entry.taxon + entry.determination_before, + entry.determination_after ) - switch (entry.subtype) { + switch (entry.kind) { case 'class_masking': { - // Link only a list that still exists; a deleted one shows its id as plain text. - if (entry.taxa_list) { - stats.push({ - label: translate(STRING.HISTORY_SPECIES_LIST), - value: ( - - {entry.taxa_list.name} - - ), - }) - } else if (entry.job?.config?.taxa_list_id != null) { - stats.push({ - label: translate(STRING.HISTORY_SPECIES_LIST), - value: translate(STRING.HISTORY_SPECIES_LIST_ID, { - id: `${entry.job.config.taxa_list_id}`, - }), - }) - } stats.push({ label: translate(STRING.HISTORY_EXCLUDED_PROBABILITY), - value: formatPercent(entry.payload.excluded_probability), + value: formatPercent(entry.data.excluded_probability), }) - if (entry.original_taxon) { + // The top prediction before masking: what the run's best classification replaced. + const replaced = entry.classifications.find( + (c) => c.taxon !== null + )?.replaced + if (replaced?.taxon) { stats.push({ label: translate(STRING.HISTORY_ORIGINAL_PREDICTION), value: - entry.payload.original_score !== null - ? `${ - entry.original_taxon.name - } (${entry.payload.original_score.toFixed(2)})` - : entry.original_taxon.name, + replaced.score !== null + ? `${replaced.taxon.name} (${replaced.score.toFixed(2)})` + : replaced.taxon.name, }) } break @@ -176,11 +159,11 @@ export const AlgorithmResult = ({ value: typeof threshold === 'number' ? translate(STRING.HISTORY_DETECTION_SIZE_WITH_THRESHOLD, { - size: formatPercent(entry.payload.relative_size), + size: formatPercent(entry.data.relative_size), threshold: formatPercent(threshold), }) : translate(STRING.HISTORY_DETECTION_SIZE_VALUE, { - size: formatPercent(entry.payload.relative_size), + size: formatPercent(entry.data.relative_size), }), }) break @@ -190,26 +173,25 @@ export const AlgorithmResult = ({ label: translate(STRING.HISTORY_DETECTIONS_AFFECTED), value: new Set(entry.classifications.map((c) => c.detection_id)).size, }) - getJobSettings(entry.job).forEach(({ key, value }) => { + getJobSettings(entry.job).forEach(({ key, value, ref }) => { const label = SETTING_LABELS[key] stats.push({ label: label !== undefined ? translate(label) : key, - value: formatSettingValue(key, value), + value: ref ? ( + + ) : ( + formatSettingValue(value) + ), }) }) if (entry.job) { stats.push({ label: translate(STRING.FIELD_LABEL_JOB), value: ( - - {entry.job.name} - + ), }) } diff --git a/ui/src/pages/occurrence-details/identification-card/history-stats.tsx b/ui/src/pages/occurrence-details/identification-card/history-stats.tsx index bdbdb7a82..5c8bc642c 100644 --- a/ui/src/pages/occurrence-details/identification-card/history-stats.tsx +++ b/ui/src/pages/occurrence-details/identification-card/history-stats.tsx @@ -1,4 +1,7 @@ import { Fragment, ReactNode } from 'react' +import { Link } from 'react-router-dom' +import { STRING, translate } from 'utils/language' +import { linkFor, Ref } from 'utils/references' export interface HistoryStat { label: string @@ -17,6 +20,28 @@ export const HistoryStats = ({ stats }: { stats: HistoryStat[] }) =>
) : null +/** A record the history names: a link to its page, or plain text when it was deleted or has no page. */ +export const RefValue = ({ + projectId, + reference, +}: { + projectId: string + reference: Ref +}) => { + const to = linkFor(reference, projectId) + const label = + reference.name ?? + translate(STRING.HISTORY_RECORD_ID, { id: `${reference.id}` }) + + return to ? ( + + {label} + + ) : ( + <>{label} + ) +} + export const HistoryTime = ({ label }: { label: string }) => ( {label} diff --git a/ui/src/pages/occurrence-details/identification-card/machine-prediction.tsx b/ui/src/pages/occurrence-details/identification-card/machine-prediction.tsx index 6c5b6da83..ce1336f2b 100644 --- a/ui/src/pages/occurrence-details/identification-card/machine-prediction.tsx +++ b/ui/src/pages/occurrence-details/identification-card/machine-prediction.tsx @@ -20,18 +20,23 @@ import { APP_ROUTES } from 'utils/constants' import { getFormatedDateTimeString } from 'utils/date/getFormatedDateTimeString/getFormatedDateTimeString' import { getAppRoute } from 'utils/getAppRoute' import { STRING, translate } from 'utils/language' +import { Ref } from 'utils/references' import { UserInfo, UserPermission } from 'utils/user/types' import { Agree } from '../agree/agree' +import { HistoryStats, RefValue } from './history-stats' import machineAvatar from './machine-avatar.svg' export const MachinePrediction = ({ currentUser, identification, + job, occurrence, subTitle, }: { currentUser?: UserInfo identification: Identification + /** The job that wrote the prediction, shown as a row when the history names one. */ + job?: Ref occurrence: Occurrence /** Replaces the terminal/intermediate label, e.g. to say which run superseded the prediction. */ subTitle?: string @@ -107,6 +112,18 @@ export const MachinePrediction = ({ /> )} + {job ? ( + + ), + }, + ]} + /> + ) : null} ({ + APP_ROUTES: { + ALGORITHM_DETAILS: (p: { projectId: string; algorithmId: string }) => + `/projects/${p.projectId}/algorithms/${p.algorithmId}`, + CAPTURES: (p: { projectId: string }) => `/projects/${p.projectId}/captures`, + JOB_DETAILS: (p: { projectId: string; jobId: string }) => + `/projects/${p.projectId}/jobs/${p.jobId}`, + OCCURRENCE_DETAILS: (p: { projectId: string; occurrenceId: string }) => + `/projects/${p.projectId}/occurrences/${p.occurrenceId}`, + TAXA_LIST_DETAILS: (p: { projectId: string; taxaListId: string }) => + `/projects/${p.projectId}/taxa-lists/${p.taxaListId}`, + TAXON_DETAILS: (p: { projectId: string; taxonId: string }) => + `/projects/${p.projectId}/taxa/${p.taxonId}`, + }, +})) + +describe('linkFor', () => { + it('links a known type to its page', () => { + expect(linkFor({ type: 'job', id: 7, name: 'Run' }, '1')).toBe( + '/projects/1/jobs/7' + ) + expect(linkFor({ type: 'taxa_list', id: 3, name: 'Kept' }, '1')).toBe( + '/projects/1/taxa-lists/3' + ) + expect(linkFor({ type: 'capture_set', id: 5, name: 'Set' }, '1')).toBe( + '/projects/1/captures?collections=5' + ) + }) + + it('does not link a deleted row or an unknown type', () => { + expect(linkFor({ type: 'job', id: 7, name: null }, '1')).toBeUndefined() + expect( + linkFor({ type: 'something_new', id: 1, name: 'x' }, '1') + ).toBeUndefined() + }) +}) diff --git a/ui/src/utils/references.ts b/ui/src/utils/references.ts new file mode 100644 index 000000000..f12846849 --- /dev/null +++ b/ui/src/utils/references.ts @@ -0,0 +1,31 @@ +import { APP_ROUTES } from 'utils/constants' +import { getAppRoute } from 'utils/getAppRoute' + +/** Another record the API mentions. `name` is null when the record no longer exists. */ +export interface Ref { + type: string + id: number + name: string | null +} + +/** Where each type of record is shown in the app. A type missing here is shown as text. */ +const ROUTES: Record string> = { + algorithm: (projectId, id) => + APP_ROUTES.ALGORITHM_DETAILS({ projectId, algorithmId: id }), + capture_set: (projectId, id) => + getAppRoute({ + to: APP_ROUTES.CAPTURES({ projectId }), + filters: { collections: id }, + }), + job: (projectId, id) => APP_ROUTES.JOB_DETAILS({ projectId, jobId: id }), + occurrence: (projectId, id) => + APP_ROUTES.OCCURRENCE_DETAILS({ projectId, occurrenceId: id }), + taxa_list: (projectId, id) => + APP_ROUTES.TAXA_LIST_DETAILS({ projectId, taxaListId: id }), + taxon: (projectId, id) => + APP_ROUTES.TAXON_DETAILS({ projectId, taxonId: id }), +} + +/** Where a reference links to, or undefined when the record is gone or has no page. */ +export const linkFor = (ref: Ref, projectId: string) => + ref.name === null ? undefined : ROUTES[ref.type]?.(projectId, `${ref.id}`) From 678170558e38621ff8a6919bec1ea2a61c20ea63 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 5 Oct 2026 17:03:47 -0700 Subject: [PATCH 025/134] fix(history): keep the history open for a result of an unregistered kind, and link results by id in the admin A stored result whose kind is no longer registered (renamed, or written by a branch with an extra kind) made the history endpoint fail while resolving its references; it now names no references and shows like any other result, which the UI skips when it has no card for it. The classification change page renders algorithm_result as an id input: algorithm results have no admin to search, and a select would list every result. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01APWsYj7AAYL5aiq7T3HUwq --- ami/main/admin.py | 2 ++ ami/main/models_future/history.py | 6 ++++-- ami/main/test_occurrence_history.py | 28 +++++++++++++++++++++++++ ami/ml/post_processing/class_masking.py | 3 ++- 4 files changed, 36 insertions(+), 3 deletions(-) diff --git a/ami/main/admin.py b/ami/main/admin.py index 55c84701f..47b383a00 100644 --- a/ami/main/admin.py +++ b/ami/main/admin.py @@ -605,6 +605,8 @@ class ClassificationAdmin(IdSearchAdminMixin, admin.ModelAdmin[Classification]): # every taxon / detection / classification — the latter makes the change page # unusable on a large database. autocomplete_fields = ("detection", "taxon", "algorithm", "category_map", "applied_to") + # Algorithm results have no admin to search, so the link is an id input rather than a select. + raw_id_fields = ("algorithm_result",) # A digit term jumps to that classification by id (IdSearchAdminMixin); text searches taxon name. search_fields = ("taxon__name",) # Order by -id (indexed PK) rather than the model's -created_at (no index). diff --git a/ami/main/models_future/history.py b/ami/main/models_future/history.py index 18e40b2bc..9b4269756 100644 --- a/ami/main/models_future/history.py +++ b/ami/main/models_future/history.py @@ -15,7 +15,7 @@ from ami.main.models import Classification, Identification, Occurrence, Taxon, User from ami.main.models_future.references import Ref, job_setting_references, resolve_references from ami.ml.models import Algorithm, AlgorithmResult -from ami.ml.results.schemas import reference_fields +from ami.ml.results.schemas import reference_fields, result_kinds if typing.TYPE_CHECKING: from ami.jobs.models import Job @@ -165,10 +165,12 @@ def _resolve_entry_references(entries: list[OccurrenceTimelineEntry]) -> None: job_refs = { id(entry): job_setting_references(_job_settings(entry.job)) for entry in entries if entry.job is not None } + # A stored kind may no longer be registered (renamed, or written by another branch); it names no references. + kinds = set(result_kinds()) data_refs = { id(entry): [ (field, ref_type, entry.data[field]) - for field, ref_type in reference_fields(entry.kind).items() + for field, ref_type in (reference_fields(entry.kind) if entry.kind in kinds else {}).items() if isinstance(entry.data.get(field), int) ] for entry in entries diff --git a/ami/main/test_occurrence_history.py b/ami/main/test_occurrence_history.py index 1f6f58c55..3c75eb6b9 100644 --- a/ami/main/test_occurrence_history.py +++ b/ami/main/test_occurrence_history.py @@ -607,3 +607,31 @@ def test_a_result_without_a_job_has_no_settings_or_references(self): entry = next(e for e in occurrence_timeline(self.occurrence) if e.type == "algorithm_result") self.assertIsNone(entry.job) self.assertEqual(entry.job_references, {}) + + def test_a_result_of_a_kind_no_longer_registered_still_shows_in_the_history(self): + """A row written by a branch with an extra kind, or before a kind was renamed, must not break the history.""" + result = AlgorithmResult.objects.record( + occurrence=self.occurrence, algorithm=self.size_filter, kind=SIZE_FILTER, data={"relative_size": 0.001} + ) + AlgorithmResult.objects.filter(pk=result.pk).update(kind="tracking") + entry = next(e for e in self.get() if e["type"] == "algorithm_result") + self.assertEqual(entry["kind"], "tracking") + self.assertEqual(entry["data_references"], {}) + + +class ClassificationAdminTestCase(OccurrenceFixtureTestCase): + """The classification change page must not preload every algorithm result into a select.""" + + def test_the_algorithm_result_field_is_a_raw_id_input(self): + superuser = User.objects.create_superuser(email="admin-history@insectai.org") # type: ignore + algorithm = Algorithm.objects.create(name="Size filter", key="size-filter-admin-test") + result = AlgorithmResult.objects.record( + occurrence=self.occurrence, algorithm=algorithm, kind=SIZE_FILTER, data={"relative_size": 0.001} + ) + classification = Classification.objects.get(detection=self.detections[0]) + Classification.objects.filter(pk=classification.pk).update(algorithm_result=result) + self.client.force_login(superuser) + response = self.client.get(f"/admin/main/classification/{classification.pk}/change/") + self.assertEqual(response.status_code, 200) + self.assertNotContains(response, ' Date: Tue, 6 Oct 2026 08:03:25 -0700 Subject: [PATCH 098/134] fix(results): lock the occurrences before moving their results to the kept occurrence move_to_occurrence read which results on the kept occurrence were current and then moved the absorbed occurrences' results onto it. A run recording a result on the kept occurrence in between left two current results for one algorithm and kind, which the partial unique constraint rejects, failing the merge. The occurrences are now locked in primary-key order first, as record_many does. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01APWsYj7AAYL5aiq7T3HUwq --- ami/main/test_occurrence_history.py | 12 ++++++++++++ ami/ml/models/algorithm_result.py | 9 +++++++++ 2 files changed, 21 insertions(+) diff --git a/ami/main/test_occurrence_history.py b/ami/main/test_occurrence_history.py index 3c75eb6b9..6afc59e9b 100644 --- a/ami/main/test_occurrence_history.py +++ b/ami/main/test_occurrence_history.py @@ -125,6 +125,18 @@ def record(occurrence, algorithm, size): current = {pk for pk, r in results.items() if r.is_current} self.assertEqual(current, {kept_current.pk, latest.pk}) + def test_moving_results_locks_the_occurrences_before_reading_their_results(self): + """Without the lock, a run recording a result on the kept occurrence mid-merge makes two current rows.""" + absorbed = Occurrence.objects.create(project=self.project, deployment=self.deployment) + self._size_filter({"relative_size": 0.02}, occurrence=absorbed).save() + with CaptureQueriesContext(connection) as queries: + AlgorithmResult.objects.move_to_occurrence(self.occurrence, [absorbed.pk]) + sql = [q["sql"] for q in queries.captured_queries] + lock = next((i for i, q in enumerate(sql) if "FOR UPDATE" in q and "main_occurrence" in q), None) + self.assertIsNotNone(lock, "the occurrences are not locked") + first_read = next(i for i, q in enumerate(sql) if "ml_algorithmresult" in q) + self.assertLess(lock, first_read) + def test_the_database_holds_one_current_result_per_occurrence_algorithm_and_kind(self): self._size_filter({"relative_size": 0.01}).save() with transaction.atomic(), self.assertRaises(IntegrityError): diff --git a/ami/ml/models/algorithm_result.py b/ami/ml/models/algorithm_result.py index 1a16d1398..b0532b815 100644 --- a/ami/ml/models/algorithm_result.py +++ b/ami/ml/models/algorithm_result.py @@ -101,6 +101,15 @@ def move_to_occurrence(self, kept: "Occurrence", absorbed_ids: typing.Iterable[i if not absorbed_ids: return 0 with transaction.atomic(): + # Lock in primary-key order, as record_many does, so a run recording a result on these + # occurrences waits for the merge instead of adding a second current result. + list( + apps.get_model("main", "Occurrence") + .objects.select_for_update() + .filter(pk__in={kept.pk, *absorbed_ids}) + .order_by("pk") + .values_list("pk", flat=True) + ) current_keys = set(self.filter(occurrence=kept, is_current=True).values_list("algorithm_id", "kind")) demote: list[int] = [] for pk, algorithm_id, kind in ( From de6656b1a7445cd81115198cd17669e2df9e2ddd Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Tue, 6 Oct 2026 08:03:41 -0700 Subject: [PATCH 099/134] feat(history): label every job setting from its task's schema, and give results a value instead of a score Result cards labelled job settings from a table in the UI, so a task the table did not list, such as tracking, showed its raw setting keys. The history now returns a job's settings as [{key, label, value, ref}], where the label is the title the task's config schema gives the setting, falling back to the key. Class masking and the size filter declare titles for theirs, and the UI table is gone. A list value shows as a comma-separated list. A result entry's headline figure moves from score to value. For some kinds it is not a confidence (tracking's is a motion figure), so score now holds a prediction's score only and is null for results. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01APWsYj7AAYL5aiq7T3HUwq --- ami/main/api/serializers.py | 24 ++++--- ami/main/models_future/history.py | 41 ++++++++---- ami/main/test_occurrence_history.py | 41 +++++++++--- ami/ml/post_processing/class_masking.py | 10 +-- ami/ml/post_processing/registry.py | 9 +++ ami/ml/post_processing/small_size_filter.py | 6 +- .../models/occurrence-history.test.ts | 67 ++++++++++++------- .../models/occurrence-history.ts | 43 ++++++------ .../identification-card/algorithm-result.tsx | 28 ++++---- ui/src/utils/language.ts | 10 --- 10 files changed, 171 insertions(+), 108 deletions(-) diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index b95682288..b44c1616d 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -2130,8 +2130,17 @@ class RefSerializer(serializers.Serializer): name = serializers.CharField(allow_null=True) +class HistoryJobSettingSerializer(serializers.Serializer): + """One setting a job ran with. ``label`` is the title the task's config schema gives it, else the key.""" + + key = serializers.CharField() + label = serializers.CharField() + value = serializers.JSONField(allow_null=True) + ref = RefSerializer(allow_null=True, help_text="The record the setting names, when it names one.") + + class HistoryJobSerializer(serializers.Serializer): - """A job in the history. ``references`` is filled per entry by ``HistoryEntryBaseSerializer``.""" + """A job in the history. ``settings`` is filled per entry by ``HistoryEntryBaseSerializer``.""" id = serializers.IntegerField() name = serializers.CharField() @@ -2141,9 +2150,7 @@ class HistoryJobSerializer(serializers.Serializer): "Null for other jobs, and for a post-processing job whose stored config is not an object." ) ) - references = serializers.DictField( - child=RefSerializer(), read_only=True, help_text="Settings that name another record, by setting key." - ) + settings = HistoryJobSettingSerializer(many=True, read_only=True, help_text="The settings, in the job's order.") @extend_schema_field(serializers.JSONField(allow_null=True)) def get_config(self, job) -> dict | None: @@ -2177,9 +2184,7 @@ class HistoryEntryBaseSerializer(serializers.Serializer): algorithm = HistoryAlgorithmSerializer(allow_null=True) job = serializers.SerializerMethodField() taxon = HistoryTaxonSerializer(allow_null=True, help_text="The identified or predicted taxon.") - score = serializers.FloatField( - allow_null=True, help_text="A prediction's score, or a result's headline value (see its kind)." - ) + score = serializers.FloatField(allow_null=True, help_text="A prediction's score; null for other entries.") @extend_schema_field(HistoryJobSerializer(allow_null=True)) def get_job(self, entry) -> dict | None: @@ -2187,7 +2192,7 @@ def get_job(self, entry) -> dict | None: return None return { **HistoryJobSerializer(entry.job).data, - "references": {key: RefSerializer(ref).data for key, ref in entry.job_references.items()}, + "settings": HistoryJobSettingSerializer(entry.job_settings, many=True).data, } @@ -2224,6 +2229,9 @@ class AlgorithmResultEntrySerializer(HistoryEntryBaseSerializer): type = serializers.ChoiceField(choices=["algorithm_result"]) kind = serializers.CharField() + value = serializers.FloatField( + allow_null=True, help_text="The kind's headline figure, for sorting and filtering; not a confidence." + ) data = serializers.JSONField(help_text="The kind's figures, validated against its data model.") data_references = serializers.DictField( child=RefSerializer(), help_text="Data fields that name another record, by field." diff --git a/ami/main/models_future/history.py b/ami/main/models_future/history.py index 9b4269756..3921db41f 100644 --- a/ami/main/models_future/history.py +++ b/ami/main/models_future/history.py @@ -15,12 +15,23 @@ from ami.main.models import Classification, Identification, Occurrence, Taxon, User from ami.main.models_future.references import Ref, job_setting_references, resolve_references from ami.ml.models import Algorithm, AlgorithmResult +from ami.ml.post_processing.registry import config_setting_labels from ami.ml.results.schemas import reference_fields, result_kinds if typing.TYPE_CHECKING: from ami.jobs.models import Job +@dataclasses.dataclass +class JobSetting: + """One setting a job ran with: its label from the task's config schema, and the record it names, if any.""" + + key: str + label: str + value: typing.Any + ref: Ref | None = None + + @dataclasses.dataclass class CreatedClassification: """A classification a run created, with the one it replaced (through applied_to), if that still exists.""" @@ -40,11 +51,12 @@ class OccurrenceTimelineEntry: user: User | None = None # identification algorithm: Algorithm | None = None # result, prediction job: Job | None = None # result, prediction - job_references: dict[str, Ref] = dataclasses.field(default_factory=dict) # setting key -> Ref + job_settings: list[JobSetting] = dataclasses.field(default_factory=list) # empty without a job taxon: Taxon | None = None # identification, prediction - score: float | None = None # prediction's score; result's value + score: float | None = None # prediction # --- Algorithm result kind: str | None = None + value: float | None = None # the kind's headline figure, not a confidence data: dict = dataclasses.field(default_factory=dict) # validated per kind data_references: dict[str, Ref] = dataclasses.field(default_factory=dict) # data field -> Ref determination_before: Taxon | None = None @@ -106,7 +118,7 @@ def superseded_by(prediction: Classification) -> int | None: timestamp=result.timestamp, algorithm=result.algorithm, job=result.job, - score=result.value, + value=result.value, kind=result.kind, data=result.data, determination_before=taxa.get(result.data.get("determination_before_id")), @@ -155,16 +167,15 @@ def superseded_by(prediction: Classification) -> int | None: if prediction.pk not in created_ids ) - _resolve_entry_references(entries) + _fill_job_settings_and_references(entries) entries.sort(key=lambda entry: (entry.timestamp, entry.id), reverse=True) return entries -def _resolve_entry_references(entries: list[OccurrenceTimelineEntry]) -> None: - """Fill each entry's job and data references, resolving every id the entries name at once.""" - job_refs = { - id(entry): job_setting_references(_job_settings(entry.job)) for entry in entries if entry.job is not None - } +def _fill_job_settings_and_references(entries: list[OccurrenceTimelineEntry]) -> None: + """Fill each entry's job settings and data references, resolving every id the entries name at once.""" + configs = {id(entry): _job_settings(entry.job) for entry in entries if entry.job is not None} + job_refs = {key: job_setting_references(config) for key, config in configs.items()} # A stored kind may no longer be registered (renamed, or written by another branch); it names no references. kinds = set(result_kinds()) data_refs = { @@ -177,11 +188,15 @@ def _resolve_entry_references(entries: list[OccurrenceTimelineEntry]) -> None: if entry.type == "algorithm_result" } wanted = [(ref_type, ref_id) for refs in (*job_refs.values(), *data_refs.values()) for _, ref_type, ref_id in refs] - if not wanted: - return - resolved = resolve_references(wanted) + resolved = resolve_references(wanted) if wanted else {} for entry in entries: - entry.job_references = {key: resolved[(t, i)] for key, t, i in job_refs.get(id(entry), [])} + if entry.job is not None: + labels = config_setting_labels((entry.job.params or {}).get("task")) + refs = {key: resolved[(t, i)] for key, t, i in job_refs[id(entry)]} + entry.job_settings = [ + JobSetting(key, labels.get(key, key), value, refs.get(key)) + for key, value in configs[id(entry)].items() + ] entry.data_references = {field: resolved[(t, i)] for field, t, i in data_refs.get(id(entry), [])} diff --git a/ami/main/test_occurrence_history.py b/ami/main/test_occurrence_history.py index 6afc59e9b..a2f0341eb 100644 --- a/ami/main/test_occurrence_history.py +++ b/ami/main/test_occurrence_history.py @@ -11,7 +11,7 @@ from ami.jobs.models import Job from ami.main.models import Classification, Detection, Identification, Occurrence, SourceImage, TaxaList, Taxon -from ami.main.models_future.history import occurrence_timeline +from ami.main.models_future.history import JobSetting, occurrence_timeline from ami.main.models_future.references import Ref, job_setting_references, resolve_references from ami.ml.models import Algorithm, AlgorithmResult from ami.tests.fixtures.main import create_captures, create_taxa, setup_test_project @@ -329,12 +329,19 @@ def test_entries_are_merged_newest_first_and_a_result_carries_the_classification self.assertEqual(result["algorithm"]["key"], self.size_filter.key) self.assertEqual( result["job"], - {"id": self.job.pk, "name": self.job.name, "config": {"size_threshold": 0.01}, "references": {}}, + { + "id": self.job.pk, + "name": self.job.name, + "config": {"size_threshold": 0.01}, + # The label is the title the size filter's config schema gives the setting. + "settings": [{"key": "size_threshold", "label": "Size threshold", "value": 0.01, "ref": None}], + }, ) self.assertIsNone(result["taxon"]) self.assertEqual(result["determination_after"]["id"], self.taxon.pk) self.assertEqual(result["determination_before"]["id"], self.other_taxon.pk) - self.assertEqual(result["score"], 0.001) + self.assertIsNone(result["score"]) + self.assertEqual(result["value"], 0.001) self.assertEqual(result["data"]["relative_size"], 0.001) self.assertEqual(result["data_references"], {}) created = Classification.objects.get(algorithm=self.size_filter) @@ -418,17 +425,24 @@ def test_a_demoted_prediction_names_the_result_that_superseded_it(self): self.assertEqual(demoted["details"]["superseded_by_result_id"], result.pk) self.assertFalse(demoted["details"]["terminal"]) self.assertIsNone(predictions[None]["details"]["superseded_by_result_id"]) + settings = {setting["key"]: setting for setting in result_entry["job"]["settings"]} self.assertEqual( - result_entry["job"]["references"], + settings["taxa_list_id"], { - "taxa_list_id": {"type": "taxa_list", "id": taxa_list.pk, "name": "Kept species"}, - "algorithm_id": {"type": "algorithm", "id": classifier.pk, "name": classifier.name}, + "key": "taxa_list_id", + "label": "Species list", + "value": taxa_list.pk, + "ref": {"type": "taxa_list", "id": taxa_list.pk, "name": "Kept species"}, }, ) + self.assertEqual(settings["algorithm_id"]["label"], "Classifier") + self.assertEqual( + settings["algorithm_id"]["ref"], {"type": "algorithm", "id": classifier.pk, "name": classifier.name} + ) # The top prediction before masking is the classification the masked one replaced. self.assertEqual(result_entry["classifications"][0]["replaced"]["id"], original.pk) self.assertEqual(result_entry["classifications"][0]["replaced"]["taxon"]["id"], self.taxon.pk) - self.assertEqual(result_entry["score"], 0.4) + self.assertEqual(result_entry["value"], 0.4) def test_a_terminal_prediction_outranked_on_its_detection_is_superseded(self): """The size filter does not demote the prediction it outranks, so the link is the shared detection.""" @@ -606,7 +620,16 @@ def test_job_settings_are_resolved_to_references(self): data={"relative_size": 0.001}, ) entry = next(e for e in occurrence_timeline(self.occurrence) if e.type == "algorithm_result") - self.assertEqual(entry.job_references, {"taxa_list_id": Ref("taxa_list", taxa_list.pk, "Kept species")}) + # The job names no task, so there is no config schema to take labels from: they fall back to the keys. + self.assertEqual( + entry.job_settings, + [ + JobSetting( + "taxa_list_id", "taxa_list_id", taxa_list.pk, Ref("taxa_list", taxa_list.pk, "Kept species") + ), + JobSetting("size_threshold", "size_threshold", 0.01, None), + ], + ) def test_a_result_without_a_job_has_no_settings_or_references(self): AlgorithmResult.objects.record( @@ -618,7 +641,7 @@ def test_a_result_without_a_job_has_no_settings_or_references(self): ) entry = next(e for e in occurrence_timeline(self.occurrence) if e.type == "algorithm_result") self.assertIsNone(entry.job) - self.assertEqual(entry.job_references, {}) + self.assertEqual(entry.job_settings, []) def test_a_result_of_a_kind_no_longer_registered_still_shows_in_the_history(self): """A row written by a branch with an extra kind, or before a kind was renamed, must not break the history.""" diff --git a/ami/ml/post_processing/class_masking.py b/ami/ml/post_processing/class_masking.py index ee5907c52..14d39c741 100644 --- a/ami/ml/post_processing/class_masking.py +++ b/ami/ml/post_processing/class_masking.py @@ -25,16 +25,16 @@ class ClassMaskingConfig(pydantic.BaseModel): # capture set is the bulk path; a single occurrence is the spot/dev path (fast # feedback while tuning a taxa list). This mirrors SmallSizeFilterConfig's # discriminated-scope shape — the shared pattern for per-occurrence triggers. - source_image_collection_id: int | None = None - occurrence_id: int | None = None + source_image_collection_id: int | None = pydantic.Field(None, title="Capture set") + occurrence_id: int | None = pydantic.Field(None, title="Occurrence") # The taxa list to keep: classes whose taxon is not in this list are masked out. - taxa_list_id: int + taxa_list_id: int = pydantic.Field(..., title="Species list") # The source classifier whose terminal classifications are re-scored. - algorithm_id: int + algorithm_id: int = pydantic.Field(..., title="Classifier") # When True (default), renormalise the kept classes' scores to sum to 1 after # masking. When False, the kept classes retain their original absolute scores and # the excluded classes are zeroed; the chosen species is identical either way. - reweight: bool = True + reweight: bool = pydantic.Field(True, title="Re-weighted scores") @pydantic.root_validator(skip_on_failure=True) def _exactly_one_scope(cls, values: dict) -> dict: diff --git a/ami/ml/post_processing/registry.py b/ami/ml/post_processing/registry.py index 308be18ae..7dbb22d5d 100644 --- a/ami/ml/post_processing/registry.py +++ b/ami/ml/post_processing/registry.py @@ -11,3 +11,12 @@ def get_postprocessing_task(key: str): """Return a post-processing task class by key.""" return POSTPROCESSING_TASKS.get(key) + + +def config_setting_labels(key: str | None) -> dict[str, str]: + """The title each setting declares in the task's config schema, for settings that declare one.""" + task_cls = POSTPROCESSING_TASKS.get(key) if isinstance(key, str) else None + if task_cls is None: + return {} + fields = task_cls.config_schema.__fields__ + return {name: field.field_info.title for name, field in fields.items() if field.field_info.title} diff --git a/ami/ml/post_processing/small_size_filter.py b/ami/ml/post_processing/small_size_filter.py index 29ea7b89f..8e82c0738 100644 --- a/ami/ml/post_processing/small_size_filter.py +++ b/ami/ml/post_processing/small_size_filter.py @@ -15,9 +15,9 @@ class SmallSizeFilterConfig(pydantic.BaseModel): # set is the bulk path; a single occurrence is the spot/dev path (fast feedback # while tuning a filter). This discriminated-scope shape is the pattern other # post-processing tasks copy when they gain per-occurrence / per-event triggers. - source_image_collection_id: int | None = None - occurrence_id: int | None = None - size_threshold: float = 0.0008 + source_image_collection_id: int | None = pydantic.Field(None, title="Capture set") + occurrence_id: int | None = pydantic.Field(None, title="Occurrence") + size_threshold: float = pydantic.Field(0.0008, title="Size threshold") @pydantic.validator("size_threshold") def _threshold_in_unit_interval(cls, v: float) -> float: diff --git a/ui/src/data-services/models/occurrence-history.test.ts b/ui/src/data-services/models/occurrence-history.test.ts index de008a69c..25066f7ab 100644 --- a/ui/src/data-services/models/occurrence-history.test.ts +++ b/ui/src/data-services/models/occurrence-history.test.ts @@ -28,7 +28,14 @@ const JOB = { config: { taxa_list_id: 2 }, id: 30, name: 'Masking run', - references: { taxa_list_id: { type: 'taxa_list', id: 2, name: 'Kept' } }, + settings: [ + { + key: 'taxa_list_id', + label: 'Species list', + value: 2, + ref: { type: 'taxa_list', id: 2, name: 'Kept' }, + }, + ], } const identificationEntry = (id: number): ServerOccurrenceHistoryEntry => ({ @@ -101,8 +108,9 @@ const classMasking: ServerOccurrenceHistoryEntry = { is_current: true, job: JOB, kind: 'class_masking', - score: 0.38, + score: null, type: 'algorithm_result', + value: 0.38, } const ownIdentification: HumanIdentification = { @@ -330,39 +338,50 @@ describe('getOccurrenceHistoryQueryKey', () => { }) describe('getJobSettings', () => { - test('lists the settings a job ran with and the records they name, leaving out unset ones', () => { - const deletedList = { type: 'taxa_list', id: 2, name: null } + test('lists the settings a job ran with, with their labels and records, leaving out unset ones', () => { + const deletedList: { type: string; id: number; name: string | null } = { + type: 'taxa_list', + id: 2, + name: null, + } + const setting = ( + key: string, + value: unknown, + ref: typeof deletedList | null = null + ) => ({ + key, + label: `Label of ${key}`, + value, + ref, + }) expect( getJobSettings({ - config: { - occurrence_id: 4, - reweight: true, - size_threshold: 0.01, - source_image_collection_id: null, - taxa_list_id: 2, - }, + config: null, id: 1, name: 'Size filter', - references: { - occurrence_id: { type: 'occurrence', id: 4, name: '#4' }, - taxa_list_id: deletedList, - }, + settings: [ + setting('occurrence_id', 4, { + type: 'occurrence', + id: 4, + name: '#4', + }), + setting('reweight', true), + setting('size_threshold', 0.01), + setting('source_image_collection_id', null), + // A deleted list keeps its reference with no name, so the card shows its id as text. + setting('taxa_list_id', 2, deletedList), + ], }) ).toEqual([ - { - key: 'occurrence_id', - value: 4, - ref: { type: 'occurrence', id: 4, name: '#4' }, - }, - { key: 'reweight', value: true, ref: undefined }, - // A deleted list keeps its reference with no name, so the card shows its id as text. - { key: 'taxa_list_id', value: 2, ref: deletedList }, + setting('occurrence_id', 4, { type: 'occurrence', id: 4, name: '#4' }), + setting('reweight', true), + setting('taxa_list_id', 2, deletedList), ]) }) test('is empty for a job without settings', () => { expect( - getJobSettings({ config: null, id: 1, name: 'Pipeline', references: {} }) + getJobSettings({ config: null, id: 1, name: 'Pipeline', settings: [] }) ).toEqual([]) expect(getJobSettings(null)).toEqual([]) }) diff --git a/ui/src/data-services/models/occurrence-history.ts b/ui/src/data-services/models/occurrence-history.ts index bcf378d96..bf627a449 100644 --- a/ui/src/data-services/models/occurrence-history.ts +++ b/ui/src/data-services/models/occurrence-history.ts @@ -27,13 +27,21 @@ export interface ServerHistoryAlgorithm { name: string } +/** One setting a job ran with. `label` is the title the task's config schema gives it, else the key. */ +export interface ServerHistoryJobSetting { + key: string + label: string + /** The record the setting names, when it names one. */ + ref: Ref | null + value: unknown +} + export interface ServerHistoryJob { /** The settings a post-processing job ran with; null for other jobs. */ config: Record | null id: number name: string - /** Settings that name another record, by setting key. */ - references: Record + settings: ServerHistoryJobSetting[] } /** The classification a run's classification replaced, when there was one and it still exists. */ @@ -93,7 +101,7 @@ interface ServerHistoryEntryBase { algorithm: ServerHistoryAlgorithm | null id: number job: ServerHistoryJob | null - /** A prediction's score, or a result's headline value. */ + /** A prediction's score; null for other entries. */ score: number | null /** The identified or predicted taxon; null for a result. */ taxon: ServerHistoryTaxon | null @@ -114,6 +122,8 @@ interface ServerResultEntry is_current: boolean kind: Kind type: 'algorithm_result' + /** The kind's headline figure, for sorting and filtering; not a confidence. */ + value: number | null } export type ClassMaskingResultEntry = ServerResultEntry< @@ -396,23 +406,16 @@ export const getResultPrediction = ( } } -export interface JobSetting { - key: string - value: unknown - /** The record the setting names, when it names one. */ - ref?: Ref -} - /** Settings the result card already shows in a row of their own. */ const SETTINGS_SHOWN_ELSEWHERE = ['size_threshold'] -/** A job's settings with the records they name, leaving out unset ones and those shown elsewhere. */ -export const getJobSettings = (job: ServerHistoryJob | null): JobSetting[] => - Object.entries(job?.config ?? {}) - .filter( - ([key, value]) => - value !== null && - value !== undefined && - !SETTINGS_SHOWN_ELSEWHERE.includes(key) - ) - .map(([key, value]) => ({ key, value, ref: job?.references[key] })) +/** A job's settings to show as rows, leaving out unset ones and those shown elsewhere. */ +export const getJobSettings = ( + job: ServerHistoryJob | null +): ServerHistoryJobSetting[] => + (job?.settings ?? []).filter( + ({ key, value }) => + value !== null && + value !== undefined && + !SETTINGS_SHOWN_ELSEWHERE.includes(key) + ) diff --git a/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx b/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx index 221682916..f08bc6491 100644 --- a/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx +++ b/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx @@ -34,15 +34,6 @@ const KINDS = { size_filter: { icon: RulerIcon, label: STRING.HISTORY_SIZE_FILTER }, } -/** Labels for the job settings shown as rows; a setting not listed here shows its raw key. */ -const SETTING_LABELS: Partial> = { - algorithm_id: STRING.HISTORY_SETTING_CLASSIFIER, - occurrence_id: STRING.HISTORY_SETTING_OCCURRENCE, - reweight: STRING.HISTORY_SETTING_REWEIGHT, - source_image_collection_id: STRING.HISTORY_SETTING_CAPTURE_SET, - taxa_list_id: STRING.HISTORY_SPECIES_LIST, -} - /** What to call a result's kind, e.g. "Class masking", for the card and for predictions it superseded. */ export const getResultKindLabel = (entry: AlgorithmResultEntry) => translate(KINDS[entry.kind].label) @@ -51,6 +42,9 @@ const formatSettingValue = (value: unknown) => { if (typeof value === 'boolean') { return translate(value ? STRING.YES : STRING.NO) } + if (Array.isArray(value)) { + return value.join(', ') + } return typeof value === 'object' ? JSON.stringify(value) : `${value}` } @@ -70,13 +64,16 @@ const getRefLabel = (reference?: Ref) => { /** The card's subtitle: for class masking the classifier and species list, otherwise the algorithm's name. */ const getSubTitle = (entry: AlgorithmResultEntry) => { if (entry.kind === 'class_masking') { - const references = entry.job?.references ?? {} + const refFor = (key: string) => + entry.job?.settings.find((setting) => setting.key === key)?.ref ?? + undefined + const classifier = refFor('algorithm_id') return translate(STRING.HISTORY_MASKING_SUBTITLE, { - algorithm: references.algorithm_id - ? getRefLabel(references.algorithm_id) + algorithm: classifier + ? getRefLabel(classifier) : entry.algorithm?.name ?? translate(STRING.VALUE_NOT_AVAILABLE), - list: getRefLabel(references.taxa_list_id), + list: getRefLabel(refFor('taxa_list_id')), }) } @@ -173,10 +170,9 @@ export const AlgorithmResult = ({ label: translate(STRING.HISTORY_DETECTIONS_AFFECTED), value: new Set(entry.classifications.map((c) => c.detection_id)).size, }) - getJobSettings(entry.job).forEach(({ key, value, ref }) => { - const label = SETTING_LABELS[key] + getJobSettings(entry.job).forEach(({ label, value, ref }) => { stats.push({ - label: label !== undefined ? translate(label) : key, + label, value: ref ? ( ) : ( diff --git a/ui/src/utils/language.ts b/ui/src/utils/language.ts index 23670a25b..b1b78ad02 100644 --- a/ui/src/utils/language.ts +++ b/ui/src/utils/language.ts @@ -341,12 +341,7 @@ export enum STRING { HISTORY_ORIGINAL_PREDICTION, HISTORY_LOAD_ERROR, HISTORY_RECORD_ID, - HISTORY_SETTING_CAPTURE_SET, - HISTORY_SETTING_CLASSIFIER, - HISTORY_SETTING_OCCURRENCE, - HISTORY_SETTING_REWEIGHT, HISTORY_SIZE_FILTER, - HISTORY_SPECIES_LIST, HISTORY_SUPERSEDED_BY, ID_APPLIED, INFO, @@ -806,12 +801,7 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.HISTORY_LOAD_ERROR]: 'Could not load the full history. Showing identifications and predictions only.', [STRING.HISTORY_RECORD_ID]: '#{{id}}', - [STRING.HISTORY_SETTING_CAPTURE_SET]: 'Capture set', - [STRING.HISTORY_SETTING_CLASSIFIER]: 'Classifier', - [STRING.HISTORY_SETTING_OCCURRENCE]: 'Occurrence', - [STRING.HISTORY_SETTING_REWEIGHT]: 'Re-weighted scores', [STRING.HISTORY_SIZE_FILTER]: 'Size filter', - [STRING.HISTORY_SPECIES_LIST]: 'Species list', [STRING.HISTORY_SUPERSEDED_BY]: 'Superseded by {{name}}', [STRING.ID_APPLIED]: 'ID applied', [STRING.INFO]: 'Info', From 6c1bb09e6f915e6b4939f17bcd5e65bf0d8bb684 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Tue, 6 Oct 2026 09:23:08 -0700 Subject: [PATCH 100/134] refactor(results): read job settings through one parser, and name each kind on its data model Job settings were parsed in four places: the history's config reader, the serializer's config field, a hard-coded map of setting keys to reference types, and a label lookup. A task's config schema now declares both on the field, with reference("taxa_list", ..., title="Species list"), and the history derives each setting's label, order and referenced record from it in one function. Settings follow the schema's order, because Postgres reorders a stored JSON object's keys. A job whose params are not an object shows no settings instead of failing the history. Each result data model now carries its kind, and the registry is built from the models, so AlgorithmResult.Kind no longer repeats the list. Guard tests check that every reference type a kind or task declares has a mapping and that no kind's schema emits #/definitions refs. The history also stops loading each job's progress and logs, which it never shows. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01APWsYj7AAYL5aiq7T3HUwq --- ami/main/api/serializers.py | 5 +- ami/main/models_future/history.py | 54 ++++++--- ami/main/models_future/references.py | 35 +++--- ami/main/test_occurrence_history.py | 106 +++++++++++++----- ami/main/tests.py | 3 +- ami/ml/models/algorithm_result.py | 4 - ami/ml/post_processing/class_masking.py | 12 +- ami/ml/post_processing/registry.py | 9 -- ami/ml/post_processing/small_size_filter.py | 8 +- .../tests/test_small_size_filter.py | 3 +- ami/ml/results/schemas.py | 45 ++++++-- ami/ml/results/tests.py | 7 ++ 12 files changed, 195 insertions(+), 96 deletions(-) diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index b44c1616d..c4cd70a4e 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -2154,8 +2154,9 @@ class HistoryJobSerializer(serializers.Serializer): @extend_schema_field(serializers.JSONField(allow_null=True)) def get_config(self, job) -> dict | None: - config = (job.params or {}).get("config") - return config if isinstance(config, dict) else None + from ami.main.models_future.history import job_config + + return job_config(job) class ReplacedClassificationSerializer(serializers.Serializer): diff --git a/ami/main/models_future/history.py b/ami/main/models_future/history.py index 3921db41f..4879a8b02 100644 --- a/ami/main/models_future/history.py +++ b/ami/main/models_future/history.py @@ -13,14 +13,17 @@ import typing from ami.main.models import Classification, Identification, Occurrence, Taxon, User -from ami.main.models_future.references import Ref, job_setting_references, resolve_references +from ami.main.models_future.references import Ref, is_record_id, resolve_references from ami.ml.models import Algorithm, AlgorithmResult -from ami.ml.post_processing.registry import config_setting_labels -from ami.ml.results.schemas import reference_fields, result_kinds +from ami.ml.post_processing.registry import get_postprocessing_task +from ami.ml.results.schemas import field_references, field_titles, reference_fields, result_kinds if typing.TYPE_CHECKING: from ami.jobs.models import Job +# A job's progress and logs can be large JSON documents, and the history shows neither. +JOB_FIELDS_NOT_SHOWN = ("job__progress", "job__logs") + @dataclasses.dataclass class JobSetting: @@ -81,6 +84,7 @@ def occurrence_timeline(occurrence: Occurrence) -> list[OccurrenceTimelineEntry] results = list( AlgorithmResult.objects.filter(occurrence=occurrence) .select_related("algorithm", "job") + .defer(*JOB_FIELDS_NOT_SHOWN) .order_by("-timestamp", "-pk") ) taxa = _by_id( @@ -174,36 +178,54 @@ def superseded_by(prediction: Classification) -> int | None: def _fill_job_settings_and_references(entries: list[OccurrenceTimelineEntry]) -> None: """Fill each entry's job settings and data references, resolving every id the entries name at once.""" - configs = {id(entry): _job_settings(entry.job) for entry in entries if entry.job is not None} - job_refs = {key: job_setting_references(config) for key, config in configs.items()} + settings = {id(entry): _job_settings(entry.job) for entry in entries if entry.job is not None} # A stored kind may no longer be registered (renamed, or written by another branch); it names no references. kinds = set(result_kinds()) data_refs = { id(entry): [ (field, ref_type, entry.data[field]) for field, ref_type in (reference_fields(entry.kind) if entry.kind in kinds else {}).items() - if isinstance(entry.data.get(field), int) + if is_record_id(entry.data.get(field)) ] for entry in entries if entry.type == "algorithm_result" } - wanted = [(ref_type, ref_id) for refs in (*job_refs.values(), *data_refs.values()) for _, ref_type, ref_id in refs] + wanted = [(ref_type, value) for specs in settings.values() for _, _, value, ref_type in specs if ref_type] + wanted += [(ref_type, ref_id) for refs in data_refs.values() for _, ref_type, ref_id in refs] resolved = resolve_references(wanted) if wanted else {} for entry in entries: if entry.job is not None: - labels = config_setting_labels((entry.job.params or {}).get("task")) - refs = {key: resolved[(t, i)] for key, t, i in job_refs[id(entry)]} entry.job_settings = [ - JobSetting(key, labels.get(key, key), value, refs.get(key)) - for key, value in configs[id(entry)].items() + JobSetting(key, label, value, resolved[(ref_type, value)] if ref_type else None) + for key, label, value, ref_type in settings[id(entry)] ] entry.data_references = {field: resolved[(t, i)] for field, t, i in data_refs.get(id(entry), [])} -def _job_settings(job: Job | None) -> dict: - """The settings a post-processing job ran with, or an empty dict for any other job.""" - config = (job.params or {}).get("config") if job is not None else None - return config if isinstance(config, dict) else {} +def job_config(job: Job | None) -> dict | None: + """The settings a post-processing job ran with, or None for any other job or malformed params.""" + params = job.params if job is not None else None + config = params.get("config") if isinstance(params, dict) else None + return config if isinstance(config, dict) else None + + +def _job_settings(job: Job) -> list[tuple[str, str, typing.Any, str | None]]: + """``(key, label, value, reference type)`` per setting, labelled, typed and ordered by the task's config schema. + + The stored config's key order is not kept (Postgres orders a JSON object's keys), so settings follow the + schema and any key it does not declare comes after. A job whose task is not registered shows its + settings by key, naming no records. + """ + config = job_config(job) or {} + task_key = job.params.get("task") if config else None + task = get_postprocessing_task(task_key) if isinstance(task_key, str) else None + titles = field_titles(task.config_schema) if task else {} + references = field_references(task.config_schema) if task else {} + keys = [key for key in titles if key in config] + [key for key in config if key not in titles] + return [ + (key, titles.get(key) or key, config[key], references.get(key) if is_record_id(config[key]) else None) + for key in keys + ] def _by_id(model, ids: set) -> dict: @@ -263,7 +285,7 @@ def rank(prediction: Classification) -> tuple: return (score, prediction.terminal, prediction.created_at, prediction.pk) best: dict[int | None, Classification] = {} - for prediction in occurrence.predictions().select_related("job").defer("scores", "logits"): + for prediction in occurrence.predictions().select_related("job").defer("scores", "logits", *JOB_FIELDS_NOT_SHOWN): current = best.get(prediction.algorithm_id) if current is None or rank(prediction) > rank(current): best[prediction.algorithm_id] = prediction diff --git a/ami/main/models_future/references.py b/ami/main/models_future/references.py index d88f88cd5..8300329c7 100644 --- a/ami/main/models_future/references.py +++ b/ami/main/models_future/references.py @@ -1,7 +1,8 @@ """Ids the occurrence history shows, resolved to ``{type, id, name}`` so the UI can link them with one table. ``name`` is None when the row no longer exists; the UI then shows the id as text. Resolution reads names -only, one query per type, and is used for ids stored on the occurrence's own results and jobs. +only, one query per type, and is used for ids stored on the occurrence's own results and jobs. Result data +models and task config schemas declare which fields hold an id with ``ami.ml.results.schemas.reference``. """ import collections @@ -28,24 +29,24 @@ class Ref: "taxon": ("main.Taxon", "name"), } -# Job settings (post-processing configs) that hold another record's id. -JOB_SETTING_REFERENCES: dict[str, str] = { - "source_image_collection_id": "capture_set", - "taxa_list_id": "taxa_list", - "algorithm_id": "algorithm", - "occurrence_id": "occurrence", -} +def is_record_id(value: typing.Any) -> bool: + """Whether a stored value can be a primary key: an int that is not a bool.""" + return isinstance(value, int) and not isinstance(value, bool) + + +def unmapped_reference_types(models: typing.Iterable[type] | None = None) -> set[str]: + """Reference types that result data models or task config schemas declare but this module does not map. + + Defaults to every registered result kind and post-processing task; a test keeps the answer empty. + """ + from ami.ml.results.schemas import ALGORITHM_RESULT_DATA_MODELS, field_references + + if models is None: + from ami.ml.post_processing.registry import POSTPROCESSING_TASKS -def job_setting_references(config: typing.Any) -> list[tuple[str, str, int]]: - """``(setting key, reference type, id)`` for each job setting that names another record.""" - if not isinstance(config, dict): - return [] - return [ - (key, ref_type, config[key]) - for key, ref_type in JOB_SETTING_REFERENCES.items() - if isinstance(config.get(key), int) and not isinstance(config.get(key), bool) - ] + models = [*ALGORITHM_RESULT_DATA_MODELS, *(task.config_schema for task in POSTPROCESSING_TASKS.values())] + return {ref_type for model in models for ref_type in field_references(model).values()} - set(REFERENCE_TYPES) def resolve_references(wanted: typing.Iterable[tuple[str, int]]) -> dict[tuple[str, int], Ref]: diff --git a/ami/main/test_occurrence_history.py b/ami/main/test_occurrence_history.py index a2f0341eb..a590768b2 100644 --- a/ami/main/test_occurrence_history.py +++ b/ami/main/test_occurrence_history.py @@ -2,6 +2,7 @@ import contextlib import datetime +import re from cachalot.api import cachalot_disabled from django.db import IntegrityError, connection, transaction @@ -10,21 +11,31 @@ from rest_framework.test import APITestCase from ami.jobs.models import Job -from ami.main.models import Classification, Detection, Identification, Occurrence, SourceImage, TaxaList, Taxon -from ami.main.models_future.history import JobSetting, occurrence_timeline -from ami.main.models_future.references import Ref, job_setting_references, resolve_references +from ami.main.models import ( + Classification, + Detection, + Identification, + Occurrence, + SourceImage, + SourceImageCollection, + TaxaList, + Taxon, +) +from ami.main.models_future.history import JobSetting, OccurrenceTimelineEntry, occurrence_timeline +from ami.main.models_future.references import Ref, resolve_references, unmapped_reference_types from ami.ml.models import Algorithm, AlgorithmResult +from ami.ml.results.schemas import AlgorithmResultData, ClassMaskingResultData, SizeFilterResultData, reference from ami.tests.fixtures.main import create_captures, create_taxa, setup_test_project from ami.users.models import User from ami.users.roles import BasicMember, ProjectManager # Measured: two savepoints, the project, the occurrence, the results with their algorithm and job, # their taxa, the classifications they created, the classifications those replaced, identifications, -# predictions with their jobs, and one query per type of record the entries name (here species lists). +# predictions with their jobs, and one query per type of record the entries name (here capture sets). HISTORY_QUERIES = 11 -SIZE_FILTER = AlgorithmResult.Kind.SIZE_FILTER -CLASS_MASKING = AlgorithmResult.Kind.CLASS_MASKING +SIZE_FILTER = SizeFilterResultData.kind +CLASS_MASKING = ClassMaskingResultData.kind @contextlib.contextmanager @@ -180,14 +191,14 @@ def test_resolves_each_type_in_one_query_and_marks_missing_rows(self): self.assertEqual(refs[("algorithm", algorithm.pk)].name, "Classifier") self.assertEqual(refs[("taxa_list", 999999)], Ref("taxa_list", 999999, None)) - def test_job_settings_yield_only_known_integer_references(self): - config = {"taxa_list_id": 3, "source_image_collection_id": 5, "size_threshold": 0.01, "algorithm_id": "x"} - self.assertEqual( - sorted(job_setting_references(config)), - [("source_image_collection_id", "capture_set", 5), ("taxa_list_id", "taxa_list", 3)], - ) - self.assertEqual(job_setting_references(None), []) - self.assertEqual(job_setting_references(["not", "a", "dict"]), []) # type: ignore[arg-type] + def test_every_declared_reference_type_is_mapped(self): + """A kind or task declaring a reference type with no mapping would fail the history at request time.""" + + class Probe(AlgorithmResultData): + merged_into_id: int | None = reference("not_a_type") + + self.assertEqual(unmapped_reference_types([Probe]), {"not_a_type"}) + self.assertEqual(unmapped_reference_types(), set()) class OccurrenceHistorySchemaTestCase(TestCase): @@ -500,7 +511,11 @@ def test_query_count_does_not_grow_with_the_entries_and_no_score_arrays_are_load self.client.force_authenticate(user=self.reader) # A setting that names a record, and created classifications that replaced one, so the # reference and replaced-classification queries are counted too. - self.job.params = {"config": {"size_threshold": 0.01, "taxa_list_id": TaxaList.objects.create(name="L").pk}} + scope = SourceImageCollection.objects.create(project=self.project, name="Scope") + self.job.params = { + "task": "small_size_filter", + "config": {"size_threshold": 0.01, "source_image_collection_id": scope.pk}, + } self.job.save() original = Classification.objects.get(detection=self.detections[0]) now = datetime.datetime.now() @@ -515,6 +530,14 @@ def test_query_count_does_not_grow_with_the_entries_and_no_score_arrays_are_load self.assertFalse( [q["sql"] for q in queries.captured_queries if '"main_classification"."scores"' in q["sql"]] ) + # A job's progress and logs can be large JSON documents; the history shows neither. + self.assertFalse( + [ + q["sql"] + for q in queries.captured_queries + if re.search(r'"jobs_job"\."(progress|logs)"', q["sql"]) + ] + ) def test_visible_to_whoever_can_open_the_occurrence(self): """Member, non-member, anonymous and superuser, on a public and on a draft project.""" @@ -608,29 +631,62 @@ def test_predictions_carry_the_job_that_wrote_them(self): self.assertTrue(predictions) self.assertTrue(all(e.job == self.job for e in predictions)) - def test_job_settings_are_resolved_to_references(self): - taxa_list = TaxaList.objects.create(name="Kept species") - self.job.params = {"config": {"taxa_list_id": taxa_list.pk, "size_threshold": 0.01}} - self.job.save() + def _size_filter_result(self, job: Job | None) -> OccurrenceTimelineEntry: AlgorithmResult.objects.record( occurrence=self.occurrence, algorithm=self.size_filter, - job=self.job, + job=job, kind=SIZE_FILTER, data={"relative_size": 0.001}, ) - entry = next(e for e in occurrence_timeline(self.occurrence) if e.type == "algorithm_result") - # The job names no task, so there is no config schema to take labels from: they fall back to the keys. + return next(e for e in occurrence_timeline(self.occurrence) if e.type == "algorithm_result") + + def test_job_settings_take_labels_and_references_from_the_task_config_schema(self): + taxa_list = TaxaList.objects.create(name="Kept species") + classifier = Algorithm.objects.create(name="Classifier", key="settings-test-classifier") + self.job.params = { + "task": "class_masking", + "config": { + "source_image_collection_id": None, + "occurrence_id": True, + "taxa_list_id": taxa_list.pk, + "algorithm_id": classifier.pk, + "reweight": True, + }, + } + self.job.save() self.assertEqual( - entry.job_settings, + self._size_filter_result(self.job).job_settings, [ + JobSetting("source_image_collection_id", "Capture set", None, None), + # A value that is not an id names no record, whatever the schema declares. + JobSetting("occurrence_id", "Occurrence", True, None), JobSetting( - "taxa_list_id", "taxa_list_id", taxa_list.pk, Ref("taxa_list", taxa_list.pk, "Kept species") + "taxa_list_id", "Species list", taxa_list.pk, Ref("taxa_list", taxa_list.pk, "Kept species") ), - JobSetting("size_threshold", "size_threshold", 0.01, None), + JobSetting("algorithm_id", "Classifier", classifier.pk, Ref("algorithm", classifier.pk, "Classifier")), + JobSetting("reweight", "Re-weighted scores", True, None), + ], + ) + + def test_a_job_without_a_registered_task_shows_its_settings_by_key_with_no_references(self): + self.job.params = {"config": {"taxa_list_id": 3, "size_threshold": 0.01}} + self.job.save() + self.assertEqual( + self._size_filter_result(self.job).job_settings, + [ + JobSetting("taxa_list_id", "taxa_list_id", 3, None), + JobSetting("size_threshold", "size_threshold", 0.01), ], ) + def test_a_job_whose_params_are_not_an_object_has_no_settings(self): + Job.objects.filter(pk=self.job.pk).update(params=["not", "an", "object"]) + self.job.refresh_from_db() + self.assertEqual(self._size_filter_result(self.job).job_settings, []) + result = next(e for e in self.get() if e["type"] == "algorithm_result") + self.assertIsNone(result["job"]["config"]) + def test_a_result_without_a_job_has_no_settings_or_references(self): AlgorithmResult.objects.record( occurrence=self.occurrence, diff --git a/ami/main/tests.py b/ami/main/tests.py index ca9796179..fc1399baa 100644 --- a/ami/main/tests.py +++ b/ami/main/tests.py @@ -46,6 +46,7 @@ from ami.ml.models.pipeline import Pipeline from ami.ml.models.processing_service import ProcessingService from ami.ml.models.project_pipeline_config import ProjectPipelineConfig +from ami.ml.results.schemas import SizeFilterResultData from ami.tests.fixtures.main import ( create_captures, create_captures_from_files, @@ -7453,7 +7454,7 @@ def setUp(self): occurrence=self.occ_result, algorithm=self.algorithm, job=self.job, - kind=AlgorithmResult.Kind.SIZE_FILTER, + kind=SizeFilterResultData.kind, data={"relative_size": 0.001}, ) diff --git a/ami/ml/models/algorithm_result.py b/ami/ml/models/algorithm_result.py index b0532b815..ff153a4eb 100644 --- a/ami/ml/models/algorithm_result.py +++ b/ami/ml/models/algorithm_result.py @@ -142,10 +142,6 @@ class AlgorithmResult(BaseModel): rank roll-ups are the next kinds expected. See #1431. """ - class Kind(models.TextChoices): - CLASS_MASKING = "class_masking" - SIZE_FILTER = "size_filter" - # Copied from the occurrence when the result is written, so per-project # queries and permission checks need no join. project = models.ForeignKey("main.Project", on_delete=models.CASCADE, related_name="algorithm_results") diff --git a/ami/ml/post_processing/class_masking.py b/ami/ml/post_processing/class_masking.py index 14d39c741..e5a9ad3d0 100644 --- a/ami/ml/post_processing/class_masking.py +++ b/ami/ml/post_processing/class_masking.py @@ -10,8 +10,8 @@ from ami.main.models import Classification, Occurrence, SourceImageCollection, TaxaList from ami.ml.models.algorithm import Algorithm, AlgorithmTaskType -from ami.ml.models.algorithm_result import AlgorithmResult from ami.ml.post_processing.base import BasePostProcessingTask +from ami.ml.results.schemas import ClassMaskingResultData, reference from ami.ml.results.writer import AlgorithmResultWriter if typing.TYPE_CHECKING: @@ -25,12 +25,12 @@ class ClassMaskingConfig(pydantic.BaseModel): # capture set is the bulk path; a single occurrence is the spot/dev path (fast # feedback while tuning a taxa list). This mirrors SmallSizeFilterConfig's # discriminated-scope shape — the shared pattern for per-occurrence triggers. - source_image_collection_id: int | None = pydantic.Field(None, title="Capture set") - occurrence_id: int | None = pydantic.Field(None, title="Occurrence") + source_image_collection_id: int | None = reference("capture_set", None, title="Capture set") + occurrence_id: int | None = reference("occurrence", None, title="Occurrence") # The taxa list to keep: classes whose taxon is not in this list are masked out. - taxa_list_id: int = pydantic.Field(..., title="Species list") + taxa_list_id: int = reference("taxa_list", ..., title="Species list") # The source classifier whose terminal classifications are re-scored. - algorithm_id: int = pydantic.Field(..., title="Classifier") + algorithm_id: int = reference("algorithm", ..., title="Classifier") # When True (default), renormalise the kept classes' scores to sum to 1 after # masking. When False, the kept classes retain their original absolute scores and # the excluded classes are zeroed; the chosen species is identical either way. @@ -132,7 +132,7 @@ def make_classifications_filtered_by_taxa_list( timestamp = timezone.now() masked_count = 0 results = AlgorithmResultWriter( - kind=AlgorithmResult.Kind.CLASS_MASKING, + kind=ClassMaskingResultData.kind, algorithm=new_algorithm, job=job, value_field="excluded_probability", diff --git a/ami/ml/post_processing/registry.py b/ami/ml/post_processing/registry.py index 7dbb22d5d..308be18ae 100644 --- a/ami/ml/post_processing/registry.py +++ b/ami/ml/post_processing/registry.py @@ -11,12 +11,3 @@ def get_postprocessing_task(key: str): """Return a post-processing task class by key.""" return POSTPROCESSING_TASKS.get(key) - - -def config_setting_labels(key: str | None) -> dict[str, str]: - """The title each setting declares in the task's config schema, for settings that declare one.""" - task_cls = POSTPROCESSING_TASKS.get(key) if isinstance(key, str) else None - if task_cls is None: - return {} - fields = task_cls.config_schema.__fields__ - return {name: field.field_info.title for name, field in fields.items() if field.field_info.title} diff --git a/ami/ml/post_processing/small_size_filter.py b/ami/ml/post_processing/small_size_filter.py index 8e82c0738..5dc1f2df4 100644 --- a/ami/ml/post_processing/small_size_filter.py +++ b/ami/ml/post_processing/small_size_filter.py @@ -4,8 +4,8 @@ from django.utils import timezone from ami.main.models import Classification, Detection, Occurrence, SourceImageCollection, Taxon, TaxonRank -from ami.ml.models.algorithm_result import AlgorithmResult from ami.ml.post_processing.base import BasePostProcessingTask +from ami.ml.results.schemas import SizeFilterResultData, reference from ami.ml.results.writer import AlgorithmResultWriter from ami.ml.schemas import BoundingBox @@ -15,8 +15,8 @@ class SmallSizeFilterConfig(pydantic.BaseModel): # set is the bulk path; a single occurrence is the spot/dev path (fast feedback # while tuning a filter). This discriminated-scope shape is the pattern other # post-processing tasks copy when they gain per-occurrence / per-event triggers. - source_image_collection_id: int | None = pydantic.Field(None, title="Capture set") - occurrence_id: int | None = pydantic.Field(None, title="Occurrence") + source_image_collection_id: int | None = reference("capture_set", None, title="Capture set") + occurrence_id: int | None = reference("occurrence", None, title="Occurrence") size_threshold: float = pydantic.Field(0.0008, title="Size threshold") @pydantic.validator("size_threshold") @@ -95,7 +95,7 @@ def run(self) -> None: checked = 0 # One algorithm result per flagged occurrence, written with the batch that flags it. results = AlgorithmResultWriter( - kind=AlgorithmResult.Kind.SIZE_FILTER, + kind=SizeFilterResultData.kind, algorithm=self.algorithm, job=self.job, value_field="relative_size", diff --git a/ami/ml/post_processing/tests/test_small_size_filter.py b/ami/ml/post_processing/tests/test_small_size_filter.py index 5e324eb79..3ea443182 100644 --- a/ami/ml/post_processing/tests/test_small_size_filter.py +++ b/ami/ml/post_processing/tests/test_small_size_filter.py @@ -9,6 +9,7 @@ from ami.main.models import Detection, Occurrence, SourceImage, Taxon from ami.ml.models import AlgorithmResult from ami.ml.post_processing.small_size_filter import SmallSizeFilterTask +from ami.ml.results.schemas import SizeFilterResultData from ami.tests.fixtures.main import create_captures, create_taxa, setup_test_project logger = logging.getLogger(__name__) @@ -98,7 +99,7 @@ class SizeFilterResultsTestCase(SizeFilterTestCase): """The size filter leaves one algorithm result per occurrence it flags, written with the batch that flags it.""" def records(self): - return AlgorithmResult.objects.filter(kind=AlgorithmResult.Kind.SIZE_FILTER).order_by("pk") + return AlgorithmResult.objects.filter(kind=SizeFilterResultData.kind).order_by("pk") def test_a_flagged_occurrence_gets_one_result_linked_to_the_classifications_it_created(self): occurrence = self._singleton(self.captures[0], [0, 0, 10, 10]) diff --git a/ami/ml/results/schemas.py b/ami/ml/results/schemas.py index 270d562f5..656f741e4 100644 --- a/ami/ml/results/schemas.py +++ b/ami/ml/results/schemas.py @@ -3,21 +3,30 @@ A kind's model holds only what its run alone knows; settings live on the job, and the new taxon on the classifications the run created. ``extra`` takes whatever else a method returns and is never read for logic. This is the only module that calls pydantic's API for results, and it imports nothing from -Django, so models, writers and serializers can all import it. Adding a kind = a model + a registry entry. +Django, so models, writers, serializers and settings can all import it. Adding a kind = a model with its +``kind`` + an entry in ``ALGORITHM_RESULT_DATA_MODELS``; see docs/claude/reference/adding-a-result-kind.md. +The field helpers below also read post-processing task config schemas, which declare setting titles and +references the same way. """ import json -from typing import Any +from typing import Any, ClassVar import pydantic -def reference(ref_type: str, default: Any = None) -> Any: - """Declare a data field that holds another record's id; the history resolves it to ``{type, id, name}``.""" - return pydantic.Field(default, reference=ref_type) +def reference(ref_type: str, default: Any = None, **field_options: Any) -> Any: + """Declare a field that holds another record's id; the history resolves it to ``{type, id, name}``. + + Used by result data models and by task config schemas; ``field_options`` go to ``pydantic.Field``. + """ + return pydantic.Field(default, reference=ref_type, **field_options) class AlgorithmResultData(pydantic.BaseModel): + # The kind this model validates; each concrete model sets it, and the registry is keyed by it. + kind: ClassVar[str] + # Stored, shown and exported only, never read for logic; a value a feature needs becomes a typed field. extra: dict[str, Any] = {} @@ -38,6 +47,8 @@ class DeterminationSnapshot(AlgorithmResultData): class ClassMaskingResultData(DeterminationSnapshot): """Figures from the occurrence's winning detection: the one whose masked classification scores highest.""" + kind: ClassVar[str] = "class_masking" + # One minus the kept mass of the unmasked softmax: what the list removed, not an out-of-distribution score. excluded_probability: float # Where the class that wins after masking ranked before it; 1 means it was already the top. @@ -47,13 +58,16 @@ class ClassMaskingResultData(DeterminationSnapshot): class SizeFilterResultData(DeterminationSnapshot): """Figures from the occurrence's smallest filtered detection.""" + kind: ClassVar[str] = "size_filter" + # The detection's box area as a fraction of its image. relative_size: float +ALGORITHM_RESULT_DATA_MODELS: tuple[type[AlgorithmResultData], ...] = (ClassMaskingResultData, SizeFilterResultData) + ALGORITHM_RESULT_DATA_SCHEMAS: dict[str, type[AlgorithmResultData]] = { - "class_masking": ClassMaskingResultData, - "size_filter": SizeFilterResultData, + model.kind: model for model in ALGORITHM_RESULT_DATA_MODELS } @@ -83,16 +97,25 @@ def validate_result_data(kind: str, data: dict | AlgorithmResultData | None) -> return json.loads(schema.parse_obj(values).json()) -def reference_fields(kind: str) -> dict[str, str]: - """The kind's data fields that hold another record's id, mapped to the reference type.""" - schema = _schema_for(kind) +def field_references(model: type[pydantic.BaseModel]) -> dict[str, str]: + """The model's fields declared with ``reference()``, mapped to the reference type.""" return { name: field.field_info.extra["reference"] - for name, field in schema.__fields__.items() + for name, field in model.__fields__.items() if "reference" in field.field_info.extra } +def field_titles(model: type[pydantic.BaseModel]) -> dict[str, str | None]: + """Every field of the model, in declaration order, mapped to its ``title`` or None.""" + return {name: field.field_info.title for name, field in model.__fields__.items()} + + +def reference_fields(kind: str) -> dict[str, str]: + """The kind's data fields that hold another record's id, mapped to the reference type.""" + return field_references(_schema_for(kind)) + + def data_json_schema(kind: str) -> dict: """The kind's data model as a JSON schema, for its OpenAPI component.""" return _schema_for(kind).schema() diff --git a/ami/ml/results/tests.py b/ami/ml/results/tests.py index 21e8c91b6..c5bef1f42 100644 --- a/ami/ml/results/tests.py +++ b/ami/ml/results/tests.py @@ -13,6 +13,13 @@ def test_every_kind_has_a_closed_json_schema(self): self.assertEqual(schema["type"], "object") self.assertFalse(schema.get("additionalProperties", True)) self.assertIn("extra", schema["properties"]) + # A nested model would emit #/definitions refs, which do not resolve inside OpenAPI. + self.assertNotIn("definitions", schema) + + def test_each_kind_is_registered_under_its_models_kind(self): + for kind, model in schemas.ALGORITHM_RESULT_DATA_SCHEMAS.items(): + with self.subTest(kind): + self.assertEqual(model.kind, kind) def test_reference_fields_come_from_field_metadata(self): class Probe(schemas.AlgorithmResultData): From a417b74d4b01d26c73dcb3d36e2c6437092c909d Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Tue, 6 Oct 2026 09:23:12 -0700 Subject: [PATCH 101/134] refactor(ui): compute a reference's label in one place Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01APWsYj7AAYL5aiq7T3HUwq --- .../identification-card/algorithm-result.tsx | 14 +------------- .../identification-card/history-stats.tsx | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx b/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx index f08bc6491..f262af5c1 100644 --- a/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx +++ b/ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx @@ -18,13 +18,13 @@ import { APP_ROUTES } from 'utils/constants' import { getFormatedDateTimeString } from 'utils/date/getFormatedDateTimeString/getFormatedDateTimeString' import { getAppRoute } from 'utils/getAppRoute' import { STRING, translate } from 'utils/language' -import { Ref } from 'utils/references' import { UserInfo, UserPermission } from 'utils/user/types' import { Agree } from '../agree/agree' import { HistoryStat, HistoryStats, HistoryTime, + getRefLabel, HistoryTypeBadge, RefValue, } from './history-stats' @@ -49,18 +49,6 @@ const formatSettingValue = (value: unknown) => { return typeof value === 'object' ? JSON.stringify(value) : `${value}` } -/** A reference's name, or its id when the record was deleted. */ -const getRefLabel = (reference?: Ref) => { - if (!reference) { - return translate(STRING.VALUE_NOT_AVAILABLE) - } - - return ( - reference.name ?? - translate(STRING.HISTORY_RECORD_ID, { id: `${reference.id}` }) - ) -} - /** The card's subtitle: for class masking the classifier and species list, otherwise the algorithm's name. */ const getSubTitle = (entry: AlgorithmResultEntry) => { if (entry.kind === 'class_masking') { diff --git a/ui/src/pages/occurrence-details/identification-card/history-stats.tsx b/ui/src/pages/occurrence-details/identification-card/history-stats.tsx index 5c8bc642c..b1deae9c5 100644 --- a/ui/src/pages/occurrence-details/identification-card/history-stats.tsx +++ b/ui/src/pages/occurrence-details/identification-card/history-stats.tsx @@ -20,6 +20,18 @@ export const HistoryStats = ({ stats }: { stats: HistoryStat[] }) => ) : null +/** A reference's name, its id when the record was deleted, or "not available" when there is none. */ +export const getRefLabel = (reference?: Ref) => { + if (!reference) { + return translate(STRING.VALUE_NOT_AVAILABLE) + } + + return ( + reference.name ?? + translate(STRING.HISTORY_RECORD_ID, { id: `${reference.id}` }) + ) +} + /** A record the history names: a link to its page, or plain text when it was deleted or has no page. */ export const RefValue = ({ projectId, @@ -29,9 +41,7 @@ export const RefValue = ({ reference: Ref }) => { const to = linkFor(reference, projectId) - const label = - reference.name ?? - translate(STRING.HISTORY_RECORD_ID, { id: `${reference.id}` }) + const label = getRefLabel(reference) return to ? ( From 8b004affeb32622f4d7f185d59f7f8860027def8 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Tue, 6 Oct 2026 09:24:15 -0700 Subject: [PATCH 102/134] docs: explain how to add an algorithm result kind, with rank roll-up as the example Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01APWsYj7AAYL5aiq7T3HUwq --- docs/claude/INDEX.md | 1 + docs/claude/reference/adding-a-result-kind.md | 176 ++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100644 docs/claude/reference/adding-a-result-kind.md diff --git a/docs/claude/INDEX.md b/docs/claude/INDEX.md index 61f0980d8..5be331041 100644 --- a/docs/claude/INDEX.md +++ b/docs/claude/INDEX.md @@ -11,6 +11,7 @@ archived. |---|---| | `reference/canonical-patterns.md` | Existing helpers/patterns to reuse before writing new ones, with file:line refs (SingleParamSerializer, ProjectMixin, permissions, schemas, fixtures). Keywords: reuse, helpers, conventions, DRF | | `reference/query-patterns.md` | DB model relationship table, composite indexes, prefetch/select_related patterns, full custom QuerySet method catalog, query anti-patterns. Keywords: N+1, indexes, ORM, performance | +| `reference/adding-a-result-kind.md` | How to add an algorithm result kind (2026-10-06): data model with `kind` + `reference()`/`title` fields in `ami/ml/results/schemas.py`, task config schema titles/references, `AlgorithmResultWriter` batch pattern (note/start_batch/finish_batch, applied_to, demote only the replaced row), UI card (`ALGORITHM_RESULT_KINDS`, `KINDS`), tests; worked example = rank roll-up (#1361). Keywords: AlgorithmResult, kind, registry, post-processing, history, OpenAPI | | `reference/api-stats-pattern.md` | How to add aggregate/leaderboard/chart endpoints (`//stats//`): GenericViewSet + @action, pure querysets in models_future. Keywords: stats, charts, aggregation | | `reference/monitoring-async-jobs.md` | Monitoring/debugging async_api (NATS JetStream) jobs: ORM, REST, consumer state, Redis counters, worker logs. Keywords: NATS, async, jobs, monitoring | | `reference/worktree-testing.md` | Testing git-worktree changes against the main Docker stack: bind-mount route vs duplicate-stack route, caveats, cleanup. Keywords: worktree, docker compose, override | diff --git a/docs/claude/reference/adding-a-result-kind.md b/docs/claude/reference/adding-a-result-kind.md new file mode 100644 index 000000000..235e9204e --- /dev/null +++ b/docs/claude/reference/adding-a-result-kind.md @@ -0,0 +1,176 @@ +# Adding an algorithm result kind + +An algorithm result (`ami/ml/models/algorithm_result.py`) records what a run decided about one +occurrence: the figures only that run knew, and the occurrence's determination before and after. It +is a record, never an input: the determination still comes only from identifications and +classifications. Each kind of result (`class_masking`, `size_filter`, and next `tracking` and +`rank_rollup`) has its own data model. This guide walks through adding one, using rank roll-up +(draft #1361) as the example. + +Adding a kind needs no migration, no serializer change and no OpenAPI change. It touches the data +model, the task that writes it, and the UI card. + +## 1. Describe the result's data + +Add a pydantic model in `ami/ml/results/schemas.py` and list it in `ALGORITHM_RESULT_DATA_MODELS`. + +```python +class RankRollupResultData(DeterminationSnapshot): + """Figures from the occurrence's winning detection: the one whose rolled-up classification scores highest.""" + + kind: ClassVar[str] = "rank_rollup" + + rolled_up_rank: str + # The summed probability at the new rank; the result's value. + rolled_up_score: float + # The threshold that rank cleared. + threshold: float + # Whether the rolled-up taxon is an ancestor of the source classifier's taxon. + within_lineage: bool + # The best summed score at each rank tried, e.g. {"SPECIES": 0.41, "GENUS": 0.72}. + rank_scores: dict[str, float] = {} + + +ALGORITHM_RESULT_DATA_MODELS = (ClassMaskingResultData, SizeFilterResultData, RankRollupResultData) +``` + +Rules for the model: + +- **Record only what the run alone knows.** Settings belong on the job and are shown from there. The + new taxon and score are on the classifications the run creates. The prediction a classification + replaced is its `applied_to`, which the history shows as `replaced`. Do not copy any of these into + `data`. +- **Subclass `DeterminationSnapshot`** if the run can change the determination. The writer fills + `determination_before_id` and `determination_after_id`. +- **Keep fields flat.** Use scalars, lists and string-keyed dicts of scalars. A nested model or an + `Enum` makes pydantic emit `#/definitions/...` references that do not resolve inside the OpenAPI + schema; `ami/ml/results/tests.py` fails if a kind does. +- **Declare ids of other records with `reference()`**, for example + `merged_into_id: int | None = reference("occurrence")`. The history returns such a field as + `{type, id, name}` under `data_references`, and the UI links it. The type must be a key of + `REFERENCE_TYPES` in `ami/main/models_future/references.py`. To add a type, add it there and add its + route to `ROUTES` in `ui/src/utils/references.ts`. A guard test fails when a declared type has no + mapping. +- **`extra`** (inherited) takes anything else the method returns. Nothing may read it for logic; when + a feature needs a value from it, make that value a typed field. +- **Pick one figure as the result's `value`:** the number lists sort and filter on (here + `rolled_up_score`). It is not necessarily a confidence; for tracking it is a motion figure. + +The kind's OpenAPI component (`RankRollupResultEntry`) and its enum name come from the registry +automatically. + +## 2. Describe the task's settings + +The task's config schema (the pydantic model `config_schema` on the task) is where the history gets +each setting's label, order and referenced record. Give every field a `title`, and declare ids with +`reference()`: + +```python +from ami.ml.results.schemas import reference + + +class RankRollupConfig(pydantic.BaseModel): + source_image_collection_id: int | None = reference("capture_set", None, title="Capture set") + occurrence_id: int | None = reference("occurrence", None, title="Occurrence") + thresholds: dict[str, float] = pydantic.Field(DEFAULT_THRESHOLDS, title="Thresholds by rank") + rollup_order: list[str] = pydantic.Field(DEFAULT_ORDER, title="Ranks to try, in order") +``` + +A field without a `title` shows its key on the result card. Register the task in +`ami/ml/post_processing/registry.py`; the history finds the schema through the job's +`params["task"]`. + +## 3. Write results from the task + +Use `AlgorithmResultWriter` (`ami/ml/results/writer.py`), the same way the size filter and class +masking do. The writer creates one result per occurrence per run, in the same transaction as the +batch that changes the occurrence, so a batch never lands without its record. + +```python +results = AlgorithmResultWriter( + kind=RankRollupResultData.kind, + algorithm=self.algorithm, + job=self.job, + value_field="rolled_up_score", +) + +for classification in scope: # batched; see class_masking.py for the batching and progress pattern + ... + if rolled_up: + new = Classification( + detection=classification.detection, + taxon=winner, + score=summed_score, + terminal=True, + algorithm=self.algorithm, + applied_to=classification, # the prediction this one replaces + job=self.job, + timestamp=classification.timestamp, + ) + classification.terminal = False # demote only the classification this run replaced + to_create.append(new) + to_demote.append(classification) + occurrences.add(classification.detection.occurrence) + # The occurrence's result keeps the figures of its highest-ranked detection. + results.note(classification.detection.occurrence, figures, rank=summed_score) + +def flush(): + with transaction.atomic(): + results.start_batch(occurrences, to_create) # creates results; points the new rows at them + Classification.objects.bulk_create(to_create) + Classification.objects.bulk_update(to_demote, ["terminal"]) + for occurrence in occurrences: + occurrence.save(update_determination=True) + results.finish_batch(occurrences) # records the determination after the saves +``` + +Rules for the writer: + +- **Call `note()` before `start_batch()` for every occurrence in the batch.** `start_batch` reads the + noted figures, and the figures must contain `value_field`. +- **Set `applied_to`** on a classification that replaces another. The history then marks the replaced + prediction as superseded by this run and shows it as the result's original prediction. A + classification that replaces nothing (like the size filter's "Not identifiable") instead supersedes + terminal predictions it outranks on the same detection. +- **Demote only what the run replaced.** Demoting other classifiers' rows leaves predictions in the + history that look superseded with nothing saying why. +- **Set `job=self.job`** on every row the run creates. +- **Do not call `.distinct()`** on a classification queryset that includes the `scores` or `logits` + arrays; de-duplicating sorts the arrays (see #1376). +- **One current result per occurrence, algorithm and kind.** A re-run replaces the current result and + keeps the old one as history. When results from different sources should coexist (masking a second + classifier, say), give each source its own `Algorithm`, as class masking does. +- **Occurrences without a project** get no result and a warning; the run carries on (#1188). + +## 4. Show it in the UI + +- `ui/src/data-services/models/occurrence-history.ts`: add the data interface (mirroring the pydantic + model), the entry type (`ServerResultEntry<'rank_rollup', RankRollupResultData>`) to the + `AlgorithmResultEntry` union, and the kind to `ALGORITHM_RESULT_KINDS`. Until the kind is in that + list, the timeline skips its results, so the server side can ship first. +- `ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx`: add an icon and label + to `KINDS`, and a `case` in the stats switch for the kind's own figures. The determination row, the + new prediction with its Confirm button, the original prediction (from `replaced`), the detections + changed, the settings and the job row are shared by every kind. +- `ui/src/utils/language.ts`: add the strings the card uses. + +## 5. Test it + +- The run writes one result per touched occurrence, with the expected `data` and `value`. +- Each created classification has `algorithm_result` and `job` set, and `applied_to` where it + replaced one. +- The history shows the result with its classifications, the replaced prediction as superseded, and + `replaced` filled. +- A second run makes the first result history (`is_current` false). +- An occurrence with several affected detections gets one result, with the figures of the + highest-ranked detection. +- `python manage.py test ami.ml.results ami.main.test_occurrence_history` covers the registry guards + and the history contract. + +## What you do not need to touch + +- **Migrations:** `kind` is a plain `CharField` without `choices`; every write path validates `data` + against the registry instead. +- **Serializers and OpenAPI:** the history serializer and the `oneOf` union are generated from the + registry, including `SPECTACULAR_SETTINGS["ENUM_NAME_OVERRIDES"]`. +- **The determination:** it never reads results. From eba6978a4ba29b26ba56a6a4331b5099dd839b09 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Tue, 6 Oct 2026 12:02:57 -0700 Subject: [PATCH 103/134] docs(results): keep the algorithm results guide next to the code, and explain adding a history entry type The guide moves to ami/ml/results/README.md, following processing_services/README.md, and opens with what an algorithm result is and how it relates to per-detection outputs such as feature vectors. A new section lists the five places a new kind of history entry touches. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01APWsYj7AAYL5aiq7T3HUwq --- .../ml/results/README.md | 73 +++++++++++++++---- ami/ml/results/schemas.py | 2 +- docs/claude/INDEX.md | 2 +- 3 files changed, 59 insertions(+), 18 deletions(-) rename docs/claude/reference/adding-a-result-kind.md => ami/ml/results/README.md (68%) diff --git a/docs/claude/reference/adding-a-result-kind.md b/ami/ml/results/README.md similarity index 68% rename from docs/claude/reference/adding-a-result-kind.md rename to ami/ml/results/README.md index 235e9204e..d674148b8 100644 --- a/docs/claude/reference/adding-a-result-kind.md +++ b/ami/ml/results/README.md @@ -1,18 +1,29 @@ -# Adding an algorithm result kind +# Algorithm results -An algorithm result (`ami/ml/models/algorithm_result.py`) records what a run decided about one -occurrence: the figures only that run knew, and the occurrence's determination before and after. It -is a record, never an input: the determination still comes only from identifications and -classifications. Each kind of result (`class_masking`, `size_filter`, and next `tracking` and -`rank_rollup`) has its own data model. This guide walks through adding one, using rank roll-up -(draft #1361) as the example. +An algorithm result (`ami/ml/models/algorithm_result.py`) is the standard record of what an +algorithm decided about one occurrence: the figures only that run knew, its headline `value`, and the +occurrence's determination before and after, with the algorithm and job that produced it. Any +algorithm can write one, whether a post-processing method or a pipeline step. It is a record, never an +input: the determination still comes only from identifications and classifications. -Adding a kind needs no migration, no serializer change and no OpenAPI change. It touches the data -model, the task that writes it, and the UI card. +Each kind of result (`class_masking`, `size_filter`, and next `tracking` and `rank_rollup`) has its +own data model in `schemas.py` in this package. The occurrence history +(`ami/main/models_future/history.py`, `GET /occurrences/{id}/history/`) shows each result next to the +occurrence's identifications and predictions, and the API publishes each kind's data as a typed +OpenAPI component. -## 1. Describe the result's data +Outputs that belong to a single detection keep their own tables: classifications, and feature vectors +(`DetectionEmbedding`). They carry the same provenance (algorithm and job) but are not results. -Add a pydantic model in `ami/ml/results/schemas.py` and list it in `ALGORITHM_RESULT_DATA_MODELS`. +This guide covers adding a result kind, using rank roll-up (draft #1361) as the example, and then +adding another kind of history entry. Adding a kind needs no migration, no serializer change and no +OpenAPI change: it touches the data model, the task that writes it, and the UI card. + +## Adding a result kind + +### 1. Describe the result's data + +Add a pydantic model in `schemas.py` and list it in `ALGORITHM_RESULT_DATA_MODELS`. ```python class RankRollupResultData(DeterminationSnapshot): @@ -59,7 +70,7 @@ Rules for the model: The kind's OpenAPI component (`RankRollupResultEntry`) and its enum name come from the registry automatically. -## 2. Describe the task's settings +### 2. Describe the task's settings The task's config schema (the pydantic model `config_schema` on the task) is where the history gets each setting's label, order and referenced record. Give every field a `title`, and declare ids with @@ -80,7 +91,7 @@ A field without a `title` shows its key on the result card. Register the task in `ami/ml/post_processing/registry.py`; the history finds the schema through the job's `params["task"]`. -## 3. Write results from the task +### 3. Write results from the task Use `AlgorithmResultWriter` (`ami/ml/results/writer.py`), the same way the size filter and class masking do. The writer creates one result per occurrence per run, in the same transaction as the @@ -142,7 +153,7 @@ Rules for the writer: classifier, say), give each source its own `Algorithm`, as class masking does. - **Occurrences without a project** get no result and a warning; the run carries on (#1188). -## 4. Show it in the UI +### 4. Show it in the UI - `ui/src/data-services/models/occurrence-history.ts`: add the data interface (mirroring the pydantic model), the entry type (`ServerResultEntry<'rank_rollup', RankRollupResultData>`) to the @@ -154,7 +165,7 @@ Rules for the writer: changed, the settings and the job row are shared by every kind. - `ui/src/utils/language.ts`: add the strings the card uses. -## 5. Test it +### 5. Test it - The run writes one result per touched occurrence, with the expected `data` and `value`. - Each created classification has `algorithm_result` and `job` set, and `applied_to` where it @@ -167,10 +178,40 @@ Rules for the writer: - `python manage.py test ami.ml.results ami.main.test_occurrence_history` covers the registry guards and the history contract. -## What you do not need to touch +### What you do not need to touch - **Migrations:** `kind` is a plain `CharField` without `choices`; every write path validates `data` against the registry instead. - **Serializers and OpenAPI:** the history serializer and the `oneOf` union are generated from the registry, including `SPECTACULAR_SETTINGS["ENUM_NAME_OVERRIDES"]`. - **The determination:** it never reads results. + +## Adding another kind of history entry + +The history merges entries from several tables: algorithm results, identifications and predictions. +Another source, such as the feature vectors a pipeline added to an occurrence's detections, becomes a +new entry type rather than a result kind when it records an output and not a decision. Add it in five +places: + +1. **`ami/main/models_future/history.py`:** a section of fields on `OccurrenceTimelineEntry` (if the + shared fields are not enough), and a loader in `occurrence_timeline` that adds the entries with a + fixed number of queries, whatever the occurrence's size. Entries with a `job` get its settings + automatically. For vectors, group by job and algorithm, and never select the vector column: + ```python + DetectionEmbedding.objects.filter(detection__occurrence=occurrence) + .values("algorithm", "job", "key") + .annotate(detections=Count("detection", distinct=True), timestamp=Max("timestamp")) + ``` +2. **`ami/main/api/serializers.py`:** an `EntrySerializer(HistoryEntryBaseSerializer)` with + `type = serializers.ChoiceField(choices=[""])` and its own fields. Add it to the serializers + of `OCCURRENCE_HISTORY_ENTRY_SCHEMA` and to `HISTORY_ENTRY_SERIALIZERS`. +3. **`config/settings/base.py`:** a `"EntryTypeEnum": [""]` entry in + `SPECTACULAR_SETTINGS["ENUM_NAME_OVERRIDES"]`, so the literal `type` gets a readable enum name. +4. **The UI:** the entry interface in `ui/src/data-services/models/occurrence-history.ts`, added to + `ServerOccurrenceHistoryEntry` and to `TimelineItem`, with a `case` in `getTimelineItems`; a card + component in `ui/src/pages/occurrence-details/identification-card/`, rendered in + `occurrence-timeline.tsx`. `getTimelineItems` skips entry types it does not know, so the server + side can ship first. +5. **Tests in `ami/main/test_occurrence_history.py`:** the new entries in the endpoint response, the + strict `HISTORY_QUERIES` count raised by the loader's queries (and still the same at two history + sizes), and the new component in the OpenAPI union. diff --git a/ami/ml/results/schemas.py b/ami/ml/results/schemas.py index 656f741e4..cc34e89b3 100644 --- a/ami/ml/results/schemas.py +++ b/ami/ml/results/schemas.py @@ -4,7 +4,7 @@ the classifications the run created. ``extra`` takes whatever else a method returns and is never read for logic. This is the only module that calls pydantic's API for results, and it imports nothing from Django, so models, writers, serializers and settings can all import it. Adding a kind = a model with its -``kind`` + an entry in ``ALGORITHM_RESULT_DATA_MODELS``; see docs/claude/reference/adding-a-result-kind.md. +``kind`` + an entry in ``ALGORITHM_RESULT_DATA_MODELS``; see README.md in this package. The field helpers below also read post-processing task config schemas, which declare setting titles and references the same way. """ diff --git a/docs/claude/INDEX.md b/docs/claude/INDEX.md index 5be331041..253ba2c71 100644 --- a/docs/claude/INDEX.md +++ b/docs/claude/INDEX.md @@ -11,7 +11,7 @@ archived. |---|---| | `reference/canonical-patterns.md` | Existing helpers/patterns to reuse before writing new ones, with file:line refs (SingleParamSerializer, ProjectMixin, permissions, schemas, fixtures). Keywords: reuse, helpers, conventions, DRF | | `reference/query-patterns.md` | DB model relationship table, composite indexes, prefetch/select_related patterns, full custom QuerySet method catalog, query anti-patterns. Keywords: N+1, indexes, ORM, performance | -| `reference/adding-a-result-kind.md` | How to add an algorithm result kind (2026-10-06): data model with `kind` + `reference()`/`title` fields in `ami/ml/results/schemas.py`, task config schema titles/references, `AlgorithmResultWriter` batch pattern (note/start_batch/finish_batch, applied_to, demote only the replaced row), UI card (`ALGORITHM_RESULT_KINDS`, `KINDS`), tests; worked example = rank roll-up (#1361). Keywords: AlgorithmResult, kind, registry, post-processing, history, OpenAPI | +| `../../ami/ml/results/README.md` | (Developer guide, lives next to the code.) Algorithm results overview + how to add a result kind (data model with `kind` and `reference()`/`title` fields, task config schema, `AlgorithmResultWriter` batch pattern, UI card, tests; worked example rank roll-up #1361) + how to add another history entry type (e.g. vectors from #1462). Keywords: AlgorithmResult, kind, registry, post-processing, history, OpenAPI, DetectionEmbedding | | `reference/api-stats-pattern.md` | How to add aggregate/leaderboard/chart endpoints (`//stats//`): GenericViewSet + @action, pure querysets in models_future. Keywords: stats, charts, aggregation | | `reference/monitoring-async-jobs.md` | Monitoring/debugging async_api (NATS JetStream) jobs: ORM, REST, consumer state, Redis counters, worker logs. Keywords: NATS, async, jobs, monitoring | | `reference/worktree-testing.md` | Testing git-worktree changes against the main Docker stack: bind-mount route vs duplicate-stack route, caveats, cleanup. Keywords: worktree, docker compose, override | From 6fce35b47a8ed63030ebda4c40d6f8af94f2c99e Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Tue, 6 Oct 2026 12:07:04 -0700 Subject: [PATCH 104/134] docs(results): explain how algorithm results fit the post-processing framework [skip ci] Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01APWsYj7AAYL5aiq7T3HUwq --- ami/ml/results/README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/ami/ml/results/README.md b/ami/ml/results/README.md index d674148b8..c46067085 100644 --- a/ami/ml/results/README.md +++ b/ami/ml/results/README.md @@ -19,6 +19,29 @@ This guide covers adding a result kind, using rank roll-up (draft #1361) as the adding another kind of history entry. Adding a kind needs no migration, no serializer change and no OpenAPI change: it touches the data model, the task that writes it, and the UI card. +## How results fit the post-processing framework + +The post-processing framework (`ami/ml/post_processing/`) runs a method over data Antenna already has. +Algorithm results are its output side. + +- **Input:** a task subclasses `BasePostProcessingTask` (`base.py`) with a `key`, a `name` and a + pydantic `config_schema`, and is registered in `registry.py`. An admin action + (`admin/actions.py`, `make_post_processing_action`) creates a `post_processing` job whose params are + `{"task": , "config": {...}}`. `PostProcessingJob.run` validates the config against the schema, + stores the validated config (defaults included) back on the job, and runs the task with + `self.job` and `self.algorithm` (one `Algorithm` row per method; class masking makes one per source + classifier and species list). +- **Output:** what the task changes goes through the classifications it creates, each carrying its + `job`, its `algorithm_result` and, when it replaces one, `applied_to`. What it decided about each + occurrence goes into one algorithm result of the task's kind, written by `AlgorithmResultWriter` in + the same transaction as each batch. +- **Shown together:** the history shows each result with the job's settings, labelled, ordered and + linked from the same `config_schema`. One schema therefore describes a task's settings for + validation, the admin form, the stored job and the result card. + +Results are not tied to the framework. A pipeline step that decides something per occurrence can write +them the same way, with `AlgorithmResult.objects.record_many`. + ## Adding a result kind ### 1. Describe the result's data From 6366c574cadb3844767f08c6fcdc19cce6d28993 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Tue, 6 Oct 2026 12:08:06 -0700 Subject: [PATCH 105/134] docs(results): note that result kinds are not tied to post-processing tasks [skip ci] Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01APWsYj7AAYL5aiq7T3HUwq --- ami/ml/results/README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ami/ml/results/README.md b/ami/ml/results/README.md index c46067085..5bb72a96c 100644 --- a/ami/ml/results/README.md +++ b/ami/ml/results/README.md @@ -42,6 +42,13 @@ Algorithm results are its output side. Results are not tied to the framework. A pipeline step that decides something per occurrence can write them the same way, with `AlgorithmResult.objects.record_many`. +A kind is not tied to a task either; today's two happen to be one each. A task can write several kinds +(one `AlgorithmResultWriter` per kind), and several tasks or pipeline steps can write the same kind and +share its data model and card. Each method keeps its own current result, because "current" is per +occurrence, algorithm and kind. The history labels settings from the job's task, not from the kind. +One rule when a run writes several kinds: pass the classifications it creates to only one writer's +`start_batch`, because `start_batch` points each of them at that writer's result for its occurrence. + ## Adding a result kind ### 1. Describe the result's data From 7656cfbf748b7bee36b1662ceb95677d3a74d94a Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Tue, 6 Oct 2026 15:05:05 -0700 Subject: [PATCH 106/134] feat(post-processing): tasks declare the result models they write Each post-processing task now lists the algorithm result data models it writes in `result_models`, next to `config_schema` for its settings. It is the first step toward pipeline stages that declare their inputs and outputs. A test checks that every declared model is a registered kind, and the size filter's tests check that what it writes is declared. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01APWsYj7AAYL5aiq7T3HUwq --- ami/ml/post_processing/base.py | 4 ++++ ami/ml/post_processing/class_masking.py | 1 + ami/ml/post_processing/small_size_filter.py | 1 + ami/ml/post_processing/tests/test_small_size_filter.py | 1 + ami/ml/results/tests.py | 7 +++++++ 5 files changed, 14 insertions(+) diff --git a/ami/ml/post_processing/base.py b/ami/ml/post_processing/base.py index 8f197f192..0344742b8 100644 --- a/ami/ml/post_processing/base.py +++ b/ami/ml/post_processing/base.py @@ -8,6 +8,7 @@ from ami.ml.models import Algorithm from ami.ml.models.algorithm import AlgorithmTaskType +from ami.ml.results.schemas import AlgorithmResultData if typing.TYPE_CHECKING: from ami.jobs.models import Job @@ -27,6 +28,9 @@ class BasePostProcessingTask(abc.ABC): key: str name: str config_schema: type[pydantic.BaseModel] + # The result data models (ami/ml/results/schemas.py) of the algorithm results the task writes; + # empty for a task that writes none. Its settings are declared by ``config_schema``. + result_models: tuple[type[AlgorithmResultData], ...] = () def __init_subclass__(cls, **kwargs): super().__init_subclass__(**kwargs) diff --git a/ami/ml/post_processing/class_masking.py b/ami/ml/post_processing/class_masking.py index e5a9ad3d0..84ef2a187 100644 --- a/ami/ml/post_processing/class_masking.py +++ b/ami/ml/post_processing/class_masking.py @@ -273,6 +273,7 @@ class ClassMaskingTask(BasePostProcessingTask): key = "class_masking" name = "Class masking" config_schema = ClassMaskingConfig + result_models = (ClassMaskingResultData,) def _get_or_create_masking_algorithm( self, source_algorithm: Algorithm, taxa_list: TaxaList, *, reweight: bool diff --git a/ami/ml/post_processing/small_size_filter.py b/ami/ml/post_processing/small_size_filter.py index 5dc1f2df4..824ec5688 100644 --- a/ami/ml/post_processing/small_size_filter.py +++ b/ami/ml/post_processing/small_size_filter.py @@ -40,6 +40,7 @@ class SmallSizeFilterTask(BasePostProcessingTask): key = "small_size_filter" name = "Small size filter" config_schema = SmallSizeFilterConfig + result_models = (SizeFilterResultData,) def _scoped_detections(self, config: SmallSizeFilterConfig) -> tuple[QuerySet[Detection], str]: """Resolve the detections to examine from whichever scope the config carries. diff --git a/ami/ml/post_processing/tests/test_small_size_filter.py b/ami/ml/post_processing/tests/test_small_size_filter.py index 3ea443182..11fd74140 100644 --- a/ami/ml/post_processing/tests/test_small_size_filter.py +++ b/ami/ml/post_processing/tests/test_small_size_filter.py @@ -116,6 +116,7 @@ def test_a_flagged_occurrence_gets_one_result_linked_to_the_classifications_it_c detection_ids = sorted(occurrence.detections.values_list("pk", flat=True)) self.assertEqual((result.occurrence_id, result.project_id), (occurrence.pk, self.project.pk)) self.assertEqual(result.algorithm.key, "small_size_filter") + self.assertIn(result.kind, {model.kind for model in SmallSizeFilterTask.result_models}) # The smallest flagged detection, 10 by 10 pixels on a 1000 by 1000 image, represents the occurrence. self.assertAlmostEqual(result.value, 0.0001) self.assertAlmostEqual(result.data.pop("relative_size"), 0.0001) diff --git a/ami/ml/results/tests.py b/ami/ml/results/tests.py index c5bef1f42..f8ebdcd47 100644 --- a/ami/ml/results/tests.py +++ b/ami/ml/results/tests.py @@ -1,5 +1,6 @@ from django.test import SimpleTestCase +from ami.ml.post_processing.registry import POSTPROCESSING_TASKS from ami.ml.results import schemas @@ -16,6 +17,12 @@ def test_every_kind_has_a_closed_json_schema(self): # A nested model would emit #/definitions refs, which do not resolve inside OpenAPI. self.assertNotIn("definitions", schema) + def test_every_result_model_a_task_declares_is_a_registered_kind(self): + for key, task in POSTPROCESSING_TASKS.items(): + for model in task.result_models: + with self.subTest(task=key, model=model.__name__): + self.assertIs(schemas.ALGORITHM_RESULT_DATA_SCHEMAS.get(model.kind), model) + def test_each_kind_is_registered_under_its_models_kind(self): for kind, model in schemas.ALGORITHM_RESULT_DATA_SCHEMAS.items(): with self.subTest(kind): From a786479dbe37db78a06ef5e21879f0edd02250a1 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Tue, 6 Oct 2026 15:05:12 -0700 Subject: [PATCH 107/134] refactor(results): keep every run's result instead of marking one as current Results are now only added. Running a method twice, such as the size filter with two thresholds, leaves two results on the occurrence, one per job, and the history shows both. Nothing read `is_current`, and keeping it cost a partial unique constraint, an occurrence lock on every write, and the reconciling logic in `move_to_occurrence`, which is now a single update. A retried job no longer adds a second result: the writer finds the results the job already wrote for an occurrence and reuses them. The history entry and its API field drop `is_current`. The ml/0029 migration, not yet merged, is edited to its final shape. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01APWsYj7AAYL5aiq7T3HUwq --- ami/main/api/serializers.py | 3 - ami/main/models_future/history.py | 2 - ami/main/test_occurrence_history.py | 54 +++------- ami/ml/migrations/0029_algorithm_result.py | 13 +-- ami/ml/models/algorithm_result.py | 100 +++--------------- .../tests/test_class_masking.py | 2 +- .../tests/test_small_size_filter.py | 17 ++- ami/ml/results/writer.py | 21 +++- .../models/occurrence-history.test.ts | 1 - .../models/occurrence-history.ts | 1 - 10 files changed, 68 insertions(+), 146 deletions(-) diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index c4cd70a4e..0e6835377 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -2242,9 +2242,6 @@ class AlgorithmResultEntrySerializer(HistoryEntryBaseSerializer): classifications = CreatedClassificationSerializer( many=True, help_text="The classifications the run created, best score first." ) - is_current = serializers.BooleanField( - help_text="Whether it is the latest of its kind, not replaced by a later run." - ) def _result_entry_component(kind: str) -> type[serializers.Serializer]: diff --git a/ami/main/models_future/history.py b/ami/main/models_future/history.py index 4879a8b02..4fad40f53 100644 --- a/ami/main/models_future/history.py +++ b/ami/main/models_future/history.py @@ -65,7 +65,6 @@ class OccurrenceTimelineEntry: determination_before: Taxon | None = None determination_after: Taxon | None = None classifications: list[CreatedClassification] = dataclasses.field(default_factory=list) - is_current: bool | None = None # --- Identification and prediction details: dict = dataclasses.field(default_factory=dict) @@ -128,7 +127,6 @@ def superseded_by(prediction: Classification) -> int | None: determination_before=taxa.get(result.data.get("determination_before_id")), determination_after=taxa.get(result.data.get("determination_after_id")), classifications=created_by_result.get(result.pk, []), - is_current=result.is_current, ) for result in results ] diff --git a/ami/main/test_occurrence_history.py b/ami/main/test_occurrence_history.py index a590768b2..631be11e3 100644 --- a/ami/main/test_occurrence_history.py +++ b/ami/main/test_occurrence_history.py @@ -5,7 +5,7 @@ import re from cachalot.api import cachalot_disabled -from django.db import IntegrityError, connection, transaction +from django.db import connection from django.test import TestCase from django.test.utils import CaptureQueriesContext from rest_framework.test import APITestCase @@ -92,7 +92,7 @@ def test_data_that_does_not_fit_its_kind_is_refused_on_every_write_path(self): with self.assertRaises(ValueError): AlgorithmResult.objects.bulk_update([result], ["data"]) - def test_a_new_result_becomes_current_and_the_earlier_one_stays_as_history(self): + def test_each_run_adds_a_result_beside_the_earlier_ones(self): first = AlgorithmResult.objects.record( occurrence=self.occurrence, algorithm=self.algorithm, @@ -106,14 +106,10 @@ def test_a_new_result_becomes_current_and_the_earlier_one_stays_as_history(self) self._size_filter({"relative_size": 0.02}, occurrence=other_occurrence), ] ) - first.refresh_from_db() - self.assertFalse(first.is_current) - self.assertTrue(second.is_current and untouched.is_current) - self.assertEqual(self.occurrence.algorithm_results.count(), 2) - self.assertEqual(self.occurrence.algorithm_results.current().get().pk, second.pk) - - def test_merging_moves_results_and_keeps_one_current_per_algorithm_and_kind(self): - """The kept occurrence's current result wins a collision; otherwise the latest moved one stays current.""" + self.assertEqual(set(self.occurrence.algorithm_results.values_list("pk", flat=True)), {first.pk, second.pk}) + self.assertEqual(list(other_occurrence.algorithm_results.values_list("pk", flat=True)), [untouched.pk]) + + def test_merging_moves_every_result_to_the_kept_occurrence(self): other_algorithm = Algorithm.objects.create(name="Second size filter", key="size-filter-merge-test") absorbed = [Occurrence.objects.create(project=self.project, deployment=self.deployment) for _ in range(3)] @@ -122,36 +118,21 @@ def record(occurrence, algorithm, size): occurrence=occurrence, algorithm=algorithm, kind=SIZE_FILTER, data={"relative_size": size} ) - kept_current = record(self.occurrence, self.algorithm, 0.01) - history = record(absorbed[0], self.algorithm, 0.02) - colliding = record(absorbed[0], self.algorithm, 0.03) - older = record(absorbed[1], other_algorithm, 0.04) - latest = record(absorbed[2], other_algorithm, 0.05) + kept_own = record(self.occurrence, self.algorithm, 0.01) + moved_results = [ + record(absorbed[0], self.algorithm, 0.02), + record(absorbed[0], self.algorithm, 0.03), + record(absorbed[1], other_algorithm, 0.04), + record(absorbed[2], other_algorithm, 0.05), + ] moved = AlgorithmResult.objects.move_to_occurrence(self.occurrence, [o.pk for o in absorbed]) self.assertEqual(moved, 4) - results = {r.pk: r for r in AlgorithmResult.objects.filter(occurrence=self.occurrence)} - self.assertEqual(set(results), {kept_current.pk, history.pk, colliding.pk, older.pk, latest.pk}) - current = {pk for pk, r in results.items() if r.is_current} - self.assertEqual(current, {kept_current.pk, latest.pk}) - - def test_moving_results_locks_the_occurrences_before_reading_their_results(self): - """Without the lock, a run recording a result on the kept occurrence mid-merge makes two current rows.""" - absorbed = Occurrence.objects.create(project=self.project, deployment=self.deployment) - self._size_filter({"relative_size": 0.02}, occurrence=absorbed).save() - with CaptureQueriesContext(connection) as queries: - AlgorithmResult.objects.move_to_occurrence(self.occurrence, [absorbed.pk]) - sql = [q["sql"] for q in queries.captured_queries] - lock = next((i for i, q in enumerate(sql) if "FOR UPDATE" in q and "main_occurrence" in q), None) - self.assertIsNotNone(lock, "the occurrences are not locked") - first_read = next(i for i, q in enumerate(sql) if "ml_algorithmresult" in q) - self.assertLess(lock, first_read) - - def test_the_database_holds_one_current_result_per_occurrence_algorithm_and_kind(self): - self._size_filter({"relative_size": 0.01}).save() - with transaction.atomic(), self.assertRaises(IntegrityError): - self._size_filter({"relative_size": 0.02}).save() + self.assertEqual( + set(AlgorithmResult.objects.filter(occurrence=self.occurrence).values_list("pk", flat=True)), + {kept_own.pk, *(r.pk for r in moved_results)}, + ) def test_project_comes_from_the_occurrence_and_a_result_without_one_is_skipped(self): result = self._size_filter({"relative_size": 0.01}) @@ -336,7 +317,6 @@ def test_entries_are_merged_newest_first_and_a_result_carries_the_classification self.assertEqual(identification["taxon"]["id"], self.other_taxon.pk) self.assertEqual(identification["details"]["comment"], "round 0") self.assertEqual(result["kind"], "size_filter") - self.assertTrue(result["is_current"]) self.assertEqual(result["algorithm"]["key"], self.size_filter.key) self.assertEqual( result["job"], diff --git a/ami/ml/migrations/0029_algorithm_result.py b/ami/ml/migrations/0029_algorithm_result.py index c35257979..3613f9cec 100644 --- a/ami/ml/migrations/0029_algorithm_result.py +++ b/ami/ml/migrations/0029_algorithm_result.py @@ -22,7 +22,6 @@ class Migration(migrations.Migration): ("kind", models.CharField(max_length=32)), ("value", models.FloatField(blank=True, null=True)), ("data", models.JSONField(blank=True, default=dict)), - ("is_current", models.BooleanField(default=True)), ("timestamp", models.DateTimeField(default=django.utils.timezone.now)), ( "algorithm", @@ -65,20 +64,12 @@ class Migration(migrations.Migration): "indexes": [ models.Index(fields=["occurrence", "-timestamp"], name="algorithm_result_occ_time"), models.Index( - condition=models.Q(("is_current", True), ("value__isnull", False)), + condition=models.Q(("value__isnull", False)), fields=["project", "kind", "value"], - name="algorithm_result_current_value", + name="algorithm_result_value", ), models.Index(fields=["job", "occurrence"], name="algorithm_result_job_occ"), ], }, ), - migrations.AddConstraint( - model_name="algorithmresult", - constraint=models.UniqueConstraint( - condition=models.Q(("is_current", True)), - fields=("occurrence", "algorithm", "kind"), - name="algorithm_result_current_occurrence", - ), - ), ] diff --git a/ami/ml/models/algorithm_result.py b/ami/ml/models/algorithm_result.py index ff153a4eb..41c822350 100644 --- a/ami/ml/models/algorithm_result.py +++ b/ami/ml/models/algorithm_result.py @@ -1,9 +1,7 @@ -import collections import logging import typing -from django.apps import apps -from django.db import models, transaction +from django.db import models from django.db.models import Q from django.utils import timezone @@ -17,9 +15,6 @@ class AlgorithmResultQuerySet(BaseQuerySet): - def current(self): - return self.filter(is_current=True) - def bulk_create(self, objs, *args, **kwargs): objs = list(objs) for obj in objs: @@ -34,25 +29,20 @@ def bulk_update(self, objs, fields, *args, **kwargs): return super().bulk_update(objs, fields, *args, **kwargs) def record(self, **fields) -> "AlgorithmResult": - """Insert one result as the current one for its occurrence, algorithm and kind.""" + """Insert one result.""" results = self.record_many([self.model(**fields)]) if not results: raise ValueError("The occurrence has no project.") return results[0] def record_many(self, results: typing.Iterable["AlgorithmResult"]) -> list["AlgorithmResult"]: - """Insert results as current, keeping each occurrence's earlier result of the same algorithm and kind. + """Insert results, each with its project copied from its occurrence. - Rows are only inserted, never rewritten, apart from clearing ``is_current`` on the result - each new one replaces. A result whose occurrence has no project is skipped with a warning - rather than failing a run that has already changed data. The returned list holds the - results that were written, in order. + Results are only ever added: a later run's result sits beside the earlier ones. A result + whose occurrence has no project is skipped with a warning rather than failing a run that + has already changed data. The returned list holds the results that were written, in order. """ - results = list(results) - if not results: - return [] kept: list[AlgorithmResult] = [] - groups: dict[tuple[int, str], set[int]] = collections.defaultdict(set) for result in results: if result.project_id is None: result.project_id = result.occurrence.project_id @@ -63,72 +53,24 @@ def record_many(self, results: typing.Iterable["AlgorithmResult"]) -> list["Algo "it has no project." ) continue - key = (result.algorithm_id, result.kind) - if result.occurrence_id in groups[key]: - raise ValueError( - f"Two {result.kind} results for Occurrence #{result.occurrence_id} " - "from one algorithm in one write." - ) - groups[key].add(result.occurrence_id) - result.is_current = True kept.append(result) - if not kept: - return [] - with transaction.atomic(): - # Lock the occurrences in a stable order, so two runs writing results for the same - # occurrence wait for each other instead of both inserting a current row. - list( - apps.get_model("main", "Occurrence") - .objects.select_for_update() - .filter(pk__in={result.occurrence_id for result in kept}) - .order_by("pk") - .values_list("pk", flat=True) - ) - for (algorithm_id, kind), occurrence_ids in groups.items(): - self.filter( - occurrence_id__in=occurrence_ids, algorithm_id=algorithm_id, kind=kind, is_current=True - ).update(is_current=False) - return self.bulk_create(kept) + return self.bulk_create(kept) if kept else [] def move_to_occurrence(self, kept: "Occurrence", absorbed_ids: typing.Iterable[int]) -> int: """Move the results of occurrences merged into ``kept``, so their history follows the merge. - Each algorithm and kind keeps one current result on ``kept``: its own if it has one, - otherwise the latest moved one. The other moved results stay as history. Returns the - number of results moved. + Call it before deleting the absorbed occurrences, whose results would otherwise be deleted + with them. Returns the number of results moved. """ absorbed_ids = set(absorbed_ids) - {kept.pk} if not absorbed_ids: return 0 - with transaction.atomic(): - # Lock in primary-key order, as record_many does, so a run recording a result on these - # occurrences waits for the merge instead of adding a second current result. - list( - apps.get_model("main", "Occurrence") - .objects.select_for_update() - .filter(pk__in={kept.pk, *absorbed_ids}) - .order_by("pk") - .values_list("pk", flat=True) - ) - current_keys = set(self.filter(occurrence=kept, is_current=True).values_list("algorithm_id", "kind")) - demote: list[int] = [] - for pk, algorithm_id, kind in ( - self.filter(occurrence_id__in=absorbed_ids, is_current=True) - .order_by("-timestamp", "-pk") - .values_list("pk", "algorithm_id", "kind") - ): - if (algorithm_id, kind) in current_keys: - demote.append(pk) - else: - current_keys.add((algorithm_id, kind)) - if demote: - self.filter(pk__in=demote).update(is_current=False) - return self.filter(occurrence_id__in=absorbed_ids).update(occurrence=kept) + return self.filter(occurrence_id__in=absorbed_ids).update(occurrence=kept) @typing.final class AlgorithmResult(BaseModel): - """What a post-processing run decided about one occurrence: a record of the run, never an input to it. + """What a post-processing run decided about one occurrence, with the figures only that run knew. The determination never reads this table. A run that changes an occurrence's taxon does so through the ``Classification`` rows it creates, and those rows point back here through @@ -136,10 +78,10 @@ class AlgorithmResult(BaseModel): ``data`` holds the run's own figures, validated against the model for ``kind`` (ami/ml/results/schemas.py), and ``value`` repeats the one figure lists filter and sort on. The ``extra`` object inside ``data`` is stored, shown and exported only; nothing reads it for - logic, and a value a feature needs becomes a typed field. A new result for the same - occurrence, algorithm and kind becomes the current one and the earlier ones stay as - history. Write through ``AlgorithmResult.objects.record`` or ``record_many``. Tracking and - rank roll-ups are the next kinds expected. See #1431. + logic, and a value a feature needs becomes a typed field. Every run adds its own result, + so running a method twice leaves two results on the occurrence, one per job. Write through + ``AlgorithmResult.objects.record`` or ``record_many``. Tracking and rank roll-ups are the + next kinds expected. See #1431. """ # Copied from the occurrence when the result is written, so per-project @@ -162,25 +104,17 @@ class AlgorithmResult(BaseModel): # the list for class masking, the relative size for the size filter. value = models.FloatField(null=True, blank=True) data = models.JSONField(default=dict, blank=True) - is_current = models.BooleanField(default=True) timestamp = models.DateTimeField(default=timezone.now) objects = AlgorithmResultQuerySet.as_manager() class Meta: - constraints = [ - models.UniqueConstraint( - fields=["occurrence", "algorithm", "kind"], - condition=Q(is_current=True), - name="algorithm_result_current_occurrence", - ), - ] indexes = [ models.Index(fields=["occurrence", "-timestamp"], name="algorithm_result_occ_time"), models.Index( fields=["project", "kind", "value"], - condition=Q(is_current=True, value__isnull=False), - name="algorithm_result_current_value", + condition=Q(value__isnull=False), + name="algorithm_result_value", ), # Leads with the job, for the ?job= occurrence filter and for deleting a job. models.Index(fields=["job", "occurrence"], name="algorithm_result_job_occ"), diff --git a/ami/ml/post_processing/tests/test_class_masking.py b/ami/ml/post_processing/tests/test_class_masking.py index 7e0f3064e..bd82dc1aa 100644 --- a/ami/ml/post_processing/tests/test_class_masking.py +++ b/ami/ml/post_processing/tests/test_class_masking.py @@ -403,7 +403,7 @@ def test_a_rescored_occurrence_gets_one_result_linked_to_the_classification_it_c ClassMaskingTask(occurrence_id=occ.pk, taxa_list_id=taxa_list.pk, algorithm_id=self.algorithm.pk).run() result = AlgorithmResult.objects.get(occurrence=occ, kind="class_masking") - self.assertTrue(result.is_current) + self.assertIn(result.kind, {model.kind for model in ClassMaskingTask.result_models}) self.assertEqual(result.project_id, self.project.pk) self.assertTrue(result.algorithm.key.startswith(f"{self.algorithm.key}_filtered_by_taxa_list_{taxa_list.pk}")) # Index 2 held e^5 / (e^2 + e^1 + e^5) of the probability and was excluded; index 0 was second. diff --git a/ami/ml/post_processing/tests/test_small_size_filter.py b/ami/ml/post_processing/tests/test_small_size_filter.py index 11fd74140..ebec0f6b1 100644 --- a/ami/ml/post_processing/tests/test_small_size_filter.py +++ b/ami/ml/post_processing/tests/test_small_size_filter.py @@ -94,6 +94,17 @@ def test_a_run_records_its_job_on_each_classification_it_writes(self): self.assertEqual(job.classifications.get().pk, classification.pk) self.assertEqual(job.algorithm_results.get().pk, classification.algorithm_result_id) + def test_a_retried_job_reuses_its_own_result_instead_of_adding_another(self): + occurrence = self._singleton(self.captures[0], [0, 0, 10, 10]) + job = self._job(occurrence, size_threshold=0.01) + + PostProcessingJob.run(job) + first = job.algorithm_results.get() + PostProcessingJob.run(job) + + self.assertEqual(list(job.algorithm_results.values_list("pk", flat=True)), [first.pk]) + self.assertEqual(occurrence.algorithm_results.count(), 1) + class SizeFilterResultsTestCase(SizeFilterTestCase): """The size filter leaves one algorithm result per occurrence it flags, written with the batch that flags it.""" @@ -130,12 +141,12 @@ def test_a_flagged_occurrence_gets_one_result_linked_to_the_classifications_it_c ) self.assertEqual(sorted(result.classifications.values_list("detection_id", flat=True)), detection_ids) - def test_a_second_run_replaces_the_current_result_and_keeps_the_first(self): + def test_each_run_adds_its_own_result(self): + """Running the filter again, say with another threshold, shows up as a second result in the history.""" occurrence = self._singleton(self.captures[0], [0, 0, 10, 10]) for _ in range(2): self.run_filter(occurrence) - first, second = self.records() - self.assertEqual((first.is_current, second.is_current), (False, True)) + self.assertEqual(self.records().filter(occurrence=occurrence).count(), 2) def test_an_occurrence_with_nothing_flagged_gets_no_result(self): occurrence = self._singleton(self.captures[0], [0, 0, 500, 500]) diff --git a/ami/ml/results/writer.py b/ami/ml/results/writer.py index bb04aff63..4732cdd6e 100644 --- a/ami/ml/results/writer.py +++ b/ami/ml/results/writer.py @@ -3,10 +3,10 @@ A run that re-scores or flags detections changes classifications and determinations in batches, each in its own transaction. ``AlgorithmResultWriter`` writes the result rows in those same transactions, so a batch either lands with its results or not at all. Every occurrence the run touches gets -exactly one current result: it is created by the first batch that touches the occurrence, before -the classifications that batch inserts so they can point at it, and later batches update it with -the figures of the detection that represents the occurrence and the determination after their -saves. +exactly one result from the run: it is created by the first batch that touches the occurrence, +before the classifications that batch inserts so they can point at it, and later batches update it +with the figures of the detection that represents the occurrence and the determination after their +saves. A retried job reuses the results it already wrote rather than adding a second one each. """ from __future__ import annotations @@ -57,6 +57,9 @@ def start_batch(self, occurrences: typing.Iterable[Occurrence], classifications: occurrences are saved, so a result's ``determination_before_id`` is what the run found. """ new = [o for o in occurrences if o.pk not in self.results and o.pk not in self.skipped] + for result in self._written_by_this_job(new): + self.results[result.occurrence_id] = result + new = [o for o in new if o.pk not in self.results] written = AlgorithmResult.objects.record_many( AlgorithmResult( occurrence=occurrence, @@ -75,6 +78,16 @@ def start_batch(self, occurrences: typing.Iterable[Occurrence], classifications: for classification in classifications: classification.algorithm_result = self.results.get(classification.detection.occurrence_id) + def _written_by_this_job(self, occurrences: list[Occurrence]) -> list[AlgorithmResult]: + """Results an earlier attempt of the same job wrote for these occurrences, latest last.""" + if self.job is None or not occurrences: + return [] + return list( + AlgorithmResult.objects.filter( + job=self.job, algorithm=self.algorithm, kind=self.kind, occurrence__in=[o.pk for o in occurrences] + ).order_by("timestamp", "pk") + ) + def finish_batch(self, occurrences: typing.Iterable[Occurrence]) -> None: """Record each occurrence's determination after the batch's saves and its best figures so far.""" touched = [] diff --git a/ui/src/data-services/models/occurrence-history.test.ts b/ui/src/data-services/models/occurrence-history.test.ts index 25066f7ab..4b67af04f 100644 --- a/ui/src/data-services/models/occurrence-history.test.ts +++ b/ui/src/data-services/models/occurrence-history.test.ts @@ -105,7 +105,6 @@ const classMasking: ServerOccurrenceHistoryEntry = { determination_after: NOCTUA, determination_before: XESTIA, id: 5, - is_current: true, job: JOB, kind: 'class_masking', score: null, diff --git a/ui/src/data-services/models/occurrence-history.ts b/ui/src/data-services/models/occurrence-history.ts index bf627a449..e46b961dd 100644 --- a/ui/src/data-services/models/occurrence-history.ts +++ b/ui/src/data-services/models/occurrence-history.ts @@ -119,7 +119,6 @@ interface ServerResultEntry determination_after: ServerHistoryTaxon | null determination_before: ServerHistoryTaxon | null /** Whether it is the latest of its kind, not replaced by a later run. */ - is_current: boolean kind: Kind type: 'algorithm_result' /** The kind's headline figure, for sorting and filtering; not a confidence. */ From 2b7d14ad5e5e4cf095406a4892167a2e27f3311e Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Tue, 6 Oct 2026 15:05:18 -0700 Subject: [PATCH 108/134] docs(results): say where results can be written, and which algorithm a run records under The guide no longer claims that a pipeline step can write results: the processing-service contract has no place for them and must not name occurrences. It now explains that a run gets its own algorithm only when it changes what can be output, that every run adds its own result, and that a task declares the kinds it writes in `result_models`. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01APWsYj7AAYL5aiq7T3HUwq --- ami/ml/results/README.md | 41 ++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/ami/ml/results/README.md b/ami/ml/results/README.md index 5bb72a96c..16cb90087 100644 --- a/ami/ml/results/README.md +++ b/ami/ml/results/README.md @@ -2,9 +2,9 @@ An algorithm result (`ami/ml/models/algorithm_result.py`) is the standard record of what an algorithm decided about one occurrence: the figures only that run knew, its headline `value`, and the -occurrence's determination before and after, with the algorithm and job that produced it. Any -algorithm can write one, whether a post-processing method or a pipeline step. It is a record, never an -input: the determination still comes only from identifications and classifications. +occurrence's determination before and after, with the algorithm and job that produced it. Results are +written by methods that run inside Antenna once the occurrence exists, which today means post-processing +tasks. The determination never reads them: it still comes only from identifications and classifications. Each kind of result (`class_masking`, `size_filter`, and next `tracking` and `rank_rollup`) has its own data model in `schemas.py` in this package. The occurrence history @@ -39,16 +39,29 @@ Algorithm results are its output side. linked from the same `config_schema`. One schema therefore describes a task's settings for validation, the admin form, the stored job and the result card. -Results are not tied to the framework. A pipeline step that decides something per occurrence can write -them the same way, with `AlgorithmResult.objects.record_many`. +Processing services cannot write results: their contract (`ami/ml/schemas.py`) has no place for them +and must not name occurrences, which Antenna creates. A stage that runs inside Antenna after occurrences +exist can, through `AlgorithmResultWriter` or `AlgorithmResult.objects.record_many`. -A kind is not tied to a task either; today's two happen to be one each. A task can write several kinds -(one `AlgorithmResultWriter` per kind), and several tasks or pipeline steps can write the same kind and -share its data model and card. Each method keeps its own current result, because "current" is per -occurrence, algorithm and kind. The history labels settings from the job's task, not from the kind. +A task declares the kinds it writes in `result_models`, next to `config_schema` for its settings. A kind +is not tied to one task: a task can write several kinds (one `AlgorithmResultWriter` per kind), and +several tasks can write the same kind and share its data model and card. The history labels settings +from the job's task, not from the kind. One rule when a run writes several kinds: pass the classifications it creates to only one writer's `start_batch`, because `start_batch` points each of them at that writer's result for its occurrence. +### Which algorithm a run records under + +A run gets its own `Algorithm` only when it changes what can be output. Class masking makes one per +source classifier and species list, because the list changes which species the masked classifier can +name (its category map). Every other setting stays on the job: the size filter uses one algorithm +whatever its threshold. + +Results are only ever added. Running a method twice, for example the size filter with two thresholds, +leaves two results on the occurrence, one per job, and the history shows both; compare runs by their +jobs. A retried job reuses the results it already wrote. A list that needs each algorithm's latest value +on an occurrence takes the latest by timestamp when it reads. + ## Adding a result kind ### 1. Describe the result's data @@ -117,7 +130,8 @@ class RankRollupConfig(pydantic.BaseModel): rollup_order: list[str] = pydantic.Field(DEFAULT_ORDER, title="Ranks to try, in order") ``` -A field without a `title` shows its key on the result card. Register the task in +A field without a `title` shows its key on the result card. Declare the kinds the task writes with +`result_models = (RankRollupResultData,)`, and register the task in `ami/ml/post_processing/registry.py`; the history finds the schema through the job's `params["task"]`. @@ -178,9 +192,8 @@ Rules for the writer: - **Set `job=self.job`** on every row the run creates. - **Do not call `.distinct()`** on a classification queryset that includes the `scores` or `logits` arrays; de-duplicating sorts the arrays (see #1376). -- **One current result per occurrence, algorithm and kind.** A re-run replaces the current result and - keeps the old one as history. When results from different sources should coexist (masking a second - classifier, say), give each source its own `Algorithm`, as class masking does. +- **One result per occurrence per run.** The writer creates it with the first batch that touches the + occurrence and updates it after later batches; a retried job reuses it, and another run adds its own. - **Occurrences without a project** get no result and a warning; the run carries on (#1188). ### 4. Show it in the UI @@ -202,7 +215,7 @@ Rules for the writer: replaced one. - The history shows the result with its classifications, the replaced prediction as superseded, and `replaced` filled. -- A second run makes the first result history (`is_current` false). +- A second run adds a second result, and a retried job does not. - An occurrence with several affected detections gets one result, with the figures of the highest-ranked detection. - `python manage.py test ami.ml.results ami.main.test_occurrence_history` covers the registry guards From 5c122ad2294a72951d7960a7c61ce6a19806fbff Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Tue, 6 Oct 2026 15:56:41 -0700 Subject: [PATCH 109/134] test(results): build the history and size filter fixtures once per class The occurrence history and size filter test classes built their project, captures and taxa before every test. Each `setup_test_project()` call also registers a processing service, which makes HTTP calls to the test ML backend. Building the fixtures once per class with `setUpTestData` takes the two files from about 26 seconds to under 10 on a local run; every test still starts from the same rows, because each runs in a transaction that is rolled back and Django gives each test its own copy of the class's objects. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01APWsYj7AAYL5aiq7T3HUwq --- ami/main/test_occurrence_history.py | 71 ++++++++++--------- .../tests/test_small_size_filter.py | 17 ++--- 2 files changed, 46 insertions(+), 42 deletions(-) diff --git a/ami/main/test_occurrence_history.py b/ami/main/test_occurrence_history.py index 631be11e3..605b52312 100644 --- a/ami/main/test_occurrence_history.py +++ b/ami/main/test_occurrence_history.py @@ -52,10 +52,11 @@ def no_query_cache(): class AlgorithmResultTestCase(TestCase): """A result's data fits its kind, its project is its occurrence's, and a new result replaces the current one.""" - def setUp(self) -> None: - self.project, self.deployment = setup_test_project(reuse=False) - self.occurrence = Occurrence.objects.create(project=self.project, deployment=self.deployment) - self.algorithm = Algorithm.objects.create(name="Size filter", key="size-filter-result-test") + @classmethod + def setUpTestData(cls) -> None: + cls.project, cls.deployment = setup_test_project(reuse=False) + cls.occurrence = Occurrence.objects.create(project=cls.project, deployment=cls.deployment) + cls.algorithm = Algorithm.objects.create(name="Size filter", key="size-filter-result-test") def _size_filter(self, data: dict | None, **fields) -> AlgorithmResult: fields.setdefault("occurrence", self.occurrence) @@ -159,8 +160,9 @@ def test_the_model_lives_in_the_ml_app(self): class ReferenceTestCase(TestCase): """Ids shown in the history become {type, id, name}; a deleted row keeps its id with no name.""" - def setUp(self): - self.project, self.deployment = setup_test_project(reuse=False) + @classmethod + def setUpTestData(cls) -> None: + cls.project, cls.deployment = setup_test_project(reuse=False) def test_resolves_each_type_in_one_query_and_marks_missing_rows(self): taxa_list = TaxaList.objects.create(name="Kept species") @@ -215,47 +217,48 @@ def resolve(schema: dict) -> dict: class OccurrenceFixtureTestCase(APITestCase): """One occurrence of four classified detections, with a project manager and a basic member.""" - def setUp(self) -> None: - self.project, self.deployment = setup_test_project(reuse=False) - create_taxa(project=self.project) - create_captures(deployment=self.deployment, num_nights=1, images_per_night=4, interval_minutes=1) - captures = list(SourceImage.objects.filter(deployment=self.deployment).order_by("timestamp")) - self.taxon = Taxon.objects.filter(projects=self.project).order_by("pk").first() - - self.manager = User.objects.create_user(email="history-manager@insectai.org") # type: ignore - self.reader = User.objects.create_user(email="history-reader@insectai.org") # type: ignore - ProjectManager.assign_user(self.manager, self.project) - BasicMember.assign_user(self.reader, self.project) - - self.occurrence = Occurrence.objects.create( - event=captures[0].event, deployment=self.deployment, project=self.project + @classmethod + def setUpTestData(cls) -> None: + cls.project, cls.deployment = setup_test_project(reuse=False) + create_taxa(project=cls.project) + create_captures(deployment=cls.deployment, num_nights=1, images_per_night=4, interval_minutes=1) + captures = list(SourceImage.objects.filter(deployment=cls.deployment).order_by("timestamp")) + cls.taxon = Taxon.objects.filter(projects=cls.project).order_by("pk").first() + + cls.manager = User.objects.create_user(email="history-manager@insectai.org") # type: ignore + cls.reader = User.objects.create_user(email="history-reader@insectai.org") # type: ignore + ProjectManager.assign_user(cls.manager, cls.project) + BasicMember.assign_user(cls.reader, cls.project) + + cls.occurrence = Occurrence.objects.create( + event=captures[0].event, deployment=cls.deployment, project=cls.project ) - self.detections = [] + cls.detections = [] for capture in captures[:4]: detection = Detection.objects.create( - source_image=capture, timestamp=capture.timestamp, bbox=[10, 10, 40, 40], occurrence=self.occurrence + source_image=capture, timestamp=capture.timestamp, bbox=[10, 10, 40, 40], occurrence=cls.occurrence ) - detection.classifications.create(taxon=self.taxon, score=0.9, timestamp=capture.timestamp) - self.detections.append(detection) - self.occurrence.save() - return super().setUp() + detection.classifications.create(taxon=cls.taxon, score=0.9, timestamp=capture.timestamp) + cls.detections.append(detection) + cls.occurrence.save() class OccurrenceHistoryEndpointTestCase(OccurrenceFixtureTestCase): """GET /occurrences/{id}/history/ merges results, reviews, identifications and predictions, newest first.""" - def setUp(self) -> None: - super().setUp() - self.size_filter = Algorithm.objects.create(name="Small size filter", key="size-filter-history-test") - self.job = Job.objects.create( - project=self.project, + @classmethod + def setUpTestData(cls) -> None: + super().setUpTestData() + cls.size_filter = Algorithm.objects.create(name="Small size filter", key="size-filter-history-test") + cls.job = Job.objects.create( + project=cls.project, name="Size filter run", job_type_key="post_processing", params={"task": "small_size_filter", "config": {"size_threshold": 0.01}}, ) - self.other_taxon = Taxon.objects.filter(projects=self.project).exclude(pk=self.taxon.pk).first() - self.superuser = User.objects.create_superuser(email="history-super@insectai.org") # type: ignore - self.outsider = User.objects.create_user(email="history-outsider@insectai.org") # type: ignore + cls.other_taxon = Taxon.objects.filter(projects=cls.project).exclude(pk=cls.taxon.pk).first() + cls.superuser = User.objects.create_superuser(email="history-super@insectai.org") # type: ignore + cls.outsider = User.objects.create_user(email="history-outsider@insectai.org") # type: ignore def url(self, occurrence: Occurrence | None = None) -> str: return f"/api/v2/occurrences/{(occurrence or self.occurrence).pk}/history/?project_id={self.project.pk}" diff --git a/ami/ml/post_processing/tests/test_small_size_filter.py b/ami/ml/post_processing/tests/test_small_size_filter.py index ebec0f6b1..f2c55435c 100644 --- a/ami/ml/post_processing/tests/test_small_size_filter.py +++ b/ami/ml/post_processing/tests/test_small_size_filter.py @@ -16,14 +16,15 @@ class SizeFilterTestCase(TestCase): - def setUp(self) -> None: - self.project, self.deployment = setup_test_project(reuse=False) - create_taxa(project=self.project) - create_captures(deployment=self.deployment, num_nights=1, images_per_night=3, interval_minutes=1) - self.captures = list(SourceImage.objects.filter(deployment=self.deployment).order_by("timestamp")) - SourceImage.objects.filter(pk__in=[c.pk for c in self.captures]).update(width=1000, height=1000) - self.event = self.captures[0].event - self.taxon = Taxon.objects.filter(projects=self.project).order_by("pk").first() + @classmethod + def setUpTestData(cls) -> None: + cls.project, cls.deployment = setup_test_project(reuse=False) + create_taxa(project=cls.project) + create_captures(deployment=cls.deployment, num_nights=1, images_per_night=3, interval_minutes=1) + cls.captures = list(SourceImage.objects.filter(deployment=cls.deployment).order_by("timestamp")) + SourceImage.objects.filter(pk__in=[c.pk for c in cls.captures]).update(width=1000, height=1000) + cls.event = cls.captures[0].event + cls.taxon = Taxon.objects.filter(projects=cls.project).order_by("pk").first() def _singleton(self, capture: SourceImage, bbox: list[int]) -> Occurrence: """One detection in an occurrence of its own, classified as the fixture taxon.""" From 8651e359a1eefd136a1752193896e4f827a66598 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Tue, 6 Oct 2026 18:00:23 -0700 Subject: [PATCH 110/134] refactor(results): derive value from data, refuse cross-project moves, and publish result data as JSON Changes from an independent review of the PR and an open CodeRabbit finding: - `move_to_occurrence` refuses absorbed occurrences from another project, so a result is never filed under a project other than its occurrence's. - Each result kind names the data field its headline figure comes from (`value_field`), and every write path copies it into `value`, so the two cannot disagree. Writers no longer pass the value by hand. - The history schema publishes a result's `data` as JSON instead of one typed component per kind. Nothing consumed the typed components yet, and naming their enums made settings import application code. #1482 tracks bringing them back once the UI generates its types from the schema; the UI keeps its hand-written types per kind. - `data_references` is removed: no kind declares a linked id in its data. Settings that name records still resolve to links on the result card. - The history view's docstring now says what it does for occurrences the default filters hide. The result-model tests move to the ml app, and a leftover `is_current` comment in the UI type is removed. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01APWsYj7AAYL5aiq7T3HUwq --- ami/main/api/serializers.py | 23 +-- ami/main/api/views.py | 3 +- ami/main/models_future/history.py | 18 +- ami/main/models_future/references.py | 8 +- ami/main/test_occurrence_history.py | 139 +--------------- ami/ml/models/algorithm_result.py | 24 ++- ami/ml/post_processing/class_masking.py | 1 - ami/ml/post_processing/small_size_filter.py | 1 - ami/ml/results/schemas.py | 29 ++-- ami/ml/results/tests.py | 154 +++++++++++++++--- ami/ml/results/writer.py | 5 - config/settings/base.py | 2 - .../models/occurrence-history.test.ts | 1 - .../models/occurrence-history.ts | 2 - 14 files changed, 180 insertions(+), 230 deletions(-) diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index 0e6835377..7e38f0790 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -15,7 +15,6 @@ from ami.jobs.models import Job from ami.main.models import Tag from ami.ml.models import Algorithm, Pipeline -from ami.ml.results.schemas import data_json_schema, result_kind_title, result_kinds from ami.ml.serializers import AlgorithmSerializer, PipelineNestedSerializer from ami.users.models import User from ami.users.roles import ProjectManager @@ -2234,9 +2233,6 @@ class AlgorithmResultEntrySerializer(HistoryEntryBaseSerializer): allow_null=True, help_text="The kind's headline figure, for sorting and filtering; not a confidence." ) data = serializers.JSONField(help_text="The kind's figures, validated against its data model.") - data_references = serializers.DictField( - child=RefSerializer(), help_text="Data fields that name another record, by field." - ) determination_before = HistoryTaxonSerializer(allow_null=True) determination_after = HistoryTaxonSerializer(allow_null=True) classifications = CreatedClassificationSerializer( @@ -2244,23 +2240,12 @@ class AlgorithmResultEntrySerializer(HistoryEntryBaseSerializer): ) -def _result_entry_component(kind: str) -> type[serializers.Serializer]: - """An OpenAPI-only serializer for one kind: ``kind`` is a literal and ``data`` is the kind's schema.""" - name = result_kind_title(kind) - data_field = extend_schema_field(data_json_schema(kind))(type(f"{name}DataField", (serializers.JSONField,), {})) - return type( - f"{name}ResultEntrySerializer", - (AlgorithmResultEntrySerializer,), - {"kind": serializers.ChoiceField(choices=[kind]), "data": data_field()}, - ) - - -# The history endpoint's response: a plain oneOf with literal ``type`` and ``kind`` fields, one -# component per result kind, so a client generated from the schema can narrow ``data`` by kind. +# The history endpoint's response: a plain oneOf with a literal ``type`` field. A result's ``data`` is +# published as JSON; the server validates it against its kind's model, and the UI types it by kind. +# Publishing one typed component per kind waits for a client generated from the schema (#1482). OCCURRENCE_HISTORY_ENTRY_SCHEMA = PolymorphicProxySerializer( component_name="OccurrenceHistoryEntry", - serializers=[IdentificationEntrySerializer, PredictionEntrySerializer] - + [_result_entry_component(kind) for kind in result_kinds()], + serializers=[IdentificationEntrySerializer, PredictionEntrySerializer, AlgorithmResultEntrySerializer], resource_type_field_name=None, many=True, ) diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 52bac0823..7d7e6ec86 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -1590,7 +1590,8 @@ def history(self, request: Request, pk=None) -> Response: """Everything that happened to this occurrence, newest first. Merges post-processing results with the classifications they created, identifications - and predictions into one list. Visible to whoever can open the occurrence itself. + and predictions into one list. Available to members of the occurrence's project, including + for an occurrence that the project's default filters hide from lists and the detail view. """ occurrence = self.get_object() entries = occurrence_timeline(occurrence) diff --git a/ami/main/models_future/history.py b/ami/main/models_future/history.py index 4fad40f53..6e564a044 100644 --- a/ami/main/models_future/history.py +++ b/ami/main/models_future/history.py @@ -16,7 +16,7 @@ from ami.main.models_future.references import Ref, is_record_id, resolve_references from ami.ml.models import Algorithm, AlgorithmResult from ami.ml.post_processing.registry import get_postprocessing_task -from ami.ml.results.schemas import field_references, field_titles, reference_fields, result_kinds +from ami.ml.results.schemas import field_references, field_titles if typing.TYPE_CHECKING: from ami.jobs.models import Job @@ -61,7 +61,6 @@ class OccurrenceTimelineEntry: kind: str | None = None value: float | None = None # the kind's headline figure, not a confidence data: dict = dataclasses.field(default_factory=dict) # validated per kind - data_references: dict[str, Ref] = dataclasses.field(default_factory=dict) # data field -> Ref determination_before: Taxon | None = None determination_after: Taxon | None = None classifications: list[CreatedClassification] = dataclasses.field(default_factory=list) @@ -175,21 +174,9 @@ def superseded_by(prediction: Classification) -> int | None: def _fill_job_settings_and_references(entries: list[OccurrenceTimelineEntry]) -> None: - """Fill each entry's job settings and data references, resolving every id the entries name at once.""" + """Fill each entry's job settings, resolving every record id the settings name at once.""" settings = {id(entry): _job_settings(entry.job) for entry in entries if entry.job is not None} - # A stored kind may no longer be registered (renamed, or written by another branch); it names no references. - kinds = set(result_kinds()) - data_refs = { - id(entry): [ - (field, ref_type, entry.data[field]) - for field, ref_type in (reference_fields(entry.kind) if entry.kind in kinds else {}).items() - if is_record_id(entry.data.get(field)) - ] - for entry in entries - if entry.type == "algorithm_result" - } wanted = [(ref_type, value) for specs in settings.values() for _, _, value, ref_type in specs if ref_type] - wanted += [(ref_type, ref_id) for refs in data_refs.values() for _, ref_type, ref_id in refs] resolved = resolve_references(wanted) if wanted else {} for entry in entries: if entry.job is not None: @@ -197,7 +184,6 @@ def _fill_job_settings_and_references(entries: list[OccurrenceTimelineEntry]) -> JobSetting(key, label, value, resolved[(ref_type, value)] if ref_type else None) for key, label, value, ref_type in settings[id(entry)] ] - entry.data_references = {field: resolved[(t, i)] for field, t, i in data_refs.get(id(entry), [])} def job_config(job: Job | None) -> dict | None: diff --git a/ami/main/models_future/references.py b/ami/main/models_future/references.py index 8300329c7..929409219 100644 --- a/ami/main/models_future/references.py +++ b/ami/main/models_future/references.py @@ -36,16 +36,16 @@ def is_record_id(value: typing.Any) -> bool: def unmapped_reference_types(models: typing.Iterable[type] | None = None) -> set[str]: - """Reference types that result data models or task config schemas declare but this module does not map. + """Reference types that task config schemas declare but this module does not map. - Defaults to every registered result kind and post-processing task; a test keeps the answer empty. + Defaults to every registered post-processing task; a test keeps the answer empty. """ - from ami.ml.results.schemas import ALGORITHM_RESULT_DATA_MODELS, field_references + from ami.ml.results.schemas import field_references if models is None: from ami.ml.post_processing.registry import POSTPROCESSING_TASKS - models = [*ALGORITHM_RESULT_DATA_MODELS, *(task.config_schema for task in POSTPROCESSING_TASKS.values())] + models = [task.config_schema for task in POSTPROCESSING_TASKS.values()] return {ref_type for model in models for ref_type in field_references(model).values()} - set(REFERENCE_TYPES) diff --git a/ami/main/test_occurrence_history.py b/ami/main/test_occurrence_history.py index 605b52312..39fe3b6ad 100644 --- a/ami/main/test_occurrence_history.py +++ b/ami/main/test_occurrence_history.py @@ -49,114 +49,6 @@ def no_query_cache(): context.__exit__(None, None, None) -class AlgorithmResultTestCase(TestCase): - """A result's data fits its kind, its project is its occurrence's, and a new result replaces the current one.""" - - @classmethod - def setUpTestData(cls) -> None: - cls.project, cls.deployment = setup_test_project(reuse=False) - cls.occurrence = Occurrence.objects.create(project=cls.project, deployment=cls.deployment) - cls.algorithm = Algorithm.objects.create(name="Size filter", key="size-filter-result-test") - - def _size_filter(self, data: dict | None, **fields) -> AlgorithmResult: - fields.setdefault("occurrence", self.occurrence) - return AlgorithmResult(algorithm=self.algorithm, kind=SIZE_FILTER, data=data, **fields) - - def test_data_that_does_not_fit_its_kind_is_refused_on_every_write_path(self): - for label, data in ( - ("missing field", {}), - ("wrong type", {"relative_size": "small"}), - ("unknown field", {"relative_size": 0.001, "note": "x"}), - ("extra that is not JSON", {"relative_size": 0.001, "extra": {"when": object()}}), - ): - with self.subTest(label): - with self.assertRaises(ValueError): - self._size_filter(data).save() - with self.assertRaises(ValueError): - AlgorithmResult.objects.record_many([self._size_filter(data)]) - with self.assertRaises(ValueError): - AlgorithmResult.objects.record( - occurrence=self.occurrence, algorithm=self.algorithm, kind="unregistered", data={} - ) - self.assertFalse(AlgorithmResult.objects.exists()) - - result = AlgorithmResult.objects.record( - occurrence=self.occurrence, - algorithm=self.algorithm, - kind=SIZE_FILTER, - value=0.001, - data={"relative_size": 0.001, "extra": {"service": "size-filter 1.2"}}, - ) - self.assertEqual(result.value, 0.001) - self.assertEqual(result.data["extra"], {"service": "size-filter 1.2"}) - result.data["relative_size"] = "small" - with self.assertRaises(ValueError): - AlgorithmResult.objects.bulk_update([result], ["data"]) - - def test_each_run_adds_a_result_beside_the_earlier_ones(self): - first = AlgorithmResult.objects.record( - occurrence=self.occurrence, - algorithm=self.algorithm, - kind=SIZE_FILTER, - data={"relative_size": 0.01}, - ) - other_occurrence = Occurrence.objects.create(project=self.project, deployment=self.deployment) - second, untouched = AlgorithmResult.objects.record_many( - [ - self._size_filter({"relative_size": 0.02}), - self._size_filter({"relative_size": 0.02}, occurrence=other_occurrence), - ] - ) - self.assertEqual(set(self.occurrence.algorithm_results.values_list("pk", flat=True)), {first.pk, second.pk}) - self.assertEqual(list(other_occurrence.algorithm_results.values_list("pk", flat=True)), [untouched.pk]) - - def test_merging_moves_every_result_to_the_kept_occurrence(self): - other_algorithm = Algorithm.objects.create(name="Second size filter", key="size-filter-merge-test") - absorbed = [Occurrence.objects.create(project=self.project, deployment=self.deployment) for _ in range(3)] - - def record(occurrence, algorithm, size): - return AlgorithmResult.objects.record( - occurrence=occurrence, algorithm=algorithm, kind=SIZE_FILTER, data={"relative_size": size} - ) - - kept_own = record(self.occurrence, self.algorithm, 0.01) - moved_results = [ - record(absorbed[0], self.algorithm, 0.02), - record(absorbed[0], self.algorithm, 0.03), - record(absorbed[1], other_algorithm, 0.04), - record(absorbed[2], other_algorithm, 0.05), - ] - - moved = AlgorithmResult.objects.move_to_occurrence(self.occurrence, [o.pk for o in absorbed]) - - self.assertEqual(moved, 4) - self.assertEqual( - set(AlgorithmResult.objects.filter(occurrence=self.occurrence).values_list("pk", flat=True)), - {kept_own.pk, *(r.pk for r in moved_results)}, - ) - - def test_project_comes_from_the_occurrence_and_a_result_without_one_is_skipped(self): - result = self._size_filter({"relative_size": 0.01}) - result.save() - self.assertEqual(result.project_id, self.project.pk) - - orphan = Occurrence.objects.create(project=None, deployment=self.deployment) - with self.assertLogs("ami.ml.models.algorithm_result", level="WARNING"): - written = AlgorithmResult.objects.record_many( - [ - self._size_filter({"relative_size": 0.02}), - self._size_filter({"relative_size": 0.03}, occurrence=orphan), - ] - ) - # The occurrence with a project still gets its result; the other is skipped, not fatal. - self.assertEqual([r.occurrence_id for r in written], [self.occurrence.pk]) - with self.assertRaises(ValueError): - self._size_filter({"relative_size": 0.03}, occurrence=orphan).save() - - def test_the_model_lives_in_the_ml_app(self): - self.assertEqual(AlgorithmResult._meta.app_label, "ml") - - class ReferenceTestCase(TestCase): """Ids shown in the history become {type, id, name}; a deleted row keeps its id with no name.""" @@ -185,33 +77,16 @@ class Probe(AlgorithmResultData): class OccurrenceHistorySchemaTestCase(TestCase): - """The OpenAPI schema publishes one history-entry component per result kind, typed by its data model.""" + """The OpenAPI schema publishes the history as a oneOf of identification, prediction and result entries.""" - def test_every_result_kind_has_a_typed_component(self): + def test_the_history_entries_are_a_oneof_tagged_by_type(self): from drf_spectacular.generators import SchemaGenerator - from ami.ml.results.schemas import result_kinds - components = SchemaGenerator(api_version="api").get_schema(request=None, public=True)["components"]["schemas"] - - def resolve(schema: dict) -> dict: - # drf-spectacular moves every enum into its own named component. - return components[schema["$ref"].rsplit("/", 1)[-1]] if "$ref" in schema else schema - - entry_names = [ref["$ref"].rsplit("/", 1)[-1] for ref in components["OccurrenceHistoryEntry"]["oneOf"]] - for kind in result_kinds(): - name = "".join(part.title() for part in kind.split("_")) + "ResultEntry" - with self.subTest(kind): - self.assertIn(name, entry_names) - props = components[name]["properties"] - self.assertEqual(resolve(props["kind"])["enum"], [kind]) - self.assertEqual(resolve(props["type"])["enum"], ["algorithm_result"]) - self.assertIn("extra", props["data"]["properties"]) - self.assertIn("IdentificationEntry", entry_names) - self.assertIn("PredictionEntry", entry_names) - self.assertIn( - "excluded_probability", components["ClassMaskingResultEntry"]["properties"]["data"]["properties"] - ) + entry_names = {ref["$ref"].rsplit("/", 1)[-1] for ref in components["OccurrenceHistoryEntry"]["oneOf"]} + self.assertEqual(entry_names, {"IdentificationEntry", "PredictionEntry", "AlgorithmResultEntry"}) + # Named enums, not drf-spectacular's hashed fallback names. + self.assertIn("AlgorithmResultEntryTypeEnum", components) class OccurrenceFixtureTestCase(APITestCase): @@ -337,7 +212,6 @@ def test_entries_are_merged_newest_first_and_a_result_carries_the_classification self.assertIsNone(result["score"]) self.assertEqual(result["value"], 0.001) self.assertEqual(result["data"]["relative_size"], 0.001) - self.assertEqual(result["data_references"], {}) created = Classification.objects.get(algorithm=self.size_filter) self.assertEqual( result["classifications"], @@ -690,7 +564,6 @@ def test_a_result_of_a_kind_no_longer_registered_still_shows_in_the_history(self AlgorithmResult.objects.filter(pk=result.pk).update(kind="tracking") entry = next(e for e in self.get() if e["type"] == "algorithm_result") self.assertEqual(entry["kind"], "tracking") - self.assertEqual(entry["data_references"], {}) class ClassificationAdminTestCase(OccurrenceFixtureTestCase): diff --git a/ami/ml/models/algorithm_result.py b/ami/ml/models/algorithm_result.py index 41c822350..407545d9e 100644 --- a/ami/ml/models/algorithm_result.py +++ b/ami/ml/models/algorithm_result.py @@ -6,7 +6,7 @@ from django.utils import timezone from ami.base.models import BaseModel, BaseQuerySet -from ami.ml.results.schemas import validate_result_data +from ami.ml.results.schemas import result_value, validate_result_data if typing.TYPE_CHECKING: from ami.main.models import Occurrence @@ -18,14 +18,15 @@ class AlgorithmResultQuerySet(BaseQuerySet): def bulk_create(self, objs, *args, **kwargs): objs = list(objs) for obj in objs: - obj.data = validate_result_data(obj.kind, obj.data) + obj.fill_from_data() return super().bulk_create(objs, *args, **kwargs) def bulk_update(self, objs, fields, *args, **kwargs): objs = list(objs) if "data" in fields: for obj in objs: - obj.data = validate_result_data(obj.kind, obj.data) + obj.fill_from_data() + fields = [*fields, "value"] if "value" not in fields else fields return super().bulk_update(objs, fields, *args, **kwargs) def record(self, **fields) -> "AlgorithmResult": @@ -60,11 +61,15 @@ def move_to_occurrence(self, kept: "Occurrence", absorbed_ids: typing.Iterable[i """Move the results of occurrences merged into ``kept``, so their history follows the merge. Call it before deleting the absorbed occurrences, whose results would otherwise be deleted - with them. Returns the number of results moved. + with them. Occurrences of another project are refused, so a result never ends up filed under + a project other than its occurrence's. Returns the number of results moved. """ absorbed_ids = set(absorbed_ids) - {kept.pk} if not absorbed_ids: return 0 + occurrences = self.model._meta.get_field("occurrence").related_model.objects + if occurrences.filter(pk__in=absorbed_ids).exclude(project_id=kept.project_id).exists(): + raise ValueError(f"Cannot move results into Occurrence #{kept.pk} from occurrences of another project.") return self.filter(occurrence_id__in=absorbed_ids).update(occurrence=kept) @@ -100,8 +105,8 @@ class AlgorithmResult(BaseModel): # No ``choices``: the registry of data models in ami/ml/results/schemas.py is the list of kinds, # and every write path rejects a kind without one, so a new kind needs no migration. kind = models.CharField(max_length=32) - # The kind's headline figure, for filtering and sorting: the share of probability outside - # the list for class masking, the relative size for the size filter. + # The kind's headline figure, for filtering and sorting, copied from the data field its model + # names in ``value_field`` on every write. value = models.FloatField(null=True, blank=True) data = models.JSONField(default=dict, blank=True) timestamp = models.DateTimeField(default=timezone.now) @@ -123,8 +128,13 @@ class Meta: def __str__(self) -> str: return f"#{self.pk} {self.kind} for Occurrence #{self.occurrence_id} from Algorithm #{self.algorithm_id}" - def save(self, *args, **kwargs): + def fill_from_data(self) -> None: + """Validate ``data`` against the kind's model and set ``value`` to the figure the kind names.""" self.data = validate_result_data(self.kind, self.data) + self.value = result_value(self.kind, self.data) + + def save(self, *args, **kwargs): + self.fill_from_data() if self.project_id is None: self.project_id = self.occurrence.project_id if self.project_id is None: diff --git a/ami/ml/post_processing/class_masking.py b/ami/ml/post_processing/class_masking.py index 84ef2a187..40439d82c 100644 --- a/ami/ml/post_processing/class_masking.py +++ b/ami/ml/post_processing/class_masking.py @@ -135,7 +135,6 @@ def make_classifications_filtered_by_taxa_list( kind=ClassMaskingResultData.kind, algorithm=new_algorithm, job=job, - value_field="excluded_probability", timestamp=timestamp, ) diff --git a/ami/ml/post_processing/small_size_filter.py b/ami/ml/post_processing/small_size_filter.py index 824ec5688..c806bf50c 100644 --- a/ami/ml/post_processing/small_size_filter.py +++ b/ami/ml/post_processing/small_size_filter.py @@ -99,7 +99,6 @@ def run(self) -> None: kind=SizeFilterResultData.kind, algorithm=self.algorithm, job=self.job, - value_field="relative_size", ) def flush(i: int) -> None: diff --git a/ami/ml/results/schemas.py b/ami/ml/results/schemas.py index cc34e89b3..f4be31b83 100644 --- a/ami/ml/results/schemas.py +++ b/ami/ml/results/schemas.py @@ -16,9 +16,9 @@ def reference(ref_type: str, default: Any = None, **field_options: Any) -> Any: - """Declare a field that holds another record's id; the history resolves it to ``{type, id, name}``. + """Declare a settings field that holds another record's id; the history resolves it to ``{type, id, name}``. - Used by result data models and by task config schemas; ``field_options`` go to ``pydantic.Field``. + Used by post-processing task config schemas; ``field_options`` go to ``pydantic.Field``. """ return pydantic.Field(default, reference=ref_type, **field_options) @@ -26,6 +26,8 @@ def reference(ref_type: str, default: Any = None, **field_options: Any) -> Any: class AlgorithmResultData(pydantic.BaseModel): # The kind this model validates; each concrete model sets it, and the registry is keyed by it. kind: ClassVar[str] + # The data field the result's ``value`` repeats, for filtering and sorting; None for no value. + value_field: ClassVar[str | None] = None # Stored, shown and exported only, never read for logic; a value a feature needs becomes a typed field. extra: dict[str, Any] = {} @@ -48,6 +50,7 @@ class ClassMaskingResultData(DeterminationSnapshot): """Figures from the occurrence's winning detection: the one whose masked classification scores highest.""" kind: ClassVar[str] = "class_masking" + value_field: ClassVar[str | None] = "excluded_probability" # One minus the kept mass of the unmasked softmax: what the list removed, not an out-of-distribution score. excluded_probability: float @@ -59,6 +62,7 @@ class SizeFilterResultData(DeterminationSnapshot): """Figures from the occurrence's smallest filtered detection.""" kind: ClassVar[str] = "size_filter" + value_field: ClassVar[str | None] = "relative_size" # The detection's box area as a fraction of its image. relative_size: float @@ -75,11 +79,6 @@ def result_kinds() -> list[str]: return list(ALGORITHM_RESULT_DATA_SCHEMAS) -def result_kind_title(kind: str) -> str: - """The kind in CamelCase, for OpenAPI component names: ``class_masking`` -> ``ClassMasking``.""" - return "".join(part.title() for part in kind.split("_")) - - def _schema_for(kind: str) -> type[AlgorithmResultData]: schema = ALGORITHM_RESULT_DATA_SCHEMAS.get(kind) if schema is None: @@ -97,6 +96,12 @@ def validate_result_data(kind: str, data: dict | AlgorithmResultData | None) -> return json.loads(schema.parse_obj(values).json()) +def result_value(kind: str, data: dict) -> float | None: + """The figure the kind's ``value_field`` names in validated ``data``, which a result stores as ``value``.""" + field = _schema_for(kind).value_field + return data.get(field) if field else None + + def field_references(model: type[pydantic.BaseModel]) -> dict[str, str]: """The model's fields declared with ``reference()``, mapped to the reference type.""" return { @@ -109,13 +114,3 @@ def field_references(model: type[pydantic.BaseModel]) -> dict[str, str]: def field_titles(model: type[pydantic.BaseModel]) -> dict[str, str | None]: """Every field of the model, in declaration order, mapped to its ``title`` or None.""" return {name: field.field_info.title for name, field in model.__fields__.items()} - - -def reference_fields(kind: str) -> dict[str, str]: - """The kind's data fields that hold another record's id, mapped to the reference type.""" - return field_references(_schema_for(kind)) - - -def data_json_schema(kind: str) -> dict: - """The kind's data model as a JSON schema, for its OpenAPI component.""" - return _schema_for(kind).schema() diff --git a/ami/ml/results/tests.py b/ami/ml/results/tests.py index f8ebdcd47..6bf624b6b 100644 --- a/ami/ml/results/tests.py +++ b/ami/ml/results/tests.py @@ -1,21 +1,17 @@ -from django.test import SimpleTestCase +from django.test import SimpleTestCase, TestCase +from ami.main.models import Occurrence +from ami.ml.models import Algorithm, AlgorithmResult from ami.ml.post_processing.registry import POSTPROCESSING_TASKS from ami.ml.results import schemas +from ami.ml.results.schemas import SizeFilterResultData +from ami.tests.fixtures.main import setup_test_project +SIZE_FILTER = SizeFilterResultData.kind -class ResultKindRegistryTest(SimpleTestCase): - """Each registered kind publishes its data schema and declares which fields reference other records.""" - def test_every_kind_has_a_closed_json_schema(self): - for kind in schemas.result_kinds(): - with self.subTest(kind): - schema = schemas.data_json_schema(kind) - self.assertEqual(schema["type"], "object") - self.assertFalse(schema.get("additionalProperties", True)) - self.assertIn("extra", schema["properties"]) - # A nested model would emit #/definitions refs, which do not resolve inside OpenAPI. - self.assertNotIn("definitions", schema) +class ResultKindRegistryTest(SimpleTestCase): + """Each registered kind is keyed by its model's kind, and tasks declare only registered kinds.""" def test_every_result_model_a_task_declares_is_a_registered_kind(self): for key, task in POSTPROCESSING_TASKS.items(): @@ -28,13 +24,129 @@ def test_each_kind_is_registered_under_its_models_kind(self): with self.subTest(kind): self.assertEqual(model.kind, kind) - def test_reference_fields_come_from_field_metadata(self): - class Probe(schemas.AlgorithmResultData): - merged_into_id: int | None = schemas.reference("occurrence") - schemas.ALGORITHM_RESULT_DATA_SCHEMAS["probe"] = Probe - try: - self.assertEqual(schemas.reference_fields("probe"), {"merged_into_id": "occurrence"}) - self.assertEqual(schemas.reference_fields("class_masking"), {}) - finally: - del schemas.ALGORITHM_RESULT_DATA_SCHEMAS["probe"] +class AlgorithmResultTestCase(TestCase): + """A result's data fits its kind, its value follows its data, and its project is its occurrence's.""" + + @classmethod + def setUpTestData(cls) -> None: + cls.project, cls.deployment = setup_test_project(reuse=False) + cls.occurrence = Occurrence.objects.create(project=cls.project, deployment=cls.deployment) + cls.algorithm = Algorithm.objects.create(name="Size filter", key="size-filter-result-test") + + def _size_filter(self, data: dict | None, **fields) -> AlgorithmResult: + fields.setdefault("occurrence", self.occurrence) + return AlgorithmResult(algorithm=self.algorithm, kind=SIZE_FILTER, data=data, **fields) + + def test_data_that_does_not_fit_its_kind_is_refused_on_every_write_path(self): + for label, data in ( + ("missing field", {}), + ("wrong type", {"relative_size": "small"}), + ("unknown field", {"relative_size": 0.001, "note": "x"}), + ("extra that is not JSON", {"relative_size": 0.001, "extra": {"when": object()}}), + ): + with self.subTest(label): + with self.assertRaises(ValueError): + self._size_filter(data).save() + with self.assertRaises(ValueError): + AlgorithmResult.objects.record_many([self._size_filter(data)]) + with self.assertRaises(ValueError): + AlgorithmResult.objects.record( + occurrence=self.occurrence, algorithm=self.algorithm, kind="unregistered", data={} + ) + self.assertFalse(AlgorithmResult.objects.exists()) + + result = AlgorithmResult.objects.record( + occurrence=self.occurrence, + algorithm=self.algorithm, + kind=SIZE_FILTER, + value=0.001, + data={"relative_size": 0.001, "extra": {"service": "size-filter 1.2"}}, + ) + self.assertEqual(result.value, 0.001) + self.assertEqual(result.data["extra"], {"service": "size-filter 1.2"}) + result.data["relative_size"] = "small" + with self.assertRaises(ValueError): + AlgorithmResult.objects.bulk_update([result], ["data"]) + + def test_each_run_adds_a_result_beside_the_earlier_ones(self): + first = AlgorithmResult.objects.record( + occurrence=self.occurrence, + algorithm=self.algorithm, + kind=SIZE_FILTER, + data={"relative_size": 0.01}, + ) + other_occurrence = Occurrence.objects.create(project=self.project, deployment=self.deployment) + second, untouched = AlgorithmResult.objects.record_many( + [ + self._size_filter({"relative_size": 0.02}), + self._size_filter({"relative_size": 0.02}, occurrence=other_occurrence), + ] + ) + self.assertEqual(set(self.occurrence.algorithm_results.values_list("pk", flat=True)), {first.pk, second.pk}) + self.assertEqual(list(other_occurrence.algorithm_results.values_list("pk", flat=True)), [untouched.pk]) + + def test_merging_moves_every_result_to_the_kept_occurrence(self): + other_algorithm = Algorithm.objects.create(name="Second size filter", key="size-filter-merge-test") + absorbed = [Occurrence.objects.create(project=self.project, deployment=self.deployment) for _ in range(3)] + + def record(occurrence, algorithm, size): + return AlgorithmResult.objects.record( + occurrence=occurrence, algorithm=algorithm, kind=SIZE_FILTER, data={"relative_size": size} + ) + + kept_own = record(self.occurrence, self.algorithm, 0.01) + moved_results = [ + record(absorbed[0], self.algorithm, 0.02), + record(absorbed[0], self.algorithm, 0.03), + record(absorbed[1], other_algorithm, 0.04), + record(absorbed[2], other_algorithm, 0.05), + ] + + moved = AlgorithmResult.objects.move_to_occurrence(self.occurrence, [o.pk for o in absorbed]) + + self.assertEqual(moved, 4) + self.assertEqual( + set(AlgorithmResult.objects.filter(occurrence=self.occurrence).values_list("pk", flat=True)), + {kept_own.pk, *(r.pk for r in moved_results)}, + ) + + def test_project_comes_from_the_occurrence_and_a_result_without_one_is_skipped(self): + result = self._size_filter({"relative_size": 0.01}) + result.save() + self.assertEqual(result.project_id, self.project.pk) + + orphan = Occurrence.objects.create(project=None, deployment=self.deployment) + with self.assertLogs("ami.ml.models.algorithm_result", level="WARNING"): + written = AlgorithmResult.objects.record_many( + [ + self._size_filter({"relative_size": 0.02}), + self._size_filter({"relative_size": 0.03}, occurrence=orphan), + ] + ) + # The occurrence with a project still gets its result; the other is skipped, not fatal. + self.assertEqual([r.occurrence_id for r in written], [self.occurrence.pk]) + with self.assertRaises(ValueError): + self._size_filter({"relative_size": 0.03}, occurrence=orphan).save() + + def test_value_is_the_figure_the_kind_names_on_every_write_path(self): + saved = self._size_filter({"relative_size": 0.004}, value=0.9) + saved.save() + [created] = AlgorithmResult.objects.bulk_create( + [self._size_filter({"relative_size": 0.005}, value=None, project=self.project)] + ) + created.data = {"relative_size": 0.006} + AlgorithmResult.objects.bulk_update([created], ["data"]) + created.refresh_from_db() + self.assertEqual((saved.value, created.value), (0.004, 0.006)) + + def test_results_cannot_be_moved_from_an_occurrence_of_another_project(self): + other_project, other_deployment = setup_test_project(reuse=False) + absorbed = Occurrence.objects.create(project=other_project, deployment=other_deployment) + result = AlgorithmResult.objects.record( + occurrence=absorbed, algorithm=self.algorithm, kind=SIZE_FILTER, data={"relative_size": 0.01} + ) + with self.assertRaises(ValueError): + AlgorithmResult.objects.move_to_occurrence(self.occurrence, [absorbed.pk]) + result.refresh_from_db() + self.assertEqual((result.occurrence_id, result.project_id), (absorbed.pk, other_project.pk)) diff --git a/ami/ml/results/writer.py b/ami/ml/results/writer.py index 4732cdd6e..4a05b0eca 100644 --- a/ami/ml/results/writer.py +++ b/ami/ml/results/writer.py @@ -29,14 +29,11 @@ def __init__( kind: str, algorithm: Algorithm, job: Job | None, - value_field: str, timestamp: datetime.datetime | None = None, ): - """``value_field`` names the figure in the result's data that is also stored in ``value``.""" self.kind = kind self.algorithm = algorithm self.job = job - self.value_field = value_field self.timestamp = timestamp self.results: dict[int, AlgorithmResult] = {} # Occurrences with no project; tried once, then left alone. @@ -66,7 +63,6 @@ def start_batch(self, occurrences: typing.Iterable[Occurrence], classifications: algorithm=self.algorithm, job=self.job, kind=self.kind, - value=self.figures[occurrence.pk][1][self.value_field], data={**self.figures[occurrence.pk][1], "determination_before_id": occurrence.determination_id}, **({"timestamp": self.timestamp} if self.timestamp else {}), ) @@ -97,7 +93,6 @@ def finish_batch(self, occurrences: typing.Iterable[Occurrence]) -> None: continue figures = self.figures[occurrence.pk][1] result.data = {**result.data, **figures, "determination_after_id": occurrence.determination_id} - result.value = figures[self.value_field] touched.append(result) if touched: AlgorithmResult.objects.bulk_update(touched, ["data", "value"]) diff --git a/config/settings/base.py b/config/settings/base.py index b4570c5b9..e054af422 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -10,7 +10,6 @@ import environ # A leaf module (pydantic only, no Django imports), so settings can read the result kinds. -from ami.ml.results.schemas import result_kind_title, result_kinds # Monkeypatching Django, so stubs will work for all generics, # see: https://github.com/typeddjango/django-stubs @@ -563,7 +562,6 @@ def _celery_result_backend_url(redis_url): "AlgorithmResultEntryTypeEnum": ["algorithm_result"], "IdentificationEntryTypeEnum": ["identification"], "PredictionEntryTypeEnum": ["prediction"], - **{f"{result_kind_title(kind)}ResultKindEnum": [kind] for kind in result_kinds()}, }, } # Your stuff... diff --git a/ui/src/data-services/models/occurrence-history.test.ts b/ui/src/data-services/models/occurrence-history.test.ts index 4b67af04f..be8ba5440 100644 --- a/ui/src/data-services/models/occurrence-history.test.ts +++ b/ui/src/data-services/models/occurrence-history.test.ts @@ -101,7 +101,6 @@ const classMasking: ServerOccurrenceHistoryEntry = { extra: {}, new_winner_original_rank: 2, }, - data_references: {}, determination_after: NOCTUA, determination_before: XESTIA, id: 5, diff --git a/ui/src/data-services/models/occurrence-history.ts b/ui/src/data-services/models/occurrence-history.ts index e46b961dd..45d7fc660 100644 --- a/ui/src/data-services/models/occurrence-history.ts +++ b/ui/src/data-services/models/occurrence-history.ts @@ -115,10 +115,8 @@ interface ServerResultEntry classifications: ServerCreatedClassification[] data: Data /** Data fields that name another record, by field. */ - data_references: Record determination_after: ServerHistoryTaxon | null determination_before: ServerHistoryTaxon | null - /** Whether it is the latest of its kind, not replaced by a later run. */ kind: Kind type: 'algorithm_result' /** The kind's headline figure, for sorting and filtering; not a confidence. */ From 46d3db87d9522b0a3b78c39f58bf7524ae63116d Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Tue, 6 Oct 2026 18:00:24 -0700 Subject: [PATCH 111/134] docs(results): cut the algorithm results guide to the steps and rules The guide restated much of the PR description and sketched features that belong to later PRs. It now covers what a result is, the five steps to add a kind, and the rules a new method must follow, in about fifty lines. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01APWsYj7AAYL5aiq7T3HUwq --- ami/ml/results/README.md | 302 ++++++--------------------------------- 1 file changed, 47 insertions(+), 255 deletions(-) diff --git a/ami/ml/results/README.md b/ami/ml/results/README.md index 16cb90087..91e9eec64 100644 --- a/ami/ml/results/README.md +++ b/ami/ml/results/README.md @@ -1,260 +1,52 @@ # Algorithm results -An algorithm result (`ami/ml/models/algorithm_result.py`) is the standard record of what an -algorithm decided about one occurrence: the figures only that run knew, its headline `value`, and the -occurrence's determination before and after, with the algorithm and job that produced it. Results are -written by methods that run inside Antenna once the occurrence exists, which today means post-processing -tasks. The determination never reads them: it still comes only from identifications and classifications. - -Each kind of result (`class_masking`, `size_filter`, and next `tracking` and `rank_rollup`) has its -own data model in `schemas.py` in this package. The occurrence history -(`ami/main/models_future/history.py`, `GET /occurrences/{id}/history/`) shows each result next to the -occurrence's identifications and predictions, and the API publishes each kind's data as a typed -OpenAPI component. - -Outputs that belong to a single detection keep their own tables: classifications, and feature vectors -(`DetectionEmbedding`). They carry the same provenance (algorithm and job) but are not results. - -This guide covers adding a result kind, using rank roll-up (draft #1361) as the example, and then -adding another kind of history entry. Adding a kind needs no migration, no serializer change and no -OpenAPI change: it touches the data model, the task that writes it, and the UI card. - -## How results fit the post-processing framework - -The post-processing framework (`ami/ml/post_processing/`) runs a method over data Antenna already has. -Algorithm results are its output side. - -- **Input:** a task subclasses `BasePostProcessingTask` (`base.py`) with a `key`, a `name` and a - pydantic `config_schema`, and is registered in `registry.py`. An admin action - (`admin/actions.py`, `make_post_processing_action`) creates a `post_processing` job whose params are - `{"task": , "config": {...}}`. `PostProcessingJob.run` validates the config against the schema, - stores the validated config (defaults included) back on the job, and runs the task with - `self.job` and `self.algorithm` (one `Algorithm` row per method; class masking makes one per source - classifier and species list). -- **Output:** what the task changes goes through the classifications it creates, each carrying its - `job`, its `algorithm_result` and, when it replaces one, `applied_to`. What it decided about each - occurrence goes into one algorithm result of the task's kind, written by `AlgorithmResultWriter` in - the same transaction as each batch. -- **Shown together:** the history shows each result with the job's settings, labelled, ordered and - linked from the same `config_schema`. One schema therefore describes a task's settings for - validation, the admin form, the stored job and the result card. +An algorithm result (`ami/ml/models/algorithm_result.py`) records what one run of a method decided +about one occurrence: the figures only that run knew, a headline `value`, and the occurrence's +determination before and after, with the algorithm and job that produced it. Post-processing tasks +write them today. The determination never reads them; it still comes only from identifications and +classifications. The occurrence history (`ami/main/models_future/history.py`, +`GET /occurrences/{id}/history/`) shows each result with the classifications its run created. Processing services cannot write results: their contract (`ami/ml/schemas.py`) has no place for them -and must not name occurrences, which Antenna creates. A stage that runs inside Antenna after occurrences -exist can, through `AlgorithmResultWriter` or `AlgorithmResult.objects.record_many`. - -A task declares the kinds it writes in `result_models`, next to `config_schema` for its settings. A kind -is not tied to one task: a task can write several kinds (one `AlgorithmResultWriter` per kind), and -several tasks can write the same kind and share its data model and card. The history labels settings -from the job's task, not from the kind. -One rule when a run writes several kinds: pass the classifications it creates to only one writer's -`start_batch`, because `start_batch` points each of them at that writer's result for its occurrence. - -### Which algorithm a run records under - -A run gets its own `Algorithm` only when it changes what can be output. Class masking makes one per -source classifier and species list, because the list changes which species the masked classifier can -name (its category map). Every other setting stays on the job: the size filter uses one algorithm -whatever its threshold. - -Results are only ever added. Running a method twice, for example the size filter with two thresholds, -leaves two results on the occurrence, one per job, and the history shows both; compare runs by their -jobs. A retried job reuses the results it already wrote. A list that needs each algorithm's latest value -on an occurrence takes the latest by timestamp when it reads. - -## Adding a result kind - -### 1. Describe the result's data - -Add a pydantic model in `schemas.py` and list it in `ALGORITHM_RESULT_DATA_MODELS`. - -```python -class RankRollupResultData(DeterminationSnapshot): - """Figures from the occurrence's winning detection: the one whose rolled-up classification scores highest.""" - - kind: ClassVar[str] = "rank_rollup" - - rolled_up_rank: str - # The summed probability at the new rank; the result's value. - rolled_up_score: float - # The threshold that rank cleared. - threshold: float - # Whether the rolled-up taxon is an ancestor of the source classifier's taxon. - within_lineage: bool - # The best summed score at each rank tried, e.g. {"SPECIES": 0.41, "GENUS": 0.72}. - rank_scores: dict[str, float] = {} - - -ALGORITHM_RESULT_DATA_MODELS = (ClassMaskingResultData, SizeFilterResultData, RankRollupResultData) -``` - -Rules for the model: - -- **Record only what the run alone knows.** Settings belong on the job and are shown from there. The - new taxon and score are on the classifications the run creates. The prediction a classification - replaced is its `applied_to`, which the history shows as `replaced`. Do not copy any of these into - `data`. -- **Subclass `DeterminationSnapshot`** if the run can change the determination. The writer fills - `determination_before_id` and `determination_after_id`. -- **Keep fields flat.** Use scalars, lists and string-keyed dicts of scalars. A nested model or an - `Enum` makes pydantic emit `#/definitions/...` references that do not resolve inside the OpenAPI - schema; `ami/ml/results/tests.py` fails if a kind does. -- **Declare ids of other records with `reference()`**, for example - `merged_into_id: int | None = reference("occurrence")`. The history returns such a field as - `{type, id, name}` under `data_references`, and the UI links it. The type must be a key of - `REFERENCE_TYPES` in `ami/main/models_future/references.py`. To add a type, add it there and add its - route to `ROUTES` in `ui/src/utils/references.ts`. A guard test fails when a declared type has no - mapping. -- **`extra`** (inherited) takes anything else the method returns. Nothing may read it for logic; when - a feature needs a value from it, make that value a typed field. -- **Pick one figure as the result's `value`:** the number lists sort and filter on (here - `rolled_up_score`). It is not necessarily a confidence; for tracking it is a motion figure. - -The kind's OpenAPI component (`RankRollupResultEntry`) and its enum name come from the registry -automatically. - -### 2. Describe the task's settings - -The task's config schema (the pydantic model `config_schema` on the task) is where the history gets -each setting's label, order and referenced record. Give every field a `title`, and declare ids with -`reference()`: - -```python -from ami.ml.results.schemas import reference - - -class RankRollupConfig(pydantic.BaseModel): - source_image_collection_id: int | None = reference("capture_set", None, title="Capture set") - occurrence_id: int | None = reference("occurrence", None, title="Occurrence") - thresholds: dict[str, float] = pydantic.Field(DEFAULT_THRESHOLDS, title="Thresholds by rank") - rollup_order: list[str] = pydantic.Field(DEFAULT_ORDER, title="Ranks to try, in order") -``` - -A field without a `title` shows its key on the result card. Declare the kinds the task writes with -`result_models = (RankRollupResultData,)`, and register the task in -`ami/ml/post_processing/registry.py`; the history finds the schema through the job's -`params["task"]`. - -### 3. Write results from the task - -Use `AlgorithmResultWriter` (`ami/ml/results/writer.py`), the same way the size filter and class -masking do. The writer creates one result per occurrence per run, in the same transaction as the -batch that changes the occurrence, so a batch never lands without its record. - -```python -results = AlgorithmResultWriter( - kind=RankRollupResultData.kind, - algorithm=self.algorithm, - job=self.job, - value_field="rolled_up_score", -) - -for classification in scope: # batched; see class_masking.py for the batching and progress pattern - ... - if rolled_up: - new = Classification( - detection=classification.detection, - taxon=winner, - score=summed_score, - terminal=True, - algorithm=self.algorithm, - applied_to=classification, # the prediction this one replaces - job=self.job, - timestamp=classification.timestamp, - ) - classification.terminal = False # demote only the classification this run replaced - to_create.append(new) - to_demote.append(classification) - occurrences.add(classification.detection.occurrence) - # The occurrence's result keeps the figures of its highest-ranked detection. - results.note(classification.detection.occurrence, figures, rank=summed_score) - -def flush(): - with transaction.atomic(): - results.start_batch(occurrences, to_create) # creates results; points the new rows at them - Classification.objects.bulk_create(to_create) - Classification.objects.bulk_update(to_demote, ["terminal"]) - for occurrence in occurrences: - occurrence.save(update_determination=True) - results.finish_batch(occurrences) # records the determination after the saves -``` - -Rules for the writer: - -- **Call `note()` before `start_batch()` for every occurrence in the batch.** `start_batch` reads the - noted figures, and the figures must contain `value_field`. -- **Set `applied_to`** on a classification that replaces another. The history then marks the replaced - prediction as superseded by this run and shows it as the result's original prediction. A - classification that replaces nothing (like the size filter's "Not identifiable") instead supersedes - terminal predictions it outranks on the same detection. -- **Demote only what the run replaced.** Demoting other classifiers' rows leaves predictions in the - history that look superseded with nothing saying why. -- **Set `job=self.job`** on every row the run creates. -- **Do not call `.distinct()`** on a classification queryset that includes the `scores` or `logits` - arrays; de-duplicating sorts the arrays (see #1376). -- **One result per occurrence per run.** The writer creates it with the first batch that touches the - occurrence and updates it after later batches; a retried job reuses it, and another run adds its own. +and must not name occurrences, which Antenna creates. Classifications and feature vectors keep their +own tables, with the same algorithm and job provenance. + +## Adding a kind + +1. **Data model** in `schemas.py`, listed in `ALGORITHM_RESULT_DATA_MODELS`. Set `kind` and + `value_field` (the figure lists filter and sort on; every write copies it into `value`). + Subclass `DeterminationSnapshot` if the run can change the determination. Record only what the + run alone knows: settings stay on the job, the new taxon on the classifications it creates, and + the prediction it replaced on their `applied_to`. Anything else the method returns goes in + `extra`, which nothing reads for logic. +2. **Task**: declare the model in the task's `result_models`, give every `config_schema` field a + `title`, and declare settings that hold a record id with `reference("capture_set", ...)`; the + history labels and links them. A new reference type goes in `REFERENCE_TYPES` + (`ami/main/models_future/references.py`) and `ui/src/utils/references.ts`. +3. **Writing**: use `AlgorithmResultWriter` (`writer.py`) as `class_masking.py` and + `small_size_filter.py` do. Per batch, inside one transaction: `note()` each changed detection's + figures, `start_batch()` before inserting classifications (it points them at their result), then + save the occurrences and call `finish_batch()`. +4. **UI**: add the data interface and the kind to `ui/src/data-services/models/occurrence-history.ts` + and a stats case to `ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx`. + The timeline skips kinds it does not know, so the server side can ship first. +5. **Tests**: one result per touched occurrence with the expected `data` and `value`; created + classifications carry `algorithm_result`, `job` and, where they replaced one, `applied_to`; the + history shows the result. `python manage.py test ami.ml.results ami.main.test_occurrence_history`. + +No migration, serializer or OpenAPI change is needed: `kind` is a plain `CharField`, every write path +validates `data` against the registry, and the API publishes `data` as JSON. + +## Rules + +- **Results are only added.** Each run adds one result per occurrence it touches; running a method + twice leaves two. A retried job reuses its own results. A list that needs an algorithm's latest + value takes the latest by timestamp. +- **A run gets its own `Algorithm` only when it changes what can be output.** Class masking makes one + per source classifier and species list, because the list changes which species can be named. Other + settings stay on the job: the size filter uses one algorithm whatever its threshold. +- **Demote only what the run replaced**, set `job` on every row it creates, and never call + `.distinct()` on a classification queryset that includes the `scores` or `logits` arrays (#1376). +- **Merges**: call `AlgorithmResult.objects.move_to_occurrence(kept, absorbed_ids)` before deleting + absorbed occurrences. Occurrences of another project are refused. - **Occurrences without a project** get no result and a warning; the run carries on (#1188). - -### 4. Show it in the UI - -- `ui/src/data-services/models/occurrence-history.ts`: add the data interface (mirroring the pydantic - model), the entry type (`ServerResultEntry<'rank_rollup', RankRollupResultData>`) to the - `AlgorithmResultEntry` union, and the kind to `ALGORITHM_RESULT_KINDS`. Until the kind is in that - list, the timeline skips its results, so the server side can ship first. -- `ui/src/pages/occurrence-details/identification-card/algorithm-result.tsx`: add an icon and label - to `KINDS`, and a `case` in the stats switch for the kind's own figures. The determination row, the - new prediction with its Confirm button, the original prediction (from `replaced`), the detections - changed, the settings and the job row are shared by every kind. -- `ui/src/utils/language.ts`: add the strings the card uses. - -### 5. Test it - -- The run writes one result per touched occurrence, with the expected `data` and `value`. -- Each created classification has `algorithm_result` and `job` set, and `applied_to` where it - replaced one. -- The history shows the result with its classifications, the replaced prediction as superseded, and - `replaced` filled. -- A second run adds a second result, and a retried job does not. -- An occurrence with several affected detections gets one result, with the figures of the - highest-ranked detection. -- `python manage.py test ami.ml.results ami.main.test_occurrence_history` covers the registry guards - and the history contract. - -### What you do not need to touch - -- **Migrations:** `kind` is a plain `CharField` without `choices`; every write path validates `data` - against the registry instead. -- **Serializers and OpenAPI:** the history serializer and the `oneOf` union are generated from the - registry, including `SPECTACULAR_SETTINGS["ENUM_NAME_OVERRIDES"]`. -- **The determination:** it never reads results. - -## Adding another kind of history entry - -The history merges entries from several tables: algorithm results, identifications and predictions. -Another source, such as the feature vectors a pipeline added to an occurrence's detections, becomes a -new entry type rather than a result kind when it records an output and not a decision. Add it in five -places: - -1. **`ami/main/models_future/history.py`:** a section of fields on `OccurrenceTimelineEntry` (if the - shared fields are not enough), and a loader in `occurrence_timeline` that adds the entries with a - fixed number of queries, whatever the occurrence's size. Entries with a `job` get its settings - automatically. For vectors, group by job and algorithm, and never select the vector column: - ```python - DetectionEmbedding.objects.filter(detection__occurrence=occurrence) - .values("algorithm", "job", "key") - .annotate(detections=Count("detection", distinct=True), timestamp=Max("timestamp")) - ``` -2. **`ami/main/api/serializers.py`:** an `EntrySerializer(HistoryEntryBaseSerializer)` with - `type = serializers.ChoiceField(choices=[""])` and its own fields. Add it to the serializers - of `OCCURRENCE_HISTORY_ENTRY_SCHEMA` and to `HISTORY_ENTRY_SERIALIZERS`. -3. **`config/settings/base.py`:** a `"EntryTypeEnum": [""]` entry in - `SPECTACULAR_SETTINGS["ENUM_NAME_OVERRIDES"]`, so the literal `type` gets a readable enum name. -4. **The UI:** the entry interface in `ui/src/data-services/models/occurrence-history.ts`, added to - `ServerOccurrenceHistoryEntry` and to `TimelineItem`, with a `case` in `getTimelineItems`; a card - component in `ui/src/pages/occurrence-details/identification-card/`, rendered in - `occurrence-timeline.tsx`. `getTimelineItems` skips entry types it does not know, so the server - side can ship first. -5. **Tests in `ami/main/test_occurrence_history.py`:** the new entries in the endpoint response, the - strict `HISTORY_QUERIES` count raised by the loader's queries (and still the same at two history - sizes), and the new component in the OpenAPI union. From ead5e18870eb555d7c27ae58c976e74a8b85d1e2 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Tue, 6 Oct 2026 18:18:16 -0700 Subject: [PATCH 112/134] fix(post-processing): leave detections the size filter already flagged alone Running the size filter again, or retrying its job, wrote another identical "Not identifiable" classification for every detection it had already flagged, so one detection could carry several. The end-to-end check of this PR found a retry doubling a job's classifications (14 to 28). The filter now skips detections that already carry a classification from its own algorithm, so a second run with a larger threshold records only what it newly flags, and a retried job changes nothing it already did. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01APWsYj7AAYL5aiq7T3HUwq --- ami/ml/post_processing/small_size_filter.py | 7 ++++- .../tests/test_small_size_filter.py | 30 ++++++++++++++++--- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/ami/ml/post_processing/small_size_filter.py b/ami/ml/post_processing/small_size_filter.py index c806bf50c..df2da35f3 100644 --- a/ami/ml/post_processing/small_size_filter.py +++ b/ami/ml/post_processing/small_size_filter.py @@ -1,6 +1,6 @@ import pydantic from django.db import transaction -from django.db.models import QuerySet +from django.db.models import Exists, OuterRef, QuerySet from django.utils import timezone from ami.main.models import Classification, Detection, Occurrence, SourceImageCollection, Taxon, TaxonRank @@ -82,6 +82,11 @@ def run(self) -> None: self.logger.info(f"=== Starting {self.name} ===") detections, scope_desc = self._scoped_detections(config) + # A detection this filter already flagged keeps its flag. Flagging it again would only add an + # identical classification, and a retried job would repeat its own work. + detections = detections.exclude( + Exists(Classification.objects.filter(detection=OuterRef("pk"), algorithm=self.algorithm)) + ) total = detections.count() self.logger.info(f"Found {total} detections in {scope_desc}") diff --git a/ami/ml/post_processing/tests/test_small_size_filter.py b/ami/ml/post_processing/tests/test_small_size_filter.py index f2c55435c..6f8fb0242 100644 --- a/ami/ml/post_processing/tests/test_small_size_filter.py +++ b/ami/ml/post_processing/tests/test_small_size_filter.py @@ -6,7 +6,7 @@ from django.test import TestCase from ami.jobs.models import Job, PostProcessingJob -from ami.main.models import Detection, Occurrence, SourceImage, Taxon +from ami.main.models import Classification, Detection, Occurrence, SourceImage, Taxon from ami.ml.models import AlgorithmResult from ami.ml.post_processing.small_size_filter import SmallSizeFilterTask from ami.ml.results.schemas import SizeFilterResultData @@ -105,6 +105,7 @@ def test_a_retried_job_reuses_its_own_result_instead_of_adding_another(self): self.assertEqual(list(job.algorithm_results.values_list("pk", flat=True)), [first.pk]) self.assertEqual(occurrence.algorithm_results.count(), 1) + self.assertEqual(job.classifications.count(), 1) class SizeFilterResultsTestCase(SizeFilterTestCase): @@ -142,12 +143,33 @@ def test_a_flagged_occurrence_gets_one_result_linked_to_the_classifications_it_c ) self.assertEqual(sorted(result.classifications.values_list("detection_id", flat=True)), detection_ids) - def test_each_run_adds_its_own_result(self): - """Running the filter again, say with another threshold, shows up as a second result in the history.""" + def test_a_second_run_with_a_larger_threshold_adds_a_result_for_what_it_newly_flags(self): + """Each run that flags something shows up in the history; a detection already flagged is left alone.""" + occurrence = self._singleton(self.captures[0], [0, 0, 10, 10]) + medium = Detection.objects.create( + source_image=self.captures[1], + bbox=[0, 0, 150, 150], + timestamp=self.captures[1].timestamp, + occurrence=occurrence, + ) + medium.classifications.create(taxon=self.taxon, score=0.9, timestamp=self.captures[1].timestamp) + for threshold in (0.01, 0.05): + SmallSizeFilterTask(logger=logger, occurrence_id=occurrence.pk, size_threshold=threshold).run() + + self.assertEqual(self.records().filter(occurrence=occurrence).count(), 2) + flags = Classification.objects.filter(detection__occurrence=occurrence, taxon__name="Not identifiable") + # Each detection carries exactly one flag: the second run did not flag the small one again. + self.assertEqual( + sorted(flags.values_list("detection_id", flat=True)), + sorted(occurrence.detections.values_list("pk", flat=True)), + ) + + def test_running_again_with_the_same_threshold_flags_nothing_twice(self): occurrence = self._singleton(self.captures[0], [0, 0, 10, 10]) for _ in range(2): self.run_filter(occurrence) - self.assertEqual(self.records().filter(occurrence=occurrence).count(), 2) + self.assertEqual(self.records().filter(occurrence=occurrence).count(), 1) + self.assertEqual(occurrence.detections.get().classifications.filter(taxon__name="Not identifiable").count(), 1) def test_an_occurrence_with_nothing_flagged_gets_no_result(self): occurrence = self._singleton(self.captures[0], [0, 0, 500, 500]) From e52d959d22ef30db4ff1c512817084afda5595dd Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Tue, 6 Oct 2026 18:20:15 -0700 Subject: [PATCH 113/134] test(results): use a kind no branch registers as the unregistered example The history test for a result whose kind is no longer registered used "tracking", which the tracking PR registers. It now uses "rank_rollup", so the test means the same thing on both branches and they do not conflict. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01APWsYj7AAYL5aiq7T3HUwq --- ami/main/test_occurrence_history.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ami/main/test_occurrence_history.py b/ami/main/test_occurrence_history.py index 39fe3b6ad..47abf424a 100644 --- a/ami/main/test_occurrence_history.py +++ b/ami/main/test_occurrence_history.py @@ -561,9 +561,9 @@ def test_a_result_of_a_kind_no_longer_registered_still_shows_in_the_history(self result = AlgorithmResult.objects.record( occurrence=self.occurrence, algorithm=self.size_filter, kind=SIZE_FILTER, data={"relative_size": 0.001} ) - AlgorithmResult.objects.filter(pk=result.pk).update(kind="tracking") + AlgorithmResult.objects.filter(pk=result.pk).update(kind="rank_rollup") entry = next(e for e in self.get() if e["type"] == "algorithm_result") - self.assertEqual(entry["kind"], "tracking") + self.assertEqual(entry["kind"], "rank_rollup") class ClassificationAdminTestCase(OccurrenceFixtureTestCase): From 756fafc5fd621adba3ec66a2f068adb87a6130b7 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 7 Oct 2026 15:36:43 -0700 Subject: [PATCH 114/134] fix(ui): keep history labels narrower than their values so labels wrap first The label column sized itself to the longest label, so a long setting name squeezed the values. Labels now take at most 40% of the card width. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01C7Xf6VPbwWtTumhjjF15g8 --- .../identification-card/history-stats.tsx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ui/src/pages/occurrence-details/identification-card/history-stats.tsx b/ui/src/pages/occurrence-details/identification-card/history-stats.tsx index b1deae9c5..d3da3ef9a 100644 --- a/ui/src/pages/occurrence-details/identification-card/history-stats.tsx +++ b/ui/src/pages/occurrence-details/identification-card/history-stats.tsx @@ -8,13 +8,16 @@ export interface HistoryStat { value: ReactNode } +// Labels get at most 40% of the width and wrap first, so values keep the room they need. export const HistoryStats = ({ stats }: { stats: HistoryStat[] }) => stats.length ? ( -
+
{stats.map(({ label, value }) => ( {label} - {value} + + {value} + ))}
From f8b47aa2e53ee66b7bc5a4fbfc1418f0a943f72a Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Thu, 8 Oct 2026 16:06:21 -0700 Subject: [PATCH 115/134] refactor(results): give references and job config their own homes, and simplify the history API Structure: - Record references move to ami/base/references.py. A model declares its reference_type and reference_name_field, so the base layer never names an app's model; reference(), field_references() and field_titles() move with it. - A job's config as labelled, linked rows moves to ami/jobs/job_config.py (JobConfigField, job_config_fields), so the jobs panel can reuse it. The word is "config" throughout, matching params["config"] and config_schema. - The occurrence history moves from ami/main/models_future to ami/main/api/occurrence_history.py: it reads ami.ml, which the domain layer must not import. - AlgorithmResult.move_to_occurrence leaves this PR; tracking (#1469) takes it over as merge_occurrences. History API: - History taxa carry their parents, so every card shows ranks. - job.config is the list of config fields; the duplicate raw config and superseded_by_result_id are gone. A demoted prediction shows as intermediate. Tests: one shared no_query_cache helper replaces three copies; reference and job config tests sit beside their modules; a test that could not fail and an admin-configuration test are cut. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01APWsYj7AAYL5aiq7T3HUwq --- ami/base/references.py | 91 ++++++++++ ami/base/serializers.py | 8 + ami/jobs/job_config.py | 75 ++++++++ ami/jobs/tests/test_job_config.py | 62 +++++++ ami/jobs/tests/test_jobs.py | 10 +- .../history.py => api/occurrence_history.py} | 91 +--------- ami/main/api/serializers.py | 54 ++---- ami/main/api/views.py | 2 +- ami/main/models.py | 9 + ami/main/models_future/references.py | 70 ------- ami/main/test_occurrence_history.py | 171 ++---------------- ami/main/tests.py | 9 +- ami/ml/models/algorithm.py | 3 + ami/ml/models/algorithm_result.py | 18 -- ami/ml/post_processing/class_masking.py | 3 +- ami/ml/post_processing/small_size_filter.py | 3 +- ami/ml/results/README.md | 17 +- ami/ml/results/schemas.py | 36 +--- ami/ml/results/tests.py | 43 +---- ami/tests/fixtures/queries.py | 17 ++ ami/tests/test_references.py | 33 ++++ config/settings/base.py | 2 - docs/claude/INDEX.md | 2 +- .../identifications/identification-card.tsx | 26 +-- 24 files changed, 368 insertions(+), 487 deletions(-) create mode 100644 ami/base/references.py create mode 100644 ami/jobs/job_config.py create mode 100644 ami/jobs/tests/test_job_config.py rename ami/main/{models_future/history.py => api/occurrence_history.py} (67%) delete mode 100644 ami/main/models_future/references.py create mode 100644 ami/tests/fixtures/queries.py create mode 100644 ami/tests/test_references.py diff --git a/ami/base/references.py b/ami/base/references.py new file mode 100644 index 000000000..d47feaf6c --- /dev/null +++ b/ami/base/references.py @@ -0,0 +1,91 @@ +"""Record ids stored in JSON, resolved to ``{type, id, name}`` so the UI can show and link them. + +A model opts in by declaring ``reference_type`` (and ``reference_name_field``, the attribute shown +as its name; None shows ``#``). A pydantic schema marks a field that holds such an id with +``reference()``. ``resolve_references`` reads the names with one query per type, and the UI maps +each type to a page in ``ui/src/utils/references.ts``. +""" + +import collections +import dataclasses +import functools +import typing + +import pydantic +from django.apps import apps + + +@dataclasses.dataclass(frozen=True) +class Ref: + """A record a stored value names. ``name`` is None when the record no longer exists.""" + + type: str + id: int + name: str | None + + +def reference(ref_type: str, default: typing.Any = None, **field_options: typing.Any) -> typing.Any: + """Declare a pydantic field that holds the id of a record of ``ref_type``. + + ``field_options`` go to ``pydantic.Field``. + """ + return pydantic.Field(default, reference=ref_type, **field_options) + + +def field_references(schema: type[pydantic.BaseModel]) -> dict[str, str]: + """The schema's fields declared with ``reference()``, mapped to the reference type.""" + return { + name: field.field_info.extra["reference"] + for name, field in schema.__fields__.items() + if "reference" in field.field_info.extra + } + + +def field_titles(schema: type[pydantic.BaseModel]) -> dict[str, str | None]: + """Every field of the schema, in declaration order, mapped to its ``title`` or None.""" + return {name: field.field_info.title for name, field in schema.__fields__.items()} + + +def is_record_id(value: typing.Any) -> bool: + """Whether a stored value can be a primary key: an int that is not a bool.""" + return isinstance(value, int) and not isinstance(value, bool) + + +@functools.cache +def reference_types() -> dict[str, tuple[type, str | None]]: + """Each declared reference type mapped to its model and name field.""" + types: dict[str, tuple[type, str | None]] = {} + for model in apps.get_models(): + ref_type = getattr(model, "reference_type", None) + if ref_type is None: + continue + if ref_type in types: + raise ValueError(f"Reference type {ref_type!r} is declared by {types[ref_type][0]} and {model}.") + types[ref_type] = (model, getattr(model, "reference_name_field", None)) + return types + + +def unmapped_reference_types(schemas: typing.Iterable[type[pydantic.BaseModel]]) -> set[str]: + """Reference types the schemas declare that no model declares; resolving them would fail.""" + declared = {ref_type for schema in schemas for ref_type in field_references(schema).values()} + return declared - set(reference_types()) + + +def resolve_references(wanted: typing.Iterable[tuple[str, int]]) -> dict[tuple[str, int], Ref]: + """Each ``(type, id)`` as a ``Ref``, reading names with one query per type. + + An unknown type raises KeyError: a schema declared a reference type no model declares. + """ + ids_by_type: dict[str, set[int]] = collections.defaultdict(set) + for ref_type, ref_id in wanted: + ids_by_type[ref_type].add(ref_id) + resolved: dict[tuple[str, int], Ref] = {} + for ref_type, ids in ids_by_type.items(): + model, name_field = reference_types()[ref_type] + if name_field is None: + names = {pk: f"#{pk}" for pk in model.objects.filter(pk__in=ids).values_list("pk", flat=True)} + else: + names = dict(model.objects.filter(pk__in=ids).values_list("pk", name_field)) + for ref_id in ids: + resolved[(ref_type, ref_id)] = Ref(ref_type, ref_id, names.get(ref_id)) + return resolved diff --git a/ami/base/serializers.py b/ami/base/serializers.py index e39ede52b..4d102f7e5 100644 --- a/ami/base/serializers.py +++ b/ami/base/serializers.py @@ -180,3 +180,11 @@ def clean(self) -> dict[str, typing.Any]: if self.is_valid(raise_exception=True): return self.validated_data raise api_exceptions.ValidationError("Invalid filter parameters") + + +class RefSerializer(serializers.Serializer): + """A record a stored value names (see ``ami.base.references``). ``name`` is null when it no longer exists.""" + + type = serializers.CharField(help_text="What kind of record: taxa_list, capture_set, algorithm, ...") + id = serializers.IntegerField() + name = serializers.CharField(allow_null=True) diff --git a/ami/jobs/job_config.py b/ami/jobs/job_config.py new file mode 100644 index 000000000..64eb6e0d4 --- /dev/null +++ b/ami/jobs/job_config.py @@ -0,0 +1,75 @@ +"""A job's config as labelled rows, with the records it names resolved, for showing it to people. + +A post-processing job stores its config in ``params["config"]``, validated by its task's +``config_schema``. The schema gives each field its title, its order and, through ``reference()``, +the type of record an id names. +""" + +import dataclasses +import typing + +from rest_framework import serializers + +from ami.base.references import Ref, field_references, field_titles, is_record_id, resolve_references +from ami.base.serializers import RefSerializer +from ami.jobs.models import Job +from ami.ml.post_processing.registry import get_postprocessing_task + + +@dataclasses.dataclass +class JobConfigField: + """One field of a job's config: its title from the task's config schema, and the record it names, if any.""" + + key: str + label: str + value: typing.Any + ref: Ref | None = None + + +class JobConfigFieldSerializer(serializers.Serializer): + """One field of a job's config. ``label`` is the title the task's config schema gives it, else the key.""" + + key = serializers.CharField() + label = serializers.CharField() + value = serializers.JSONField(allow_null=True) + ref = RefSerializer(allow_null=True, help_text="The record the field names, when it names one.") + + +def job_config(job: Job | None) -> dict | None: + """The config a post-processing job ran with, or None for any other job or malformed params.""" + params = job.params if job is not None else None + config = params.get("config") if isinstance(params, dict) else None + return config if isinstance(config, dict) else None + + +def job_config_fields(jobs: typing.Iterable[Job]) -> dict[int, list[JobConfigField]]: + """Each job's config fields by job id, resolving every record id they name with one query per type.""" + specs = {job.pk: _field_specs(job) for job in jobs} + wanted = [(ref_type, value) for fields in specs.values() for _, _, value, ref_type in fields if ref_type] + resolved = resolve_references(wanted) if wanted else {} + return { + job_id: [ + JobConfigField(key, label, value, resolved[(ref_type, value)] if ref_type else None) + for key, label, value, ref_type in fields + ] + for job_id, fields in specs.items() + } + + +def _field_specs(job: Job) -> list[tuple[str, str, typing.Any, str | None]]: + """``(key, label, value, reference type)`` per field, labelled, typed and ordered by the task's config schema. + + The stored config's key order is not kept (Postgres orders a JSON object's keys), so fields follow the + schema and any key it does not declare comes after. A job whose task is not registered shows its + config by key, naming no records. + """ + config = job_config(job) or {} + task_key = job.params.get("task") if config else None + task = get_postprocessing_task(task_key) if isinstance(task_key, str) else None + titles = field_titles(task.config_schema) if task else {} + references = field_references(task.config_schema) if task else {} + keys = [key for key in titles if key in config] + [key for key in config if key not in titles] + return [ + (key, titles.get(key) or key, config[key], references.get(key) if is_record_id(config[key]) else None) + for key in keys + ] diff --git a/ami/jobs/tests/test_job_config.py b/ami/jobs/tests/test_job_config.py new file mode 100644 index 000000000..878a2869d --- /dev/null +++ b/ami/jobs/tests/test_job_config.py @@ -0,0 +1,62 @@ +from django.test import TestCase + +from ami.base.references import Ref, unmapped_reference_types +from ami.jobs.job_config import JobConfigField, job_config_fields +from ami.jobs.models import Job +from ami.main.models import TaxaList +from ami.ml.models import Algorithm +from ami.ml.post_processing.registry import POSTPROCESSING_TASKS +from ami.tests.fixtures.main import setup_test_project + + +class JobConfigFieldsTestCase(TestCase): + """A job's config as labelled rows, in its task's schema order, with the records it names resolved.""" + + @classmethod + def setUpTestData(cls) -> None: + cls.project, _ = setup_test_project(reuse=False) + + def _fields(self, params) -> list[JobConfigField]: + job = Job.objects.create(project=self.project, name="Run", job_type_key="post_processing", params=params) + return job_config_fields([job])[job.pk] + + def test_labels_order_and_references_come_from_the_task_config_schema(self): + taxa_list = TaxaList.objects.create(name="Kept species") + classifier = Algorithm.objects.create(name="Classifier", key="config-test-classifier") + config = { + "reweight": True, + "algorithm_id": classifier.pk, + "taxa_list_id": taxa_list.pk, + "occurrence_id": True, + "source_image_collection_id": None, + } + self.assertEqual( + self._fields({"task": "class_masking", "config": config}), + [ + JobConfigField("source_image_collection_id", "Capture set", None, None), + # A value that is not an id names no record, whatever the schema declares. + JobConfigField("occurrence_id", "Occurrence", True, None), + JobConfigField( + "taxa_list_id", "Species list", taxa_list.pk, Ref("taxa_list", taxa_list.pk, "Kept species") + ), + JobConfigField( + "algorithm_id", "Classifier", classifier.pk, Ref("algorithm", classifier.pk, "Classifier") + ), + JobConfigField("reweight", "Re-weighted scores", True, None), + ], + ) + + def test_a_job_without_a_registered_task_shows_its_config_by_key_with_no_references(self): + self.assertEqual( + self._fields({"config": {"taxa_list_id": 3, "size_threshold": 0.01}}), + [ + JobConfigField("taxa_list_id", "taxa_list_id", 3, None), + JobConfigField("size_threshold", "size_threshold", 0.01), + ], + ) + + def test_a_job_whose_params_are_not_an_object_has_no_config(self): + self.assertEqual(self._fields(["not", "an", "object"]), []) + + def test_every_reference_a_task_declares_can_be_resolved(self): + self.assertEqual(unmapped_reference_types(task.config_schema for task in POSTPROCESSING_TASKS.values()), set()) diff --git a/ami/jobs/tests/test_jobs.py b/ami/jobs/tests/test_jobs.py index fbed09b8f..c8c80a85e 100644 --- a/ami/jobs/tests/test_jobs.py +++ b/ami/jobs/tests/test_jobs.py @@ -1799,10 +1799,11 @@ def test_query_count_does_not_grow_with_the_number_of_jobs(self): """No per-job work: listing eight jobs runs the same queries as listing three, for a member whose permissions would otherwise be resolved on every row. Cachalot is off so every query counts.""" - from cachalot.api import cachalot_disabled from django.db import connection from django.test.utils import CaptureQueriesContext + from ami.tests.fixtures.queries import no_query_cache + self.client.force_authenticate(self.user) # The owner is a member of their project. def list_choices() -> tuple[int, int]: @@ -1811,17 +1812,12 @@ def list_choices() -> tuple[int, int]: self.assertEqual(response.status_code, status.HTTP_200_OK) return len(response.json()["results"]), len(queries.captured_queries) - disabled = cachalot_disabled() - disabled.__enter__() - try: + with no_query_cache(): three_rows, three_rows_queries = list_choices() Job.objects.bulk_create( Job(project=self.project, name=f"More {index}", job_type_key=MLJob.key) for index in range(5) ) eight_rows, eight_rows_queries = list_choices() - finally: - # cachalot_disabled() does not restore itself when the block raises. - disabled.__exit__(None, None, None) self.assertEqual((three_rows, eight_rows), (3, 8)) self.assertEqual(eight_rows_queries, three_rows_queries) diff --git a/ami/main/models_future/history.py b/ami/main/api/occurrence_history.py similarity index 67% rename from ami/main/models_future/history.py rename to ami/main/api/occurrence_history.py index 6e564a044..1e232bae0 100644 --- a/ami/main/models_future/history.py +++ b/ami/main/api/occurrence_history.py @@ -12,11 +12,9 @@ import datetime import typing +from ami.jobs.job_config import JobConfigField, job_config_fields from ami.main.models import Classification, Identification, Occurrence, Taxon, User -from ami.main.models_future.references import Ref, is_record_id, resolve_references from ami.ml.models import Algorithm, AlgorithmResult -from ami.ml.post_processing.registry import get_postprocessing_task -from ami.ml.results.schemas import field_references, field_titles if typing.TYPE_CHECKING: from ami.jobs.models import Job @@ -25,16 +23,6 @@ JOB_FIELDS_NOT_SHOWN = ("job__progress", "job__logs") -@dataclasses.dataclass -class JobSetting: - """One setting a job ran with: its label from the task's config schema, and the record it names, if any.""" - - key: str - label: str - value: typing.Any - ref: Ref | None = None - - @dataclasses.dataclass class CreatedClassification: """A classification a run created, with the one it replaced (through applied_to), if that still exists.""" @@ -54,7 +42,7 @@ class OccurrenceTimelineEntry: user: User | None = None # identification algorithm: Algorithm | None = None # result, prediction job: Job | None = None # result, prediction - job_settings: list[JobSetting] = dataclasses.field(default_factory=list) # empty without a job + job_config: list[JobConfigField] = dataclasses.field(default_factory=list) # empty without a job taxon: Taxon | None = None # identification, prediction score: float | None = None # prediction # --- Algorithm result @@ -73,10 +61,8 @@ def occurrence_timeline(occurrence: Occurrence) -> list[OccurrenceTimelineEntry] A result comes with the classifications its run created, so the history shows the run and what it changed as one entry; those classifications are not listed again as predictions. - Every other algorithm contributes one prediction, its best. A prediction names the result - that superseded it when a result's classification re-scored it, or when a result's - classification on the same detection outranks it as a terminal prediction. Ids the entries - show are resolved to references with one query per type. The query count does not grow + Every other algorithm contributes one prediction, its best. Ids the entries show are resolved + to references with one query per type. The query count does not grow with the entries. """ results = list( @@ -91,28 +77,6 @@ def occurrence_timeline(occurrence: Occurrence) -> list[OccurrenceTimelineEntry] ) created_by_result = _classifications_created_by(results, occurrence) created_ids = {c.classification.pk for created in created_by_result.values() for c in created} - rescored_by = { - c.classification.applied_to_id: result_id - for result_id, created in created_by_result.items() - for c in created - if c.classification.applied_to_id is not None - } - # Only a classification that re-scored nothing (the size filter's) supersedes by detection; a - # re-scored one names its original through applied_to, so other classifiers stay unmarked. - outranked_by = { - c.classification.detection_id: result_id - for result_id, created in created_by_result.items() - for c in created - if c.classification.applied_to_id is None - } - - def superseded_by(prediction: Classification) -> int | None: - if prediction.pk in rescored_by: - return rescored_by[prediction.pk] - if prediction.terminal: - return outranked_by.get(prediction.detection_id) - return None - entries = [ OccurrenceTimelineEntry( type="algorithm_result", @@ -161,55 +125,18 @@ def superseded_by(prediction: Classification) -> int | None: "detection_id": prediction.detection_id, "terminal": prediction.terminal, "applied_to_id": prediction.applied_to_id, - "superseded_by_result_id": superseded_by(prediction), }, ) for prediction in _one_prediction_per_algorithm(occurrence) if prediction.pk not in created_ids ) - _fill_job_settings_and_references(entries) - entries.sort(key=lambda entry: (entry.timestamp, entry.id), reverse=True) - return entries - - -def _fill_job_settings_and_references(entries: list[OccurrenceTimelineEntry]) -> None: - """Fill each entry's job settings, resolving every record id the settings name at once.""" - settings = {id(entry): _job_settings(entry.job) for entry in entries if entry.job is not None} - wanted = [(ref_type, value) for specs in settings.values() for _, _, value, ref_type in specs if ref_type] - resolved = resolve_references(wanted) if wanted else {} + config_fields = job_config_fields({entry.job.pk: entry.job for entry in entries if entry.job}.values()) for entry in entries: if entry.job is not None: - entry.job_settings = [ - JobSetting(key, label, value, resolved[(ref_type, value)] if ref_type else None) - for key, label, value, ref_type in settings[id(entry)] - ] - - -def job_config(job: Job | None) -> dict | None: - """The settings a post-processing job ran with, or None for any other job or malformed params.""" - params = job.params if job is not None else None - config = params.get("config") if isinstance(params, dict) else None - return config if isinstance(config, dict) else None - - -def _job_settings(job: Job) -> list[tuple[str, str, typing.Any, str | None]]: - """``(key, label, value, reference type)`` per setting, labelled, typed and ordered by the task's config schema. - - The stored config's key order is not kept (Postgres orders a JSON object's keys), so settings follow the - schema and any key it does not declare comes after. A job whose task is not registered shows its - settings by key, naming no records. - """ - config = job_config(job) or {} - task_key = job.params.get("task") if config else None - task = get_postprocessing_task(task_key) if isinstance(task_key, str) else None - titles = field_titles(task.config_schema) if task else {} - references = field_references(task.config_schema) if task else {} - keys = [key for key in titles if key in config] + [key for key in config if key not in titles] - return [ - (key, titles.get(key) or key, config[key], references.get(key) if is_record_id(config[key]) else None) - for key in keys - ] + entry.job_config = config_fields[entry.job.pk] + entries.sort(key=lambda entry: (entry.timestamp, entry.id), reverse=True) + return entries def _by_id(model, ids: set) -> dict: @@ -244,7 +171,7 @@ def _classifications_created_by( c.pk: c for c in Classification.objects.filter(pk__in=replaced_ids) .select_related("taxon") - .only("pk", "score", "taxon", "taxon__name", "taxon__rank") + .only("pk", "score", "taxon", "taxon__name", "taxon__rank", "taxon__parents_json") } if replaced_ids else {} diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index 7e38f0790..05f75aadf 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -12,6 +12,7 @@ from ami.base.permissions import add_m2m_object_permissions from ami.base.serializers import DefaultSerializer, MinimalNestedModelSerializer, reverse_with_params from ami.base.views import get_active_project +from ami.jobs.job_config import JobConfigFieldSerializer from ami.jobs.models import Job from ami.main.models import Tag from ami.ml.models import Algorithm, Pipeline @@ -2119,43 +2120,17 @@ class HistoryTaxonSerializer(serializers.Serializer): id = serializers.IntegerField() name = serializers.CharField() rank = serializers.CharField() - - -class RefSerializer(serializers.Serializer): - """Another record the history mentions. ``name`` is null when it no longer exists.""" - - type = serializers.CharField(help_text="What kind of record: job, taxa_list, capture_set, algorithm, ...") - id = serializers.IntegerField() - name = serializers.CharField(allow_null=True) - - -class HistoryJobSettingSerializer(serializers.Serializer): - """One setting a job ran with. ``label`` is the title the task's config schema gives it, else the key.""" - - key = serializers.CharField() - label = serializers.CharField() - value = serializers.JSONField(allow_null=True) - ref = RefSerializer(allow_null=True, help_text="The record the setting names, when it names one.") + parents = TaxonParentSerializer(many=True, read_only=True, source="parents_json") class HistoryJobSerializer(serializers.Serializer): - """A job in the history. ``settings`` is filled per entry by ``HistoryEntryBaseSerializer``.""" + """A job in the history. ``config`` is filled per entry by ``HistoryEntryBaseSerializer``.""" id = serializers.IntegerField() name = serializers.CharField() - config = serializers.SerializerMethodField( - help_text=( - "The settings a post-processing job ran with, as its task validated them. " - "Null for other jobs, and for a post-processing job whose stored config is not an object." - ) + config = JobConfigFieldSerializer( + many=True, read_only=True, help_text="The job's config, in the order its task's config schema declares it." ) - settings = HistoryJobSettingSerializer(many=True, read_only=True, help_text="The settings, in the job's order.") - - @extend_schema_field(serializers.JSONField(allow_null=True)) - def get_config(self, job) -> dict | None: - from ami.main.models_future.history import job_config - - return job_config(job) class ReplacedClassificationSerializer(serializers.Serializer): @@ -2192,7 +2167,7 @@ def get_job(self, entry) -> dict | None: return None return { **HistoryJobSerializer(entry.job).data, - "settings": HistoryJobSettingSerializer(entry.job_settings, many=True).data, + "config": JobConfigFieldSerializer(entry.job_config, many=True).data, } @@ -2209,9 +2184,6 @@ class PredictionDetailsSerializer(serializers.Serializer): applied_to_id = serializers.IntegerField( allow_null=True, help_text="The classification this one re-scored and demoted, when there is one." ) - superseded_by_result_id = serializers.IntegerField( - allow_null=True, help_text="The algorithm result whose classification replaced this one." - ) class IdentificationEntrySerializer(HistoryEntryBaseSerializer): @@ -2240,22 +2212,22 @@ class AlgorithmResultEntrySerializer(HistoryEntryBaseSerializer): ) +HISTORY_ENTRY_SERIALIZERS = { + "identification": IdentificationEntrySerializer, + "prediction": PredictionEntrySerializer, + "algorithm_result": AlgorithmResultEntrySerializer, +} + # The history endpoint's response: a plain oneOf with a literal ``type`` field. A result's ``data`` is # published as JSON; the server validates it against its kind's model, and the UI types it by kind. # Publishing one typed component per kind waits for a client generated from the schema (#1482). OCCURRENCE_HISTORY_ENTRY_SCHEMA = PolymorphicProxySerializer( component_name="OccurrenceHistoryEntry", - serializers=[IdentificationEntrySerializer, PredictionEntrySerializer, AlgorithmResultEntrySerializer], + serializers=list(HISTORY_ENTRY_SERIALIZERS.values()), resource_type_field_name=None, many=True, ) -HISTORY_ENTRY_SERIALIZERS = { - "identification": IdentificationEntrySerializer, - "prediction": PredictionEntrySerializer, - "algorithm_result": AlgorithmResultEntrySerializer, -} - def serialize_history(entries, context) -> list[dict]: return [HISTORY_ENTRY_SERIALIZERS[entry.type](entry, context=context).data for entry in entries] diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 7d7e6ec86..914fc236f 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -32,9 +32,9 @@ from ami.base.permissions import IsActiveStaffOrReadOnly, IsProjectMemberOrReadOnly, ObjectPermission from ami.base.serializers import FilterParamsSerializer, SingleParamSerializer from ami.base.views import ProjectMixin +from ami.main.api.occurrence_history import occurrence_timeline from ami.main.api.schemas import limit_doc_param, project_id_doc_param from ami.main.api.serializers import TagSerializer -from ami.main.models_future.history import occurrence_timeline from ami.main.models_future.identifications import create_identifications_batch, resolve_occurrences from ami.main.models_future.occurrence import model_agreement_for_project, top_identifiers_for_project from ami.ml.models.algorithm import Algorithm diff --git a/ami/main/models.py b/ami/main/models.py index 15d26536e..1556d1751 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -3788,6 +3788,9 @@ def get_queryset(self): class Occurrence(BaseModel): """An occurrence of a taxon, a sequence of one or more detections""" + reference_type = "occurrence" + reference_name_field = None + # @TODO change Determination to a nested field with a Taxon, User, Identification, etc like the serializer # this could be a OneToOneField to a Determination model or a JSONField validated by a Pydantic model determination = models.ForeignKey("Taxon", on_delete=models.SET_NULL, null=True, related_name="occurrences") @@ -4951,6 +4954,9 @@ class TaxaListManager(models.Manager.from_queryset(TaxaListQuerySet)): class TaxaList(BaseModel): """A checklist of taxa""" + reference_type = "taxa_list" + reference_name_field = "name" + name = models.CharField(max_length=255) description = models.TextField(blank=True) @@ -5135,6 +5141,9 @@ class SourceImageCollection(BaseModel): """ + reference_type = "capture_set" + reference_name_field = "name" + name = models.CharField(max_length=255) description = models.TextField(blank=True) # dataset_type = models.CharField( diff --git a/ami/main/models_future/references.py b/ami/main/models_future/references.py deleted file mode 100644 index 929409219..000000000 --- a/ami/main/models_future/references.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Ids the occurrence history shows, resolved to ``{type, id, name}`` so the UI can link them with one table. - -``name`` is None when the row no longer exists; the UI then shows the id as text. Resolution reads names -only, one query per type, and is used for ids stored on the occurrence's own results and jobs. Result data -models and task config schemas declare which fields hold an id with ``ami.ml.results.schemas.reference``. -""" - -import collections -import dataclasses -import typing - -from django.apps import apps - - -@dataclasses.dataclass(frozen=True) -class Ref: - type: str - id: int - name: str | None - - -# Reference type -> (model, attribute holding its display name; None shows "#"). -REFERENCE_TYPES: dict[str, tuple[str, str | None]] = { - "algorithm": ("ml.Algorithm", "name"), - "capture_set": ("main.SourceImageCollection", "name"), - "job": ("jobs.Job", "name"), - "occurrence": ("main.Occurrence", None), - "taxa_list": ("main.TaxaList", "name"), - "taxon": ("main.Taxon", "name"), -} - - -def is_record_id(value: typing.Any) -> bool: - """Whether a stored value can be a primary key: an int that is not a bool.""" - return isinstance(value, int) and not isinstance(value, bool) - - -def unmapped_reference_types(models: typing.Iterable[type] | None = None) -> set[str]: - """Reference types that task config schemas declare but this module does not map. - - Defaults to every registered post-processing task; a test keeps the answer empty. - """ - from ami.ml.results.schemas import field_references - - if models is None: - from ami.ml.post_processing.registry import POSTPROCESSING_TASKS - - models = [task.config_schema for task in POSTPROCESSING_TASKS.values()] - return {ref_type for model in models for ref_type in field_references(model).values()} - set(REFERENCE_TYPES) - - -def resolve_references(wanted: typing.Iterable[tuple[str, int]]) -> dict[tuple[str, int], Ref]: - """Each ``(type, id)`` as a ``Ref``, reading names with one query per type. - - An unknown type raises KeyError: a result kind declared a reference type nobody mapped. - """ - ids_by_type: dict[str, set[int]] = collections.defaultdict(set) - for ref_type, ref_id in wanted: - ids_by_type[ref_type].add(ref_id) - resolved: dict[tuple[str, int], Ref] = {} - for ref_type, ids in ids_by_type.items(): - model_label, name_attr = REFERENCE_TYPES[ref_type] - model = apps.get_model(model_label) - if name_attr is None: - names = {pk: f"#{pk}" for pk in model.objects.filter(pk__in=ids).values_list("pk", flat=True)} - else: - names = dict(model.objects.filter(pk__in=ids).values_list("pk", name_attr)) - for ref_id in ids: - resolved[(ref_type, ref_id)] = Ref(ref_type, ref_id, names.get(ref_id)) - return resolved diff --git a/ami/main/test_occurrence_history.py b/ami/main/test_occurrence_history.py index 47abf424a..b4020f358 100644 --- a/ami/main/test_occurrence_history.py +++ b/ami/main/test_occurrence_history.py @@ -1,16 +1,15 @@ """Algorithm results and the occurrence history endpoint that reads them.""" -import contextlib import datetime import re -from cachalot.api import cachalot_disabled from django.db import connection from django.test import TestCase from django.test.utils import CaptureQueriesContext from rest_framework.test import APITestCase from ami.jobs.models import Job +from ami.main.api.occurrence_history import occurrence_timeline from ami.main.models import ( Classification, Detection, @@ -21,11 +20,10 @@ TaxaList, Taxon, ) -from ami.main.models_future.history import JobSetting, OccurrenceTimelineEntry, occurrence_timeline -from ami.main.models_future.references import Ref, resolve_references, unmapped_reference_types from ami.ml.models import Algorithm, AlgorithmResult -from ami.ml.results.schemas import AlgorithmResultData, ClassMaskingResultData, SizeFilterResultData, reference +from ami.ml.results.schemas import ClassMaskingResultData, SizeFilterResultData from ami.tests.fixtures.main import create_captures, create_taxa, setup_test_project +from ami.tests.fixtures.queries import no_query_cache from ami.users.models import User from ami.users.roles import BasicMember, ProjectManager @@ -38,44 +36,6 @@ CLASS_MASKING = ClassMaskingResultData.kind -@contextlib.contextmanager -def no_query_cache(): - """``cachalot_disabled`` that restores the cache even when the block raises, so one failure stays one.""" - context = cachalot_disabled() - context.__enter__() - try: - yield - finally: - context.__exit__(None, None, None) - - -class ReferenceTestCase(TestCase): - """Ids shown in the history become {type, id, name}; a deleted row keeps its id with no name.""" - - @classmethod - def setUpTestData(cls) -> None: - cls.project, cls.deployment = setup_test_project(reuse=False) - - def test_resolves_each_type_in_one_query_and_marks_missing_rows(self): - taxa_list = TaxaList.objects.create(name="Kept species") - algorithm = Algorithm.objects.create(name="Classifier", key="ref-test-classifier") - wanted = [("taxa_list", taxa_list.pk), ("algorithm", algorithm.pk), ("taxa_list", 999999)] - with no_query_cache(), self.assertNumQueries(2): - refs = resolve_references(wanted) - self.assertEqual(refs[("taxa_list", taxa_list.pk)], Ref("taxa_list", taxa_list.pk, "Kept species")) - self.assertEqual(refs[("algorithm", algorithm.pk)].name, "Classifier") - self.assertEqual(refs[("taxa_list", 999999)], Ref("taxa_list", 999999, None)) - - def test_every_declared_reference_type_is_mapped(self): - """A kind or task declaring a reference type with no mapping would fail the history at request time.""" - - class Probe(AlgorithmResultData): - merged_into_id: int | None = reference("not_a_type") - - self.assertEqual(unmapped_reference_types([Probe]), {"not_a_type"}) - self.assertEqual(unmapped_reference_types(), set()) - - class OccurrenceHistorySchemaTestCase(TestCase): """The OpenAPI schema publishes the history as a oneOf of identification, prediction and result entries.""" @@ -119,7 +79,7 @@ def setUpTestData(cls) -> None: class OccurrenceHistoryEndpointTestCase(OccurrenceFixtureTestCase): - """GET /occurrences/{id}/history/ merges results, reviews, identifications and predictions, newest first.""" + """GET /occurrences/{id}/history/ merges results, identifications and predictions, newest first.""" @classmethod def setUpTestData(cls) -> None: @@ -201,9 +161,8 @@ def test_entries_are_merged_newest_first_and_a_result_carries_the_classification { "id": self.job.pk, "name": self.job.name, - "config": {"size_threshold": 0.01}, # The label is the title the size filter's config schema gives the setting. - "settings": [{"key": "size_threshold", "label": "Size threshold", "value": 0.01, "ref": None}], + "config": [{"key": "size_threshold", "label": "Size threshold", "value": 0.01, "ref": None}], }, ) self.assertIsNone(result["taxon"]) @@ -218,7 +177,7 @@ def test_entries_are_merged_newest_first_and_a_result_carries_the_classification [ { "id": created.pk, - "taxon": {"id": self.taxon.pk, "name": self.taxon.name, "rank": self.taxon.rank}, + "taxon": {"id": self.taxon.pk, "name": self.taxon.name, "rank": self.taxon.rank, "parents": []}, "score": 0.1, "terminal": True, "detection_id": self.detections[0].pk, @@ -228,7 +187,7 @@ def test_entries_are_merged_newest_first_and_a_result_carries_the_classification ) self.assertNotIn("email", str(data)) - def test_a_demoted_prediction_names_the_result_that_superseded_it(self): + def test_a_masked_prediction_shows_inside_its_result_and_the_original_stays_as_a_prediction(self): """Class masking demotes the original classification and links its replacement to the result.""" classifier = Algorithm.objects.create(name="Classifier", key="classifier-history-test") masker = Algorithm.objects.create(name="Masked classifier", key="masker-history-test") @@ -279,7 +238,7 @@ def test_a_demoted_prediction_names_the_result_that_superseded_it(self): result_entry = next(entry for entry in data if entry["type"] == "algorithm_result") self.assertEqual([c["id"] for c in result_entry["classifications"]], [masked.pk]) # The masked classification shows inside the result, not as a prediction; the original - # still has its card, marked as superseded by the run. + # keeps its card, demoted to non-terminal. predictions = { entry["algorithm"]["key"] if entry["algorithm"] else None: entry for entry in data @@ -287,15 +246,12 @@ def test_a_demoted_prediction_names_the_result_that_superseded_it(self): } self.assertEqual(set(predictions), {None, classifier.key, bystander_algorithm.key}) self.assertEqual(predictions[bystander_algorithm.key]["id"], bystander.pk) - self.assertIsNone(predictions[bystander_algorithm.key]["details"]["superseded_by_result_id"]) demoted = predictions[classifier.key] self.assertEqual(demoted["id"], original.pk) - self.assertEqual(demoted["details"]["superseded_by_result_id"], result.pk) self.assertFalse(demoted["details"]["terminal"]) - self.assertIsNone(predictions[None]["details"]["superseded_by_result_id"]) - settings = {setting["key"]: setting for setting in result_entry["job"]["settings"]} + config = {field["key"]: field for field in result_entry["job"]["config"]} self.assertEqual( - settings["taxa_list_id"], + config["taxa_list_id"], { "key": "taxa_list_id", "label": "Species list", @@ -303,40 +259,15 @@ def test_a_demoted_prediction_names_the_result_that_superseded_it(self): "ref": {"type": "taxa_list", "id": taxa_list.pk, "name": "Kept species"}, }, ) - self.assertEqual(settings["algorithm_id"]["label"], "Classifier") + self.assertEqual(config["algorithm_id"]["label"], "Classifier") self.assertEqual( - settings["algorithm_id"]["ref"], {"type": "algorithm", "id": classifier.pk, "name": classifier.name} + config["algorithm_id"]["ref"], {"type": "algorithm", "id": classifier.pk, "name": classifier.name} ) # The top prediction before masking is the classification the masked one replaced. self.assertEqual(result_entry["classifications"][0]["replaced"]["id"], original.pk) self.assertEqual(result_entry["classifications"][0]["replaced"]["taxon"]["id"], self.taxon.pk) self.assertEqual(result_entry["value"], 0.4) - def test_a_terminal_prediction_outranked_on_its_detection_is_superseded(self): - """The size filter does not demote the prediction it outranks, so the link is the shared detection.""" - # The fixture's tied predictions resolve to the latest one, on the last detection. - outranked = self.detections[3].classifications.get() - result = AlgorithmResult.objects.record( - occurrence=self.occurrence, - algorithm=self.size_filter, - job=self.job, - kind=SIZE_FILTER, - data={"relative_size": 0.001}, - ) - Classification.objects.create( - detection=self.detections[3], - taxon=self.other_taxon, - score=1.0, - algorithm=self.size_filter, - timestamp=datetime.datetime.now(), - algorithm_result=result, - ) - data = self.get() - - prediction = next(entry for entry in data if entry["type"] == "prediction") - self.assertEqual(prediction["id"], outranked.pk) - self.assertEqual(prediction["details"]["superseded_by_result_id"], result.pk) - def test_each_algorithm_shows_one_prediction_preferring_terminal_then_latest(self): """Tied top scores would otherwise show the same prediction once per detection.""" classifier = Algorithm.objects.create(name="Tied classifier", key="tied-classifier-history-test") @@ -488,63 +419,7 @@ def test_predictions_carry_the_job_that_wrote_them(self): self.assertTrue(predictions) self.assertTrue(all(e.job == self.job for e in predictions)) - def _size_filter_result(self, job: Job | None) -> OccurrenceTimelineEntry: - AlgorithmResult.objects.record( - occurrence=self.occurrence, - algorithm=self.size_filter, - job=job, - kind=SIZE_FILTER, - data={"relative_size": 0.001}, - ) - return next(e for e in occurrence_timeline(self.occurrence) if e.type == "algorithm_result") - - def test_job_settings_take_labels_and_references_from_the_task_config_schema(self): - taxa_list = TaxaList.objects.create(name="Kept species") - classifier = Algorithm.objects.create(name="Classifier", key="settings-test-classifier") - self.job.params = { - "task": "class_masking", - "config": { - "source_image_collection_id": None, - "occurrence_id": True, - "taxa_list_id": taxa_list.pk, - "algorithm_id": classifier.pk, - "reweight": True, - }, - } - self.job.save() - self.assertEqual( - self._size_filter_result(self.job).job_settings, - [ - JobSetting("source_image_collection_id", "Capture set", None, None), - # A value that is not an id names no record, whatever the schema declares. - JobSetting("occurrence_id", "Occurrence", True, None), - JobSetting( - "taxa_list_id", "Species list", taxa_list.pk, Ref("taxa_list", taxa_list.pk, "Kept species") - ), - JobSetting("algorithm_id", "Classifier", classifier.pk, Ref("algorithm", classifier.pk, "Classifier")), - JobSetting("reweight", "Re-weighted scores", True, None), - ], - ) - - def test_a_job_without_a_registered_task_shows_its_settings_by_key_with_no_references(self): - self.job.params = {"config": {"taxa_list_id": 3, "size_threshold": 0.01}} - self.job.save() - self.assertEqual( - self._size_filter_result(self.job).job_settings, - [ - JobSetting("taxa_list_id", "taxa_list_id", 3, None), - JobSetting("size_threshold", "size_threshold", 0.01), - ], - ) - - def test_a_job_whose_params_are_not_an_object_has_no_settings(self): - Job.objects.filter(pk=self.job.pk).update(params=["not", "an", "object"]) - self.job.refresh_from_db() - self.assertEqual(self._size_filter_result(self.job).job_settings, []) - result = next(e for e in self.get() if e["type"] == "algorithm_result") - self.assertIsNone(result["job"]["config"]) - - def test_a_result_without_a_job_has_no_settings_or_references(self): + def test_a_result_without_a_job_has_no_config(self): AlgorithmResult.objects.record( occurrence=self.occurrence, algorithm=self.size_filter, @@ -554,7 +429,7 @@ def test_a_result_without_a_job_has_no_settings_or_references(self): ) entry = next(e for e in occurrence_timeline(self.occurrence) if e.type == "algorithm_result") self.assertIsNone(entry.job) - self.assertEqual(entry.job_settings, []) + self.assertEqual(entry.job_config, []) def test_a_result_of_a_kind_no_longer_registered_still_shows_in_the_history(self): """A row written by a branch with an extra kind, or before a kind was renamed, must not break the history.""" @@ -564,21 +439,3 @@ def test_a_result_of_a_kind_no_longer_registered_still_shows_in_the_history(self AlgorithmResult.objects.filter(pk=result.pk).update(kind="rank_rollup") entry = next(e for e in self.get() if e["type"] == "algorithm_result") self.assertEqual(entry["kind"], "rank_rollup") - - -class ClassificationAdminTestCase(OccurrenceFixtureTestCase): - """The classification change page must not preload every algorithm result into a select.""" - - def test_the_algorithm_result_field_is_a_raw_id_input(self): - superuser = User.objects.create_superuser(email="admin-history@insectai.org") # type: ignore - algorithm = Algorithm.objects.create(name="Size filter", key="size-filter-admin-test") - result = AlgorithmResult.objects.record( - occurrence=self.occurrence, algorithm=algorithm, kind=SIZE_FILTER, data={"relative_size": 0.001} - ) - classification = Classification.objects.get(detection=self.detections[0]) - Classification.objects.filter(pk=classification.pk).update(algorithm_result=result) - self.client.force_login(superuser) - response = self.client.get(f"/admin/main/classification/{classification.pk}/change/") - self.assertEqual(response.status_code, 200) - self.assertNotContains(response, '