From 2361de9b58beadb11bf04e6dae03fd27f9f9dc83 Mon Sep 17 00:00:00 2001 From: Tarkadia <161824332+tarkadia@users.noreply.github.com> Date: Thu, 23 Apr 2026 15:30:45 +0200 Subject: [PATCH 1/5] Fix Perf for ComplianceAssessment objects Processing --- backend/core/models.py | 72 +++++++++++++++---- .../core/tests/test_implementation_groups.py | 62 ++++++++++++++++ 2 files changed, 122 insertions(+), 12 deletions(-) diff --git a/backend/core/models.py b/backend/core/models.py index 010d9e8a3a..8ce31ad7a9 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -7841,20 +7841,68 @@ def assign_attributes(target, attributes): ) return requirement_assessments, assessment_source_dict - @property - def progress(self) -> int: - requirement_assessments = list( - self.get_requirement_assessments(include_non_assessable=False) + def _get_progress_counts(self) -> tuple[int, int]: + """ + Return (total, assessed) counts for assessable requirements. + + This keeps the progress calculation lightweight by avoiding the + heavy prefetch graph used to render a full assessment tree. + """ + requirements = RequirementAssessment.objects.filter( + compliance_assessment=self, requirement__assessable=True ) - total_cnt = len(requirement_assessments) - assessed_cnt = len( - [ - r - for r in requirement_assessments - if (r.result != RequirementAssessment.Result.NOT_ASSESSED) - or r.score != None - ] + + if not self.selected_implementation_groups: + # Fast path: when no IG filter is active, the DB can compute both + # counters directly without hydrating the requirement tree. + counts = requirements.aggregate( + total=Count("id"), + assessed=Count( + "id", + filter=~Q(result=RequirementAssessment.Result.NOT_ASSESSED) + | Q(score__isnull=False), + ), + ) + return counts["total"] or 0, counts["assessed"] or 0 + + selected_groups = set(self.selected_implementation_groups) + total = 0 + assessed = 0 + lightweight_requirements = ( + # IG membership lives in a JSONField. To stay DB-portable, stream only + # the small set of fields needed for the progress calculation and + # apply the set-intersection in Python. + requirements.select_related("requirement") + .only( + "result", + "score", + "requirement_id", + "requirement__implementation_groups", + ) + .iterator() ) + + for requirement_assessment in lightweight_requirements: + requirement_groups = set( + requirement_assessment.requirement.implementation_groups or [] + ) + if not (selected_groups & requirement_groups): + continue + + total += 1 + # Keep the existing semantics: a requirement counts as "assessed" + # as soon as it has either a non-default result or an explicit score. + if ( + requirement_assessment.result + != RequirementAssessment.Result.NOT_ASSESSED + ) or requirement_assessment.score is not None: + assessed += 1 + + return total, assessed + + @property + def progress(self) -> int: + total_cnt, assessed_cnt = self._get_progress_counts() return int((assessed_cnt / total_cnt) * 100) if total_cnt > 0 else 0 @property diff --git a/backend/core/tests/test_implementation_groups.py b/backend/core/tests/test_implementation_groups.py index 2f2dcbdd9d..d79e1c5223 100644 --- a/backend/core/tests/test_implementation_groups.py +++ b/backend/core/tests/test_implementation_groups.py @@ -540,3 +540,65 @@ def test_upsert_daily_metrics_with_selected_ig(self, dynamic_framework_setup): # save() calls upsert_daily_metrics() internally — must not crash d["ca"].refresh_from_db() assert d["ca"].selected_implementation_groups == ["base"] + + +@pytest.mark.django_db +class TestComplianceAssessmentProgress: + def test_progress_counts_score_only_requirements_as_assessed( + self, dynamic_framework_setup + ): + d = dynamic_framework_setup + d["ra"].result = RequirementAssessment.Result.NOT_ASSESSED + d["ra"].score = 10 + d["ra"].save(update_fields=["result", "score"]) + + second_requirement = RequirementNode.objects.create( + framework=d["framework"], + urn="urn:test:ig:req:002", + ref_id="IG-REQ-2", + assessable=True, + folder=d["folder"], + is_published=True, + ) + RequirementAssessment.objects.create( + compliance_assessment=d["ca"], + requirement=second_requirement, + folder=d["folder"], + result=RequirementAssessment.Result.NOT_ASSESSED, + ) + + # One assessed requirement out of two assessable ones -> 50%. + assert d["ca"].progress == 50 + + def test_progress_filters_selected_implementation_groups_lightweight( + self, dynamic_framework_setup + ): + d = dynamic_framework_setup + d["requirement_node"].implementation_groups = ["base"] + d["requirement_node"].save(update_fields=["implementation_groups"]) + d["ra"].result = RequirementAssessment.Result.NOT_ASSESSED + d["ra"].score = None + d["ra"].save(update_fields=["result", "score"]) + + advanced_requirement = RequirementNode.objects.create( + framework=d["framework"], + urn="urn:test:ig:req:003", + ref_id="IG-REQ-3", + assessable=True, + implementation_groups=["advanced"], + folder=d["folder"], + is_published=True, + ) + RequirementAssessment.objects.create( + compliance_assessment=d["ca"], + requirement=advanced_requirement, + folder=d["folder"], + result=RequirementAssessment.Result.COMPLIANT, + ) + + # Only the "base" requirement is in scope, and it is still untouched. + d["ca"].selected_implementation_groups = ["base"] + d["ca"].save() + d["ca"].refresh_from_db() + + assert d["ca"].progress == 0 From 8887753a59710b7351c163e5ad704373aa1aa9a0 Mon Sep 17 00:00:00 2001 From: Tarkadia <161824332+tarkadia@users.noreply.github.com> Date: Thu, 23 Apr 2026 15:31:38 +0200 Subject: [PATCH 2/5] Remove Test for my specific changes --- .../core/tests/test_implementation_groups.py | 62 ------------------- 1 file changed, 62 deletions(-) diff --git a/backend/core/tests/test_implementation_groups.py b/backend/core/tests/test_implementation_groups.py index d79e1c5223..2f2dcbdd9d 100644 --- a/backend/core/tests/test_implementation_groups.py +++ b/backend/core/tests/test_implementation_groups.py @@ -540,65 +540,3 @@ def test_upsert_daily_metrics_with_selected_ig(self, dynamic_framework_setup): # save() calls upsert_daily_metrics() internally — must not crash d["ca"].refresh_from_db() assert d["ca"].selected_implementation_groups == ["base"] - - -@pytest.mark.django_db -class TestComplianceAssessmentProgress: - def test_progress_counts_score_only_requirements_as_assessed( - self, dynamic_framework_setup - ): - d = dynamic_framework_setup - d["ra"].result = RequirementAssessment.Result.NOT_ASSESSED - d["ra"].score = 10 - d["ra"].save(update_fields=["result", "score"]) - - second_requirement = RequirementNode.objects.create( - framework=d["framework"], - urn="urn:test:ig:req:002", - ref_id="IG-REQ-2", - assessable=True, - folder=d["folder"], - is_published=True, - ) - RequirementAssessment.objects.create( - compliance_assessment=d["ca"], - requirement=second_requirement, - folder=d["folder"], - result=RequirementAssessment.Result.NOT_ASSESSED, - ) - - # One assessed requirement out of two assessable ones -> 50%. - assert d["ca"].progress == 50 - - def test_progress_filters_selected_implementation_groups_lightweight( - self, dynamic_framework_setup - ): - d = dynamic_framework_setup - d["requirement_node"].implementation_groups = ["base"] - d["requirement_node"].save(update_fields=["implementation_groups"]) - d["ra"].result = RequirementAssessment.Result.NOT_ASSESSED - d["ra"].score = None - d["ra"].save(update_fields=["result", "score"]) - - advanced_requirement = RequirementNode.objects.create( - framework=d["framework"], - urn="urn:test:ig:req:003", - ref_id="IG-REQ-3", - assessable=True, - implementation_groups=["advanced"], - folder=d["folder"], - is_published=True, - ) - RequirementAssessment.objects.create( - compliance_assessment=d["ca"], - requirement=advanced_requirement, - folder=d["folder"], - result=RequirementAssessment.Result.COMPLIANT, - ) - - # Only the "base" requirement is in scope, and it is still untouched. - d["ca"].selected_implementation_groups = ["base"] - d["ca"].save() - d["ca"].refresh_from_db() - - assert d["ca"].progress == 0 From 853d3c7e530530e7ab28e907aaaea1b0921feba3 Mon Sep 17 00:00:00 2001 From: Tarkadia <161824332+tarkadia@users.noreply.github.com> Date: Fri, 24 Apr 2026 12:59:57 +0200 Subject: [PATCH 3/5] Fix Comments --- backend/core/models.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/backend/core/models.py b/backend/core/models.py index 8ce31ad7a9..40efb75b5d 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -7843,18 +7843,14 @@ def assign_attributes(target, attributes): def _get_progress_counts(self) -> tuple[int, int]: """ - Return (total, assessed) counts for assessable requirements. - - This keeps the progress calculation lightweight by avoiding the - heavy prefetch graph used to render a full assessment tree. + Return (total, assessed) counts for assessable requirements """ + requirements = RequirementAssessment.objects.filter( compliance_assessment=self, requirement__assessable=True ) if not self.selected_implementation_groups: - # Fast path: when no IG filter is active, the DB can compute both - # counters directly without hydrating the requirement tree. counts = requirements.aggregate( total=Count("id"), assessed=Count( @@ -7869,9 +7865,6 @@ def _get_progress_counts(self) -> tuple[int, int]: total = 0 assessed = 0 lightweight_requirements = ( - # IG membership lives in a JSONField. To stay DB-portable, stream only - # the small set of fields needed for the progress calculation and - # apply the set-intersection in Python. requirements.select_related("requirement") .only( "result", @@ -7890,8 +7883,6 @@ def _get_progress_counts(self) -> tuple[int, int]: continue total += 1 - # Keep the existing semantics: a requirement counts as "assessed" - # as soon as it has either a non-default result or an explicit score. if ( requirement_assessment.result != RequirementAssessment.Result.NOT_ASSESSED From b1454aa2f9db670abb1d65b2e79d521ea0592f13 Mon Sep 17 00:00:00 2001 From: Tarkadia <161824332+tarkadia@users.noreply.github.com> Date: Fri, 24 Apr 2026 13:30:04 +0200 Subject: [PATCH 4/5] Simplify Code --- backend/core/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/core/models.py b/backend/core/models.py index 40efb75b5d..a77f25f122 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -7859,7 +7859,7 @@ def _get_progress_counts(self) -> tuple[int, int]: | Q(score__isnull=False), ), ) - return counts["total"] or 0, counts["assessed"] or 0 + return counts["total"], counts["assessed"] selected_groups = set(self.selected_implementation_groups) total = 0 @@ -7879,7 +7879,7 @@ def _get_progress_counts(self) -> tuple[int, int]: requirement_groups = set( requirement_assessment.requirement.implementation_groups or [] ) - if not (selected_groups & requirement_groups): + if selected_groups.isdisjoint(requirement_groups): continue total += 1 From cf14a96ad27e220d43ca39ecb06dd054ec713e13 Mon Sep 17 00:00:00 2001 From: Tarkadia <161824332+tarkadia@users.noreply.github.com> Date: Fri, 24 Apr 2026 13:32:48 +0200 Subject: [PATCH 5/5] Formatting --- backend/core/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/core/models.py b/backend/core/models.py index a77f25f122..175978d656 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -7845,7 +7845,7 @@ def _get_progress_counts(self) -> tuple[int, int]: """ Return (total, assessed) counts for assessable requirements """ - + requirements = RequirementAssessment.objects.filter( compliance_assessment=self, requirement__assessable=True )