Merge pull request #5105 from kobotoolbox/fix-bad-performance-with-fo…

…rmdisclaimer

Fix bad performance introduced in `2.024.25`

kobo/apps/openrosa/apps/logger/migrations/0035_add_index_on_logger_xform__kpi_asset_uid.py

@@ -0,0 +1,56 @@
+# Generated by Django 4.2.15 on 2024-09-11 19:33
+
+from django.conf import settings
+from django.db import migrations, models
+
+
+def manually_create_indexes_instructions(apps, schema_editor):
+    print(
+        """
+        !!! ATTENTION !!!
+        If you have existing projects you need to run the SQL queries below in PostgreSQL directly:
+
+           > CREATE INDEX CONCURRENTLY "logger_xform_kpi_asset_uid_e5e6b08d" ON "logger_xform" ("kpi_asset_uid");
+           > CREATE INDEX CONCURRENTLY "logger_xform_kpi_asset_uid_e5e6b08d_like" ON "logger_xform" ("kpi_asset_uid" varchar_pattern_ops);
+
+
+        Otherwise, project views will perform very poorly.
+        """
+    )
+
+
+def manually_drop_indexes_instructions(apps, schema_editor):
+    print(
+        """
+        !!! ATTENTION !!!
+        Run the SQL queries below in PostgreSQL directly:
+
+           > DROP INDEX IF EXISTS "logger_xform_kpi_asset_uid_e5e6b08d_like";
+           > DROP INDEX IF EXISTS "logger_xform_kpi_asset_uid_e5e6b08d";
+
+        """
+    )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('logger', '0034_set_require_auth_at_project_level'),
+    ]
+
+    if settings.SKIP_HEAVY_MIGRATIONS:
+        operations = [
+            migrations.RunPython(
+                manually_create_indexes_instructions,
+                manually_drop_indexes_instructions,
+            )
+        ]
+    else:
+        operations = [
+            migrations.AlterField(
+                model_name='xform',
+                name='kpi_asset_uid',
+                field=models.CharField(db_index=True, max_length=32, null=True),
+            ),
+        ]

kobo/apps/openrosa/apps/logger/models/xform.py

@@ -101,7 +101,7 @@ class XForm(BaseModel):
     tags = TaggableManager()
 
     has_kpi_hooks = LazyDefaultBooleanField(default=False)
-    kpi_asset_uid = models.CharField(max_length=32, null=True)
+    kpi_asset_uid = models.CharField(max_length=32, null=True, db_index=True)
     pending_delete = models.BooleanField(default=False)
 
     class Meta:
@@ -132,11 +132,16 @@ def asset(self):
         See kpi.utils.xml.XMLFormWithDisclaimer for more details.
         """
         if not hasattr(self, '_cache_asset'):
+            # We only need to load the PK because XMLFormWithDisclaimer
+            # uses an Asset object only to narrow down a query with a filter,
+            # thus uses only asset PK
             try:
-                asset = Asset.objects.get(uid=self.kpi_asset_uid)
+                asset = Asset.objects.only('pk').get(uid=self.kpi_asset_uid)
             except Asset.DoesNotExist:
                 try:
-                    asset = Asset.objects.get(_deployment_data__formid=self.pk)
+                    asset = Asset.objects.only('pk').get(
+                        _deployment_data__formid=self.pk
+                    )
                 except Asset.DoesNotExist:
                     # An `Asset` object needs to be returned to avoid 500 while
                     # Enketo is fetching for project XML (e.g: /formList, /manifest)

kpi/migrations/0057_add_index_on_kpi_asset__deployment_data__formid.py

@@ -0,0 +1,55 @@
+# Generated by Django 4.2.15 on 2024-09-11 19:21
+
+import django.contrib.postgres.indexes
+from django.conf import settings
+from django.db import migrations, models
+
+
+def manually_create_indexes_instructions(apps, schema_editor):
+    print(
+        """
+        !!! ATTENTION !!!
+        If you have existing projects you need to run the SQL queries below in PostgreSQL directly:
+
+           > CREATE INDEX CONCURRENTLY "deployment_data__formid_idx" ON kpi_asset USING btree ((("_deployment_data" -> 'formid')));
+
+        Otherwise, project views will perform very poorly.
+        """
+    )
+
+
+def manually_drop_indexes_instructions(apps, schema_editor):
+    print(
+        """
+        !!! ATTENTION !!!
+        Run the SQL queries below in PostgreSQL directly:
+
+           > DROP INDEX IF EXISTS "deployment_data__formid_idx";
+
+        """
+    )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('kpi', '0056_fix_add_submission_bad_permission_assignment'),
+    ]
+
+    if settings.SKIP_HEAVY_MIGRATIONS:
+        operations = [
+            migrations.RunPython(
+                manually_create_indexes_instructions,
+                manually_drop_indexes_instructions,
+            )
+        ]
+    else:
+        operations = [
+            migrations.AddIndex(
+                model_name='asset',
+                index=django.contrib.postgres.indexes.BTreeIndex(
+                    models.F('_deployment_data__formid'),
+                    name='deployment_data__formid_idx',
+                ),
+            ),
+        ]

kpi/models/asset.py

@@ -8,7 +8,7 @@
 
 from django.conf import settings
 from django.contrib.auth.models import Permission
-from django.contrib.postgres.indexes import GinIndex
+from django.contrib.postgres.indexes import BTreeIndex, GinIndex
 from django.db import models
 from django.db import transaction
 from django.db.models import Prefetch, Q, F
@@ -231,6 +231,9 @@ class Meta:
             GinIndex(
                 F('settings__country_codes'), name='settings__country_codes_idx'
             ),
+            BTreeIndex(
+                F('_deployment_data__formid'), name='deployment_data__formid_idx'
+            ),
         ]
 
         # Example in Django documentation  represents `ordering` as a list