Skip to content

Commit

Permalink
Merge pull request #5002 from kobotoolbox/fix-attachment-link-keyerror
Browse files Browse the repository at this point in the history 
[TASK-880] Fix 500 error when filtering fields with data endpoint
  • Loading branch information
@rajpatel24
rajpatel24 authored Jul 11, 2024
2 parents 794dab3 + a1cf23b commit fd68e8a
Show file tree
Hide file tree
Showing 2 changed files with 100 additions and 22 deletions.
105 changes: 85 additions & 20 deletions kpi/tests/test_mock_data_exports.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -579,10 +579,10 @@ def test_csv_export_hierarchy_in_labels(self):
def test_csv_export_filter_fields(self):
export_options = {'fields': ["start", "end", "Do_you_descend_from_unicellular_organism", "_index"]}
expected_lines = [
'"start";"end";"Do you descend from an ancestral unicellular organism?";"_index"',
'"2017-10-23T05:40:39.000-04:00";"2017-10-23T05:41:13.000-04:00";"No";"1"',
'"2017-10-23T05:41:14.000-04:00";"2017-10-23T05:41:32.000-04:00";"No";"2"',
'"2017-10-23T05:41:32.000-04:00";"2017-10-23T05:42:05.000-04:00";"Yes";"3"',
'"start";"end";"Do you descend from an ancestral unicellular organism?";"_uuid";"_index"',
'"2017-10-23T05:40:39.000-04:00";"2017-10-23T05:41:13.000-04:00";"No";"48583952-1892-4931-8d9c-869e7b49bafb";"1"',
'"2017-10-23T05:41:14.000-04:00";"2017-10-23T05:41:32.000-04:00";"No";"317ba7b7-bea4-4a8c-8620-a483c3079c4b";"2"',
'"2017-10-23T05:41:32.000-04:00";"2017-10-23T05:42:05.000-04:00";"Yes";"3f15cdfe-3eab-4678-8352-7806febf158d";"3"',
]
self.run_csv_export_test(expected_lines, export_options)

Expand Down Expand Up @@ -647,32 +647,97 @@ def test_xls_export_multiple_select_details(self):
self.run_xls_export_test(expected_data, export_options)

def test_xls_export_filter_fields(self):
export_options = {'fields': ['start', 'end', 'Do_you_descend_from_unicellular_organism', '_index']}
expected_data = {self.asset.name: [
['start', 'end', 'Do you descend from an ancestral unicellular organism?', '_index'],
['2017-10-23T05:40:39.000-04:00', '2017-10-23T05:41:13.000-04:00', 'No', 1.0],
['2017-10-23T05:41:14.000-04:00', '2017-10-23T05:41:32.000-04:00', 'No', 2.0],
['2017-10-23T05:41:32.000-04:00', '2017-10-23T05:42:05.000-04:00', 'Yes', 3.0],
]}
export_options = {
'fields': [
'start',
'end',
'Do_you_descend_from_unicellular_organism',
'_index',
]
}
expected_data = {
self.asset.name: [
[
'start',
'end',
'Do you descend from an ancestral unicellular organism?',
'_uuid',
'_index',
],
[
'2017-10-23T05:40:39.000-04:00',
'2017-10-23T05:41:13.000-04:00',
'No',
'48583952-1892-4931-8d9c-869e7b49bafb',
1.0,
],
[
'2017-10-23T05:41:14.000-04:00',
'2017-10-23T05:41:32.000-04:00',
'No',
'317ba7b7-bea4-4a8c-8620-a483c3079c4b',
2.0,
],
[
'2017-10-23T05:41:32.000-04:00',
'2017-10-23T05:42:05.000-04:00',
'Yes',
'3f15cdfe-3eab-4678-8352-7806febf158d',
3.0,
],
]
}
self.run_xls_export_test(expected_data, export_options)

def test_xls_export_filter_fields_without_index(self):
export_options = {'fields': ['start', 'end', 'Do_you_descend_from_unicellular_organism']}
expected_data = {self.asset.name: [
['start', 'end', 'Do you descend from an ancestral unicellular organism?'],
['2017-10-23T05:40:39.000-04:00', '2017-10-23T05:41:13.000-04:00', 'No'],
['2017-10-23T05:41:14.000-04:00', '2017-10-23T05:41:32.000-04:00', 'No'],
['2017-10-23T05:41:32.000-04:00', '2017-10-23T05:42:05.000-04:00', 'Yes'],
]}
export_options = {
'fields': [
'start',
'end',
'Do_you_descend_from_unicellular_organism',
]
}
expected_data = {
self.asset.name: [
[
'start',
'end',
'Do you descend from an ancestral unicellular organism?',
'_uuid',
],
[
'2017-10-23T05:40:39.000-04:00',
'2017-10-23T05:41:13.000-04:00',
'No',
'48583952-1892-4931-8d9c-869e7b49bafb',
],
[
'2017-10-23T05:41:14.000-04:00',
'2017-10-23T05:41:32.000-04:00',
'No',
'317ba7b7-bea4-4a8c-8620-a483c3079c4b',
],
[
'2017-10-23T05:41:32.000-04:00',
'2017-10-23T05:42:05.000-04:00',
'Yes',
'3f15cdfe-3eab-4678-8352-7806febf158d',
],
]
}
self.run_xls_export_test(expected_data, export_options)

def test_xls_export_filter_fields_with_media_url(self):
asset_name = 'Simple media'
export_options = {'fields': ['an_image'], 'include_media_url': True}
expected_data = {
asset_name: [
['Submit an image', 'Submit an image_URL'],
['image.png', 'http://testserver/image.png'],
['Submit an image', 'Submit an image_URL', '_uuid'],
[
'image.png',
'http://testserver/image.png',
'f80be949-89b5-4af1-a42d-7d292b2bc0cd',
],
]
}
self.run_xls_export_test(
Expand Down
17 changes: 15 additions & 2 deletions kpi/utils/mongo_helper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ class MongoHelper:

# Match KoBoCAT's variables of ParsedInstance class
USERFORM_ID = '_userform_id'
SUBMISSION_UUID = '_uuid'
DEFAULT_BATCHSIZE = 1000

@classmethod
Expand Down Expand Up @@ -340,7 +341,7 @@ def _convert_permissions(
def _get_cursor_and_count(
cls,
mongo_userform_id,
fields: Optional[dict] = None,
fields: Optional[list, dict] = None,
query: Optional[dict] = None,
submission_ids: Optional[list] = None,
permission_filters=None,
Expand All @@ -361,10 +362,22 @@ def _get_cursor_and_count(
query = cls.to_safe_dict(query, reading=True)

if fields is not None and len(fields) > 0:
# `cls.SUBMISSION_UUID` is mandatory.
# It is needed to build the attachment link on fly on API response
if cls.SUBMISSION_UUID not in fields:
if isinstance(fields, list):
fields.append(cls.SUBMISSION_UUID)
else:
fields[cls.SUBMISSION_UUID] = 1

# Retrieve only specified fields from Mongo. Remove
# `cls.USERFORM_ID` from those fields in case users try to add it.
if cls.USERFORM_ID in fields:
fields.remove(cls.USERFORM_ID)
if isinstance(fields, list):
fields.remove(cls.USERFORM_ID)
else:
del fields[cls.USERFORM_ID]

fields_to_select = dict(
[(cls.encode(field), 1) for field in fields]
)
Expand Down

0 comments on commit fd68e8a

Please sign in to comment.