Merge branch 'bugfix/conditional_prob_data' into 'master'

Conditional probability data update

See merge request caimira/caimira!499

caimira/apps/calculator/__init__.py

@@ -193,7 +193,7 @@ async def post(self) -> None:
         )
         # Re-generate the report with the conditional probability of infection plot
         if self.get_cookie('conditional_plot'):
-            form.conditional_probability_plot = True if self.get_cookie('conditional_plot') == '1' else False
+            form.conditional_probability_viral_loads = True if self.get_cookie('conditional_plot') == '1' else False
             self.clear_cookie('conditional_plot') # Clears cookie after changing the form value.
 
         report_task = executor.submit(

caimira/apps/calculator/defaults.py

@@ -15,7 +15,6 @@
     'precise_activity': '{}',
     'calculator_version': NO_DEFAULT,
     'ceiling_height': 0.,
-    'conditional_probability_plot': False,
     'conditional_probability_viral_loads': False,
     'CO2_fitting_result': '{}',
     'exposed_coffee_break_option': 'coffee_break_0',

caimira/apps/calculator/model_generator.py

@@ -31,7 +31,6 @@ class VirusFormData(FormData):
     arve_sensors_option: bool
     precise_activity: dict
     ceiling_height: float
-    conditional_probability_plot: bool
     conditional_probability_viral_loads: bool
     CO2_fitting_result: dict
     floor_area: float
@@ -497,7 +496,6 @@ def baseline_raw_form_data() -> typing.Dict[str, typing.Union[str, float]]:
         'air_changes': '',
         'air_supply': '',
         'ceiling_height': '',
-        'conditional_probability_plot': '0',
         'conditional_probability_viral_loads': '0',
         'exposed_coffee_break_option': 'coffee_break_4',
         'exposed_coffee_duration': '10',

caimira/apps/calculator/report_generator.py

@@ -171,25 +171,14 @@ def calculate_report_data(form: VirusFormData, model: models.ExposureModel, exec
     expected_new_cases = np.array(model.expected_new_cases()).mean()
     exposed_presence_intervals = [list(interval) for interval in model.exposed.presence_interval().boundaries()]
 
-    if (model.data_registry.virological_data['virus_distributions'][form.virus_type]['viral_load_in_sputum'] == ViralLoads.COVID_OVERALL.value # type: ignore
-        and form.conditional_probability_plot): # Only generate this data if covid_overall_vl_data is selected.
-
-        viral_load_in_sputum: models._VectorisedFloat = model.concentration_model.infected.virus.viral_load_in_sputum
-        viral_loads, pi_means, lower_percentiles, upper_percentiles = manufacture_conditional_probability_data(model, prob)
-
-        uncertainties_plot_src = img2base64(_figure2bytes(uncertainties_plot(prob, viral_load_in_sputum, viral_loads, 
-                                                                             pi_means, lower_percentiles, upper_percentiles)))
-        conditional_probability_data = {key: value for key, value in
-                                    zip(('viral_loads', 'pi_means', 'lower_percentiles', 'upper_percentiles'),
-                                    (viral_loads, pi_means, lower_percentiles, upper_percentiles))}
-        vl_dist = list(np.log10(viral_load_in_sputum))
-
-    else:
-        uncertainties_plot_src = None
-        conditional_probability_data = None
-        vl = model.concentration_model.virus.viral_load_in_sputum
-        if isinstance(vl, np.ndarray): vl_dist = list(np.log10(model.concentration_model.virus.viral_load_in_sputum))
-        else: vl_dist = np.log10(model.concentration_model.virus.viral_load_in_sputum)
+    conditional_probability_data = None
+    uncertainties_plot_src = None
+    if (form.conditional_probability_viral_loads and 
+        model.data_registry.virological_data['virus_distributions'][form.virus_type]['viral_load_in_sputum'] == ViralLoads.COVID_OVERALL.value): # type: ignore
+            # Generate all the required data for the conditional probability plot
+            conditional_probability_data = manufacture_conditional_probability_data(model, prob)
+            # Generate the matplotlib image based on the received data
+            uncertainties_plot_src = img2base64(_figure2bytes(uncertainties_plot(prob, conditional_probability_data)))
 
     return {
         "model_repr": repr(model),
@@ -208,10 +197,9 @@ def calculate_report_data(form: VirusFormData, model: models.ExposureModel, exec
         "prob_hist_bins": list(prob_dist_bins),
         "prob_probabilistic_exposure": prob_probabilistic_exposure,
         "expected_new_cases": expected_new_cases,
-        "uncertainties_plot_src": uncertainties_plot_src,
         "CO2_concentrations": CO2_concentrations,
-        "vl_dist": vl_dist,
         "conditional_probability_data": conditional_probability_data,
+        "uncertainties_plot_src": uncertainties_plot_src,
     }
 
 
@@ -235,7 +223,6 @@ def generate_permalink(base_url, get_root_url,  get_root_calculator_url, form: V
 
 
 def conditional_prob_inf_given_vl_dist(
-        data_registry: DataRegistry,
         infection_probability: models._VectorisedFloat,
         viral_loads: np.ndarray,
         specific_vl: float,
@@ -247,7 +234,9 @@ def conditional_prob_inf_given_vl_dist(
     upper_percentiles = []
 
     for vl_log in viral_loads:
+        # Probability of infection corresponding to a certain viral load value in the distribution
         specific_prob = infection_probability[np.where((vl_log-step/2-specific_vl)*(vl_log+step/2-specific_vl)<0)[0]] #type: ignore
+
         pi_means.append(specific_prob.mean())
         lower_percentiles.append(np.quantile(specific_prob, 0.05))
         upper_percentiles.append(np.quantile(specific_prob, 0.95))
@@ -259,69 +248,74 @@ def manufacture_conditional_probability_data(
     exposure_model: models.ExposureModel,
     infection_probability: models._VectorisedFloat
 ):
-    data_registry: DataRegistry = exposure_model.data_registry
-
     min_vl = 2
     max_vl = 10
     step = (max_vl - min_vl)/100
     viral_loads = np.arange(min_vl, max_vl, step)
     specific_vl = np.log10(exposure_model.concentration_model.virus.viral_load_in_sputum)
-    pi_means, lower_percentiles, upper_percentiles = conditional_prob_inf_given_vl_dist(data_registry, infection_probability, viral_loads,
+    pi_means, lower_percentiles, upper_percentiles = conditional_prob_inf_given_vl_dist(infection_probability, viral_loads,
                                                                                         specific_vl, step)
-
-    return list(viral_loads), list(pi_means), list(lower_percentiles), list(upper_percentiles)
+    log10_vl_in_sputum = np.log10(exposure_model.concentration_model.infected.virus.viral_load_in_sputum)
+
+    return {
+        'viral_loads': list(viral_loads), 
+        'pi_means': list(pi_means), 
+        'lower_percentiles': list(lower_percentiles),
+        'upper_percentiles': list(upper_percentiles),
+        'log10_vl_in_sputum': list(log10_vl_in_sputum),
+    }
 
 
 def uncertainties_plot(infection_probability: models._VectorisedFloat,
-                       viral_load_in_sputum: models._VectorisedFloat,
-                       viral_loads: models._VectorisedFloat,
-                       pi_means: models._VectorisedFloat,
-                       lower_percentiles: models._VectorisedFloat,
-                       upper_percentiles: models._VectorisedFloat):
+                       conditional_probability_data: dict):
 
-    fig, axes = plt.subplots(2, 3, 
+    viral_loads: list = conditional_probability_data['viral_loads']
+    pi_means: list = conditional_probability_data['pi_means']
+    lower_percentiles: list = conditional_probability_data['lower_percentiles']
+    upper_percentiles: list = conditional_probability_data['upper_percentiles']
+    log10_vl_in_sputum: list = conditional_probability_data['log10_vl_in_sputum']
+
+    fig, ((axs00, axs01, axs02), (axs10, axs11, axs12)) = plt.subplots(nrows=2, ncols=3, # type: ignore
         gridspec_kw={'width_ratios': [5, 0.5] + [1],
             'height_ratios': [3, 1], 'wspace': 0},
         sharey='row',
         sharex='col')
 
-    # Type hint for axs
-    axs: np.ndarray = np.array(axes)
-
-    for y, x in [(0, 1)] + [(1, i + 1) for i in range(2)]:
-        axs[y, x].axis('off')
+    axs01.axis('off')
+    axs11.axis('off')
+    axs12.axis('off')
 
-    axs[0, 1].set_visible(False)
+    axs01.set_visible(False)
 
-    axs[0, 0].plot(viral_loads, np.array(pi_means)/100, label='Predictive total probability')
-    axs[0, 0].fill_between(viral_loads, np.array(lower_percentiles)/100, np.array(upper_percentiles)/100, alpha=0.1, label='5ᵗʰ and 95ᵗʰ percentile')
+    axs00.plot(viral_loads, np.array(pi_means), label='Predictive total probability')
+    axs00.fill_between(viral_loads, np.array(lower_percentiles), np.array(upper_percentiles), alpha=0.1, label='5ᵗʰ and 95ᵗʰ percentile')
 
-    axs[0, 2].hist(infection_probability, bins=30, orientation='horizontal')
-    axs[0, 2].set_xticks([])
-    axs[0, 2].set_xticklabels([])
-    axs[0, 2].set_facecolor("lightgrey")
+    axs02.hist(infection_probability, bins=30, orientation='horizontal')
+    axs02.set_xticks([])
+    axs02.set_xticklabels([])
+    axs02.set_facecolor("lightgrey")
 
-    highest_bar = axs[0, 2].get_xlim()[1]
-    axs[0, 2].set_xlim(0, highest_bar)
+    highest_bar = axs02.get_xlim()[1]
+    axs02.set_xlim(0, highest_bar)
 
-    axs[0, 2].text(highest_bar * 0.5, 0.5,
-                        rf"$\bf{np.round(np.mean(infection_probability), 1)}$%", ha='center', va='center')
-    axs[1, 0].hist(np.log10(viral_load_in_sputum),
+    axs02.text(highest_bar * 0.5, 50,
+                        "$P(I)=$\n" + rf"$\bf{np.round(np.mean(infection_probability), 1)}$%", ha='center', va='center')
+    axs10.hist(log10_vl_in_sputum,
                     bins=150, range=(2, 10), color='grey')
-    axs[1, 0].set_facecolor("lightgrey")
-    axs[1, 0].set_yticks([])
-    axs[1, 0].set_yticklabels([])
-    axs[1, 0].set_xticks([i for i in range(2, 13, 2)])
-    axs[1, 0].set_xticklabels(['$10^{' + str(i) + '}$' for i in range(2, 13, 2)])
-    axs[1, 0].set_xlim(2, 10)
-    axs[1, 0].set_xlabel('Viral load\n(RNA copies)', fontsize=12)
-    axs[0, 0].set_ylabel('Conditional Probability\nof Infection', fontsize=12)
-
-    axs[0, 0].text(9.5, -0.01, '$(i)$')
-    axs[1, 0].text(9.5, axs[1, 0].get_ylim()[1] * 0.8, '$(ii)$')
-    axs[0, 2].set_title('$(iii)$', fontsize=10)
-
-    axs[0, 0].legend()
+    axs10.set_facecolor("lightgrey")
+    axs10.set_yticks([])
+    axs10.set_yticklabels([])
+    axs10.set_xticks([i for i in range(2, 13, 2)])
+    axs10.set_xticklabels(['$10^{' + str(i) + '}$' for i in range(2, 13, 2)])
+    axs10.set_xlim(2, 10)
+    axs10.set_xlabel('Viral load\n(RNA copies)', fontsize=12)
+    axs00.set_ylabel('Conditional Probability\nof Infection', fontsize=12)
+
+    axs00.text(9.5, -0.01, '$(i)$')
+    axs10.text(9.5, axs10.get_ylim()[1] * 0.8, '$(ii)$')
+    axs02.set_title('$(iii)$', fontsize=10)
+
+    axs00.legend()
     return fig
 
 

caimira/apps/calculator/static/js/report.js

@@ -1,6 +1,6 @@
-function on_report_load(conditional_probability_plot) {
+function on_report_load(conditional_probability_viral_loads) {
     // Check/uncheck uncertainties image generation
-    document.getElementById('conditional_probability_plot').checked = conditional_probability_plot
+    document.getElementById('conditional_probability_viral_loads').checked = conditional_probability_viral_loads
 }
 
 /* Generate the concentration plot using d3 library. */
@@ -1164,14 +1164,14 @@ function display_rename_column(bool, id) {
     else document.getElementById(id).style.display = 'none';
 }
 
-function conditional_probability_plot(value, is_generated) {
+function conditional_probability_viral_loads(value, is_generated) {
     // If the image was previously generated, there is no need to reload the page.
     if (value && is_generated == 1) {
         document.getElementById('conditional_probability_div').style.display = 'block'
     }
     else if (value && is_generated == 0) {
-        document.getElementById('label_conditional_probability_plot').innerHTML = `<span id="loading_spinner" class="spinner-border spinner-border-sm mr-2 mt-0" role="status" aria-hidden="true"></span>Loading...`;
-        document.getElementById('conditional_probability_plot').setAttribute('disabled', true);
+        document.getElementById('label_conditional_probability_viral_loads').innerHTML = `<span id="loading_spinner" class="spinner-border spinner-border-sm mr-2 mt-0" role="status" aria-hidden="true"></span>Loading...`;
+        document.getElementById('conditional_probability_viral_loads').setAttribute('disabled', true);
         document.cookie = `conditional_plot= 1; path=/`;
         window.location.reload();
     }

caimira/apps/templates/base/calculator.form.html.j2

@@ -495,7 +495,7 @@
     </div>
 
     <div class="form-check d-none">
-      <input type="checkbox" id="conditional_probability_plot" class="tabbed form-check-input" name="conditional_probability_plot" value="0" disabled>
+      <input type="checkbox" id="conditional_probability_viral_loads" class="tabbed form-check-input" name="conditional_probability_viral_loads" value="0" disabled>
     </div>
 
     <span id="training_limit_error" class="red_text" hidden>Conference/Training activities limited to 1 infected<br></span>

caimira/apps/templates/base/calculator.report.html.j2

@@ -15,7 +15,7 @@
 
 </head>
 
-<body id="body" onload="on_report_load({{ form.conditional_probability_plot | int }})">
+<body id="body" onload="on_report_load({{ form.conditional_probability_viral_loads | int }})">
 
 <!-- MODEL REPR - Available in the developer tools once the report is generated. Useful to re-create the model using an interpreter that has CAiMIRA installed:
 
@@ -214,13 +214,13 @@
 									draw_histogram("prob_inf_hist", {{ prob_inf }}, {{ prob_inf_sd }});
 								</script>
 								<br>
-								{% if model.data_registry.virological_data['virus_distributions'][form.virus_type]['viral_load_in_sputum'] == 'Ref: Viral load - covid_overal_vl_data' %}
+								{% if model.data_registry.virological_data['virus_distributions'][form.virus_type]['viral_load_in_sputum'] == 'Ref: Viral load - covid overal viral load data' %}
 									<div class="form-check">
-										<input type="checkbox" id="conditional_probability_plot" class="tabbed form-check-input" name="conditional_probability_plot" value="1" onClick="conditional_probability_plot(this.checked, {{ form.conditional_probability_plot | int }});">
-										<label id="label_conditional_probability_plot" for="conditional_probability_plot" class="form-check-label col-sm-12">Generate full uncertainty data (as function of the viral load)</label>
+										<input type="checkbox" id="conditional_probability_viral_loads" class="tabbed form-check-input" name="conditional_probability_viral_loads" value="1" onClick="conditional_probability_viral_loads(this.checked, {{ form.conditional_probability_viral_loads | int }});">
+										<label id="label_conditional_probability_viral_loads" for="conditional_probability_viral_loads" class="form-check-label col-sm-12">Generate full uncertainty data (as function of the viral load)</label>
 									</div>
 								{% endif %}
-								{% if form.conditional_probability_plot %}
+								{% if form.conditional_probability_viral_loads %}
 								<div id="conditional_probability_div">
 									<img src= "{{ uncertainties_plot_src }}" />
 									<div class="ml-5">

caimira/tests/test_conditional_probability.py

@@ -72,7 +72,7 @@ def test_conditional_prob_inf_given_vl_dist(data_registry, baseline_exposure_mod
     specific_vl = np.log10(mc_model.concentration_model.infected.virus.viral_load_in_sputum)
     step = 8/100
     actual_pi_means, actual_lower_percentiles, actual_upper_percentiles = (
-        report_generator.conditional_prob_inf_given_vl_dist(data_registry, infection_probability, viral_loads, specific_vl, step)
+        report_generator.conditional_prob_inf_given_vl_dist(infection_probability, viral_loads, specific_vl, step)
     )
 
     assert np.allclose(actual_pi_means, expected_pi_means, atol=0.002)