Merge branch 'changes/schema_update' into 'master'

Data registry update (schema v2.1.1)

See merge request caimira/caimira!487

caimira/apps/calculator/__init__.py

@@ -42,7 +42,7 @@
 # calculator version. If the calculator needs to make breaking changes (e.g. change
 # form attributes) then it can also increase its MAJOR version without needing to
 # increase the overall CAiMIRA version (found at ``caimira.__version__``).
-__version__ = "4.15.2"
+__version__ = "4.15.3"
 
 LOG = logging.getLogger("Calculator")
 
@@ -573,7 +573,11 @@ def make_app(
 
     data_registry = DataRegistry()
     data_service = None
-    data_service_enabled = os.environ.get('DATA_SERVICE_ENABLED', 0)
+    try:
+        data_service_enabled = int(os.environ.get('DATA_SERVICE_ENABLED', 0))
+    except ValueError:
+        data_service_enabled = None
+
     if data_service_enabled: data_service = DataService.create()
 
     return Application(

caimira/apps/calculator/co2_model_generator.py

@@ -159,7 +159,7 @@ def ventilation_transition_times(self) -> typing.Tuple[float, ...]:
             return tuple((self.CO2_data['times'][0], self.CO2_data['times'][-1]))
 
     def build_model(self, size=None) -> models.CO2DataModel: # type: ignore
-        size = size or self.data_registry.monte_carlo_sample_size
+        size = size or self.data_registry.monte_carlo['sample_size']
         # Build a simple infected and exposed population for the case when presence
         # intervals and number of people are dynamic. Activity type is not needed.
         infected_presence = self.infected_present_interval()

caimira/apps/calculator/model_generator.py

@@ -200,10 +200,10 @@ def initialize_room(self) -> models.Room:
 
         if self.arve_sensors_option == False:
             if self.room_heating_option:
-                humidity = self.data_registry.room['defaults']['humidity_with_heating']
+                humidity = self.data_registry.room['humidity_with_heating']
             else:
-                humidity = self.data_registry.room['defaults']['humidity_without_heating']
-            inside_temp = self.data_registry.room['defaults']['inside_temp']
+                humidity = self.data_registry.room['humidity_without_heating']
+            inside_temp = self.data_registry.room['inside_temp']
         else:
             humidity = float(self.humidity)
             inside_temp = self.inside_temp
@@ -245,11 +245,11 @@ def build_mc_model(self) -> mc.ExposureModel:
         )
 
     def build_model(self, sample_size=None) -> models.ExposureModel:
-        sample_size = sample_size or self.data_registry.monte_carlo_sample_size
+        sample_size = sample_size or self.data_registry.monte_carlo['sample_size']
         return self.build_mc_model().build_model(size=sample_size)
 
     def build_CO2_model(self, sample_size=None) -> models.CO2ConcentrationModel:
-        sample_size = sample_size or self.data_registry.monte_carlo_sample_size
+        sample_size = sample_size or self.data_registry.monte_carlo['sample_size']
         infected_population: models.InfectedPopulation = self.infected_population().build_model(sample_size)
         exposed_population: models.Population = self.exposed_population().build_model(sample_size)
 

caimira/apps/calculator/report_generator.py

@@ -19,6 +19,7 @@
 from ... import monte_carlo as mc
 from .model_generator import VirusFormData
 from ... import dataclass_utils
+from caimira.enums import ViralLoads
 
 
 def model_start_end(model: models.ExposureModel):
@@ -168,12 +169,27 @@ def calculate_report_data(form: VirusFormData, model: models.ExposureModel, exec
     prob_dist_count, prob_dist_bins = np.histogram(prob/100, bins=100, density=True)
     prob_probabilistic_exposure = np.array(model.total_probability_rule()).mean()
     expected_new_cases = np.array(model.expected_new_cases()).mean()
-    uncertainties_plot_src = img2base64(_figure2bytes(uncertainties_plot(model, prob))) if form.conditional_probability_plot else None
     exposed_presence_intervals = [list(interval) for interval in model.exposed.presence_interval().boundaries()]
-    conditional_probability_data = {key: value for key, value in
-                                    zip(('viral_loads', 'pi_means', 'lower_percentiles', 'upper_percentiles'),
-                                        manufacture_conditional_probability_data(model, prob))}
 
+    if (model.data_registry.virological_data['virus_distributions'][form.virus_type]['viral_load_in_sputum'] == ViralLoads.COVID_OVERALL.value # type: ignore
+        and form.conditional_probability_plot): # Only generate this data if covid_overall_vl_data is selected.
+
+        viral_load_in_sputum: models._VectorisedFloat = model.concentration_model.infected.virus.viral_load_in_sputum
+        viral_loads, pi_means, lower_percentiles, upper_percentiles = manufacture_conditional_probability_data(model, prob)
+
+        uncertainties_plot_src = img2base64(_figure2bytes(uncertainties_plot(prob, viral_load_in_sputum, viral_loads, 
+                                                                             pi_means, lower_percentiles, upper_percentiles)))
+        conditional_probability_data = {key: value for key, value in
+                                    zip(('viral_loads', 'pi_means', 'lower_percentiles', 'upper_percentiles'),
+                                    (viral_loads, pi_means, lower_percentiles, upper_percentiles))}
+        vl_dist = list(np.log10(viral_load_in_sputum))
+
+    else:
+        uncertainties_plot_src = None
+        conditional_probability_data = None
+        vl = model.concentration_model.virus.viral_load_in_sputum
+        if isinstance(vl, np.ndarray): vl_dist = list(np.log10(model.concentration_model.virus.viral_load_in_sputum))
+        else: vl_dist = np.log10(model.concentration_model.virus.viral_load_in_sputum)
 
     return {
         "model_repr": repr(model),
@@ -194,7 +210,7 @@ def calculate_report_data(form: VirusFormData, model: models.ExposureModel, exec
         "expected_new_cases": expected_new_cases,
         "uncertainties_plot_src": uncertainties_plot_src,
         "CO2_concentrations": CO2_concentrations,
-        "vl_dist": list(np.log10(model.concentration_model.virus.viral_load_in_sputum)),
+        "vl_dist": vl_dist,
         "conditional_probability_data": conditional_probability_data,
     }
 
@@ -233,8 +249,8 @@ def conditional_prob_inf_given_vl_dist(
     for vl_log in viral_loads:
         specific_prob = infection_probability[np.where((vl_log-step/2-specific_vl)*(vl_log+step/2-specific_vl)<0)[0]] #type: ignore
         pi_means.append(specific_prob.mean())
-        lower_percentiles.append(np.quantile(specific_prob, data_registry.conditional_prob_inf_given_viral_load['lower_percentile']))
-        upper_percentiles.append(np.quantile(specific_prob, data_registry.conditional_prob_inf_given_viral_load['upper_percentile']))
+        lower_percentiles.append(np.quantile(specific_prob, 0.05))
+        upper_percentiles.append(np.quantile(specific_prob, 0.95))
 
     return pi_means, lower_percentiles, upper_percentiles
 
@@ -245,8 +261,8 @@ def manufacture_conditional_probability_data(
 ):
     data_registry: DataRegistry = exposure_model.data_registry
 
-    min_vl = data_registry.conditional_prob_inf_given_viral_load['min_vl']
-    max_vl = data_registry.conditional_prob_inf_given_viral_load['max_vl']
+    min_vl = 2
+    max_vl = 10
     step = (max_vl - min_vl)/100
     viral_loads = np.arange(min_vl, max_vl, step)
     specific_vl = np.log10(exposure_model.concentration_model.virus.viral_load_in_sputum)
@@ -256,11 +272,12 @@ def manufacture_conditional_probability_data(
     return list(viral_loads), list(pi_means), list(lower_percentiles), list(upper_percentiles)
 
 
-def uncertainties_plot(exposure_model: models.ExposureModel, prob: models._VectorisedFloat):
-    fig = plt.figure(figsize=(4, 7), dpi=110)
-
-    infection_probability = prob / 100
-    viral_loads, pi_means, lower_percentiles, upper_percentiles = manufacture_conditional_probability_data(exposure_model, infection_probability)
+def uncertainties_plot(infection_probability: models._VectorisedFloat,
+                       viral_load_in_sputum: models._VectorisedFloat,
+                       viral_loads: models._VectorisedFloat,
+                       pi_means: models._VectorisedFloat,
+                       lower_percentiles: models._VectorisedFloat,
+                       upper_percentiles: models._VectorisedFloat):
 
     fig, axs = plt.subplots(2, 3,
         gridspec_kw={'width_ratios': [5, 0.5] + [1],
@@ -273,8 +290,8 @@ def uncertainties_plot(exposure_model: models.ExposureModel, prob: models._Vecto
 
     axs[0, 1].set_visible(False)
 
-    axs[0, 0].plot(viral_loads, pi_means, label='Predictive total probability')
-    axs[0, 0].fill_between(viral_loads, lower_percentiles, upper_percentiles, alpha=0.1, label='5ᵗʰ and 95ᵗʰ percentile')
+    axs[0, 0].plot(viral_loads, np.array(pi_means)/100, label='Predictive total probability')
+    axs[0, 0].fill_between(viral_loads, np.array(lower_percentiles)/100, np.array(upper_percentiles)/100, alpha=0.1, label='5ᵗʰ and 95ᵗʰ percentile')
 
     axs[0, 2].hist(infection_probability, bins=30, orientation='horizontal')
     axs[0, 2].set_xticks([])
@@ -285,8 +302,8 @@ def uncertainties_plot(exposure_model: models.ExposureModel, prob: models._Vecto
     axs[0, 2].set_xlim(0, highest_bar)
 
     axs[0, 2].text(highest_bar * 0.5, 0.5,
-                        rf"$\bf{np.round(np.mean(infection_probability) * 100, 1)}$%", ha='center', va='center')
-    axs[1, 0].hist(np.log10(exposure_model.concentration_model.infected.virus.viral_load_in_sputum),
+                        rf"$\bf{np.round(np.mean(infection_probability), 1)}$%", ha='center', va='center')
+    axs[1, 0].hist(np.log10(viral_load_in_sputum),
                     bins=150, range=(2, 10), color='grey')
     axs[1, 0].set_facecolor("lightgrey")
     axs[1, 0].set_yticks([])
@@ -442,7 +459,7 @@ def scenario_statistics(
     sample_times: typing.List[float],
     compute_prob_exposure: bool
 ):
-    model = mc_model.build_model(size=mc_model.data_registry.monte_carlo_sample_size)
+    model = mc_model.build_model(size=mc_model.data_registry.monte_carlo['sample_size'])
     if (compute_prob_exposure):
         # It means we have data to calculate the total_probability_rule
         prob_probabilistic_exposure = model.total_probability_rule()

caimira/apps/templates/base/calculator.report.html.j2

@@ -214,11 +214,12 @@
 									draw_histogram("prob_inf_hist", {{ prob_inf }}, {{ prob_inf_sd }});
 								</script>
 								<br>
-
-								<div class="form-check">
-									<input type="checkbox" id="conditional_probability_plot" class="tabbed form-check-input" name="conditional_probability_plot" value="1" onClick="conditional_probability_plot(this.checked, {{ form.conditional_probability_plot | int }});">
-									<label id="label_conditional_probability_plot" for="conditional_probability_plot" class="form-check-label col-sm-12">Generate full uncertainty data (as function of the viral load)</label>
-								</div>
+								{% if model.data_registry.virological_data['virus_distributions'][form.virus_type]['viral_load_in_sputum'] == 'Ref: Viral load - covid_overal_vl_data' %}
+									<div class="form-check">
+										<input type="checkbox" id="conditional_probability_plot" class="tabbed form-check-input" name="conditional_probability_plot" value="1" onClick="conditional_probability_plot(this.checked, {{ form.conditional_probability_plot | int }});">
+										<label id="label_conditional_probability_plot" for="conditional_probability_plot" class="form-check-label col-sm-12">Generate full uncertainty data (as function of the viral load)</label>
+									</div>
+								{% endif %}
 								{% if form.conditional_probability_plot %}
 								<div id="conditional_probability_div">
 									<img src= "{{ uncertainties_plot_src }}" />

caimira/enums.py

@@ -3,11 +3,3 @@
 class ViralLoads(Enum):
     COVID_OVERALL = "Ref: Viral load - covid_overal_vl_data"
     SYMPTOMATIC_FREQUENCIES = "Ref: Viral load - symptomatic_vl_frequencies"
-
-
-class InfectiousDoses(Enum):
-    DISTRIBUTION  = "Ref: Infectious dose - infectious_dose_distribution"
-
-
-class ViableToRNARatios(Enum):
-    DISTRIBUTION  = "Ref: Viable to RNA ratio - viable_to_RNA_ratio_distribution"

caimira/models.py

@@ -879,7 +879,7 @@ def fraction_of_infectious_virus(self) -> _VectorisedFloat:
         The fraction of infectious virus.
 
         """
-        return self.data_registry.population_with_virus['fraction_of_infectious_virus'] # type: ignore
+        return 1
 
     def aerosols(self):
         """
@@ -1052,7 +1052,7 @@ def min_background_concentration(self) -> _VectorisedFloat:
         (in the same unit as the concentration). Its the value towards which
         the concentration will decay to.
         """
-        return self.data_registry.concentration_model['min_background_concentration'] # type: ignore
+        return self.data_registry.concentration_model['virus_concentration_model']['min_background_concentration'] # type: ignore
 
     def normalization_factor(self) -> _VectorisedFloat:
         """
@@ -1242,7 +1242,7 @@ class ConcentrationModel(_ConcentrationModelBase):
 
     def __post_init__(self):
         if self.evaporation_factor is None:
-            self.evaporation_factor = self.data_registry.particle['evaporation_factor']
+            self.evaporation_factor = self.data_registry.expiration_particle['particle']['evaporation_factor']
 
     @property
     def population(self) -> InfectedPopulation:
@@ -1335,7 +1335,7 @@ def dilution_factor(self) -> _VectorisedFloat:
         '''
         The dilution factor for the respective expiratory activity type.
         '''
-        _dilution_factor = self.data_registry.short_range_model['dilution_factor']
+        _dilution_factor = self.data_registry.short_range_model['dilution_factor'] 
         # Average mouth opening diameter (m)
         mouth_diameter: float = _dilution_factor['mouth_diameter'] # type: ignore
 
@@ -1355,11 +1355,14 @@ def dilution_factor(self) -> _VectorisedFloat:
         # Initial velocity of the exhalation airflow (m/s)
         u0 = np.array(Q_exh/Am)
 
-        # Duration of the expiration period(s), assuming a 4s breath-cycle
-        tstar: float = _dilution_factor['tstar'] # type: ignore
+        # Duration of one breathing cycle
+        breathing_cicle: float = _dilution_factor['breathing_cycle'] # type: ignore
+
+        # Duration of the expiration period(s)
+        tstar: float = breathing_cicle / 2
 
         # Streamwise and radial penetration coefficients
-        _df_pc = _dilution_factor['penetration_coefficients']
+        _df_pc = _dilution_factor['penetration_coefficients'] # type: ignore
         𝛽r1: float = _df_pc['𝛽r1'] # type: ignore
         𝛽r2: float = _df_pc['𝛽r2'] # type: ignore
         𝛽x1: float = _df_pc['𝛽x1'] # type: ignore
@@ -1585,7 +1588,7 @@ class ExposureModel:
     #: The number of times the exposure event is repeated (default 1).
     @property
     def repeats(self) -> int:
-        return self.data_registry.exposure_model['repeats'] # type: ignore
+        return 1
 
     def __post_init__(self):
         """