diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index eada20ee..e8626a3b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,6 +24,7 @@ jobs: env: PROJECT_ROOT: ./ PROJECT_NAME: caimira + CAIMIRA_TESTS_CALCULATOR_TIMEOUT: 30 steps: - name: Checkout uses: actions/checkout@v2 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 13cf58bb..85ffd32f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,24 +15,55 @@ variables: # ################################################################################################### -# Test code +# Test code - CAiMIRA (model) and CERN CAiMIRA (CERN's UI) -# A full installation of CAiMIRA, tested with pytest. + # A full installation of CAiMIRA, tested with pytest. test_install: extends: .acc_py_full_test + variables: + project_root: ./caimira + project_name: caimira # A development installation of CAiMIRA tested with pytest. test_dev: extends: .acc_py_dev_test + variables: + project_root: ./caimira + project_name: caimira # A development installation of CAiMIRA tested with pytest. test_dev-39: variables: PY_VERSION: "3.9" + project_root: ./caimira + project_name: caimira + extends: .acc_py_dev_test + +# A full installation of CERN CAiMIRA, tested with pytest. +test_install: + extends: .acc_py_full_test + variables: + project_root: ./_cern_caimira + project_name: cern_caimira + + +# A development installation of CERN CAiMIRA tested with pytest. +test_dev: extends: .acc_py_dev_test + variables: + project_root: ./_cern_caimira + project_name: cern_caimira + +# A development installation of CERN CAiMIRA tested with pytest. +test_dev-39: + variables: + PY_VERSION: "3.9" + project_root: ./_cern_caimira + project_name: cern_caimira + extends: .acc_py_dev_test # ################################################################################################### # Test OpenShift config diff --git a/README.md b/README.md index 9a4a8346..5e334b0d 100644 --- a/README.md +++ b/README.md @@ -103,25 +103,426 @@ pip install -e . # At the root of the repository ### Running the Calculator app in development mode ``` -python -m caimira.apps.calculator +python -m ui.apps.calculator ``` To run with a specific template theme created: ``` -python -m caimira.apps.calculator --theme=caimira/apps/templates/{theme} +python -m ui.apps.calculator --theme=ui/apps/templates/{theme} ``` To run the entire app in a different `APPLICATION_ROOT` path: ``` -python -m caimira.apps.calculator --app_root=/myroot +python -m ui.apps.calculator --app_root=/myroot ``` To run the calculator on a different URL path: ``` -python -m caimira.apps.calculator --prefix=/mycalc +python -m ui.apps.calculator --prefix=/mycalc +``` + +Each of these commands will start a local version of CAiMIRA, which can be visited at http://localhost:8080/. + +### How to compile and read the documentation + +In order to generate the documentation, CAiMIRA must be installed first with the `doc` dependencies: + +``` +pip install -e .[doc] +``` + +To generate the HTML documentation page, the command `make html` should be executed in the `caimira/docs` directory. +If any of the `.rst` files under the `caimira/docs` folder is changed, this command should be executed again. + +Then, right click on `caimira/docs/_build/html/index.html` and select `Open with` your preferred web browser. + +### Running the CAiMIRA Expert-App or CO2-App apps in development mode + +#### Disclaimer + +The `ExpertApplication` and `CO2Application` are no longer actively maintained but will remain in the codebase for legacy purposes. +Please note that the functionality of these applications might be compromised due to deprecation issues. + +#### Running the Applications + +These applications only work within Jupyter notebooks. Attempting to run them outside of a Jupyter environment may result in errors or degraded functionality. + +##### Prerequisites + +Make sure you have the needed dependencies intalled: + +``` +pip install notebook jupyterlab +``` + +Running with Visual Studio Code (VSCode): + +1. Ensure you have the following extensions installed in VSCode: `Jupyter` and `Python`. + +2. Open VSCode and navigate to the directory containing the notebook. + +3. Open the notebook (e.g. `caimira/apps/expert/caimira.ipynb`) and run the cells by clicking the `run` button next to each cell. + +### Running the tests + +``` +pip install -e .[test] +pytest ./caimira +``` + +### Running the profiler + +The profiler is enabled when the environment variable `CAIMIRA_PROFILER_ENABLED` is set to 1. + +When visiting http://localhost:8080/profiler, you can start a new session and choose between [PyInstrument](https://github.com/joerick/pyinstrument) or [cProfile](https://docs.python.org/3/library/profile.html#module-cProfile). The app includes two different profilers, mainly because they can give different information. + +Keep the profiler page open. Then, in another window, navigate to any page in CAiMIRA, for example generate a new report. Refresh the profiler page, and click on the `Report` link to see the profiler output. + +The sessions are stored in a local file in the `/tmp` folder. To share it across multiple web nodes, a shared storage should be added to all web nodes. The folder can be customized via the environment variable `CAIMIRA_PROFILER_CACHE_DIR`. + +### Building the whole environment for local development + +**Simulate the docker build that takes place on openshift with:** + +``` +s2i build file://$(pwd) --copy --keep-symlinks --context-dir ./app-config/nginx/ centos/nginx-112-centos7 caimira-nginx-app +docker build . -f ./app-config/calculator-app/Dockerfile -t calculator-app +docker build ./app-config/auth-service -t auth-service +``` + +Get the client secret from the CERN Application portal for the `caimira-test` app. See [CERN-SSO-integration](#cern-sso-integration) for more info. +``` +read CLIENT_SECRET +``` + +Define some env vars (copy/paste): +``` +export COOKIE_SECRET=$(openssl rand -hex 50) +export OIDC_SERVER=https://auth.cern.ch/auth +export OIDC_REALM=CERN +export CLIENT_ID=caimira-test +``` + +Run docker-compose: +``` +cd app-config +CURRENT_UID=$(id -u):$(id -g) docker-compose up +``` + +Then visit http://localhost:8080/. + +### Setting up the application on openshift + +The https://cern.ch/caimira application is running on CERN's OpenShift platform. In order to set it up for the first time, we followed the documentation at https://paas.docs.cern.ch/. In particular we: + + * Added the OpenShift application deploy key to the GitLab repository + * Created a Python 3.6 (the highest possible at the time of writing) application in OpenShift + * Configured a generic webhook on OpenShift, and call that from the CI of the GitLab repository + +### Updating the caimira-test.web.cern.ch instance + +We have a replica of https://caimira.web.cern.ch running on http://caimira-test.web.cern.ch. Its purpose is to simulate what will happen when +a feature is merged. To push your changes to caimira-test, simply push your branch to `live/caimira-test` and the CI pipeline will trigger the +deployment. To push to this branch, there is a good chance that you will need to force push - you should always force push with care and +understanding why you are doing it. Syntactically, it will look something like (assuming that you have "upstream" as your remote name, +but it may be origin if you haven't configured it differently): + + git push --force upstream name-of-local-branch:live/caimira-test + + +## OpenShift templates + +### First setup + +First, get the [oc](https://docs.okd.io/3.11/cli_reference/get_started_cli.html) client and then login: + +```console +$ oc login https://api.paas.okd.cern.ch +``` + +Then, switch to the project that you want to update: + +```console +$ oc project caimira-test +``` + +Create a new service account in OpenShift to use GitLab container registry: + +```console +$ oc create serviceaccount gitlabci-deployer +serviceaccount "gitlabci-deployer" created + +$ oc policy add-role-to-user registry-editor -z gitlabci-deployer + +# We will refer to the output of this command as `test-token` +$ oc serviceaccounts get-token gitlabci-deployer +<...test-token...> +``` + +Add the token to GitLab to allow GitLab to access OpenShift and define/change image stream tags. Go to `Settings` -> `CI / CD` -> `Variables` -> click on `Expand` button and create the variable `OPENSHIFT_CAIMIRA_TEST_DEPLOY_TOKEN`: insert the token `<...test-token...>`. + +Then, create the webhook secret to be able to trigger automatic builds from GitLab. + +Create and store the secret. Copy the secret above and add it to the GitLab project under `CI /CD` -> `Variables` with the name `OPENSHIFT_CAIMIRA_TEST_WEBHOOK_SECRET`. + +```console +$ WEBHOOKSECRET=$(openssl rand -hex 50) +$ oc create secret generic \ + --from-literal="WebHookSecretKey=$WEBHOOKSECRET" \ + gitlab-caimira-webhook-secret +``` + +For CI usage, we also suggest creating a service account: + +```console +oc create sa gitlab-config-checker +``` + +Under ``User Management`` -> ``RoleBindings`` create a new `RoleBinding` to grant `View` access to the `gitlab-config-checker` service account: + +* name: `gitlab-config-checker-view-role` +* role name: `view` +* service account: `gitlab-config-checker` + +To get this new user's authentication token go to ``User Management`` -> ``Service Accounts`` -> `gitlab-config-checker` and locate the token in the newly created secret associated with the user (in this case ``gitlab-config-checker-token-XXXX``). Copy the `token` value from `Data`. + +Create the various configurations: + +```console +$ cd app-config/openshift + +$ oc process -f configmap.yaml | oc create -f - +$ oc process -f services.yaml | oc create -f - +$ oc process -f imagestreams.yaml | oc create -f - +$ oc process -f buildconfig.yaml --param GIT_BRANCH='live/caimira-test' | oc create -f - +$ oc process -f deploymentconfig.yaml --param PROJECT_NAME='caimira-test' | oc create -f - +``` + +Manually create the **route** to access the website, see `routes.example.yaml`. +After having created the route, make sure that you extend the HTTP request timeout annotation: the +report generation can take more time than the default 30 seconds. + +``` +$ oc annotate route caimira-route --overwrite haproxy.router.openshift.io/timeout=60s +``` + +### CERN SSO integration + +The SSO integration uses OpenID credentials configured in [CERN Applications portal](https://application-portal.web.cern.ch/). +How to configure the application: + +* Application Identifier: `caimira-test` +* Homepage: `https://caimira-test.web.cern.ch` +* Administrators: `caimira-dev` +* SSO Registration: + * Protocol: `OpenID (OIDC)` + * Redirect URI: `https://caimira-test.web.cern.ch/auth/authorize` + * Leave unchecked all the other checkboxes +* Define new roles: + * Name: `CERN Users` + * Role Identifier: `external-users` + * Leave unchecked checkboxes + * Minimum Level Of Assurance: `CERN (highest)` + * Assign role to groups: `cern-accounts-primary` e-group + * Name: `External accounts` + * Role Identifier: `admin` + * Leave unchecked checkboxes + * Minimum Level Of Assurance: `Any (no restrictions)` + * Assign role to groups: `caimira-app-external-access` e-group + * Name: `Allowed users` + * Role Identifier: `allowed-users` + * Check `This role is required to access my application` + * Minimum Level Of Assurance:`Any (no restrictions)` + * Assign role to groups: `cern-accounts-primary` and `caimira-app-external-access` e-groups + +Copy the client id and client secret and use it below. + +```console +$ COOKIE_SECRET=$(openssl rand -hex 50) +$ oc create secret generic \ + --from-literal="CLIENT_ID=$CLIENT_ID" \ + --from-literal="CLIENT_SECRET=$CLIENT_SECRET" \ + --from-literal="COOKIE_SECRET=$COOKIE_SECRET" \ + auth-service-secrets +``` + +### External APIs + +- **Geographical location:** +There is one external API call to fetch required information related to the geographical location inserted by a user. +The documentation for this geocoding service is available at https://developers.arcgis.com/rest/geocode/api-reference/geocoding-suggest.htm . +Please note that there is no need for keys on this API call. It is **free-of-charge**. + +- **Humidity and Inside Temperature:** +There is the possibility of using one external API call to fetch information related to a location specified in the UI. The data is related to the inside temperature and humidity taken from an indoor measurement device. Note that the API currently used from ARVE is only available for the `CERN theme` as the authorised sensors are installed at CERN." + +- **ARVE:** + +The ARVE Swiss Air Quality System provides trusted air data for commercial buildings in real-time and analyzes it with the help of AI and machine learning algorithms to create actionable insights. + +Create secret: + +```console +$ read ARVE_CLIENT_ID +$ read ARVE_CLIENT_SECRET +$ read ARVE_API_KEY +$ oc create secret generic \ + --from-literal="ARVE_CLIENT_ID=$ARVE_CLIENT_ID" \ + --from-literal="ARVE_CLIENT_SECRET=$ARVE_CLIENT_SECRET" \ + --from-literal="ARVE_API_KEY=$ARVE_API_KEY" \ + arve-api +``` + +- **CERN Data Service:** + +The CERN data service collects data from various sources and expose them via a REST API endpoint. + +The service is enabled when the environment variable `DATA_SERVICE_ENABLED` is set to 1. + +## Update configuration + +If you need to **update** existing configuration, then modify this repository and after having logged in, run: + +```console +$ cd app-config/openshift + + +$ oc process -f configmap.yaml | oc replace -f - +$ oc process -f services.yaml | oc replace -f - +$ oc process -f imagestreams.yaml | oc replace -f - +$ oc process -f buildconfig.yaml --param GIT_BRANCH='live/caimira-test' | oc replace -f - +$ oc process -f deploymentconfig.yaml --param PROJECT_NAME='caimira-test' | oc replace -f - +``` + +Be aware that if you create/recreate the environment you must manually create a **route** in OpenShift, +specifying the respective annotation to be exposed outside CERN. +# CAiMIRA - CERN Airborne Model for Risk Assessment + +CAiMIRA is a risk assessment tool developed to model the concentration of viruses in enclosed spaces, in order to inform space-management decisions. + +CAiMIRA models the concentration profile of potential virions in enclosed spaces , both as background (room) concentration and during close-proximity interations, with clear and intuitive graphs. +The user can set a number of parameters, including room volume, exposure time, activity type, mask-wearing and ventilation. +The report generated indicates how to avoid exceeding critical concentrations and chains of airborne transmission in spaces such as individual offices, meeting rooms and labs. + +The risk assessment tool simulates the airborne spread SARS-CoV-2 virus in a finite volume, assuming a homogenous mixture and a two-stage exhaled jet model, and estimates the risk of COVID-19 infection therein. +The results DO NOT include the other known modes of SARS-CoV-2 transmission, such as fomite or blood-bound. +Hence, the output from this model is only valid when the other recommended public health & safety instructions are observed, such as good hand hygiene and other barrier measures. + +The model used is based on scientific publications relating to airborne transmission of infectious diseases, dose-response exposures and aerosol science, as of February 2022. +It can be used to compare the effectiveness of different airborne-related risk mitigation measures. + +Note that this model applies a deterministic approach, i.e., it is assumed at least one person is infected and shedding viruses into the simulated volume. +Nonetheless, it is also important to understand that the absolute risk of infection is uncertain, as it will depend on the probability that someone infected attends the event. +The model is most useful for comparing the impact and effectiveness of different mitigation measures such as ventilation, filtration, exposure time, physical activity, amount and nature of close-range interactions and +the size of the room, considering both long- and short-range airborne transmission modes of COVID-19 in indoor settings. + +This tool is designed to be informative, allowing the user to adapt different settings and model the relative impact on the estimated infection probabilities. +The objective is to facilitate targeted decision-making and investment through comparisons, rather than a singular determination of absolute risk. +While the SARS-CoV-2 virus is in circulation among the population, the notion of 'zero risk' or 'completely safe scenario' does not exist. +Each event modelled is unique, and the results generated therein are only as accurate as the inputs and assumptions. + +## Authors +CAiMIRA was developed by following members of CERN - European Council for Nuclear Research (visit https://home.cern/): + +Andre Henriques1, Luis Aleixo1, Marco Andreini1, Gabriella Azzopardi2, James Devine3, Philip Elson4, Nicolas Mounet2, Markus Kongstein Rognlien2,6, Nicola Tarocco5 + +1HSE Unit, Occupational Health & Safety Group, CERN
+2Beams Department, Accelerators and Beam Physics Group, CERN
+3Experimental Physics Department, Safety Office, CERN
+4Beams Department, Controls Group, CERN
+5Information Technology Department, Collaboration, Devices & Applications Group, CERN
+6Norwegian University of Science and Technology (NTNU)
+ +### Reference and Citation + +**For the use of the CAiMIRA web app** + +CAiMIRA – CERN Airborne Model for Indoor Risk Assessment tool + +[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.6520431.svg)](https://doi.org/10.5281/zenodo.6520431) + +© Copyright 2020-2021 CERN. All rights not expressly granted are reserved. + +**For use of the CAiMIRA model** + +Henriques A, Mounet N, Aleixo L, Elson P, Devine J, Azzopardi G, Andreini M, Rognlien M, Tarocco N, Tang J. (2022). Modelling airborne transmission of SARS-CoV-2 using CARA: risk assessment for enclosed spaces. _Interface Focus 20210076_. https://doi.org/10.1098/rsfs.2021.0076 + +Reference on the Short-range expiratory jet model from: +Jia W, Wei J, Cheng P, Wang Q, Li Y. (2022). Exposure and respiratory infection risk via the short-range airborne route. _Building and Environment_ *219*: 109166. +https://doi.org/10.1016/j.buildenv.2022.109166 + +***Open Source Acknowledgments*** + +For a detailed list of the open-source dependencies used in this project along with their respective licenses, please refer to [License Information](open-source-licences/README.md). This includes both the core dependencies specified in the project's requirements and their transitive dependencies. + +The information also features a distribution diagram of licenses and a brief description of each of them. + +## Applications + +### Calculator + +A risk assessment tool which simulates the airborne spread of the SARS-CoV-2 virus for space managers. + + +### CAiMIRA Expert App and CO₂ App + +A tool to interact with various parameters of the CAiMIRA model. + + +## Disclaimer + +CAiMIRA has not undergone review, approval or certification by competent authorities, and as a result, it cannot be considered as a fully endorsed and reliable tool, namely in the assessment of potential viral emissions from infected hosts to be modelled. + +The software is provided "as is", without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and non-infringement. +In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the software. + + +## Running CAiMIRA locally + +The easiest way to run a version of CAiMIRA Calculator is to use docker. A pre-built +image of CAiMIRA is made available at https://gitlab.cern.ch/caimira/caimira/container_registry. +In order to run CAiMIRA locally with docker, run the following: + + $ docker run -it -p 8080:8080 gitlab-registry.cern.ch/caimira/caimira/calculator + +This will start a local version of CAiMIRA, which can be visited at http://localhost:8080/. + + +## Development guide + +CAiMIRA is also mirrored to Github if you wish to collaborate on development and can be found at: https://github.com/CERN/caimira + +### Installing CAiMIRA in editable mode + +``` +pip install -e . # At the root of the repository +``` + +### Running the Calculator app in development mode + +``` +python -m cern_caimira.apps.calculator +``` + +To run with a specific template theme created: + +``` +python -m cern_caimira.apps.calculator --theme=cern_caimira/apps/templates/{theme} +``` + +To run the entire app in a different `APPLICATION_ROOT` path: + +``` +python -m cern_caimira.apps.calculator --app_root=/myroot +``` + +To run the calculator on a different URL path: + +``` +python -m cern_caimira.apps.calculator --prefix=/mycalc ``` Each of these commands will start a local version of CAiMIRA, which can be visited at http://localhost:8080/. diff --git a/app-config/caimira-public-docker-image/run_caimira.sh b/app-config/caimira-public-docker-image/run_caimira.sh index 00a61542..86a994f9 100755 --- a/app-config/caimira-public-docker-image/run_caimira.sh +++ b/app-config/caimira-public-docker-image/run_caimira.sh @@ -7,4 +7,4 @@ nginx -c /opt/caimira/nginx.conf cd /opt/caimira/src/caimira # Run the calculator in the foreground. -/opt/caimira/app/bin/python -m caimira.apps.calculator --port 8081 --no-debug +/opt/caimira/app/bin/python -m ui.apps.calculator --port 8081 --no-debug diff --git a/app-config/calculator-app/app.sh b/app-config/calculator-app/app.sh index f3a05bd7..ec7beb8f 100755 --- a/app-config/calculator-app/app.sh +++ b/app-config/calculator-app/app.sh @@ -26,8 +26,8 @@ if [[ "$APP_NAME" == "calculator-app" ]]; then export "DATA_SERVICE_ENABLED"="${DATA_SERVICE_ENABLED:=0}" export "CAIMIRA_PROFILER_ENABLED"="${CAIMIRA_PROFILER_ENABLED:=0}" - echo "Starting the caimira webservice with: python -m caimira.apps.calculator ${args[@]}" - python -m caimira.apps.calculator "${args[@]}" + echo "Starting the caimira webservice with: python -m ui.apps.calculator ${args[@]}" + python -m ui.apps.calculator "${args[@]}" else echo "No APP_NAME specified" diff --git a/app-config/docker-compose.yml b/app-config/docker-compose.yml index b44ba2dd..329e886c 100644 --- a/app-config/docker-compose.yml +++ b/app-config/docker-compose.yml @@ -1,6 +1,5 @@ version: "3.8" services: - calculator-app: image: calculator-app environment: @@ -8,7 +7,7 @@ services: - APP_NAME=calculator-app - APPLICATION_ROOT=/ - CAIMIRA_CALCULATOR_PREFIX=/calculator-cern - - CAIMIRA_THEME=caimira/apps/templates/cern + - CAIMIRA_THEME=ui/apps/templates/cern - DATA_SERVICE_ENABLED=0 - CAIMIRA_PROFILER_ENABLED=0 user: ${CURRENT_UID} diff --git a/app-config/openshift/deploymentconfig.yaml b/app-config/openshift/deploymentconfig.yaml index ca2f363f..61c50e72 100644 --- a/app-config/openshift/deploymentconfig.yaml +++ b/app-config/openshift/deploymentconfig.yaml @@ -303,3 +303,4 @@ - name: PROJECT_NAME description: The name of this project, e.g. caimira-test required: true + \ No newline at end of file diff --git a/.readthedocs.yaml b/caimira/.readthedocs.yaml similarity index 84% rename from .readthedocs.yaml rename to caimira/.readthedocs.yaml index edd0ab51..c489761c 100644 --- a/.readthedocs.yaml +++ b/caimira/.readthedocs.yaml @@ -14,4 +14,4 @@ sphinx: python: install: - - requirements: caimira/docs/requirements.txt \ No newline at end of file + - requirements: caimira/docs/requirements.txt diff --git a/.zenodo.json b/caimira/.zenodo.json similarity index 100% rename from .zenodo.json rename to caimira/.zenodo.json diff --git a/LICENSE b/caimira/LICENSE similarity index 100% rename from LICENSE rename to caimira/LICENSE diff --git a/caimira/store/__init__.py b/caimira/README.md similarity index 100% rename from caimira/store/__init__.py rename to caimira/README.md diff --git a/caimira/apps/__init__.py b/caimira/apps/__init__.py deleted file mode 100644 index 26b7f5d3..00000000 --- a/caimira/apps/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .expert import ExpertApplication -from .expert_co2 import CO2Application - -__all__ = ['ExpertApplication', 'CO2Application'] diff --git a/caimira/pyproject.toml b/caimira/pyproject.toml new file mode 100644 index 00000000..d98ad751 --- /dev/null +++ b/caimira/pyproject.toml @@ -0,0 +1,110 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "caimira" +version = "2.0.0" +description = "CAiMIRA - CERN Airborne Model for Indoor Risk Assessment" +readme = "README.md" +license = { text = "Apache-2.0" } +authors = [ + { name = "Andre Henriques", email = "andre.henriques@cern.ch" } +] +classifiers = [ + "Programming Language :: Python :: 3", + "Operating System :: OS Independent", + "License :: OSI Approved :: Apache Software License", +] +requires-python = ">=3.9" +dependencies = [ + "ipykernel", + "ipympl>=0.9.0", + "ipywidgets<8.0", + "Jinja2", + "loky", + "matplotlib", + "memoization", + "mistune", + "numpy", + "pandas", + "psutil", + "pyinstrument", + "pyjwt", + "python-dateutil", + "retry", + "ruptures", + "scipy", + "scikit-learn", + "timezonefinder", + "tornado", + "types-retry", +] + +[project.optional-dependencies] +dev = [] +test = [ + "pytest", + "pytest-mypy >= 0.10.3", + "mypy >= 1.0.0", + "pytest-tornasync", + "numpy-stubs @ git+https://github.com/numpy/numpy-stubs.git", + "types-dataclasses", + "types-python-dateutil", + "types-requests" +] +doc = [ + "sphinx", + "sphinx_rtd_theme" +] + +[project.urls] +Homepage = "https://github.com/cern/caimira" + +[tool.setuptools] +packages = ["caimira"] +package-dir = {"" = "src"} + +[tool.pytest.ini_options] +addopts = "--mypy" + +[tool.mypy] +no_warn_no_return = true +exclude = "caimira/profiler.py" +ignore_missing_imports = true # TODO what to do here? + +[tool.mypy-loky] +ignore_missing_imports = true + +[tool.mypy-ipympl] +ignore_missing_imports = true + +[tool.mypy-ipywidgets] +ignore_missing_imports = true + +[tool.mypy-matplotlib] +ignore_missing_imports = true + +[tool.mypy-mistune] +ignore_missing_imports = true + +[tool.mypy-qrcode] +ignore_missing_imports = true + +[tool.mypy-scipy] +ignore_missing_imports = true + +[tool.mypy-timezonefinder] +ignore_missing_imports = true + +[tool.mypy-pandas] +ignore_missing_imports = true + +[tool.mypy-pstats] +follow_imports = "skip" + +[tool.mypy-tabulate] +ignore_missing_imports = true + +[tool.mypy-ruptures] +ignore_missing_imports = true diff --git a/caimira/setup.cfg b/caimira/setup.cfg new file mode 100644 index 00000000..5d55bc97 --- /dev/null +++ b/caimira/setup.cfg @@ -0,0 +1,2 @@ +[tool:pytest] +addopts = --mypy diff --git a/caimira/src/caimira/__init__.py b/caimira/src/caimira/__init__.py new file mode 100644 index 00000000..2c02b644 --- /dev/null +++ b/caimira/src/caimira/__init__.py @@ -0,0 +1,3 @@ +import importlib.metadata + +__version__ = importlib.metadata.version(__package__ or __name__) diff --git a/caimira/src/caimira/api/__init__.py b/caimira/src/caimira/api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/caimira/src/caimira/api/app.py b/caimira/src/caimira/api/app.py new file mode 100644 index 00000000..db26c9d4 --- /dev/null +++ b/caimira/src/caimira/api/app.py @@ -0,0 +1,31 @@ +# """ +# Entry point for the CAiMIRA application +# """ + +import tornado.ioloop +import tornado.web +import tornado.log +from tornado.options import define, options +import logging + +from caimira.api.routes.report_routes import ReportHandler + +define("port", default=8088, help="Port to listen on", type=int) + +logging.basicConfig(format="%(message)s", level=logging.INFO) + +class Application(tornado.web.Application): + def __init__(self): + handlers = [ + (r"/report", ReportHandler), + ] + settings = dict( + debug=True, + ) + super(Application, self).__init__(handlers, **settings) + +if __name__ == "__main__": + app = Application() + app.listen(options.port) + logging.info(f"Tornado server is running on port {options.port}") + tornado.ioloop.IOLoop.current().start() diff --git a/caimira/src/caimira/api/controller/__init__.py b/caimira/src/caimira/api/controller/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/caimira/src/caimira/api/controller/report_controller.py b/caimira/src/caimira/api/controller/report_controller.py new file mode 100644 index 00000000..be151555 --- /dev/null +++ b/caimira/src/caimira/api/controller/report_controller.py @@ -0,0 +1,30 @@ +import concurrent.futures +import functools + +from caimira.calculator.validators.virus.virus_validator import VirusFormData +from caimira.calculator.store.data_registry import DataRegistry +import caimira.calculator.report.report_generator as rg + + +def generate_form_obj(form_data, data_registry): + return VirusFormData.from_dict(form_data, data_registry) + + +def generate_model(form_obj): + return form_obj.build_model(250_000) + + +def generate_report_results(form_obj, model): + return rg.calculate_report_data(form=form_obj, model=model, executor_factory=functools.partial( + concurrent.futures.ThreadPoolExecutor, None, # TODO define report_parallelism + ),) + + +def submit_virus_form(form_data): + data_registry = DataRegistry + + form_obj = generate_form_obj(form_data, data_registry) + model = generate_model(form_obj) + report_data = generate_report_results(form_obj, model=model) + + return report_data diff --git a/caimira/src/caimira/api/routes/__init__.py b/caimira/src/caimira/api/routes/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/caimira/src/caimira/api/routes/report_routes.py b/caimira/src/caimira/api/routes/report_routes.py new file mode 100644 index 00000000..7b6a51cb --- /dev/null +++ b/caimira/src/caimira/api/routes/report_routes.py @@ -0,0 +1,28 @@ +import json +import traceback +import tornado.web + +from caimira.api.controller.report_controller import submit_virus_form + +class ReportHandler(tornado.web.RequestHandler): + def set_default_headers(self): + self.set_header("Access-Control-Allow-Origin", "*") + self.set_header("Access-Control-Allow-Headers", "x-requested-with") + self.set_header("Access-Control-Allow-Methods", "POST, GET, OPTIONS") + + def post(self): + try: + form_data = json.loads(self.request.body) + report_data = submit_virus_form(form_data) + + response_data = { + "status": "success", + "message": "Results generated successfully", + "report_data": report_data, + } + + self.write(response_data) + except Exception as e: + traceback.print_exc() + self.set_status(400) + self.write({"message": str(e)}) diff --git a/caimira/src/caimira/calculator/__init__.py b/caimira/src/caimira/calculator/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/caimira/docs/Makefile b/caimira/src/caimira/calculator/docs/Makefile similarity index 100% rename from caimira/docs/Makefile rename to caimira/src/caimira/calculator/docs/Makefile diff --git a/caimira/docs/UML-CAiMIRA.png b/caimira/src/caimira/calculator/docs/UML-CAiMIRA.png similarity index 100% rename from caimira/docs/UML-CAiMIRA.png rename to caimira/src/caimira/calculator/docs/UML-CAiMIRA.png diff --git a/caimira/docs/caimira.apps.calculator.rst b/caimira/src/caimira/calculator/docs/caimira.apps.calculator.rst similarity index 100% rename from caimira/docs/caimira.apps.calculator.rst rename to caimira/src/caimira/calculator/docs/caimira.apps.calculator.rst diff --git a/caimira/docs/caimira.apps.rst b/caimira/src/caimira/calculator/docs/caimira.apps.rst similarity index 100% rename from caimira/docs/caimira.apps.rst rename to caimira/src/caimira/calculator/docs/caimira.apps.rst diff --git a/caimira/docs/caimira.data.rst b/caimira/src/caimira/calculator/docs/caimira.data.rst similarity index 100% rename from caimira/docs/caimira.data.rst rename to caimira/src/caimira/calculator/docs/caimira.data.rst diff --git a/caimira/docs/caimira.monte_carlo.rst b/caimira/src/caimira/calculator/docs/caimira.monte_carlo.rst similarity index 100% rename from caimira/docs/caimira.monte_carlo.rst rename to caimira/src/caimira/calculator/docs/caimira.monte_carlo.rst diff --git a/caimira/docs/caimira.rst b/caimira/src/caimira/calculator/docs/caimira.rst similarity index 100% rename from caimira/docs/caimira.rst rename to caimira/src/caimira/calculator/docs/caimira.rst diff --git a/caimira/docs/caimira.tests.apps.calculator.rst b/caimira/src/caimira/calculator/docs/caimira.tests.apps.calculator.rst similarity index 100% rename from caimira/docs/caimira.tests.apps.calculator.rst rename to caimira/src/caimira/calculator/docs/caimira.tests.apps.calculator.rst diff --git a/caimira/docs/caimira.tests.apps.rst b/caimira/src/caimira/calculator/docs/caimira.tests.apps.rst similarity index 100% rename from caimira/docs/caimira.tests.apps.rst rename to caimira/src/caimira/calculator/docs/caimira.tests.apps.rst diff --git a/caimira/docs/caimira.tests.data.rst b/caimira/src/caimira/calculator/docs/caimira.tests.data.rst similarity index 100% rename from caimira/docs/caimira.tests.data.rst rename to caimira/src/caimira/calculator/docs/caimira.tests.data.rst diff --git a/caimira/docs/caimira.tests.models.rst b/caimira/src/caimira/calculator/docs/caimira.tests.models.rst similarity index 100% rename from caimira/docs/caimira.tests.models.rst rename to caimira/src/caimira/calculator/docs/caimira.tests.models.rst diff --git a/caimira/docs/caimira.tests.rst b/caimira/src/caimira/calculator/docs/caimira.tests.rst similarity index 100% rename from caimira/docs/caimira.tests.rst rename to caimira/src/caimira/calculator/docs/caimira.tests.rst diff --git a/caimira/docs/conf.py b/caimira/src/caimira/calculator/docs/conf.py similarity index 100% rename from caimira/docs/conf.py rename to caimira/src/caimira/calculator/docs/conf.py diff --git a/caimira/docs/full_diameter_dependence.rst b/caimira/src/caimira/calculator/docs/full_diameter_dependence.rst similarity index 100% rename from caimira/docs/full_diameter_dependence.rst rename to caimira/src/caimira/calculator/docs/full_diameter_dependence.rst diff --git a/caimira/docs/index.rst b/caimira/src/caimira/calculator/docs/index.rst similarity index 100% rename from caimira/docs/index.rst rename to caimira/src/caimira/calculator/docs/index.rst diff --git a/caimira/docs/make.bat b/caimira/src/caimira/calculator/docs/make.bat similarity index 100% rename from caimira/docs/make.bat rename to caimira/src/caimira/calculator/docs/make.bat diff --git a/caimira/docs/requirements.txt b/caimira/src/caimira/calculator/docs/requirements.txt similarity index 80% rename from caimira/docs/requirements.txt rename to caimira/src/caimira/calculator/docs/requirements.txt index 6074cd93..b1b732ea 100644 --- a/caimira/docs/requirements.txt +++ b/caimira/src/caimira/calculator/docs/requirements.txt @@ -3,4 +3,4 @@ sphinx-rtd-theme==1.2.2 pillow==5.4.1 mock==1.0.1 commonmark==0.9.1 -recommonmark==0.5.0 \ No newline at end of file +recommonmark==0.5.0 diff --git a/caimira/__init__.py b/caimira/src/caimira/calculator/models/__init__.py similarity index 100% rename from caimira/__init__.py rename to caimira/src/caimira/calculator/models/__init__.py diff --git a/caimira/data/__init__.py b/caimira/src/caimira/calculator/models/data/__init__.py similarity index 98% rename from caimira/data/__init__.py rename to caimira/src/caimira/calculator/models/data/__init__.py index 8c0a1390..3d539df4 100644 --- a/caimira/data/__init__.py +++ b/caimira/src/caimira/calculator/models/data/__init__.py @@ -1,6 +1,6 @@ import numpy as np -from caimira import models -from caimira.data.weather import wx_data, nearest_wx_station +from caimira.calculator.models import models +from .weather import wx_data, nearest_wx_station MONTH_NAMES = [ 'January', 'February', 'March', 'April', 'May', 'June', 'July', diff --git a/caimira/data/global_weather_set.json b/caimira/src/caimira/calculator/models/data/global_weather_set.json similarity index 100% rename from caimira/data/global_weather_set.json rename to caimira/src/caimira/calculator/models/data/global_weather_set.json diff --git a/caimira/data/hadisd_station_fullinfo_v311_202001p.txt b/caimira/src/caimira/calculator/models/data/hadisd_station_fullinfo_v311_202001p.txt similarity index 100% rename from caimira/data/hadisd_station_fullinfo_v311_202001p.txt rename to caimira/src/caimira/calculator/models/data/hadisd_station_fullinfo_v311_202001p.txt diff --git a/caimira/data/weather.py b/caimira/src/caimira/calculator/models/data/weather.py similarity index 100% rename from caimira/data/weather.py rename to caimira/src/caimira/calculator/models/data/weather.py diff --git a/caimira/dataclass_utils.py b/caimira/src/caimira/calculator/models/dataclass_utils.py similarity index 100% rename from caimira/dataclass_utils.py rename to caimira/src/caimira/calculator/models/dataclass_utils.py diff --git a/caimira/enums.py b/caimira/src/caimira/calculator/models/enums.py similarity index 100% rename from caimira/enums.py rename to caimira/src/caimira/calculator/models/enums.py diff --git a/caimira/models.py b/caimira/src/caimira/calculator/models/models.py similarity index 99% rename from caimira/models.py rename to caimira/src/caimira/calculator/models/models.py index 88f9ffb1..b49f6692 100644 --- a/caimira/models.py +++ b/caimira/src/caimira/calculator/models/models.py @@ -40,7 +40,7 @@ import scipy.stats as sct from scipy.optimize import minimize -from caimira.store.data_registry import DataRegistry +from caimira.calculator.store.data_registry import DataRegistry if not typing.TYPE_CHECKING: from memoization import cached diff --git a/caimira/monte_carlo/__init__.py b/caimira/src/caimira/calculator/models/monte_carlo/__init__.py similarity index 100% rename from caimira/monte_carlo/__init__.py rename to caimira/src/caimira/calculator/models/monte_carlo/__init__.py diff --git a/caimira/monte_carlo/__init__.pyi b/caimira/src/caimira/calculator/models/monte_carlo/__init__.pyi similarity index 100% rename from caimira/monte_carlo/__init__.pyi rename to caimira/src/caimira/calculator/models/monte_carlo/__init__.pyi diff --git a/caimira/monte_carlo/data.py b/caimira/src/caimira/calculator/models/monte_carlo/data.py similarity index 98% rename from caimira/monte_carlo/data.py rename to caimira/src/caimira/calculator/models/monte_carlo/data.py index acca3207..7b503cb0 100644 --- a/caimira/monte_carlo/data.py +++ b/caimira/src/caimira/calculator/models/monte_carlo/data.py @@ -7,11 +7,11 @@ from scipy import special as sp from scipy.stats import weibull_min -from caimira.enums import ViralLoads +from ..enums import ViralLoads -import caimira.monte_carlo.models as mc -from caimira.monte_carlo.sampleable import LogCustom, LogNormal, Normal, LogCustomKernel, CustomKernel, Uniform, Custom -from caimira.store.data_registry import DataRegistry +import caimira.calculator.models.monte_carlo.models as mc +from caimira.calculator.models.monte_carlo.sampleable import LogCustom, LogNormal, Normal, LogCustomKernel, CustomKernel, Uniform, Custom +from caimira.calculator.store.data_registry import DataRegistry def evaluate_vl(root: typing.Dict, value: str, data_registry: DataRegistry): diff --git a/caimira/monte_carlo/models.py b/caimira/src/caimira/calculator/models/monte_carlo/models.py similarity index 78% rename from caimira/monte_carlo/models.py rename to caimira/src/caimira/calculator/models/monte_carlo/models.py index 7215db16..f4ad09e2 100644 --- a/caimira/monte_carlo/models.py +++ b/caimira/src/caimira/calculator/models/monte_carlo/models.py @@ -3,7 +3,7 @@ import sys import typing -import caimira.models +from caimira.calculator.models import models from .sampleable import SampleableDistribution, _VectorisedFloatOrSampleable @@ -57,7 +57,7 @@ def _build_mc_model(model: dataclass_instance) -> typing.Type[MCModelBase[_Model # Note: deepcopy not needed here as we aren't mutating entities beyond # the top level. new_field = copy.copy(field) - if field.type is caimira.models._VectorisedFloat: # noqa + if field.type is models._VectorisedFloat: # noqa new_field.type = _VectorisedFloatOrSampleable # type: ignore field_type: typing.Any = new_field.type @@ -65,30 +65,30 @@ def _build_mc_model(model: dataclass_instance) -> typing.Type[MCModelBase[_Model if getattr(field_type, '__origin__', None) in [typing.Union, typing.Tuple]: # It is challenging to generalise this code, so we provide specific transformations, # and raise for unforseen cases. - if new_field.type == typing.Tuple[caimira.models._VentilationBase, ...]: + if new_field.type == typing.Tuple[models._VentilationBase, ...]: VB = getattr(sys.modules[__name__], "_VentilationBase") - field_type = typing.Tuple[typing.Union[caimira.models._VentilationBase, VB], ...] - elif new_field.type == typing.Tuple[caimira.models._ExpirationBase, ...]: + field_type = typing.Tuple[typing.Union[models._VentilationBase, VB], ...] + elif new_field.type == typing.Tuple[models._ExpirationBase, ...]: EB = getattr(sys.modules[__name__], "_ExpirationBase") - field_type = typing.Tuple[typing.Union[caimira.models._ExpirationBase, EB], ...] - elif new_field.type == typing.Tuple[caimira.models.SpecificInterval, ...]: + field_type = typing.Tuple[typing.Union[models._ExpirationBase, EB], ...] + elif new_field.type == typing.Tuple[models.SpecificInterval, ...]: SI = getattr(sys.modules[__name__], "SpecificInterval") - field_type = typing.Tuple[typing.Union[caimira.models.SpecificInterval, SI], ...] + field_type = typing.Tuple[typing.Union[models.SpecificInterval, SI], ...] - elif new_field.type == typing.Union[int, caimira.models.IntPiecewiseConstant]: + elif new_field.type == typing.Union[int, models.IntPiecewiseConstant]: IPC = getattr(sys.modules[__name__], "IntPiecewiseConstant") - field_type = typing.Union[int, caimira.models.IntPiecewiseConstant, IPC] - elif new_field.type == typing.Union[caimira.models.Interval, None]: + field_type = typing.Union[int, models.IntPiecewiseConstant, IPC] + elif new_field.type == typing.Union[models.Interval, None]: I = getattr(sys.modules[__name__], "Interval") - field_type = typing.Union[None, caimira.models.Interval, I] + field_type = typing.Union[None, models.Interval, I] else: # Check that we don't need to do anything with this type. for item in new_field.type.__args__: - if getattr(item, '__module__', None) == 'caimira.models': + if getattr(item, '__module__', None) == 'source.models.models': raise ValueError( f"unsupported type annotation transformation required for {new_field.type}") - elif field_type.__module__ == 'caimira.models': + elif field_type.__module__ == 'source.models.models': mc_model = getattr(sys.modules[__name__], new_field.type.__name__) field_type = typing.Union[new_field.type, mc_model] @@ -119,7 +119,7 @@ def _build_mc_model(model: dataclass_instance) -> typing.Type[MCModelBase[_Model _MODEL_CLASSES = [ - cls for cls in vars(caimira.models).values() + cls for cls in vars(models).values() if dataclasses.is_dataclass(cls) ] diff --git a/caimira/monte_carlo/sampleable.py b/caimira/src/caimira/calculator/models/monte_carlo/sampleable.py similarity index 98% rename from caimira/monte_carlo/sampleable.py rename to caimira/src/caimira/calculator/models/monte_carlo/sampleable.py index 4bbc4c35..eddb4d24 100644 --- a/caimira/monte_carlo/sampleable.py +++ b/caimira/src/caimira/calculator/models/monte_carlo/sampleable.py @@ -3,7 +3,7 @@ import numpy as np from sklearn.neighbors import KernelDensity # type: ignore -import caimira.models +from caimira.calculator.models import models # Declare a float array type of a given size. # There is no better way to declare this currently, unfortunately. @@ -158,5 +158,5 @@ def generate_samples(self, size: int) -> float_array_size_n: _VectorisedFloatOrSampleable = typing.Union[ - SampleableDistribution, caimira.models._VectorisedFloat, + SampleableDistribution, models._VectorisedFloat, ] diff --git a/caimira/profiler.py b/caimira/src/caimira/calculator/models/profiler.py similarity index 100% rename from caimira/profiler.py rename to caimira/src/caimira/calculator/models/profiler.py diff --git a/caimira/utils.py b/caimira/src/caimira/calculator/models/utils.py similarity index 100% rename from caimira/utils.py rename to caimira/src/caimira/calculator/models/utils.py diff --git a/caimira/src/caimira/calculator/report/__init__.py b/caimira/src/caimira/calculator/report/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/caimira/apps/calculator/report_generator.py b/caimira/src/caimira/calculator/report/report_generator.py similarity index 54% rename from caimira/apps/calculator/report_generator.py rename to caimira/src/caimira/calculator/report/report_generator.py index 587c3b64..7c6f5cea 100644 --- a/caimira/apps/calculator/report_generator.py +++ b/caimira/src/caimira/calculator/report/report_generator.py @@ -1,26 +1,18 @@ import concurrent.futures import base64 import dataclasses -from datetime import datetime import io -import json import typing -import urllib -import zlib - -import jinja2 import numpy as np import matplotlib.pyplot as plt -from caimira import models -from caimira.apps.calculator import markdown_tools -from caimira.profiler import profile -from caimira.store.data_registry import DataRegistry -from ... import monte_carlo as mc -from .model_generator import VirusFormData -from ... import dataclass_utils -from caimira.enums import ViralLoads - +from caimira.calculator.models import models +# from caimira.apps.calculator import markdown_tools +# from caimira.profiler import profile +from caimira.calculator.store.data_registry import DataRegistry +from caimira.calculator.validators.virus.virus_validator import VirusFormData +from caimira.calculator.models import dataclass_utils +from caimira.calculator.models.enums import ViralLoads def model_start_end(model: models.ExposureModel): t_start = min(model.exposed.presence_interval().boundaries()[0][0], @@ -82,7 +74,6 @@ def walk_model(model, name=""): # such as PeriodicIntervals, which extend beyond the model itself). return sorted(time for time in change_times if (t_start <= time <= t_end)) - def interesting_times(model: models.ExposureModel, approx_n_pts: typing.Optional[int] = None) -> typing.List[float]: """ Pick approximately ``approx_n_pts`` time points which are interesting for the @@ -104,6 +95,7 @@ def interesting_times(model: models.ExposureModel, approx_n_pts: typing.Optional return nice_times + def concentrations_with_sr_breathing(form: VirusFormData, model: models.ExposureModel, times: typing.List[float], short_range_intervals: typing.List) -> typing.List[float]: lower_concentrations = [] for time in times: @@ -115,16 +107,20 @@ def concentrations_with_sr_breathing(form: VirusFormData, model: models.Exposure lower_concentrations.append(np.array(model.concentration_model.concentration(float(time))).mean()) return lower_concentrations + def _calculate_deposited_exposure(model, time1, time2, fn_name=None): return np.array(model.deposited_exposure_between_bounds(float(time1), float(time2))).mean(),fn_name + def _calculate_long_range_deposited_exposure(model, time1, time2, fn_name=None): return np.array(model.long_range_deposited_exposure_between_bounds(float(time1), float(time2))).mean(), fn_name + def _calculate_co2_concentration(CO2_model, time, fn_name=None): return np.array(CO2_model.concentration(float(time))).mean(), fn_name -@profile + +# @profile def calculate_report_data(form: VirusFormData, model: models.ExposureModel, executor_factory: typing.Callable[[], concurrent.futures.Executor]) -> typing.Dict[str, typing.Any]: times = interesting_times(model) short_range_intervals = [interaction.presence.boundaries()[0] for interaction in model.short_range] @@ -215,25 +211,6 @@ def calculate_report_data(form: VirusFormData, model: models.ExposureModel, exec } -def generate_permalink(base_url, get_root_url, get_root_calculator_url, form: VirusFormData): - form_dict = VirusFormData.to_dict(form, strip_defaults=True) - - # Generate the calculator URL arguments that would be needed to re-create this - # form. - args = urllib.parse.urlencode(form_dict) - - # Then zlib compress + base64 encode the string. To be inverted by the - # /_c/ endpoint. - compressed_args = base64.b64encode(zlib.compress(args.encode())).decode() - qr_url = f"{base_url}{get_root_url()}/_c/{compressed_args}" - url = f"{base_url}{get_root_calculator_url()}?{args}" - - return { - 'link': url, - 'shortened': qr_url, - } - - def conditional_prob_inf_given_vl_dist( data_registry: DataRegistry, infection_probability: models._VectorisedFloat, @@ -279,7 +256,7 @@ def uncertainties_plot(infection_probability: models._VectorisedFloat, lower_percentiles: models._VectorisedFloat, upper_percentiles: models._VectorisedFloat): - fig, axes = plt.subplots(2, 3, + fig, axes = plt.subplots(2, 3, gridspec_kw={'width_ratios': [5, 0.5] + [1], 'height_ratios': [3, 1], 'wspace': 0}, sharey='row', @@ -325,13 +302,6 @@ def uncertainties_plot(infection_probability: models._VectorisedFloat, return fig -def _img2bytes(figure): - # Draw the image - img_data = io.BytesIO() - figure.save(img_data, format='png', bbox_inches="tight") - return img_data - - def _figure2bytes(figure): # Draw the image img_data = io.BytesIO() @@ -344,249 +314,3 @@ def img2base64(img_data) -> str: pic_hash = base64.b64encode(img_data.read()).decode('ascii') # A src suitable for a tag such as f'. return f'data:image/png;base64,{pic_hash}' - - -def minutes_to_time(minutes: int) -> str: - minute_string = str(minutes % 60) - minute_string = "0" * (2 - len(minute_string)) + minute_string - hour_string = str(minutes // 60) - hour_string = "0" * (2 - len(hour_string)) + hour_string - - return f"{hour_string}:{minute_string}" - - -def readable_minutes(minutes: int) -> str: - time = float(minutes) - unit = " minute" - if time % 60 == 0: - time = minutes/60 - unit = " hour" - if time != 1: - unit += "s" - - if time.is_integer(): - time_str = "{:0.0f}".format(time) - else: - time_str = "{0:.2f}".format(time) - - return time_str + unit - - -def hour_format(hour: float) -> str: - # Convert float hour to HH:MM format - hours = int(hour) - minutes = int(hour % 1 * 60) - return f"{hours}:{minutes if minutes != 0 else '00'}" - - -def percentage(absolute: float) -> float: - return absolute * 100 - - -def non_zero_percentage(percentage: int) -> str: - if percentage < 0.01: - return "<0.01%" - elif percentage < 1: - return "{:0.2f}%".format(percentage) - elif percentage > 99.9 or np.isnan(percentage): - return ">99.9%" - else: - return "{:0.1f}%".format(percentage) - - -def manufacture_viral_load_scenarios_percentiles(model: mc.ExposureModel) -> typing.Dict[str, mc.ExposureModel]: - viral_load = model.concentration_model.infected.virus.viral_load_in_sputum - scenarios = {} - for percentil in (0.01, 0.05, 0.25, 0.5, 0.75, 0.95, 0.99): - vl = np.quantile(viral_load, percentil) - specific_vl_scenario = dataclass_utils.nested_replace(model, - {'concentration_model.infected.virus.viral_load_in_sputum': vl} - ) - scenarios[str(vl)] = np.mean(specific_vl_scenario.infection_probability()) - return scenarios - - -def manufacture_alternative_scenarios(form: VirusFormData) -> typing.Dict[str, mc.ExposureModel]: - scenarios = {} - if (form.short_range_option == "short_range_no"): - # Two special option cases - HEPA and/or FFP2 masks. - FFP2_being_worn = bool(form.mask_wearing_option == 'mask_on' and form.mask_type == 'FFP2') - if FFP2_being_worn and form.hepa_option: - FFP2andHEPAalternative = dataclass_utils.replace(form, mask_type='Type I') - if not (form.hepa_option and form.mask_wearing_option == 'mask_on' and form.mask_type == 'Type I'): - scenarios['Base scenario with HEPA filter and Type I masks'] = FFP2andHEPAalternative.build_mc_model() - if not FFP2_being_worn and form.hepa_option: - noHEPAalternative = dataclass_utils.replace(form, mask_type = 'FFP2') - noHEPAalternative = dataclass_utils.replace(noHEPAalternative, mask_wearing_option = 'mask_on') - noHEPAalternative = dataclass_utils.replace(noHEPAalternative, hepa_option=False) - if not (not form.hepa_option and FFP2_being_worn): - scenarios['Base scenario without HEPA filter, with FFP2 masks'] = noHEPAalternative.build_mc_model() - - # The remaining scenarios are based on Type I masks (possibly not worn) - # and no HEPA filtration. - form = dataclass_utils.replace(form, mask_type='Type I') - if form.hepa_option: - form = dataclass_utils.replace(form, hepa_option=False) - - with_mask = dataclass_utils.replace(form, mask_wearing_option='mask_on') - without_mask = dataclass_utils.replace(form, mask_wearing_option='mask_off') - - if form.ventilation_type == 'mechanical_ventilation': - #scenarios['Mechanical ventilation with Type I masks'] = with_mask.build_mc_model() - if not (form.mask_wearing_option == 'mask_off'): - scenarios['Mechanical ventilation without masks'] = without_mask.build_mc_model() - - elif form.ventilation_type == 'natural_ventilation': - #scenarios['Windows open with Type I masks'] = with_mask.build_mc_model() - if not (form.mask_wearing_option == 'mask_off'): - scenarios['Windows open without masks'] = without_mask.build_mc_model() - - # No matter the ventilation scheme, we include scenarios which don't have any ventilation. - with_mask_no_vent = dataclass_utils.replace(with_mask, ventilation_type='no_ventilation') - without_mask_or_vent = dataclass_utils.replace(without_mask, ventilation_type='no_ventilation') - - if not (form.mask_wearing_option == 'mask_on' and form.mask_type == 'Type I' and form.ventilation_type == 'no_ventilation'): - scenarios['No ventilation with Type I masks'] = with_mask_no_vent.build_mc_model() - if not (form.mask_wearing_option == 'mask_off' and form.ventilation_type == 'no_ventilation'): - scenarios['Neither ventilation nor masks'] = without_mask_or_vent.build_mc_model() - - else: - no_short_range_alternative = dataclass_utils.replace(form, short_range_interactions=[], total_people=form.total_people - form.short_range_occupants) - scenarios['Base scenario without short-range interactions'] = no_short_range_alternative.build_mc_model() - - return scenarios - - -def scenario_statistics( - mc_model: mc.ExposureModel, - sample_times: typing.List[float], - compute_prob_exposure: bool -): - model = mc_model.build_model(size=mc_model.data_registry.monte_carlo['sample_size']) - if (compute_prob_exposure): - # It means we have data to calculate the total_probability_rule - prob_probabilistic_exposure = model.total_probability_rule() - else: - prob_probabilistic_exposure = 0. - - return { - 'probability_of_infection': np.mean(model.infection_probability()), - 'expected_new_cases': np.mean(model.expected_new_cases()), - 'concentrations': [ - np.mean(model.concentration(time)) - for time in sample_times - ], - 'prob_probabilistic_exposure': prob_probabilistic_exposure, - } - - -def comparison_report( - form: VirusFormData, - report_data: typing.Dict[str, typing.Any], - scenarios: typing.Dict[str, mc.ExposureModel], - sample_times: typing.List[float], - executor_factory: typing.Callable[[], concurrent.futures.Executor], -): - if (form.short_range_option == "short_range_no"): - statistics = { - 'Current scenario' : { - 'probability_of_infection': report_data['prob_inf'], - 'expected_new_cases': report_data['expected_new_cases'], - 'concentrations': report_data['concentrations'], - } - } - else: - statistics = {} - - if (form.short_range_option == "short_range_yes" and form.exposure_option == "p_probabilistic_exposure"): - compute_prob_exposure = True - else: - compute_prob_exposure = False - - with executor_factory() as executor: - results = executor.map( - scenario_statistics, - scenarios.values(), - [sample_times] * len(scenarios), - [compute_prob_exposure] * len(scenarios), - timeout=60, - ) - - for (name, model), model_stats in zip(scenarios.items(), results): - statistics[name] = model_stats - - return { - 'stats': statistics, - } - - -@dataclasses.dataclass -class ReportGenerator: - jinja_loader: jinja2.BaseLoader - get_root_url: typing.Any - get_root_calculator_url: typing.Any - - def build_report( - self, - base_url: str, - form: VirusFormData, - executor_factory: typing.Callable[[], concurrent.futures.Executor], - ) -> str: - model = form.build_model() - context = self.prepare_context(base_url, model, form, executor_factory=executor_factory) - return self.render(context) - - def prepare_context( - self, - base_url: str, - model: models.ExposureModel, - form: VirusFormData, - executor_factory: typing.Callable[[], concurrent.futures.Executor], - ) -> dict: - now = datetime.utcnow().astimezone() - time = now.strftime("%Y-%m-%d %H:%M:%S UTC") - - data_registry_version = f"v{model.data_registry.version}" if model.data_registry.version else None - context = { - 'model': model, - 'form': form, - 'creation_date': time, - 'data_registry_version': data_registry_version, - } - - scenario_sample_times = interesting_times(model) - report_data = calculate_report_data(form, model, executor_factory=executor_factory) - context.update(report_data) - - alternative_scenarios = manufacture_alternative_scenarios(form) - context['alternative_viral_load'] = manufacture_viral_load_scenarios_percentiles(model) if form.conditional_probability_viral_loads else None - context['alternative_scenarios'] = comparison_report( - form, report_data, alternative_scenarios, scenario_sample_times, executor_factory=executor_factory, - ) - context['permalink'] = generate_permalink(base_url, self.get_root_url, self.get_root_calculator_url, form) - context['get_url'] = self.get_root_url - context['get_calculator_url'] = self.get_root_calculator_url - - return context - - def _template_environment(self) -> jinja2.Environment: - env = jinja2.Environment( - loader=self.jinja_loader, - undefined=jinja2.StrictUndefined, - ) - env.globals["common_text"] = markdown_tools.extract_rendered_markdown_blocks( - env.get_template('common_text.md.j2') - ) - env.filters['non_zero_percentage'] = non_zero_percentage - env.filters['readable_minutes'] = readable_minutes - env.filters['minutes_to_time'] = minutes_to_time - env.filters['hour_format'] = hour_format - env.filters['float_format'] = "{0:.2f}".format - env.filters['int_format'] = "{:0.0f}".format - env.filters['percentage'] = percentage - env.filters['JSONify'] = json.dumps - return env - - def render(self, context: dict) -> str: - template = self._template_environment().get_template("calculator.report.html.j2") - return template.render(**context, text_blocks=template.globals["common_text"]) diff --git a/caimira/store/data_registry.py b/caimira/src/caimira/calculator/store/data_registry.py similarity index 99% rename from caimira/store/data_registry.py rename to caimira/src/caimira/calculator/store/data_registry.py index 570a4fd9..5b549858 100644 --- a/caimira/store/data_registry.py +++ b/caimira/src/caimira/calculator/store/data_registry.py @@ -1,4 +1,4 @@ -from caimira.enums import ViralLoads +from ..models.enums import ViralLoads class DataRegistry: diff --git a/caimira/store/data_service.py b/caimira/src/caimira/calculator/store/data_service.py similarity index 96% rename from caimira/store/data_service.py rename to caimira/src/caimira/calculator/store/data_service.py index 1ecbac82..7271e708 100644 --- a/caimira/store/data_service.py +++ b/caimira/src/caimira/calculator/store/data_service.py @@ -2,7 +2,7 @@ import typing import requests -from caimira.store.data_registry import DataRegistry +from ..store.data_registry import DataRegistry logger = logging.getLogger("DATA") diff --git a/caimira/src/caimira/calculator/validators/__init__.py b/caimira/src/caimira/calculator/validators/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/caimira/src/caimira/calculator/validators/co2/__init__.py b/caimira/src/caimira/calculator/validators/co2/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/caimira/apps/calculator/co2_model_generator.py b/caimira/src/caimira/calculator/validators/co2/co2_validator.py similarity index 97% rename from caimira/apps/calculator/co2_model_generator.py rename to caimira/src/caimira/calculator/validators/co2/co2_validator.py index a4a6a9c2..a630e774 100644 --- a/caimira/apps/calculator/co2_model_generator.py +++ b/caimira/src/caimira/calculator/validators/co2/co2_validator.py @@ -6,11 +6,11 @@ import matplotlib.pyplot as plt import re -from caimira import models -from caimira.store.data_registry import DataRegistry -from .form_data import FormData, cast_class_fields -from .defaults import NO_DEFAULT -from .report_generator import img2base64, _figure2bytes +from ..form_validator import FormData, cast_class_fields +from ..defaults import NO_DEFAULT +from ...store.data_registry import DataRegistry +from ...models import models +from ...report.report_generator import img2base64, _figure2bytes minutes_since_midnight = typing.NewType('minutes_since_midnight', int) diff --git a/caimira/apps/calculator/defaults.py b/caimira/src/caimira/calculator/validators/defaults.py similarity index 100% rename from caimira/apps/calculator/defaults.py rename to caimira/src/caimira/calculator/validators/defaults.py diff --git a/caimira/apps/calculator/form_data.py b/caimira/src/caimira/calculator/validators/form_validator.py similarity index 79% rename from caimira/apps/calculator/form_data.py rename to caimira/src/caimira/calculator/validators/form_validator.py index a0a96b67..94cf32b4 100644 --- a/caimira/apps/calculator/form_data.py +++ b/caimira/src/caimira/calculator/validators/form_validator.py @@ -7,9 +7,9 @@ import numpy as np -from caimira import models -from caimira.store.data_registry import DataRegistry from .defaults import DEFAULTS, NO_DEFAULT, COFFEE_OPTIONS_INT +from ..models import models +from ..store.data_registry import DataRegistry LOG = logging.getLogger(__name__) @@ -26,13 +26,16 @@ class FormData: exposed_lunch_option: bool exposed_lunch_start: minutes_since_midnight exposed_start: minutes_since_midnight - infected_coffee_break_option: str #Used if infected_dont_have_breaks_with_exposed - infected_coffee_duration: int #Used if infected_dont_have_breaks_with_exposed + # Used if infected_dont_have_breaks_with_exposed + infected_coffee_break_option: str + infected_coffee_duration: int # Used if infected_dont_have_breaks_with_exposed infected_dont_have_breaks_with_exposed: bool infected_finish: minutes_since_midnight - infected_lunch_finish: minutes_since_midnight #Used if infected_dont_have_breaks_with_exposed - infected_lunch_option: bool #Used if infected_dont_have_breaks_with_exposed - infected_lunch_start: minutes_since_midnight #Used if infected_dont_have_breaks_with_exposed + # Used if infected_dont_have_breaks_with_exposed + infected_lunch_finish: minutes_since_midnight + infected_lunch_option: bool # Used if infected_dont_have_breaks_with_exposed + # Used if infected_dont_have_breaks_with_exposed + infected_lunch_start: minutes_since_midnight infected_people: int infected_start: minutes_since_midnight room_volume: float @@ -47,7 +50,6 @@ def from_dict(cls, form_data: typing.Dict, data_registry: DataRegistry): # Take a copy of the form data so that we can mutate it. form_data = form_data.copy() form_data.pop('_xsrf', None) - # Don't let arbitrary unescaped HTML through the net. for key, value in form_data.items(): if isinstance(value, str): @@ -64,7 +66,8 @@ def from_dict(cls, form_data: typing.Dict, data_registry: DataRegistry): form_data[key] = _CAST_RULES_FORM_ARG_TO_NATIVE[key](value) if key not in cls._DEFAULTS: - raise ValueError(f'Invalid argument "{html.escape(key)}" given') + raise ValueError( + f'Invalid argument "{html.escape(key)}" given') instance = cls(**form_data, data_registry=data_registry) instance.validate() @@ -93,7 +96,8 @@ def to_dict(cls, form: "FormData", strip_defaults: bool = False) -> dict: def validate_population_parameters(self): # Validate number of infected <= number of total people if self.infected_people >= self.total_people: - raise ValueError('Number of infected people cannot be greater or equal to the number of total people.') + raise ValueError( + 'Number of infected people cannot be greater or equal to the number of total people.') # Validate time intervals selected by user time_intervals = [ @@ -101,9 +105,11 @@ def validate_population_parameters(self): ['infected_start', 'infected_finish'], ] if self.exposed_lunch_option: - time_intervals.append(['exposed_lunch_start', 'exposed_lunch_finish']) + time_intervals.append( + ['exposed_lunch_start', 'exposed_lunch_finish']) if self.infected_dont_have_breaks_with_exposed and self.infected_lunch_option: - time_intervals.append(['infected_lunch_start', 'infected_lunch_finish']) + time_intervals.append( + ['infected_lunch_start', 'infected_lunch_finish']) for start_name, end_name in time_intervals: start = getattr(self, start_name) @@ -116,29 +122,33 @@ def validate_lunch(start, finish): lunch_start = getattr(self, f'{population}_lunch_start') lunch_finish = getattr(self, f'{population}_lunch_finish') return (start <= lunch_start <= finish and - start <= lunch_finish <= finish) + start <= lunch_finish <= finish) def get_lunch_mins(population): lunch_mins = 0 if getattr(self, f'{population}_lunch_option'): - lunch_mins = getattr(self, f'{population}_lunch_finish') - getattr(self, f'{population}_lunch_start') + lunch_mins = getattr( + self, f'{population}_lunch_finish') - getattr(self, f'{population}_lunch_start') return lunch_mins def get_coffee_mins(population): coffee_mins = 0 if getattr(self, f'{population}_coffee_break_option') != 'coffee_break_0': - coffee_mins = COFFEE_OPTIONS_INT[getattr(self, f'{population}_coffee_break_option')] * getattr(self, f'{population}_coffee_duration') + coffee_mins = COFFEE_OPTIONS_INT[getattr( + self, f'{population}_coffee_break_option')] * getattr(self, f'{population}_coffee_duration') return coffee_mins def get_activity_mins(population): return getattr(self, f'{population}_finish') - getattr(self, f'{population}_start') - populations = ['exposed', 'infected'] if self.infected_dont_have_breaks_with_exposed else ['exposed'] + populations = [ + 'exposed', 'infected'] if self.infected_dont_have_breaks_with_exposed else ['exposed'] for population in populations: # Validate lunch time within the activity times. if (getattr(self, f'{population}_lunch_option') and - not validate_lunch(getattr(self, f'{population}_start'), getattr(self, f'{population}_finish')) - ): + not validate_lunch(getattr(self, f'{population}_start'), getattr( + self, f'{population}_finish')) + ): raise ValueError( f"{population} lunch break must be within presence times." ) @@ -152,7 +162,8 @@ def get_activity_mins(population): for attr_name, valid_set in [('exposed_coffee_break_option', COFFEE_OPTIONS_INT), ('infected_coffee_break_option', COFFEE_OPTIONS_INT)]: if getattr(self, attr_name) not in valid_set: - raise ValueError(f"{getattr(self, attr_name)} is not a valid value for {attr_name}") + raise ValueError( + f"{getattr(self, attr_name)} is not a valid value for {attr_name}") def validate(self): raise NotImplementedError("Subclass must implement") @@ -161,7 +172,8 @@ def build_model(self, sample_size=None): raise NotImplementedError("Subclass must implement") def _compute_breaks_in_interval(self, start, finish, n_breaks, duration) -> models.BoundarySequence_t: - break_delay = ((finish - start) - (n_breaks * duration)) // (n_breaks+1) + break_delay = ((finish - start) - + (n_breaks * duration)) // (n_breaks+1) break_times = [] end = start for n in range(n_breaks): @@ -173,14 +185,16 @@ def _compute_breaks_in_interval(self, start, finish, n_breaks, duration) -> mode def exposed_lunch_break_times(self) -> models.BoundarySequence_t: result = [] if self.exposed_lunch_option: - result.append((self.exposed_lunch_start, self.exposed_lunch_finish)) + result.append((self.exposed_lunch_start, + self.exposed_lunch_finish)) return tuple(result) def infected_lunch_break_times(self) -> models.BoundarySequence_t: if self.infected_dont_have_breaks_with_exposed: result = [] if self.infected_lunch_option: - result.append((self.infected_lunch_start, self.infected_lunch_finish)) + result.append((self.infected_lunch_start, + self.infected_lunch_finish)) return tuple(result) else: return self.exposed_lunch_break_times() @@ -194,7 +208,8 @@ def infected_number_of_coffee_breaks(self) -> int: def _coffee_break_times(self, activity_start, activity_finish, coffee_breaks, coffee_duration, lunch_start, lunch_finish) -> models.BoundarySequence_t: time_before_lunch = lunch_start - activity_start time_after_lunch = activity_finish - lunch_finish - before_lunch_frac = time_before_lunch / (time_before_lunch + time_after_lunch) + before_lunch_frac = time_before_lunch / \ + (time_before_lunch + time_after_lunch) n_morning_breaks = round(coffee_breaks * before_lunch_frac) breaks = ( self._compute_breaks_in_interval( @@ -211,9 +226,11 @@ def exposed_coffee_break_times(self) -> models.BoundarySequence_t: if exposed_coffee_breaks == 0: return () if self.exposed_lunch_option: - breaks = self._coffee_break_times(self.exposed_start, self.exposed_finish, exposed_coffee_breaks, self.exposed_coffee_duration, self.exposed_lunch_start, self.exposed_lunch_finish) + breaks = self._coffee_break_times(self.exposed_start, self.exposed_finish, exposed_coffee_breaks, + self.exposed_coffee_duration, self.exposed_lunch_start, self.exposed_lunch_finish) else: - breaks = self._compute_breaks_in_interval(self.exposed_start, self.exposed_finish, exposed_coffee_breaks, self.exposed_coffee_duration) + breaks = self._compute_breaks_in_interval( + self.exposed_start, self.exposed_finish, exposed_coffee_breaks, self.exposed_coffee_duration) return breaks def infected_coffee_break_times(self) -> models.BoundarySequence_t: @@ -222,9 +239,11 @@ def infected_coffee_break_times(self) -> models.BoundarySequence_t: if infected_coffee_breaks == 0: return () if self.infected_lunch_option: - breaks = self._coffee_break_times(self.infected_start, self.infected_finish, infected_coffee_breaks, self.infected_coffee_duration, self.infected_lunch_start, self.infected_lunch_finish) + breaks = self._coffee_break_times(self.infected_start, self.infected_finish, infected_coffee_breaks, + self.infected_coffee_duration, self.infected_lunch_start, self.infected_lunch_finish) else: - breaks = self._compute_breaks_in_interval(self.infected_start, self.infected_finish, infected_coffee_breaks, self.infected_coffee_duration) + breaks = self._compute_breaks_in_interval( + self.infected_start, self.infected_finish, infected_coffee_breaks, self.infected_coffee_duration) return breaks else: return self.exposed_coffee_break_times() @@ -232,13 +251,14 @@ def infected_coffee_break_times(self) -> models.BoundarySequence_t: def generate_specific_break_times(self, breaks_dict: dict, target: str) -> models.BoundarySequence_t: break_times = [] for n in breaks_dict[f'{target}_breaks']: - # Parse break times. + # Parse break times. begin = time_string_to_minutes(n["start_time"]) end = time_string_to_minutes(n["finish_time"]) for time in [begin, end]: # For a specific break, the infected and exposed presence is the same. if not getattr(self, f'{target}_start') < time < getattr(self, f'{target}_finish'): - raise ValueError(f'All breaks should be within the simulation time. Got {time_minutes_to_string(time)}.') + raise ValueError( + f'All breaks should be within the simulation time. Got {time_minutes_to_string(time)}.') break_times.append((begin, end)) return tuple(break_times) @@ -260,7 +280,8 @@ def present_interval( # Order the breaks by their start-time, and ensure that they are monotonic # and that the start of one break happens after the end of another. - break_boundaries: models.BoundarySequence_t = tuple(sorted(breaks, key=lambda break_pair: break_pair[0])) + break_boundaries: models.BoundarySequence_t = tuple( + sorted(breaks, key=lambda break_pair: break_pair[0])) for break_start, break_end in break_boundaries: if break_start >= break_end: @@ -269,13 +290,15 @@ def present_interval( prev_break_end = break_boundaries[0][1] for break_start, break_end in break_boundaries[1:]: if prev_break_end >= break_start: - raise ValueError(f"A break starts before another ends ({break_start}, {break_end}, {prev_break_end}).") + raise ValueError( + f"A break starts before another ends ({break_start}, {break_end}, {prev_break_end}).") prev_break_end = break_end present_intervals = [] current_time = start - LOG.debug(f"starting time march at {_hours2timestring(current_time/60)} to {_hours2timestring(finish/60)}") + LOG.debug( + f"starting time march at {_hours2timestring(current_time/60)} to {_hours2timestring(finish/60)}") # As we step through the breaks. For each break there are 6 important cases # we must cover. Let S=start; E=end; Bs=Break start; Be=Break end: @@ -336,8 +359,9 @@ def present_interval( return models.SpecificInterval(tuple(present_intervals)) def infected_present_interval(self) -> models.Interval: - if self.specific_breaks != {}: # It means the breaks are specific and not predefined - breaks = self.generate_specific_break_times(breaks_dict=self.specific_breaks, target='exposed') + if self.specific_breaks != {}: # It means the breaks are specific and not predefined + breaks = self.generate_specific_break_times( + breaks_dict=self.specific_breaks, target='exposed') else: breaks = self.infected_lunch_break_times() + self.infected_coffee_break_times() return self.present_interval( @@ -346,14 +370,17 @@ def infected_present_interval(self) -> models.Interval: ) def population_present_interval(self) -> models.Interval: - state_change_times = set(self.infected_present_interval().transition_times()) - state_change_times.update(self.exposed_present_interval().transition_times()) + state_change_times = set( + self.infected_present_interval().transition_times()) + state_change_times.update( + self.exposed_present_interval().transition_times()) all_state_changes = sorted(state_change_times) return models.SpecificInterval(tuple(zip(all_state_changes[:-1], all_state_changes[1:]))) def exposed_present_interval(self) -> models.Interval: - if self.specific_breaks != {}: # It means the breaks are specific and not predefined - breaks = self.generate_specific_break_times(breaks_dict=self.specific_breaks, target='exposed') + if self.specific_breaks != {}: # It means the breaks are specific and not predefined + breaks = self.generate_specific_break_times( + breaks_dict=self.specific_breaks, target='exposed') else: breaks = self.exposed_lunch_break_times() + self.exposed_coffee_break_times() return self.present_interval( @@ -382,7 +409,7 @@ def time_minutes_to_string(time: int) -> str: :param time: The number of minutes between 'time' and 00:00 :return: A string of the form "HH:MM" representing a time of day """ - return "{0:0=2d}".format(int(time/60)) + ":" + "{0:0=2d}".format(time%60) + return "{0:0=2d}".format(int(time/60)) + ":" + "{0:0=2d}".format(time % 60) def string_to_list(s: str) -> list: @@ -409,7 +436,8 @@ def _safe_int_cast(value) -> int: elif isinstance(value, str) and value.isdecimal(): return int(value) else: - raise TypeError(f"Unable to safely cast {value} ({type(value)} type) to int") + raise TypeError( + f"Unable to safely cast {value} ({type(value)} type) to int") #: Mapping of field name to a callable which can convert values from form @@ -420,6 +448,7 @@ def _safe_int_cast(value) -> int: #: that can be encoded to URL arguments. _CAST_RULES_NATIVE_TO_FORM_ARG: typing.Dict[str, typing.Callable] = {} + def cast_class_fields(cls): for _field in dataclasses.fields(cls): if _field.type is minutes_since_midnight: @@ -439,4 +468,5 @@ def cast_class_fields(cls): _CAST_RULES_FORM_ARG_TO_NATIVE[_field.name] = string_to_dict _CAST_RULES_NATIVE_TO_FORM_ARG[_field.name] = dict_to_string + cast_class_fields(FormData) diff --git a/caimira/src/caimira/calculator/validators/virus/__init__.py b/caimira/src/caimira/calculator/validators/virus/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/caimira/apps/calculator/model_generator.py b/caimira/src/caimira/calculator/validators/virus/virus_validator.py similarity index 78% rename from caimira/apps/calculator/model_generator.py rename to caimira/src/caimira/calculator/validators/virus/virus_validator.py index aa35ba1e..74f10427 100644 --- a/caimira/apps/calculator/model_generator.py +++ b/caimira/src/caimira/calculator/validators/virus/virus_validator.py @@ -6,17 +6,13 @@ import numpy as np -from caimira import models -from caimira import data -import caimira.data.weather -import caimira.monte_carlo as mc -from .. import calculator -from .form_data import FormData, cast_class_fields, time_string_to_minutes -from caimira.monte_carlo.data import activity_distributions, virus_distributions, mask_distributions, short_range_distances -from caimira.monte_carlo.data import expiration_distribution, expiration_BLO_factors, expiration_distributions, short_range_expiration_distributions -from .defaults import (DEFAULTS, CONFIDENCE_LEVEL_OPTIONS, - MECHANICAL_VENTILATION_TYPES, MASK_WEARING_OPTIONS, MONTH_NAMES, VACCINE_BOOSTER_TYPE, VACCINE_TYPE, - VENTILATION_TYPES, VOLUME_TYPES, WINDOWS_OPENING_REGIMES, WINDOWS_TYPES) +from ..form_validator import FormData, cast_class_fields, time_string_to_minutes +from ..defaults import (DEFAULTS, CONFIDENCE_LEVEL_OPTIONS, + MECHANICAL_VENTILATION_TYPES, MASK_WEARING_OPTIONS, MONTH_NAMES, VACCINE_BOOSTER_TYPE, VACCINE_TYPE, + VENTILATION_TYPES, VOLUME_TYPES, WINDOWS_OPENING_REGIMES, WINDOWS_TYPES) +from ...models import models, data, monte_carlo as mc +from ...models.monte_carlo.data import activity_distributions, virus_distributions, mask_distributions, short_range_distances +from ...models.monte_carlo.data import expiration_distribution, expiration_BLO_factors, expiration_distributions, short_range_expiration_distributions LOG = logging.getLogger("MODEL") @@ -77,15 +73,17 @@ class VirusFormData(FormData): _DEFAULTS: typing.ClassVar[typing.Dict[str, typing.Any]] = DEFAULTS def validate(self): - # Validate population parameters self.validate_population_parameters() validation_tuples = [('activity_type', self.data_registry.population_scenario_activity.keys()), - ('mechanical_ventilation_type', MECHANICAL_VENTILATION_TYPES), - ('mask_type', list(mask_distributions(self.data_registry).keys())), + ('mechanical_ventilation_type', + MECHANICAL_VENTILATION_TYPES), + ('mask_type', list(mask_distributions( + self.data_registry).keys())), ('mask_wearing_option', MASK_WEARING_OPTIONS), ('ventilation_type', VENTILATION_TYPES), - ('virus_type', list(virus_distributions(self.data_registry).keys())), + ('virus_type', list(virus_distributions( + self.data_registry).keys())), ('volume_type', VOLUME_TYPES), ('window_opening_regime', WINDOWS_OPENING_REGIMES), ('window_type', WINDOWS_TYPES), @@ -96,11 +94,13 @@ def validate(self): for attr_name, valid_set in validation_tuples: if getattr(self, attr_name) not in valid_set: - raise ValueError(f"{getattr(self, attr_name)} is not a valid value for {attr_name}") + raise ValueError( + f"{getattr(self, attr_name)} is not a valid value for {attr_name}") # Validate number of infected people == 1 when activity is Conference/Training. if self.activity_type == 'training' and self.infected_people > 1: - raise ValueError('Conference/Training activities are limited to 1 infected.') + raise ValueError( + 'Conference/Training activities are limited to 1 infected.') # Validate ventilation parameters if self.ventilation_type == 'natural_ventilation': @@ -115,7 +115,7 @@ def validate(self): "ventilation_type is 'natural_ventilation'" ) if (self.window_opening_regime == 'windows_open_periodically' and - self.windows_duration > self.windows_frequency): + self.windows_duration > self.windows_frequency): raise ValueError( 'Duration cannot be bigger than frequency.' ) @@ -128,61 +128,78 @@ def validate(self): # Validate specific inputs - breaks (exposed and infected) if self.specific_breaks != {}: if type(self.specific_breaks) is not dict: - raise TypeError('The specific breaks should be in a dictionary.') + raise TypeError( + 'The specific breaks should be in a dictionary.') dict_keys = list(self.specific_breaks.keys()) if "exposed_breaks" not in dict_keys: - raise TypeError(f'Unable to fetch "exposed_breaks" key. Got "{dict_keys[0]}".') + raise TypeError( + f'Unable to fetch "exposed_breaks" key. Got "{dict_keys[0]}".') if "infected_breaks" not in dict_keys: - raise TypeError(f'Unable to fetch "infected_breaks" key. Got "{dict_keys[1]}".') + raise TypeError( + f'Unable to fetch "infected_breaks" key. Got "{dict_keys[1]}".') for population_breaks in ['exposed_breaks', 'infected_breaks']: if self.specific_breaks[population_breaks] != []: if type(self.specific_breaks[population_breaks]) is not list: - raise TypeError(f'All breaks should be in a list. Got {type(self.specific_breaks[population_breaks])}.') + raise TypeError( + f'All breaks should be in a list. Got {type(self.specific_breaks[population_breaks])}.') for input_break in self.specific_breaks[population_breaks]: # Input validations. if type(input_break) is not dict: - raise TypeError(f'Each break should be a dictionary. Got {type(input_break)}.') + raise TypeError( + f'Each break should be a dictionary. Got {type(input_break)}.') dict_keys = list(input_break.keys()) if "start_time" not in input_break: - raise TypeError(f'Unable to fetch "start_time" key. Got "{dict_keys[0]}".') + raise TypeError( + f'Unable to fetch "start_time" key. Got "{dict_keys[0]}".') if "finish_time" not in input_break: - raise TypeError(f'Unable to fetch "finish_time" key. Got "{dict_keys[1]}".') + raise TypeError( + f'Unable to fetch "finish_time" key. Got "{dict_keys[1]}".') for time in input_break.values(): if not re.compile("^(2[0-3]|[01]?[0-9]):([0-5]?[0-9])$").match(time): - raise TypeError(f'Wrong time format - "HH:MM". Got "{time}".') + raise TypeError( + f'Wrong time format - "HH:MM". Got "{time}".') # Validate specific inputs - precise activity if self.precise_activity != {}: if type(self.precise_activity) is not dict: - raise TypeError('The precise activities should be in a dictionary.') + raise TypeError( + 'The precise activities should be in a dictionary.') dict_keys = list(self.precise_activity.keys()) if "physical_activity" not in dict_keys: - raise TypeError(f'Unable to fetch "physical_activity" key. Got "{dict_keys[0]}".') + raise TypeError( + f'Unable to fetch "physical_activity" key. Got "{dict_keys[0]}".') if "respiratory_activity" not in dict_keys: - raise TypeError(f'Unable to fetch "respiratory_activity" key. Got "{dict_keys[1]}".') + raise TypeError( + f'Unable to fetch "respiratory_activity" key. Got "{dict_keys[1]}".') if type(self.precise_activity['physical_activity']) is not str: - raise TypeError('The physical activities should be a single string.') + raise TypeError( + 'The physical activities should be a single string.') if type(self.precise_activity['respiratory_activity']) is not list: - raise TypeError('The respiratory activities should be in a list.') + raise TypeError( + 'The respiratory activities should be in a list.') total_percentage = 0 for respiratory_activity in self.precise_activity['respiratory_activity']: if type(respiratory_activity) is not dict: - raise TypeError('Each respiratory activity should be defined in a dictionary.') + raise TypeError( + 'Each respiratory activity should be defined in a dictionary.') dict_keys = list(respiratory_activity.keys()) if "type" not in dict_keys: - raise TypeError(f'Unable to fetch "type" key. Got "{dict_keys[0]}".') + raise TypeError( + f'Unable to fetch "type" key. Got "{dict_keys[0]}".') if "percentage" not in dict_keys: - raise TypeError(f'Unable to fetch "percentage" key. Got "{dict_keys[1]}".') + raise TypeError( + f'Unable to fetch "percentage" key. Got "{dict_keys[1]}".') total_percentage += respiratory_activity['percentage'] if total_percentage != 100: - raise ValueError(f'The sum of all respiratory activities should be 100. Got {total_percentage}.') + raise ValueError( + f'The sum of all respiratory activities should be 100. Got {total_percentage}.') # Validate number of people with short-range interactions max_occupants_for_sr = self.total_people - self.infected_people @@ -219,7 +236,8 @@ def build_mc_model(self) -> mc.ExposureModel: for interaction in self.short_range_interactions: short_range.append(mc.ShortRangeModel( data_registry=self.data_registry, - expiration=short_range_expiration_distributions(self.data_registry)[interaction['expiration']], + expiration=short_range_expiration_distributions( + self.data_registry)[interaction['expiration']], activity=infected_population.activity, presence=self.short_range_interval(interaction), distance=short_range_distances(self.data_registry), @@ -234,7 +252,7 @@ def build_mc_model(self) -> mc.ExposureModel: infected=infected_population, evaporation_factor=0.3, ), - short_range = tuple(short_range), + short_range=tuple(short_range), exposed=self.exposed_population(), geographical_data=mc.Cases( geographic_population=self.geographic_population, @@ -250,11 +268,14 @@ def build_model(self, sample_size=None) -> models.ExposureModel: def build_CO2_model(self, sample_size=None) -> models.CO2ConcentrationModel: sample_size = sample_size or self.data_registry.monte_carlo['sample_size'] - infected_population: models.InfectedPopulation = self.infected_population().build_model(sample_size) + infected_population: models.InfectedPopulation = self.infected_population( + ).build_model(sample_size) exposed_population: models.Population = self.exposed_population().build_model(sample_size) - state_change_times = set(infected_population.presence_interval().transition_times()) - state_change_times.update(exposed_population.presence_interval().transition_times()) + state_change_times = set( + infected_population.presence_interval().transition_times()) + state_change_times.update( + exposed_population.presence_interval().transition_times()) transition_times = sorted(state_change_times) total_people = [infected_population.people_present(stop) + exposed_population.people_present(stop) @@ -263,10 +284,12 @@ def build_CO2_model(self, sample_size=None) -> models.CO2ConcentrationModel: if (self.activity_type == 'precise'): activity_defn, _ = self.generate_precise_activity_expiration() else: - activity_defn = self.data_registry.population_scenario_activity[self.activity_type]['activity'] + activity_defn = self.data_registry.population_scenario_activity[ + self.activity_type]['activity'] population = mc.SimplePopulation( - number=models.IntPiecewiseConstant(transition_times=tuple(transition_times), values=tuple(total_people)), + number=models.IntPiecewiseConstant(transition_times=tuple( + transition_times), values=tuple(total_people)), presence=None, activity=activity_distributions(self.data_registry)[activity_defn], ) @@ -286,7 +309,7 @@ def tz_name_and_utc_offset(self) -> typing.Tuple[str, float]: """ month = MONTH_NAMES.index(self.event_month) + 1 - timezone = caimira.data.weather.timezone_at( + timezone = data.weather.timezone_at( latitude=self.location_latitude, longitude=self.location_longitude, ) # We choose the first of the month for the current year. @@ -307,7 +330,8 @@ def outside_temp(self) -> models.PiecewiseConstant: month = MONTH_NAMES.index(self.event_month) + 1 wx_station = self.nearest_weather_station() - temp_profile = caimira.data.weather.mean_hourly_temperatures(wx_station = wx_station[0], month = MONTH_NAMES.index(self.event_month) + 1) + temp_profile = data.weather.mean_hourly_temperatures( + wx_station=wx_station[0], month=MONTH_NAMES.index(self.event_month) + 1) _, utc_offset = self.tz_name_and_utc_offset() @@ -315,13 +339,14 @@ def outside_temp(self) -> models.PiecewiseConstant: # result the first data value may no longer be a midnight, and the hours # no longer ordered modulo 24). source_times = np.arange(24) + utc_offset - times, temp_profile = caimira.data.weather.refine_hourly_data( + times, temp_profile = data.weather.refine_hourly_data( source_times, temp_profile, npts=24*10, # 10 steps per hour => 6 min steps ) outside_temp = models.PiecewiseConstant( - tuple(float(t) for t in times), tuple(float(t) for t in temp_profile), + tuple(float(t) for t in times), tuple(float(t) + for t in temp_profile), ) return outside_temp @@ -337,7 +362,8 @@ def ventilation(self) -> models._VentilationBase: ventilations.append(models.AirChange(active=models.SpecificInterval(present_times=((start, stop), )), air_exch=self.CO2_fitting_result['ventilation_values'][index])) else: - ventilations.append(models.AirChange(active=always_on, air_exch=self.CO2_fitting_result['ventilation_values'][0])) + ventilations.append(models.AirChange( + active=always_on, air_exch=self.CO2_fitting_result['ventilation_values'][0])) return models.MultipleVentilation(tuple(ventilations)) # Initializes a ventilation instance as a window if 'natural_ventilation' is selected, or as a HEPA-filter otherwise @@ -373,7 +399,8 @@ def ventilation(self) -> models._VentilationBase: ventilation = models.AirChange(active=always_on, air_exch=0.) else: if self.mechanical_ventilation_type == 'mech_type_air_changes': - ventilation = models.AirChange(active=always_on, air_exch=self.air_changes) + ventilation = models.AirChange( + active=always_on, air_exch=self.air_changes) else: ventilation = models.HVACMechanical( active=always_on, q_air_mech=self.air_supply) @@ -382,16 +409,18 @@ def ventilation(self) -> models._VentilationBase: # to the air infiltration from the outside. # See CERN-OPEN-2021-004, p. 12. residual_vent: float = self.data_registry.ventilation['infiltration_ventilation'] # type: ignore - infiltration_ventilation = models.AirChange(active=always_on, air_exch=residual_vent) + infiltration_ventilation = models.AirChange( + active=always_on, air_exch=residual_vent) if self.hepa_option: - hepa = models.HEPAFilter(active=always_on, q_air_mech=self.hepa_amount) + hepa = models.HEPAFilter( + active=always_on, q_air_mech=self.hepa_amount) return models.MultipleVentilation((ventilation, hepa, infiltration_ventilation)) else: return models.MultipleVentilation((ventilation, infiltration_ventilation)) - def nearest_weather_station(self) -> caimira.data.weather.WxStationRecordType: + def nearest_weather_station(self) -> data.weather.WxStationRecordType: """Return the nearest weather station (which has valid data) for this form""" - return caimira.data.weather.nearest_wx_station( + return data.weather.nearest_wx_station( longitude=self.location_longitude, latitude=self.location_latitude ) @@ -405,11 +434,13 @@ def mask(self) -> models.Mask: return mask def generate_precise_activity_expiration(self) -> typing.Tuple[typing.Any, ...]: - if self.precise_activity == {}: # It means the precise activity is not defined by a specific input. + # It means the precise activity is not defined by a specific input. + if self.precise_activity == {}: return () respiratory_dict = {} for respiratory_activity in self.precise_activity['respiratory_activity']: - respiratory_dict[respiratory_activity['type']] = respiratory_activity['percentage'] + respiratory_dict[respiratory_activity['type'] + ] = respiratory_activity['percentage'] return (self.precise_activity['physical_activity'], respiratory_dict) @@ -417,11 +448,14 @@ def infected_population(self) -> mc.InfectedPopulation: # Initializes the virus virus = virus_distributions(self.data_registry)[self.virus_type] - activity_defn = self.data_registry.population_scenario_activity[self.activity_type]['activity'] - expiration_defn = self.data_registry.population_scenario_activity[self.activity_type]['expiration'] + activity_defn = self.data_registry.population_scenario_activity[ + self.activity_type]['activity'] + expiration_defn = self.data_registry.population_scenario_activity[ + self.activity_type]['expiration'] if (self.activity_type == 'smallmeeting'): # Conversation of N people is approximately 1/N% of the time speaking. - expiration_defn = {'Speaking': 1, 'Breathing': self.total_people - 1} + expiration_defn = {'Speaking': 1, + 'Breathing': self.total_people - 1} elif (self.activity_type == 'precise'): activity_defn, expiration_defn = self.generate_precise_activity_expiration() @@ -438,7 +472,8 @@ def infected_population(self) -> mc.InfectedPopulation: mask=self.mask(), activity=activity, expiration=expiration, - host_immunity=0., # Vaccination status does not affect the infected population (for now) + # Vaccination status does not affect the infected population (for now) + host_immunity=0., ) return infected @@ -456,8 +491,8 @@ def exposed_population(self) -> mc.Population: if (self.vaccine_option): if (self.vaccine_booster_option and self.vaccine_booster_type != 'Other'): host_immunity = [vaccine['VE'] for vaccine in data.vaccine_booster_host_immunity if - vaccine['primary series vaccine'] == self.vaccine_type and - vaccine['booster vaccine'] == self.vaccine_booster_type][0] + vaccine['primary series vaccine'] == self.vaccine_type and + vaccine['booster vaccine'] == self.vaccine_booster_type][0] else: host_immunity = data.vaccine_primary_host_immunity[self.vaccine_type] else: @@ -484,9 +519,10 @@ def build_expiration(data_registry, expiration_definition) -> mc._ExpirationBase elif isinstance(expiration_definition, dict): total_weight = sum(expiration_definition.values()) BLO_factors = np.sum([ - np.array(expiration_BLO_factors(data_registry)[exp_type]) * weight/total_weight + np.array(expiration_BLO_factors(data_registry) + [exp_type]) * weight/total_weight for exp_type, weight in expiration_definition.items() - ], axis=0) + ], axis=0) return expiration_distribution(data_registry=data_registry, BLO_factors=tuple(BLO_factors)) @@ -529,7 +565,7 @@ def baseline_raw_form_data() -> typing.Dict[str, typing.Union[str, float]]: 'mask_type': 'Type I', 'mask_wearing_option': 'mask_off', 'mechanical_ventilation_type': '', - 'calculator_version': calculator.__version__, + 'calculator_version': '4.17.0', #TODO different version for API and calculator form? 'opening_distance': '0.2', 'event_month': 'January', 'room_heating_option': '0', @@ -555,4 +591,5 @@ def baseline_raw_form_data() -> typing.Dict[str, typing.Union[str, float]]: 'short_range_interactions': '[]', } + cast_class_fields(VirusFormData) diff --git a/caimira/scripts/data/vaccine_effectiveness.py b/caimira/src/caimira/scripts/data/vaccine_effectiveness.py similarity index 97% rename from caimira/scripts/data/vaccine_effectiveness.py rename to caimira/src/caimira/scripts/data/vaccine_effectiveness.py index ee98ac00..87be7e6b 100644 --- a/caimira/scripts/data/vaccine_effectiveness.py +++ b/caimira/src/caimira/scripts/data/vaccine_effectiveness.py @@ -1,5 +1,5 @@ -import pandas as pd -from tabulate import tabulate +# import pandas as pd +# from tabulate import tabulate ''' Script file to generate the vaccine effectiveness values. diff --git a/caimira/scripts/themes/base/caimira_script.command b/caimira/src/caimira/scripts/themes/base/caimira_script.command similarity index 100% rename from caimira/scripts/themes/base/caimira_script.command rename to caimira/src/caimira/scripts/themes/base/caimira_script.command diff --git a/caimira/scripts/themes/base/caimira_script.sh b/caimira/src/caimira/scripts/themes/base/caimira_script.sh similarity index 100% rename from caimira/scripts/themes/base/caimira_script.sh rename to caimira/src/caimira/scripts/themes/base/caimira_script.sh diff --git a/caimira/scripts/themes/cern/caimira_script.command b/caimira/src/caimira/scripts/themes/cern/caimira_script.command similarity index 100% rename from caimira/scripts/themes/cern/caimira_script.command rename to caimira/src/caimira/scripts/themes/cern/caimira_script.command diff --git a/caimira/scripts/themes/cern/caimira_script.sh b/caimira/src/caimira/scripts/themes/cern/caimira_script.sh similarity index 100% rename from caimira/scripts/themes/cern/caimira_script.sh rename to caimira/src/caimira/scripts/themes/cern/caimira_script.sh diff --git a/caimira/tests/apps/calculator/conftest.py b/caimira/tests/apps/calculator/conftest.py index d774e333..d1ac45d1 100644 --- a/caimira/tests/apps/calculator/conftest.py +++ b/caimira/tests/apps/calculator/conftest.py @@ -1,22 +1,13 @@ import pytest -from caimira.apps.calculator import model_generator +from caimira.calculator.validators.virus import virus_validator @pytest.fixture def baseline_form_data(): - return model_generator.baseline_raw_form_data() + return virus_validator.baseline_raw_form_data() @pytest.fixture def baseline_form(baseline_form_data, data_registry): - return model_generator.VirusFormData.from_dict(baseline_form_data, data_registry) - - -@pytest.fixture -def baseline_form_with_sr(baseline_form_data, data_registry): - form_data_sr = baseline_form_data - form_data_sr['short_range_option'] = 'short_range_yes' - form_data_sr['short_range_interactions'] = '[{"expiration": "Shouting", "start_time": "10:30", "duration": "30"}]' - form_data_sr['short_range_occupants'] = 5 - return model_generator.VirusFormData.from_dict(form_data_sr, data_registry) \ No newline at end of file + return virus_validator.VirusFormData.from_dict(baseline_form_data, data_registry) diff --git a/caimira/tests/apps/calculator/test_model_generator.py b/caimira/tests/apps/calculator/test_model_generator.py index bcd9e064..435ecd39 100644 --- a/caimira/tests/apps/calculator/test_model_generator.py +++ b/caimira/tests/apps/calculator/test_model_generator.py @@ -6,24 +6,24 @@ import pytest from retry import retry -from caimira.apps.calculator import model_generator -from caimira.apps.calculator.form_data import (_hours2timestring, minutes_since_midnight, +from caimira.calculator.validators.virus import virus_validator +from caimira.calculator.validators.form_validator import (_hours2timestring, minutes_since_midnight, _CAST_RULES_FORM_ARG_TO_NATIVE, _CAST_RULES_NATIVE_TO_FORM_ARG) -from caimira import models -from caimira.monte_carlo.data import expiration_distributions -from caimira.apps.calculator.defaults import NO_DEFAULT -from caimira.store.data_registry import DataRegistry +from caimira.calculator.models import models +from caimira.calculator.models.monte_carlo.data import expiration_distributions +from caimira.calculator.validators.defaults import NO_DEFAULT +from caimira.calculator.store.data_registry import DataRegistry def test_model_from_dict(baseline_form_data, data_registry): - form = model_generator.VirusFormData.from_dict(baseline_form_data, data_registry) + form = virus_validator.VirusFormData.from_dict(baseline_form_data, data_registry) assert isinstance(form.build_model(), models.ExposureModel) def test_model_from_dict_invalid(baseline_form_data, data_registry): baseline_form_data['invalid_item'] = 'foobar' with pytest.raises(ValueError, match='Invalid argument "invalid_item" given'): - model_generator.VirusFormData.from_dict(baseline_form_data, data_registry) + virus_validator.VirusFormData.from_dict(baseline_form_data, data_registry) @retry(tries=10) @@ -39,14 +39,14 @@ def test_blend_expiration(data_registry, mask_type): SAMPLE_SIZE = 250000 TOLERANCE = 0.02 blend = {'Breathing': 2, 'Speaking': 1} - r = model_generator.build_expiration(data_registry, blend).build_model(SAMPLE_SIZE) + r = virus_validator.build_expiration(data_registry, blend).build_model(SAMPLE_SIZE) mask = models.Mask.types[mask_type] expected = (expiration_distributions(data_registry)['Breathing'].build_model(SAMPLE_SIZE).aerosols(mask).mean()*2/3. + expiration_distributions(data_registry)['Speaking'].build_model(SAMPLE_SIZE).aerosols(mask).mean()/3.) npt.assert_allclose(r.aerosols(mask).mean(), expected, rtol=TOLERANCE) -def test_ventilation_slidingwindow(data_registry: DataRegistry, baseline_form: model_generator.VirusFormData): +def test_ventilation_slidingwindow(data_registry: DataRegistry, baseline_form: virus_validator.VirusFormData): baseline_form.ventilation_type = 'natural_ventilation' baseline_form.windows_duration = 10 baseline_form.windows_frequency = 120 @@ -77,7 +77,7 @@ def test_ventilation_slidingwindow(data_registry: DataRegistry, baseline_form: m assert ventilation == baseline_vent -def test_ventilation_hingedwindow(baseline_form: model_generator.VirusFormData): +def test_ventilation_hingedwindow(baseline_form: virus_validator.VirusFormData): baseline_form.ventilation_type = 'natural_ventilation' baseline_form.windows_duration = 10 baseline_form.windows_frequency = 120 @@ -107,7 +107,7 @@ def test_ventilation_hingedwindow(baseline_form: model_generator.VirusFormData): assert ventilation == baseline_vent -def test_ventilation_mechanical(baseline_form: model_generator.VirusFormData): +def test_ventilation_mechanical(baseline_form: virus_validator.VirusFormData): room = models.Room(volume=75, inside_temp=models.PiecewiseConstant((0, 24), (293,))) mech = models.HVACMechanical( active=models.PeriodicInterval(period=120, duration=120), @@ -122,7 +122,7 @@ def test_ventilation_mechanical(baseline_form: model_generator.VirusFormData): np.array([baseline_form.ventilation().air_exchange(room, t) for t in ts])) -def test_ventilation_airchanges(baseline_form: model_generator.VirusFormData): +def test_ventilation_airchanges(baseline_form: virus_validator.VirusFormData): room = models.Room(75, inside_temp=models.PiecewiseConstant((0, 24), (293,))) airchange = models.AirChange( active=models.PeriodicInterval(period=120, duration=120), @@ -137,7 +137,7 @@ def test_ventilation_airchanges(baseline_form: model_generator.VirusFormData): np.array([baseline_form.ventilation().air_exchange(room, t) for t in ts])) -def test_ventilation_window_hepa(data_registry: DataRegistry, baseline_form: model_generator.VirusFormData): +def test_ventilation_window_hepa(data_registry: DataRegistry, baseline_form: virus_validator.VirusFormData): baseline_form.ventilation_type = 'natural_ventilation' baseline_form.windows_duration = 10 baseline_form.windows_frequency = 120 @@ -181,7 +181,7 @@ def test_ventilation_window_hepa(data_registry: DataRegistry, baseline_form: mod ] ) def test_infected_less_than_total_people(activity, total_people, infected_people, error, - baseline_form: model_generator.VirusFormData, + baseline_form: virus_validator.VirusFormData, data_registry: DataRegistry): baseline_form.activity_type = activity baseline_form.total_people = total_people @@ -195,7 +195,7 @@ def present_times(interval: models.Interval) -> models.BoundarySequence_t: return interval.present_times -def test_infected_present_intervals(baseline_form: model_generator.VirusFormData): +def test_infected_present_intervals(baseline_form: virus_validator.VirusFormData): baseline_form.infected_dont_have_breaks_with_exposed = False baseline_form.exposed_coffee_duration = 15 baseline_form.exposed_coffee_break_option = 'coffee_break_2' @@ -209,7 +209,7 @@ def test_infected_present_intervals(baseline_form: model_generator.VirusFormData assert present_times(baseline_form.infected_present_interval()) == correct -def test_exposed_present_intervals(baseline_form: model_generator.VirusFormData): +def test_exposed_present_intervals(baseline_form: virus_validator.VirusFormData): baseline_form.exposed_coffee_duration = 15 baseline_form.exposed_coffee_break_option = 'coffee_break_2' baseline_form.exposed_start = minutes_since_midnight(9 * 60) @@ -220,7 +220,7 @@ def test_exposed_present_intervals(baseline_form: model_generator.VirusFormData) assert present_times(baseline_form.exposed_present_interval()) == correct -def test_present_intervals_common_breaks(baseline_form: model_generator.VirusFormData): +def test_present_intervals_common_breaks(baseline_form: virus_validator.VirusFormData): baseline_form.infected_dont_have_breaks_with_exposed = False baseline_form.infected_coffee_duration = baseline_form.exposed_coffee_duration = 15 baseline_form.infected_coffee_break_option = baseline_form.exposed_coffee_break_option = 'coffee_break_2' @@ -236,7 +236,7 @@ def test_present_intervals_common_breaks(baseline_form: model_generator.VirusFor assert present_times(baseline_form.infected_present_interval()) == correct_infected -def test_present_intervals_split_breaks(baseline_form: model_generator.VirusFormData): +def test_present_intervals_split_breaks(baseline_form: virus_validator.VirusFormData): baseline_form.infected_dont_have_breaks_with_exposed = True baseline_form.infected_coffee_duration = baseline_form.exposed_coffee_duration = 15 baseline_form.infected_coffee_break_option = baseline_form.exposed_coffee_break_option = 'coffee_break_2' @@ -252,7 +252,7 @@ def test_present_intervals_split_breaks(baseline_form: model_generator.VirusForm assert present_times(baseline_form.infected_present_interval()) == correct_infected -def test_exposed_present_intervals_starting_with_lunch(baseline_form: model_generator.VirusFormData): +def test_exposed_present_intervals_starting_with_lunch(baseline_form: virus_validator.VirusFormData): baseline_form.exposed_coffee_break_option = 'coffee_break_0' baseline_form.exposed_start = baseline_form.exposed_lunch_start = minutes_since_midnight(13 * 60) baseline_form.exposed_finish = minutes_since_midnight(18 * 60) @@ -261,7 +261,7 @@ def test_exposed_present_intervals_starting_with_lunch(baseline_form: model_gene assert present_times(baseline_form.exposed_present_interval()) == correct -def test_exposed_present_intervals_ending_with_lunch(baseline_form: model_generator.VirusFormData): +def test_exposed_present_intervals_ending_with_lunch(baseline_form: virus_validator.VirusFormData): baseline_form.exposed_coffee_break_option = 'coffee_break_0' baseline_form.exposed_start = minutes_since_midnight(11 * 60) baseline_form.exposed_finish = baseline_form.exposed_lunch_start = minutes_since_midnight(13 * 60) @@ -270,7 +270,7 @@ def test_exposed_present_intervals_ending_with_lunch(baseline_form: model_genera assert present_times(baseline_form.exposed_present_interval()) == correct -def test_exposed_present_lunch_end_before_beginning(baseline_form: model_generator.VirusFormData, data_registry: DataRegistry): +def test_exposed_present_lunch_end_before_beginning(baseline_form: virus_validator.VirusFormData, data_registry: DataRegistry): baseline_form.exposed_coffee_break_option = 'coffee_break_0' baseline_form.exposed_lunch_start = minutes_since_midnight(14 * 60) baseline_form.exposed_lunch_finish = minutes_since_midnight(13 * 60) @@ -287,7 +287,7 @@ def test_exposed_present_lunch_end_before_beginning(baseline_form: model_generat [9, 20], # lunch_finish after the presence finishing ], ) -def test_exposed_presence_lunch_break(baseline_form: model_generator.VirusFormData, data_registry: DataRegistry, exposed_lunch_start, exposed_lunch_finish): +def test_exposed_presence_lunch_break(baseline_form: virus_validator.VirusFormData, data_registry: DataRegistry, exposed_lunch_start, exposed_lunch_finish): baseline_form.exposed_lunch_start = minutes_since_midnight(exposed_lunch_start * 60) baseline_form.exposed_lunch_finish = minutes_since_midnight(exposed_lunch_finish * 60) with pytest.raises(ValueError, match='exposed lunch break must be within presence times.'): @@ -303,14 +303,14 @@ def test_exposed_presence_lunch_break(baseline_form: model_generator.VirusFormDa [9, 20], # lunch_finish after the presence finishing ], ) -def test_infected_presence_lunch_break(baseline_form: model_generator.VirusFormData, data_registry: DataRegistry, infected_lunch_start, infected_lunch_finish): +def test_infected_presence_lunch_break(baseline_form: virus_validator.VirusFormData, data_registry: DataRegistry, infected_lunch_start, infected_lunch_finish): baseline_form.infected_lunch_start = minutes_since_midnight(infected_lunch_start * 60) baseline_form.infected_lunch_finish = minutes_since_midnight(infected_lunch_finish * 60) with pytest.raises(ValueError, match='infected lunch break must be within presence times.'): baseline_form.validate() -def test_exposed_breaks_length(baseline_form: model_generator.VirusFormData, data_registry: DataRegistry): +def test_exposed_breaks_length(baseline_form: virus_validator.VirusFormData, data_registry: DataRegistry): baseline_form.exposed_coffee_break_option = 'coffee_break_4' baseline_form.exposed_coffee_duration = 30 baseline_form.exposed_start = minutes_since_midnight(10 * 60) @@ -320,7 +320,7 @@ def test_exposed_breaks_length(baseline_form: model_generator.VirusFormData, dat baseline_form.validate() -def test_infected_breaks_length(baseline_form: model_generator.VirusFormData, data_registry: DataRegistry): +def test_infected_breaks_length(baseline_form: virus_validator.VirusFormData, data_registry: DataRegistry): baseline_form.infected_start = minutes_since_midnight(9 * 60) baseline_form.infected_finish = minutes_since_midnight(12 * 60) baseline_form.infected_lunch_start = minutes_since_midnight(10 * 60) @@ -332,7 +332,7 @@ def test_infected_breaks_length(baseline_form: model_generator.VirusFormData, da @pytest.fixture -def coffee_break_between_1045_and_1115(baseline_form: model_generator.VirusFormData): +def coffee_break_between_1045_and_1115(baseline_form: virus_validator.VirusFormData): baseline_form.exposed_coffee_break_option = 'coffee_break_1' baseline_form.exposed_coffee_duration = 30 baseline_form.exposed_start = minutes_since_midnight(10 * 60) @@ -390,7 +390,7 @@ def assert_boundaries(interval, boundaries_in_time_string_form): @pytest.fixture -def breaks_every_25_mins_for_20_mins(baseline_form: model_generator.VirusFormData): +def breaks_every_25_mins_for_20_mins(baseline_form: virus_validator.VirusFormData): baseline_form.exposed_coffee_break_option = 'coffee_break_4' baseline_form.exposed_coffee_duration = 20 baseline_form.exposed_start = time2mins("10:00") @@ -435,7 +435,7 @@ def test_present_only_during_second_break(breaks_every_25_mins_for_20_mins): assert_boundaries(interval, []) -def test_valid_no_lunch(baseline_form: model_generator.VirusFormData, data_registry: DataRegistry): +def test_valid_no_lunch(baseline_form: virus_validator.VirusFormData, data_registry: DataRegistry): # Check that it is valid to have a 0 length lunch if no lunch is selected. baseline_form.exposed_lunch_option = False baseline_form.exposed_lunch_start = minutes_since_midnight(0) @@ -443,7 +443,7 @@ def test_valid_no_lunch(baseline_form: model_generator.VirusFormData, data_regis assert baseline_form.validate() is None -def test_no_breaks(baseline_form: model_generator.VirusFormData): +def test_no_breaks(baseline_form: virus_validator.VirusFormData): # Check that the times are correct in the absence of breaks. baseline_form.infected_dont_have_breaks_with_exposed = False baseline_form.exposed_lunch_option = False @@ -458,7 +458,7 @@ def test_no_breaks(baseline_form: model_generator.VirusFormData): assert present_times(baseline_form.infected_present_interval()) == infected_correct -def test_coffee_lunch_breaks(baseline_form: model_generator.VirusFormData): +def test_coffee_lunch_breaks(baseline_form: virus_validator.VirusFormData): baseline_form.exposed_coffee_duration = 30 baseline_form.exposed_coffee_break_option = 'coffee_break_4' baseline_form.exposed_start = minutes_since_midnight(9 * 60) @@ -470,7 +470,7 @@ def test_coffee_lunch_breaks(baseline_form: model_generator.VirusFormData): np.testing.assert_allclose(present_times(baseline_form.exposed_present_interval()), correct, rtol=1e-14) -def test_coffee_lunch_breaks_unbalance(baseline_form: model_generator.VirusFormData): +def test_coffee_lunch_breaks_unbalance(baseline_form: virus_validator.VirusFormData): baseline_form.exposed_coffee_duration = 30 baseline_form.exposed_coffee_break_option = 'coffee_break_2' baseline_form.exposed_start = minutes_since_midnight(9 * 60) @@ -481,7 +481,7 @@ def test_coffee_lunch_breaks_unbalance(baseline_form: model_generator.VirusFormD np.testing.assert_allclose(present_times(baseline_form.exposed_present_interval()), correct, rtol=1e-14) -def test_coffee_breaks(baseline_form: model_generator.VirusFormData): +def test_coffee_breaks(baseline_form: virus_validator.VirusFormData): baseline_form.exposed_coffee_duration = 10 baseline_form.exposed_coffee_break_option = 'coffee_break_4' baseline_form.exposed_start = minutes_since_midnight(9 * 60) @@ -494,24 +494,24 @@ def test_coffee_breaks(baseline_form: model_generator.VirusFormData): def test_key_validation(baseline_form_data, data_registry): baseline_form_data['activity_type'] = 'invalid key' with pytest.raises(ValueError): - model_generator.VirusFormData.from_dict(baseline_form_data, data_registry) + virus_validator.VirusFormData.from_dict(baseline_form_data, data_registry) def test_key_validation_natural_ventilation_window_type_na(baseline_form_data, data_registry): baseline_form_data['ventilation_type'] = 'natural_ventilation' baseline_form_data['window_type'] = 'not-applicable' with pytest.raises(ValueError, match='window_type cannot be \'not-applicable\''): - model_generator.VirusFormData.from_dict(baseline_form_data, data_registry) + virus_validator.VirusFormData.from_dict(baseline_form_data, data_registry) def test_key_validation_natural_ventilation_window_opening_regime_na(baseline_form_data, data_registry): baseline_form_data['ventilation_type'] = 'natural_ventilation' baseline_form_data['window_opening_regime'] = 'not-applicable' with pytest.raises(ValueError, match='window_opening_regime cannot be \'not-applicable\''): - model_generator.VirusFormData.from_dict(baseline_form_data, data_registry) + virus_validator.VirusFormData.from_dict(baseline_form_data, data_registry) -def test_natural_ventilation_window_opening_periodically(baseline_form: model_generator.VirusFormData, data_registry: DataRegistry): +def test_natural_ventilation_window_opening_periodically(baseline_form: virus_validator.VirusFormData, data_registry: DataRegistry): baseline_form.window_opening_regime = 'windows_open_periodically' baseline_form.windows_duration = 20 baseline_form.windows_frequency = 10 @@ -523,20 +523,20 @@ def test_key_validation_mech_ventilation_type_na(baseline_form_data, data_regist baseline_form_data['ventilation_type'] = 'mechanical_ventilation' baseline_form_data['mechanical_ventilation_type'] = 'not-applicable' with pytest.raises(ValueError, match='mechanical_ventilation_type cannot be \'not-applicable\''): - model_generator.VirusFormData.from_dict(baseline_form_data, data_registry) + virus_validator.VirusFormData.from_dict(baseline_form_data, data_registry) def test_key_validation_event_month(baseline_form_data, data_registry): baseline_form_data['event_month'] = 'invalid month' with pytest.raises(ValueError, match='invalid month is not a valid value for event_month'): - model_generator.VirusFormData.from_dict(baseline_form_data, data_registry) + virus_validator.VirusFormData.from_dict(baseline_form_data, data_registry) def test_default_types(): # Validate that VirusFormData._DEFAULTS are complete and of the correct type. # Validate that we have the right types and matching attributes to the DEFAULTS. - fields = {field.name: field for field in dataclasses.fields(model_generator.VirusFormData)} - for field, value in model_generator.VirusFormData._DEFAULTS.items(): + fields = {field.name: field for field in dataclasses.fields(virus_validator.VirusFormData)} + for field, value in virus_validator.VirusFormData._DEFAULTS.items(): if field not in fields: raise ValueError(f"Unmatched default {field}") @@ -557,7 +557,7 @@ def test_default_types(): for field in fields.values(): if field.name == "data_registry": continue # Skip the assertion for the "data_registry" field - assert field.name in model_generator.VirusFormData._DEFAULTS, f"No default set for field name {field.name}" + assert field.name in virus_validator.VirusFormData._DEFAULTS, f"No default set for field name {field.name}" def test_form_to_dict(baseline_form): @@ -566,7 +566,7 @@ def test_form_to_dict(baseline_form): assert 1 < len(stripped) < len(full) assert 'exposed_coffee_break_option' in stripped # If we set the value to the default one, it should no longer turn up in the dictionary. - baseline_form.exposed_coffee_break_option = model_generator.VirusFormData._DEFAULTS['exposed_coffee_break_option'] + baseline_form.exposed_coffee_break_option = virus_validator.VirusFormData._DEFAULTS['exposed_coffee_break_option'] assert 'exposed_coffee_break_option' not in baseline_form.to_dict(baseline_form, strip_defaults=True) @@ -584,7 +584,7 @@ def test_form_timezone(baseline_form_data, data_registry, longitude, latitude, m baseline_form_data['location_latitude'] = latitude baseline_form_data['location_longitude'] = longitude baseline_form_data['event_month'] = month - form = model_generator.VirusFormData.from_dict(baseline_form_data, data_registry) + form = virus_validator.VirusFormData.from_dict(baseline_form_data, data_registry) name, offset = form.tz_name_and_utc_offset() assert name == expected_tz_name assert offset == expected_offset diff --git a/caimira/tests/apps/calculator/test_report_json.py b/caimira/tests/apps/calculator/test_report_json.py deleted file mode 100644 index 95cb60a1..00000000 --- a/caimira/tests/apps/calculator/test_report_json.py +++ /dev/null @@ -1,32 +0,0 @@ -import json - -import tornado.testing - -import caimira.apps.calculator -from caimira.apps.calculator import model_generator - -_TIMEOUT = 40. - - -class TestCalculatorJsonResponse(tornado.testing.AsyncHTTPTestCase): - def setUp(self): - super().setUp() - self.http_client.defaults['request_timeout'] = _TIMEOUT - - def get_app(self): - return caimira.apps.calculator.make_app() - - @tornado.testing.gen_test(timeout=_TIMEOUT) - def test_json_response(self): - response = yield self.http_client.fetch( - request=self.get_url("/calculator/report-json"), - method="POST", - headers={'content-type': 'application/json'}, - body=json.dumps(model_generator.baseline_raw_form_data()) - ) - self.assertEqual(response.code, 200) - - data = json.loads(response.body) - self.assertIsInstance(data['prob_inf'], float) - self.assertIsInstance(data['expected_new_cases'], float) - diff --git a/caimira/tests/apps/calculator/test_specific_model_generator.py b/caimira/tests/apps/calculator/test_specific_model_generator.py index e8a6b977..6945a2c1 100644 --- a/caimira/tests/apps/calculator/test_specific_model_generator.py +++ b/caimira/tests/apps/calculator/test_specific_model_generator.py @@ -2,8 +2,8 @@ import numpy as np import pytest -from caimira.apps.calculator import model_generator -from caimira.store.data_registry import DataRegistry +from caimira.calculator.validators.virus import virus_validator +from caimira.calculator.store.data_registry import DataRegistry @pytest.mark.parametrize( @@ -14,7 +14,7 @@ [{"exposed_breaks": [], "ifected_breaks": []}, 'Unable to fetch "infected_breaks" key. Got "ifected_breaks".'], ] ) -def test_specific_break_structure(break_input, error, baseline_form: model_generator.VirusFormData, data_registry: DataRegistry): +def test_specific_break_structure(break_input, error, baseline_form: virus_validator.VirusFormData, data_registry: DataRegistry): baseline_form.specific_breaks = break_input with pytest.raises(TypeError, match=error): baseline_form.validate() @@ -31,7 +31,7 @@ def test_specific_break_structure(break_input, error, baseline_form: model_gener [[{"start_time": "10:00", "finish_time": "11"}], 'Wrong time format - "HH:MM". Got "11".'], ] ) -def test_specific_population_break_data_structure(population_break_input, error, baseline_form: model_generator.VirusFormData, data_registry: DataRegistry): +def test_specific_population_break_data_structure(population_break_input, error, baseline_form: virus_validator.VirusFormData, data_registry: DataRegistry): baseline_form.specific_breaks = {'exposed_breaks': population_break_input, 'infected_breaks': population_break_input} with pytest.raises(TypeError, match=error): baseline_form.validate() @@ -46,7 +46,7 @@ def test_specific_population_break_data_structure(population_break_input, error, [{'exposed_breaks': [], 'infected_breaks': [{"start_time": "08:00", "finish_time": "11:00"}, {"start_time": "14:00", "finish_time": "15:00"}, ]}, "All breaks should be within the simulation time. Got 08:00."], ] ) -def test_specific_break_time(break_input, error, baseline_form: model_generator.VirusFormData): +def test_specific_break_time(break_input, error, baseline_form: virus_validator.VirusFormData): with pytest.raises(ValueError, match=error): baseline_form.generate_specific_break_times(breaks_dict=break_input, target='exposed') baseline_form.generate_specific_break_times(breaks_dict=break_input, target='infected') @@ -65,7 +65,7 @@ def test_specific_break_time(break_input, error, baseline_form: model_generator. [{"physical_activity": "Light activity", "respiratory_activity": [{"type": "Breathing", "percentag": 50}, {"type": "Speaking", "percentage": 50}]}, 'Unable to fetch "percentage" key. Got "percentag".'], ] ) -def test_precise_activity_structure(precise_activity_input, error, baseline_form: model_generator.VirusFormData, data_registry: DataRegistry): +def test_precise_activity_structure(precise_activity_input, error, baseline_form: virus_validator.VirusFormData, data_registry: DataRegistry): baseline_form.precise_activity = precise_activity_input with pytest.raises(TypeError, match=error): baseline_form.validate() @@ -80,7 +80,7 @@ def test_precise_activity_structure(precise_activity_input, error, baseline_form [{"physical_activity": "Light activity", "respiratory_activity": [{"type": "Breathing", "percentage": 50}]}, 'The sum of all respiratory activities should be 100. Got 50.'], ] ) -def test_sum_precise_activity(precise_activity_input, error, baseline_form: model_generator.VirusFormData): +def test_sum_precise_activity(precise_activity_input, error, baseline_form: virus_validator.VirusFormData): baseline_form.precise_activity = precise_activity_input with pytest.raises(ValueError, match=error): baseline_form.validate() diff --git a/caimira/tests/conftest.py b/caimira/tests/conftest.py index 4142ff79..aa8a390b 100644 --- a/caimira/tests/conftest.py +++ b/caimira/tests/conftest.py @@ -1,11 +1,9 @@ -from caimira import models -import caimira.data -import caimira.dataclass_utils - import pytest -from caimira.store.data_registry import DataRegistry - +from caimira.calculator.models import models +import caimira.calculator.models.data +import caimira.calculator.models.dataclass_utils +from caimira.calculator.store.data_registry import DataRegistry @pytest.fixture def data_registry(): @@ -61,12 +59,12 @@ def baseline_exposure_model(data_registry, baseline_concentration_model, baselin @pytest.fixture def exposure_model_w_outside_temp_changes(data_registry, baseline_exposure_model: models.ExposureModel): - exp_model = caimira.dataclass_utils.nested_replace( + exp_model = caimira.calculator.models.dataclass_utils.nested_replace( baseline_exposure_model, { 'concentration_model.ventilation': models.SlidingWindow( data_registry=data_registry, active=models.PeriodicInterval(2.2 * 60, 1.8 * 60), - outside_temp=caimira.data.GenevaTemperatures['Jan'], + outside_temp=caimira.calculator.models.data.GenevaTemperatures['Jan'], window_height=1.6, opening_length=0.6, ) diff --git a/caimira/tests/data/test_weather.py b/caimira/tests/data/test_weather.py index 03eb29c3..7cdab2a8 100644 --- a/caimira/tests/data/test_weather.py +++ b/caimira/tests/data/test_weather.py @@ -5,7 +5,7 @@ import numpy.testing import pytest -import caimira.data.weather as wx +import caimira.calculator.models.data.weather as wx def test_nearest_wx_station(): diff --git a/caimira/tests/models/test_co2_concentration_model.py b/caimira/tests/models/test_co2_concentration_model.py index ce57110b..d6010032 100644 --- a/caimira/tests/models/test_co2_concentration_model.py +++ b/caimira/tests/models/test_co2_concentration_model.py @@ -1,7 +1,7 @@ import numpy.testing as npt import pytest -from caimira import models +from caimira.calculator.models import models @pytest.fixture diff --git a/caimira/tests/models/test_concentration_model.py b/caimira/tests/models/test_concentration_model.py index 88d796d7..d2d5020c 100644 --- a/caimira/tests/models/test_concentration_model.py +++ b/caimira/tests/models/test_concentration_model.py @@ -5,8 +5,8 @@ import pytest from dataclasses import dataclass -from caimira import models -from caimira.store.data_registry import DataRegistry +from caimira.calculator.models import models +from caimira.calculator.store.data_registry import DataRegistry @dataclass(frozen=True) class KnownConcentrationModelBase(models._ConcentrationModelBase): diff --git a/caimira/tests/models/test_dynamic_population.py b/caimira/tests/models/test_dynamic_population.py index 79c89980..d7de725b 100644 --- a/caimira/tests/models/test_dynamic_population.py +++ b/caimira/tests/models/test_dynamic_population.py @@ -4,8 +4,8 @@ import numpy.testing as npt import pytest -from caimira import models -import caimira.dataclass_utils as dc_utils +from caimira.calculator.models import models +from caimira.calculator.models import dataclass_utils as dc_utils @pytest.fixture def full_exposure_model(data_registry): diff --git a/caimira/tests/models/test_exposure_model.py b/caimira/tests/models/test_exposure_model.py index 99e7ca1e..37fc7450 100644 --- a/caimira/tests/models/test_exposure_model.py +++ b/caimira/tests/models/test_exposure_model.py @@ -5,11 +5,11 @@ import pytest from dataclasses import dataclass -from caimira import models -from caimira.models import ExposureModel -from caimira.dataclass_utils import replace -from caimira.monte_carlo.data import expiration_distributions -from caimira.store.data_registry import DataRegistry +from caimira.calculator.models import models +from caimira.calculator.models.models import ExposureModel +from caimira.calculator.models.dataclass_utils import replace +from caimira.calculator.models.monte_carlo.data import expiration_distributions +from caimira.calculator.store.data_registry import DataRegistry @dataclass(frozen=True) class KnownNormedconcentration(models.ConcentrationModel): diff --git a/caimira/tests/models/test_fitting_algorithm.py b/caimira/tests/models/test_fitting_algorithm.py index 65b6f447..bb4886de 100644 --- a/caimira/tests/models/test_fitting_algorithm.py +++ b/caimira/tests/models/test_fitting_algorithm.py @@ -2,7 +2,7 @@ import numpy.testing as npt import pytest -from caimira import models +from caimira.calculator.models import models @pytest.mark.parametrize( diff --git a/caimira/tests/models/test_mask.py b/caimira/tests/models/test_mask.py index 5d87ac61..3c32a8dc 100644 --- a/caimira/tests/models/test_mask.py +++ b/caimira/tests/models/test_mask.py @@ -2,7 +2,7 @@ import numpy.testing as npt import pytest -from caimira import models +from caimira.calculator.models import models @pytest.mark.parametrize( diff --git a/caimira/tests/models/test_piecewiseconstant.py b/caimira/tests/models/test_piecewiseconstant.py index 74c8a056..93347bf4 100644 --- a/caimira/tests/models/test_piecewiseconstant.py +++ b/caimira/tests/models/test_piecewiseconstant.py @@ -1,8 +1,8 @@ import numpy as np import pytest -from caimira import models -from caimira import data +from caimira.calculator.models import models +from caimira.calculator.models import data def test_piecewiseconstantfunction_wrongarguments(): diff --git a/caimira/tests/models/test_short_range_model.py b/caimira/tests/models/test_short_range_model.py index 7bf5f429..0382369a 100644 --- a/caimira/tests/models/test_short_range_model.py +++ b/caimira/tests/models/test_short_range_model.py @@ -3,10 +3,10 @@ import numpy as np import pytest -from caimira import models -import caimira.monte_carlo as mc_models -from caimira.apps.calculator.model_generator import build_expiration -from caimira.monte_carlo.data import short_range_expiration_distributions,\ +from caimira.calculator.models import models +import caimira.calculator.models.monte_carlo as mc_models +from caimira.calculator.validators.virus.virus_validator import build_expiration +from caimira.calculator.models.monte_carlo.data import short_range_expiration_distributions,\ expiration_distributions, short_range_distances, activity_distributions SAMPLE_SIZE = 250_000 diff --git a/caimira/tests/models/test_virus.py b/caimira/tests/models/test_virus.py deleted file mode 100644 index 060ae6af..00000000 --- a/caimira/tests/models/test_virus.py +++ /dev/null @@ -1,25 +0,0 @@ -import numpy as np -import numpy.testing as npt -import pytest - -from caimira import models - - -@pytest.mark.parametrize( - "inside_temp, humidity, expected_halflife, expected_decay_constant", - [ - [293.15, 0.5, 0.5947447349860315, 1.1654532436949188], - [272.15, 0.7, 1.6070844193207476, 0.4313072619127947], - [300.15, 1., 0.17367078830147223, 3.9911558376571805], - [300.15, 0., 6.43, 0.10779893943389507], - [np.array([272.15, 300.15]), np.array([0.7, 0.]), - np.array([1.60708442, 6.43]), np.array([0.43130726, 0.10779894])], - [np.array([293.15, 300.15]), np.array([0.5, 1.]), - np.array([0.59474473, 0.17367079]), np.array([1.16545324, 3.99115584])] - ], -) -def test_decay_constant(inside_temp, humidity, expected_halflife, expected_decay_constant): - npt.assert_almost_equal(models.Virus.types['SARS_CoV_2'].halflife(humidity, inside_temp), - expected_halflife) - npt.assert_almost_equal(models.Virus.types['SARS_CoV_2'].decay_constant(humidity, inside_temp), - expected_decay_constant) \ No newline at end of file diff --git a/caimira/tests/test_caimira.py b/caimira/tests/test_caimira.py index c99882be..e9f43c36 100644 --- a/caimira/tests/test_caimira.py +++ b/caimira/tests/test_caimira.py @@ -3,8 +3,8 @@ """ -import caimira +import caimira.calculator.models def test_version(): - assert caimira.__version__ is not None + assert caimira.calculator.models.__version__ is not None diff --git a/caimira/tests/test_conditional_probability.py b/caimira/tests/test_conditional_probability.py index 0e616ae7..447a8fe3 100644 --- a/caimira/tests/test_conditional_probability.py +++ b/caimira/tests/test_conditional_probability.py @@ -2,11 +2,11 @@ import pytest from retry import retry -import caimira.monte_carlo as mc -from caimira import models -from caimira.dataclass_utils import nested_replace -from caimira.apps.calculator import report_generator -from caimira.monte_carlo.data import activity_distributions, virus_distributions, expiration_distributions +import caimira.calculator.models.monte_carlo as mc +from caimira.calculator.models import models +from caimira.calculator.models.dataclass_utils import nested_replace +from caimira.calculator.report import report_generator +from caimira.calculator.models.monte_carlo.data import activity_distributions, virus_distributions, expiration_distributions @pytest.fixture diff --git a/caimira/tests/test_data_service.py b/caimira/tests/test_data_service.py index c6c21343..d6c5025b 100644 --- a/caimira/tests/test_data_service.py +++ b/caimira/tests/test_data_service.py @@ -1,7 +1,7 @@ import unittest from unittest.mock import Mock, patch -from caimira.store.data_service import DataService +from caimira.calculator.store.data_service import DataService class DataServiceTests(unittest.TestCase): diff --git a/caimira/tests/test_dataclass_utils.py b/caimira/tests/test_dataclass_utils.py index ac575052..14088018 100644 --- a/caimira/tests/test_dataclass_utils.py +++ b/caimira/tests/test_dataclass_utils.py @@ -1,6 +1,6 @@ import dataclasses -from caimira.dataclass_utils import nested_replace, walk_dataclass +from caimira.calculator.models.dataclass_utils import nested_replace, walk_dataclass @dataclasses.dataclass(frozen=True) diff --git a/caimira/tests/test_expiration.py b/caimira/tests/test_expiration.py index 0cda581b..03842149 100644 --- a/caimira/tests/test_expiration.py +++ b/caimira/tests/test_expiration.py @@ -5,8 +5,8 @@ import pytest from retry import retry -from caimira import models -from caimira.monte_carlo.data import expiration_distribution +from caimira.calculator.models import models +from caimira.calculator.models.monte_carlo.data import expiration_distribution def test_multiple_wrong_weight_size(): diff --git a/caimira/tests/test_full_algorithm.py b/caimira/tests/test_full_algorithm.py index 112b1082..adb93df0 100644 --- a/caimira/tests/test_full_algorithm.py +++ b/caimira/tests/test_full_algorithm.py @@ -8,11 +8,11 @@ import pytest from retry import retry -import caimira.monte_carlo as mc -from caimira import models -from caimira.utils import method_cache -from caimira.models import _VectorisedFloat,Interval,SpecificInterval -from caimira.monte_carlo.data import (expiration_distributions, +import caimira.calculator.models.monte_carlo as mc +from caimira.calculator.models import models +from caimira.calculator.models.utils import method_cache +from caimira.calculator.models.models import _VectorisedFloat,Interval,SpecificInterval +from caimira.calculator.models.monte_carlo.data import (expiration_distributions, expiration_BLO_factors,short_range_expiration_distributions, short_range_distances,virus_distributions,activity_distributions) diff --git a/caimira/tests/test_infected_population.py b/caimira/tests/test_infected_population.py index 192441e4..2a48e8df 100644 --- a/caimira/tests/test_infected_population.py +++ b/caimira/tests/test_infected_population.py @@ -1,7 +1,7 @@ import numpy as np import pytest -import caimira.models +import caimira.calculator.models.models @pytest.mark.parametrize( @@ -17,26 +17,26 @@ def test_infected_population_vectorisation(override_params, data_registry): } defaults.update(override_params) - office_hours = caimira.models.SpecificInterval(present_times=[(8,17)]) - infected = caimira.models.InfectedPopulation( + office_hours = caimira.calculator.models.models.SpecificInterval(present_times=[(8,17)]) + infected = caimira.calculator.models.models.InfectedPopulation( data_registry=data_registry, number=1, presence=office_hours, - mask=caimira.models.Mask( + mask=caimira.calculator.models.models.Mask( factor_exhale=0.95, η_inhale=0.3, ), - activity=caimira.models.Activity( + activity=caimira.calculator.models.models.Activity( 0.51, defaults['exhalation_rate'], ), - virus=caimira.models.SARSCoV2( + virus=caimira.calculator.models.models.SARSCoV2( viral_load_in_sputum=defaults['viral_load_in_sputum'], infectious_dose=50., viable_to_RNA_ratio = 0.5, transmissibility_factor=1.0, ), - expiration=caimira.models._ExpirationBase.types['Breathing'], + expiration=caimira.calculator.models.models._ExpirationBase.types['Breathing'], host_immunity=0., ) emission_rate = infected.emission_rate(10) diff --git a/caimira/tests/test_known_quantities.py b/caimira/tests/test_known_quantities.py index ba7e451a..c3943157 100644 --- a/caimira/tests/test_known_quantities.py +++ b/caimira/tests/test_known_quantities.py @@ -2,8 +2,8 @@ import numpy.testing as npt import pytest -import caimira.models as models -import caimira.data as data +import caimira.calculator.models.models as models +import caimira.calculator.models.data as data def test_no_mask_superspeading_emission_rate(baseline_concentration_model): diff --git a/caimira/tests/test_model.py b/caimira/tests/test_model.py index eec01921..956d0c92 100644 --- a/caimira/tests/test_model.py +++ b/caimira/tests/test_model.py @@ -1,5 +1,4 @@ -import caimira.models -from caimira.dataclass_utils import nested_replace +from caimira.calculator.models.dataclass_utils import nested_replace def test_exposure_r0(baseline_exposure_model): diff --git a/caimira/tests/test_monte_carlo.py b/caimira/tests/test_monte_carlo.py index 656450ea..ece38bd7 100644 --- a/caimira/tests/test_monte_carlo.py +++ b/caimira/tests/test_monte_carlo.py @@ -3,12 +3,12 @@ import numpy as np import pytest -import caimira.models -import caimira.monte_carlo.models as mc_models -import caimira.monte_carlo.sampleable +import caimira.calculator.models +import caimira.calculator.models.models +import caimira.calculator.models.monte_carlo.sampleable MODEL_CLASSES = [ - cls for cls in vars(caimira.models).values() + cls for cls in vars(caimira.calculator.models).values() if dataclasses.is_dataclass(cls) ] @@ -21,11 +21,11 @@ def test_type_annotations(): # runtime execution. missing = [] for cls in MODEL_CLASSES: - if not hasattr(caimira.monte_carlo, cls.__name__): + if not hasattr(caimira.calculator.models.monte_carlo, cls.__name__): missing.append(cls.__name__) continue - mc_cls = getattr(caimira.monte_carlo, cls.__name__) - assert issubclass(mc_cls, caimira.monte_carlo.MCModelBase) + mc_cls = getattr(caimira.calculator.models.monte_carlo, cls.__name__) + assert issubclass(mc_cls, caimira.calculator.models.monte_carlo.MCModelBase) if missing: msg = ( @@ -37,25 +37,25 @@ def test_type_annotations(): @pytest.fixture -def baseline_mc_concentration_model(data_registry) -> caimira.monte_carlo.ConcentrationModel: - mc_model = caimira.monte_carlo.ConcentrationModel( +def baseline_mc_concentration_model(data_registry) -> caimira.calculator.models.monte_carlo.ConcentrationModel: + mc_model = caimira.calculator.models.monte_carlo.ConcentrationModel( data_registry=data_registry, - room=caimira.monte_carlo.Room(volume=caimira.monte_carlo.sampleable.Normal(75, 20), - inside_temp=caimira.models.PiecewiseConstant((0., 24.), (293,))), - ventilation=caimira.monte_carlo.SlidingWindow( + room=caimira.calculator.models.monte_carlo.Room(volume=caimira.calculator.models.monte_carlo.sampleable.Normal(75, 20), + inside_temp=caimira.calculator.models.models.PiecewiseConstant((0., 24.), (293,))), + ventilation=caimira.calculator.models.monte_carlo.SlidingWindow( data_registry=data_registry, - active=caimira.models.PeriodicInterval(period=120, duration=120), - outside_temp=caimira.models.PiecewiseConstant((0., 24.), (283,)), + active=caimira.calculator.models.models.PeriodicInterval(period=120, duration=120), + outside_temp=caimira.calculator.models.models.PiecewiseConstant((0., 24.), (283,)), window_height=1.6, opening_length=0.6, ), - infected=caimira.models.InfectedPopulation( + infected=caimira.calculator.models.models.InfectedPopulation( data_registry=data_registry, number=1, - virus=caimira.models.Virus.types['SARS_CoV_2'], - presence=caimira.models.SpecificInterval(((0., 4.), (5., 8.))), - mask=caimira.models.Mask.types['No mask'], - activity=caimira.models.Activity.types['Light activity'], - expiration=caimira.models.Expiration.types['Breathing'], + virus=caimira.calculator.models.models.Virus.types['SARS_CoV_2'], + presence=caimira.calculator.models.models.SpecificInterval(((0., 4.), (5., 8.))), + mask=caimira.calculator.models.models.Mask.types['No mask'], + activity=caimira.calculator.models.models.Activity.types['Light activity'], + expiration=caimira.calculator.models.models.Expiration.types['Breathing'], host_immunity=0., ), evaporation_factor=0.3, @@ -64,39 +64,39 @@ def baseline_mc_concentration_model(data_registry) -> caimira.monte_carlo.Concen @pytest.fixture -def baseline_mc_sr_model() -> caimira.monte_carlo.ShortRangeModel: +def baseline_mc_sr_model() -> caimira.calculator.models.monte_carlo.ShortRangeModel: return () @pytest.fixture -def baseline_mc_exposure_model(data_registry, baseline_mc_concentration_model, baseline_mc_sr_model) -> caimira.monte_carlo.ExposureModel: - return caimira.monte_carlo.ExposureModel( +def baseline_mc_exposure_model(data_registry, baseline_mc_concentration_model, baseline_mc_sr_model) -> caimira.calculator.models.monte_carlo.ExposureModel: + return caimira.calculator.models.monte_carlo.ExposureModel( data_registry, baseline_mc_concentration_model, baseline_mc_sr_model, - exposed=caimira.models.Population( + exposed=caimira.calculator.models.models.Population( number=10, presence=baseline_mc_concentration_model.infected.presence, activity=baseline_mc_concentration_model.infected.activity, mask=baseline_mc_concentration_model.infected.mask, host_immunity=0., ), - geographical_data=caimira.models.Cases(), + geographical_data=caimira.calculator.models.models.Cases(), ) -def test_build_concentration_model(baseline_mc_concentration_model: caimira.monte_carlo.ConcentrationModel): +def test_build_concentration_model(baseline_mc_concentration_model: caimira.calculator.models.monte_carlo.ConcentrationModel): model = baseline_mc_concentration_model.build_model(7) - assert isinstance(model, caimira.models.ConcentrationModel) + assert isinstance(model, caimira.calculator.models.models.ConcentrationModel) assert isinstance(model.concentration(time=0.), float) conc = model.concentration(time=1.) assert isinstance(conc, np.ndarray) assert conc.shape == (7, ) -def test_build_exposure_model(baseline_mc_exposure_model: caimira.monte_carlo.ExposureModel): +def test_build_exposure_model(baseline_mc_exposure_model: caimira.calculator.models.monte_carlo.ExposureModel): model = baseline_mc_exposure_model.build_model(7) - assert isinstance(model, caimira.models.ExposureModel) + assert isinstance(model, caimira.calculator.models.models.ExposureModel) prob = model.deposited_exposure() assert isinstance(prob, np.ndarray) assert prob.shape == (7, ) diff --git a/caimira/tests/test_monte_carlo_full_models.py b/caimira/tests/test_monte_carlo_full_models.py index ec4f6496..948ce45e 100644 --- a/caimira/tests/test_monte_carlo_full_models.py +++ b/caimira/tests/test_monte_carlo_full_models.py @@ -3,10 +3,10 @@ import pytest from retry import retry -import caimira.monte_carlo as mc -from caimira import models,data -from caimira.monte_carlo.data import activity_distributions, virus_distributions, expiration_distributions, infectious_dose_distribution, viable_to_RNA_ratio_distribution -from caimira.apps.calculator.model_generator import build_expiration +import caimira.calculator.models.monte_carlo as mc +from caimira.calculator.models import models, data +from caimira.calculator.models.monte_carlo.data import activity_distributions, virus_distributions, expiration_distributions, infectious_dose_distribution, viable_to_RNA_ratio_distribution +from caimira.calculator.validators.virus.virus_validator import build_expiration SAMPLE_SIZE = 500_000 TOLERANCE = 0.05 diff --git a/caimira/tests/test_predefined_distributions.py b/caimira/tests/test_predefined_distributions.py index b75e4ce5..c0248b17 100644 --- a/caimira/tests/test_predefined_distributions.py +++ b/caimira/tests/test_predefined_distributions.py @@ -2,8 +2,7 @@ import numpy.testing as npt import pytest -from caimira.monte_carlo.data import activity_distributions, virus_distributions -from caimira.store import data_registry +from caimira.calculator.models.monte_carlo.data import activity_distributions, virus_distributions # Mean & std deviations from https://doi.org/10.1101/2021.10.14.21264988 (Table 3) diff --git a/caimira/tests/test_sampleable_distribution.py b/caimira/tests/test_sampleable_distribution.py index 98f0a9ca..add9dba2 100644 --- a/caimira/tests/test_sampleable_distribution.py +++ b/caimira/tests/test_sampleable_distribution.py @@ -3,7 +3,7 @@ import pytest from retry import retry -from caimira.monte_carlo import sampleable +from caimira.calculator.models.monte_carlo import sampleable @retry(tries=10) diff --git a/caimira/tests/test_ventilation.py b/caimira/tests/test_ventilation.py index 4d322cfa..82c9f682 100644 --- a/caimira/tests/test_ventilation.py +++ b/caimira/tests/test_ventilation.py @@ -4,7 +4,7 @@ import numpy.testing as npt import pytest -from caimira import models +from caimira.calculator.models import models @pytest.fixture diff --git a/cern_caimira/LICENSE b/cern_caimira/LICENSE new file mode 100644 index 00000000..de49c2af --- /dev/null +++ b/cern_caimira/LICENSE @@ -0,0 +1,13 @@ +Copyright 2020-2021 CERN. All rights not expressly granted are reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/cern_caimira/README.md b/cern_caimira/README.md new file mode 100644 index 00000000..4ff16ab5 --- /dev/null +++ b/cern_caimira/README.md @@ -0,0 +1,401 @@ +# CAiMIRA - CERN Airborne Model for Risk Assessment + +CAiMIRA is a risk assessment tool developed to model the concentration of viruses in enclosed spaces, in order to inform space-management decisions. + +CAiMIRA models the concentration profile of potential virions in enclosed spaces , both as background (room) concentration and during close-proximity interations, with clear and intuitive graphs. +The user can set a number of parameters, including room volume, exposure time, activity type, mask-wearing and ventilation. +The report generated indicates how to avoid exceeding critical concentrations and chains of airborne transmission in spaces such as individual offices, meeting rooms and labs. + +The risk assessment tool simulates the airborne spread SARS-CoV-2 virus in a finite volume, assuming a homogenous mixture and a two-stage exhaled jet model, and estimates the risk of COVID-19 infection therein. +The results DO NOT include the other known modes of SARS-CoV-2 transmission, such as fomite or blood-bound. +Hence, the output from this model is only valid when the other recommended public health & safety instructions are observed, such as good hand hygiene and other barrier measures. + +The model used is based on scientific publications relating to airborne transmission of infectious diseases, dose-response exposures and aerosol science, as of February 2022. +It can be used to compare the effectiveness of different airborne-related risk mitigation measures. + +Note that this model applies a deterministic approach, i.e., it is assumed at least one person is infected and shedding viruses into the simulated volume. +Nonetheless, it is also important to understand that the absolute risk of infection is uncertain, as it will depend on the probability that someone infected attends the event. +The model is most useful for comparing the impact and effectiveness of different mitigation measures such as ventilation, filtration, exposure time, physical activity, amount and nature of close-range interactions and +the size of the room, considering both long- and short-range airborne transmission modes of COVID-19 in indoor settings. + +This tool is designed to be informative, allowing the user to adapt different settings and model the relative impact on the estimated infection probabilities. +The objective is to facilitate targeted decision-making and investment through comparisons, rather than a singular determination of absolute risk. +While the SARS-CoV-2 virus is in circulation among the population, the notion of 'zero risk' or 'completely safe scenario' does not exist. +Each event modelled is unique, and the results generated therein are only as accurate as the inputs and assumptions. + +## Authors +CAiMIRA was developed by following members of CERN - European Council for Nuclear Research (visit https://home.cern/): + +Andre Henriques1, Luis Aleixo1, Marco Andreini1, Gabriella Azzopardi2, James Devine3, Philip Elson4, Nicolas Mounet2, Markus Kongstein Rognlien2,6, Nicola Tarocco5 + +1HSE Unit, Occupational Health & Safety Group, CERN
+2Beams Department, Accelerators and Beam Physics Group, CERN
+3Experimental Physics Department, Safety Office, CERN
+4Beams Department, Controls Group, CERN
+5Information Technology Department, Collaboration, Devices & Applications Group, CERN
+6Norwegian University of Science and Technology (NTNU)
+ +### Reference and Citation + +**For the use of the CAiMIRA web app** + +CAiMIRA – CERN Airborne Model for Indoor Risk Assessment tool + +[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.6520431.svg)](https://doi.org/10.5281/zenodo.6520431) + +© Copyright 2020-2021 CERN. All rights not expressly granted are reserved. + +**For use of the CAiMIRA model** + +Henriques A, Mounet N, Aleixo L, Elson P, Devine J, Azzopardi G, Andreini M, Rognlien M, Tarocco N, Tang J. (2022). Modelling airborne transmission of SARS-CoV-2 using CARA: risk assessment for enclosed spaces. _Interface Focus 20210076_. https://doi.org/10.1098/rsfs.2021.0076 + +Reference on the Short-range expiratory jet model from: +Jia W, Wei J, Cheng P, Wang Q, Li Y. (2022). Exposure and respiratory infection risk via the short-range airborne route. _Building and Environment_ *219*: 109166. +https://doi.org/10.1016/j.buildenv.2022.109166 + +***Open Source Acknowledgments*** + +For a detailed list of the open-source dependencies used in this project along with their respective licenses, please refer to [License Information](open-source-licences/README.md). This includes both the core dependencies specified in the project's requirements and their transitive dependencies. + +The information also features a distribution diagram of licenses and a brief description of each of them. + +## Applications + +### Calculator + +A risk assessment tool which simulates the airborne spread of the SARS-CoV-2 virus for space managers. + + +### CAiMIRA Expert App and CO₂ App + +A tool to interact with various parameters of the CAiMIRA model. + + +## Disclaimer + +CAiMIRA has not undergone review, approval or certification by competent authorities, and as a result, it cannot be considered as a fully endorsed and reliable tool, namely in the assessment of potential viral emissions from infected hosts to be modelled. + +The software is provided "as is", without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and non-infringement. +In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the software. + + +## Running CAiMIRA locally + +The easiest way to run a version of CAiMIRA Calculator is to use docker. A pre-built +image of CAiMIRA is made available at https://gitlab.cern.ch/caimira/caimira/container_registry. +In order to run CAiMIRA locally with docker, run the following: + + $ docker run -it -p 8080:8080 gitlab-registry.cern.ch/caimira/caimira/calculator + +This will start a local version of CAiMIRA, which can be visited at http://localhost:8080/. + + +## Development guide + +CAiMIRA is also mirrored to Github if you wish to collaborate on development and can be found at: https://github.com/CERN/caimira + +### Installing CAiMIRA in editable mode + +``` +pip install -e . # At the root of the repository +``` + +### Running the Calculator app in development mode + +``` +python -m cern_caimira.apps.calculator +``` + +To run with a specific template theme created: + +``` +python -m cern_caimira.apps.calculator --theme=cern_caimira/apps/templates/{theme} +``` + +To run the entire app in a different `APPLICATION_ROOT` path: + +``` +python -m cern_caimira.apps.calculator --app_root=/myroot +``` + +To run the calculator on a different URL path: + +``` +python -m cern_caimira.apps.calculator --prefix=/mycalc +``` + +Each of these commands will start a local version of CAiMIRA, which can be visited at http://localhost:8080/. + +### How to compile and read the documentation + +In order to generate the documentation, CAiMIRA must be installed first with the `doc` dependencies: + +``` +pip install -e .[doc] +``` + +To generate the HTML documentation page, the command `make html` should be executed in the `caimira/docs` directory. +If any of the `.rst` files under the `caimira/docs` folder is changed, this command should be executed again. + +Then, right click on `caimira/docs/_build/html/index.html` and select `Open with` your preferred web browser. + +### Running the CAiMIRA Expert-App or CO2-App apps in development mode + +#### Disclaimer + +The `ExpertApplication` and `CO2Application` are no longer actively maintained but will remain in the codebase for legacy purposes. +Please note that the functionality of these applications might be compromised due to deprecation issues. + +#### Running the Applications + +These applications only work within Jupyter notebooks. Attempting to run them outside of a Jupyter environment may result in errors or degraded functionality. + +##### Prerequisites + +Make sure you have the needed dependencies intalled: + +``` +pip install notebook jupyterlab +``` + +Running with Visual Studio Code (VSCode): + +1. Ensure you have the following extensions installed in VSCode: `Jupyter` and `Python`. + +2. Open VSCode and navigate to the directory containing the notebook. + +3. Open the notebook (e.g. `caimira/apps/expert/caimira.ipynb`) and run the cells by clicking the `run` button next to each cell. + +### Running the tests + +``` +pip install -e .[test] +pytest ./caimira +``` + +### Running the profiler + +The profiler is enabled when the environment variable `CAIMIRA_PROFILER_ENABLED` is set to 1. + +When visiting http://localhost:8080/profiler, you can start a new session and choose between [PyInstrument](https://github.com/joerick/pyinstrument) or [cProfile](https://docs.python.org/3/library/profile.html#module-cProfile). The app includes two different profilers, mainly because they can give different information. + +Keep the profiler page open. Then, in another window, navigate to any page in CAiMIRA, for example generate a new report. Refresh the profiler page, and click on the `Report` link to see the profiler output. + +The sessions are stored in a local file in the `/tmp` folder. To share it across multiple web nodes, a shared storage should be added to all web nodes. The folder can be customized via the environment variable `CAIMIRA_PROFILER_CACHE_DIR`. + +### Building the whole environment for local development + +**Simulate the docker build that takes place on openshift with:** + +``` +s2i build file://$(pwd) --copy --keep-symlinks --context-dir ./app-config/nginx/ centos/nginx-112-centos7 caimira-nginx-app +docker build . -f ./app-config/calculator-app/Dockerfile -t calculator-app +docker build ./app-config/auth-service -t auth-service +``` + +Get the client secret from the CERN Application portal for the `caimira-test` app. See [CERN-SSO-integration](#cern-sso-integration) for more info. +``` +read CLIENT_SECRET +``` + +Define some env vars (copy/paste): +``` +export COOKIE_SECRET=$(openssl rand -hex 50) +export OIDC_SERVER=https://auth.cern.ch/auth +export OIDC_REALM=CERN +export CLIENT_ID=caimira-test +``` + +Run docker-compose: +``` +cd app-config +CURRENT_UID=$(id -u):$(id -g) docker-compose up +``` + +Then visit http://localhost:8080/. + +### Setting up the application on openshift + +The https://cern.ch/caimira application is running on CERN's OpenShift platform. In order to set it up for the first time, we followed the documentation at https://paas.docs.cern.ch/. In particular we: + + * Added the OpenShift application deploy key to the GitLab repository + * Created a Python 3.6 (the highest possible at the time of writing) application in OpenShift + * Configured a generic webhook on OpenShift, and call that from the CI of the GitLab repository + +### Updating the caimira-test.web.cern.ch instance + +We have a replica of https://caimira.web.cern.ch running on http://caimira-test.web.cern.ch. Its purpose is to simulate what will happen when +a feature is merged. To push your changes to caimira-test, simply push your branch to `live/caimira-test` and the CI pipeline will trigger the +deployment. To push to this branch, there is a good chance that you will need to force push - you should always force push with care and +understanding why you are doing it. Syntactically, it will look something like (assuming that you have "upstream" as your remote name, +but it may be origin if you haven't configured it differently): + + git push --force upstream name-of-local-branch:live/caimira-test + + +## OpenShift templates + +### First setup + +First, get the [oc](https://docs.okd.io/3.11/cli_reference/get_started_cli.html) client and then login: + +```console +$ oc login https://api.paas.okd.cern.ch +``` + +Then, switch to the project that you want to update: + +```console +$ oc project caimira-test +``` + +Create a new service account in OpenShift to use GitLab container registry: + +```console +$ oc create serviceaccount gitlabci-deployer +serviceaccount "gitlabci-deployer" created + +$ oc policy add-role-to-user registry-editor -z gitlabci-deployer + +# We will refer to the output of this command as `test-token` +$ oc serviceaccounts get-token gitlabci-deployer +<...test-token...> +``` + +Add the token to GitLab to allow GitLab to access OpenShift and define/change image stream tags. Go to `Settings` -> `CI / CD` -> `Variables` -> click on `Expand` button and create the variable `OPENSHIFT_CAIMIRA_TEST_DEPLOY_TOKEN`: insert the token `<...test-token...>`. + +Then, create the webhook secret to be able to trigger automatic builds from GitLab. + +Create and store the secret. Copy the secret above and add it to the GitLab project under `CI /CD` -> `Variables` with the name `OPENSHIFT_CAIMIRA_TEST_WEBHOOK_SECRET`. + +```console +$ WEBHOOKSECRET=$(openssl rand -hex 50) +$ oc create secret generic \ + --from-literal="WebHookSecretKey=$WEBHOOKSECRET" \ + gitlab-caimira-webhook-secret +``` + +For CI usage, we also suggest creating a service account: + +```console +oc create sa gitlab-config-checker +``` + +Under ``User Management`` -> ``RoleBindings`` create a new `RoleBinding` to grant `View` access to the `gitlab-config-checker` service account: + +* name: `gitlab-config-checker-view-role` +* role name: `view` +* service account: `gitlab-config-checker` + +To get this new user's authentication token go to ``User Management`` -> ``Service Accounts`` -> `gitlab-config-checker` and locate the token in the newly created secret associated with the user (in this case ``gitlab-config-checker-token-XXXX``). Copy the `token` value from `Data`. + +Create the various configurations: + +```console +$ cd app-config/openshift + +$ oc process -f configmap.yaml | oc create -f - +$ oc process -f services.yaml | oc create -f - +$ oc process -f imagestreams.yaml | oc create -f - +$ oc process -f buildconfig.yaml --param GIT_BRANCH='live/caimira-test' | oc create -f - +$ oc process -f deploymentconfig.yaml --param PROJECT_NAME='caimira-test' | oc create -f - +``` + +Manually create the **route** to access the website, see `routes.example.yaml`. +After having created the route, make sure that you extend the HTTP request timeout annotation: the +report generation can take more time than the default 30 seconds. + +``` +$ oc annotate route caimira-route --overwrite haproxy.router.openshift.io/timeout=60s +``` + +### CERN SSO integration + +The SSO integration uses OpenID credentials configured in [CERN Applications portal](https://application-portal.web.cern.ch/). +How to configure the application: + +* Application Identifier: `caimira-test` +* Homepage: `https://caimira-test.web.cern.ch` +* Administrators: `caimira-dev` +* SSO Registration: + * Protocol: `OpenID (OIDC)` + * Redirect URI: `https://caimira-test.web.cern.ch/auth/authorize` + * Leave unchecked all the other checkboxes +* Define new roles: + * Name: `CERN Users` + * Role Identifier: `external-users` + * Leave unchecked checkboxes + * Minimum Level Of Assurance: `CERN (highest)` + * Assign role to groups: `cern-accounts-primary` e-group + * Name: `External accounts` + * Role Identifier: `admin` + * Leave unchecked checkboxes + * Minimum Level Of Assurance: `Any (no restrictions)` + * Assign role to groups: `caimira-app-external-access` e-group + * Name: `Allowed users` + * Role Identifier: `allowed-users` + * Check `This role is required to access my application` + * Minimum Level Of Assurance:`Any (no restrictions)` + * Assign role to groups: `cern-accounts-primary` and `caimira-app-external-access` e-groups + +Copy the client id and client secret and use it below. + +```console +$ COOKIE_SECRET=$(openssl rand -hex 50) +$ oc create secret generic \ + --from-literal="CLIENT_ID=$CLIENT_ID" \ + --from-literal="CLIENT_SECRET=$CLIENT_SECRET" \ + --from-literal="COOKIE_SECRET=$COOKIE_SECRET" \ + auth-service-secrets +``` + +### External APIs + +- **Geographical location:** +There is one external API call to fetch required information related to the geographical location inserted by a user. +The documentation for this geocoding service is available at https://developers.arcgis.com/rest/geocode/api-reference/geocoding-suggest.htm . +Please note that there is no need for keys on this API call. It is **free-of-charge**. + +- **Humidity and Inside Temperature:** +There is the possibility of using one external API call to fetch information related to a location specified in the UI. The data is related to the inside temperature and humidity taken from an indoor measurement device. Note that the API currently used from ARVE is only available for the `CERN theme` as the authorised sensors are installed at CERN." + +- **ARVE:** + +The ARVE Swiss Air Quality System provides trusted air data for commercial buildings in real-time and analyzes it with the help of AI and machine learning algorithms to create actionable insights. + +Create secret: + +```console +$ read ARVE_CLIENT_ID +$ read ARVE_CLIENT_SECRET +$ read ARVE_API_KEY +$ oc create secret generic \ + --from-literal="ARVE_CLIENT_ID=$ARVE_CLIENT_ID" \ + --from-literal="ARVE_CLIENT_SECRET=$ARVE_CLIENT_SECRET" \ + --from-literal="ARVE_API_KEY=$ARVE_API_KEY" \ + arve-api +``` + +- **CERN Data Service:** + +The CERN data service collects data from various sources and expose them via a REST API endpoint. + +The service is enabled when the environment variable `DATA_SERVICE_ENABLED` is set to 1. + +## Update configuration + +If you need to **update** existing configuration, then modify this repository and after having logged in, run: + +```console +$ cd app-config/openshift + + +$ oc process -f configmap.yaml | oc replace -f - +$ oc process -f services.yaml | oc replace -f - +$ oc process -f imagestreams.yaml | oc replace -f - +$ oc process -f buildconfig.yaml --param GIT_BRANCH='live/caimira-test' | oc replace -f - +$ oc process -f deploymentconfig.yaml --param PROJECT_NAME='caimira-test' | oc replace -f - +``` + +Be aware that if you create/recreate the environment you must manually create a **route** in OpenShift, +specifying the respective annotation to be exposed outside CERN. diff --git a/cern_caimira/pyproject.toml b/cern_caimira/pyproject.toml new file mode 100644 index 00000000..70d1600b --- /dev/null +++ b/cern_caimira/pyproject.toml @@ -0,0 +1,109 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "cern-caimira" +version = "2.0.0" +description = "CAiMIRA - CERN Airborne Model for Indoor Risk Assessment" +readme = "README.md" +license = { text = "Apache-2.0" } +authors = [ + { name = "Andre Henriques", email = "andre.henriques@cern.ch" } +] +classifiers = [ + "Programming Language :: Python :: 3", + "Operating System :: OS Independent", + "License :: OSI Approved :: Apache Software License", +] +requires-python = ">=3.9" +dependencies = [ + "ipykernel", + "ipympl>=0.9.0", + "ipywidgets<8.0", + "Jinja2", + "loky", + "matplotlib", + "memoization", + "mistune", + "numpy", + "pandas", + "psutil", + "pyinstrument", + "pyjwt", + "python-dateutil", + "retry", + "ruptures", + "scipy", + "scikit-learn", + "timezonefinder", + "tornado", + "types-retry", +] + +[project.optional-dependencies] +dev = [] +test = [ + "pytest", + "pytest-mypy >= 0.10.3", + "mypy >= 1.0.0", + "pytest-tornasync", + "numpy-stubs @ git+https://github.com/numpy/numpy-stubs.git", + "types-dataclasses", + "types-python-dateutil", + "types-requests" +] +doc = [ + "sphinx", + "sphinx_rtd_theme" +] + +[project.urls] +Homepage = "https://cern.ch/caimira" + +[tool.setuptools] +packages = ["cern_caimira"] +package-dir = {"" = "src"} + +[tool.pytest.ini_options] +addopts = "--mypy" + +[tool.mypy] +no_warn_no_return = true +ignore_missing_imports = true # TODO what to do here? + +[tool.mypy-loky] +ignore_missing_imports = true + +[tool.mypy-ipympl] +ignore_missing_imports = true + +[tool.mypy-ipywidgets] +ignore_missing_imports = true + +[tool.mypy-matplotlib] +ignore_missing_imports = true + +[tool.mypy-mistune] +ignore_missing_imports = true + +[tool.mypy-qrcode] +ignore_missing_imports = true + +[tool.mypy-scipy] +ignore_missing_imports = true + +[tool.mypy-timezonefinder] +ignore_missing_imports = true + +[tool.mypy-pandas] +ignore_missing_imports = true + +[tool.mypy-pstats] +follow_imports = "skip" + +[tool.mypy-tabulate] +ignore_missing_imports = true + +[tool.mypy-ruptures] +ignore_missing_imports = true diff --git a/requirements.txt b/cern_caimira/requirements.txt similarity index 100% rename from requirements.txt rename to cern_caimira/requirements.txt diff --git a/cern_caimira/src/cern_caimira/apps/__.init__.py b/cern_caimira/src/cern_caimira/apps/__.init__.py new file mode 100644 index 00000000..2c02b644 --- /dev/null +++ b/cern_caimira/src/cern_caimira/apps/__.init__.py @@ -0,0 +1,3 @@ +import importlib.metadata + +__version__ = importlib.metadata.version(__package__ or __name__) diff --git a/caimira/apps/calculator/__init__.py b/cern_caimira/src/cern_caimira/apps/calculator/__init__.py similarity index 93% rename from caimira/apps/calculator/__init__.py rename to cern_caimira/src/cern_caimira/apps/calculator/__init__.py index 98f75762..b16b7c7a 100644 --- a/caimira/apps/calculator/__init__.py +++ b/cern_caimira/src/cern_caimira/apps/calculator/__init__.py @@ -25,14 +25,17 @@ from tornado.web import Application, RequestHandler, StaticFileHandler from tornado.httpclient import AsyncHTTPClient, HTTPRequest import tornado.log -from caimira.profiler import CaimiraProfiler, Profilers -from caimira.store.data_registry import DataRegistry +from caimira.calculator.models.profiler import CaimiraProfiler, Profilers +from caimira.calculator.store.data_registry import DataRegistry +from caimira.calculator.store.data_service import DataService -from caimira.store.data_service import DataService +from caimira.api.controller.report_controller import generate_form_obj, generate_model, generate_report_results +from caimira.calculator.report.report_generator import calculate_report_data +from caimira.calculator.validators.virus import virus_validator +from caimira.calculator.validators.co2 import co2_validator +from .report import ReportGenerator from . import markdown_tools -from . import model_generator, co2_model_generator -from .report_generator import ReportGenerator, calculate_report_data from .user import AuthenticatedUser, AnonymousUser # The calculator version is based on a combination of the model version and the @@ -42,7 +45,7 @@ # calculator version. If the calculator needs to make breaking changes (e.g. change # form attributes) then it can also increase its MAJOR version without needing to # increase the overall CAiMIRA version (found at ``caimira.__version__``). -__version__ = "4.16.1" +__version__ = "4.17.0" LOG = logging.getLogger("Calculator") @@ -177,27 +180,27 @@ async def post(self) -> None: LOG.debug(pformat(requested_model_config)) try: - form = model_generator.VirusFormData.from_dict(requested_model_config, data_registry) + form = generate_form_obj(requested_model_config, data_registry) + model = generate_model(form) + report_data = generate_report_results(form, model) + except Exception as err: LOG.exception(err) response_json = {'code': 400, 'error': f'Your request was invalid {html.escape(str(err))}'} self.set_status(400) self.finish(json.dumps(response_json)) return - + base_url = self.request.protocol + "://" + self.request.host report_generator: ReportGenerator = self.settings['report_generator'] executor = loky.get_reusable_executor( max_workers=self.settings['handler_worker_pool_size'], timeout=300, ) - # Re-generate the report with the conditional probability of infection plot - if self.get_cookie('conditional_plot'): - form.conditional_probability_plot = True if self.get_cookie('conditional_plot') == '1' else False - self.clear_cookie('conditional_plot') # Clears cookie after changing the form value. report_task = executor.submit( report_generator.build_report, base_url, form, + model, report_data, executor_factory=functools.partial( concurrent.futures.ThreadPoolExecutor, self.settings['report_generation_parallelism'], @@ -231,7 +234,7 @@ async def post(self) -> None: LOG.debug(pformat(requested_model_config)) try: - form = model_generator.VirusFormData.from_dict(requested_model_config, data_registry) + form = generate_form_obj(requested_model_config, data_registry) except Exception as err: LOG.exception(err) response_json = {'code': 400, 'error': f'Your request was invalid {html.escape(str(err))}'} @@ -243,7 +246,7 @@ async def post(self) -> None: max_workers=self.settings['handler_worker_pool_size'], timeout=300, ) - model = form.build_model() + model = generate_model(form) report_data_task = executor.submit(calculate_report_data, form, model, executor_factory=functools.partial( concurrent.futures.ThreadPoolExecutor, @@ -262,14 +265,17 @@ async def get(self) -> None: if data_service: data_service.update_registry(data_registry) - form = model_generator.VirusFormData.from_dict(model_generator.baseline_raw_form_data(), data_registry) + form = generate_form_obj(virus_validator.baseline_raw_form_data(), data_registry) + model = generate_model(form) + report_data = generate_report_results(form, model) + base_url = self.request.protocol + "://" + self.request.host report_generator: ReportGenerator = self.settings['report_generator'] executor = loky.get_reusable_executor(max_workers=self.settings['handler_worker_pool_size']) report_task = executor.submit( report_generator.build_report, base_url, form, - executor_factory=functools.partial( + model, report_data, executor_factory=functools.partial( concurrent.futures.ThreadPoolExecutor, self.settings['report_generation_parallelism'], ), @@ -442,7 +448,7 @@ async def post(self, endpoint: str) -> None: requested_model_config = tornado.escape.json_decode(self.request.body) try: - form = co2_model_generator.CO2FormData.from_dict(requested_model_config, data_registry) + form = co2_validator.CO2FormData.from_dict(requested_model_config, data_registry) except Exception as err: if self.settings.get("debug", False): import traceback @@ -453,8 +459,8 @@ async def post(self, endpoint: str) -> None: return if endpoint.rstrip('/') == 'plot': - transition_times = co2_model_generator.CO2FormData.find_change_points_with_pelt(form.CO2_data) - self.finish({'CO2_plot': co2_model_generator.CO2FormData.generate_ventilation_plot(form.CO2_data, transition_times), + transition_times = co2_validator.CO2FormData.find_change_points_with_pelt(form.CO2_data) + self.finish({'CO2_plot': co2_validator.CO2FormData.generate_ventilation_plot(form.CO2_data, transition_times), 'transition_times': [round(el, 2) for el in transition_times]}) else: executor = loky.get_reusable_executor( @@ -462,7 +468,7 @@ async def post(self, endpoint: str) -> None: timeout=300, ) report_task = executor.submit( - co2_model_generator.CO2FormData.build_model, form, + co2_validator.CO2FormData.build_model, form, ) report = await asyncio.wrap_future(report_task) @@ -471,7 +477,7 @@ async def post(self, endpoint: str) -> None: result['fitting_ventilation_type'] = form.fitting_ventilation_type result['transition_times'] = ventilation_transition_times - result['CO2_plot'] = co2_model_generator.CO2FormData.generate_ventilation_plot(CO2_data=form.CO2_data, + result['CO2_plot'] = co2_validator.CO2FormData.generate_ventilation_plot(CO2_data=form.CO2_data, transition_times=ventilation_transition_times[:-1], predictive_CO2=result['predictive_CO2']) self.finish(result) diff --git a/caimira/apps/calculator/__main__.py b/cern_caimira/src/cern_caimira/apps/calculator/__main__.py similarity index 100% rename from caimira/apps/calculator/__main__.py rename to cern_caimira/src/cern_caimira/apps/calculator/__main__.py diff --git a/caimira/apps/calculator/markdown_tools.py b/cern_caimira/src/cern_caimira/apps/calculator/markdown_tools.py similarity index 100% rename from caimira/apps/calculator/markdown_tools.py rename to cern_caimira/src/cern_caimira/apps/calculator/markdown_tools.py diff --git a/cern_caimira/src/cern_caimira/apps/calculator/report.py b/cern_caimira/src/cern_caimira/apps/calculator/report.py new file mode 100644 index 00000000..71249a81 --- /dev/null +++ b/cern_caimira/src/cern_caimira/apps/calculator/report.py @@ -0,0 +1,379 @@ +from datetime import datetime +import dataclasses + +import concurrent.futures +import json +import typing +import jinja2 +import numpy as np +import urllib +import base64 +import zlib + +from . import markdown_tools + +from caimira.calculator.models import dataclass_utils, models, monte_carlo as mc +from caimira.calculator.validators.virus.virus_validator import VirusFormData + + +def generate_permalink(base_url, get_root_url, get_root_calculator_url, form: VirusFormData): + form_dict = VirusFormData.to_dict(form, strip_defaults=True) + + # Generate the calculator URL arguments that would be needed to re-create this + # form. + args = urllib.parse.urlencode(form_dict) + + # Then zlib compress + base64 encode the string. To be inverted by the + # /_c/ endpoint. + compressed_args = base64.b64encode(zlib.compress(args.encode())).decode() + qr_url = f"{base_url}{get_root_url()}/_c/{compressed_args}" + url = f"{base_url}{get_root_calculator_url()}?{args}" + + return { + 'link': url, + 'shortened': qr_url, + } + + +def model_start_end(model: models.ExposureModel): + t_start = min(model.exposed.presence_interval().boundaries()[0][0], + model.concentration_model.infected.presence_interval().boundaries()[0][0]) + t_end = max(model.exposed.presence_interval().boundaries()[-1][1], + model.concentration_model.infected.presence_interval().boundaries()[-1][1]) + return t_start, t_end + + +def fill_big_gaps(array, gap_size): + """ + Insert values into the given sorted list if there is a gap of more than ``gap_size``. + All values in the given array are preserved, even if they are within the ``gap_size`` of one another. + + >>> fill_big_gaps([1, 2, 4], gap_size=0.75) + [1, 1.75, 2, 2.75, 3.5, 4] + + """ + result = [] + if len(array) == 0: + raise ValueError("Input array must be len > 0") + + last_value = array[0] + for value in array: + while value - last_value > gap_size + 1e-15: + last_value = last_value + gap_size + result.append(last_value) + result.append(value) + last_value = value + return result + + +def non_temp_transition_times(model: models.ExposureModel): + """ + Return the non-temperature (and PiecewiseConstant) based transition times. + + """ + def walk_model(model, name=""): + # Extend walk_dataclass to handle lists of dataclasses + # (e.g. in MultipleVentilation). + for name, obj in dataclass_utils.walk_dataclass(model, name=name): + if name.endswith('.ventilations') and isinstance(obj, (list, tuple)): + for i, item in enumerate(obj): + fq_name_i = f'{name}[{i}]' + yield fq_name_i, item + if dataclasses.is_dataclass(item): + yield from dataclass_utils.walk_dataclass(item, name=fq_name_i) + else: + yield name, obj + + t_start, t_end = model_start_end(model) + + change_times = {t_start, t_end} + for name, obj in walk_model(model, name="exposure"): + if isinstance(obj, models.Interval): + change_times |= obj.transition_times() + + # Only choose times that are in the range of the model (removes things + # such as PeriodicIntervals, which extend beyond the model itself). + return sorted(time for time in change_times if (t_start <= time <= t_end)) + + +def interesting_times(model: models.ExposureModel, approx_n_pts: typing.Optional[int] = None) -> typing.List[float]: + """ + Pick approximately ``approx_n_pts`` time points which are interesting for the + given model. If not provided by argument, ``approx_n_pts`` is set to be 15 times + the number of hours of the simulation. + + Initially the times are seeded by important state change times (excluding + outside temperature), and the times are then subsequently expanded to ensure + that the step size is at most ``(t_end - t_start) / approx_n_pts``. + + """ + times = non_temp_transition_times(model) + sim_duration = max(times) - min(times) + if not approx_n_pts: + approx_n_pts = sim_duration * 15 + + # Expand the times list to ensure that we have a maximum gap size between + # the key times. + nice_times = fill_big_gaps(times, gap_size=(sim_duration) / approx_n_pts) + return nice_times + + +def manufacture_viral_load_scenarios_percentiles(model: mc.ExposureModel) -> typing.Dict[str, mc.ExposureModel]: + viral_load = model.concentration_model.infected.virus.viral_load_in_sputum + scenarios = {} + for percentil in (0.01, 0.05, 0.25, 0.5, 0.75, 0.95, 0.99): + vl = np.quantile(viral_load, percentil) + specific_vl_scenario = dataclass_utils.nested_replace(model, + {'concentration_model.infected.virus.viral_load_in_sputum': vl} + ) + scenarios[str(vl)] = np.mean( + specific_vl_scenario.infection_probability()) + return scenarios + +def manufacture_alternative_scenarios(form: VirusFormData) -> typing.Dict[str, mc.ExposureModel]: + scenarios = {} + if (form.short_range_option == "short_range_no"): + # Two special option cases - HEPA and/or FFP2 masks. + FFP2_being_worn = bool(form.mask_wearing_option == + 'mask_on' and form.mask_type == 'FFP2') + if FFP2_being_worn and form.hepa_option: + FFP2andHEPAalternative = dataclass_utils.replace( + form, mask_type='Type I') + if not (form.hepa_option and form.mask_wearing_option == 'mask_on' and form.mask_type == 'Type I'): + scenarios['Base scenario with HEPA filter and Type I masks'] = FFP2andHEPAalternative.build_mc_model() + if not FFP2_being_worn and form.hepa_option: + noHEPAalternative = dataclass_utils.replace(form, mask_type='FFP2') + noHEPAalternative = dataclass_utils.replace( + noHEPAalternative, mask_wearing_option='mask_on') + noHEPAalternative = dataclass_utils.replace( + noHEPAalternative, hepa_option=False) + if not (not form.hepa_option and FFP2_being_worn): + scenarios['Base scenario without HEPA filter, with FFP2 masks'] = noHEPAalternative.build_mc_model() + + # The remaining scenarios are based on Type I masks (possibly not worn) + # and no HEPA filtration. + form = dataclass_utils.replace(form, mask_type='Type I') + if form.hepa_option: + form = dataclass_utils.replace(form, hepa_option=False) + + with_mask = dataclass_utils.replace( + form, mask_wearing_option='mask_on') + without_mask = dataclass_utils.replace( + form, mask_wearing_option='mask_off') + + if form.ventilation_type == 'mechanical_ventilation': + # scenarios['Mechanical ventilation with Type I masks'] = with_mask.build_mc_model() + if not (form.mask_wearing_option == 'mask_off'): + scenarios['Mechanical ventilation without masks'] = without_mask.build_mc_model( + ) + + elif form.ventilation_type == 'natural_ventilation': + # scenarios['Windows open with Type I masks'] = with_mask.build_mc_model() + if not (form.mask_wearing_option == 'mask_off'): + scenarios['Windows open without masks'] = without_mask.build_mc_model() + + # No matter the ventilation scheme, we include scenarios which don't have any ventilation. + with_mask_no_vent = dataclass_utils.replace( + with_mask, ventilation_type='no_ventilation') + without_mask_or_vent = dataclass_utils.replace( + without_mask, ventilation_type='no_ventilation') + + if not (form.mask_wearing_option == 'mask_on' and form.mask_type == 'Type I' and form.ventilation_type == 'no_ventilation'): + scenarios['No ventilation with Type I masks'] = with_mask_no_vent.build_mc_model() + if not (form.mask_wearing_option == 'mask_off' and form.ventilation_type == 'no_ventilation'): + scenarios['Neither ventilation nor masks'] = without_mask_or_vent.build_mc_model() + + else: + no_short_range_alternative = dataclass_utils.replace(form, short_range_interactions=[], + total_people=form.total_people - form.short_range_occupants) + scenarios['Base scenario without short-range interactions'] = no_short_range_alternative.build_mc_model() + + return scenarios + + +def scenario_statistics( + mc_model: mc.ExposureModel, + sample_times: typing.List[float], + compute_prob_exposure: bool +): + model = mc_model.build_model( + size=mc_model.data_registry.monte_carlo['sample_size']) + if (compute_prob_exposure): + # It means we have data to calculate the total_probability_rule + prob_probabilistic_exposure = model.total_probability_rule() + else: + prob_probabilistic_exposure = 0. + + return { + 'probability_of_infection': np.mean(model.infection_probability()), + 'expected_new_cases': np.mean(model.expected_new_cases()), + 'concentrations': [ + np.mean(model.concentration(time)) + for time in sample_times + ], + 'prob_probabilistic_exposure': prob_probabilistic_exposure, + } + + +def comparison_report( + form: VirusFormData, + report_data: typing.Dict[str, typing.Any], + scenarios: typing.Dict[str, mc.ExposureModel], + sample_times: typing.List[float], + executor_factory: typing.Callable[[], concurrent.futures.Executor], +): + if (form.short_range_option == "short_range_no"): + statistics = { + 'Current scenario': { + 'probability_of_infection': report_data['prob_inf'], + 'expected_new_cases': report_data['expected_new_cases'], + 'concentrations': report_data['concentrations'], + } + } + else: + statistics = {} + + if (form.short_range_option == "short_range_yes" and form.exposure_option == "p_probabilistic_exposure"): + compute_prob_exposure = True + else: + compute_prob_exposure = False + + with executor_factory() as executor: + results = executor.map( + scenario_statistics, + scenarios.values(), + [sample_times] * len(scenarios), + [compute_prob_exposure] * len(scenarios), + timeout=60, + ) + + for (name, model), model_stats in zip(scenarios.items(), results): + statistics[name] = model_stats + + return { + 'stats': statistics, + } + + +def minutes_to_time(minutes: int) -> str: + minute_string = str(minutes % 60) + minute_string = "0" * (2 - len(minute_string)) + minute_string + hour_string = str(minutes // 60) + hour_string = "0" * (2 - len(hour_string)) + hour_string + + return f"{hour_string}:{minute_string}" + + +def readable_minutes(minutes: int) -> str: + time = float(minutes) + unit = " minute" + if time % 60 == 0: + time = minutes/60 + unit = " hour" + if time != 1: + unit += "s" + + if time.is_integer(): + time_str = "{:0.0f}".format(time) + else: + time_str = "{0:.2f}".format(time) + + return time_str + unit + + +def hour_format(hour: float) -> str: + # Convert float hour to HH:MM format + hours = int(hour) + minutes = int(hour % 1 * 60) + return f"{hours}:{minutes if minutes != 0 else '00'}" + + +def percentage(absolute: float) -> float: + return absolute * 100 + + +def non_zero_percentage(percentage: int) -> str: + if percentage < 0.01: + return "<0.01%" + elif percentage < 1: + return "{:0.2f}%".format(percentage) + elif percentage > 99.9 or np.isnan(percentage): + return ">99.9%" + else: + return "{:0.1f}%".format(percentage) + + +@dataclasses.dataclass +class ReportGenerator: + jinja_loader: jinja2.BaseLoader + get_root_url: typing.Any + get_root_calculator_url: typing.Any + + def build_report( + self, + base_url: str, + form: VirusFormData, + model: models.ExposureModel, + report_data: dict, + executor_factory: typing.Callable[[], concurrent.futures.Executor], + ) -> str: + context = self.prepare_context( + base_url, form, model, report_data, executor_factory=executor_factory) + return self.render(context) + + def prepare_context( + self, + base_url: str, + form: VirusFormData, + model: models.ExposureModel, + report_data: dict, + executor_factory: typing.Callable[[], concurrent.futures.Executor], + ) -> dict: + now = datetime.utcnow().astimezone() + time = now.strftime("%Y-%m-%d %H:%M:%S UTC") + + data_registry_version = f"v{model.data_registry.version}" if model.data_registry.version else None + context = { + 'model': model, + 'form': form, + 'creation_date': time, + 'data_registry_version': data_registry_version, + } + + scenario_sample_times = interesting_times(model) + context.update(report_data) + + alternative_scenarios = manufacture_alternative_scenarios(form) + context['alternative_viral_load'] = manufacture_viral_load_scenarios_percentiles( + model) if form.conditional_probability_viral_loads else None + context['alternative_scenarios'] = comparison_report( + form, report_data, alternative_scenarios, scenario_sample_times, executor_factory=executor_factory, + ) + context['permalink'] = generate_permalink( + base_url, self.get_root_url, self.get_root_calculator_url, form) + context['get_url'] = self.get_root_url + context['get_calculator_url'] = self.get_root_calculator_url + + return context + + def _template_environment(self) -> jinja2.Environment: + env = jinja2.Environment( + loader=self.jinja_loader, + undefined=jinja2.StrictUndefined, + ) + env.globals["common_text"] = markdown_tools.extract_rendered_markdown_blocks( + env.get_template('common_text.md.j2') + ) + env.filters['non_zero_percentage'] = non_zero_percentage + env.filters['readable_minutes'] = readable_minutes + env.filters['minutes_to_time'] = minutes_to_time + env.filters['hour_format'] = hour_format + env.filters['float_format'] = "{0:.2f}".format + env.filters['int_format'] = "{:0.0f}".format + env.filters['percentage'] = percentage + env.filters['JSONify'] = json.dumps + return env + + def render(self, context: dict) -> str: + template = self._template_environment().get_template("calculator.report.html.j2") + return template.render(**context, text_blocks=template.globals["common_text"]) diff --git a/caimira/apps/calculator/static/css/form.css b/cern_caimira/src/cern_caimira/apps/calculator/static/css/form.css similarity index 100% rename from caimira/apps/calculator/static/css/form.css rename to cern_caimira/src/cern_caimira/apps/calculator/static/css/form.css diff --git a/caimira/apps/calculator/static/css/report.css b/cern_caimira/src/cern_caimira/apps/calculator/static/css/report.css similarity index 100% rename from caimira/apps/calculator/static/css/report.css rename to cern_caimira/src/cern_caimira/apps/calculator/static/css/report.css diff --git a/caimira/apps/calculator/static/icons/favicon.ico b/cern_caimira/src/cern_caimira/apps/calculator/static/icons/favicon.ico similarity index 100% rename from caimira/apps/calculator/static/icons/favicon.ico rename to cern_caimira/src/cern_caimira/apps/calculator/static/icons/favicon.ico diff --git a/caimira/apps/calculator/static/images/disclaimer.jpg b/cern_caimira/src/cern_caimira/apps/calculator/static/images/disclaimer.jpg similarity index 100% rename from caimira/apps/calculator/static/images/disclaimer.jpg rename to cern_caimira/src/cern_caimira/apps/calculator/static/images/disclaimer.jpg diff --git a/caimira/apps/calculator/static/images/warning_scale/green-1.png b/cern_caimira/src/cern_caimira/apps/calculator/static/images/warning_scale/green-1.png similarity index 100% rename from caimira/apps/calculator/static/images/warning_scale/green-1.png rename to cern_caimira/src/cern_caimira/apps/calculator/static/images/warning_scale/green-1.png diff --git a/caimira/apps/calculator/static/images/warning_scale/orange-3.png b/cern_caimira/src/cern_caimira/apps/calculator/static/images/warning_scale/orange-3.png similarity index 100% rename from caimira/apps/calculator/static/images/warning_scale/orange-3.png rename to cern_caimira/src/cern_caimira/apps/calculator/static/images/warning_scale/orange-3.png diff --git a/caimira/apps/calculator/static/images/warning_scale/red-4.png b/cern_caimira/src/cern_caimira/apps/calculator/static/images/warning_scale/red-4.png similarity index 100% rename from caimira/apps/calculator/static/images/warning_scale/red-4.png rename to cern_caimira/src/cern_caimira/apps/calculator/static/images/warning_scale/red-4.png diff --git a/caimira/apps/calculator/static/images/warning_scale/yellow-2.png b/cern_caimira/src/cern_caimira/apps/calculator/static/images/warning_scale/yellow-2.png similarity index 100% rename from caimira/apps/calculator/static/images/warning_scale/yellow-2.png rename to cern_caimira/src/cern_caimira/apps/calculator/static/images/warning_scale/yellow-2.png diff --git a/caimira/apps/calculator/static/images/window_opening.png b/cern_caimira/src/cern_caimira/apps/calculator/static/images/window_opening.png similarity index 100% rename from caimira/apps/calculator/static/images/window_opening.png rename to cern_caimira/src/cern_caimira/apps/calculator/static/images/window_opening.png diff --git a/caimira/apps/calculator/static/images/window_type.PNG b/cern_caimira/src/cern_caimira/apps/calculator/static/images/window_type.PNG similarity index 100% rename from caimira/apps/calculator/static/images/window_type.PNG rename to cern_caimira/src/cern_caimira/apps/calculator/static/images/window_type.PNG diff --git a/caimira/apps/calculator/static/js/co2_form.js b/cern_caimira/src/cern_caimira/apps/calculator/static/js/co2_form.js similarity index 100% rename from caimira/apps/calculator/static/js/co2_form.js rename to cern_caimira/src/cern_caimira/apps/calculator/static/js/co2_form.js diff --git a/caimira/apps/calculator/static/js/form.js b/cern_caimira/src/cern_caimira/apps/calculator/static/js/form.js similarity index 100% rename from caimira/apps/calculator/static/js/form.js rename to cern_caimira/src/cern_caimira/apps/calculator/static/js/form.js diff --git a/caimira/apps/calculator/static/js/report.js b/cern_caimira/src/cern_caimira/apps/calculator/static/js/report.js similarity index 100% rename from caimira/apps/calculator/static/js/report.js rename to cern_caimira/src/cern_caimira/apps/calculator/static/js/report.js diff --git a/caimira/apps/calculator/user.py b/cern_caimira/src/cern_caimira/apps/calculator/user.py similarity index 100% rename from caimira/apps/calculator/user.py rename to cern_caimira/src/cern_caimira/apps/calculator/user.py diff --git a/cern_caimira/src/cern_caimira/apps/expert_apps/__init__.py b/cern_caimira/src/cern_caimira/apps/expert_apps/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/caimira/apps/expert.py b/cern_caimira/src/cern_caimira/apps/expert_apps/expert.py similarity index 99% rename from caimira/apps/expert.py rename to cern_caimira/src/cern_caimira/apps/expert_apps/expert.py index 79c64c2b..d60e33e4 100644 --- a/caimira/apps/expert.py +++ b/cern_caimira/src/cern_caimira/apps/expert_apps/expert.py @@ -14,8 +14,11 @@ import pandas as pd import logging -from caimira import data, models, state -from caimira.store.data_registry import DataRegistry +from . import state +from caimira.calculator.models import data, models +from caimira.calculator.store.data_registry import DataRegistry + +LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__) @@ -935,7 +938,7 @@ def __init__(self) -> None: LOG.warning( "ExpertApplication is currently deactivated and will no longer be maintained. It remains in the codebase for legacy purposes." ) - + self._data_registry = DataRegistry() #: A list of scenario name and ModelState instances. This is intended to be #: mutated. Any mutation should notify the appropriate Views for handling. diff --git a/caimira/apps/expert/caimira.ipynb b/cern_caimira/src/cern_caimira/apps/expert_apps/expert/caimira.ipynb similarity index 95% rename from caimira/apps/expert/caimira.ipynb rename to cern_caimira/src/cern_caimira/apps/expert_apps/expert/caimira.ipynb index 7153e257..dcb93b2c 100644 --- a/caimira/apps/expert/caimira.ipynb +++ b/cern_caimira/src/cern_caimira/apps/expert_apps/expert/caimira.ipynb @@ -37,9 +37,9 @@ } ], "source": [ - "import caimira.apps\n", + "import cern_caimira.apps as apps\n", "\n", - "app = caimira.apps.ExpertApplication()\n", + "app = apps.ExpertApplication()\n", "app.widget\n" ] } diff --git a/caimira/apps/expert/static/images/header_image.png b/cern_caimira/src/cern_caimira/apps/expert_apps/expert/static/images/header_image.png similarity index 100% rename from caimira/apps/expert/static/images/header_image.png rename to cern_caimira/src/cern_caimira/apps/expert_apps/expert/static/images/header_image.png diff --git a/caimira/apps/expert_co2.py b/cern_caimira/src/cern_caimira/apps/expert_apps/expert_co2.py similarity index 99% rename from caimira/apps/expert_co2.py rename to cern_caimira/src/cern_caimira/apps/expert_apps/expert_co2.py index 29964bf9..4a0a9ba5 100644 --- a/caimira/apps/expert_co2.py +++ b/cern_caimira/src/cern_caimira/apps/expert_apps/expert_co2.py @@ -4,13 +4,17 @@ import numpy as np import logging -from caimira import data, models, state -from caimira.store.data_registry import DataRegistry +from . import state +from .expert import generate_presence_widget, collapsible, ipympl_canvas, WidgetGroup, CAIMIRAStateBuilder +from caimira.calculator.models import data, models +from caimira.calculator.store.data_registry import DataRegistry + import matplotlib import matplotlib.figure import matplotlib.lines as mlines import matplotlib.patches as patches -from .expert import generate_presence_widget, collapsible, ipympl_canvas, WidgetGroup, CAIMIRAStateBuilder + +LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__) @@ -194,7 +198,7 @@ def __init__(self) -> None: LOG.warning( "CO2Application is currently deactivated and will no longer be maintained. It remains in the codebase for legacy purposes." ) - + self._data_registry = DataRegistry() # self._debug_output = widgets.Output() diff --git a/caimira/apps/expert_co2/caimira.ipynb b/cern_caimira/src/cern_caimira/apps/expert_apps/expert_co2/caimira.ipynb similarity index 95% rename from caimira/apps/expert_co2/caimira.ipynb rename to cern_caimira/src/cern_caimira/apps/expert_apps/expert_co2/caimira.ipynb index 259328ac..52117834 100644 --- a/caimira/apps/expert_co2/caimira.ipynb +++ b/cern_caimira/src/cern_caimira/apps/expert_apps/expert_co2/caimira.ipynb @@ -33,9 +33,9 @@ } ], "source": [ - "import caimira.apps\n", + "import cern_caimira.apps as apps\n", "\n", - "app = caimira.apps.CO2Application()\n", + "app = apps.CO2Application()\n", "app.widget" ] } diff --git a/caimira/apps/expert_co2/static/images/header_image.png b/cern_caimira/src/cern_caimira/apps/expert_apps/expert_co2/static/images/header_image.png similarity index 100% rename from caimira/apps/expert_co2/static/images/header_image.png rename to cern_caimira/src/cern_caimira/apps/expert_apps/expert_co2/static/images/header_image.png diff --git a/caimira/state.py b/cern_caimira/src/cern_caimira/apps/expert_apps/state.py similarity index 100% rename from caimira/state.py rename to cern_caimira/src/cern_caimira/apps/expert_apps/state.py diff --git a/caimira/apps/static/css/style.css b/cern_caimira/src/cern_caimira/apps/static/css/style.css similarity index 100% rename from caimira/apps/static/css/style.css rename to cern_caimira/src/cern_caimira/apps/static/css/style.css diff --git a/caimira/apps/static/icons/calculator.svg b/cern_caimira/src/cern_caimira/apps/static/icons/calculator.svg similarity index 100% rename from caimira/apps/static/icons/calculator.svg rename to cern_caimira/src/cern_caimira/apps/static/icons/calculator.svg diff --git a/caimira/apps/static/icons/expert.svg b/cern_caimira/src/cern_caimira/apps/static/icons/expert.svg similarity index 100% rename from caimira/apps/static/icons/expert.svg rename to cern_caimira/src/cern_caimira/apps/static/icons/expert.svg diff --git a/caimira/apps/static/icons/favicon.ico b/cern_caimira/src/cern_caimira/apps/static/icons/favicon.ico similarity index 100% rename from caimira/apps/static/icons/favicon.ico rename to cern_caimira/src/cern_caimira/apps/static/icons/favicon.ico diff --git a/caimira/apps/static/images/CAiMIRA_1_Vs3_Colour.jpg b/cern_caimira/src/cern_caimira/apps/static/images/CAiMIRA_1_Vs3_Colour.jpg similarity index 100% rename from caimira/apps/static/images/CAiMIRA_1_Vs3_Colour.jpg rename to cern_caimira/src/cern_caimira/apps/static/images/CAiMIRA_1_Vs3_Colour.jpg diff --git a/caimira/apps/static/images/caimira_full_logo.png b/cern_caimira/src/cern_caimira/apps/static/images/caimira_full_logo.png similarity index 100% rename from caimira/apps/static/images/caimira_full_logo.png rename to cern_caimira/src/cern_caimira/apps/static/images/caimira_full_logo.png diff --git a/caimira/apps/static/images/caimira_full_text.png b/cern_caimira/src/cern_caimira/apps/static/images/caimira_full_text.png similarity index 100% rename from caimira/apps/static/images/caimira_full_text.png rename to cern_caimira/src/cern_caimira/apps/static/images/caimira_full_text.png diff --git a/caimira/apps/static/images/caimira_logo.200x200.png b/cern_caimira/src/cern_caimira/apps/static/images/caimira_logo.200x200.png similarity index 100% rename from caimira/apps/static/images/caimira_logo.200x200.png rename to cern_caimira/src/cern_caimira/apps/static/images/caimira_logo.200x200.png diff --git a/caimira/apps/static/images/caimira_logo_white_text.png b/cern_caimira/src/cern_caimira/apps/static/images/caimira_logo_white_text.png similarity index 100% rename from caimira/apps/static/images/caimira_logo_white_text.png rename to cern_caimira/src/cern_caimira/apps/static/images/caimira_logo_white_text.png diff --git a/caimira/apps/static/images/long_range_anim.png b/cern_caimira/src/cern_caimira/apps/static/images/long_range_anim.png similarity index 100% rename from caimira/apps/static/images/long_range_anim.png rename to cern_caimira/src/cern_caimira/apps/static/images/long_range_anim.png diff --git a/caimira/apps/static/images/masks/cloth.png b/cern_caimira/src/cern_caimira/apps/static/images/masks/cloth.png similarity index 100% rename from caimira/apps/static/images/masks/cloth.png rename to cern_caimira/src/cern_caimira/apps/static/images/masks/cloth.png diff --git a/caimira/apps/static/images/masks/ffp2.png b/cern_caimira/src/cern_caimira/apps/static/images/masks/ffp2.png similarity index 100% rename from caimira/apps/static/images/masks/ffp2.png rename to cern_caimira/src/cern_caimira/apps/static/images/masks/ffp2.png diff --git a/caimira/apps/static/images/masks/t1.png b/cern_caimira/src/cern_caimira/apps/static/images/masks/t1.png similarity index 100% rename from caimira/apps/static/images/masks/t1.png rename to cern_caimira/src/cern_caimira/apps/static/images/masks/t1.png diff --git a/caimira/apps/static/images/nat_vent_dimensions.png b/cern_caimira/src/cern_caimira/apps/static/images/nat_vent_dimensions.png similarity index 100% rename from caimira/apps/static/images/nat_vent_dimensions.png rename to cern_caimira/src/cern_caimira/apps/static/images/nat_vent_dimensions.png diff --git a/caimira/apps/static/images/short_range_anim.png b/cern_caimira/src/cern_caimira/apps/static/images/short_range_anim.png similarity index 100% rename from caimira/apps/static/images/short_range_anim.png rename to cern_caimira/src/cern_caimira/apps/static/images/short_range_anim.png diff --git a/caimira/apps/static/js/ScrollMagic.min.js b/cern_caimira/src/cern_caimira/apps/static/js/ScrollMagic.min.js similarity index 100% rename from caimira/apps/static/js/ScrollMagic.min.js rename to cern_caimira/src/cern_caimira/apps/static/js/ScrollMagic.min.js diff --git a/caimira/apps/static/js/jquery.colorbox-min.js b/cern_caimira/src/cern_caimira/apps/static/js/jquery.colorbox-min.js similarity index 100% rename from caimira/apps/static/js/jquery.colorbox-min.js rename to cern_caimira/src/cern_caimira/apps/static/js/jquery.colorbox-min.js diff --git a/caimira/apps/static/js/js_packaged_for_theme.js b/cern_caimira/src/cern_caimira/apps/static/js/js_packaged_for_theme.js similarity index 100% rename from caimira/apps/static/js/js_packaged_for_theme.js rename to cern_caimira/src/cern_caimira/apps/static/js/js_packaged_for_theme.js diff --git a/caimira/apps/static/js/usage-tracking.js b/cern_caimira/src/cern_caimira/apps/static/js/usage-tracking.js similarity index 100% rename from caimira/apps/static/js/usage-tracking.js rename to cern_caimira/src/cern_caimira/apps/static/js/usage-tracking.js diff --git a/caimira/apps/templates/about.html.j2 b/cern_caimira/src/cern_caimira/apps/templates/about.html.j2 similarity index 100% rename from caimira/apps/templates/about.html.j2 rename to cern_caimira/src/cern_caimira/apps/templates/about.html.j2 diff --git a/caimira/apps/templates/base/calculator.form.html.j2 b/cern_caimira/src/cern_caimira/apps/templates/base/calculator.form.html.j2 similarity index 100% rename from caimira/apps/templates/base/calculator.form.html.j2 rename to cern_caimira/src/cern_caimira/apps/templates/base/calculator.form.html.j2 diff --git a/caimira/apps/templates/base/calculator.report.html.j2 b/cern_caimira/src/cern_caimira/apps/templates/base/calculator.report.html.j2 similarity index 99% rename from caimira/apps/templates/base/calculator.report.html.j2 rename to cern_caimira/src/cern_caimira/apps/templates/base/calculator.report.html.j2 index 041cf3ef..b47aa63a 100644 --- a/caimira/apps/templates/base/calculator.report.html.j2 +++ b/cern_caimira/src/cern_caimira/apps/templates/base/calculator.report.html.j2 @@ -615,11 +615,11 @@ diff --git a/caimira/apps/templates/base/index.html.j2 b/cern_caimira/src/cern_caimira/apps/templates/base/index.html.j2 similarity index 100% rename from caimira/apps/templates/base/index.html.j2 rename to cern_caimira/src/cern_caimira/apps/templates/base/index.html.j2 diff --git a/caimira/apps/templates/base/layout.html.j2 b/cern_caimira/src/cern_caimira/apps/templates/base/layout.html.j2 similarity index 96% rename from caimira/apps/templates/base/layout.html.j2 rename to cern_caimira/src/cern_caimira/apps/templates/base/layout.html.j2 index e64664bd..9e743f72 100644 --- a/caimira/apps/templates/base/layout.html.j2 +++ b/cern_caimira/src/cern_caimira/apps/templates/base/layout.html.j2 @@ -45,8 +45,8 @@ diff --git a/caimira/apps/templates/base/userguide.html.j2 b/cern_caimira/src/cern_caimira/apps/templates/base/userguide.html.j2 similarity index 100% rename from caimira/apps/templates/base/userguide.html.j2 rename to cern_caimira/src/cern_caimira/apps/templates/base/userguide.html.j2 diff --git a/caimira/apps/templates/calculator.form.html.j2 b/cern_caimira/src/cern_caimira/apps/templates/calculator.form.html.j2 similarity index 100% rename from caimira/apps/templates/calculator.form.html.j2 rename to cern_caimira/src/cern_caimira/apps/templates/calculator.form.html.j2 diff --git a/caimira/apps/templates/calculator.report.html.j2 b/cern_caimira/src/cern_caimira/apps/templates/calculator.report.html.j2 similarity index 100% rename from caimira/apps/templates/calculator.report.html.j2 rename to cern_caimira/src/cern_caimira/apps/templates/calculator.report.html.j2 diff --git a/caimira/apps/templates/cern/calculator.form.html.j2 b/cern_caimira/src/cern_caimira/apps/templates/cern/calculator.form.html.j2 similarity index 100% rename from caimira/apps/templates/cern/calculator.form.html.j2 rename to cern_caimira/src/cern_caimira/apps/templates/cern/calculator.form.html.j2 diff --git a/caimira/apps/templates/cern/calculator.report.html.j2 b/cern_caimira/src/cern_caimira/apps/templates/cern/calculator.report.html.j2 similarity index 100% rename from caimira/apps/templates/cern/calculator.report.html.j2 rename to cern_caimira/src/cern_caimira/apps/templates/cern/calculator.report.html.j2 diff --git a/caimira/apps/templates/cern/index.html.j2 b/cern_caimira/src/cern_caimira/apps/templates/cern/index.html.j2 similarity index 100% rename from caimira/apps/templates/cern/index.html.j2 rename to cern_caimira/src/cern_caimira/apps/templates/cern/index.html.j2 diff --git a/caimira/apps/templates/cern/layout.html.j2 b/cern_caimira/src/cern_caimira/apps/templates/cern/layout.html.j2 similarity index 100% rename from caimira/apps/templates/cern/layout.html.j2 rename to cern_caimira/src/cern_caimira/apps/templates/cern/layout.html.j2 diff --git a/caimira/apps/templates/cern/userguide.html.j2 b/cern_caimira/src/cern_caimira/apps/templates/cern/userguide.html.j2 similarity index 100% rename from caimira/apps/templates/cern/userguide.html.j2 rename to cern_caimira/src/cern_caimira/apps/templates/cern/userguide.html.j2 diff --git a/caimira/apps/templates/common_text.md.j2 b/cern_caimira/src/cern_caimira/apps/templates/common_text.md.j2 similarity index 100% rename from caimira/apps/templates/common_text.md.j2 rename to cern_caimira/src/cern_caimira/apps/templates/common_text.md.j2 diff --git a/caimira/apps/templates/error.html.j2 b/cern_caimira/src/cern_caimira/apps/templates/error.html.j2 similarity index 100% rename from caimira/apps/templates/error.html.j2 rename to cern_caimira/src/cern_caimira/apps/templates/error.html.j2 diff --git a/caimira/apps/templates/expert-app.html.j2 b/cern_caimira/src/cern_caimira/apps/templates/expert-app.html.j2 similarity index 97% rename from caimira/apps/templates/expert-app.html.j2 rename to cern_caimira/src/cern_caimira/apps/templates/expert-app.html.j2 index 5e18a638..93c81e47 100644 --- a/caimira/apps/templates/expert-app.html.j2 +++ b/cern_caimira/src/cern_caimira/apps/templates/expert-app.html.j2 @@ -15,4 +15,4 @@ For any query, please let us know by sending an email to None: @@ -18,17 +19,24 @@ def test_generate_report(baseline_form) -> None: # generate a report for it. Because this is what happens in the caimira # calculator, we confirm that the generation happens within a reasonable # time threshold. - time_limit: float = float(os.environ.get("CAIMIRA_TESTS_CALCULATOR_TIMEOUT", 10.)) + time_limit: float = float(os.environ.get( + "CAIMIRA_TESTS_CALCULATOR_TIMEOUT", 10.)) start = time.perf_counter() generator: ReportGenerator = make_app().settings['report_generator'] - report = generator.build_report("", baseline_form, partial( + + model = generate_model(baseline_form) + report_data = generate_report_results(baseline_form, model) + + report = generator.build_report("", baseline_form, model, report_data, partial( concurrent.futures.ThreadPoolExecutor, 1, )) + end = time.perf_counter() total = end-start - print(f"Time limit: {time_limit} | Time taken: {end} - {start} = {total} < {time_limit}") + print( + f"Time limit: {time_limit} | Time taken: {end} - {start} = {total} < {time_limit}") assert report != "" assert end - start < time_limit @@ -54,8 +62,10 @@ def test_fill_big_gaps(): def test_fill_big_gaps__float_tolerance(): # Ensure that there is some float tolerance to the gap size check. - assert rep_gen.fill_big_gaps([0, 2 + 1e-15, 4], gap_size=2) == [0, 2 + 1e-15, 4] - assert rep_gen.fill_big_gaps([0, 2 + 1e-14, 4], gap_size=2) == [0, 2, 2 + 1e-14, 4] + assert rep_gen.fill_big_gaps( + [0, 2 + 1e-15, 4], gap_size=2) == [0, 2 + 1e-15, 4] + assert rep_gen.fill_big_gaps( + [0, 2 + 1e-14, 4], gap_size=2) == [0, 2, 2 + 1e-14, 4] def test_non_temp_transition_times(baseline_exposure_model): @@ -65,7 +75,8 @@ def test_non_temp_transition_times(baseline_exposure_model): def test_interesting_times_many(baseline_exposure_model): - result = rep_gen.interesting_times(baseline_exposure_model, approx_n_pts=100) + result = rep_gen.interesting_times( + baseline_exposure_model, approx_n_pts=100) assert 100 <= len(result) <= 120 assert np.abs(np.diff(result)).max() < 8.1/100. @@ -73,7 +84,8 @@ def test_interesting_times_many(baseline_exposure_model): def test_interesting_times_small(baseline_exposure_model): expected = [0.0, 0.8, 1.6, 2.4, 3.2, 4.0, 4.8, 5.0, 5.8, 6.6, 7.4, 8.0] # Ask for more data than there is in the transition times. - result = rep_gen.interesting_times(baseline_exposure_model, approx_n_pts=10) + result = rep_gen.interesting_times( + baseline_exposure_model, approx_n_pts=10) np.testing.assert_allclose(result, expected, atol=1e-04) @@ -81,12 +93,14 @@ def test_interesting_times_small(baseline_exposure_model): def test_interesting_times_w_temp(exposure_model_w_outside_temp_changes): # Ensure that the state change times are returned (minus the temperature changes) by # requesting n_points=1. - result = rep_gen.interesting_times(exposure_model_w_outside_temp_changes, approx_n_pts=1) + result = rep_gen.interesting_times( + exposure_model_w_outside_temp_changes, approx_n_pts=1) expected = [0., 1.8, 2.2, 4., 4.4, 5., 6.2, 6.6, 8.] np.testing.assert_allclose(result, expected) # Now request more than the state-change times. - result = rep_gen.interesting_times(exposure_model_w_outside_temp_changes, approx_n_pts=20) + result = rep_gen.interesting_times( + exposure_model_w_outside_temp_changes, approx_n_pts=20) expected = [ 0., 0.4, 0.8, 1.2, 1.6, 1.8, 2.2, 2.6, 3., 3.4, 3.8, 4., 4.4, 4.8, 5., 5.4, 5.8, 6.2, 6.6, 7., 7.4, 7.8, 8. @@ -94,7 +108,7 @@ def test_interesting_times_w_temp(exposure_model_w_outside_temp_changes): np.testing.assert_allclose(result, expected) -def test_expected_new_cases(baseline_form_with_sr: VirusFormData): +def test_expected_new_cases(baseline_form_with_sr: VirusFormData): model = baseline_form_with_sr.build_model() executor_factory = partial( @@ -102,12 +116,12 @@ def test_expected_new_cases(baseline_form_with_sr: VirusFormData): ) # Short- and Long-range contributions - report_data = calculate_report_data(baseline_form_with_sr, model, executor_factory) + report_data = rep_gen.calculate_report_data(baseline_form_with_sr, model, executor_factory) sr_lr_expected_new_cases = report_data['expected_new_cases'] sr_lr_prob_inf = report_data['prob_inf']/100 # Long-range contributions alone - scenario_sample_times = interesting_times(model) + scenario_sample_times = rep_gen.interesting_times(model) alternative_scenarios = manufacture_alternative_scenarios(baseline_form_with_sr) alternative_statistics = comparison_report( baseline_form_with_sr, report_data, alternative_scenarios, scenario_sample_times, executor_factory=executor_factory, @@ -115,3 +129,4 @@ def test_expected_new_cases(baseline_form_with_sr: VirusFormData): lr_expected_new_cases = alternative_statistics['stats']['Base scenario without short-range interactions']['expected_new_cases'] np.testing.assert_almost_equal(sr_lr_expected_new_cases, lr_expected_new_cases + sr_lr_prob_inf * baseline_form_with_sr.short_range_occupants, 2) + \ No newline at end of file diff --git a/caimira/tests/test_state.py b/cern_caimira/tests/test_state.py similarity index 99% rename from caimira/tests/test_state.py rename to cern_caimira/tests/test_state.py index ac0341fe..2ca854b3 100644 --- a/caimira/tests/test_state.py +++ b/cern_caimira/tests/test_state.py @@ -4,7 +4,7 @@ import pytest -from caimira import state +from cern_caimira.apps.expert_apps.expert import state @dataclass diff --git a/caimira/tests/apps/calculator/test_webapp.py b/cern_caimira/tests/test_webapp.py similarity index 82% rename from caimira/tests/apps/calculator/test_webapp.py rename to cern_caimira/tests/test_webapp.py index a9fd52f3..a5e0944d 100644 --- a/caimira/tests/apps/calculator/test_webapp.py +++ b/cern_caimira/tests/test_webapp.py @@ -4,15 +4,15 @@ import pytest import tornado.testing -import caimira.apps.calculator -from caimira.apps.calculator.report_generator import generate_permalink +import cern_caimira.apps.calculator +from cern_caimira.apps.calculator.report import generate_permalink _TIMEOUT = float(os.environ.get("CAIMIRA_TESTS_CALCULATOR_TIMEOUT", 10.)) @pytest.fixture def app(): - return caimira.apps.calculator.make_app() + return cern_caimira.apps.calculator.make_app() async def test_homepage(http_server_client): @@ -36,7 +36,7 @@ async def test_404(http_server_client): class TestBasicApp(tornado.testing.AsyncHTTPTestCase): def get_app(self): - return caimira.apps.calculator.make_app() + return cern_caimira.apps.calculator.make_app() @tornado.testing.gen_test(timeout=_TIMEOUT) def test_report(self): @@ -64,8 +64,8 @@ def end_time(resp): class TestCernApp(tornado.testing.AsyncHTTPTestCase): def get_app(self): - cern_theme = Path(caimira.apps.calculator.__file__).parent.parent / 'themes' / 'cern' - return caimira.apps.calculator.make_app(theme_dir=cern_theme) + cern_theme = Path(cern_caimira.apps.calculator.__file__).parent.parent / 'themes' / 'cern' + return cern_caimira.apps.calculator.make_app(theme_dir=cern_theme) @tornado.testing.gen_test(timeout=_TIMEOUT) def test_report(self): @@ -76,7 +76,7 @@ def test_report(self): class TestOpenApp(tornado.testing.AsyncHTTPTestCase): def get_app(self): - return caimira.apps.calculator.make_app(calculator_prefix="/mycalc") + return cern_caimira.apps.calculator.make_app(calculator_prefix="/mycalc") @tornado.testing.gen_test(timeout=_TIMEOUT) def test_report(self): @@ -121,10 +121,10 @@ async def test_invalid_compressed_url(http_server_client, baseline_form): class TestError500(tornado.testing.AsyncHTTPTestCase): def get_app(self): - class ProcessingErrorPage(caimira.apps.calculator.BaseRequestHandler): + class ProcessingErrorPage(cern_caimira.apps.calculator.BaseRequestHandler): def get(self): raise ValueError('some unexpected error') - app = caimira.apps.calculator.make_app() + app = cern_caimira.apps.calculator.make_app() page = [ (r'/', ProcessingErrorPage), ] @@ -138,11 +138,11 @@ def test_500(self): class TestCERNGenericPage(tornado.testing.AsyncHTTPTestCase): def get_app(self): - cern_theme = Path(caimira.apps.calculator.__file__).parent.parent / 'themes' / 'cern' - app = caimira.apps.calculator.make_app(theme_dir=cern_theme) + cern_theme = Path(cern_caimira.apps.calculator.__file__).parent.parent / 'themes' / 'cern' + app = cern_caimira.apps.calculator.make_app(theme_dir=cern_theme) pages = [ - (r'/calculator/user-guide', caimira.apps.calculator.GenericExtraPage, {'active_page': 'calculator/user-guide', 'filename': 'userguide.html.j2'}), - (r'/about', caimira.apps.calculator.GenericExtraPage, {'active_page': 'about', 'filename': 'about.html.j2'}), + (r'/calculator/user-guide', cern_caimira.apps.calculator.GenericExtraPage, {'active_page': 'calculator/user-guide', 'filename': 'userguide.html.j2'}), + (r'/about', cern_caimira.apps.calculator.GenericExtraPage, {'active_page': 'about', 'filename': 'about.html.j2'}), ] return tornado.web.Application(pages, **app.settings) diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 5ed8feac..00000000 --- a/setup.cfg +++ /dev/null @@ -1,42 +0,0 @@ -[tool:pytest] -addopts = --mypy - -[mypy] -no_warn_no_return = True -exclude = caimira/profiler.py - -[mypy-loky.*] -ignore_missing_imports = True - -[mypy-ipympl.*] -ignore_missing_imports = True - -[mypy-ipywidgets.*] -ignore_missing_imports = True - -[mypy-matplotlib.*] -ignore_missing_imports = True - -[mypy-mistune.*] -ignore_missing_imports = True - -[mypy-qrcode.*] -ignore_missing_imports = True - -[mypy-scipy.*] -ignore_missing_imports = True - -[mypy-timezonefinder.*] -ignore_missing_imports = True - -[mypy-pandas.*] -ignore_missing_imports = True - -[mypy-pstats.*] -follow_imports = skip - -[mypy-tabulate.*] -ignore_missing_imports = True - -[mypy-ruptures.*] -ignore_missing_imports = True diff --git a/setup.py b/setup.py deleted file mode 100644 index 42954280..00000000 --- a/setup.py +++ /dev/null @@ -1,102 +0,0 @@ -# This module is part of CAiMIRA. Please see the repository at -# https://gitlab.cern.ch/caimira/caimira for details of the license and terms of use. -""" -setup.py for CAiMIRA. - -For reference see -https://packaging.python.org/guides/distributing-packages-using-setuptools/ - -""" -from pathlib import Path -from setuptools import setup, find_packages - - -HERE = Path(__file__).parent.absolute() -with (HERE / 'README.md').open('rt') as fh: - LONG_DESCRIPTION = fh.read().strip() - - -REQUIREMENTS: dict = { - 'core': [ - 'dataclasses; python_version < "3.7"', - 'ipykernel', - 'ipympl >= 0.9.0', - 'ipywidgets < 8.0', - 'Jinja2', - 'loky', - 'matplotlib', - 'memoization', - 'mistune', - 'numpy', - 'pandas', - 'psutil', - 'pyinstrument', - 'pyjwt', - 'python-dateutil', - 'retry', - 'ruptures', - 'scipy', - 'scikit-learn', - 'timezonefinder', - 'tornado', - 'types-retry', - ], - 'app': [], - 'test': [ - 'pytest < 8.2', - 'pytest-mypy >= 0.10.3', - 'mypy >= 1.0.0', - 'pytest-tornasync', - 'numpy-stubs @ git+https://github.com/numpy/numpy-stubs.git', - 'types-dataclasses', - 'types-python-dateutil', - 'types-requests', - ], - 'dev': [ - ], - 'doc': [ - 'sphinx', - 'sphinx_rtd_theme', - ], -} - - -setup( - name='CAiMIRA', - version="1.0.0", - - maintainer='Andre Henriques', - maintainer_email='andre.henriques@cern.ch', - description='COVID Airborne Risk Assessment', - long_description=LONG_DESCRIPTION, - long_description_content_type='text/markdown', - url='cern.ch/caimira', - - packages=find_packages(), - python_requires='~=3.9', - classifiers=[ - "Programming Language :: Python :: 3", - "Operating System :: OS Independent", - "License :: OSI Approved :: Apache Software License", # Apache 2.0 - ], - - install_requires=REQUIREMENTS['core'], - extras_require={ - **REQUIREMENTS, - # The 'dev' extra is the union of 'test' and 'doc', with an option - # to have explicit development dependencies listed. - 'dev': [req - for extra in ['dev', 'test', 'doc'] - for req in REQUIREMENTS.get(extra, [])], - # The 'all' extra is the union of all requirements. - 'all': [req for reqs in REQUIREMENTS.values() for req in reqs], - }, - package_data={'caimira': [ - 'apps/*/*', - 'apps/*/*/*', - 'apps/*/*/*/*', - 'apps/*/*/*/*/*', - 'data/*.json', - 'data/*.txt', - ]}, -)