add PyInstrument and cProfile profilers

README.md

@@ -55,7 +55,7 @@ https://doi.org/10.1016/j.buildenv.2022.109166
 
 ***Open Source Acknowledgments***
 
-For a detailed list of the open-source dependencies used in this project along with their respective licenses, please refer to [License Information](open-source-licences/README.md). This includes both the core dependencies specified in the project's requirements and their transitive dependencies. 
+For a detailed list of the open-source dependencies used in this project along with their respective licenses, please refer to [License Information](open-source-licences/README.md). This includes both the core dependencies specified in the project's requirements and their transitive dependencies.
 
 The information also features a distribution diagram of licenses and a brief description of each of them.
 
@@ -147,14 +147,25 @@ voila caimira/apps/expert/caimira.ipynb --port=8080
 
 Then visit http://localhost:8080.
 
-
 ### Running the tests
 
 ```
 pip install -e .[test]
 pytest ./caimira
 ```
 
+### Running the profiler
+
+The profiler is enabled in one of the following cases:
+- the calculator app is running in `debug` mode
+- the environment variable `CAIMIRA_PROFILER_ENABLED` is set to 1
+
+When visiting http://localhost:8080/profiler, you can start a new session and choose between [PyInstrument](https://github.com/joerick/pyinstrument) or [cProfile](https://docs.python.org/3/library/profile.html#module-cProfile). The app includes two different profilers, mainly because they can give different information.
+
+Keep the profiler page open. Then, in another window, navigate to any page in CAiMIRA, for example generate a new report. Refresh the profiler page, and click on the `Report` link to see the profiler output.
+
+The sessions are stored in a local file in the `/tmp` folder. To share it across multiple web nodes, a shared storage should be added to all web nodes. The folder can be customized via the environment variable `CAIMIRA_PROFILER_CACHE_DIR`.
+
 ### Building the whole environment for local development
 
 **Simulate the docker build that takes place on openshift with:**

caimira/apps/calculator/__init__.py

@@ -25,6 +25,7 @@
 from tornado.web import Application, RequestHandler, StaticFileHandler
 from tornado.httpclient import AsyncHTTPClient, HTTPRequest
 import tornado.log
+from caimira.profiler import CaimiraProfiler, Profilers
 from caimira.store.data_registry import DataRegistry
 
 from caimira.store.data_service import DataService
@@ -43,7 +44,53 @@
 # increase the overall CAiMIRA version (found at ``caimira.__version__``).
 __version__ = "4.14.3"
 
-LOG = logging.getLogger("APP")
+LOG = logging.getLogger("Calculator")
+
+
+class ProfilerPage(RequestHandler):
+    """Render the profiler page.
+
+    This class does not inherit from BaseRequestHandler to avoid profiling the
+    profiler page itself.
+    """
+    def get(self) -> None:
+        profiler = CaimiraProfiler()
+
+        template_environment = self.settings["template_environment"]
+        template = template_environment.get_template("profiler.html.j2")
+        report = template.render(
+            user=AnonymousUser(),
+            active_page="Profiler",
+            xsrf_form_html=self.xsrf_form_html(),
+            is_active=profiler.is_active,
+            sessions=profiler.sessions,
+        )
+        self.finish(report)
+
+    def post(self) -> None:
+        profiler = CaimiraProfiler()
+
+        if self.get_argument("start", None) is not None:
+            name = self.get_argument("name", None)
+            profiler_type = Profilers.from_str(self.get_argument("profiler_type", ""))
+            profiler.start_session(name, profiler_type)
+        elif self.get_argument("stop", None) is not None:
+            profiler.stop_session()
+        elif self.get_argument("clear", None) is not None:
+            profiler.clear_sessions()
+
+        self.redirect(CaimiraProfiler.ROOT_URL)
+
+
+class ProfilerReport(RequestHandler):
+    """Render the profiler HTML report."""
+    def get(self, report_id) -> None:
+        profiler = CaimiraProfiler()
+        _, report_html = profiler.get_report(report_id)
+        if report_html:
+            self.finish(report_html)
+        else:
+            self.send_error(404)
 
 
 class BaseRequestHandler(RequestHandler):
@@ -64,6 +111,22 @@ async def prepare(self):
         else:
             self.current_user = AnonymousUser()
 
+        profiler = CaimiraProfiler()
+        if profiler.is_active and not self.request.path.startswith(CaimiraProfiler.ROOT_URL):
+            self._request_profiler = profiler.start_profiler()
+
+    def on_finish(self) -> None:
+        """Called at the end of the request."""
+        profiler = CaimiraProfiler()
+        if profiler.is_active and self._request_profiler:
+            profiler.stop_profiler(
+                profiler=self._request_profiler,
+                uri=self.request.uri or "",
+                path=self.request.path,
+                query=self.request.query,
+                method=self.request.method,
+            )
+
     def write_error(self, status_code: int, **kwargs) -> None:
         template = self.settings["template_environment"].get_template(
             "error.html.j2")
@@ -199,6 +262,7 @@ async def get(self) -> None:
         base_url = self.request.protocol + "://" + self.request.host
         report_generator: ReportGenerator = self.settings['report_generator']
         executor = loky.get_reusable_executor(max_workers=self.settings['handler_worker_pool_size'])
+
         report_task = executor.submit(
             report_generator.build_report, base_url, form,
             executor_factory=functools.partial(
@@ -452,6 +516,13 @@ def make_app(
             'filename': 'userguide.html.j2'}),
     ]
 
+    profiler = os.environ.get('CAIMIRA_PROFILER_ENABLED', 0)
+    if debug or profiler:
+        urls += [
+            (get_root_url(CaimiraProfiler.ROOT_URL), ProfilerPage),
+            (get_root_url(r'{root_url}/(.*)'.format(root_url=CaimiraProfiler.ROOT_URL)), ProfilerReport),
+        ]
+
     interface: str = os.environ.get('CAIMIRA_THEME', '<undefined>')
     if interface != '<undefined>' and (interface != '<undefined>' and 'cern' not in interface): urls = list(filter(lambda i: i in base_urls, urls))
 

caimira/apps/calculator/__main__.py

@@ -58,7 +58,12 @@ def main():
         theme_dir = Path(theme_dir).absolute()
         assert theme_dir.exists()
 
-    app = make_app(debug=debug, APPLICATION_ROOT=args.app_root, calculator_prefix=args.prefix, theme_dir=theme_dir)
+    app = make_app(
+        debug=debug,
+        APPLICATION_ROOT=args.app_root,
+        calculator_prefix=args.prefix,
+        theme_dir=theme_dir
+    )
     app.listen(args.port)
     IOLoop.current().start()
 

caimira/apps/calculator/report_generator.py

@@ -14,6 +14,7 @@
 
 from caimira import models
 from caimira.apps.calculator import markdown_tools
+from caimira.profiler import profile
 from caimira.store.data_registry import DataRegistry
 from ... import monte_carlo as mc
 from .model_generator import VirusFormData
@@ -114,6 +115,7 @@ def concentrations_with_sr_breathing(form: VirusFormData, model: models.Exposure
     return lower_concentrations
 
 
+@profile
 def calculate_report_data(form: VirusFormData, model: models.ExposureModel) -> typing.Dict[str, typing.Any]:
     times = interesting_times(model)
     short_range_intervals = [interaction.presence.boundaries()[0] for interaction in model.short_range]

caimira/apps/templates/profiler.html.j2

@@ -0,0 +1,62 @@
+{% extends "layout.html.j2" %}
+
+{% block main %}
+<div class="container mt-5 mb-5">
+    <form method="POST">
+
+        <h1>Profiler</h1>
+        {% if is_active %}
+            <div class="form-group mt-3">
+                The profiler is running.
+            </div>
+            <button type="submit" class="btn btn-primary" name="stop">Stop current session</button>
+        {% else %}
+            The profiler is not running.
+            <div class="form-group mt-3">
+                <input type="text" class="form-control" name="name" placeholder="Enter a name for the new session">
+
+                <label class="mt-3">Choose the profiler:</label>
+
+                <div class="btn-group btn-group-toggle" data-toggle="buttons">
+                    <label class="btn btn-secondary active">
+                        <input type="radio" name="profiler_type" id="pyinstrument" value="pyinstrument" autocomplete="off" checked> PyInstrument
+                    </label>
+                    <label class="btn btn-secondary">
+                        <input type="radio" name="profiler_type" id="cprofiler" value="cprofiler" autocomplete="off"> CProfiler
+                    </label>
+                </div>
+            </div>
+            <button type="submit" class="btn btn-primary mt-3" name="start">Start new session</button>
+        {% endif %}
+
+        {{ xsrf_form_html }}
+
+        <h3 class="mt-5">Sessions</h3>
+        {% if sessions %}
+            <ol>
+            {% for name, reports in sessions.items() %}
+                <li>Name: {{ name }}</li>
+                <ul>
+                    {% if reports %}
+                        {% for report in reports %}
+                            <li>{{ report["ts"] }} - {{ report["method"] }} {{ report["uri"] }} - <a href="/profiler/{{ report["report_id"] }}" target="_blank">Report</a></li>
+                        {% endfor %}
+                    {% else %}
+                    <i>No reports yet!</i>
+                    {% endif %}
+                </ul>
+            {% endfor %}
+            </ol>
+
+            {% if not is_active %}
+            <div class="mt-3">
+                <button type="submit" class="btn btn-danger btn-sm" name="clear">Clear all sessions</button>
+            </div>
+            {% endif %}
+
+        {% else %}
+        <i>No sessions yet!</i>
+        {% endif %}
+    </form>
+</div>
+{% endblock main %}