Clerk in Babashka

bb-runtime.edn

@@ -0,0 +1,26 @@
+{:min-bb-version "1.0.164"
+ :paths ["src" "notebooks" "resources"]
+ :deps {hiccup/hiccup {:mvn/version "2.0.0-alpha2"}
+        org.babashka/cli {:mvn/version "0.5.40"}
+        io.github.nextjournal/markdown {:mvn/version "0.4.126"}
+        io.github.nextjournal/clerk-slideshow {:git/sha "562f634494a1e1a9149ed78d5d39fd9486cc00ba"}}
+ :tasks
+ {dev
+  {:requires ([babashka.fs :as fs]
+              [babashka.nrepl.server :as srv]
+              [nextjournal.clerk :as clerk]
+              [babashka.cli :as cli])
+   :task (do (srv/start-server! {:host "localhost" :port 1339})
+             (spit ".nrepl-port" "1339")
+             (nextjournal.clerk/serve! (cli/parse-opts *command-line-args*))
+             (-> (Runtime/getRuntime)
+               (.addShutdownHook (Thread. (fn []
+                                            (nextjournal.clerk/halt!)
+                                            (fs/delete ".nrepl-port")))))
+             (deref (promise)))}
+
+  build
+  {:requires ([nextjournal.clerk :as clerk]
+              [babashka.cli :as cli])
+   :task (let [spec (-> (resolve 'nextjournal.clerk/build!) meta :org.babashka/cli)]
+           (clerk/build! (cli/parse-opts *command-line-args* spec)))}}}

resources/viewer-js-hash

@@ -1 +1 @@
-46k5SeDmWgR6seZFSVzwdiCJ7BEi
+jTDHst3WuaZZ65Kbk7vW4w7315u

src/nextjournal/beholder.bb

@@ -0,0 +1,5 @@
+(ns nextjournal.beholder
+  "Babashka runtime no-op stubs")
+
+(defn watch [cb & args] nil)
+(defn stop [w] nil)

src/nextjournal/clerk/analyzer.bb

@@ -0,0 +1,6 @@
+(ns nextjournal.clerk.analyzer
+  "Babashka runtime no-op stubs")
+
+;; TODO: consider using this in eval
+(defn valuehash [val] "valuehash")
+(defn ->hash-str [val] (valuehash val))

src/nextjournal/clerk/builder.clj

@@ -2,9 +2,7 @@
   "Clerk's Static App Builder."
   (:require [babashka.fs :as fs]
             [clojure.java.browse :as browse]
-            [clojure.set :as set]
             [clojure.string :as str]
-            [nextjournal.clerk.analyzer :as analyzer]
             [nextjournal.clerk.builder-ui :as builder-ui]
             [nextjournal.clerk.eval :as eval]
             [nextjournal.clerk.parser :as parser]
@@ -197,7 +195,7 @@
         {state :result duration :time-ms} (eval/time-ms (mapv (comp (partial parser/parse-file {:doc? true}) :file) state))
         _ (report-fn {:stage :parsed :state state :duration duration})
         {state :result duration :time-ms} (eval/time-ms (reduce (fn [state doc]
-                                                                  (try (conj state (-> doc analyzer/build-graph analyzer/hash))
+                                                                  (try (conj state (eval/analyze-doc doc))
                                                                        (catch Exception e
                                                                          (reduced {:error e}))))
                                                                 []

src/nextjournal/clerk/eval.bb

@@ -0,0 +1,168 @@
+(ns nextjournal.clerk.eval
+  "Clerk's incremental evaluation (Babashka Edition) with in-memory caching layer."
+  (:refer-clojure :exclude [read-string])
+  (:require [clojure.string :as str]
+            [edamame.core :as edamame]
+            [nextjournal.clerk.config :as config]
+            [nextjournal.clerk.parser :as parser]
+            [nextjournal.clerk.viewer :as v]))
+
+(defn wrapped-with-metadata [value hash]
+  (cond-> {:nextjournal/value value}
+    ;; TODO: maybe fix hash for blob serving
+    hash (assoc :nextjournal/blob-id (cond-> hash (not (string? hash)) str #_ multihash/base58))))
+
+#_(wrap-with-blob-id :test "foo")
+
+(defn elapsed-ms [from]
+  (/ (double (- (. System (nanoTime)) from)) 1000000.0))
+
+(defmacro time-ms
+  "Pure version of `clojure.core/time`. Returns a map with `:result` and `:time-ms` keys."
+  [expr]
+  `(let [start# (System/nanoTime)
+         ret# ~expr]
+     {:result ret#
+      :time-ms (elapsed-ms start#)}))
+
+(defn ^:private var-from-def [var]
+  (let [resolved-var (cond (var? var)
+                           var
+
+                           (symbol? var)
+                           (find-var var)
+
+                           :else
+                           (throw (ex-info "Unable to resolve into a variable" {:data var})))]
+    {:nextjournal.clerk/var-from-def resolved-var}))
+
+(defn ^:private eval-form [{:keys [form var no-cache?]} hash]
+  (try
+    (let [{:keys [result]} (time-ms (binding [config/*in-clerk* true]
+                                      (eval form)))
+          result (if (and (nil? result) var (= 'defonce (first form)))
+                   (find-var var)
+                   result)
+          var-value (cond-> result (and var (var? result)) deref)
+          no-cache? (or no-cache? config/cache-disabled?)]
+      (let [blob-id (cond no-cache? "valuehash" #_#_ TODO?/valuehash (analyzer/->hash-str var-value)
+                          (fn? var-value) nil
+                          :else hash)
+            result (if var (var-from-def var) result)]
+        (wrapped-with-metadata result blob-id)))
+    (catch Throwable t
+      (throw (ex-info (ex-message t) (Throwable->map t))))))
+
+(defn maybe-eval-viewers [{:as opts :nextjournal/keys [viewer viewers]}]
+  (cond-> opts
+    viewer
+    (update :nextjournal/viewer eval)
+    viewers
+    (update :nextjournal/viewers eval)))
+
+(defn read+eval-cached [{:as _doc :keys [blob->result]} {:as codeblock :keys [form vars var ns-effect? no-cache?]}]
+  (let [no-cache? (or ns-effect? no-cache?)
+        hash (.encodeToString (java.util.Base64/getEncoder) (.getBytes (str form)))
+        opts-from-form-meta (-> (meta form)
+                                (select-keys [:nextjournal.clerk/viewer :nextjournal.clerk/viewers :nextjournal.clerk/width :nextjournal.clerk/opts])
+                                v/normalize-viewer-opts
+                                maybe-eval-viewers)]
+    (cond-> (or (when-let [cached-result (and (not no-cache?) (get-in blob->result [hash :nextjournal/value]))]
+                  (wrapped-with-metadata cached-result hash))
+                (eval-form codeblock hash))
+      (seq opts-from-form-meta)
+      (merge opts-from-form-meta))))
+
+(defn eval-analyzed-doc [{:as analyzed-doc :keys [ns blocks]}]
+  (let [{:as evaluated-doc :keys [blob-ids]}
+        (reduce (fn [state {:as cell :keys [type]}]
+                  (let [{:as result :nextjournal/keys [blob-id]} (when (= :code type) (read+eval-cached state cell))]
+                    (cond-> (update state :blocks conj (cond-> cell result (assoc :result result)))
+                      blob-id (update :blob-ids conj blob-id)
+                      blob-id (assoc-in [:blob->result blob-id] result))))
+                (assoc analyzed-doc :blocks [] :blob-ids #{})
+                blocks)]
+    (-> evaluated-doc
+        (cond-> (not ns) (assoc :ns (find-ns 'user)))
+        (update :blob->result select-keys blob-ids)
+        (dissoc :blob-ids))))
+
+(defn read-string [s]
+  (edamame/parse-string s
+                        {:all true
+                         :readers *data-readers*
+                         :read-cond :allow
+                         :regex #(list `re-pattern %)
+                         :features #{:clj}
+                         :auto-resolve (as-> (ns-aliases (or *ns* (find-ns 'user))) $
+                                         (zipmap (keys $) (map ns-name (vals $)))
+                                         (assoc $ :current (ns-name *ns*)))}))
+
+(defn deflike? [form] (and (seq? form) (symbol? (first form)) (str/starts-with? (name (first form)) "def")))
+#_(deflike? (read-string "(def ^{:doc \"aloha\"} foo 123)"))
+#_(deflike? (read-string "(def ^{:doc \"aloha\"} foo 123)"))
+(defn no-cache-from-meta [form]
+  (when (contains? (meta form) :nextjournal.clerk/no-cache)
+    (-> form meta :nextjournal.clerk/no-cache)))
+(defn no-cache? [& subjects] (or (some no-cache-from-meta subjects) false))
+(defn deref? [form]
+  (and (seq? form)
+       (= (first form) `deref)
+       (= 2 (count form))))
+
+(defn read-forms [doc]
+  (binding [*ns* *ns*]
+    (reduce (fn [doc {:as b :keys [type text]}]
+              (let [form (read-string text)
+                    ns? (= 'ns (when (list? form) (first form)))
+                    var (when (and (deflike? form) (symbol? (second form))) (second form))]
+                (when ns? (eval form))
+                (-> doc
+                    (cond-> (and ns? (not (:ns doc))) (assoc :ns *ns*))
+                    (update :blocks conj
+                            (cond-> b
+                              (= :code type) (assoc :form form)
+                              (or ns? (deref? form) (no-cache? form var *ns*)) (assoc :no-cache? true)
+                              var (assoc :var (symbol (name (ns-name *ns*)) (name var))))))))
+            (assoc doc :blocks [])
+            (:blocks doc))))
+
+#_(read-forms
+   (parser/parse-file "notebooks/hello.clj"))
+
+;; used in builder
+(def analyze-doc read-forms)
+
+(defn +eval-results
+  "Evaluates the given `parsed-doc` using the `in-memory-cache` and augments it with the results."
+  [in-memory-cache parsed-doc]
+  (let [{:as doc :keys [ns]} (read-forms parsed-doc)]
+    (binding [*ns* (or ns *ns*)]
+      (-> doc
+          (assoc :blob->result in-memory-cache)
+          eval-analyzed-doc))))
+
+(defn eval-doc
+  "Evaluates the given `doc`."
+  ([doc] (eval-doc {} doc))
+  ([in-memory-cache doc] (+eval-results in-memory-cache doc)))
+
+(defn eval-file
+  "Reads given `file` (using `slurp`) and evaluates it."
+  ([file] (eval-file {} file))
+  ([in-memory-cache file]
+   (->> file
+        (parser/parse-file {:doc? true})
+        (eval-doc in-memory-cache))))
+
+#_(eval-file "notebooks/hello.clj")
+#_(eval-file "notebooks/rule_30.clj")
+#_(eval-file "notebooks/visibility.clj")
+
+(defn eval-string
+  "Evaluated the given `code-string` using the optional `in-memory-cache` map."
+  ([code-string] (eval-string {} code-string))
+  ([in-memory-cache code-string]
+   (eval-doc in-memory-cache (parser/parse-clojure-string {:doc? true} code-string))))
+
+#_(eval-string "(+ 39 3)")

src/nextjournal/clerk/eval.clj

@@ -192,6 +192,9 @@
         (update :blob->result select-keys blob-ids)
         (dissoc :blob-ids))))
 
+;; TODO: used in builder to drop analyzer dependency, cfr. below
+(defn analyze-doc [doc] (-> doc analyzer/build-graph analyzer/hash))
+
 (defn +eval-results
   "Evaluates the given `parsed-doc` using the `in-memory-cache` and augments it with the results."
   [in-memory-cache parsed-doc]
@@ -226,4 +229,3 @@
    (eval-doc in-memory-cache (parser/parse-clojure-string {:doc? true} code-string))))
 
 #_(eval-string "(+ 39 3)")
-

src/nextjournal/clerk/sci_viewer.cljs

@@ -10,6 +10,7 @@
             [goog.string :as gstring]
             [nextjournal.clerk.viewer :as viewer :refer [code md plotly tex table vl row col with-viewer with-viewers]]
             [nextjournal.clerk.parser :as clerk.parser]
+            [nextjournal.markdown :as markdown]
             [nextjournal.markdown.transform :as md.transform]
             [nextjournal.ui.components.icon :as icon]
             [nextjournal.ui.components.localstorage :as ls]
@@ -752,6 +753,7 @@
    'col col
    'html html-render
    'md md
+   'md->hiccup markdown/->hiccup
    'plotly plotly
    'row row
    'table table

src/nextjournal/clerk/viewer.cljc

@@ -14,7 +14,10 @@
                        [applied-science.js-interop :as j]])
             [nextjournal.markdown :as md]
             [nextjournal.markdown.transform :as md.transform])
-  #?(:clj (:import (com.pngencoder PngEncoder)
+  #?(:bb (:import (java.nio.file Files StandardOpenOption)
+                  (java.util Base64)
+                  (java.lang Throwable))
+     :clj (:import (com.pngencoder PngEncoder)
                    (clojure.lang IDeref)
                    (java.lang Throwable)
                    (java.awt.image BufferedImage)
@@ -605,7 +608,8 @@
   {:pred (fn [e] (instance? #?(:clj Throwable :cljs js/Error) e))
    :name :error :render-fn 'v/throwable-viewer :transform-fn (comp mark-presented (update-val (comp demunge-ex-data datafy/datafy)))})
 
-(def buffered-image-viewer #?(:clj {:pred #(instance? BufferedImage %)
+(def buffered-image-viewer #?(:bb {}
+                              :clj {:pred #(instance? BufferedImage %)
                                     :transform-fn (fn [{image :nextjournal/value}]
                                                     (let [w (.getWidth image)
                                                           h (.getHeight image)
@@ -620,15 +624,17 @@
                                     :render-fn '(fn [blob] (v/html [:figure.flex.flex-col.items-center.not-prose [:img {:src (v/url-for blob)}]]))}))
 
 (def ideref-viewer
-  {:pred #(instance? IDeref %)
-   :transform-fn (update-val (fn [ideref]
-                               (with-viewer :tagged-value
-                                 {:tag "object"
-                                  :value (vector (symbol (pr-str (type ideref)))
-                                                 #?(:clj (with-viewer :number-hex (System/identityHashCode ideref)))
-                                                 (if-let [deref-as-map (resolve 'clojure.core/deref-as-map)]
-                                                   (deref-as-map ideref)
-                                                   ideref))})))})
+  #?(:bb {}
+     :clj
+     {:pred #(instance? IDeref %)
+      :transform-fn (update-val (fn [ideref]
+                                  (with-viewer :tagged-value
+                                    {:tag "object"
+                                     :value (vector (symbol (pr-str (type ideref)))
+                                                    #?(:clj (with-viewer :number-hex (System/identityHashCode ideref)))
+                                                    (if-let [deref-as-map (resolve 'clojure.core/deref-as-map)]
+                                                      (deref-as-map ideref)
+                                                      ideref))})))}))
 
 (def regex-viewer
   {:pred #?(:clj (partial instance? java.util.regex.Pattern) :cljs regexp?)

src/nextjournal/clerk/webserver.clj

@@ -3,14 +3,14 @@
             [clojure.edn :as edn]
             [clojure.pprint :as pprint]
             [clojure.string :as str]
-            [lambdaisland.uri :as uri]
             [nextjournal.clerk.view :as view]
             [nextjournal.clerk.viewer :as v]
             [nextjournal.markdown :as md]
             [org.httpkit.server :as httpkit]))
 
 (def help-doc
-  {:blocks [{:type :markdown :doc (md/parse "Use `nextjournal.clerk/show!` to make your notebook appear…")}]})
+  {:ns *ns*
+   :blocks [{:type :markdown :doc (md/parse "Use `nextjournal.clerk/show!` to make your notebook appear…")}]})
 
 (defonce !clients (atom #{}))
 (defonce !doc (atom help-doc))

src/nextjournal/markdown.bb

@@ -0,0 +1,44 @@
+(ns nextjournal.markdown
+  "Babashka runtime stubs"
+  (:require [babashka.fs :as fs]
+            [babashka.process :as p]
+            [clojure.data.json :as json]
+            [clojure.java.io :as io]
+            [clojure.string :as str]
+            [nextjournal.markdown.parser :as md.parser]))
+
+(defn assert-quickjs! [] (assert (= 0 (:exit @(p/process '[which qjs]))) "QuickJS needs to be installed (brew install quickjs)"))
+(def !md-mod-temp-dir (atom nil))
+(defn md-mod-temp-dir []
+  (or @!md-mod-temp-dir
+      (let [tempdir (fs/create-temp-dir)]
+        (assert-quickjs!)
+        (spit (fs/file tempdir "markdown.mjs") (slurp (io/resource "js/markdown.mjs")))
+        (reset! !md-mod-temp-dir (str tempdir)))))
+
+(defn escape [t] (-> t (str/replace "\\" "\\\\\\") (str/replace "`" "\\`") (str/replace "'" "\\'")))
+(defn tokenize [text]
+  (some-> (p/shell {:out :string :err :string :dir (md-mod-temp-dir)}
+                   (str "qjs -e 'import(\"./markdown.mjs\").then((mod) => {print(mod.default.tokenizeJSON(`" (escape text) "`))})"
+                        ".catch((e) => {import(\"std\").then((std) => { std.err.puts(\"cant find markdown module\"); std.exit(1)})})'"))
+          deref :out not-empty
+          (json/read-str {:key-fn keyword})))
+
+(defn parse [md] {:type :doc :content []} (some-> md tokenize md.parser/parse))
+
+(comment
+  (assert-quickjs!)
+  (md-mod-temp-dir)
+  (tokenize "# Hello")
+  (parse "# Hello
+* `this`
+* _is_ Some $\\mathfrak{M}$ formula
+* crazy as [hello](https://hell.is)
+
+---
+```clojure
+and this is code
+```
+")
+  (try (parse (slurp "notebooks/markdown.md"))
+       (catch Exception e (:err (ex-data e)))))