fix the case when a giant is deleted in the same tx

juji-io · Jan 20, 2023 · 8ba7521 · 8ba7521
1 parent 60f75d2
commit 8ba7521
Show file tree

Hide file tree

Showing 3 changed files with 63 additions and 36 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,9 @@
 # Change Log
 
+## WIP
+### Fixed
+- [Datalog] error when large `:db/fulltext` value is added then removed in the same transaction.
+
 ## 0.8.2 (2023-01-19)
 ### Fixed
 - [Search] `search-utils/create-ngram-token-filter` now works. #164

diff --git a/src/datalevin/storage.cljc b/src/datalevin/storage.cljc
@@ -354,23 +354,23 @@
 
   (load-datoms [this datoms]
     (locking (lmdb/write-txn lmdb)
-      (let [;; fulltext [:a [e aid v]], [:d [e aid v]], [:g [gt v]] or [:r gt]
-            ft-ds  (transient [])
+      ;; fulltext [:a [e aid v]], [:d [e aid v]], [:g [gt v]] or [:r gt]
+      (let [ft-ds (transient [])
+            ;; needed because a giant may be deleted in the same tx
+            giants (volatile! {})
             add-fn (fn [holder datom]
                      (if (d/datom-added datom)
-                       (let [res (insert-data this datom ft-ds)]
-                         (when (nth res 1) (advance-max-gt this))
-                         (reduce conj! holder (nth res 0)))
-                       (reduce conj! holder (delete-data this datom ft-ds))))
-            txs    (persistent!
-                     (conj! (reduce add-fn (transient []) datoms)
-                            [:put c/meta :max-tx (advance-max-tx this)
-                             :attr :long]))]
-        (lmdb/transact-kv lmdb txs)
+                       (reduce conj! holder
+                               (insert-data this datom ft-ds giants))
+                       (reduce conj! holder
+                               (delete-data this datom ft-ds giants))))]
+        (lmdb/transact-kv
+          lmdb (persistent! (reduce add-fn (transient []) datoms)))
         (fulltext-index search-engine ft-ds)
-        (lmdb/transact-kv lmdb [[:put c/meta :last-modified
-                                 (System/currentTimeMillis)
-                                 :attr :long]]))))
+        (lmdb/transact-kv
+          lmdb [[:put c/meta :max-tx (advance-max-tx this) :attr :long]
+                [:put c/meta :last-modified (System/currentTimeMillis)
+                 :attr :long]]))))
 
   (fetch [_ datom]
     (mapv (partial retrieved->datom lmdb attrs)
@@ -566,7 +566,7 @@
       :pass-through)))
 
 (defn- insert-data
-  [^Store store ^Datom d ft-ds]
+  [^Store store ^Datom d ft-ds giants]
   (let [attr  (.-a d)
         props (or ((schema store) attr)
                   (swap-attr store attr identity))
@@ -582,38 +582,38 @@
         (u/raise "Invalid data, expecting " vt {:input v}))
     (if (b/giant? i)
       (let [max-gt (max-gt store)]
+        (advance-max-gt store)
+        (vswap! giants assoc e max-gt)
         (when ft?
           (let [v (str v)]
             (when-not (str/blank? v) (conj! ft-ds [:g [max-gt v]]))))
-        [(cond-> [[:put c/eav i max-gt :eav :id]
-                  [:put c/ave i max-gt :ave :id]
-                  [:put c/giants max-gt d :id :datom [:append]]]
-           ref? (conj [:put c/vea i max-gt :vea :id]))
-         true])
+        (cond-> [[:put c/eav i max-gt :eav :id]
+                 [:put c/ave i max-gt :ave :id]
+                 [:put c/giants max-gt d :id :datom [:append]]]
+          ref? (conj [:put c/vea i max-gt :vea :id])))
       (do (when ft?
             (let [v (str v)]
               (when-not (str/blank? v) (conj! ft-ds [:a [e aid v]]))))
-          [(cond-> [[:put c/eav i c/normal :eav :id]
-                    [:put c/ave i c/normal :ave :id]]
-             ref? (conj [:put c/vea i c/normal :vea :id]))
-           false]))))
+          (cond-> [[:put c/eav i c/normal :eav :id]
+                   [:put c/ave i c/normal :ave :id]]
+            ref? (conj [:put c/vea i c/normal :vea :id]))))))
 
 (defn- delete-data
-  [^Store store ^Datom d ft-ds]
-  (let [props  ((schema store) (.-a d))
-        vt     (:db/valueType props)
-        ref?   (= :db.type/ref vt)
-        e      (.-e d)
-        aid    (:db/aid props)
-        v      (.-v d)
-        i      (b/indexable e aid v vt)
-        giant? (b/giant? i)
-        gt     (when giant?
-                 (lmdb/get-value (.-lmdb store) c/eav i :eav :id))]
+  [^Store store ^Datom d ft-ds giants]
+  (let [props ((schema store) (.-a d))
+        vt    (:db/valueType props)
+        ref?  (= :db.type/ref vt)
+        e     (.-e d)
+        aid   (:db/aid props)
+        v     (.-v d)
+        i     (b/indexable e aid v vt)
+        gt    (when (b/giant? i)
+                (or (@giants e)
+                    (lmdb/get-value (.-lmdb store) c/eav i :eav :id)))]
     (when (:db/fulltext props)
       (let [v (str v)]
         (when-not (str/blank? v)
-          (conj! ft-ds (if giant? [:r gt] [:d [e aid v]])))))
+          (conj! ft-ds (if gt [:r gt] [:d [e aid v]])))))
     (cond-> [[:del c/eav i :eav]
              [:del c/ave i :ave]]
       ref? (conj [:del c/vea i :vea])

diff --git a/test/data/data.json b/test/data/data.json
@@ -1,4 +1,27 @@
 [
+  {
+    "id": 132012987,
+    "name": "lutece-apps-module-quiz-exportdirectory",
+    "organization_name": "lutece-platform",
+    "platform": "GitHub",
+    "repository_url": "https://github.com/lutece-platform/lutece-apps-module-quiz-exportdirectory",
+    "description": "Lutece is an open source platform developed by the City of Paris to help you develop digital solutions. Honed for cities’ use through almost 20 years of development, Lutece has evolved into multifaceted platform with more than 400 plugins and modular architecture that will enable your specific needs and enhance your users' digital experience. In 2021, the team is getting ready to release version 7 of Lutece Core, a version that further marks the desire to integrate new core functions while lightening its code.  Lutece harnesses the power of almost 20 years of iterations and constant evolution. Any institution seeking to adopt a new technology for its site must ensure the highest level of guarantees to avoid the risk of being left alone in trouble. Is the code open? Is the project monitored and sustainable? Can I contribute upstream, can the community help me... are these questions essential for a good choice of a framework. Developers can contribute to Open Source projects that directly impact residents, collaborate with a global community and work on cutting-edge technology challenges.",
+    "default_branch": "master",
+    "is_fork": false,
+    "is_archived": false,
+    "creation_date": "2018-05-03T15:11:53Z",
+    "last_update": "2018-05-03T15:21:55Z",
+    "last_modification": "2018-05-03T15:22:32Z",
+    "homepage": "",
+    "stars_count": 0,
+    "forks_count": 0,
+    "license": "",
+    "open_issues_count": 0,
+    "language": "Java",
+    "topics": "",
+    "software_heritage_exists": "",
+    "software_heritage_url": ""
+  },
   {
     "id": 132012987,
     "name": "lutece-apps-module-quiz-exportdirectory",