properly handle large values in

juji-io · Sep 18, 2020 · 29607e4 · 29607e4
1 parent fa236fc
commit 29607e4
Show file tree

Hide file tree

Showing 5 changed files with 36 additions and 14 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,10 @@
 # Change Log
 All notable changes to this project will be documented in this file. This change log follows the conventions of [keepachangelog.com](http://keepachangelog.com/).
 
+## WIP
+### Fixed
+-  Correctly handle `init-max-eid` for large values as well.
+
 ## 0.3.7
 ### Fixed
 - Fixed regression introduced by 0.3.6, where :ignore value was not considered [#25]

diff --git a/src/datalevin/bits.clj b/src/datalevin/bits.clj
@@ -453,6 +453,8 @@
 
 (deftype ^:no-doc Retrieved [e a v])
 
+(def ^:no-doc ^:const overflown-key (->Retrieved c/e0 c/overflown c/overflown))
+
 (defn- indexable->retrieved
   [^Indexable i]
   (->Retrieved (.-e i) (.-a i) (.-v i)))

diff --git a/src/datalevin/lmdb.clj b/src/datalevin/lmdb.clj
@@ -7,7 +7,6 @@
   (:import [org.lmdbjava Env EnvFlags Env$MapFullException Stat Dbi DbiFlags
             PutFlags Txn CursorIterable CursorIterable$KeyVal KeyRange]
            [clojure.lang IMapEntry]
-           [datalevin.bits Retrieved]
            [java.util Iterator]
            [java.util.concurrent ConcurrentHashMap]
            [java.nio.charset StandardCharsets]
@@ -503,7 +502,7 @@
    (read-key kv k-type v false))
   ([kv k-type v rewind?]
    (if (and v (not= v c/normal) (c/index-types k-type))
-     (b/->Retrieved c/e0 c/overflown c/overflown)
+     b/overflown-key
      (b/read-buffer (if rewind?
                       (.rewind ^ByteBuffer (key kv))
                       (key kv))
@@ -688,7 +687,7 @@
                    (put-key dbi k kt)
                    (del dbi txn))))
         (.commit txn))
-      (catch Env$MapFullException e
+      (catch Env$MapFullException _
         (up-db-size env)
         (transact this txs))
       (catch Exception e

diff --git a/src/datalevin/storage.clj b/src/datalevin/storage.clj
@@ -3,6 +3,7 @@
   (:require [datalevin.lmdb :as lmdb]
             [datalevin.util :as u]
             [datalevin.bits :as b]
+            ;; [taoensso.timbre :as log]
             [datalevin.constants :as c]
             [datalevin.datom :as d])
   (:import [java.util UUID]
@@ -192,8 +193,8 @@
 (deftype Store [^LMDB lmdb
                 ^:volatile-mutable schema
                 ^:volatile-mutable attrs
-                ^:volatile-mutable ^long max-aid
-                ^:volatile-mutable ^long max-gt]
+                ^:volatile-mutable max-aid
+                ^:volatile-mutable max-gt]
   IStore
   (dir [_]
     (.-dir lmdb))
@@ -207,7 +208,7 @@
     max-gt)
 
   (advance-max-gt [_]
-    (set! max-gt (inc max-gt)))
+    (set! max-gt (inc ^long max-gt)))
 
   (max-aid [_]
     max-aid)
@@ -224,8 +225,10 @@
     attrs)
 
   (init-max-eid [_]
-    (or (when-let [[r _] (lmdb/get-first lmdb c/eav [:all-back] :eav :ignore)]
-          (.-e ^Retrieved r))
+    (or (when-let [[k v] (lmdb/get-first lmdb c/eav [:all-back] :eav :id)]
+          (if (= c/overflown (.-a ^Retrieved k))
+            (.-e ^Datom (lmdb/get-value lmdb c/giants v :id :datom))
+            (.-e ^Retrieved k)))
         c/e0))
 
   (swap-attr [this attr f]
@@ -235,7 +238,7 @@
   (swap-attr [_ attr f x y]
     (let [o (or (schema attr)
                 (let [m {:db/aid max-aid}]
-                  (set! max-aid (inc max-aid))
+                  (set! max-aid (inc ^long max-aid))
                   m))
           p (cond
               (and x y) (f o x y)

diff --git a/test/datalevin/storage_test.clj b/test/datalevin/storage_test.clj
@@ -159,15 +159,29 @@
                           (d/datom c/e0 :a c/vmax))))))
 
 (deftest giants-data-test
-  (let [dir    (u/tmp-dir (str "datalevin-giants-data-test-" (UUID/randomUUID)))
-        store  (sut/open dir)
-        v      (apply str (repeat 10000 (UUID/randomUUID)))
-        d      (d/datom c/e0 :a v)]
+  (let [dir   (u/tmp-dir (str "datalevin-giants-data-test-" (UUID/randomUUID)))
+        store (sut/open dir)
+        v     (apply str (repeat 10000 (UUID/randomUUID)))
+        d     (d/datom c/e0 :a v)
+        d1    (d/datom (inc c/e0) :b v)]
     (sut/load-datoms store [d])
     (is (= [d] (sut/fetch store d)))
     (is (= [d] (sut/slice store :eavt
                           (d/datom c/e0 :a c/v0)
-                          (d/datom c/e0 :a c/vmax))))))
+                          (d/datom c/e0 :a c/vmax))))
+    (sut/close store)
+
+    (let [store' (sut/open dir)]
+      (is (sut/populated? store' :eav
+                          (d/datom c/e0 :a c/v0)
+                          (d/datom c/e0 :a c/vmax)))
+      (is (= [d] (sut/fetch store' d)))
+      (is (= [d] (sut/slice store' :eavt
+                            (d/datom c/e0 :a c/v0)
+                            (d/datom c/e0 :a c/vmax))))
+      (sut/load-datoms store' [d1])
+      (is (= 1 (sut/init-max-eid store'))))
+    ))
 
 (deftest false-value-test
   (let [d (d/datom c/e0 :a false)]