diff --git a/CHANGELOG.md b/CHANGELOG.md index efe20965..0f80948e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ # Change Log All notable changes to this project will be documented in this file. This change log follows the conventions of [keepachangelog.com](http://keepachangelog.com/). +## WIP +### Fixed +- Correctly handle `init-max-eid` for large values as well. + ## 0.3.7 ### Fixed - Fixed regression introduced by 0.3.6, where :ignore value was not considered [#25] diff --git a/src/datalevin/bits.clj b/src/datalevin/bits.clj index 3ca8377f..9e3514e5 100644 --- a/src/datalevin/bits.clj +++ b/src/datalevin/bits.clj @@ -453,6 +453,8 @@ (deftype ^:no-doc Retrieved [e a v]) +(def ^:no-doc ^:const overflown-key (->Retrieved c/e0 c/overflown c/overflown)) + (defn- indexable->retrieved [^Indexable i] (->Retrieved (.-e i) (.-a i) (.-v i))) diff --git a/src/datalevin/lmdb.clj b/src/datalevin/lmdb.clj index 790e22b5..69df9b17 100644 --- a/src/datalevin/lmdb.clj +++ b/src/datalevin/lmdb.clj @@ -7,7 +7,6 @@ (:import [org.lmdbjava Env EnvFlags Env$MapFullException Stat Dbi DbiFlags PutFlags Txn CursorIterable CursorIterable$KeyVal KeyRange] [clojure.lang IMapEntry] - [datalevin.bits Retrieved] [java.util Iterator] [java.util.concurrent ConcurrentHashMap] [java.nio.charset StandardCharsets] @@ -503,7 +502,7 @@ (read-key kv k-type v false)) ([kv k-type v rewind?] (if (and v (not= v c/normal) (c/index-types k-type)) - (b/->Retrieved c/e0 c/overflown c/overflown) + b/overflown-key (b/read-buffer (if rewind? (.rewind ^ByteBuffer (key kv)) (key kv)) @@ -688,7 +687,7 @@ (put-key dbi k kt) (del dbi txn)))) (.commit txn)) - (catch Env$MapFullException e + (catch Env$MapFullException _ (up-db-size env) (transact this txs)) (catch Exception e diff --git a/src/datalevin/storage.clj b/src/datalevin/storage.clj index 58237055..59d434f5 100644 --- a/src/datalevin/storage.clj +++ b/src/datalevin/storage.clj @@ -3,6 +3,7 @@ (:require [datalevin.lmdb :as lmdb] [datalevin.util :as u] [datalevin.bits :as b] + ;; [taoensso.timbre :as log] [datalevin.constants :as c] [datalevin.datom :as d]) (:import [java.util UUID] @@ -192,8 +193,8 @@ (deftype Store [^LMDB lmdb ^:volatile-mutable schema ^:volatile-mutable attrs - ^:volatile-mutable ^long max-aid - ^:volatile-mutable ^long max-gt] + ^:volatile-mutable max-aid + ^:volatile-mutable max-gt] IStore (dir [_] (.-dir lmdb)) @@ -207,7 +208,7 @@ max-gt) (advance-max-gt [_] - (set! max-gt (inc max-gt))) + (set! max-gt (inc ^long max-gt))) (max-aid [_] max-aid) @@ -224,8 +225,10 @@ attrs) (init-max-eid [_] - (or (when-let [[r _] (lmdb/get-first lmdb c/eav [:all-back] :eav :ignore)] - (.-e ^Retrieved r)) + (or (when-let [[k v] (lmdb/get-first lmdb c/eav [:all-back] :eav :id)] + (if (= c/overflown (.-a ^Retrieved k)) + (.-e ^Datom (lmdb/get-value lmdb c/giants v :id :datom)) + (.-e ^Retrieved k))) c/e0)) (swap-attr [this attr f] @@ -235,7 +238,7 @@ (swap-attr [_ attr f x y] (let [o (or (schema attr) (let [m {:db/aid max-aid}] - (set! max-aid (inc max-aid)) + (set! max-aid (inc ^long max-aid)) m)) p (cond (and x y) (f o x y) diff --git a/test/datalevin/storage_test.clj b/test/datalevin/storage_test.clj index 7c378111..937d6429 100644 --- a/test/datalevin/storage_test.clj +++ b/test/datalevin/storage_test.clj @@ -159,15 +159,29 @@ (d/datom c/e0 :a c/vmax)))))) (deftest giants-data-test - (let [dir (u/tmp-dir (str "datalevin-giants-data-test-" (UUID/randomUUID))) - store (sut/open dir) - v (apply str (repeat 10000 (UUID/randomUUID))) - d (d/datom c/e0 :a v)] + (let [dir (u/tmp-dir (str "datalevin-giants-data-test-" (UUID/randomUUID))) + store (sut/open dir) + v (apply str (repeat 10000 (UUID/randomUUID))) + d (d/datom c/e0 :a v) + d1 (d/datom (inc c/e0) :b v)] (sut/load-datoms store [d]) (is (= [d] (sut/fetch store d))) (is (= [d] (sut/slice store :eavt (d/datom c/e0 :a c/v0) - (d/datom c/e0 :a c/vmax)))))) + (d/datom c/e0 :a c/vmax)))) + (sut/close store) + + (let [store' (sut/open dir)] + (is (sut/populated? store' :eav + (d/datom c/e0 :a c/v0) + (d/datom c/e0 :a c/vmax))) + (is (= [d] (sut/fetch store' d))) + (is (= [d] (sut/slice store' :eavt + (d/datom c/e0 :a c/v0) + (d/datom c/e0 :a c/vmax)))) + (sut/load-datoms store' [d1]) + (is (= 1 (sut/init-max-eid store')))) + )) (deftest false-value-test (let [d (d/datom c/e0 :a false)]