From 7313608f96bfbba79151a8437b8a47e5a9376770 Mon Sep 17 00:00:00 2001 From: James Reeves Date: Fri, 8 Nov 2024 16:00:27 +0000 Subject: [PATCH] Refactor to work with Duct main Refactor codebase to work with new 'Duct main' design. Update dependecies and change migrator to work with a single edn file. --- project.clj | 17 +- src/duct/migrator/ragtime.clj | 60 ++--- test/duct/migrator/example.txt | 1 - .../migrations/01-create-foo.down.sql | 1 - .../migrator/migrations/01-create-foo.up.sql | 1 - .../migrations/02-create-bar.down.sql | 1 - .../migrator/migrations/02-create-bar.up.sql | 1 - test/duct/migrator/migrations1.edn | 2 + test/duct/migrator/migrations2.edn | 2 + test/duct/migrator/migrations3.edn | 1 + test/duct/migrator/ragtime_test.clj | 218 +++++------------- 11 files changed, 83 insertions(+), 222 deletions(-) delete mode 100644 test/duct/migrator/example.txt delete mode 100644 test/duct/migrator/migrations/01-create-foo.down.sql delete mode 100644 test/duct/migrator/migrations/01-create-foo.up.sql delete mode 100644 test/duct/migrator/migrations/02-create-bar.down.sql delete mode 100644 test/duct/migrator/migrations/02-create-bar.up.sql create mode 100644 test/duct/migrator/migrations1.edn create mode 100644 test/duct/migrator/migrations2.edn create mode 100644 test/duct/migrator/migrations3.edn diff --git a/project.clj b/project.clj index 3c77014..d0417f1 100644 --- a/project.clj +++ b/project.clj @@ -3,13 +3,12 @@ :url "https://github.com/duct-framework/migrator.ragtime" :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} - :dependencies [[org.clojure/clojure "1.10.1"] - [duct/core "0.8.0"] - [duct/logger "0.3.0"] - [integrant "0.8.0"] - [pandect "0.6.1"] - [ragtime "0.8.0"]] + :dependencies [[org.clojure/clojure "1.11.4"] + [org.duct-framework/logger "0.4.0"] + [dev.weavejester/ragtime "0.10.1"] + [integrant "0.13.1"] + [pandect "1.0.2"]] :profiles - {:dev {:dependencies [[duct/database.sql "0.1.0"] - [org.clojure/java.jdbc "0.7.11"] - [org.xerial/sqlite-jdbc "3.30.1"]]}}) + {:dev {:dependencies [[org.duct-framework/database.sql "0.3.0"] + [com.github.seancorfield/next.jdbc "1.3.955"] + [org.xerial/sqlite-jdbc "3.47.0.0"]]}}) diff --git a/src/duct/migrator/ragtime.clj b/src/duct/migrator/ragtime.clj index 92da59b..cd46b35 100644 --- a/src/duct/migrator/ragtime.clj +++ b/src/duct/migrator/ragtime.clj @@ -1,19 +1,18 @@ (ns duct.migrator.ragtime - (:require [clojure.java.io :as io] - [duct.core :as duct] - [duct.logger :as logger] + (:require [duct.logger :as logger] [integrant.core :as ig] [pandect.algo.sha1 :refer [sha1]] [ragtime.core :as ragtime] - [ragtime.jdbc :as jdbc] + [ragtime.sql :as sql] + [ragtime.next-jdbc :as jdbc] [ragtime.reporter :as reporter] [ragtime.strategy :as strategy])) (defn logger-reporter [logger] (fn [_ op id] (case op - :up (logger/log logger :report ::applying id) - :down (logger/log logger :report ::rolling-back id)))) + :up (logger/report logger ::applying {:id id}) + :down (logger/report logger ::rolling-back {:id id})))) (def strategies {:apply-new strategy/apply-new @@ -21,25 +20,6 @@ :rebase strategy/rebase :ignore-future strategy/ignore-future}) -(defprotocol StringSource - (get-string [source])) - -(extend-protocol StringSource - String - (get-string [s] s) - java.net.URL - (get-string [s] (slurp s)) - duct.core.resource.Resource - (get-string [s] (slurp s))) - -(defn- singularize [coll] - (if (= (count coll) 1) (first coll) coll)) - -(defn- clean-key [base key] - (if (vector? key) - (singularize (remove #{base} key)) - key)) - (def ^:private colon (.getBytes ":" "US-ASCII")) (def ^:private comma (.getBytes "," "US-ASCII")) (def ^:private u= (.getBytes "u=" "US-ASCII")) @@ -62,8 +42,12 @@ (defn- add-hash-to-id [migration] (update migration :id str "#" (subs (hash-migration migration) 0 8))) -(defn- get-database [{{:keys [spec]} :database :as opts}] - (jdbc/sql-database spec (select-keys opts [:migrations-table]))) +(defn- load-migrations [{:keys [migrations-file]}] + (->> (sql/load-migrations migrations-file) + (map (comp jdbc/sql-migration add-hash-to-id)))) + +(defn- get-database [{{:keys [datasource]} :database :as opts}] + (jdbc/sql-database datasource (select-keys opts [:migrations-table]))) (defn- get-strategy [{:keys [strategy] :or {strategy :raise-error}}] (strategies strategy)) @@ -71,11 +55,11 @@ (defn- get-reporter [{:keys [logger]}] (if logger (logger-reporter logger) reporter/print)) -(defn- migrate [index {:keys [migrations] :as opts}] - (let [db (get-database opts) - strat (get-strategy opts) - rep (get-reporter opts) - migs (flatten migrations)] +(defn- migrate [index options] + (let [db (get-database options) + strat (get-strategy options) + rep (get-reporter options) + migs (load-migrations options)] (ragtime/migrate-all db index migs {:reporter rep, :strategy strat}) (ragtime/into-index index migs))) @@ -84,15 +68,3 @@ (defmethod ig/resume-key :duct.migrator/ragtime [_ options _ index] (migrate index options)) - -(defmethod ig/init-key ::sql [key {:keys [up down] :as opts}] - (-> (jdbc/sql-migration {:id (:id opts (clean-key ::sql key)) - :up (mapv get-string up) - :down (mapv get-string down)}) - (add-hash-to-id))) - -(defmethod ig/init-key ::resources [_ {:keys [path]}] - (->> (jdbc/load-resources path) (map add-hash-to-id))) - -(defmethod ig/init-key ::directory [_ {:keys [path]}] - (->> (jdbc/load-directory path) (map add-hash-to-id))) diff --git a/test/duct/migrator/example.txt b/test/duct/migrator/example.txt deleted file mode 100644 index 257cc56..0000000 --- a/test/duct/migrator/example.txt +++ /dev/null @@ -1 +0,0 @@ -foo diff --git a/test/duct/migrator/migrations/01-create-foo.down.sql b/test/duct/migrator/migrations/01-create-foo.down.sql deleted file mode 100644 index 0df01c0..0000000 --- a/test/duct/migrator/migrations/01-create-foo.down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP TABLE foo; \ No newline at end of file diff --git a/test/duct/migrator/migrations/01-create-foo.up.sql b/test/duct/migrator/migrations/01-create-foo.up.sql deleted file mode 100644 index 1c53de2..0000000 --- a/test/duct/migrator/migrations/01-create-foo.up.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE TABLE foo (id int); \ No newline at end of file diff --git a/test/duct/migrator/migrations/02-create-bar.down.sql b/test/duct/migrator/migrations/02-create-bar.down.sql deleted file mode 100644 index a9bbc12..0000000 --- a/test/duct/migrator/migrations/02-create-bar.down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP TABLE bar; \ No newline at end of file diff --git a/test/duct/migrator/migrations/02-create-bar.up.sql b/test/duct/migrator/migrations/02-create-bar.up.sql deleted file mode 100644 index 1ef5456..0000000 --- a/test/duct/migrator/migrations/02-create-bar.up.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE TABLE bar (id int); \ No newline at end of file diff --git a/test/duct/migrator/migrations1.edn b/test/duct/migrator/migrations1.edn new file mode 100644 index 0000000..04db586 --- /dev/null +++ b/test/duct/migrator/migrations1.edn @@ -0,0 +1,2 @@ +[[:create-table foo (id "INTEGER")] + [:create-table bar (id "INTEGER")]] diff --git a/test/duct/migrator/migrations2.edn b/test/duct/migrator/migrations2.edn new file mode 100644 index 0000000..c714a9e --- /dev/null +++ b/test/duct/migrator/migrations2.edn @@ -0,0 +1,2 @@ +[[:create-table foo (id "INTEGER")] + [:create-table baz (id "INTEGER")]] diff --git a/test/duct/migrator/migrations3.edn b/test/duct/migrator/migrations3.edn new file mode 100644 index 0000000..9a2db05 --- /dev/null +++ b/test/duct/migrator/migrations3.edn @@ -0,0 +1 @@ +[[:create-table foo (id "INTEGER")]] \ No newline at end of file diff --git a/test/duct/migrator/ragtime_test.clj b/test/duct/migrator/ragtime_test.clj index 87c39be..af86035 100644 --- a/test/duct/migrator/ragtime_test.clj +++ b/test/duct/migrator/ragtime_test.clj @@ -1,178 +1,68 @@ (ns duct.migrator.ragtime-test - (:require [clojure.java.jdbc :as jdbc] - [clojure.test :refer :all] - [duct.core :as duct] - [duct.database.sql :as sql] + (:require [clojure.test :refer [deftest is testing]] [duct.logger :as logger] - [duct.migrator.ragtime :as ragtime] [integrant.core :as ig] - [clojure.java.io :as io])) - -(duct/load-hierarchy) + [next.jdbc :as jdbc])) (defrecord TestLogger [logs] logger/Logger - (-log [_ level ns-str file line id event data] + (-log [_ _level _ns-str _file _line _id event data] (swap! logs conj [event data]))) -(def logs - (atom [])) - -(def db-spec - {:connection (jdbc/get-connection {:connection-uri "jdbc:sqlite:"})}) - -(def config - {:duct.database/sql db-spec - - :duct.migrator/ragtime - {:database (ig/ref :duct.database/sql) - :strategy :rebase - :logger (->TestLogger logs) - :migrations [(ig/ref ::create-foo) - (ig/ref ::create-bar)]} - - [:duct.migrator.ragtime/sql ::create-foo] - {:up ["CREATE TABLE foo (id int);"] - :down ["DROP TABLE foo;"]} - - [:duct.migrator.ragtime/sql ::create-bar] - {:up ["CREATE TABLE bar (id int);"] - :down ["DROP TABLE bar;"]}}) - -(def config-resources - {:duct.database/sql db-spec - - :duct.migrator/ragtime - {:database (ig/ref :duct.database/sql) - :strategy :rebase - :logger (->TestLogger logs) - :migrations (ig/ref :duct.migrator.ragtime/resources)} - - :duct.migrator.ragtime/resources - {:path "duct/migrator/migrations"}}) - -(def config-directory - {:duct.database/sql db-spec - - :duct.migrator/ragtime - {:database (ig/ref :duct.database/sql) - :strategy :rebase - :logger (->TestLogger logs) - :migrations (ig/ref :duct.migrator.ragtime/directory)} - - :duct.migrator.ragtime/directory - {:path "test/duct/migrator/migrations"}}) - -(def config-mixed - {:duct.database/sql db-spec +(defmethod ig/init-key ::logger [_ _] + (->TestLogger (atom []))) +(def base-config + {::logger {} + :duct.database/sql {} :duct.migrator/ragtime - {:database (ig/ref :duct.database/sql) - :strategy :rebase - :logger (->TestLogger logs) - :migrations [(ig/ref :duct.migrator.ragtime/resources) - (ig/ref ::create-baz)]} - - :duct.migrator.ragtime/resources - {:path "duct/migrator/migrations"} - - [:duct.migrator.ragtime/sql ::create-baz] - {:up ["CREATE TABLE baz (id int);"] - :down ["DROP TABLE baz;"]}}) - -(defn- find-tables [] - (jdbc/query db-spec ["SELECT name FROM sqlite_master WHERE type='table'"])) - -(defn- drop-all-tables [] - (doseq [t (find-tables)] - (jdbc/execute! db-spec [(str "DROP TABLE " (:name t))]))) + {:logger (ig/ref ::logger) + :database (ig/ref :duct.database/sql) + :strategy :rebase + :migrations-file "test/duct/migrator/migrations1.edn"}}) -(defn reset-test-state [] - (reset! logs []) - (drop-all-tables)) +(defn- find-tables [db] + (jdbc/execute! db ["SELECT name FROM sqlite_master WHERE type='table'"])) (deftest migration-test - (testing "default migrations table name" - (reset-test-state) - (let [system (ig/init config)] - (is (= (find-tables) - [{:name "ragtime_migrations"} {:name "foo"} {:name "bar"}])) - (is (= @logs - [[::ragtime/applying ":duct.migrator.ragtime-test/create-foo#f1480e44"] - [::ragtime/applying ":duct.migrator.ragtime-test/create-bar#6d969ce8"]])))) - - (testing "custom migrations table name" - (reset-test-state) - (let [migrations-table "custom_migrations" - system (-> config - (assoc-in [:duct.migrator/ragtime :migrations-table] migrations-table) - ig/init)] - (is (= (find-tables) - [{:name migrations-table} {:name "foo"} {:name "bar"}])) - (is (= @logs - [[::ragtime/applying ":duct.migrator.ragtime-test/create-foo#f1480e44"] - [::ragtime/applying ":duct.migrator.ragtime-test/create-bar#6d969ce8"]])))) - - (testing "migrations from resource directory" - (reset-test-state) - (let [system (ig/init config-resources)] - (is (= (find-tables) - [{:name "ragtime_migrations"} {:name "foo"} {:name "bar"}])) - (is (= @logs - [[::ragtime/applying "01-create-foo#f1480e44"] - [::ragtime/applying "02-create-bar#6d969ce8"]])))) - - (testing "migrations from filesystem directory" - (reset-test-state) - (let [system (ig/init config-directory)] - (is (= (find-tables) - [{:name "ragtime_migrations"} {:name "foo"} {:name "bar"}])) - (is (= @logs - [[::ragtime/applying "01-create-foo#f1480e44"] - [::ragtime/applying "02-create-bar#6d969ce8"]])))) - - (testing "migrations from multiple sources" - (reset-test-state) - (let [system (ig/init config-mixed)] - (is (= (find-tables) - [{:name "ragtime_migrations"} {:name "foo"} {:name "bar"} {:name "baz"}])) - (is (= @logs - [[::ragtime/applying "01-create-foo#f1480e44"] - [::ragtime/applying "02-create-bar#6d969ce8"] - [::ragtime/applying ":duct.migrator.ragtime-test/create-baz#01f7277d"]]))))) - -(deftest remove-and-resume-test - (reset-test-state) - (let [system (ig/init config) - config' (update-in config [:duct.migrator/ragtime :migrations] pop) - system' (ig/resume config' system)] - (is (= (find-tables) - [{:name "ragtime_migrations"} - {:name "foo"}])) - (is (= @logs - [[::ragtime/applying ":duct.migrator.ragtime-test/create-foo#f1480e44"] - [::ragtime/applying ":duct.migrator.ragtime-test/create-bar#6d969ce8"] - [::ragtime/rolling-back ":duct.migrator.ragtime-test/create-bar#6d969ce8"]])))) - -(deftest change-and-resume-test - (reset-test-state) - (let [system (ig/init config) - config' (assoc config - [:duct.migrator.ragtime/sql ::create-bar] - {:up ["CREATE TABLE barbaz (id int)"] - :down ["DROP TABLE barbaz"]}) - system' (ig/resume config' system)] - (is (= (find-tables) - [{:name "ragtime_migrations"} - {:name "foo"} - {:name "barbaz"}])) - (is (= @logs - [[::ragtime/applying ":duct.migrator.ragtime-test/create-foo#f1480e44"] - [::ragtime/applying ":duct.migrator.ragtime-test/create-bar#6d969ce8"] - [::ragtime/rolling-back ":duct.migrator.ragtime-test/create-bar#6d969ce8"] - [::ragtime/applying ":duct.migrator.ragtime-test/create-bar#66068fd2"]])))) + (let [tempfile (java.io.File/createTempFile "duct" "db") + jdbc-url (str "jdbc:sqlite:" tempfile) + config (-> base-config + (doto (ig/load-namespaces)) + (assoc-in [:duct.database/sql :jdbcUrl] jdbc-url)) + system (atom (ig/init config))] + + (testing "initial migrations" + (let [logs (-> @system ::logger :logs) + db (-> @system :duct.database/sql :datasource)] + (is (= ["ragtime_migrations" "foo" "bar"] + (map :sqlite_master/name (find-tables db)))) + (is (= [[:duct.migrator.ragtime/applying {:id "create-table-foo#52bfa531"}] + [:duct.migrator.ragtime/applying {:id "create-table-bar#3e718b28"}]] + @logs)))) + + (testing "change migration" + (let [config (assoc-in config [:duct.migrator/ragtime :migrations-file] + "test/duct/migrator/migrations2.edn") + system (swap! system (fn [sys] (ig/suspend! sys) (ig/resume config sys))) + logs (-> system ::logger :logs) + db (-> system :duct.database/sql :datasource)] + (is (= ["ragtime_migrations" "foo" "baz"] + (map :sqlite_master/name (find-tables db)))) + (is (= [[:duct.migrator.ragtime/rolling-back {:id "create-table-bar#3e718b28"}] + [:duct.migrator.ragtime/applying {:id "create-table-baz#055a605e"}]] + @logs)))) + + (testing "remove migration" + (let [config (assoc-in config [:duct.migrator/ragtime :migrations-file] + "test/duct/migrator/migrations3.edn") + system (swap! system (fn [sys] (ig/suspend! sys) (ig/resume config sys))) + logs (-> system ::logger :logs) + db (-> system :duct.database/sql :datasource)] + (is (= ["ragtime_migrations" "foo"] + (map :sqlite_master/name (find-tables db)))) + (is (= [[:duct.migrator.ragtime/rolling-back {:id "create-table-baz#055a605e"}]] + @logs)))) + + (.delete tempfile))) -(deftest string-source-test - (is (= "foo\n" (ragtime/get-string "foo\n"))) - (is (= "foo\n" (ragtime/get-string (io/resource "duct/migrator/example.txt")))) - (is (= "foo\n" (ragtime/get-string (duct/resource "duct/migrator/example.txt")))))