migration: rehash during migrate-db #81

2 files changed, 37 insertions, 18 deletions

diff --git a/src/weepinbell/migration.clj b/src/weepinbell/migration.clj
index c59910a..701ed8f 100644
--- a/src/weepinbell/migration.clj
+++ b/src/weepinbell/migration.clj
@@ -6,6 +6,7 @@
             [weepinbell.storage.edn-storage :as edns]

             [weepinbell.storage.spec]

             [weepinbell.metadata.microformats :refer [parse-microformats]]

+            [weepinbell.utils.digest :refer [digest]]

             [clojure.java.io :as io]

             [fast-edn.core :as edn]))

 

@@ -18,6 +19,15 @@
       (assoc :jf2 (parse-microformats (:microformats wm)))

       (dissoc :microformats)))

 

+(defn rehash

+  "Given a Webmention object, recalculate its hash and return a new map."

+  [wm]

+  {:pre [(map? wm) (contains? wm :webmention)]}

+  (assoc wm :webmention

+         (assoc (:webmention wm) :id

+                (digest (str (get-in wm [:webmention :source])

+                             (get-in wm [:webmention :target]))))))

+

 (defn edn->h2

   "Given an EDN file and a SQLite DB path, migrate data from that

    edn file to SQLite DB. It appends data to DB so the destination

@@ -35,7 +45,8 @@
             (println "Appending webmention" id "to database")

             (try

               (let [wm (if (contains? wm :microformats)

-                         (mf->jf2 wm) wm)]

+                         (mf->jf2 wm) wm)

+                    wm (rehash wm)]

                 (h2/insert-webmention ds wm))

               (catch Exception e

                 (println "Something unexpected happened.")

@@ -43,7 +54,7 @@
     (println "Done")))

 

 (defn- file? [file]

-  (or (string? file) 

+  (or (string? file)

       (instance? java.io.File file)

       (instance? java.net.URL file)))

 

diff --git a/test/weepinbell/migration_test.clj b/test/weepinbell/migration_test.clj
index 07a11d6..833c066 100644
--- a/test/weepinbell/migration_test.clj
+++ b/test/weepinbell/migration_test.clj
@@ -3,14 +3,16 @@
             [weepinbell.metadata.jf2-spec]

             [fast-edn.core :as edn]

             [weepinbell.storage.h2 :as h2]

+            [weepinbell.utils.digest :refer [digest]]

             [next.jdbc :as jdbc]

+            [next.jdbc.sql :as sql]

             [clojure.spec.alpha :as s]

             [clojure.test :refer [deftest testing is]]

             [clojure.java.io :as io]))

 

 (def test-source-edn (io/resource "webmention-test.edn"))

 

-(def webmention-test-edn 

+(def webmention-test-edn

   (with-open [rdr (io/reader test-source-edn)]

     (->> rdr line-seq (map edn/read-string) doall)))

 

@@ -22,36 +24,42 @@
           converted (m/mf->jf2 data)]

       (is (contains? converted :jf2))

       (is (not (contains? converted :microformats)))

-      (is (s/valid? :weepinbell.metadata.jf2-spec/jf2 

+      (is (s/valid? :weepinbell.metadata.jf2-spec/jf2

                     (:jf2 converted))))))

 

 (deftest test-migration-from-edn-to-h2

-  (testing "Migrating EDN file storage to H2 database storage"

+  (testing "Migrating EDN file storage to H2 database storage:"

     (let [target-h2 "tmp/webmention-test-h2"]

-      ;; assert migration succeed with no error

-      (is (nil? (try

-        (m/edn->h2 test-source-edn target-h2)

-        (catch Throwable _ :fail))))

-      ;; assert data count 

-      (let [ds (h2/make-datasource target-h2)

-            result (jdbc/execute! ds ["SELECT count(*) FROM webmention"])

-            len (get (first result) (keyword "count(*)"))]

-        (is (= len (count webmention-test-edn))))

+      (testing "no error or exception should be thrown"

+        (is (nil? (try

+                    (m/edn->h2 test-source-edn target-h2)

+                    (catch Throwable _ :fail)))))

+      (testing "there should be no data loss"

+        (let [ds (h2/make-datasource target-h2)

+              result (jdbc/execute! ds ["SELECT count(*) FROM webmention"])

+              len (get (first result) (keyword "count(*)"))]

+          (is (= len (count webmention-test-edn)))))

+      (testing "should recalculate hash/id of webmentions"

+        (let [ds (h2/make-datasource target-h2)

+              result (jdbc/execute! ds ["SELECT hash, wm_source, wm_target FROM webmention"])

+              match? (fn [{:keys [hash wm_source wm_target]}]

+                       (= hash (digest (str wm_source wm_target))))]

+          (is (every? match? result))))

       (io/delete-file (str target-h2 ".mv.db"))))

   (testing "Exceptions thrown during migration does not abort program"

-    (with-redefs [weepinbell.storage.h2/insert-webmention 

+    (with-redefs [weepinbell.storage.h2/insert-webmention

                   (fn [& _] (throw (Exception. "Test exception")))]

       (let [target-h2 "tmp/webmention-test-exception-h2"]

         (is (nil? (try

                     (m/edn->h2 test-source-edn target-h2)

                     (catch Exception _ :fail)))))))

   (testing "Errors thrown during migration should fail migration"

-    (with-redefs [weepinbell.storage.h2/insert-webmention 

+    (with-redefs [weepinbell.storage.h2/insert-webmention

                   (fn [& _] (throw (Error. "Test error")))]

       (let [target-h2 "tmp/webmention-test-exception-h2"]

         (is (= :fail (try

-                    (m/edn->h2 test-source-edn target-h2)

-                    (catch Error _ :fail))))))))

+                       (m/edn->h2 test-source-edn target-h2)

+                       (catch Error _ :fail))))))))

 

 (deftest test-edn-jf2ify

   (testing "Convert webmenion-test.edn to webmention-test-jf2.edn with jf2 markup"