verifier, http.client: retry timeout HTTP requests

- add weepinbell.http.client/with-retry macro
  - retry only requests that returns :error with type TimeoutException
  - retry is done asynchronously
  - with-retry should return a Future
- retry verifier requesting source and target, 3 times with 5 seconds
backoff

2 files changed, 29 insertions, 4 deletions

diff --git a/src/weepinbell/http/client.clj b/src/weepinbell/http/client.clj
index 7a3bc06..99514f4 100644
--- a/src/weepinbell/http/client.clj
+++ b/src/weepinbell/http/client.clj
@@ -4,11 +4,13 @@
   All HTTP client related changeability and complexity is containend within here,

   for instance, all Exceptions are caught inside and returned as normal value."

   (:refer-clojure :exclude [get])

-  (:require [org.httpkit.client :as hk]))

+  (:require [org.httpkit.client :as hk]

+            [taoensso.timbre :as timbre])

+  (:import [org.httpkit.client TimeoutException]))

 

 (def default-options

   {:user-agent "WebmentionBot"

-   :timeout 15000})

+   :timeout (* 20 1000)})

 

 (defn- wrap-response [response]

   (assoc response :content-length (or (get-in response [:headers "Content-Length"])

@@ -67,3 +69,25 @@
    @(head url options nil))

   ([url options callbacks]

    @(head url options callbacks)))

+

+(defn retryable? [error]

+  (let [retryables #{TimeoutException}]

+    (some #(instance? % error) retryables)))

+

+(defmacro with-retry 

+  "Retry request after failure with backoff, asynchronously."

+  [times backoff & body]

+  `(future (loop [~'current-try 0

+          ~'backoff ~backoff]

+     (let [~'result (try 

+       (deref ~@body) 

+       (catch Throwable ~'e {:error ~'e}))]

+       (if (and (:error ~'result)

+                (retryable? (:error ~'result))

+                (< ~'current-try ~times))

+         (do

+           (timbre/warn "request failed. retry after" ~'backoff "seconds")

+           (Thread/sleep (* ~'backoff 1000)) 

+           (recur (inc ~'current-try) (* 2 ~'backoff)))

+         ~'result)))))

+

diff --git a/src/weepinbell/processor/verifier.clj b/src/weepinbell/processor/verifier.clj
index 37b91be..3053b82 100644
--- a/src/weepinbell/processor/verifier.clj
+++ b/src/weepinbell/processor/verifier.clj
@@ -60,8 +60,9 @@
   (when (:ok? data)

     (let [{:keys [id wm new?]} data

           {:keys [s t]} wm

-          source-resp (client/get s {:headers {"Range" "bytes=0-1048575"}})

-          target-resp (client/head t)

+          source-resp (client/with-retry 3 5

+                        (client/get s {:headers {"Range" "bytes=0-1048575"}}))

+          target-resp (client/with-retry 3 5 (client/head t))

           verified? (verify-mention id @source-resp @target-resp s t)]

       (-> data

           (assoc :verified? verified?)