Add delete benchmarking techniques

Author: dgrnbrg

src/tree/bench.clj

@@ -19,6 +19,7 @@
   [b backend]
   {:structure (core/b-tree (core/->Config b b 0))
    :insert core/insert
+   :delete core/delete
    :flush (fn [x] (core/flush-tree x backend))})
 
 (defn msg-b-tree
@@ -27,13 +28,15 @@
   (let [sqrt-b (long (Math/sqrt b))]
     {:structure (core/b-tree(core/->Config sqrt-b b (- b sqrt-b)))
      :insert msg/insert
+     :delete msg/delete
      :flush (fn [x] (core/flush-tree x backend))}))
 
 (defn sorted-set-repr
   "Returns a sorted set"
   []
   {:structure (sorted-set)
    :insert conj
+   :delete disj
    :flush (fn [set]
             {:tree set
              :stats (atom {})})})
@@ -58,20 +61,24 @@
    flush-freq is the number of keys per flush
    datastruct is the test data structure
    out is the stream to write the results to (as well as stdout)"
-  [n dataset flush-freq datastruct out]
-  (let [{:keys [structure insert flush]} datastruct]
-    (loop [[x & data] (take n (:data dataset))
+  [n dataset flush-freq datastruct out delete-xform]
+  (let [{:keys [structure delete insert flush]} datastruct
+        dataset (take n (:data dataset))]
+    (loop [[x & data] dataset
            t 0
            tree structure
            last-flush nil
            i 0
+           inserting? true
            outputs []]
       (let [i' (inc i)
             {flushed-tree :tree
              stats :stats} (when (zero? (mod i' flush-freq))
                              (flush tree))
             before (System/nanoTime)
-            tree' (insert (or flushed-tree tree) x)
+            tree' (if inserting?
+                    (insert (or flushed-tree tree) x)
+                    (delete (or flushed-tree tree) x))
             after (System/nanoTime) 
             log-inserts (zero? (mod i' (quot n 100)))
             updated-outputs (atom outputs)]
@@ -80,35 +87,53 @@
             (let [ks (sort (keys last-flush))
                   avg-ns (float (/ t (quot n 100)))]
               (when (zero? i)
-                (println (str "elements,insert_took_avg_ns,"
+                (println (str "elements,op,insert_took_avg_ns,"
                               (str/join "," ks))))
-              (println (str i' "," avg-ns
+              (println (str i' "," (if inserting? "insert" "delete") "," avg-ns
                             "," (str/join "," (map #(get last-flush %) ks))))
               (swap! updated-outputs conj (-> (into {} last-flush)
-                                              (assoc :avg-ns avg-ns
-                                                     :n i))))))
-        (if (seq data)
+                                              (assoc :ins-avg-ns avg-ns
+                                                     (if inserting?
+                                                       :insert
+                                                       :delete) true
+                                                     :n i'))))))
+        (cond
+          (seq data)
           (recur data
                  (if log-inserts
                    0
                    (+ t (- after before)))
                  tree'
                  (if stats (merge-with + last-flush @stats) last-flush)
                  i'
+                 inserting?
                  @updated-outputs)
+          inserting?
+          (recur (delete-xform dataset)
+                 0
+                 tree'
+                 nil
+                 i'
+                 false
+                 @updated-outputs)
+          :else
           @updated-outputs)))))
 
 (def options
   [["-n" "--num-operations NUM_OPS" "The number of elements that will be applied to the data structure"
     :default 100000
     :parse-fn #(Long. %)
     :validate [pos? "n must be positive"]]
-   [nil "--data-structure" "Which data structure to run the test on"
+   [nil "--data-structure STRUCT" "Which data structure to run the test on"
     :default "fractal"
     :validate [#(#{"fractal" "b-tree" "sorted-set"} %) "Data structure must be fractal, b-tree, or sorted set"]]
    [nil "--backend testing" "Runs the benchmark with the specified backend"
     :default "testing"
     :validate [#(#{"redis" "testing"} %) "Backend must be redis or testing"]] 
+   ["-d" "--delete-pattern PATTERN" "Specifies how the operations will be reordered on delete"
+    :default "forward"
+    :validate [#(#{"forward" "reverse" "shuffle" "zero"} %) "Incorrect delete pattern"]
+    ]
    [nil "--sorted-set" "Runs the benchmarks on a sorted set"]
    ["-b" "--tree-width" "Determines the width of the trees. Fractal trees use sqrt(b) child pointers; the rest is for messages."
     :default 300
@@ -138,35 +163,45 @@
              "Options:"
              options-summary
              ""
+             "Delete patterns:"
+             "forward: we delete the elements in the order they were inserted"
+             "reverse: we delete the elements in the reverse order they were inserted"
+             "shuffle: we delete the elements in a random order"
+             "zero: we repeatedly attempt to delete 0, thus never actually deleting"
+             ""
              "Backends:"
              "testing: this backend serializes nothing, just using an extra indirection"
              "redis: this backend uses a local redis server"]))
 
 (defn make-template-for-one-tree-freq-combo
-  [list-of-benchmark-results]
-    (clojure.pprint/pprint list-of-benchmark-results)
+  [list-of-benchmark-results filter-by]
+  ;(clojure.pprint/pprint list-of-benchmark-results)
   (assert (= 2 (count list-of-benchmark-results)) "Should be random and ordered")
   (let [indexed (group-by :ds list-of-benchmark-results)]
-    (map #(vector (:n %1) (:avg-ns %1) (:writes %1) (:avg-ns %2) (:writes %2))
-         (:results (first (get indexed "in-order")))
-         (:results (first (get indexed "random"))))))
+    (map #(vector (:n %1) (:ins-avg-ns %1) (:writes %1) (:ins-avg-ns %2) (:writes %2))
+         (filter filter-by (:results (first (get indexed "in-order"))))
+         (filter filter-by (:results (first (get indexed "random")))))))
 
 (defn template-one-sheet
   [pair-of-results-for-one-ds-config]
-  (let [{:keys [tree ds freq n b results]} (first pair-of-results-for-one-ds-config)
+  (let [{:keys [tree ds freq n b results delete-pattern]}
+        (first pair-of-results-for-one-ds-config)
         x {;:sheet-name (str (name tree) " " ds " flushed every " freq)
            0 [["Data Structure" (name tree) "" "n" n]]
-           1 [["Flush Frequency" freq "" "b" b]]
-           [5 104] (make-template-for-one-tree-freq-combo pair-of-results-for-one-ds-config)}]
+           1 [["Flush Frequency" freq "" "b" b "" "delete pattern" delete-pattern]]
+           [5 18] (make-template-for-one-tree-freq-combo pair-of-results-for-one-ds-config :insert)
+           [22 35] (make-template-for-one-tree-freq-combo pair-of-results-for-one-ds-config :delete)}]
     x))
 
 (defn -main
   [root & args]
-  (doseq [args (->> args
-                    (partition-by #(= % "--"))
-                    (map-indexed vector)
-                    (filter (comp even? first))
-                    (map second))]
+  (doseq [args (or (->> args
+                        (partition-by #(= % "--"))
+                        (map-indexed vector)
+                        (filter (comp even? first))
+                        (map second)
+                        (seq))
+                   [[]])] ; always do one iteration
     (let [{:keys [options arguments errors summary]} (parse-opts args options)
           tree-to-test (atom {})
           results (atom [])]
@@ -178,38 +213,49 @@
                       "testing" (core/->TestingBackend)
                       "redis" (redis/->RedisBackend
                                 #_(java.util.concurrent.Executors/newFixedThreadPool 4)))
+            delete-xform (case (:delete-pattern options)
+                           "forward" identity
+                           "reverse" reverse
+                           "shuffle" shuffle
+                           "zero" #(repeat (count %) 0.0))
             [tree-name structure]
             (case (:data-structure options)
               "b-tree" ["b-tree" (core-b-tree (:tree-width options) backend)]
               "fractal" ["fractal" (msg-b-tree (:tree-width options) backend)]
               "sorted-set" ["sorted-set" (sorted-set-repr)])
             flush-freq (:flush-freq options)
             codename (str tree-name
-                          "_flush"
+                          "__flush_"
                           flush-freq
-                          "_b"
+                          "__b_"
                           (:tree-width options)
-                          "_"
+                          "__"
                           (:backend options)
-                          "_n"
-                          (:num-operations options))]
+                          "__n_"
+                          (:num-operations options)
+                          "__del_"
+                          (:delete-pattern options))]
         (doseq [ds (generate-test-datasets)
                 :let [codename (str codename
                                     "_"
                                     (:name ds))
                       out (create-output-dir
                             root
-                            codename)]]
-                (println "Doing" codename)
+                            codename)
+                      _ (println "Doing" codename)
+                      bench-res (benchmark (:num-operations options) ds flush-freq structure out delete-xform)]]
           (swap! results conj 
                  {:tree tree-name
                   :ds (:name ds)
                   :freq flush-freq
                   :n (:num-operations options)
                   :b (:tree-width options)
-                  :results (benchmark (:num-operations options) ds flush-freq structure out)}))
-      (excel/render-to-file
-        "template_benchmark.xlsx"
-        (.getPath (File. root (str codename "_analysis.xlsx")))
-        {"SingleDS"
-         (template-one-sheet @results)})))))
+                  :delete-pattern (:delete-pattern options)
+                  :results bench-res}))
+        ;(println "results")
+        ;(clojure.pprint/pprint @results)
+        (excel/render-to-file
+          "template_benchmark.xlsx"
+          (.getPath (File. root (str codename "_analysis.xlsx")))
+          {"SingleDS"
+           (template-one-sheet @results)})))))