add cl-quil-benchmarking system quilc-perf for perf. benchmarking

This is part of issue "Investigate performance at 72-150 qubits #565".

new modules:

  benchmarking/quilc-perf.lisp - primarily exports benchmark-nq
    function to get timings as nQ increases for various chip and
    program types. This also introduces a "chip preparation" step
    before running any benchmarks. This removes any overhead from
    running the first compile on a chip out of the first benchmark
    run. Currently, there is significant but not tremendous amount of
    initial overhead, but a PR under consideration might add
    significant additional preparation overhead, and these should not
    be be charged to the first run.

  benchmarking/quilc-mon-prof.lisp - builds on quilc-perf.lisp and is
    for monitoring and profiling, as opposed to simple benchmarks, and
    is a WIP of sorts in that, while it's been useful to run for
    probing and experimenting in a REPL, we ;; so far lack really good
    theory of operation and associated modes of ;; running. We hope
    with time to get there. Also, note that some of this is SBCL-only.

Makefile - add targets to run benchmark-nq a couple of ways: `make
  benchmark-nq` calls benchmark-nq one time, and `make
  benchmark-nq-2x` calls it twice. The idea is to capture the
  sometimes significant jump in performance observed between a first
  and second run within the same Lisp environment, due to the effects
  of "warming", caching, and the like.

cl-quil-benchmarking.asd - add new module quilc-perf and depend on 3
  additional systems: metering, qvm-app, sb-sprof

Author: David, Mark H

Makefile

@@ -147,13 +147,24 @@ test-ccl:
 ###############################################################################
 # BENCHMARKS
 ###############################################################################
-.PHONY: benchmark-qasm
+.PHONY: benchmark-qasm benchmark-nq benchmark-nq-2x
 
 benchmark-qasm:
 	$(QUICKLISP) \
 		--eval "(ql:quickload :cl-quil-benchmarking)" \
 		--eval "(cl-quil-benchmarking::benchmark-qasm-suite)"
 
+benchmark-nq:
+	$(QUICKLISP) \
+		--eval "(ql:quickload :cl-quil-benchmarking)" \
+		--eval "(cl-quil-benchmarking::benchmark-nq)"
+
+benchmark-nq-2x:
+	$(QUICKLISP) \
+		--eval "(ql:quickload :cl-quil-benchmarking)" \
+		--eval "(cl-quil-benchmarking::benchmark-nq)" \
+		--eval "(cl-quil-benchmarking::benchmark-nq)"
+
 ###############################################################################
 # CLEAN
 ###############################################################################

benchmarking/qasm-benchmarks.lisp

@@ -60,7 +60,8 @@
             :do (quil::append-instruction-to-lschedule lschedule instr)
           :finally (return (quil::lscheduler-calculate-depth lschedule)))))
 
-(defun benchmark-qasm-suite (&key (timeout 30))
+(defun benchmark-qasm-suite (&key (timeout 30) named)
+  "Run benchmarks from qasm suite. If NAMED is not nil, the specified test(s) will be the ONLY one(s) run; otherwise, all the tests are run. NAMED should be a short name (as shown in the output) of a test, either as a symbol or string, or a list thereof (i.e., matching mutiple tests), to be compared using string-equal. TIMEOUT specifies a timeout in seconds, defaulting to 30 seconds."
   (let ((timed-out nil))
     (flet ((print-rule ()
              (format t "+------------------+----------+-------+----------+~%")))
@@ -73,28 +74,33 @@
             (quil::*addresser-use-1q-queues* t)
             (quil::*safe-include-directory* (asdf:system-relative-pathname :cl-quil "tests/qasm-files/")))
         (dolist (file (qasm-test-files))
-          (format t "| ~Va " 16 (trim-long-string (pathname-name file) 16))
-          (finish-output)
-          (handler-case
-              (let ((text (alexandria:read-file-into-string file)))
-                (tg:gc :full t)
-                (bordeaux-threads:with-timeout (timeout)
-                  (with-stopwatch elapsed-time
-                    (multiple-value-bind (cpp swaps)
-                        (quil::compiler-hook (quil::parse text
-                                                          :originating-file file)
-                                             chip
-                                             :protoquil t
-                                             :destructive t)
-                      (format t "| ~Vf | ~Vd | ~Vd |~%"
-                              8 (/ elapsed-time 1000)
-                              5 swaps
-                              8 (calculate-multiqubit-gate-depth (parsed-program-executable-code cpp)))))))
-            (bt:timeout ()
-              (format t "| ~8,'>d | ????? | ???????? |~%"
-                      timeout)
-              (push (pathname-name file) timed-out)))
-          (finish-output)))
+          (let ((short-name (trim-long-string (pathname-name file) 16)))
+            (when (or (null named)
+                      (if (atom named)
+                          (string-equal named short-name)
+                          (member short-name named :test 'string-equal)))
+              (format t "| ~Va " 16 short-name)
+              (finish-output)
+              (handler-case
+                  (let ((text (alexandria:read-file-into-string file)))
+                    (tg:gc :full t)
+                    (bordeaux-threads:with-timeout (timeout)
+                      (with-stopwatch elapsed-time
+                        (multiple-value-bind (cpp swaps)
+                            (quil::compiler-hook (quil::parse text
+                                                              :originating-file file)
+                                                 chip
+                                                 :protoquil t
+                                                 :destructive t)
+                          (format t "| ~Vf | ~Vd | ~Vd |~%"
+                                  8 (/ elapsed-time 1000)
+                                  5 swaps
+                                  8 (calculate-multiqubit-gate-depth (parsed-program-executable-code cpp)))))))
+                (bt:timeout ()
+                  (format t "| ~8,'>d | ????? | ???????? |~%"
+                          timeout)
+                  (push (pathname-name file) timed-out)))
+              (finish-output)))))
       (print-rule)
       (terpri)
       (when timed-out

benchmarking/quilc-mon-prof.lisp

@@ -0,0 +1,186 @@
+;;;; quilc-mon-prof.lisp
+;;;;
+;;;; Author: Mark David
+
+(in-package #:cl-quil-benchmarking)
+
+
+
+;;;; Monitoring and Profiling
+
+;; This builds on quilc-perf.lisp and is for monitoring and profiling,
+;; as opposed to simple benchmarks, and is a WIP in that we have not
+;; established a makefile entry to run this. That is because, while
+;; it's been useful to run for probing and experimenting in a REPL, we
+;; so far lack really good theory of operation and associated modes of
+;; running. We hope with time to get there.
+
+
+(defparameter *monitor-types*
+  '(:mon #+sbcl :sb-sprof))
+
+(defun monitor-run (program-type chip-type nq repeats monitor
+                    &optional report-type sample-interval)
+  "Do REPEATS perf runs for PROGRAM-TYPE and CHIP-TYPE (as documented
+   at the top of QUILC-PERF module) using the specified MONITOR, which
+   must be one of those in the list *monitor-types*.  Optional args
+   REPORT-TYPE and SAMPLE-INTERVAL are only relevant when MONITOR is
+   :SB-SPROF in which case REPORT-TYPE is passed as the same-named arg
+   to SB-SPROF:REPORT, and SAMPLE-INTERVAL is passed as the same-named
+   arg to SB-SPROF:START-PROFILING. Note as well that for :SB-SPROF
+   monitor, only the current thread is profiled. This returns no
+   useful value. It's just run in order to get output from the
+   monitor."
+  (when (not (member monitor *monitor-types*))
+    (unless (null monitor) ; if so, just silently default with no warning
+      (warn "unrecognized monitor, should be one of ~s; using ~s"
+            *monitor-types* (first *monitor-types*)))
+    (setq monitor (first *monitor-types*)))
+  (let (program chip)
+    (format t "~2%****~%Building ~a program (nQ = ~d)... " program-type nq)
+    (setq program (get-or-build-benchmark-program nq program-type))
+    (format t "DONE (~a).~%" program)
+    (format t "Building ~a chip (nQ = ~d)... " chip-type nq)
+    (setq chip (get-or-build-benchmark-chip nq chip-type))
+    (format t "DONE (~a).~%" chip)
+    (format t "** Doing ~d run~p... **~%" repeats repeats)
+    (let* ((*package* (find-package :cl-quil))
+           (thunk
+             #'(lambda ()                 
+                 (dotimes (i repeats)
+                   (prepare-environment-for-perf-run)
+                   (format t "#~d: Compiling program/chip ... " i)
+                   (let* ((t1 (get-internal-run-time))
+                          (t2 (progn (do-one-quilc-perf-run program chip)
+                                     (get-internal-run-time)))
+                          (elapsed-time (- t2 t1)))
+                     (format
+                      t
+                      "DONE (~,2f sec compiler, ~2df sec real time w/overhead).~%"
+                      ;; -- 2nd timing is real time, as opposed to
+                      ;; internal run time. 'Overhead' is primarily GC +
+                      ;; warming.
+                      (internal-time-to-seconds elapsed-time)
+                      (internal-time-to-seconds (- t2 t1))))))))
+      (ecase monitor
+        (:mon (mon:monitor-form (funcall thunk)))
+        #+sbcl
+        (:sb-sprof
+         (progn
+           (sb-sprof:reset)
+           (sb-sprof:start-profiling
+            :sample-interval (or sample-interval 0.005)    ; default = 0.01
+            :threads (list sb-thread:*current-thread*))
+           (funcall thunk)
+           (sb-sprof:report
+            :min-percent 3
+            :type (ecase report-type
+                    ((nil :flat) :flat)
+                    (:graph :graph)))))))
+    (format t "** DONE with ~d runs. **~%" repeats)))
+
+(defun do-monitor-runs (&key start step end
+                             repeats monitor
+                             report-type sam sample-interval)
+  (or repeats (setq repeats 3))
+  (loop :for nq := (or start 10)
+          :then (+ nq (or step 10))
+        :when (> nq (or end start))
+          :do (return)
+        :do (format t "~%**** NQ: ~d ****~%" nq)
+            (loop :for program-type :in *benchmark-program-types*
+                  :do (loop :for chip-type :in *benchmark-chip-connectedness-types*
+                            :do (monitor-run
+                                 program-type chip-type nq repeats monitor 
+                                 report-type sample-interval)))))
+
+;; Try this on SBCL: (do-monitor-runs :monitor ':sb-sprof) 
+
+
+
+
+(defun do-one (nq)
+  (do-one-nq-program-chip nq :hadamard :fully-connected))
+
+(defparameter *min-sb-sprof-perf-pct* 1)
+
+(defun do-one-nq-program-chip (nq program-type chip-type)
+  (let ((program (build-benchmark-program nq :hadamard))
+        (chip (build-benchmark-chip nq :fully-connected)))
+    (sb-sprof:reset)
+    (tg:gc :full t)
+    (sb-sprof:start-profiling :threads (list sb-thread:*current-thread*))
+    (time (benchmark-one-quilc-perf-run program chip))
+    (sb-sprof:stop-profiling)
+    (sb-sprof:report :type :graph :min-percent *min-sb-sprof-perf-pct*)
+    (sb-sprof:report :type :flat :min-percent *min-sb-sprof-perf-pct*)))
+
+(defun do-one-mon (nq &optional program-type)
+  (let ((program (build-benchmark-program nq (or program-type :hadamard)))
+        (chip (build-benchmark-chip nq :fully-connected)))
+    (tg:gc :full t)
+    (sb-sprof:with-profiling (:max-samples 1000
+                              :report :flat
+                              :loop nil
+                                   
+                              :reset t
+                              :sample-interval 0.01) ; default .01
+      (benchmark-one-quilc-perf-run program chip))))
+
+
+
+;;;; SB-SPROF Runs of Various Flavors
+
+(defparameter *default-sb-sprof-run-nqs*
+  '(20 50 80 110)
+  "Default list of nQ values for running various SB-SPROF calls below.")
+
+(defun sb-sprof-run ()
+  (loop :for nq :in *default-sb-sprof-run-nqs*
+        :do (do-monitor-runs
+              :start nq 
+              :monitor :sb-sprof
+              :repeats 1
+              :report-type :flat)))
+
+(defun sb-sprof-graph ()
+  (loop :for nq :in *default-sb-sprof-run-nqs*
+        :do (do-monitor-runs
+              :start nq 
+              :monitor :sb-sprof
+              :repeats 1
+              :report-type :flat)))
+
+(defun sb-sprof-run-high-sample ()
+  (loop :for nq :in *default-sb-sprof-run-nqs*
+        :do (do-monitor-runs
+              :start nq 
+              :monitor :sb-sprof
+              :repeats 1
+             
+              :sample-interval 0.001    ; our default: 0.005
+              :report-type :flat)))
+
+(defun sb-sprof-run-high-sample-plus-graph ()
+  (loop :for nq :in *default-sb-sprof-run-nqs*
+        :do (do-monitor-runs
+              :start nq 
+              :monitor :sb-sprof
+              :repeats 1
+             
+              :sample-interval 0.001    ; our default: 0.005
+              :report-type :graph)))     ; our default: :flat
+
+(defun sb-sprof-run-mon ()
+  (loop :for nq :in *default-sb-sprof-run-nqs*
+        :do (do-monitor-runs
+              :start nq 
+              :monitor :mon
+              :repeats 1
+             
+              :sample-interval 0.001    ; our default: 0.005
+              :report-type :graph)))     ; our default: :flat
+
+(defun one-monitor-run-bill-linear (nq)
+  (monitor-run :bell :linear nq 1 :mon :flat .0001))
+