diff --git a/Makefile b/Makefile index 90672d59e..be34a8c78 100644 --- a/Makefile +++ b/Makefile @@ -147,13 +147,24 @@ test-ccl: ############################################################################### # BENCHMARKS ############################################################################### -.PHONY: benchmark-qasm +.PHONY: benchmark-qasm benchmark-nq benchmark-nq-2x benchmark-qasm: $(QUICKLISP) \ --eval "(ql:quickload :cl-quil-benchmarking)" \ --eval "(cl-quil-benchmarking::benchmark-qasm-suite)" +benchmark-nq: + $(QUICKLISP) \ + --eval "(ql:quickload :cl-quil-benchmarking)" \ + --eval "(cl-quil-benchmarking::benchmark-nq)" + +benchmark-nq-2x: + $(QUICKLISP) \ + --eval "(ql:quickload :cl-quil-benchmarking)" \ + --eval "(cl-quil-benchmarking::benchmark-nq)" \ + --eval "(cl-quil-benchmarking::benchmark-nq)" + ############################################################################### # CLEAN ############################################################################### diff --git a/benchmarking/qasm-benchmarks.lisp b/benchmarking/qasm-benchmarks.lisp index e44c0e624..48983556b 100644 --- a/benchmarking/qasm-benchmarks.lisp +++ b/benchmarking/qasm-benchmarks.lisp @@ -60,7 +60,8 @@ :do (quil::append-instruction-to-lschedule lschedule instr) :finally (return (quil::lscheduler-calculate-depth lschedule))))) -(defun benchmark-qasm-suite (&key (timeout 30)) +(defun benchmark-qasm-suite (&key (timeout 30) named) + "Run benchmarks from qasm suite. If NAMED is not nil, the specified test(s) will be the ONLY one(s) run; otherwise, all the tests are run. NAMED should be a short name (as shown in the output) of a test, either as a symbol or string, or a list thereof (i.e., matching mutiple tests), to be compared using string-equal. TIMEOUT specifies a timeout in seconds, defaulting to 30 seconds." (let ((timed-out nil)) (flet ((print-rule () (format t "+------------------+----------+-------+----------+~%"))) @@ -73,28 +74,33 @@ (quil::*addresser-use-1q-queues* t) (quil::*safe-include-directory* (asdf:system-relative-pathname :cl-quil "tests/qasm-files/"))) (dolist (file (qasm-test-files)) - (format t "| ~Va " 16 (trim-long-string (pathname-name file) 16)) - (finish-output) - (handler-case - (let ((text (alexandria:read-file-into-string file))) - (tg:gc :full t) - (bordeaux-threads:with-timeout (timeout) - (with-stopwatch elapsed-time - (multiple-value-bind (cpp swaps) - (quil::compiler-hook (quil::parse text - :originating-file file) - chip - :protoquil t - :destructive t) - (format t "| ~Vf | ~Vd | ~Vd |~%" - 8 (/ elapsed-time 1000) - 5 swaps - 8 (calculate-multiqubit-gate-depth (parsed-program-executable-code cpp))))))) - (bt:timeout () - (format t "| ~8,'>d | ????? | ???????? |~%" - timeout) - (push (pathname-name file) timed-out))) - (finish-output))) + (let ((short-name (trim-long-string (pathname-name file) 16))) + (when (or (null named) + (if (atom named) + (string-equal named short-name) + (member short-name named :test 'string-equal))) + (format t "| ~Va " 16 short-name) + (finish-output) + (handler-case + (let ((text (alexandria:read-file-into-string file))) + (tg:gc :full t) + (bordeaux-threads:with-timeout (timeout) + (with-stopwatch elapsed-time + (multiple-value-bind (cpp swaps) + (quil::compiler-hook (quil::parse text + :originating-file file) + chip + :protoquil t + :destructive t) + (format t "| ~Vf | ~Vd | ~Vd |~%" + 8 (/ elapsed-time 1000) + 5 swaps + 8 (calculate-multiqubit-gate-depth (parsed-program-executable-code cpp))))))) + (bt:timeout () + (format t "| ~8,'>d | ????? | ???????? |~%" + timeout) + (push (pathname-name file) timed-out))) + (finish-output))))) (print-rule) (terpri) (when timed-out diff --git a/benchmarking/quilc-mon-prof.lisp b/benchmarking/quilc-mon-prof.lisp new file mode 100644 index 000000000..c3b1e0534 --- /dev/null +++ b/benchmarking/quilc-mon-prof.lisp @@ -0,0 +1,186 @@ +;;;; quilc-mon-prof.lisp +;;;; +;;;; Author: Mark David + +(in-package #:cl-quil-benchmarking) + + + +;;;; Monitoring and Profiling + +;; This builds on quilc-perf.lisp and is for monitoring and profiling, +;; as opposed to simple benchmarks, and is a WIP in that we have not +;; established a makefile entry to run this. That is because, while +;; it's been useful to run for probing and experimenting in a REPL, we +;; so far lack really good theory of operation and associated modes of +;; running. We hope with time to get there. + + +(defparameter *monitor-types* + '(:mon #+sbcl :sb-sprof)) + +(defun monitor-run (program-type chip-type nq repeats monitor + &optional report-type sample-interval) + "Do REPEATS perf runs for PROGRAM-TYPE and CHIP-TYPE (as documented + at the top of QUILC-PERF module) using the specified MONITOR, which + must be one of those in the list *monitor-types*. Optional args + REPORT-TYPE and SAMPLE-INTERVAL are only relevant when MONITOR is + :SB-SPROF in which case REPORT-TYPE is passed as the same-named arg + to SB-SPROF:REPORT, and SAMPLE-INTERVAL is passed as the same-named + arg to SB-SPROF:START-PROFILING. Note as well that for :SB-SPROF + monitor, only the current thread is profiled. This returns no + useful value. It's just run in order to get output from the + monitor." + (when (not (member monitor *monitor-types*)) + (unless (null monitor) ; if so, just silently default with no warning + (warn "unrecognized monitor, should be one of ~s; using ~s" + *monitor-types* (first *monitor-types*))) + (setq monitor (first *monitor-types*))) + (let (program chip) + (format t "~2%****~%Building ~a program (nQ = ~d)... " program-type nq) + (setq program (get-or-build-benchmark-program nq program-type)) + (format t "DONE (~a).~%" program) + (format t "Building ~a chip (nQ = ~d)... " chip-type nq) + (setq chip (get-or-build-benchmark-chip nq chip-type)) + (format t "DONE (~a).~%" chip) + (format t "** Doing ~d run~p... **~%" repeats repeats) + (let* ((*package* (find-package :cl-quil)) + (thunk + #'(lambda () + (dotimes (i repeats) + (prepare-environment-for-perf-run) + (format t "#~d: Compiling program/chip ... " i) + (let* ((t1 (get-internal-run-time)) + (t2 (progn (do-one-quilc-perf-run program chip) + (get-internal-run-time))) + (elapsed-time (- t2 t1))) + (format + t + "DONE (~,2f sec compiler, ~2df sec real time w/overhead).~%" + ;; -- 2nd timing is real time, as opposed to + ;; internal run time. 'Overhead' is primarily GC + + ;; warming. + (internal-time-to-seconds elapsed-time) + (internal-time-to-seconds (- t2 t1)))))))) + (ecase monitor + (:mon (mon:monitor-form (funcall thunk))) + #+sbcl + (:sb-sprof + (progn + (sb-sprof:reset) + (sb-sprof:start-profiling + :sample-interval (or sample-interval 0.005) ; default = 0.01 + :threads (list sb-thread:*current-thread*)) + (funcall thunk) + (sb-sprof:report + :min-percent 3 + :type (ecase report-type + ((nil :flat) :flat) + (:graph :graph))))))) + (format t "** DONE with ~d runs. **~%" repeats))) + +(defun do-monitor-runs (&key start step end + repeats monitor + report-type sam sample-interval) + (or repeats (setq repeats 3)) + (loop :for nq := (or start 10) + :then (+ nq (or step 10)) + :when (> nq (or end start)) + :do (return) + :do (format t "~%**** NQ: ~d ****~%" nq) + (loop :for program-type :in *benchmark-program-types* + :do (loop :for chip-type :in *benchmark-chip-connectedness-types* + :do (monitor-run + program-type chip-type nq repeats monitor + report-type sample-interval))))) + +;; Try this on SBCL: (do-monitor-runs :monitor ':sb-sprof) + + + + +(defun do-one (nq) + (do-one-nq-program-chip nq :hadamard :fully-connected)) + +(defparameter *min-sb-sprof-perf-pct* 1) + +(defun do-one-nq-program-chip (nq program-type chip-type) + (let ((program (build-benchmark-program nq :hadamard)) + (chip (build-benchmark-chip nq :fully-connected))) + (sb-sprof:reset) + (tg:gc :full t) + (sb-sprof:start-profiling :threads (list sb-thread:*current-thread*)) + (time (benchmark-one-quilc-perf-run program chip)) + (sb-sprof:stop-profiling) + (sb-sprof:report :type :graph :min-percent *min-sb-sprof-perf-pct*) + (sb-sprof:report :type :flat :min-percent *min-sb-sprof-perf-pct*))) + +(defun do-one-mon (nq &optional program-type) + (let ((program (build-benchmark-program nq (or program-type :hadamard))) + (chip (build-benchmark-chip nq :fully-connected))) + (tg:gc :full t) + (sb-sprof:with-profiling (:max-samples 1000 + :report :flat + :loop nil + + :reset t + :sample-interval 0.01) ; default .01 + (benchmark-one-quilc-perf-run program chip)))) + + + +;;;; SB-SPROF Runs of Various Flavors + +(defparameter *default-sb-sprof-run-nqs* + '(20 50 80 110) + "Default list of nQ values for running various SB-SPROF calls below.") + +(defun sb-sprof-run () + (loop :for nq :in *default-sb-sprof-run-nqs* + :do (do-monitor-runs + :start nq + :monitor :sb-sprof + :repeats 1 + :report-type :flat))) + +(defun sb-sprof-graph () + (loop :for nq :in *default-sb-sprof-run-nqs* + :do (do-monitor-runs + :start nq + :monitor :sb-sprof + :repeats 1 + :report-type :flat))) + +(defun sb-sprof-run-high-sample () + (loop :for nq :in *default-sb-sprof-run-nqs* + :do (do-monitor-runs + :start nq + :monitor :sb-sprof + :repeats 1 + + :sample-interval 0.001 ; our default: 0.005 + :report-type :flat))) + +(defun sb-sprof-run-high-sample-plus-graph () + (loop :for nq :in *default-sb-sprof-run-nqs* + :do (do-monitor-runs + :start nq + :monitor :sb-sprof + :repeats 1 + + :sample-interval 0.001 ; our default: 0.005 + :report-type :graph))) ; our default: :flat + +(defun sb-sprof-run-mon () + (loop :for nq :in *default-sb-sprof-run-nqs* + :do (do-monitor-runs + :start nq + :monitor :mon + :repeats 1 + + :sample-interval 0.001 ; our default: 0.005 + :report-type :graph))) ; our default: :flat + +(defun one-monitor-run-bill-linear (nq) + (monitor-run :bell :linear nq 1 :mon :flat .0001)) + diff --git a/benchmarking/quilc-perf.lisp b/benchmarking/quilc-perf.lisp new file mode 100644 index 000000000..b2386d9e4 --- /dev/null +++ b/benchmarking/quilc-perf.lisp @@ -0,0 +1,307 @@ +;;;; quilc-perf.lisp +;;;; +;;;; Author: Mark David + +;; Based on qasm-benchmarks.lisp by Eric Peterson, and uses some of +;; its functionality. + +(in-package #:cl-quil-benchmarking) + + +;;; This module has benchmarking tools primarily to detect performance +;;; bottlenecks as the number of qubits (nQ, hereafter) increases. +;;; +;;; We provide a few kinds of chips and Quil programs and several +;;; methods of benchmarking as a function of nQ. +;;; +;;; The chip types: :fully-connected, :linear +;;; +;;; The program types: +;;; +;;; :static - a small rather trivial Quil program, namely: +;;; +;;; H 0; CNOT 2 0; H 1; CNOT 0 1; X 0 +;;; +;;; Note that the program is the same (static) no matter the value +;;; of nQ. +;;; +;;; :bell - the result of (qvm-app::bell-program nQ) +;;; +;;; :qft - the result of (qvm-app::qft-program nQ) +;;; +;;; :hadamard - the result of (qvm-app::hadamard-program nQ) + +;;; BENCHMARK-NQ: run the default series of nQ values on all the chip +;;; types for all the program types, printing out results on +;;; *standard-output* as CSV lines in two forms: (1) raw timings (in +;;; seconds) and (2) max-of-series style. + +(defparameter *default-benchmark-nq-start* 10 + "Starting nQ value used by benchmark-nq.") + +(defparameter *default-benchmark-nq-step* 20 + "Step to next nQ used by benchmark-nq.") + +(defparameter *default-benchmark-nq-end* 70 + "End nQ value used by benchmark-nq.") + +(defvar *benchmark-quilc-perf-series* '() + "Initially an empty list, gets set to the list of results by each + run of benchmark-nq, provided as a developer convenience for later + perusal, e.g., in a REPL. These results are in a list of sublists + of the form + + ((program-type chip-type) . timings) + + where timings is a list of lists of the form (nQ time), where time + is an integer as a multiple of internal-time-units-per-second. Each + sublist has the same length and contains the same nQ values.") + +(defun benchmark-nq () + (setq *benchmark-quilc-perf-series* + (benchmark-quilc-perf + :start *default-benchmark-nq-start* + :step *default-benchmark-nq-step* + :end *default-benchmark-nq-end*)) + (csv-timings *benchmark-quilc-perf-series*) + (terpri) + (csv-timings *benchmark-quilc-perf-series* :max-of-series-style t) + *benchmark-quilc-perf-series*) + +(defparameter *benchmark-program-types* + '(:static :bell :qft :hadamard)) + +(defparameter *benchmark-chip-connectedness-types* + '(:fully-connected :linear)) + +(defun build-benchmark-chip (nq type) + (let ((chip-spec + (ecase type + (:fully-connected + (cl-quil::build-nq-fully-connected-chip nq)) + (:linear + (cl-quil::build-nq-linear-chip nq))))) + (prepare-chip-for-benchmarking chip-spec) + chip-spec)) + +(defun build-benchmark-program (nq type) + (ecase type + (:static + (quil:parse "H 0; CNOT 2 0; H 1; CNOT 0 1; X 0")) + (:bell + (qvm-app::bell-program nq)) + (:qft + (qvm-app::qft-program nq)) + (:hadamard + (qvm-app::hadamard-program nq)))) + + +(defun benchmark-print-rule (&optional stream) + (when (null stream) + (setq stream *standard-output*)) + (format stream "+----------------------------+----------+-------+----------+~%")) + +(defun internal-time-to-seconds (internal-time-units) + "Convert integer INTERNAL-TIME-UNITS to number of seconds as float." + (/ (float internal-time-units) internal-time-units-per-second)) + +(defun prepare-environment-for-perf-run () + (tg:gc :full t)) + +(defun do-one-quilc-perf-run (program chip) + "Run compiler on PROGRAM for CHIP; see above re required pre-warm." + (quil::compiler-hook + program chip + :protoquil t + :destructive t)) + +(defun benchmark-one-quilc-perf-run (program chip) + (handler-case + (progn + (prepare-environment-for-perf-run) + (let* (cpp + swaps + (time ; in cl:internal-time-units-per-second + (let ((t1 (get-internal-run-time))) + (multiple-value-setq (cpp swaps) + (do-one-quilc-perf-run program chip)) + (- (get-internal-run-time) t1))) + (2q-depth + (calculate-multiqubit-gate-depth + (parsed-program-executable-code cpp)))) + (values time swaps 2q-depth))) + (error (condition) + (values 'error condition)))) + + + +(defparameter *enable-quilc-perf-chip-cache* t + "Enable (if true) or disable (if false) caching chips for + benchmarking/performance runs. This defaults to true. If enabled, + get-or-build-benchmark-chip uses *quilc-perf-chip-cache* to cache + chips.") + +(defvar *quilc-perf-chip-cache* (make-hash-table :test 'equal) + "Hash table whose keys are ( nQ) lists, tested using + equal, mapping to chip-specification instances created by + build-benchmark-chip for these performance runs.") + +(defun get-or-build-benchmark-chip (nq chip-type) + (if *enable-quilc-perf-chip-cache* + (let ((key (list chip-type nq))) + (or (gethash key *quilc-perf-chip-cache*) + (setf (gethash key *quilc-perf-chip-cache*) + (build-benchmark-chip nq chip-type)))) + (build-benchmark-chip nq chip-type))) + + +(defparameter *enable-quilc-perf-program-cache* t + "Enable (if true) or disable (if false) caching programs for + benchmarking/performance runs. This defaults to true. If enabled, + get-or-build-benchmark-program uses *quilc-perf-program-cache* to + cache programs.") + +(defvar *quilc-perf-program-cache* (make-hash-table :test 'equal) + "Hash table whose keys are ( nQ) lists, tested using + equal, mapping to parses created by build-benchmark-program for + these performance runs.") + +(defun get-or-build-benchmark-program (nq program-type) + (if *enable-quilc-perf-program-cache* + (let ((key (list program-type nq))) + (or (gethash key *quilc-perf-program-cache*) + (setf (gethash key *quilc-perf-program-cache*) + (build-benchmark-program nq program-type)))) + (build-benchmark-program nq program-type))) + + + +(defun get-epsilon-program () + "Get a 'null' or 'nop' program, i.e., the most minimal program that + makes the compiler go through its paces, e.g., doing any warming or + caching that may take considerable time and that would normally be + avoided on subsequent runs. For now, the program is merely \"I 0\"." + (quil:parse "I 0")) + +(defun prepare-chip-for-benchmarking (chip-spec) + "Do normal preparations for chip benchmarking, namely, by doing a + compilation on a minimal program, which presumably may result in + 'warming up' and/or caching. The idea is to NOT have time for this + dominate the results of the first program benchmarked." + (do-one-quilc-perf-run (get-epsilon-program) chip-spec)) + +;; Consider moving the above function (or some variant) to cl-quil and +;; then making it a standard or optional part of building a chip +;; and/or at least calling it from places, e.g., from the app. + + +(defun benchmark-one-quilc-perf (nq program-type chip-type) + (let* ((program (get-or-build-benchmark-program nq program-type)) + (chip (get-or-build-benchmark-chip nq chip-type))) + (benchmark-one-quilc-perf-run program chip))) + +(defun benchmark-quilc-perf-series (start step end program-type chip-type) + (benchmark-print-rule) + (format t "| NAME | TIME (s) | SWAPS | 2Q DEPTH |~%") + (benchmark-print-rule) + (loop :with time :with swaps :with 2q-depth + :for nq := start :then (+ nq step) + :while (<= nq end) + :do (format t "| ~Va " 26 (format nil "~a/~a ~d" program-type chip-type nq)) + (finish-output) + (multiple-value-setq (time swaps 2q-depth) + (benchmark-one-quilc-perf nq program-type chip-type)) + (cond + ((eq time 'error) + (print (list 'error time))) ; for now (do something better?) + (t + (format t "| ~V,2f | ~Vd | ~Vd |~%" + 8 (internal-time-to-seconds time) + 5 swaps + 8 2q-depth))) + :collect `(,nq ,time) :into results + :finally + (benchmark-print-rule) + (return results))) + +(defun benchmark-quilc-perf (&key start step end skip exclusively) + ;; (print-configuration) ; a bit messy, so leave off for now + (loop :with start := (or start 10) + :with step := (or step 10) + :with end := (or end 30) + :for chip-type :in *benchmark-chip-connectedness-types* + :nconc (loop :for program-type :in *benchmark-program-types* + :as skip-this + := (and (member program-type skip :key #'first) + (member chip-type skip :key #'second)) + :as exclusively-this + := (and (member program-type exclusively :key #'first) + (member chip-type exclusively :key #'second)) + :if skip-this + :do (format t "(** Skipping ~a/~a **)~%" program-type chip-type) + :else :nconc (and (or (null exclusively) + exclusively-this) + (let ((perf-series + (benchmark-quilc-perf-series + start step end + program-type chip-type))) + `(((,Program-type ,chip-type) + ,@perf-series))))))) + +(defun csv-timings (x &key max-of-series-style) + "Output timing data X as comma-separated values like so + +header row: + \"nQ\", program/chip-1, ..., program/chip-n +data rows: + ,