1 ;;; 8sync --- Asynchronous programming for Guile
2 ;;; Copyright © 2015, 2016, 2017 Christopher Allan Webber <cwebber@dustycloud.org>
4 ;;; This file is part of 8sync.
6 ;;; 8sync is free software: you can redistribute it and/or modify it
7 ;;; under the terms of the GNU Lesser General Public License as
8 ;;; published by the Free Software Foundation, either version 3 of the
9 ;;; License, or (at your option) any later version.
11 ;;; 8sync is distributed in the hope that it will be useful,
12 ;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
13 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 ;;; GNU Lesser General Public License for more details.
16 ;;; You should have received a copy of the GNU Lesser General Public
17 ;;; License along with 8sync. If not, see <http://www.gnu.org/licenses/>.
19 (define-module (8sync agenda)
20 #:use-module (srfi srfi-1)
21 #:use-module (srfi srfi-9)
22 #:use-module (ice-9 q)
23 #:use-module (ice-9 match)
24 #:use-module (ice-9 receive)
25 #:use-module (ice-9 suspendable-ports)
28 agenda-queue agenda-prompt-tag
29 agenda-read-port-map agenda-write-port-map
37 make-schedule schedule?
38 schedule-add! schedule-empty?
42 schedule-segments-split schedule-extract-until!
43 add-segments-contents-to-queue!
46 make-run-request run-request?
47 run-request-proc run-request-when
49 run-it wrap wrap-apply run run-at run-delay
54 ;; used for introspecting the error, but a method for making
57 wrapped-exception-key wrapped-exception-args
58 wrapped-exception-stacks
60 print-error-and-continue
64 %current-agenda-prompt
65 run-agenda agenda-run-once!))
71 ;;; The agenda consists of:
72 ;;; - a queue of immediate items to handle
73 ;;; - sheduled future events to be added to a future queue
74 ;;; - a tag by which running processes can escape for some asynchronous
75 ;;; operation (from which they can be returned later)
76 ;;; - a mapping of ports to various handler procedures
78 ;;; @@: Is this next part deprecated?
79 ;;; The goal, maybe eventually, is for this all to be immutable and functional.
80 ;;; However, we aren't there yet. Some tricky things:
81 ;;; - The schedule needs to be immutable, yet reasonably efficient.
82 ;;; - Need to use immutable queues (ijp's pfds library?)
83 ;;; - Modeling reading from ports as something repeatable,
84 ;;; and with reasonable separation from functional components?
86 ;; TODO: Tear out the immutable agenda aspect until we're actually ready
88 (define-record-type <agenda>
89 (make-agenda-intern queue prompt-tag
90 read-port-map write-port-map
91 schedule catch-handler pre-unwind-handler)
93 (queue agenda-queue set-agenda-queue!)
94 (prompt-tag agenda-prompt-tag)
95 (read-port-map agenda-read-port-map)
96 (write-port-map agenda-write-port-map)
97 (schedule agenda-schedule)
98 (catch-handler agenda-catch-handler)
99 (pre-unwind-handler agenda-pre-unwind-handler))
101 (define (make-async-prompt-tag)
102 "Make an async prompt tag for an agenda.
104 Generally done automatically for the user through (make-agenda)."
105 (make-prompt-tag "prompt"))
107 (define* (make-agenda #:key
109 (prompt (make-prompt-tag))
110 (read-port-map (make-hash-table))
111 (write-port-map (make-hash-table))
112 (schedule (make-schedule))
114 (pre-unwind-handler print-error-and-continue))
115 ;; TODO: document arguments
116 "Make a fresh agenda."
117 (make-agenda-intern queue prompt
118 read-port-map write-port-map
119 schedule catch-handler pre-unwind-handler))
121 ;; helper for making queues for an agenda
122 (define (list->q lst)
123 "Makes a queue composed of LST items"
131 (define (make-q* . args)
132 "Makes a queue and populates it with this invocation's ARGS"
139 ;;; This is where we handle timed events for the future
141 ;; This section totally borrows from the ideas in SICP
144 ;; NOTE: time is a cons of (seconds . microseconds)
146 (define-record-type <time-segment>
147 (make-time-segment-intern time queue)
149 (time time-segment-time)
150 (queue time-segment-queue))
152 (define* (make-time-segment time #:optional (queue (make-q)))
153 "Make a time segment of TIME and QUEUE
155 No automatic conversion is done, so you might have to
156 run (time-segment-right-format) first."
157 (make-time-segment-intern time queue))
159 (define (time< time1 time2)
160 "Check if TIME1 is less than TIME2"
161 (cond ((< (car time1)
171 (define (time= time1 time2)
172 "Check whether TIME1 and TIME2 are equivalent"
173 (and (= (car time1) (car time2))
174 (= (cdr time1) (cdr time2))))
176 (define (time<= time1 time2)
177 "Check if TIME1 is less than or equal to TIME2"
178 (or (time< time1 time2)
179 (time= time1 time2)))
181 ;; @@: Maybe we should use floor/ here?
182 (define (time-carry-correct time)
183 "Corrects/handles time microsecond carry.
184 Will produce (0 . 0) instead of a negative number, if needed."
185 (cond ((>= (cdr time) 1000000)
188 (- (cdr time) 1000000)))
194 (+ (cdr time) 1000000))))
197 (define (time-minus time1 time2)
198 "Subtract TIME2 from TIME1"
200 (cons (- (car time1) (car time2))
201 (- (cdr time1) (cdr time2)))))
204 (define (time-plus time1 time2)
205 "Add TIME1 and TIME2"
207 (cons (+ (car time1) (car time2))
208 (+ (cdr time1) (cdr time2)))))
210 (define-record-type <schedule>
211 (make-schedule-intern segments)
213 (segments schedule-segments set-schedule-segments!))
215 (define* (make-schedule #:optional segments)
216 "Make a schedule, optionally pre-composed of SEGMENTS"
217 (make-schedule-intern (or segments '())))
219 (define (schedule-soonest-time schedule)
220 "Return a cons of (sec . usec) for next time segement, or #f if none"
221 (let ((segments (schedule-segments schedule)))
222 (if (eq? segments '())
224 (time-segment-time (car segments)))))
226 ;; TODO: This code is reasonably easy to read but it
227 ;; mutates AND is worst case of O(n) in both space and time :(
228 ;; but at least it'll be reasonably easy to refactor to
229 ;; a more functional setup?
230 (define (schedule-add! schedule time proc)
231 "Mutate SCHEDULE, adding PROC at an appropriate time segment for TIME"
232 (define (new-time-segment)
234 (make-time-segment time)))
235 (enq! (time-segment-queue new-segment) proc)
237 (define (loop segments)
238 (define (segment-equals-time? segment)
239 (time= time (time-segment-time segment)))
241 (define (segment-more-than-time? segment)
242 (time< time (time-segment-time segment)))
244 ;; We could switch this out to be more mutate'y
245 ;; and avoid the O(n) of space... is that over-optimizing?
247 ;; If we're at the end of the list, time to make a new
249 ('() (cons (new-time-segment) '()))
250 ;; If the segment's time is exactly our time, good news
251 ;; everyone! Let's append our stuff to its queue
252 (((? segment-equals-time? first) rest ...)
253 (enq! (time-segment-queue first) proc)
255 ;; If the first segment is more than our time,
256 ;; ours belongs before this one, so add it and
257 ;; start consing our way back
258 (((? segment-more-than-time? first) rest ...)
259 (cons (new-time-segment) segments))
260 ;; Otherwise, build up recursive result
262 (cons first (loop rest)))))
263 (set-schedule-segments!
265 (loop (schedule-segments schedule))))
267 (define (schedule-empty? schedule)
268 "Check if the SCHEDULE is currently empty"
269 (eq? (schedule-segments schedule) '()))
271 (define (schedule-segments-split schedule time)
272 "Does a multiple value return of time segments before/at and after TIME"
273 (define (segment-is-now? segment)
274 (time= (time-segment-time segment) time))
275 (define (segment-is-before-now? segment)
276 (time< (time-segment-time segment) time))
278 (let loop ((segments-before '())
279 (segments-left (schedule-segments schedule)))
281 ;; end of the line, return
283 (values (reverse segments-before) '()))
285 ;; It's right now, so time to stop, but include this one in before
286 ;; but otherwise return
287 (((? segment-is-now? first) rest ...)
288 (values (reverse (cons first segments-before)) rest))
290 ;; This is prior or at now, so add it and keep going
291 (((? segment-is-before-now? first) rest ...)
292 (loop (cons first segments-before) rest))
294 ;; Otherwise it's past now, just return what we have
296 (values segments-before segments-after)))))
298 (define (schedule-extract-until! schedule time)
299 "Extract all segments until TIME from SCHEDULE, and pop old segments off"
300 (receive (segments-before segments-after)
301 (schedule-segments-split schedule time)
302 (set-schedule-segments! schedule segments-after)
305 (define (add-segments-contents-to-queue! segments queue)
308 (let ((seg-queue (time-segment-queue segment)))
309 (while (not (q-empty? seg-queue))
310 (enq! queue (deq! seg-queue)))))
315 ;;; Request to run stuff
316 ;;; ====================
318 (define-record-type <run-request>
319 (make-run-request proc when)
321 (proc run-request-proc)
322 (when run-request-when))
324 (define* (run-it proc #:optional when)
325 "Make a request to run PROC (possibly at WHEN)"
326 (make-run-request proc when))
328 (define-syntax-rule (wrap body ...)
329 "Wrap contents in a procedure"
333 (define-syntax-rule (wrap-apply body)
334 "Wrap possibly multi-value function in a procedure, applies all arguments"
339 ;; @@: Do we really want `body ...' here?
340 ;; what about just `body'?
341 (define-syntax-rule (run body ...)
342 "Run everything in BODY but wrap in a convenient procedure"
343 (make-run-request (wrap body ...) #f))
345 (define-syntax-rule (run-at body ... when)
347 (make-run-request (wrap body ...) when))
349 (define-syntax-rule (run-delay body ... delay-time)
350 "Run BODY at DELAY-TIME time from now"
351 (make-run-request (wrap body ...) (delayed-time delay-time)))
353 ;;; Procedures that are delimited continuations being resumed are
354 ;;; wrapped in a <kontinue>. This is so that we don't accidentally
355 ;;; wrap in another catch statement every time we resume them, which
358 (define-record-type <kontinue>
361 (kont kontinue-kont))
365 ;;; Asynchronous escape to run things
366 ;;; =================================
368 (define-syntax-rule (8sync-abort-to-prompt async-request)
369 (abort-to-prompt (%current-agenda-prompt)
372 ;; Async port request and run-request meta-requests
373 (define (make-async-request proc)
374 "Wrap PROC in an async-request
376 The purpose of this is to make sure that users don't accidentally
377 return the wrong thing via (8sync) and trip themselves up."
378 (cons '*async-request* proc))
380 (define (setup-async-request resume-kont async-request)
381 "Complete an async request for agenda-run-once's continuation handling"
383 (('*async-request* . async-setup-proc)
384 (async-setup-proc resume-kont))
385 ;; TODO: deliver more helpful errors depending on what the user
387 (_ (throw 'invalid-async-request
388 "Invalid request passed back via an (8sync) procedure."
391 (define-syntax-rule (8sync body ...)
392 "Run body asynchronously but ignore its result...
393 forge ahead in our current function!"
394 (8sync-abort-to-prompt
397 (list (make-run-request
399 (make-run-request (lambda () body ...) #f))))))
401 (define (delayed-time in-secs)
402 "Calculate a cons of '(sec . usec) of IN-SECS from current time"
403 (define cur-time (gettimeofday))
404 (define cur-secs (car cur-time))
405 (define cur-usecs (cdr cur-time))
406 (define (convert-non-integer)
407 (define next-time-in-usecs
408 (+ (* (+ in-secs cur-secs) ; add our seconds to current seconds
409 1000000) ; and turn into usecs
410 cur-usecs)) ; then add in current usecs
411 ;; convert into sec / usec pair
412 (receive (secs usecs)
413 (floor/ next-time-in-usecs 1000000)
414 (cons (inexact->exact secs)
415 (inexact->exact (floor usecs)))))
416 (define (convert-integer)
417 (cons (+ in-secs cur-secs) cur-usecs))
418 (if (integer? in-secs)
420 (convert-non-integer)))
422 ;; TODO: Rewrite when we move to this being just `sleep'.
423 (define (8sleep secs)
424 "Like sleep, but asynchronous."
425 (8sync-abort-to-prompt
428 (make-run-request (kontinue kont) (delayed-time secs))))))
430 ;; Voluntarily yield execution
432 "Voluntarily yield execution to the scheduler."
433 (8sync-abort-to-prompt
436 (make-run-request (kontinue kont) #f)))))
439 ;;; Execution of agenda, and current agenda
440 ;;; =======================================
442 (define %current-agenda-prompt (make-parameter #f))
444 (define (update-agenda-from-select! agenda)
445 "Potentially (select) on ports specified in agenda, adding items to queue.
447 Also handles sleeping when all we have to do is wait on the schedule."
448 (define (hash-keys hash)
449 (hash-map->list (lambda (k v) k) hash))
450 (define (get-wait-time)
451 ;; TODO: we need to figure this out based on whether there's anything
452 ;; in the queue, and if not, how long till the next scheduled item
453 (let ((soonest-time (schedule-soonest-time (agenda-schedule agenda))))
455 ((not (q-empty? (agenda-queue agenda)))
457 (soonest-time ; ie, the agenda is non-empty
458 (let* ((current-time (gettimeofday)))
459 (if (time<= soonest-time current-time)
460 ;; Well there's something due so let's select
461 ;; (this avoids a (possible?) race condition chance)
463 (time-minus soonest-time current-time))))
467 ;; TODO: support usecond wait time too
468 (match (get-wait-time)
470 (catch #t ; expect: wrong-type-arg (open port), system-error
472 (select (hash-keys (agenda-read-port-map agenda))
473 (hash-keys (agenda-write-port-map agenda))
476 (lambda (key . rest-args)
477 (unless (and (memq key '(system-error wrong-type-arg))
479 (((or "select" "get-u8" "get-bytevector-n" "lookahead-u8"
480 "put-u8" "put-bytevector") arg ...) #t)
482 (apply throw key rest-args))
486 (_ (error "Unhandled error in select!" key rest-args))))))))
487 (define (get-procs-to-run)
488 (define (ports->procs ports port-map)
489 (lambda (initial-procs)
492 (define proc (hashq-ref port-map port))
493 ;; Now that we've selected on this port, it can be removed
494 (hashq-remove! port-map port)
499 ((read-ports write-ports except-ports) ; except-ports not used
500 ((compose (ports->procs
502 (agenda-read-port-map agenda))
505 (agenda-write-port-map agenda)))
507 (define (update-agenda)
508 (let ((procs-to-run (get-procs-to-run))
509 (q (agenda-queue agenda)))
515 (define (ports-to-select?)
516 (define (has-items? selector)
518 ;; ... we could use hash-for-each and a continuation to jump
519 ;; out with a #t at first indication of an item
520 (not (= (hash-count (const #t)
523 (or (has-items? agenda-read-port-map)
524 (has-items? agenda-write-port-map)))
526 (if (or (ports-to-select?)
527 ;; select doubles as sleep...
528 (not (schedule-empty? (agenda-schedule agenda))))
532 (define-record-type <read-request>
533 (make-read-request port proc)
535 (port read-request-port)
536 (proc read-request-proc))
538 (define-record-type <write-request>
539 (make-write-request port proc)
541 (port write-request-port)
542 (proc write-request-proc))
544 (define (agenda-handle-read-request! agenda read-request)
545 "Handle <read-request>, which is a request to add this port to the poll/select
546 on suspendable ports."
547 (hashq-set! (agenda-read-port-map agenda)
548 (read-request-port read-request)
549 (read-request-proc read-request)))
551 (define (agenda-handle-write-request! agenda write-request)
552 (hashq-set! (agenda-write-port-map agenda)
553 (write-request-port write-request)
554 (write-request-proc write-request)))
556 (define (stop-on-nothing-to-do agenda)
557 (and (q-empty? (agenda-queue agenda))
558 (schedule-empty? (agenda-schedule agenda))
559 (= 0 (hash-count (const #t) (agenda-read-port-map agenda)))
560 (= 0 (hash-count (const #t) (agenda-write-port-map agenda)))))
563 (define* (run-agenda agenda
564 #:key (stop-condition stop-on-nothing-to-do)
565 ;; For live hacking madness, etc
567 ;; TODO: Document fields
568 "Start up the AGENDA"
569 (install-suspendable-ports!)
570 (parameterize ((%current-agenda-prompt (agenda-prompt-tag agenda))
571 (current-read-waiter wait-for-readable)
572 (current-write-waiter wait-for-writable))
573 (while (not (stop-condition agenda))
574 (agenda-run-once! agenda)
575 (update-agenda-from-select! agenda)
576 (add-segments-contents-to-queue!
577 (schedule-extract-until! (agenda-schedule agenda) (gettimeofday))
578 (agenda-queue agenda))
580 (post-run-hook agenda))))
583 (define (print-error-and-continue key . args)
584 "Frequently used as pre-unwind-handler for agenda"
586 ((eq? key '8sync-caught-error)
588 ((orig-key orig-args stacks)
589 (display "\n*** Caught async exception. ***\n")
590 (format (current-error-port)
591 "* Original key '~s and arguments: ~s *\n"
593 (display "* Caught stacks below (ending with original) *\n\n")
596 (display-backtrace s (current-error-port))
597 (newline (current-error-port)))
600 (format (current-error-port)
601 "\n*** Caught exception with key '~s and arguments: ~s ***\n"
603 (display-backtrace (make-stack #t 1 0)
604 (current-error-port))
605 (newline (current-error-port)))))
607 (define-syntax-rule (maybe-catch-all (catch-handler pre-unwind-handler)
609 (if (or catch-handler pre-unwind-handler)
614 (or catch-handler (lambda _ #f))
615 (or pre-unwind-handler (lambda _ #f)))
618 (define (wait-for-readable port)
619 (8sync-abort-to-prompt
622 (make-read-request port (wrap (kont #f)))))))
624 (define (wait-for-writable port)
625 (8sync-abort-to-prompt
628 (make-write-request port (wrap (kont #f)))))))
630 (define (agenda-run-once! agenda)
631 "Run once through the agenda, and produce a new agenda
632 based on the results"
633 ;; @@: Maybe rename proc -> run-this ?
634 (define (call-proc proc)
636 (agenda-prompt-tag agenda)
639 ;; Resume continuation.
640 ;; We need to pass in #f, otherwise we sometimes get errors like
641 ;; "Zero values returned to single-valued continuation""
642 ((kontinue-kont proc) #f)
643 ;; Otherwise call the procedure and set up error catching.
645 ((agenda-catch-handler agenda)
646 (agenda-pre-unwind-handler agenda))
648 (lambda (kont async-request)
649 (setup-async-request kont async-request))))
651 (let ((queue (agenda-queue agenda))
652 (next-queue (make-q)))
653 (while (not (q-empty? queue))
654 (let* ((proc (q-pop! queue))
655 (proc-result (call-proc proc))
657 (lambda (run-request)
658 (let ((request-time (run-request-when run-request)))
660 (schedule-add! (agenda-schedule agenda) request-time
661 (run-request-proc run-request))
662 (enq! next-queue (run-request-proc run-request)))))))
663 (define (handle-individual result)
664 ;; @@: Could maybe optimize by switching to an explicit cond...
666 ((? run-request? new-proc)
668 ((? read-request? read-request)
669 (agenda-handle-read-request! agenda read-request))
670 ((? write-request? write-request)
671 (agenda-handle-write-request! agenda write-request))
673 ;; Remember, we don't throw an error here because procedures can
674 ;; return a run request, eg with run-it, at the end of their
675 ;; evaluation to keep looping.
676 ;; @@: Though is this really a useful feature?
678 ;; @@: We might support delay-wrapped procedures here
681 (for-each handle-individual results))
682 (one-result (handle-individual one-result)))))
683 (set-agenda-queue! agenda next-queue)))