Using Emacs to visualize speaking and target words/phrases in real-ish time
| audio, speech-recognition, emacs, learning, visualization
When I'm working with a tutor to improve my French, I sometimes need to push myself to provide more details when I'm sharing and ask more follow-up questions in response to what they're saying, instead of falling back to « D'accord » and « C'est cool. » Visualizing the timeline in real-time lets me see if the turn-taking looks good. I might even need to take a more active role in controlling the conversation if I happen to be meeting a tutor who tends to go on monologues.
When I'm talking out loud on my own for practice, this might also nudge me to reduce the pauses and enrich my vocabulary by practising saying things often enough that they become more automatic.
1. Voice activity detection and speech recognition
This code uses this server vad-file.py, which I generated with Google Gemini. It emits JSONL for voice activity detection events (start/stop) and uses Orukeet to transcribe the speech once the voice activity detection notices it's finished. I also use spacy to look up token information because I want to someday colour the nouns by gender, which might allow me to correct myself in the next sentence instead of hesitating in the previous one.
;; Report on the balance between my tutor and me
(defvar sacha-vad-tutor-process nil)
(defvar sacha-vad-tutor-dir "~/vendor/orukeet")
(defvar sacha-vad-tutor-command `(,(expand-file-name ".venv/bin/python" sacha-vad-tutor-dir)
"asr.py"
"--spacy-model"
"fr_dep_news_trf"
"-s" "300" "-t"
"VirtualMicSink.monitor=sacha" "alsa_output.usb-AT_AT2040USB_202011110001-00.analog-stereo.monitor=tutor"))
(defvar sacha-vad-tutor-events nil "List of events as an list of alists (label start stop duration).")
(defvar sacha-vad-tutor-labels '("sacha" "tutor"))
(defvar sacha-vad-tutor-summary '(("sacha" . 0)
("tutor" . 0)))
(defvar sacha-vad-tutor-session-length (* 60 60) "Number of seconds to use for scale.")
(defun sacha-vad-tutor-update-view ()
"Visualize the last 3 minutes and the overview of the session.
Also keep track of target phrases."
(interactive)
(with-current-buffer (get-buffer-create "*tutor stats*")
(erase-buffer)
(let* ((total (apply '+ (mapcar 'cdr sacha-vad-tutor-summary)))
(height 30)
(width (window-width nil t))
(max-seconds
(if sacha-vad-tutor-events
(apply 'max (mapcar (lambda (o) (alist-get 'end o))
sacha-vad-tutor-events))
0))
(threshold 180) ; last three minutes
(threshold-seconds (- max-seconds threshold))
(recent
(seq-keep
(lambda (o)
(when (>= (alist-get 'start o)
threshold-seconds)
(let-alist o
`((label . ,.label)
(start . ,(- .start threshold-seconds))
(end . ,(- .end threshold-seconds))))))
(copy-sequence sacha-vad-tutor-events)))
(recent-total (apply '+ (mapcar (lambda (o)
(-
(alist-get 'end o)
(alist-get 'start o)))
recent))))
(save-excursion
(insert
(format "Last %d minute%s:\n"
(/ threshold 60)
(if (> (/ threshold 60) 1)
"s"
""))
(mapconcat
(lambda (o)
(format "%d%% %s\n"
(/ (* 100.0 (cdr o)) recent-total)
(car o)))
(seq-map
(lambda (group)
(cons (car group)
(apply '+ (mapcar (lambda (o)
(-
(alist-get 'end o)
(alist-get 'start o)))
(cdr group)))))
(seq-group-by (lambda (o) (alist-get 'label o)) recent)))
"\n")
(svg-insert-image
(sacha-vad-visualize
sacha-vad-tutor-labels
recent
t
width height))
(insert
"\nTotal:\n"
(mapconcat
(lambda (o)
(format "%d%% %s\n"
(/ (* 100.0 (cdr o)) total)
(car o)))
sacha-vad-tutor-summary))
(svg-insert-image
(sacha-vad-visualize sacha-vad-tutor-labels sacha-vad-tutor-events t
width height
sacha-vad-tutor-session-length))
(insert "\n\n" (sacha-learn-lang-bingo-summary))))))
(defun sacha-vad-handle-end-event (data)
"Handle the end event."
(let ((event (list
(cons 'label (alist-get 'label data))
(cons 'start (- (alist-get 'timestamp data)
(alist-get 'duration data)
(or sacha-vad-tutor-start-seconds 0)))
(cons 'end (- (alist-get 'timestamp data)
(or sacha-vad-tutor-start-seconds 0))))))
(push event sacha-vad-tutor-events)
(let-alist data
(incf (alist-get (alist-get 'label data) sacha-vad-tutor-summary 0 nil 'string=)
.duration))
(sacha-vad-tutor-update-view)))
(defun sacha-vad-tutor-filter (proc string)
"Add and parse."
(when (buffer-live-p (process-buffer proc))
;; split the string by newline?
(with-current-buffer (process-buffer proc)
(goto-char (point-max))
(save-excursion
(insert string))
(catch 'waiting
(unless (or (eobp) (looking-at "\n*$"))
(let ((data (json-read)))
(when (string= (alist-get 'event data) "end")
(sacha-vad-handle-end-event data))
(when (string= (alist-get 'event data) "final")
(sacha-vad-handle-text-event data))))))))
(defvar sacha-vad-last-speaker nil "Previous speaker.")
(defun sacha-vad-handle-text-event (data)
(with-current-buffer (get-buffer-create "*Transcript*")
(set (make-local-variable 'window-point-insertion-type) t)
(save-excursion
(goto-char (point-max))
;; insert break
(insert
(if (string= sacha-vad-last-speaker (alist-get 'label data))
" "
(concat (if (bobp) "" "\n\n") (alist-get 'label data) ": "))
(alist-get 'text data))
(setq sacha-vad-last-speaker (alist-get 'label data))
(when (string= (alist-get 'label data) "sacha")
(sacha-learn-lang-notice-bingo-words (alist-get 'text data))))))
(defun sacha-vad-tutor-start ()
"Start the process."
(interactive)
(sacha-vad-tutor-ensure)
(setq sacha-vad-tutor-summary '(("sacha" . 0)
("tutor" . 0))
sacha-vad-tutor-events nil)
(with-current-buffer (get-buffer-create "*Transcript*")
(erase-buffer)
(display-buffer (current-buffer)))
(display-buffer (get-buffer-create "*tutor stats*")))
(defun sacha-vad-tutor-ensure ()
"Start the process if it's not already running."
(interactive)
(unless (process-live-p sacha-vad-tutor-process)
(setq sacha-vad-tutor-process-start-time (current-time))
(setq sacha-vad-tutor-start-seconds 0)
(let ((default-directory sacha-vad-tutor-dir))
(setq sacha-vad-tutor-process
(make-process
:name "vad-tutor"
:command sacha-vad-tutor-command
:connection-type 'pipe
:buffer (get-buffer-create "*vad-tutor*")
:stderr (get-buffer-create "*vad-tutor-err*")
:filter #'sacha-vad-tutor-filter)))))
(defun sacha-vad-tutor-stop ()
"Stop the process."
(interactive)
(interrupt-process sacha-vad-tutor-process))
;;;###autoload
(defun sacha-js-parse-jsonl-file (file)
"Parse jsonl as a list"
(let (results)
(with-temp-buffer
(insert-file-contents file)
(goto-char (point-min))
(while (not (eobp))
(condition-case nil
(push (json-read) results)
(error (goto-char (point-max))))))
(nreverse results)))
(defvar sacha-vad-tutor-start-seconds 0 "Number of seconds to consider as the start.")
(defvar sacha-vad-tutor-process-start-time nil "Time that the process started.")
;;;###autoload
(defun sacha-vad-tutor-reset-clock ()
"Reset the start of the session."
(interactive)
(setq sacha-vad-tutor-start-seconds
(time-to-seconds (time-subtract (current-time)
sacha-vad-tutor-process-start-time)))
(mapcar (lambda (o) (setcdr o 0)) sacha-vad-tutor-summary)
(setq sacha-vad-tutor-events nil))
(defun sacha-vad-visualize (labels list &optional output-file width height max-seconds)
(setq height (or height 25))
(setq width (or width (window-width nil t)))
(let* ((svg (svg-create width height))
(max-seconds (max
(or max-seconds 0)
(if list
(apply 'max (mapcar (lambda (o) (alist-get 'end o))
list))
0)))
(pixels-per-second (/ (* 1.0 width) max-seconds))
(row-height (/ height (length labels))))
(dolist (o list)
(let-alist o
(let ((y (* row-height (or (seq-position labels .label 'string=) -1))))
(when (>= y 0)
(svg-rectangle
svg
(* .start pixels-per-second)
y
(* (- .end .start) pixels-per-second)
row-height :fill "green")))))
(cond
((stringp output-file)
(with-temp-file output-file
(svg-print svg)))
(output-file
svg)
(t
(with-current-buffer (get-buffer-create "*vad*")
(goto-char (point-min))
(display-buffer (current-buffer))
(erase-buffer)
(svg-insert-image svg))))))
(defun sacha-vad-stats (labels list)
"Return the balance of time talking as (label total percentage)."
(let* ((max-seconds (apply '+ (mapcar
(lambda (o) (let-alist o (- .end .start)))
list)))
(durations (mapcar (lambda (o)
(cons
(car o)
(mapcar (lambda (e) (let-alist e (- .end .start)))
(cdr o))))
(seq-group-by (lambda (o) (alist-get 'label o))
list))))
(seq-map
(lambda (label)
(let* ((row (cdr (assoc label durations)))
(total (apply '+ row)))
`((label . ,label)
(total . ,total)
(count . ,(length row))
(percentage . ,(and total max-seconds (> max-seconds 0) (* 100.0 (/ total max-seconds)))))))
labels)))
;; (sacha-vad-stats '("sacha" "tutor") (seq-take (sacha-js-parse-jsonl-file "~/proj/speech/output2.log") 5))
;; (sacha-vad-stats '("sacha" "tutor") (sacha-js-parse-jsonl-file "~/proj/speech/output.log"))
;; (sacha-vad-visualize '("sacha" "tutor") (sacha-js-parse-jsonl-file "~/proj/speech/output.log"))
;; (sacha-vad-visualize '("sacha" "tutor") (seq-take (sacha-js-parse-jsonl-file "~/proj/speech/output2.log") 30))
(defun sacha-vad-tutor-group-output (files)
(let ((grouped-files (seq-group-by
(lambda (o)
(substring (file-name-base o) 0 10))
files)))
(mapcar
(lambda (group)
(let ((start 0))
(cons (car (cdr group))
(seq-reduce
(lambda (prev file)
(append prev
(let ((max-seconds 0))
(prog1 (mapcar (lambda (event)
(incf (alist-get 'start event) start)
(incf (alist-get 'end event) start)
(setq max-seconds (max (alist-get 'end event) max-seconds))
event)
(sacha-js-parse-jsonl-file file))
(setq start max-seconds)))))
(cdr group)
nil))))
grouped-files)))
(defun sacha-vad-tutor-make-table (files &optional width height)
"Make a summary table."
(let ((labels sacha-vad-tutor-labels)
(width (or width 500))
(height (or height 30)))
;; combine data as needed
(append
'(("Date" "Tutor" "Minutes" "%" "Visualization") hline)
(seq-map
(lambda (group)
(let* ((list (cdr group))
(base (file-name-base (car group))))
(if list
(let-alist (car (sacha-vad-stats labels list))
(let ((svg-name
(expand-file-name
(concat (file-name-base (car group)) "-" .label ".svg")
"~/proj/french/conversation")))
(sacha-vad-visualize
(list .label)
list
svg-name
width
height)
(list
(format-time-string "%Y-%m-%d"
(seconds-to-time
(sacha-filename-timestamp (car group))))
;; (org-link-make-string (concat "file:" o) "file")
(if (string-match "-conversation-\\([^\\.]+\\)" base)
(s-capitalize (substring (match-string 1 base) 0 1))
"")
(round (/ .total 60))
(format "%d%%" .percentage)
(org-link-make-string (concat "file:" svg-name)
(concat "file:" svg-name))
)))
(list "?"))))
(sacha-vad-tutor-group-output files))
nil)))
2. Noticing bingo phrases
Here's the code that reminds me to use the words or expressions on my to-do list, and then keeps track of how often I've used them, prioritizes the ones that I haven't used yet, and displays these. It can work with either my Whisper dictation workflow or the VAD events server described above.
I keep the list of expressions in file:///home/sacha/proj/french/bingo.txt.
(defvar sacha-learn-lang-bingo-word-file "~/proj/french/bingo.txt")
(defvar sacha-learn-lang-bingo-words nil "List of words.")
(defvar sacha-learn-lang-bingo-words-hash nil "Data for the counts.")
;;;###autoload
(defun sacha-learn-lang-bingo-words ()
"Load the list of words from the file."
(interactive)
(setq sacha-learn-lang-bingo-words
(seq-map (lambda (o)
(when (= (length o) 1)
(setq o (list (car o) 0)))
(setq o (append o (list
(string-trim (replace-regexp-in-string
" *\\[.+?\\] *" " "
(car o))))))
o)
(pcsv-parse-file sacha-learn-lang-bingo-word-file)))
sacha-learn-lang-bingo-words)
;;;###autoload
(defun sacha-learn-lang-bingo-handler (event)
"Manually register the event."
(interactive "e")
(let* ((pos (posn-point (event-end event)))
(val (get-text-property pos 'bingo)))
(when val
(sacha-learn-lang-notice-bingo-words val))))
(defvar-keymap sacha-learn-lang-bingo-map
"<mouse-1>" #'sacha-learn-lang-bingo-handler)
;;;###autoload
(defun sacha-learn-lang-bingo-summary ()
"Summarize the words to say."
(let ((target (seq-filter (lambda (row) (= (elt row 1) 0))
sacha-learn-lang-bingo-words))
(yay (seq-filter (lambda (row) (> (elt row 1) 0))
sacha-learn-lang-bingo-words)))
(concat
(if target
(concat "Target: "
(mapconcat (lambda (row)
(propertize
(car row)
'keymap sacha-learn-lang-bingo-map
'bingo (car row)))
target
"; ")
"\n\n")
"")
(if yay
(concat
"Yay! "
(mapconcat (lambda (row)
(propertize (format "%s (%s)" (car row) (elt row 1))
'keymap sacha-learn-lang-bingo-map
'bingo (car row)))
(seq-filter (lambda (row) (> (elt row 1) 0))
sacha-learn-lang-bingo-words)
"; ")
"\n\n")
""))))
;;;###autoload
(defun sacha-learn-lang-notice-bingo-words (text)
"Look for the bingo words in TEXT and celebrate when I use them.
Add to `whisper-insert-text-at-point'."
(when text
(unless sacha-learn-lang-bingo-words (sacha-learn-lang-bingo-words))
(let ((found (seq-find
(lambda (o)
(and (elt o 2)
(string-match (concat "\\<" (elt o 2)) text)))
sacha-learn-lang-bingo-words)))
(when found
(incf (elt found 1))
(sacha-vad-tutor-update-view)
(with-current-buffer (get-buffer-create "*Bingo*")
(erase-buffer)
(insert (sacha-learn-lang-bingo-summary))))))
text)
This is part of my EmacsConfiguration.
You can e-mail me at sacha@sachachua.com.