Rename, recolor, and file my sketches automatically
| geek, supernote, python, drawing
I want to make it easier to process the sketchnotes I make on my
Supernote. I write IDs of the form yyyy-mm-dd-nn
to identify my
sketches. To avoid duplicates, I get these IDs from the web-based
journaling system I wrote. I've started putting the titles and tags
into those journal entries as well so that I can reuse them in
scripts. When I export a sketch to PNG and synchronize it, the file
appears in my ~/Dropbox/Supernote/EXPORT
directory on my laptop.
Then it goes through this process:
- I use Google Cloud Vision to detect handwriting so that I can find the ID.
- I retrieve the matching entry from my journal system and rename the file based on the title and tags.
- If there's no matching entry, I rename the file based on the ID.
- If there are other tags or references in the sketch, I add those to the filename as well.
- I recolor it based on the tags, so parenting-related posts are a little purple, tech/Emacs-related posts are blue, and things are generally highlighted in yellow otherwise.
- I move it to a directory based on the tags.
- If it's a private sketch, I move it to the directory for my private sketches.
- If it's a public sketch, I move it to the directory that will eventually get synchronized to sketches.sachachua.com, and I reload the list of sketches after some delay.
The following code does that processing.
supernote-daemon source code
#!/usr/bin/python3 # -*- mode: python -*- # (c) 2022-2023 Sacha Chua (sacha@sachachua.com) - MIT License # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation files # (the "Software"), to deal in the Software without restriction, # including without limitation the rights to use, copy, modify, merge, # publish, distribute, sublicense, and/or sell copies of the Software, # and to permit persons to whom the Software is furnished to do so, # subject to the following conditions: # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS # BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN # ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import os import json import re import requests import time from dotenv import load_dotenv # Import the Google Cloud client libraries from google.cloud import vision from google.cloud.vision_v1 import AnnotateImageResponse import sys sys.path.append("/home/sacha/proj/supernote/") import recolor # noqa: E402 # muffles flake8 error about import load_dotenv() # Set the folder path where the png files are located folder_path = '/home/sacha/Dropbox/Supernote/EXPORT/' public_sketch_dir = '/home/sacha/sync/sketches/' private_sketch_dir = '/home/sacha/sync/private-sketches/' # Initialize the Google Cloud Vision client client = vision.ImageAnnotatorClient() refresh_counter = 0 def extract_text(client, file): json_file = file[:-3] + 'json' # TODO Preprocess to keep only black text with open(file, 'rb') as image_file: content = image_file.read() # Convert the png file to a Google Cloud Vision image object image = vision.Image(content=content) # Extract handwriting from the image using the Google Cloud Vision API response = client.document_text_detection(image=image) response_json = AnnotateImageResponse.to_json(response) json_response = json.loads(response_json) # Save the response to a json file with the same name as the png file with open(json_file, "w") as f: json.dump(json_response, f) def maybe_rename(file): # TODO Match on ID json_file = file[:-3] + 'json' with open(json_file, 'r') as f: data = json.load(f) # Extract the text from the json file text = data['fullTextAnnotation']['text'] # Check if the text contains a string matching the regex pattern pattern = r'(?<!ref:)[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{2}' match = re.search(pattern, text) if match: # Get the matched string matched_string = match.group(0) new_name = matched_string from_zid = get_journal_entry(matched_string).strip() if from_zid: new_name = matched_string + ' ' + from_zid tags = get_tags(new_name, text) if tags: new_name = new_name + ' ' + tags ref = get_references(text) if ref: new_name = new_name + ' ' + ref print('Renaming ' + file + ' to ' + new_name) # Rename the png and json files to the matched string new_filename = os.path.join(os.path.dirname(file), new_name + '.png') rename_set(file, new_filename) return new_filename def get_tags(filename, text): tags = re.findall(r'(^|\W)#[ \n\t]+', text) return ' '.join(filter(lambda x: x not in filename, tags)) def get_references(text): refs = re.findall(r'!ref:[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{2}', text) return ' '.join(refs) def get_journal_entry(zid): resp = requests.get('https://' + os.environ['JOURNAL_USER'] + ':' + os.environ['JOURNAL_PASS'] + '@journal.sachachua.com/api/entries/' + zid) j = resp.json() if j and not re.search('^I thought about', j['Note']): return j['Note'] def get_color_map(filename, text=None): if text: together = filename + ' ' + text else: together = filename if re.search('r#(parenting|purple|life)', together): return {'9d9d9d': '8754a1', 'c9c9c9': 'e4c1d9'} # parenting is purplish elif re.search(r'#(emacs|geek|tech|blue)', together): return {'9d9d9d': '2b64a9', 'c9c9c9': 'b3e3f1'} # geeky stuff in light/dark blue else: return {'9d9d9d': '884636', 'c9c9c9': 'f6f396'} # yellow highlighter, dark brown def rename_set(old_name, new_name): if old_name != new_name: old_json = old_name[:-3] + 'json' new_json = new_name[:-3] + 'json' os.rename(old_name, new_name) os.rename(old_json, new_json) def recolor_based_on_filename(filename): color_map = get_color_map(filename) recolored = recolor.map_colors(filename, color_map) # possibly rename based on the filename new_filename = re.sub(' #(purple|blue)', '', filename) rename_set(filename, new_filename) recolored.save(new_filename) def move_processed_sketch(file): global refresh_counter if '#private' in file: output_dir = private_sketch_dir elif '#' in file: output_dir = public_sketch_dir refresh_counter = 3 else: return file new_filename = os.path.join(output_dir, os.path.basename(file)) rename_set(file, new_filename) return new_filename def process_file(file): json_file = file[:-3] + 'json' # Check if a corresponding json file already exists if not os.path.exists(json_file): extract_text(client, file) if not re.search('[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{2} ', file): file = maybe_rename(file) recolor_based_on_filename(file) move_processed_sketch(file) def process_dir(folder_path): global processed_files # Iterate through all png files in the specified folder files = sorted(os.listdir(folder_path)) for file in files: if file.endswith('.png') and '_' in file: print("Processing ", file) process_file(os.path.join(folder_path, file)) def daemon(folder_path, wait): global refresh_counter while True: process_dir(folder_path) time.sleep(wait) if refresh_counter > 0: refresh_counter = refresh_counter - 1 if refresh_counter == 0: print("Reloading sketches") requests.get('https://' + os.environ['JOURNAL_USER'] + ':' + os.environ['JOURNAL_PASS'] + '@sketches.sachachua.com/reload?python=1') if __name__ == '__main__': # Create a set to store the names of processed files processed_files = set() if len(sys.argv) > 1: if os.path.isdir(sys.argv[1]): folder_path = sys.argv[1] daemon(folder_path, 300) else: for f in sys.argv[1:]: process_file(f) else: daemon(folder_path, 300)
It uses this script I wrote to recolor my sketches with Python.
I'm contemplating writing some annotation tools to make it easier to turn the detected text into useful text for searching or writing about because the sketches throw off the recognition (misrecognized text, low confidence) and the columns mess up the line wrapping. Low priority, though.
My handwriting (at least for numbers) is probably simple enough that I might be able to train Tesseract OCR to process that someday. And who knows, maybe some organization will release a pre-trained model for offline handwriting recognition that'll be as useful as OpenAI Whisper is for audio files. That would be neat!