just process a markdown file for text.

process_markdown.py

@@ -0,0 +1,79 @@
+from fugashi_parser import *
+
+import os
+from pathlib import Path
+import shutil
+import hashlib
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    #parser.add_argument("--vocab-file", required=True)
+    parser.add_argument("--markdown-file", required=True)
+    args = parser.parse_args()
+
+    destination_path = Path(args.markdown_file).parent
+    if not destination_path.joinpath('vocab').exists():
+        destination_path.joinpath('vocab').mkdir(parents=True, exist_ok=True)
+    # if args.text:
+    #     text = " ".join(args.text)
+    # else:
+    #     text = sys.stdin.read()
+
+    #print(args)
+    vocab = {} #load_vocab('foo.json')
+
+    ## Currently just doing this line by line
+    with open(args.markdown_file, "r", encoding="utf-8") as f:
+        lines = f.readlines()
+
+    i = 0 
+    file_changed = False
+    while i < len(lines):
+        ## Skip lines that start with certain elements
+        if lines[i][0:6] == '<span>' or lines[i][0] in ('#','>'):
+            i+=1
+            continue
+        ## If the line has japanese
+        if has_japanese(lines[i]):
+            print(lines[i])
+            result = analyze(lines[i],vocab)
+            ## If the line changed at all
+            if lines[i] != result['ruby_html']:
+                lines[i] = '<span>' + result['ruby_html'] + '</span>' + "\n> [!summary]- Vocab\n>" + result['notes_link'] + "\n\n" # lost the new line
+                file_changed = True
+        i+=1
+
+    if file_changed == False:
+        return
+    ## vocab now
+    ## Lookup vocab
+    for word in vocab:
+        #vocab_hash = hashlib.shake_256((word + dictionary_name).encode()).hexdigest(4)
+        if Path(destination_path.joinpath('vocab/' + word + '.md')).exists() == False:
+            definitions = anki_fields_term(word)
+            if definitions.get("fields"):
+                with open(destination_path.as_posix() + '/vocab/' + word + '.md', 'w') as vocab_note:
+                    for entry in definitions.get("fields"):
+                        #vocab_note.write("\n")
+                        #print(entry['expression'], file=vocab_note)
+                        #print(entry['furigana-plain'], file=vocab_note)
+                        print(entry['furigana'], file=vocab_note)
+                        if entry['expression'] != entry['reading']:
+                           print(entry['reading'], file=vocab_note)
+                        print("> ",re.sub(r'<.*?>', '', entry['glossary-plain-no-dictionary']), "\n", file=vocab_note)
+                        #print(f"Part of Speech: {entry['part-of-speech']}", file=vocab_note)
+                        print(f"Frequency: {entry['frequency-average-rank']}", file=vocab_note)
+                        #print(entry['glossary-plain-no-dictionary'], file=vocab_note)
+                        vocab_note.write(entry['glossary-first'])
+                        #print("##### Other Definitions", file=vocab_note)
+                        print("\n", file=vocab_note)
+    
+    ## Output file with changes if changes
+    with open(args.markdown_file, "w", encoding="utf-8") as f:
+        f.writelines(lines)
+
+
+if __name__ == "__main__":
+    main()
+

subtitle_processor.py

@@ -43,6 +43,7 @@ def main():
     ## First lien has some unicode bom bs
     i = 0 
     while i < len(lines):
+        ## If the line starts with a single number build a header
         if re.fullmatch(r'\d+', lines[i].strip()) or i == 0:
             notes_lines.append('##### ' + lines[i].strip() + " `" + lines[i+1].strip() + "`" + "\n")
             i+=2