Dustin/jp_text_parsing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

function word parser, path arguments for splitting the kind of things that can use this stuff.

Browse Source
This commit is contained in:
Nitsud Yarg 2026-05-15 12:29:09 -07:00
parent 7bb75ae7d9
commit e2d0e08346
2 changed files with 53 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
fugashi_parser.py
View File

@ -7,6 +7,7 @@ import argparse
import html import html
import json import json
import os import os
from pathlib import Path
import re import re
import sys import sys
from dataclasses import dataclass, asdict from dataclasses import dataclass, asdict
@ -360,3 +361,25 @@ def analyze(text: str, vocab: Dict[str, Dict]):
"token_readings": [asdict(t) for t in tokens], "token_readings": [asdict(t) for t in tokens],
"predicate_chunks" : predicate_chunks, "predicate_chunks" : predicate_chunks,
} }
def write_word_note(word: str, destination_dir: Path):
if Path(destination_dir.joinpath('/' + word + '.md')).exists() == False:
definitions = anki_fields_term(word)
if definitions.get("fields"):
with open(destination_dir.as_posix() + '/' + word + '.md', 'w') as vocab_note:
for entry in definitions.get("fields"):
#vocab_note.write("\n")
#print(entry['expression'], file=vocab_note)
#print(entry['furigana-plain'], file=vocab_note)
print(entry['furigana'], file=vocab_note)
if entry['expression'] != entry['reading']:
print(entry['reading'], file=vocab_note)
print("> ",re.sub(r'<.*?>', '', entry['glossary-plain-no-dictionary']), "\n", file=vocab_note)
#print(f"Part of Speech: {entry['part-of-speech']}", file=vocab_note)
print(f"Frequency: {entry['frequency-average-rank']}", file=vocab_note)
#print(entry['glossary-plain-no-dictionary'], file=vocab_note)
vocab_note.write(entry['glossary-first'])
#print("##### Other Definitions", file=vocab_note)
print("\n", file=vocab_note)

47
process_markdown.py
View File

@ -5,16 +5,23 @@ from pathlib import Path
import shutil import shutil
import hashlib import hashlib
def main(): def main():
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
#parser.add_argument("--vocab-file", required=True) #parser.add_argument("--vocab-file", required=True)
parser.add_argument("--markdown-file", required=True) parser.add_argument("--markdown-file", required=True)
parser.add_argument("--vocab-folder", required=False)
args = parser.parse_args() args = parser.parse_args()
if args.vocab_folder:
destination_path = Path(args.vocab_folder)
if destination_path.is_dir() == False:
raise ValueError("Vocab Folder is not a directory.")
return
else:
destination_path = Path(args.markdown_file).parent destination_path = Path(args.markdown_file).parent
if not destination_path.joinpath('vocab').exists(): if not destination_path.joinpath('vocab').exists():
destination_path.joinpath('vocab').mkdir(parents=True, exist_ok=True) destination_path.joinpath('vocab').mkdir(parents=True, exist_ok=True)
destination_path = destination_path.joinpath('vocab')
# if args.text: # if args.text:
# text = " ".join(args.text) # text = " ".join(args.text)
# else: # else:
@ -49,25 +56,25 @@ def main():
## vocab now ## vocab now
## Lookup vocab ## Lookup vocab
for word in vocab: for word in vocab:
#vocab_hash = hashlib.shake_256((word + dictionary_name).encode()).hexdigest(4) write_word_note(word, destination_path)
if Path(destination_path.joinpath('vocab/' + word + '.md')).exists() == False: # if Path(destination_path.joinpath('/' + word + '.md')).exists() == False:
definitions = anki_fields_term(word) # definitions = anki_fields_term(word)
if definitions.get("fields"): # if definitions.get("fields"):
with open(destination_path.as_posix() + '/vocab/' + word + '.md', 'w') as vocab_note: # with open(destination_path.as_posix() + '/' + word + '.md', 'w') as vocab_note:
for entry in definitions.get("fields"): # for entry in definitions.get("fields"):
#vocab_note.write("\n") # #vocab_note.write("\n")
#print(entry['expression'], file=vocab_note) # #print(entry['expression'], file=vocab_note)
#print(entry['furigana-plain'], file=vocab_note) # #print(entry['furigana-plain'], file=vocab_note)
print(entry['furigana'], file=vocab_note) # print(entry['furigana'], file=vocab_note)
if entry['expression'] != entry['reading']: # if entry['expression'] != entry['reading']:
print(entry['reading'], file=vocab_note) # print(entry['reading'], file=vocab_note)
print("> ",re.sub(r'<.*?>', '', entry['glossary-plain-no-dictionary']), "\n", file=vocab_note) # print("> ",re.sub(r'<.*?>', '', entry['glossary-plain-no-dictionary']), "\n", file=vocab_note)
#print(f"Part of Speech: {entry['part-of-speech']}", file=vocab_note) # #print(f"Part of Speech: {entry['part-of-speech']}", file=vocab_note)
print(f"Frequency: {entry['frequency-average-rank']}", file=vocab_note) # print(f"Frequency: {entry['frequency-average-rank']}", file=vocab_note)
#print(entry['glossary-plain-no-dictionary'], file=vocab_note) # #print(entry['glossary-plain-no-dictionary'], file=vocab_note)
vocab_note.write(entry['glossary-first']) # vocab_note.write(entry['glossary-first'])
#print("##### Other Definitions", file=vocab_note) # #print("##### Other Definitions", file=vocab_note)
print("\n", file=vocab_note) # print("\n", file=vocab_note)
## Output file with changes if changes ## Output file with changes if changes
with open(args.markdown_file, "w", encoding="utf-8") as f: with open(args.markdown_file, "w", encoding="utf-8") as f: