Dustin/jp_text_parsing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

function word parser, path arguments for splitting the kind of things that can use this stuff.

Browse Source
This commit is contained in:
Nitsud Yarg 2026-05-15 12:29:09 -07:00
parent 7bb75ae7d9
commit e2d0e08346
2 changed files with 53 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
fugashi_parser.py
View File

@ -7,6 +7,7 @@ import argparse
import html
import json
import os
from pathlib import Path
import re
import sys
from dataclasses import dataclass, asdict
@ -360,3 +361,25 @@ def analyze(text: str, vocab: Dict[str, Dict]):
"token_readings": [asdict(t) for t in tokens],
"predicate_chunks" : predicate_chunks,
}
def write_word_note(word: str, destination_dir: Path):
if Path(destination_dir.joinpath('/' + word + '.md')).exists() == False:
definitions = anki_fields_term(word)
if definitions.get("fields"):
with open(destination_dir.as_posix() + '/' + word + '.md', 'w') as vocab_note:
for entry in definitions.get("fields"):
#vocab_note.write("\n")
#print(entry['expression'], file=vocab_note)
#print(entry['furigana-plain'], file=vocab_note)
print(entry['furigana'], file=vocab_note)
if entry['expression'] != entry['reading']:
print(entry['reading'], file=vocab_note)
print("> ",re.sub(r'<.*?>', '', entry['glossary-plain-no-dictionary']), "\n", file=vocab_note)
#print(f"Part of Speech: {entry['part-of-speech']}", file=vocab_note)
print(f"Frequency: {entry['frequency-average-rank']}", file=vocab_note)
#print(entry['glossary-plain-no-dictionary'], file=vocab_note)
vocab_note.write(entry['glossary-first'])
#print("##### Other Definitions", file=vocab_note)
print("\n", file=vocab_note)

53
process_markdown.py
View File

@ -5,16 +5,23 @@ from pathlib import Path
import shutil
import hashlib
def main():
parser = argparse.ArgumentParser()
#parser.add_argument("--vocab-file", required=True)
parser.add_argument("--markdown-file", required=True)
parser.add_argument("--vocab-folder", required=False)
args = parser.parse_args()
destination_path = Path(args.markdown_file).parent
if not destination_path.joinpath('vocab').exists():
destination_path.joinpath('vocab').mkdir(parents=True, exist_ok=True)
if args.vocab_folder:
destination_path = Path(args.vocab_folder)
if destination_path.is_dir() == False:
raise ValueError("Vocab Folder is not a directory.")
return
else:
destination_path = Path(args.markdown_file).parent
if not destination_path.joinpath('vocab').exists():
destination_path.joinpath('vocab').mkdir(parents=True, exist_ok=True)
destination_path = destination_path.joinpath('vocab')
# if args.text:
# text = " ".join(args.text)
# else:
@ -49,25 +56,25 @@ def main():
## vocab now
## Lookup vocab
for word in vocab:
#vocab_hash = hashlib.shake_256((word + dictionary_name).encode()).hexdigest(4)
if Path(destination_path.joinpath('vocab/' + word + '.md')).exists() == False:
definitions = anki_fields_term(word)
if definitions.get("fields"):
with open(destination_path.as_posix() + '/vocab/' + word + '.md', 'w') as vocab_note:
for entry in definitions.get("fields"):
#vocab_note.write("\n")
#print(entry['expression'], file=vocab_note)
#print(entry['furigana-plain'], file=vocab_note)
print(entry['furigana'], file=vocab_note)
if entry['expression'] != entry['reading']:
print(entry['reading'], file=vocab_note)
print("> ",re.sub(r'<.*?>', '', entry['glossary-plain-no-dictionary']), "\n", file=vocab_note)
#print(f"Part of Speech: {entry['part-of-speech']}", file=vocab_note)
print(f"Frequency: {entry['frequency-average-rank']}", file=vocab_note)
#print(entry['glossary-plain-no-dictionary'], file=vocab_note)
vocab_note.write(entry['glossary-first'])
#print("##### Other Definitions", file=vocab_note)
print("\n", file=vocab_note)
write_word_note(word, destination_path)
# if Path(destination_path.joinpath('/' + word + '.md')).exists() == False:
# definitions = anki_fields_term(word)
# if definitions.get("fields"):
# with open(destination_path.as_posix() + '/' + word + '.md', 'w') as vocab_note:
# for entry in definitions.get("fields"):
# #vocab_note.write("\n")
# #print(entry['expression'], file=vocab_note)
# #print(entry['furigana-plain'], file=vocab_note)
# print(entry['furigana'], file=vocab_note)
# if entry['expression'] != entry['reading']:
# print(entry['reading'], file=vocab_note)
# print("> ",re.sub(r'<.*?>', '', entry['glossary-plain-no-dictionary']), "\n", file=vocab_note)
# #print(f"Part of Speech: {entry['part-of-speech']}", file=vocab_note)
# print(f"Frequency: {entry['frequency-average-rank']}", file=vocab_note)
# #print(entry['glossary-plain-no-dictionary'], file=vocab_note)
# vocab_note.write(entry['glossary-first'])
# #print("##### Other Definitions", file=vocab_note)
# print("\n", file=vocab_note)
## Output file with changes if changes
with open(args.markdown_file, "w", encoding="utf-8") as f: