Dustin/jp_text_parsing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

just playing with stuff now. words output pos now

Browse Source
This commit is contained in:
Nitsud Yarg 2026-05-23 21:47:40 -07:00
parent 5cc5985e91
commit 5eb0f88bc2
3 changed files with 66 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
fugashi_parser.py
View File

@ -98,6 +98,12 @@ def term_entries(term :str) -> dict:
response = requests.post(request_url + "/termEntries", json = params, timeout = request_timeout)
return json.loads(response.text)
CONTENT_LINK_TAG = {
'名詞': 'meishi', # noun
'動詞': 'doshi', # verb
'形容詞': 'keiyoshi' # Adjective
#'副詞' : 'fukushi' #adverb
}
def is_content_word(pos1: str) -> bool:
return pos1 in {"名詞", "動詞", "形容詞", "副詞"}
@ -137,11 +143,12 @@ def build_kana(surface: str, reading: str) -> str:
return f"「{reading}」"
def build_vocab_link(surface: str, reading: str, pos) -> str:
pos1, pos2, pos3, pos4 = pos
if not is_vocab_token(pos, keep_pronouns=True,keep_numbers=False):
return surface
if surface == reading:
return f"[[{surface}]]"
return f"[[{reading}|{surface}]]"
# if surface == reading:
# return f"[[{surface}]]"
return f"[[{reading}_{pos1}|{surface}]]"
## Predicate filtering Verbs
def is_predicate_start(pos):
@ -362,12 +369,44 @@ def analyze(text: str, vocab: Dict[str, Dict]):
"predicate_chunks" : predicate_chunks,
}
def write_word_note(word: str, destination_dir: Path):
if Path(destination_dir.joinpath(word + '.md')).exists() == False:
YOMITAN_TO_MICAB_POS = {
'interjection': 'foo',
'noun': 'meishi',
'5-dan': '',
'1-dan': '',
'prefix': '',
'intransitive': '',
'aux-verb': '',
'suru': '',
'transitive': '',
'counter': '',
'na-adj': '',
'exp': '',
'adjective': '',
'suffix': '',
'conjunction': '',
'interjection': '',
'adverb': '',
'pronoun': '',
'unclass': '',
}
def write_word_note(word: str, destination_dir: Path, part_of_speech:str='', overwrite:bool=False):
word_file_name = word
# Not ready for different file names yet
if part_of_speech != '':
word_file_name = word + '_' + part_of_speech
if (Path(destination_dir.joinpath(word_file_name + '.md')).exists() == False) or overwrite == True:
definitions = anki_fields_term(word)
if definitions.get("fields"):
with open(destination_dir.joinpath(word + '.md'), 'w') as vocab_note:
with open(destination_dir.joinpath(word_file_name + '.md'), 'w') as vocab_note:
for entry in definitions.get("fields"):
#pos = entry['part-of-speech'] ## POS is always 'Unknown' for some reason
pos = re.match(r'.*<span.*?"part-of-speech-info".*?>(.*?)<',entry['glossary-first'])
if pos:
pos = pos.group(1)
# if part_of_speech != '' and pos:
# print(pos)
#vocab_note.write("\n")
#print(entry['expression'], file=vocab_note)
#print(entry['furigana-plain'], file=vocab_note)
@ -381,5 +420,7 @@ def write_word_note(word: str, destination_dir: Path):
vocab_note.write(entry['glossary-first'])
#print("##### Other Definitions", file=vocab_note)
print("\n", file=vocab_note)
# if CONTENT_LINK_TAG.get(part_of_speech, '') != '':
# print(f"#{CONTENT_LINK_TAG[part_of_speech]}", file=vocab_note)

9
make_dialog_notes.py
View File

@ -7,6 +7,7 @@ import hashlib
destination_directory_name = '/run/media/deck/YF8SD/Sync/Notebooks/Obsidian/Japanese/Yu-Gi-Oh/'
destination_path = Path(destination_directory_name)
vocab_destination_path = Path('/run/media/deck/YF8SD/Sync/Notebooks/Obsidian/Japanese/vocab')
with open("scripts_dump2.json", "r", encoding="utf-8") as f:
dump = json.load(f)
@ -31,6 +32,14 @@ for title in dump:
text_J = dump[title][block]['text_J']
text_J = re.sub(r'\$R(.*?)\((.*?)\)',r'<ruby>\1<rt>\2</rt></ruby>',text_J)
print(f"<details><summary>{text_J}</summary>{dump[title][block]['text_E']}</details>",file=card_note)
text_J = dump[title][block]['text_J']
text_J = re.sub(r'\$R(.*?)\((.*?)\)',r'\1',text_J)
text_J = text_J.replace('<br>', '')
vocab = {} ## flush the vocab variable each time
result = analyze(text_J, vocab)
print(f"> [!summary]- Vocab\n>{result['notes_link']}",file=card_note)
for word in vocab:
write_word_note(word, vocab_destination_path, vocab[word]['pos'][0])
# ## add card names as properties
# card_note.write("---\n")
# card_note.write(f"aliases: [{dump[cd]['name_J']},{dump[cd]['name_E']}]\n")

22
process_markdown.py
View File

@ -13,19 +13,17 @@ def main():
args = parser.parse_args()
if args.vocab_folder:
destination_path = Path(args.vocab_folder)
if destination_path.is_dir() == False:
vocab_destination_path = Path(args.vocab_folder)
if vocab_destination_path.is_dir() == False:
raise ValueError("Vocab Folder is not a directory.")
return
else:
destination_path = Path(args.markdown_file).parent
if not destination_path.joinpath('vocab').exists():
destination_path.joinpath('vocab').mkdir(parents=True, exist_ok=True)
destination_path = destination_path.joinpath('vocab')
# if args.text:
# text = " ".join(args.text)
# else:
# text = sys.stdin.read()
vocab_destination_path = Path(args.markdown_file).parent
if not vocab_destination_path.joinpath('vocab').exists():
vocab_destination_path.joinpath('vocab').mkdir(parents=True, exist_ok=True)
vocab_destination_path = vocab_destination_path.joinpath('vocab')
destination_path = Path(args.markdown_file).parent.joinpath(Path(args.markdown_file).stem + '_STUDY.md')
#print(args)
vocab = {} #load_vocab('foo.json')
@ -56,10 +54,10 @@ def main():
## vocab now
## Lookup vocab
for word in vocab:
write_word_note(word, destination_path)
write_word_note(word, vocab_destination_path, vocab[word]['pos'][0])
## Output file with changes if changes
with open(args.markdown_file, "w", encoding="utf-8") as f:
with open(destination_path, "w", encoding="utf-8") as f:
f.writelines(lines)