Dustin/jp_text_parsing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

better output and note page generation.

Browse Source
This commit is contained in:
Nitsud Yarg 2026-04-29 20:29:15 -07:00
parent 0c1eebd071
commit 26e43ea184
2 changed files with 41 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
add-readings.py
View File

@ -45,6 +45,8 @@ def annotate_phrase(text: str) -> str:
node = tagger.parseToNode(text) node = tagger.parseToNode(text)
vocab = () vocab = ()
parts = [] parts = []
# <ruby>漢<rt>かん</rt></ruby><ruby>字<rt>じ</rt></ruby>
furi = []
while node: while node:
surface = node.surface surface = node.surface
if surface: if surface:
@ -52,11 +54,13 @@ def annotate_phrase(text: str) -> str:
if has_kanji(surface): if has_kanji(surface):
reading = get_node_reading(node) reading = get_node_reading(node)
parts.append(f"{surface}({reading})") parts.append(f"{surface}({reading})")
furi.append(f"<ruby>{surface}<rt>{reading}</rt></ruby>")
else: else:
parts.append(surface) parts.append(surface)
furi.append(surface)
node = node.next node = node.next
return {'reading':("".join(parts)),'vocab':vocab} return {'reading':("".join(parts)),'vocab':vocab,"furigana":("".join(furi))}
def elide(text: str) -> str: def elide(text: str) -> str:
elide_max_length = 100 elide_max_length = 100
@ -141,11 +145,12 @@ for cd in dump:
# node = node.next # node = node.next
anottated = annotate_phrase(dump[cd]['name_J']) anottated = annotate_phrase(dump[cd]['name_J'])
if anottated['reading'] != dump[cd]['name_J']: if anottated['reading'] != dump[cd]['name_J']:
print(anottated['reading']) #print(anottated['reading'])
dump[cd]['kana'] = anottated['reading'] dump[cd]['kana'] = anottated['reading']
dump[cd]['furi'] = anottated['furigana']
cd_nm = dump[cd]['name_J'] cd_nm = dump[cd]['name_J']
if (anottated.get('vocab')): if (anottated.get('vocab')):
dump[cd]['glossary_Name_J'] = list() dump[cd]['glossary_Name_J'] = {}
for word in anottated['vocab']: for word in anottated['vocab']:
# Skip punctuation words hopefully # Skip punctuation words hopefully
if len(word) == 1 and has_kanji(word) == False: if len(word) == 1 and has_kanji(word) == False:
@ -160,16 +165,18 @@ for cd in dump:
vocab_dict[word]['count'] = vocab_dict[word]['count'] + 1 vocab_dict[word]['count'] = vocab_dict[word]['count'] + 1
if cd_nm not in vocab_dict[word]['Examples']: if cd_nm not in vocab_dict[word]['Examples']:
vocab_dict[word]['Examples'] += (cd_nm,) vocab_dict[word]['Examples'] += (cd_nm,)
if vocab_dict.get(word): if vocab_dict.get(word) and dump[cd]['glossary_Name_J'].get(word) is None:
dump[cd]['glossary_Name_J'].append(f"{word}: {vocab_dict[word]['Def.']}") dump[cd]['glossary_Name_J'][word] = vocab_dict[word]['Def.']
#append(f"{word}: {vocab_dict[word]['Def.']}")
print() print()
phrase = dump[cd]['desc_J'] phrase = dump[cd]['desc_J']
anottated = annotate_phrase(phrase) anottated = annotate_phrase(phrase)
if anottated['reading'] != phrase: if anottated['reading'] != phrase:
#print(anottated['reading']) #print(anottated['reading'])
dump[cd]['desc_Furi'] = anottated['reading'] dump[cd]['desc_kana'] = anottated['reading']
dump[cd]['desc_Furi'] = anottated['furigana']
if (anottated.get('vocab')): if (anottated.get('vocab')):
dump[cd]['glossary_Desc_J'] = list() dump[cd]['glossary_Desc_J'] = {}
for word in anottated['vocab']: for word in anottated['vocab']:
if len(word) == 1 and has_kanji(word) == False: if len(word) == 1 and has_kanji(word) == False:
continue continue
@ -184,13 +191,13 @@ for cd in dump:
if len(vocab_dict[word]['Examples']) < 5: if len(vocab_dict[word]['Examples']) < 5:
if cd_nm not in vocab_dict[word]['Examples']: if cd_nm not in vocab_dict[word]['Examples']:
vocab_dict[word]['Examples'] += (cd_nm,) vocab_dict[word]['Examples'] += (cd_nm,)
if vocab_dict.get(word): if vocab_dict.get(word) and dump[cd]['glossary_Desc_J'].get(word) is None:
dump[cd]['glossary_Desc_J'].append(f"{word}: {vocab_dict[word]['Def.']}") dump[cd]['glossary_Desc_J'][word] = vocab_dict[word]['Def.']
vocab_dict = sorted(vocab_dict.items(), key= lambda x: x[1]['count'], reverse=True) vocab_dict = sorted(vocab_dict.items(), key= lambda x: x[1]['count'], reverse=True)
with open("dump2.json", "w", encoding="utf-16") as outfile: with open("dump-withFuri.json", "w", encoding="utf-16") as outfile:
json.dump(dump, outfile, ensure_ascii=False, indent=2) json.dump(dump, outfile, ensure_ascii=False, indent=2)
with open("dump-vocab.json", "w", encoding="utf-16") as outfile: #with open("dump-vocab.json", "w", encoding="utf-16") as outfile:
json.dump(vocab_dict, outfile, ensure_ascii=False, indent=2) # json.dump(vocab_dict, outfile, ensure_ascii=False, indent=2)

33
create-notes.py
View File

@ -82,7 +82,7 @@ if not destination_path.exists():
if not destination_path.joinpath('_attachments').exists(): if not destination_path.joinpath('_attachments').exists():
destination_path.joinpath('_attachments').mkdir(parents=True, exist_ok=True) destination_path.joinpath('_attachments').mkdir(parents=True, exist_ok=True)
with open("dump2.json", "r", encoding="utf-16") as f: with open("dump-withFuri.json", "r", encoding="utf-16") as f:
dump = json.load(f) dump = json.load(f)
for cd in dump: for cd in dump:
@ -97,22 +97,33 @@ for cd in dump:
card_note.write("---\n") card_note.write("---\n")
## Name at the top ## Name at the top
card_note.write(f"![[{cd}.jpg]]") card_note.write(f"![[{cd}.jpg]]")
card_note.write("\n<details><summary>\n") card_note.write("\n##### Name\n")
card_note.write("<details><summary>")
card_note.write(dump[cd]['name_J']) card_note.write(dump[cd]['name_J'])
card_note.write("\n</summary>\n") card_note.write("</summary>")
card_note.write(dump[cd]['name_E']) card_note.write(dump[cd]['name_E'])
card_note.write("\n</details>\n") card_note.write("</details>")
if(dump[cd].get("kana")):
card_note.write(f"{dump[cd]['kana']}\n")
## the description ## the description
card_note.write("\n##### Details\n")
card_note.write("<details><summary>")
card_note.write(dump[cd]['desc_J']) card_note.write(dump[cd]['desc_J'])
card_note.write("<details><summary>\n") card_note.write("</summary>")
card_note.write("Translation")
card_note.write("\n</summary>\n")
card_note.write(dump[cd]['desc_E']) card_note.write(dump[cd]['desc_E'])
card_note.write("\n</details>\n") card_note.write("</details>")
# desc_Furi # desc_Furi
card_note.write("\n##### Reading\n")
if(dump[cd].get("furi")):
card_note.write(f"{dump[cd]['furi']}\n")
else:
card_note.write(dump[cd]['name_J'])
card_note.write("\n")
if(dump[cd].get("desc_Furi")): if(dump[cd].get("desc_Furi")):
card_note.write(f"{dump[cd]['desc_Furi']}\n") card_note.write(f"{dump[cd]['desc_Furi']}\n")
else:
card_note.write(dump[cd]['desc_J'])
card_note.write("\n")
card_note.write("\n##### Glossary\n")
for vocab in dump[cd]["glossary_Desc_J"]: for vocab in dump[cd]["glossary_Desc_J"]:
print(vocab, file=card_note) print(f"{vocab} : {dump[cd]["glossary_Desc_J"][vocab]}", file=card_note)
#print('>' + dump[cd]["glossary_Desc_J"][vocab], file=card_note)