Dustin/jp_text_parsing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

these two are it

Browse Source
This commit is contained in:
Nitsud Yarg 2026-04-22 22:11:44 -07:00
parent 100a25b6d9
commit db0682203c
2 changed files with 11 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
add-readings.py
View File

@ -58,27 +58,24 @@ def annotate_phrase(text: str) -> str:
#print(annotate_phrase("私は学校へ行きます")) #print(annotate_phrase("私は学校へ行きます"))
with open("words-readings.json", "r", encoding="utf-16le") as f: with open("dump2.json", "r", encoding="utf-16") as f:
readings = json.load(f)
with open("dump.json", "r", encoding="utf-16le") as f:
dump = json.load(f) dump = json.load(f)
vocab_dict = {} vocab_dict = {}
## A micab phrase based word parser ## A micab phrase based word parser
for cd in dump: for cd in dump:
print(dump[cd]['nm']) print(dump[cd]['name_J'])
# node = tagger.parseToNode(dump[cd]['nm']) # node = tagger.parseToNode(dump[cd]['name_J'])
# while node: # while node:
# # node.surface is the word; node.feature contains POS tags # # node.surface is the word; node.feature contains POS tags
# if node.surface: # if node.surface:
# print(f"{node.surface}\t{node.feature}") # print(f"{node.surface}\t{node.feature}")
# node = node.next # node = node.next
anottated = annotate_phrase(dump[cd]['nm']) anottated = annotate_phrase(dump[cd]['name_J'])
if anottated['reading'] != dump[cd]['nm']: if anottated['reading'] != dump[cd]['name_J']:
print(anottated['reading']) print(anottated['reading'])
dump[cd]['kana'] = anottated['reading'] dump[cd]['kana'] = anottated['reading']
cd_nm = dump[cd]['nm'] cd_nm = dump[cd]['name_J']
for word in anottated['vocab']: for word in anottated['vocab']:
# Skip punctuation words hopefully # Skip punctuation words hopefully
if len(word) == 1 and has_kanji(word) == False: if len(word) == 1 and has_kanji(word) == False:
@ -93,11 +90,11 @@ for cd in dump:
if cd_nm not in vocab_dict[word]['Examples']: if cd_nm not in vocab_dict[word]['Examples']:
vocab_dict[word]['Examples'] += (cd_nm,) vocab_dict[word]['Examples'] += (cd_nm,)
print() print()
phrase = dump[cd]['de'] phrase = dump[cd]['desc_J']
anottated = annotate_phrase(phrase) anottated = annotate_phrase(phrase)
if anottated['reading'] != phrase: if anottated['reading'] != phrase:
#print(anottated['reading']) #print(anottated['reading'])
dump[cd]['furi_de'] = anottated['reading'] dump[cd]['desc_Furi'] = anottated['reading']
for word in anottated['vocab']: for word in anottated['vocab']:
if len(word) == 1 and has_kanji(word) == False: if len(word) == 1 and has_kanji(word) == False:
continue continue
@ -114,7 +111,7 @@ for cd in dump:
vocab_dict = sorted(vocab_dict.items(), key= lambda x: x[1]['count'], reverse=True) vocab_dict = sorted(vocab_dict.items(), key= lambda x: x[1]['count'], reverse=True)
with open("dump-withkana2.json", "w", encoding="utf-16") as outfile: with open("dump2.json", "w", encoding="utf-16") as outfile:
json.dump(dump, outfile, ensure_ascii=False, indent=2) json.dump(dump, outfile, ensure_ascii=False, indent=2)
with open("dump-vocab.json", "w", encoding="utf-16") as outfile: with open("dump-vocab.json", "w", encoding="utf-16") as outfile:

3
get-properties.py
View File

@ -1,6 +1,7 @@
import mmap import mmap
import struct import struct
from pathlib import Path from pathlib import Path
import json
prop_data = Path(f"YGO_2020/bin/CARD_Prop.bin").read_bytes() # Get all the bytes prop_data = Path(f"YGO_2020/bin/CARD_Prop.bin").read_bytes() # Get all the bytes
@ -86,7 +87,7 @@ desc_data = Path(f"YGO_2020/bin/CARD_Desc_{language_code}.bin").read_bytes()
get_props() get_props()
with open("dump2.json", "w", encoding="utf-16le") as outfile: with open("dump2.json", "w", encoding="utf-16") as outfile:
json.dump(cd_db, outfile, ensure_ascii=False, indent=2) json.dump(cd_db, outfile, ensure_ascii=False, indent=2)
print() print()