121 lines
3.7 KiB
Python
121 lines
3.7 KiB
Python
import json
|
||
import MeCab
|
||
import re
|
||
|
||
# Initialize the Tagger
|
||
#tagger = MeCab.Tagger()
|
||
|
||
# Generate Hiragana and Katakana characters
|
||
hiragana = tuple(chr(i) for i in range(12353, 12436)) # \u3041 - \u3094
|
||
katakana = tuple(chr(i) for i in range(12450, 12532)) # \u30A2 - \u30F4
|
||
punctuation = tuple(chr(i) for i in range(65281, 65382))
|
||
|
||
# Combine both into one tuple
|
||
all_kana = hiragana + katakana + punctuation + (' ', 'ー', '・')
|
||
|
||
KANJI_RE = re.compile(r"[\u4E00-\u9FFF]")
|
||
|
||
def katakana_to_hiragana(text: str) -> str:
|
||
return "".join(
|
||
chr(ord(ch) - 0x60) if 0x30A1 <= ord(ch) <= 0x30F6 else ch
|
||
for ch in text
|
||
)
|
||
|
||
def get_node_reading(node) -> str:
|
||
features = node.feature.split(",")
|
||
print()
|
||
if len(features) >= 8 and features[7] != "*":
|
||
return katakana_to_hiragana(features[7])
|
||
return node.surface
|
||
|
||
def annotate_token(surface: str, reading: str) -> str:
|
||
if not KANJI_RE.search(surface):
|
||
return surface
|
||
|
||
# full-kanji token like 学校 -> 学校(がっこう)
|
||
if all(KANJI_RE.match(ch) for ch in surface):
|
||
return f"{surface}({reading})"
|
||
|
||
# prefix kanji + suffix kana, e.g. 行きます / 食べる
|
||
m = re.match(r"^([\u4E00-\u9FFF]+)([ぁ-んァ-ンー]*)$", surface)
|
||
if m:
|
||
kanji_part, kana_suffix = m.groups()
|
||
hira_suffix = katakana_to_hiragana(kana_suffix)
|
||
if reading.endswith(hira_suffix):
|
||
kanji_reading = reading[:len(reading) - len(hira_suffix)] if hira_suffix else reading
|
||
return f"{kanji_part}({kanji_reading}){kana_suffix}"
|
||
|
||
# fallback
|
||
return f"{surface}({reading})"
|
||
|
||
def annotate_phrase(text: str) -> str:
|
||
tagger = MeCab.Tagger()
|
||
node = tagger.parseToNode(text)
|
||
|
||
parts = []
|
||
while node:
|
||
surface = node.surface
|
||
if surface:
|
||
reading = get_node_reading(node)
|
||
parts.append(annotate_token(surface, reading))
|
||
node = node.next
|
||
|
||
return "".join(parts)
|
||
|
||
#print(annotate_phrase("私は学校へ行きます"))
|
||
|
||
with open("words-readings.json", "r", encoding="utf-16le") as f:
|
||
readings = json.load(f)
|
||
|
||
with open("dump.json", "r", encoding="utf-16le") as f:
|
||
dump = json.load(f)
|
||
|
||
## A micab phrase based word parser
|
||
for cd in dump:
|
||
print(dump[cd]['nm'])
|
||
# node = tagger.parseToNode(dump[cd]['nm'])
|
||
# while node:
|
||
# # node.surface is the word; node.feature contains POS tags
|
||
# if node.surface:
|
||
# print(f"{node.surface}\t{node.feature}")
|
||
# node = node.next
|
||
print(annotate_phrase(dump[cd]['nm']))
|
||
print()
|
||
|
||
exit()
|
||
|
||
## Iterates through all the keys not values
|
||
for cd in dump:
|
||
word = ''
|
||
new_name = ''
|
||
for char in dump[cd]['nm'] + '・': ## hacky char add to prevent dupe logic
|
||
if char in all_kana:
|
||
if len(word) > 0:
|
||
#print(word)
|
||
if readings.get(word) is not None:
|
||
kana = None
|
||
try:
|
||
#print("<--", readings[word]['kana'])
|
||
kana = readings[word]['kana']
|
||
new_name += '(' + kana + ')'
|
||
#print()
|
||
except TypeError as e:
|
||
kana = None
|
||
#words[word] = (words[word] + 1)
|
||
else:
|
||
print(word, " <- Not Found")
|
||
word = ''
|
||
else:
|
||
word += char
|
||
new_name += char
|
||
if new_name[:-1]!= dump[cd]['nm']:
|
||
print (dump[cd]['nm'])
|
||
print (new_name[:-1])
|
||
dump[cd]['kana'] = new_name[:-1]
|
||
## Might need catch for end of string still
|
||
|
||
print()
|
||
|
||
with open("dump-withkana.json", "w", encoding="utf-16le") as outfile:
|
||
json.dump(dump, outfile, ensure_ascii=False, indent=2)
|