diff --git a/add-readings.py b/add-readings.py index 8c24f1f..cd33011 100644 --- a/add-readings.py +++ b/add-readings.py @@ -1,4 +1,9 @@ import json +import MeCab +import re + +# Initialize the Tagger +#tagger = MeCab.Tagger() # Generate Hiragana and Katakana characters hiragana = tuple(chr(i) for i in range(12353, 12436)) # \u3041 - \u3094 @@ -8,6 +13,56 @@ punctuation = tuple(chr(i) for i in range(65281, 65382)) # Combine both into one tuple all_kana = hiragana + katakana + punctuation + (' ', 'ー', '・') +KANJI_RE = re.compile(r"[\u4E00-\u9FFF]") + +def katakana_to_hiragana(text: str) -> str: + return "".join( + chr(ord(ch) - 0x60) if 0x30A1 <= ord(ch) <= 0x30F6 else ch + for ch in text + ) + +def get_node_reading(node) -> str: + features = node.feature.split(",") + print() + if len(features) >= 8 and features[7] != "*": + return katakana_to_hiragana(features[7]) + return node.surface + +def annotate_token(surface: str, reading: str) -> str: + if not KANJI_RE.search(surface): + return surface + + # full-kanji token like 学校 -> 学校(がっこう) + if all(KANJI_RE.match(ch) for ch in surface): + return f"{surface}({reading})" + + # prefix kanji + suffix kana, e.g. 行きます / 食べる + m = re.match(r"^([\u4E00-\u9FFF]+)([ぁ-んァ-ンー]*)$", surface) + if m: + kanji_part, kana_suffix = m.groups() + hira_suffix = katakana_to_hiragana(kana_suffix) + if reading.endswith(hira_suffix): + kanji_reading = reading[:len(reading) - len(hira_suffix)] if hira_suffix else reading + return f"{kanji_part}({kanji_reading}){kana_suffix}" + + # fallback + return f"{surface}({reading})" + +def annotate_phrase(text: str) -> str: + tagger = MeCab.Tagger() + node = tagger.parseToNode(text) + + parts = [] + while node: + surface = node.surface + if surface: + reading = get_node_reading(node) + parts.append(annotate_token(surface, reading)) + node = node.next + + return "".join(parts) + +#print(annotate_phrase("私は学校へ行きます")) with open("words-readings.json", "r", encoding="utf-16le") as f: readings = json.load(f) @@ -15,6 +70,20 @@ with open("words-readings.json", "r", encoding="utf-16le") as f: with open("dump.json", "r", encoding="utf-16le") as f: dump = json.load(f) +## A micab phrase based word parser +for cd in dump: + print(dump[cd]['nm']) + # node = tagger.parseToNode(dump[cd]['nm']) + # while node: + # # node.surface is the word; node.feature contains POS tags + # if node.surface: + # print(f"{node.surface}\t{node.feature}") + # node = node.next + print(annotate_phrase(dump[cd]['nm'])) + print() + +exit() + ## Iterates through all the keys not values for cd in dump: word = ''