mecab reading version

add-readings.py

@@ -1,4 +1,9 @@
 import json
+import MeCab
+import re
+
+# Initialize the Tagger
+#tagger = MeCab.Tagger()
 
 # Generate Hiragana and Katakana characters
 hiragana = tuple(chr(i) for i in range(12353, 12436)) # \u3041 - \u3094
@@ -8,6 +13,56 @@ punctuation = tuple(chr(i) for i in range(65281, 65382))
 # Combine both into one tuple
 all_kana = hiragana + katakana + punctuation + ('　', 'ー', '・')
 
+KANJI_RE = re.compile(r"[\u4E00-\u9FFF]")
+
+def katakana_to_hiragana(text: str) -> str:
+    return "".join(
+        chr(ord(ch) - 0x60) if 0x30A1 <= ord(ch) <= 0x30F6 else ch
+        for ch in text
+    )
+
+def get_node_reading(node) -> str:
+    features = node.feature.split(",")
+    print()
+    if len(features) >= 8 and features[7] != "*":
+        return katakana_to_hiragana(features[7])
+    return node.surface
+
+def annotate_token(surface: str, reading: str) -> str:
+    if not KANJI_RE.search(surface):
+        return surface
+
+    # full-kanji token like 学校 -> 学校(がっこう)
+    if all(KANJI_RE.match(ch) for ch in surface):
+        return f"{surface}({reading})"
+
+    # prefix kanji + suffix kana, e.g. 行きます / 食べる
+    m = re.match(r"^([\u4E00-\u9FFF]+)([ぁ-んァ-ンー]*)$", surface)
+    if m:
+        kanji_part, kana_suffix = m.groups()
+        hira_suffix = katakana_to_hiragana(kana_suffix)
+        if reading.endswith(hira_suffix):
+            kanji_reading = reading[:len(reading) - len(hira_suffix)] if hira_suffix else reading
+            return f"{kanji_part}({kanji_reading}){kana_suffix}"
+
+    # fallback
+    return f"{surface}({reading})"
+
+def annotate_phrase(text: str) -> str:
+    tagger = MeCab.Tagger()
+    node = tagger.parseToNode(text)
+
+    parts = []
+    while node:
+        surface = node.surface
+        if surface:
+            reading = get_node_reading(node)
+            parts.append(annotate_token(surface, reading))
+        node = node.next
+
+    return "".join(parts)
+
+#print(annotate_phrase("私は学校へ行きます"))
 
 with open("words-readings.json", "r", encoding="utf-16le") as f:
     readings = json.load(f)
@@ -15,6 +70,20 @@ with open("words-readings.json", "r", encoding="utf-16le") as f:
 with open("dump.json", "r", encoding="utf-16le") as f:
     dump = json.load(f)
 
+## A micab phrase based word parser
+for cd in dump:
+    print(dump[cd]['nm'])
+    # node = tagger.parseToNode(dump[cd]['nm'])
+    # while node:
+    #     # node.surface is the word; node.feature contains POS tags
+    #     if node.surface:
+    #         print(f"{node.surface}\t{node.feature}")
+    #     node = node.next
+    print(annotate_phrase(dump[cd]['nm']))
+    print()
+
+exit()
+
 ## Iterates through all the keys not values
 for cd in dump:
     word = ''