import json import MeCab import re # Initialize the Tagger #tagger = MeCab.Tagger() # Generate Hiragana and Katakana characters hiragana = tuple(chr(i) for i in range(12353, 12436)) # \u3041 - \u3094 katakana = tuple(chr(i) for i in range(12450, 12532)) # \u30A2 - \u30F4 punctuation = tuple(chr(i) for i in range(65281, 65382)) # Combine both into one tuple all_kana = hiragana + katakana + punctuation + (' ', 'ー', '・') KANJI_RE = re.compile(r"[\u4E00-\u9FFF]") def katakana_to_hiragana(text: str) -> str: return "".join( chr(ord(ch) - 0x60) if 0x30A1 <= ord(ch) <= 0x30F6 else ch for ch in text ) def get_node_reading(node) -> str: features = node.feature.split(",") print() if len(features) >= 8 and features[7] != "*": return katakana_to_hiragana(features[7]) return node.surface def annotate_token(surface: str, reading: str) -> str: if not KANJI_RE.search(surface): return surface # full-kanji token like 学校 -> 学校(がっこう) if all(KANJI_RE.match(ch) for ch in surface): return f"{surface}({reading})" # prefix kanji + suffix kana, e.g. 行きます / 食べる m = re.match(r"^([\u4E00-\u9FFF]+)([ぁ-んァ-ンー]*)$", surface) if m: kanji_part, kana_suffix = m.groups() hira_suffix = katakana_to_hiragana(kana_suffix) if reading.endswith(hira_suffix): kanji_reading = reading[:len(reading) - len(hira_suffix)] if hira_suffix else reading return f"{kanji_part}({kanji_reading}){kana_suffix}" # fallback return f"{surface}({reading})" def annotate_phrase(text: str) -> str: tagger = MeCab.Tagger() node = tagger.parseToNode(text) parts = [] while node: surface = node.surface if surface: reading = get_node_reading(node) parts.append(annotate_token(surface, reading)) node = node.next return "".join(parts) #print(annotate_phrase("私は学校へ行きます")) with open("words-readings.json", "r", encoding="utf-16le") as f: readings = json.load(f) with open("dump.json", "r", encoding="utf-16le") as f: dump = json.load(f) ## A micab phrase based word parser for cd in dump: print(dump[cd]['nm']) # node = tagger.parseToNode(dump[cd]['nm']) # while node: # # node.surface is the word; node.feature contains POS tags # if node.surface: # print(f"{node.surface}\t{node.feature}") # node = node.next print(annotate_phrase(dump[cd]['nm'])) print() exit() ## Iterates through all the keys not values for cd in dump: word = '' new_name = '' for char in dump[cd]['nm'] + '・': ## hacky char add to prevent dupe logic if char in all_kana: if len(word) > 0: #print(word) if readings.get(word) is not None: kana = None try: #print("<--", readings[word]['kana']) kana = readings[word]['kana'] new_name += '(' + kana + ')' #print() except TypeError as e: kana = None #words[word] = (words[word] + 1) else: print(word, " <- Not Found") word = '' else: word += char new_name += char if new_name[:-1]!= dump[cd]['nm']: print (dump[cd]['nm']) print (new_name[:-1]) dump[cd]['kana'] = new_name[:-1] ## Might need catch for end of string still print() with open("dump-withkana.json", "w", encoding="utf-16le") as outfile: json.dump(dump, outfile, ensure_ascii=False, indent=2)