import json import MeCab import re # Initialize the Tagger #tagger = MeCab.Tagger() # Generate Hiragana and Katakana characters hiragana = tuple(chr(i) for i in range(12353, 12436)) # \u3041 - \u3094 katakana = tuple(chr(i) for i in range(12450, 12532)) # \u30A2 - \u30F4 punctuation = tuple(chr(i) for i in range(65281, 65382)) # Combine both into one tuple all_kana = hiragana + katakana + punctuation + (' ', 'ー', '・') KANJI_RE = re.compile(r"[\u4E00-\u9FFF]") def katakana_to_hiragana(text: str) -> str: return "".join( chr(ord(ch) - 0x60) if 0x30A1 <= ord(ch) <= 0x30F6 else ch for ch in text ) def get_node_reading(node) -> str: features = node.feature.split(",") print() if len(features) >= 8 and features[7] != "*": return katakana_to_hiragana(features[7]) return node.surface def has_kanji(text: str) -> bool: return bool(KANJI_RE.search(text)) def get_node_reading(node) -> str: features = node.feature.split(",") ## These depend on what dictionary is being used. if len(features) > 6 and features[6] != "*": return katakana_to_hiragana(features[6]) return node.surface def annotate_phrase(text: str) -> str: tagger = MeCab.Tagger() node = tagger.parseToNode(text) vocab = () parts = [] while node: surface = node.surface if surface: vocab += (surface,) if has_kanji(surface): reading = get_node_reading(node) parts.append(f"{surface}({reading})") else: parts.append(surface) node = node.next return {'reading':("".join(parts)),'vocab':vocab} #print(annotate_phrase("私は学校へ行きます")) with open("words-readings.json", "r", encoding="utf-16le") as f: readings = json.load(f) with open("dump.json", "r", encoding="utf-16le") as f: dump = json.load(f) ## A micab phrase based word parser for cd in dump: print(dump[cd]['nm']) # node = tagger.parseToNode(dump[cd]['nm']) # while node: # # node.surface is the word; node.feature contains POS tags # if node.surface: # print(f"{node.surface}\t{node.feature}") # node = node.next anottated = annotate_phrase(dump[cd]['nm']) if anottated['reading'] != dump[cd]['nm']: print(anottated['reading']) dump[cd]['kana'] = anottated['reading'] #dump[cd]['vocab'] = anottated['vocab'] print() exit() ## Iterates through all the keys not values for cd in dump: word = '' new_name = '' for char in dump[cd]['nm'] + '・': ## hacky char add to prevent dupe logic if char in all_kana: if len(word) > 0: #print(word) if readings.get(word) is not None: kana = None try: #print("<--", readings[word]['kana']) kana = readings[word]['kana'] new_name += '(' + kana + ')' #print() except TypeError as e: kana = None #words[word] = (words[word] + 1) else: print(word, " <- Not Found") word = '' else: word += char new_name += char if new_name[:-1]!= dump[cd]['nm']: print (dump[cd]['nm']) print (new_name[:-1]) dump[cd]['kana'] = new_name[:-1] ## Might need catch for end of string still print() with open("dump-withkana.json", "w", encoding="utf-16le") as outfile: json.dump(dump, outfile, ensure_ascii=False, indent=2)