119 lines
3.8 KiB
Python
119 lines
3.8 KiB
Python
import json
|
||
import MeCab
|
||
import re
|
||
|
||
# Initialize the Tagger
|
||
#tagger = MeCab.Tagger()
|
||
|
||
# Generate Hiragana and Katakana characters
|
||
hiragana = tuple(chr(i) for i in range(12353, 12436)) # \u3041 - \u3094
|
||
katakana = tuple(chr(i) for i in range(12450, 12532)) # \u30A2 - \u30F4
|
||
punctuation = tuple(chr(i) for i in range(65281, 65382))
|
||
|
||
# Combine both into one tuple
|
||
all_kana = hiragana + katakana + punctuation + (' ', 'ー', '・')
|
||
|
||
KANJI_RE = re.compile(r"[\u4E00-\u9FFF]")
|
||
|
||
def katakana_to_hiragana(text: str) -> str:
|
||
return "".join(
|
||
chr(ord(ch) - 0x60) if 0x30A1 <= ord(ch) <= 0x30F6 else ch
|
||
for ch in text
|
||
)
|
||
|
||
def get_node_reading(node) -> str:
|
||
features = node.feature.split(",")
|
||
print()
|
||
if len(features) >= 8 and features[7] != "*":
|
||
return katakana_to_hiragana(features[7])
|
||
return node.surface
|
||
|
||
def has_kanji(text: str) -> bool:
|
||
return bool(KANJI_RE.search(text))
|
||
|
||
def get_node_reading(node) -> str:
|
||
features = node.feature.split(",")
|
||
## These depend on what dictionary is being used.
|
||
if len(features) > 6 and features[6] != "*":
|
||
return katakana_to_hiragana(features[6])
|
||
return node.surface
|
||
|
||
def annotate_phrase(text: str) -> str:
|
||
tagger = MeCab.Tagger()
|
||
node = tagger.parseToNode(text)
|
||
vocab = ()
|
||
parts = []
|
||
while node:
|
||
surface = node.surface
|
||
if surface:
|
||
vocab += (surface,)
|
||
if has_kanji(surface):
|
||
reading = get_node_reading(node)
|
||
parts.append(f"{surface}({reading})")
|
||
else:
|
||
parts.append(surface)
|
||
node = node.next
|
||
|
||
return {'reading':("".join(parts)),'vocab':vocab}
|
||
|
||
#print(annotate_phrase("私は学校へ行きます"))
|
||
|
||
with open("dump2.json", "r", encoding="utf-16") as f:
|
||
dump = json.load(f)
|
||
|
||
vocab_dict = {}
|
||
## A micab phrase based word parser
|
||
for cd in dump:
|
||
print(dump[cd]['name_J'])
|
||
# node = tagger.parseToNode(dump[cd]['name_J'])
|
||
# while node:
|
||
# # node.surface is the word; node.feature contains POS tags
|
||
# if node.surface:
|
||
# print(f"{node.surface}\t{node.feature}")
|
||
# node = node.next
|
||
anottated = annotate_phrase(dump[cd]['name_J'])
|
||
if anottated['reading'] != dump[cd]['name_J']:
|
||
print(anottated['reading'])
|
||
dump[cd]['kana'] = anottated['reading']
|
||
cd_nm = dump[cd]['name_J']
|
||
for word in anottated['vocab']:
|
||
# Skip punctuation words hopefully
|
||
if len(word) == 1 and has_kanji(word) == False:
|
||
continue
|
||
if vocab_dict.get(word) is None:
|
||
vocab_dict[word] = {
|
||
'count': 0,
|
||
'Examples': (cd_nm,)
|
||
}
|
||
else:
|
||
vocab_dict[word]['count'] = vocab_dict[word]['count'] + 1
|
||
if cd_nm not in vocab_dict[word]['Examples']:
|
||
vocab_dict[word]['Examples'] += (cd_nm,)
|
||
print()
|
||
phrase = dump[cd]['desc_J']
|
||
anottated = annotate_phrase(phrase)
|
||
if anottated['reading'] != phrase:
|
||
#print(anottated['reading'])
|
||
dump[cd]['desc_Furi'] = anottated['reading']
|
||
for word in anottated['vocab']:
|
||
if len(word) == 1 and has_kanji(word) == False:
|
||
continue
|
||
if vocab_dict.get(word) is None:
|
||
vocab_dict[word] = {
|
||
'count': 0,
|
||
'Examples': (cd_nm,)
|
||
}
|
||
else:
|
||
vocab_dict[word]['count'] = vocab_dict[word]['count'] + 1
|
||
if len(vocab_dict[word]['Examples']) < 5:
|
||
if cd_nm not in vocab_dict[word]['Examples']:
|
||
vocab_dict[word]['Examples'] += (cd_nm,)
|
||
|
||
vocab_dict = sorted(vocab_dict.items(), key= lambda x: x[1]['count'], reverse=True)
|
||
|
||
with open("dump2.json", "w", encoding="utf-16") as outfile:
|
||
json.dump(dump, outfile, ensure_ascii=False, indent=2)
|
||
|
||
with open("dump-vocab.json", "w", encoding="utf-16") as outfile:
|
||
json.dump(vocab_dict, outfile, ensure_ascii=False, indent=2)
|