83 lines
3.2 KiB
Python
83 lines
3.2 KiB
Python
from parser import *
|
|
|
|
#print(annotate_phrase("私は学校へ行きます"))
|
|
|
|
with open("dump2.json", "r", encoding="utf-16") as f:
|
|
dump = json.load(f)
|
|
|
|
# foo = anki_fields_term(surface)
|
|
# for entry in foo.get("fields"):
|
|
# entry['expression']
|
|
# entry['glossary-first-brief']
|
|
# entry['glossary-plain-no-dictionary']
|
|
# entry['frequency-average-rank']
|
|
|
|
vocab_dict = {}
|
|
## A micab phrase based word parser
|
|
for cd in dump:
|
|
print(dump[cd]['name_J'])
|
|
# node = tagger.parseToNode(dump[cd]['name_J'])
|
|
# while node:
|
|
# # node.surface is the word; node.feature contains POS tags
|
|
# if node.surface:
|
|
# print(f"{node.surface}\t{node.feature}")
|
|
# node = node.next
|
|
anottated = annotate_phrase(dump[cd]['name_J'])
|
|
if anottated['reading'] != dump[cd]['name_J']:
|
|
#print(anottated['reading'])
|
|
dump[cd]['kana'] = anottated['reading']
|
|
dump[cd]['furi'] = anottated['furigana']
|
|
cd_nm = dump[cd]['name_J']
|
|
if (anottated.get('vocab')):
|
|
dump[cd]['glossary_Name_J'] = {}
|
|
for word in anottated['vocab']:
|
|
# Skip punctuation words hopefully
|
|
if len(word) == 1 and has_kanji(word) == False:
|
|
continue
|
|
if vocab_dict.get(word) is None:
|
|
vocab_dict[word] = {
|
|
'Def.': "; ".join(get_glossary(word)),
|
|
'count': 0,
|
|
'Examples': (cd_nm,)
|
|
}
|
|
else:
|
|
vocab_dict[word]['count'] = vocab_dict[word]['count'] + 1
|
|
if cd_nm not in vocab_dict[word]['Examples']:
|
|
vocab_dict[word]['Examples'] += (cd_nm,)
|
|
if vocab_dict.get(word) and dump[cd]['glossary_Name_J'].get(word) is None:
|
|
dump[cd]['glossary_Name_J'][word] = vocab_dict[word]['Def.']
|
|
#append(f"{word}: {vocab_dict[word]['Def.']}")
|
|
print()
|
|
phrase = dump[cd]['desc_J']
|
|
anottated = annotate_phrase(phrase)
|
|
if anottated['reading'] != phrase:
|
|
#print(anottated['reading'])
|
|
dump[cd]['desc_kana'] = anottated['reading']
|
|
dump[cd]['desc_Furi'] = anottated['furigana']
|
|
if (anottated.get('vocab')):
|
|
dump[cd]['glossary_Desc_J'] = {}
|
|
for word in anottated['vocab']:
|
|
if len(word) == 1 and has_kanji(word) == False:
|
|
continue
|
|
if vocab_dict.get(word) is None:
|
|
vocab_dict[word] = {
|
|
'Def.': "; ".join(get_glossary(word)),
|
|
'count': 0,
|
|
'Examples': (cd_nm,)
|
|
}
|
|
else:
|
|
vocab_dict[word]['count'] = vocab_dict[word]['count'] + 1
|
|
if len(vocab_dict[word]['Examples']) < 5:
|
|
if cd_nm not in vocab_dict[word]['Examples']:
|
|
vocab_dict[word]['Examples'] += (cd_nm,)
|
|
if vocab_dict.get(word) and dump[cd]['glossary_Desc_J'].get(word) is None:
|
|
dump[cd]['glossary_Desc_J'][word] = vocab_dict[word]['Def.']
|
|
|
|
vocab_dict = sorted(vocab_dict.items(), key= lambda x: x[1]['count'], reverse=True)
|
|
|
|
#with open("dump-withFuri.json", "w", encoding="utf-16") as outfile:
|
|
# json.dump(dump, outfile, ensure_ascii=False, indent=2)
|
|
|
|
#with open("dump-vocab.json", "w", encoding="utf-16") as outfile:
|
|
# json.dump(vocab_dict, outfile, ensure_ascii=False, indent=2)
|