poc pulling from yomitan

define-vocab.py

@@ -0,0 +1,125 @@
+import json
+import MeCab
+import re
+
+import requests
+
+request_url = "http://127.0.0.1:19633"
+request_timeout = 10
+
+# Initialize the Tagger
+#tagger = MeCab.Tagger()
+
+# Generate Hiragana and Katakana characters
+hiragana = tuple(chr(i) for i in range(12353, 12436)) # \u3041 - \u3094
+katakana = tuple(chr(i) for i in range(12450, 12532)) # \u30A2 - \u30F4
+punctuation = tuple(chr(i) for i in range(65281, 65382))
+
+def elide(text: str) -> str:
+    elide_max_length = 100
+    if len(text) > elide_max_length:
+        return text[:100] + "..."
+    return text
+
+def anki_fields_term(term : str) -> dict:
+    params = {
+        "text": term,
+        "type": "term",
+        "markers": ["audio", "cloze-body-kana", "conjugation", "expression", "furigana", "furigana-plain", "glossary", "glossary-brief", "glossary-no-dictionary", "glossary-first", "glossary-first-brief", "glossary-first-no-dictionary", "part-of-speech", "phonetic-transcriptions", "pitch-accents", "pitch-accent-graphs", "pitch-accent-graphs-jj", "pitch-accent-positions", "pitch-accent-categories", "reading", "tags", "clipboard-image", "clipboard-text", "cloze-body", "cloze-prefix", "cloze-suffix", "dictionary", "dictionary-alias", "document-title", "frequencies", "frequency-harmonic-rank", "frequency-harmonic-occurrence", "frequency-average-rank", "frequency-average-occurrence", "screenshot", "search-query", "popup-selection-text", "sentence", "sentence-furigana", "sentence-furigana-plain", "url"],
+        "maxEntries": 1,
+        "includeMedia": False,
+    }
+    response = requests.post(request_url + "/ankiFields", json = params, timeout = request_timeout)
+    return json.loads(response.text)
+
+def term_entries(term :str) -> dict:
+    print("Requesting termEntries:")
+    params = {
+        "term": term,
+        "maxEntries": 1,
+        "includeMedia": False,
+    }
+    response = requests.post(request_url + "/termEntries", json = params, timeout = request_timeout)
+    return json.loads(response.text)
+
+# perhaps set up a filter parameter.
+def flatten_string(node):
+    if isinstance(node,str):
+        yield node
+    elif isinstance(node, list):
+        for item in node:
+            yield from flatten_string(item)
+    elif isinstance(node,dict):
+        if isinstance(node.get("content"), str):
+            yield node["content"]
+        else:
+            for v in node.values():
+                yield from flatten_string(v)
+
+def extract_text_by_type(node, target_type):
+    if isinstance(node,dict):
+        data = node.get("data")
+        if isinstance(data,dict) and data.get("content") == target_type:
+            yield from flatten_string(node.get("content"))
+            #yield from ",".join(node.get("content"))
+        for value in node.values():
+            yield from extract_text_by_type(value, target_type)
+    elif isinstance(node,list):
+        for item in node:
+            yield from extract_text_by_type(item,target_type)
+
+# Combine both into one tuple
+all_kana = hiragana + katakana + punctuation + ('　', 'ー', '・')
+
+KANJI_RE = re.compile(r"[\u4E00-\u9FFF]")
+
+with open("dump-vocab.json", "r", encoding="utf-16") as f:
+    vocab = json.load(f)
+
+for item in vocab:
+    #print(word[0])
+    word = item[0]
+    print("Requesting termEntries:")
+    # params = {
+    #     "term": word,
+    # }
+    # response = requests.post(request_url + "/termEntries", json = params, timeout = request_timeout)
+    # print(response)
+    # print(elide(response.text))
+    # print(response.json()) # Dumps json
+    dict_items = term_entries(word)
+    # for dic in dict_items['dictionaryEntries']:
+    #     print(dic)
+    ## Recurssive content -> data pipelines
+    print(word)
+    glossary = list()
+
+    entry = next(iter(dict_items.get("dictionaryEntries",[])),None)
+    if entry:
+        definition = next(iter(entry.get("definitions",[])),None)
+        if definition:
+            def_entry = next(iter(definition.get("entries", [])),None)
+            if def_entry:
+                glossary.extend( list(extract_text_by_type(def_entry, "glossary")) )
+    glossary[:2]
+#    for entry in dict_items.get("dictionaryEntries", []):
+#        for definition in entry.get("definitions", []):
+            # first = next(iter(definition.get("entries", [])),None)
+            # if first:
+            #     glossary = list(extract_text_by_type(first, "glossary"))
+            #     if len(glossary) > 0:
+            #         break
+#            for def_entry in definition.get("entries", []):
+#                glossary.extend( list(extract_text_by_type(def_entry, "glossary")) )
+                # for content in def_entry.get("content", []):
+                #     if content.get('data') is not None:
+                #         data = content['data']
+                #         print(data['content']) # type of content
+            
+    #anki_card = anki_fields_term(word)
+    item.append("; ".join(glossary))
+    print()
+
+with open("dump-vocab.json", "w", encoding="utf-16") as outfile:
+    json.dump(vocab, outfile, ensure_ascii=False, indent=2)
+