progress subs to notes

add-readings.py

@@ -1,118 +1,82 @@
-import json
-import MeCab
-import re
-
-# Initialize the Tagger
-#tagger = MeCab.Tagger()
-
-# Generate Hiragana and Katakana characters
-hiragana = tuple(chr(i) for i in range(12353, 12436)) # \u3041 - \u3094
-katakana = tuple(chr(i) for i in range(12450, 12532)) # \u30A2 - \u30F4
-punctuation = tuple(chr(i) for i in range(65281, 65382))
-
-# Combine both into one tuple
-all_kana = hiragana + katakana + punctuation + ('　', 'ー', '・')
-
-KANJI_RE = re.compile(r"[\u4E00-\u9FFF]")
-
-def katakana_to_hiragana(text: str) -> str:
-    return "".join(
-        chr(ord(ch) - 0x60) if 0x30A1 <= ord(ch) <= 0x30F6 else ch
-        for ch in text
-    )
-
-def get_node_reading(node) -> str:
-    features = node.feature.split(",")
-    print()
-    if len(features) >= 8 and features[7] != "*":
-        return katakana_to_hiragana(features[7])
-    return node.surface
-
-def has_kanji(text: str) -> bool:
-    return bool(KANJI_RE.search(text))
-
-def get_node_reading(node) -> str:
-    features = node.feature.split(",")
-    ## These depend on what dictionary is being used.
-    if len(features) > 6 and features[6] != "*":
-        return katakana_to_hiragana(features[6])
-    return node.surface
-
-def annotate_phrase(text: str) -> str:
-    tagger = MeCab.Tagger()
-    node = tagger.parseToNode(text)
-    vocab = ()
-    parts = []
-    while node:
-        surface = node.surface
-        if surface:
-            vocab += (surface,)
-            if has_kanji(surface):
-                reading = get_node_reading(node)
-                parts.append(f"{surface}({reading})")
-            else:
-                parts.append(surface)
-        node = node.next
-
-    return {'reading':("".join(parts)),'vocab':vocab}
+from parser import *
 
 #print(annotate_phrase("私は学校へ行きます"))
 
-with open("words-readings.json", "r", encoding="utf-16le") as f:
-    readings = json.load(f)
-
-with open("dump.json", "r", encoding="utf-16le") as f:
+with open("dump2.json", "r", encoding="utf-16") as f:
     dump = json.load(f)
 
+# foo = anki_fields_term(surface)
+# for entry in foo.get("fields"):
+#     entry['expression']
+#     entry['glossary-first-brief']
+    # entry['glossary-plain-no-dictionary']
+    # entry['frequency-average-rank']
+
+vocab_dict = {}
 ## A micab phrase based word parser
 for cd in dump:
-    print(dump[cd]['nm'])
-    # node = tagger.parseToNode(dump[cd]['nm'])
+    print(dump[cd]['name_J'])
+    # node = tagger.parseToNode(dump[cd]['name_J'])
     # while node:
     #     # node.surface is the word; node.feature contains POS tags
     #     if node.surface:
     #         print(f"{node.surface}\t{node.feature}")
     #     node = node.next
-    anottated = annotate_phrase(dump[cd]['nm'])
-    if anottated['reading'] != dump[cd]['nm']:
-        print(anottated['reading'])
+    anottated = annotate_phrase(dump[cd]['name_J'])
+    if anottated['reading'] != dump[cd]['name_J']:
+        #print(anottated['reading'])
         dump[cd]['kana'] = anottated['reading']
-        #dump[cd]['vocab'] = anottated['vocab']
-    print()
-
-exit()
-
-## Iterates through all the keys not values
-for cd in dump:
-    word = ''
-    new_name = ''
-    for char in dump[cd]['nm'] + '・': ## hacky char add to prevent dupe logic
-        if char in all_kana:
-            if len(word) > 0:
-                #print(word)
-                if readings.get(word) is not None:
-                    kana = None
-                    try:
-                        #print("<--", readings[word]['kana'])
-                        kana = readings[word]['kana']
-                        new_name += '(' + kana + ')'
-                        #print()
-                    except TypeError as e:
-                        kana = None
-                #words[word] = (words[word] + 1)
-                else:
-                    print(word, " <- Not Found")
-            word = ''
+        dump[cd]['furi'] = anottated['furigana']
+    cd_nm = dump[cd]['name_J']
+    if (anottated.get('vocab')):
+        dump[cd]['glossary_Name_J'] = {}
+    for word in anottated['vocab']:
+        # Skip punctuation words hopefully
+        if len(word) == 1 and has_kanji(word) == False:
+            continue
+        if vocab_dict.get(word) is None:
+            vocab_dict[word] = {
+                'Def.': "; ".join(get_glossary(word)),
+                'count': 0,
+                'Examples': (cd_nm,)
+            }
         else:
-            word += char
-        new_name += char
-    if new_name[:-1]!= dump[cd]['nm']:
-        print (dump[cd]['nm'])
-        print (new_name[:-1])
-        dump[cd]['kana'] = new_name[:-1]
-        ## Might need catch for end of string still
+            vocab_dict[word]['count'] = vocab_dict[word]['count'] + 1
+        if cd_nm not in vocab_dict[word]['Examples']:
+            vocab_dict[word]['Examples'] += (cd_nm,)
+        if vocab_dict.get(word) and dump[cd]['glossary_Name_J'].get(word) is None:
+            dump[cd]['glossary_Name_J'][word] = vocab_dict[word]['Def.']
+            #append(f"{word}: {vocab_dict[word]['Def.']}")
+    print()
+    phrase = dump[cd]['desc_J']
+    anottated = annotate_phrase(phrase)
+    if anottated['reading'] != phrase:
+        #print(anottated['reading'])
+        dump[cd]['desc_kana'] = anottated['reading']
+        dump[cd]['desc_Furi'] = anottated['furigana']
+    if (anottated.get('vocab')):
+        dump[cd]['glossary_Desc_J'] = {}
+    for word in anottated['vocab']:
+        if len(word) == 1 and has_kanji(word) == False:
+            continue
+        if vocab_dict.get(word) is None:
+            vocab_dict[word] = {
+                'Def.': "; ".join(get_glossary(word)),
+                'count': 0,
+                'Examples': (cd_nm,)
+            }
+        else:
+            vocab_dict[word]['count'] = vocab_dict[word]['count'] + 1
+        if len(vocab_dict[word]['Examples']) < 5:
+            if cd_nm not in vocab_dict[word]['Examples']:
+                vocab_dict[word]['Examples'] += (cd_nm,)
+        if vocab_dict.get(word) and dump[cd]['glossary_Desc_J'].get(word) is None:
+            dump[cd]['glossary_Desc_J'][word] = vocab_dict[word]['Def.']
 
-print()
+vocab_dict = sorted(vocab_dict.items(), key= lambda x: x[1]['count'], reverse=True)
 
-with open("dump-withkana.json", "w", encoding="utf-16le") as outfile:
-    json.dump(dump, outfile, ensure_ascii=False, indent=2)
+#with open("dump-withFuri.json", "w", encoding="utf-16") as outfile:
+#    json.dump(dump, outfile, ensure_ascii=False, indent=2)
+
+#with open("dump-vocab.json", "w", encoding="utf-16") as outfile:
+#    json.dump(vocab_dict, outfile, ensure_ascii=False, indent=2)

create-notes.py

@@ -0,0 +1,96 @@
+from parser import *
+
+import os
+from pathlib import Path
+import shutil
+import hashlib
+
+destination_directory_name = '/run/media/deck/YF8SD/Sync/Notebooks/Obsidian/Japanese/Yu-Gi-Oh/Cards/'
+destination_path = Path(destination_directory_name)
+
+#print(annotate_phrase("私は学校へ行きます"))
+
+## Make sure directories exist
+if not destination_path.exists():
+    destination_path.mkdir(parents=True, exist_ok=True)
+if not destination_path.joinpath('_attachments').exists():
+    destination_path.joinpath('_attachments').mkdir(parents=True, exist_ok=True)
+if not destination_path.joinpath('vocab').exists():
+    destination_path.joinpath('vocab').mkdir(parents=True, exist_ok=True)
+
+with open("dump-withFuri.json", "r", encoding="utf-16") as f:
+    dump = json.load(f)
+
+for cd in dump:
+    print(dump[cd]['name_J'])
+    if Path('YGO_2020/2020.full.illust_j.jpg.zib/' + cd + '.jpg').exists():
+        ##TODO: Ew
+        if Path(destination_path.joinpath('_attachments/' + cd + '.jpg')).exists() == False:
+            shutil.copy2(Path('YGO_2020/2020.full.illust_j.jpg.zib/' + cd + '.jpg'), destination_path.joinpath('_attachments'))
+    with open(destination_path.as_posix() + '/' + cd + '.md', 'w') as card_note:
+        card_note.write("---\n")
+        card_note.write(f"aliases: [{dump[cd]['name_J']},{dump[cd]['name_E']}]\n")
+        card_note.write("---\n")
+        ## Name at the top
+        card_note.write(f"![[{cd}.jpg]]")
+        card_note.write("\n#### Name\n")
+        card_note.write("<details><summary>")
+        card_note.write(dump[cd]['name_J'])
+        card_note.write("</summary>")
+        card_note.write(dump[cd]['name_E'])
+        card_note.write("</details>")
+        ## the description
+        card_note.write("\n#### Details\n")
+        card_note.write("<details><summary>")
+        card_note.write(dump[cd]['desc_J'])
+        card_note.write("</summary>")
+        card_note.write(dump[cd]['desc_E'])
+        card_note.write("</details>")
+        # desc_Furi
+        card_note.write("\n#### Reading\n")
+        if(dump[cd].get("furi")):
+            card_note.write(f"{dump[cd]['furi']}\n")
+        else:
+            card_note.write(dump[cd]['name_J'])
+            card_note.write("\n")
+        card_note.write("##### Glossary\n")
+        for vocab in dump[cd]["glossary_Name_J"]:
+            print(f"{vocab} : {dump[cd]["glossary_Name_J"][vocab]}", file=card_note)
+        card_note.write("\n")
+        if(dump[cd].get("desc_Furi")):
+            card_note.write(f"{dump[cd]['desc_Furi']}\n")
+        else:
+            card_note.write(dump[cd]['desc_J'])
+            card_note.write("\n")
+        card_note.write("##### Glossary\n")
+        for vocab in dump[cd]["glossary_Desc_J"]:
+            vocab_hash = hashlib.shake_256((vocab + 'Jitendex.org [2026-04-04]').encode()).hexdigest(4)
+            ## Only search and create a vocab file if it doesn't exist
+            if Path(destination_path.joinpath('vocab/' + vocab_hash + '.md')).exists() == False:
+                definitions = anki_fields_term(vocab)
+                if definitions.get("fields"):
+                    with open(destination_path.as_posix() + '/vocab/' + vocab_hash + '.md', 'w') as vocab_note:
+                        for entry in definitions.get("fields"):
+                            vocab_note.write("\n")
+                            print(entry['expression'], file=vocab_note)
+                            #print(entry['furigana-plain'], file=vocab_note)
+                            if entry['expression'] != entry['reading']:
+                                print(entry['reading'], file=vocab_note)
+                            print(entry['furigana'], file=vocab_note)
+                            #print(f"Part of Speech: {entry['part-of-speech']}", file=vocab_note)
+                            print(f"Frequency: {entry['frequency-average-rank']}", file=vocab_note)
+                            #print(entry['glossary-plain-no-dictionary'], file=vocab_note)
+                            vocab_note.write(entry['glossary-first'])
+                            print("##### Other Definitions", file=vocab_note)
+                            print(entry['glossary-plain-no-dictionary'], file=vocab_note)
+            print(f"[[{vocab_hash}|{vocab}]] : {dump[cd]["glossary_Desc_J"][vocab]}", file=card_note)
+            #print('>' + dump[cd]["glossary_Desc_J"][vocab], file=card_note)
+        card_note.write("\n")
+
+# foo = anki_fields_term(surface)
+# for entry in foo.get("fields"):
+#     entry['expression']
+#     entry['glossary-first-brief']
+    # entry['glossary-plain-no-dictionary']
+    # entry['frequency-average-rank']
+

define-vocab.py

@@ -0,0 +1,123 @@
+import json
+import MeCab
+import re
+
+import requests
+
+request_url = "http://127.0.0.1:19633"
+request_timeout = 10
+
+# Initialize the Tagger
+#tagger = MeCab.Tagger()
+
+# Generate Hiragana and Katakana characters
+hiragana = tuple(chr(i) for i in range(12353, 12436)) # \u3041 - \u3094
+katakana = tuple(chr(i) for i in range(12450, 12532)) # \u30A2 - \u30F4
+punctuation = tuple(chr(i) for i in range(65281, 65382))
+
+def elide(text: str) -> str:
+    elide_max_length = 100
+    if len(text) > elide_max_length:
+        return text[:100] + "..."
+    return text
+
+def anki_fields_term(term : str) -> dict:
+    params = {
+        "text": term,
+        "type": "term",
+        "markers": ["audio", "cloze-body-kana", "conjugation", "expression", "furigana", "furigana-plain", "glossary", "glossary-brief", "glossary-no-dictionary", "glossary-first", "glossary-first-brief", "glossary-first-no-dictionary", "part-of-speech", "phonetic-transcriptions", "pitch-accents", "pitch-accent-graphs", "pitch-accent-graphs-jj", "pitch-accent-positions", "pitch-accent-categories", "reading", "tags", "clipboard-image", "clipboard-text", "cloze-body", "cloze-prefix", "cloze-suffix", "dictionary", "dictionary-alias", "document-title", "frequencies", "frequency-harmonic-rank", "frequency-harmonic-occurrence", "frequency-average-rank", "frequency-average-occurrence", "screenshot", "search-query", "popup-selection-text", "sentence", "sentence-furigana", "sentence-furigana-plain", "url"],
+        "maxEntries": 1,
+        "includeMedia": False,
+    }
+    response = requests.post(request_url + "/ankiFields", json = params, timeout = request_timeout)
+    return json.loads(response.text)
+
+def term_entries(term :str) -> dict:
+    print("Requesting termEntries:")
+    params = {
+        "term": term,
+        "maxEntries": 1,
+        "includeMedia": False,
+    }
+    response = requests.post(request_url + "/termEntries", json = params, timeout = request_timeout)
+    return json.loads(response.text)
+
+# perhaps set up a filter parameter.
+def flatten_string(node):
+    if isinstance(node,str):
+        yield node
+    elif isinstance(node, list):
+        for item in node:
+            yield from flatten_string(item)
+    elif isinstance(node,dict):
+        if isinstance(node.get("content"), str):
+            yield node["content"]
+        else:
+            for v in node.values():
+                yield from flatten_string(v)
+
+def extract_text_by_type(node, target_type):
+    if isinstance(node,dict):
+        data = node.get("data")
+        if isinstance(data,dict) and data.get("content") == target_type:
+            yield from flatten_string(node.get("content"))
+            #yield from ",".join(node.get("content"))
+        for value in node.values():
+            yield from extract_text_by_type(value, target_type)
+    elif isinstance(node,list):
+        for item in node:
+            yield from extract_text_by_type(item,target_type)
+
+# Combine both into one tuple
+all_kana = hiragana + katakana + punctuation + ('　', 'ー', '・')
+
+KANJI_RE = re.compile(r"[\u4E00-\u9FFF]")
+
+with open("dump-vocab.json", "r", encoding="utf-16") as f:
+    vocab = json.load(f)
+
+def get_glossary(word :str) -> list:
+    glossary = list()
+    dict_items = term_entries(word)
+
+    entry = next(iter(dict_items.get("dictionaryEntries",[])),None)
+    if entry:
+        definition = next(iter(entry.get("definitions",[])),None)
+        if definition:
+            def_entry = next(iter(definition.get("entries", [])),None)
+            if def_entry:
+                glossary.extend( list(extract_text_by_type(def_entry, "glossary")) )
+    return glossary[:2]
+
+for item in vocab:
+    #print(word[0])
+    word = item[0]
+    print("Requesting termEntries:")
+    # params = {
+    #     "term": word,
+    # }
+    # response = requests.post(request_url + "/termEntries", json = params, timeout = request_timeout)
+    # print(response)
+    # print(elide(response.text))
+    # print(response.json()) # Dumps json
+#    for entry in dict_items.get("dictionaryEntries", []):
+#        for definition in entry.get("definitions", []):
+            # first = next(iter(definition.get("entries", [])),None)
+            # if first:
+            #     glossary = list(extract_text_by_type(first, "glossary"))
+            #     if len(glossary) > 0:
+            #         break
+#            for def_entry in definition.get("entries", []):
+#                glossary.extend( list(extract_text_by_type(def_entry, "glossary")) )
+                # for content in def_entry.get("content", []):
+                #     if content.get('data') is not None:
+                #         data = content['data']
+                #         print(data['content']) # type of content
+            
+    #anki_card = anki_fields_term(word)
+    item.append("; ".join(get_glossary(word)))
+    print()
+
+with open("dump-vocab.json", "w", encoding="utf-16") as outfile:
+    json.dump(vocab, outfile, ensure_ascii=False, indent=2)
+

fugashi_parser.py

@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+import argparse
+import html
+import json
+import os
+import re
+import sys
+from dataclasses import dataclass, asdict
+from typing import Dict, List, Any
+
+import requests
+
+request_url = "http://127.0.0.1:19633"
+request_timeout = 10
+
+from fugashi import Tagger
+
+
+# Ensure UTF-8 output
+if hasattr(sys.stdout, "reconfigure"):
+    sys.stdout.reconfigure(encoding="utf-8", errors="backslashreplace")
+
+
+KATAKANA_START = ord("ァ")
+KATAKANA_END = ord("ヶ")
+KATAKANA_TO_HIRAGANA_OFFSET = ord("ぁ") - ord("ァ")
+
+KANJI_RE = re.compile(r"[\u4E00-\u9FFF]")
+
+def katakana_to_hiragana(text: str) -> str:
+    return "".join(
+        chr(ord(ch) + KATAKANA_TO_HIRAGANA_OFFSET) if KATAKANA_START <= ord(ch) <= KATAKANA_END else ch
+        for ch in text
+    )
+
+def has_kanji(text: str) -> bool:
+    return bool(KANJI_RE.search(text))
+
+def has_japanese(text: str) -> bool:
+    return any(
+        0x3040 <= ord(ch) <= 0x30FF or
+        0x3400 <= ord(ch) <= 0x9FFF
+        for ch in text
+    )
+
+
+def is_content_word(pos1: str) -> bool:
+    return pos1 in {"名詞", "動詞", "形容詞", "副詞"}
+
+
+def should_skip(surface: str, pos1: str) -> bool:
+    return not surface.strip() or pos1 == "補助記号"
+
+
+def clean_lemma(lemma: str) -> str:
+    return re.split(r"-", lemma, maxsplit=1)[0] if lemma else lemma
+
+
+def get_feature(feature, name: str, default=""):
+    return getattr(feature, name, default) or default
+
+
+def build_ruby(surface: str, reading: str) -> str:
+    if not reading or surface == reading or not has_japanese(surface):
+        return html.escape(surface)
+    return f"<ruby>{html.escape(surface)}<rt>{html.escape(reading)}</rt></ruby>"
+
+def build_kana(surface: str, reading: str) -> str:
+    if not reading or surface == reading or not has_japanese(surface):
+        return surface
+    return f"「{reading}」"
+
+def build_vocab_link(surface: str, reading: str, pos1: str) -> str:
+    if should_skip(surface, pos1) or not is_content_word(pos1):
+        return surface
+    if surface == reading:
+        return f"[[{surface}]]"
+    return f"[[{reading}|{surface}]]"
+
+
+@dataclass
+class TokenReading:
+    surface: str
+    reading_hiragana: str
+    lemma: str
+    lemma_reading_hiragana: str
+    pos: List[str]
+
+
+# ---------- Persistent vocab handling ----------
+
+def load_vocab(path: str) -> Dict[str, Dict]:
+    if not os.path.exists(path):
+        return {}
+
+    with open(path, "r", encoding="utf-8") as f:
+        raw = json.load(f)
+
+    # normalize + convert surfaces_seen → set
+    out = {}
+    for lemma, entry in raw.items():
+        out[lemma] = {
+            "lemma": entry.get("lemma", lemma),
+            "lemma_reading_hiragana": entry.get("lemma_reading_hiragana", ""),
+            "pos": entry.get("pos", []),
+            "mention_count": int(entry.get("mention_count", 0)),
+            "surfaces_seen": set(entry.get("surfaces_seen", [])),  # ← key change
+        }
+    return out
+
+
+def save_vocab(path: str, vocab: Dict[str, Dict]):
+    os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
+
+    # convert sets → sorted lists
+    serializable = {
+        lemma: {
+            **entry,
+            "surfaces_seen": sorted(entry.get("surfaces_seen", []))
+        }
+        for lemma, entry in vocab.items()
+    }
+
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(serializable, f, ensure_ascii=False, indent=2)
+
+
+def update_vocab(vocab: Dict[str, Dict], tokens: List[TokenReading]):
+    for t in tokens:
+        pos1 = t.pos[0] if t.pos else ""
+
+        if should_skip(t.surface, pos1) or not is_content_word(pos1):
+            continue
+
+        entry = vocab.get(t.lemma)
+
+        if entry is None:
+            vocab[t.lemma] = {
+                "lemma": t.lemma,
+                "lemma_reading_hiragana": t.lemma_reading_hiragana,
+                "pos": t.pos,
+                "mention_count": 1,
+                "surfaces_seen": {t.surface},  # ← set
+            }
+        else:
+            entry["mention_count"] += 1
+            entry.setdefault("surfaces_seen", set()).add(t.surface)
+
+
+# ---------- Main analysis ----------
+
+def analyze(text: str, vocab: Dict[str, Dict]):
+    tagger = Tagger()
+
+    tokens: List[TokenReading] = []
+    ruby_parts = []
+    kana_parts = []
+    vocab_link_parts = []
+
+    for word in tagger(text):
+        f = word.feature
+        surface = word.surface
+
+        pos = [
+            get_feature(f, "pos1"),
+            get_feature(f, "pos2"),
+            get_feature(f, "pos3"),
+            get_feature(f, "pos4"),
+        ]
+
+        kana = get_feature(f, "kana")
+        #reading = katakana_to_hiragana(kana) if kana else ""
+        if kana:
+            if kana == surface:
+                reading = ""
+            else:
+                reading = katakana_to_hiragana(kana)
+        else:
+            reading = ""
+
+        lemma = clean_lemma(get_feature(f, "lemma") or surface)
+        vocab_link_parts.append(build_vocab_link(surface, lemma, pos[0]))
+
+        lform = get_feature(f, "lForm")
+        kana_base = get_feature(f, "kanaBase")
+        lemma_reading = katakana_to_hiragana(lform or kana_base or kana or lemma)
+
+        token = TokenReading(surface, reading, lemma, lemma_reading, pos)
+        tokens.append(token)
+
+        ruby_parts.append(build_ruby(surface, reading))
+        kana_parts.append(build_kana(surface, reading))
+
+    update_vocab(vocab, tokens)
+
+    return {
+        "original_text": text,
+        "kana_reading": "".join(kana_parts),
+        "ruby_html": "".join(ruby_parts),
+        "notes_link":"".join(vocab_link_parts),
+        "token_readings": [asdict(t) for t in tokens],
+    }

get-properties.py

@@ -1,49 +1,93 @@
 import mmap
 import struct
 from pathlib import Path
+import json
 
 prop_data = Path(f"YGO_2020/bin/CARD_Prop.bin").read_bytes() # Get all the bytes
-language_code = 'E'
-offsets_data = Path(f"YGO_2020/bin/CARD_Indx_{language_code}.bin").read_bytes() # Get all the bytes
-name_data = Path(f"YGO_2020/bin/CARD_Name_{language_code}.bin").read_bytes() # Get all the bytes
 
 print(prop_data[8]) # one byte supposedly
 
-# 8 bytes at a time
-test = 0
-for r in range(0,len(prop_data),8):
-    print(prop_data[r:r+8].hex(' ')) # print the byte range
-    card_prop = prop_data[r:r+8]    
-    card_prop = card_prop[::-1]
-    #print(card_prop.hex(' '))
-    card_first = prop_data[r:r+4] #Get first 4 bytes in range
-    card_second = prop_data[r+4:r+8] #Get first 4 bytes in range
-    # offsets_data[4:8].hex()
-    card_index_name_data =  int.from_bytes( offsets_data[r:r+4], byteorder='little' )
-    card_index_desc_data =  int.from_bytes( offsets_data[r+4:r+8], byteorder='little' )
-    #card_id = card_id[::-1]
-    card_bytes = int.from_bytes(card_first,byteorder='little')
-    card_bytes_2 = int.from_bytes(card_second,byteorder='little')
-    card_id = card_bytes & 16383
-    card_attack = ((card_bytes >> 14) & 511) * 10
-    card_defense = ((card_bytes >> 18) & 511) * 10
-    
-    print(card_id , card_attack, card_attack, 'idx', card_index_name_data, card_index_desc_data)
-    #read until null found
-    end = card_index_name_data
-    while end + 1 < len(name_data):
+def read_until_null(indx: int, data: tuple) -> int:
+    end = indx
+    while end + 1 < len(data):
         ## Read 2 bytes at a time until null found
-        if name_data[end:end+2] == b"\x00\x00":
+        if data[end:end+2] == b"\x00\x00":
             break
         end += 2
-    name = name_data[card_index_name_data:end].decode("utf-16le", errors="replace")
-    print(name)
-    #print(card_id.hex(' '))
-    # 2. Get last 14 bits using mask 0x3FFF (binary 11 1111 1111 1111)
-    #print([bin(x) for x in card_first])
-    test += 1
-    # if test > 3:
-    #     break;
-    #print(prop_data[r:r+8])
+    return end
 
+# 8 bytes at a time
+cd_db = {}
+def get_props():
+    test = 1
+    for r in range(0,len(prop_data),8):
+        print(prop_data[r:r+8].hex(' ')) # print the byte range
+        card_prop = prop_data[r:r+8]    
+        card_prop = card_prop[::-1]
+        #print(card_prop.hex(' '))
+        card_first = prop_data[r:r+4] #Get first 4 bytes in range
+        card_second = prop_data[r+4:r+8] #Get first 4 bytes in range
+        # offsets_data[4:8].hex()
+        card_index_name_data =  int.from_bytes( offsets_data[r:r+4], byteorder='little' )
+        card_index_desc_data =  int.from_bytes( offsets_data[r+4:r+8], byteorder='little' )
+        #card_id = card_id[::-1]
+        card_bytes = int.from_bytes(card_first,byteorder='little')
+        card_bytes_2 = int.from_bytes(card_second,byteorder='little')
+        card_id = card_bytes & 16383
+        if card_id == 0:
+            continue
+        card_attack = ((card_bytes >> 14) & 511) * 10
+        card_defense = ((card_bytes >> 18) & 511) * 10
+        
+        print(card_id , card_attack, card_attack, 'idx', card_index_name_data, card_index_desc_data)
+        #read until null found
+        # end = card_index_name_data
+        # while end + 1 < len(name_data):
+        #     ## Read 2 bytes at a time until null found
+        #     if name_data[end:end+2] == b"\x00\x00":
+        #         break
+        #     end += 2
+        # name = name_data[card_index_name_data:end].decode("utf-16le", errors="replace")
+        end_index = read_until_null(card_index_name_data, name_data)
+        name = name_data[card_index_name_data:end_index].decode("utf-16le", errors="replace")
+        end_index = read_until_null(card_index_desc_data, desc_data)
+        desc = desc_data[card_index_desc_data:end_index].decode("utf-16le", errors="replace")
+        print(name)
+        #print(card_id.hex(' '))
+        # 2. Get last 14 bits using mask 0x3FFF (binary 11 1111 1111 1111)
+        #print([bin(x) for x in card_first])
+        test += 1
+        # if test > 3:
+        #     break;
+        #print(prop_data[r:r+8])
+        if cd_db.get(card_id) is None:
+            cd_db[card_id] = {}
+        # {
+        #     f"name_{language_code}":name, 
+        #     f"desc_{language_code}": 'foo', 
+        #     f"name_addr_{language_code}": hex(card_index_name_data), 
+        #     f"desc_addr_{language_code}": hex(card_index_desc_data)
+        # }
+        cd_db[card_id][f"name_{language_code}"] = name 
+        cd_db[card_id][f"desc_{language_code}"] = desc 
+        cd_db[card_id][f"name_addr_{language_code}"] = hex(card_index_name_data)
+        cd_db[card_id][f"desc_addr_{language_code}"] = hex(card_index_desc_data)
 
+language_code = 'E'
+offsets_data = Path(f"YGO_2020/bin/CARD_Indx_{language_code}.bin").read_bytes() # Get all the bytes
+name_data = Path(f"YGO_2020/bin/CARD_Name_{language_code}.bin").read_bytes() # Get all the bytes
+desc_data = Path(f"YGO_2020/bin/CARD_Desc_{language_code}.bin").read_bytes()
+
+get_props()
+
+language_code = 'J'
+offsets_data = Path(f"YGO_2020/bin/CARD_Indx_{language_code}.bin").read_bytes() # Get all the bytes
+name_data = Path(f"YGO_2020/bin/CARD_Name_{language_code}.bin").read_bytes() # Get all the bytes
+desc_data = Path(f"YGO_2020/bin/CARD_Desc_{language_code}.bin").read_bytes()
+
+get_props()
+
+with open("dump2.json", "w", encoding="utf-16") as outfile:
+    json.dump(cd_db, outfile, ensure_ascii=False, indent=2)
+
+print()

jp_readings.py

@@ -0,0 +1,25 @@
+from fugashi_parser import *
+
+# ---------- CLI ----------
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--vocab-file", required=True)
+    parser.add_argument("text", nargs="*")
+    args = parser.parse_args()
+
+    if args.text:
+        text = " ".join(args.text)
+    else:
+        text = sys.stdin.read()
+
+    vocab = load_vocab(args.vocab_file)
+    result = analyze(text, vocab)
+    save_vocab(args.vocab_file, vocab)
+
+    result["vocab_file"] = args.vocab_file
+    print(json.dumps(result, ensure_ascii=False, indent=2))
+
+
+if __name__ == "__main__":
+    main()

parser.py

@@ -0,0 +1,130 @@
+import json
+import MeCab
+import re
+
+import requests
+
+request_url = "http://127.0.0.1:19633"
+request_timeout = 10
+
+# Generate Hiragana and Katakana characters
+hiragana = tuple(chr(i) for i in range(12353, 12436)) # \u3041 - \u3094
+katakana = tuple(chr(i) for i in range(12450, 12532)) # \u30A2 - \u30F4
+punctuation = tuple(chr(i) for i in range(65281, 65382))
+
+# Combine both into one tuple
+all_kana = hiragana + katakana + punctuation + ('　', 'ー', '・')
+
+KANJI_RE = re.compile(r"[\u4E00-\u9FFF]")
+
+def katakana_to_hiragana(text: str) -> str:
+    return "".join(
+        chr(ord(ch) - 0x60) if 0x30A1 <= ord(ch) <= 0x30F6 else ch
+        for ch in text
+    )
+
+# def get_node_reading(node) -> str:
+#     features = node.feature.split(",")
+#     print()
+#     if len(features) >= 8 and features[7] != "*":
+#         return katakana_to_hiragana(features[7])
+#     return node.surface
+
+def has_kanji(text: str) -> bool:
+    return bool(KANJI_RE.search(text))
+
+def get_node_reading(node) -> str:
+    features = node.feature.split(",")
+    ## These depend on what dictionary is being used.
+    if len(features) > 6 and features[6] != "*":
+        return katakana_to_hiragana(features[6])
+    return node.surface
+
+def annotate_phrase(text: str) -> str:
+    tagger = MeCab.Tagger()
+    node = tagger.parseToNode(text)
+    vocab = ()
+    parts = []
+    # <ruby>漢<rt>かん</rt></ruby><ruby>字<rt>じ</rt></ruby>
+    furi = []
+    while node:
+        surface = node.surface
+        if surface:
+            vocab += (surface,)
+            if has_kanji(surface):
+                reading = get_node_reading(node)
+                parts.append(f"{surface}({reading})")
+                furi.append(f"<ruby>{surface}<rt>{reading}</rt></ruby>")
+            else:
+                parts.append(surface)
+                furi.append(surface)
+        node = node.next
+
+    return {'reading':("".join(parts)),'vocab':vocab,"furigana":("".join(furi))}
+
+def elide(text: str) -> str:
+    elide_max_length = 100
+    if len(text) > elide_max_length:
+        return text[:100] + "..."
+    return text
+
+def anki_fields_term(term : str) -> dict:
+    params = {
+        "text": term,
+        "type": "term",
+        #"markers": ["audio", "cloze-body-kana", "conjugation", "expression", "furigana", "furigana-plain", "glossary", "glossary-brief", "glossary-no-dictionary", "glossary-first", "glossary-first-brief", "glossary-first-no-dictionary", "part-of-speech", "phonetic-transcriptions", "pitch-accents", "pitch-accent-graphs", "pitch-accent-graphs-jj", "pitch-accent-positions", "pitch-accent-categories", "reading", "tags", "clipboard-image", "clipboard-text", "cloze-body", "cloze-prefix", "cloze-suffix", "dictionary", "dictionary-alias", "document-title", "frequencies", "frequency-harmonic-rank", "frequency-harmonic-occurrence", "frequency-average-rank", "frequency-average-occurrence", "screenshot", "search-query", "popup-selection-text", "sentence", "sentence-furigana", "sentence-furigana-plain", "url"],
+        "markers": ["audio", "expression", "cloze-body" , "furigana","furigana-plain", "part-of-speech", "reading", "glossary-first", "glossary-plain-no-dictionary" , "glossary-first-no-dictionary" , "dictionary", "frequencies", "frequency-average-rank", "frequency-average-occurrence", "screenshot", "sentence", "sentence-furigana", "sentence-furigana-plain" ],
+        "maxEntries": 2,
+        "includeMedia": False,
+    }
+    response = requests.post(request_url + "/ankiFields", json = params, timeout = request_timeout)
+    return json.loads(response.text)
+
+def term_entries(term :str) -> dict:
+    #print("Requesting termEntries:")
+    params = {
+        "term": term,
+        "maxEntries": 1,
+        "includeMedia": False,
+    }
+    response = requests.post(request_url + "/termEntries", json = params, timeout = request_timeout)
+    return json.loads(response.text)
+
+# perhaps set up a filter parameter.
+def flatten_string(node):
+    if isinstance(node,str):
+        yield node
+    elif isinstance(node, list):
+        for item in node:
+            yield from flatten_string(item)
+    elif isinstance(node,dict):
+        if isinstance(node.get("content"), str):
+            yield node["content"]
+        else:
+            for v in node.values():
+                yield from flatten_string(v)
+
+def extract_text_by_type(node, target_type):
+    if isinstance(node,dict):
+        data = node.get("data")
+        if isinstance(data,dict) and data.get("content") == target_type:
+            yield from flatten_string(node.get("content"))
+            #yield from ",".join(node.get("content"))
+        for value in node.values():
+            yield from extract_text_by_type(value, target_type)
+    elif isinstance(node,list):
+        for item in node:
+            yield from extract_text_by_type(item,target_type)
+
+def get_glossary(word :str) -> list:
+    glossary = list()
+    dict_items = term_entries(word)
+
+    entry = next(iter(dict_items.get("dictionaryEntries",[])),None)
+    if entry:
+        definition = next(iter(entry.get("definitions",[])),None)
+        if definition:
+            def_entry = next(iter(definition.get("entries", [])),None)
+            if def_entry:
+                glossary.extend( list(extract_text_by_type(def_entry, "glossary")) )
+    return glossary[:2]

subtitle_processor.py

@@ -0,0 +1,53 @@
+from fugashi_parser import *
+
+# ---------- CLI ----------
+
+def main():
+#    parser = argparse.ArgumentParser()
+#    parser.add_argument("--vocab-file", required=True)
+#    parser.add_argument("text", nargs="*")
+#    args = parser.parse_args()
+
+    vocab = load_vocab('foo.json')
+
+    with open("/run/media/deck/YF8SD/Video/[Retimed and Corrected] Yu-Gi-Oh! Duel Monsters - 001.srt", "r", encoding="utf-8") as f:
+        lines = f.readlines()
+
+    
+    notes_lines = []
+
+    # Example: Modify the 3rd line (index 2)
+    #lines[2] = "This is the new subtitle text\n"
+    ## First lien has some unicode bom bs 
+    for i in range(len(lines)):
+        if re.fullmatch(r'\d+', lines[i].strip()) or i == 0:
+            notes_lines.append('##### ' + lines[i].strip() + " `" + lines[i+1].strip() + "`" + "\n")
+            i+=1
+            continue
+        if has_japanese(lines[i]):
+            print(lines[i])
+            result = analyze(lines[i],vocab)
+            print(result['kana_reading'])
+            notes_lines.append(result["ruby_html"] + "\n" )
+            notes_lines.append (result['notes_link'] + "\n")
+            if lines[i] != result['kana_reading']:
+                lines[i] = result['kana_reading'] + "\n" # lost the new line
+        else:
+           notes_lines.append("`" + lines[i].strip() + "`" + "\n")
+
+    with open("subtitle.srt", "w", encoding="utf-8") as f:
+        f.writelines(lines)
+
+    with open("notestest.md", "w",  encoding="utf-8") as f:
+        f.writelines(notes_lines)
+
+    #vocab = load_vocab(args.vocab_file)
+    #result = analyze(text, vocab)
+    #save_vocab('foo.json', vocab)
+
+    #result["vocab_file"] = args.vocab_file
+    #print(json.dumps(result, ensure_ascii=False, indent=2))
+
+
+if __name__ == "__main__":
+    main()