stuff

append_vocab_tab.py

@@ -0,0 +1,28 @@
+from fugashi_parser import *
+
+import os
+from pathlib import Path
+import shutil
+import hashlib
+
+vocab_destination_path = Path('/run/media/deck/YF8SD/Sync/Notebooks/Obsidian/Japanese/vocab')
+
+pos_list = list()
+for file_path in vocab_destination_path.iterdir():
+    if file_path.is_file():  # Ensure it's a file, not a subdirectory
+        pos = file_path.stem.split('_')[1]
+        pos_list.append(pos)
+        if CONTENT_LINK_TAG.get(pos):
+            #do a thing
+            with open(file_path, "a") as file:
+                #file.write("This text will be added to the end.")
+                print(f"#{CONTENT_LINK_TAG[pos]}", file=file)
+        else:
+            print(pos, ' - no tag')
+
+        # with file_path.open('r') as file:
+        #     content = file.read()
+        #     print(f"Content of {file_path.name}: {content}")
+
+unique_values = list(set(pos_list))
+print(unique_values)

create-notes.py

@@ -1,4 +1,4 @@
-from parser import *
+from fugashi_parser import *
 
 import os
 from pathlib import Path
@@ -21,18 +21,23 @@ if not destination_path.joinpath('vocab').exists():
 with open("dump-withFuri.json", "r", encoding="utf-16") as f:
     dump = json.load(f)
 
+vocab = {} #load_vocab('foo.json')
+
 for cd in dump:
+    ## The japanese card name
     print(dump[cd]['name_J'])
+    ## copy the image if it isn't already there
     if Path('YGO_2020/2020.full.illust_j.jpg.zib/' + cd + '.jpg').exists():
         ##TODO: Ew
         if Path(destination_path.joinpath('_attachments/' + cd + '.jpg')).exists() == False:
             shutil.copy2(Path('YGO_2020/2020.full.illust_j.jpg.zib/' + cd + '.jpg'), destination_path.joinpath('_attachments'))
     with open(destination_path.as_posix() + '/' + cd + '.md', 'w') as card_note:
+        ## add card names as properties
         card_note.write("---\n")
         card_note.write(f"aliases: [{dump[cd]['name_J']},{dump[cd]['name_E']}]\n")
         card_note.write("---\n")
         ## Name at the top
-        card_note.write(f"![[{cd}.jpg]]")
+        card_note.write(f"![[{cd}.jpg]]") ## card image
         card_note.write("\n#### Name\n")
         card_note.write("<details><summary>")
         card_note.write(dump[cd]['name_J'])
@@ -48,42 +53,70 @@ for cd in dump:
         card_note.write("</details>")
         # desc_Furi
         card_note.write("\n#### Reading\n")
-        if(dump[cd].get("furi")):
+        vocab = {} ## flush the vocab variable each time
-            card_note.write(f"{dump[cd]['furi']}\n")
+        result = analyze(dump[cd]['name_J'],vocab)
-        else:
+        print(f"<span>{result['ruby_html']}</span>\n> [!summary]- Vocab\n>{result['notes_link']}\n",file=card_note)
-            card_note.write(dump[cd]['name_J'])
+        ## Generate the vocab item
-            card_note.write("\n")
+        for word in vocab:
-        card_note.write("##### Glossary\n")
+            if Path(destination_path.joinpath('vocab/' + word + '.md')).exists() == False:
-        for vocab in dump[cd]["glossary_Name_J"]:
+                definitions = anki_fields_term(word)
-            print(f"{vocab} : {dump[cd]["glossary_Name_J"][vocab]}", file=card_note)
-        card_note.write("\n")
-        if(dump[cd].get("desc_Furi")):
-            card_note.write(f"{dump[cd]['desc_Furi']}\n")
-        else:
-            card_note.write(dump[cd]['desc_J'])
-            card_note.write("\n")
-        card_note.write("##### Glossary\n")
-        for vocab in dump[cd]["glossary_Desc_J"]:
-            vocab_hash = hashlib.shake_256((vocab + 'Jitendex.org [2026-04-04]').encode()).hexdigest(4)
-            ## Only search and create a vocab file if it doesn't exist
-            if Path(destination_path.joinpath('vocab/' + vocab_hash + '.md')).exists() == False:
-                definitions = anki_fields_term(vocab)
                 if definitions.get("fields"):
-                    with open(destination_path.as_posix() + '/vocab/' + vocab_hash + '.md', 'w') as vocab_note:
+                    with open(destination_path.as_posix() + '/vocab/' + word + '.md', 'w') as vocab_note:
                         for entry in definitions.get("fields"):
-                            vocab_note.write("\n")
+                            print(entry['furigana'], file=vocab_note)
-                            print(entry['expression'], file=vocab_note)
-                            #print(entry['furigana-plain'], file=vocab_note)
                             if entry['expression'] != entry['reading']:
                                 print(entry['reading'], file=vocab_note)
-                            print(entry['furigana'], file=vocab_note)
+                            print("> ",re.sub(r'<.*?>', '', entry['glossary-plain-no-dictionary']), "\n", file=vocab_note)
-                            #print(f"Part of Speech: {entry['part-of-speech']}", file=vocab_note)
                             print(f"Frequency: {entry['frequency-average-rank']}", file=vocab_note)
-                            #print(entry['glossary-plain-no-dictionary'], file=vocab_note)
                             vocab_note.write(entry['glossary-first'])
-                            print("##### Other Definitions", file=vocab_note)
+                            print("\n", file=vocab_note)
-                            print(entry['glossary-plain-no-dictionary'], file=vocab_note)
+        # card_note.write("##### Glossary\n")
-            print(f"[[{vocab_hash}|{vocab}]] : {dump[cd]["glossary_Desc_J"][vocab]}", file=card_note)
+        # for vocab in dump[cd]["glossary_Name_J"]:
+        #     print(f"{vocab} : {dump[cd]["glossary_Name_J"][vocab]}", file=card_note)
+        vocab = {} ## flush the vocab variable each time
+        result = analyze(dump[cd]['desc_J'],vocab)
+        #card_note.write("\n")
+        print(f"<span>{result['ruby_html']}</span>\n> [!summary]- Vocab\n>{result['notes_link']}\n",file=card_note)
+        for word in vocab:
+            if Path(destination_path.joinpath('vocab/' + word + '.md')).exists() == False:
+                definitions = anki_fields_term(word)
+                if definitions.get("fields"):
+                    with open(destination_path.as_posix() + '/vocab/' + word + '.md', 'w') as vocab_note:
+                        for entry in definitions.get("fields"):
+                            print(entry['furigana'], file=vocab_note)
+                            if entry['expression'] != entry['reading']:
+                                print(entry['reading'], file=vocab_note)
+                            print("> ",re.sub(r'<.*?>', '', entry['glossary-plain-no-dictionary']), "\n", file=vocab_note)
+                            print(f"Frequency: {entry['frequency-average-rank']}", file=vocab_note)
+                            vocab_note.write(entry['glossary-first'])
+                            print("\n", file=vocab_note)
+        # if(dump[cd].get("desc_Furi")):
+        #     card_note.write(f"{dump[cd]['desc_Furi']}\n")
+        # else:
+        #     card_note.write(dump[cd]['desc_J'])
+        #     card_note.write("\n")
+        # card_note.write("##### Glossary\n")
+        # for vocab in dump[cd]["glossary_Desc_J"]:
+        #     vocab_hash = hashlib.shake_256((vocab + 'Jitendex.org [2026-04-04]').encode()).hexdigest(4)
+        #     ## Only search and create a vocab file if it doesn't exist
+        #     if Path(destination_path.joinpath('vocab/' + vocab_hash + '.md')).exists() == False:
+        #         definitions = anki_fields_term(vocab)
+        #         if definitions.get("fields"):
+        #             with open(destination_path.as_posix() + '/vocab/' + vocab_hash + '.md', 'w') as vocab_note:
+        #                 for entry in definitions.get("fields"):
+        #                     vocab_note.write("\n")
+        #                     print(entry['expression'], file=vocab_note)
+        #                     #print(entry['furigana-plain'], file=vocab_note)
+        #                     if entry['expression'] != entry['reading']:
+        #                         print(entry['reading'], file=vocab_note)
+        #                     print(entry['furigana'], file=vocab_note)
+        #                     #print(f"Part of Speech: {entry['part-of-speech']}", file=vocab_note)
+        #                     print(f"Frequency: {entry['frequency-average-rank']}", file=vocab_note)
+        #                     #print(entry['glossary-plain-no-dictionary'], file=vocab_note)
+        #                     vocab_note.write(entry['glossary-first'])
+        #                     print("##### Other Definitions", file=vocab_note)
+        #                     print(entry['glossary-plain-no-dictionary'], file=vocab_note)
+        #     print(f"[[{vocab_hash}|{vocab}]] : {dump[cd]["glossary_Desc_J"][vocab]}", file=card_note)
             #print('>' + dump[cd]["glossary_Desc_J"][vocab], file=card_note)
         card_note.write("\n")
 

fugashi_parser.py

@@ -7,6 +7,7 @@ import argparse
 import html
 import json
 import os
+from pathlib import Path
 import re
 import sys
 from dataclasses import dataclass, asdict
@@ -30,7 +31,24 @@ KATAKANA_START = ord("ァ")
 KATAKANA_END = ord("ヶ")
 KATAKANA_TO_HIRAGANA_OFFSET = ord("ぁ") - ord("ァ")
 
-KANJI_RE = re.compile(r"[\u4E00-\u9FFF]")
+
+CONTENT_POS1 = {"名詞", "動詞", "形容詞", "形状詞", "副詞", "代名詞"}
+
+def is_vocab_token(pos, keep_pronouns=False, keep_numbers=False):
+    pos1, pos2, pos3, pos4 = pos
+
+    if pos1 not in CONTENT_POS1:
+        return False
+
+    if pos1 == "代名詞" and not keep_pronouns:
+        return False
+
+    if pos1 == "名詞" and pos2 == "数詞" and not keep_numbers:
+        return False
+
+    return True
+
+
 
 def katakana_to_hiragana(text: str) -> str:
     return "".join(
@@ -38,9 +56,19 @@ def katakana_to_hiragana(text: str) -> str:
         for ch in text
     )
 
+KANJI_RE = re.compile(r"[\u4E00-\u9FFF]")
 def has_kanji(text: str) -> bool:
     return bool(KANJI_RE.search(text))
 
+# def has_kanji(text):
+#     return any(
+#         0x3400 <= ord(ch) <= 0x4DBF or
+#         0x4E00 <= ord(ch) <= 0x9FFF or
+#         0xF900 <= ord(ch) <= 0xFAFF
+#         for ch in text
+#     )
+
+
 def has_japanese(text: str) -> bool:
     return any(
         0x3040 <= ord(ch) <= 0x30FF or
@@ -70,6 +98,16 @@ def term_entries(term :str) -> dict:
     response = requests.post(request_url + "/termEntries", json = params, timeout = request_timeout)
     return json.loads(response.text)
 
+CONTENT_LINK_TAG = {
+    '名詞': 'meishi', # noun
+    '動詞': 'doshi', # verb
+    '形容詞': 'keiyoshi', # Adjective
+    '形状詞': 'keijoshi', # form word
+    '代名詞':'daimeishi', # pronoun
+    '感動詞': 'kandoshi', # interjection
+    '副詞' : 'fukushi' # adverb
+    #'副詞' : 'fukushi' #adverb
+}
 
 def is_content_word(pos1: str) -> bool:
     return pos1 in {"名詞", "動詞", "形容詞", "副詞"}
@@ -86,6 +124,17 @@ def clean_lemma(lemma: str) -> str:
 def get_feature(feature, name: str, default=""):
     return getattr(feature, name, default) or default
 
+def should_ruby(surface, reading_hiragana):
+    if not reading_hiragana:
+        return False
+
+    if surface == reading_hiragana:
+        return False
+
+    if not has_kanji(surface):
+        return False
+
+    return True
 
 def build_ruby(surface: str, reading: str) -> str:
     if not reading or surface == reading or not has_japanese(surface):
@@ -97,13 +146,105 @@ def build_kana(surface: str, reading: str) -> str:
         return surface
     return f"「{reading}」"
 
-def build_vocab_link(surface: str, reading: str, pos1: str) -> str:
+def build_vocab_link(surface: str, reading: str, pos) -> str:
-    if should_skip(surface, pos1) or not is_content_word(pos1):
+    pos1, pos2, pos3, pos4 = pos
+    if not is_vocab_token(pos, keep_pronouns=True,keep_numbers=False):
         return surface
-    if surface == reading:
+    # if surface == reading:
-        return f"[[{surface}]]"
+    #     return f"[[{surface}]]"
-    return f"[[{reading}|{surface}]]"
+    return f"[[{reading}_{pos1}|{surface}]]"
 
+## Predicate filtering Verbs
+def is_predicate_start(pos):
+    pos1, pos2, pos3, pos4 = pos
+
+    return (
+        pos1 == "動詞" and pos2 == "一般"
+    ) or (
+        pos1 == "形容詞"
+    ) or (
+        pos1 == "形状詞"
+    )
+
+def attaches_to_predicate(pos):
+    pos1, pos2, pos3, pos4 = pos
+
+    # ます, た, ない, れる, られる, たい, etc.
+    if pos1 == "助動詞":
+        return True
+
+    # helper verbs like いる, ある, しまう in constructions
+    if pos1 == "動詞" and pos2 == "非自立可能":
+        return True
+
+    # te-form connector in 読んでいる, 食べている
+    if pos1 == "助詞" and pos2 == "接続助詞":
+        return True
+
+    # suffixes attached to predicates
+    if pos1 == "接尾辞":
+        return True
+
+    return False
+
+def chunk_predicates(tokens):
+    """
+    tokens should be a list of TokenReading objects:
+      token.surface
+      token.lemma
+      token.pos
+    """
+    chunks = []
+    i = 0
+
+    while i < len(tokens):
+        token = tokens[i]
+
+        if not is_predicate_start(token.pos):
+            i += 1
+            continue
+
+        chunk = [token]
+        j = i + 1
+
+        while j < len(tokens) and attaches_to_predicate(tokens[j].pos):
+            chunk.append(tokens[j])
+            j += 1
+
+        chunks.append({
+            "surface": "".join(t.surface for t in chunk),
+            "head_lemma": token.lemma,
+            "head_reading_hiragana": token.lemma_reading_hiragana,
+            "pos": token.pos,
+            "parts": [
+                {
+                    "surface": t.surface,
+                    "lemma": t.lemma,
+                    "reading_hiragana": t.reading_hiragana,
+                    "pos": t.pos,
+                }
+                for t in chunk
+            ],
+        })
+
+        i = j
+
+    return chunks
+
+## Vocab Filtering
+def is_vocab_token(pos, keep_pronouns=False, keep_numbers=False):
+    pos1, pos2, pos3, pos4 = pos
+
+    if pos1 not in {"名詞", "動詞", "形容詞", "形状詞", "副詞", "代名詞", "感動詞"}:
+        return False
+
+    if pos1 == "代名詞" and not keep_pronouns:
+        return False
+
+    if pos1 == "名詞" and pos2 == "数詞" and not keep_numbers:
+        return False
+
+    return True
 
 @dataclass
 class TokenReading:
@@ -156,7 +297,9 @@ def update_vocab(vocab: Dict[str, Dict], tokens: List[TokenReading]):
     for t in tokens:
         pos1 = t.pos[0] if t.pos else ""
 
-        if should_skip(t.surface, pos1) or not is_content_word(pos1):
+        # if should_skip(t.surface, pos1) or not is_content_word(pos1):
+        #     continue
+        if not is_vocab_token(t.pos, keep_pronouns=True,keep_numbers=False):
             continue
 
         entry = vocab.get(t.lemma)
@@ -206,7 +349,7 @@ def analyze(text: str, vocab: Dict[str, Dict]):
             reading = ""
 
         lemma = clean_lemma(get_feature(f, "lemma") or surface)
-        vocab_link_parts.append(build_vocab_link(surface, lemma, pos[0]))
+        vocab_link_parts.append(build_vocab_link(surface, lemma, pos))
 
         lform = get_feature(f, "lForm")
         kana_base = get_feature(f, "kanaBase")
@@ -217,6 +360,7 @@ def analyze(text: str, vocab: Dict[str, Dict]):
 
         ruby_parts.append(build_ruby(surface, reading))
         kana_parts.append(build_kana(surface, reading))
+        predicate_chunks = chunk_predicates(tokens)
 
     update_vocab(vocab, tokens)
 
@@ -226,4 +370,61 @@ def analyze(text: str, vocab: Dict[str, Dict]):
         "ruby_html": "".join(ruby_parts),
         "notes_link":"".join(vocab_link_parts),
         "token_readings": [asdict(t) for t in tokens],
+        "predicate_chunks" : predicate_chunks,
     }
+
+YOMITAN_TO_MICAB_POS = {
+    'interjection': 'foo',
+    'noun': 'meishi',
+    '5-dan': '',
+    '1-dan': '',
+    'prefix': '',
+    'intransitive': '',
+    'aux-verb': '',
+    'suru': '',
+    'transitive': '',
+    'counter': '',
+    'na-adj': '',
+    'exp': '',
+    'adjective': '',
+    'suffix': '',
+    'conjunction': '',
+    'interjection': '',
+    'adverb': '',
+    'pronoun': '',
+    'unclass': '',
+}
+
+def write_word_note(word: str, destination_dir: Path, part_of_speech:str='', overwrite:bool=False):
+    word_file_name = word
+    # Not ready for different file names yet
+    if part_of_speech != '':
+        word_file_name = word + '_' + part_of_speech
+    if (Path(destination_dir.joinpath(word_file_name + '.md')).exists() == False) or overwrite == True:
+        definitions = anki_fields_term(word)
+        if definitions.get("fields"):
+            with open(destination_dir.joinpath(word_file_name + '.md'), 'w') as vocab_note:
+                for entry in definitions.get("fields"):
+                    #pos = entry['part-of-speech'] ## POS is always 'Unknown' for some reason
+                    pos = re.match(r'.*<span.*?"part-of-speech-info".*?>(.*?)<',entry['glossary-first'])
+                    if pos:
+                        pos = pos.group(1)
+                    # if part_of_speech != '' and pos:
+                    #    print(pos)
+                    #vocab_note.write("\n")
+                    #print(entry['expression'], file=vocab_note)
+                    #print(entry['furigana-plain'], file=vocab_note)
+                    print(entry['furigana'], file=vocab_note)
+                    if entry['expression'] != entry['reading']:
+                        print(entry['reading'], file=vocab_note)
+                    print("> ",re.sub(r'<.*?>', '', entry['glossary-plain-no-dictionary']), "\n", file=vocab_note)
+                    #print(f"Part of Speech: {entry['part-of-speech']}", file=vocab_note)
+                    print(f"Frequency: {entry['frequency-average-rank']}", file=vocab_note)
+                    #print(entry['glossary-plain-no-dictionary'], file=vocab_note)
+                    vocab_note.write(entry['glossary-first'])
+                    #print("##### Other Definitions", file=vocab_note)
+                    print("\n", file=vocab_note)
+                    if CONTENT_LINK_TAG.get(part_of_speech, '') != '':
+                        print(f"#{CONTENT_LINK_TAG[part_of_speech]}", file=vocab_note)
+
+

getting_dialog.py

@@ -0,0 +1,283 @@
+import mmap
+import struct
+from pathlib import Path
+import json
+
+def read_until_null(indx: int, data: tuple) -> int:
+    end = indx
+    while end + 1 < len(data):
+        ## Read 2 bytes at a time until null found
+        if data[end:end+2] == b"\x00\x00":
+            break
+        end += 2
+    return end
+
+def read_until_space(indx: int, data: tuple) -> int:
+    end = indx
+    while end + 1 < len(data):
+        ## Read 2 bytes at a time until null found
+        d = data[end:end+1]
+        if data[end:end+1] == b"\x00":
+            break
+        end += 1
+    return end
+
+
+# 8 bytes at a time
+cd_db = {}
+def get_props():
+    test = 0
+    for r in range(0,len(offsets_data),8):
+        test += 1
+        # offsets_data[4:8].hex()
+        card_index_name_data =  int.from_bytes( offsets_data[r:r+4], byteorder='little', signed=False )
+        if card_index_name_data == 0:
+            continue
+        #card_index_desc_data =  int.from_bytes( offsets_data[r+4:r+8], byteorder='little' )
+        #card_id = card_id[::-1]
+        #read until null found
+        # end = card_index_name_data
+        # while end + 1 < len(name_data):
+        #     ## Read 2 bytes at a time until null found
+        #     if name_data[end:end+2] == b"\x00\x00":
+        #         break
+        #     end += 2
+        # name = name_data[card_index_name_data:end].decode("utf-16le", errors="replace")
+        end_index = read_until_null(card_index_name_data, name_data)
+        name = name_data[card_index_name_data:end_index].decode("utf-16le", errors="replace")
+        print(name)
+        if name == '':
+            continue
+        #print(card_id.hex(' '))
+        # 2. Get last 14 bits using mask 0x3FFF (binary 11 1111 1111 1111)
+        #print([bin(x) for x in card_first])
+        # if test > 3:
+        #     break;
+        #print(prop_data[r:r+8])
+        if cd_db.get(test) is None:
+            cd_db[test] = {}
+        # {
+        #     f"name_{language_code}":name, 
+        #     f"desc_{language_code}": 'foo', 
+        #     f"name_addr_{language_code}": hex(card_index_name_data), 
+        #     f"desc_addr_{language_code}": hex(card_index_desc_data)
+        # }
+        cd_db[test][f"name_{language_code}"] = name 
+        #cd_db[card_id][f"desc_{language_code}"] = desc 
+        cd_db[test][f"name_addr_{language_code}"] = hex(card_index_name_data)
+        #cd_db[card_id][f"desc_addr_{language_code}"] = hex(card_index_desc_data)
+
+language_code = 'E'
+#offsets_data = Path(f"YGO_2020/bin/DLG_Indx_{language_code}.bin").read_bytes() # Get all the bytes
+e_name_data = Path(f"YGO_2020/main/howto_db/howtoplay_{language_code}.bin").read_bytes() # Get all the bytes
+#get_props()
+language_code = 'J'
+j_name_data = Path(f"YGO_2020/main/howto_db/howtoplay_{language_code}.bin").read_bytes() # Get all the bytes
+
+
+scripts = {}
+
+def getScriptData():
+    offset = struct.unpack("<Q", script_data[0:8])[0] # 11914
+    titlecount = struct.unpack("<I", script_data[8:12])[0] # 307
+    scriptblockcount = struct.unpack("<I", script_data[12:16])[0] # 7340
+
+    titleStartPos = list()
+    titleEndPos = list()
+    titleOffset = list()
+    for p in range(titlecount):
+        i = p * 16
+        titleStartPos.append(struct.unpack("<I", script_data[i:i+4])[0])
+        titleEndPos.append(struct.unpack("<I", script_data[i+4:i+8])[0])
+        titleOffset.append(struct.unpack("<Q", script_data[i+8:i+16])[0])
+    ## 307 items? Is that good
+
+    script_titles = list()
+    for o in range(len(titleOffset)):
+        start = titleOffset[o]
+        end = read_until_space(start,script_data)
+        script_titles.append(script_data[start:end].decode("utf-8"))
+        print( script_data[start:end].decode("utf-8") )
+
+    ## Script Data extraction!
+    for p in range(titlecount):
+        #scripts[script_titles[p]] = {"title":script_titles[p], "blocks" : list()}
+        if script_titles[p] == '':
+            continue
+        if scripts.get(script_titles[p]) == None:
+            scripts[script_titles[p]] = {}
+        script_block = {}
+        blockCount = titleEndPos[p] - titleStartPos[p] #+1
+        #start = offset + titleOffset[p] + titleStartPos[p]
+        for j in range(blockCount):
+            seeker = offset + titleStartPos[p] * 32 + j * 32
+            start_char = struct.unpack("<Q", script_data[seeker:seeker+8])[0]
+            end = read_until_space(start_char, script_data)
+            script_block['character'] = script_data[start_char:end].decode('utf-8')
+            #index = f"{j}_{language_code}_{script_block['character']}"
+            index = f"{j}_{script_block['character']}"
+
+            seeker = seeker + 8
+            start_arg1 = struct.unpack("<Q", script_data[seeker:seeker+8])[0]
+            end = read_until_space(start_arg1, script_data)
+            script_block['arg1'] = script_data[start_arg1:end].decode('utf-8')
+
+            seeker = seeker + 8
+            start_arg2 = struct.unpack("<Q", script_data[seeker:seeker+8])[0]
+            end = read_until_space(start_arg2, script_data)
+            script_block['arg2'] = script_data[start_arg2:end].decode('utf-8')
+
+            seeker = seeker + 8
+            start_text = struct.unpack("<Q", script_data[seeker:seeker+8])[0]
+            end = read_until_space(start_text, script_data) ## These strings seem to always start with /0x00
+            script_block['text'] = script_data[start_text:end].decode('utf-8')
+
+            if script_block['text'] != '':
+                if scripts[script_titles[p]].get(index) == None:
+                    scripts[script_titles[p]][index] = {}
+                scripts[script_titles[p]][index]['index'] = j
+                scripts[script_titles[p]][index]['arg1'] = script_block['arg1']
+                scripts[script_titles[p]][index]['arg2'] = script_block['arg2']
+                #scripts[script_titles[p]][index]['text'] = script_block['text']
+                scripts[script_titles[p]][index][f"text_{language_code}"] = script_block['text']
+
+            #scripts[p]['blocks'].append(script_block.copy())
+            print(script_block)
+
+language_code = 'J'
+script_data = Path(f"YGO_2020/main/scriptdata_{language_code}.bin").read_bytes() # Get all the bytes
+getScriptData()
+language_code = 'E'
+script_data = Path(f"YGO_2020/main/scriptdata_{language_code}.bin").read_bytes() # Get all the bytes
+getScriptData()
+
+print('again?')
+#scripts = sorted(scripts.items(), key= lambda x: x[1]['index'], reverse=False)
+
+# for key in scripts:
+#     scripts[key] = sorted(scripts[key].items(), key = lambda x: x[1]['index'], reverse=False)
+
+with open(f"scripts_dump2.json", "w", encoding="utf-8") as outfile:
+   json.dump(scripts, outfile, ensure_ascii=False, indent=2)
+exit()
+
+print("now what?")
+
+count_of_items = int.from_bytes(e_name_data[0:4], byteorder='big') # 202
+## 202 * 8 = 1616 (Thats about where this started)
+
+
+help_dump = {}
+i_e = 1624
+i_j = 1624
+c = 1
+while (i_e < len(e_name_data) and i_j < len(j_name_data)):
+    help_dump[c] = {}
+    e = read_until_null(i_e,e_name_data)
+    help_dump[c]['en'] = e_name_data[i_e:e].decode("utf-16be", errors="replace")
+    print(help_dump[c]['en'] )
+    i_e = e + 2
+    e = read_until_null(i_j,j_name_data)
+    help_dump[c]['ja'] = j_name_data[i_j:e].decode("utf-16be", errors="replace")
+    print(help_dump[c]['ja'])
+    i_j = e + 2
+    c+=1
+print(c, " - strings extracted.")
+# c = 333 not sure how to index these anymore
+with open("helptext_dump.json", "w", encoding="utf-8") as outfile:
+   json.dump(help_dump, outfile, ensure_ascii=False, indent=2)
+
+exit()
+
+## Attempt to get ids or sense out of the beginning block, no luck so far
+for r in range(0,len(name_data),8):
+    #id = int.from_bytes( name_data[r:r+4], byteorder='big', signed=False )
+    #id = int(name_data[r:r+4], 16) & 0xFFFFFFFF 
+    id = struct.unpack('<I', name_data[r:r+4])[0] ## Only one that seemed to match the hex editor values for some reason. 
+    print(r, ": ", (name_data[r:r+4]), " - " , id)
+    if (id == 4): ## Why four?
+        break
+
+# What's 1616/4 == 404 
+read_until_null(1624,name_data)
+name_data[1624:read_until_null(1624,name_data)].decode("utf-16be", errors="replace")
+
+
+
+## Old attempts at getting dlg file extracted, leaving block for reference
+language_code = 'J'
+offsets_data = Path(f"YGO_2020/bin/DLG_Indx_{language_code}.bin").read_bytes() # Get all the bytes
+name_data = Path(f"YGO_2020/bin/DLG_Text_{language_code}.bin").read_bytes() # Get all the bytes
+get_props()
+
+# with open("dump2.json", "w", encoding="utf-16") as outfile:
+#     json.dump(cd_db, outfile, ensure_ascii=False, indent=2)
+
+print()
+
+### For dialog scripts eventually
+# Int ReadInt(std::ifstream &file){
+#     Int value;
+#     std::array<Byte,4> bytes;
+#     file.read((char*)(bytes.data()),4);
+#     value = bytes[0] | (bytes[1]<<8) | (bytes[2]<<16) | (bytes[3]<<24);
+#     return value;
+# }
+# Long ReadLong(std::ifstream &file){
+#     Long value;
+#     std::array<Byte,8> bytes;
+#     file.read((char*)(bytes.data()),8);
+#     value = bytes[0] | (bytes[1]<<8) | (bytes[2]<<16) | (bytes[3]<<24) | (bytes[4]<<32) | (bytes[5]<<40) | (bytes[6]<<48) | (bytes[7]<<56);
+#     return value;
+# }
+
+    # if (scriptFile.is_open()){
+    # Long pointerToPointers=ReadLong(scriptFile);
+    # Int titleCount=ReadInt(scriptFile);
+    # Int scriptBlockCount=ReadInt(scriptFile);
+    # std::vector<Int> titleStartPos;
+    # std::vector<Int> titleEndPos;
+    # std::vector<Long> titlePointer;
+    # for (Int i=0; i<titleCount;i++){
+    #     titleStartPos.push_back(ReadInt(scriptFile));     # 1
+    #     titleEndPos.push_back(ReadInt(scriptFile));       # 2
+    #     titlePointer.push_back(ReadLong(scriptFile));     # 3
+    # }
+    # std::vector<std::string> titleNames;
+    # for (Int i=0; i<titleCount;i++){
+    #     scriptFile.seekg(titlePointer[i],scriptFile.beg);
+    #     titleNames.push_back(ReadByteString(scriptFile));
+    # }
+
+    # std::vector<std::vector<SCRIPT_BLOCK> > script;
+    # for (Int i=0; i< titleCount; i++){
+    #     std::vector<SCRIPT_BLOCK> scriptBlocks;
+    #     Int blockCount=titleEndPos[i]-titleStartPos[i]+1;
+    #     int pointer;
+    #     for (Int j=0; j<blockCount;j++){
+    #         SCRIPT_BLOCK scriptBlock;
+    #         scriptFile.seekg(pointerToPointers+titleStartPos[i]*32+j*32);
+    #         pointer=ReadLong(scriptFile);
+    #         scriptFile.seekg(pointer,scriptFile.beg);
+    #         scriptBlock.chara=ReadByteString(scriptFile);
+    #         scriptFile.seekg(pointerToPointers+titleStartPos[i]*32+j*32+8);
+    #         pointer=ReadLong(scriptFile);
+    #         scriptFile.seekg(pointer,scriptFile.beg);
+    #         scriptBlock.arg1=ReadByteString(scriptFile);
+    #         scriptFile.seekg(pointerToPointers+titleStartPos[i]*32+j*32+16);
+    #         pointer=ReadLong(scriptFile);
+    #         scriptFile.seekg(pointer,scriptFile.beg);
+    #         scriptBlock.arg2=ReadByteString(scriptFile);
+    #         scriptFile.seekg(pointerToPointers+titleStartPos[i]*32+j*32+24);
+    #         pointer=ReadLong(scriptFile);
+    #         scriptFile.seekg(pointer,scriptFile.beg);
+    #         scriptBlock.text=ReadByteString(scriptFile);
+    #         scriptBlocks.push_back(scriptBlock);
+    #     }
+    #     script.push_back(scriptBlocks);
+    # }
+    # scriptFile.close();
+
+
+
+#> 

jp_readings.py

@@ -4,7 +4,7 @@ from fugashi_parser import *
 
 def main():
     parser = argparse.ArgumentParser()
-    parser.add_argument("--vocab-file", required=True)
+    parser.add_argument("--vocab-file", required=False)
     parser.add_argument("text", nargs="*")
     args = parser.parse_args()
 
@@ -13,12 +13,15 @@ def main():
     else:
         text = sys.stdin.read()
 
-    vocab = load_vocab(args.vocab_file)
+    #vocab = load_vocab(args.vocab_file)
+    vocab = {}
     result = analyze(text, vocab)
-    save_vocab(args.vocab_file, vocab)
+    #save_vocab(args.vocab_file, vocab)
 
     result["vocab_file"] = args.vocab_file
     print(json.dumps(result, ensure_ascii=False, indent=2))
+    print(json.dumps(list(vocab), ensure_ascii=False, indent=2))
+    #print(vocab)
 
 
 if __name__ == "__main__":

make_dialog_notes.py

@@ -0,0 +1,101 @@
+from fugashi_parser import *
+
+import os
+from pathlib import Path
+import shutil
+import hashlib
+
+destination_directory_name = '/run/media/deck/YF8SD/Sync/Notebooks/Obsidian/Japanese/Yu-Gi-Oh/'
+destination_path = Path(destination_directory_name)
+vocab_destination_path = Path('/run/media/deck/YF8SD/Sync/Notebooks/Obsidian/Japanese/vocab')
+
+with open("scripts_dump2.json", "r", encoding="utf-8") as f:
+    dump = json.load(f)
+
+if not destination_path.exists():
+    destination_path.mkdir(parents=True, exist_ok=True)
+if not destination_path.joinpath('YLOTD_dialog').exists():
+    destination_path.joinpath('YLOTD_dialog').mkdir(parents=True, exist_ok=True)
+
+counter = 0
+destination_path = destination_path.joinpath('YLOTD_dialog')
+for title in dump:
+    print(title)
+    counter+=1
+    ## copy the image if it isn't already there
+    with open(destination_path.as_posix() + f"/{counter:03}_{title}.md", 'w') as card_note:
+        for block in dump[title]:
+            print(block)
+            card_note.write(f"\n###### {block}\n\n")
+            #reformatted = re.sub(r'(\d{2})/(\d{2})/(\d{4})', r'\3-\1-\2', date)
+            #re.replace('\$R')
+            text_J = dump[title][block]['text_J']
+            text_J = re.sub(r'\$R(.*?)\((.*?)\)',r'<ruby>\1<rt>\2</rt></ruby>',text_J)
+            print(f"<details><summary>{text_J}</summary>{dump[title][block]['text_E']}</details>",file=card_note)
+            text_J = dump[title][block]['text_J']
+            text_J = re.sub(r'\$R(.*?)\((.*?)\)',r'\1',text_J)
+            text_J = text_J.replace('<br>', '')
+            vocab = {} ## flush the vocab variable each time
+            result = analyze(text_J, vocab)
+            print(f"> [!summary]- Vocab\n>{result['notes_link']}",file=card_note)
+            for word in vocab:
+                write_word_note(word, vocab_destination_path, vocab[word]['pos'][0])
+        # ## add card names as properties
+        # card_note.write("---\n")
+        # card_note.write(f"aliases: [{dump[cd]['name_J']},{dump[cd]['name_E']}]\n")
+        # card_note.write("---\n")
+        ## Name at the top
+        # card_note.write(f"![[{cd}.jpg]]") ## card image
+        # card_note.write("\n#### Name\n")
+        # card_note.write("<details><summary>")
+        # card_note.write(dump[cd]['name_J'])
+        # card_note.write("</summary>")
+        # card_note.write(dump[cd]['name_E'])
+        # card_note.write("</details>")
+        # ## the description
+        # card_note.write("\n#### Details\n")
+        # card_note.write("<details><summary>")
+        # card_note.write(dump[cd]['desc_J'])
+        # card_note.write("</summary>")
+        # card_note.write(dump[cd]['desc_E'])
+        # card_note.write("</details>")
+        # # desc_Furi
+        # card_note.write("\n#### Reading\n")
+        # vocab = {} ## flush the vocab variable each time
+        # result = analyze(dump[cd]['name_J'],vocab)
+        # print(f"<span>{result['ruby_html']}</span>\n> [!summary]- Vocab\n>{result['notes_link']}\n",file=card_note)
+        # ## Generate the vocab item
+        # for word in vocab:
+        #     if Path(destination_path.joinpath('vocab/' + word + '.md')).exists() == False:
+        #         definitions = anki_fields_term(word)
+        #         if definitions.get("fields"):
+        #             with open(destination_path.as_posix() + '/vocab/' + word + '.md', 'w') as vocab_note:
+        #                 for entry in definitions.get("fields"):
+        #                     print(entry['furigana'], file=vocab_note)
+        #                     if entry['expression'] != entry['reading']:
+        #                         print(entry['reading'], file=vocab_note)
+        #                     print("> ",re.sub(r'<.*?>', '', entry['glossary-plain-no-dictionary']), "\n", file=vocab_note)
+        #                     print(f"Frequency: {entry['frequency-average-rank']}", file=vocab_note)
+        #                     vocab_note.write(entry['glossary-first'])
+        #                     print("\n", file=vocab_note)
+        # # card_note.write("##### Glossary\n")
+        # # for vocab in dump[cd]["glossary_Name_J"]:
+        # #     print(f"{vocab} : {dump[cd]["glossary_Name_J"][vocab]}", file=card_note)
+        # vocab = {} ## flush the vocab variable each time
+        # result = analyze(dump[cd]['desc_J'],vocab)
+        # #card_note.write("\n")
+        # print(f"<span>{result['ruby_html']}</span>\n> [!summary]- Vocab\n>{result['notes_link']}\n",file=card_note)
+        # for word in vocab:
+        #     if Path(destination_path.joinpath('vocab/' + word + '.md')).exists() == False:
+        #         definitions = anki_fields_term(word)
+        #         if definitions.get("fields"):
+        #             with open(destination_path.as_posix() + '/vocab/' + word + '.md', 'w') as vocab_note:
+        #                 for entry in definitions.get("fields"):
+        #                     print(entry['furigana'], file=vocab_note)
+        #                     if entry['expression'] != entry['reading']:
+        #                         print(entry['reading'], file=vocab_note)
+        #                     print("> ",re.sub(r'<.*?>', '', entry['glossary-plain-no-dictionary']), "\n", file=vocab_note)
+        #                     print(f"Frequency: {entry['frequency-average-rank']}", file=vocab_note)
+        #                     vocab_note.write(entry['glossary-first'])
+        #                     print("\n", file=vocab_note)
+        # card_note.write("\n")

process_markdown.py

@@ -0,0 +1,66 @@
+from fugashi_parser import *
+
+import os
+from pathlib import Path
+import shutil
+import hashlib
+
+def main():
+    parser = argparse.ArgumentParser()
+    #parser.add_argument("--vocab-file", required=True)
+    parser.add_argument("--markdown-file", required=True)
+    parser.add_argument("--vocab-folder", required=False)
+    args = parser.parse_args()
+
+    if args.vocab_folder:
+        vocab_destination_path = Path(args.vocab_folder)
+        if vocab_destination_path.is_dir() == False:
+            raise ValueError("Vocab Folder is not a directory.")
+            return
+    else:
+        vocab_destination_path = Path(args.markdown_file).parent
+        if not vocab_destination_path.joinpath('vocab').exists():
+            vocab_destination_path.joinpath('vocab').mkdir(parents=True, exist_ok=True)
+        vocab_destination_path = vocab_destination_path.joinpath('vocab')
+
+    destination_path = Path(args.markdown_file).parent.joinpath(Path(args.markdown_file).stem + '_STUDY.md')
+
+    #print(args)
+    vocab = {} #load_vocab('foo.json')
+
+    ## Currently just doing this line by line
+    with open(args.markdown_file, "r", encoding="utf-8") as f:
+        lines = f.readlines()
+
+    i = 0 
+    file_changed = False
+    while i < len(lines):
+        ## Skip lines that start with certain elements
+        if lines[i][0:6] == '<span>' or lines[i][0] in ('#','>','['):
+            i+=1
+            continue
+        ## If the line has japanese
+        if has_japanese(lines[i]):
+            print(lines[i])
+            result = analyze(lines[i],vocab)
+            ## If the line changed at all
+            if lines[i] != result['ruby_html']:
+                lines[i] = '<span>' + result['ruby_html'] + '</span>' + "\n> [!summary]- Vocab\n>" + result['notes_link'] + "\n\n" # lost the new line
+                file_changed = True
+        i+=1
+
+    if file_changed == False:
+        return
+    ## vocab now
+    ## Lookup vocab
+    for word in vocab:
+        write_word_note(word, vocab_destination_path, vocab[word]['pos'][0])
+    
+    ## Output file with changes if changes
+    with open(destination_path, "w", encoding="utf-8") as f:
+        f.writelines(lines)
+
+
+if __name__ == "__main__":
+    main()
+

subtitle_processor.py

@@ -6,33 +6,48 @@ import shutil
 import hashlib
 
 
-destination_directory_name = '/run/media/deck/YF8SD/Sync/Notebooks/Obsidian/Japanese/Yu-Gi-Oh/AnimeSubs/'
+# destination_directory_name = '/run/media/deck/YF8SD/Sync/Notebooks/Obsidian/Japanese/Yu-Gi-Oh/AnimeSubs/'
-destination_path = Path(destination_directory_name)
+# destination_path = Path(destination_directory_name)
 
-dictionary_name = 'Jitendex.org [2026-04-04]'
+# dictionary_name = 'Jitendex.org [2026-04-04]'
 
-#print(annotate_phrase("私は学校へ行きます"))
+# #print(annotate_phrase("私は学校へ行きます"))
 
-## Make sure directories exist
+# ## Make sure directories exist
-if not destination_path.exists():
+# if not destination_path.exists():
-    destination_path.mkdir(parents=True, exist_ok=True)
+#     destination_path.mkdir(parents=True, exist_ok=True)
-if not destination_path.joinpath('_attachments').exists():
+# if not destination_path.joinpath('_attachments').exists():
-    destination_path.joinpath('_attachments').mkdir(parents=True, exist_ok=True)
+#     destination_path.joinpath('_attachments').mkdir(parents=True, exist_ok=True)
-if not destination_path.joinpath('vocab').exists():
+# if not destination_path.joinpath('vocab').exists():
-    destination_path.joinpath('vocab').mkdir(parents=True, exist_ok=True)
+#     destination_path.joinpath('vocab').mkdir(parents=True, exist_ok=True)
 
 
 # ---------- CLI ----------
 
 def main():
-#    parser = argparse.ArgumentParser()
+    parser = argparse.ArgumentParser()
     #parser.add_argument("--vocab-file", required=True)
-#    parser.add_argument("text", nargs="*")
+    parser.add_argument("--subtitle-file", required=True)
-#    args = parser.parse_args()
+    parser.add_argument("--vocab-folder", required=False)
+    args = parser.parse_args()
 
-    vocab = load_vocab('foo.json')
+    if args.vocab_folder:
+        destination_path = Path(args.vocab_folder)
+        if destination_path.is_dir() == False:
+            raise ValueError("Vocab Folder is not a directory.")
+            return
+    else:
+        destination_path = Path(args.subtitle_file).parent
+        if not destination_path.joinpath('vocab').exists():
+            destination_path.joinpath('vocab').mkdir(parents=True, exist_ok=True)
+        destination_path = destination_path.joinpath('vocab')
 
-    with open("/run/media/deck/YF8SD/Video/[Retimed and Corrected] Yu-Gi-Oh! Duel Monsters - 001.srt", "r", encoding="utf-8") as f:
+    # with open("/run/media/deck/YF8SD/Video/Yu-Gi-Oh! Duel Monsters - 001.vtt", "r", encoding="utf-8") as f:
+    #     lines = f.readlines()
+    vocab = {} #load_vocab('foo.json')
+
+    ## Currently just doing this line by line
+    with open(args.subtitle_file, "r", encoding="utf-8") as f:
         lines = f.readlines()
 
     
@@ -43,18 +58,25 @@ def main():
     ## First lien has some unicode bom bs
     i = 0 
     while i < len(lines):
+        ## If the line starts with a single number build a header
         if re.fullmatch(r'\d+', lines[i].strip()) or i == 0:
             notes_lines.append('##### ' + lines[i].strip() + " `" + lines[i+1].strip() + "`" + "\n")
             i+=2
             continue
+        if re.fullmatch(r'(\d{2,}:)*\d{2,}:\d{2,}[,.]\d{3,}\s*-->\s*(\d{2,}:)*\d{2,}:\d{2,}[,.]\d{3,}' , lines[i].strip()):
+            notes_lines.append('##### ' " `" + lines[i].strip() + "`" + "\n")
+            i+=1
+            continue
         if has_japanese(lines[i]):
             print(lines[i])
             result = analyze(lines[i],vocab)
             print(result['kana_reading'])
             notes_lines.append(result["ruby_html"] + "\n" )
             notes_lines.append (result['notes_link'] + "\n")
-            if lines[i] != result['kana_reading']:
+            if lines[i] != result['ruby_html']:
-                lines[i] = result['kana_reading'] + "\n" # lost the new line
+                lines[i] = result['ruby_html'] + "\n" # lost the new line
+            # if lines[i] != result['kana_reading']:
+            #     lines[i] = result['kana_reading'] + "\n" # lost the new line
         else:
            #notes_lines.append("`" + lines[i].strip() + "`" + "\n")
            notes_lines.append(lines[i])
@@ -62,29 +84,20 @@ def main():
 
     ## Lookup vocab
     for word in vocab:
-        #vocab_hash = hashlib.shake_256((word + dictionary_name).encode()).hexdigest(4)
+        write_word_note(word, destination_path)
-        if Path(destination_path.joinpath('vocab/' + word + '.md')).exists() == False:
+    return
-            definitions = anki_fields_term(word)
-            if definitions.get("fields"):
-                with open(destination_path.as_posix() + '/vocab/' + word + '.md', 'w') as vocab_note:
-                    for entry in definitions.get("fields"):
-                        vocab_note.write("\n")
-                        print(entry['expression'], file=vocab_note)
-                        #print(entry['furigana-plain'], file=vocab_note)
-                        if entry['expression'] != entry['reading']:
-                            print(entry['reading'], file=vocab_note)
-                        print(entry['furigana'], file=vocab_note)
-                        #print(f"Part of Speech: {entry['part-of-speech']}", file=vocab_note)
-                        print(f"Frequency: {entry['frequency-average-rank']}", file=vocab_note)
-                        #print(entry['glossary-plain-no-dictionary'], file=vocab_note)
-                        vocab_note.write(entry['glossary-first'])
-                        print("##### Other Definitions", file=vocab_note)
-                        print(entry['glossary-plain-no-dictionary'], file=vocab_note)
 
-    with open("subtitle.srt", "w", encoding="utf-8") as f:
+    # with open("/run/media/deck/YF8SD/Video/Yu-Gi-Oh! Duel Monsters - 001.vtt", "w", encoding="utf-8") as f:
+    #     f.writelines(lines)
+
+    ## Output file with changes if changes
+    out_dir = Path(args.subtitle_file).parent
+    changed_file = out_dir.joinpath(Path(args.subtitle_file).stem + '.txt')
+    with open(changed_file, "w", encoding="utf-8") as f:
         f.writelines(lines)
 
-    with open("notestest.md", "w",  encoding="utf-8") as f:
+    note_file = out_dir.joinpath(Path(args.subtitle_file).stem + '.md')
+    with open(note_file, "w",  encoding="utf-8") as f:
         f.writelines(notes_lines)
 
     #vocab = load_vocab(args.vocab_file)