finally set up a repo

add-readings.py

@@ -0,0 +1,51 @@
+import json
+
+# Generate Hiragana and Katakana characters
+hiragana = tuple(chr(i) for i in range(12353, 12436)) # \u3041 - \u3094
+katakana = tuple(chr(i) for i in range(12450, 12532)) # \u30A2 - \u30F4
+punctuation = tuple(chr(i) for i in range(65281, 65382))
+
+# Combine both into one tuple
+all_kana = hiragana + katakana + punctuation + ('　', 'ー', '・')
+
+
+with open("words-readings.json", "r", encoding="utf-16le") as f:
+    readings = json.load(f)
+
+with open("dump.json", "r", encoding="utf-16le") as f:
+    dump = json.load(f)
+
+## Iterates through all the keys not values
+for cd in dump:
+    word = ''
+    new_name = ''
+    for char in dump[cd]['nm'] + '・': ## hacky char add to prevent dupe logic
+        if char in all_kana:
+            if len(word) > 0:
+                #print(word)
+                if readings.get(word) is not None:
+                    kana = None
+                    try:
+                        #print("<--", readings[word]['kana'])
+                        kana = readings[word]['kana']
+                        new_name += '(' + kana + ')'
+                        #print()
+                    except TypeError as e:
+                        kana = None
+                #words[word] = (words[word] + 1)
+                else:
+                    print(word, " <- Not Found")
+            word = ''
+        else:
+            word += char
+        new_name += char
+    if new_name[:-1]!= dump[cd]['nm']:
+        print (dump[cd]['nm'])
+        print (new_name[:-1])
+        dump[cd]['kana'] = new_name[:-1]
+        ## Might need catch for end of string still
+
+print()
+
+with open("dump-withkana.json", "w", encoding="utf-16le") as outfile:
+    json.dump(dump, outfile, ensure_ascii=False, indent=2)

extract_cards.py

@@ -0,0 +1,156 @@
+import mmap
+import struct
+import json
+from pathlib import Path
+
+# Generate Hiragana and Katakana characters
+hiragana = tuple(chr(i) for i in range(12353, 12436)) # \u3041 - \u3094
+katakana = tuple(chr(i) for i in range(12450, 12532)) # \u30A2 - \u30F4
+
+# Combine both into one tuple
+all_kana = hiragana + katakana
+
+# Print example
+print(all_kana)
+
+prop_data = Path(f"YGO_2020/bin/CARD_Prop.bin").read_bytes() # Get all the bytes
+
+language_code = 'J'
+offsets_data = Path(f"YGO_2020/bin/CARD_Indx_{language_code}.bin").read_bytes() # Get all the bytes
+name_data = Path(f"YGO_2020/bin/CARD_Name_{language_code}.bin").read_bytes() # Get all the bytes
+desc_data = Path(f"YGO_2020/bin/CARD_Desc_{language_code}.bin").read_bytes()
+offsets = [x[0] for x in struct.iter_unpack("<I", offsets_data)] ## load all bytes into file.
+
+
+#fruits = ("apple", "banana", "cherry")
+#print("orange" not in fruits)  # Output: True
+# if key not in my_dict:
+
+char_db = {}
+words = {}
+cd_db = {}
+cd_db_ctr = 3900
+starting_offset = 2
+off = starting_offset
+## Read two bytes at a time 
+#while off+2 < len(offsets_data): ## didnt work
+#while offsets[off+2] and off < 10:
+while off+2 < len(offsets):
+    name_start_off = offsets[off]
+    desc_start_off = offsets[off + 1]
+
+    #read until null found
+    end = name_start_off
+    while end + 1 < len(name_data):
+        ## Read 2 bytes at a time until null found
+        if name_data[end:end+2] == b"\x00\x00":
+            break
+        end += 2
+    name = name_data[name_start_off:end].decode("utf-16le", errors="replace")
+    print(name)
+    # for c in  range(name_start_off, end, 2):
+    #     c_value = name_data[c:c+2]
+    #     if char_db.get(c_value) is None:
+    #         char_db[c_value] = 0
+    #         print(c_value, c_value.decode("utf-16le", errors="replace"))
+    #     char_db[c_value] = (char_db[c_value] + 1 )
+    for char in name:
+        if char_db.get(char) is None:
+            char_db[char] = 0
+            print(char)
+        char_db[char] = (char_db[char] + 1 )
+
+    word = ''
+    for char in name:
+        if char in all_kana:
+            if len(word) > 0:
+                if words.get(word) is None:
+                    words[word] = 0
+                    print(word)
+                words[word] = (words[word] + 1)
+                word = ''
+        else:
+            word += char
+    if len(word) > 0:
+        if words.get(word) is None:
+            words[word] = 0
+            print(word)
+        words[word] = (words[word] + 1)
+        
+
+
+    end = desc_start_off
+    while end + 1 < len(desc_data):
+        ## Read 2 bytes at a time until null found
+        if desc_data[end:end+2] == b"\x00\x00":
+            break
+        end += 2
+    desc = desc_data[desc_start_off:end].decode("utf-16le", errors="replace")
+
+
+    #print(name)
+    cd_db[cd_db_ctr] = {"nm":name, "de": desc, "nm_addr":hex(name_start_off), "de_addr":hex(desc_start_off)}
+    cd_db_ctr += 1
+    #print()
+    ## Iterate offset
+    off += 2
+
+language_code = 'E'
+offsets_data = Path(f"YGO_2020/bin/CARD_Indx_{language_code}.bin").read_bytes() # Get all the bytes
+name_data = Path(f"YGO_2020/bin/CARD_Name_{language_code}.bin").read_bytes() # Get all the bytes
+desc_data = Path(f"YGO_2020/bin/CARD_Desc_{language_code}.bin").read_bytes()
+offsets = [x[0] for x in struct.iter_unpack("<I", offsets_data)] ## load all bytes into file.
+
+cd_db_ctr = 3900
+starting_offset = 2
+off = starting_offset
+## Read two bytes at a time 
+#while off+2 < len(offsets_data): ## didnt work
+#while offsets[off+2] and off < 10:
+while off+2 < len(offsets):
+    name_start_off = offsets[off]
+    desc_start_off = offsets[off + 1]
+
+    #read until null found
+    end = name_start_off
+    while end + 1 < len(name_data):
+        ## Read 2 bytes at a time until null found
+        if name_data[end:end+2] == b"\x00\x00":
+            break
+        end += 2
+    name = name_data[name_start_off:end].decode("utf-16le", errors="replace")
+    print(name)
+
+    end = desc_start_off
+    while end + 1 < len(desc_data):
+        ## Read 2 bytes at a time until null found
+        if desc_data[end:end+2] == b"\x00\x00":
+            break
+        end += 2
+    desc = desc_data[desc_start_off:end].decode("utf-16le", errors="replace")
+
+
+    #print(name)
+    cd_db[cd_db_ctr]["nm_e"] = name 
+    cd_db[cd_db_ctr]["de_e"] = desc 
+    cd_db[cd_db_ctr]["nm_addr_e"] = hex(name_start_off)
+    cd_db[cd_db_ctr]["de_addr_e"] = hex(desc_start_off)
+    cd_db_ctr += 1
+    #print()
+    ## Iterate offset
+    off += 2
+
+
+# Writing to a file
+with open("dump.json", "w", encoding="utf-16le") as outfile:
+    json.dump(cd_db, outfile, ensure_ascii=False, indent=2)
+
+with open("words.json", "w", encoding="utf-16le") as outfile:
+    json.dump(words, outfile, ensure_ascii=False, indent=2)
+
+with open("char_db.json", "w", encoding="utf-16le") as outfile:
+    json.dump(char_db, outfile, ensure_ascii=False, indent=2)
+
+
+print("how do?")
+

get-properties.py

@@ -0,0 +1,49 @@
+import mmap
+import struct
+from pathlib import Path
+
+prop_data = Path(f"YGO_2020/bin/CARD_Prop.bin").read_bytes() # Get all the bytes
+language_code = 'E'
+offsets_data = Path(f"YGO_2020/bin/CARD_Indx_{language_code}.bin").read_bytes() # Get all the bytes
+name_data = Path(f"YGO_2020/bin/CARD_Name_{language_code}.bin").read_bytes() # Get all the bytes
+
+print(prop_data[8]) # one byte supposedly
+
+# 8 bytes at a time
+test = 0
+for r in range(0,len(prop_data),8):
+    print(prop_data[r:r+8].hex(' ')) # print the byte range
+    card_prop = prop_data[r:r+8]    
+    card_prop = card_prop[::-1]
+    #print(card_prop.hex(' '))
+    card_first = prop_data[r:r+4] #Get first 4 bytes in range
+    card_second = prop_data[r+4:r+8] #Get first 4 bytes in range
+    # offsets_data[4:8].hex()
+    card_index_name_data =  int.from_bytes( offsets_data[r:r+4], byteorder='little' )
+    card_index_desc_data =  int.from_bytes( offsets_data[r+4:r+8], byteorder='little' )
+    #card_id = card_id[::-1]
+    card_bytes = int.from_bytes(card_first,byteorder='little')
+    card_bytes_2 = int.from_bytes(card_second,byteorder='little')
+    card_id = card_bytes & 16383
+    card_attack = ((card_bytes >> 14) & 511) * 10
+    card_defense = ((card_bytes >> 18) & 511) * 10
+    
+    print(card_id , card_attack, card_attack, 'idx', card_index_name_data, card_index_desc_data)
+    #read until null found
+    end = card_index_name_data
+    while end + 1 < len(name_data):
+        ## Read 2 bytes at a time until null found
+        if name_data[end:end+2] == b"\x00\x00":
+            break
+        end += 2
+    name = name_data[card_index_name_data:end].decode("utf-16le", errors="replace")
+    print(name)
+    #print(card_id.hex(' '))
+    # 2. Get last 14 bits using mask 0x3FFF (binary 11 1111 1111 1111)
+    #print([bin(x) for x in card_first])
+    test += 1
+    # if test > 3:
+    #     break;
+    #print(prop_data[r:r+8])
+
+

get_readings.py

@@ -0,0 +1,33 @@
+import json
+from janome.tokenizer import Tokenizer
+from kotobase import Kotobase
+
+kb = Kotobase()
+tokenizer = Tokenizer()
+
+parts = []
+for token in tokenizer.tokenize('魔草トークン'):
+    print(token)
+    print(token.reading)
+    reading = token.reading
+    if reading == "*":
+        reading = token.surface
+    parts.append(reading)
+
+print(parts)
+
+with open("words.json", "r", encoding="utf-16le") as f:
+    words = json.load(f)
+
+for word in words:
+    result = kb.lookup(word)
+    if result:
+        if len(result.entries) > 0:
+            kana = result.entries[0].kana[0]
+            print (result.word, ": ", kana)
+            words[word] = {"kana":kana,"count":words[word]}
+
+with open("words-readings.json", "w", encoding="utf-16le") as outfile:
+    json.dump(words, outfile, ensure_ascii=False, indent=2)
+
+print()

get_readings2.py

@@ -0,0 +1,67 @@
+import json
+from janome.tokenizer import Tokenizer
+from kotobase import Kotobase
+
+kb = Kotobase()
+tokenizer = Tokenizer()
+
+with open("words.json", "r", encoding="utf-16le") as f:
+    words = json.load(f)
+
+# Generate Hiragana and Katakana characters
+hiragana = tuple(chr(i) for i in range(12353, 12436)) # \u3041 - \u3094
+katakana = tuple(chr(i) for i in range(12450, 12532)) # \u30A2 - \u30F4
+punctuation = tuple(chr(i) for i in range(65281, 65382))
+
+# Combine both into one tuple
+all_kana = hiragana + katakana + punctuation + ('　', 'ー', '・')
+
+
+with open("dump.json", "r", encoding="utf-16le") as f:
+    dump = json.load(f)
+
+readings = {}
+## Iterates through all the keys not values
+for cd in dump:
+    word = ''
+    for char in dump[cd]['nm'] + '・': ## hacky char add to prevent dupe logic
+        if char in all_kana:
+            if len(word) > 0:
+                #print(word)
+                if readings.get(word) is None:
+                    kana = None
+                    try:
+                        result = kb.lookup(word)
+                        if result:
+                            if len(result.entries) > 0:
+                                kana = result.entries[0].kana[0]
+                                print (result.word, ": ", kana)
+                                readings[word] = {"kana":kana}
+                            else:
+                                print("no entries")
+                                for k in result.kanji:
+                                    if len(k.kunyomi) > 0:
+                                        print(k.kunyomi[0])
+                    except TypeError as e:
+                        kana = None
+                #words[word] = (words[word] + 1)
+                word = ''
+        else:
+            word += char
+        ## Might need catch for end of string still
+
+
+print()
+
+# for word in words:
+#     result = kb.lookup(word)
+#     if result:
+#         if len(result.entries) > 0:
+#             kana = result.entries[0].kana[0]
+#             print (result.word, ": ", kana)
+#             words[word] = {"kana":kana,"count":words[word]}
+
+with open("words-readings.json", "w", encoding="utf-16le") as outfile:
+    json.dump(readings, outfile, ensure_ascii=False, indent=2)
+
+print()

kana_assc.py

@@ -0,0 +1,27 @@
+
+import mmap
+import struct
+from pathlib import Path
+
+
+kana1_data = Path("YGO_2020/CARD_Kana1_J.bin").read_bytes()
+kana2_data = Path("YGO_2020/CARD_Kana2_J.bin").read_bytes()
+kana3_data = Path("YGO_2020/CARD_Kana3_J.bin").read_bytes()
+
+## The offsets pulled from the file
+items1 = [x[0] for x in struct.iter_unpack("<I", kana1_data)]
+
+#print(f"Foo {[f'0x{x:x}' for x in offsets[2:4]]}") # Getting range and convert to hex
+#print(", ".join("{:04x}".format(x) for x in items1[6:8]))
+print(", ".join("{:04x}".format(x) for x in kana1_data[6:8]))
+kana1_data[6:8].decode("utf-16le", errors="replace")
+
+print()
+
+
+# kana1_data[2:4].decode("utf-16le", errors="replace")
+# 'い'
+# kana2_data[2:4].decode("utf-16le", errors="replace")
+# 'ん'
+# kana3_data[2:4].decode("utf-16le", errors="replace")
+# 'せ'

parse_strings.py

@@ -0,0 +1,82 @@
+import mmap
+import struct
+from pathlib import Path
+
+def iter_offsets(offset_file: str):
+    skip_header = False
+    with open(offset_file, "rb") as f:
+        if skip_header == False:
+            print(f.read(8))
+            skip_header = True
+        while chunk := f.read(4):
+            if len(chunk) != 4:
+                raise ValueError("Incomplete 4-byte offset at end of file")
+            #yield struct.unpact("<I", chunk)[0]
+            print(chunk)
+
+# with open("YGO_2020/CARD_Name_E.bin", "rb") as tf:
+#     with mmap.mmap(tf.fileno(), 0, access=mmap.ACCESS_READ) as mm:
+#         for off in iter_offsets("YGO_2020\CARD_Indx_E.bin"):
+#             print(off)
+
+## Test iter_offsets
+#iter_offsets("YGO_2020/CARD_Indx_E.bin")
+
+chr(65322) # int to char
+ord('Ｌ') # char to unicode val
+
+
+# offsets_data = Path("YGO_2020/CARD_Indx_E.bin").read_bytes()
+# text_data = Path("YGO_2020/CARD_Name_E.bin").read_bytes()
+
+offsets_data = Path("YGO_2020/CARD_Indx_J.bin").read_bytes()
+text_data = Path("YGO_2020/CARD_Name_J.bin").read_bytes()
+
+
+offsets = [x[0] for x in struct.iter_unpack("<I", offsets_data)]
+## Getting ranges of offsets
+offsets[:2] # first 2 bytes
+offsets[4:8] # 8 bytes from the 4th 
+
+## Printing values for debugging
+print(f"Foo {[f'0x{x:x}' for x in offsets[2:4]]}") # Getting range and convert to hex
+print(", ".join("{:04x}".format(x) for x in offsets[6:8]))
+
+# for off in offsets[:10]:
+#     print(f"offset={off} hex{off:08X}")
+#     print(text_data[off:off+32].hex(" "))
+#     print()
+
+cd_db = {}
+cd_db_ctr = 3900
+starting_offset = 2
+off = starting_offset
+## Read two bytes at a time 
+#while off+2 < len(offsets_data): ## didnt work
+#while offsets[off+2] and off < 10:
+while off+2 < len(offsets):
+    print("Offset Pair:", ", ".join("{:04x}".format(x) for x in offsets[off:off+2]))
+    name_start_off = offsets[off]
+    ## Peek method
+    # name_end_off = offsets[off+2] # peek to next offset
+    # print((text_data[name_start_off:(name_end_off-1)]).hex(" "))
+    # print((text_data[name_start_off:(name_end_off-1)]).decode("utf-16le", errors="replace"))
+    # print()
+
+    #read until null found
+    end = name_start_off
+    while end + 1 < len(text_data):
+        ## Read 2 bytes at a time until null found
+        if text_data[end:end+2] == b"\x00\x00":
+            break
+        end += 2
+    text = text_data[name_start_off:end].decode("utf-16le", errors="replace")
+    print(text)
+    cd_db[cd_db_ctr] = {"nm":text, "addr":hex(name_start_off)}
+    cd_db_ctr += 1
+    print()
+    ## Iterate offset
+    off += 2
+
+print("how do?")
+