better things

add-readings.py

@@ -64,6 +64,7 @@ with open("words-readings.json", "r", encoding="utf-16le") as f:
 with open("dump.json", "r", encoding="utf-16le") as f:
     dump = json.load(f)
 
+vocab_dict = {}
 ## A micab phrase based word parser
 for cd in dump:
     print(dump[cd]['nm'])
@@ -77,42 +78,44 @@ for cd in dump:
     if anottated['reading'] != dump[cd]['nm']:
         print(anottated['reading'])
         dump[cd]['kana'] = anottated['reading']
-        #dump[cd]['vocab'] = anottated['vocab']
+    cd_nm = dump[cd]['nm']
-    print()
+    for word in anottated['vocab']:
-
+        # Skip punctuation words hopefully
-exit()
+        if len(word) == 1 and has_kanji(word) == False:
-
+            continue
-## Iterates through all the keys not values
+        if vocab_dict.get(word) is None:
-for cd in dump:
+            vocab_dict[word] = {
-    word = ''
+                'count': 0,
-    new_name = ''
+                'Examples': (cd_nm,)
-    for char in dump[cd]['nm'] + '・': ## hacky char add to prevent dupe logic
+            }
-        if char in all_kana:
-            if len(word) > 0:
-                #print(word)
-                if readings.get(word) is not None:
-                    kana = None
-                    try:
-                        #print("<--", readings[word]['kana'])
-                        kana = readings[word]['kana']
-                        new_name += '(' + kana + ')'
-                        #print()
-                    except TypeError as e:
-                        kana = None
-                #words[word] = (words[word] + 1)
         else:
-                    print(word, " <- Not Found")
+            vocab_dict[word]['count'] = vocab_dict[word]['count'] + 1
-            word = ''
+        if cd_nm not in vocab_dict[word]['Examples']:
-        else:
+            vocab_dict[word]['Examples'] += (cd_nm,)
-            word += char
-        new_name += char
-    if new_name[:-1]!= dump[cd]['nm']:
-        print (dump[cd]['nm'])
-        print (new_name[:-1])
-        dump[cd]['kana'] = new_name[:-1]
-        ## Might need catch for end of string still
-
     print()
+    phrase = dump[cd]['de']
+    anottated = annotate_phrase(phrase)
+    if anottated['reading'] != phrase:
+        #print(anottated['reading'])
+        dump[cd]['furi_de'] = anottated['reading']
+    for word in anottated['vocab']:
+        if len(word) == 1 and has_kanji(word) == False:
+            continue
+        if vocab_dict.get(word) is None:
+            vocab_dict[word] = {
+                'count': 0,
+                'Examples': (cd_nm,)
+            }
+        else:
+            vocab_dict[word]['count'] = vocab_dict[word]['count'] + 1
+        if len(vocab_dict[word]['Examples']) < 5:
+            if cd_nm not in vocab_dict[word]['Examples']:
+                vocab_dict[word]['Examples'] += (cd_nm,)
 
-with open("dump-withkana.json", "w", encoding="utf-16le") as outfile:
+vocab_dict = sorted(vocab_dict.items(), key= lambda x: x[1]['count'], reverse=True)
+
+with open("dump-withkana2.json", "w", encoding="utf-16") as outfile:
     json.dump(dump, outfile, ensure_ascii=False, indent=2)
+
+with open("dump-vocab.json", "w", encoding="utf-16") as outfile:
+    json.dump(vocab_dict, outfile, ensure_ascii=False, indent=2)

get-properties.py

@@ -3,14 +3,22 @@ import struct
 from pathlib import Path
 
 prop_data = Path(f"YGO_2020/bin/CARD_Prop.bin").read_bytes() # Get all the bytes
-language_code = 'E'
-offsets_data = Path(f"YGO_2020/bin/CARD_Indx_{language_code}.bin").read_bytes() # Get all the bytes
-name_data = Path(f"YGO_2020/bin/CARD_Name_{language_code}.bin").read_bytes() # Get all the bytes
 
 print(prop_data[8]) # one byte supposedly
 
+def read_until_null(indx: int, data: tuple) -> int:
+    end = indx
+    while end + 1 < len(data):
+        ## Read 2 bytes at a time until null found
+        if data[end:end+2] == b"\x00\x00":
+            break
+        end += 2
+    return end
+
 # 8 bytes at a time
-test = 0
+cd_db = {}
+def get_props():
+    test = 1
     for r in range(0,len(prop_data),8):
         print(prop_data[r:r+8].hex(' ')) # print the byte range
         card_prop = prop_data[r:r+8]    
@@ -25,18 +33,24 @@ for r in range(0,len(prop_data),8):
         card_bytes = int.from_bytes(card_first,byteorder='little')
         card_bytes_2 = int.from_bytes(card_second,byteorder='little')
         card_id = card_bytes & 16383
+        if card_id == 0:
+            continue
         card_attack = ((card_bytes >> 14) & 511) * 10
         card_defense = ((card_bytes >> 18) & 511) * 10
         
         print(card_id , card_attack, card_attack, 'idx', card_index_name_data, card_index_desc_data)
         #read until null found
-    end = card_index_name_data
+        # end = card_index_name_data
-    while end + 1 < len(name_data):
+        # while end + 1 < len(name_data):
-        ## Read 2 bytes at a time until null found
+        #     ## Read 2 bytes at a time until null found
-        if name_data[end:end+2] == b"\x00\x00":
+        #     if name_data[end:end+2] == b"\x00\x00":
-            break
+        #         break
-        end += 2
+        #     end += 2
-    name = name_data[card_index_name_data:end].decode("utf-16le", errors="replace")
+        # name = name_data[card_index_name_data:end].decode("utf-16le", errors="replace")
+        end_index = read_until_null(card_index_name_data, name_data)
+        name = name_data[card_index_name_data:end_index].decode("utf-16le", errors="replace")
+        end_index = read_until_null(card_index_desc_data, desc_data)
+        desc = desc_data[card_index_desc_data:end_index].decode("utf-16le", errors="replace")
         print(name)
         #print(card_id.hex(' '))
         # 2. Get last 14 bits using mask 0x3FFF (binary 11 1111 1111 1111)
@@ -45,5 +59,34 @@ for r in range(0,len(prop_data),8):
         # if test > 3:
         #     break;
         #print(prop_data[r:r+8])
+        if cd_db.get(card_id) is None:
+            cd_db[card_id] = {}
+        # {
+        #     f"name_{language_code}":name, 
+        #     f"desc_{language_code}": 'foo', 
+        #     f"name_addr_{language_code}": hex(card_index_name_data), 
+        #     f"desc_addr_{language_code}": hex(card_index_desc_data)
+        # }
+        cd_db[card_id][f"name_{language_code}"] = name 
+        cd_db[card_id][f"desc_{language_code}"] = desc 
+        cd_db[card_id][f"name_addr_{language_code}"] = hex(card_index_name_data)
+        cd_db[card_id][f"desc_addr_{language_code}"] = hex(card_index_desc_data)
 
+language_code = 'E'
+offsets_data = Path(f"YGO_2020/bin/CARD_Indx_{language_code}.bin").read_bytes() # Get all the bytes
+name_data = Path(f"YGO_2020/bin/CARD_Name_{language_code}.bin").read_bytes() # Get all the bytes
+desc_data = Path(f"YGO_2020/bin/CARD_Desc_{language_code}.bin").read_bytes()
 
+get_props()
+
+language_code = 'J'
+offsets_data = Path(f"YGO_2020/bin/CARD_Indx_{language_code}.bin").read_bytes() # Get all the bytes
+name_data = Path(f"YGO_2020/bin/CARD_Name_{language_code}.bin").read_bytes() # Get all the bytes
+desc_data = Path(f"YGO_2020/bin/CARD_Desc_{language_code}.bin").read_bytes()
+
+get_props()
+
+with open("dump2.json", "w", encoding="utf-16le") as outfile:
+    json.dump(cd_db, outfile, ensure_ascii=False, indent=2)
+
+print()