diff --git a/add-readings.py b/add-readings.py index b234209..7dc3fcb 100644 --- a/add-readings.py +++ b/add-readings.py @@ -64,6 +64,7 @@ with open("words-readings.json", "r", encoding="utf-16le") as f: with open("dump.json", "r", encoding="utf-16le") as f: dump = json.load(f) +vocab_dict = {} ## A micab phrase based word parser for cd in dump: print(dump[cd]['nm']) @@ -77,42 +78,44 @@ for cd in dump: if anottated['reading'] != dump[cd]['nm']: print(anottated['reading']) dump[cd]['kana'] = anottated['reading'] - #dump[cd]['vocab'] = anottated['vocab'] - print() - -exit() - -## Iterates through all the keys not values -for cd in dump: - word = '' - new_name = '' - for char in dump[cd]['nm'] + '・': ## hacky char add to prevent dupe logic - if char in all_kana: - if len(word) > 0: - #print(word) - if readings.get(word) is not None: - kana = None - try: - #print("<--", readings[word]['kana']) - kana = readings[word]['kana'] - new_name += '(' + kana + ')' - #print() - except TypeError as e: - kana = None - #words[word] = (words[word] + 1) - else: - print(word, " <- Not Found") - word = '' + cd_nm = dump[cd]['nm'] + for word in anottated['vocab']: + # Skip punctuation words hopefully + if len(word) == 1 and has_kanji(word) == False: + continue + if vocab_dict.get(word) is None: + vocab_dict[word] = { + 'count': 0, + 'Examples': (cd_nm,) + } else: - word += char - new_name += char - if new_name[:-1]!= dump[cd]['nm']: - print (dump[cd]['nm']) - print (new_name[:-1]) - dump[cd]['kana'] = new_name[:-1] - ## Might need catch for end of string still + vocab_dict[word]['count'] = vocab_dict[word]['count'] + 1 + if cd_nm not in vocab_dict[word]['Examples']: + vocab_dict[word]['Examples'] += (cd_nm,) + print() + phrase = dump[cd]['de'] + anottated = annotate_phrase(phrase) + if anottated['reading'] != phrase: + #print(anottated['reading']) + dump[cd]['furi_de'] = anottated['reading'] + for word in anottated['vocab']: + if len(word) == 1 and has_kanji(word) == False: + continue + if vocab_dict.get(word) is None: + vocab_dict[word] = { + 'count': 0, + 'Examples': (cd_nm,) + } + else: + vocab_dict[word]['count'] = vocab_dict[word]['count'] + 1 + if len(vocab_dict[word]['Examples']) < 5: + if cd_nm not in vocab_dict[word]['Examples']: + vocab_dict[word]['Examples'] += (cd_nm,) -print() +vocab_dict = sorted(vocab_dict.items(), key= lambda x: x[1]['count'], reverse=True) -with open("dump-withkana.json", "w", encoding="utf-16le") as outfile: +with open("dump-withkana2.json", "w", encoding="utf-16") as outfile: json.dump(dump, outfile, ensure_ascii=False, indent=2) + +with open("dump-vocab.json", "w", encoding="utf-16") as outfile: + json.dump(vocab_dict, outfile, ensure_ascii=False, indent=2) diff --git a/get-properties.py b/get-properties.py index 62c329e..5d5cb25 100644 --- a/get-properties.py +++ b/get-properties.py @@ -3,47 +3,90 @@ import struct from pathlib import Path prop_data = Path(f"YGO_2020/bin/CARD_Prop.bin").read_bytes() # Get all the bytes -language_code = 'E' -offsets_data = Path(f"YGO_2020/bin/CARD_Indx_{language_code}.bin").read_bytes() # Get all the bytes -name_data = Path(f"YGO_2020/bin/CARD_Name_{language_code}.bin").read_bytes() # Get all the bytes print(prop_data[8]) # one byte supposedly -# 8 bytes at a time -test = 0 -for r in range(0,len(prop_data),8): - print(prop_data[r:r+8].hex(' ')) # print the byte range - card_prop = prop_data[r:r+8] - card_prop = card_prop[::-1] - #print(card_prop.hex(' ')) - card_first = prop_data[r:r+4] #Get first 4 bytes in range - card_second = prop_data[r+4:r+8] #Get first 4 bytes in range - # offsets_data[4:8].hex() - card_index_name_data = int.from_bytes( offsets_data[r:r+4], byteorder='little' ) - card_index_desc_data = int.from_bytes( offsets_data[r+4:r+8], byteorder='little' ) - #card_id = card_id[::-1] - card_bytes = int.from_bytes(card_first,byteorder='little') - card_bytes_2 = int.from_bytes(card_second,byteorder='little') - card_id = card_bytes & 16383 - card_attack = ((card_bytes >> 14) & 511) * 10 - card_defense = ((card_bytes >> 18) & 511) * 10 - - print(card_id , card_attack, card_attack, 'idx', card_index_name_data, card_index_desc_data) - #read until null found - end = card_index_name_data - while end + 1 < len(name_data): +def read_until_null(indx: int, data: tuple) -> int: + end = indx + while end + 1 < len(data): ## Read 2 bytes at a time until null found - if name_data[end:end+2] == b"\x00\x00": + if data[end:end+2] == b"\x00\x00": break end += 2 - name = name_data[card_index_name_data:end].decode("utf-16le", errors="replace") - print(name) - #print(card_id.hex(' ')) - # 2. Get last 14 bits using mask 0x3FFF (binary 11 1111 1111 1111) - #print([bin(x) for x in card_first]) - test += 1 - # if test > 3: - # break; - #print(prop_data[r:r+8]) + return end +# 8 bytes at a time +cd_db = {} +def get_props(): + test = 1 + for r in range(0,len(prop_data),8): + print(prop_data[r:r+8].hex(' ')) # print the byte range + card_prop = prop_data[r:r+8] + card_prop = card_prop[::-1] + #print(card_prop.hex(' ')) + card_first = prop_data[r:r+4] #Get first 4 bytes in range + card_second = prop_data[r+4:r+8] #Get first 4 bytes in range + # offsets_data[4:8].hex() + card_index_name_data = int.from_bytes( offsets_data[r:r+4], byteorder='little' ) + card_index_desc_data = int.from_bytes( offsets_data[r+4:r+8], byteorder='little' ) + #card_id = card_id[::-1] + card_bytes = int.from_bytes(card_first,byteorder='little') + card_bytes_2 = int.from_bytes(card_second,byteorder='little') + card_id = card_bytes & 16383 + if card_id == 0: + continue + card_attack = ((card_bytes >> 14) & 511) * 10 + card_defense = ((card_bytes >> 18) & 511) * 10 + + print(card_id , card_attack, card_attack, 'idx', card_index_name_data, card_index_desc_data) + #read until null found + # end = card_index_name_data + # while end + 1 < len(name_data): + # ## Read 2 bytes at a time until null found + # if name_data[end:end+2] == b"\x00\x00": + # break + # end += 2 + # name = name_data[card_index_name_data:end].decode("utf-16le", errors="replace") + end_index = read_until_null(card_index_name_data, name_data) + name = name_data[card_index_name_data:end_index].decode("utf-16le", errors="replace") + end_index = read_until_null(card_index_desc_data, desc_data) + desc = desc_data[card_index_desc_data:end_index].decode("utf-16le", errors="replace") + print(name) + #print(card_id.hex(' ')) + # 2. Get last 14 bits using mask 0x3FFF (binary 11 1111 1111 1111) + #print([bin(x) for x in card_first]) + test += 1 + # if test > 3: + # break; + #print(prop_data[r:r+8]) + if cd_db.get(card_id) is None: + cd_db[card_id] = {} + # { + # f"name_{language_code}":name, + # f"desc_{language_code}": 'foo', + # f"name_addr_{language_code}": hex(card_index_name_data), + # f"desc_addr_{language_code}": hex(card_index_desc_data) + # } + cd_db[card_id][f"name_{language_code}"] = name + cd_db[card_id][f"desc_{language_code}"] = desc + cd_db[card_id][f"name_addr_{language_code}"] = hex(card_index_name_data) + cd_db[card_id][f"desc_addr_{language_code}"] = hex(card_index_desc_data) +language_code = 'E' +offsets_data = Path(f"YGO_2020/bin/CARD_Indx_{language_code}.bin").read_bytes() # Get all the bytes +name_data = Path(f"YGO_2020/bin/CARD_Name_{language_code}.bin").read_bytes() # Get all the bytes +desc_data = Path(f"YGO_2020/bin/CARD_Desc_{language_code}.bin").read_bytes() + +get_props() + +language_code = 'J' +offsets_data = Path(f"YGO_2020/bin/CARD_Indx_{language_code}.bin").read_bytes() # Get all the bytes +name_data = Path(f"YGO_2020/bin/CARD_Name_{language_code}.bin").read_bytes() # Get all the bytes +desc_data = Path(f"YGO_2020/bin/CARD_Desc_{language_code}.bin").read_bytes() + +get_props() + +with open("dump2.json", "w", encoding="utf-16le") as outfile: + json.dump(cd_db, outfile, ensure_ascii=False, indent=2) + +print()