better things
This commit is contained in:
parent
dddc6a0b0e
commit
100a25b6d9
|
|
@ -64,6 +64,7 @@ with open("words-readings.json", "r", encoding="utf-16le") as f:
|
||||||
with open("dump.json", "r", encoding="utf-16le") as f:
|
with open("dump.json", "r", encoding="utf-16le") as f:
|
||||||
dump = json.load(f)
|
dump = json.load(f)
|
||||||
|
|
||||||
|
vocab_dict = {}
|
||||||
## A micab phrase based word parser
|
## A micab phrase based word parser
|
||||||
for cd in dump:
|
for cd in dump:
|
||||||
print(dump[cd]['nm'])
|
print(dump[cd]['nm'])
|
||||||
|
|
@ -77,42 +78,44 @@ for cd in dump:
|
||||||
if anottated['reading'] != dump[cd]['nm']:
|
if anottated['reading'] != dump[cd]['nm']:
|
||||||
print(anottated['reading'])
|
print(anottated['reading'])
|
||||||
dump[cd]['kana'] = anottated['reading']
|
dump[cd]['kana'] = anottated['reading']
|
||||||
#dump[cd]['vocab'] = anottated['vocab']
|
cd_nm = dump[cd]['nm']
|
||||||
print()
|
for word in anottated['vocab']:
|
||||||
|
# Skip punctuation words hopefully
|
||||||
exit()
|
if len(word) == 1 and has_kanji(word) == False:
|
||||||
|
continue
|
||||||
## Iterates through all the keys not values
|
if vocab_dict.get(word) is None:
|
||||||
for cd in dump:
|
vocab_dict[word] = {
|
||||||
word = ''
|
'count': 0,
|
||||||
new_name = ''
|
'Examples': (cd_nm,)
|
||||||
for char in dump[cd]['nm'] + '・': ## hacky char add to prevent dupe logic
|
}
|
||||||
if char in all_kana:
|
|
||||||
if len(word) > 0:
|
|
||||||
#print(word)
|
|
||||||
if readings.get(word) is not None:
|
|
||||||
kana = None
|
|
||||||
try:
|
|
||||||
#print("<--", readings[word]['kana'])
|
|
||||||
kana = readings[word]['kana']
|
|
||||||
new_name += '(' + kana + ')'
|
|
||||||
#print()
|
|
||||||
except TypeError as e:
|
|
||||||
kana = None
|
|
||||||
#words[word] = (words[word] + 1)
|
|
||||||
else:
|
|
||||||
print(word, " <- Not Found")
|
|
||||||
word = ''
|
|
||||||
else:
|
else:
|
||||||
word += char
|
vocab_dict[word]['count'] = vocab_dict[word]['count'] + 1
|
||||||
new_name += char
|
if cd_nm not in vocab_dict[word]['Examples']:
|
||||||
if new_name[:-1]!= dump[cd]['nm']:
|
vocab_dict[word]['Examples'] += (cd_nm,)
|
||||||
print (dump[cd]['nm'])
|
print()
|
||||||
print (new_name[:-1])
|
phrase = dump[cd]['de']
|
||||||
dump[cd]['kana'] = new_name[:-1]
|
anottated = annotate_phrase(phrase)
|
||||||
## Might need catch for end of string still
|
if anottated['reading'] != phrase:
|
||||||
|
#print(anottated['reading'])
|
||||||
|
dump[cd]['furi_de'] = anottated['reading']
|
||||||
|
for word in anottated['vocab']:
|
||||||
|
if len(word) == 1 and has_kanji(word) == False:
|
||||||
|
continue
|
||||||
|
if vocab_dict.get(word) is None:
|
||||||
|
vocab_dict[word] = {
|
||||||
|
'count': 0,
|
||||||
|
'Examples': (cd_nm,)
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
vocab_dict[word]['count'] = vocab_dict[word]['count'] + 1
|
||||||
|
if len(vocab_dict[word]['Examples']) < 5:
|
||||||
|
if cd_nm not in vocab_dict[word]['Examples']:
|
||||||
|
vocab_dict[word]['Examples'] += (cd_nm,)
|
||||||
|
|
||||||
print()
|
vocab_dict = sorted(vocab_dict.items(), key= lambda x: x[1]['count'], reverse=True)
|
||||||
|
|
||||||
with open("dump-withkana.json", "w", encoding="utf-16le") as outfile:
|
with open("dump-withkana2.json", "w", encoding="utf-16") as outfile:
|
||||||
json.dump(dump, outfile, ensure_ascii=False, indent=2)
|
json.dump(dump, outfile, ensure_ascii=False, indent=2)
|
||||||
|
|
||||||
|
with open("dump-vocab.json", "w", encoding="utf-16") as outfile:
|
||||||
|
json.dump(vocab_dict, outfile, ensure_ascii=False, indent=2)
|
||||||
|
|
|
||||||
|
|
@ -3,47 +3,90 @@ import struct
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
prop_data = Path(f"YGO_2020/bin/CARD_Prop.bin").read_bytes() # Get all the bytes
|
prop_data = Path(f"YGO_2020/bin/CARD_Prop.bin").read_bytes() # Get all the bytes
|
||||||
language_code = 'E'
|
|
||||||
offsets_data = Path(f"YGO_2020/bin/CARD_Indx_{language_code}.bin").read_bytes() # Get all the bytes
|
|
||||||
name_data = Path(f"YGO_2020/bin/CARD_Name_{language_code}.bin").read_bytes() # Get all the bytes
|
|
||||||
|
|
||||||
print(prop_data[8]) # one byte supposedly
|
print(prop_data[8]) # one byte supposedly
|
||||||
|
|
||||||
# 8 bytes at a time
|
def read_until_null(indx: int, data: tuple) -> int:
|
||||||
test = 0
|
end = indx
|
||||||
for r in range(0,len(prop_data),8):
|
while end + 1 < len(data):
|
||||||
print(prop_data[r:r+8].hex(' ')) # print the byte range
|
|
||||||
card_prop = prop_data[r:r+8]
|
|
||||||
card_prop = card_prop[::-1]
|
|
||||||
#print(card_prop.hex(' '))
|
|
||||||
card_first = prop_data[r:r+4] #Get first 4 bytes in range
|
|
||||||
card_second = prop_data[r+4:r+8] #Get first 4 bytes in range
|
|
||||||
# offsets_data[4:8].hex()
|
|
||||||
card_index_name_data = int.from_bytes( offsets_data[r:r+4], byteorder='little' )
|
|
||||||
card_index_desc_data = int.from_bytes( offsets_data[r+4:r+8], byteorder='little' )
|
|
||||||
#card_id = card_id[::-1]
|
|
||||||
card_bytes = int.from_bytes(card_first,byteorder='little')
|
|
||||||
card_bytes_2 = int.from_bytes(card_second,byteorder='little')
|
|
||||||
card_id = card_bytes & 16383
|
|
||||||
card_attack = ((card_bytes >> 14) & 511) * 10
|
|
||||||
card_defense = ((card_bytes >> 18) & 511) * 10
|
|
||||||
|
|
||||||
print(card_id , card_attack, card_attack, 'idx', card_index_name_data, card_index_desc_data)
|
|
||||||
#read until null found
|
|
||||||
end = card_index_name_data
|
|
||||||
while end + 1 < len(name_data):
|
|
||||||
## Read 2 bytes at a time until null found
|
## Read 2 bytes at a time until null found
|
||||||
if name_data[end:end+2] == b"\x00\x00":
|
if data[end:end+2] == b"\x00\x00":
|
||||||
break
|
break
|
||||||
end += 2
|
end += 2
|
||||||
name = name_data[card_index_name_data:end].decode("utf-16le", errors="replace")
|
return end
|
||||||
print(name)
|
|
||||||
#print(card_id.hex(' '))
|
|
||||||
# 2. Get last 14 bits using mask 0x3FFF (binary 11 1111 1111 1111)
|
|
||||||
#print([bin(x) for x in card_first])
|
|
||||||
test += 1
|
|
||||||
# if test > 3:
|
|
||||||
# break;
|
|
||||||
#print(prop_data[r:r+8])
|
|
||||||
|
|
||||||
|
# 8 bytes at a time
|
||||||
|
cd_db = {}
|
||||||
|
def get_props():
|
||||||
|
test = 1
|
||||||
|
for r in range(0,len(prop_data),8):
|
||||||
|
print(prop_data[r:r+8].hex(' ')) # print the byte range
|
||||||
|
card_prop = prop_data[r:r+8]
|
||||||
|
card_prop = card_prop[::-1]
|
||||||
|
#print(card_prop.hex(' '))
|
||||||
|
card_first = prop_data[r:r+4] #Get first 4 bytes in range
|
||||||
|
card_second = prop_data[r+4:r+8] #Get first 4 bytes in range
|
||||||
|
# offsets_data[4:8].hex()
|
||||||
|
card_index_name_data = int.from_bytes( offsets_data[r:r+4], byteorder='little' )
|
||||||
|
card_index_desc_data = int.from_bytes( offsets_data[r+4:r+8], byteorder='little' )
|
||||||
|
#card_id = card_id[::-1]
|
||||||
|
card_bytes = int.from_bytes(card_first,byteorder='little')
|
||||||
|
card_bytes_2 = int.from_bytes(card_second,byteorder='little')
|
||||||
|
card_id = card_bytes & 16383
|
||||||
|
if card_id == 0:
|
||||||
|
continue
|
||||||
|
card_attack = ((card_bytes >> 14) & 511) * 10
|
||||||
|
card_defense = ((card_bytes >> 18) & 511) * 10
|
||||||
|
|
||||||
|
print(card_id , card_attack, card_attack, 'idx', card_index_name_data, card_index_desc_data)
|
||||||
|
#read until null found
|
||||||
|
# end = card_index_name_data
|
||||||
|
# while end + 1 < len(name_data):
|
||||||
|
# ## Read 2 bytes at a time until null found
|
||||||
|
# if name_data[end:end+2] == b"\x00\x00":
|
||||||
|
# break
|
||||||
|
# end += 2
|
||||||
|
# name = name_data[card_index_name_data:end].decode("utf-16le", errors="replace")
|
||||||
|
end_index = read_until_null(card_index_name_data, name_data)
|
||||||
|
name = name_data[card_index_name_data:end_index].decode("utf-16le", errors="replace")
|
||||||
|
end_index = read_until_null(card_index_desc_data, desc_data)
|
||||||
|
desc = desc_data[card_index_desc_data:end_index].decode("utf-16le", errors="replace")
|
||||||
|
print(name)
|
||||||
|
#print(card_id.hex(' '))
|
||||||
|
# 2. Get last 14 bits using mask 0x3FFF (binary 11 1111 1111 1111)
|
||||||
|
#print([bin(x) for x in card_first])
|
||||||
|
test += 1
|
||||||
|
# if test > 3:
|
||||||
|
# break;
|
||||||
|
#print(prop_data[r:r+8])
|
||||||
|
if cd_db.get(card_id) is None:
|
||||||
|
cd_db[card_id] = {}
|
||||||
|
# {
|
||||||
|
# f"name_{language_code}":name,
|
||||||
|
# f"desc_{language_code}": 'foo',
|
||||||
|
# f"name_addr_{language_code}": hex(card_index_name_data),
|
||||||
|
# f"desc_addr_{language_code}": hex(card_index_desc_data)
|
||||||
|
# }
|
||||||
|
cd_db[card_id][f"name_{language_code}"] = name
|
||||||
|
cd_db[card_id][f"desc_{language_code}"] = desc
|
||||||
|
cd_db[card_id][f"name_addr_{language_code}"] = hex(card_index_name_data)
|
||||||
|
cd_db[card_id][f"desc_addr_{language_code}"] = hex(card_index_desc_data)
|
||||||
|
|
||||||
|
language_code = 'E'
|
||||||
|
offsets_data = Path(f"YGO_2020/bin/CARD_Indx_{language_code}.bin").read_bytes() # Get all the bytes
|
||||||
|
name_data = Path(f"YGO_2020/bin/CARD_Name_{language_code}.bin").read_bytes() # Get all the bytes
|
||||||
|
desc_data = Path(f"YGO_2020/bin/CARD_Desc_{language_code}.bin").read_bytes()
|
||||||
|
|
||||||
|
get_props()
|
||||||
|
|
||||||
|
language_code = 'J'
|
||||||
|
offsets_data = Path(f"YGO_2020/bin/CARD_Indx_{language_code}.bin").read_bytes() # Get all the bytes
|
||||||
|
name_data = Path(f"YGO_2020/bin/CARD_Name_{language_code}.bin").read_bytes() # Get all the bytes
|
||||||
|
desc_data = Path(f"YGO_2020/bin/CARD_Desc_{language_code}.bin").read_bytes()
|
||||||
|
|
||||||
|
get_props()
|
||||||
|
|
||||||
|
with open("dump2.json", "w", encoding="utf-16le") as outfile:
|
||||||
|
json.dump(cd_db, outfile, ensure_ascii=False, indent=2)
|
||||||
|
|
||||||
|
print()
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user