Dustin/jp_text_parsing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

better things

Browse Source
This commit is contained in:
Nitsud Yarg 2026-04-22 21:58:52 -07:00
parent dddc6a0b0e
commit 100a25b6d9
2 changed files with 117 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
add-readings.py
View File

@ -64,6 +64,7 @@ with open("words-readings.json", "r", encoding="utf-16le") as f:
with open("dump.json", "r", encoding="utf-16le") as f:
dump = json.load(f)
vocab_dict = {}
## A micab phrase based word parser
for cd in dump:
print(dump[cd]['nm'])
@ -77,42 +78,44 @@ for cd in dump:
if anottated['reading'] != dump[cd]['nm']:
print(anottated['reading'])
dump[cd]['kana'] = anottated['reading']
#dump[cd]['vocab'] = anottated['vocab']
print()
exit()
## Iterates through all the keys not values
for cd in dump:
word = ''
new_name = ''
for char in dump[cd]['nm'] + '・': ## hacky char add to prevent dupe logic
if char in all_kana:
if len(word) > 0:
#print(word)
if readings.get(word) is not None:
kana = None
try:
#print("<--", readings[word]['kana'])
kana = readings[word]['kana']
new_name += '(' + kana + ')'
#print()
except TypeError as e:
kana = None
#words[word] = (words[word] + 1)
cd_nm = dump[cd]['nm']
for word in anottated['vocab']:
# Skip punctuation words hopefully
if len(word) == 1 and has_kanji(word) == False:
continue
if vocab_dict.get(word) is None:
vocab_dict[word] = {
'count': 0,
'Examples': (cd_nm,)
}
else:
print(word, " <- Not Found")
word = ''
else:
word += char
new_name += char
if new_name[:-1]!= dump[cd]['nm']:
print (dump[cd]['nm'])
print (new_name[:-1])
dump[cd]['kana'] = new_name[:-1]
## Might need catch for end of string still
vocab_dict[word]['count'] = vocab_dict[word]['count'] + 1
if cd_nm not in vocab_dict[word]['Examples']:
vocab_dict[word]['Examples'] += (cd_nm,)
print()
phrase = dump[cd]['de']
anottated = annotate_phrase(phrase)
if anottated['reading'] != phrase:
#print(anottated['reading'])
dump[cd]['furi_de'] = anottated['reading']
for word in anottated['vocab']:
if len(word) == 1 and has_kanji(word) == False:
continue
if vocab_dict.get(word) is None:
vocab_dict[word] = {
'count': 0,
'Examples': (cd_nm,)
}
else:
vocab_dict[word]['count'] = vocab_dict[word]['count'] + 1
if len(vocab_dict[word]['Examples']) < 5:
if cd_nm not in vocab_dict[word]['Examples']:
vocab_dict[word]['Examples'] += (cd_nm,)
with open("dump-withkana.json", "w", encoding="utf-16le") as outfile:
vocab_dict = sorted(vocab_dict.items(), key= lambda x: x[1]['count'], reverse=True)
with open("dump-withkana2.json", "w", encoding="utf-16") as outfile:
json.dump(dump, outfile, ensure_ascii=False, indent=2)
with open("dump-vocab.json", "w", encoding="utf-16") as outfile:
json.dump(vocab_dict, outfile, ensure_ascii=False, indent=2)

65
get-properties.py
View File

@ -3,14 +3,22 @@ import struct
from pathlib import Path
prop_data = Path(f"YGO_2020/bin/CARD_Prop.bin").read_bytes() # Get all the bytes
language_code = 'E'
offsets_data = Path(f"YGO_2020/bin/CARD_Indx_{language_code}.bin").read_bytes() # Get all the bytes
name_data = Path(f"YGO_2020/bin/CARD_Name_{language_code}.bin").read_bytes() # Get all the bytes
print(prop_data[8]) # one byte supposedly
def read_until_null(indx: int, data: tuple) -> int:
end = indx
while end + 1 < len(data):
## Read 2 bytes at a time until null found
if data[end:end+2] == b"\x00\x00":
break
end += 2
return end
# 8 bytes at a time
test = 0
cd_db = {}
def get_props():
test = 1
for r in range(0,len(prop_data),8):
print(prop_data[r:r+8].hex(' ')) # print the byte range
card_prop = prop_data[r:r+8]
@ -25,18 +33,24 @@ for r in range(0,len(prop_data),8):
card_bytes = int.from_bytes(card_first,byteorder='little')
card_bytes_2 = int.from_bytes(card_second,byteorder='little')
card_id = card_bytes & 16383
if card_id == 0:
continue
card_attack = ((card_bytes >> 14) & 511) * 10
card_defense = ((card_bytes >> 18) & 511) * 10
print(card_id , card_attack, card_attack, 'idx', card_index_name_data, card_index_desc_data)
#read until null found
end = card_index_name_data
while end + 1 < len(name_data):
## Read 2 bytes at a time until null found
if name_data[end:end+2] == b"\x00\x00":
break
end += 2
name = name_data[card_index_name_data:end].decode("utf-16le", errors="replace")
# end = card_index_name_data
# while end + 1 < len(name_data):
# ## Read 2 bytes at a time until null found
# if name_data[end:end+2] == b"\x00\x00":
# break
# end += 2
# name = name_data[card_index_name_data:end].decode("utf-16le", errors="replace")
end_index = read_until_null(card_index_name_data, name_data)
name = name_data[card_index_name_data:end_index].decode("utf-16le", errors="replace")
end_index = read_until_null(card_index_desc_data, desc_data)
desc = desc_data[card_index_desc_data:end_index].decode("utf-16le", errors="replace")
print(name)
#print(card_id.hex(' '))
# 2. Get last 14 bits using mask 0x3FFF (binary 11 1111 1111 1111)
@ -45,5 +59,34 @@ for r in range(0,len(prop_data),8):
# if test > 3:
# break;
#print(prop_data[r:r+8])
if cd_db.get(card_id) is None:
cd_db[card_id] = {}
# {
# f"name_{language_code}":name,
# f"desc_{language_code}": 'foo',
# f"name_addr_{language_code}": hex(card_index_name_data),
# f"desc_addr_{language_code}": hex(card_index_desc_data)
# }
cd_db[card_id][f"name_{language_code}"] = name
cd_db[card_id][f"desc_{language_code}"] = desc
cd_db[card_id][f"name_addr_{language_code}"] = hex(card_index_name_data)
cd_db[card_id][f"desc_addr_{language_code}"] = hex(card_index_desc_data)
language_code = 'E'
offsets_data = Path(f"YGO_2020/bin/CARD_Indx_{language_code}.bin").read_bytes() # Get all the bytes
name_data = Path(f"YGO_2020/bin/CARD_Name_{language_code}.bin").read_bytes() # Get all the bytes
desc_data = Path(f"YGO_2020/bin/CARD_Desc_{language_code}.bin").read_bytes()
get_props()
language_code = 'J'
offsets_data = Path(f"YGO_2020/bin/CARD_Indx_{language_code}.bin").read_bytes() # Get all the bytes
name_data = Path(f"YGO_2020/bin/CARD_Name_{language_code}.bin").read_bytes() # Get all the bytes
desc_data = Path(f"YGO_2020/bin/CARD_Desc_{language_code}.bin").read_bytes()
get_props()
with open("dump2.json", "w", encoding="utf-16le") as outfile:
json.dump(cd_db, outfile, ensure_ascii=False, indent=2)
print()