finally set up a repo
This commit is contained in:
commit
138c1467a4
51
add-readings.py
Normal file
51
add-readings.py
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
import json
|
||||
|
||||
# Generate Hiragana and Katakana characters
|
||||
hiragana = tuple(chr(i) for i in range(12353, 12436)) # \u3041 - \u3094
|
||||
katakana = tuple(chr(i) for i in range(12450, 12532)) # \u30A2 - \u30F4
|
||||
punctuation = tuple(chr(i) for i in range(65281, 65382))
|
||||
|
||||
# Combine both into one tuple
|
||||
all_kana = hiragana + katakana + punctuation + (' ', 'ー', '・')
|
||||
|
||||
|
||||
with open("words-readings.json", "r", encoding="utf-16le") as f:
|
||||
readings = json.load(f)
|
||||
|
||||
with open("dump.json", "r", encoding="utf-16le") as f:
|
||||
dump = json.load(f)
|
||||
|
||||
## Iterates through all the keys not values
|
||||
for cd in dump:
|
||||
word = ''
|
||||
new_name = ''
|
||||
for char in dump[cd]['nm'] + '・': ## hacky char add to prevent dupe logic
|
||||
if char in all_kana:
|
||||
if len(word) > 0:
|
||||
#print(word)
|
||||
if readings.get(word) is not None:
|
||||
kana = None
|
||||
try:
|
||||
#print("<--", readings[word]['kana'])
|
||||
kana = readings[word]['kana']
|
||||
new_name += '(' + kana + ')'
|
||||
#print()
|
||||
except TypeError as e:
|
||||
kana = None
|
||||
#words[word] = (words[word] + 1)
|
||||
else:
|
||||
print(word, " <- Not Found")
|
||||
word = ''
|
||||
else:
|
||||
word += char
|
||||
new_name += char
|
||||
if new_name[:-1]!= dump[cd]['nm']:
|
||||
print (dump[cd]['nm'])
|
||||
print (new_name[:-1])
|
||||
dump[cd]['kana'] = new_name[:-1]
|
||||
## Might need catch for end of string still
|
||||
|
||||
print()
|
||||
|
||||
with open("dump-withkana.json", "w", encoding="utf-16le") as outfile:
|
||||
json.dump(dump, outfile, ensure_ascii=False, indent=2)
|
||||
156
extract_cards.py
Normal file
156
extract_cards.py
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
import mmap
|
||||
import struct
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
# Generate Hiragana and Katakana characters
|
||||
hiragana = tuple(chr(i) for i in range(12353, 12436)) # \u3041 - \u3094
|
||||
katakana = tuple(chr(i) for i in range(12450, 12532)) # \u30A2 - \u30F4
|
||||
|
||||
# Combine both into one tuple
|
||||
all_kana = hiragana + katakana
|
||||
|
||||
# Print example
|
||||
print(all_kana)
|
||||
|
||||
prop_data = Path(f"YGO_2020/bin/CARD_Prop.bin").read_bytes() # Get all the bytes
|
||||
|
||||
language_code = 'J'
|
||||
offsets_data = Path(f"YGO_2020/bin/CARD_Indx_{language_code}.bin").read_bytes() # Get all the bytes
|
||||
name_data = Path(f"YGO_2020/bin/CARD_Name_{language_code}.bin").read_bytes() # Get all the bytes
|
||||
desc_data = Path(f"YGO_2020/bin/CARD_Desc_{language_code}.bin").read_bytes()
|
||||
offsets = [x[0] for x in struct.iter_unpack("<I", offsets_data)] ## load all bytes into file.
|
||||
|
||||
|
||||
#fruits = ("apple", "banana", "cherry")
|
||||
#print("orange" not in fruits) # Output: True
|
||||
# if key not in my_dict:
|
||||
|
||||
char_db = {}
|
||||
words = {}
|
||||
cd_db = {}
|
||||
cd_db_ctr = 3900
|
||||
starting_offset = 2
|
||||
off = starting_offset
|
||||
## Read two bytes at a time
|
||||
#while off+2 < len(offsets_data): ## didnt work
|
||||
#while offsets[off+2] and off < 10:
|
||||
while off+2 < len(offsets):
|
||||
name_start_off = offsets[off]
|
||||
desc_start_off = offsets[off + 1]
|
||||
|
||||
#read until null found
|
||||
end = name_start_off
|
||||
while end + 1 < len(name_data):
|
||||
## Read 2 bytes at a time until null found
|
||||
if name_data[end:end+2] == b"\x00\x00":
|
||||
break
|
||||
end += 2
|
||||
name = name_data[name_start_off:end].decode("utf-16le", errors="replace")
|
||||
print(name)
|
||||
# for c in range(name_start_off, end, 2):
|
||||
# c_value = name_data[c:c+2]
|
||||
# if char_db.get(c_value) is None:
|
||||
# char_db[c_value] = 0
|
||||
# print(c_value, c_value.decode("utf-16le", errors="replace"))
|
||||
# char_db[c_value] = (char_db[c_value] + 1 )
|
||||
for char in name:
|
||||
if char_db.get(char) is None:
|
||||
char_db[char] = 0
|
||||
print(char)
|
||||
char_db[char] = (char_db[char] + 1 )
|
||||
|
||||
word = ''
|
||||
for char in name:
|
||||
if char in all_kana:
|
||||
if len(word) > 0:
|
||||
if words.get(word) is None:
|
||||
words[word] = 0
|
||||
print(word)
|
||||
words[word] = (words[word] + 1)
|
||||
word = ''
|
||||
else:
|
||||
word += char
|
||||
if len(word) > 0:
|
||||
if words.get(word) is None:
|
||||
words[word] = 0
|
||||
print(word)
|
||||
words[word] = (words[word] + 1)
|
||||
|
||||
|
||||
|
||||
end = desc_start_off
|
||||
while end + 1 < len(desc_data):
|
||||
## Read 2 bytes at a time until null found
|
||||
if desc_data[end:end+2] == b"\x00\x00":
|
||||
break
|
||||
end += 2
|
||||
desc = desc_data[desc_start_off:end].decode("utf-16le", errors="replace")
|
||||
|
||||
|
||||
#print(name)
|
||||
cd_db[cd_db_ctr] = {"nm":name, "de": desc, "nm_addr":hex(name_start_off), "de_addr":hex(desc_start_off)}
|
||||
cd_db_ctr += 1
|
||||
#print()
|
||||
## Iterate offset
|
||||
off += 2
|
||||
|
||||
language_code = 'E'
|
||||
offsets_data = Path(f"YGO_2020/bin/CARD_Indx_{language_code}.bin").read_bytes() # Get all the bytes
|
||||
name_data = Path(f"YGO_2020/bin/CARD_Name_{language_code}.bin").read_bytes() # Get all the bytes
|
||||
desc_data = Path(f"YGO_2020/bin/CARD_Desc_{language_code}.bin").read_bytes()
|
||||
offsets = [x[0] for x in struct.iter_unpack("<I", offsets_data)] ## load all bytes into file.
|
||||
|
||||
cd_db_ctr = 3900
|
||||
starting_offset = 2
|
||||
off = starting_offset
|
||||
## Read two bytes at a time
|
||||
#while off+2 < len(offsets_data): ## didnt work
|
||||
#while offsets[off+2] and off < 10:
|
||||
while off+2 < len(offsets):
|
||||
name_start_off = offsets[off]
|
||||
desc_start_off = offsets[off + 1]
|
||||
|
||||
#read until null found
|
||||
end = name_start_off
|
||||
while end + 1 < len(name_data):
|
||||
## Read 2 bytes at a time until null found
|
||||
if name_data[end:end+2] == b"\x00\x00":
|
||||
break
|
||||
end += 2
|
||||
name = name_data[name_start_off:end].decode("utf-16le", errors="replace")
|
||||
print(name)
|
||||
|
||||
end = desc_start_off
|
||||
while end + 1 < len(desc_data):
|
||||
## Read 2 bytes at a time until null found
|
||||
if desc_data[end:end+2] == b"\x00\x00":
|
||||
break
|
||||
end += 2
|
||||
desc = desc_data[desc_start_off:end].decode("utf-16le", errors="replace")
|
||||
|
||||
|
||||
#print(name)
|
||||
cd_db[cd_db_ctr]["nm_e"] = name
|
||||
cd_db[cd_db_ctr]["de_e"] = desc
|
||||
cd_db[cd_db_ctr]["nm_addr_e"] = hex(name_start_off)
|
||||
cd_db[cd_db_ctr]["de_addr_e"] = hex(desc_start_off)
|
||||
cd_db_ctr += 1
|
||||
#print()
|
||||
## Iterate offset
|
||||
off += 2
|
||||
|
||||
|
||||
# Writing to a file
|
||||
with open("dump.json", "w", encoding="utf-16le") as outfile:
|
||||
json.dump(cd_db, outfile, ensure_ascii=False, indent=2)
|
||||
|
||||
with open("words.json", "w", encoding="utf-16le") as outfile:
|
||||
json.dump(words, outfile, ensure_ascii=False, indent=2)
|
||||
|
||||
with open("char_db.json", "w", encoding="utf-16le") as outfile:
|
||||
json.dump(char_db, outfile, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
print("how do?")
|
||||
|
||||
49
get-properties.py
Normal file
49
get-properties.py
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
import mmap
|
||||
import struct
|
||||
from pathlib import Path
|
||||
|
||||
prop_data = Path(f"YGO_2020/bin/CARD_Prop.bin").read_bytes() # Get all the bytes
|
||||
language_code = 'E'
|
||||
offsets_data = Path(f"YGO_2020/bin/CARD_Indx_{language_code}.bin").read_bytes() # Get all the bytes
|
||||
name_data = Path(f"YGO_2020/bin/CARD_Name_{language_code}.bin").read_bytes() # Get all the bytes
|
||||
|
||||
print(prop_data[8]) # one byte supposedly
|
||||
|
||||
# 8 bytes at a time
|
||||
test = 0
|
||||
for r in range(0,len(prop_data),8):
|
||||
print(prop_data[r:r+8].hex(' ')) # print the byte range
|
||||
card_prop = prop_data[r:r+8]
|
||||
card_prop = card_prop[::-1]
|
||||
#print(card_prop.hex(' '))
|
||||
card_first = prop_data[r:r+4] #Get first 4 bytes in range
|
||||
card_second = prop_data[r+4:r+8] #Get first 4 bytes in range
|
||||
# offsets_data[4:8].hex()
|
||||
card_index_name_data = int.from_bytes( offsets_data[r:r+4], byteorder='little' )
|
||||
card_index_desc_data = int.from_bytes( offsets_data[r+4:r+8], byteorder='little' )
|
||||
#card_id = card_id[::-1]
|
||||
card_bytes = int.from_bytes(card_first,byteorder='little')
|
||||
card_bytes_2 = int.from_bytes(card_second,byteorder='little')
|
||||
card_id = card_bytes & 16383
|
||||
card_attack = ((card_bytes >> 14) & 511) * 10
|
||||
card_defense = ((card_bytes >> 18) & 511) * 10
|
||||
|
||||
print(card_id , card_attack, card_attack, 'idx', card_index_name_data, card_index_desc_data)
|
||||
#read until null found
|
||||
end = card_index_name_data
|
||||
while end + 1 < len(name_data):
|
||||
## Read 2 bytes at a time until null found
|
||||
if name_data[end:end+2] == b"\x00\x00":
|
||||
break
|
||||
end += 2
|
||||
name = name_data[card_index_name_data:end].decode("utf-16le", errors="replace")
|
||||
print(name)
|
||||
#print(card_id.hex(' '))
|
||||
# 2. Get last 14 bits using mask 0x3FFF (binary 11 1111 1111 1111)
|
||||
#print([bin(x) for x in card_first])
|
||||
test += 1
|
||||
# if test > 3:
|
||||
# break;
|
||||
#print(prop_data[r:r+8])
|
||||
|
||||
|
||||
33
get_readings.py
Normal file
33
get_readings.py
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
import json
|
||||
from janome.tokenizer import Tokenizer
|
||||
from kotobase import Kotobase
|
||||
|
||||
kb = Kotobase()
|
||||
tokenizer = Tokenizer()
|
||||
|
||||
parts = []
|
||||
for token in tokenizer.tokenize('魔草トークン'):
|
||||
print(token)
|
||||
print(token.reading)
|
||||
reading = token.reading
|
||||
if reading == "*":
|
||||
reading = token.surface
|
||||
parts.append(reading)
|
||||
|
||||
print(parts)
|
||||
|
||||
with open("words.json", "r", encoding="utf-16le") as f:
|
||||
words = json.load(f)
|
||||
|
||||
for word in words:
|
||||
result = kb.lookup(word)
|
||||
if result:
|
||||
if len(result.entries) > 0:
|
||||
kana = result.entries[0].kana[0]
|
||||
print (result.word, ": ", kana)
|
||||
words[word] = {"kana":kana,"count":words[word]}
|
||||
|
||||
with open("words-readings.json", "w", encoding="utf-16le") as outfile:
|
||||
json.dump(words, outfile, ensure_ascii=False, indent=2)
|
||||
|
||||
print()
|
||||
67
get_readings2.py
Normal file
67
get_readings2.py
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
import json
|
||||
from janome.tokenizer import Tokenizer
|
||||
from kotobase import Kotobase
|
||||
|
||||
kb = Kotobase()
|
||||
tokenizer = Tokenizer()
|
||||
|
||||
with open("words.json", "r", encoding="utf-16le") as f:
|
||||
words = json.load(f)
|
||||
|
||||
# Generate Hiragana and Katakana characters
|
||||
hiragana = tuple(chr(i) for i in range(12353, 12436)) # \u3041 - \u3094
|
||||
katakana = tuple(chr(i) for i in range(12450, 12532)) # \u30A2 - \u30F4
|
||||
punctuation = tuple(chr(i) for i in range(65281, 65382))
|
||||
|
||||
# Combine both into one tuple
|
||||
all_kana = hiragana + katakana + punctuation + (' ', 'ー', '・')
|
||||
|
||||
|
||||
with open("dump.json", "r", encoding="utf-16le") as f:
|
||||
dump = json.load(f)
|
||||
|
||||
readings = {}
|
||||
## Iterates through all the keys not values
|
||||
for cd in dump:
|
||||
word = ''
|
||||
for char in dump[cd]['nm'] + '・': ## hacky char add to prevent dupe logic
|
||||
if char in all_kana:
|
||||
if len(word) > 0:
|
||||
#print(word)
|
||||
if readings.get(word) is None:
|
||||
kana = None
|
||||
try:
|
||||
result = kb.lookup(word)
|
||||
if result:
|
||||
if len(result.entries) > 0:
|
||||
kana = result.entries[0].kana[0]
|
||||
print (result.word, ": ", kana)
|
||||
readings[word] = {"kana":kana}
|
||||
else:
|
||||
print("no entries")
|
||||
for k in result.kanji:
|
||||
if len(k.kunyomi) > 0:
|
||||
print(k.kunyomi[0])
|
||||
except TypeError as e:
|
||||
kana = None
|
||||
#words[word] = (words[word] + 1)
|
||||
word = ''
|
||||
else:
|
||||
word += char
|
||||
## Might need catch for end of string still
|
||||
|
||||
|
||||
print()
|
||||
|
||||
# for word in words:
|
||||
# result = kb.lookup(word)
|
||||
# if result:
|
||||
# if len(result.entries) > 0:
|
||||
# kana = result.entries[0].kana[0]
|
||||
# print (result.word, ": ", kana)
|
||||
# words[word] = {"kana":kana,"count":words[word]}
|
||||
|
||||
with open("words-readings.json", "w", encoding="utf-16le") as outfile:
|
||||
json.dump(readings, outfile, ensure_ascii=False, indent=2)
|
||||
|
||||
print()
|
||||
27
kana_assc.py
Normal file
27
kana_assc.py
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
|
||||
import mmap
|
||||
import struct
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
kana1_data = Path("YGO_2020/CARD_Kana1_J.bin").read_bytes()
|
||||
kana2_data = Path("YGO_2020/CARD_Kana2_J.bin").read_bytes()
|
||||
kana3_data = Path("YGO_2020/CARD_Kana3_J.bin").read_bytes()
|
||||
|
||||
## The offsets pulled from the file
|
||||
items1 = [x[0] for x in struct.iter_unpack("<I", kana1_data)]
|
||||
|
||||
#print(f"Foo {[f'0x{x:x}' for x in offsets[2:4]]}") # Getting range and convert to hex
|
||||
#print(", ".join("{:04x}".format(x) for x in items1[6:8]))
|
||||
print(", ".join("{:04x}".format(x) for x in kana1_data[6:8]))
|
||||
kana1_data[6:8].decode("utf-16le", errors="replace")
|
||||
|
||||
print()
|
||||
|
||||
|
||||
# kana1_data[2:4].decode("utf-16le", errors="replace")
|
||||
# 'い'
|
||||
# kana2_data[2:4].decode("utf-16le", errors="replace")
|
||||
# 'ん'
|
||||
# kana3_data[2:4].decode("utf-16le", errors="replace")
|
||||
# 'せ'
|
||||
82
parse_strings.py
Normal file
82
parse_strings.py
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
import mmap
|
||||
import struct
|
||||
from pathlib import Path
|
||||
|
||||
def iter_offsets(offset_file: str):
|
||||
skip_header = False
|
||||
with open(offset_file, "rb") as f:
|
||||
if skip_header == False:
|
||||
print(f.read(8))
|
||||
skip_header = True
|
||||
while chunk := f.read(4):
|
||||
if len(chunk) != 4:
|
||||
raise ValueError("Incomplete 4-byte offset at end of file")
|
||||
#yield struct.unpact("<I", chunk)[0]
|
||||
print(chunk)
|
||||
|
||||
# with open("YGO_2020/CARD_Name_E.bin", "rb") as tf:
|
||||
# with mmap.mmap(tf.fileno(), 0, access=mmap.ACCESS_READ) as mm:
|
||||
# for off in iter_offsets("YGO_2020\CARD_Indx_E.bin"):
|
||||
# print(off)
|
||||
|
||||
## Test iter_offsets
|
||||
#iter_offsets("YGO_2020/CARD_Indx_E.bin")
|
||||
|
||||
chr(65322) # int to char
|
||||
ord('Ｌ') # char to unicode val
|
||||
|
||||
|
||||
# offsets_data = Path("YGO_2020/CARD_Indx_E.bin").read_bytes()
|
||||
# text_data = Path("YGO_2020/CARD_Name_E.bin").read_bytes()
|
||||
|
||||
offsets_data = Path("YGO_2020/CARD_Indx_J.bin").read_bytes()
|
||||
text_data = Path("YGO_2020/CARD_Name_J.bin").read_bytes()
|
||||
|
||||
|
||||
offsets = [x[0] for x in struct.iter_unpack("<I", offsets_data)]
|
||||
## Getting ranges of offsets
|
||||
offsets[:2] # first 2 bytes
|
||||
offsets[4:8] # 8 bytes from the 4th
|
||||
|
||||
## Printing values for debugging
|
||||
print(f"Foo {[f'0x{x:x}' for x in offsets[2:4]]}") # Getting range and convert to hex
|
||||
print(", ".join("{:04x}".format(x) for x in offsets[6:8]))
|
||||
|
||||
# for off in offsets[:10]:
|
||||
# print(f"offset={off} hex{off:08X}")
|
||||
# print(text_data[off:off+32].hex(" "))
|
||||
# print()
|
||||
|
||||
cd_db = {}
|
||||
cd_db_ctr = 3900
|
||||
starting_offset = 2
|
||||
off = starting_offset
|
||||
## Read two bytes at a time
|
||||
#while off+2 < len(offsets_data): ## didnt work
|
||||
#while offsets[off+2] and off < 10:
|
||||
while off+2 < len(offsets):
|
||||
print("Offset Pair:", ", ".join("{:04x}".format(x) for x in offsets[off:off+2]))
|
||||
name_start_off = offsets[off]
|
||||
## Peek method
|
||||
# name_end_off = offsets[off+2] # peek to next offset
|
||||
# print((text_data[name_start_off:(name_end_off-1)]).hex(" "))
|
||||
# print((text_data[name_start_off:(name_end_off-1)]).decode("utf-16le", errors="replace"))
|
||||
# print()
|
||||
|
||||
#read until null found
|
||||
end = name_start_off
|
||||
while end + 1 < len(text_data):
|
||||
## Read 2 bytes at a time until null found
|
||||
if text_data[end:end+2] == b"\x00\x00":
|
||||
break
|
||||
end += 2
|
||||
text = text_data[name_start_off:end].decode("utf-16le", errors="replace")
|
||||
print(text)
|
||||
cd_db[cd_db_ctr] = {"nm":text, "addr":hex(name_start_off)}
|
||||
cd_db_ctr += 1
|
||||
print()
|
||||
## Iterate offset
|
||||
off += 2
|
||||
|
||||
print("how do?")
|
||||
|
||||
Loading…
Reference in New Issue
Block a user