Compare commits
No commits in common. "c14eb1b4c402238389d6e99154c2b085a92f453a" and "abaf271bf26f95f38d172c3577acad32c74dae61" have entirely different histories.
c14eb1b4c4
...
abaf271bf2
|
|
@ -1,28 +0,0 @@
|
|||
from fugashi_parser import *
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import hashlib
|
||||
|
||||
vocab_destination_path = Path('/run/media/deck/YF8SD/Sync/Notebooks/Obsidian/Japanese/vocab')
|
||||
|
||||
pos_list = list()
|
||||
for file_path in vocab_destination_path.iterdir():
|
||||
if file_path.is_file(): # Ensure it's a file, not a subdirectory
|
||||
pos = file_path.stem.split('_')[1]
|
||||
pos_list.append(pos)
|
||||
if CONTENT_LINK_TAG.get(pos):
|
||||
#do a thing
|
||||
with open(file_path, "a") as file:
|
||||
#file.write("This text will be added to the end.")
|
||||
print(f"#{CONTENT_LINK_TAG[pos]}", file=file)
|
||||
else:
|
||||
print(pos, ' - no tag')
|
||||
|
||||
# with file_path.open('r') as file:
|
||||
# content = file.read()
|
||||
# print(f"Content of {file_path.name}: {content}")
|
||||
|
||||
unique_values = list(set(pos_list))
|
||||
print(unique_values)
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
from fugashi_parser import *
|
||||
from parser import *
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
|
@ -21,23 +21,18 @@ if not destination_path.joinpath('vocab').exists():
|
|||
with open("dump-withFuri.json", "r", encoding="utf-16") as f:
|
||||
dump = json.load(f)
|
||||
|
||||
vocab = {} #load_vocab('foo.json')
|
||||
|
||||
for cd in dump:
|
||||
## The japanese card name
|
||||
print(dump[cd]['name_J'])
|
||||
## copy the image if it isn't already there
|
||||
if Path('YGO_2020/2020.full.illust_j.jpg.zib/' + cd + '.jpg').exists():
|
||||
##TODO: Ew
|
||||
if Path(destination_path.joinpath('_attachments/' + cd + '.jpg')).exists() == False:
|
||||
shutil.copy2(Path('YGO_2020/2020.full.illust_j.jpg.zib/' + cd + '.jpg'), destination_path.joinpath('_attachments'))
|
||||
with open(destination_path.as_posix() + '/' + cd + '.md', 'w') as card_note:
|
||||
## add card names as properties
|
||||
card_note.write("---\n")
|
||||
card_note.write(f"aliases: [{dump[cd]['name_J']},{dump[cd]['name_E']}]\n")
|
||||
card_note.write("---\n")
|
||||
## Name at the top
|
||||
card_note.write(f"![[{cd}.jpg]]") ## card image
|
||||
card_note.write(f"![[{cd}.jpg]]")
|
||||
card_note.write("\n#### Name\n")
|
||||
card_note.write("<details><summary>")
|
||||
card_note.write(dump[cd]['name_J'])
|
||||
|
|
@ -53,70 +48,42 @@ for cd in dump:
|
|||
card_note.write("</details>")
|
||||
# desc_Furi
|
||||
card_note.write("\n#### Reading\n")
|
||||
vocab = {} ## flush the vocab variable each time
|
||||
result = analyze(dump[cd]['name_J'],vocab)
|
||||
print(f"<span>{result['ruby_html']}</span>\n> [!summary]- Vocab\n>{result['notes_link']}\n",file=card_note)
|
||||
## Generate the vocab item
|
||||
for word in vocab:
|
||||
if Path(destination_path.joinpath('vocab/' + word + '.md')).exists() == False:
|
||||
definitions = anki_fields_term(word)
|
||||
if(dump[cd].get("furi")):
|
||||
card_note.write(f"{dump[cd]['furi']}\n")
|
||||
else:
|
||||
card_note.write(dump[cd]['name_J'])
|
||||
card_note.write("\n")
|
||||
card_note.write("##### Glossary\n")
|
||||
for vocab in dump[cd]["glossary_Name_J"]:
|
||||
print(f"{vocab} : {dump[cd]["glossary_Name_J"][vocab]}", file=card_note)
|
||||
card_note.write("\n")
|
||||
if(dump[cd].get("desc_Furi")):
|
||||
card_note.write(f"{dump[cd]['desc_Furi']}\n")
|
||||
else:
|
||||
card_note.write(dump[cd]['desc_J'])
|
||||
card_note.write("\n")
|
||||
card_note.write("##### Glossary\n")
|
||||
for vocab in dump[cd]["glossary_Desc_J"]:
|
||||
vocab_hash = hashlib.shake_256((vocab + 'Jitendex.org [2026-04-04]').encode()).hexdigest(4)
|
||||
## Only search and create a vocab file if it doesn't exist
|
||||
if Path(destination_path.joinpath('vocab/' + vocab_hash + '.md')).exists() == False:
|
||||
definitions = anki_fields_term(vocab)
|
||||
if definitions.get("fields"):
|
||||
with open(destination_path.as_posix() + '/vocab/' + word + '.md', 'w') as vocab_note:
|
||||
with open(destination_path.as_posix() + '/vocab/' + vocab_hash + '.md', 'w') as vocab_note:
|
||||
for entry in definitions.get("fields"):
|
||||
print(entry['furigana'], file=vocab_note)
|
||||
vocab_note.write("\n")
|
||||
print(entry['expression'], file=vocab_note)
|
||||
#print(entry['furigana-plain'], file=vocab_note)
|
||||
if entry['expression'] != entry['reading']:
|
||||
print(entry['reading'], file=vocab_note)
|
||||
print("> ",re.sub(r'<.*?>', '', entry['glossary-plain-no-dictionary']), "\n", file=vocab_note)
|
||||
print(f"Frequency: {entry['frequency-average-rank']}", file=vocab_note)
|
||||
vocab_note.write(entry['glossary-first'])
|
||||
print("\n", file=vocab_note)
|
||||
# card_note.write("##### Glossary\n")
|
||||
# for vocab in dump[cd]["glossary_Name_J"]:
|
||||
# print(f"{vocab} : {dump[cd]["glossary_Name_J"][vocab]}", file=card_note)
|
||||
vocab = {} ## flush the vocab variable each time
|
||||
result = analyze(dump[cd]['desc_J'],vocab)
|
||||
#card_note.write("\n")
|
||||
print(f"<span>{result['ruby_html']}</span>\n> [!summary]- Vocab\n>{result['notes_link']}\n",file=card_note)
|
||||
for word in vocab:
|
||||
if Path(destination_path.joinpath('vocab/' + word + '.md')).exists() == False:
|
||||
definitions = anki_fields_term(word)
|
||||
if definitions.get("fields"):
|
||||
with open(destination_path.as_posix() + '/vocab/' + word + '.md', 'w') as vocab_note:
|
||||
for entry in definitions.get("fields"):
|
||||
print(entry['furigana'], file=vocab_note)
|
||||
if entry['expression'] != entry['reading']:
|
||||
print(entry['reading'], file=vocab_note)
|
||||
print("> ",re.sub(r'<.*?>', '', entry['glossary-plain-no-dictionary']), "\n", file=vocab_note)
|
||||
#print(f"Part of Speech: {entry['part-of-speech']}", file=vocab_note)
|
||||
print(f"Frequency: {entry['frequency-average-rank']}", file=vocab_note)
|
||||
vocab_note.write(entry['glossary-first'])
|
||||
print("\n", file=vocab_note)
|
||||
# if(dump[cd].get("desc_Furi")):
|
||||
# card_note.write(f"{dump[cd]['desc_Furi']}\n")
|
||||
# else:
|
||||
# card_note.write(dump[cd]['desc_J'])
|
||||
# card_note.write("\n")
|
||||
# card_note.write("##### Glossary\n")
|
||||
# for vocab in dump[cd]["glossary_Desc_J"]:
|
||||
# vocab_hash = hashlib.shake_256((vocab + 'Jitendex.org [2026-04-04]').encode()).hexdigest(4)
|
||||
# ## Only search and create a vocab file if it doesn't exist
|
||||
# if Path(destination_path.joinpath('vocab/' + vocab_hash + '.md')).exists() == False:
|
||||
# definitions = anki_fields_term(vocab)
|
||||
# if definitions.get("fields"):
|
||||
# with open(destination_path.as_posix() + '/vocab/' + vocab_hash + '.md', 'w') as vocab_note:
|
||||
# for entry in definitions.get("fields"):
|
||||
# vocab_note.write("\n")
|
||||
# print(entry['expression'], file=vocab_note)
|
||||
# #print(entry['furigana-plain'], file=vocab_note)
|
||||
# if entry['expression'] != entry['reading']:
|
||||
# print(entry['reading'], file=vocab_note)
|
||||
# print(entry['furigana'], file=vocab_note)
|
||||
# #print(f"Part of Speech: {entry['part-of-speech']}", file=vocab_note)
|
||||
# print(f"Frequency: {entry['frequency-average-rank']}", file=vocab_note)
|
||||
# #print(entry['glossary-plain-no-dictionary'], file=vocab_note)
|
||||
# vocab_note.write(entry['glossary-first'])
|
||||
# print("##### Other Definitions", file=vocab_note)
|
||||
#print(entry['glossary-plain-no-dictionary'], file=vocab_note)
|
||||
# print(f"[[{vocab_hash}|{vocab}]] : {dump[cd]["glossary_Desc_J"][vocab]}", file=card_note)
|
||||
vocab_note.write(entry['glossary-first'])
|
||||
print("##### Other Definitions", file=vocab_note)
|
||||
print(entry['glossary-plain-no-dictionary'], file=vocab_note)
|
||||
print(f"[[{vocab_hash}|{vocab}]] : {dump[cd]["glossary_Desc_J"][vocab]}", file=card_note)
|
||||
#print('>' + dump[cd]["glossary_Desc_J"][vocab], file=card_note)
|
||||
card_note.write("\n")
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ import argparse
|
|||
import html
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass, asdict
|
||||
|
|
@ -31,24 +30,7 @@ KATAKANA_START = ord("ァ")
|
|||
KATAKANA_END = ord("ヶ")
|
||||
KATAKANA_TO_HIRAGANA_OFFSET = ord("ぁ") - ord("ァ")
|
||||
|
||||
|
||||
CONTENT_POS1 = {"名詞", "動詞", "形容詞", "形状詞", "副詞", "代名詞"}
|
||||
|
||||
def is_vocab_token(pos, keep_pronouns=False, keep_numbers=False):
|
||||
pos1, pos2, pos3, pos4 = pos
|
||||
|
||||
if pos1 not in CONTENT_POS1:
|
||||
return False
|
||||
|
||||
if pos1 == "代名詞" and not keep_pronouns:
|
||||
return False
|
||||
|
||||
if pos1 == "名詞" and pos2 == "数詞" and not keep_numbers:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
KANJI_RE = re.compile(r"[\u4E00-\u9FFF]")
|
||||
|
||||
def katakana_to_hiragana(text: str) -> str:
|
||||
return "".join(
|
||||
|
|
@ -56,19 +38,9 @@ def katakana_to_hiragana(text: str) -> str:
|
|||
for ch in text
|
||||
)
|
||||
|
||||
KANJI_RE = re.compile(r"[\u4E00-\u9FFF]")
|
||||
def has_kanji(text: str) -> bool:
|
||||
return bool(KANJI_RE.search(text))
|
||||
|
||||
# def has_kanji(text):
|
||||
# return any(
|
||||
# 0x3400 <= ord(ch) <= 0x4DBF or
|
||||
# 0x4E00 <= ord(ch) <= 0x9FFF or
|
||||
# 0xF900 <= ord(ch) <= 0xFAFF
|
||||
# for ch in text
|
||||
# )
|
||||
|
||||
|
||||
def has_japanese(text: str) -> bool:
|
||||
return any(
|
||||
0x3040 <= ord(ch) <= 0x30FF or
|
||||
|
|
@ -98,16 +70,6 @@ def term_entries(term :str) -> dict:
|
|||
response = requests.post(request_url + "/termEntries", json = params, timeout = request_timeout)
|
||||
return json.loads(response.text)
|
||||
|
||||
CONTENT_LINK_TAG = {
|
||||
'名詞': 'meishi', # noun
|
||||
'動詞': 'doshi', # verb
|
||||
'形容詞': 'keiyoshi', # Adjective
|
||||
'形状詞': 'keijoshi', # form word
|
||||
'代名詞':'daimeishi', # pronoun
|
||||
'感動詞': 'kandoshi', # interjection
|
||||
'副詞' : 'fukushi' # adverb
|
||||
#'副詞' : 'fukushi' #adverb
|
||||
}
|
||||
|
||||
def is_content_word(pos1: str) -> bool:
|
||||
return pos1 in {"名詞", "動詞", "形容詞", "副詞"}
|
||||
|
|
@ -124,17 +86,6 @@ def clean_lemma(lemma: str) -> str:
|
|||
def get_feature(feature, name: str, default=""):
|
||||
return getattr(feature, name, default) or default
|
||||
|
||||
def should_ruby(surface, reading_hiragana):
|
||||
if not reading_hiragana:
|
||||
return False
|
||||
|
||||
if surface == reading_hiragana:
|
||||
return False
|
||||
|
||||
if not has_kanji(surface):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def build_ruby(surface: str, reading: str) -> str:
|
||||
if not reading or surface == reading or not has_japanese(surface):
|
||||
|
|
@ -146,105 +97,13 @@ def build_kana(surface: str, reading: str) -> str:
|
|||
return surface
|
||||
return f"「{reading}」"
|
||||
|
||||
def build_vocab_link(surface: str, reading: str, pos) -> str:
|
||||
pos1, pos2, pos3, pos4 = pos
|
||||
if not is_vocab_token(pos, keep_pronouns=True,keep_numbers=False):
|
||||
def build_vocab_link(surface: str, reading: str, pos1: str) -> str:
|
||||
if should_skip(surface, pos1) or not is_content_word(pos1):
|
||||
return surface
|
||||
# if surface == reading:
|
||||
# return f"[[{surface}]]"
|
||||
return f"[[{reading}_{pos1}|{surface}]]"
|
||||
if surface == reading:
|
||||
return f"[[{surface}]]"
|
||||
return f"[[{reading}|{surface}]]"
|
||||
|
||||
## Predicate filtering Verbs
|
||||
def is_predicate_start(pos):
|
||||
pos1, pos2, pos3, pos4 = pos
|
||||
|
||||
return (
|
||||
pos1 == "動詞" and pos2 == "一般"
|
||||
) or (
|
||||
pos1 == "形容詞"
|
||||
) or (
|
||||
pos1 == "形状詞"
|
||||
)
|
||||
|
||||
def attaches_to_predicate(pos):
|
||||
pos1, pos2, pos3, pos4 = pos
|
||||
|
||||
# ます, た, ない, れる, られる, たい, etc.
|
||||
if pos1 == "助動詞":
|
||||
return True
|
||||
|
||||
# helper verbs like いる, ある, しまう in constructions
|
||||
if pos1 == "動詞" and pos2 == "非自立可能":
|
||||
return True
|
||||
|
||||
# te-form connector in 読んでいる, 食べている
|
||||
if pos1 == "助詞" and pos2 == "接続助詞":
|
||||
return True
|
||||
|
||||
# suffixes attached to predicates
|
||||
if pos1 == "接尾辞":
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def chunk_predicates(tokens):
|
||||
"""
|
||||
tokens should be a list of TokenReading objects:
|
||||
token.surface
|
||||
token.lemma
|
||||
token.pos
|
||||
"""
|
||||
chunks = []
|
||||
i = 0
|
||||
|
||||
while i < len(tokens):
|
||||
token = tokens[i]
|
||||
|
||||
if not is_predicate_start(token.pos):
|
||||
i += 1
|
||||
continue
|
||||
|
||||
chunk = [token]
|
||||
j = i + 1
|
||||
|
||||
while j < len(tokens) and attaches_to_predicate(tokens[j].pos):
|
||||
chunk.append(tokens[j])
|
||||
j += 1
|
||||
|
||||
chunks.append({
|
||||
"surface": "".join(t.surface for t in chunk),
|
||||
"head_lemma": token.lemma,
|
||||
"head_reading_hiragana": token.lemma_reading_hiragana,
|
||||
"pos": token.pos,
|
||||
"parts": [
|
||||
{
|
||||
"surface": t.surface,
|
||||
"lemma": t.lemma,
|
||||
"reading_hiragana": t.reading_hiragana,
|
||||
"pos": t.pos,
|
||||
}
|
||||
for t in chunk
|
||||
],
|
||||
})
|
||||
|
||||
i = j
|
||||
|
||||
return chunks
|
||||
|
||||
## Vocab Filtering
|
||||
def is_vocab_token(pos, keep_pronouns=False, keep_numbers=False):
|
||||
pos1, pos2, pos3, pos4 = pos
|
||||
|
||||
if pos1 not in {"名詞", "動詞", "形容詞", "形状詞", "副詞", "代名詞", "感動詞"}:
|
||||
return False
|
||||
|
||||
if pos1 == "代名詞" and not keep_pronouns:
|
||||
return False
|
||||
|
||||
if pos1 == "名詞" and pos2 == "数詞" and not keep_numbers:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
@dataclass
|
||||
class TokenReading:
|
||||
|
|
@ -297,9 +156,7 @@ def update_vocab(vocab: Dict[str, Dict], tokens: List[TokenReading]):
|
|||
for t in tokens:
|
||||
pos1 = t.pos[0] if t.pos else ""
|
||||
|
||||
# if should_skip(t.surface, pos1) or not is_content_word(pos1):
|
||||
# continue
|
||||
if not is_vocab_token(t.pos, keep_pronouns=True,keep_numbers=False):
|
||||
if should_skip(t.surface, pos1) or not is_content_word(pos1):
|
||||
continue
|
||||
|
||||
entry = vocab.get(t.lemma)
|
||||
|
|
@ -349,7 +206,7 @@ def analyze(text: str, vocab: Dict[str, Dict]):
|
|||
reading = ""
|
||||
|
||||
lemma = clean_lemma(get_feature(f, "lemma") or surface)
|
||||
vocab_link_parts.append(build_vocab_link(surface, lemma, pos))
|
||||
vocab_link_parts.append(build_vocab_link(surface, lemma, pos[0]))
|
||||
|
||||
lform = get_feature(f, "lForm")
|
||||
kana_base = get_feature(f, "kanaBase")
|
||||
|
|
@ -360,7 +217,6 @@ def analyze(text: str, vocab: Dict[str, Dict]):
|
|||
|
||||
ruby_parts.append(build_ruby(surface, reading))
|
||||
kana_parts.append(build_kana(surface, reading))
|
||||
predicate_chunks = chunk_predicates(tokens)
|
||||
|
||||
update_vocab(vocab, tokens)
|
||||
|
||||
|
|
@ -370,61 +226,4 @@ def analyze(text: str, vocab: Dict[str, Dict]):
|
|||
"ruby_html": "".join(ruby_parts),
|
||||
"notes_link":"".join(vocab_link_parts),
|
||||
"token_readings": [asdict(t) for t in tokens],
|
||||
"predicate_chunks" : predicate_chunks,
|
||||
}
|
||||
|
||||
YOMITAN_TO_MICAB_POS = {
|
||||
'interjection': 'foo',
|
||||
'noun': 'meishi',
|
||||
'5-dan': '',
|
||||
'1-dan': '',
|
||||
'prefix': '',
|
||||
'intransitive': '',
|
||||
'aux-verb': '',
|
||||
'suru': '',
|
||||
'transitive': '',
|
||||
'counter': '',
|
||||
'na-adj': '',
|
||||
'exp': '',
|
||||
'adjective': '',
|
||||
'suffix': '',
|
||||
'conjunction': '',
|
||||
'interjection': '',
|
||||
'adverb': '',
|
||||
'pronoun': '',
|
||||
'unclass': '',
|
||||
}
|
||||
|
||||
def write_word_note(word: str, destination_dir: Path, part_of_speech:str='', overwrite:bool=False):
|
||||
word_file_name = word
|
||||
# Not ready for different file names yet
|
||||
if part_of_speech != '':
|
||||
word_file_name = word + '_' + part_of_speech
|
||||
if (Path(destination_dir.joinpath(word_file_name + '.md')).exists() == False) or overwrite == True:
|
||||
definitions = anki_fields_term(word)
|
||||
if definitions.get("fields"):
|
||||
with open(destination_dir.joinpath(word_file_name + '.md'), 'w') as vocab_note:
|
||||
for entry in definitions.get("fields"):
|
||||
#pos = entry['part-of-speech'] ## POS is always 'Unknown' for some reason
|
||||
pos = re.match(r'.*<span.*?"part-of-speech-info".*?>(.*?)<',entry['glossary-first'])
|
||||
if pos:
|
||||
pos = pos.group(1)
|
||||
# if part_of_speech != '' and pos:
|
||||
# print(pos)
|
||||
#vocab_note.write("\n")
|
||||
#print(entry['expression'], file=vocab_note)
|
||||
#print(entry['furigana-plain'], file=vocab_note)
|
||||
print(entry['furigana'], file=vocab_note)
|
||||
if entry['expression'] != entry['reading']:
|
||||
print(entry['reading'], file=vocab_note)
|
||||
print("> ",re.sub(r'<.*?>', '', entry['glossary-plain-no-dictionary']), "\n", file=vocab_note)
|
||||
#print(f"Part of Speech: {entry['part-of-speech']}", file=vocab_note)
|
||||
print(f"Frequency: {entry['frequency-average-rank']}", file=vocab_note)
|
||||
#print(entry['glossary-plain-no-dictionary'], file=vocab_note)
|
||||
vocab_note.write(entry['glossary-first'])
|
||||
#print("##### Other Definitions", file=vocab_note)
|
||||
print("\n", file=vocab_note)
|
||||
if CONTENT_LINK_TAG.get(part_of_speech, '') != '':
|
||||
print(f"#{CONTENT_LINK_TAG[part_of_speech]}", file=vocab_note)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,283 +0,0 @@
|
|||
import mmap
|
||||
import struct
|
||||
from pathlib import Path
|
||||
import json
|
||||
|
||||
def read_until_null(indx: int, data: tuple) -> int:
|
||||
end = indx
|
||||
while end + 1 < len(data):
|
||||
## Read 2 bytes at a time until null found
|
||||
if data[end:end+2] == b"\x00\x00":
|
||||
break
|
||||
end += 2
|
||||
return end
|
||||
|
||||
def read_until_space(indx: int, data: tuple) -> int:
|
||||
end = indx
|
||||
while end + 1 < len(data):
|
||||
## Read 2 bytes at a time until null found
|
||||
d = data[end:end+1]
|
||||
if data[end:end+1] == b"\x00":
|
||||
break
|
||||
end += 1
|
||||
return end
|
||||
|
||||
|
||||
# 8 bytes at a time
|
||||
cd_db = {}
|
||||
def get_props():
|
||||
test = 0
|
||||
for r in range(0,len(offsets_data),8):
|
||||
test += 1
|
||||
# offsets_data[4:8].hex()
|
||||
card_index_name_data = int.from_bytes( offsets_data[r:r+4], byteorder='little', signed=False )
|
||||
if card_index_name_data == 0:
|
||||
continue
|
||||
#card_index_desc_data = int.from_bytes( offsets_data[r+4:r+8], byteorder='little' )
|
||||
#card_id = card_id[::-1]
|
||||
#read until null found
|
||||
# end = card_index_name_data
|
||||
# while end + 1 < len(name_data):
|
||||
# ## Read 2 bytes at a time until null found
|
||||
# if name_data[end:end+2] == b"\x00\x00":
|
||||
# break
|
||||
# end += 2
|
||||
# name = name_data[card_index_name_data:end].decode("utf-16le", errors="replace")
|
||||
end_index = read_until_null(card_index_name_data, name_data)
|
||||
name = name_data[card_index_name_data:end_index].decode("utf-16le", errors="replace")
|
||||
print(name)
|
||||
if name == '':
|
||||
continue
|
||||
#print(card_id.hex(' '))
|
||||
# 2. Get last 14 bits using mask 0x3FFF (binary 11 1111 1111 1111)
|
||||
#print([bin(x) for x in card_first])
|
||||
# if test > 3:
|
||||
# break;
|
||||
#print(prop_data[r:r+8])
|
||||
if cd_db.get(test) is None:
|
||||
cd_db[test] = {}
|
||||
# {
|
||||
# f"name_{language_code}":name,
|
||||
# f"desc_{language_code}": 'foo',
|
||||
# f"name_addr_{language_code}": hex(card_index_name_data),
|
||||
# f"desc_addr_{language_code}": hex(card_index_desc_data)
|
||||
# }
|
||||
cd_db[test][f"name_{language_code}"] = name
|
||||
#cd_db[card_id][f"desc_{language_code}"] = desc
|
||||
cd_db[test][f"name_addr_{language_code}"] = hex(card_index_name_data)
|
||||
#cd_db[card_id][f"desc_addr_{language_code}"] = hex(card_index_desc_data)
|
||||
|
||||
language_code = 'E'
|
||||
#offsets_data = Path(f"YGO_2020/bin/DLG_Indx_{language_code}.bin").read_bytes() # Get all the bytes
|
||||
e_name_data = Path(f"YGO_2020/main/howto_db/howtoplay_{language_code}.bin").read_bytes() # Get all the bytes
|
||||
#get_props()
|
||||
language_code = 'J'
|
||||
j_name_data = Path(f"YGO_2020/main/howto_db/howtoplay_{language_code}.bin").read_bytes() # Get all the bytes
|
||||
|
||||
|
||||
scripts = {}
|
||||
|
||||
def getScriptData():
|
||||
offset = struct.unpack("<Q", script_data[0:8])[0] # 11914
|
||||
titlecount = struct.unpack("<I", script_data[8:12])[0] # 307
|
||||
scriptblockcount = struct.unpack("<I", script_data[12:16])[0] # 7340
|
||||
|
||||
titleStartPos = list()
|
||||
titleEndPos = list()
|
||||
titleOffset = list()
|
||||
for p in range(titlecount):
|
||||
i = p * 16
|
||||
titleStartPos.append(struct.unpack("<I", script_data[i:i+4])[0])
|
||||
titleEndPos.append(struct.unpack("<I", script_data[i+4:i+8])[0])
|
||||
titleOffset.append(struct.unpack("<Q", script_data[i+8:i+16])[0])
|
||||
## 307 items? Is that good
|
||||
|
||||
script_titles = list()
|
||||
for o in range(len(titleOffset)):
|
||||
start = titleOffset[o]
|
||||
end = read_until_space(start,script_data)
|
||||
script_titles.append(script_data[start:end].decode("utf-8"))
|
||||
print( script_data[start:end].decode("utf-8") )
|
||||
|
||||
## Script Data extraction!
|
||||
for p in range(titlecount):
|
||||
#scripts[script_titles[p]] = {"title":script_titles[p], "blocks" : list()}
|
||||
if script_titles[p] == '':
|
||||
continue
|
||||
if scripts.get(script_titles[p]) == None:
|
||||
scripts[script_titles[p]] = {}
|
||||
script_block = {}
|
||||
blockCount = titleEndPos[p] - titleStartPos[p] #+1
|
||||
#start = offset + titleOffset[p] + titleStartPos[p]
|
||||
for j in range(blockCount):
|
||||
seeker = offset + titleStartPos[p] * 32 + j * 32
|
||||
start_char = struct.unpack("<Q", script_data[seeker:seeker+8])[0]
|
||||
end = read_until_space(start_char, script_data)
|
||||
script_block['character'] = script_data[start_char:end].decode('utf-8')
|
||||
#index = f"{j}_{language_code}_{script_block['character']}"
|
||||
index = f"{j}_{script_block['character']}"
|
||||
|
||||
seeker = seeker + 8
|
||||
start_arg1 = struct.unpack("<Q", script_data[seeker:seeker+8])[0]
|
||||
end = read_until_space(start_arg1, script_data)
|
||||
script_block['arg1'] = script_data[start_arg1:end].decode('utf-8')
|
||||
|
||||
seeker = seeker + 8
|
||||
start_arg2 = struct.unpack("<Q", script_data[seeker:seeker+8])[0]
|
||||
end = read_until_space(start_arg2, script_data)
|
||||
script_block['arg2'] = script_data[start_arg2:end].decode('utf-8')
|
||||
|
||||
seeker = seeker + 8
|
||||
start_text = struct.unpack("<Q", script_data[seeker:seeker+8])[0]
|
||||
end = read_until_space(start_text, script_data) ## These strings seem to always start with /0x00
|
||||
script_block['text'] = script_data[start_text:end].decode('utf-8')
|
||||
|
||||
if script_block['text'] != '':
|
||||
if scripts[script_titles[p]].get(index) == None:
|
||||
scripts[script_titles[p]][index] = {}
|
||||
scripts[script_titles[p]][index]['index'] = j
|
||||
scripts[script_titles[p]][index]['arg1'] = script_block['arg1']
|
||||
scripts[script_titles[p]][index]['arg2'] = script_block['arg2']
|
||||
#scripts[script_titles[p]][index]['text'] = script_block['text']
|
||||
scripts[script_titles[p]][index][f"text_{language_code}"] = script_block['text']
|
||||
|
||||
#scripts[p]['blocks'].append(script_block.copy())
|
||||
print(script_block)
|
||||
|
||||
language_code = 'J'
|
||||
script_data = Path(f"YGO_2020/main/scriptdata_{language_code}.bin").read_bytes() # Get all the bytes
|
||||
getScriptData()
|
||||
language_code = 'E'
|
||||
script_data = Path(f"YGO_2020/main/scriptdata_{language_code}.bin").read_bytes() # Get all the bytes
|
||||
getScriptData()
|
||||
|
||||
print('again?')
|
||||
#scripts = sorted(scripts.items(), key= lambda x: x[1]['index'], reverse=False)
|
||||
|
||||
# for key in scripts:
|
||||
# scripts[key] = sorted(scripts[key].items(), key = lambda x: x[1]['index'], reverse=False)
|
||||
|
||||
with open(f"scripts_dump2.json", "w", encoding="utf-8") as outfile:
|
||||
json.dump(scripts, outfile, ensure_ascii=False, indent=2)
|
||||
exit()
|
||||
|
||||
print("now what?")
|
||||
|
||||
count_of_items = int.from_bytes(e_name_data[0:4], byteorder='big') # 202
|
||||
## 202 * 8 = 1616 (Thats about where this started)
|
||||
|
||||
|
||||
help_dump = {}
|
||||
i_e = 1624
|
||||
i_j = 1624
|
||||
c = 1
|
||||
while (i_e < len(e_name_data) and i_j < len(j_name_data)):
|
||||
help_dump[c] = {}
|
||||
e = read_until_null(i_e,e_name_data)
|
||||
help_dump[c]['en'] = e_name_data[i_e:e].decode("utf-16be", errors="replace")
|
||||
print(help_dump[c]['en'] )
|
||||
i_e = e + 2
|
||||
e = read_until_null(i_j,j_name_data)
|
||||
help_dump[c]['ja'] = j_name_data[i_j:e].decode("utf-16be", errors="replace")
|
||||
print(help_dump[c]['ja'])
|
||||
i_j = e + 2
|
||||
c+=1
|
||||
print(c, " - strings extracted.")
|
||||
# c = 333 not sure how to index these anymore
|
||||
with open("helptext_dump.json", "w", encoding="utf-8") as outfile:
|
||||
json.dump(help_dump, outfile, ensure_ascii=False, indent=2)
|
||||
|
||||
exit()
|
||||
|
||||
## Attempt to get ids or sense out of the beginning block, no luck so far
|
||||
for r in range(0,len(name_data),8):
|
||||
#id = int.from_bytes( name_data[r:r+4], byteorder='big', signed=False )
|
||||
#id = int(name_data[r:r+4], 16) & 0xFFFFFFFF
|
||||
id = struct.unpack('<I', name_data[r:r+4])[0] ## Only one that seemed to match the hex editor values for some reason.
|
||||
print(r, ": ", (name_data[r:r+4]), " - " , id)
|
||||
if (id == 4): ## Why four?
|
||||
break
|
||||
|
||||
# What's 1616/4 == 404
|
||||
read_until_null(1624,name_data)
|
||||
name_data[1624:read_until_null(1624,name_data)].decode("utf-16be", errors="replace")
|
||||
|
||||
|
||||
|
||||
## Old attempts at getting dlg file extracted, leaving block for reference
|
||||
language_code = 'J'
|
||||
offsets_data = Path(f"YGO_2020/bin/DLG_Indx_{language_code}.bin").read_bytes() # Get all the bytes
|
||||
name_data = Path(f"YGO_2020/bin/DLG_Text_{language_code}.bin").read_bytes() # Get all the bytes
|
||||
get_props()
|
||||
|
||||
# with open("dump2.json", "w", encoding="utf-16") as outfile:
|
||||
# json.dump(cd_db, outfile, ensure_ascii=False, indent=2)
|
||||
|
||||
print()
|
||||
|
||||
### For dialog scripts eventually
|
||||
# Int ReadInt(std::ifstream &file){
|
||||
# Int value;
|
||||
# std::array<Byte,4> bytes;
|
||||
# file.read((char*)(bytes.data()),4);
|
||||
# value = bytes[0] | (bytes[1]<<8) | (bytes[2]<<16) | (bytes[3]<<24);
|
||||
# return value;
|
||||
# }
|
||||
# Long ReadLong(std::ifstream &file){
|
||||
# Long value;
|
||||
# std::array<Byte,8> bytes;
|
||||
# file.read((char*)(bytes.data()),8);
|
||||
# value = bytes[0] | (bytes[1]<<8) | (bytes[2]<<16) | (bytes[3]<<24) | (bytes[4]<<32) | (bytes[5]<<40) | (bytes[6]<<48) | (bytes[7]<<56);
|
||||
# return value;
|
||||
# }
|
||||
|
||||
# if (scriptFile.is_open()){
|
||||
# Long pointerToPointers=ReadLong(scriptFile);
|
||||
# Int titleCount=ReadInt(scriptFile);
|
||||
# Int scriptBlockCount=ReadInt(scriptFile);
|
||||
# std::vector<Int> titleStartPos;
|
||||
# std::vector<Int> titleEndPos;
|
||||
# std::vector<Long> titlePointer;
|
||||
# for (Int i=0; i<titleCount;i++){
|
||||
# titleStartPos.push_back(ReadInt(scriptFile)); # 1
|
||||
# titleEndPos.push_back(ReadInt(scriptFile)); # 2
|
||||
# titlePointer.push_back(ReadLong(scriptFile)); # 3
|
||||
# }
|
||||
# std::vector<std::string> titleNames;
|
||||
# for (Int i=0; i<titleCount;i++){
|
||||
# scriptFile.seekg(titlePointer[i],scriptFile.beg);
|
||||
# titleNames.push_back(ReadByteString(scriptFile));
|
||||
# }
|
||||
|
||||
# std::vector<std::vector<SCRIPT_BLOCK> > script;
|
||||
# for (Int i=0; i< titleCount; i++){
|
||||
# std::vector<SCRIPT_BLOCK> scriptBlocks;
|
||||
# Int blockCount=titleEndPos[i]-titleStartPos[i]+1;
|
||||
# int pointer;
|
||||
# for (Int j=0; j<blockCount;j++){
|
||||
# SCRIPT_BLOCK scriptBlock;
|
||||
# scriptFile.seekg(pointerToPointers+titleStartPos[i]*32+j*32);
|
||||
# pointer=ReadLong(scriptFile);
|
||||
# scriptFile.seekg(pointer,scriptFile.beg);
|
||||
# scriptBlock.chara=ReadByteString(scriptFile);
|
||||
# scriptFile.seekg(pointerToPointers+titleStartPos[i]*32+j*32+8);
|
||||
# pointer=ReadLong(scriptFile);
|
||||
# scriptFile.seekg(pointer,scriptFile.beg);
|
||||
# scriptBlock.arg1=ReadByteString(scriptFile);
|
||||
# scriptFile.seekg(pointerToPointers+titleStartPos[i]*32+j*32+16);
|
||||
# pointer=ReadLong(scriptFile);
|
||||
# scriptFile.seekg(pointer,scriptFile.beg);
|
||||
# scriptBlock.arg2=ReadByteString(scriptFile);
|
||||
# scriptFile.seekg(pointerToPointers+titleStartPos[i]*32+j*32+24);
|
||||
# pointer=ReadLong(scriptFile);
|
||||
# scriptFile.seekg(pointer,scriptFile.beg);
|
||||
# scriptBlock.text=ReadByteString(scriptFile);
|
||||
# scriptBlocks.push_back(scriptBlock);
|
||||
# }
|
||||
# script.push_back(scriptBlocks);
|
||||
# }
|
||||
# scriptFile.close();
|
||||
|
||||
|
||||
|
||||
#>
|
||||
|
|
@ -4,7 +4,7 @@ from fugashi_parser import *
|
|||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--vocab-file", required=False)
|
||||
parser.add_argument("--vocab-file", required=True)
|
||||
parser.add_argument("text", nargs="*")
|
||||
args = parser.parse_args()
|
||||
|
||||
|
|
@ -13,15 +13,12 @@ def main():
|
|||
else:
|
||||
text = sys.stdin.read()
|
||||
|
||||
#vocab = load_vocab(args.vocab_file)
|
||||
vocab = {}
|
||||
vocab = load_vocab(args.vocab_file)
|
||||
result = analyze(text, vocab)
|
||||
#save_vocab(args.vocab_file, vocab)
|
||||
save_vocab(args.vocab_file, vocab)
|
||||
|
||||
result["vocab_file"] = args.vocab_file
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
print(json.dumps(list(vocab), ensure_ascii=False, indent=2))
|
||||
#print(vocab)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -1,101 +0,0 @@
|
|||
from fugashi_parser import *
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import hashlib
|
||||
|
||||
destination_directory_name = '/run/media/deck/YF8SD/Sync/Notebooks/Obsidian/Japanese/Yu-Gi-Oh/'
|
||||
destination_path = Path(destination_directory_name)
|
||||
vocab_destination_path = Path('/run/media/deck/YF8SD/Sync/Notebooks/Obsidian/Japanese/vocab')
|
||||
|
||||
with open("scripts_dump2.json", "r", encoding="utf-8") as f:
|
||||
dump = json.load(f)
|
||||
|
||||
if not destination_path.exists():
|
||||
destination_path.mkdir(parents=True, exist_ok=True)
|
||||
if not destination_path.joinpath('YLOTD_dialog').exists():
|
||||
destination_path.joinpath('YLOTD_dialog').mkdir(parents=True, exist_ok=True)
|
||||
|
||||
counter = 0
|
||||
destination_path = destination_path.joinpath('YLOTD_dialog')
|
||||
for title in dump:
|
||||
print(title)
|
||||
counter+=1
|
||||
## copy the image if it isn't already there
|
||||
with open(destination_path.as_posix() + f"/{counter:03}_{title}.md", 'w') as card_note:
|
||||
for block in dump[title]:
|
||||
print(block)
|
||||
card_note.write(f"\n###### {block}\n\n")
|
||||
#reformatted = re.sub(r'(\d{2})/(\d{2})/(\d{4})', r'\3-\1-\2', date)
|
||||
#re.replace('\$R')
|
||||
text_J = dump[title][block]['text_J']
|
||||
text_J = re.sub(r'\$R(.*?)\((.*?)\)',r'<ruby>\1<rt>\2</rt></ruby>',text_J)
|
||||
print(f"<details><summary>{text_J}</summary>{dump[title][block]['text_E']}</details>",file=card_note)
|
||||
text_J = dump[title][block]['text_J']
|
||||
text_J = re.sub(r'\$R(.*?)\((.*?)\)',r'\1',text_J)
|
||||
text_J = text_J.replace('<br>', '')
|
||||
vocab = {} ## flush the vocab variable each time
|
||||
result = analyze(text_J, vocab)
|
||||
print(f"> [!summary]- Vocab\n>{result['notes_link']}",file=card_note)
|
||||
for word in vocab:
|
||||
write_word_note(word, vocab_destination_path, vocab[word]['pos'][0])
|
||||
# ## add card names as properties
|
||||
# card_note.write("---\n")
|
||||
# card_note.write(f"aliases: [{dump[cd]['name_J']},{dump[cd]['name_E']}]\n")
|
||||
# card_note.write("---\n")
|
||||
## Name at the top
|
||||
# card_note.write(f"![[{cd}.jpg]]") ## card image
|
||||
# card_note.write("\n#### Name\n")
|
||||
# card_note.write("<details><summary>")
|
||||
# card_note.write(dump[cd]['name_J'])
|
||||
# card_note.write("</summary>")
|
||||
# card_note.write(dump[cd]['name_E'])
|
||||
# card_note.write("</details>")
|
||||
# ## the description
|
||||
# card_note.write("\n#### Details\n")
|
||||
# card_note.write("<details><summary>")
|
||||
# card_note.write(dump[cd]['desc_J'])
|
||||
# card_note.write("</summary>")
|
||||
# card_note.write(dump[cd]['desc_E'])
|
||||
# card_note.write("</details>")
|
||||
# # desc_Furi
|
||||
# card_note.write("\n#### Reading\n")
|
||||
# vocab = {} ## flush the vocab variable each time
|
||||
# result = analyze(dump[cd]['name_J'],vocab)
|
||||
# print(f"<span>{result['ruby_html']}</span>\n> [!summary]- Vocab\n>{result['notes_link']}\n",file=card_note)
|
||||
# ## Generate the vocab item
|
||||
# for word in vocab:
|
||||
# if Path(destination_path.joinpath('vocab/' + word + '.md')).exists() == False:
|
||||
# definitions = anki_fields_term(word)
|
||||
# if definitions.get("fields"):
|
||||
# with open(destination_path.as_posix() + '/vocab/' + word + '.md', 'w') as vocab_note:
|
||||
# for entry in definitions.get("fields"):
|
||||
# print(entry['furigana'], file=vocab_note)
|
||||
# if entry['expression'] != entry['reading']:
|
||||
# print(entry['reading'], file=vocab_note)
|
||||
# print("> ",re.sub(r'<.*?>', '', entry['glossary-plain-no-dictionary']), "\n", file=vocab_note)
|
||||
# print(f"Frequency: {entry['frequency-average-rank']}", file=vocab_note)
|
||||
# vocab_note.write(entry['glossary-first'])
|
||||
# print("\n", file=vocab_note)
|
||||
# # card_note.write("##### Glossary\n")
|
||||
# # for vocab in dump[cd]["glossary_Name_J"]:
|
||||
# # print(f"{vocab} : {dump[cd]["glossary_Name_J"][vocab]}", file=card_note)
|
||||
# vocab = {} ## flush the vocab variable each time
|
||||
# result = analyze(dump[cd]['desc_J'],vocab)
|
||||
# #card_note.write("\n")
|
||||
# print(f"<span>{result['ruby_html']}</span>\n> [!summary]- Vocab\n>{result['notes_link']}\n",file=card_note)
|
||||
# for word in vocab:
|
||||
# if Path(destination_path.joinpath('vocab/' + word + '.md')).exists() == False:
|
||||
# definitions = anki_fields_term(word)
|
||||
# if definitions.get("fields"):
|
||||
# with open(destination_path.as_posix() + '/vocab/' + word + '.md', 'w') as vocab_note:
|
||||
# for entry in definitions.get("fields"):
|
||||
# print(entry['furigana'], file=vocab_note)
|
||||
# if entry['expression'] != entry['reading']:
|
||||
# print(entry['reading'], file=vocab_note)
|
||||
# print("> ",re.sub(r'<.*?>', '', entry['glossary-plain-no-dictionary']), "\n", file=vocab_note)
|
||||
# print(f"Frequency: {entry['frequency-average-rank']}", file=vocab_note)
|
||||
# vocab_note.write(entry['glossary-first'])
|
||||
# print("\n", file=vocab_note)
|
||||
# card_note.write("\n")
|
||||
|
|
@ -1,66 +0,0 @@
|
|||
from fugashi_parser import *
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import hashlib
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
#parser.add_argument("--vocab-file", required=True)
|
||||
parser.add_argument("--markdown-file", required=True)
|
||||
parser.add_argument("--vocab-folder", required=False)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.vocab_folder:
|
||||
vocab_destination_path = Path(args.vocab_folder)
|
||||
if vocab_destination_path.is_dir() == False:
|
||||
raise ValueError("Vocab Folder is not a directory.")
|
||||
return
|
||||
else:
|
||||
vocab_destination_path = Path(args.markdown_file).parent
|
||||
if not vocab_destination_path.joinpath('vocab').exists():
|
||||
vocab_destination_path.joinpath('vocab').mkdir(parents=True, exist_ok=True)
|
||||
vocab_destination_path = vocab_destination_path.joinpath('vocab')
|
||||
|
||||
destination_path = Path(args.markdown_file).parent.joinpath(Path(args.markdown_file).stem + '_STUDY.md')
|
||||
|
||||
#print(args)
|
||||
vocab = {} #load_vocab('foo.json')
|
||||
|
||||
## Currently just doing this line by line
|
||||
with open(args.markdown_file, "r", encoding="utf-8") as f:
|
||||
lines = f.readlines()
|
||||
|
||||
i = 0
|
||||
file_changed = False
|
||||
while i < len(lines):
|
||||
## Skip lines that start with certain elements
|
||||
if lines[i][0:6] == '<span>' or lines[i][0] in ('#','>','['):
|
||||
i+=1
|
||||
continue
|
||||
## If the line has japanese
|
||||
if has_japanese(lines[i]):
|
||||
print(lines[i])
|
||||
result = analyze(lines[i],vocab)
|
||||
## If the line changed at all
|
||||
if lines[i] != result['ruby_html']:
|
||||
lines[i] = '<span>' + result['ruby_html'] + '</span>' + "\n> [!summary]- Vocab\n>" + result['notes_link'] + "\n\n" # lost the new line
|
||||
file_changed = True
|
||||
i+=1
|
||||
|
||||
if file_changed == False:
|
||||
return
|
||||
## vocab now
|
||||
## Lookup vocab
|
||||
for word in vocab:
|
||||
write_word_note(word, vocab_destination_path, vocab[word]['pos'][0])
|
||||
|
||||
## Output file with changes if changes
|
||||
with open(destination_path, "w", encoding="utf-8") as f:
|
||||
f.writelines(lines)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
|
|
@ -6,48 +6,33 @@ import shutil
|
|||
import hashlib
|
||||
|
||||
|
||||
# destination_directory_name = '/run/media/deck/YF8SD/Sync/Notebooks/Obsidian/Japanese/Yu-Gi-Oh/AnimeSubs/'
|
||||
# destination_path = Path(destination_directory_name)
|
||||
destination_directory_name = '/run/media/deck/YF8SD/Sync/Notebooks/Obsidian/Japanese/Yu-Gi-Oh/AnimeSubs/'
|
||||
destination_path = Path(destination_directory_name)
|
||||
|
||||
# dictionary_name = 'Jitendex.org [2026-04-04]'
|
||||
dictionary_name = 'Jitendex.org [2026-04-04]'
|
||||
|
||||
# #print(annotate_phrase("私は学校へ行きます"))
|
||||
#print(annotate_phrase("私は学校へ行きます"))
|
||||
|
||||
# ## Make sure directories exist
|
||||
# if not destination_path.exists():
|
||||
# destination_path.mkdir(parents=True, exist_ok=True)
|
||||
# if not destination_path.joinpath('_attachments').exists():
|
||||
# destination_path.joinpath('_attachments').mkdir(parents=True, exist_ok=True)
|
||||
# if not destination_path.joinpath('vocab').exists():
|
||||
# destination_path.joinpath('vocab').mkdir(parents=True, exist_ok=True)
|
||||
## Make sure directories exist
|
||||
if not destination_path.exists():
|
||||
destination_path.mkdir(parents=True, exist_ok=True)
|
||||
if not destination_path.joinpath('_attachments').exists():
|
||||
destination_path.joinpath('_attachments').mkdir(parents=True, exist_ok=True)
|
||||
if not destination_path.joinpath('vocab').exists():
|
||||
destination_path.joinpath('vocab').mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
# ---------- CLI ----------
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
# parser = argparse.ArgumentParser()
|
||||
# parser.add_argument("--vocab-file", required=True)
|
||||
parser.add_argument("--subtitle-file", required=True)
|
||||
parser.add_argument("--vocab-folder", required=False)
|
||||
args = parser.parse_args()
|
||||
# parser.add_argument("text", nargs="*")
|
||||
# args = parser.parse_args()
|
||||
|
||||
if args.vocab_folder:
|
||||
destination_path = Path(args.vocab_folder)
|
||||
if destination_path.is_dir() == False:
|
||||
raise ValueError("Vocab Folder is not a directory.")
|
||||
return
|
||||
else:
|
||||
destination_path = Path(args.subtitle_file).parent
|
||||
if not destination_path.joinpath('vocab').exists():
|
||||
destination_path.joinpath('vocab').mkdir(parents=True, exist_ok=True)
|
||||
destination_path = destination_path.joinpath('vocab')
|
||||
vocab = load_vocab('foo.json')
|
||||
|
||||
# with open("/run/media/deck/YF8SD/Video/Yu-Gi-Oh! Duel Monsters - 001.vtt", "r", encoding="utf-8") as f:
|
||||
# lines = f.readlines()
|
||||
vocab = {} #load_vocab('foo.json')
|
||||
|
||||
## Currently just doing this line by line
|
||||
with open(args.subtitle_file, "r", encoding="utf-8") as f:
|
||||
with open("/run/media/deck/YF8SD/Video/[Retimed and Corrected] Yu-Gi-Oh! Duel Monsters - 001.srt", "r", encoding="utf-8") as f:
|
||||
lines = f.readlines()
|
||||
|
||||
|
||||
|
|
@ -58,25 +43,18 @@ def main():
|
|||
## First lien has some unicode bom bs
|
||||
i = 0
|
||||
while i < len(lines):
|
||||
## If the line starts with a single number build a header
|
||||
if re.fullmatch(r'\d+', lines[i].strip()) or i == 0:
|
||||
notes_lines.append('##### ' + lines[i].strip() + " `" + lines[i+1].strip() + "`" + "\n")
|
||||
i+=2
|
||||
continue
|
||||
if re.fullmatch(r'(\d{2,}:)*\d{2,}:\d{2,}[,.]\d{3,}\s*-->\s*(\d{2,}:)*\d{2,}:\d{2,}[,.]\d{3,}' , lines[i].strip()):
|
||||
notes_lines.append('##### ' " `" + lines[i].strip() + "`" + "\n")
|
||||
i+=1
|
||||
continue
|
||||
if has_japanese(lines[i]):
|
||||
print(lines[i])
|
||||
result = analyze(lines[i],vocab)
|
||||
print(result['kana_reading'])
|
||||
notes_lines.append(result["ruby_html"] + "\n" )
|
||||
notes_lines.append (result['notes_link'] + "\n")
|
||||
if lines[i] != result['ruby_html']:
|
||||
lines[i] = result['ruby_html'] + "\n" # lost the new line
|
||||
# if lines[i] != result['kana_reading']:
|
||||
# lines[i] = result['kana_reading'] + "\n" # lost the new line
|
||||
if lines[i] != result['kana_reading']:
|
||||
lines[i] = result['kana_reading'] + "\n" # lost the new line
|
||||
else:
|
||||
#notes_lines.append("`" + lines[i].strip() + "`" + "\n")
|
||||
notes_lines.append(lines[i])
|
||||
|
|
@ -84,20 +62,29 @@ def main():
|
|||
|
||||
## Lookup vocab
|
||||
for word in vocab:
|
||||
write_word_note(word, destination_path)
|
||||
return
|
||||
#vocab_hash = hashlib.shake_256((word + dictionary_name).encode()).hexdigest(4)
|
||||
if Path(destination_path.joinpath('vocab/' + word + '.md')).exists() == False:
|
||||
definitions = anki_fields_term(word)
|
||||
if definitions.get("fields"):
|
||||
with open(destination_path.as_posix() + '/vocab/' + word + '.md', 'w') as vocab_note:
|
||||
for entry in definitions.get("fields"):
|
||||
vocab_note.write("\n")
|
||||
print(entry['expression'], file=vocab_note)
|
||||
#print(entry['furigana-plain'], file=vocab_note)
|
||||
if entry['expression'] != entry['reading']:
|
||||
print(entry['reading'], file=vocab_note)
|
||||
print(entry['furigana'], file=vocab_note)
|
||||
#print(f"Part of Speech: {entry['part-of-speech']}", file=vocab_note)
|
||||
print(f"Frequency: {entry['frequency-average-rank']}", file=vocab_note)
|
||||
#print(entry['glossary-plain-no-dictionary'], file=vocab_note)
|
||||
vocab_note.write(entry['glossary-first'])
|
||||
print("##### Other Definitions", file=vocab_note)
|
||||
print(entry['glossary-plain-no-dictionary'], file=vocab_note)
|
||||
|
||||
# with open("/run/media/deck/YF8SD/Video/Yu-Gi-Oh! Duel Monsters - 001.vtt", "w", encoding="utf-8") as f:
|
||||
# f.writelines(lines)
|
||||
|
||||
## Output file with changes if changes
|
||||
out_dir = Path(args.subtitle_file).parent
|
||||
changed_file = out_dir.joinpath(Path(args.subtitle_file).stem + '.txt')
|
||||
with open(changed_file, "w", encoding="utf-8") as f:
|
||||
with open("subtitle.srt", "w", encoding="utf-8") as f:
|
||||
f.writelines(lines)
|
||||
|
||||
note_file = out_dir.joinpath(Path(args.subtitle_file).stem + '.md')
|
||||
with open(note_file, "w", encoding="utf-8") as f:
|
||||
with open("notestest.md", "w", encoding="utf-8") as f:
|
||||
f.writelines(notes_lines)
|
||||
|
||||
#vocab = load_vocab(args.vocab_file)
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user