jp_text_parsing/process_markdown.py

69 lines
2.1 KiB
Python

from fugashi_parser import *
import os
from pathlib import Path
import shutil
import hashlib
def main():
parser = argparse.ArgumentParser()
#parser.add_argument("--vocab-file", required=True)
parser.add_argument("--markdown-file", required=True)
parser.add_argument("--vocab-folder", required=False)
args = parser.parse_args()
if args.vocab_folder:
destination_path = Path(args.vocab_folder)
if destination_path.is_dir() == False:
raise ValueError("Vocab Folder is not a directory.")
return
else:
destination_path = Path(args.markdown_file).parent
if not destination_path.joinpath('vocab').exists():
destination_path.joinpath('vocab').mkdir(parents=True, exist_ok=True)
destination_path = destination_path.joinpath('vocab')
# if args.text:
# text = " ".join(args.text)
# else:
# text = sys.stdin.read()
#print(args)
vocab = {} #load_vocab('foo.json')
## Currently just doing this line by line
with open(args.markdown_file, "r", encoding="utf-8") as f:
lines = f.readlines()
i = 0
file_changed = False
while i < len(lines):
## Skip lines that start with certain elements
if lines[i][0:6] == '<span>' or lines[i][0] in ('#','>','['):
i+=1
continue
## If the line has japanese
if has_japanese(lines[i]):
print(lines[i])
result = analyze(lines[i],vocab)
## If the line changed at all
if lines[i] != result['ruby_html']:
lines[i] = '<span>' + result['ruby_html'] + '</span>' + "\n> [!summary]- Vocab\n>" + result['notes_link'] + "\n\n" # lost the new line
file_changed = True
i+=1
if file_changed == False:
return
## vocab now
## Lookup vocab
for word in vocab:
write_word_note(word, destination_path)
## Output file with changes if changes
with open(args.markdown_file, "w", encoding="utf-8") as f:
f.writelines(lines)
if __name__ == "__main__":
main()