diff --git a/fugashi_parser.py b/fugashi_parser.py index 92f76d4..5ccca21 100644 --- a/fugashi_parser.py +++ b/fugashi_parser.py @@ -30,7 +30,24 @@ KATAKANA_START = ord("ァ") KATAKANA_END = ord("ヶ") KATAKANA_TO_HIRAGANA_OFFSET = ord("ぁ") - ord("ァ") -KANJI_RE = re.compile(r"[\u4E00-\u9FFF]") + +CONTENT_POS1 = {"名詞", "動詞", "形容詞", "形状詞", "副詞", "代名詞"} + +def is_vocab_token(pos, keep_pronouns=False, keep_numbers=False): + pos1, pos2, pos3, pos4 = pos + + if pos1 not in CONTENT_POS1: + return False + + if pos1 == "代名詞" and not keep_pronouns: + return False + + if pos1 == "名詞" and pos2 == "数詞" and not keep_numbers: + return False + + return True + + def katakana_to_hiragana(text: str) -> str: return "".join( @@ -38,9 +55,19 @@ def katakana_to_hiragana(text: str) -> str: for ch in text ) +KANJI_RE = re.compile(r"[\u4E00-\u9FFF]") def has_kanji(text: str) -> bool: return bool(KANJI_RE.search(text)) +# def has_kanji(text): +# return any( +# 0x3400 <= ord(ch) <= 0x4DBF or +# 0x4E00 <= ord(ch) <= 0x9FFF or +# 0xF900 <= ord(ch) <= 0xFAFF +# for ch in text +# ) + + def has_japanese(text: str) -> bool: return any( 0x3040 <= ord(ch) <= 0x30FF or @@ -86,6 +113,17 @@ def clean_lemma(lemma: str) -> str: def get_feature(feature, name: str, default=""): return getattr(feature, name, default) or default +def should_ruby(surface, reading_hiragana): + if not reading_hiragana: + return False + + if surface == reading_hiragana: + return False + + if not has_kanji(surface): + return False + + return True def build_ruby(surface: str, reading: str) -> str: if not reading or surface == reading or not has_japanese(surface): @@ -97,13 +135,104 @@ def build_kana(surface: str, reading: str) -> str: return surface return f"「{reading}」" -def build_vocab_link(surface: str, reading: str, pos1: str) -> str: - if should_skip(surface, pos1) or not is_content_word(pos1): +def build_vocab_link(surface: str, reading: str, pos) -> str: + if not is_vocab_token(pos, keep_pronouns=True,keep_numbers=False): return surface if surface == reading: return f"[[{surface}]]" return f"[[{reading}|{surface}]]" +## Predicate filtering Verbs +def is_predicate_start(pos): + pos1, pos2, pos3, pos4 = pos + + return ( + pos1 == "動詞" and pos2 == "一般" + ) or ( + pos1 == "形容詞" + ) or ( + pos1 == "形状詞" + ) + +def attaches_to_predicate(pos): + pos1, pos2, pos3, pos4 = pos + + # ます, た, ない, れる, られる, たい, etc. + if pos1 == "助動詞": + return True + + # helper verbs like いる, ある, しまう in constructions + if pos1 == "動詞" and pos2 == "非自立可能": + return True + + # te-form connector in 読んでいる, 食べている + if pos1 == "助詞" and pos2 == "接続助詞": + return True + + # suffixes attached to predicates + if pos1 == "接尾辞": + return True + + return False + +def chunk_predicates(tokens): + """ + tokens should be a list of TokenReading objects: + token.surface + token.lemma + token.pos + """ + chunks = [] + i = 0 + + while i < len(tokens): + token = tokens[i] + + if not is_predicate_start(token.pos): + i += 1 + continue + + chunk = [token] + j = i + 1 + + while j < len(tokens) and attaches_to_predicate(tokens[j].pos): + chunk.append(tokens[j]) + j += 1 + + chunks.append({ + "surface": "".join(t.surface for t in chunk), + "head_lemma": token.lemma, + "head_reading_hiragana": token.lemma_reading_hiragana, + "pos": token.pos, + "parts": [ + { + "surface": t.surface, + "lemma": t.lemma, + "reading_hiragana": t.reading_hiragana, + "pos": t.pos, + } + for t in chunk + ], + }) + + i = j + + return chunks + +## Vocab Filtering +def is_vocab_token(pos, keep_pronouns=False, keep_numbers=False): + pos1, pos2, pos3, pos4 = pos + + if pos1 not in {"名詞", "動詞", "形容詞", "形状詞", "副詞", "代名詞"}: + return False + + if pos1 == "代名詞" and not keep_pronouns: + return False + + if pos1 == "名詞" and pos2 == "数詞" and not keep_numbers: + return False + + return True @dataclass class TokenReading: @@ -156,7 +285,9 @@ def update_vocab(vocab: Dict[str, Dict], tokens: List[TokenReading]): for t in tokens: pos1 = t.pos[0] if t.pos else "" - if should_skip(t.surface, pos1) or not is_content_word(pos1): + # if should_skip(t.surface, pos1) or not is_content_word(pos1): + # continue + if not is_vocab_token(t.pos, keep_pronouns=True,keep_numbers=False): continue entry = vocab.get(t.lemma) @@ -206,7 +337,7 @@ def analyze(text: str, vocab: Dict[str, Dict]): reading = "" lemma = clean_lemma(get_feature(f, "lemma") or surface) - vocab_link_parts.append(build_vocab_link(surface, lemma, pos[0])) + vocab_link_parts.append(build_vocab_link(surface, lemma, pos)) lform = get_feature(f, "lForm") kana_base = get_feature(f, "kanaBase") @@ -217,6 +348,7 @@ def analyze(text: str, vocab: Dict[str, Dict]): ruby_parts.append(build_ruby(surface, reading)) kana_parts.append(build_kana(surface, reading)) + predicate_chunks = chunk_predicates(tokens) update_vocab(vocab, tokens) @@ -226,4 +358,5 @@ def analyze(text: str, vocab: Dict[str, Dict]): "ruby_html": "".join(ruby_parts), "notes_link":"".join(vocab_link_parts), "token_readings": [asdict(t) for t in tokens], + "predicate_chunks" : predicate_chunks, } diff --git a/subtitle_processor.py b/subtitle_processor.py index ce88236..0c0521e 100644 --- a/subtitle_processor.py +++ b/subtitle_processor.py @@ -84,7 +84,7 @@ def main(): with open("subtitle.srt", "w", encoding="utf-8") as f: f.writelines(lines) - with open("notestest.md", "w", encoding="utf-8") as f: + with open("notestest-newchunks.md", "w", encoding="utf-8") as f: f.writelines(notes_lines) #vocab = load_vocab(args.vocab_file)