better token parsing with predicate extraction that might be useful some day.

fugashi_parser.py

@@ -30,7 +30,24 @@ KATAKANA_START = ord("ァ")
 KATAKANA_END = ord("ヶ")
 KATAKANA_TO_HIRAGANA_OFFSET = ord("ぁ") - ord("ァ")
 
-KANJI_RE = re.compile(r"[\u4E00-\u9FFF]")
+
+CONTENT_POS1 = {"名詞", "動詞", "形容詞", "形状詞", "副詞", "代名詞"}
+
+def is_vocab_token(pos, keep_pronouns=False, keep_numbers=False):
+    pos1, pos2, pos3, pos4 = pos
+
+    if pos1 not in CONTENT_POS1:
+        return False
+
+    if pos1 == "代名詞" and not keep_pronouns:
+        return False
+
+    if pos1 == "名詞" and pos2 == "数詞" and not keep_numbers:
+        return False
+
+    return True
+
+
 
 def katakana_to_hiragana(text: str) -> str:
     return "".join(
@@ -38,9 +55,19 @@ def katakana_to_hiragana(text: str) -> str:
         for ch in text
     )
 
+KANJI_RE = re.compile(r"[\u4E00-\u9FFF]")
 def has_kanji(text: str) -> bool:
     return bool(KANJI_RE.search(text))
 
+# def has_kanji(text):
+#     return any(
+#         0x3400 <= ord(ch) <= 0x4DBF or
+#         0x4E00 <= ord(ch) <= 0x9FFF or
+#         0xF900 <= ord(ch) <= 0xFAFF
+#         for ch in text
+#     )
+
+
 def has_japanese(text: str) -> bool:
     return any(
         0x3040 <= ord(ch) <= 0x30FF or
@@ -86,6 +113,17 @@ def clean_lemma(lemma: str) -> str:
 def get_feature(feature, name: str, default=""):
     return getattr(feature, name, default) or default
 
+def should_ruby(surface, reading_hiragana):
+    if not reading_hiragana:
+        return False
+
+    if surface == reading_hiragana:
+        return False
+
+    if not has_kanji(surface):
+        return False
+
+    return True
 
 def build_ruby(surface: str, reading: str) -> str:
     if not reading or surface == reading or not has_japanese(surface):
@@ -97,13 +135,104 @@ def build_kana(surface: str, reading: str) -> str:
         return surface
     return f"「{reading}」"
 
-def build_vocab_link(surface: str, reading: str, pos1: str) -> str:
-    if should_skip(surface, pos1) or not is_content_word(pos1):
+def build_vocab_link(surface: str, reading: str, pos) -> str:
+    if not is_vocab_token(pos, keep_pronouns=True,keep_numbers=False):
         return surface
     if surface == reading:
         return f"[[{surface}]]"
     return f"[[{reading}|{surface}]]"
 
+## Predicate filtering Verbs
+def is_predicate_start(pos):
+    pos1, pos2, pos3, pos4 = pos
+
+    return (
+        pos1 == "動詞" and pos2 == "一般"
+    ) or (
+        pos1 == "形容詞"
+    ) or (
+        pos1 == "形状詞"
+    )
+
+def attaches_to_predicate(pos):
+    pos1, pos2, pos3, pos4 = pos
+
+    # ます, た, ない, れる, られる, たい, etc.
+    if pos1 == "助動詞":
+        return True
+
+    # helper verbs like いる, ある, しまう in constructions
+    if pos1 == "動詞" and pos2 == "非自立可能":
+        return True
+
+    # te-form connector in 読んでいる, 食べている
+    if pos1 == "助詞" and pos2 == "接続助詞":
+        return True
+
+    # suffixes attached to predicates
+    if pos1 == "接尾辞":
+        return True
+
+    return False
+
+def chunk_predicates(tokens):
+    """
+    tokens should be a list of TokenReading objects:
+      token.surface
+      token.lemma
+      token.pos
+    """
+    chunks = []
+    i = 0
+
+    while i < len(tokens):
+        token = tokens[i]
+
+        if not is_predicate_start(token.pos):
+            i += 1
+            continue
+
+        chunk = [token]
+        j = i + 1
+
+        while j < len(tokens) and attaches_to_predicate(tokens[j].pos):
+            chunk.append(tokens[j])
+            j += 1
+
+        chunks.append({
+            "surface": "".join(t.surface for t in chunk),
+            "head_lemma": token.lemma,
+            "head_reading_hiragana": token.lemma_reading_hiragana,
+            "pos": token.pos,
+            "parts": [
+                {
+                    "surface": t.surface,
+                    "lemma": t.lemma,
+                    "reading_hiragana": t.reading_hiragana,
+                    "pos": t.pos,
+                }
+                for t in chunk
+            ],
+        })
+
+        i = j
+
+    return chunks
+
+## Vocab Filtering
+def is_vocab_token(pos, keep_pronouns=False, keep_numbers=False):
+    pos1, pos2, pos3, pos4 = pos
+
+    if pos1 not in {"名詞", "動詞", "形容詞", "形状詞", "副詞", "代名詞"}:
+        return False
+
+    if pos1 == "代名詞" and not keep_pronouns:
+        return False
+
+    if pos1 == "名詞" and pos2 == "数詞" and not keep_numbers:
+        return False
+
+    return True
 
 @dataclass
 class TokenReading:
@@ -156,7 +285,9 @@ def update_vocab(vocab: Dict[str, Dict], tokens: List[TokenReading]):
     for t in tokens:
         pos1 = t.pos[0] if t.pos else ""
 
-        if should_skip(t.surface, pos1) or not is_content_word(pos1):
+        # if should_skip(t.surface, pos1) or not is_content_word(pos1):
+        #     continue
+        if not is_vocab_token(t.pos, keep_pronouns=True,keep_numbers=False):
             continue
 
         entry = vocab.get(t.lemma)
@@ -206,7 +337,7 @@ def analyze(text: str, vocab: Dict[str, Dict]):
             reading = ""
 
         lemma = clean_lemma(get_feature(f, "lemma") or surface)
-        vocab_link_parts.append(build_vocab_link(surface, lemma, pos[0]))
+        vocab_link_parts.append(build_vocab_link(surface, lemma, pos))
 
         lform = get_feature(f, "lForm")
         kana_base = get_feature(f, "kanaBase")
@@ -217,6 +348,7 @@ def analyze(text: str, vocab: Dict[str, Dict]):
 
         ruby_parts.append(build_ruby(surface, reading))
         kana_parts.append(build_kana(surface, reading))
+        predicate_chunks = chunk_predicates(tokens)
 
     update_vocab(vocab, tokens)
 
@@ -226,4 +358,5 @@ def analyze(text: str, vocab: Dict[str, Dict]):
         "ruby_html": "".join(ruby_parts),
         "notes_link":"".join(vocab_link_parts),
         "token_readings": [asdict(t) for t in tokens],
+        "predicate_chunks" : predicate_chunks,
     }

subtitle_processor.py

@@ -84,7 +84,7 @@ def main():
     with open("subtitle.srt", "w", encoding="utf-8") as f:
         f.writelines(lines)
 
-    with open("notestest.md", "w",  encoding="utf-8") as f:
+    with open("notestest-newchunks.md", "w",  encoding="utf-8") as f:
         f.writelines(notes_lines)
 
     #vocab = load_vocab(args.vocab_file)