wordcloudの固有名詞等を学習する (#10)

Author: yuito-it

.python-version

@@ -1 +1 @@
-3.11
+3.14.3

Dockerfile

@@ -1,22 +1,32 @@
-FROM ghcr.io/astral-sh/uv:debian-slim
+FROM ghcr.io/astral-sh/uv:python3.14-trixie-slim
 
 WORKDIR /app
 
-RUN apt update
-RUN apt install -y ca-certificates
+# 1. ビルドに必要なツール一式をインストール
+# gcc と libc6-dev がないと、Rustコンパイラがあってもビルドに失敗します
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  ca-certificates \
+  gcc \
+  libc6-dev \
+  rustc \
+  cargo \
+  && rm -rf /var/lib/apt/lists/*
 
-# 1. Pythonの出力をリアルタイムで表示させる設定（重要）
+# 2. 環境設定
 ENV PYTHONUNBUFFERED=1
-# 2. .pycファイルを作成しない設定（コンテナを軽量に保つ）
 ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 
 
+# 3. 依存関係のインストール
 COPY pyproject.toml .
 COPY uv.lock* ./
 
-# Disable development dependencies
 ENV UV_NO_DEV=1
-RUN uv sync --locked
+# --locked を使いつつ同期
+RUN uv sync --locked --no-install-project
 
 COPY . .
 
+RUN uv sync --locked
+
 CMD ["uv", "run", "/app/src/main.py"]

debug_sudachi.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+"""Debug script to examine Sudachi tokenization and POS tags."""
+
+from sudachipy import dictionary, tokenizer
+
+tokenizer_obj = dictionary.Dictionary().create()
+MODE = tokenizer.Tokenizer.SplitMode.C
+
+test_texts = [
+    "ミラノ風ドリア",
+    "ナポリタン",
+    "オムレツ",
+    "塩辛い",
+    "風",
+    "式",
+    "的",
+]
+
+print("=" * 80)
+print("Sudachi Tokenization Analysis")
+print("=" * 80)
+
+for text in test_texts:
+    print(f"\nText: 『{text}』")
+    print("-" * 60)
+    tokens = tokenizer_obj.tokenize(text, MODE)
+    
+    for i, token in enumerate(tokens):
+        surface = token.surface()
+        pos = token.part_of_speech()
+        print(f"  Token[{i}]: '{surface}'")
+        print(f"    POS[0] (major): {pos[0]}")
+        print(f"    POS[1] (minor): {pos[1]}")
+        if len(pos) > 2:
+            print(f"    POS[2]: {pos[2]}")
+        print()
+
+print("=" * 80)

pyproject.toml

@@ -6,15 +6,17 @@ readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
     "discord-py>=2.7.1",
-    "janome>=0.5.0",
     "matplotlib>=3.10.8",
     "networkx>=3.6.1",
     "pymongo>=4.16.0",
     "scipy>=1.17.1",
+    "sudachidict-core>=20260116",
+    "sudachipy>=0.6.10",
     "wordcloud>=1.9.6",
 ]
 
 [dependency-groups]
 dev = [
     "python-dotenv>=1.2.2",
+    "pytest>=9.0.2",
 ]

src/cogs/network.py

@@ -2,11 +2,15 @@
 from discord import app_commands
 from discord.ext import commands
 from typing import Optional
-from datetime import timedelta
-from collections import defaultdict
 
-from libs.visualize import generate_conversation_network
 from libs.embed import EmbedHelper
+from libs.network_service import (
+    build_node_labels,
+    build_conversation_edges,
+    fetch_network_documents,
+    generate_conversation_network,
+)
+from libs.wordcloud_service import parse_period_days
 
 
 class ConversationNetwork(commands.Cog):
@@ -55,20 +59,9 @@ async def generate_network(
             await interaction.response.send_message(embed=embed, ephemeral=True)
             return
 
-        # period
-        period_filter = {}
         if period:
             try:
-                period_days = int(period)
-                if period_days <= 0:
-                    raise ValueError
-                period_filter = {
-                    "timestamp": {
-                        "$gte": (
-                            discord.utils.utcnow() - timedelta(days=period_days)
-                        ).isoformat()
-                    }
-                }
+                period_days = parse_period_days(period)
             except ValueError:
                 embed = embed_helper.create_error_embed(
                     title="エラー",
@@ -84,35 +77,19 @@ async def generate_network(
                 await interaction.response.send_message(embed=embed, ephemeral=True)
                 print(f"Error processing network period: {e}")
                 return
-
-        user_filter = {}
-        if user:
-            user_filter = {"user_id": str(user.id)}
-
-        channel_filter = {}
-        if channel:
-            channel_filter = {"channel_id": str(channel.id)}
+        else:
+            period_days = None
 
         await interaction.response.defer(thinking=True)
 
         try:
-            docs = list(
-                self.bot.db.messages.find(
-                    {
-                        "guild_id": str(interaction.guild_id),
-                        **period_filter,
-                        **user_filter,
-                        **channel_filter,
-                    },
-                    {
-                        "message_id": 1,
-                        "user_id": 1,
-                        "reply_to": 1,
-                        "mentions": 1,
-                    },
-                )
-                .sort("timestamp", -1)
-                .limit(self.MAX_MESSAGE_COUNT)
+            docs = fetch_network_documents(
+                self.bot.db,
+                str(interaction.guild_id),
+                period_days=period_days,
+                user_id=str(user.id) if user else None,
+                channel_id=str(channel.id) if channel else None,
+                limit=self.MAX_MESSAGE_COUNT,
             )
         except Exception as e:
             embed = embed_helper.create_error_embed(
@@ -131,69 +108,17 @@ async def generate_network(
             await interaction.followup.send(embed=embed)
             return
 
-        # message map
-        valid_docs = []
-        invalid_doc_count = 0
-
-        for doc in docs:
-            message_id = doc.get("message_id")
-            author_id = doc.get("user_id")
-
-            if message_id is None or author_id is None:
-                invalid_doc_count += 1
-                continue
+        edges, invalid_doc_count = build_conversation_edges(docs)
+        valid_doc_count = len(docs) - invalid_doc_count
 
-            valid_docs.append(
-                {
-                    "message_id": str(message_id),
-                    "user_id": str(author_id),
-                    "reply_to": (
-                        str(doc["reply_to"])
-                        if doc.get("reply_to") is not None
-                        else None
-                    ),
-                    "mentions": [
-                        str(mentioned)
-                        for mentioned in doc.get("mentions", [])
-                        if mentioned is not None
-                    ],
-                }
-            )
-
-        if not valid_docs:
+        if valid_doc_count <= 0:
             embed = embed_helper.create_warning_embed(
                 title="データ不足",
                 description="解析に使えるメッセージがありませんでした。",
             )
             await interaction.followup.send(embed=embed)
             return
 
-        msg_map = {doc["message_id"]: doc for doc in valid_docs}
-
-        edges = defaultdict(int)
-
-        for msg in valid_docs:
-
-            author = msg.get("user_id")
-            if author is None:
-                continue
-
-            # reply
-            reply_to = msg.get("reply_to")
-            if reply_to and reply_to in msg_map:
-                other = msg_map[reply_to].get("user_id")
-                if author != other:
-                    edges[tuple(sorted([author, other]))] += 1
-
-            # mention
-            mentions = msg.get("mentions", [])
-            if not isinstance(mentions, list):
-                continue
-
-            for mentioned in mentions:
-                if mentioned != author:
-                    edges[tuple(sorted([author, mentioned]))] += 1
-
         if not edges:
             embed = embed_helper.create_warning_embed(
                 title="会話不足",
@@ -202,29 +127,16 @@ async def generate_network(
             await interaction.followup.send(embed=embed)
             return
 
-        # user id → name
-        user_map = {}
-
-        for a, b in edges.keys():
-            if a not in user_map:
-                try:
-                    member = interaction.guild.get_member(int(a))
-                    user_map[a] = member.display_name if member else a
-                except (TypeError, ValueError):
-                    user_map[a] = a
-
-            if b not in user_map:
-                try:
-                    member = interaction.guild.get_member(int(b))
-                    user_map[b] = member.display_name if member else b
-                except (TypeError, ValueError):
-                    user_map[b] = b
+        def resolve_name(user_id: str) -> str:
+            try:
+                member = interaction.guild.get_member(int(user_id))
+                return member.display_name if member else user_id
+            except (TypeError, ValueError):
+                return user_id
 
-        named_edges = {
-            (user_map[a], user_map[b]): count for (a, b), count in edges.items()
-        }
+        node_labels = build_node_labels(edges, resolve_name)
 
-        if not named_edges:
+        if not node_labels:
             embed = embed_helper.create_warning_embed(
                 title="会話不足",
                 description="ネットワーク図に変換できるデータがありませんでした。",
@@ -233,7 +145,7 @@ async def generate_network(
             return
 
         try:
-            image_buffer = generate_conversation_network(named_edges)
+            image_buffer = generate_conversation_network(edges, labels=node_labels)
         except ValueError:
             embed = embed_helper.create_warning_embed(
                 title="会話不足",
@@ -260,7 +172,7 @@ async def generate_network(
         embed = embed_helper.create_success_embed(
             title="会話ネットワーク生成",
             description=(
-                f"{len(valid_docs)}件のメッセージを解析しました"
+                f"{valid_doc_count}件のメッセージを解析しました"
                 + (
                     f"\n不正データ {invalid_doc_count}件 は自動でスキップしました"
                     if invalid_doc_count

src/cogs/optout.py

@@ -3,6 +3,7 @@
 from discord.ext import commands
 from discord import app_commands
 from libs.embed import EmbedHelper
+from libs.message_store import delete_messages_by_query
 
 
 class Optout(commands.Cog):
@@ -16,10 +17,13 @@ def __init__(self, bot: commands.Bot):
 
     async def _delete_messages_background(self, query: dict, scope: str):
         try:
-            # pymongo is synchronous, so run in a thread to avoid blocking the event loop.
-            result = await asyncio.to_thread(self.bot.db.messages.delete_many, query)
+            deleted_count = await asyncio.to_thread(
+                delete_messages_by_query,
+                self.bot.db,
+                query,
+            )
             print(
-                f"[Optout] Background recursive delete completed for {scope}, deleted={result.deleted_count}"
+                f"[Optout] Background recursive delete completed for {scope}, deleted={deleted_count}"
             )
         except Exception as e:
             print(f"[Optout] Background recursive delete failed for {scope}: {e}")