feat: preserve child tombstones

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## 0.8.3 - Unreleased
 
+- Preserve comments, pull-request reviews, review threads, and commit references as explicit tombstones; comments and review threads retain edit history, and child hydration now merges by identity so only a sourced deletion event deletes an item while a later live observation restores it.
+
 ## 0.8.2 - 2026-07-18
 
 ### Highlights

docs/maintainer-archive.md

@@ -40,6 +40,8 @@ The first command mirrors open issues and pull requests into SQLite. The targete
 
 Use `--include-comments` when issue discussion matters. Use `--with pr-details` for pull requests that need files, commits, checks, workflow runs, and review-thread state. Those detail rows are useful for review work, but they cost more GitHub API calls than metadata-only sync.
 
+Child hydration is lossless by default. Gitcrawl merges comments, reviews, review threads, and pull-request commit references by their stable GitHub identity; an item missing from one response is not treated as deleted. Deletion therefore requires an explicit sourced record rather than inference from GitHub list responses. Explicit deletion records remain as tombstones with their reason, edited comments and review threads retain revisions, and a later live observation restores their current row without discarding that revision history. Commit references are immutable identities, so they retain tombstone state but do not have an edit-revision table.
+
 ## Search with bounded staleness
 
 Use local search for discovery and add a staleness bound when an agent or script should refresh before answering:

internal/cli/app_test.go

@@ -3697,10 +3697,10 @@ func TestDoctorJSONReportsCurrentSchemaDiagnosticsWithoutMutation(t *testing.T)
 	if got := schema["state"]; got != "current" {
 		t.Fatalf("db_schema.state = %#v, payload=%#v", got, schema)
 	}
-	if got := schema["current_version"]; got != float64(12) {
+	if got := schema["current_version"]; got != float64(13) {
 		t.Fatalf("db_schema.current_version = %#v, payload=%#v", got, schema)
 	}
-	if got := schema["supported_version"]; got != float64(12) {
+	if got := schema["supported_version"]; got != float64(13) {
 		t.Fatalf("db_schema.supported_version = %#v, payload=%#v", got, schema)
 	}
 	if got := schema["child_observation_reservations"]; got != true {
@@ -3962,7 +3962,7 @@ func TestDoctorJSONReportsLegacyPendingSchemaWithoutMutation(t *testing.T) {
 		t.Fatalf("pr_details.duplicate_path_files_supported = %#v, payload=%#v", got, prDetails)
 	}
 	pending := doctorStringList(t, schema, "pending_migrations")
-	if !doctorListContains(pending, "schema_version_3_to_12") ||
+	if !doctorListContains(pending, "schema_version_3_to_13") ||
 		!doctorListContains(pending, "pull_request_files_position_key") ||
 		!doctorListContains(pending, "thread_child_observation_reservations_table") {
 		t.Fatalf("pending_migrations = %#v", pending)

internal/store/comments.go

@@ -2,8 +2,11 @@ package store
 
 import (
 	"context"
+	"database/sql"
 	"encoding/json"
 	"fmt"
+	"strings"
+	"time"
 
 	"github.com/openclaw/gitcrawl/internal/store/storedb"
 )
@@ -21,30 +24,128 @@ type Comment struct {
 	RawJSON         string `json:"-"`
 	CreatedAtGitHub string `json:"created_at_gh,omitempty"`
 	UpdatedAtGitHub string `json:"updated_at_gh,omitempty"`
+	DeletedAt       string `json:"deleted_at,omitempty"`
+	DeletionReason  string `json:"deletion_reason,omitempty"`
 }
 
 func (s *Store) UpsertComment(ctx context.Context, comment Comment) (int64, error) {
+	if err := validateTombstone(comment.DeletedAt, comment.DeletionReason); err != nil {
+		return 0, fmt.Errorf("upsert comment: %w", err)
+	}
+	if s.queries == nil {
+		var id int64
+		err := s.WithTx(ctx, func(tx *Store) error {
+			var err error
+			id, err = tx.upsertComment(ctx, comment)
+			return err
+		})
+		return id, err
+	}
+	return s.upsertComment(ctx, comment)
+}
+
+func (s *Store) upsertComment(ctx context.Context, comment Comment) (int64, error) {
+	if comment.DeletedAt != "" {
+		id, applied, err := s.tombstoneComment(ctx, comment.ThreadID, comment.CommentType, comment.GitHubID, comment.DeletedAt, comment.DeletionReason)
+		if err != nil {
+			return 0, err
+		}
+		if applied {
+			return id, nil
+		}
+	}
 	id, err := s.qsql().UpsertComment(ctx, storedb.UpsertCommentParams{
-		ThreadID:    comment.ThreadID,
-		GithubID:    comment.GitHubID,
-		CommentType: comment.CommentType,
-		AuthorLogin: nullString(comment.AuthorLogin),
-		AuthorType:  nullString(comment.AuthorType),
-		Body:        comment.Body,
-		IsBot:       int64(boolInt(comment.IsBot)),
-		RawJson:     comment.RawJSON,
-		CreatedAtGh: nullString(comment.CreatedAtGitHub),
-		UpdatedAtGh: nullString(comment.UpdatedAtGitHub),
+		ThreadID:       comment.ThreadID,
+		GithubID:       comment.GitHubID,
+		CommentType:    comment.CommentType,
+		AuthorLogin:    nullString(comment.AuthorLogin),
+		AuthorType:     nullString(comment.AuthorType),
+		Body:           comment.Body,
+		IsBot:          int64(boolInt(comment.IsBot)),
+		RawJson:        comment.RawJSON,
+		CreatedAtGh:    nullString(comment.CreatedAtGitHub),
+		UpdatedAtGh:    nullString(comment.UpdatedAtGitHub),
+		DeletedAt:      nullString(comment.DeletedAt),
+		DeletionReason: nullString(comment.DeletionReason),
 	})
 	if err != nil {
 		return 0, fmt.Errorf("upsert comment: %w", err)
 	}
+	if err := s.recordCommentRevision(ctx, id, time.Now().UTC().Format(timeLayout)); err != nil {
+		return 0, err
+	}
 	return id, nil
 }
 
-func (s *Store) DeleteCommentsForThread(ctx context.Context, threadID int64) error {
-	if _, err := s.q().ExecContext(ctx, `delete from comments where thread_id = ?`, threadID); err != nil {
-		return fmt.Errorf("delete comments for thread: %w", err)
+func (s *Store) TombstoneComment(ctx context.Context, threadID int64, commentType, githubID, deletedAt, reason string) (bool, error) {
+	if err := validateTombstone(deletedAt, reason); err != nil {
+		return false, fmt.Errorf("tombstone comment: %w", err)
+	}
+	if deletedAt == "" {
+		return false, fmt.Errorf("tombstone comment: deleted_at is required")
+	}
+	if s.queries == nil {
+		var applied bool
+		err := s.WithTx(ctx, func(tx *Store) error {
+			var err error
+			_, applied, err = tx.tombstoneComment(ctx, threadID, commentType, githubID, deletedAt, reason)
+			return err
+		})
+		return applied, err
+	}
+	_, applied, err := s.tombstoneComment(ctx, threadID, commentType, githubID, deletedAt, reason)
+	return applied, err
+}
+
+func (s *Store) tombstoneComment(ctx context.Context, threadID int64, commentType, githubID, deletedAt, reason string) (int64, bool, error) {
+	var id int64
+	err := s.q().QueryRowContext(ctx, `
+		update comments
+		set deleted_at = ?, deletion_reason = ?
+		where thread_id = ? and comment_type = ? and github_id = ?
+		returning id
+	`, deletedAt, reason, threadID, commentType, githubID).Scan(&id)
+	if err == sql.ErrNoRows {
+		return 0, false, nil
+	}
+	if err != nil {
+		return 0, false, fmt.Errorf("tombstone comment: %w", err)
+	}
+	if err := s.recordCommentRevision(ctx, id, time.Now().UTC().Format(timeLayout)); err != nil {
+		return 0, false, err
+	}
+	return id, true, nil
+}
+
+func (s *Store) recordCommentRevision(ctx context.Context, commentID int64, recordedAt string) error {
+	if _, err := s.q().ExecContext(ctx, `
+		insert into comment_revisions(
+			comment_id, author_login, author_type, body, is_bot, raw_json,
+			created_at_gh, updated_at_gh, deleted_at, deletion_reason, recorded_at
+		)
+		select c.id, c.author_login, c.author_type, c.body, c.is_bot, c.raw_json,
+			c.created_at_gh, c.updated_at_gh, c.deleted_at, c.deletion_reason, ?
+		from comments c
+		where c.id = ?
+			and not exists (
+				select 1
+				from comment_revisions r
+				where r.id = (
+					select max(latest.id) from comment_revisions latest
+					where latest.comment_id = c.id
+				)
+					and r.author_login is c.author_login
+					and r.author_type is c.author_type
+					and r.body is c.body
+					and r.is_bot is c.is_bot
+					and r.raw_json is c.raw_json
+					and r.created_at_gh is c.created_at_gh
+					and r.updated_at_gh is c.updated_at_gh
+					and r.deleted_at is c.deleted_at
+					and r.deletion_reason is c.deletion_reason
+			)
+	`, recordedAt, commentID); err != nil {
+		return fmt.Errorf("record comment revision: %w", err)
 	}
 	return nil
 }
@@ -72,11 +173,31 @@ func (s *Store) ListComments(ctx context.Context, threadID int64) ([]Comment, er
 			RawJSON:         row.RawJson,
 			CreatedAtGitHub: stringValue(row.CreatedAtGh),
 			UpdatedAtGitHub: stringValue(row.UpdatedAtGh),
+			DeletedAt:       stringValue(row.DeletedAt),
+			DeletionReason:  stringValue(row.DeletionReason),
 		})
 	}
 	return comments, nil
 }
 
+func validateTombstone(deletedAt, reason string) error {
+	normalizedDeletedAt := strings.TrimSpace(deletedAt)
+	normalizedReason := strings.TrimSpace(reason)
+	if deletedAt != normalizedDeletedAt || reason != normalizedReason {
+		return fmt.Errorf("deleted_at and deletion_reason must not have surrounding whitespace")
+	}
+	if (normalizedDeletedAt == "") != (normalizedReason == "") {
+		return fmt.Errorf("deleted_at and deletion_reason must be set together")
+	}
+	if normalizedDeletedAt == "" {
+		return nil
+	}
+	if _, err := time.Parse(timeLayout, normalizedDeletedAt); err != nil {
+		return fmt.Errorf("deleted_at must be RFC3339: %w", err)
+	}
+	return nil
+}
+
 func reviewStateFromRawJSON(raw string) string {
 	var value struct {
 		State string `json:"state"`

internal/store/comments_test.go

@@ -3,6 +3,7 @@ package store
 import (
 	"context"
 	"path/filepath"
+	"strings"
 	"testing"
 )
 
@@ -50,4 +51,121 @@ func TestUpsertComment(t *testing.T) {
 	if len(comments) != 2 || comments[0].GitHubID != "c0" || !comments[0].IsBot || comments[1].GitHubID != "c1" {
 		t.Fatalf("comments = %+v", comments)
 	}
+
+	edited := comments[1]
+	edited.Body = "same edited bug here"
+	edited.UpdatedAtGitHub = "2026-04-26T00:02:00Z"
+	if _, err := st.UpsertComment(ctx, edited); err != nil {
+		t.Fatalf("edit comment: %v", err)
+	}
+	deletedAt := "2026-04-26T00:03:00Z"
+	if _, err := st.UpsertComment(ctx, Comment{
+		ThreadID: threadID, GitHubID: "c1", CommentType: "issue_comment",
+		DeletedAt: deletedAt, DeletionReason: "explicit-source-delete",
+	}); err != nil {
+		t.Fatalf("import sparse comment tombstone: %v", err)
+	}
+	comments, err = st.ListComments(ctx, threadID)
+	if err != nil {
+		t.Fatalf("list comments after tombstone: %v", err)
+	}
+	if len(comments) != 1 || comments[0].GitHubID != "c0" {
+		t.Fatalf("tombstoned comment remained visible: %+v", comments)
+	}
+	var revisionCount int
+	if err := st.DB().QueryRowContext(ctx, `
+		select count(*) from comment_revisions where comment_id = ?
+	`, id).Scan(&revisionCount); err != nil {
+		t.Fatalf("comment revisions: %v", err)
+	}
+	if revisionCount != 3 {
+		t.Fatalf("comment revisions = %d, want create/edit/delete", revisionCount)
+	}
+	var retainedBody string
+	if err := st.DB().QueryRowContext(ctx, `select body from comments where id = ?`, id).Scan(&retainedBody); err != nil {
+		t.Fatalf("retained tombstone body: %v", err)
+	}
+	if retainedBody != edited.Body {
+		t.Fatalf("sparse tombstone replaced body with %q", retainedBody)
+	}
+	edited.DeletedAt = ""
+	edited.DeletionReason = ""
+	edited.UpdatedAtGitHub = "2026-04-26T00:04:00Z"
+	if _, err := st.UpsertComment(ctx, edited); err != nil {
+		t.Fatalf("restore comment: %v", err)
+	}
+	comments, err = st.ListComments(ctx, threadID)
+	if err != nil {
+		t.Fatalf("list restored comments: %v", err)
+	}
+	if len(comments) != 2 || comments[1].GitHubID != "c1" || comments[1].Body != edited.Body {
+		t.Fatalf("restored comments = %+v", comments)
+	}
+	if err := st.DB().QueryRowContext(ctx, `
+		select count(*) from comment_revisions where comment_id = ?
+	`, id).Scan(&revisionCount); err != nil {
+		t.Fatalf("restored comment revisions: %v", err)
+	}
+	if revisionCount != 4 {
+		t.Fatalf("comment revisions after restore = %d, want 4", revisionCount)
+	}
+}
+
+func TestTombstoneFieldsRejectSurroundingWhitespace(t *testing.T) {
+	ctx := context.Background()
+	st, err := Open(ctx, filepath.Join(t.TempDir(), "gitcrawl.db"))
+	if err != nil {
+		t.Fatalf("open store: %v", err)
+	}
+	defer st.Close()
+
+	repoID, err := st.UpsertRepository(ctx, Repository{Owner: "openclaw", Name: "gitcrawl", FullName: "openclaw/gitcrawl", RawJSON: "{}", UpdatedAt: "2026-05-01T00:00:00Z"})
+	if err != nil {
+		t.Fatalf("repo: %v", err)
+	}
+	threadID, err := st.UpsertThread(ctx, Thread{
+		RepoID: repoID, GitHubID: "2", Number: 2, Kind: "issue", State: "open",
+		Title: "whitespace", HTMLURL: "https://github.com/openclaw/gitcrawl/issues/2",
+		LabelsJSON: "[]", AssigneesJSON: "[]", RawJSON: "{}", ContentHash: "hash", UpdatedAt: "2026-05-01T00:00:00Z",
+	})
+	if err != nil {
+		t.Fatalf("thread: %v", err)
+	}
+
+	for _, test := range []struct {
+		name      string
+		deletedAt string
+		reason    string
+	}{
+		{name: "whitespace-only pair", deletedAt: " ", reason: " "},
+		{name: "timestamp padding", deletedAt: " 2026-05-01T00:01:00Z", reason: "explicit-source-delete"},
+		{name: "reason padding", deletedAt: "2026-05-01T00:01:00Z", reason: "explicit-source-delete "},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			_, err := st.UpsertComment(ctx, Comment{
+				ThreadID: threadID, GitHubID: "space", CommentType: "issue_comment",
+				Body: "body", RawJSON: `{}`, DeletedAt: test.deletedAt, DeletionReason: test.reason,
+			})
+			if err == nil || !strings.Contains(err.Error(), "surrounding whitespace") {
+				t.Fatalf("upsert comment error = %v, want surrounding whitespace rejection", err)
+			}
+		})
+	}
+	if _, err := st.UpsertComment(ctx, Comment{
+		ThreadID: threadID, GitHubID: "direct", CommentType: "issue_comment",
+		Body: "direct tombstone", RawJSON: `{}`,
+	}); err != nil {
+		t.Fatalf("seed direct tombstone comment: %v", err)
+	}
+	applied, err := st.TombstoneComment(ctx, threadID, "issue_comment", "direct", "2026-05-01T00:02:00Z", "explicit-source-delete")
+	if err != nil || !applied {
+		t.Fatalf("direct comment tombstone = %t, %v", applied, err)
+	}
+	applied, err = st.TombstoneComment(ctx, threadID, "issue_comment", "missing", "2026-05-01T00:02:00Z", "explicit-source-delete")
+	if err != nil || applied {
+		t.Fatalf("missing comment tombstone = %t, %v", applied, err)
+	}
+	if _, err := st.TombstoneComment(ctx, threadID, "issue_comment", "direct", "", ""); err == nil || !strings.Contains(err.Error(), "deleted_at is required") {
+		t.Fatalf("empty comment tombstone error = %v", err)
+	}
 }