feat: duplicate message scanner bot (#479)

* feat: duplicate message scanner

* test: duplicate scanner caching

* chore: disabled bot for star helpers/mvp

* chore: limited channels, added logging and cooldown

Author: Simplyalex99

package-lock.json

@@ -45,6 +45,7 @@
     "xss": "^1.0.15"
   },
   "devDependencies": {
+    "@total-typescript/shoehorn": "^0.1.2",
     "@types/node": "20.14.2",
     "@types/node-cron": "3.0.11",
     "@types/node-fetch": "2.6.11",

package.json

@@ -0,0 +1,93 @@
+import { describe, it, expect, beforeEach, vi } from "vitest";
+import { messageDuplicateChecker } from "./duplicate-scanner";
+import { User } from "discord.js";
+import { fromPartial } from "@total-typescript/shoehorn";
+import { duplicateCache } from "./duplicate-scanner";
+
+const maxMessagesPerUser = 5;
+const maxCacheSize = 100;
+const maxTrivialCharacters = 10;
+// Mock dependencies
+const mockBot = {
+  channels: {
+    fetch: vi.fn().mockResolvedValue({
+      type: "GUILD_TEXT",
+      send: vi.fn().mockResolvedValue({
+        delete: vi.fn().mockResolvedValue(undefined),
+      }),
+    }),
+  },
+};
+const mockMessage = (content: string, authorId: string, isBot = false) => {
+  return {
+    content,
+    author: { id: authorId, bot: isBot } as User,
+    delete: vi.fn(),
+    channel: {
+      send: vi.fn().mockResolvedValue({
+        delete: vi.fn().mockResolvedValue(undefined),
+      }),
+    },
+  };
+};
+describe("Duplicate Scanner Tests", () => {
+  beforeEach(() => {
+    // Reset the cache before each test
+    duplicateCache.clear();
+  });
+  it(`should not store messages less than ${maxTrivialCharacters} characters`, async () => {
+    const msg = mockMessage("Help me", "user1");
+    const bot = mockBot;
+    messageDuplicateChecker.handleMessage?.(fromPartial({ msg, bot }));
+    const userMessages = duplicateCache.get("user1");
+    expect(userMessages).toBeUndefined();
+  });
+
+  it("should store messages correctly in the cache", async () => {
+    const msg = mockMessage("Hello world", "user1");
+    const bot = mockBot;
+
+    messageDuplicateChecker.handleMessage?.(fromPartial({ msg, bot }));
+
+    const userMessages = duplicateCache.get("user1");
+    expect(userMessages).toBeDefined();
+    expect(userMessages?.has("hello world")).toBe(true);
+  });
+
+  it(`should enforce max size of ${maxMessagesPerUser} messages per user`, async () => {
+    const bot = mockBot;
+    for (let i = 1; i <= maxMessagesPerUser; i++) {
+      const msg = mockMessage(`Message to delete ${i}`, "user1");
+      await messageDuplicateChecker.handleMessage?.(fromPartial({ msg, bot }));
+    }
+
+    const userMessages = duplicateCache.get("user1");
+    expect(userMessages).toBeDefined();
+    expect(userMessages?.size).toBe(maxMessagesPerUser);
+
+    const msg = mockMessage("New Message", "user1");
+    await messageDuplicateChecker.handleMessage?.(fromPartial({ msg, bot }));
+
+    expect(userMessages?.size).toBe(maxMessagesPerUser);
+    expect(userMessages?.has("message 1")).toBe(false); // First message should be removed
+    expect(userMessages?.has("new message")).toBe(true); // New message should be added
+  });
+
+  it(`should enforce max size of ${maxCacheSize} users in the cache`, async () => {
+    const bot = mockBot;
+
+    for (let i = 1; i <= maxCacheSize; i++) {
+      const msg = mockMessage("Hello world", `user${i}`);
+      await messageDuplicateChecker.handleMessage?.(fromPartial({ msg, bot }));
+    }
+
+    expect(duplicateCache.size).toBe(maxCacheSize);
+
+    const msg = mockMessage("Hello world", "user101");
+    await messageDuplicateChecker.handleMessage?.(fromPartial({ msg, bot }));
+
+    expect(duplicateCache.size).toBe(maxCacheSize);
+    expect(duplicateCache.has("user1")).toBe(false);
+    expect(duplicateCache.has("user101")).toBe(true);
+  });
+});

src/features/duplicate-scanner/duplicate-scanner.test.ts

@@ -0,0 +1,93 @@
+import type { ChannelHandlers, HandleMessageArgs } from "../../types/index.js";
+import { EmbedType } from "discord.js";
+
+import { EMBED_COLOR } from "../commands.js";
+import { LRUCache } from "lru-cache";
+import { isStaff, isHelpful } from "../../helpers/discord.js";
+import { logger } from "../log.js";
+import { truncateMessage } from "../../helpers/modLog.js";
+import { formatWithEllipsis } from "./helper.js";
+import cooldown from "../cooldown.js";
+const maxMessagesPerUser = 5; // Maximum number of messages per user to track
+// Time (ms) to keep track of duplicates (e.g., 30 sec)
+export const duplicateCache = new LRUCache<string, Set<string>>({
+  max: 100,
+  ttl: 1000 * 60 * 0.5,
+  dispose: (value) => {
+    value.clear();
+  },
+});
+const maxTrivialCharacters = 10;
+const removeFirstElement = (messages: Set<string>) => {
+  const iterator = messages.values();
+  const firstElement = iterator.next().value;
+  if (firstElement) {
+    messages.delete(firstElement);
+  }
+};
+
+const handleDuplicateMessage = async ({
+  msg,
+  userId,
+}: HandleMessageArgs & { userId: string }) => {
+  await msg.delete().catch(console.error);
+  const cooldownKey = `resume-${msg.channelId}`;
+  if (cooldown.hasCooldown(userId, cooldownKey)) {
+    return;
+  }
+  cooldown.addCooldown(userId, cooldownKey);
+  const warningMsg = `Hey <@${userId}>, it looks like you've posted this message in another channel already. Please avoid cross-posting.`;
+  const warning = await msg.channel.send({
+    embeds: [
+      {
+        title: "Duplicate Message Detected",
+        type: EmbedType.Rich,
+        description: warningMsg,
+        color: EMBED_COLOR,
+      },
+    ],
+  });
+
+  // Auto-delete warning after 30 seconds
+  setTimeout(() => {
+    warning.delete().catch(console.error);
+  }, 30_000);
+
+  logger.log(
+    "duplicate message detected",
+    `${msg.author.username} in <#${msg.channel.id}> \n${formatWithEllipsis(truncateMessage(msg.content, 100))}`,
+  );
+  return;
+};
+const normalizeContent = (content: string) =>
+  content.trim().toLowerCase().replace(/\s+/g, " ");
+export const messageDuplicateChecker: ChannelHandlers = {
+  handleMessage: async ({ msg, bot }) => {
+    if (msg.author.bot || isStaff(msg.member) || isHelpful(msg.member)) return;
+
+    const content = normalizeContent(msg.content);
+    const userId = msg.author.id;
+
+    if (content.length < maxTrivialCharacters) return;
+
+    const userMessages = duplicateCache.get(userId);
+
+    if (!userMessages) {
+      const messages = new Set<string>();
+      messages.add(content);
+      duplicateCache.set(userId, messages);
+      return;
+    }
+
+    if (userMessages.has(content)) {
+      await handleDuplicateMessage({ msg, bot, userId });
+      return;
+    }
+
+    if (userMessages.size >= maxMessagesPerUser) {
+      removeFirstElement(userMessages);
+    }
+
+    userMessages.add(content);
+  },
+};

src/features/duplicate-scanner/duplicate-scanner.ts

@@ -0,0 +1,11 @@
+export const formatWithEllipsis = (sentences: string): string => {
+  const ellipsis = "...";
+
+  if (sentences.length === 0) {
+    return ellipsis;
+  }
+  if (sentences.charAt(sentences.length - 1) !== ".") {
+    return sentences + ellipsis;
+  }
+  return sentences + "..";
+};

src/features/duplicate-scanner/helper.ts

@@ -42,6 +42,9 @@ import { recommendBookCommand } from "./features/book-list.js";
 import { mdnSearch } from "./features/mdn.js";
 import "./server.js";
 import { jobScanner } from "./features/job-scanner.js";
+
+import { messageDuplicateChecker } from "./features/duplicate-scanner/duplicate-scanner.js";
+
 import { getMessage } from "./helpers/discord.js";
 
 export const bot = new Client({
@@ -228,7 +231,10 @@ const threadChannels = [CHANNELS.helpJs, CHANNELS.helpThreadsReact];
 addHandler(threadChannels, autothread);
 
 addHandler(CHANNELS.resumeReview, resumeReviewPdf);
-
+addHandler(
+  [CHANNELS.helpReact, CHANNELS.generalReact, CHANNELS.generalTech],
+  messageDuplicateChecker,
+);
 bot.on("ready", () => {
   deployCommands(bot);
   jobsMod(bot);