refactor: update robots.ts to use array for disallowed paths and format sitemap

Author: kingRayhan

public/robots.txt

@@ -4,9 +4,14 @@ export default function robots(): MetadataRoute.Robots {
   return {
     rules: {
       userAgent: "*",
+      crawlDelay: 10,
       allow: "/",
-      disallow: "/dashboard",
+      disallow: ["/dashboard", "/backdoor"],
     },
     host: "https://www.techdiary.dev",
+    sitemap: [
+      "https://www.techdiary.dev/sitemaps/articles/sitemap.xml",
+      "https://www.techdiary.dev/sitemaps/profiles/sitemap.xml",
+    ],
   };
 }