Remove known extensions from source document IDs

Author: Ndpnt

src/archivist/services/sourceDocument.js

@@ -1,3 +1,7 @@
+import path from 'path';
+
+import mime from 'mime';
+
 export default class SourceDocument {
   /**
    * Represents a source document containing web content and metadata for extraction.
@@ -63,7 +67,11 @@ export default class SourceDocument {
   static generateId(location) {
     const ILLEGAL_CHARACTERS = /[\\:"<>|*?]/g; // Characters forbidden in filenames for cross-platform compatibility; see https://github.com/actions/toolkit/blob/main/packages/artifact/src/internal/upload/path-and-artifact-name-validation.ts
 
-    return decodeURIComponent(new URL(location).pathname)
+    const pathname = decodeURIComponent(new URL(location).pathname);
+    const extension = path.extname(pathname);
+    const pathnameWithoutExtension = mime.getType(extension) ? pathname.slice(0, -extension.length) : pathname; // Remove file extension when it corresponds to a known MIME type, as the extension is not part of the document's identity but a web server implementation detail
+
+    return pathnameWithoutExtension
       .split('/')
       .filter(Boolean)
       .join('-')

src/archivist/services/sourceDocument.test.js

@@ -170,6 +170,17 @@ describe('SourceDocument', () => {
       expect(new SourceDocument({ location: 'https://example.com/terms%20of%20service' }).id).to.equal('terms of service');
     });
 
+    it('removes known file extension from URL pathname', () => {
+      expect(new SourceDocument({ location: 'https://example.com/en.html' }).id).to.equal('en');
+    });
+
+    it('removes only the last file extension', () => {
+      expect(new SourceDocument({ location: 'https://example.com/terms.backup.html' }).id).to.equal('terms.backup');
+    });
+
+    it('keeps unknown extension in URL pathname', () => {
+      expect(new SourceDocument({ location: 'https://example.com/terms.of.service' }).id).to.equal('terms.of.service');
+    });
     it('decodes URL-encoded characters before replacing illegal ones', () => {
       expect(new SourceDocument({ location: 'https://example.com/terms%3Aof%3Aservice' }).id).to.equal('terms_of_service');
     });