Fix source document ID generation for filesystem compatibility (#1240)

Author: Ndpnt

CHANGELOG.md

@@ -2,6 +2,15 @@
 
 All changes that impact users of this module are documented in this file, in the [Common Changelog](https://common-changelog.org) format with some additional specifications defined in the CONTRIBUTING file. This codebase adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased [patch]
+
+> Development of this release was supported by [Reset Tech](https://www.reset.tech).
+
+### Fixed
+
+- Fix snapshot storage failure when source document URLs contain characters forbidden in filenames
+- Fix duplicate file extension in snapshot filenames when source document URLs end with an extension
+
 ## 11.0.1 - 2026-04-06
 
 > Development of this release was supported by [Reset Tech](https://www.reset.tech).

src/archivist/services/sourceDocument.js

@@ -1,3 +1,7 @@
+import path from 'path';
+
+import mime from 'mime';
+
 export default class SourceDocument {
   /**
    * Represents a source document containing web content and metadata for extraction.
@@ -21,7 +25,7 @@ export default class SourceDocument {
     this.filters = filters;
     this.content = content;
     this.mimeType = mimeType;
-    this.id = new URL(location).pathname.split('/').filter(Boolean).join('-');
+    this.id = SourceDocument.generateId(location);
   }
 
   get cssSelectors() {
@@ -60,6 +64,20 @@ export default class SourceDocument {
     return [selector];
   }
 
+  static generateId(location) {
+    const ILLEGAL_CHARACTERS = /[\\:"<>|*?]/g; // Characters forbidden in filenames for cross-platform compatibility; see https://github.com/actions/toolkit/blob/main/packages/artifact/src/internal/upload/path-and-artifact-name-validation.ts
+
+    const pathname = decodeURIComponent(new URL(location).pathname);
+    const extension = path.extname(pathname);
+    const pathnameWithoutExtension = mime.getType(extension) ? pathname.slice(0, -extension.length) : pathname; // Remove file extension when it corresponds to a known MIME type, as the extension is not part of the document's identity but a web server implementation detail
+
+    return pathnameWithoutExtension
+      .split('/')
+      .filter(Boolean)
+      .join('-')
+      .replace(ILLEGAL_CHARACTERS, '_');
+  }
+
   toPersistence() {
     return {
       fetch: this.location,

src/archivist/services/sourceDocument.test.js

@@ -157,6 +157,63 @@ describe('SourceDocument', () => {
     });
   });
 
+  describe('#generateId', () => {
+    it('generates ID from URL pathname', () => {
+      expect(new SourceDocument({ location: 'https://example.com/legal/terms' }).id).to.equal('legal-terms');
+    });
+
+    it('returns empty string for root URL', () => {
+      expect(new SourceDocument({ location: 'https://example.com/' }).id).to.equal('');
+    });
+
+    it('decodes URL-encoded characters', () => {
+      expect(new SourceDocument({ location: 'https://example.com/terms%20of%20service' }).id).to.equal('terms of service');
+    });
+
+    it('removes known file extension from URL pathname', () => {
+      expect(new SourceDocument({ location: 'https://example.com/en.html' }).id).to.equal('en');
+    });
+
+    it('removes only the last file extension', () => {
+      expect(new SourceDocument({ location: 'https://example.com/terms.backup.html' }).id).to.equal('terms.backup');
+    });
+
+    it('keeps unknown extension in URL pathname', () => {
+      expect(new SourceDocument({ location: 'https://example.com/terms.of.service' }).id).to.equal('terms.of.service');
+    });
+    it('decodes URL-encoded characters before replacing illegal ones', () => {
+      expect(new SourceDocument({ location: 'https://example.com/terms%3Aof%3Aservice' }).id).to.equal('terms_of_service');
+    });
+
+    context('replaces characters that are illegal in filenames for cross-platform compatibility', () => {
+      const ILLEGAL_CHARACTERS_IN_URL_PATHNAME = {
+        ':': 'colon',
+        '"': 'double quote',
+        '<': 'less than',
+        '>': 'greater than',
+        '|': 'vertical bar',
+        '*': 'asterisk',
+      };
+
+      for (const [ character, name ] of Object.entries(ILLEGAL_CHARACTERS_IN_URL_PATHNAME)) {
+        it(`replaces ${name} "${character}"`, () => {
+          expect(new SourceDocument({ location: `https://example.com/before${character}after` }).id).to.equal('before_after');
+        });
+      }
+
+      const ILLEGAL_CHARACTERS_ENCODED_IN_URL = {
+        '%5C': 'backslash',
+        '%3F': 'question mark',
+      };
+
+      for (const [ encoded, name ] of Object.entries(ILLEGAL_CHARACTERS_ENCODED_IN_URL)) {
+        it(`replaces ${name} decoded from "${encoded}"`, () => {
+          expect(new SourceDocument({ location: `https://example.com/before${encoded}after` }).id).to.equal('before_after');
+        });
+      }
+    });
+  });
+
   describe('#toPersistence', () => {
     it('converts basic source document declarations into JSON representation', () => {
       const result = new SourceDocument({