Add tool to clean existing keys for a certain upload year in the database with the new purify method. Usage example: npm run clean 2023.

Author: toberndo

package.json

@@ -20,7 +20,8 @@
     "test:integration": "mocha --exit --require ./test/setup.js --recursive ./test/integration",
     "release": "npm run release:install && npm run release:archive",
     "release:install": "rm -rf node_modules/ && npm ci --production",
-    "release:archive": "zip -rq release.zip package.json package-lock.json node_modules/ *.js src/ config/ locales/"
+    "release:archive": "zip -rq release.zip package.json package-lock.json node_modules/ *.js src/ config/ locales/",
+    "clean": "node src/tools/clean"
   },
   "dependencies": {
     "@hapi/boom": "^10.0.1",

src/modules/mongo.js

@@ -6,7 +6,7 @@
 'use strict';
 
 const log = require('../lib/log');
-const MongoClient = require('mongodb').MongoClient;
+const {MongoClient} = require('mongodb');
 
 /**
  * A simple wrapper around the official MongoDB client.
@@ -22,8 +22,9 @@ class Mongo {
   async init({uri, user, pass}) {
     log.info('Connecting to MongoDB ...');
     const url = `mongodb://${user}:${pass}@${uri}`;
-    this._client = await MongoClient.connect(url);
+    this._client = new MongoClient(url);
     this._client.on('commandFailed', event => log.error('MongoDB command failed\n%s', event));
+    await this._client.connect();
     this._db = this._client.db();
   }
 
@@ -134,6 +135,29 @@ class Mongo {
     const col = this._db.collection(type);
     return col.deleteMany({});
   }
+
+  /**
+   * Aggregate documents from a collection
+   * @param  {Array} pipeline                 The aggregation pipeline
+   * @param  {String} type                    The collection to use e.g. 'publickey'
+   * @return {Promise<AggregationCursor<T>>}  The operation result
+   */
+  aggregate(pipeline, type) {
+    const col = this._db.collection(type);
+    return col.aggregate(pipeline);
+  }
+
+  /**
+   * Replace one document
+   * @param  {Object} filter       The filter used to select the document to replace
+   * @param  {Object} replacement  The Document that replaces the matching document
+   * @param  {String} type         The collection to use e.g. 'publickey'
+   * @return {Promise<Document>}
+   */
+  replace(filter, replacement, type) {
+    const col = this._db.collection(type);
+    return col.replaceOne(filter, replacement);
+  }
 }
 
 module.exports = Mongo;

src/tools/clean.js

@@ -0,0 +1,78 @@
+/**
+ * Copyright (C) 2024 Mailvelope GmbH
+ * Licensed under the GNU Affero General Public License version 3
+ */
+
+'use strict';
+
+const config = require('../../config/config');
+const Mongo = require('../modules/mongo');
+const PurifyKey = require('../modules/purify-key');
+const PGP = require('../modules/pgp');
+
+const DB_TYPE = 'publickey';
+const KEY_SIZE = 1; // divided by 4/3 gives binary size of key
+const MAX_UPLOAD_DATE = new Date(new Date().setDate(new Date().getDate() - config.publicKey.purgeTimeInDays)); // now - purgeTimeInDays
+const YEAR = parseInt(process.argv[2] ?? MAX_UPLOAD_DATE.getFullYear());
+
+let mongo;
+let pgp;
+
+async function init() {
+  mongo = new Mongo();
+  await mongo.init(config.mongo);
+  const purify = new PurifyKey({...config.purify, maxNumUserEmail: 60, maxNumSubkey: 25, maxSizeKey: 64 * 1024});
+  pgp = new PGP(purify);
+}
+
+function aggregate() {
+  return mongo.aggregate([
+    {$match: {uploaded: {$gte: new Date(YEAR, 0, 1), $lt: new Date(YEAR + 1, 0, 1)}}},
+    {$match: {uploaded: {$lt: MAX_UPLOAD_DATE}}},
+    {$project: {keySize: {$binarySize: '$publicKeyArmored'}}},
+    {$match: {keySize: {$gt: KEY_SIZE}}}
+  ], DB_TYPE);
+}
+
+async function clean() {
+  try {
+    console.log(`Start cleaning year ${YEAR}...`);
+    await init();
+    const result = await aggregate();
+    let count = 0;
+    for await (const document of result) {
+      await cleanKey(document);
+      count++;
+    }
+    console.log('Number of keys processed:', count);
+  } catch (e) {
+    console.log('Error while traversing keys:', e);
+  } finally {
+    await mongo.disconnect();
+  }
+}
+
+async function cleanKey({_id}) {
+  const key = await mongo.get({_id}, DB_TYPE);
+  if (!key.publicKeyArmored) {
+    console.log('No armored key. Key is not yet verified. Skip');
+    return;
+  }
+  try {
+    const purified = await pgp.parseKey(key.publicKeyArmored);
+    // filter out all unverified user ID and those that are not in the purified set
+    key.userIds = key.userIds.filter(userId => userId.verified && purified.userIds.some(id => id.email === userId.email));
+    if (!key.userIds.length) {
+      throw new Error('No user ID after comparing with purified key.');
+    }
+    const publicKeyArmored = await pgp.filterKeyByUserIds(key.userIds, purified.publicKeyArmored);
+    key.publicKeyArmored = publicKeyArmored;
+    await mongo.replace({_id}, key, DB_TYPE);
+  } catch (e) {
+    console.log('Parsing of key failed:', e.message);
+    await mongo.remove({_id}, DB_TYPE);
+    console.log(`Key ${key.fingerprint} removed.`);
+  }
+}
+
+clean();