From 2221311182844864045ee62719a6bd01644c691e Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Tue, 7 Apr 2026 14:27:04 +0530 Subject: [PATCH 1/2] fix: retry DB connection in RDS init lambda --- demos/rds-init-fn-code/index.js | 59 +++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 3 deletions(-) diff --git a/demos/rds-init-fn-code/index.js b/demos/rds-init-fn-code/index.js index 51bf748..4a72d9d 100644 --- a/demos/rds-init-fn-code/index.js +++ b/demos/rds-init-fn-code/index.js @@ -7,6 +7,9 @@ const fs = require('fs') const path = require('path') require('dotenv').config(); +const DB_CONNECT_RETRY_MAX_ATTEMPTS = Number.parseInt(process.env.DB_CONNECT_RETRY_MAX_ATTEMPTS || '24', 10) +const DB_CONNECT_RETRY_DELAY_MS = Number.parseInt(process.env.DB_CONNECT_RETRY_DELAY_MS || '5000', 10) + // the env AWS_ENDPOINT_URL is automatically injected and available const endpoint = process.env.AWS_ENDPOINT_URL; const url = new URL(endpoint); @@ -21,10 +24,11 @@ const secrets = new AWS.SecretsManager({ }) exports.handler = async (e) => { + let connection try { const { config } = e.params const { password, username, dbname, port } = await getSecretValue(config.credsSecretName) - const connection = mysql.createConnection({ + connection = await createConnectionWithRetry({ host: hostname, user: username, database: dbname, @@ -33,8 +37,6 @@ exports.handler = async (e) => { multipleStatements: true }) - connection.connect() - const sqlScript = fs.readFileSync(path.join(__dirname, 'script.sql')).toString() const res = await query(connection, sqlScript) @@ -48,6 +50,10 @@ exports.handler = async (e) => { err, message: err.message } + } finally { + if (connection) { + await closeConnection(connection) + } } } @@ -61,6 +67,53 @@ function query (connection, sql) { }) } +async function createConnectionWithRetry (connectionConfig) { + let lastError + for (let attempt = 1; attempt <= DB_CONNECT_RETRY_MAX_ATTEMPTS; attempt += 1) { + const connection = mysql.createConnection(connectionConfig) + try { + await connect(connection) + return connection + } catch (error) { + connection.destroy() + lastError = error + if (!shouldRetryConnectionError(error) || attempt === DB_CONNECT_RETRY_MAX_ATTEMPTS) { + break + } + + const retryInSeconds = DB_CONNECT_RETRY_DELAY_MS / 1000 + console.log(`Database connection attempt ${attempt}/${DB_CONNECT_RETRY_MAX_ATTEMPTS} failed with '${error.code || error.message}'. Retrying in ${retryInSeconds}s...`) + await sleep(DB_CONNECT_RETRY_DELAY_MS) + } + } + + throw lastError +} + +function connect (connection) { + return new Promise((resolve, reject) => { + connection.connect((error) => { + if (error) return reject(error) + + return resolve() + }) + }) +} + +function shouldRetryConnectionError (error) { + return ['ECONNREFUSED', 'ETIMEDOUT', 'EHOSTUNREACH', 'ENOTFOUND', 'PROTOCOL_CONNECTION_LOST'].includes(error?.code) +} + +function sleep (delayMs) { + return new Promise((resolve) => setTimeout(resolve, delayMs)) +} + +function closeConnection (connection) { + return new Promise((resolve) => { + connection.end(() => resolve()) + }) +} + function getSecretValue (secretId) { return new Promise((resolve, reject) => { secrets.getSecretValue({ SecretId: secretId }, (err, data) => { From f40ec8dc24bbaab99cefd3538b396ecb05effefe Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Tue, 7 Apr 2026 14:36:24 +0530 Subject: [PATCH 2/2] fix: wait for secret attachment before DB init --- demos/rds-init-example.ts | 4 ++++ demos/rds-init-fn-code/index.js | 22 +++++++++++----------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/demos/rds-init-example.ts b/demos/rds-init-example.ts index 7774622..77c6266 100644 --- a/demos/rds-init-example.ts +++ b/demos/rds-init-example.ts @@ -72,6 +72,10 @@ export class RdsInitStackExample extends Stack { }) // manage resources dependency initializer.customResource.node.addDependency(dbServer) + const credsAttachment = creds.node.tryFindChild('Attachment') + if (credsAttachment) { + initializer.customResource.node.addDependency(credsAttachment) + } // allow the initializer function to connect to the RDS instance dbServer.connections.allowFrom(initializer.function, Port.tcp(3306)) // note: not required for LocalStack diff --git a/demos/rds-init-fn-code/index.js b/demos/rds-init-fn-code/index.js index 4a72d9d..c83ae80 100644 --- a/demos/rds-init-fn-code/index.js +++ b/demos/rds-init-fn-code/index.js @@ -27,15 +27,7 @@ exports.handler = async (e) => { let connection try { const { config } = e.params - const { password, username, dbname, port } = await getSecretValue(config.credsSecretName) - connection = await createConnectionWithRetry({ - host: hostname, - user: username, - database: dbname, - port, - password, - multipleStatements: true - }) + connection = await createConnectionWithRetry(config.credsSecretName) const sqlScript = fs.readFileSync(path.join(__dirname, 'script.sql')).toString() const res = await query(connection, sqlScript) @@ -70,7 +62,15 @@ function query (connection, sql) { async function createConnectionWithRetry (connectionConfig) { let lastError for (let attempt = 1; attempt <= DB_CONNECT_RETRY_MAX_ATTEMPTS; attempt += 1) { - const connection = mysql.createConnection(connectionConfig) + const { password, username, dbname, port } = await getSecretValue(connectionConfig) + const connection = mysql.createConnection({ + host: hostname, + user: username, + database: dbname, + port, + password, + multipleStatements: true + }) try { await connect(connection) return connection @@ -82,7 +82,7 @@ async function createConnectionWithRetry (connectionConfig) { } const retryInSeconds = DB_CONNECT_RETRY_DELAY_MS / 1000 - console.log(`Database connection attempt ${attempt}/${DB_CONNECT_RETRY_MAX_ATTEMPTS} failed with '${error.code || error.message}'. Retrying in ${retryInSeconds}s...`) + console.log(`Database connection attempt ${attempt}/${DB_CONNECT_RETRY_MAX_ATTEMPTS} failed (port=${port}) with '${error.code || error.message}'. Retrying in ${retryInSeconds}s...`) await sleep(DB_CONNECT_RETRY_DELAY_MS) } }