Skip to content

Commit a33bb1f

Browse files
ImTotemclaude
ImTotem
and
claude
committed
perf(workflow): use Sheets API batchUpdate/pasteData for sync
Replace Clear + row-by-row Append with single batchUpdate call containing updateCells (clear) + pasteData (CSV paste). Per changed table: 1 Sheets API call instead of 2+. Unchanged tables: 0 calls (mod count change detection). Flow: PG mod counts → filter changed → get sheet IDs → loop: ensure sheet → dump PG → CSV → batchUpdate(clear+paste) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 19bdd5c commit a33bb1f

1 file changed

Lines changed: 76 additions & 43 deletions

File tree

‎workflows/pg_sheets_sync.json‎

Lines changed: 76 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
},
3030
{
3131
"parameters": {
32-
"jsCode": "const staticData = $getWorkflowStaticData('global');\nif (!staticData.mods) staticData.mods = {};\n\nconst changed = [];\nfor (const item of $input.all()) {\n const name = item.json.table_name;\n const mods = item.json.mod_count;\n if (staticData.mods[name] !== mods) {\n staticData.mods[name] = mods;\n changed.push({ json: { table_name: name } });\n }\n}\n\nif (changed.length === 0) {\n return [{ json: { table_name: '__none__', skipped: true } }];\n}\nreturn changed;"
32+
"jsCode": "const staticData = $getWorkflowStaticData('global');\nif (!staticData.mods) staticData.mods = {};\n\nconst changed = [];\nfor (const item of $input.all()) {\n const name = item.json.table_name;\n const mods = item.json.mod_count;\n if (staticData.mods[name] !== mods) {\n staticData.mods[name] = mods;\n changed.push({ json: { table_name: name } });\n }\n}\n\nif (changed.length === 0) return [{ json: { _skip: true } }];\nreturn changed;"
3333
},
3434
"id": "filter-changed",
3535
"name": "Filter Changed Tables",
@@ -41,31 +41,48 @@
4141
"parameters": {
4242
"conditions": {
4343
"options": {"caseSensitive": true, "leftValue": "", "typeValidation": "strict"},
44-
"conditions": [
45-
{
46-
"id": "not-skipped",
47-
"leftValue": "={{ $json.skipped }}",
48-
"rightValue": true,
49-
"operator": {"type": "boolean", "operation": "notEquals"}
50-
}
51-
],
44+
"conditions": [{"id": "c1", "leftValue": "={{ $json._skip }}", "rightValue": true, "operator": {"type": "boolean", "operation": "notEquals"}}],
5245
"combinator": "and"
5346
},
5447
"options": {}
5548
},
56-
"id": "if-has-changes",
49+
"id": "if-changed",
5750
"name": "Any Changes?",
5851
"type": "n8n-nodes-base.if",
5952
"typeVersion": 2.3,
6053
"position": [660, 0]
6154
},
55+
{
56+
"parameters": {
57+
"method": "GET",
58+
"url": "=https://sheets.googleapis.com/v4/spreadsheets/{{ $env.GOOGLE_SHEETS_ID }}?fields=sheets.properties",
59+
"authentication": "predefinedCredentialType",
60+
"nodeCredentialType": "googleSheetsApi",
61+
"options": {}
62+
},
63+
"id": "get-sheet-ids",
64+
"name": "Get Sheet IDs",
65+
"type": "n8n-nodes-base.httpRequest",
66+
"typeVersion": 4.2,
67+
"position": [880, -100]
68+
},
69+
{
70+
"parameters": {
71+
"jsCode": "const sheetsResponse = $('Get Sheet IDs').first().json;\nconst sheetMap = {};\nfor (const s of sheetsResponse.sheets) {\n sheetMap[s.properties.title] = s.properties.sheetId;\n}\n\nconst tables = $('Any Changes?').all().map(i => i.json.table_name);\nconst result = tables.map(t => ({\n json: { table_name: t, sheet_id: sheetMap[t] ?? null, exists: sheetMap[t] != null }\n}));\nreturn result;"
72+
},
73+
"id": "map-sheet-ids",
74+
"name": "Map Sheet IDs",
75+
"type": "n8n-nodes-base.code",
76+
"typeVersion": 2,
77+
"position": [1100, -100]
78+
},
6279
{
6380
"parameters": {"options": {}},
6481
"id": "split-batches",
6582
"name": "Loop Over Tables",
6683
"type": "n8n-nodes-base.splitInBatches",
6784
"typeVersion": 3,
68-
"position": [880, -100]
85+
"position": [1320, -100]
6986
},
7087
{
7188
"parameters": {
@@ -74,24 +91,12 @@
7491
"title": "={{ $json.table_name }}"
7592
},
7693
"id": "sheets-create",
77-
"name": "Ensure Sheet Exists",
94+
"name": "Ensure Sheet",
7895
"type": "n8n-nodes-base.googleSheets",
7996
"typeVersion": 4.7,
80-
"position": [1100, -100],
97+
"position": [1540, -100],
8198
"onError": "continueRegularOutput"
8299
},
83-
{
84-
"parameters": {
85-
"operation": "clear",
86-
"documentId": {"__rl": true, "mode": "id", "value": "={{ $env.GOOGLE_SHEETS_ID }}"},
87-
"sheetName": {"__rl": true, "mode": "name", "value": "={{ $('Loop Over Tables').item.json.table_name }}"}
88-
},
89-
"id": "sheets-clear",
90-
"name": "Clear Sheet",
91-
"type": "n8n-nodes-base.googleSheets",
92-
"typeVersion": 4.7,
93-
"position": [1320, -100]
94-
},
95100
{
96101
"parameters": {
97102
"operation": "executeQuery",
@@ -102,36 +107,64 @@
102107
"name": "Dump Table",
103108
"type": "n8n-nodes-base.postgres",
104109
"typeVersion": 2.6,
105-
"position": [1540, -100]
110+
"position": [1760, -100]
106111
},
107112
{
108113
"parameters": {
109-
"operation": "append",
110-
"documentId": {"__rl": true, "mode": "id", "value": "={{ $env.GOOGLE_SHEETS_ID }}"},
111-
"sheetName": {"__rl": true, "mode": "name", "value": "={{ $('Loop Over Tables').item.json.table_name }}"},
112-
"columns": {
113-
"mappingMode": "autoMapInputData",
114-
"value": {}
114+
"jsCode": "const rows = $input.all().map(i => i.json);\nconst tableName = $('Loop Over Tables').item.json.table_name;\nlet sheetId = $('Loop Over Tables').item.json.sheet_id;\n\nif (rows.length === 0) {\n return [{ json: { _empty: true, table_name: tableName } }];\n}\n\nconst headers = Object.keys(rows[0]);\nconst csvLines = [headers.join(',')];\nfor (const row of rows) {\n const line = headers.map(h => {\n const v = String(row[h] ?? '');\n if (v.includes(',') || v.includes('\"') || v.includes('\\n')) {\n return '\"' + v.replace(/\"/g, '\"\"') + '\"';\n }\n return v;\n }).join(',');\n csvLines.push(line);\n}\n\nconst requests = [];\nif (sheetId != null) {\n requests.push({ updateCells: { range: { sheetId: sheetId }, fields: 'userEnteredValue' } });\n}\nrequests.push({\n pasteData: {\n coordinate: { sheetId: sheetId ?? 0, rowIndex: 0, columnIndex: 0 },\n data: csvLines.join('\\n'),\n type: 'PASTE_NORMAL',\n delimiter: ','\n }\n});\n\nreturn [{ json: { _empty: false, table_name: tableName, body: { requests } } }];"
115+
},
116+
"id": "to-csv",
117+
"name": "Build CSV + Payload",
118+
"type": "n8n-nodes-base.code",
119+
"typeVersion": 2,
120+
"position": [1980, -100]
121+
},
122+
{
123+
"parameters": {
124+
"conditions": {
125+
"options": {"caseSensitive": true, "leftValue": "", "typeValidation": "strict"},
126+
"conditions": [{"id": "c1", "leftValue": "={{ $json._empty }}", "rightValue": true, "operator": {"type": "boolean", "operation": "notEquals"}}],
127+
"combinator": "and"
115128
},
116129
"options": {}
117130
},
118-
"id": "sheets-append",
119-
"name": "Write to Sheet",
120-
"type": "n8n-nodes-base.googleSheets",
121-
"typeVersion": 4.7,
122-
"position": [1760, -100]
131+
"id": "if-not-empty",
132+
"name": "Has Rows?",
133+
"type": "n8n-nodes-base.if",
134+
"typeVersion": 2.3,
135+
"position": [2200, -100]
136+
},
137+
{
138+
"parameters": {
139+
"method": "POST",
140+
"url": "=https://sheets.googleapis.com/v4/spreadsheets/{{ $env.GOOGLE_SHEETS_ID }}:batchUpdate",
141+
"authentication": "predefinedCredentialType",
142+
"nodeCredentialType": "googleSheetsApi",
143+
"sendBody": true,
144+
"specifyBody": "json",
145+
"jsonBody": "={{ JSON.stringify($json.body) }}",
146+
"options": {}
147+
},
148+
"id": "paste-data",
149+
"name": "Paste to Sheet",
150+
"type": "n8n-nodes-base.httpRequest",
151+
"typeVersion": 4.2,
152+
"position": [2420, -200]
123153
}
124154
],
125155
"connections": {
126156
"Every 5 Minutes": {"main": [[{"node": "List Tables + Mod Counts", "type": "main", "index": 0}]]},
127157
"List Tables + Mod Counts": {"main": [[{"node": "Filter Changed Tables", "type": "main", "index": 0}]]},
128158
"Filter Changed Tables": {"main": [[{"node": "Any Changes?", "type": "main", "index": 0}]]},
129-
"Any Changes?": {"main": [[{"node": "Loop Over Tables", "type": "main", "index": 0}], []]},
130-
"Loop Over Tables": {"main": [[], [{"node": "Ensure Sheet Exists", "type": "main", "index": 0}]]},
131-
"Ensure Sheet Exists": {"main": [[{"node": "Clear Sheet", "type": "main", "index": 0}]]},
132-
"Clear Sheet": {"main": [[{"node": "Dump Table", "type": "main", "index": 0}]]},
133-
"Dump Table": {"main": [[{"node": "Write to Sheet", "type": "main", "index": 0}]]},
134-
"Write to Sheet": {"main": [[{"node": "Loop Over Tables", "type": "main", "index": 0}]]}
159+
"Any Changes?": {"main": [[{"node": "Get Sheet IDs", "type": "main", "index": 0}], []]},
160+
"Get Sheet IDs": {"main": [[{"node": "Map Sheet IDs", "type": "main", "index": 0}]]},
161+
"Map Sheet IDs": {"main": [[{"node": "Loop Over Tables", "type": "main", "index": 0}]]},
162+
"Loop Over Tables": {"main": [[], [{"node": "Ensure Sheet", "type": "main", "index": 0}]]},
163+
"Ensure Sheet": {"main": [[{"node": "Dump Table", "type": "main", "index": 0}]]},
164+
"Dump Table": {"main": [[{"node": "Build CSV + Payload", "type": "main", "index": 0}]]},
165+
"Build CSV + Payload": {"main": [[{"node": "Has Rows?", "type": "main", "index": 0}]]},
166+
"Has Rows?": {"main": [[{"node": "Paste to Sheet", "type": "main", "index": 0}], [{"node": "Loop Over Tables", "type": "main", "index": 0}]]},
167+
"Paste to Sheet": {"main": [[{"node": "Loop Over Tables", "type": "main", "index": 0}]]}
135168
},
136169
"settings": {"executionOrder": "v1"}
137170
}

0 commit comments

Comments
 (0)