tweaks

- add version command
- fix large uploads
- conditionally load .env
- fix local paths

Author: robb-j

cli/README.md

@@ -0,0 +1,68 @@
+# maps cli
+
+This directory is a Node.js CLI for extracting a set of pmtiles
+and uploading them to an S3 bucket.
+
+## usage
+
+You can run the `main.ts` as an executable if you have Node.js 22+ installed.
+It will load the `.env` next to it
+
+```
+main.ts <command>
+
+Commands:
+  main.ts config  Output config and usage
+  main.ts run     Run the tool
+
+Options:
+  --version  Show version number                                       [boolean]
+  --help     Show help                                                 [boolean]
+```
+
+## configuration
+
+These are the configurations you can set for how the CLI behaves.
+Set them in a `config.json` in this directory or with the corresponding CLI flags or environment variables.
+powered by [Gruber](https://gruber.r0b.io).
+
+| name               | type   | flag | variable               | fallback                                         |
+| ------------------ | ------ | ---- | ---------------------- | ------------------------------------------------ |
+| env                | string | ~    | NODE_ENV               | production                                       |
+| meta.name          | string | ~    | APP_NAME               | @openlab/protomaps-cli                           |
+| meta.version       | string | ~    | APP_VERSION            | 0.0.0                                            |
+| protomaps.builds   | url    | ~    | PROTOMAPS_BUILDS_URL   | https://build.protomaps.com/                     |
+| protomaps.metadata | url    | ~    | PROTOMAPS_METADATA_URL | https://build-metadata.protomaps.dev/builds.json |
+| s3.accessKey       | string | ~    | S3_ACCESS_KEY          | gruber://unset                                   |
+| s3.bucketName      | string | ~    | S3_BUCKET_NAME         | gruber://unset                                   |
+| s3.endpoint        | url    | ~    | S3_ENDPOINT            | gruber://unset                                   |
+| s3.prefix          | string | ~    | S3_PREFIX              |                                                  |
+| s3.secretKey       | string | ~    | S3_SECRET_KEY          | gruber://unset                                   |
+
+Default:
+
+```json
+{
+  "env": "development",
+  "meta": {
+    "name": "@openlab/protomaps-cli",
+    "version": "0.0.0"
+  },
+  "targets": [],
+  "protomaps": {
+    "metadata": "https://build-metadata.protomaps.dev/builds.json",
+    "builds": "https://build.protomaps.com/"
+  },
+  "s3": {
+    "prefix": "",
+    "accessKey": "gruber://unset",
+    "secretKey": "gruber://unset",
+    "bucketName": "gruber://unset",
+    "endpoint": "gruber://unset"
+  }
+}
+```
+
+## notes
+
+The CLI relies on minio `8.0.4` **specifically**, this is to remedy [this issue](https://github.com/minio/minio-js/issues/1395).

cli/config.json

@@ -3,6 +3,10 @@
 		{
 			"name": "ncl.pmtiles",
 			"bbox": [-2.072468, 54.730692, -1.112537, 55.248329]
+		},
+		{
+			"name": "uk.pmtiles",
+			"bbox": [-13.736773, 49.69262, 5.55685, 61.29407]
 		}
 	],
 	"s3": {

cli/config.ts

@@ -1,6 +1,8 @@
 import { getConfiguration, Structure } from "gruber";
 import pkg from "./package.json" with { type: "json" };
+import { loadEnvFile } from "./lib.ts";
 
+loadEnvFile();
 const config = getConfiguration();
 
 const UNSET = "gruber://unset";

cli/lib.ts

@@ -1,4 +1,10 @@
 import cp from "node:child_process";
+import fs from "node:fs";
+import process from "node:process";
 import { promisify } from "node:util";
 
 export const exec = promisify(cp.exec);
+
+export function loadEnvFile() {
+	if (fs.existsSync(".env")) process.loadEnvFile(".env");
+}

cli/main.ts

@@ -7,12 +7,13 @@ import process from "node:process";
 import yargs from "yargs";
 import { hideBin } from "yargs/helpers";
 
-import { outputConfiguration } from "./config.ts";
+import { appConfig, outputConfiguration } from "./config.ts";
 import { runTool } from "./run.ts";
 
 const cli = yargs()
 	.demandCommand(1, "A command is required")
 	.recommendCommands()
+	.version(appConfig.meta.version)
 	.help();
 
 cli.command(

cli/package.json

@@ -9,7 +9,7 @@
 	},
 	"dependencies": {
 		"gruber": "^0.9.0-beta.1",
-		"minio": "^8.0.5",
+		"minio": "8.0.4",
 		"yargs": "^18.0.0"
 	},
 	"devDependencies": {

cli/run.ts

@@ -1,4 +1,5 @@
 import fs from "node:fs";
+import { fileURLToPath } from "node:url";
 import { appConfig } from "./config.ts";
 import * as Minio from "minio";
 import { exec } from "./lib.ts";
@@ -62,22 +63,22 @@ export async function runTool(options: RunOptions) {
 		// Work out where to put the build locally & see if it already exists
 		const local = new URL(`../tiles/${target.name}`, import.meta.url);
 		let stat = await fs.promises.stat(local).catch(() => null);
-		console.log("local=%o", stat?.isFile() ?? false);
+		console.log("local=%o ", stat?.isFile() ?? false);
 
 		// If it doesn't exist, use the pmtiles CLI to extract and download it
 		if (!stat?.isFile()) {
 			const command = [
 				"pmtiles",
 				"extract",
 				url.toString(),
-				local,
+				fileURLToPath(local),
 				"--bbox=" + target.bbox.join(","),
 			].join(" ");
 
 			if (options.dryRun) {
 				console.log("dry-run:\n  ", command);
 			} else {
-				console.log("downloading file");
+				console.log("downloading file=%o", fileURLToPath(local));
 				await exec(command);
 			}
 		}
@@ -96,16 +97,24 @@ export async function runTool(options: RunOptions) {
 		if (options.dryRun) {
 			console.log("dry-run: upload to s3 object=%o", tilesName);
 		} else {
-			console.log("upload", tilesName);
+			const metadata = {
+				...appConfig.s3.objectMetadata,
+				"content-type": "application/vnd.pmtiles",
+			};
+
+			console.log(
+				"upload name=%o chunk=%o metadata=%O",
+				tilesName,
+				s3.calculatePartSize(stat!.size),
+				metadata,
+			);
+
 			await s3.putObject(
 				appConfig.s3.bucketName,
 				tilesName,
 				stream,
 				stat!.size,
-				{
-					...appConfig.s3.objectMetadata,
-					"content-type": "application/vnd.pmtiles",
-				},
+				metadata,
 			);
 		}