another supporting blog

Author: paddymul

docs/example-notebooks/big_file_blog/BigFile-initial.ipynb

@@ -0,0 +1,125 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "4bacd1ad-6658-4e69-b596-d9db5a0a2201",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Buckaroo has been enabled as the default DataFrame viewer.  To return to default dataframe visualization use `from buckaroo import disable; disable()`\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "import polars as pl\n",
+    "import buckaroo\n",
+    "JULY_FILE = \"~/NPPES_Data_Dissemination_July_2025/npidata_pfile_20050523-20250713.csv\"\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2c21c2b4-8d86-4d2e-a7df-5022a0ed296c",
+   "metadata": {},
+   "source": [
+    "# Lets investigate this file\n",
+    "We are going to use some unix command line utils.  These are generally going to be very fast and memory efficient"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "8eabb58f-ce10-4ac3-b5eb-99e1a2ada843",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      " 10G\t/Users/paddy/NPPES_Data_Dissemination_July_2025/npidata_pfile_20050523-20250713.csv\n"
+     ]
+    }
+   ],
+   "source": [
+    "!du -h /Users/paddy/NPPES_Data_Dissemination_July_2025/npidata_pfile_20050523-20250713.csv"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "1338b817-097d-4155-a131-cf5b011a8ccc",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "cat  > /dev/null  0.04s user 1.67s system 43% cpu 3.995 total\n"
+     ]
+    }
+   ],
+   "source": [
+    "!time cat /Users/paddy/NPPES_Data_Dissemination_July_2025/npidata_pfile_20050523-20250713.csv > /dev/null"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "15037796-b479-493d-b9ee-00ddbe69189b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      " 9026997 /Users/paddy/NPPES_Data_Dissemination_July_2025/npidata_pfile_20050523-20250713.csv\n",
+      "wc -l ~/NPPES_Data_Dissemination_July_2025/npidata_pfile_20050523-20250713.cs  4.12s user 1.70s system 87% cpu 6.675 total\n"
+     ]
+    }
+   ],
+   "source": [
+    "!time wc -l ~/NPPES_Data_Dissemination_July_2025/npidata_pfile_20050523-20250713.csv"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "74450ee4-c29a-4230-8ed3-e2e3fa987485",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.8"
+  },
+  "widgets": {
+   "application/vnd.jupyter.widget-state+json": {
+    "state": {},
+    "version_major": 2,
+    "version_minor": 0
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}