Skip to content

Commit 9281088

Browse files
ArtmannArtmann
authored
feat: Watch snapshot files and update notebook outputs (#331)
1 parent b99e42e commit 9281088

8 files changed

Lines changed: 1685 additions & 244 deletions

‎src/kernels/deepnote/environments/deepnoteExtensionSidecarWriter.node.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { inject, injectable } from 'inversify';
2-
import * as yaml from 'js-yaml';
2+
import * as YAML from 'yaml';
33
import { env, NotebookDocument, Uri, workspace } from 'vscode';
44

55
import { IExtensionSyncActivationService } from '../../../platform/activation/types';
@@ -307,7 +307,7 @@ export class DeepnoteExtensionSidecarWriter implements IExtensionSyncActivationS
307307
private async readProjectIdFromFile(fileUri: Uri): Promise<string | undefined> {
308308
try {
309309
const raw = await workspace.fs.readFile(fileUri);
310-
const parsed = yaml.load(Buffer.from(raw).toString('utf-8')) as { project?: { id?: string } } | undefined;
310+
const parsed = YAML.parse(Buffer.from(raw).toString('utf-8')) as { project?: { id?: string } } | undefined;
311311
return parsed?.project?.id;
312312
} catch {
313313
return undefined;

‎src/notebooks/deepnote/deepnoteDataConverter.ts‎

Lines changed: 161 additions & 142 deletions
Original file line numberDiff line numberDiff line change
@@ -55,23 +55,6 @@ export class DeepnoteDataConverter {
5555
this.registry.register(new VisualizationBlockConverter());
5656
}
5757

58-
/**
59-
* Initialize async dependencies like vega-lite.
60-
* Must be called before using output conversion methods.
61-
*/
62-
async initialize(): Promise<void> {
63-
await ensureVegaLiteLoaded();
64-
}
65-
66-
/**
67-
* Finds a converter for the given block type.
68-
* @param blockType The type of block to find a converter for
69-
* @returns The converter if found, undefined otherwise
70-
*/
71-
public findConverter(blockType: string): BlockConverter | undefined {
72-
return this.registry.findConverter(blockType);
73-
}
74-
7558
/**
7659
* Converts Deepnote blocks to VS Code notebook cells.
7760
* Sorts blocks by sortingKey before conversion to maintain proper order.
@@ -178,135 +161,24 @@ export class DeepnoteDataConverter {
178161
return cells.map((cell, index) => this.convertCellToBlock(cell, index));
179162
}
180163

181-
private base64ToUint8Array(base64: string): Uint8Array {
182-
const binaryString = atob(base64);
183-
const bytes = new Uint8Array(binaryString.length);
184-
185-
for (let i = 0; i < binaryString.length; i++) {
186-
bytes[i] = binaryString.charCodeAt(i);
187-
}
188-
189-
return bytes;
190-
}
191-
192-
private createFallbackBlock(cell: NotebookCellData, index: number): DeepnoteBlock {
193-
return {
194-
blockGroup: uuidUtils.generateUuid(),
195-
id: generateBlockId(),
196-
sortingKey: generateSortingKey(index),
197-
type: cell.kind === NotebookCellKind.Code ? 'code' : 'markdown',
198-
content: cell.value || '',
199-
metadata: {}
200-
};
201-
}
202-
203-
private createFallbackCell(block: DeepnoteBlock): NotebookCellData {
204-
const cell = new NotebookCellData(NotebookCellKind.Markup, block.content || '', 'markdown');
205-
206-
cell.metadata = {
207-
deepnoteBlockId: block.id,
208-
deepnoteBlockType: block.type,
209-
deepnoteSortingKey: block.sortingKey,
210-
deepnoteMetadata: block.metadata
211-
};
212-
213-
return cell;
164+
/**
165+
* Finds a converter for the given block type.
166+
* @param blockType The type of block to find a converter for
167+
* @returns The converter if found, undefined otherwise
168+
*/
169+
public findConverter(blockType: string): BlockConverter | undefined {
170+
return this.registry.findConverter(blockType);
214171
}
215172

216-
private transformOutputsForDeepnote(outputs: NotebookCellOutput[]): DeepnoteOutput[] {
217-
return outputs.map((output) => {
218-
// Check if this is an error output
219-
const errorItem = output.items.find((item) => item.mime === 'application/vnd.code.notebook.error');
220-
221-
if (errorItem) {
222-
try {
223-
const errorData = JSON.parse(new TextDecoder().decode(errorItem.data));
224-
225-
return {
226-
ename: errorData.name || 'Error',
227-
evalue: errorData.message || '',
228-
output_type: 'error',
229-
traceback: errorData.stack ? errorData.stack.split('\n') : []
230-
} as DeepnoteOutput;
231-
} catch {
232-
return {
233-
ename: 'Error',
234-
evalue: '',
235-
output_type: 'error',
236-
traceback: []
237-
} as DeepnoteOutput;
238-
}
239-
}
240-
241-
// Check if this is a stream output
242-
const stdoutItem = output.items.find((item) => item.mime === 'application/vnd.code.notebook.stdout');
243-
const stderrItem = output.items.find((item) => item.mime === 'application/vnd.code.notebook.stderr');
244-
245-
if (stdoutItem || stderrItem) {
246-
const item = stdoutItem || stderrItem;
247-
const text = new TextDecoder().decode(item!.data);
248-
249-
return {
250-
name: stderrItem ? 'stderr' : 'stdout',
251-
output_type: 'stream',
252-
text
253-
} as DeepnoteOutput;
254-
}
255-
256-
// Rich output (execute_result or display_data)
257-
const data: Record<string, unknown> = {};
258-
259-
for (const item of output.items) {
260-
if (item.mime === 'text/plain') {
261-
data['text/plain'] = new TextDecoder().decode(item.data);
262-
} else if (item.mime === 'text/markdown') {
263-
data['text/markdown'] = new TextDecoder().decode(item.data);
264-
} else if (item.mime === 'text/html') {
265-
data['text/html'] = new TextDecoder().decode(item.data);
266-
} else if (item.mime === 'application/json') {
267-
data['application/json'] = JSON.parse(new TextDecoder().decode(item.data));
268-
} else if (item.mime === 'image/png') {
269-
data['image/png'] = this.uint8ArrayToBase64(item.data);
270-
} else if (item.mime === 'image/jpeg') {
271-
data['image/jpeg'] = this.uint8ArrayToBase64(item.data);
272-
} else if (item.mime === 'application/vnd.deepnote.dataframe.v3+json') {
273-
data['application/vnd.deepnote.dataframe.v3+json'] = JSON.parse(
274-
new TextDecoder().decode(item.data)
275-
);
276-
} else if (item.mime === 'application/vnd.vega.v6+json') {
277-
data['application/vnd.vega.v6+json'] = JSON.parse(new TextDecoder().decode(item.data));
278-
} else if (item.mime === 'application/vnd.vega.v5+json') {
279-
data['application/vnd.vega.v5+json'] = JSON.parse(new TextDecoder().decode(item.data));
280-
} else if (item.mime === 'application/vnd.plotly.v1+json') {
281-
data['application/vnd.plotly.v1+json'] = JSON.parse(new TextDecoder().decode(item.data));
282-
} else if (item.mime === 'application/vnd.deepnote.sql-output-metadata+json') {
283-
data['application/vnd.deepnote.sql-output-metadata+json'] = JSON.parse(
284-
new TextDecoder().decode(item.data)
285-
);
286-
}
287-
}
288-
289-
const deepnoteOutput: DeepnoteOutput = {
290-
data,
291-
execution_count: (output.metadata?.executionCount as number) || 0,
292-
output_type: 'execute_result'
293-
};
294-
295-
// Add metadata if present (excluding executionCount which we already handled)
296-
if (output.metadata) {
297-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
298-
const { executionCount, ...restMetadata } = output.metadata;
299-
300-
if (Object.keys(restMetadata).length > 0) {
301-
(deepnoteOutput as DeepnoteOutput & { metadata?: Record<string, unknown> }).metadata = restMetadata;
302-
}
303-
}
304-
305-
return deepnoteOutput;
306-
});
173+
/**
174+
* Initialize async dependencies like vega-lite.
175+
* Must be called before using output conversion methods.
176+
*/
177+
async initialize(): Promise<void> {
178+
await ensureVegaLiteLoaded();
307179
}
308180

309-
private transformOutputsForVsCode(
181+
public transformOutputsForVsCode(
310182
outputs: DeepnoteOutput[],
311183
cellIndex: number,
312184
cellId: string,
@@ -552,6 +424,153 @@ export class DeepnoteDataConverter {
552424
});
553425
}
554426

427+
private base64ToUint8Array(base64: string): Uint8Array {
428+
const binaryString = atob(base64);
429+
const bytes = new Uint8Array(binaryString.length);
430+
431+
for (let i = 0; i < binaryString.length; i++) {
432+
bytes[i] = binaryString.charCodeAt(i);
433+
}
434+
435+
return bytes;
436+
}
437+
438+
private createFallbackBlock(cell: NotebookCellData, index: number): DeepnoteBlock {
439+
const meta = cell.metadata as Record<string, unknown> | undefined;
440+
const preservedId = (meta?.id ?? meta?.__deepnoteBlockId ?? meta?.deepnoteBlockId) as string | undefined;
441+
const preservedSortingKey = (meta?.sortingKey ?? meta?.deepnoteSortingKey) as string | undefined;
442+
const preservedBlockGroup = meta?.blockGroup as string | undefined;
443+
444+
return {
445+
blockGroup: preservedBlockGroup ?? uuidUtils.generateUuid(),
446+
id: preservedId ?? generateBlockId(),
447+
sortingKey: preservedSortingKey ?? generateSortingKey(index),
448+
type: cell.kind === NotebookCellKind.Code ? 'code' : 'markdown',
449+
content: cell.value || '',
450+
metadata: {}
451+
};
452+
}
453+
454+
private createFallbackCell(block: DeepnoteBlock): NotebookCellData {
455+
const cell = new NotebookCellData(NotebookCellKind.Markup, block.content || '', 'markdown');
456+
457+
cell.metadata = {
458+
...(block.metadata ?? {}),
459+
id: block.id,
460+
__deepnoteBlockId: block.id,
461+
type: block.type,
462+
sortingKey: block.sortingKey,
463+
deepnoteBlockId: block.id,
464+
deepnoteBlockType: block.type,
465+
deepnoteSortingKey: block.sortingKey,
466+
deepnoteMetadata: block.metadata
467+
};
468+
469+
return cell;
470+
}
471+
472+
private transformOutputsForDeepnote(outputs: NotebookCellOutput[]): DeepnoteOutput[] {
473+
return outputs.map((output) => {
474+
// Check if this is an error output
475+
const errorItem = output.items.find((item) => item.mime === 'application/vnd.code.notebook.error');
476+
477+
if (errorItem) {
478+
try {
479+
const errorData = JSON.parse(new TextDecoder().decode(errorItem.data));
480+
481+
return {
482+
ename: errorData.name || 'Error',
483+
evalue: errorData.message || '',
484+
output_type: 'error',
485+
traceback: errorData.stack ? errorData.stack.split('\n') : []
486+
} as DeepnoteOutput;
487+
} catch {
488+
return {
489+
ename: 'Error',
490+
evalue: '',
491+
output_type: 'error',
492+
traceback: []
493+
} as DeepnoteOutput;
494+
}
495+
}
496+
497+
// Check if this is a stream output
498+
const stdoutItem = output.items.find((item) => item.mime === 'application/vnd.code.notebook.stdout');
499+
const stderrItem = output.items.find((item) => item.mime === 'application/vnd.code.notebook.stderr');
500+
501+
if (stdoutItem || stderrItem) {
502+
const item = stdoutItem || stderrItem;
503+
const text = new TextDecoder().decode(item!.data);
504+
505+
return {
506+
name: stderrItem ? 'stderr' : 'stdout',
507+
output_type: 'stream',
508+
text
509+
} as DeepnoteOutput;
510+
}
511+
512+
// Rich output (execute_result or display_data)
513+
const data: Record<string, unknown> = {};
514+
515+
for (const item of output.items) {
516+
try {
517+
if (item.mime === 'text/plain') {
518+
data['text/plain'] = new TextDecoder().decode(item.data);
519+
} else if (item.mime === 'text/markdown') {
520+
data['text/markdown'] = new TextDecoder().decode(item.data);
521+
} else if (item.mime === 'text/html') {
522+
data['text/html'] = new TextDecoder().decode(item.data);
523+
} else if (item.mime === CHART_BIG_NUMBER_MIME_TYPE) {
524+
data['text/plain'] = new TextDecoder().decode(item.data);
525+
} else if (item.mime === 'application/json') {
526+
data['application/json'] = JSON.parse(new TextDecoder().decode(item.data));
527+
} else if (item.mime === 'image/png') {
528+
data['image/png'] = this.uint8ArrayToBase64(item.data);
529+
} else if (item.mime === 'image/jpeg') {
530+
data['image/jpeg'] = this.uint8ArrayToBase64(item.data);
531+
} else if (item.mime === 'application/vnd.deepnote.dataframe.v3+json') {
532+
data['application/vnd.deepnote.dataframe.v3+json'] = JSON.parse(
533+
new TextDecoder().decode(item.data)
534+
);
535+
} else if (item.mime === 'application/vnd.vega.v6+json') {
536+
data['application/vnd.vega.v6+json'] = JSON.parse(new TextDecoder().decode(item.data));
537+
} else if (item.mime === 'application/vnd.vega.v5+json') {
538+
data['application/vnd.vega.v5+json'] = JSON.parse(new TextDecoder().decode(item.data));
539+
} else if (item.mime === 'application/vnd.plotly.v1+json') {
540+
data['application/vnd.plotly.v1+json'] = JSON.parse(new TextDecoder().decode(item.data));
541+
} else if (item.mime === 'application/vnd.deepnote.sql-output-metadata+json') {
542+
data['application/vnd.deepnote.sql-output-metadata+json'] = JSON.parse(
543+
new TextDecoder().decode(item.data)
544+
);
545+
}
546+
} catch (e) {
547+
console.warn(`Failed to convert output item mime=${item.mime}`, e);
548+
}
549+
}
550+
551+
const deepnoteOutput: DeepnoteOutput = {
552+
data,
553+
execution_count: (output.metadata?.executionCount as number) || 0,
554+
output_type: 'execute_result'
555+
};
556+
557+
// Add metadata if present (excluding executionCount which we already handled)
558+
if (output.metadata) {
559+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
560+
const { executionCount, cellId, cellIndex, ...restMetadata } = output.metadata as Record<
561+
string,
562+
unknown
563+
>;
564+
565+
if (Object.keys(restMetadata).length > 0) {
566+
(deepnoteOutput as DeepnoteOutput & { metadata?: Record<string, unknown> }).metadata = restMetadata;
567+
}
568+
}
569+
570+
return deepnoteOutput;
571+
});
572+
}
573+
555574
/**
556575
* Converts a Uint8Array to a base64 string without causing stack overflow.
557576
* Uses chunked processing to avoid call stack limits with large arrays.

0 commit comments

Comments
 (0)