mirror of
https://github.com/microsoft/vscode.git
synced 2025-12-23 19:59:37 +00:00
Split up ipynb serializer file
This commit is contained in:
42
extensions/ipynb/src/common.ts
Normal file
42
extensions/ipynb/src/common.ts
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
/*---------------------------------------------------------------------------------------------
|
||||||
|
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||||
|
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||||
|
*--------------------------------------------------------------------------------------------*/
|
||||||
|
|
||||||
|
import { nbformat } from '@jupyterlab/coreutils';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Metadata we store in VS Code cell output items.
|
||||||
|
* This contains the original metadata from the Jupyter outputs.
|
||||||
|
*/
|
||||||
|
export interface CellOutputMetadata {
|
||||||
|
/**
|
||||||
|
* Cell output metadata.
|
||||||
|
*/
|
||||||
|
metadata?: any;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Transient data from Jupyter.
|
||||||
|
*/
|
||||||
|
transient?: {
|
||||||
|
/**
|
||||||
|
* This is used for updating the output in other cells.
|
||||||
|
* We don't know of other properties, but this is definitely used.
|
||||||
|
*/
|
||||||
|
display_id?: string;
|
||||||
|
} & any;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Original cell output type
|
||||||
|
*/
|
||||||
|
outputType: nbformat.OutputType | string;
|
||||||
|
|
||||||
|
executionCount?: nbformat.IExecuteResult['ExecutionCount'];
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Whether the original Mime data is JSON or not.
|
||||||
|
* This properly only exists in metadata for NotebookCellOutputItems
|
||||||
|
* (this is something we have added)
|
||||||
|
*/
|
||||||
|
__isJson?: boolean;
|
||||||
|
}
|
||||||
347
extensions/ipynb/src/deserializers.ts
Normal file
347
extensions/ipynb/src/deserializers.ts
Normal file
@@ -0,0 +1,347 @@
|
|||||||
|
/*---------------------------------------------------------------------------------------------
|
||||||
|
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||||
|
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||||
|
*--------------------------------------------------------------------------------------------*/
|
||||||
|
|
||||||
|
import { nbformat } from '@jupyterlab/coreutils';
|
||||||
|
import { extensions, NotebookCellData, NotebookCellExecutionSummary, NotebookCellKind, NotebookCellOutput, NotebookCellOutputItem, NotebookData } from 'vscode';
|
||||||
|
import { CellOutputMetadata } from './common';
|
||||||
|
|
||||||
|
const jupyterLanguageToMonacoLanguageMapping = new Map([
|
||||||
|
['c#', 'csharp'],
|
||||||
|
['f#', 'fsharp'],
|
||||||
|
['q#', 'qsharp'],
|
||||||
|
['c++11', 'c++'],
|
||||||
|
['c++12', 'c++'],
|
||||||
|
['c++14', 'c++']
|
||||||
|
]);
|
||||||
|
|
||||||
|
export function getPreferredLanguage(metadata?: nbformat.INotebookMetadata) {
|
||||||
|
const jupyterLanguage =
|
||||||
|
metadata?.language_info?.name ||
|
||||||
|
(metadata?.kernelspec as any)?.language;
|
||||||
|
|
||||||
|
// Default to python language only if the Python extension is installed.
|
||||||
|
const defaultLanguage = extensions.getExtension('ms-python.python') ? 'python' : 'plaintext';
|
||||||
|
|
||||||
|
// Note, whatever language is returned here, when the user selects a kernel, the cells (of blank documents) get updated based on that kernel selection.
|
||||||
|
return translateKernelLanguageToMonaco(jupyterLanguage || defaultLanguage);
|
||||||
|
}
|
||||||
|
|
||||||
|
function translateKernelLanguageToMonaco(language: string): string {
|
||||||
|
language = language.toLowerCase();
|
||||||
|
if (language.length === 2 && language.endsWith('#')) {
|
||||||
|
return `${language.substring(0, 1)}sharp`;
|
||||||
|
}
|
||||||
|
return jupyterLanguageToMonacoLanguageMapping.get(language) || language;
|
||||||
|
}
|
||||||
|
|
||||||
|
const orderOfMimeTypes = [
|
||||||
|
'application/vnd.*',
|
||||||
|
'application/vdom.*',
|
||||||
|
'application/geo+json',
|
||||||
|
'application/x-nteract-model-debug+json',
|
||||||
|
'text/html',
|
||||||
|
'application/javascript',
|
||||||
|
'image/gif',
|
||||||
|
'text/latex',
|
||||||
|
'text/markdown',
|
||||||
|
'image/png',
|
||||||
|
'image/svg+xml',
|
||||||
|
'image/jpeg',
|
||||||
|
'application/json',
|
||||||
|
'text/plain'
|
||||||
|
];
|
||||||
|
|
||||||
|
function sortOutputItemsBasedOnDisplayOrder(outputItems: NotebookCellOutputItem[]): NotebookCellOutputItem[] {
|
||||||
|
return outputItems.sort((outputItemA, outputItemB) => {
|
||||||
|
const isMimeTypeMatch = (value: string, compareWith: string) => {
|
||||||
|
if (value.endsWith('.*')) {
|
||||||
|
value = value.substr(0, value.indexOf('.*'));
|
||||||
|
}
|
||||||
|
return compareWith.startsWith(value);
|
||||||
|
};
|
||||||
|
const indexOfMimeTypeA = orderOfMimeTypes.findIndex(mime => isMimeTypeMatch(outputItemA.mime, mime));
|
||||||
|
const indexOfMimeTypeB = orderOfMimeTypes.findIndex(mime => isMimeTypeMatch(outputItemB.mime, mime));
|
||||||
|
return indexOfMimeTypeA - indexOfMimeTypeB;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
enum CellOutputMimeTypes {
|
||||||
|
error = 'application/vnd.code.notebook.error',
|
||||||
|
stderr = 'application/vnd.code.notebook.stderr',
|
||||||
|
stdout = 'application/vnd.code.notebook.stdout'
|
||||||
|
}
|
||||||
|
|
||||||
|
const textMimeTypes = ['text/plain', 'text/markdown', CellOutputMimeTypes.stderr, CellOutputMimeTypes.stdout];
|
||||||
|
|
||||||
|
function concatMultilineString(str: string | string[], trim?: boolean): string {
|
||||||
|
const nonLineFeedWhiteSpaceTrim = /(^[\t\f\v\r ]+|[\t\f\v\r ]+$)/g;
|
||||||
|
if (Array.isArray(str)) {
|
||||||
|
let result = '';
|
||||||
|
for (let i = 0; i < str.length; i += 1) {
|
||||||
|
const s = str[i];
|
||||||
|
if (i < str.length - 1 && !s.endsWith('\n')) {
|
||||||
|
result = result.concat(`${s}\n`);
|
||||||
|
} else {
|
||||||
|
result = result.concat(s);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Just trim whitespace. Leave \n in place
|
||||||
|
return trim ? result.replace(nonLineFeedWhiteSpaceTrim, '') : result;
|
||||||
|
}
|
||||||
|
return trim ? str.toString().replace(nonLineFeedWhiteSpaceTrim, '') : str.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
function convertJupyterOutputToBuffer(mime: string, value: unknown): NotebookCellOutputItem {
|
||||||
|
if (!value) {
|
||||||
|
return NotebookCellOutputItem.text('', mime);
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
if (
|
||||||
|
(mime.startsWith('text/') || textMimeTypes.includes(mime)) &&
|
||||||
|
(Array.isArray(value) || typeof value === 'string')
|
||||||
|
) {
|
||||||
|
const stringValue = Array.isArray(value) ? concatMultilineString(value) : value;
|
||||||
|
return NotebookCellOutputItem.text(stringValue, mime);
|
||||||
|
} else if (mime.startsWith('image/') && typeof value === 'string' && mime !== 'image/svg+xml') {
|
||||||
|
// Images in Jupyter are stored in base64 encoded format.
|
||||||
|
// VS Code expects bytes when rendering images.
|
||||||
|
if (typeof Buffer !== 'undefined' && typeof Buffer.from === 'function') {
|
||||||
|
return new NotebookCellOutputItem(Buffer.from(value, 'base64'), mime);
|
||||||
|
} else {
|
||||||
|
const data = Uint8Array.from(atob(value), c => c.charCodeAt(0));
|
||||||
|
return new NotebookCellOutputItem(data, mime);
|
||||||
|
}
|
||||||
|
} else if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
|
||||||
|
return NotebookCellOutputItem.text(JSON.stringify(value), mime);
|
||||||
|
} else {
|
||||||
|
// For everything else, treat the data as strings (or multi-line strings).
|
||||||
|
value = Array.isArray(value) ? concatMultilineString(value) : value;
|
||||||
|
return NotebookCellOutputItem.text(value as string, mime);
|
||||||
|
}
|
||||||
|
} catch (ex) {
|
||||||
|
return NotebookCellOutputItem.error(ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Metadata we store in VS Code cells.
|
||||||
|
* This contains the original metadata from the Jupyuter cells.
|
||||||
|
*/
|
||||||
|
interface CellMetadata {
|
||||||
|
/**
|
||||||
|
* Stores attachments for cells.
|
||||||
|
*/
|
||||||
|
attachments?: nbformat.IAttachments;
|
||||||
|
/**
|
||||||
|
* Stores cell metadata.
|
||||||
|
*/
|
||||||
|
metadata?: Partial<nbformat.ICellMetadata>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getNotebookCellMetadata(cell: nbformat.IBaseCell): CellMetadata {
|
||||||
|
// We put this only for VSC to display in diff view.
|
||||||
|
// Else we don't use this.
|
||||||
|
const propertiesToClone: (keyof CellMetadata)[] = ['metadata', 'attachments'];
|
||||||
|
const custom: CellMetadata = {};
|
||||||
|
propertiesToClone.forEach((propertyToClone) => {
|
||||||
|
if (cell[propertyToClone]) {
|
||||||
|
custom[propertyToClone] = JSON.parse(JSON.stringify(cell[propertyToClone]));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return custom;
|
||||||
|
}
|
||||||
|
function getOutputMetadata(output: nbformat.IOutput): CellOutputMetadata {
|
||||||
|
// Add on transient data if we have any. This should be removed by our save functions elsewhere.
|
||||||
|
const metadata: CellOutputMetadata = {
|
||||||
|
outputType: output.output_type
|
||||||
|
};
|
||||||
|
if (output.transient) {
|
||||||
|
metadata.transient = output.transient;
|
||||||
|
}
|
||||||
|
|
||||||
|
switch (output.output_type as nbformat.OutputType) {
|
||||||
|
case 'display_data':
|
||||||
|
case 'execute_result':
|
||||||
|
case 'update_display_data': {
|
||||||
|
metadata.executionCount = output.execution_count;
|
||||||
|
metadata.metadata = output.metadata ? JSON.parse(JSON.stringify(output.metadata)) : {};
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
return metadata;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
function translateDisplayDataOutput(
|
||||||
|
output: nbformat.IDisplayData | nbformat.IDisplayUpdate | nbformat.IExecuteResult
|
||||||
|
): NotebookCellOutput {
|
||||||
|
// Metadata could be as follows:
|
||||||
|
// We'll have metadata specific to each mime type as well as generic metadata.
|
||||||
|
/*
|
||||||
|
IDisplayData = {
|
||||||
|
output_type: 'display_data',
|
||||||
|
data: {
|
||||||
|
'image/jpg': '/////'
|
||||||
|
'image/png': '/////'
|
||||||
|
'text/plain': '/////'
|
||||||
|
},
|
||||||
|
metadata: {
|
||||||
|
'image/png': '/////',
|
||||||
|
'background': true,
|
||||||
|
'xyz': '///
|
||||||
|
}
|
||||||
|
}
|
||||||
|
*/
|
||||||
|
const metadata = getOutputMetadata(output);
|
||||||
|
const items: NotebookCellOutputItem[] = [];
|
||||||
|
if (output.data) {
|
||||||
|
for (const key in output.data) {
|
||||||
|
items.push(convertJupyterOutputToBuffer(key, output.data[key]));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return new NotebookCellOutput(sortOutputItemsBasedOnDisplayOrder(items), metadata);
|
||||||
|
}
|
||||||
|
|
||||||
|
function translateErrorOutput(output?: nbformat.IError): NotebookCellOutput {
|
||||||
|
output = output || { output_type: 'error', ename: '', evalue: '', traceback: [] };
|
||||||
|
return new NotebookCellOutput(
|
||||||
|
[
|
||||||
|
NotebookCellOutputItem.error({
|
||||||
|
name: output?.ename || '',
|
||||||
|
message: output?.evalue || '',
|
||||||
|
stack: (output?.traceback || []).join('\n')
|
||||||
|
})
|
||||||
|
],
|
||||||
|
{ ...getOutputMetadata(output), originalError: output }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function translateStreamOutput(output: nbformat.IStream): NotebookCellOutput {
|
||||||
|
const value = concatMultilineString(output.text);
|
||||||
|
const item = output.name === 'stderr' ? NotebookCellOutputItem.stderr(value) : NotebookCellOutputItem.stdout(value);
|
||||||
|
return new NotebookCellOutput([item], getOutputMetadata(output));
|
||||||
|
}
|
||||||
|
|
||||||
|
const cellOutputMappers = new Map<nbformat.OutputType, (output: any) => NotebookCellOutput>();
|
||||||
|
cellOutputMappers.set('display_data', translateDisplayDataOutput);
|
||||||
|
cellOutputMappers.set('execute_result', translateDisplayDataOutput);
|
||||||
|
cellOutputMappers.set('update_display_data', translateDisplayDataOutput);
|
||||||
|
cellOutputMappers.set('error', translateErrorOutput);
|
||||||
|
cellOutputMappers.set('stream', translateStreamOutput);
|
||||||
|
|
||||||
|
function jupyterCellOutputToCellOutput(output: nbformat.IOutput): NotebookCellOutput {
|
||||||
|
/**
|
||||||
|
* Stream, `application/x.notebook.stream`
|
||||||
|
* Error, `application/x.notebook.error-traceback`
|
||||||
|
* Rich, { mime: value }
|
||||||
|
*
|
||||||
|
* outputs: [
|
||||||
|
new vscode.NotebookCellOutput([
|
||||||
|
new vscode.NotebookCellOutputItem('application/x.notebook.stream', 2),
|
||||||
|
new vscode.NotebookCellOutputItem('application/x.notebook.stream', 3),
|
||||||
|
]),
|
||||||
|
new vscode.NotebookCellOutput([
|
||||||
|
new vscode.NotebookCellOutputItem('text/markdown', '## header 2'),
|
||||||
|
new vscode.NotebookCellOutputItem('image/svg+xml', [
|
||||||
|
"<svg baseProfile=\"full\" height=\"200\" version=\"1.1\" width=\"300\" xmlns=\"http://www.w3.org/2000/svg\">\n",
|
||||||
|
" <rect fill=\"blue\" height=\"100%\" width=\"100%\"/>\n",
|
||||||
|
" <circle cx=\"150\" cy=\"100\" fill=\"green\" r=\"80\"/>\n",
|
||||||
|
" <text fill=\"white\" font-size=\"60\" text-anchor=\"middle\" x=\"150\" y=\"125\">SVG</text>\n",
|
||||||
|
"</svg>"
|
||||||
|
]),
|
||||||
|
]),
|
||||||
|
]
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
const fn = cellOutputMappers.get(output.output_type as nbformat.OutputType);
|
||||||
|
let result: NotebookCellOutput;
|
||||||
|
if (fn) {
|
||||||
|
result = fn(output);
|
||||||
|
} else {
|
||||||
|
result = translateDisplayDataOutput(output as any);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
function createNotebookCellDataFromRawCell(cell: nbformat.IRawCell): NotebookCellData {
|
||||||
|
const cellData = new NotebookCellData(NotebookCellKind.Code, concatMultilineString(cell.source), 'raw');
|
||||||
|
cellData.outputs = [];
|
||||||
|
cellData.metadata = { custom: getNotebookCellMetadata(cell) };
|
||||||
|
return cellData;
|
||||||
|
}
|
||||||
|
function createNotebookCellDataFromMarkdownCell(cell: nbformat.IMarkdownCell): NotebookCellData {
|
||||||
|
const cellData = new NotebookCellData(
|
||||||
|
NotebookCellKind.Markup,
|
||||||
|
concatMultilineString(cell.source),
|
||||||
|
'markdown'
|
||||||
|
);
|
||||||
|
cellData.outputs = [];
|
||||||
|
cellData.metadata = { custom: getNotebookCellMetadata(cell) };
|
||||||
|
return cellData;
|
||||||
|
}
|
||||||
|
function createNotebookCellDataFromCodeCell(cell: nbformat.ICodeCell, cellLanguage: string): NotebookCellData {
|
||||||
|
const cellOutputs = Array.isArray(cell.outputs) ? cell.outputs : [];
|
||||||
|
const outputs = cellOutputs.map(jupyterCellOutputToCellOutput);
|
||||||
|
const hasExecutionCount = typeof cell.execution_count === 'number' && cell.execution_count > 0;
|
||||||
|
|
||||||
|
const source = concatMultilineString(cell.source);
|
||||||
|
|
||||||
|
const executionSummary: NotebookCellExecutionSummary = hasExecutionCount
|
||||||
|
? { executionOrder: cell.execution_count as number }
|
||||||
|
: {};
|
||||||
|
|
||||||
|
const cellData = new NotebookCellData(NotebookCellKind.Code, source, cellLanguage);
|
||||||
|
|
||||||
|
cellData.outputs = outputs;
|
||||||
|
cellData.metadata = { custom: getNotebookCellMetadata(cell) };
|
||||||
|
cellData.executionSummary = executionSummary;
|
||||||
|
return cellData;
|
||||||
|
}
|
||||||
|
|
||||||
|
function createNotebookCellDataFromJupyterCell(
|
||||||
|
cellLanguage: string,
|
||||||
|
cell: nbformat.IBaseCell
|
||||||
|
): NotebookCellData | undefined {
|
||||||
|
switch (cell.cell_type) {
|
||||||
|
case 'raw': {
|
||||||
|
return createNotebookCellDataFromRawCell(cell as nbformat.IRawCell);
|
||||||
|
}
|
||||||
|
case 'markdown': {
|
||||||
|
return createNotebookCellDataFromMarkdownCell(cell as nbformat.IMarkdownCell);
|
||||||
|
}
|
||||||
|
case 'code': {
|
||||||
|
return createNotebookCellDataFromCodeCell(cell as nbformat.ICodeCell, cellLanguage);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Converts a NotebookModel into VS Code format.
|
||||||
|
*/
|
||||||
|
export function jupyterNotebookModelToNotebookData(
|
||||||
|
notebookContent: Partial<nbformat.INotebookContent>,
|
||||||
|
preferredLanguage: string
|
||||||
|
): NotebookData {
|
||||||
|
const notebookContentWithoutCells = { ...notebookContent, cells: [] };
|
||||||
|
if (!notebookContent.cells || notebookContent.cells.length === 0) {
|
||||||
|
throw new Error('Notebook content is missing cells');
|
||||||
|
}
|
||||||
|
|
||||||
|
const cells = notebookContent.cells
|
||||||
|
.map(cell => createNotebookCellDataFromJupyterCell(preferredLanguage, cell))
|
||||||
|
.filter((item): item is NotebookCellData => !!item);
|
||||||
|
|
||||||
|
const notebookData = new NotebookData(cells);
|
||||||
|
notebookData.metadata = { custom: notebookContentWithoutCells };
|
||||||
|
return notebookData;
|
||||||
|
}
|
||||||
@@ -4,7 +4,7 @@
|
|||||||
*--------------------------------------------------------------------------------------------*/
|
*--------------------------------------------------------------------------------------------*/
|
||||||
|
|
||||||
import * as vscode from 'vscode';
|
import * as vscode from 'vscode';
|
||||||
import { NotebookSerializer } from './serializer';
|
import { NotebookSerializer } from './notebookSerializer';
|
||||||
|
|
||||||
export function activate(context: vscode.ExtensionContext) {
|
export function activate(context: vscode.ExtensionContext) {
|
||||||
context.subscriptions.push(vscode.workspace.registerNotebookSerializer('jupyter-notebook', new NotebookSerializer(context), {
|
context.subscriptions.push(vscode.workspace.registerNotebookSerializer('jupyter-notebook', new NotebookSerializer(context), {
|
||||||
|
|||||||
@@ -7,7 +7,8 @@ import type { nbformat } from '@jupyterlab/coreutils';
|
|||||||
import * as detectIndent from 'detect-indent';
|
import * as detectIndent from 'detect-indent';
|
||||||
import * as vscode from 'vscode';
|
import * as vscode from 'vscode';
|
||||||
import { defaultNotebookFormat } from './constants';
|
import { defaultNotebookFormat } from './constants';
|
||||||
import { createJupyterCellFromNotebookCell, getPreferredLanguage, jupyterNotebookModelToNotebookData, pruneCell } from './helpers';
|
import { getPreferredLanguage, jupyterNotebookModelToNotebookData } from './deserializers';
|
||||||
|
import { createJupyterCellFromNotebookCell, pruneCell } from './serializers';
|
||||||
import * as fnv from '@enonic/fnv-plus';
|
import * as fnv from '@enonic/fnv-plus';
|
||||||
|
|
||||||
export class NotebookSerializer implements vscode.NotebookSerializer {
|
export class NotebookSerializer implements vscode.NotebookSerializer {
|
||||||
@@ -4,69 +4,10 @@
|
|||||||
*--------------------------------------------------------------------------------------------*/
|
*--------------------------------------------------------------------------------------------*/
|
||||||
|
|
||||||
import { nbformat } from '@jupyterlab/coreutils';
|
import { nbformat } from '@jupyterlab/coreutils';
|
||||||
import { extensions, NotebookCellData, NotebookCellExecutionSummary, NotebookCellKind, NotebookCellOutput, NotebookCellOutputItem, NotebookData } from 'vscode';
|
import { NotebookCellData, NotebookCellKind, NotebookCellOutput } from 'vscode';
|
||||||
|
import { CellOutputMetadata } from './common';
|
||||||
const jupyterLanguageToMonacoLanguageMapping = new Map([
|
|
||||||
['c#', 'csharp'],
|
|
||||||
['f#', 'fsharp'],
|
|
||||||
['q#', 'qsharp'],
|
|
||||||
['c++11', 'c++'],
|
|
||||||
['c++12', 'c++'],
|
|
||||||
['c++14', 'c++']
|
|
||||||
]);
|
|
||||||
|
|
||||||
export function getPreferredLanguage(metadata?: nbformat.INotebookMetadata) {
|
|
||||||
const jupyterLanguage =
|
|
||||||
metadata?.language_info?.name ||
|
|
||||||
(metadata?.kernelspec as any)?.language;
|
|
||||||
|
|
||||||
// Default to python language only if the Python extension is installed.
|
|
||||||
const defaultLanguage = extensions.getExtension('ms-python.python') ? 'python' : 'plaintext';
|
|
||||||
|
|
||||||
// Note, whatever language is returned here, when the user selects a kernel, the cells (of blank documents) get updated based on that kernel selection.
|
|
||||||
return translateKernelLanguageToMonaco(jupyterLanguage || defaultLanguage);
|
|
||||||
}
|
|
||||||
|
|
||||||
const textDecoder = new TextDecoder();
|
const textDecoder = new TextDecoder();
|
||||||
function translateKernelLanguageToMonaco(language: string): string {
|
|
||||||
language = language.toLowerCase();
|
|
||||||
if (language.length === 2 && language.endsWith('#')) {
|
|
||||||
return `${language.substring(0, 1)}sharp`;
|
|
||||||
}
|
|
||||||
return jupyterLanguageToMonacoLanguageMapping.get(language) || language;
|
|
||||||
}
|
|
||||||
|
|
||||||
const orderOfMimeTypes = [
|
|
||||||
'application/vnd.*',
|
|
||||||
'application/vdom.*',
|
|
||||||
'application/geo+json',
|
|
||||||
'application/x-nteract-model-debug+json',
|
|
||||||
'text/html',
|
|
||||||
'application/javascript',
|
|
||||||
'image/gif',
|
|
||||||
'text/latex',
|
|
||||||
'text/markdown',
|
|
||||||
'image/png',
|
|
||||||
'image/svg+xml',
|
|
||||||
'image/jpeg',
|
|
||||||
'application/json',
|
|
||||||
'text/plain'
|
|
||||||
];
|
|
||||||
|
|
||||||
function sortOutputItemsBasedOnDisplayOrder(outputItems: NotebookCellOutputItem[]): NotebookCellOutputItem[] {
|
|
||||||
return outputItems.sort((outputItemA, outputItemB) => {
|
|
||||||
const isMimeTypeMatch = (value: string, compareWith: string) => {
|
|
||||||
if (value.endsWith('.*')) {
|
|
||||||
value = value.substr(0, value.indexOf('.*'));
|
|
||||||
}
|
|
||||||
return compareWith.startsWith(value);
|
|
||||||
};
|
|
||||||
const indexOfMimeTypeA = orderOfMimeTypes.findIndex(mime => isMimeTypeMatch(outputItemA.mime, mime));
|
|
||||||
const indexOfMimeTypeB = orderOfMimeTypes.findIndex(mime => isMimeTypeMatch(outputItemB.mime, mime));
|
|
||||||
return indexOfMimeTypeA - indexOfMimeTypeB;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
enum CellOutputMimeTypes {
|
enum CellOutputMimeTypes {
|
||||||
error = 'application/vnd.code.notebook.error',
|
error = 'application/vnd.code.notebook.error',
|
||||||
@@ -76,57 +17,6 @@ enum CellOutputMimeTypes {
|
|||||||
|
|
||||||
const textMimeTypes = ['text/plain', 'text/markdown', CellOutputMimeTypes.stderr, CellOutputMimeTypes.stdout];
|
const textMimeTypes = ['text/plain', 'text/markdown', CellOutputMimeTypes.stderr, CellOutputMimeTypes.stdout];
|
||||||
|
|
||||||
function concatMultilineString(str: string | string[], trim?: boolean): string {
|
|
||||||
const nonLineFeedWhiteSpaceTrim = /(^[\t\f\v\r ]+|[\t\f\v\r ]+$)/g;
|
|
||||||
if (Array.isArray(str)) {
|
|
||||||
let result = '';
|
|
||||||
for (let i = 0; i < str.length; i += 1) {
|
|
||||||
const s = str[i];
|
|
||||||
if (i < str.length - 1 && !s.endsWith('\n')) {
|
|
||||||
result = result.concat(`${s}\n`);
|
|
||||||
} else {
|
|
||||||
result = result.concat(s);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Just trim whitespace. Leave \n in place
|
|
||||||
return trim ? result.replace(nonLineFeedWhiteSpaceTrim, '') : result;
|
|
||||||
}
|
|
||||||
return trim ? str.toString().replace(nonLineFeedWhiteSpaceTrim, '') : str.toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
function convertJupyterOutputToBuffer(mime: string, value: unknown): NotebookCellOutputItem {
|
|
||||||
if (!value) {
|
|
||||||
return NotebookCellOutputItem.text('', mime);
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
if (
|
|
||||||
(mime.startsWith('text/') || textMimeTypes.includes(mime)) &&
|
|
||||||
(Array.isArray(value) || typeof value === 'string')
|
|
||||||
) {
|
|
||||||
const stringValue = Array.isArray(value) ? concatMultilineString(value) : value;
|
|
||||||
return NotebookCellOutputItem.text(stringValue, mime);
|
|
||||||
} else if (mime.startsWith('image/') && typeof value === 'string' && mime !== 'image/svg+xml') {
|
|
||||||
// Images in Jupyter are stored in base64 encoded format.
|
|
||||||
// VS Code expects bytes when rendering images.
|
|
||||||
if (typeof Buffer !== 'undefined' && typeof Buffer.from === 'function') {
|
|
||||||
return new NotebookCellOutputItem(Buffer.from(value, 'base64'), mime);
|
|
||||||
} else {
|
|
||||||
const data = Uint8Array.from(atob(value), c => c.charCodeAt(0));
|
|
||||||
return new NotebookCellOutputItem(data, mime);
|
|
||||||
}
|
|
||||||
} else if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
|
|
||||||
return NotebookCellOutputItem.text(JSON.stringify(value), mime);
|
|
||||||
} else {
|
|
||||||
// For everything else, treat the data as strings (or multi-line strings).
|
|
||||||
value = Array.isArray(value) ? concatMultilineString(value) : value;
|
|
||||||
return NotebookCellOutputItem.text(value as string, mime);
|
|
||||||
}
|
|
||||||
} catch (ex) {
|
|
||||||
return NotebookCellOutputItem.error(ex);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export function createJupyterCellFromNotebookCell(
|
export function createJupyterCellFromNotebookCell(
|
||||||
vscCell: NotebookCellData
|
vscCell: NotebookCellData
|
||||||
): nbformat.IRawCell | nbformat.IMarkdownCell | nbformat.ICodeCell {
|
): nbformat.IRawCell | nbformat.IMarkdownCell | nbformat.ICodeCell {
|
||||||
@@ -186,39 +76,6 @@ function splitMultilineString(source: nbformat.MultilineString): string[] {
|
|||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Metadata we store in VS Code cell output items.
|
|
||||||
* This contains the original metadata from the Jupyuter Outputs.
|
|
||||||
*/
|
|
||||||
interface CellOutputMetadata {
|
|
||||||
/**
|
|
||||||
* Cell output metadata.
|
|
||||||
*/
|
|
||||||
metadata?: any;
|
|
||||||
/**
|
|
||||||
* Transient data from Jupyter.
|
|
||||||
*/
|
|
||||||
transient?: {
|
|
||||||
/**
|
|
||||||
* This is used for updating the output in other cells.
|
|
||||||
* We don't know of others properties, but this is definitely used.
|
|
||||||
*/
|
|
||||||
display_id?: string;
|
|
||||||
} & any;
|
|
||||||
/**
|
|
||||||
* Original cell output type
|
|
||||||
*/
|
|
||||||
outputType: nbformat.OutputType | string;
|
|
||||||
executionCount?: nbformat.IExecuteResult['ExecutionCount'];
|
|
||||||
/**
|
|
||||||
* Whether the original Mime data is JSON or not.
|
|
||||||
* This properly only exists in metadata for NotebookCellOutputItems
|
|
||||||
* (this is something we have added)
|
|
||||||
*/
|
|
||||||
__isJson?: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
function translateCellDisplayOutput(output: NotebookCellOutput): JupyterOutput {
|
function translateCellDisplayOutput(output: NotebookCellOutput): JupyterOutput {
|
||||||
const customMetadata = output.metadata as CellOutputMetadata | undefined;
|
const customMetadata = output.metadata as CellOutputMetadata | undefined;
|
||||||
let result: JupyterOutput;
|
let result: JupyterOutput;
|
||||||
@@ -512,207 +369,3 @@ function fixupOutput(output: nbformat.IOutput): nbformat.IOutput {
|
|||||||
}
|
}
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
function getNotebookCellMetadata(cell: nbformat.IBaseCell): CellMetadata {
|
|
||||||
// We put this only for VSC to display in diff view.
|
|
||||||
// Else we don't use this.
|
|
||||||
const propertiesToClone: (keyof CellMetadata)[] = ['metadata', 'attachments'];
|
|
||||||
const custom: CellMetadata = {};
|
|
||||||
propertiesToClone.forEach((propertyToClone) => {
|
|
||||||
if (cell[propertyToClone]) {
|
|
||||||
custom[propertyToClone] = JSON.parse(JSON.stringify(cell[propertyToClone]));
|
|
||||||
}
|
|
||||||
});
|
|
||||||
return custom;
|
|
||||||
}
|
|
||||||
function getOutputMetadata(output: nbformat.IOutput): CellOutputMetadata {
|
|
||||||
// Add on transient data if we have any. This should be removed by our save functions elsewhere.
|
|
||||||
const metadata: CellOutputMetadata = {
|
|
||||||
outputType: output.output_type
|
|
||||||
};
|
|
||||||
if (output.transient) {
|
|
||||||
metadata.transient = output.transient;
|
|
||||||
}
|
|
||||||
|
|
||||||
switch (output.output_type as nbformat.OutputType) {
|
|
||||||
case 'display_data':
|
|
||||||
case 'execute_result':
|
|
||||||
case 'update_display_data': {
|
|
||||||
metadata.executionCount = output.execution_count;
|
|
||||||
metadata.metadata = output.metadata ? JSON.parse(JSON.stringify(output.metadata)) : {};
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
return metadata;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
function translateDisplayDataOutput(
|
|
||||||
output: nbformat.IDisplayData | nbformat.IDisplayUpdate | nbformat.IExecuteResult
|
|
||||||
): NotebookCellOutput {
|
|
||||||
// Metadata could be as follows:
|
|
||||||
// We'll have metadata specific to each mime type as well as generic metadata.
|
|
||||||
/*
|
|
||||||
IDisplayData = {
|
|
||||||
output_type: 'display_data',
|
|
||||||
data: {
|
|
||||||
'image/jpg': '/////'
|
|
||||||
'image/png': '/////'
|
|
||||||
'text/plain': '/////'
|
|
||||||
},
|
|
||||||
metadata: {
|
|
||||||
'image/png': '/////',
|
|
||||||
'background': true,
|
|
||||||
'xyz': '///
|
|
||||||
}
|
|
||||||
}
|
|
||||||
*/
|
|
||||||
const metadata = getOutputMetadata(output);
|
|
||||||
const items: NotebookCellOutputItem[] = [];
|
|
||||||
if (output.data) {
|
|
||||||
for (const key in output.data) {
|
|
||||||
items.push(convertJupyterOutputToBuffer(key, output.data[key]));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return new NotebookCellOutput(sortOutputItemsBasedOnDisplayOrder(items), metadata);
|
|
||||||
}
|
|
||||||
|
|
||||||
function translateErrorOutput(output?: nbformat.IError): NotebookCellOutput {
|
|
||||||
output = output || { output_type: 'error', ename: '', evalue: '', traceback: [] };
|
|
||||||
return new NotebookCellOutput(
|
|
||||||
[
|
|
||||||
NotebookCellOutputItem.error({
|
|
||||||
name: output?.ename || '',
|
|
||||||
message: output?.evalue || '',
|
|
||||||
stack: (output?.traceback || []).join('\n')
|
|
||||||
})
|
|
||||||
],
|
|
||||||
{ ...getOutputMetadata(output), originalError: output }
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function translateStreamOutput(output: nbformat.IStream): NotebookCellOutput {
|
|
||||||
const value = concatMultilineString(output.text);
|
|
||||||
const item = output.name === 'stderr' ? NotebookCellOutputItem.stderr(value) : NotebookCellOutputItem.stdout(value);
|
|
||||||
return new NotebookCellOutput([item], getOutputMetadata(output));
|
|
||||||
}
|
|
||||||
|
|
||||||
const cellOutputMappers = new Map<nbformat.OutputType, (output: any) => NotebookCellOutput>();
|
|
||||||
cellOutputMappers.set('display_data', translateDisplayDataOutput);
|
|
||||||
cellOutputMappers.set('execute_result', translateDisplayDataOutput);
|
|
||||||
cellOutputMappers.set('update_display_data', translateDisplayDataOutput);
|
|
||||||
cellOutputMappers.set('error', translateErrorOutput);
|
|
||||||
cellOutputMappers.set('stream', translateStreamOutput);
|
|
||||||
|
|
||||||
function jupyterCellOutputToCellOutput(output: nbformat.IOutput): NotebookCellOutput {
|
|
||||||
/**
|
|
||||||
* Stream, `application/x.notebook.stream`
|
|
||||||
* Error, `application/x.notebook.error-traceback`
|
|
||||||
* Rich, { mime: value }
|
|
||||||
*
|
|
||||||
* outputs: [
|
|
||||||
new vscode.NotebookCellOutput([
|
|
||||||
new vscode.NotebookCellOutputItem('application/x.notebook.stream', 2),
|
|
||||||
new vscode.NotebookCellOutputItem('application/x.notebook.stream', 3),
|
|
||||||
]),
|
|
||||||
new vscode.NotebookCellOutput([
|
|
||||||
new vscode.NotebookCellOutputItem('text/markdown', '## header 2'),
|
|
||||||
new vscode.NotebookCellOutputItem('image/svg+xml', [
|
|
||||||
"<svg baseProfile=\"full\" height=\"200\" version=\"1.1\" width=\"300\" xmlns=\"http://www.w3.org/2000/svg\">\n",
|
|
||||||
" <rect fill=\"blue\" height=\"100%\" width=\"100%\"/>\n",
|
|
||||||
" <circle cx=\"150\" cy=\"100\" fill=\"green\" r=\"80\"/>\n",
|
|
||||||
" <text fill=\"white\" font-size=\"60\" text-anchor=\"middle\" x=\"150\" y=\"125\">SVG</text>\n",
|
|
||||||
"</svg>"
|
|
||||||
]),
|
|
||||||
]),
|
|
||||||
]
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
const fn = cellOutputMappers.get(output.output_type as nbformat.OutputType);
|
|
||||||
let result: NotebookCellOutput;
|
|
||||||
if (fn) {
|
|
||||||
result = fn(output);
|
|
||||||
} else {
|
|
||||||
result = translateDisplayDataOutput(output as any);
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
function createNotebookCellDataFromRawCell(cell: nbformat.IRawCell): NotebookCellData {
|
|
||||||
const cellData = new NotebookCellData(NotebookCellKind.Code, concatMultilineString(cell.source), 'raw');
|
|
||||||
cellData.outputs = [];
|
|
||||||
cellData.metadata = { custom: getNotebookCellMetadata(cell) };
|
|
||||||
return cellData;
|
|
||||||
}
|
|
||||||
function createNotebookCellDataFromMarkdownCell(cell: nbformat.IMarkdownCell): NotebookCellData {
|
|
||||||
const cellData = new NotebookCellData(
|
|
||||||
NotebookCellKind.Markup,
|
|
||||||
concatMultilineString(cell.source),
|
|
||||||
'markdown'
|
|
||||||
);
|
|
||||||
cellData.outputs = [];
|
|
||||||
cellData.metadata = { custom: getNotebookCellMetadata(cell) };
|
|
||||||
return cellData;
|
|
||||||
}
|
|
||||||
function createNotebookCellDataFromCodeCell(cell: nbformat.ICodeCell, cellLanguage: string): NotebookCellData {
|
|
||||||
const cellOutputs = Array.isArray(cell.outputs) ? cell.outputs : [];
|
|
||||||
const outputs = cellOutputs.map(jupyterCellOutputToCellOutput);
|
|
||||||
const hasExecutionCount = typeof cell.execution_count === 'number' && cell.execution_count > 0;
|
|
||||||
|
|
||||||
const source = concatMultilineString(cell.source);
|
|
||||||
|
|
||||||
const executionSummary: NotebookCellExecutionSummary = hasExecutionCount
|
|
||||||
? { executionOrder: cell.execution_count as number }
|
|
||||||
: {};
|
|
||||||
|
|
||||||
const cellData = new NotebookCellData(NotebookCellKind.Code, source, cellLanguage);
|
|
||||||
|
|
||||||
cellData.outputs = outputs;
|
|
||||||
cellData.metadata = { custom: getNotebookCellMetadata(cell) };
|
|
||||||
cellData.executionSummary = executionSummary;
|
|
||||||
return cellData;
|
|
||||||
}
|
|
||||||
|
|
||||||
function createNotebookCellDataFromJupyterCell(
|
|
||||||
cellLanguage: string,
|
|
||||||
cell: nbformat.IBaseCell
|
|
||||||
): NotebookCellData | undefined {
|
|
||||||
switch (cell.cell_type) {
|
|
||||||
case 'raw': {
|
|
||||||
return createNotebookCellDataFromRawCell(cell as nbformat.IRawCell);
|
|
||||||
}
|
|
||||||
case 'markdown': {
|
|
||||||
return createNotebookCellDataFromMarkdownCell(cell as nbformat.IMarkdownCell);
|
|
||||||
}
|
|
||||||
case 'code': {
|
|
||||||
return createNotebookCellDataFromCodeCell(cell as nbformat.ICodeCell, cellLanguage);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Converts a NotebookModel into VS Code format.
|
|
||||||
*/
|
|
||||||
export function jupyterNotebookModelToNotebookData(
|
|
||||||
notebookContent: Partial<nbformat.INotebookContent>,
|
|
||||||
preferredLanguage: string
|
|
||||||
): NotebookData {
|
|
||||||
const notebookContentWithoutCells = { ...notebookContent, cells: [] };
|
|
||||||
if (!notebookContent.cells || notebookContent.cells.length === 0) {
|
|
||||||
throw new Error('Notebook content is missing cells');
|
|
||||||
}
|
|
||||||
|
|
||||||
const cells = notebookContent.cells
|
|
||||||
.map(cell => createNotebookCellDataFromJupyterCell(preferredLanguage, cell))
|
|
||||||
.filter((item): item is NotebookCellData => !!item);
|
|
||||||
|
|
||||||
const notebookData = new NotebookData(cells);
|
|
||||||
notebookData.metadata = { custom: notebookContentWithoutCells };
|
|
||||||
return notebookData;
|
|
||||||
}
|
|
||||||
Reference in New Issue
Block a user