From 8b6547aa2c5ae331abae2e1344dc1154cdbb2458 Mon Sep 17 00:00:00 2001 From: Matt Bierner Date: Tue, 17 Aug 2021 09:57:57 -0700 Subject: [PATCH] Transfer notebook output as VSBuffer (#130452) * Transfer notebook output data as a VSBuffer This PR transfers notebook output as an VSBuffer instead of as a array of numbers. This significantly reduces the message size as well as the serialize/deserialize times To accomplish this, I've introduced a new `SerializableObjectWithBuffers` type. This specially marks rpc objects that contain VSBuffers. This is needed as our current RPC implementation can only efficently transfer VSBuffers that appears as top level arguments. The rpcProtocol now can also identify top level `SerializableObjectWithBuffers` arguments and ensure these are serialized more efficently. This is easier than trying to extract the `VSBuffer` proeprties to top level arguments. It also lets us easily support return types that may contain buffers * use SerializableObjectWithBuffers when dealing with (old) notebook controllers Co-authored-by: Johannes Rieken --- src/vs/base/common/marshalling.ts | 1 + .../api/browser/mainThreadNotebook.ts | 7 +- .../browser/mainThreadNotebookDocuments.ts | 3 +- .../mainThreadNotebookDocumentsAndEditors.ts | 3 +- .../api/browser/mainThreadNotebookDto.ts | 4 +- .../api/browser/mainThreadNotebookKernels.ts | 4 +- .../workbench/api/common/extHost.protocol.ts | 16 +- .../workbench/api/common/extHostNotebook.ts | 45 ++-- .../api/common/extHostNotebookDocuments.ts | 5 +- .../api/common/extHostNotebookKernels.ts | 3 +- .../api/common/extHostTypeConverters.ts | 5 +- .../browser/interactive.contribution.ts | 4 +- .../contrib/codeRenderer/codeRenderer.ts | 6 +- .../browser/diff/diffElementViewModel.ts | 6 +- .../notebook/browser/notebook.contribution.ts | 4 +- .../view/output/transforms/richTransform.ts | 5 +- .../view/renderers/backLayerWebView.ts | 2 +- .../common/model/notebookTextModel.ts | 2 +- .../contrib/notebook/common/notebookCommon.ts | 3 +- .../common/services/notebookSimpleWorker.ts | 2 +- .../notebook/test/notebookDiff.test.ts | 29 +-- .../notebook/test/notebookTextModel.test.ts | 7 +- .../notebook/test/testNotebookEditor.ts | 5 +- .../extensions/common/proxyIdentifier.ts | 9 + .../services/extensions/common/rpcProtocol.ts | 214 ++++++++++++++---- .../test/common/rpcProtocol.test.ts | 29 ++- .../test/browser/api/extHostNotebook.test.ts | 49 ++-- .../api/extHostNotebookConcatDocument.test.ts | 55 ++--- .../browser/api/extHostNotebookKernel.test.ts | 19 +- .../browser/api/extHostTypeConverter.test.ts | 4 +- .../test/browser/api/testRPCProtocol.ts | 15 +- 31 files changed, 378 insertions(+), 187 deletions(-) diff --git a/src/vs/base/common/marshalling.ts b/src/vs/base/common/marshalling.ts index e8732b18ed2..789cdf08a16 100644 --- a/src/vs/base/common/marshalling.ts +++ b/src/vs/base/common/marshalling.ts @@ -51,6 +51,7 @@ function replacer(key: string, value: any): any { type Deserialize = T extends UriComponents ? URI + : T extends VSBuffer ? VSBuffer : T extends object ? Revived : T; diff --git a/src/vs/workbench/api/browser/mainThreadNotebook.ts b/src/vs/workbench/api/browser/mainThreadNotebook.ts index cc20644d23b..72ca5b6a0be 100644 --- a/src/vs/workbench/api/browser/mainThreadNotebook.ts +++ b/src/vs/workbench/api/browser/mainThreadNotebook.ts @@ -13,6 +13,7 @@ import { extHostNamedCustomer } from 'vs/workbench/api/common/extHostCustomers'; import { INotebookCellStatusBarService } from 'vs/workbench/contrib/notebook/common/notebookCellStatusBarService'; import { INotebookCellStatusBarItemProvider, INotebookContributionData, NotebookData as NotebookData, TransientCellMetadata, TransientDocumentMetadata, TransientOptions } from 'vs/workbench/contrib/notebook/common/notebookCommon'; import { INotebookContentProvider, INotebookService } from 'vs/workbench/contrib/notebook/common/notebookService'; +import { SerializableObjectWithBuffers } from 'vs/workbench/services/extensions/common/proxyIdentifier'; import { ExtHostContext, ExtHostNotebookShape, IExtHostContext, MainContext, MainThreadNotebookShape, NotebookExtensionDescription } from '../common/extHost.protocol'; @extHostNamedCustomer(MainContext.MainThreadNotebook) @@ -57,7 +58,7 @@ export class MainThreadNotebooks implements MainThreadNotebookShape { open: async (uri: URI, backupId: string | undefined, untitledDocumentData: VSBuffer | undefined, token: CancellationToken) => { const data = await this._proxy.$openNotebook(viewType, uri, backupId, untitledDocumentData, token); return { - data: NotebookDto.fromNotebookDataDto(data), + data: NotebookDto.fromNotebookDataDto(data.value), transientOptions: contentOptions }; }, @@ -107,10 +108,10 @@ export class MainThreadNotebooks implements MainThreadNotebookShape { options, dataToNotebook: async (data: VSBuffer): Promise => { const dto = await this._proxy.$dataToNotebook(handle, data, CancellationToken.None); - return NotebookDto.fromNotebookDataDto(dto); + return NotebookDto.fromNotebookDataDto(dto.value); }, notebookToData: (data: NotebookData): Promise => { - return this._proxy.$notebookToData(handle, NotebookDto.toNotebookDataDto(data), CancellationToken.None); + return this._proxy.$notebookToData(handle, new SerializableObjectWithBuffers(NotebookDto.toNotebookDataDto(data)), CancellationToken.None); } }); const disposables = new DisposableStore(); diff --git a/src/vs/workbench/api/browser/mainThreadNotebookDocuments.ts b/src/vs/workbench/api/browser/mainThreadNotebookDocuments.ts index f70b4905751..e3de991dafb 100644 --- a/src/vs/workbench/api/browser/mainThreadNotebookDocuments.ts +++ b/src/vs/workbench/api/browser/mainThreadNotebookDocuments.ts @@ -15,6 +15,7 @@ import { IUriIdentityService } from 'vs/workbench/services/uriIdentity/common/ur import { ExtHostContext, ExtHostNotebookDocumentsShape, IExtHostContext, MainThreadNotebookDocumentsShape, NotebookCellDto, NotebookCellsChangedEventDto, NotebookDataDto } from '../common/extHost.protocol'; import { MainThreadNotebooksAndEditors } from 'vs/workbench/api/browser/mainThreadNotebookDocumentsAndEditors'; import { NotebookDto } from 'vs/workbench/api/browser/mainThreadNotebookDto'; +import { SerializableObjectWithBuffers } from 'vs/workbench/services/extensions/common/proxyIdentifier'; export class MainThreadNotebookDocuments implements MainThreadNotebookDocumentsShape { @@ -104,7 +105,7 @@ export class MainThreadNotebookDocuments implements MainThreadNotebookDocumentsS // is marked as dirty and that another event is fired this._proxy.$acceptModelChanged( textModel.uri, - eventDto, + new SerializableObjectWithBuffers(eventDto), this._notebookEditorModelResolverService.isDirty(textModel.uri) ); diff --git a/src/vs/workbench/api/browser/mainThreadNotebookDocumentsAndEditors.ts b/src/vs/workbench/api/browser/mainThreadNotebookDocumentsAndEditors.ts index 239cd0fba9c..ad86a0d4ea6 100644 --- a/src/vs/workbench/api/browser/mainThreadNotebookDocumentsAndEditors.ts +++ b/src/vs/workbench/api/browser/mainThreadNotebookDocumentsAndEditors.ts @@ -20,6 +20,7 @@ import { INotebookService } from 'vs/workbench/contrib/notebook/common/notebookS import { IEditorGroupsService } from 'vs/workbench/services/editor/common/editorGroupsService'; import { IEditorService } from 'vs/workbench/services/editor/common/editorService'; import { ExtHostContext, ExtHostNotebookShape, IExtHostContext, INotebookDocumentsAndEditorsDelta, INotebookEditorAddData, INotebookModelAddedData, MainContext } from '../common/extHost.protocol'; +import { SerializableObjectWithBuffers } from 'vs/workbench/services/extensions/common/proxyIdentifier'; interface INotebookAndEditorDelta { removedDocuments: URI[]; @@ -190,7 +191,7 @@ export class MainThreadNotebooksAndEditors { }; // send to extension FIRST - this._proxy.$acceptDocumentAndEditorsDelta(dto); + this._proxy.$acceptDocumentAndEditorsDelta(new SerializableObjectWithBuffers(dto)); // handle internally this._onDidRemoveEditors.fire(delta.removedEditors); diff --git a/src/vs/workbench/api/browser/mainThreadNotebookDto.ts b/src/vs/workbench/api/browser/mainThreadNotebookDto.ts index 693ff376cc4..5d582e16f86 100644 --- a/src/vs/workbench/api/browser/mainThreadNotebookDto.ts +++ b/src/vs/workbench/api/browser/mainThreadNotebookDto.ts @@ -13,7 +13,7 @@ export namespace NotebookDto { export function toNotebookOutputItemDto(item: notebookCommon.IOutputItemDto): extHostProtocol.NotebookOutputItemDto { return { mime: item.mime, - valueBytes: Array.from(item.data) + valueBytes: item.data }; } @@ -47,7 +47,7 @@ export namespace NotebookDto { export function fromNotebookOutputItemDto(item: extHostProtocol.NotebookOutputItemDto): notebookCommon.IOutputItemDto { return { mime: item.mime, - data: new Uint8Array(item.valueBytes) + data: item.valueBytes }; } diff --git a/src/vs/workbench/api/browser/mainThreadNotebookKernels.ts b/src/vs/workbench/api/browser/mainThreadNotebookKernels.ts index efc7566861f..a2a2f03736f 100644 --- a/src/vs/workbench/api/browser/mainThreadNotebookKernels.ts +++ b/src/vs/workbench/api/browser/mainThreadNotebookKernels.ts @@ -16,6 +16,7 @@ import { INotebookEditor } from 'vs/workbench/contrib/notebook/browser/notebookB import { INotebookEditorService } from 'vs/workbench/contrib/notebook/browser/notebookEditorService'; import { INotebookCellExecution, INotebookExecutionService } from 'vs/workbench/contrib/notebook/common/notebookExecutionService'; import { INotebookKernel, INotebookKernelChangeEvent, INotebookKernelService } from 'vs/workbench/contrib/notebook/common/notebookKernelService'; +import { SerializableObjectWithBuffers } from 'vs/workbench/services/extensions/common/proxyIdentifier'; import { ExtHostContext, ExtHostNotebookKernelsShape, ICellExecuteUpdateDto, IExtHostContext, INotebookKernelDto2, MainContext, MainThreadNotebookKernelsShape } from '../common/extHost.protocol'; abstract class MainThreadKernel implements INotebookKernel { @@ -233,7 +234,8 @@ export class MainThreadNotebookKernels implements MainThreadNotebookKernelsShape this._executions.set(handle, execution); } - $updateExecutions(updates: ICellExecuteUpdateDto[]): void { + $updateExecutions(data: SerializableObjectWithBuffers): void { + const updates = data.value; const groupedUpdates = groupBy(updates, (a, b) => a.executionHandle - b.executionHandle); groupedUpdates.forEach(datas => { const first = datas[0]; diff --git a/src/vs/workbench/api/common/extHost.protocol.ts b/src/vs/workbench/api/common/extHost.protocol.ts index 5bcf5dda003..6cbfc9a805d 100644 --- a/src/vs/workbench/api/common/extHost.protocol.ts +++ b/src/vs/workbench/api/common/extHost.protocol.ts @@ -65,7 +65,7 @@ import { InternalTimelineOptions, Timeline, TimelineChangeEvent, TimelineOptions import { TypeHierarchyItem } from 'vs/workbench/contrib/typeHierarchy/common/typeHierarchy'; import { EditorGroupColumn } from 'vs/workbench/services/editor/common/editorGroupColumn'; import { ActivationKind, ExtensionHostKind, MissingExtensionDependency } from 'vs/workbench/services/extensions/common/extensions'; -import { createExtHostContextProxyIdentifier as createExtId, createMainContextProxyIdentifier as createMainId, IRPCProtocol } from 'vs/workbench/services/extensions/common/proxyIdentifier'; +import { createExtHostContextProxyIdentifier as createExtId, createMainContextProxyIdentifier as createMainId, IRPCProtocol, SerializableObjectWithBuffers } from 'vs/workbench/services/extensions/common/proxyIdentifier'; import { CandidatePort } from 'vs/workbench/services/remote/common/remoteExplorerService'; import * as search from 'vs/workbench/services/search/common/search'; import * as statusbar from 'vs/workbench/services/statusbar/common/statusbar'; @@ -944,7 +944,7 @@ export interface MainThreadNotebookKernelsShape extends IDisposable { $updateNotebookPriority(handle: number, uri: UriComponents, value: number | undefined): void; $addExecution(handle: number, uri: UriComponents, cellHandle: number): void; - $updateExecutions(data: ICellExecuteUpdateDto[]): void; + $updateExecutions(data: SerializableObjectWithBuffers): void; $removeExecution(handle: number): void; } @@ -1952,7 +1952,7 @@ export interface INotebookDocumentsAndEditorsDelta { export interface NotebookOutputItemDto { readonly mime: string; - readonly valueBytes: number[]; // todo@jrieken ugly, should be VSBuffer + readonly valueBytes: VSBuffer; } export interface NotebookOutputDto { @@ -1993,13 +1993,13 @@ export interface ExtHostNotebookShape extends ExtHostNotebookDocumentsAndEditors $provideNotebookCellStatusBarItems(handle: number, uri: UriComponents, index: number, token: CancellationToken): Promise; $releaseNotebookCellStatusBarItems(id: number): void; - $openNotebook(viewType: string, uri: UriComponents, backupId: string | undefined, untitledDocumentData: VSBuffer | undefined, token: CancellationToken): Promise; + $openNotebook(viewType: string, uri: UriComponents, backupId: string | undefined, untitledDocumentData: VSBuffer | undefined, token: CancellationToken): Promise>; $saveNotebook(viewType: string, uri: UriComponents, token: CancellationToken): Promise; $saveNotebookAs(viewType: string, uri: UriComponents, target: UriComponents, token: CancellationToken): Promise; $backupNotebook(viewType: string, uri: UriComponents, cancellation: CancellationToken): Promise; - $dataToNotebook(handle: number, data: VSBuffer, token: CancellationToken): Promise; - $notebookToData(handle: number, data: NotebookDataDto, token: CancellationToken): Promise; + $dataToNotebook(handle: number, data: VSBuffer, token: CancellationToken): Promise>; + $notebookToData(handle: number, data: SerializableObjectWithBuffers, token: CancellationToken): Promise; } export interface ExtHostNotebookRenderersShape { @@ -2007,7 +2007,7 @@ export interface ExtHostNotebookRenderersShape { } export interface ExtHostNotebookDocumentsAndEditorsShape { - $acceptDocumentAndEditorsDelta(delta: INotebookDocumentsAndEditorsDelta): void; + $acceptDocumentAndEditorsDelta(delta: SerializableObjectWithBuffers): void; } export type NotebookRawContentEventDto = @@ -2050,7 +2050,7 @@ export type NotebookCellsChangedEventDto = { }; export interface ExtHostNotebookDocumentsShape { - $acceptModelChanged(uriComponents: UriComponents, event: NotebookCellsChangedEventDto, isDirty: boolean): void; + $acceptModelChanged(uriComponents: UriComponents, event: SerializableObjectWithBuffers, isDirty: boolean): void; $acceptDirtyStateChanged(uriComponents: UriComponents, isDirty: boolean): void; $acceptModelSaved(uriComponents: UriComponents): void; $acceptDocumentPropertiesChanged(uriComponents: UriComponents, data: INotebookDocumentPropertiesChangeData): void; diff --git a/src/vs/workbench/api/common/extHostNotebook.ts b/src/vs/workbench/api/common/extHostNotebook.ts index 85f177666f2..9c699d13746 100644 --- a/src/vs/workbench/api/common/extHostNotebook.ts +++ b/src/vs/workbench/api/common/extHostNotebook.ts @@ -24,6 +24,7 @@ import { IExtensionStoragePaths } from 'vs/workbench/api/common/extHostStoragePa import * as typeConverters from 'vs/workbench/api/common/extHostTypeConverters'; import * as extHostTypes from 'vs/workbench/api/common/extHostTypes'; import { INotebookExclusiveDocumentFilter, INotebookContributionData } from 'vs/workbench/contrib/notebook/common/notebookCommon'; +import { SerializableObjectWithBuffers } from 'vs/workbench/services/extensions/common/proxyIdentifier'; import type * as vscode from 'vscode'; import { ExtHostCell, ExtHostNotebookDocument } from './extHostNotebookDocument'; import { ExtHostNotebookEditor } from './extHostNotebookEditor'; @@ -334,33 +335,33 @@ export class ExtHostNotebookController implements ExtHostNotebookShape { }); } - async $dataToNotebook(handle: number, bytes: VSBuffer, token: CancellationToken): Promise { + async $dataToNotebook(handle: number, bytes: VSBuffer, token: CancellationToken): Promise> { const serializer = this._notebookSerializer.get(handle); if (!serializer) { throw new Error('NO serializer found'); } const data = await serializer.deserializeNotebook(bytes.buffer, token); - return typeConverters.NotebookData.from(data); + return new SerializableObjectWithBuffers(typeConverters.NotebookData.from(data)); } - async $notebookToData(handle: number, data: NotebookDataDto, token: CancellationToken): Promise { + async $notebookToData(handle: number, data: SerializableObjectWithBuffers, token: CancellationToken): Promise { const serializer = this._notebookSerializer.get(handle); if (!serializer) { throw new Error('NO serializer found'); } - const bytes = await serializer.serializeNotebook(typeConverters.NotebookData.to(data), token); + const bytes = await serializer.serializeNotebook(typeConverters.NotebookData.to(data.value), token); return VSBuffer.wrap(bytes); } // --- open, save, saveAs, backup - async $openNotebook(viewType: string, uri: UriComponents, backupId: string | undefined, untitledDocumentData: VSBuffer | undefined, token: CancellationToken): Promise { + async $openNotebook(viewType: string, uri: UriComponents, backupId: string | undefined, untitledDocumentData: VSBuffer | undefined, token: CancellationToken): Promise> { const { provider } = this._getProviderData(viewType); const data = await provider.openNotebook(URI.revive(uri), { backupId, untitledDocumentData: untitledDocumentData?.buffer }, token); - return { + return new SerializableObjectWithBuffers({ metadata: data.metadata ?? Object.create(null), cells: data.cells.map(typeConverters.NotebookCellData.from), - }; + }); } async $saveNotebook(viewType: string, uri: UriComponents, token: CancellationToken): Promise { @@ -411,10 +412,10 @@ export class ExtHostNotebookController implements ExtHostNotebookShape { this._editors.set(editorId, editor); } - $acceptDocumentAndEditorsDelta(delta: INotebookDocumentsAndEditorsDelta): void { + $acceptDocumentAndEditorsDelta(delta: SerializableObjectWithBuffers): void { - if (delta.removedDocuments) { - for (const uri of delta.removedDocuments) { + if (delta.value.removedDocuments) { + for (const uri of delta.value.removedDocuments) { const revivedUri = URI.revive(uri); const document = this._documents.get(revivedUri); @@ -433,11 +434,11 @@ export class ExtHostNotebookController implements ExtHostNotebookShape { } } - if (delta.addedDocuments) { + if (delta.value.addedDocuments) { const addedCellDocuments: IModelAddedData[] = []; - for (const modelData of delta.addedDocuments) { + for (const modelData of delta.value.addedDocuments) { const uri = URI.revive(modelData.uri); if (this._documents.has(uri)) { @@ -478,8 +479,8 @@ export class ExtHostNotebookController implements ExtHostNotebookShape { } } - if (delta.addedEditors) { - for (const editorModelData of delta.addedEditors) { + if (delta.value.addedEditors) { + for (const editorModelData of delta.value.addedEditors) { if (this._editors.has(editorModelData.id)) { return; } @@ -495,8 +496,8 @@ export class ExtHostNotebookController implements ExtHostNotebookShape { const removedEditors: ExtHostNotebookEditor[] = []; - if (delta.removedEditors) { - for (const editorid of delta.removedEditors) { + if (delta.value.removedEditors) { + for (const editorid of delta.value.removedEditors) { const editor = this._editors.get(editorid); if (editor) { @@ -511,8 +512,8 @@ export class ExtHostNotebookController implements ExtHostNotebookShape { } } - if (delta.visibleEditors) { - this._visibleNotebookEditors = delta.visibleEditors.map(id => this._editors.get(id)!).filter(editor => !!editor) as ExtHostNotebookEditor[]; + if (delta.value.visibleEditors) { + this._visibleNotebookEditors = delta.value.visibleEditors.map(id => this._editors.get(id)!).filter(editor => !!editor) as ExtHostNotebookEditor[]; const visibleEditorsSet = new Set(); this._visibleNotebookEditors.forEach(editor => visibleEditorsSet.add(editor.id)); @@ -525,13 +526,13 @@ export class ExtHostNotebookController implements ExtHostNotebookShape { this._onDidChangeVisibleNotebookEditors.fire(this.visibleNotebookEditors); } - if (delta.newActiveEditor === null) { + if (delta.value.newActiveEditor === null) { // clear active notebook as current active editor is non-notebook editor this._activeNotebookEditor = undefined; - } else if (delta.newActiveEditor) { - this._activeNotebookEditor = this._editors.get(delta.newActiveEditor); + } else if (delta.value.newActiveEditor) { + this._activeNotebookEditor = this._editors.get(delta.value.newActiveEditor); } - if (delta.newActiveEditor !== undefined) { + if (delta.value.newActiveEditor !== undefined) { this._onDidChangeActiveNotebookEditor.fire(this._activeNotebookEditor?.apiEditor); } } diff --git a/src/vs/workbench/api/common/extHostNotebookDocuments.ts b/src/vs/workbench/api/common/extHostNotebookDocuments.ts index c4c22029379..70db04b1181 100644 --- a/src/vs/workbench/api/common/extHostNotebookDocuments.ts +++ b/src/vs/workbench/api/common/extHostNotebookDocuments.ts @@ -8,6 +8,7 @@ import { URI, UriComponents } from 'vs/base/common/uri'; import { ILogService } from 'vs/platform/log/common/log'; import * as extHostProtocol from 'vs/workbench/api/common/extHost.protocol'; import { ExtHostNotebookController } from 'vs/workbench/api/common/extHostNotebook'; +import { SerializableObjectWithBuffers } from 'vs/workbench/services/extensions/common/proxyIdentifier'; import type * as vscode from 'vscode'; export class ExtHostNotebookDocuments implements extHostProtocol.ExtHostNotebookDocumentsShape { @@ -23,9 +24,9 @@ export class ExtHostNotebookDocuments implements extHostProtocol.ExtHostNotebook private readonly _notebooksAndEditors: ExtHostNotebookController, ) { } - $acceptModelChanged(uri: UriComponents, event: extHostProtocol.NotebookCellsChangedEventDto, isDirty: boolean): void { + $acceptModelChanged(uri: UriComponents, event: SerializableObjectWithBuffers, isDirty: boolean): void { const document = this._notebooksAndEditors.getNotebookDocument(URI.revive(uri)); - document.acceptModelChanged(event, isDirty); + document.acceptModelChanged(event.value, isDirty); } $acceptDirtyStateChanged(uri: UriComponents, isDirty: boolean): void { diff --git a/src/vs/workbench/api/common/extHostNotebookKernels.ts b/src/vs/workbench/api/common/extHostNotebookKernels.ts index c8b57133dba..d35df6e4211 100644 --- a/src/vs/workbench/api/common/extHostNotebookKernels.ts +++ b/src/vs/workbench/api/common/extHostNotebookKernels.ts @@ -21,6 +21,7 @@ import { NotebookCellOutput } from 'vs/workbench/api/common/extHostTypes'; import { asWebviewUri } from 'vs/workbench/api/common/shared/webview'; import { CellExecutionUpdateType } from 'vs/workbench/contrib/notebook/common/notebookExecutionService'; import { checkProposedApiEnabled } from 'vs/workbench/services/extensions/common/extensions'; +import { SerializableObjectWithBuffers } from 'vs/workbench/services/extensions/common/proxyIdentifier'; import * as vscode from 'vscode'; interface IKernelData { @@ -362,7 +363,7 @@ class NotebookCellExecutionTask extends Disposable { private async update(update: ICellExecuteUpdateDto | ICellExecuteUpdateDto[]): Promise { const updates = Array.isArray(update) ? update : [update]; - return this._proxy.$updateExecutions(updates); + return this._proxy.$updateExecutions(new SerializableObjectWithBuffers(updates)); } private verifyStateForOutput() { diff --git a/src/vs/workbench/api/common/extHostTypeConverters.ts b/src/vs/workbench/api/common/extHostTypeConverters.ts index e88fa62a559..1c39f5887b9 100644 --- a/src/vs/workbench/api/common/extHostTypeConverters.ts +++ b/src/vs/workbench/api/common/extHostTypeConverters.ts @@ -4,6 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { coalesce, isNonEmptyArray } from 'vs/base/common/arrays'; +import { VSBuffer } from 'vs/base/common/buffer'; import * as htmlContent from 'vs/base/common/htmlContent'; import { DisposableStore } from 'vs/base/common/lifecycle'; import * as marked from 'vs/base/common/marked/marked'; @@ -1494,12 +1495,12 @@ export namespace NotebookCellOutputItem { export function from(item: types.NotebookCellOutputItem): extHostProtocol.NotebookOutputItemDto { return { mime: item.mime, - valueBytes: Array.from(item.data), //todo@jrieken this HACKY and SLOW... hoist VSBuffer instead + valueBytes: VSBuffer.wrap(item.data), }; } export function to(item: extHostProtocol.NotebookOutputItemDto): types.NotebookCellOutputItem { - return new types.NotebookCellOutputItem(new Uint8Array(item.valueBytes), item.mime); + return new types.NotebookCellOutputItem(item.valueBytes.buffer, item.mime); } } diff --git a/src/vs/workbench/contrib/interactive/browser/interactive.contribution.ts b/src/vs/workbench/contrib/interactive/browser/interactive.contribution.ts index d768558a3a0..fef2ccb414c 100644 --- a/src/vs/workbench/contrib/interactive/browser/interactive.contribution.ts +++ b/src/vs/workbench/contrib/interactive/browser/interactive.contribution.ts @@ -104,7 +104,7 @@ export class InteractiveDocumentContribution extends Disposable implements IWork outputId: output.outputId, outputs: output.outputs.map(ot => ({ mime: ot.mime, - data: Uint8Array.from(ot.data) + data: ot.data })) })) : [], @@ -145,7 +145,7 @@ export class InteractiveDocumentContribution extends Disposable implements IWork outputId: output.outputId, outputs: output.outputs.map(ot => ({ mime: ot.mime, - data: Array.from(ot.data) + data: ot.data })) }; }), diff --git a/src/vs/workbench/contrib/notebook/browser/contrib/codeRenderer/codeRenderer.ts b/src/vs/workbench/contrib/notebook/browser/contrib/codeRenderer/codeRenderer.ts index f586e91e977..1751fbc799a 100644 --- a/src/vs/workbench/contrib/notebook/browser/contrib/codeRenderer/codeRenderer.ts +++ b/src/vs/workbench/contrib/notebook/browser/contrib/codeRenderer/codeRenderer.ts @@ -95,7 +95,7 @@ export class NotebookCodeRendererContribution extends Disposable { } render(output: ICellOutputViewModel, item: IOutputItemDto, container: HTMLElement): IRenderOutput { - const str = getStringValue(item); + const str = item.data.toString(); return this._render(output, container, str, languageId); } }); @@ -121,10 +121,6 @@ workbenchContributionsRegistry.registerWorkbenchContribution(NotebookCodeRendere // --- utils --- -function getStringValue(item: IOutputItemDto): string { - // todo@jrieken NOT proper, should be VSBuffer - return new TextDecoder().decode(item.data); -} function getOutputSimpleEditorOptions(): IEditorConstructionOptions { return { diff --git a/src/vs/workbench/contrib/notebook/browser/diff/diffElementViewModel.ts b/src/vs/workbench/contrib/notebook/browser/diff/diffElementViewModel.ts index 26e5cc8395c..5a0073f9c92 100644 --- a/src/vs/workbench/contrib/notebook/browser/diff/diffElementViewModel.ts +++ b/src/vs/workbench/contrib/notebook/browser/diff/diffElementViewModel.ts @@ -515,12 +515,12 @@ function outputsEqual(original: ICellOutput[], modified: ICellOutput[]) { return false; } - if (aOutputItem.data.length !== bOutputItem.data.length) { + if (aOutputItem.data.buffer.length !== bOutputItem.data.buffer.length) { return false; } - for (let k = 0; k < aOutputItem.data.length; k++) { - if (aOutputItem.data[k] !== bOutputItem.data[k]) { + for (let k = 0; k < aOutputItem.data.buffer.length; k++) { + if (aOutputItem.data.buffer[k] !== bOutputItem.data.buffer[k]) { return false; } } diff --git a/src/vs/workbench/contrib/notebook/browser/notebook.contribution.ts b/src/vs/workbench/contrib/notebook/browser/notebook.contribution.ts index 4e83c9c7ba2..93e7a8a8caa 100644 --- a/src/vs/workbench/contrib/notebook/browser/notebook.contribution.ts +++ b/src/vs/workbench/contrib/notebook/browser/notebook.contribution.ts @@ -372,7 +372,7 @@ class CellInfoContentProvider { const sameStream = !outputs.find(op => op.mime !== mime); if (sameStream) { - return outputs.map(opit => new TextDecoder().decode(opit.data)).join(''); + return outputs.map(opit => opit.data.toString()).join(''); } else { return null; } @@ -413,7 +413,7 @@ class CellInfoContentProvider { metadata: output.metadata, outputItems: output.outputs.map(opit => ({ mimeType: opit.mime, - data: new TextDecoder().decode(opit.data) + data: opit.data.toString() })) }))); const edits = format(content, undefined, {}); diff --git a/src/vs/workbench/contrib/notebook/browser/view/output/transforms/richTransform.ts b/src/vs/workbench/contrib/notebook/browser/view/output/transforms/richTransform.ts index ed87e351236..6d96e9d2feb 100644 --- a/src/vs/workbench/contrib/notebook/browser/view/output/transforms/richTransform.ts +++ b/src/vs/workbench/contrib/notebook/browser/view/output/transforms/richTransform.ts @@ -254,7 +254,7 @@ class ImgRendererContrib extends Disposable implements IOutputRendererContributi render(output: ICellOutputViewModel, item: IOutputItemDto, container: HTMLElement, notebookUri: URI): IRenderOutput { const disposable = new DisposableStore(); - const blob = new Blob([item.data], { type: item.mime }); + const blob = new Blob([item.data.buffer], { type: item.mime }); const src = URL.createObjectURL(blob); disposable.add(toDisposable(() => URL.revokeObjectURL(src))); @@ -281,6 +281,5 @@ OutputRendererRegistry.registerOutputTransform(StderrRendererContrib); // --- utils --- export function getStringValue(item: IOutputItemDto): string { - // todo@jrieken NOT proper, should be VSBuffer - return new TextDecoder().decode(item.data); + return item.data.toString(); } diff --git a/src/vs/workbench/contrib/notebook/browser/view/renderers/backLayerWebView.ts b/src/vs/workbench/contrib/notebook/browser/view/renderers/backLayerWebView.ts index f51aecc215a..d591c3036cb 100644 --- a/src/vs/workbench/contrib/notebook/browser/view/renderers/backLayerWebView.ts +++ b/src/vs/workbench/contrib/notebook/browser/view/renderers/backLayerWebView.ts @@ -980,7 +980,7 @@ export class BackLayerWebView extends Disposable { type: RenderOutputType.Extension, outputId: output.outputId, mimeType: first.mime, - valueBytes: first.data, + valueBytes: first.data.buffer, metadata: output.metadata, }, }; diff --git a/src/vs/workbench/contrib/notebook/common/model/notebookTextModel.ts b/src/vs/workbench/contrib/notebook/common/model/notebookTextModel.ts index f7e0cab9601..941d32bdf6b 100644 --- a/src/vs/workbench/contrib/notebook/common/model/notebookTextModel.ts +++ b/src/vs/workbench/contrib/notebook/common/model/notebookTextModel.ts @@ -1017,7 +1017,7 @@ class OutputSequence implements ISequence { return this.outputs.map(output => { return hash(output.outputs.map(output => ({ mime: output.mime, - data: Array.from(output.data) + data: output.data }))); }); } diff --git a/src/vs/workbench/contrib/notebook/common/notebookCommon.ts b/src/vs/workbench/contrib/notebook/common/notebookCommon.ts index 7a2ad86414e..d534b8081a0 100644 --- a/src/vs/workbench/contrib/notebook/common/notebookCommon.ts +++ b/src/vs/workbench/contrib/notebook/common/notebookCommon.ts @@ -3,6 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ +import { VSBuffer } from 'vs/base/common/buffer'; import { CancellationToken } from 'vs/base/common/cancellation'; import { IDiffResult, ISequence } from 'vs/base/common/diff/diff'; import { Event } from 'vs/base/common/event'; @@ -157,7 +158,7 @@ export interface IOrderedMimeType { export interface IOutputItemDto { readonly mime: string; - readonly data: Uint8Array; + readonly data: VSBuffer; } export interface IOutputDto { diff --git a/src/vs/workbench/contrib/notebook/common/services/notebookSimpleWorker.ts b/src/vs/workbench/contrib/notebook/common/services/notebookSimpleWorker.ts index 27cd8990a68..4313fb84348 100644 --- a/src/vs/workbench/contrib/notebook/common/services/notebookSimpleWorker.ts +++ b/src/vs/workbench/contrib/notebook/common/services/notebookSimpleWorker.ts @@ -74,7 +74,7 @@ class MirrorCell { this._hash = hash([hash(this.language), hash(this.getValue()), this.metadata, this.internalMetadata, this.outputs.map(op => ({ outputs: op.outputs.map(output => ({ mime: output.mime, - data: Array.from(output.data) + data: output.data })), metadata: op.metadata }))]); diff --git a/src/vs/workbench/contrib/notebook/test/notebookDiff.test.ts b/src/vs/workbench/contrib/notebook/test/notebookDiff.test.ts index bf204572dd3..2bbf84eef4b 100644 --- a/src/vs/workbench/contrib/notebook/test/notebookDiff.test.ts +++ b/src/vs/workbench/contrib/notebook/test/notebookDiff.test.ts @@ -4,6 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import * as assert from 'assert'; +import { VSBuffer } from 'vs/base/common/buffer'; import { LcsDiff } from 'vs/base/common/diff/diff'; import { Mimes } from 'vs/base/common/mime'; import { NotebookDiffEditorEventDispatcher } from 'vs/workbench/contrib/notebook/browser/diff/eventDispatcher'; @@ -15,9 +16,9 @@ suite('NotebookCommon', () => { test('diff different source', async () => { await withTestNotebookDiffModel([ - ['x', 'javascript', CellKind.Code, [{ outputId: 'someOtherId', outputs: [{ mime: Mimes.text, data: new Uint8Array([3]) }] }], { custom: { metadata: { collapsed: false } }, executionOrder: 3 }], + ['x', 'javascript', CellKind.Code, [{ outputId: 'someOtherId', outputs: [{ mime: Mimes.text, data: VSBuffer.wrap(new Uint8Array([3])) }] }], { custom: { metadata: { collapsed: false } }, executionOrder: 3 }], ], [ - ['y', 'javascript', CellKind.Code, [{ outputId: 'someOtherId', outputs: [{ mime: Mimes.text, data: new Uint8Array([3]) }] }], { custom: { metadata: { collapsed: false } }, executionOrder: 3 }], + ['y', 'javascript', CellKind.Code, [{ outputId: 'someOtherId', outputs: [{ mime: Mimes.text, data: VSBuffer.wrap(new Uint8Array([3])) }] }], { custom: { metadata: { collapsed: false } }, executionOrder: 3 }], ], (model, accessor) => { const diff = new LcsDiff(new CellSequence(model.original.notebook), new CellSequence(model.modified.notebook)); const diffResult = diff.ComputeDiff(false); @@ -45,10 +46,10 @@ suite('NotebookCommon', () => { test('diff different output', async () => { await withTestNotebookDiffModel([ - ['x', 'javascript', CellKind.Code, [{ outputId: 'someId', outputs: [{ mime: Mimes.text, data: new Uint8Array([5]) }] }], { custom: { metadata: { collapsed: false } }, executionOrder: 5 }], + ['x', 'javascript', CellKind.Code, [{ outputId: 'someId', outputs: [{ mime: Mimes.text, data: VSBuffer.wrap(new Uint8Array([5])) }] }], { custom: { metadata: { collapsed: false } }, executionOrder: 5 }], ['', 'javascript', CellKind.Code, [], {}] ], [ - ['x', 'javascript', CellKind.Code, [{ outputId: 'someOtherId', outputs: [{ mime: Mimes.text, data: new Uint8Array([3]) }] }], { custom: { metadata: { collapsed: false } }, executionOrder: 3 }], + ['x', 'javascript', CellKind.Code, [{ outputId: 'someOtherId', outputs: [{ mime: Mimes.text, data: VSBuffer.wrap(new Uint8Array([3])) }] }], { custom: { metadata: { collapsed: false } }, executionOrder: 3 }], ['', 'javascript', CellKind.Code, [], {}] ], (model, accessor) => { const diff = new LcsDiff(new CellSequence(model.original.notebook), new CellSequence(model.modified.notebook)); @@ -146,12 +147,12 @@ suite('NotebookCommon', () => { test('diff foo/foe', async () => { await withTestNotebookDiffModel([ - [['def foe(x, y):\n', ' return x + y\n', 'foe(3, 2)'].join(''), 'javascript', CellKind.Code, [{ outputId: 'someId', outputs: [{ mime: Mimes.text, data: new Uint8Array([6]) }] }], { custom: { metadata: { collapsed: false } }, executionOrder: 5 }], - [['def foo(x, y):\n', ' return x * y\n', 'foo(1, 2)'].join(''), 'javascript', CellKind.Code, [{ outputId: 'someId', outputs: [{ mime: Mimes.text, data: new Uint8Array([2]) }] }], { custom: { metadata: { collapsed: false } }, executionOrder: 6 }], + [['def foe(x, y):\n', ' return x + y\n', 'foe(3, 2)'].join(''), 'javascript', CellKind.Code, [{ outputId: 'someId', outputs: [{ mime: Mimes.text, data: VSBuffer.wrap(new Uint8Array([6])) }] }], { custom: { metadata: { collapsed: false } }, executionOrder: 5 }], + [['def foo(x, y):\n', ' return x * y\n', 'foo(1, 2)'].join(''), 'javascript', CellKind.Code, [{ outputId: 'someId', outputs: [{ mime: Mimes.text, data: VSBuffer.wrap(new Uint8Array([2])) }] }], { custom: { metadata: { collapsed: false } }, executionOrder: 6 }], ['', 'javascript', CellKind.Code, [], {}] ], [ - [['def foo(x, y):\n', ' return x * y\n', 'foo(1, 2)'].join(''), 'javascript', CellKind.Code, [{ outputId: 'someId', outputs: [{ mime: Mimes.text, data: new Uint8Array([6]) }] }], { custom: { metadata: { collapsed: false } }, executionOrder: 5 }], - [['def foe(x, y):\n', ' return x + y\n', 'foe(3, 2)'].join(''), 'javascript', CellKind.Code, [{ outputId: 'someId', outputs: [{ mime: Mimes.text, data: new Uint8Array([2]) }] }], { custom: { metadata: { collapsed: false } }, executionOrder: 6 }], + [['def foo(x, y):\n', ' return x * y\n', 'foo(1, 2)'].join(''), 'javascript', CellKind.Code, [{ outputId: 'someId', outputs: [{ mime: Mimes.text, data: VSBuffer.wrap(new Uint8Array([6])) }] }], { custom: { metadata: { collapsed: false } }, executionOrder: 5 }], + [['def foe(x, y):\n', ' return x + y\n', 'foe(3, 2)'].join(''), 'javascript', CellKind.Code, [{ outputId: 'someId', outputs: [{ mime: Mimes.text, data: VSBuffer.wrap(new Uint8Array([2])) }] }], { custom: { metadata: { collapsed: false } }, executionOrder: 6 }], ['', 'javascript', CellKind.Code, [], {}] ], (model, accessor) => { const diff = new LcsDiff(new CellSequence(model.original.notebook), new CellSequence(model.modified.notebook)); @@ -273,13 +274,13 @@ suite('NotebookCommon', () => { await withTestNotebookDiffModel([ ['# Description', 'markdown', CellKind.Markup, [], { custom: { metadata: {} } }], ['x = 3', 'javascript', CellKind.Code, [], { custom: { metadata: { collapsed: true } }, executionOrder: 1 }], - ['x', 'javascript', CellKind.Code, [{ outputId: 'someId', outputs: [{ mime: Mimes.text, data: new Uint8Array([3]) }] }], { custom: { metadata: { collapsed: false } }, executionOrder: 1 }], + ['x', 'javascript', CellKind.Code, [{ outputId: 'someId', outputs: [{ mime: Mimes.text, data: VSBuffer.wrap(new Uint8Array([3])) }] }], { custom: { metadata: { collapsed: false } }, executionOrder: 1 }], ['x', 'javascript', CellKind.Code, [], { custom: { metadata: { collapsed: false } } }] ], [ ['# Description', 'markdown', CellKind.Markup, [], { custom: { metadata: {} } }], ['x = 3', 'javascript', CellKind.Code, [], { custom: { metadata: { collapsed: true } }, executionOrder: 1 }], ['x', 'javascript', CellKind.Code, [], { custom: { metadata: { collapsed: false } } }], - ['x', 'javascript', CellKind.Code, [{ outputId: 'someId', outputs: [{ mime: Mimes.text, data: new Uint8Array([3]) }] }], { custom: { metadata: { collapsed: false } }, executionOrder: 1 }] + ['x', 'javascript', CellKind.Code, [{ outputId: 'someId', outputs: [{ mime: Mimes.text, data: VSBuffer.wrap(new Uint8Array([3])) }] }], { custom: { metadata: { collapsed: false } }, executionOrder: 1 }] ], async (model) => { const diff = new LcsDiff(new CellSequence(model.original.notebook), new CellSequence(model.modified.notebook)); const diffResult = diff.ComputeDiff(false); @@ -306,18 +307,18 @@ suite('NotebookCommon', () => { await withTestNotebookDiffModel([ ['# Description', 'markdown', CellKind.Markup, [], { custom: { metadata: {} } }], ['x = 3', 'javascript', CellKind.Code, [], { custom: { metadata: { collapsed: true } }, executionOrder: 1 }], - ['x', 'javascript', CellKind.Code, [{ outputId: 'someId', outputs: [{ mime: Mimes.text, data: new Uint8Array([3]) }] }], { custom: { metadata: { collapsed: false } }, executionOrder: 1 }], + ['x', 'javascript', CellKind.Code, [{ outputId: 'someId', outputs: [{ mime: Mimes.text, data: VSBuffer.wrap(new Uint8Array([3])) }] }], { custom: { metadata: { collapsed: false } }, executionOrder: 1 }], ['x', 'javascript', CellKind.Code, [], { custom: { metadata: { collapsed: false } } }], ['x = 5', 'javascript', CellKind.Code, [], {}], ['x', 'javascript', CellKind.Code, [], {}], - ['x', 'javascript', CellKind.Code, [{ outputId: 'someId', outputs: [{ mime: Mimes.text, data: new Uint8Array([5]) }] }], {}], + ['x', 'javascript', CellKind.Code, [{ outputId: 'someId', outputs: [{ mime: Mimes.text, data: VSBuffer.wrap(new Uint8Array([5])) }] }], {}], ], [ ['# Description', 'markdown', CellKind.Markup, [], { custom: { metadata: {} } }], ['x = 3', 'javascript', CellKind.Code, [], { custom: { metadata: { collapsed: true } }, executionOrder: 1 }], ['x', 'javascript', CellKind.Code, [], { custom: { metadata: { collapsed: false } } }], - ['x', 'javascript', CellKind.Code, [{ outputId: 'someId', outputs: [{ mime: Mimes.text, data: new Uint8Array([3]) }] }], { custom: { metadata: { collapsed: false } }, executionOrder: 1 }], + ['x', 'javascript', CellKind.Code, [{ outputId: 'someId', outputs: [{ mime: Mimes.text, data: VSBuffer.wrap(new Uint8Array([3])) }] }], { custom: { metadata: { collapsed: false } }, executionOrder: 1 }], ['x = 5', 'javascript', CellKind.Code, [], {}], - ['x', 'javascript', CellKind.Code, [{ outputId: 'someId', outputs: [{ mime: Mimes.text, data: new Uint8Array([5]) }] }], {}], + ['x', 'javascript', CellKind.Code, [{ outputId: 'someId', outputs: [{ mime: Mimes.text, data: VSBuffer.wrap(new Uint8Array([5])) }] }], {}], ['x', 'javascript', CellKind.Code, [], {}], ], async (model) => { const diff = new LcsDiff(new CellSequence(model.original.notebook), new CellSequence(model.modified.notebook)); diff --git a/src/vs/workbench/contrib/notebook/test/notebookTextModel.test.ts b/src/vs/workbench/contrib/notebook/test/notebookTextModel.test.ts index ba773781edd..a5e3bc53ade 100644 --- a/src/vs/workbench/contrib/notebook/test/notebookTextModel.test.ts +++ b/src/vs/workbench/contrib/notebook/test/notebookTextModel.test.ts @@ -4,6 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import * as assert from 'assert'; +import { VSBuffer } from 'vs/base/common/buffer'; import { Mimes } from 'vs/base/common/mime'; import { IModeService } from 'vs/editor/common/services/modeService'; import { IUndoRedoService } from 'vs/platform/undoRedo/common/undoRedo'; @@ -426,7 +427,7 @@ suite('NotebookTextModel', () => { const success1 = model.applyEdits( [{ editType: CellEditType.Output, index: 0, outputs: [ - { outputId: 'out1', outputs: [{ mime: 'application/x.notebook.stream', data: new Uint8Array([1]) }] } + { outputId: 'out1', outputs: [{ mime: 'application/x.notebook.stream', data: VSBuffer.wrap(new Uint8Array([1])) }] } ], append: false }], true, undefined, () => undefined, undefined, false @@ -438,7 +439,7 @@ suite('NotebookTextModel', () => { const success2 = model.applyEdits( [{ editType: CellEditType.Output, index: 0, outputs: [ - { outputId: 'out2', outputs: [{ mime: 'application/x.notebook.stream', data: new Uint8Array([1]) }] } + { outputId: 'out2', outputs: [{ mime: 'application/x.notebook.stream', data: VSBuffer.wrap(new Uint8Array([1])) }] } ], append: true }], true, undefined, () => undefined, undefined, false @@ -465,7 +466,7 @@ suite('NotebookTextModel', () => { const success = model.applyEdits( [{ editType: CellEditType.Output, index: 0, outputs: [ - { outputId: 'out1', outputs: [{ mime: 'application/x.notebook.stream', data: new Uint8Array([1]) }] } + { outputId: 'out1', outputs: [{ mime: 'application/x.notebook.stream', data: VSBuffer.wrap(new Uint8Array([1])) }] } ], append: false }], true, undefined, () => undefined, undefined, false diff --git a/src/vs/workbench/contrib/notebook/test/testNotebookEditor.ts b/src/vs/workbench/contrib/notebook/test/testNotebookEditor.ts index c5322cef25f..0d2f290df58 100644 --- a/src/vs/workbench/contrib/notebook/test/testNotebookEditor.ts +++ b/src/vs/workbench/contrib/notebook/test/testNotebookEditor.ts @@ -47,6 +47,7 @@ import { NotebookOptions } from 'vs/workbench/contrib/notebook/common/notebookOp import { ViewContext } from 'vs/workbench/contrib/notebook/browser/viewModel/viewContext'; import { OutputRenderer } from 'vs/workbench/contrib/notebook/browser/view/output/outputRenderer'; import { Mimes } from 'vs/base/common/mime'; +import { VSBuffer } from 'vs/base/common/buffer'; export class TestCell extends NotebookCellTextModel { constructor( @@ -345,6 +346,6 @@ export function createNotebookCellList(instantiationService: TestInstantiationSe return cellList; } -export function valueBytesFromString(value: string) { - return new TextEncoder().encode(value); +export function valueBytesFromString(value: string): VSBuffer { + return VSBuffer.fromString(value); } diff --git a/src/vs/workbench/services/extensions/common/proxyIdentifier.ts b/src/vs/workbench/services/extensions/common/proxyIdentifier.ts index 2a850898788..a591d963ab3 100644 --- a/src/vs/workbench/services/extensions/common/proxyIdentifier.ts +++ b/src/vs/workbench/services/extensions/common/proxyIdentifier.ts @@ -57,3 +57,12 @@ export function createExtHostContextProxyIdentifier(identifier: string): Prox export function getStringIdentifierForProxy(nid: number): string { return identifiers[nid].sid; } + +/** + * Marks the object as containing buffers that should be serialized more efficently. + */ +export class SerializableObjectWithBuffers { + constructor( + public readonly value: T + ) { } +} diff --git a/src/vs/workbench/services/extensions/common/rpcProtocol.ts b/src/vs/workbench/services/extensions/common/rpcProtocol.ts index 8e1d9ee523f..db5188fdbc3 100644 --- a/src/vs/workbench/services/extensions/common/rpcProtocol.ts +++ b/src/vs/workbench/services/extensions/common/rpcProtocol.ts @@ -4,17 +4,17 @@ *--------------------------------------------------------------------------------------------*/ import { RunOnceScheduler } from 'vs/base/common/async'; +import { VSBuffer } from 'vs/base/common/buffer'; import { CancellationToken, CancellationTokenSource } from 'vs/base/common/cancellation'; import { CharCode } from 'vs/base/common/charCode'; import * as errors from 'vs/base/common/errors'; import { Emitter, Event } from 'vs/base/common/event'; import { Disposable } from 'vs/base/common/lifecycle'; +import { MarshalledId, MarshalledObject } from 'vs/base/common/marshalling'; import { IURITransformer, transformIncomingURIs } from 'vs/base/common/uriIpc'; import { IMessagePassingProtocol } from 'vs/base/parts/ipc/common/ipc'; import { LazyPromise } from 'vs/workbench/services/extensions/common/lazyPromise'; -import { IRPCProtocol, ProxyIdentifier, getStringIdentifierForProxy } from 'vs/workbench/services/extensions/common/proxyIdentifier'; -import { VSBuffer } from 'vs/base/common/buffer'; -import { MarshalledId } from 'vs/base/common/marshalling'; +import { getStringIdentifierForProxy, IRPCProtocol, ProxyIdentifier, SerializableObjectWithBuffers } from 'vs/workbench/services/extensions/common/proxyIdentifier'; export interface JSONStringifyReplacer { (key: string, value: any): any; @@ -28,6 +28,55 @@ function safeStringify(obj: any, replacer: JSONStringifyReplacer | null): string } } +const refSymbolName = '$$ref$$'; +const undefinedRef = { [refSymbolName]: -1 } as const; + +class StringifiedJsonWithBufferRefs { + constructor( + public readonly jsonString: string, + public readonly referencedBuffers: readonly VSBuffer[], + ) { } +} + +export function stringifyJsonWithBufferRefs(obj: T, replacer: JSONStringifyReplacer | null = null, useSafeStringify = false): StringifiedJsonWithBufferRefs { + const foundBuffers: VSBuffer[] = []; + const serialized = (useSafeStringify ? safeStringify : JSON.stringify)(obj, (key, value) => { + if (typeof value === 'undefined') { + return undefinedRef; // JSON.stringify normally converts 'undefined' to 'null' + } else if (typeof value === 'object') { + if (value instanceof VSBuffer) { + const bufferIndex = foundBuffers.push(value) - 1; + return { [refSymbolName]: bufferIndex }; + } + if (replacer) { + return replacer(key, value); + } + } + return value; + }); + return { + jsonString: serialized, + referencedBuffers: foundBuffers + }; +} + +export function parseJsonAndRestoreBufferRefs(jsonString: string, buffers: readonly VSBuffer[], uriTransformer: IURITransformer | null): any { + return JSON.parse(jsonString, (_key, value) => { + if (value) { + const ref = value[refSymbolName]; + if (typeof ref === 'number') { + return buffers[ref]; + } + + if (uriTransformer && (value).$mid === MarshalledId.Uri) { + return uriTransformer.transformIncoming(value); + } + } + return value; + }); +} + + function stringify(obj: any, replacer: JSONStringifyReplacer | null): string { return JSON.stringify(obj, <(key: string, value: any) => any>replacer); } @@ -278,6 +327,11 @@ export class RPCProtocol extends Disposable implements IRPCProtocol { this._receiveReply(msgLength, req, value); break; } + case MessageType.ReplyOKJSONWithBuffers: { + const value = MessageIO.deserializeReplyOKJSONWithBuffers(buff, this._uriTransformer); + this._receiveReply(msgLength, req, value); + break; + } case MessageType.ReplyOKVSBuffer: { let value = MessageIO.deserializeReplyOKVSBuffer(buff); this._receiveReply(msgLength, req, value); @@ -558,19 +612,25 @@ class MessageBuffer { return buff; } - public static sizeMixedArray(arr: VSBuffer[], arrType: ArgType[]): number { + public static sizeMixedArray(arr: readonly MixedArg[]): number { let size = 0; size += 1; // arr length for (let i = 0, len = arr.length; i < len; i++) { const el = arr[i]; - const elType = arrType[i]; size += 1; // arg type - switch (elType) { + switch (el.type) { case ArgType.String: - size += this.sizeLongString(el); + size += this.sizeLongString(el.value); break; case ArgType.VSBuffer: - size += this.sizeVSBuffer(el); + size += this.sizeVSBuffer(el.value); + break; + case ArgType.SerializedObjectWithBuffers: + size += this.sizeUInt8(); // buffer count + size += this.sizeLongString(el.value); + for (let i = 0; i < el.buffers.length; ++i) { + size += this.sizeVSBuffer(el.buffers[i]); + } break; case ArgType.Undefined: // empty... @@ -580,19 +640,26 @@ class MessageBuffer { return size; } - public writeMixedArray(arr: VSBuffer[], arrType: ArgType[]): void { + public writeMixedArray(arr: readonly MixedArg[]): void { this._buff.writeUInt8(arr.length, this._offset); this._offset += 1; for (let i = 0, len = arr.length; i < len; i++) { const el = arr[i]; - const elType = arrType[i]; - switch (elType) { + switch (el.type) { case ArgType.String: this.writeUInt8(ArgType.String); - this.writeLongString(el); + this.writeLongString(el.value); break; case ArgType.VSBuffer: this.writeUInt8(ArgType.VSBuffer); - this.writeVSBuffer(el); + this.writeVSBuffer(el.value); + break; + case ArgType.SerializedObjectWithBuffers: + this.writeUInt8(ArgType.SerializedObjectWithBuffers); + this.writeUInt8(el.buffers.length); + this.writeLongString(el.value); + for (let i = 0; i < el.buffers.length; ++i) { + this.writeBuffer(el.buffers[i]); + } break; case ArgType.Undefined: this.writeUInt8(ArgType.Undefined); @@ -601,9 +668,9 @@ class MessageBuffer { } } - public readMixedArray(): Array { + public readMixedArray(): Array | undefined> { const arrLen = this._buff.readUInt8(this._offset); this._offset += 1; - let arr: Array = new Array(arrLen); + let arr: Array | undefined> = new Array(arrLen); for (let i = 0; i < arrLen; i++) { const argType = this.readUInt8(); switch (argType) { @@ -613,6 +680,15 @@ class MessageBuffer { case ArgType.VSBuffer: arr[i] = this.readVSBuffer(); break; + case ArgType.SerializedObjectWithBuffers: + const bufferCount = this.readUInt8(); + const jsonString = this.readLongString(); + const buffers: VSBuffer[] = []; + for (let i = 0; i < bufferCount; ++i) { + buffers.push(this.readVSBuffer()); + } + arr[i] = new SerializableObjectWithBuffers(parseJsonAndRestoreBufferRefs(jsonString, buffers, null)); + break; case ArgType.Undefined: arr[i] = undefined; break; @@ -622,15 +698,26 @@ class MessageBuffer { } } -type SerializedRequestArguments = { type: 'mixed'; args: VSBuffer[]; argsType: ArgType[]; } | { type: 'simple'; args: string; }; +const enum SerializedRequestArgumentType { + Simple, + Mixed, +} + +type SerializedRequestArguments = + | { readonly type: SerializedRequestArgumentType.Simple; args: string; } + | { readonly type: SerializedRequestArgumentType.Mixed; args: MixedArg[] }; + class MessageIO { - private static _arrayContainsBufferOrUndefined(arr: any[]): boolean { + private static _useMixedArgSerialization(arr: any[]): boolean { for (let i = 0, len = arr.length; i < len; i++) { if (arr[i] instanceof VSBuffer) { return true; } + if (arr[i] instanceof SerializableObjectWithBuffers) { + return true; + } if (typeof arr[i] === 'undefined') { return true; } @@ -639,39 +726,39 @@ class MessageIO { } public static serializeRequestArguments(args: any[], replacer: JSONStringifyReplacer | null): SerializedRequestArguments { - if (this._arrayContainsBufferOrUndefined(args)) { - let massagedArgs: VSBuffer[] = []; - let massagedArgsType: ArgType[] = []; + if (this._useMixedArgSerialization(args)) { + const massagedArgs: MixedArg[] = []; for (let i = 0, len = args.length; i < len; i++) { const arg = args[i]; if (arg instanceof VSBuffer) { - massagedArgs[i] = arg; - massagedArgsType[i] = ArgType.VSBuffer; + massagedArgs[i] = { type: ArgType.VSBuffer, value: arg }; } else if (typeof arg === 'undefined') { - massagedArgs[i] = VSBuffer.alloc(0); - massagedArgsType[i] = ArgType.Undefined; + massagedArgs[i] = { type: ArgType.Undefined }; + } else if (arg instanceof SerializableObjectWithBuffers) { + const { jsonString, referencedBuffers } = stringifyJsonWithBufferRefs(arg.value, replacer); + massagedArgs[i] = { type: ArgType.SerializedObjectWithBuffers, value: VSBuffer.fromString(jsonString), buffers: referencedBuffers }; } else { - massagedArgs[i] = VSBuffer.fromString(stringify(arg, replacer)); - massagedArgsType[i] = ArgType.String; + massagedArgs[i] = { type: ArgType.String, value: VSBuffer.fromString(stringify(arg, replacer)) }; } } return { - type: 'mixed', + type: SerializedRequestArgumentType.Mixed, args: massagedArgs, - argsType: massagedArgsType }; } return { - type: 'simple', + type: SerializedRequestArgumentType.Simple, args: stringify(args, replacer) }; } public static serializeRequest(req: number, rpcId: number, method: string, serializedArgs: SerializedRequestArguments, usesCancellationToken: boolean): VSBuffer { - if (serializedArgs.type === 'mixed') { - return this._requestMixedArgs(req, rpcId, method, serializedArgs.args, serializedArgs.argsType, usesCancellationToken); + switch (serializedArgs.type) { + case SerializedRequestArgumentType.Simple: + return this._requestJSONArgs(req, rpcId, method, serializedArgs.args, usesCancellationToken); + case SerializedRequestArgumentType.Mixed: + return this._requestMixedArgs(req, rpcId, method, serializedArgs.args, usesCancellationToken); } - return this._requestJSONArgs(req, rpcId, method, serializedArgs.args, usesCancellationToken); } private static _requestJSONArgs(req: number, rpcId: number, method: string, args: string, usesCancellationToken: boolean): VSBuffer { @@ -701,18 +788,18 @@ class MessageIO { }; } - private static _requestMixedArgs(req: number, rpcId: number, method: string, args: VSBuffer[], argsType: ArgType[], usesCancellationToken: boolean): VSBuffer { + private static _requestMixedArgs(req: number, rpcId: number, method: string, args: readonly MixedArg[], usesCancellationToken: boolean): VSBuffer { const methodBuff = VSBuffer.fromString(method); let len = 0; len += MessageBuffer.sizeUInt8(); len += MessageBuffer.sizeShortString(methodBuff); - len += MessageBuffer.sizeMixedArray(args, argsType); + len += MessageBuffer.sizeMixedArray(args); let result = MessageBuffer.alloc(usesCancellationToken ? MessageType.RequestMixedArgsWithCancellation : MessageType.RequestMixedArgs, req, len); result.writeUInt8(rpcId); result.writeShortString(methodBuff); - result.writeMixedArray(args, argsType); + result.writeMixedArray(args); return result.buffer; } @@ -747,11 +834,14 @@ class MessageIO { public static serializeReplyOK(req: number, res: any, replacer: JSONStringifyReplacer | null): VSBuffer { if (typeof res === 'undefined') { return this._serializeReplyOKEmpty(req); - } - if (res instanceof VSBuffer) { + } else if (res instanceof VSBuffer) { return this._serializeReplyOKVSBuffer(req, res); + } else if (res instanceof SerializableObjectWithBuffers) { + const { jsonString, referencedBuffers } = stringifyJsonWithBufferRefs(res.value, replacer, true); + return this._serializeReplyOKJSONWithBuffers(req, jsonString, referencedBuffers); + } else { + return this._serializeReplyOKJSON(req, safeStringify(res, replacer)); } - return this._serializeReplyOKJSON(req, safeStringify(res, replacer)); } private static _serializeReplyOKEmpty(req: number): VSBuffer { @@ -782,11 +872,43 @@ class MessageIO { return result.buffer; } + private static _serializeReplyOKJSONWithBuffers(req: number, res: string, buffers: readonly VSBuffer[]): VSBuffer { + const resBuff = VSBuffer.fromString(res); + + let len = 0; + len += MessageBuffer.sizeUInt8(); // buffer count + len += MessageBuffer.sizeLongString(resBuff); + for (const buffer of buffers) { + len += MessageBuffer.sizeVSBuffer(buffer); + } + + let result = MessageBuffer.alloc(MessageType.ReplyOKJSONWithBuffers, req, len); + result.writeUInt8(buffers.length); + result.writeLongString(resBuff); + for (const buffer of buffers) { + result.writeBuffer(buffer); + } + + return result.buffer; + } + public static deserializeReplyOKJSON(buff: MessageBuffer): any { const res = buff.readLongString(); return JSON.parse(res); } + public static deserializeReplyOKJSONWithBuffers(buff: MessageBuffer, uriTransformer: IURITransformer | null): SerializableObjectWithBuffers { + const bufferCount = buff.readUInt8(); + const res = buff.readLongString(); + + const buffers: VSBuffer[] = []; + for (let i = 0; i < bufferCount; ++i) { + buffers.push(buff.readVSBuffer()); + } + + return new SerializableObjectWithBuffers(parseJsonAndRestoreBufferRefs(res, buffers, uriTransformer)); + } + public static serializeReplyErr(req: number, err: any): VSBuffer { if (err) { return this._serializeReplyErrEror(req, err); @@ -825,12 +947,22 @@ const enum MessageType { ReplyOKEmpty = 7, ReplyOKVSBuffer = 8, ReplyOKJSON = 9, - ReplyErrError = 10, - ReplyErrEmpty = 11, + ReplyOKJSONWithBuffers = 10, + ReplyErrError = 11, + ReplyErrEmpty = 12, } const enum ArgType { String = 1, VSBuffer = 2, - Undefined = 3 + SerializedObjectWithBuffers = 3, + Undefined = 4, } + + +type MixedArg = + | { readonly type: ArgType.String, readonly value: VSBuffer } + | { readonly type: ArgType.VSBuffer, readonly value: VSBuffer } + | { readonly type: ArgType.SerializedObjectWithBuffers, readonly value: VSBuffer, readonly buffers: readonly VSBuffer[] } + | { readonly type: ArgType.Undefined } + ; diff --git a/src/vs/workbench/services/extensions/test/common/rpcProtocol.test.ts b/src/vs/workbench/services/extensions/test/common/rpcProtocol.test.ts index 5264e3d7801..f500096ba1a 100644 --- a/src/vs/workbench/services/extensions/test/common/rpcProtocol.test.ts +++ b/src/vs/workbench/services/extensions/test/common/rpcProtocol.test.ts @@ -7,7 +7,7 @@ import * as assert from 'assert'; import { CancellationToken, CancellationTokenSource } from 'vs/base/common/cancellation'; import { Emitter, Event } from 'vs/base/common/event'; import { IMessagePassingProtocol } from 'vs/base/parts/ipc/common/ipc'; -import { ProxyIdentifier } from 'vs/workbench/services/extensions/common/proxyIdentifier'; +import { ProxyIdentifier, SerializableObjectWithBuffers } from 'vs/workbench/services/extensions/common/proxyIdentifier'; import { RPCProtocol } from 'vs/workbench/services/extensions/common/rpcProtocol'; import { VSBuffer } from 'vs/base/common/buffer'; @@ -211,4 +211,31 @@ suite('RPCProtocol', () => { bProxy.$m(badObject, '2'); }); }); + + test('SerializableObjectWithBuffers is correctly transfered', function (done) { + delegate = (a1: SerializableObjectWithBuffers<{ string: string, buff: VSBuffer }>, a2: number) => { + return new SerializableObjectWithBuffers({ string: a1.value.string + ' world', buff: a1.value.buff }); + }; + + const b = VSBuffer.alloc(4); + b.buffer[0] = 1; + b.buffer[1] = 2; + b.buffer[2] = 3; + b.buffer[3] = 4; + + bProxy.$m(new SerializableObjectWithBuffers({ string: 'hello', buff: b }), undefined).then((res: SerializableObjectWithBuffers) => { + assert.ok(res instanceof SerializableObjectWithBuffers); + assert.strictEqual(res.value.string, 'hello world'); + + assert.ok(res.value.buff instanceof VSBuffer); + + const bufferValues = Array.from(res.value.buff.buffer); + + assert.strictEqual(bufferValues[0], 1); + assert.strictEqual(bufferValues[1], 2); + assert.strictEqual(bufferValues[2], 3); + assert.strictEqual(bufferValues[3], 4); + done(null); + }, done); + }); }); diff --git a/src/vs/workbench/test/browser/api/extHostNotebook.test.ts b/src/vs/workbench/test/browser/api/extHostNotebook.test.ts index 79c6a109595..e2cd5aa061a 100644 --- a/src/vs/workbench/test/browser/api/extHostNotebook.test.ts +++ b/src/vs/workbench/test/browser/api/extHostNotebook.test.ts @@ -23,6 +23,7 @@ import { IExtensionStoragePaths } from 'vs/workbench/api/common/extHostStoragePa import { generateUuid } from 'vs/base/common/uuid'; import { Event } from 'vs/base/common/event'; import { ExtHostNotebookDocuments } from 'vs/workbench/api/common/extHostNotebookDocuments'; +import { SerializableObjectWithBuffers } from 'vs/workbench/services/extensions/common/proxyIdentifier'; suite('NotebookCell#Document', function () { @@ -63,7 +64,7 @@ suite('NotebookCell#Document', function () { let reg = extHostNotebooks.registerNotebookContentProvider(nullExtensionDescription, 'test', new class extends mock() { // async openNotebook() { } }); - extHostNotebooks.$acceptDocumentAndEditorsDelta({ + extHostNotebooks.$acceptDocumentAndEditorsDelta(new SerializableObjectWithBuffers({ addedDocuments: [{ uri: notebookUri, viewType: 'test', @@ -92,8 +93,8 @@ suite('NotebookCell#Document', function () { selections: [{ start: 0, end: 1 }], visibleRanges: [] }] - }); - extHostNotebooks.$acceptDocumentAndEditorsDelta({ newActiveEditor: '_notebook_editor_0' }); + })); + extHostNotebooks.$acceptDocumentAndEditorsDelta(new SerializableObjectWithBuffers({ newActiveEditor: '_notebook_editor_0' })); notebook = extHostNotebooks.notebookDocuments[0]!; @@ -134,7 +135,7 @@ suite('NotebookCell#Document', function () { removedCellUris.push(doc.uri.toString()); }); - extHostNotebooks.$acceptDocumentAndEditorsDelta({ removedDocuments: [notebook.uri] }); + extHostNotebooks.$acceptDocumentAndEditorsDelta(new SerializableObjectWithBuffers({ removedDocuments: [notebook.uri] })); reg.dispose(); assert.strictEqual(removedCellUris.length, 2); @@ -167,7 +168,7 @@ suite('NotebookCell#Document', function () { }); }); - extHostNotebookDocuments.$acceptModelChanged(notebookUri, { + extHostNotebookDocuments.$acceptModelChanged(notebookUri, new SerializableObjectWithBuffers({ versionId: notebook.apiNotebook.version + 1, rawEvents: [ { @@ -191,7 +192,7 @@ suite('NotebookCell#Document', function () { }]]] } ] - }, false); + }), false); await p; @@ -229,7 +230,7 @@ suite('NotebookCell#Document', function () { } // close notebook -> docs are closed - extHostNotebooks.$acceptDocumentAndEditorsDelta({ removedDocuments: [notebook.uri] }); + extHostNotebooks.$acceptDocumentAndEditorsDelta(new SerializableObjectWithBuffers({ removedDocuments: [notebook.uri] })); for (let cell of notebook.apiNotebook.getCells()) { assert.throws(() => extHostDocuments.getDocument(cell.document.uri)); } @@ -243,7 +244,7 @@ suite('NotebookCell#Document', function () { assert.strictEqual(notebook.apiNotebook.cellCount, 2); const [cell1, cell2] = notebook.apiNotebook.getCells(); - extHostNotebookDocuments.$acceptModelChanged(notebook.uri, { + extHostNotebookDocuments.$acceptModelChanged(notebook.uri, new SerializableObjectWithBuffers({ versionId: 2, rawEvents: [ { @@ -251,7 +252,7 @@ suite('NotebookCell#Document', function () { changes: [[0, 1, []]] } ] - }, false); + }), false); assert.strictEqual(notebook.apiNotebook.cellCount, 1); assert.strictEqual(cell1.document.isClosed, true); // ref still alive! @@ -274,18 +275,18 @@ suite('NotebookCell#Document', function () { assert.strictEqual(second.index, 1); // remove first cell - extHostNotebookDocuments.$acceptModelChanged(notebook.uri, { + extHostNotebookDocuments.$acceptModelChanged(notebook.uri, new SerializableObjectWithBuffers({ versionId: notebook.apiNotebook.version + 1, rawEvents: [{ kind: NotebookCellsChangeType.ModelChange, changes: [[0, 1, []]] }] - }, false); + }), false); assert.strictEqual(notebook.apiNotebook.cellCount, 1); assert.strictEqual(second.index, 0); - extHostNotebookDocuments.$acceptModelChanged(notebookUri, { + extHostNotebookDocuments.$acceptModelChanged(notebookUri, new SerializableObjectWithBuffers({ versionId: notebook.apiNotebook.version + 1, rawEvents: [{ kind: NotebookCellsChangeType.ModelChange, @@ -307,7 +308,7 @@ suite('NotebookCell#Document', function () { outputs: [], }]]] }] - }, false); + }), false); assert.strictEqual(notebook.apiNotebook.cellCount, 3); assert.strictEqual(second.index, 2); @@ -320,7 +321,7 @@ suite('NotebookCell#Document', function () { // DON'T call this, make sure the cell-documents have not been created yet // assert.strictEqual(notebook.notebookDocument.cellCount, 2); - extHostNotebookDocuments.$acceptModelChanged(notebook.uri, { + extHostNotebookDocuments.$acceptModelChanged(notebook.uri, new SerializableObjectWithBuffers({ versionId: 100, rawEvents: [{ kind: NotebookCellsChangeType.ModelChange, @@ -342,7 +343,7 @@ suite('NotebookCell#Document', function () { outputs: [], }]]] }] - }, false); + }), false); assert.strictEqual(notebook.apiNotebook.cellCount, 2); @@ -363,18 +364,18 @@ suite('NotebookCell#Document', function () { let count = 0; extHostNotebooks.onDidChangeActiveNotebookEditor(() => count += 1); - extHostNotebooks.$acceptDocumentAndEditorsDelta({ + extHostNotebooks.$acceptDocumentAndEditorsDelta(new SerializableObjectWithBuffers({ addedEditors: [{ documentUri: notebookUri, id: '_notebook_editor_2', selections: [{ start: 0, end: 1 }], visibleRanges: [] }] - }); + })); - extHostNotebooks.$acceptDocumentAndEditorsDelta({ + extHostNotebooks.$acceptDocumentAndEditorsDelta(new SerializableObjectWithBuffers({ newActiveEditor: '_notebook_editor_2' - }); + })); assert.strictEqual(count, 1); }); @@ -384,13 +385,13 @@ suite('NotebookCell#Document', function () { const editor = extHostNotebooks.activeNotebookEditor; assert.ok(editor !== undefined); - extHostNotebooks.$acceptDocumentAndEditorsDelta({ newActiveEditor: undefined }); + extHostNotebooks.$acceptDocumentAndEditorsDelta(new SerializableObjectWithBuffers({ newActiveEditor: undefined })); assert.ok(extHostNotebooks.activeNotebookEditor === editor); - extHostNotebooks.$acceptDocumentAndEditorsDelta({}); + extHostNotebooks.$acceptDocumentAndEditorsDelta(new SerializableObjectWithBuffers({})); assert.ok(extHostNotebooks.activeNotebookEditor === editor); - extHostNotebooks.$acceptDocumentAndEditorsDelta({ newActiveEditor: null }); + extHostNotebooks.$acceptDocumentAndEditorsDelta(new SerializableObjectWithBuffers({ newActiveEditor: null })); assert.ok(extHostNotebooks.activeNotebookEditor === undefined); }); @@ -403,13 +404,13 @@ suite('NotebookCell#Document', function () { const removed = Event.toPromise(extHostDocuments.onDidRemoveDocument); const added = Event.toPromise(extHostDocuments.onDidAddDocument); - extHostNotebookDocuments.$acceptModelChanged(notebook.uri, { + extHostNotebookDocuments.$acceptModelChanged(notebook.uri, new SerializableObjectWithBuffers({ versionId: 12, rawEvents: [{ kind: NotebookCellsChangeType.ChangeLanguage, index: 0, language: 'fooLang' }] - }, false); + }), false); const removedDoc = await removed; const addedDoc = await added; diff --git a/src/vs/workbench/test/browser/api/extHostNotebookConcatDocument.test.ts b/src/vs/workbench/test/browser/api/extHostNotebookConcatDocument.test.ts index 6ff7b0f14af..b969649d253 100644 --- a/src/vs/workbench/test/browser/api/extHostNotebookConcatDocument.test.ts +++ b/src/vs/workbench/test/browser/api/extHostNotebookConcatDocument.test.ts @@ -23,6 +23,7 @@ import { DisposableStore } from 'vs/base/common/lifecycle'; import { IExtensionStoragePaths } from 'vs/workbench/api/common/extHostStoragePaths'; import { generateUuid } from 'vs/base/common/uuid'; import { ExtHostNotebookDocuments } from 'vs/workbench/api/common/extHostNotebookDocuments'; +import { SerializableObjectWithBuffers } from 'vs/workbench/services/extensions/common/proxyIdentifier'; suite('NotebookConcatDocument', function () { @@ -60,7 +61,7 @@ suite('NotebookConcatDocument', function () { let reg = extHostNotebooks.registerNotebookContentProvider(nullExtensionDescription, 'test', new class extends mock() { // async openNotebook() { } }); - extHostNotebooks.$acceptDocumentAndEditorsDelta({ + extHostNotebooks.$acceptDocumentAndEditorsDelta(new SerializableObjectWithBuffers({ addedDocuments: [{ uri: notebookUri, viewType: 'test', @@ -81,8 +82,8 @@ suite('NotebookConcatDocument', function () { selections: [{ start: 0, end: 1 }], visibleRanges: [] }] - }); - extHostNotebooks.$acceptDocumentAndEditorsDelta({ newActiveEditor: '_notebook_editor_0' }); + })); + extHostNotebooks.$acceptDocumentAndEditorsDelta(new SerializableObjectWithBuffers({ newActiveEditor: '_notebook_editor_0' })); notebook = extHostNotebooks.notebookDocuments[0]!; @@ -127,7 +128,7 @@ suite('NotebookConcatDocument', function () { const cellUri1 = CellUri.generate(notebook.uri, 1); const cellUri2 = CellUri.generate(notebook.uri, 2); - extHostNotebookDocuments.$acceptModelChanged(notebookUri, { + extHostNotebookDocuments.$acceptModelChanged(notebookUri, new SerializableObjectWithBuffers({ versionId: notebook.apiNotebook.version + 1, rawEvents: [{ kind: NotebookCellsChangeType.ModelChange, @@ -150,7 +151,7 @@ suite('NotebookConcatDocument', function () { }]] ] }] - }, false); + }), false); assert.strictEqual(notebook.apiNotebook.cellCount, 1 + 2); // markdown and code @@ -164,7 +165,7 @@ suite('NotebookConcatDocument', function () { test('location, position mapping', function () { - extHostNotebookDocuments.$acceptModelChanged(notebookUri, { + extHostNotebookDocuments.$acceptModelChanged(notebookUri, new SerializableObjectWithBuffers({ versionId: notebook.apiNotebook.version + 1, rawEvents: [ { @@ -188,7 +189,7 @@ suite('NotebookConcatDocument', function () { }]]] } ] - }, false); + }), false); assert.strictEqual(notebook.apiNotebook.cellCount, 1 + 2); // markdown and code @@ -209,7 +210,7 @@ suite('NotebookConcatDocument', function () { let doc = new ExtHostNotebookConcatDocument(extHostNotebooks, extHostDocuments, notebook.apiNotebook, undefined); // UPDATE 1 - extHostNotebookDocuments.$acceptModelChanged(notebookUri, { + extHostNotebookDocuments.$acceptModelChanged(notebookUri, new SerializableObjectWithBuffers({ versionId: notebook.apiNotebook.version + 1, rawEvents: [ { @@ -225,7 +226,7 @@ suite('NotebookConcatDocument', function () { }]]] } ] - }, false); + }), false); assert.strictEqual(notebook.apiNotebook.cellCount, 1 + 1); assert.strictEqual(doc.version, 1); assertLines(doc, 'Hello', 'World', 'Hello World!'); @@ -236,7 +237,7 @@ suite('NotebookConcatDocument', function () { // UPDATE 2 - extHostNotebookDocuments.$acceptModelChanged(notebookUri, { + extHostNotebookDocuments.$acceptModelChanged(notebookUri, new SerializableObjectWithBuffers({ versionId: notebook.apiNotebook.version + 1, rawEvents: [ { @@ -252,7 +253,7 @@ suite('NotebookConcatDocument', function () { }]]] } ] - }, false); + }), false); assert.strictEqual(notebook.apiNotebook.cellCount, 1 + 2); assert.strictEqual(doc.version, 2); @@ -264,7 +265,7 @@ suite('NotebookConcatDocument', function () { assertLocation(doc, new Position(5, 12), new Location(notebook.apiNotebook.cellAt(1).document.uri, new Position(2, 11)), false); // don't check identity because position will be clamped // UPDATE 3 (remove cell #2 again) - extHostNotebookDocuments.$acceptModelChanged(notebookUri, { + extHostNotebookDocuments.$acceptModelChanged(notebookUri, new SerializableObjectWithBuffers({ versionId: notebook.apiNotebook.version + 1, rawEvents: [ { @@ -272,7 +273,7 @@ suite('NotebookConcatDocument', function () { changes: [[1, 1, []]] } ] - }, false); + }), false); assert.strictEqual(notebook.apiNotebook.cellCount, 1 + 1); assert.strictEqual(doc.version, 3); assertLines(doc, 'Hello', 'World', 'Hello World!'); @@ -286,7 +287,7 @@ suite('NotebookConcatDocument', function () { let doc = new ExtHostNotebookConcatDocument(extHostNotebooks, extHostDocuments, notebook.apiNotebook, undefined); // UPDATE 1 - extHostNotebookDocuments.$acceptModelChanged(notebookUri, { + extHostNotebookDocuments.$acceptModelChanged(notebookUri, new SerializableObjectWithBuffers({ versionId: notebook.apiNotebook.version + 1, rawEvents: [ { @@ -311,7 +312,7 @@ suite('NotebookConcatDocument', function () { }]]] } ] - }, false); + }), false); assert.strictEqual(notebook.apiNotebook.cellCount, 1 + 2); assert.strictEqual(doc.version, 1); @@ -347,7 +348,7 @@ suite('NotebookConcatDocument', function () { test('selector', function () { - extHostNotebookDocuments.$acceptModelChanged(notebookUri, { + extHostNotebookDocuments.$acceptModelChanged(notebookUri, new SerializableObjectWithBuffers({ versionId: notebook.apiNotebook.version + 1, rawEvents: [ { @@ -371,7 +372,7 @@ suite('NotebookConcatDocument', function () { }]]] } ] - }, false); + }), false); const mixedDoc = new ExtHostNotebookConcatDocument(extHostNotebooks, extHostDocuments, notebook.apiNotebook, undefined); const fooLangDoc = new ExtHostNotebookConcatDocument(extHostNotebooks, extHostDocuments, notebook.apiNotebook, 'fooLang'); @@ -381,7 +382,7 @@ suite('NotebookConcatDocument', function () { assertLines(fooLangDoc, 'fooLang-document'); assertLines(barLangDoc, 'barLang-document'); - extHostNotebookDocuments.$acceptModelChanged(notebookUri, { + extHostNotebookDocuments.$acceptModelChanged(notebookUri, new SerializableObjectWithBuffers({ versionId: notebook.apiNotebook.version + 1, rawEvents: [ { @@ -397,7 +398,7 @@ suite('NotebookConcatDocument', function () { }]]] } ] - }, false); + }), false); assertLines(mixedDoc, 'fooLang-document', 'barLang-document', 'barLang-document2'); assertLines(fooLangDoc, 'fooLang-document'); @@ -419,7 +420,7 @@ suite('NotebookConcatDocument', function () { test('offsetAt(position) <-> positionAt(offset)', function () { - extHostNotebookDocuments.$acceptModelChanged(notebookUri, { + extHostNotebookDocuments.$acceptModelChanged(notebookUri, new SerializableObjectWithBuffers({ versionId: notebook.apiNotebook.version + 1, rawEvents: [ { @@ -443,7 +444,7 @@ suite('NotebookConcatDocument', function () { }]]] } ] - }, false); + }), false); assert.strictEqual(notebook.apiNotebook.cellCount, 1 + 2); // markdown and code @@ -476,7 +477,7 @@ suite('NotebookConcatDocument', function () { test('locationAt(position) <-> positionAt(location)', function () { - extHostNotebookDocuments.$acceptModelChanged(notebookUri, { + extHostNotebookDocuments.$acceptModelChanged(notebookUri, new SerializableObjectWithBuffers({ versionId: notebook.apiNotebook.version + 1, rawEvents: [ { @@ -500,7 +501,7 @@ suite('NotebookConcatDocument', function () { }]]] } ] - }, false); + }), false); assert.strictEqual(notebook.apiNotebook.cellCount, 1 + 2); // markdown and code @@ -517,7 +518,7 @@ suite('NotebookConcatDocument', function () { test('getText(range)', function () { - extHostNotebookDocuments.$acceptModelChanged(notebookUri, { + extHostNotebookDocuments.$acceptModelChanged(notebookUri, new SerializableObjectWithBuffers({ versionId: notebook.apiNotebook.version + 1, rawEvents: [ { @@ -549,7 +550,7 @@ suite('NotebookConcatDocument', function () { }]]] } ] - }, false); + }), false); assert.strictEqual(notebook.apiNotebook.cellCount, 1 + 3); // markdown and code @@ -564,7 +565,7 @@ suite('NotebookConcatDocument', function () { test('validateRange/Position', function () { - extHostNotebookDocuments.$acceptModelChanged(notebookUri, { + extHostNotebookDocuments.$acceptModelChanged(notebookUri, new SerializableObjectWithBuffers({ versionId: notebook.apiNotebook.version + 1, rawEvents: [ { @@ -588,7 +589,7 @@ suite('NotebookConcatDocument', function () { }]]] } ] - }, false); + }), false); assert.strictEqual(notebook.apiNotebook.cellCount, 1 + 2); // markdown and code diff --git a/src/vs/workbench/test/browser/api/extHostNotebookKernel.test.ts b/src/vs/workbench/test/browser/api/extHostNotebookKernel.test.ts index d7405c50298..252913294ad 100644 --- a/src/vs/workbench/test/browser/api/extHostNotebookKernel.test.ts +++ b/src/vs/workbench/test/browser/api/extHostNotebookKernel.test.ts @@ -24,6 +24,7 @@ import { NotebookCellOutput, NotebookCellOutputItem } from 'vs/workbench/api/com import { CellKind, CellUri, NotebookCellsChangeType } from 'vs/workbench/contrib/notebook/common/notebookCommon'; import { CellExecutionUpdateType } from 'vs/workbench/contrib/notebook/common/notebookExecutionService'; import { nullExtensionDescription } from 'vs/workbench/services/extensions/common/extensions'; +import { SerializableObjectWithBuffers } from 'vs/workbench/services/extensions/common/proxyIdentifier'; import { TestRPCProtocol } from 'vs/workbench/test/browser/api/testRPCProtocol'; import { mock } from 'vs/workbench/test/common/workbenchTestServices'; @@ -65,8 +66,8 @@ suite('NotebookKernel', function () { assert.strictEqual(kernelData.has(handle), true); kernelData.set(handle, { ...kernelData.get(handle)!, ...data, }); } - override $updateExecutions(data: ICellExecuteUpdateDto[]): void { - cellExecuteUpdates.push(...data); + override $updateExecutions(data: SerializableObjectWithBuffers): void { + cellExecuteUpdates.push(...data.value); } }); rpcProtocol.set(MainContext.MainThreadNotebookDocuments, new class extends mock() { @@ -87,7 +88,7 @@ suite('NotebookKernel', function () { extHostNotebookDocuments = new ExtHostNotebookDocuments(new NullLogService(), extHostNotebooks); - extHostNotebooks.$acceptDocumentAndEditorsDelta({ + extHostNotebooks.$acceptDocumentAndEditorsDelta(new SerializableObjectWithBuffers({ addedDocuments: [{ uri: notebookUri, viewType: 'test', @@ -116,8 +117,8 @@ suite('NotebookKernel', function () { selections: [{ start: 0, end: 1 }], visibleRanges: [] }] - }); - extHostNotebooks.$acceptDocumentAndEditorsDelta({ newActiveEditor: '_notebook_editor_0' }); + })); + extHostNotebooks.$acceptDocumentAndEditorsDelta(new SerializableObjectWithBuffers({ newActiveEditor: '_notebook_editor_0' })); notebook = extHostNotebooks.notebookDocuments[0]!; @@ -206,13 +207,13 @@ suite('NotebookKernel', function () { const cell1 = notebook.apiNotebook.cellAt(0); extHostNotebookKernels.$acceptNotebookAssociation(0, notebook.uri, true); - extHostNotebookDocuments.$acceptModelChanged(notebook.uri, { + extHostNotebookDocuments.$acceptModelChanged(notebook.uri, new SerializableObjectWithBuffers({ versionId: 12, rawEvents: [{ kind: NotebookCellsChangeType.ModelChange, changes: [[0, notebook.apiNotebook.cellCount, []]] }] - }, true); + }), true); assert.strictEqual(cell1.index, -1); @@ -274,7 +275,7 @@ suite('NotebookKernel', function () { assert.strictEqual(edit.append, false); assert.strictEqual(edit.outputs.length, 1); assert.strictEqual(edit.outputs[0].items.length, 1); - assert.deepStrictEqual(edit.outputs[0].items[0].valueBytes, Array.from(new TextEncoder().encode('canceled'))); + assert.deepStrictEqual(Array.from(edit.outputs[0].items[0].valueBytes.buffer), Array.from(new TextEncoder().encode('canceled'))); found = true; } } @@ -308,7 +309,7 @@ suite('NotebookKernel', function () { assert.strictEqual(edit.append, false); assert.strictEqual(edit.outputs.length, 1); assert.strictEqual(edit.outputs[0].items.length, 1); - assert.deepStrictEqual(edit.outputs[0].items[0].valueBytes, Array.from(new TextEncoder().encode('interrupted'))); + assert.deepStrictEqual(Array.from(edit.outputs[0].items[0].valueBytes.buffer), Array.from(new TextEncoder().encode('interrupted'))); found = true; } } diff --git a/src/vs/workbench/test/browser/api/extHostTypeConverter.test.ts b/src/vs/workbench/test/browser/api/extHostTypeConverter.test.ts index a635a390c67..20ef8c11b6c 100644 --- a/src/vs/workbench/test/browser/api/extHostTypeConverter.test.ts +++ b/src/vs/workbench/test/browser/api/extHostTypeConverter.test.ts @@ -104,11 +104,11 @@ suite('ExtHostTypeConverter', function () { const dto = NotebookCellOutputItem.from(item); assert.strictEqual(dto.mime, 'foo/bar'); - assert.deepStrictEqual(dto.valueBytes, Array.from(new TextEncoder().encode('Hello'))); + assert.deepStrictEqual(Array.from(dto.valueBytes.buffer), Array.from(new TextEncoder().encode('Hello'))); const item2 = NotebookCellOutputItem.to(dto); assert.strictEqual(item2.mime, item.mime); - assert.deepStrictEqual(item2.data, item.data); + assert.deepStrictEqual(Array.from(item2.data), Array.from(item.data)); }); }); diff --git a/src/vs/workbench/test/browser/api/testRPCProtocol.ts b/src/vs/workbench/test/browser/api/testRPCProtocol.ts index 196bff1c56d..89ab443712a 100644 --- a/src/vs/workbench/test/browser/api/testRPCProtocol.ts +++ b/src/vs/workbench/test/browser/api/testRPCProtocol.ts @@ -3,12 +3,13 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { ProxyIdentifier } from 'vs/workbench/services/extensions/common/proxyIdentifier'; +import { ProxyIdentifier, SerializableObjectWithBuffers } from 'vs/workbench/services/extensions/common/proxyIdentifier'; import { CharCode } from 'vs/base/common/charCode'; import { IExtHostContext } from 'vs/workbench/api/common/extHost.protocol'; import { isThenable } from 'vs/base/common/async'; import { IExtHostRpcService } from 'vs/workbench/api/common/extHostRpcService'; import { ExtensionHostKind } from 'vs/workbench/services/extensions/common/extensions'; +import { parseJsonAndRestoreBufferRefs, stringifyJsonWithBufferRefs } from 'vs/workbench/services/extensions/common/rpcProtocol'; export function SingleProxyRPCProtocol(thing: any): IExtHostContext & IExtHostRpcService { return { @@ -143,5 +144,15 @@ function simulateWireTransfer(obj: T): T { if (!obj) { return obj; } - return JSON.parse(JSON.stringify(obj)); + + if (Array.isArray(obj)) { + return obj.map(simulateWireTransfer) as any; + } + + if (obj instanceof SerializableObjectWithBuffers) { + const { jsonString, referencedBuffers } = stringifyJsonWithBufferRefs(obj); + return parseJsonAndRestoreBufferRefs(jsonString, referencedBuffers, null); + } else { + return JSON.parse(JSON.stringify(obj)); + } }