From ac75e58f8bb0b0848dde2cdd5fd8dc7fca3f11bc Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Mon, 2 May 2016 14:46:12 +0200 Subject: [PATCH] Further cleanup in editorCommon --- src/vs/editor/browser/standalone/colorizer.ts | 5 +- .../browser/standalone/standaloneEditor.ts | 3 - .../editor/browser/widget/codeEditorWidget.ts | 3 +- .../editor/browser/widget/diffEditorWidget.ts | 2 +- src/vs/editor/common/commonCodeEditor.ts | 2 +- .../common/config/commonEditorConfig.ts | 16 +- .../common/controller/cursorMoveHelper.ts | 16 +- src/vs/editor/common/controller/oneCursor.ts | 10 +- src/vs/editor/common/core/viewLineToken.ts | 85 +++++ src/vs/editor/common/editorCommon.ts | 128 +------- src/vs/editor/common/model/lineToken.ts | 47 +++ src/vs/editor/common/model/modelLine.ts | 124 ++++--- .../common/model/textModelWithTokens.ts | 115 ++----- .../model/textModelWithTokensHelpers.ts | 62 +--- src/vs/editor/common/model/tokenIterator.ts | 40 +-- .../common/model/tokensBinaryEncoding.ts | 307 +++++++++--------- .../editor/common/viewLayout/viewLineParts.ts | 2 +- .../common/viewLayout/viewLineRenderer.ts | 2 +- .../common/viewModel/filteredLineTokens.ts | 7 +- .../common/viewModel/splitLinesCollection.ts | 2 +- src/vs/editor/common/viewModel/viewModel.ts | 80 +---- .../editor/common/viewModel/viewModelImpl.ts | 7 +- .../test/common/model/model.line.test.ts | 8 +- src/vs/editor/test/common/model/model.test.ts | 39 +-- .../common/viewLayout/viewLineParts.test.ts | 2 +- .../viewLayout/viewLineRenderer.test.ts | 2 +- .../viewModel/splitLinesCollection.test.ts | 2 +- .../languages/html/test/common/html.test.ts | 10 +- 28 files changed, 494 insertions(+), 634 deletions(-) create mode 100644 src/vs/editor/common/core/viewLineToken.ts create mode 100644 src/vs/editor/common/model/lineToken.ts diff --git a/src/vs/editor/browser/standalone/colorizer.ts b/src/vs/editor/browser/standalone/colorizer.ts index 616c2ffa501..b17e2f21cf9 100644 --- a/src/vs/editor/browser/standalone/colorizer.ts +++ b/src/vs/editor/browser/standalone/colorizer.ts @@ -10,8 +10,7 @@ import {IModel} from 'vs/editor/common/editorCommon'; import {ILineTokens, IMode} from 'vs/editor/common/modes'; import {IModeService} from 'vs/editor/common/services/modeService'; import {RenderLineOutput, renderLine, RenderLineInput} from 'vs/editor/common/viewLayout/viewLineRenderer'; -import * as TokensBinaryEncoding from 'vs/editor/common/model/tokensBinaryEncoding'; -import {ViewLineToken} from 'vs/editor/common/viewModel/viewModel'; +import {ViewLineToken} from 'vs/editor/common/core/viewLineToken'; export interface IColorizerOptions { tabSize?: number; @@ -108,7 +107,7 @@ export class Colorizer { public static colorizeModelLine(model:IModel, lineNumber:number, tabSize:number = 4): string { var content = model.getLineContent(lineNumber); var tokens = model.getLineTokens(lineNumber, false); - var inflatedTokens = TokensBinaryEncoding.inflateArr(tokens.getBinaryEncodedTokensMap(), tokens.getBinaryEncodedTokens()); + var inflatedTokens = tokens.inflate(); return this.colorizeLine(content, inflatedTokens, tabSize); } } diff --git a/src/vs/editor/browser/standalone/standaloneEditor.ts b/src/vs/editor/browser/standalone/standaloneEditor.ts index 3944c9b134b..409bedae472 100644 --- a/src/vs/editor/browser/standalone/standaloneEditor.ts +++ b/src/vs/editor/browser/standalone/standaloneEditor.ts @@ -13,7 +13,6 @@ import {ClassNames, ContentWidgetPositionPreference, OverlayWidgetPositionPrefer import {Colorizer} from 'vs/editor/browser/standalone/colorizer'; import * as standaloneCodeEditor from 'vs/editor/browser/standalone/standaloneCodeEditor'; import {ILanguageDef} from 'vs/editor/standalone-languages/types'; -import * as TokensBinaryEncoding from 'vs/editor/common/model/tokensBinaryEncoding'; var global:any = self; if (!global.Monaco) { @@ -38,11 +37,9 @@ Monaco.Editor.colorizeModelLine = Colorizer.colorizeModelLine; // -- export common constants Monaco.Editor.SelectionDirection = editorCommon.SelectionDirection; Monaco.Editor.WrappingIndent = editorCommon.WrappingIndent; -Monaco.Editor.wrappingIndentFromString = editorCommon.wrappingIndentFromString; Monaco.Editor.OverviewRulerLane = editorCommon.OverviewRulerLane; Monaco.Editor.EndOfLinePreference = editorCommon.EndOfLinePreference; Monaco.Editor.EndOfLineSequence = editorCommon.EndOfLineSequence; -Monaco.Editor.LineTokensBinaryEncoding = TokensBinaryEncoding; Monaco.Editor.TrackedRangeStickiness = editorCommon.TrackedRangeStickiness; Monaco.Editor.VerticalRevealType = editorCommon.VerticalRevealType; Monaco.Editor.MouseTargetType = editorCommon.MouseTargetType; diff --git a/src/vs/editor/browser/widget/codeEditorWidget.ts b/src/vs/editor/browser/widget/codeEditorWidget.ts index d9d9f7c139e..29c2630daa4 100644 --- a/src/vs/editor/browser/widget/codeEditorWidget.ts +++ b/src/vs/editor/browser/widget/codeEditorWidget.ts @@ -25,7 +25,6 @@ import * as editorBrowser from 'vs/editor/browser/editorBrowser'; import {EditorBrowserRegistry} from 'vs/editor/browser/editorBrowserExtensions'; import {Colorizer} from 'vs/editor/browser/standalone/colorizer'; import {View} from 'vs/editor/browser/view/viewImpl'; -import * as TokensBinaryEncoding from 'vs/editor/common/model/tokensBinaryEncoding'; export class CodeEditorWidget extends CommonCodeEditor implements editorBrowser.ICodeEditor { @@ -107,7 +106,7 @@ export class CodeEditorWidget extends CommonCodeEditor implements editorBrowser. } var content = model.getLineContent(lineNumber); var tokens = model.getLineTokens(lineNumber, false); - var inflatedTokens = TokensBinaryEncoding.inflateArr(tokens.getBinaryEncodedTokensMap(), tokens.getBinaryEncodedTokens()); + var inflatedTokens = tokens.inflate(); var tabSize = model.getOptions().tabSize; return Colorizer.colorizeLine(content, inflatedTokens, tabSize); } diff --git a/src/vs/editor/browser/widget/diffEditorWidget.ts b/src/vs/editor/browser/widget/diffEditorWidget.ts index c6dc131868b..59b579e3a9b 100644 --- a/src/vs/editor/browser/widget/diffEditorWidget.ts +++ b/src/vs/editor/browser/widget/diffEditorWidget.ts @@ -23,7 +23,7 @@ import {createLineParts} from 'vs/editor/common/viewLayout/viewLineParts'; import {renderLine, RenderLineInput} from 'vs/editor/common/viewLayout/viewLineRenderer'; import * as editorBrowser from 'vs/editor/browser/editorBrowser'; import {CodeEditorWidget} from 'vs/editor/browser/widget/codeEditorWidget'; -import {ViewLineToken, ViewLineTokens} from 'vs/editor/common/viewModel/viewModel'; +import {ViewLineToken, ViewLineTokens} from 'vs/editor/common/core/viewLineToken'; interface IEditorScrollEvent { scrollLeft: number; diff --git a/src/vs/editor/common/commonCodeEditor.ts b/src/vs/editor/common/commonCodeEditor.ts index 43fec30d05f..9372cbc42fa 100644 --- a/src/vs/editor/common/commonCodeEditor.ts +++ b/src/vs/editor/common/commonCodeEditor.ts @@ -693,7 +693,7 @@ export abstract class CommonCodeEditor extends EventEmitter implements IActionPr this.model.getOptions().tabSize, this._configuration.editor.wrappingInfo.wrappingColumn, this._configuration.editor.fontInfo.typicalFullwidthCharacterWidth / this._configuration.editor.fontInfo.typicalHalfwidthCharacterWidth, - editorCommon.wrappingIndentFromString(this._configuration.editor.wrappingIndent) + this._configuration.editor.wrappingIndent ); this.viewModel = new ViewModel( diff --git a/src/vs/editor/common/config/commonEditorConfig.ts b/src/vs/editor/common/config/commonEditorConfig.ts index 3b9b4664ef6..65a0a91ce1c 100644 --- a/src/vs/editor/common/config/commonEditorConfig.ts +++ b/src/vs/editor/common/config/commonEditorConfig.ts @@ -84,7 +84,7 @@ export class InternalEditorOptions implements editorCommon.IInternalEditorOption fontLigatures:boolean; hideCursorInOverviewRuler:boolean; scrollBeyondLastLine:boolean; - wrappingIndent: string; + wrappingIndent: editorCommon.WrappingIndent; wordWrapBreakBeforeCharacters: string; wordWrapBreakAfterCharacters: string; wordWrapBreakObtrusiveCharacters: string; @@ -150,7 +150,7 @@ export class InternalEditorOptions implements editorCommon.IInternalEditorOption this.fontLigatures = Boolean(input.fontLigatures); this.hideCursorInOverviewRuler = Boolean(input.hideCursorInOverviewRuler); this.scrollBeyondLastLine = Boolean(input.scrollBeyondLastLine); - this.wrappingIndent = String(input.wrappingIndent); + this.wrappingIndent = input.wrappingIndent; this.wordWrapBreakBeforeCharacters = String(input.wordWrapBreakBeforeCharacters); this.wordWrapBreakAfterCharacters = String(input.wordWrapBreakAfterCharacters); this.wordWrapBreakObtrusiveCharacters = String(input.wordWrapBreakObtrusiveCharacters); @@ -335,7 +335,7 @@ class InternalEditorOptionsHelper { fontLigatures: toBoolean(opts.fontLigatures), hideCursorInOverviewRuler: toBoolean(opts.hideCursorInOverviewRuler), scrollBeyondLastLine: toBoolean(opts.scrollBeyondLastLine), - wrappingIndent: opts.wrappingIndent, + wrappingIndent: wrappingIndentFromString(opts.wrappingIndent), wordWrapBreakBeforeCharacters: opts.wordWrapBreakBeforeCharacters, wordWrapBreakAfterCharacters: opts.wordWrapBreakAfterCharacters, wordWrapBreakObtrusiveCharacters: opts.wordWrapBreakObtrusiveCharacters, @@ -536,6 +536,16 @@ function toSortedIntegerArray(source:any): number[] { return r; } +function wrappingIndentFromString(wrappingIndent:string): editorCommon.WrappingIndent { + if (wrappingIndent === 'indent') { + return editorCommon.WrappingIndent.Indent; + } else if (wrappingIndent === 'same') { + return editorCommon.WrappingIndent.Same; + } else { + return editorCommon.WrappingIndent.None; + } +} + function toIntegerWithDefault(source:any, defaultValue:number): number { if (typeof source === 'undefined') { return defaultValue; diff --git a/src/vs/editor/common/controller/cursorMoveHelper.ts b/src/vs/editor/common/controller/cursorMoveHelper.ts index 43f5e04e1b9..e0b3a35ff90 100644 --- a/src/vs/editor/common/controller/cursorMoveHelper.ts +++ b/src/vs/editor/common/controller/cursorMoveHelper.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ 'use strict'; -import {IInternalIndentationOptions, IPosition, IEditorSelection} from 'vs/editor/common/editorCommon'; +import {IPosition, IEditorSelection} from 'vs/editor/common/editorCommon'; import {Selection} from 'vs/editor/common/core/selection'; export interface IMoveResult { @@ -32,6 +32,20 @@ export interface ICursorMoveHelperModel { getLineContent(lineNumber:number): string; } +/** + * Internal indentation options (computed) for the editor. + */ +export interface IInternalIndentationOptions { + /** + * Tab size in spaces. This is used for rendering and for editing. + */ + tabSize:number; + /** + * Insert spaces instead of tabs when indenting or when auto-indenting. + */ + insertSpaces:boolean; +} + export interface IConfiguration { getIndentationOptions(): IInternalIndentationOptions; } diff --git a/src/vs/editor/common/controller/oneCursor.ts b/src/vs/editor/common/controller/oneCursor.ts index 4f014c459c3..2be869f8f2e 100644 --- a/src/vs/editor/common/controller/oneCursor.ts +++ b/src/vs/editor/common/controller/oneCursor.ts @@ -239,20 +239,20 @@ export class OneCursor { } public adjustBracketDecorations(): void { - let bracketMatch: editorCommon.IMatchBracketResult = null; + let bracketMatch: [editorCommon.IEditorRange,editorCommon.IEditorRange] = null; let selection = this.getSelection(); if (selection.isEmpty()) { - bracketMatch = this.model.matchBracket(this.position, /*inaccurateResultAcceptable*/true); + bracketMatch = this.model.matchBracket(this.position); } let newDecorations: editorCommon.IModelDeltaDecoration[] = []; - if (bracketMatch && bracketMatch.brackets) { + if (bracketMatch) { let options: editorCommon.IModelDecorationOptions = { stickiness: editorCommon.TrackedRangeStickiness.NeverGrowsWhenTypingAtEdges, className: 'bracket-match' }; - newDecorations.push({ range: bracketMatch.brackets[0], options: options }); - newDecorations.push({ range: bracketMatch.brackets[1], options: options }); + newDecorations.push({ range: bracketMatch[0], options: options }); + newDecorations.push({ range: bracketMatch[1], options: options }); } this.bracketDecorations = this.model.deltaDecorations(this.bracketDecorations, newDecorations, this.editorId); diff --git a/src/vs/editor/common/core/viewLineToken.ts b/src/vs/editor/common/core/viewLineToken.ts new file mode 100644 index 00000000000..2fa93a70048 --- /dev/null +++ b/src/vs/editor/common/core/viewLineToken.ts @@ -0,0 +1,85 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +'use strict'; + +import {Arrays} from 'vs/editor/common/core/arrays'; + +/** + * A token on a line. + */ +export class ViewLineToken { + public _viewLineTokenTrait: void; + + public startIndex:number; + public type:string; + + constructor(startIndex:number, type:string) { + this.startIndex = startIndex|0;// @perf + this.type = type.replace(/[^a-z0-9\-]/gi, ' '); + } + + public equals(other:ViewLineToken): boolean { + return ( + this.startIndex === other.startIndex + && this.type === other.type + ); + } + + public static findIndexInSegmentsArray(arr:ViewLineToken[], desiredIndex: number): number { + return Arrays.findIndexInSegmentsArray(arr, desiredIndex); + } + + public static equalsArray(a:ViewLineToken[], b:ViewLineToken[]): boolean { + let aLen = a.length; + let bLen = b.length; + if (aLen !== bLen) { + return false; + } + for (let i = 0; i < aLen; i++) { + if (!a[i].equals(b[i])) { + return false; + } + } + return true; + } +} + +export class ViewLineTokens { + _viewLineTokensTrait: void; + + private _lineTokens:ViewLineToken[]; + private _fauxIndentLength:number; + private _textLength:number; + + constructor(lineTokens:ViewLineToken[], fauxIndentLength:number, textLength:number) { + this._lineTokens = lineTokens; + this._fauxIndentLength = fauxIndentLength|0; + this._textLength = textLength|0; + } + + public getTokens(): ViewLineToken[] { + return this._lineTokens; + } + + public getFauxIndentLength(): number { + return this._fauxIndentLength; + } + + public getTextLength(): number { + return this._textLength; + } + + public equals(other:ViewLineTokens): boolean { + return ( + this._fauxIndentLength === other._fauxIndentLength + && this._textLength === other._textLength + && ViewLineToken.equalsArray(this._lineTokens, other._lineTokens) + ); + } + + public findIndexOfOffset(offset:number): number { + return ViewLineToken.findIndexInSegmentsArray(this._lineTokens, offset); + } +} diff --git a/src/vs/editor/common/editorCommon.ts b/src/vs/editor/common/editorCommon.ts index d699cecc77b..85a2f8c7722 100644 --- a/src/vs/editor/common/editorCommon.ts +++ b/src/vs/editor/common/editorCommon.ts @@ -13,7 +13,7 @@ import URI from 'vs/base/common/uri'; import {TPromise} from 'vs/base/common/winjs.base'; import {IInstantiationService, IConstructorSignature1, IConstructorSignature2} from 'vs/platform/instantiation/common/instantiation'; import {ILineContext, IMode, IModeTransition, IToken} from 'vs/editor/common/modes'; -import {Arrays} from 'vs/editor/common/core/arrays'; +import {ViewLineToken} from 'vs/editor/common/core/viewLineToken'; export type KeyCode = KeyCode; export type KeyMod = KeyMod; @@ -257,16 +257,6 @@ export enum WrappingIndent { Indent = 2 } -export function wrappingIndentFromString(wrappingIndent:string): WrappingIndent { - if (wrappingIndent === 'indent') { - return WrappingIndent.Indent; - } else if (wrappingIndent === 'same') { - return WrappingIndent.Same; - } else { - return WrappingIndent.None; - } -} - /** * Configuration options for the editor. */ @@ -563,20 +553,6 @@ export interface IDiffEditorOptions extends IEditorOptions { originalEditable?: boolean; } -/** - * Internal indentation options (computed) for the editor. - */ -export interface IInternalIndentationOptions { - /** - * Tab size in spaces. This is used for rendering and for editing. - */ - tabSize:number; - /** - * Insert spaces instead of tabs when indenting or when auto-indenting. - */ - insertSpaces:boolean; -} - export interface IInternalEditorScrollbarOptions { arrowSize:number; vertical:string; @@ -622,7 +598,7 @@ export interface IInternalEditorOptions { fontLigatures:boolean; hideCursorInOverviewRuler:boolean; scrollBeyondLastLine:boolean; - wrappingIndent: string; + wrappingIndent: WrappingIndent; wordWrapBreakBeforeCharacters: string; wordWrapBreakAfterCharacters: string; wordWrapBreakObtrusiveCharacters: string; @@ -1024,21 +1000,6 @@ export enum EndOfLineSequence { CRLF = 1 } -/** - * The result of a matchBracket operation. - */ -export interface IMatchBracketResult { - /** - * The two ranges describing matching brackets, or null - */ - brackets:IEditorRange[]; - /** - * Indicates that the bracket match result is not accurate because the search - * hit some untokenized lines. - */ - isAccurate:boolean; -} - /** * A read-only line marker in the model. */ @@ -1162,7 +1123,6 @@ export interface IIdentifiedSingleEditOperation { forceMoveMarkers: boolean; } - /** * A callback that can compute the cursor state after applying a series of edit operations. */ @@ -1173,80 +1133,10 @@ export interface ICursorStateComputer { (inverseEditOperations:IIdentifiedSingleEditOperation[]): IEditorSelection[]; } -/** - * A token on a line. - */ -export class LineToken { - public _lineTokenTrait: void; - - public startIndex:number; - public type:string; - - constructor(startIndex:number, type:string) { - this.startIndex = startIndex|0;// @perf - this.type = type; - } - - public equals(other:LineToken): boolean { - return ( - this.startIndex === other.startIndex - && this.type === other.type - ); - } - - public static findIndexInSegmentsArray(arr:LineToken[], desiredIndex: number): number { - return Arrays.findIndexInSegmentsArray(arr, desiredIndex); - } - - public static equalsArray(a:LineToken[], b:LineToken[]): boolean { - let aLen = a.length; - let bLen = b.length; - if (aLen !== bLen) { - return false; - } - for (let i = 0; i < aLen; i++) { - if (!a[i].equals(b[i])) { - return false; - } - } - return true; - } -} - -export interface ITokensInflatorMap { - _inflate:string[]; - _deflate: { [token:string]:number; }; -} - -export interface ILineTokensBinaryEncoding { - START_INDEX_MASK: number; - TYPE_MASK: number; - START_INDEX_OFFSET: number; - TYPE_OFFSET: number; - - deflateArr(map:ITokensInflatorMap, tokens:IToken[]): number[]; - inflate(map:ITokensInflatorMap, binaryEncodedToken:number): IToken; - getStartIndex(binaryEncodedToken:number): number; - getType(map:ITokensInflatorMap, binaryEncodedToken:number): string; - inflateArr(map:ITokensInflatorMap, binaryEncodedTokens:number[]): IToken[]; - findIndexOfOffset(binaryEncodedTokens:number[], offset:number): number; - sliceAndInflate(map:ITokensInflatorMap, binaryEncodedTokens:number[], startOffset:number, endOffset:number, deltaStartIndex:number): IToken[]; -} - /** * A list of tokens on a line. */ export interface ILineTokens { - /** - * Get the binary representation of tokens. - */ - getBinaryEncodedTokens(): number[]; - - /** - * A map to help decoding the token type. - */ - getBinaryEncodedTokensMap(): ITokensInflatorMap; - getTokenCount(): number; getTokenStartIndex(tokenIndex:number): number; getTokenType(tokenIndex:number): string; @@ -1268,6 +1158,10 @@ export interface ILineTokens { * @return The index of the token containing the offset. */ findIndexOfOffset(offset:number): number; + + sliceAndInflate(startOffset:number, endOffset:number, deltaStartIndex:number): ViewLineToken[]; + + inflate(): ViewLineToken[]; } /** @@ -1538,14 +1432,6 @@ export interface ITokenizedModel extends ITextModel { */ getWordUntilPosition(position:IPosition): IWordAtPosition; - /** - * Get the words on line `lineNumber`. - * @param lineNumber The lineNumber - * @param skipSyntaxTokens Ignore syntax tokens, as identified by the mode. - * @return All the words on the line. - */ - getWords(lineNumber:number): IWordRange[]; - /** * Returns an iterator that can be used to read * next and previous tokens from the provided position. @@ -1581,7 +1467,7 @@ export interface ITokenizedModel extends ITextModel { * find the matching bracket of that bracket and return the ranges of both brackets. * @param position The position at which to look for a bracket. */ - matchBracket(position:IPosition, inaccurateResultAcceptable?:boolean): IMatchBracketResult; + matchBracket(position:IPosition): [IEditorRange,IEditorRange]; /** * No mode supports allowed on this model because it is simply too large. diff --git a/src/vs/editor/common/model/lineToken.ts b/src/vs/editor/common/model/lineToken.ts new file mode 100644 index 00000000000..6229af2c4cf --- /dev/null +++ b/src/vs/editor/common/model/lineToken.ts @@ -0,0 +1,47 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +'use strict'; + +import {Arrays} from 'vs/editor/common/core/arrays'; + +/** + * A token on a line. + */ +export class LineToken { + public _lineTokenTrait: void; + + public startIndex:number; + public type:string; + + constructor(startIndex:number, type:string) { + this.startIndex = startIndex|0;// @perf + this.type = type; + } + + public equals(other:LineToken): boolean { + return ( + this.startIndex === other.startIndex + && this.type === other.type + ); + } + + public static findIndexInSegmentsArray(arr:LineToken[], desiredIndex: number): number { + return Arrays.findIndexInSegmentsArray(arr, desiredIndex); + } + + public static equalsArray(a:LineToken[], b:LineToken[]): boolean { + let aLen = a.length; + let bLen = b.length; + if (aLen !== bLen) { + return false; + } + for (let i = 0; i < aLen; i++) { + if (!a[i].equals(b[i])) { + return false; + } + } + return true; + } +} \ No newline at end of file diff --git a/src/vs/editor/common/model/modelLine.ts b/src/vs/editor/common/model/modelLine.ts index cbb37bf6515..fd516abcd34 100644 --- a/src/vs/editor/common/model/modelLine.ts +++ b/src/vs/editor/common/model/modelLine.ts @@ -5,10 +5,17 @@ 'use strict'; import * as strings from 'vs/base/common/strings'; -import {ILineTokens, IReadOnlyLineMarker, ITokensInflatorMap, LineToken} from 'vs/editor/common/editorCommon'; +import {ILineTokens, IReadOnlyLineMarker} from 'vs/editor/common/editorCommon'; import {IMode, IState} from 'vs/editor/common/modes'; -import * as TokensBinaryEncoding from 'vs/editor/common/model/tokensBinaryEncoding'; +import {TokensBinaryEncoding, TokensInflatorMap} from 'vs/editor/common/model/tokensBinaryEncoding'; import {ModeTransition} from 'vs/editor/common/core/modeTransition'; +import {LineToken} from 'vs/editor/common/model/lineToken'; +import {ViewLineToken} from 'vs/editor/common/core/viewLineToken'; + +const START_INDEX_MASK = TokensBinaryEncoding.START_INDEX_MASK; +const TYPE_MASK = TokensBinaryEncoding.TYPE_MASK; +const START_INDEX_OFFSET = TokensBinaryEncoding.START_INDEX_OFFSET; +const TYPE_OFFSET = TokensBinaryEncoding.TYPE_OFFSET; export interface ILineEdit { startColumn: number; @@ -76,8 +83,8 @@ export class ModelLine { public set isInvalid(value:boolean) { this._isInvalid = value; } private _state:IState; - private _modeTransitions:ModeTransition[]; - private _lineTokens:ILineTokens; + private _modeTransitions: ModeTransition[]; + private _lineTokens: LineTokens; private _markers:ILineMarker[]; constructor(lineNumber:number, text:string) { @@ -116,12 +123,12 @@ export class ModelLine { // --- BEGIN TOKENS - public setTokens(map: ITokensInflatorMap, tokens: LineToken[], topLevelMode:IMode, modeTransitions:ModeTransition[]): void { + public setTokens(map: TokensInflatorMap, tokens: LineToken[], topLevelMode:IMode, modeTransitions:ModeTransition[]): void { this._lineTokens = toLineTokensFromInflated(map, tokens, this._text.length); this._modeTransitions = toModeTransitions(topLevelMode, modeTransitions); } - private _setLineTokensFromDeflated(map:ITokensInflatorMap, tokens:number[]): void { + private _setLineTokensFromDeflated(map:TokensInflatorMap, tokens:number[]): void { this._lineTokens = toLineTokensFromDeflated(map, tokens, this._text.length); } @@ -145,7 +152,6 @@ export class ModelLine { var lineTokens = this._lineTokens; - let BIN = TokensBinaryEncoding; let tokens = lineTokens.getBinaryEncodedTokens(); let tokensLength = tokens.length; let tokensIndex = 0; @@ -161,14 +167,14 @@ export class ModelLine { if (currentTokenStartIndex > 0 && delta !== 0) { // adjust token's `startIndex` by `delta` - let deflatedType = (tokens[tokensIndex] / BIN.TYPE_OFFSET) & BIN.TYPE_MASK; + let deflatedType = (tokens[tokensIndex] / TYPE_OFFSET) & TYPE_MASK; let newStartIndex = Math.max(minimumAllowedIndex, currentTokenStartIndex + delta); - let newToken = deflatedType * BIN.TYPE_OFFSET + newStartIndex * BIN.START_INDEX_OFFSET; + let newToken = deflatedType * TYPE_OFFSET + newStartIndex * START_INDEX_OFFSET; if (delta < 0) { // pop all previous tokens that have become `collapsed` while (tokensIndex > 0) { - let prevTokenStartIndex = (tokens[tokensIndex - 1] / BIN.START_INDEX_OFFSET) & BIN.START_INDEX_MASK; + let prevTokenStartIndex = (tokens[tokensIndex - 1] / START_INDEX_OFFSET) & START_INDEX_MASK; if (prevTokenStartIndex >= newStartIndex) { // Token at `tokensIndex` - 1 is now `collapsed` => pop it tokens.splice(tokensIndex - 1, 1); @@ -185,7 +191,7 @@ export class ModelLine { tokensIndex++; if (tokensIndex < tokensLength) { - currentTokenStartIndex = (tokens[tokensIndex] / BIN.START_INDEX_OFFSET) & BIN.START_INDEX_MASK; + currentTokenStartIndex = (tokens[tokensIndex] / START_INDEX_OFFSET) & START_INDEX_MASK; } } // console.log('after call: tokensIndex: ' + tokensIndex + ': ' + String(this.getTokens())); @@ -205,14 +211,13 @@ export class ModelLine { this._text = text; if (this._lineTokens) { - let BIN = TokensBinaryEncoding, - map = this._lineTokens.getBinaryEncodedTokensMap(), + let map = this._lineTokens.getBinaryEncodedTokensMap(), tokens = this._lineTokens.getBinaryEncodedTokens(), lineTextLength = this._text.length; // Remove overflowing tokens while (tokens.length > 0) { - let lastTokenStartIndex = (tokens[tokens.length - 1] / BIN.START_INDEX_OFFSET) & BIN.START_INDEX_MASK; + let lastTokenStartIndex = (tokens[tokens.length - 1] / START_INDEX_OFFSET) & START_INDEX_MASK; if (lastTokenStartIndex < lineTextLength) { // Valid token break; @@ -456,15 +461,13 @@ export class ModelLine { // Adjust other tokens if (thisTextLength > 0) { - let BIN = TokensBinaryEncoding; - for (let i = 0, len = otherTokens.length; i < len; i++) { let token = otherTokens[i]; - let deflatedStartIndex = (token / BIN.START_INDEX_OFFSET) & BIN.START_INDEX_MASK; - let deflatedType = (token / BIN.TYPE_OFFSET) & BIN.TYPE_MASK; + let deflatedStartIndex = (token / START_INDEX_OFFSET) & START_INDEX_MASK; + let deflatedType = (token / TYPE_OFFSET) & TYPE_MASK; let newStartIndex = deflatedStartIndex + thisTextLength; - let newToken = deflatedType * BIN.TYPE_OFFSET + newStartIndex * BIN.START_INDEX_OFFSET; + let newToken = deflatedType * TYPE_OFFSET + newStartIndex * START_INDEX_OFFSET; otherTokens[i] = newToken; } @@ -625,7 +628,7 @@ export class ModelLine { } } -function toLineTokensFromInflated(map:ITokensInflatorMap, tokens:LineToken[], textLength:number): ILineTokens { +function toLineTokensFromInflated(map:TokensInflatorMap, tokens:LineToken[], textLength:number): LineTokens { if (textLength === 0) { return null; } @@ -642,7 +645,7 @@ function toLineTokensFromInflated(map:ITokensInflatorMap, tokens:LineToken[], te return new LineTokens(map, deflated); } -function toLineTokensFromDeflated(map:ITokensInflatorMap, tokens:number[], textLength:number): ILineTokens { +function toLineTokensFromDeflated(map:TokensInflatorMap, tokens:number[], textLength:number): LineTokens { if (textLength === 0) { return null; } @@ -657,21 +660,17 @@ function toLineTokensFromDeflated(map:ITokensInflatorMap, tokens:number[], textL return new LineTokens(map, tokens); } -var getStartIndex = TokensBinaryEncoding.getStartIndex; -var getType = TokensBinaryEncoding.getType; -var findIndexOfOffset = TokensBinaryEncoding.findIndexOfOffset; - export class LineTokens implements ILineTokens { - private map:ITokensInflatorMap; + private map:TokensInflatorMap; private _tokens:number[]; - constructor(map:ITokensInflatorMap, tokens:number[]) { + constructor(map:TokensInflatorMap, tokens:number[]) { this.map = map; this._tokens = tokens; } - public getBinaryEncodedTokensMap(): ITokensInflatorMap { + public getBinaryEncodedTokensMap(): TokensInflatorMap { return this.map; } @@ -684,41 +683,57 @@ export class LineTokens implements ILineTokens { } public getTokenStartIndex(tokenIndex:number): number { - return getStartIndex(this._tokens[tokenIndex]); + return TokensBinaryEncoding.getStartIndex(this._tokens[tokenIndex]); } public getTokenType(tokenIndex:number): string { - return getType(this.map, this._tokens[tokenIndex]); + return TokensBinaryEncoding.getType(this.map, this._tokens[tokenIndex]); } public getTokenEndIndex(tokenIndex:number, textLength:number): number { if (tokenIndex + 1 < this._tokens.length) { - return getStartIndex(this._tokens[tokenIndex + 1]); + return TokensBinaryEncoding.getStartIndex(this._tokens[tokenIndex + 1]); } return textLength; } public equals(other:ILineTokens): boolean { - return this === other; + if (other instanceof LineTokens) { + if (this.map !== other.map) { + return false; + } + if (this._tokens.length !== other._tokens.length) { + return false; + } + for (let i = 0, len = this._tokens.length; i < len; i++) { + if (this._tokens[i] !== other._tokens[i]) { + return false; + } + } + return true; + } + if (!(other instanceof LineTokens)) { + return false; + } } public findIndexOfOffset(offset:number): number { - return findIndexOfOffset(this._tokens, offset); + return TokensBinaryEncoding.findIndexOfOffset(this._tokens, offset); } + + public inflate(): ViewLineToken[] { + return TokensBinaryEncoding.inflateArr(this.map, this._tokens); + } + + public sliceAndInflate(startOffset:number, endOffset:number, deltaStartIndex:number): ViewLineToken[] { + return TokensBinaryEncoding.sliceAndInflate(this.map, this._tokens, startOffset, endOffset, deltaStartIndex); + } + } class EmptyLineTokens implements ILineTokens { public static INSTANCE = new EmptyLineTokens(); - private static TOKENS = []; - - public getBinaryEncodedTokens(): number[] { - return EmptyLineTokens.TOKENS; - } - - public getBinaryEncodedTokensMap(): ITokensInflatorMap { - return null; - } public getTokenCount(): number { return 0; @@ -743,20 +758,19 @@ class EmptyLineTokens implements ILineTokens { public findIndexOfOffset(offset:number): number { return 0; } + + public inflate(): ViewLineToken[] { + return []; + } + + public sliceAndInflate(startOffset:number, endOffset:number, deltaStartIndex:number): ViewLineToken[] { + return []; + } } export class DefaultLineTokens implements ILineTokens { public static INSTANCE = new DefaultLineTokens(); - private static TOKENS = [0]; - - public getBinaryEncodedTokensMap(): ITokensInflatorMap { - return null; - } - - public getBinaryEncodedTokens(): number[] { - return DefaultLineTokens.TOKENS; - } public getTokenCount(): number { return 1; @@ -782,6 +796,14 @@ export class DefaultLineTokens implements ILineTokens { return 0; } + public inflate(): ViewLineToken[] { + return [new ViewLineToken(0, '')]; + } + + public sliceAndInflate(startOffset:number, endOffset:number, deltaStartIndex:number): ViewLineToken[] { + return [new ViewLineToken(0, '')]; + } + } function toModeTransitions(topLevelMode:IMode, modeTransitions:ModeTransition[]): ModeTransition[] { diff --git a/src/vs/editor/common/model/textModelWithTokens.ts b/src/vs/editor/common/model/textModelWithTokens.ts index 88216a54e2e..1c173141585 100644 --- a/src/vs/editor/common/model/textModelWithTokens.ts +++ b/src/vs/editor/common/model/textModelWithTokens.ts @@ -22,22 +22,9 @@ import {ILineContext, ILineTokens, IToken, IModeTransition, IMode, IState} from import {NullMode, NullState, nullTokenize} from 'vs/editor/common/modes/nullMode'; import {ignoreBracketsInToken} from 'vs/editor/common/modes/supports'; import {BracketsUtils} from 'vs/editor/common/modes/supports/richEditBrackets'; -import * as TokensBinaryEncoding from 'vs/editor/common/model/tokensBinaryEncoding'; import {ModeTransition} from 'vs/editor/common/core/modeTransition'; - -export class TokensInflatorMap implements editorCommon.ITokensInflatorMap { - - public _inflate:string[]; - - public _deflate: { - [token:string]:number; - }; - - constructor() { - this._inflate = [ '' ]; - this._deflate = { '': 0 }; - } -} +import {LineToken} from 'vs/editor/common/model/lineToken'; +import {TokensInflatorMap} from 'vs/editor/common/model/tokensBinaryEncoding'; class ModeToModelBinder implements IDisposable { @@ -190,7 +177,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke private _mode: IMode; private _modeListener: IDisposable; private _modeToModelBinder:ModeToModelBinder; - private _tokensInflatorMap:editorCommon.ITokensInflatorMap; + private _tokensInflatorMap:TokensInflatorMap; private _stopLineTokenizationAfter:number; private _invalidLineStartIndex:number; @@ -535,16 +522,16 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke } } - private static _toLineTokens(tokens:IToken[]): editorCommon.LineToken[] { + private static _toLineTokens(tokens:IToken[]): LineToken[] { if (!tokens || tokens.length === 0) { return []; } - if (tokens[0] instanceof editorCommon.LineToken) { - return tokens; + if (tokens[0] instanceof LineToken) { + return tokens; } - let result:editorCommon.LineToken[] = []; + let result:LineToken[] = []; for (let i = 0, len = tokens.length; i < len; i++) { - result[i] = new editorCommon.LineToken(tokens[i].startIndex, tokens[i].type); + result[i] = new LineToken(tokens[i].startIndex, tokens[i].type); } return result; } @@ -563,7 +550,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke return result; } - private _updateLineTokens(lineIndex:number, map:editorCommon.ITokensInflatorMap, topLevelMode:IMode, r:ILineTokens): void { + private _updateLineTokens(lineIndex:number, map:TokensInflatorMap, topLevelMode:IMode, r:ILineTokens): void { this._lines[lineIndex].setTokens(map, TextModelWithTokens._toLineTokens(r.tokens), topLevelMode, TextModelWithTokens._toModeTransitions(r.modeTransitions)); } @@ -820,14 +807,6 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke }; } - public getWords(lineNumber:number): editorCommon.IWordRange[] { - if (lineNumber < 1 || lineNumber > this.getLineCount()) { - throw new Error('Illegal value ' + lineNumber + ' for `lineNumber`'); - } - - return WordHelper.getWords(this, this.validateLineNumber(lineNumber)); - } - public tokenIterator(position:editorCommon.IPosition, callback:(it:TokenIterator)=>any): any { var iter = new TokenIterator(this, this.validatePosition(position)); var result = callback(iter); @@ -855,19 +834,17 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke return this._findMatchingBracketUp(data, position); } - public matchBracket(position:editorCommon.IPosition, inaccurateResultAcceptable:boolean = false): editorCommon.IMatchBracketResult { + public matchBracket(position:editorCommon.IPosition): [editorCommon.IEditorRange,editorCommon.IEditorRange] { return this._matchBracket(this.validatePosition(position)); } - private _matchBracket(position:editorCommon.IEditorPosition): editorCommon.IMatchBracketResult { - let tokensMap = this._tokensInflatorMap; + private _matchBracket(position:editorCommon.IEditorPosition): [editorCommon.IEditorRange,editorCommon.IEditorRange] { let lineNumber = position.lineNumber; let lineText = this._lines[lineNumber - 1].text; let lineTokens = this._lines[lineNumber - 1].getTokens(); - let tokens = lineTokens.getBinaryEncodedTokens(); let currentTokenIndex = lineTokens.findIndexOfOffset(position.column - 1); - let currentTokenStart = getStartIndex(tokens[currentTokenIndex]); + let currentTokenStart = lineTokens.getTokenStartIndex(currentTokenIndex); let modeTransitions = this._lines[lineNumber - 1].getModeTransitions(this._mode); let currentModeIndex = ModeTransition.findIndexInSegmentsArray(modeTransitions, position.column - 1); @@ -877,11 +854,11 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke // If position is in between two tokens, try first looking in the previous token if (currentTokenIndex > 0 && currentTokenStart === position.column - 1) { let prevTokenIndex = currentTokenIndex - 1; - let prevTokenType = getType(tokensMap, tokens[prevTokenIndex]); + let prevTokenType = lineTokens.getTokenType(prevTokenIndex); // check that previous token is not to be ignored if (!ignoreBracketsInToken(prevTokenType)) { - let prevTokenStart = getStartIndex(tokens[prevTokenIndex]); + let prevTokenStart = lineTokens.getTokenStartIndex(prevTokenIndex); let prevMode = currentMode; let prevModeBrackets = currentModeBrackets; @@ -912,14 +889,14 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke } // check that the token is not to be ignored - if (!ignoreBracketsInToken(getType(tokensMap, tokens[currentTokenIndex]))) { + if (!ignoreBracketsInToken(lineTokens.getTokenType(currentTokenIndex))) { if (currentModeBrackets) { // limit search to not go before `maxBracketLength` currentTokenStart = Math.max(currentTokenStart, position.column - 1 - currentModeBrackets.maxBracketLength); // limit search to not go after `maxBracketLength` - let currentTokenEnd = (currentTokenIndex + 1 < tokens.length ? getStartIndex(tokens[currentTokenIndex + 1]) : lineText.length); + let currentTokenEnd = lineTokens.getTokenEndIndex(currentTokenIndex, lineText.length); currentTokenEnd = Math.min(currentTokenEnd, position.column - 1 + currentModeBrackets.maxBracketLength); // it might still be the case that [currentTokenStart -> currentTokenEnd] contains multiple brackets @@ -946,28 +923,19 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke } } - return { - brackets: null, - isAccurate: true - }; + return null; } - private _matchFoundBracket(foundBracket:Range, data:editorCommon.IRichEditBracket, isOpen:boolean): editorCommon.IMatchBracketResult { + private _matchFoundBracket(foundBracket:Range, data:editorCommon.IRichEditBracket, isOpen:boolean): [editorCommon.IEditorRange,editorCommon.IEditorRange] { if (isOpen) { let matched = this._findMatchingBracketDown(data, foundBracket.getEndPosition()); if (matched) { - return { - brackets: [foundBracket, matched], - isAccurate: true - }; + return [foundBracket, matched]; } } else { let matched = this._findMatchingBracketUp(data, foundBracket.getStartPosition()); if (matched) { - return { - brackets: [foundBracket, matched], - isAccurate: true - }; + return [foundBracket, matched]; } } @@ -978,20 +946,18 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke // console.log('_findMatchingBracketUp: ', 'bracket: ', JSON.stringify(bracket), 'startPosition: ', String(position)); let modeId = bracket.modeId; - let tokensMap = this._tokensInflatorMap; let reversedBracketRegex = bracket.reversedRegex; let count = -1; for (let lineNumber = position.lineNumber; lineNumber >= 1; lineNumber--) { let lineTokens = this._lines[lineNumber - 1].getTokens(); let lineText = this._lines[lineNumber - 1].text; - let tokens = lineTokens.getBinaryEncodedTokens(); let modeTransitions = this._lines[lineNumber - 1].getModeTransitions(this._mode); let currentModeIndex = modeTransitions.length - 1; let currentModeStart = modeTransitions[currentModeIndex].startIndex; let currentModeId = modeTransitions[currentModeIndex].mode.getId(); - let tokensLength = tokens.length - 1; + let tokensLength = lineTokens.getTokenCount() - 1; let currentTokenEnd = lineText.length; if (lineNumber === position.lineNumber) { tokensLength = lineTokens.findIndexOfOffset(position.column - 1); @@ -1003,9 +969,8 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke } for (let tokenIndex = tokensLength; tokenIndex >= 0; tokenIndex--) { - let currentToken = tokens[tokenIndex]; - let currentTokenType = getType(tokensMap, currentToken); - let currentTokenStart = getStartIndex(currentToken); + let currentTokenType = lineTokens.getTokenType(tokenIndex); + let currentTokenStart = lineTokens.getTokenStartIndex(tokenIndex); if (currentTokenStart < currentModeStart) { currentModeIndex--; @@ -1048,21 +1013,19 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke // console.log('_findMatchingBracketDown: ', 'bracket: ', JSON.stringify(bracket), 'startPosition: ', String(position)); let modeId = bracket.modeId; - let tokensMap = this._tokensInflatorMap; let bracketRegex = bracket.forwardRegex; let count = 1; for (let lineNumber = position.lineNumber, lineCount = this.getLineCount(); lineNumber <= lineCount; lineNumber++) { let lineTokens = this._lines[lineNumber - 1].getTokens(); let lineText = this._lines[lineNumber - 1].text; - let tokens = lineTokens.getBinaryEncodedTokens(); let modeTransitions = this._lines[lineNumber - 1].getModeTransitions(this._mode); let currentModeIndex = 0; let nextModeStart = (currentModeIndex + 1 < modeTransitions.length ? modeTransitions[currentModeIndex + 1].startIndex : lineText.length + 1); let currentModeId = modeTransitions[currentModeIndex].mode.getId(); let startTokenIndex = 0; - let currentTokenStart = getStartIndex(startTokenIndex); + let currentTokenStart = lineTokens.getTokenStartIndex(startTokenIndex); if (lineNumber === position.lineNumber) { startTokenIndex = lineTokens.findIndexOfOffset(position.column - 1); currentTokenStart = Math.max(currentTokenStart, position.column - 1); @@ -1072,10 +1035,9 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke currentModeId = modeTransitions[currentModeIndex].mode.getId(); } - for (let tokenIndex = startTokenIndex, tokensLength = tokens.length; tokenIndex < tokensLength; tokenIndex++) { - let currentToken = tokens[tokenIndex]; - let currentTokenType = getType(tokensMap, currentToken); - let currentTokenEnd = tokenIndex + 1 < tokensLength ? getStartIndex(tokens[tokenIndex + 1]) : lineText.length; + for (let tokenIndex = startTokenIndex, tokensLength = lineTokens.getTokenCount(); tokenIndex < tokensLength; tokenIndex++) { + let currentTokenType = lineTokens.getTokenType(tokenIndex); + let currentTokenEnd = lineTokens.getTokenEndIndex(tokenIndex, lineText.length); if (currentTokenStart >= nextModeStart) { currentModeIndex++; @@ -1116,15 +1078,13 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke public findPrevBracket(_position:editorCommon.IPosition): editorCommon.IFoundBracket { let position = this.validatePosition(_position); - let tokensMap = this._tokensInflatorMap; let reversedBracketRegex = /[\(\)\[\]\{\}]/; // TODO@Alex: use mode's brackets for (let lineNumber = position.lineNumber; lineNumber >= 1; lineNumber--) { let lineTokens = this._lines[lineNumber - 1].getTokens(); let lineText = this._lines[lineNumber - 1].text; - let tokens = lineTokens.getBinaryEncodedTokens(); - let tokensLength = tokens.length - 1; + let tokensLength = lineTokens.getTokenCount() - 1; let currentTokenEnd = lineText.length; if (lineNumber === position.lineNumber) { tokensLength = lineTokens.findIndexOfOffset(position.column - 1); @@ -1132,9 +1092,8 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke } for (let tokenIndex = tokensLength; tokenIndex >= 0; tokenIndex--) { - let currentToken = tokens[tokenIndex]; - let currentTokenType = getType(tokensMap, currentToken); - let currentTokenStart = getStartIndex(currentToken); + let currentTokenType = lineTokens.getTokenType(tokenIndex); + let currentTokenStart = lineTokens.getTokenStartIndex(tokenIndex); if (!ignoreBracketsInToken(currentTokenType)) { let r = BracketsUtils.findPrevBracketInToken(reversedBracketRegex, lineNumber, lineText, currentTokenStart, currentTokenEnd); @@ -1153,25 +1112,22 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke public findNextBracket(_position:editorCommon.IPosition): editorCommon.IFoundBracket { let position = this.validatePosition(_position); - let tokensMap = this._tokensInflatorMap; let bracketRegex = /[\(\)\[\]\{\}]/; // TODO@Alex: use mode's brackets for (let lineNumber = position.lineNumber, lineCount = this.getLineCount(); lineNumber <= lineCount; lineNumber++) { let lineTokens = this._lines[lineNumber - 1].getTokens(); let lineText = this._lines[lineNumber - 1].text; - let tokens = lineTokens.getBinaryEncodedTokens(); let startTokenIndex = 0; - let currentTokenStart = getStartIndex(startTokenIndex); + let currentTokenStart = lineTokens.getTokenStartIndex(startTokenIndex); if (lineNumber === position.lineNumber) { startTokenIndex = lineTokens.findIndexOfOffset(position.column - 1); currentTokenStart = Math.max(currentTokenStart, position.column - 1); } - for (let tokenIndex = startTokenIndex, tokensLength = tokens.length; tokenIndex < tokensLength; tokenIndex++) { - let currentToken = tokens[tokenIndex]; - let currentTokenType = getType(tokensMap, currentToken); - let currentTokenEnd = tokenIndex + 1 < tokensLength ? getStartIndex(tokens[tokenIndex + 1]) : lineText.length; + for (let tokenIndex = startTokenIndex, tokensLength = lineTokens.getTokenCount(); tokenIndex < tokensLength; tokenIndex++) { + let currentTokenType = lineTokens.getTokenType(tokenIndex); + let currentTokenEnd = lineTokens.getTokenEndIndex(tokenIndex, lineText.length); if (!ignoreBracketsInToken(currentTokenType)) { let r = BracketsUtils.findNextBracketInToken(bracketRegex, lineNumber, lineText, currentTokenStart, currentTokenEnd); @@ -1206,6 +1162,3 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke return null; } } - -var getType = TokensBinaryEncoding.getType; -var getStartIndex = TokensBinaryEncoding.getStartIndex; diff --git a/src/vs/editor/common/model/textModelWithTokensHelpers.ts b/src/vs/editor/common/model/textModelWithTokensHelpers.ts index ff0deb7f12e..f306132d810 100644 --- a/src/vs/editor/common/model/textModelWithTokensHelpers.ts +++ b/src/vs/editor/common/model/textModelWithTokensHelpers.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ 'use strict'; -import {ILineTokens, IPosition, IWordAtPosition, IWordRange} from 'vs/editor/common/editorCommon'; +import {IPosition, IWordAtPosition, IWordRange} from 'vs/editor/common/editorCommon'; import {IMode, IModeTransition} from 'vs/editor/common/modes'; import {NullMode} from 'vs/editor/common/modes/nullMode'; import {ModeTransition} from 'vs/editor/common/core/modeTransition'; @@ -15,15 +15,9 @@ export interface ITextSource { getLineContent(lineNumber:number): string; - getLineCount(): number; - getMode(): IMode; - getModeAtPosition(lineNumber:number, column:number): IMode; - _getLineModeTransitions(lineNumber:number): ModeTransition[]; - - getLineTokens(lineNumber:number, inaccurateTokensAcceptable:boolean): ILineTokens; } export interface INonWordTokenMap { @@ -63,60 +57,6 @@ export class WordHelper { return WordHelper.ensureValidWordDefinition(WordHelper._safeGetWordDefinition(mode)); } - public static getWords(textSource:ITextSource, lineNumber:number): IWordRange[] { - if (!textSource._lineIsTokenized(lineNumber)) { - return WordHelper._getWordsInText(textSource.getLineContent(lineNumber), WordHelper.massageWordDefinitionOf(textSource.getMode())); - } - - var r: IWordRange[] = [], - txt = textSource.getLineContent(lineNumber); - - if (txt.length > 0) { - - var modeTransitions = textSource._getLineModeTransitions(lineNumber), - i:number, - len:number, - k:number, - lenK:number, - currentModeStartIndex: number, - currentModeEndIndex: number, - currentWordDefinition:RegExp, - currentModeText: string, - words: RegExpMatchArray, - startWord: number, - endWord: number, - word: string; - - // Go through all the modes - for (i = 0, currentModeStartIndex = 0, len = modeTransitions.length; i < len; i++) { - currentWordDefinition = WordHelper.massageWordDefinitionOf(modeTransitions[i].mode); - currentModeStartIndex = modeTransitions[i].startIndex; - currentModeEndIndex = (i + 1 < len ? modeTransitions[i + 1].startIndex : txt.length); - currentModeText = txt.substring(currentModeStartIndex, currentModeEndIndex); - words = currentModeText.match(currentWordDefinition); - - if (!words) { - continue; - } - - endWord = 0; - for (k = 0, lenK = words.length; k < lenK; k++) { - word = words[k]; - if (word.length > 0) { - startWord = currentModeText.indexOf(word, endWord); - endWord = startWord + word.length; - - r.push({ - start: currentModeStartIndex + startWord, - end: currentModeStartIndex + endWord - }); - } - } - } - } - - return r; - } static _getWordsInText(text:string, wordDefinition:RegExp): IWordRange[] { var words = text.match(wordDefinition) || [], diff --git a/src/vs/editor/common/model/tokenIterator.ts b/src/vs/editor/common/model/tokenIterator.ts index d45b2d9cebd..156526bb8be 100644 --- a/src/vs/editor/common/model/tokenIterator.ts +++ b/src/vs/editor/common/model/tokenIterator.ts @@ -5,10 +5,6 @@ 'use strict'; import * as editorCommon from 'vs/editor/common/editorCommon'; -import * as TokensBinaryEncoding from 'vs/editor/common/model/tokensBinaryEncoding'; - -var getStartIndex = TokensBinaryEncoding.getStartIndex; -var inflate = TokensBinaryEncoding.inflate; export class TokenIterator implements editorCommon.ITokenIterator { @@ -16,8 +12,6 @@ export class TokenIterator implements editorCommon.ITokenIterator { private _currentLineNumber:number; private _currentTokenIndex:number; private _currentLineTokens:editorCommon.ILineTokens; - private _currentTokens:number[]; - private _map:editorCommon.ITokensInflatorMap; private _next:editorCommon.ITokenInfo; private _prev:editorCommon.ITokenInfo; @@ -32,27 +26,28 @@ export class TokenIterator implements editorCommon.ITokenIterator { // start with a position to next/prev run var columnIndex = position.column - 1, tokenEndIndex = Number.MAX_VALUE; - for (var i = this._currentTokens.length - 1; i >= 0; i--) { - if (getStartIndex(this._currentTokens[i]) <= columnIndex && columnIndex <= tokenEndIndex) { + for (var i = this._currentLineTokens.getTokenCount() - 1; i >= 0; i--) { + let tokenStartIndex = this._currentLineTokens.getTokenStartIndex(i); + + if (tokenStartIndex <= columnIndex && columnIndex <= tokenEndIndex) { + this._currentTokenIndex = i; this._next = this._current(); this._prev = this._current(); break; } - tokenEndIndex = getStartIndex(this._currentTokens[i]); + tokenEndIndex = tokenStartIndex; } } private _readLineTokens(lineNumber:number): void { this._currentLineTokens = this._model.getLineTokens(lineNumber, false); - this._currentTokens = this._currentLineTokens.getBinaryEncodedTokens(); - this._map = this._currentLineTokens.getBinaryEncodedTokensMap(); } private _advanceNext() { this._prev = this._next; this._next = null; - if (this._currentTokenIndex + 1 < this._currentTokens.length) { + if (this._currentTokenIndex + 1 < this._currentLineTokens.getTokenCount()) { // There are still tokens on current line this._currentTokenIndex++; this._next = this._current(); @@ -62,7 +57,7 @@ export class TokenIterator implements editorCommon.ITokenIterator { while (this._currentLineNumber + 1 <= this._model.getLineCount()) { this._currentLineNumber++; this._readLineTokens(this._currentLineNumber); - if (this._currentTokens.length > 0) { + if (this._currentLineTokens.getTokenCount() > 0) { this._currentTokenIndex = 0; this._next = this._current(); break; @@ -71,7 +66,7 @@ export class TokenIterator implements editorCommon.ITokenIterator { if (this._next === null) { // prepare of a previous run this._readLineTokens(this._currentLineNumber); - this._currentTokenIndex = this._currentTokens.length; + this._currentTokenIndex = this._currentLineTokens.getTokenCount(); this._advancePrev(); this._next = null; } @@ -91,8 +86,8 @@ export class TokenIterator implements editorCommon.ITokenIterator { while (this._currentLineNumber > 1) { this._currentLineNumber--; this._readLineTokens(this._currentLineNumber); - if (this._currentTokens.length > 0) { - this._currentTokenIndex = this._currentTokens.length - 1; + if (this._currentLineTokens.getTokenCount() > 0) { + this._currentTokenIndex = this._currentLineTokens.getTokenCount() - 1; this._prev = this._current(); break; } @@ -101,11 +96,18 @@ export class TokenIterator implements editorCommon.ITokenIterator { } private _current(): editorCommon.ITokenInfo { + let startIndex = this._currentLineTokens.getTokenStartIndex(this._currentTokenIndex); + let type = this._currentLineTokens.getTokenType(this._currentTokenIndex); + let endIndex = this._currentLineTokens.getTokenEndIndex(this._currentTokenIndex, this._model.getLineContent(this._currentLineNumber).length); + return { - token: inflate(this._map, this._currentTokens[this._currentTokenIndex]), + token: { + startIndex: startIndex, + type: type + }, lineNumber: this._currentLineNumber, - startColumn: getStartIndex(this._currentTokens[this._currentTokenIndex]) + 1, - endColumn: this._currentTokenIndex + 1 < this._currentTokens.length ? getStartIndex(this._currentTokens[this._currentTokenIndex + 1]) + 1 : this._model.getLineContent(this._currentLineNumber).length + 1 + startColumn: startIndex + 1, + endColumn: endIndex + 1 }; } diff --git a/src/vs/editor/common/model/tokensBinaryEncoding.ts b/src/vs/editor/common/model/tokensBinaryEncoding.ts index 94b786ef24d..a329514d822 100644 --- a/src/vs/editor/common/model/tokensBinaryEncoding.ts +++ b/src/vs/editor/common/model/tokensBinaryEncoding.ts @@ -6,183 +6,192 @@ import {onUnexpectedError} from 'vs/base/common/errors'; import * as strings from 'vs/base/common/strings'; -import {ITokensInflatorMap, LineToken} from 'vs/editor/common/editorCommon'; -import {ViewLineToken} from 'vs/editor/common/viewModel/viewModel'; +import {ViewLineToken} from 'vs/editor/common/core/viewLineToken'; +import {LineToken} from 'vs/editor/common/model/lineToken'; -export const START_INDEX_MASK = 0xffffffff; -export const TYPE_MASK = 0xffff; -export const START_INDEX_OFFSET = 1; -export const TYPE_OFFSET = Math.pow(2, 32); +const START_INDEX_MASK = 0xffffffff; +const TYPE_MASK = 0xffff; +const START_INDEX_OFFSET = 1; +const TYPE_OFFSET = Math.pow(2, 32); -const DEFAULT_TOKEN = new LineToken(0, ''); const DEFAULT_VIEW_TOKEN = new ViewLineToken(0, ''); const INFLATED_TOKENS_EMPTY_TEXT:ViewLineToken[] = []; const DEFLATED_TOKENS_EMPTY_TEXT:number[] = []; const INFLATED_TOKENS_NON_EMPTY_TEXT:ViewLineToken[] = [DEFAULT_VIEW_TOKEN]; const DEFLATED_TOKENS_NON_EMPTY_TEXT:number[] = [0]; -export function deflateArr(map:ITokensInflatorMap, tokens:LineToken[]): number[] { - if (tokens.length === 0) { - return DEFLATED_TOKENS_EMPTY_TEXT; - } - if (tokens.length === 1 && tokens[0].startIndex === 0 && !tokens[0].type) { - return DEFLATED_TOKENS_NON_EMPTY_TEXT; +export class TokensInflatorMap { + + public _inflate:string[]; + + public _deflate: { + [token:string]:number; + }; + + constructor() { + this._inflate = [ '' ]; + this._deflate = { '': 0 }; } +} - var i:number, - len:number, - deflatedToken:number, - deflated:number, - token:LineToken, - inflateMap = map._inflate, - deflateMap = map._deflate, - prevStartIndex:number = -1, - result:number[] = new Array(tokens.length); +export class TokensBinaryEncoding { + public static START_INDEX_MASK = START_INDEX_MASK; + public static TYPE_MASK = TYPE_MASK; + public static START_INDEX_OFFSET = START_INDEX_OFFSET; + public static TYPE_OFFSET = TYPE_OFFSET; - for (i = 0, len = tokens.length; i < len; i++) { - token = tokens[i]; - - if (token.startIndex <= prevStartIndex) { - token.startIndex = prevStartIndex + 1; - onUnexpectedError({ - message: 'Invalid tokens detected', - tokens: tokens - }); + public static deflateArr(map:TokensInflatorMap, tokens:LineToken[]): number[] { + if (tokens.length === 0) { + return DEFLATED_TOKENS_EMPTY_TEXT; + } + if (tokens.length === 1 && tokens[0].startIndex === 0 && !tokens[0].type) { + return DEFLATED_TOKENS_NON_EMPTY_TEXT; } - if (deflateMap.hasOwnProperty(token.type)) { - deflatedToken = deflateMap[token.type]; - } else { - deflatedToken = inflateMap.length; - deflateMap[token.type] = deflatedToken; - inflateMap.push(token.type); + var i:number, + len:number, + deflatedToken:number, + deflated:number, + token:LineToken, + inflateMap = map._inflate, + deflateMap = map._deflate, + prevStartIndex:number = -1, + result:number[] = new Array(tokens.length); + + for (i = 0, len = tokens.length; i < len; i++) { + token = tokens[i]; + + if (token.startIndex <= prevStartIndex) { + token.startIndex = prevStartIndex + 1; + onUnexpectedError({ + message: 'Invalid tokens detected', + tokens: tokens + }); + } + + if (deflateMap.hasOwnProperty(token.type)) { + deflatedToken = deflateMap[token.type]; + } else { + deflatedToken = inflateMap.length; + deflateMap[token.type] = deflatedToken; + inflateMap.push(token.type); + } + + // http://stackoverflow.com/a/2803010 + // All numbers in JavaScript are actually IEEE-754 compliant floating-point doubles. + // These have a 53-bit mantissa which should mean that any integer value with a magnitude + // of approximately 9 quadrillion or less -- more specifically, 9,007,199,254,740,991 -- + // will be represented accurately. + + // http://stackoverflow.com/a/6729252 + // Bitwise operations cast numbers to 32bit representation in JS + + // 32 bits for startIndex => up to 2^32 = 4,294,967,296 + // 16 bits for token => up to 2^16 = 65,536 + + // [token][startIndex] + deflated = deflatedToken * TYPE_OFFSET + token.startIndex * START_INDEX_OFFSET; + + result[i] = deflated; + + prevStartIndex = token.startIndex; } - // http://stackoverflow.com/a/2803010 - // All numbers in JavaScript are actually IEEE-754 compliant floating-point doubles. - // These have a 53-bit mantissa which should mean that any integer value with a magnitude - // of approximately 9 quadrillion or less -- more specifically, 9,007,199,254,740,991 -- - // will be represented accurately. - - // http://stackoverflow.com/a/6729252 - // Bitwise operations cast numbers to 32bit representation in JS - - // 32 bits for startIndex => up to 2^32 = 4,294,967,296 - // 16 bits for token => up to 2^16 = 65,536 - - // [token][startIndex] - deflated = deflatedToken * TYPE_OFFSET + token.startIndex * START_INDEX_OFFSET; - - result[i] = deflated; - - prevStartIndex = token.startIndex; + return result; } - return result; -} - -export function inflate(map:ITokensInflatorMap, binaryEncodedToken:number): LineToken { - if (binaryEncodedToken === 0) { - return DEFAULT_TOKEN; + public static getStartIndex(binaryEncodedToken:number): number { + return (binaryEncodedToken / START_INDEX_OFFSET) & START_INDEX_MASK; } - var startIndex = (binaryEncodedToken / START_INDEX_OFFSET) & START_INDEX_MASK; - var deflatedType = (binaryEncodedToken / TYPE_OFFSET) & TYPE_MASK; - - return new LineToken(startIndex, map._inflate[deflatedType]); -} - -export function getStartIndex(binaryEncodedToken:number): number { - return (binaryEncodedToken / START_INDEX_OFFSET) & START_INDEX_MASK; -} - -export function getType(map:ITokensInflatorMap, binaryEncodedToken:number): string { - var deflatedType = (binaryEncodedToken / TYPE_OFFSET) & TYPE_MASK; - if (deflatedType === 0) { - return strings.empty; - } - return map._inflate[deflatedType]; -} - -export function inflateArr(map:ITokensInflatorMap, binaryEncodedTokens:number[]): ViewLineToken[] { - if (binaryEncodedTokens.length === 0) { - return INFLATED_TOKENS_EMPTY_TEXT; - } - if (binaryEncodedTokens.length === 1 && binaryEncodedTokens[0] === 0) { - return INFLATED_TOKENS_NON_EMPTY_TEXT; + public static getType(map:TokensInflatorMap, binaryEncodedToken:number): string { + var deflatedType = (binaryEncodedToken / TYPE_OFFSET) & TYPE_MASK; + if (deflatedType === 0) { + return strings.empty; + } + return map._inflate[deflatedType]; } - let result: ViewLineToken[] = []; - const inflateMap = map._inflate; - - for (let i = 0, len = binaryEncodedTokens.length; i < len; i++) { - let deflated = binaryEncodedTokens[i]; - - let startIndex = (deflated / START_INDEX_OFFSET) & START_INDEX_MASK; - let deflatedType = (deflated / TYPE_OFFSET) & TYPE_MASK; - - result.push(new ViewLineToken(startIndex, inflateMap[deflatedType])); - } - - return result; -} - -export function findIndexOfOffset(binaryEncodedTokens:number[], offset:number): number { - return findIndexInSegmentsArray(binaryEncodedTokens, offset); -} - -export function sliceAndInflate(map:ITokensInflatorMap, binaryEncodedTokens:number[], startOffset:number, endOffset:number, deltaStartIndex:number): ViewLineToken[] { - if (binaryEncodedTokens.length === 0) { - return INFLATED_TOKENS_EMPTY_TEXT; - } - if (binaryEncodedTokens.length === 1 && binaryEncodedTokens[0] === 0) { - return INFLATED_TOKENS_NON_EMPTY_TEXT; - } - - let startIndex = findIndexInSegmentsArray(binaryEncodedTokens, startOffset); - let result: ViewLineToken[] = []; - const inflateMap = map._inflate; - - let originalToken = binaryEncodedTokens[startIndex]; - let deflatedType = (originalToken / TYPE_OFFSET) & TYPE_MASK; - let newStartIndex = 0; - result.push(new ViewLineToken(newStartIndex, inflateMap[deflatedType])); - - for (let i = startIndex + 1, len = binaryEncodedTokens.length; i < len; i++) { - originalToken = binaryEncodedTokens[i]; - let originalStartIndex = (originalToken / START_INDEX_OFFSET) & START_INDEX_MASK; - - if (originalStartIndex >= endOffset) { - break; + public static inflateArr(map:TokensInflatorMap, binaryEncodedTokens:number[]): ViewLineToken[] { + if (binaryEncodedTokens.length === 0) { + return INFLATED_TOKENS_EMPTY_TEXT; + } + if (binaryEncodedTokens.length === 1 && binaryEncodedTokens[0] === 0) { + return INFLATED_TOKENS_NON_EMPTY_TEXT; } - deflatedType = (originalToken / TYPE_OFFSET) & TYPE_MASK; - newStartIndex = originalStartIndex - startOffset + deltaStartIndex; + let result: ViewLineToken[] = []; + const inflateMap = map._inflate; + + for (let i = 0, len = binaryEncodedTokens.length; i < len; i++) { + let deflated = binaryEncodedTokens[i]; + + let startIndex = (deflated / START_INDEX_OFFSET) & START_INDEX_MASK; + let deflatedType = (deflated / TYPE_OFFSET) & TYPE_MASK; + + result.push(new ViewLineToken(startIndex, inflateMap[deflatedType])); + } + + return result; + } + + public static findIndexOfOffset(binaryEncodedTokens:number[], offset:number): number { + return this.findIndexInSegmentsArray(binaryEncodedTokens, offset); + } + + public static sliceAndInflate(map:TokensInflatorMap, binaryEncodedTokens:number[], startOffset:number, endOffset:number, deltaStartIndex:number): ViewLineToken[] { + if (binaryEncodedTokens.length === 0) { + return INFLATED_TOKENS_EMPTY_TEXT; + } + if (binaryEncodedTokens.length === 1 && binaryEncodedTokens[0] === 0) { + return INFLATED_TOKENS_NON_EMPTY_TEXT; + } + + let startIndex = this.findIndexInSegmentsArray(binaryEncodedTokens, startOffset); + let result: ViewLineToken[] = []; + const inflateMap = map._inflate; + + let originalToken = binaryEncodedTokens[startIndex]; + let deflatedType = (originalToken / TYPE_OFFSET) & TYPE_MASK; + let newStartIndex = 0; result.push(new ViewLineToken(newStartIndex, inflateMap[deflatedType])); - } - return result; -} + for (let i = startIndex + 1, len = binaryEncodedTokens.length; i < len; i++) { + originalToken = binaryEncodedTokens[i]; + let originalStartIndex = (originalToken / START_INDEX_OFFSET) & START_INDEX_MASK; -export function findIndexInSegmentsArray(arr:number[], desiredIndex: number):number { + if (originalStartIndex >= endOffset) { + break; + } - var low = 0, - high = arr.length - 1, - mid:number, - value:number; - - while (low < high) { - - mid = low + Math.ceil((high - low)/2); - - value = arr[mid] & 0xffffffff; - - if (value > desiredIndex) { - high = mid - 1; - } else { - low = mid; + deflatedType = (originalToken / TYPE_OFFSET) & TYPE_MASK; + newStartIndex = originalStartIndex - startOffset + deltaStartIndex; + result.push(new ViewLineToken(newStartIndex, inflateMap[deflatedType])); } + + return result; } - return low; + private static findIndexInSegmentsArray(arr:number[], desiredIndex: number):number { + + var low = 0, + high = arr.length - 1, + mid:number, + value:number; + + while (low < high) { + + mid = low + Math.ceil((high - low)/2); + + value = arr[mid] & 0xffffffff; + + if (value > desiredIndex) { + high = mid - 1; + } else { + low = mid; + } + } + + return low; + } } \ No newline at end of file diff --git a/src/vs/editor/common/viewLayout/viewLineParts.ts b/src/vs/editor/common/viewLayout/viewLineParts.ts index dd17765aa32..90599c39349 100644 --- a/src/vs/editor/common/viewLayout/viewLineParts.ts +++ b/src/vs/editor/common/viewLayout/viewLineParts.ts @@ -8,7 +8,7 @@ import * as strings from 'vs/base/common/strings'; import {Arrays} from 'vs/editor/common/core/arrays'; import {IEditorRange} from 'vs/editor/common/editorCommon'; import {Range} from 'vs/editor/common/core/range'; -import {ViewLineToken, ViewLineTokens} from 'vs/editor/common/viewModel/viewModel'; +import {ViewLineToken, ViewLineTokens} from 'vs/editor/common/core/viewLineToken'; function cmpLineDecorations(a:ILineDecoration, b:ILineDecoration): number { return Range.compareRangesUsingStarts(a.range, b.range); diff --git a/src/vs/editor/common/viewLayout/viewLineRenderer.ts b/src/vs/editor/common/viewLayout/viewLineRenderer.ts index a523964b3a9..835c45d2897 100644 --- a/src/vs/editor/common/viewLayout/viewLineRenderer.ts +++ b/src/vs/editor/common/viewLayout/viewLineRenderer.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ 'use strict'; -import {ViewLineToken} from 'vs/editor/common/viewModel/viewModel'; +import {ViewLineToken} from 'vs/editor/common/core/viewLineToken'; export class RenderLineInput { public _renderLineInputTrait: void; diff --git a/src/vs/editor/common/viewModel/filteredLineTokens.ts b/src/vs/editor/common/viewModel/filteredLineTokens.ts index 4293a0a5b7b..29ae5eef517 100644 --- a/src/vs/editor/common/viewModel/filteredLineTokens.ts +++ b/src/vs/editor/common/viewModel/filteredLineTokens.ts @@ -5,15 +5,14 @@ 'use strict'; import {ILineTokens} from 'vs/editor/common/editorCommon'; -import * as TokensBinaryEncoding from 'vs/editor/common/model/tokensBinaryEncoding'; -import {ViewLineTokens} from 'vs/editor/common/viewModel/viewModel'; +import {ViewLineTokens} from 'vs/editor/common/core/viewLineToken'; export class FilteredLineTokens { /** * [startOffset; endOffset) (i.e. do not include endOffset) */ public static create(original:ILineTokens, startOffset:number, endOffset:number, deltaStartIndex:number): ViewLineTokens { - let inflatedTokens = TokensBinaryEncoding.sliceAndInflate(original.getBinaryEncodedTokensMap(), original.getBinaryEncodedTokens(), startOffset, endOffset, deltaStartIndex); + let inflatedTokens = original.sliceAndInflate(startOffset, endOffset, deltaStartIndex); return new ViewLineTokens( inflatedTokens, deltaStartIndex, @@ -25,7 +24,7 @@ export class FilteredLineTokens { export class IdentityFilteredLineTokens { public static create(original:ILineTokens, textLength:number): ViewLineTokens { - let inflatedTokens = TokensBinaryEncoding.inflateArr(original.getBinaryEncodedTokensMap(), original.getBinaryEncodedTokens()); + let inflatedTokens = original.inflate(); return new ViewLineTokens( inflatedTokens, 0, diff --git a/src/vs/editor/common/viewModel/splitLinesCollection.ts b/src/vs/editor/common/viewModel/splitLinesCollection.ts index 7d5360f7a4b..3f4f724089a 100644 --- a/src/vs/editor/common/viewModel/splitLinesCollection.ts +++ b/src/vs/editor/common/viewModel/splitLinesCollection.ts @@ -10,7 +10,7 @@ import * as editorCommon from 'vs/editor/common/editorCommon'; import {FilteredLineTokens, IdentityFilteredLineTokens} from 'vs/editor/common/viewModel/filteredLineTokens'; import {PrefixSumComputer} from 'vs/editor/common/viewModel/prefixSumComputer'; import {ILinesCollection} from 'vs/editor/common/viewModel/viewModelImpl'; -import {ViewLineTokens} from 'vs/editor/common/viewModel/viewModel'; +import {ViewLineTokens} from 'vs/editor/common/core/viewLineToken'; export class OutputPosition { _outputPositionTrait: void; diff --git a/src/vs/editor/common/viewModel/viewModel.ts b/src/vs/editor/common/viewModel/viewModel.ts index 349f14f7a06..1ce3f1b4f52 100644 --- a/src/vs/editor/common/viewModel/viewModel.ts +++ b/src/vs/editor/common/viewModel/viewModel.ts @@ -5,86 +5,8 @@ 'use strict'; import {IEventEmitter} from 'vs/base/common/eventEmitter'; -import {Arrays} from 'vs/editor/common/core/arrays'; import {IModelDecoration, IRange, IEditorRange, EndOfLinePreference, IEditorSelection, IPosition, IEditorPosition} from 'vs/editor/common/editorCommon'; - -/** - * A token on a line. - */ -export class ViewLineToken { - public _viewLineTokenTrait: void; - - public startIndex:number; - public type:string; - - constructor(startIndex:number, type:string) { - this.startIndex = startIndex|0;// @perf - this.type = type.replace(/[^a-z0-9\-]/gi, ' '); - } - - public equals(other:ViewLineToken): boolean { - return ( - this.startIndex === other.startIndex - && this.type === other.type - ); - } - - public static findIndexInSegmentsArray(arr:ViewLineToken[], desiredIndex: number): number { - return Arrays.findIndexInSegmentsArray(arr, desiredIndex); - } - - public static equalsArray(a:ViewLineToken[], b:ViewLineToken[]): boolean { - let aLen = a.length; - let bLen = b.length; - if (aLen !== bLen) { - return false; - } - for (let i = 0; i < aLen; i++) { - if (!a[i].equals(b[i])) { - return false; - } - } - return true; - } -} - -export class ViewLineTokens { - _viewLineTokensTrait: void; - - private _lineTokens:ViewLineToken[]; - private _fauxIndentLength:number; - private _textLength:number; - - constructor(lineTokens:ViewLineToken[], fauxIndentLength:number, textLength:number) { - this._lineTokens = lineTokens; - this._fauxIndentLength = fauxIndentLength|0; - this._textLength = textLength|0; - } - - public getTokens(): ViewLineToken[] { - return this._lineTokens; - } - - public getFauxIndentLength(): number { - return this._fauxIndentLength; - } - - public getTextLength(): number { - return this._textLength; - } - - public equals(other:ViewLineTokens): boolean { - return ( - this._fauxIndentLength === other._fauxIndentLength - && this._textLength === other._textLength - && ViewLineToken.equalsArray(this._lineTokens, other._lineTokens) - ); - } - - public findIndexOfOffset(offset:number): number { - return ViewLineToken.findIndexInSegmentsArray(this._lineTokens, offset); - } -} +import {ViewLineTokens} from 'vs/editor/common/core/viewLineToken'; export interface IDecorationsViewportData { decorations: IModelDecoration[]; diff --git a/src/vs/editor/common/viewModel/viewModelImpl.ts b/src/vs/editor/common/viewModel/viewModelImpl.ts index 5a1f6913438..ee972b31d5c 100644 --- a/src/vs/editor/common/viewModel/viewModelImpl.ts +++ b/src/vs/editor/common/viewModel/viewModelImpl.ts @@ -13,7 +13,8 @@ import {Selection} from 'vs/editor/common/core/selection'; import * as editorCommon from 'vs/editor/common/editorCommon'; import {ViewModelCursors} from 'vs/editor/common/viewModel/viewModelCursors'; import {ViewModelDecorations} from 'vs/editor/common/viewModel/viewModelDecorations'; -import {IDecorationsViewportData, ViewLineTokens, IViewModel} from 'vs/editor/common/viewModel/viewModel'; +import {IDecorationsViewportData, IViewModel} from 'vs/editor/common/viewModel/viewModel'; +import {ViewLineTokens} from 'vs/editor/common/core/viewLineToken'; export interface ILinesCollection { setTabSize(newTabSize:number, emit:(evenType:string, payload:any)=>void): boolean; @@ -115,8 +116,8 @@ export class ViewModel extends EventEmitter implements IViewModel { return lineMappingChanged; } - private _onWrappingIndentChange(newWrappingIndent:string): boolean { - var lineMappingChanged = this.lines.setWrappingIndent(editorCommon.wrappingIndentFromString(newWrappingIndent), (eventType:string, payload:any) => this.emit(eventType, payload)); + private _onWrappingIndentChange(newWrappingIndent:editorCommon.WrappingIndent): boolean { + var lineMappingChanged = this.lines.setWrappingIndent(newWrappingIndent, (eventType:string, payload:any) => this.emit(eventType, payload)); if (lineMappingChanged) { this.emit(editorCommon.ViewEventNames.LineMappingChangedEvent); this.decorations.onLineMappingChanged((eventType:string, payload:any) => this.emit(eventType, payload)); diff --git a/src/vs/editor/test/common/model/model.line.test.ts b/src/vs/editor/test/common/model/model.line.test.ts index 0843799ada2..32ca85b98a2 100644 --- a/src/vs/editor/test/common/model/model.line.test.ts +++ b/src/vs/editor/test/common/model/model.line.test.ts @@ -5,15 +5,15 @@ 'use strict'; import * as assert from 'assert'; -import {ILineTokens, LineToken} from 'vs/editor/common/editorCommon'; +import {ILineTokens} from 'vs/editor/common/editorCommon'; import * as modelLine from 'vs/editor/common/model/modelLine'; import {LineMarker} from 'vs/editor/common/model/textModelWithMarkers'; -import {TokensInflatorMap} from 'vs/editor/common/model/textModelWithTokens'; +import {TokensInflatorMap} from 'vs/editor/common/model/tokensBinaryEncoding'; import {IToken} from 'vs/editor/common/modes'; -import * as TokensBinaryEncoding from 'vs/editor/common/model/tokensBinaryEncoding'; +import {LineToken} from 'vs/editor/common/model/lineToken'; function assertLineTokens(actual:ILineTokens, expected:IToken[]): void { - var inflatedActual = TokensBinaryEncoding.inflateArr(actual.getBinaryEncodedTokensMap(), actual.getBinaryEncodedTokens()); + var inflatedActual = actual.inflate(); assert.deepEqual(inflatedActual, expected, 'Line tokens are equal'); } diff --git a/src/vs/editor/test/common/model/model.test.ts b/src/vs/editor/test/common/model/model.test.ts index 25cdf8b7d15..501b4d97e69 100644 --- a/src/vs/editor/test/common/model/model.test.ts +++ b/src/vs/editor/test/common/model/model.test.ts @@ -14,26 +14,20 @@ import {BracketMode} from 'vs/editor/test/common/testModes'; // --------- utils -function isNotABracket(model, lineNumber, column) { +function isNotABracket(model:Model, lineNumber:number, column:number) { var match = model.matchBracket(new Position(lineNumber, column)); - assert.equal(match.isAccurate, true, 'is not matching brackets at ' + lineNumber + ', ' + column); - assert.equal(match.brackets, null, 'is not matching brackets at ' + lineNumber + ', ' + column); + assert.equal(match, null, 'is not matching brackets at ' + lineNumber + ', ' + column); } -function isBracket(model, lineNumber1, column11, column12, lineNumber2, column21, column22) { +function isBracket(model:Model, lineNumber1:number, column11:number, column12:number, lineNumber2:number, column21:number, column22:number) { var match = model.matchBracket(new Position(lineNumber1, column11)); - assert.deepEqual(match, { - brackets: [ - new Range(lineNumber1, column11, lineNumber1, column12), - new Range(lineNumber2, column21, lineNumber2, column22) - ], - isAccurate: true - }, 'is matching brackets at ' + lineNumber1 + ', ' + column11); + assert.deepEqual(match, [ + new Range(lineNumber1, column11, lineNumber1, column12), + new Range(lineNumber2, column21, lineNumber2, column22) + ], 'is matching brackets at ' + lineNumber1 + ', ' + column11); } - - -function rangeEqual(range, startLineNumber, startColumn, endLineNumber, endColumn) { +function rangeEqual(range:Range, startLineNumber:number, startColumn:number, endLineNumber:number, endColumn:number) { assert.deepEqual(range, new Range(startLineNumber, startColumn, endLineNumber, endColumn)); } @@ -506,23 +500,6 @@ suite('Editor Model - Words', () => { thisModel.destroy(); }); - test('Get all words', () => { - var words = [ - { start: 0, end: 4 }, - { start: 5, end: 9 }, - { start: 10, end: 13 }, - { start: 14, end: 18 }, - { start: 20, end: 25 }, - { start: 25, end: 26 } - ]; - - var modelWords = thisModel.getWords(1); - - for (var i = 0; i < modelWords.length; i++) { - assert.deepEqual(modelWords[i], words[i]); - } - }); - test('Get word at position', () => { assert.deepEqual(thisModel.getWordAtPosition(new Position(1, 1)), { word: 'This', startColumn: 1, endColumn: 5 }); assert.deepEqual(thisModel.getWordAtPosition(new Position(1, 2)), { word: 'This', startColumn: 1, endColumn: 5 }); diff --git a/src/vs/editor/test/common/viewLayout/viewLineParts.test.ts b/src/vs/editor/test/common/viewLayout/viewLineParts.test.ts index 56db849f9d8..0d033ca033b 100644 --- a/src/vs/editor/test/common/viewLayout/viewLineParts.test.ts +++ b/src/vs/editor/test/common/viewLayout/viewLineParts.test.ts @@ -8,7 +8,7 @@ import * as assert from 'assert'; import {DecorationSegment, ILineDecoration, LineDecorationsNormalizer, getColumnOfLinePartOffset, createLineParts} from 'vs/editor/common/viewLayout/viewLineParts'; import {Range} from 'vs/editor/common/core/range'; import {RenderLineInput, renderLine} from 'vs/editor/common/viewLayout/viewLineRenderer'; -import {ViewLineToken, ViewLineTokens} from 'vs/editor/common/viewModel/viewModel'; +import {ViewLineToken, ViewLineTokens} from 'vs/editor/common/core/viewLineToken'; suite('Editor ViewLayout - ViewLineParts', () => { diff --git a/src/vs/editor/test/common/viewLayout/viewLineRenderer.test.ts b/src/vs/editor/test/common/viewLayout/viewLineRenderer.test.ts index 5050b476458..e029f13785f 100644 --- a/src/vs/editor/test/common/viewLayout/viewLineRenderer.test.ts +++ b/src/vs/editor/test/common/viewLayout/viewLineRenderer.test.ts @@ -6,7 +6,7 @@ import * as assert from 'assert'; import {renderLine, RenderLineInput} from 'vs/editor/common/viewLayout/viewLineRenderer'; -import {ViewLineToken} from 'vs/editor/common/viewModel/viewModel'; +import {ViewLineToken} from 'vs/editor/common/core/viewLineToken'; suite('viewLineRenderer.renderLine', () => { diff --git a/src/vs/editor/test/common/viewModel/splitLinesCollection.test.ts b/src/vs/editor/test/common/viewModel/splitLinesCollection.test.ts index e6762c256c5..0aad2fb2302 100644 --- a/src/vs/editor/test/common/viewModel/splitLinesCollection.test.ts +++ b/src/vs/editor/test/common/viewModel/splitLinesCollection.test.ts @@ -104,7 +104,7 @@ suite('Editor ViewModel - SplitLinesCollection', () => { model.getOptions().tabSize, config.editor.wrappingInfo.wrappingColumn, config.editor.fontInfo.typicalFullwidthCharacterWidth / config.editor.fontInfo.typicalHalfwidthCharacterWidth, - editorCommon.wrappingIndentFromString(config.editor.wrappingIndent) + config.editor.wrappingIndent ); linesCollection.setHiddenAreas([{ diff --git a/src/vs/languages/html/test/common/html.test.ts b/src/vs/languages/html/test/common/html.test.ts index 94caee822f6..1fc0e908554 100644 --- a/src/vs/languages/html/test/common/html.test.ts +++ b/src/vs/languages/html/test/common/html.test.ts @@ -725,15 +725,15 @@ suite('Colorizing - HTML', () => { test('matchBracket', () => { - function toString(brackets:EditorCommon.IEditorRange[]): string[] { + function toString(brackets:[EditorCommon.IEditorRange, EditorCommon.IEditorRange]): [string,string] { if (!brackets) { return null; } brackets.sort(Range.compareRangesUsingStarts); - return brackets.map(b => b.toString()); + return [brackets[0].toString(), brackets[1].toString()]; } - function assertBracket(lines:string[], lineNumber:number, column:number, expected:EditorCommon.IEditorRange[]): void { + function assertBracket(lines:string[], lineNumber:number, column:number, expected:[EditorCommon.IEditorRange, EditorCommon.IEditorRange]): void { let model = new TextModelWithTokens([], TextModel.toRawText(lines.join('\n'), TextModel.DEFAULT_CREATION_OPTIONS), false, _mode); // force tokenization model.getLineContext(model.getLineCount()); @@ -741,9 +741,7 @@ suite('Colorizing - HTML', () => { lineNumber: lineNumber, column: column }); - let actualStr = actual ? toString(actual.brackets) : null; - let expectedStr = toString(expected); - assert.deepEqual(actualStr, expectedStr, 'TEXT <<' + lines.join('\n') + '>>, POS: ' + lineNumber + ', ' + column); + assert.deepEqual(toString(actual), toString(expected), 'TEXT <<' + lines.join('\n') + '>>, POS: ' + lineNumber + ', ' + column); } assertBracket(['

'], 1, 1, [new Range(1, 1, 1, 2), new Range(1, 3, 1, 4)]);