Further cleanup in editorCommon

src/vs/editor/browser/standalone/colorizer.ts

@@ -10,8 +10,7 @@ import {IModel} from 'vs/editor/common/editorCommon';
 import {ILineTokens, IMode} from 'vs/editor/common/modes';
 import {IModeService} from 'vs/editor/common/services/modeService';
 import {RenderLineOutput, renderLine, RenderLineInput} from 'vs/editor/common/viewLayout/viewLineRenderer';
-import * as TokensBinaryEncoding from 'vs/editor/common/model/tokensBinaryEncoding';
-import {ViewLineToken} from 'vs/editor/common/viewModel/viewModel';
+import {ViewLineToken} from 'vs/editor/common/core/viewLineToken';
 
 export interface IColorizerOptions {
 	tabSize?: number;
@@ -108,7 +107,7 @@ export class Colorizer {
 	public static colorizeModelLine(model:IModel, lineNumber:number, tabSize:number = 4): string {
 		var content = model.getLineContent(lineNumber);
 		var tokens = model.getLineTokens(lineNumber, false);
-		var inflatedTokens = TokensBinaryEncoding.inflateArr(tokens.getBinaryEncodedTokensMap(), tokens.getBinaryEncodedTokens());
+		var inflatedTokens = tokens.inflate();
 		return this.colorizeLine(content, inflatedTokens, tabSize);
 	}
 }

src/vs/editor/browser/standalone/standaloneEditor.ts

@@ -13,7 +13,6 @@ import {ClassNames, ContentWidgetPositionPreference, OverlayWidgetPositionPrefer
 import {Colorizer} from 'vs/editor/browser/standalone/colorizer';
 import * as standaloneCodeEditor from 'vs/editor/browser/standalone/standaloneCodeEditor';
 import {ILanguageDef} from 'vs/editor/standalone-languages/types';
-import * as TokensBinaryEncoding from 'vs/editor/common/model/tokensBinaryEncoding';
 
 var global:any = self;
 if (!global.Monaco) {
@@ -38,11 +37,9 @@ Monaco.Editor.colorizeModelLine = Colorizer.colorizeModelLine;
 // -- export common constants
 Monaco.Editor.SelectionDirection = editorCommon.SelectionDirection;
 Monaco.Editor.WrappingIndent = editorCommon.WrappingIndent;
-Monaco.Editor.wrappingIndentFromString = editorCommon.wrappingIndentFromString;
 Monaco.Editor.OverviewRulerLane = editorCommon.OverviewRulerLane;
 Monaco.Editor.EndOfLinePreference = editorCommon.EndOfLinePreference;
 Monaco.Editor.EndOfLineSequence = editorCommon.EndOfLineSequence;
-Monaco.Editor.LineTokensBinaryEncoding = TokensBinaryEncoding;
 Monaco.Editor.TrackedRangeStickiness = editorCommon.TrackedRangeStickiness;
 Monaco.Editor.VerticalRevealType = editorCommon.VerticalRevealType;
 Monaco.Editor.MouseTargetType = editorCommon.MouseTargetType;

src/vs/editor/browser/widget/codeEditorWidget.ts

@@ -25,7 +25,6 @@ import * as editorBrowser from 'vs/editor/browser/editorBrowser';
 import {EditorBrowserRegistry} from 'vs/editor/browser/editorBrowserExtensions';
 import {Colorizer} from 'vs/editor/browser/standalone/colorizer';
 import {View} from 'vs/editor/browser/view/viewImpl';
-import * as TokensBinaryEncoding from 'vs/editor/common/model/tokensBinaryEncoding';
 
 export class CodeEditorWidget extends CommonCodeEditor implements editorBrowser.ICodeEditor {
 
@@ -107,7 +106,7 @@ export class CodeEditorWidget extends CommonCodeEditor implements editorBrowser.
 		}
 		var content = model.getLineContent(lineNumber);
 		var tokens = model.getLineTokens(lineNumber, false);
-		var inflatedTokens = TokensBinaryEncoding.inflateArr(tokens.getBinaryEncodedTokensMap(), tokens.getBinaryEncodedTokens());
+		var inflatedTokens = tokens.inflate();
 		var tabSize = model.getOptions().tabSize;
 		return Colorizer.colorizeLine(content, inflatedTokens, tabSize);
 	}

src/vs/editor/browser/widget/diffEditorWidget.ts

@@ -23,7 +23,7 @@ import {createLineParts} from 'vs/editor/common/viewLayout/viewLineParts';
 import {renderLine, RenderLineInput} from 'vs/editor/common/viewLayout/viewLineRenderer';
 import * as editorBrowser from 'vs/editor/browser/editorBrowser';
 import {CodeEditorWidget} from 'vs/editor/browser/widget/codeEditorWidget';
-import {ViewLineToken, ViewLineTokens} from 'vs/editor/common/viewModel/viewModel';
+import {ViewLineToken, ViewLineTokens} from 'vs/editor/common/core/viewLineToken';
 
 interface IEditorScrollEvent {
 	scrollLeft: number;

src/vs/editor/common/commonCodeEditor.ts

@@ -693,7 +693,7 @@ export abstract class CommonCodeEditor extends EventEmitter implements IActionPr
 				this.model.getOptions().tabSize,
 				this._configuration.editor.wrappingInfo.wrappingColumn,
 				this._configuration.editor.fontInfo.typicalFullwidthCharacterWidth / this._configuration.editor.fontInfo.typicalHalfwidthCharacterWidth,
-				editorCommon.wrappingIndentFromString(this._configuration.editor.wrappingIndent)
+				this._configuration.editor.wrappingIndent
 			);
 
 			this.viewModel = new ViewModel(

src/vs/editor/common/config/commonEditorConfig.ts

@@ -84,7 +84,7 @@ export class InternalEditorOptions implements editorCommon.IInternalEditorOption
 	fontLigatures:boolean;
 	hideCursorInOverviewRuler:boolean;
 	scrollBeyondLastLine:boolean;
-	wrappingIndent: string;
+	wrappingIndent: editorCommon.WrappingIndent;
 	wordWrapBreakBeforeCharacters: string;
 	wordWrapBreakAfterCharacters: string;
 	wordWrapBreakObtrusiveCharacters: string;
@@ -150,7 +150,7 @@ export class InternalEditorOptions implements editorCommon.IInternalEditorOption
 		this.fontLigatures = Boolean(input.fontLigatures);
 		this.hideCursorInOverviewRuler = Boolean(input.hideCursorInOverviewRuler);
 		this.scrollBeyondLastLine = Boolean(input.scrollBeyondLastLine);
-		this.wrappingIndent = String(input.wrappingIndent);
+		this.wrappingIndent = input.wrappingIndent;
 		this.wordWrapBreakBeforeCharacters = String(input.wordWrapBreakBeforeCharacters);
 		this.wordWrapBreakAfterCharacters = String(input.wordWrapBreakAfterCharacters);
 		this.wordWrapBreakObtrusiveCharacters = String(input.wordWrapBreakObtrusiveCharacters);
@@ -335,7 +335,7 @@ class InternalEditorOptionsHelper {
 			fontLigatures: toBoolean(opts.fontLigatures),
 			hideCursorInOverviewRuler: toBoolean(opts.hideCursorInOverviewRuler),
 			scrollBeyondLastLine: toBoolean(opts.scrollBeyondLastLine),
-			wrappingIndent: opts.wrappingIndent,
+			wrappingIndent: wrappingIndentFromString(opts.wrappingIndent),
 			wordWrapBreakBeforeCharacters: opts.wordWrapBreakBeforeCharacters,
 			wordWrapBreakAfterCharacters: opts.wordWrapBreakAfterCharacters,
 			wordWrapBreakObtrusiveCharacters: opts.wordWrapBreakObtrusiveCharacters,
@@ -536,6 +536,16 @@ function toSortedIntegerArray(source:any): number[] {
 	return r;
 }
 
+function wrappingIndentFromString(wrappingIndent:string): editorCommon.WrappingIndent {
+	if (wrappingIndent === 'indent') {
+		return editorCommon.WrappingIndent.Indent;
+	} else if (wrappingIndent === 'same') {
+		return editorCommon.WrappingIndent.Same;
+	} else {
+		return editorCommon.WrappingIndent.None;
+	}
+}
+
 function toIntegerWithDefault(source:any, defaultValue:number): number {
 	if (typeof source === 'undefined') {
 		return defaultValue;

src/vs/editor/common/controller/cursorMoveHelper.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 'use strict';
 
-import {IInternalIndentationOptions, IPosition, IEditorSelection} from 'vs/editor/common/editorCommon';
+import {IPosition, IEditorSelection} from 'vs/editor/common/editorCommon';
 import {Selection} from 'vs/editor/common/core/selection';
 
 export interface IMoveResult {
@@ -32,6 +32,20 @@ export interface ICursorMoveHelperModel {
 	getLineContent(lineNumber:number): string;
 }
 
+/**
+ * Internal indentation options (computed) for the editor.
+ */
+export interface IInternalIndentationOptions {
+	/**
+	 * Tab size in spaces. This is used for rendering and for editing.
+	 */
+	tabSize:number;
+	/**
+	 * Insert spaces instead of tabs when indenting or when auto-indenting.
+	 */
+	insertSpaces:boolean;
+}
+
 export interface IConfiguration {
 	getIndentationOptions(): IInternalIndentationOptions;
 }

src/vs/editor/common/controller/oneCursor.ts

@@ -239,20 +239,20 @@ export class OneCursor {
 	}
 
 	public adjustBracketDecorations(): void {
-		let bracketMatch: editorCommon.IMatchBracketResult = null;
+		let bracketMatch: [editorCommon.IEditorRange,editorCommon.IEditorRange] = null;
 		let selection = this.getSelection();
 		if (selection.isEmpty()) {
-			bracketMatch = this.model.matchBracket(this.position, /*inaccurateResultAcceptable*/true);
+			bracketMatch = this.model.matchBracket(this.position);
 		}
 
 		let newDecorations: editorCommon.IModelDeltaDecoration[] = [];
-		if (bracketMatch && bracketMatch.brackets) {
+		if (bracketMatch) {
 			let options: editorCommon.IModelDecorationOptions = {
 				stickiness: editorCommon.TrackedRangeStickiness.NeverGrowsWhenTypingAtEdges,
 				className: 'bracket-match'
 			};
-			newDecorations.push({ range: bracketMatch.brackets[0], options: options });
-			newDecorations.push({ range: bracketMatch.brackets[1], options: options });
+			newDecorations.push({ range: bracketMatch[0], options: options });
+			newDecorations.push({ range: bracketMatch[1], options: options });
 		}
 
 		this.bracketDecorations = this.model.deltaDecorations(this.bracketDecorations, newDecorations, this.editorId);

src/vs/editor/common/core/viewLineToken.ts

@@ -0,0 +1,85 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import {Arrays} from 'vs/editor/common/core/arrays';
+
+/**
+ * A token on a line.
+ */
+export class ViewLineToken {
+	public _viewLineTokenTrait: void;
+
+	public startIndex:number;
+	public type:string;
+
+	constructor(startIndex:number, type:string) {
+		this.startIndex = startIndex|0;// @perf
+		this.type = type.replace(/[^a-z0-9\-]/gi, ' ');
+	}
+
+	public equals(other:ViewLineToken): boolean {
+		return (
+			this.startIndex === other.startIndex
+			&& this.type === other.type
+		);
+	}
+
+	public static findIndexInSegmentsArray(arr:ViewLineToken[], desiredIndex: number): number {
+		return Arrays.findIndexInSegmentsArray(arr, desiredIndex);
+	}
+
+	public static equalsArray(a:ViewLineToken[], b:ViewLineToken[]): boolean {
+		let aLen = a.length;
+		let bLen = b.length;
+		if (aLen !== bLen) {
+			return false;
+		}
+		for (let i = 0; i < aLen; i++) {
+			if (!a[i].equals(b[i])) {
+				return false;
+			}
+		}
+		return true;
+	}
+}
+
+export class ViewLineTokens {
+	_viewLineTokensTrait: void;
+
+	private _lineTokens:ViewLineToken[];
+	private _fauxIndentLength:number;
+	private _textLength:number;
+
+	constructor(lineTokens:ViewLineToken[], fauxIndentLength:number, textLength:number) {
+		this._lineTokens = lineTokens;
+		this._fauxIndentLength = fauxIndentLength|0;
+		this._textLength = textLength|0;
+	}
+
+	public getTokens(): ViewLineToken[] {
+		return this._lineTokens;
+	}
+
+	public getFauxIndentLength(): number {
+		return this._fauxIndentLength;
+	}
+
+	public getTextLength(): number {
+		return this._textLength;
+	}
+
+	public equals(other:ViewLineTokens): boolean {
+		return (
+			this._fauxIndentLength === other._fauxIndentLength
+			&& this._textLength === other._textLength
+			&& ViewLineToken.equalsArray(this._lineTokens, other._lineTokens)
+		);
+	}
+
+	public findIndexOfOffset(offset:number): number {
+		return ViewLineToken.findIndexInSegmentsArray(this._lineTokens, offset);
+	}
+}

src/vs/editor/common/editorCommon.ts

@@ -13,7 +13,7 @@ import URI from 'vs/base/common/uri';
 import {TPromise} from 'vs/base/common/winjs.base';
 import {IInstantiationService, IConstructorSignature1, IConstructorSignature2} from 'vs/platform/instantiation/common/instantiation';
 import {ILineContext, IMode, IModeTransition, IToken} from 'vs/editor/common/modes';
-import {Arrays} from 'vs/editor/common/core/arrays';
+import {ViewLineToken} from 'vs/editor/common/core/viewLineToken';
 
 export type KeyCode = KeyCode;
 export type KeyMod = KeyMod;
@@ -257,16 +257,6 @@ export enum WrappingIndent {
 	Indent = 2
 }
 
-export function wrappingIndentFromString(wrappingIndent:string): WrappingIndent {
-	if (wrappingIndent === 'indent') {
-		return WrappingIndent.Indent;
-	} else if (wrappingIndent === 'same') {
-		return WrappingIndent.Same;
-	} else {
-		return WrappingIndent.None;
-	}
-}
-
 /**
  * Configuration options for the editor.
  */
@@ -563,20 +553,6 @@ export interface IDiffEditorOptions extends IEditorOptions {
 	originalEditable?: boolean;
 }
 
-/**
- * Internal indentation options (computed) for the editor.
- */
-export interface IInternalIndentationOptions {
-	/**
-	 * Tab size in spaces. This is used for rendering and for editing.
-	 */
-	tabSize:number;
-	/**
-	 * Insert spaces instead of tabs when indenting or when auto-indenting.
-	 */
-	insertSpaces:boolean;
-}
-
 export interface IInternalEditorScrollbarOptions {
 	arrowSize:number;
 	vertical:string;
@@ -622,7 +598,7 @@ export interface IInternalEditorOptions {
 	fontLigatures:boolean;
 	hideCursorInOverviewRuler:boolean;
 	scrollBeyondLastLine:boolean;
-	wrappingIndent: string;
+	wrappingIndent: WrappingIndent;
 	wordWrapBreakBeforeCharacters: string;
 	wordWrapBreakAfterCharacters: string;
 	wordWrapBreakObtrusiveCharacters: string;
@@ -1024,21 +1000,6 @@ export enum EndOfLineSequence {
 	CRLF = 1
 }
 
-/**
- * The result of a matchBracket operation.
- */
-export interface IMatchBracketResult {
-	/**
-	 * The two ranges describing matching brackets, or null
-	 */
-	brackets:IEditorRange[];
-	/**
-	 * Indicates that the bracket match result is not accurate because the search
-	 * hit some untokenized lines.
-	 */
-	isAccurate:boolean;
-}
-
 /**
  * A read-only line marker in the model.
  */
@@ -1162,7 +1123,6 @@ export interface IIdentifiedSingleEditOperation {
 	forceMoveMarkers: boolean;
 }
 
-
 /**
  * A callback that can compute the cursor state after applying a series of edit operations.
  */
@@ -1173,80 +1133,10 @@ export interface ICursorStateComputer {
 	(inverseEditOperations:IIdentifiedSingleEditOperation[]): IEditorSelection[];
 }
 
-/**
- * A token on a line.
- */
-export class LineToken {
-	public _lineTokenTrait: void;
-
-	public startIndex:number;
-	public type:string;
-
-	constructor(startIndex:number, type:string) {
-		this.startIndex = startIndex|0;// @perf
-		this.type = type;
-	}
-
-	public equals(other:LineToken): boolean {
-		return (
-			this.startIndex === other.startIndex
-			&& this.type === other.type
-		);
-	}
-
-	public static findIndexInSegmentsArray(arr:LineToken[], desiredIndex: number): number {
-		return Arrays.findIndexInSegmentsArray(arr, desiredIndex);
-	}
-
-	public static equalsArray(a:LineToken[], b:LineToken[]): boolean {
-		let aLen = a.length;
-		let bLen = b.length;
-		if (aLen !== bLen) {
-			return false;
-		}
-		for (let i = 0; i < aLen; i++) {
-			if (!a[i].equals(b[i])) {
-				return false;
-			}
-		}
-		return true;
-	}
-}
-
-export interface ITokensInflatorMap {
-	_inflate:string[];
-	_deflate: { [token:string]:number; };
-}
-
-export interface ILineTokensBinaryEncoding {
-	START_INDEX_MASK: number;
-	TYPE_MASK: number;
-	START_INDEX_OFFSET: number;
-	TYPE_OFFSET: number;
-
-	deflateArr(map:ITokensInflatorMap, tokens:IToken[]): number[];
-	inflate(map:ITokensInflatorMap, binaryEncodedToken:number): IToken;
-	getStartIndex(binaryEncodedToken:number): number;
-	getType(map:ITokensInflatorMap, binaryEncodedToken:number): string;
-	inflateArr(map:ITokensInflatorMap, binaryEncodedTokens:number[]): IToken[];
-	findIndexOfOffset(binaryEncodedTokens:number[], offset:number): number;
-	sliceAndInflate(map:ITokensInflatorMap, binaryEncodedTokens:number[], startOffset:number, endOffset:number, deltaStartIndex:number): IToken[];
-}
-
 /**
  * A list of tokens on a line.
  */
 export interface ILineTokens {
-	/**
-	 * Get the binary representation of tokens.
-	 */
-	getBinaryEncodedTokens(): number[];
-
-	/**
-	 * A map to help decoding the token type.
-	 */
-	getBinaryEncodedTokensMap(): ITokensInflatorMap;
-
 	getTokenCount(): number;
 	getTokenStartIndex(tokenIndex:number): number;
 	getTokenType(tokenIndex:number): string;
@@ -1268,6 +1158,10 @@ export interface ILineTokens {
 	 * @return The index of the token containing the offset.
 	 */
 	findIndexOfOffset(offset:number): number;
+
+	sliceAndInflate(startOffset:number, endOffset:number, deltaStartIndex:number): ViewLineToken[];
+
+	inflate(): ViewLineToken[];
 }
 
 /**
@@ -1538,14 +1432,6 @@ export interface ITokenizedModel extends ITextModel {
 	 */
 	getWordUntilPosition(position:IPosition): IWordAtPosition;
 
-	/**
-	 * Get the words on line `lineNumber`.
-	 * @param lineNumber The lineNumber
-	 * @param skipSyntaxTokens Ignore syntax tokens, as identified by the mode.
-	 * @return All the words on the line.
-	 */
-	getWords(lineNumber:number): IWordRange[];
-
 	/**
 	 * Returns an iterator that can be used to read
 	 * next and previous tokens from the provided position.
@@ -1581,7 +1467,7 @@ export interface ITokenizedModel extends ITextModel {
 	 * find the matching bracket of that bracket and return the ranges of both brackets.
 	 * @param position The position at which to look for a bracket.
 	 */
-	matchBracket(position:IPosition, inaccurateResultAcceptable?:boolean): IMatchBracketResult;
+	matchBracket(position:IPosition): [IEditorRange,IEditorRange];
 
 	/**
 	 * No mode supports allowed on this model because it is simply too large.

src/vs/editor/common/model/lineToken.ts

@@ -0,0 +1,47 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import {Arrays} from 'vs/editor/common/core/arrays';
+
+/**
+ * A token on a line.
+ */
+export class LineToken {
+	public _lineTokenTrait: void;
+
+	public startIndex:number;
+	public type:string;
+
+	constructor(startIndex:number, type:string) {
+		this.startIndex = startIndex|0;// @perf
+		this.type = type;
+	}
+
+	public equals(other:LineToken): boolean {
+		return (
+			this.startIndex === other.startIndex
+			&& this.type === other.type
+		);
+	}
+
+	public static findIndexInSegmentsArray(arr:LineToken[], desiredIndex: number): number {
+		return Arrays.findIndexInSegmentsArray(arr, desiredIndex);
+	}
+
+	public static equalsArray(a:LineToken[], b:LineToken[]): boolean {
+		let aLen = a.length;
+		let bLen = b.length;
+		if (aLen !== bLen) {
+			return false;
+		}
+		for (let i = 0; i < aLen; i++) {
+			if (!a[i].equals(b[i])) {
+				return false;
+			}
+		}
+		return true;
+	}
+}

src/vs/editor/common/model/modelLine.ts

@@ -5,10 +5,17 @@
 'use strict';
 
 import * as strings from 'vs/base/common/strings';
-import {ILineTokens, IReadOnlyLineMarker, ITokensInflatorMap, LineToken} from 'vs/editor/common/editorCommon';
+import {ILineTokens, IReadOnlyLineMarker} from 'vs/editor/common/editorCommon';
 import {IMode, IState} from 'vs/editor/common/modes';
-import * as TokensBinaryEncoding from 'vs/editor/common/model/tokensBinaryEncoding';
+import {TokensBinaryEncoding, TokensInflatorMap} from 'vs/editor/common/model/tokensBinaryEncoding';
 import {ModeTransition} from 'vs/editor/common/core/modeTransition';
+import {LineToken} from 'vs/editor/common/model/lineToken';
+import {ViewLineToken} from 'vs/editor/common/core/viewLineToken';
+
+const START_INDEX_MASK = TokensBinaryEncoding.START_INDEX_MASK;
+const TYPE_MASK = TokensBinaryEncoding.TYPE_MASK;
+const START_INDEX_OFFSET = TokensBinaryEncoding.START_INDEX_OFFSET;
+const TYPE_OFFSET = TokensBinaryEncoding.TYPE_OFFSET;
 
 export interface ILineEdit {
 	startColumn: number;
@@ -76,8 +83,8 @@ export class ModelLine {
 	public set isInvalid(value:boolean) { this._isInvalid = value; }
 
 	private _state:IState;
-	private _modeTransitions:ModeTransition[];
-	private _lineTokens:ILineTokens;
+	private _modeTransitions: ModeTransition[];
+	private _lineTokens: LineTokens;
 	private _markers:ILineMarker[];
 
 	constructor(lineNumber:number, text:string) {
@@ -116,12 +123,12 @@ export class ModelLine {
 
 	// --- BEGIN TOKENS
 
-	public setTokens(map: ITokensInflatorMap, tokens: LineToken[], topLevelMode:IMode, modeTransitions:ModeTransition[]): void {
+	public setTokens(map: TokensInflatorMap, tokens: LineToken[], topLevelMode:IMode, modeTransitions:ModeTransition[]): void {
 		this._lineTokens = toLineTokensFromInflated(map, tokens, this._text.length);
 		this._modeTransitions = toModeTransitions(topLevelMode, modeTransitions);
 	}
 
-	private _setLineTokensFromDeflated(map:ITokensInflatorMap, tokens:number[]): void {
+	private _setLineTokensFromDeflated(map:TokensInflatorMap, tokens:number[]): void {
 		this._lineTokens = toLineTokensFromDeflated(map, tokens, this._text.length);
 	}
 
@@ -145,7 +152,6 @@ export class ModelLine {
 
 		var lineTokens = this._lineTokens;
 
-		let BIN = TokensBinaryEncoding;
 		let tokens = lineTokens.getBinaryEncodedTokens();
 		let tokensLength = tokens.length;
 		let tokensIndex = 0;
@@ -161,14 +167,14 @@ export class ModelLine {
 
 				if (currentTokenStartIndex > 0 && delta !== 0) {
 					// adjust token's `startIndex` by `delta`
-					let deflatedType = (tokens[tokensIndex] / BIN.TYPE_OFFSET) & BIN.TYPE_MASK;
+					let deflatedType = (tokens[tokensIndex] / TYPE_OFFSET) & TYPE_MASK;
 					let newStartIndex = Math.max(minimumAllowedIndex, currentTokenStartIndex + delta);
-					let newToken = deflatedType * BIN.TYPE_OFFSET + newStartIndex * BIN.START_INDEX_OFFSET;
+					let newToken = deflatedType * TYPE_OFFSET + newStartIndex * START_INDEX_OFFSET;
 
 					if (delta < 0) {
 						// pop all previous tokens that have become `collapsed`
 						while (tokensIndex > 0) {
-							let prevTokenStartIndex = (tokens[tokensIndex - 1] / BIN.START_INDEX_OFFSET) & BIN.START_INDEX_MASK;
+							let prevTokenStartIndex = (tokens[tokensIndex - 1] / START_INDEX_OFFSET) & START_INDEX_MASK;
 							if (prevTokenStartIndex >= newStartIndex) {
 								// Token at `tokensIndex` - 1 is now `collapsed` => pop it
 								tokens.splice(tokensIndex - 1, 1);
@@ -185,7 +191,7 @@ export class ModelLine {
 				tokensIndex++;
 
 				if (tokensIndex < tokensLength) {
-					currentTokenStartIndex = (tokens[tokensIndex] / BIN.START_INDEX_OFFSET) & BIN.START_INDEX_MASK;
+					currentTokenStartIndex = (tokens[tokensIndex] / START_INDEX_OFFSET) & START_INDEX_MASK;
 				}
 			}
 			// console.log('after call: tokensIndex: ' + tokensIndex + ': ' + String(this.getTokens()));
@@ -205,14 +211,13 @@ export class ModelLine {
 		this._text = text;
 
 		if (this._lineTokens) {
-			let BIN = TokensBinaryEncoding,
-				map = this._lineTokens.getBinaryEncodedTokensMap(),
+			let map = this._lineTokens.getBinaryEncodedTokensMap(),
 				tokens = this._lineTokens.getBinaryEncodedTokens(),
 				lineTextLength = this._text.length;
 
 			// Remove overflowing tokens
 			while (tokens.length > 0) {
-				let lastTokenStartIndex = (tokens[tokens.length - 1] / BIN.START_INDEX_OFFSET) & BIN.START_INDEX_MASK;
+				let lastTokenStartIndex = (tokens[tokens.length - 1] / START_INDEX_OFFSET) & START_INDEX_MASK;
 				if (lastTokenStartIndex < lineTextLength) {
 					// Valid token
 					break;
@@ -456,15 +461,13 @@ export class ModelLine {
 
 			// Adjust other tokens
 			if (thisTextLength > 0) {
-				let BIN = TokensBinaryEncoding;
-
 				for (let i = 0, len = otherTokens.length; i < len; i++) {
 					let token = otherTokens[i];
 
-					let deflatedStartIndex = (token / BIN.START_INDEX_OFFSET) & BIN.START_INDEX_MASK;
-					let deflatedType = (token / BIN.TYPE_OFFSET) & BIN.TYPE_MASK;
+					let deflatedStartIndex = (token / START_INDEX_OFFSET) & START_INDEX_MASK;
+					let deflatedType = (token / TYPE_OFFSET) & TYPE_MASK;
 					let newStartIndex = deflatedStartIndex + thisTextLength;
-					let newToken = deflatedType * BIN.TYPE_OFFSET + newStartIndex * BIN.START_INDEX_OFFSET;
+					let newToken = deflatedType * TYPE_OFFSET + newStartIndex * START_INDEX_OFFSET;
 
 					otherTokens[i] = newToken;
 				}
@@ -625,7 +628,7 @@ export class ModelLine {
 	}
 }
 
-function toLineTokensFromInflated(map:ITokensInflatorMap, tokens:LineToken[], textLength:number): ILineTokens {
+function toLineTokensFromInflated(map:TokensInflatorMap, tokens:LineToken[], textLength:number): LineTokens {
 	if (textLength === 0) {
 		return null;
 	}
@@ -642,7 +645,7 @@ function toLineTokensFromInflated(map:ITokensInflatorMap, tokens:LineToken[], te
 	return new LineTokens(map, deflated);
 }
 
-function toLineTokensFromDeflated(map:ITokensInflatorMap, tokens:number[], textLength:number): ILineTokens {
+function toLineTokensFromDeflated(map:TokensInflatorMap, tokens:number[], textLength:number): LineTokens {
 	if (textLength === 0) {
 		return null;
 	}
@@ -657,21 +660,17 @@ function toLineTokensFromDeflated(map:ITokensInflatorMap, tokens:number[], textL
 	return new LineTokens(map, tokens);
 }
 
-var getStartIndex = TokensBinaryEncoding.getStartIndex;
-var getType = TokensBinaryEncoding.getType;
-var findIndexOfOffset = TokensBinaryEncoding.findIndexOfOffset;
-
 export class LineTokens implements ILineTokens {
 
-	private map:ITokensInflatorMap;
+	private map:TokensInflatorMap;
 	private _tokens:number[];
 
-	constructor(map:ITokensInflatorMap, tokens:number[]) {
+	constructor(map:TokensInflatorMap, tokens:number[]) {
 		this.map = map;
 		this._tokens = tokens;
 	}
 
-	public getBinaryEncodedTokensMap(): ITokensInflatorMap {
+	public getBinaryEncodedTokensMap(): TokensInflatorMap {
 		return this.map;
 	}
 
@@ -684,41 +683,57 @@ export class LineTokens implements ILineTokens {
 	}
 
 	public getTokenStartIndex(tokenIndex:number): number {
-		return getStartIndex(this._tokens[tokenIndex]);
+		return TokensBinaryEncoding.getStartIndex(this._tokens[tokenIndex]);
 	}
 
 	public getTokenType(tokenIndex:number): string {
-		return getType(this.map, this._tokens[tokenIndex]);
+		return TokensBinaryEncoding.getType(this.map, this._tokens[tokenIndex]);
 	}
 
 	public getTokenEndIndex(tokenIndex:number, textLength:number): number {
 		if (tokenIndex + 1 < this._tokens.length) {
-			return getStartIndex(this._tokens[tokenIndex + 1]);
+			return TokensBinaryEncoding.getStartIndex(this._tokens[tokenIndex + 1]);
 		}
 		return textLength;
 	}
 
 	public equals(other:ILineTokens): boolean {
-		return this === other;
+		if (other instanceof LineTokens) {
+			if (this.map !== other.map) {
+				return false;
+			}
+			if (this._tokens.length !== other._tokens.length) {
+				return false;
+			}
+			for (let i = 0, len = this._tokens.length; i < len; i++) {
+				if (this._tokens[i] !== other._tokens[i]) {
+					return false;
+				}
+			}
+			return true;
+		}
+		if (!(other instanceof LineTokens)) {
+			return false;
+		}
 	}
 
 	public findIndexOfOffset(offset:number): number {
-		return findIndexOfOffset(this._tokens, offset);
+		return TokensBinaryEncoding.findIndexOfOffset(this._tokens, offset);
 	}
+
+	public inflate(): ViewLineToken[] {
+		return TokensBinaryEncoding.inflateArr(this.map, this._tokens);
+	}
+
+	public sliceAndInflate(startOffset:number, endOffset:number, deltaStartIndex:number): ViewLineToken[] {
+		return TokensBinaryEncoding.sliceAndInflate(this.map, this._tokens, startOffset, endOffset, deltaStartIndex);
+	}
+
 }
 
 class EmptyLineTokens implements ILineTokens {
 
 	public static INSTANCE = new EmptyLineTokens();
-	private static TOKENS = <number[]>[];
-
-	public getBinaryEncodedTokens(): number[] {
-		return EmptyLineTokens.TOKENS;
-	}
-
-	public getBinaryEncodedTokensMap(): ITokensInflatorMap {
-		return null;
-	}
 
 	public getTokenCount(): number {
 		return 0;
@@ -743,20 +758,19 @@ class EmptyLineTokens implements ILineTokens {
 	public findIndexOfOffset(offset:number): number {
 		return 0;
 	}
+
+	public inflate(): ViewLineToken[] {
+		return [];
+	}
+
+	public sliceAndInflate(startOffset:number, endOffset:number, deltaStartIndex:number): ViewLineToken[] {
+		return [];
+	}
 }
 
 export class DefaultLineTokens implements ILineTokens {
 
 	public static INSTANCE = new DefaultLineTokens();
-	private static TOKENS = <number[]> [0];
-
-	public getBinaryEncodedTokensMap(): ITokensInflatorMap {
-		return null;
-	}
-
-	public getBinaryEncodedTokens(): number[] {
-		return DefaultLineTokens.TOKENS;
-	}
 
 	public getTokenCount(): number {
 		return 1;
@@ -782,6 +796,14 @@ export class DefaultLineTokens implements ILineTokens {
 		return 0;
 	}
 
+	public inflate(): ViewLineToken[] {
+		return [new ViewLineToken(0, '')];
+	}
+
+	public sliceAndInflate(startOffset:number, endOffset:number, deltaStartIndex:number): ViewLineToken[] {
+		return [new ViewLineToken(0, '')];
+	}
+
 }
 
 function toModeTransitions(topLevelMode:IMode, modeTransitions:ModeTransition[]): ModeTransition[] {

src/vs/editor/common/model/textModelWithTokens.ts

@@ -22,22 +22,9 @@ import {ILineContext, ILineTokens, IToken, IModeTransition, IMode, IState} from
 import {NullMode, NullState, nullTokenize} from 'vs/editor/common/modes/nullMode';
 import {ignoreBracketsInToken} from 'vs/editor/common/modes/supports';
 import {BracketsUtils} from 'vs/editor/common/modes/supports/richEditBrackets';
-import * as TokensBinaryEncoding from 'vs/editor/common/model/tokensBinaryEncoding';
 import {ModeTransition} from 'vs/editor/common/core/modeTransition';
-
-export class TokensInflatorMap implements editorCommon.ITokensInflatorMap {
-
-	public _inflate:string[];
-
-	public _deflate: {
-		[token:string]:number;
-	};
-
-	constructor() {
-		this._inflate = [ '' ];
-		this._deflate = { '': 0 };
-	}
-}
+import {LineToken} from 'vs/editor/common/model/lineToken';
+import {TokensInflatorMap} from 'vs/editor/common/model/tokensBinaryEncoding';
 
 class ModeToModelBinder implements IDisposable {
 
@@ -190,7 +177,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 	private _mode: IMode;
 	private _modeListener: IDisposable;
 	private _modeToModelBinder:ModeToModelBinder;
-	private _tokensInflatorMap:editorCommon.ITokensInflatorMap;
+	private _tokensInflatorMap:TokensInflatorMap;
 	private _stopLineTokenizationAfter:number;
 
 	private _invalidLineStartIndex:number;
@@ -535,16 +522,16 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 		}
 	}
 
-	private static _toLineTokens(tokens:IToken[]): editorCommon.LineToken[] {
+	private static _toLineTokens(tokens:IToken[]): LineToken[] {
 		if (!tokens || tokens.length === 0) {
 			return [];
 		}
-		if (tokens[0] instanceof editorCommon.LineToken) {
-			return <editorCommon.LineToken[]>tokens;
+		if (tokens[0] instanceof LineToken) {
+			return <LineToken[]>tokens;
 		}
-		let result:editorCommon.LineToken[] = [];
+		let result:LineToken[] = [];
 		for (let i = 0, len = tokens.length; i < len; i++) {
-			result[i] = new editorCommon.LineToken(tokens[i].startIndex, tokens[i].type);
+			result[i] = new LineToken(tokens[i].startIndex, tokens[i].type);
 		}
 		return result;
 	}
@@ -563,7 +550,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 		return result;
 	}
 
-	private _updateLineTokens(lineIndex:number, map:editorCommon.ITokensInflatorMap, topLevelMode:IMode, r:ILineTokens): void {
+	private _updateLineTokens(lineIndex:number, map:TokensInflatorMap, topLevelMode:IMode, r:ILineTokens): void {
 		this._lines[lineIndex].setTokens(map, TextModelWithTokens._toLineTokens(r.tokens), topLevelMode, TextModelWithTokens._toModeTransitions(r.modeTransitions));
 	}
 
@@ -820,14 +807,6 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 		};
 	}
 
-	public getWords(lineNumber:number): editorCommon.IWordRange[] {
-		if (lineNumber < 1 || lineNumber > this.getLineCount()) {
-			throw new Error('Illegal value ' + lineNumber + ' for `lineNumber`');
-		}
-
-		return WordHelper.getWords(this, this.validateLineNumber(lineNumber));
-	}
-
 	public tokenIterator(position:editorCommon.IPosition, callback:(it:TokenIterator)=>any): any {
 		var iter = new TokenIterator(this, this.validatePosition(position));
 		var result = callback(iter);
@@ -855,19 +834,17 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 		return this._findMatchingBracketUp(data, position);
 	}
 
-	public matchBracket(position:editorCommon.IPosition, inaccurateResultAcceptable:boolean = false): editorCommon.IMatchBracketResult {
+	public matchBracket(position:editorCommon.IPosition): [editorCommon.IEditorRange,editorCommon.IEditorRange] {
 		return this._matchBracket(this.validatePosition(position));
 	}
 
-	private _matchBracket(position:editorCommon.IEditorPosition): editorCommon.IMatchBracketResult {
-		let tokensMap = this._tokensInflatorMap;
+	private _matchBracket(position:editorCommon.IEditorPosition): [editorCommon.IEditorRange,editorCommon.IEditorRange] {
 		let lineNumber = position.lineNumber;
 		let lineText = this._lines[lineNumber - 1].text;
 
 		let lineTokens = this._lines[lineNumber - 1].getTokens();
-		let tokens = lineTokens.getBinaryEncodedTokens();
 		let currentTokenIndex = lineTokens.findIndexOfOffset(position.column - 1);
-		let currentTokenStart = getStartIndex(tokens[currentTokenIndex]);
+		let currentTokenStart = lineTokens.getTokenStartIndex(currentTokenIndex);
 
 		let modeTransitions = this._lines[lineNumber - 1].getModeTransitions(this._mode);
 		let currentModeIndex = ModeTransition.findIndexInSegmentsArray(modeTransitions, position.column - 1);
@@ -877,11 +854,11 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 		// If position is in between two tokens, try first looking in the previous token
 		if (currentTokenIndex > 0 && currentTokenStart === position.column - 1) {
 			let prevTokenIndex = currentTokenIndex - 1;
-			let prevTokenType = getType(tokensMap, tokens[prevTokenIndex]);
+			let prevTokenType = lineTokens.getTokenType(prevTokenIndex);
 
 			// check that previous token is not to be ignored
 			if (!ignoreBracketsInToken(prevTokenType)) {
-				let prevTokenStart = getStartIndex(tokens[prevTokenIndex]);
+				let prevTokenStart = lineTokens.getTokenStartIndex(prevTokenIndex);
 
 				let prevMode = currentMode;
 				let prevModeBrackets = currentModeBrackets;
@@ -912,14 +889,14 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 		}
 
 		// check that the token is not to be ignored
-		if (!ignoreBracketsInToken(getType(tokensMap, tokens[currentTokenIndex]))) {
+		if (!ignoreBracketsInToken(lineTokens.getTokenType(currentTokenIndex))) {
 
 			if (currentModeBrackets) {
 				// limit search to not go before `maxBracketLength`
 				currentTokenStart = Math.max(currentTokenStart, position.column - 1 - currentModeBrackets.maxBracketLength);
 
 				// limit search to not go after `maxBracketLength`
-				let currentTokenEnd = (currentTokenIndex + 1 < tokens.length ? getStartIndex(tokens[currentTokenIndex + 1]) : lineText.length);
+				let currentTokenEnd = lineTokens.getTokenEndIndex(currentTokenIndex, lineText.length);
 				currentTokenEnd = Math.min(currentTokenEnd, position.column - 1 + currentModeBrackets.maxBracketLength);
 
 				// it might still be the case that [currentTokenStart -> currentTokenEnd] contains multiple brackets
@@ -946,28 +923,19 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 			}
 		}
 
-		return {
-			brackets: null,
-			isAccurate: true
-		};
+		return null;
 	}
 
-	private _matchFoundBracket(foundBracket:Range, data:editorCommon.IRichEditBracket, isOpen:boolean): editorCommon.IMatchBracketResult {
+	private _matchFoundBracket(foundBracket:Range, data:editorCommon.IRichEditBracket, isOpen:boolean): [editorCommon.IEditorRange,editorCommon.IEditorRange] {
 		if (isOpen) {
 			let matched = this._findMatchingBracketDown(data, foundBracket.getEndPosition());
 			if (matched) {
-				return {
-					brackets: [foundBracket, matched],
-					isAccurate: true
-				};
+				return [foundBracket, matched];
 			}
 		} else {
 			let matched = this._findMatchingBracketUp(data, foundBracket.getStartPosition());
 			if (matched) {
-				return {
-					brackets: [foundBracket, matched],
-					isAccurate: true
-				};
+				return [foundBracket, matched];
 			}
 		}
 
@@ -978,20 +946,18 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 		// console.log('_findMatchingBracketUp: ', 'bracket: ', JSON.stringify(bracket), 'startPosition: ', String(position));
 
 		let modeId = bracket.modeId;
-		let tokensMap = this._tokensInflatorMap;
 		let reversedBracketRegex = bracket.reversedRegex;
 		let count = -1;
 
 		for (let lineNumber = position.lineNumber; lineNumber >= 1; lineNumber--) {
 			let lineTokens = this._lines[lineNumber - 1].getTokens();
 			let lineText = this._lines[lineNumber - 1].text;
-			let tokens = lineTokens.getBinaryEncodedTokens();
 			let modeTransitions = this._lines[lineNumber - 1].getModeTransitions(this._mode);
 			let currentModeIndex = modeTransitions.length - 1;
 			let currentModeStart = modeTransitions[currentModeIndex].startIndex;
 			let currentModeId = modeTransitions[currentModeIndex].mode.getId();
 
-			let tokensLength = tokens.length - 1;
+			let tokensLength = lineTokens.getTokenCount() - 1;
 			let currentTokenEnd = lineText.length;
 			if (lineNumber === position.lineNumber) {
 				tokensLength = lineTokens.findIndexOfOffset(position.column - 1);
@@ -1003,9 +969,8 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 			}
 
 			for (let tokenIndex = tokensLength; tokenIndex >= 0; tokenIndex--) {
-				let currentToken = tokens[tokenIndex];
-				let currentTokenType = getType(tokensMap, currentToken);
-				let currentTokenStart = getStartIndex(currentToken);
+				let currentTokenType = lineTokens.getTokenType(tokenIndex);
+				let currentTokenStart = lineTokens.getTokenStartIndex(tokenIndex);
 
 				if (currentTokenStart < currentModeStart) {
 					currentModeIndex--;
@@ -1048,21 +1013,19 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 		// console.log('_findMatchingBracketDown: ', 'bracket: ', JSON.stringify(bracket), 'startPosition: ', String(position));
 
 		let modeId = bracket.modeId;
-		let tokensMap = this._tokensInflatorMap;
 		let bracketRegex = bracket.forwardRegex;
 		let count = 1;
 
 		for (let lineNumber = position.lineNumber, lineCount = this.getLineCount(); lineNumber <= lineCount; lineNumber++) {
 			let lineTokens = this._lines[lineNumber - 1].getTokens();
 			let lineText = this._lines[lineNumber - 1].text;
-			let tokens = lineTokens.getBinaryEncodedTokens();
 			let modeTransitions = this._lines[lineNumber - 1].getModeTransitions(this._mode);
 			let currentModeIndex = 0;
 			let nextModeStart = (currentModeIndex + 1 < modeTransitions.length ? modeTransitions[currentModeIndex + 1].startIndex : lineText.length + 1);
 			let currentModeId = modeTransitions[currentModeIndex].mode.getId();
 
 			let startTokenIndex = 0;
-			let currentTokenStart = getStartIndex(startTokenIndex);
+			let currentTokenStart = lineTokens.getTokenStartIndex(startTokenIndex);
 			if (lineNumber === position.lineNumber) {
 				startTokenIndex = lineTokens.findIndexOfOffset(position.column - 1);
 				currentTokenStart = Math.max(currentTokenStart, position.column - 1);
@@ -1072,10 +1035,9 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 				currentModeId = modeTransitions[currentModeIndex].mode.getId();
 			}
 
-			for (let tokenIndex = startTokenIndex, tokensLength = tokens.length; tokenIndex < tokensLength; tokenIndex++) {
-				let currentToken = tokens[tokenIndex];
-				let currentTokenType = getType(tokensMap, currentToken);
-				let currentTokenEnd = tokenIndex + 1 < tokensLength ? getStartIndex(tokens[tokenIndex + 1]) : lineText.length;
+			for (let tokenIndex = startTokenIndex, tokensLength = lineTokens.getTokenCount(); tokenIndex < tokensLength; tokenIndex++) {
+				let currentTokenType = lineTokens.getTokenType(tokenIndex);
+				let currentTokenEnd = lineTokens.getTokenEndIndex(tokenIndex, lineText.length);
 
 				if (currentTokenStart >= nextModeStart) {
 					currentModeIndex++;
@@ -1116,15 +1078,13 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 	public findPrevBracket(_position:editorCommon.IPosition): editorCommon.IFoundBracket {
 		let position = this.validatePosition(_position);
 
-		let tokensMap = this._tokensInflatorMap;
 		let reversedBracketRegex = /[\(\)\[\]\{\}]/; // TODO@Alex: use mode's brackets
 
 		for (let lineNumber = position.lineNumber; lineNumber >= 1; lineNumber--) {
 			let lineTokens = this._lines[lineNumber - 1].getTokens();
 			let lineText = this._lines[lineNumber - 1].text;
-			let tokens = lineTokens.getBinaryEncodedTokens();
 
-			let tokensLength = tokens.length - 1;
+			let tokensLength = lineTokens.getTokenCount() - 1;
 			let currentTokenEnd = lineText.length;
 			if (lineNumber === position.lineNumber) {
 				tokensLength = lineTokens.findIndexOfOffset(position.column - 1);
@@ -1132,9 +1092,8 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 			}
 
 			for (let tokenIndex = tokensLength; tokenIndex >= 0; tokenIndex--) {
-				let currentToken = tokens[tokenIndex];
-				let currentTokenType = getType(tokensMap, currentToken);
-				let currentTokenStart = getStartIndex(currentToken);
+				let currentTokenType = lineTokens.getTokenType(tokenIndex);
+				let currentTokenStart = lineTokens.getTokenStartIndex(tokenIndex);
 
 				if (!ignoreBracketsInToken(currentTokenType)) {
 					let r = BracketsUtils.findPrevBracketInToken(reversedBracketRegex, lineNumber, lineText, currentTokenStart, currentTokenEnd);
@@ -1153,25 +1112,22 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 	public findNextBracket(_position:editorCommon.IPosition): editorCommon.IFoundBracket {
 		let position = this.validatePosition(_position);
 
-		let tokensMap = this._tokensInflatorMap;
 		let bracketRegex = /[\(\)\[\]\{\}]/; // TODO@Alex: use mode's brackets
 
 		for (let lineNumber = position.lineNumber, lineCount = this.getLineCount(); lineNumber <= lineCount; lineNumber++) {
 			let lineTokens = this._lines[lineNumber - 1].getTokens();
 			let lineText = this._lines[lineNumber - 1].text;
-			let tokens = lineTokens.getBinaryEncodedTokens();
 
 			let startTokenIndex = 0;
-			let currentTokenStart = getStartIndex(startTokenIndex);
+			let currentTokenStart = lineTokens.getTokenStartIndex(startTokenIndex);
 			if (lineNumber === position.lineNumber) {
 				startTokenIndex = lineTokens.findIndexOfOffset(position.column - 1);
 				currentTokenStart = Math.max(currentTokenStart, position.column - 1);
 			}
 
-			for (let tokenIndex = startTokenIndex, tokensLength = tokens.length; tokenIndex < tokensLength; tokenIndex++) {
-				let currentToken = tokens[tokenIndex];
-				let currentTokenType = getType(tokensMap, currentToken);
-				let currentTokenEnd = tokenIndex + 1 < tokensLength ? getStartIndex(tokens[tokenIndex + 1]) : lineText.length;
+			for (let tokenIndex = startTokenIndex, tokensLength = lineTokens.getTokenCount(); tokenIndex < tokensLength; tokenIndex++) {
+				let currentTokenType = lineTokens.getTokenType(tokenIndex);
+				let currentTokenEnd = lineTokens.getTokenEndIndex(tokenIndex, lineText.length);
 
 				if (!ignoreBracketsInToken(currentTokenType)) {
 					let r = BracketsUtils.findNextBracketInToken(bracketRegex, lineNumber, lineText, currentTokenStart, currentTokenEnd);
@@ -1206,6 +1162,3 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 		return null;
 	}
 }
-
-var getType = TokensBinaryEncoding.getType;
-var getStartIndex = TokensBinaryEncoding.getStartIndex;

src/vs/editor/common/model/textModelWithTokensHelpers.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 'use strict';
 
-import {ILineTokens, IPosition, IWordAtPosition, IWordRange} from 'vs/editor/common/editorCommon';
+import {IPosition, IWordAtPosition, IWordRange} from 'vs/editor/common/editorCommon';
 import {IMode, IModeTransition} from 'vs/editor/common/modes';
 import {NullMode} from 'vs/editor/common/modes/nullMode';
 import {ModeTransition} from 'vs/editor/common/core/modeTransition';
@@ -15,15 +15,9 @@ export interface ITextSource {
 
 	getLineContent(lineNumber:number): string;
 
-	getLineCount(): number;
-
 	getMode(): IMode;
 
-	getModeAtPosition(lineNumber:number, column:number): IMode;
-
 	_getLineModeTransitions(lineNumber:number): ModeTransition[];
-
-	getLineTokens(lineNumber:number, inaccurateTokensAcceptable:boolean): ILineTokens;
 }
 
 export interface INonWordTokenMap {
@@ -63,60 +57,6 @@ export class WordHelper {
 		return WordHelper.ensureValidWordDefinition(WordHelper._safeGetWordDefinition(mode));
 	}
 
-	public static getWords(textSource:ITextSource, lineNumber:number): IWordRange[] {
-		if (!textSource._lineIsTokenized(lineNumber)) {
-			return WordHelper._getWordsInText(textSource.getLineContent(lineNumber), WordHelper.massageWordDefinitionOf(textSource.getMode()));
-		}
-
-		var r: IWordRange[] = [],
-			txt = textSource.getLineContent(lineNumber);
-
-		if (txt.length > 0) {
-
-			var modeTransitions = textSource._getLineModeTransitions(lineNumber),
-				i:number,
-				len:number,
-				k:number,
-				lenK:number,
-				currentModeStartIndex: number,
-				currentModeEndIndex: number,
-				currentWordDefinition:RegExp,
-				currentModeText: string,
-				words: RegExpMatchArray,
-				startWord: number,
-				endWord: number,
-				word: string;
-
-			// Go through all the modes
-			for (i = 0, currentModeStartIndex = 0, len = modeTransitions.length; i < len; i++) {
-				currentWordDefinition = WordHelper.massageWordDefinitionOf(modeTransitions[i].mode);
-				currentModeStartIndex = modeTransitions[i].startIndex;
-				currentModeEndIndex = (i + 1 < len ? modeTransitions[i + 1].startIndex : txt.length);
-				currentModeText = txt.substring(currentModeStartIndex, currentModeEndIndex);
-				words = currentModeText.match(currentWordDefinition);
-
-				if (!words) {
-					continue;
-				}
-
-				endWord = 0;
-				for (k = 0, lenK = words.length; k < lenK; k++) {
-					word = words[k];
-					if (word.length > 0) {
-						startWord = currentModeText.indexOf(word, endWord);
-						endWord = startWord + word.length;
-
-						r.push({
-							start: currentModeStartIndex + startWord,
-							end: currentModeStartIndex + endWord
-						});
-					}
-				}
-			}
-		}
-
-		return r;
-	}
 
 	static _getWordsInText(text:string, wordDefinition:RegExp): IWordRange[] {
 		var words = text.match(wordDefinition) || [],

src/vs/editor/common/model/tokenIterator.ts

@@ -5,10 +5,6 @@
 'use strict';
 
 import * as editorCommon from 'vs/editor/common/editorCommon';
-import * as TokensBinaryEncoding from 'vs/editor/common/model/tokensBinaryEncoding';
-
-var getStartIndex = TokensBinaryEncoding.getStartIndex;
-var inflate = TokensBinaryEncoding.inflate;
 
 export class TokenIterator implements editorCommon.ITokenIterator {
 
@@ -16,8 +12,6 @@ export class TokenIterator implements editorCommon.ITokenIterator {
 	private _currentLineNumber:number;
 	private _currentTokenIndex:number;
 	private _currentLineTokens:editorCommon.ILineTokens;
-	private _currentTokens:number[];
-	private _map:editorCommon.ITokensInflatorMap;
 	private _next:editorCommon.ITokenInfo;
 	private _prev:editorCommon.ITokenInfo;
 
@@ -32,27 +26,28 @@ export class TokenIterator implements editorCommon.ITokenIterator {
 		// start with a position to next/prev run
 		var columnIndex = position.column - 1, tokenEndIndex = Number.MAX_VALUE;
 
-		for (var i = this._currentTokens.length - 1; i >= 0; i--) {
-			if (getStartIndex(this._currentTokens[i]) <= columnIndex && columnIndex <= tokenEndIndex) {
+		for (var i = this._currentLineTokens.getTokenCount() - 1; i >= 0; i--) {
+			let tokenStartIndex = this._currentLineTokens.getTokenStartIndex(i);
+
+			if (tokenStartIndex <= columnIndex && columnIndex <= tokenEndIndex) {
+
 				this._currentTokenIndex = i;
 				this._next = this._current();
 				this._prev = this._current();
 				break;
 			}
-			tokenEndIndex = getStartIndex(this._currentTokens[i]);
+			tokenEndIndex = tokenStartIndex;
 		}
 	}
 
 	private _readLineTokens(lineNumber:number): void {
 		this._currentLineTokens = this._model.getLineTokens(lineNumber, false);
-		this._currentTokens = this._currentLineTokens.getBinaryEncodedTokens();
-		this._map = this._currentLineTokens.getBinaryEncodedTokensMap();
 	}
 
 	private _advanceNext() {
 		this._prev = this._next;
 		this._next = null;
-		if (this._currentTokenIndex + 1 < this._currentTokens.length) {
+		if (this._currentTokenIndex + 1 < this._currentLineTokens.getTokenCount()) {
 			// There are still tokens on current line
 			this._currentTokenIndex++;
 			this._next = this._current();
@@ -62,7 +57,7 @@ export class TokenIterator implements editorCommon.ITokenIterator {
 			while (this._currentLineNumber + 1 <= this._model.getLineCount()) {
 				this._currentLineNumber++;
 				this._readLineTokens(this._currentLineNumber);
-				if (this._currentTokens.length > 0) {
+				if (this._currentLineTokens.getTokenCount() > 0) {
 					this._currentTokenIndex = 0;
 					this._next = this._current();
 					break;
@@ -71,7 +66,7 @@ export class TokenIterator implements editorCommon.ITokenIterator {
 			if (this._next === null) {
 				// prepare of a previous run
 				this._readLineTokens(this._currentLineNumber);
-				this._currentTokenIndex = this._currentTokens.length;
+				this._currentTokenIndex = this._currentLineTokens.getTokenCount();
 				this._advancePrev();
 				this._next = null;
 			}
@@ -91,8 +86,8 @@ export class TokenIterator implements editorCommon.ITokenIterator {
 			while (this._currentLineNumber > 1) {
 				this._currentLineNumber--;
 				this._readLineTokens(this._currentLineNumber);
-				if (this._currentTokens.length > 0) {
-					this._currentTokenIndex = this._currentTokens.length - 1;
+				if (this._currentLineTokens.getTokenCount() > 0) {
+					this._currentTokenIndex = this._currentLineTokens.getTokenCount() - 1;
 					this._prev = this._current();
 					break;
 				}
@@ -101,11 +96,18 @@ export class TokenIterator implements editorCommon.ITokenIterator {
 	}
 
 	private _current(): editorCommon.ITokenInfo {
+		let startIndex = this._currentLineTokens.getTokenStartIndex(this._currentTokenIndex);
+		let type = this._currentLineTokens.getTokenType(this._currentTokenIndex);
+		let endIndex = this._currentLineTokens.getTokenEndIndex(this._currentTokenIndex, this._model.getLineContent(this._currentLineNumber).length);
+
 		return {
-			token: inflate(this._map, this._currentTokens[this._currentTokenIndex]),
+			token: {
+				startIndex: startIndex,
+				type: type
+			},
 			lineNumber: this._currentLineNumber,
-			startColumn: getStartIndex(this._currentTokens[this._currentTokenIndex]) + 1,
-			endColumn: this._currentTokenIndex + 1 < this._currentTokens.length ? getStartIndex(this._currentTokens[this._currentTokenIndex + 1]) + 1 : this._model.getLineContent(this._currentLineNumber).length + 1
+			startColumn: startIndex + 1,
+			endColumn: endIndex + 1
 		};
 	}
 

src/vs/editor/common/model/tokensBinaryEncoding.ts

@@ -6,183 +6,192 @@
 
 import {onUnexpectedError} from 'vs/base/common/errors';
 import * as strings from 'vs/base/common/strings';
-import {ITokensInflatorMap, LineToken} from 'vs/editor/common/editorCommon';
-import {ViewLineToken} from 'vs/editor/common/viewModel/viewModel';
+import {ViewLineToken} from 'vs/editor/common/core/viewLineToken';
+import {LineToken} from 'vs/editor/common/model/lineToken';
 
-export const START_INDEX_MASK = 0xffffffff;
-export const TYPE_MASK = 0xffff;
-export const START_INDEX_OFFSET = 1;
-export const TYPE_OFFSET = Math.pow(2, 32);
+const START_INDEX_MASK = 0xffffffff;
+const TYPE_MASK = 0xffff;
+const START_INDEX_OFFSET = 1;
+const TYPE_OFFSET = Math.pow(2, 32);
 
-const DEFAULT_TOKEN = new LineToken(0, '');
 const DEFAULT_VIEW_TOKEN = new ViewLineToken(0, '');
 const INFLATED_TOKENS_EMPTY_TEXT:ViewLineToken[] = [];
 const DEFLATED_TOKENS_EMPTY_TEXT:number[] = [];
 const INFLATED_TOKENS_NON_EMPTY_TEXT:ViewLineToken[] = [DEFAULT_VIEW_TOKEN];
 const DEFLATED_TOKENS_NON_EMPTY_TEXT:number[] = [0];
 
-export function deflateArr(map:ITokensInflatorMap, tokens:LineToken[]): number[] {
-	if (tokens.length === 0) {
-		return DEFLATED_TOKENS_EMPTY_TEXT;
-	}
-	if (tokens.length === 1 && tokens[0].startIndex === 0 && !tokens[0].type) {
-		return DEFLATED_TOKENS_NON_EMPTY_TEXT;
+export class TokensInflatorMap {
+
+	public _inflate:string[];
+
+	public _deflate: {
+		[token:string]:number;
+	};
+
+	constructor() {
+		this._inflate = [ '' ];
+		this._deflate = { '': 0 };
 	}
+}
 
-	var i:number,
-		len:number,
-		deflatedToken:number,
-		deflated:number,
-		token:LineToken,
-		inflateMap = map._inflate,
-		deflateMap = map._deflate,
-		prevStartIndex:number = -1,
-		result:number[] = new Array(tokens.length);
+export class TokensBinaryEncoding {
+	public static START_INDEX_MASK = START_INDEX_MASK;
+	public static TYPE_MASK = TYPE_MASK;
+	public static START_INDEX_OFFSET = START_INDEX_OFFSET;
+	public static TYPE_OFFSET = TYPE_OFFSET;
 
-	for (i = 0, len = tokens.length; i < len; i++) {
-		token = tokens[i];
-
-		if (token.startIndex <= prevStartIndex) {
-			token.startIndex = prevStartIndex + 1;
-			onUnexpectedError({
-				message: 'Invalid tokens detected',
-				tokens: tokens
-			});
+	public static deflateArr(map:TokensInflatorMap, tokens:LineToken[]): number[] {
+		if (tokens.length === 0) {
+			return DEFLATED_TOKENS_EMPTY_TEXT;
+		}
+		if (tokens.length === 1 && tokens[0].startIndex === 0 && !tokens[0].type) {
+			return DEFLATED_TOKENS_NON_EMPTY_TEXT;
 		}
 
-		if (deflateMap.hasOwnProperty(token.type)) {
-			deflatedToken = deflateMap[token.type];
-		} else {
-			deflatedToken = inflateMap.length;
-			deflateMap[token.type] = deflatedToken;
-			inflateMap.push(token.type);
+		var i:number,
+			len:number,
+			deflatedToken:number,
+			deflated:number,
+			token:LineToken,
+			inflateMap = map._inflate,
+			deflateMap = map._deflate,
+			prevStartIndex:number = -1,
+			result:number[] = new Array(tokens.length);
+
+		for (i = 0, len = tokens.length; i < len; i++) {
+			token = tokens[i];
+
+			if (token.startIndex <= prevStartIndex) {
+				token.startIndex = prevStartIndex + 1;
+				onUnexpectedError({
+					message: 'Invalid tokens detected',
+					tokens: tokens
+				});
+			}
+
+			if (deflateMap.hasOwnProperty(token.type)) {
+				deflatedToken = deflateMap[token.type];
+			} else {
+				deflatedToken = inflateMap.length;
+				deflateMap[token.type] = deflatedToken;
+				inflateMap.push(token.type);
+			}
+
+			// http://stackoverflow.com/a/2803010
+			// All numbers in JavaScript are actually IEEE-754 compliant floating-point doubles.
+			// These have a 53-bit mantissa which should mean that any integer value with a magnitude
+			// of approximately 9 quadrillion or less -- more specifically, 9,007,199,254,740,991 --
+			// will be represented accurately.
+
+			// http://stackoverflow.com/a/6729252
+			// Bitwise operations cast numbers to 32bit representation in JS
+
+			// 32 bits for startIndex => up to 2^32 = 4,294,967,296
+			// 16 bits for token => up to 2^16 = 65,536
+
+			// [token][startIndex]
+			deflated = deflatedToken * TYPE_OFFSET + token.startIndex * START_INDEX_OFFSET;
+
+			result[i] = deflated;
+
+			prevStartIndex = token.startIndex;
 		}
 
-		// http://stackoverflow.com/a/2803010
-		// All numbers in JavaScript are actually IEEE-754 compliant floating-point doubles.
-		// These have a 53-bit mantissa which should mean that any integer value with a magnitude
-		// of approximately 9 quadrillion or less -- more specifically, 9,007,199,254,740,991 --
-		// will be represented accurately.
-
-		// http://stackoverflow.com/a/6729252
-		// Bitwise operations cast numbers to 32bit representation in JS
-
-		// 32 bits for startIndex => up to 2^32 = 4,294,967,296
-		// 16 bits for token => up to 2^16 = 65,536
-
-		// [token][startIndex]
-		deflated = deflatedToken * TYPE_OFFSET + token.startIndex * START_INDEX_OFFSET;
-
-		result[i] = deflated;
-
-		prevStartIndex = token.startIndex;
+		return result;
 	}
 
-	return result;
-}
-
-export function inflate(map:ITokensInflatorMap, binaryEncodedToken:number): LineToken {
-	if (binaryEncodedToken === 0) {
-		return DEFAULT_TOKEN;
+	public static getStartIndex(binaryEncodedToken:number): number {
+		return (binaryEncodedToken / START_INDEX_OFFSET) & START_INDEX_MASK;
 	}
 
-	var startIndex = (binaryEncodedToken / START_INDEX_OFFSET) & START_INDEX_MASK;
-	var deflatedType = (binaryEncodedToken / TYPE_OFFSET) & TYPE_MASK;
-
-	return new LineToken(startIndex, map._inflate[deflatedType]);
-}
-
-export function getStartIndex(binaryEncodedToken:number): number {
-	return (binaryEncodedToken / START_INDEX_OFFSET) & START_INDEX_MASK;
-}
-
-export function getType(map:ITokensInflatorMap, binaryEncodedToken:number): string {
-	var deflatedType = (binaryEncodedToken / TYPE_OFFSET) & TYPE_MASK;
-	if (deflatedType === 0) {
-		return strings.empty;
-	}
-	return map._inflate[deflatedType];
-}
-
-export function inflateArr(map:ITokensInflatorMap, binaryEncodedTokens:number[]): ViewLineToken[] {
-	if (binaryEncodedTokens.length === 0) {
-		return INFLATED_TOKENS_EMPTY_TEXT;
-	}
-	if (binaryEncodedTokens.length === 1 && binaryEncodedTokens[0] === 0) {
-		return INFLATED_TOKENS_NON_EMPTY_TEXT;
+	public static getType(map:TokensInflatorMap, binaryEncodedToken:number): string {
+		var deflatedType = (binaryEncodedToken / TYPE_OFFSET) & TYPE_MASK;
+		if (deflatedType === 0) {
+			return strings.empty;
+		}
+		return map._inflate[deflatedType];
 	}
 
-	let result: ViewLineToken[] = [];
-	const inflateMap = map._inflate;
-
-	for (let i = 0, len = binaryEncodedTokens.length; i < len; i++) {
-		let deflated = binaryEncodedTokens[i];
-
-		let startIndex = (deflated / START_INDEX_OFFSET) & START_INDEX_MASK;
-		let deflatedType = (deflated / TYPE_OFFSET) & TYPE_MASK;
-
-		result.push(new ViewLineToken(startIndex, inflateMap[deflatedType]));
-	}
-
-	return result;
-}
-
-export function findIndexOfOffset(binaryEncodedTokens:number[], offset:number): number {
-	return findIndexInSegmentsArray(binaryEncodedTokens, offset);
-}
-
-export function sliceAndInflate(map:ITokensInflatorMap, binaryEncodedTokens:number[], startOffset:number, endOffset:number, deltaStartIndex:number): ViewLineToken[] {
-	if (binaryEncodedTokens.length === 0) {
-		return INFLATED_TOKENS_EMPTY_TEXT;
-	}
-	if (binaryEncodedTokens.length === 1 && binaryEncodedTokens[0] === 0) {
-		return INFLATED_TOKENS_NON_EMPTY_TEXT;
-	}
-
-	let startIndex = findIndexInSegmentsArray(binaryEncodedTokens, startOffset);
-	let result: ViewLineToken[] = [];
-	const inflateMap = map._inflate;
-
-	let originalToken = binaryEncodedTokens[startIndex];
-	let deflatedType = (originalToken / TYPE_OFFSET) & TYPE_MASK;
-	let newStartIndex = 0;
-	result.push(new ViewLineToken(newStartIndex, inflateMap[deflatedType]));
-
-	for (let i = startIndex + 1, len = binaryEncodedTokens.length; i < len; i++) {
-		originalToken = binaryEncodedTokens[i];
-		let originalStartIndex = (originalToken / START_INDEX_OFFSET) & START_INDEX_MASK;
-
-		if (originalStartIndex >= endOffset) {
-			break;
+	public static inflateArr(map:TokensInflatorMap, binaryEncodedTokens:number[]): ViewLineToken[] {
+		if (binaryEncodedTokens.length === 0) {
+			return INFLATED_TOKENS_EMPTY_TEXT;
+		}
+		if (binaryEncodedTokens.length === 1 && binaryEncodedTokens[0] === 0) {
+			return INFLATED_TOKENS_NON_EMPTY_TEXT;
 		}
 
-		deflatedType = (originalToken / TYPE_OFFSET) & TYPE_MASK;
-		newStartIndex = originalStartIndex - startOffset + deltaStartIndex;
+		let result: ViewLineToken[] = [];
+		const inflateMap = map._inflate;
+
+		for (let i = 0, len = binaryEncodedTokens.length; i < len; i++) {
+			let deflated = binaryEncodedTokens[i];
+
+			let startIndex = (deflated / START_INDEX_OFFSET) & START_INDEX_MASK;
+			let deflatedType = (deflated / TYPE_OFFSET) & TYPE_MASK;
+
+			result.push(new ViewLineToken(startIndex, inflateMap[deflatedType]));
+		}
+
+		return result;
+	}
+
+	public static findIndexOfOffset(binaryEncodedTokens:number[], offset:number): number {
+		return this.findIndexInSegmentsArray(binaryEncodedTokens, offset);
+	}
+
+	public static sliceAndInflate(map:TokensInflatorMap, binaryEncodedTokens:number[], startOffset:number, endOffset:number, deltaStartIndex:number): ViewLineToken[] {
+		if (binaryEncodedTokens.length === 0) {
+			return INFLATED_TOKENS_EMPTY_TEXT;
+		}
+		if (binaryEncodedTokens.length === 1 && binaryEncodedTokens[0] === 0) {
+			return INFLATED_TOKENS_NON_EMPTY_TEXT;
+		}
+
+		let startIndex = this.findIndexInSegmentsArray(binaryEncodedTokens, startOffset);
+		let result: ViewLineToken[] = [];
+		const inflateMap = map._inflate;
+
+		let originalToken = binaryEncodedTokens[startIndex];
+		let deflatedType = (originalToken / TYPE_OFFSET) & TYPE_MASK;
+		let newStartIndex = 0;
 		result.push(new ViewLineToken(newStartIndex, inflateMap[deflatedType]));
-	}
 
-	return result;
-}
+		for (let i = startIndex + 1, len = binaryEncodedTokens.length; i < len; i++) {
+			originalToken = binaryEncodedTokens[i];
+			let originalStartIndex = (originalToken / START_INDEX_OFFSET) & START_INDEX_MASK;
 
-export function findIndexInSegmentsArray(arr:number[], desiredIndex: number):number {
+			if (originalStartIndex >= endOffset) {
+				break;
+			}
 
-	var low = 0,
-		high = arr.length - 1,
-		mid:number,
-		value:number;
-
-	while (low < high) {
-
-		mid = low + Math.ceil((high - low)/2);
-
-		value = arr[mid] & 0xffffffff;
-
-		if (value > desiredIndex) {
-			high = mid - 1;
-		} else {
-			low = mid;
+			deflatedType = (originalToken / TYPE_OFFSET) & TYPE_MASK;
+			newStartIndex = originalStartIndex - startOffset + deltaStartIndex;
+			result.push(new ViewLineToken(newStartIndex, inflateMap[deflatedType]));
 		}
+
+		return result;
 	}
 
-	return low;
+	private static findIndexInSegmentsArray(arr:number[], desiredIndex: number):number {
+
+		var low = 0,
+			high = arr.length - 1,
+			mid:number,
+			value:number;
+
+		while (low < high) {
+
+			mid = low + Math.ceil((high - low)/2);
+
+			value = arr[mid] & 0xffffffff;
+
+			if (value > desiredIndex) {
+				high = mid - 1;
+			} else {
+				low = mid;
+			}
+		}
+
+		return low;
+	}
 }

src/vs/editor/common/viewLayout/viewLineParts.ts

@@ -8,7 +8,7 @@ import * as strings from 'vs/base/common/strings';
 import {Arrays} from 'vs/editor/common/core/arrays';
 import {IEditorRange} from 'vs/editor/common/editorCommon';
 import {Range} from 'vs/editor/common/core/range';
-import {ViewLineToken, ViewLineTokens} from 'vs/editor/common/viewModel/viewModel';
+import {ViewLineToken, ViewLineTokens} from 'vs/editor/common/core/viewLineToken';
 
 function cmpLineDecorations(a:ILineDecoration, b:ILineDecoration): number {
 	return Range.compareRangesUsingStarts(a.range, b.range);

src/vs/editor/common/viewLayout/viewLineRenderer.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 'use strict';
 
-import {ViewLineToken} from 'vs/editor/common/viewModel/viewModel';
+import {ViewLineToken} from 'vs/editor/common/core/viewLineToken';
 
 export class RenderLineInput {
 	public _renderLineInputTrait: void;

src/vs/editor/common/viewModel/filteredLineTokens.ts

@@ -5,15 +5,14 @@
 'use strict';
 
 import {ILineTokens} from 'vs/editor/common/editorCommon';
-import * as TokensBinaryEncoding from 'vs/editor/common/model/tokensBinaryEncoding';
-import {ViewLineTokens} from 'vs/editor/common/viewModel/viewModel';
+import {ViewLineTokens} from 'vs/editor/common/core/viewLineToken';
 
 export class FilteredLineTokens {
 	/**
 	 * [startOffset; endOffset) (i.e. do not include endOffset)
 	 */
 	public static create(original:ILineTokens, startOffset:number, endOffset:number, deltaStartIndex:number): ViewLineTokens {
-		let inflatedTokens = TokensBinaryEncoding.sliceAndInflate(original.getBinaryEncodedTokensMap(), original.getBinaryEncodedTokens(), startOffset, endOffset, deltaStartIndex);
+		let inflatedTokens = original.sliceAndInflate(startOffset, endOffset, deltaStartIndex);
 		return new ViewLineTokens(
 			inflatedTokens,
 			deltaStartIndex,
@@ -25,7 +24,7 @@ export class FilteredLineTokens {
 export class IdentityFilteredLineTokens {
 
 	public static create(original:ILineTokens, textLength:number): ViewLineTokens {
-		let inflatedTokens = TokensBinaryEncoding.inflateArr(original.getBinaryEncodedTokensMap(), original.getBinaryEncodedTokens());
+		let inflatedTokens = original.inflate();
 		return new ViewLineTokens(
 			inflatedTokens,
 			0,

src/vs/editor/common/viewModel/splitLinesCollection.ts

@@ -10,7 +10,7 @@ import * as editorCommon from 'vs/editor/common/editorCommon';
 import {FilteredLineTokens, IdentityFilteredLineTokens} from 'vs/editor/common/viewModel/filteredLineTokens';
 import {PrefixSumComputer} from 'vs/editor/common/viewModel/prefixSumComputer';
 import {ILinesCollection} from 'vs/editor/common/viewModel/viewModelImpl';
-import {ViewLineTokens} from 'vs/editor/common/viewModel/viewModel';
+import {ViewLineTokens} from 'vs/editor/common/core/viewLineToken';
 
 export class OutputPosition {
 	_outputPositionTrait: void;

src/vs/editor/common/viewModel/viewModel.ts

@@ -5,86 +5,8 @@
 'use strict';
 
 import {IEventEmitter} from 'vs/base/common/eventEmitter';
-import {Arrays} from 'vs/editor/common/core/arrays';
 import {IModelDecoration, IRange, IEditorRange, EndOfLinePreference, IEditorSelection, IPosition, IEditorPosition} from 'vs/editor/common/editorCommon';
-
-/**
- * A token on a line.
- */
-export class ViewLineToken {
-	public _viewLineTokenTrait: void;
-
-	public startIndex:number;
-	public type:string;
-
-	constructor(startIndex:number, type:string) {
-		this.startIndex = startIndex|0;// @perf
-		this.type = type.replace(/[^a-z0-9\-]/gi, ' ');
-	}
-
-	public equals(other:ViewLineToken): boolean {
-		return (
-			this.startIndex === other.startIndex
-			&& this.type === other.type
-		);
-	}
-
-	public static findIndexInSegmentsArray(arr:ViewLineToken[], desiredIndex: number): number {
-		return Arrays.findIndexInSegmentsArray(arr, desiredIndex);
-	}
-
-	public static equalsArray(a:ViewLineToken[], b:ViewLineToken[]): boolean {
-		let aLen = a.length;
-		let bLen = b.length;
-		if (aLen !== bLen) {
-			return false;
-		}
-		for (let i = 0; i < aLen; i++) {
-			if (!a[i].equals(b[i])) {
-				return false;
-			}
-		}
-		return true;
-	}
-}
-
-export class ViewLineTokens {
-	_viewLineTokensTrait: void;
-
-	private _lineTokens:ViewLineToken[];
-	private _fauxIndentLength:number;
-	private _textLength:number;
-
-	constructor(lineTokens:ViewLineToken[], fauxIndentLength:number, textLength:number) {
-		this._lineTokens = lineTokens;
-		this._fauxIndentLength = fauxIndentLength|0;
-		this._textLength = textLength|0;
-	}
-
-	public getTokens(): ViewLineToken[] {
-		return this._lineTokens;
-	}
-
-	public getFauxIndentLength(): number {
-		return this._fauxIndentLength;
-	}
-
-	public getTextLength(): number {
-		return this._textLength;
-	}
-
-	public equals(other:ViewLineTokens): boolean {
-		return (
-			this._fauxIndentLength === other._fauxIndentLength
-			&& this._textLength === other._textLength
-			&& ViewLineToken.equalsArray(this._lineTokens, other._lineTokens)
-		);
-	}
-
-	public findIndexOfOffset(offset:number): number {
-		return ViewLineToken.findIndexInSegmentsArray(this._lineTokens, offset);
-	}
-}
+import {ViewLineTokens} from 'vs/editor/common/core/viewLineToken';
 
 export interface IDecorationsViewportData {
 	decorations: IModelDecoration[];

src/vs/editor/common/viewModel/viewModelImpl.ts

@@ -13,7 +13,8 @@ import {Selection} from 'vs/editor/common/core/selection';
 import * as editorCommon from 'vs/editor/common/editorCommon';
 import {ViewModelCursors} from 'vs/editor/common/viewModel/viewModelCursors';
 import {ViewModelDecorations} from 'vs/editor/common/viewModel/viewModelDecorations';
-import {IDecorationsViewportData, ViewLineTokens, IViewModel} from 'vs/editor/common/viewModel/viewModel';
+import {IDecorationsViewportData, IViewModel} from 'vs/editor/common/viewModel/viewModel';
+import {ViewLineTokens} from 'vs/editor/common/core/viewLineToken';
 
 export interface ILinesCollection {
 	setTabSize(newTabSize:number, emit:(evenType:string, payload:any)=>void): boolean;
@@ -115,8 +116,8 @@ export class ViewModel extends EventEmitter implements IViewModel {
 		return lineMappingChanged;
 	}
 
-	private _onWrappingIndentChange(newWrappingIndent:string): boolean {
-		var lineMappingChanged = this.lines.setWrappingIndent(editorCommon.wrappingIndentFromString(newWrappingIndent), (eventType:string, payload:any) => this.emit(eventType, payload));
+	private _onWrappingIndentChange(newWrappingIndent:editorCommon.WrappingIndent): boolean {
+		var lineMappingChanged = this.lines.setWrappingIndent(newWrappingIndent, (eventType:string, payload:any) => this.emit(eventType, payload));
 		if (lineMappingChanged) {
 			this.emit(editorCommon.ViewEventNames.LineMappingChangedEvent);
 			this.decorations.onLineMappingChanged((eventType:string, payload:any) => this.emit(eventType, payload));

src/vs/editor/test/common/model/model.line.test.ts

@@ -5,15 +5,15 @@
 'use strict';
 
 import * as assert from 'assert';
-import {ILineTokens, LineToken} from 'vs/editor/common/editorCommon';
+import {ILineTokens} from 'vs/editor/common/editorCommon';
 import * as modelLine from 'vs/editor/common/model/modelLine';
 import {LineMarker} from 'vs/editor/common/model/textModelWithMarkers';
-import {TokensInflatorMap} from 'vs/editor/common/model/textModelWithTokens';
+import {TokensInflatorMap} from 'vs/editor/common/model/tokensBinaryEncoding';
 import {IToken} from 'vs/editor/common/modes';
-import * as TokensBinaryEncoding from 'vs/editor/common/model/tokensBinaryEncoding';
+import {LineToken} from 'vs/editor/common/model/lineToken';
 
 function assertLineTokens(actual:ILineTokens, expected:IToken[]): void {
-	var inflatedActual = TokensBinaryEncoding.inflateArr(actual.getBinaryEncodedTokensMap(), actual.getBinaryEncodedTokens());
+	var inflatedActual = actual.inflate();
 	assert.deepEqual(inflatedActual, expected, 'Line tokens are equal');
 }
 

src/vs/editor/test/common/model/model.test.ts

@@ -14,26 +14,20 @@ import {BracketMode} from 'vs/editor/test/common/testModes';
 
 // --------- utils
 
-function isNotABracket(model, lineNumber, column) {
+function isNotABracket(model:Model, lineNumber:number, column:number) {
 	var match = model.matchBracket(new Position(lineNumber, column));
-	assert.equal(match.isAccurate, true, 'is not matching brackets at ' + lineNumber + ', ' + column);
-	assert.equal(match.brackets, null, 'is not matching brackets at ' + lineNumber + ', ' + column);
+	assert.equal(match, null, 'is not matching brackets at ' + lineNumber + ', ' + column);
 }
 
-function isBracket(model, lineNumber1, column11, column12, lineNumber2, column21, column22) {
+function isBracket(model:Model, lineNumber1:number, column11:number, column12:number, lineNumber2:number, column21:number, column22:number) {
 	var match = model.matchBracket(new Position(lineNumber1, column11));
-	assert.deepEqual(match, {
-		brackets: [
-			new Range(lineNumber1, column11, lineNumber1, column12),
-			new Range(lineNumber2, column21, lineNumber2, column22)
-		],
-		isAccurate: true
-	}, 'is matching brackets at ' + lineNumber1 + ', ' + column11);
+	assert.deepEqual(match, [
+		new Range(lineNumber1, column11, lineNumber1, column12),
+		new Range(lineNumber2, column21, lineNumber2, column22)
+	], 'is matching brackets at ' + lineNumber1 + ', ' + column11);
 }
 
-
-
-function rangeEqual(range, startLineNumber, startColumn, endLineNumber, endColumn) {
+function rangeEqual(range:Range, startLineNumber:number, startColumn:number, endLineNumber:number, endColumn:number) {
 	assert.deepEqual(range, new Range(startLineNumber, startColumn, endLineNumber, endColumn));
 }
 
@@ -506,23 +500,6 @@ suite('Editor Model - Words', () => {
 		thisModel.destroy();
 	});
 
-	test('Get all words', () => {
-		var words = [
-			{ start: 0,		end: 4 },
-			{ start: 5,		end: 9 },
-			{ start: 10,	end: 13 },
-			{ start: 14,	end: 18 },
-			{ start: 20,	end: 25 },
-			{ start: 25,	end: 26 }
-		];
-
-		var modelWords = thisModel.getWords(1);
-
-		for (var i = 0; i < modelWords.length; i++) {
-			assert.deepEqual(modelWords[i], words[i]);
-		}
-	});
-
 	test('Get word at position', () => {
 		assert.deepEqual(thisModel.getWordAtPosition(new Position(1, 1)), { word: 'This', startColumn: 1, endColumn: 5 });
 		assert.deepEqual(thisModel.getWordAtPosition(new Position(1, 2)), { word: 'This', startColumn: 1, endColumn: 5 });

src/vs/editor/test/common/viewLayout/viewLineParts.test.ts

@@ -8,7 +8,7 @@ import * as assert from 'assert';
 import {DecorationSegment, ILineDecoration, LineDecorationsNormalizer, getColumnOfLinePartOffset, createLineParts} from 'vs/editor/common/viewLayout/viewLineParts';
 import {Range} from 'vs/editor/common/core/range';
 import {RenderLineInput, renderLine} from 'vs/editor/common/viewLayout/viewLineRenderer';
-import {ViewLineToken, ViewLineTokens} from 'vs/editor/common/viewModel/viewModel';
+import {ViewLineToken, ViewLineTokens} from 'vs/editor/common/core/viewLineToken';
 
 suite('Editor ViewLayout - ViewLineParts', () => {
 

src/vs/editor/test/common/viewLayout/viewLineRenderer.test.ts

@@ -6,7 +6,7 @@
 
 import * as assert from 'assert';
 import {renderLine, RenderLineInput} from 'vs/editor/common/viewLayout/viewLineRenderer';
-import {ViewLineToken} from 'vs/editor/common/viewModel/viewModel';
+import {ViewLineToken} from 'vs/editor/common/core/viewLineToken';
 
 suite('viewLineRenderer.renderLine', () => {
 

src/vs/editor/test/common/viewModel/splitLinesCollection.test.ts

@@ -104,7 +104,7 @@ suite('Editor ViewModel - SplitLinesCollection', () => {
 			model.getOptions().tabSize,
 			config.editor.wrappingInfo.wrappingColumn,
 			config.editor.fontInfo.typicalFullwidthCharacterWidth / config.editor.fontInfo.typicalHalfwidthCharacterWidth,
-			editorCommon.wrappingIndentFromString(config.editor.wrappingIndent)
+			config.editor.wrappingIndent
 		);
 
 		linesCollection.setHiddenAreas([{

src/vs/languages/html/test/common/html.test.ts

@@ -725,15 +725,15 @@ suite('Colorizing - HTML', () => {
 
 	test('matchBracket', () => {
 
-		function toString(brackets:EditorCommon.IEditorRange[]): string[] {
+		function toString(brackets:[EditorCommon.IEditorRange, EditorCommon.IEditorRange]): [string,string] {
 			if (!brackets) {
 				return null;
 			}
 			brackets.sort(Range.compareRangesUsingStarts);
-			return brackets.map(b => b.toString());
+			return [brackets[0].toString(), brackets[1].toString()];
 		}
 
-		function assertBracket(lines:string[], lineNumber:number, column:number, expected:EditorCommon.IEditorRange[]): void {
+		function assertBracket(lines:string[], lineNumber:number, column:number, expected:[EditorCommon.IEditorRange, EditorCommon.IEditorRange]): void {
 			let model = new TextModelWithTokens([], TextModel.toRawText(lines.join('\n'), TextModel.DEFAULT_CREATION_OPTIONS), false, _mode);
 			// force tokenization
 			model.getLineContext(model.getLineCount());
@@ -741,9 +741,7 @@ suite('Colorizing - HTML', () => {
 				lineNumber: lineNumber,
 				column: column
 			});
-			let actualStr = actual ? toString(actual.brackets) : null;
-			let expectedStr = toString(expected);
-			assert.deepEqual(actualStr, expectedStr, 'TEXT <<' + lines.join('\n') + '>>, POS: ' + lineNumber + ', ' + column);
+			assert.deepEqual(toString(actual), toString(expected), 'TEXT <<' + lines.join('\n') + '>>, POS: ' + lineNumber + ', ' + column);
 		}
 
 		assertBracket(['<p></p>'], 1, 1, [new Range(1, 1, 1, 2), new Range(1, 3, 1, 4)]);