From 68026eadf8e20ee27b1a30f42f7e888ab4d28c09 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Fri, 12 Jan 2018 12:57:48 +0100 Subject: [PATCH 01/33] First cut --- .../model/chunksTextBuffer/bufferPiece.ts | 220 ++++++ .../chunksTextBuffer/chunksTextBuffer.ts | 688 ++++++++++++++++++ .../chunksTextBufferBuilder.ts | 129 ++++ src/vs/editor/common/model/textModel.ts | 5 + 4 files changed, 1042 insertions(+) create mode 100644 src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts create mode 100644 src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts create mode 100644 src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts diff --git a/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts b/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts new file mode 100644 index 00000000000..cbe35718d99 --- /dev/null +++ b/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts @@ -0,0 +1,220 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +'use strict'; + +import { CharCode } from 'vs/base/common/charCode'; + +export class LeafOffsetLenEdit { + constructor( + public readonly start: number, + public readonly length: number, + public readonly text: string + ) { } +} + +export class BufferPiece { + private readonly _str: string; + public get text(): string { return this._str; } + + private readonly _lineStarts: Uint32Array; + + constructor(str: string, lineStarts: Uint32Array = null) { + this._str = str; + if (lineStarts === null) { + this._lineStarts = createUint32Array(createLineStarts(str)); + } else { + this._lineStarts = lineStarts; + } + } + + public length(): number { + return this._str.length; + } + + public newLineCount(): number { + return this._lineStarts.length; + } + + public lineStartFor(relativeLineIndex: number): number { + return this._lineStarts[relativeLineIndex]; + } + + public charCodeAt(index: number): number { + return this._str.charCodeAt(index); + } + + public substr(from: number, length: number): string { + return this._str.substr(from, length); + } + + public static deleteLastChar(target: BufferPiece): BufferPiece { + const targetCharsLength = target.length(); + const targetLineStartsLength = target.newLineCount(); + const targetLineStarts = target._lineStarts; + + let newLineStartsLength; + if (targetLineStartsLength > 0 && targetLineStarts[targetLineStartsLength - 1] === targetCharsLength) { + newLineStartsLength = targetLineStartsLength - 1; + } else { + newLineStartsLength = targetLineStartsLength; + } + + let newLineStarts = new Uint32Array(newLineStartsLength); + newLineStarts.set(targetLineStarts); // TODO: does this work correctly? + + return new BufferPiece( + target._str.substring(0, targetCharsLength - 1), + newLineStarts + ); + } + + public static insertFirstChar(target: BufferPiece, character: number): BufferPiece { + const targetLineStartsLength = target.newLineCount(); + const targetLineStarts = target._lineStarts; + const insertLineStart = ((character === CharCode.CarriageReturn && (targetLineStartsLength === 0 || targetLineStarts[0] !== 1 || target.charCodeAt(0) !== CharCode.LineFeed)) || (character === CharCode.LineFeed)); + + const newLineStartsLength = (insertLineStart ? targetLineStartsLength + 1 : targetLineStartsLength); + let newLineStarts = new Uint32Array(newLineStartsLength); + + if (insertLineStart) { + newLineStarts[0] = 1; + for (let i = 0; i < targetLineStartsLength; i++) { + newLineStarts[i + 1] = targetLineStarts[i] + 1; + } + } else { + for (let i = 0; i < targetLineStartsLength; i++) { + newLineStarts[i] = targetLineStarts[i] + 1; + } + } + + return new BufferPiece( + String.fromCharCode(character) + target._str, + newLineStarts + ); + } + + public static join(first: BufferPiece, second: BufferPiece): BufferPiece { + const firstCharsLength = first._str.length; + + const firstLineStartsLength = first._lineStarts.length; + const secondLineStartsLength = second._lineStarts.length; + + const firstLineStarts = first._lineStarts; + const secondLineStarts = second._lineStarts; + + const newLineStartsLength = firstLineStartsLength + secondLineStartsLength; + let newLineStarts = new Uint32Array(newLineStartsLength); + newLineStarts.set(firstLineStarts, 0); + for (let i = 0; i < secondLineStartsLength; i++) { + newLineStarts[i + firstLineStartsLength] = secondLineStarts[i] + firstCharsLength; + } + + return new BufferPiece(first._str + second._str, newLineStarts); + } + + public static replaceOffsetLen(target: BufferPiece, edits: LeafOffsetLenEdit[], idealLeafLength: number, maxLeafLength: number, result: BufferPiece[]): void { + const editsSize = edits.length; + const originalCharsLength = target.length(); + if (editsSize === 1 && edits[0].text.length === 0 && edits[0].start === 0 && edits[0].length === originalCharsLength) { + // special case => deleting everything + return; + } + + let pieces: string[]; + let originalFromIndex = 0; + let piecesTextLength = 0; + for (let i = 0; i < editsSize; i++) { + const edit = edits[i]; + + const originalText = target._str.substr(originalFromIndex, edit.start - originalFromIndex); + pieces[2 * i] = originalText; + piecesTextLength += originalText.length; + + originalFromIndex = edit.start + edit.length; + pieces[2 * i + 1] = edit.text; + piecesTextLength += edit.text.length; + } + + // maintain the chars that survive to the right of the last edit + let text = target._str.substring(originalFromIndex, originalCharsLength - originalFromIndex); + pieces[2 * editsSize] = text; + piecesTextLength += text.length; + + let targetDataLength = piecesTextLength > maxLeafLength ? idealLeafLength : piecesTextLength; + let targetDataOffset = 0; + + let data: string = ''; + + for (let pieceIndex = 0, pieceCount = pieces.length; pieceIndex < pieceCount; pieceIndex++) { + const pieceText = pieces[pieceIndex]; + const pieceLength = pieceText.length; + if (pieceLength === 0) { + continue; + } + + let pieceOffset = 0; + while (pieceOffset < pieceLength) { + if (targetDataOffset >= targetDataLength) { + result.push(new BufferPiece(data)); + targetDataLength = piecesTextLength > maxLeafLength ? idealLeafLength : piecesTextLength; + targetDataOffset = 0; + data = ''; + } + + let writingCnt = min(pieceLength - pieceOffset, targetDataLength - targetDataOffset); + data += pieceText.substr(pieceOffset, writingCnt); + pieceOffset += writingCnt; + targetDataOffset += writingCnt; + piecesTextLength -= writingCnt; + + // check that the buffer piece does not end in a \r or high surrogate + if (targetDataOffset === targetDataLength && piecesTextLength > 0) { + const lastChar = data.charCodeAt(targetDataLength - 1); + if (lastChar === CharCode.CarriageReturn || (0xD800 <= lastChar && lastChar <= 0xDBFF)) { + // move lastChar over to next buffer piece + targetDataLength -= 1; + pieceOffset -= 1; + targetDataOffset -= 1; + piecesTextLength += 1; + data = data.substr(0, data.length - 1); + } + } + } + } + + result.push(new BufferPiece(data)); + } +} + +function min(a: number, b: number): number { + return (a < b ? a : b); +} + +function createUint32Array(arr: number[]): Uint32Array { + let r = new Uint32Array(arr.length); + r.set(arr, 0); + return r; +} + +function createLineStarts(str: string): number[] { + let r: number[] = [], rLength = 0; + for (let i = 0, len = str.length; i < len; i++) { + const chr = str.charCodeAt(i); + + if (chr === CharCode.CarriageReturn) { + if (i + 1 < len && str.charCodeAt(i + 1) === CharCode.LineFeed) { + // \r\n... case + r[rLength++] = i + 2; + i++; // skip \n + } else { + // \r... case + r[rLength++] = i + 1; + } + } else if (chr === CharCode.LineFeed) { + r[rLength++] = i + 1; + } + } + return r; +} diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts new file mode 100644 index 00000000000..194031e6e8a --- /dev/null +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts @@ -0,0 +1,688 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +'use strict'; + +import { CharCode } from 'vs/base/common/charCode'; +import { ITextBuffer, EndOfLinePreference, IIdentifiedSingleEditOperation, ApplyEditsResult } from 'vs/editor/common/model'; +import { BufferPiece, LeafOffsetLenEdit } from 'vs/editor/common/model/chunksTextBuffer/bufferPiece'; +import { Position } from 'vs/editor/common/core/position'; +import { Range } from 'vs/editor/common/core/range'; + +export class ChunksTextBuffer implements ITextBuffer { + + private _actual: Buffer; + + constructor(pieces: BufferPiece[], _averageChunkSize: number) { + const averageChunkSize = Math.floor(Math.min(65536.0, Math.max(128.0, _averageChunkSize))); + const delta = Math.floor(averageChunkSize / 3); + const min = averageChunkSize - delta; + const max = 2 * min; + this._actual = new Buffer(pieces, min, max); + } + + equals(other: ITextBuffer): boolean { + throw new Error('TODO'); + } + mightContainRTL(): boolean { + // TODO + return true; + } + mightContainNonBasicASCII(): boolean { + // TODO + return true; + } + getBOM(): string { + // TODO + return ''; + } + getEOL(): string { + // TODO + return '\n'; + } + getOffsetAt(lineNumber: number, column: number): number { + return this._actual.getOffsetAt(lineNumber, column); + } + getPositionAt(offset: number): Position { + throw new Error('TODO'); + } + getRangeAt(offset: number, length: number): Range { + throw new Error('TODO'); + } + getValueInRange(range: Range, eol: EndOfLinePreference): string { + // TODO + + if (range.isEmpty()) { + return ''; + } + + if (range.startLineNumber === range.endLineNumber) { + return this.getLineContent(range.startLineNumber).substring(range.startColumn - 1, range.endColumn - 1); + } + + var lineEnding = '\n',//todo this._getEndOfLine(eol), + startLineIndex = range.startLineNumber - 1, + endLineIndex = range.endLineNumber - 1, + resultLines: string[] = []; + + resultLines.push(this.getLineContent(startLineIndex + 1).substring(range.startColumn - 1)); + for (var i = startLineIndex + 1; i < endLineIndex; i++) { + resultLines.push(this.getLineContent(i + 1)); + } + resultLines.push(this.getLineContent(endLineIndex + 1).substring(0, range.endColumn - 1)); + + return resultLines.join(lineEnding); + } + getValueLengthInRange(range: Range, eol: EndOfLinePreference): number { + // TODO + return this.getValueInRange(range, eol).length; + } + getLineCount(): number { + // TODO: perhaps cache? + return this._actual.getLineCount(); + } + getLinesContent(): string[] { + return this._actual.getLinesContent(); + } + getLineContent(lineNumber: number): string { + // TODO + return this._actual.getLineContent(lineNumber).replace(/\r?\n?/, ''); + } + getLineCharCode(lineNumber: number, index: number): number { + return this.getLineContent(lineNumber).charCodeAt(index); + } + getLineLength(lineNumber: number): number { + // TODO + let content = this.getLineContent(lineNumber); + return content.length; + } + getLineFirstNonWhitespaceColumn(lineNumber: number): number { + throw new Error('TODO'); + } + getLineLastNonWhitespaceColumn(lineNumber: number): number { + throw new Error('TODO'); + } + setEOL(newEOL: string): void { + throw new Error('TODO'); + } + applyEdits(rawOperations: IIdentifiedSingleEditOperation[], recordTrimAutoWhitespace: boolean): ApplyEditsResult { + throw new Error('TODO'); + } +} + + +class BufferNodes { + + public length: Uint32Array; + public newLineCount: Uint32Array; + + constructor(count: number) { + this.length = new Uint32Array(count); + this.newLineCount = new Uint32Array(count); + } + +} + +class BufferCursor { + constructor( + public readonly offset, + public readonly leafIndex, + public readonly leafStartOffset, + public readonly leafStartNewLineCount + ) { } +} + +class OffsetLenEdit { + constructor( + public readonly initialIndex: number, + public readonly offset: number, + public length: number, + public text: string + ) { } +} + +class InternalOffsetLenEdit { + constructor( + public readonly startLeafIndex: number, + public readonly startInnerOffset: number, + public readonly endLeafIndex: number, + public readonly endInnerOffset: number, + public text: string + ) { } +} + +class LeafReplacement { + constructor( + public readonly startLeafIndex: number, + public readonly endLeafIndex: number, + public readonly replacements: BufferPiece[] + ) { } +} + +class Buffer { + + private _minLeafLength: number; + private _maxLeafLength: number; + private _idealLeafLength: number; + + private _leafs: BufferPiece[]; + private _nodes: BufferNodes; + private _nodesCount: number; + private _leafsStart: number; + private _leafsEnd: number; + + constructor(pieces: BufferPiece[], minLeafLength: number, maxLeafLength: number) { + if (!(2 * minLeafLength >= maxLeafLength)) { + throw new Error(`assertion violation`); + } + + this._minLeafLength = minLeafLength; + this._maxLeafLength = maxLeafLength; + this._idealLeafLength = (minLeafLength + maxLeafLength) >>> 1; + + this._leafs = pieces; + this._nodes = null; + this._nodesCount = 0; + this._leafsStart = 0; + this._leafsEnd = 0; + + this._rebuildNodes(); + } + + private _rebuildNodes() { + const leafsCount = this._leafs.length; + + this._nodesCount = (1 << log2(leafsCount)); + this._leafsStart = this._nodesCount; + this._leafsEnd = this._leafsStart + leafsCount; + + this._nodes = new BufferNodes(this._nodesCount); + for (let i = this._nodesCount - 1; i >= 1; i--) { + this._updateSingleNode(i); + } + } + + private _updateSingleNode(nodeIndex: number): void { + const left = LEFT_CHILD(nodeIndex); + const right = RIGHT_CHILD(nodeIndex); + + let length = 0; + let newLineCount = 0; + + if (this.IS_NODE(left)) { + length += this._nodes.length[left]; + newLineCount += this._nodes.newLineCount[left]; + } else if (this.IS_LEAF(left)) { + const leaf = this._leafs[this.NODE_TO_LEAF_INDEX(left)]; + length += leaf.length(); + newLineCount += leaf.newLineCount(); + } + + if (this.IS_NODE(right)) { + length += this._nodes.length[right]; + newLineCount += this._nodes.newLineCount[right]; + } else if (this.IS_LEAF(right)) { + const leaf = this._leafs[this.NODE_TO_LEAF_INDEX(right)]; + length += leaf.length(); + newLineCount += leaf.newLineCount(); + } + + this._nodes.length[nodeIndex] = length; + this._nodes.newLineCount[nodeIndex] = newLineCount; + } + + public findOffset(offset: number): BufferCursor { + if (offset > this._nodes.length[1]) { + return null; + } + + let it = 1; + let searchOffset = offset; + let leafStartOffset = 0; + let leafStartNewLineCount = 0; + while (!this.IS_LEAF(it)) { + const left = LEFT_CHILD(it); + const right = RIGHT_CHILD(it); + + let leftNewLineCount = 0; + let leftLength = 0; + if (this.IS_NODE(left)) { + leftNewLineCount = this._nodes.newLineCount[left]; + leftLength = this._nodes.length[left]; + } else if (this.IS_LEAF(left)) { + const leaf = this._leafs[this.NODE_TO_LEAF_INDEX(left)]; + leftNewLineCount = leaf.newLineCount(); + leftLength = leaf.length(); + } + + let rightLength = 0; + if (this.IS_NODE(right)) { + rightLength += this._nodes.length[right]; + } else if (this.IS_LEAF(right)) { + rightLength += this._leafs[this.NODE_TO_LEAF_INDEX(right)].length(); + } + + if (searchOffset < leftLength || rightLength === 0) { + // go left + it = left; + } else { + // go right + searchOffset -= leftLength; + leafStartOffset += leftLength; + leafStartNewLineCount += leftNewLineCount; + it = right; + } + } + it = this.NODE_TO_LEAF_INDEX(it); + + return new BufferCursor(offset, it, leafStartOffset, leafStartNewLineCount); + } + + private _findLineStart(lineIndex: number): BufferCursor { + if (lineIndex < 0 || lineIndex > this._nodes.newLineCount[1]) { + return null; + } + + let it = 1; + let leafStartOffset = 0; + let leafStartNewLineCount = 0; + while (!this.IS_LEAF(it)) { + const left = LEFT_CHILD(it); + const right = RIGHT_CHILD(it); + + let leftNewLineCount = 0; + let leftLength = 0; + if (this.IS_NODE(left)) { + leftNewLineCount = this._nodes.newLineCount[left]; + leftLength = this._nodes.length[left]; + } else if (this.IS_LEAF(left)) { + const leaf = this._leafs[this.NODE_TO_LEAF_INDEX(left)]; + leftNewLineCount = leaf.newLineCount(); + leftLength = leaf.length(); + } + + if (lineIndex <= leftNewLineCount) { + // go left + it = left; + continue; + } + + // go right + lineIndex -= leftNewLineCount; + leafStartOffset += leftLength; + leafStartNewLineCount += leftNewLineCount; + it = right; + } + it = this.NODE_TO_LEAF_INDEX(it); + + const innerLineStartOffset = (lineIndex === 0 ? 0 : this._leafs[it].lineStartFor(lineIndex - 1)); + + return new BufferCursor(leafStartOffset + innerLineStartOffset, it, leafStartOffset, leafStartNewLineCount); + } + + private _findLineEnd(start: BufferCursor, lineNumber: number): BufferCursor { + let innerLineIndex = lineNumber - 1 - start.leafStartNewLineCount; + const leafsCount = this._leafs.length; + + let leafIndex = start.leafIndex; + let leafStartOffset = start.leafStartOffset; + let leafStartNewLineCount = start.leafStartNewLineCount; + while (true) { + const leaf = this._leafs[leafIndex]; + + if (innerLineIndex < leaf.newLineCount()) { + const lineEndOffset = this._leafs[leafIndex].lineStartFor(innerLineIndex); + return new BufferCursor(leafStartOffset + lineEndOffset, leafIndex, leafStartOffset, leafStartNewLineCount); + } + + leafIndex++; + + if (leafIndex >= leafsCount) { + return new BufferCursor(leafStartOffset + leaf.length(), leafIndex - 1, leafStartOffset, leafStartNewLineCount); + } + + leafStartOffset += leaf.length(); + leafStartNewLineCount += leaf.newLineCount(); + innerLineIndex = 0; + } + } + + public findLine(lineNumber: number): [BufferCursor, BufferCursor] { + const innerLineIndex = lineNumber - 1; + const start = this._findLineStart(innerLineIndex); + if (!start) { + return null; + } + + const end = this._findLineEnd(start, lineNumber); + return [start, end]; + } + + public getLineCount(): number { + return this._nodes.newLineCount[1] + 1; + } + + public getLineContent(lineNumber: number): string { + const r = this.findLine(lineNumber); + if (!r) { + throw new Error(`Line not found`); + } + const [start, end] = r; + return this.extractString(start, end.offset - start.offset); + } + + public getLinesContent(): string[] { + let result: string[] = new Array(this.getLineCount()); + let resultIndex = 0; + + let currentLine = ''; + for (let leafIndex = 0, leafsCount = this._leafs.length; leafIndex < leafsCount; leafIndex++) { + const leaf = this._leafs[leafIndex]; + const leafNewLineCount = leaf.newLineCount(); + + if (leafNewLineCount === 0) { + // special case => push entire leaf text + currentLine += leaf.text; + continue; + } + + let leafSubstrOffset = 0; + for (let newLineIndex = 0; newLineIndex < leafNewLineCount; newLineIndex++) { + const newLineStart = leaf.lineStartFor(newLineIndex); + let length = newLineStart - leafSubstrOffset - 1 /*-1 for EOL*/; + + if (length > 0 && leaf.charCodeAt(leafSubstrOffset + length - 1) === CharCode.CarriageReturn) { + // \r\n case + length--; + } + currentLine += leaf.substr(leafSubstrOffset, length); + result[resultIndex++] = currentLine; + + currentLine = ''; + leafSubstrOffset = newLineStart; + } + currentLine += leaf.substr(leafSubstrOffset, leaf.length()); + } + result[resultIndex++] = currentLine; + + return result; + } + + public extractString(start: BufferCursor, len: number): string { + if (!(start.offset + len <= this._nodes.length[1])) { + throw new Error(`assertion violation`); + } + + let innerLeafOffset = start.offset - start.leafStartOffset; + let leafIndex = start.leafIndex; + let res = ''; + while (len > 0) { + const leaf = this._leafs[leafIndex]; + const cnt = Math.min(len, leaf.length() - innerLeafOffset); + res += leaf.substr(innerLeafOffset, cnt); + + len -= cnt; + innerLeafOffset = 0; + + if (len === 0) { + break; + } + + leafIndex++; + } + return res; + } + + public getOffsetAt(lineNumber: number, column: number): number { + const start = this._findLineStart(lineNumber - 1); + if (!start) { + throw new Error(`Line not found`); + } + return start.offset + column - 1; + } + + //#region Editing + + private _mergeAdjacentEdits(edits: OffsetLenEdit[]): OffsetLenEdit[] { + // Check if we must merge adjacent edits + let merged: OffsetLenEdit[] = [], mergedLength = 0; + let prev = edits[0]; + for (let i = 1, len = edits.length; i < len; i++) { + const curr = edits[i]; + if (prev.offset + prev.length === curr.offset) { + // merge into `prev` + prev.length = prev.length + curr.length; + prev.text = prev.text + curr.text; + } else { + merged[mergedLength++] = prev; + prev = curr; + } + } + merged[mergedLength++] = prev; + + return merged; + } + + private _resolveEdits(edits: OffsetLenEdit[]): InternalOffsetLenEdit[] { + edits = this._mergeAdjacentEdits(edits); + + let result: InternalOffsetLenEdit[] = []; + let tmp: BufferCursor; + for (let i = 0, len = edits.length; i < len; i++) { + const edit = edits[i]; + + let text = edit.text; + + tmp = this.findOffset(edit.offset); + let startLeafIndex = tmp.leafIndex; + let startInnerOffset = tmp.offset - tmp.leafStartOffset; + if (startInnerOffset > 0) { + const startLeaf = this._leafs[startLeafIndex]; + const charBefore = startLeaf.charCodeAt(startInnerOffset - 1); + if (charBefore === CharCode.CarriageReturn) { + // include the replacement of \r in the edit + text = '\r' + text; + + tmp = this.findOffset(edit.offset - 1); + startLeafIndex = tmp.leafIndex; + startInnerOffset = tmp.offset - tmp.leafStartOffset; + } + } + + tmp = this.findOffset(edit.offset + edit.length); + let endLeafIndex = tmp.leafIndex; + let endInnerOffset = tmp.offset - tmp.leafStartOffset; + const endLeaf = this._leafs[endLeafIndex]; + if (endInnerOffset < endLeaf.length()) { + const charAfter = endLeaf.charCodeAt(endInnerOffset); + if (charAfter === CharCode.LineFeed) { + // include the replacement of \n in the edit + text = text + '\n'; + + tmp = this.findOffset(edit.offset + edit.length + 1); + endLeafIndex = tmp.leafIndex; + endInnerOffset = tmp.offset - tmp.leafStartOffset; + } + } + + result[i] = new InternalOffsetLenEdit( + startLeafIndex, startInnerOffset, + endLeafIndex, endInnerOffset, + text + ); + } + + return result; + } + + private _pushLeafReplacement(startLeafIndex: number, endLeafIndex: number, replacements: LeafReplacement[]): LeafReplacement { + const res = new LeafReplacement(startLeafIndex, endLeafIndex, []); + replacements.push(res); + return res; + } + + private _flushLeafEdits(accumulatedLeafIndex: number, accumulatedLeafEdits: LeafOffsetLenEdit[], replacements: LeafReplacement[]): void { + if (accumulatedLeafEdits.length > 0) { + const rep = this._pushLeafReplacement(accumulatedLeafIndex, accumulatedLeafIndex, replacements); + BufferPiece.replaceOffsetLen(this._leafs[accumulatedLeafIndex], accumulatedLeafEdits, this._idealLeafLength, this._maxLeafLength, rep.replacements); + } + accumulatedLeafEdits.length = 0; + } + + private _pushLeafEdits(start: number, length: number, text: string, accumulatedLeafEdits: LeafOffsetLenEdit[]): void { + if (length !== 0 || text.length !== 0) { + accumulatedLeafEdits.push(new LeafOffsetLenEdit(start, length, text)); + } + } + + private _appendLeaf(leaf: BufferPiece, leafs: BufferPiece[], prevLeaf: BufferPiece): BufferPiece { + if (prevLeaf === null) { + leafs.push(leaf); + prevLeaf = leaf; + return prevLeaf; + } + + let prevLeafLength = prevLeaf.length(); + let currLeafLength = leaf.length(); + + if ((prevLeafLength < this._minLeafLength || currLeafLength < this._minLeafLength) && prevLeafLength + currLeafLength <= this._maxLeafLength) { + const joinedLeaf = BufferPiece.join(prevLeaf, leaf); + leafs[leafs.length - 1] = joinedLeaf; + prevLeaf = joinedLeaf; + return prevLeaf; + } + + const lastChar = prevLeaf.charCodeAt(prevLeafLength - 1); + const firstChar = leaf.charCodeAt(0); + + if ( + (lastChar >= 0xd800 && lastChar <= 0xdbff) || (lastChar === CharCode.CarriageReturn && firstChar === CharCode.LineFeed) + ) { + const modifiedPrevLeaf = BufferPiece.deleteLastChar(prevLeaf); + leafs[leafs.length - 1] = modifiedPrevLeaf; + + const modifiedLeaf = BufferPiece.insertFirstChar(leaf, lastChar); + leaf = modifiedLeaf; + } + + leafs.push(leaf); + prevLeaf = leaf; + return prevLeaf; + } + + public replaceOffsetLen(_edits: OffsetLenEdit[]): void { + const initialLeafLength = this._leafs.length; + const edits = this._resolveEdits(_edits); + + let accumulatedLeafIndex = 0; + let accumulatedLeafEdits: LeafOffsetLenEdit[]; + let replacements: LeafReplacement[]; + + for (let i = 0, len = edits.length; i < len; i++) { + const edit = edits[i]; + + const startLeafIndex = edit.startLeafIndex; + const endLeafIndex = edit.endLeafIndex; + + if (startLeafIndex !== accumulatedLeafIndex) { + this._flushLeafEdits(accumulatedLeafIndex, accumulatedLeafEdits, replacements); + accumulatedLeafIndex = startLeafIndex; + } + + const leafEditStart = edit.startInnerOffset; + const leafEditEnd = (startLeafIndex === endLeafIndex ? edit.endInnerOffset : this._leafs[startLeafIndex].length()); + this._pushLeafEdits(leafEditStart, leafEditEnd - leafEditStart, edit.text, accumulatedLeafEdits); + + if (startLeafIndex < endLeafIndex) { + this._flushLeafEdits(accumulatedLeafIndex, accumulatedLeafEdits, replacements); + accumulatedLeafIndex = endLeafIndex; + + // delete leafs in the middle + if (startLeafIndex + 1 < endLeafIndex) { + this._pushLeafReplacement(startLeafIndex + 1, endLeafIndex - 1, replacements); + } + + // delete on last leaf + const leafEditStart = 0; + const leafEditEnd = edit.endInnerOffset; + this._pushLeafEdits(leafEditStart, leafEditEnd - leafEditStart, '', accumulatedLeafEdits); + } + } + this._flushLeafEdits(accumulatedLeafIndex, accumulatedLeafEdits, replacements); + + let leafs: BufferPiece[] = []; + let leafIndex = 0; + let prevLeaf: BufferPiece = null; + + for (let i = 0, len = replacements.length; i < len; i++) { + const replaceStartLeafIndex = replacements[i].startLeafIndex; + const replaceEndLeafIndex = replacements[i].endLeafIndex; + const innerLeafs = replacements[i].replacements; + + // add leafs to the left of this replace op. + while (leafIndex < replaceStartLeafIndex) { + prevLeaf = this._appendLeaf(this._leafs[leafIndex], leafs, prevLeaf); + leafIndex++; + } + + // delete leafs that get replaced. + while (leafIndex <= replaceEndLeafIndex) { + leafIndex++; + } + + // add new leafs. + for (let j = 0, lenJ = innerLeafs.length; j < lenJ; j++) { + prevLeaf = this._appendLeaf(innerLeafs[j], leafs, prevLeaf); + } + } + + // add remaining leafs to the right of the last replacement. + while (leafIndex < initialLeafLength) { + prevLeaf = this._appendLeaf(this._leafs[leafIndex], leafs, prevLeaf); + leafIndex++; + } + + if (leafs.length === 0) { + // don't leave behind an empty leafs array + leafs.push(new BufferPiece('')); + } + + this._leafs = leafs; + this._rebuildNodes(); + } + + //#endregion + + private IS_NODE(i: number): boolean { + return (i < this._nodesCount); + } + private IS_LEAF(i: number): boolean { + return (i >= this._leafsStart && i < this._leafsEnd); + } + private NODE_TO_LEAF_INDEX(i: number): number { + return (i - this._leafsStart); + } + // private LEAF_TO_NODE_INDEX(i: number): number { + // return (i + this._leafsStart); + // } +} + +function log2(n: number): number { + let v = 1; + for (let pow = 1; ; pow++) { + v = v << 1; + if (v >= n) { + return pow; + } + } + // return -1; +} + +function LEFT_CHILD(i: number): number { + return (i << 1); +} + +function RIGHT_CHILD(i: number): number { + return (i << 1) + 1; +} diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts new file mode 100644 index 00000000000..ddd56eab383 --- /dev/null +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts @@ -0,0 +1,129 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +'use strict'; + +import * as strings from 'vs/base/common/strings'; +import { ITextBufferBuilder, ITextBufferFactory, ITextBuffer, DefaultEndOfLine } from 'vs/editor/common/model'; +import { BufferPiece } from 'vs/editor/common/model/chunksTextBuffer/bufferPiece'; +import { ChunksTextBuffer } from 'vs/editor/common/model/chunksTextBuffer/chunksTextBuffer'; + +export class TextBufferFactory implements ITextBufferFactory { + + constructor( + private readonly _pieces: BufferPiece[], + private readonly _averageChunkSize: number + ) { + } + + public create(defaultEOL: DefaultEndOfLine): ITextBuffer { + // TODO: CRLF vs LF + return new ChunksTextBuffer(this._pieces, this._averageChunkSize); + } + + public getFirstLineText(lengthLimit: number): string { + console.log(`TODO`); + return ''; + } +} + +export class ChunksTextBufferBuilder implements ITextBufferBuilder { + + private _rawPieces: BufferPiece[]; + private _hasPreviousChar: boolean; + private _previousChar: number; + private _averageChunkSize: number; + + // private totalCRCount: number; + private containsRTL: boolean; + private isBasicASCII: boolean; + + constructor() { + this._rawPieces = []; + this._hasPreviousChar = false; + this._previousChar = 0; + this._averageChunkSize = 0; + + // this.totalCRCount = 0; + this.containsRTL = false; + this.isBasicASCII = true; + } + + // private _updateCRCount(chunk: string): void { + // // Count how many \r are present in chunk to determine the majority EOL sequence + // let chunkCarriageReturnCnt = 0; + // let lastCarriageReturnIndex = -1; + // while ((lastCarriageReturnIndex = chunk.indexOf('\r', lastCarriageReturnIndex + 1)) !== -1) { + // chunkCarriageReturnCnt++; + // } + // this.totalCRCount += chunkCarriageReturnCnt; + // } + + public acceptChunk(chunk: string): void { + if (chunk.length === 0) { + return; + } + + this._averageChunkSize = (this._averageChunkSize * this._rawPieces.length + chunk.length) / (this._rawPieces.length + 1); + + const lastChar = chunk.charCodeAt(chunk.length - 1); + if (lastChar === 13 || (lastChar >= 0xd800 && lastChar <= 0xdbff)) { + // last character is \r or a high surrogate => keep it back + this._acceptChunk1(chunk.substring(0, chunk.length - 1), false); + this._hasPreviousChar = true; + this._previousChar = lastChar; + } else { + this._acceptChunk1(chunk, false); + this._hasPreviousChar = false; + this._previousChar = lastChar; + } + + if (!this.containsRTL) { + this.containsRTL = strings.containsRTL(chunk); + } + if (this.isBasicASCII) { + this.isBasicASCII = strings.isBasicASCII(chunk); + } + } + + private _acceptChunk1(chunk: string, allowEmptyStrings: boolean): void { + if (!allowEmptyStrings && chunk.length === 0) { + // Nothing to do + return; + } + + if (this._hasPreviousChar) { + this._acceptChunk2(chunk + String.fromCharCode(this._previousChar)); + } else { + this._acceptChunk2(chunk); + } + } + + private _acceptChunk2(chunk: string): void { + this._rawPieces.push(new BufferPiece(chunk)); + } + + public finish(): TextBufferFactory { + this._finish(); + return new TextBufferFactory(this._rawPieces, this._averageChunkSize); + } + + private _finish(): void { + if (this._rawPieces.length === 0) { + // no chunks => forcefully go through accept chunk + this._acceptChunk1('', true); + return; + } + + if (this._hasPreviousChar) { + this._hasPreviousChar = false; + + // recreate last chunk + const lastPiece = this._rawPieces[this._rawPieces.length - 1]; + const tmp = new BufferPiece(String.fromCharCode(this._previousChar)); + const newLastPiece = BufferPiece.join(lastPiece, tmp); + this._rawPieces[this._rawPieces.length - 1] = newLastPiece; + } + } +} diff --git a/src/vs/editor/common/model/textModel.ts b/src/vs/editor/common/model/textModel.ts index dff56a1a7a8..701c7274e71 100644 --- a/src/vs/editor/common/model/textModel.ts +++ b/src/vs/editor/common/model/textModel.ts @@ -34,9 +34,14 @@ import { TextModelSearch, SearchParams } from 'vs/editor/common/model/textModelS import { TPromise } from 'vs/base/common/winjs.base'; import { IStringStream } from 'vs/platform/files/common/files'; import { LinesTextBufferBuilder } from 'vs/editor/common/model/linesTextBuffer/linesTextBufferBuilder'; +import { ChunksTextBufferBuilder } from 'vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder'; // Here is the master switch for the text buffer implementation: +const USE_CHUNKS_TEXT_BUFFER = true; function createTextBufferBuilder() { + if (USE_CHUNKS_TEXT_BUFFER) { + return new ChunksTextBufferBuilder(); + } return new LinesTextBufferBuilder(); } From d13a91aa096c56ac867f24222b5fdab9d2cbce0f Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Fri, 12 Jan 2018 14:49:39 +0100 Subject: [PATCH 02/33] Reduce GC --- .../chunksTextBuffer/chunksTextBuffer.ts | 142 +++++++++++++----- 1 file changed, 107 insertions(+), 35 deletions(-) diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts index 194031e6e8a..90ee5d8e8b1 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts @@ -93,9 +93,7 @@ export class ChunksTextBuffer implements ITextBuffer { return this.getLineContent(lineNumber).charCodeAt(index); } getLineLength(lineNumber: number): number { - // TODO - let content = this.getLineContent(lineNumber); - return content.length; + return this._actual.getLineLength(lineNumber); } getLineFirstNonWhitespaceColumn(lineNumber: number): number { throw new Error('TODO'); @@ -126,11 +124,18 @@ class BufferNodes { class BufferCursor { constructor( - public readonly offset, - public readonly leafIndex, - public readonly leafStartOffset, - public readonly leafStartNewLineCount + public offset: number, + public leafIndex: number, + public leafStartOffset: number, + public leafStartNewLineCount: number ) { } + + public set(offset: number, leafIndex: number, leafStartOffset: number, leafStartNewLineCount: number) { + this.offset = offset; + this.leafIndex = leafIndex; + this.leafStartOffset = leafStartOffset; + this.leafStartNewLineCount = leafStartNewLineCount; + } } class OffsetLenEdit { @@ -160,6 +165,39 @@ class LeafReplacement { ) { } } +const BUFFER_CURSOR_POOL_SIZE = 10; +const BufferCursorPool = new class { + private _pool: BufferCursor[]; + private _len: number; + + constructor() { + this._pool = []; + for (let i = 0; i < BUFFER_CURSOR_POOL_SIZE; i++) { + this._pool[i] = new BufferCursor(0, 0, 0, 0); + } + this._len = this._pool.length; + } + + public put(cursor: BufferCursor): void { + if (this._len > this._pool.length) { + // oh, well + return; + } + this._pool[this._len++] = cursor; + } + + public take(): BufferCursor { + if (this._len === 0) { + // oh, well + console.log(`insufficient BufferCursor pool`); + return new BufferCursor(0, 0, 0, 0); + } + const result = this._pool[this._len - 1]; + this._pool[this._len--] = null; + return result; + } +}; + class Buffer { private _minLeafLength: number; @@ -232,9 +270,9 @@ class Buffer { this._nodes.newLineCount[nodeIndex] = newLineCount; } - public findOffset(offset: number): BufferCursor { + private _findOffset(offset: number, result: BufferCursor): boolean { if (offset > this._nodes.length[1]) { - return null; + return false; } let it = 1; @@ -276,12 +314,15 @@ class Buffer { } it = this.NODE_TO_LEAF_INDEX(it); - return new BufferCursor(offset, it, leafStartOffset, leafStartNewLineCount); + result.set(offset, it, leafStartOffset, leafStartNewLineCount); + return true; } - private _findLineStart(lineIndex: number): BufferCursor { + private _findLineStart(lineNumber: number, result: BufferCursor): boolean { + let lineIndex = lineNumber - 1; if (lineIndex < 0 || lineIndex > this._nodes.newLineCount[1]) { - return null; + result.set(0, 0, 0, 0); + return false; } let it = 1; @@ -318,10 +359,11 @@ class Buffer { const innerLineStartOffset = (lineIndex === 0 ? 0 : this._leafs[it].lineStartFor(lineIndex - 1)); - return new BufferCursor(leafStartOffset + innerLineStartOffset, it, leafStartOffset, leafStartNewLineCount); + result.set(leafStartOffset + innerLineStartOffset, it, leafStartOffset, leafStartNewLineCount); + return true; } - private _findLineEnd(start: BufferCursor, lineNumber: number): BufferCursor { + private _findLineEnd(start: BufferCursor, lineNumber: number, result: BufferCursor): void { let innerLineIndex = lineNumber - 1 - start.leafStartNewLineCount; const leafsCount = this._leafs.length; @@ -333,13 +375,15 @@ class Buffer { if (innerLineIndex < leaf.newLineCount()) { const lineEndOffset = this._leafs[leafIndex].lineStartFor(innerLineIndex); - return new BufferCursor(leafStartOffset + lineEndOffset, leafIndex, leafStartOffset, leafStartNewLineCount); + result.set(leafStartOffset + lineEndOffset, leafIndex, leafStartOffset, leafStartNewLineCount); + return; } leafIndex++; if (leafIndex >= leafsCount) { - return new BufferCursor(leafStartOffset + leaf.length(), leafIndex - 1, leafStartOffset, leafStartNewLineCount); + result.set(leafStartOffset + leaf.length(), leafIndex - 1, leafStartOffset, leafStartNewLineCount); + return; } leafStartOffset += leaf.length(); @@ -348,15 +392,13 @@ class Buffer { } } - public findLine(lineNumber: number): [BufferCursor, BufferCursor] { - const innerLineIndex = lineNumber - 1; - const start = this._findLineStart(innerLineIndex); - if (!start) { - return null; + private _findLine(lineNumber: number, start: BufferCursor, end: BufferCursor): boolean { + if (!this._findLineStart(lineNumber, start)) { + return false; } - const end = this._findLineEnd(start, lineNumber); - return [start, end]; + this._findLineEnd(start, lineNumber, end); + return true; } public getLineCount(): number { @@ -364,12 +406,37 @@ class Buffer { } public getLineContent(lineNumber: number): string { - const r = this.findLine(lineNumber); - if (!r) { + const start = BufferCursorPool.take(); + const end = BufferCursorPool.take(); + + if (!this._findLine(lineNumber, start, end)) { + BufferCursorPool.put(start); + BufferCursorPool.put(end); throw new Error(`Line not found`); } - const [start, end] = r; - return this.extractString(start, end.offset - start.offset); + + const result = this.extractString(start, end.offset - start.offset); + + BufferCursorPool.put(start); + BufferCursorPool.put(end); + return result; + } + + public getLineLength(lineNumber: number): number { + const start = BufferCursorPool.take(); + const end = BufferCursorPool.take(); + + if (!this._findLine(lineNumber, start, end)) { + BufferCursorPool.put(start); + BufferCursorPool.put(end); + throw new Error(`Line not found`); + } + + const result = end.offset - start.offset; + + BufferCursorPool.put(start); + BufferCursorPool.put(end); + return result; } public getLinesContent(): string[] { @@ -435,11 +502,16 @@ class Buffer { } public getOffsetAt(lineNumber: number, column: number): number { - const start = this._findLineStart(lineNumber - 1); - if (!start) { + const start = BufferCursorPool.take(); + + if (!this._findLineStart(lineNumber, start)) { + BufferCursorPool.put(start); throw new Error(`Line not found`); } - return start.offset + column - 1; + const result = start.offset + column - 1; + + BufferCursorPool.put(start); + return result; } //#region Editing @@ -468,13 +540,13 @@ class Buffer { edits = this._mergeAdjacentEdits(edits); let result: InternalOffsetLenEdit[] = []; - let tmp: BufferCursor; + let tmp = new BufferCursor(0, 0, 0, 0); for (let i = 0, len = edits.length; i < len; i++) { const edit = edits[i]; let text = edit.text; - tmp = this.findOffset(edit.offset); + this._findOffset(edit.offset, tmp); let startLeafIndex = tmp.leafIndex; let startInnerOffset = tmp.offset - tmp.leafStartOffset; if (startInnerOffset > 0) { @@ -484,13 +556,13 @@ class Buffer { // include the replacement of \r in the edit text = '\r' + text; - tmp = this.findOffset(edit.offset - 1); + this._findOffset(edit.offset - 1, tmp); startLeafIndex = tmp.leafIndex; startInnerOffset = tmp.offset - tmp.leafStartOffset; } } - tmp = this.findOffset(edit.offset + edit.length); + this._findOffset(edit.offset + edit.length, tmp); let endLeafIndex = tmp.leafIndex; let endInnerOffset = tmp.offset - tmp.leafStartOffset; const endLeaf = this._leafs[endLeafIndex]; @@ -500,7 +572,7 @@ class Buffer { // include the replacement of \n in the edit text = text + '\n'; - tmp = this.findOffset(edit.offset + edit.length + 1); + this._findOffset(edit.offset + edit.length + 1, tmp); endLeafIndex = tmp.leafIndex; endInnerOffset = tmp.offset - tmp.leafStartOffset; } From 88d0451e0679a64a13fdb6138a3742a5a4764b2d Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Fri, 12 Jan 2018 15:05:40 +0100 Subject: [PATCH 03/33] Better EOL handling --- .../model/chunksTextBuffer/bufferPiece.ts | 17 ++++-- .../chunksTextBuffer/chunksTextBuffer.ts | 24 +++++---- .../chunksTextBufferBuilder.ts | 53 +++++++++++++++---- 3 files changed, 71 insertions(+), 23 deletions(-) diff --git a/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts b/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts index cbe35718d99..d682c373eab 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts @@ -23,7 +23,7 @@ export class BufferPiece { constructor(str: string, lineStarts: Uint32Array = null) { this._str = str; if (lineStarts === null) { - this._lineStarts = createUint32Array(createLineStarts(str)); + this._lineStarts = createUint32Array(createLineStarts(str).lineStarts); } else { this._lineStarts = lineStarts; } @@ -192,18 +192,27 @@ function min(a: number, b: number): number { return (a < b ? a : b); } -function createUint32Array(arr: number[]): Uint32Array { +export function createUint32Array(arr: number[]): Uint32Array { let r = new Uint32Array(arr.length); r.set(arr, 0); return r; } -function createLineStarts(str: string): number[] { +export class LineStarts { + constructor( + public readonly lineStarts: number[], + public readonly carriageReturnCnt: number + ) { } +} + +export function createLineStarts(str: string): LineStarts { let r: number[] = [], rLength = 0; + let carriageReturnCnt = 0; for (let i = 0, len = str.length; i < len; i++) { const chr = str.charCodeAt(i); if (chr === CharCode.CarriageReturn) { + carriageReturnCnt++; if (i + 1 < len && str.charCodeAt(i + 1) === CharCode.LineFeed) { // \r\n... case r[rLength++] = i + 2; @@ -216,5 +225,5 @@ function createLineStarts(str: string): number[] { r[rLength++] = i + 1; } } - return r; + return new LineStarts(r, carriageReturnCnt); } diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts index 90ee5d8e8b1..12f494dd734 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts @@ -14,12 +14,12 @@ export class ChunksTextBuffer implements ITextBuffer { private _actual: Buffer; - constructor(pieces: BufferPiece[], _averageChunkSize: number) { + constructor(pieces: BufferPiece[], _averageChunkSize: number, eol: '\r\n' | '\n') { const averageChunkSize = Math.floor(Math.min(65536.0, Math.max(128.0, _averageChunkSize))); const delta = Math.floor(averageChunkSize / 3); const min = averageChunkSize - delta; const max = 2 * min; - this._actual = new Buffer(pieces, min, max); + this._actual = new Buffer(pieces, min, max, eol); } equals(other: ITextBuffer): boolean { @@ -204,13 +204,16 @@ class Buffer { private _maxLeafLength: number; private _idealLeafLength: number; + private _eol: '\r\n' | '\n'; + private _eolLength: number; + private _leafs: BufferPiece[]; private _nodes: BufferNodes; private _nodesCount: number; private _leafsStart: number; private _leafsEnd: number; - constructor(pieces: BufferPiece[], minLeafLength: number, maxLeafLength: number) { + constructor(pieces: BufferPiece[], minLeafLength: number, maxLeafLength: number, eol: '\r\n' | '\n') { if (!(2 * minLeafLength >= maxLeafLength)) { throw new Error(`assertion violation`); } @@ -219,6 +222,9 @@ class Buffer { this._maxLeafLength = maxLeafLength; this._idealLeafLength = (minLeafLength + maxLeafLength) >>> 1; + this._eol = eol; + this._eolLength = this._eol.length; + this._leafs = pieces; this._nodes = null; this._nodesCount = 0; @@ -228,6 +234,10 @@ class Buffer { this._rebuildNodes(); } + public getEOL(): string { + return this._eol; + } + private _rebuildNodes() { const leafsCount = this._leafs.length; @@ -457,13 +467,7 @@ class Buffer { let leafSubstrOffset = 0; for (let newLineIndex = 0; newLineIndex < leafNewLineCount; newLineIndex++) { const newLineStart = leaf.lineStartFor(newLineIndex); - let length = newLineStart - leafSubstrOffset - 1 /*-1 for EOL*/; - - if (length > 0 && leaf.charCodeAt(leafSubstrOffset + length - 1) === CharCode.CarriageReturn) { - // \r\n case - length--; - } - currentLine += leaf.substr(leafSubstrOffset, length); + currentLine += leaf.substr(leafSubstrOffset, newLineStart - leafSubstrOffset - this._eolLength); result[resultIndex++] = currentLine; currentLine = ''; diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts index ddd56eab383..fd3d525e368 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts @@ -6,20 +6,44 @@ import * as strings from 'vs/base/common/strings'; import { ITextBufferBuilder, ITextBufferFactory, ITextBuffer, DefaultEndOfLine } from 'vs/editor/common/model'; -import { BufferPiece } from 'vs/editor/common/model/chunksTextBuffer/bufferPiece'; +import { BufferPiece, createLineStarts, createUint32Array } from 'vs/editor/common/model/chunksTextBuffer/bufferPiece'; import { ChunksTextBuffer } from 'vs/editor/common/model/chunksTextBuffer/chunksTextBuffer'; +import { CharCode } from 'vs/base/common/charCode'; export class TextBufferFactory implements ITextBufferFactory { constructor( private readonly _pieces: BufferPiece[], - private readonly _averageChunkSize: number + private readonly _averageChunkSize: number, + private readonly _totalCRCount: number, + private readonly _totalEOLCount: number ) { } + /** + * if text source is empty or with precisely one line, returns null. No end of line is detected. + * if text source contains more lines ending with '\r\n', returns '\r\n'. + * Otherwise returns '\n'. More lines end with '\n'. + */ + private _getEOL(defaultEOL: DefaultEndOfLine): '\r\n' | '\n' { + if (this._totalEOLCount === 0) { + // This is an empty file or a file with precisely one line + return (defaultEOL === DefaultEndOfLine.LF ? '\n' : '\r\n'); + } + if (this._totalCRCount > this._totalEOLCount / 2) { + // More than half of the file contains \r\n ending lines + return '\r\n'; + } + // At least one line more ends in \n + return '\n'; + } + public create(defaultEOL: DefaultEndOfLine): ITextBuffer { - // TODO: CRLF vs LF - return new ChunksTextBuffer(this._pieces, this._averageChunkSize); + if (this._totalCRCount > 0 && this._totalCRCount !== this._totalEOLCount) { + // TODO + console.warn(`mixed line endings not handled correctly at this time!`); + } + return new ChunksTextBuffer(this._pieces, this._averageChunkSize, this._getEOL(defaultEOL)); } public getFirstLineText(lengthLimit: number): string { @@ -35,7 +59,8 @@ export class ChunksTextBufferBuilder implements ITextBufferBuilder { private _previousChar: number; private _averageChunkSize: number; - // private totalCRCount: number; + private totalCRCount: number; + private totalEOLCount: number; private containsRTL: boolean; private isBasicASCII: boolean; @@ -45,7 +70,8 @@ export class ChunksTextBufferBuilder implements ITextBufferBuilder { this._previousChar = 0; this._averageChunkSize = 0; - // this.totalCRCount = 0; + this.totalCRCount = 0; + this.totalEOLCount = 0; this.containsRTL = false; this.isBasicASCII = true; } @@ -68,7 +94,7 @@ export class ChunksTextBufferBuilder implements ITextBufferBuilder { this._averageChunkSize = (this._averageChunkSize * this._rawPieces.length + chunk.length) / (this._rawPieces.length + 1); const lastChar = chunk.charCodeAt(chunk.length - 1); - if (lastChar === 13 || (lastChar >= 0xd800 && lastChar <= 0xdbff)) { + if (lastChar === CharCode.CarriageReturn || (lastChar >= 0xd800 && lastChar <= 0xdbff)) { // last character is \r or a high surrogate => keep it back this._acceptChunk1(chunk.substring(0, chunk.length - 1), false); this._hasPreviousChar = true; @@ -101,12 +127,17 @@ export class ChunksTextBufferBuilder implements ITextBufferBuilder { } private _acceptChunk2(chunk: string): void { - this._rawPieces.push(new BufferPiece(chunk)); + const lineStarts = createLineStarts(chunk); + + this._rawPieces.push(new BufferPiece(chunk, createUint32Array(lineStarts.lineStarts))); + this.totalCRCount += lineStarts.carriageReturnCnt; + this.totalEOLCount += lineStarts.lineStarts.length; } public finish(): TextBufferFactory { this._finish(); - return new TextBufferFactory(this._rawPieces, this._averageChunkSize); + console.log(`${this.totalCRCount}, ${this.totalEOLCount}`); + return new TextBufferFactory(this._rawPieces, this._averageChunkSize, this.totalCRCount, this.totalEOLCount); } private _finish(): void { @@ -124,6 +155,10 @@ export class ChunksTextBufferBuilder implements ITextBufferBuilder { const tmp = new BufferPiece(String.fromCharCode(this._previousChar)); const newLastPiece = BufferPiece.join(lastPiece, tmp); this._rawPieces[this._rawPieces.length - 1] = newLastPiece; + if (this._previousChar === CharCode.CarriageReturn) { + this.totalCRCount++; + this.totalEOLCount++; + } } } } From 675929e543313240fd09fc4097e24a9ca9779c10 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Fri, 12 Jan 2018 15:45:23 +0100 Subject: [PATCH 04/33] Improve getValueInRange --- .../chunksTextBuffer/chunksTextBuffer.ts | 143 ++++++++++++++---- 1 file changed, 112 insertions(+), 31 deletions(-) diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts index 12f494dd734..f49a5ff3efa 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts @@ -51,30 +51,32 @@ export class ChunksTextBuffer implements ITextBuffer { throw new Error('TODO'); } getValueInRange(range: Range, eol: EndOfLinePreference): string { - // TODO - if (range.isEmpty()) { return ''; } - if (range.startLineNumber === range.endLineNumber) { - return this.getLineContent(range.startLineNumber).substring(range.startColumn - 1, range.endColumn - 1); + const text = this._actual.getValueInRange(range); + switch (eol) { + case EndOfLinePreference.TextDefined: + return text; + case EndOfLinePreference.LF: + if (this.getEOL() === '\n') { + return text; + } else { + return text.replace(/\r\n/g, '\n'); + } + case EndOfLinePreference.CRLF: + if (this.getEOL() === '\r\n') { + return text; + } else { + return text.replace(/\n/g, '\r\n'); + } } - - var lineEnding = '\n',//todo this._getEndOfLine(eol), - startLineIndex = range.startLineNumber - 1, - endLineIndex = range.endLineNumber - 1, - resultLines: string[] = []; - - resultLines.push(this.getLineContent(startLineIndex + 1).substring(range.startColumn - 1)); - for (var i = startLineIndex + 1; i < endLineIndex; i++) { - resultLines.push(this.getLineContent(i + 1)); - } - resultLines.push(this.getLineContent(endLineIndex + 1).substring(0, range.endColumn - 1)); - - return resultLines.join(lineEnding); + return null; } + getValueLengthInRange(range: Range, eol: EndOfLinePreference): number { + // TODO return this.getValueInRange(range, eol).length; } @@ -86,8 +88,7 @@ export class ChunksTextBuffer implements ITextBuffer { return this._actual.getLinesContent(); } getLineContent(lineNumber: number): string { - // TODO - return this._actual.getLineContent(lineNumber).replace(/\r?\n?/, ''); + return this._actual.getLineContent(lineNumber); } getLineCharCode(lineNumber: number, index: number): number { return this.getLineContent(lineNumber).charCodeAt(index); @@ -328,6 +329,39 @@ class Buffer { return true; } + private _findOffsetCloseAfter(offset: number, start: BufferCursor, result: BufferCursor): boolean { + if (offset > this._nodes.length[1]) { + return false; + } + + let innerOffset = offset - start.leafStartOffset; + const leafsCount = this._leafs.length; + + let leafIndex = start.leafIndex; + let leafStartOffset = start.leafStartOffset; + let leafStartNewLineCount = start.leafStartNewLineCount; + + while (true) { + const leaf = this._leafs[leafIndex]; + + if (innerOffset < leaf.length()) { + result.set(offset, leafIndex, leafStartOffset, leafStartNewLineCount); + return true; + } + + leafIndex++; + + if (leafIndex >= leafsCount) { + result.set(offset, leafIndex, leafStartOffset, leafStartNewLineCount); + return true; + } + + leafStartOffset += leaf.length(); + leafStartNewLineCount += leaf.newLineCount(); + innerOffset -= leaf.length(); + } + } + private _findLineStart(lineNumber: number, result: BufferCursor): boolean { let lineIndex = lineNumber - 1; if (lineIndex < 0 || lineIndex > this._nodes.newLineCount[1]) { @@ -425,7 +459,7 @@ class Buffer { throw new Error(`Line not found`); } - const result = this.extractString(start, end.offset - start.offset); + const result = this.extractString(start, end.offset - start.offset - this._eolLength); BufferCursorPool.put(start); BufferCursorPool.put(end); @@ -442,7 +476,7 @@ class Buffer { throw new Error(`Line not found`); } - const result = end.offset - start.offset; + const result = end.offset - start.offset - this._eolLength; BufferCursorPool.put(start); BufferCursorPool.put(end); @@ -505,16 +539,56 @@ class Buffer { return res; } - public getOffsetAt(lineNumber: number, column: number): number { - const start = BufferCursorPool.take(); + private _getOffsetAt(lineNumber: number, column: number, result: BufferCursor): boolean { + const lineStart = BufferCursorPool.take(); - if (!this._findLineStart(lineNumber, start)) { + if (!this._findLineStart(lineNumber, lineStart)) { + BufferCursorPool.put(lineStart); + return false; + } + + const startOffset = lineStart.offset + column - 1; + if (!this._findOffsetCloseAfter(startOffset, lineStart, result)) { + BufferCursorPool.put(lineStart); + return false; + } + + BufferCursorPool.put(lineStart); + return true; + } + + public getOffsetAt(lineNumber: number, column: number): number { + const offset = BufferCursorPool.take(); + + if (!this._getOffsetAt(lineNumber, column, offset)) { + BufferCursorPool.put(offset); + throw new Error(`Position not found`); + } + + BufferCursorPool.put(offset); + return offset.offset; + } + + public getValueInRange(range: Range): string { + const start = BufferCursorPool.take(); + const end = BufferCursorPool.take(); + + if (!this._getOffsetAt(range.startLineNumber, range.startColumn, start)) { BufferCursorPool.put(start); + BufferCursorPool.put(end); throw new Error(`Line not found`); } - const result = start.offset + column - 1; + + if (!this._getOffsetAt(range.endLineNumber, range.endColumn, end)) { + BufferCursorPool.put(start); + BufferCursorPool.put(end); + throw new Error(`Line not found`); + } + + const result = this.extractString(start, end.offset - start.offset); BufferCursorPool.put(start); + BufferCursorPool.put(end); return result; } @@ -545,6 +619,7 @@ class Buffer { let result: InternalOffsetLenEdit[] = []; let tmp = new BufferCursor(0, 0, 0, 0); + let tmp2 = new BufferCursor(0, 0, 0, 0); for (let i = 0, len = edits.length; i < len; i++) { const edit = edits[i]; @@ -560,9 +635,12 @@ class Buffer { // include the replacement of \r in the edit text = '\r' + text; - this._findOffset(edit.offset - 1, tmp); - startLeafIndex = tmp.leafIndex; - startInnerOffset = tmp.offset - tmp.leafStartOffset; + this._findOffsetCloseAfter(edit.offset - 1, tmp, tmp2); + startLeafIndex = tmp2.leafIndex; + startInnerOffset = tmp2.offset - tmp2.leafStartOffset; + // this._findOffset(edit.offset - 1, tmp); + // startLeafIndex = tmp.leafIndex; + // startInnerOffset = tmp.offset - tmp.leafStartOffset; } } @@ -576,9 +654,12 @@ class Buffer { // include the replacement of \n in the edit text = text + '\n'; - this._findOffset(edit.offset + edit.length + 1, tmp); - endLeafIndex = tmp.leafIndex; - endInnerOffset = tmp.offset - tmp.leafStartOffset; + this._findOffsetCloseAfter(edit.offset + edit.length + 1, tmp, tmp2); + startLeafIndex = tmp2.leafIndex; + startInnerOffset = tmp2.offset - tmp2.leafStartOffset; + // this._findOffset(edit.offset + edit.length + 1, tmp); + // endLeafIndex = tmp.leafIndex; + // endInnerOffset = tmp.offset - tmp.leafStartOffset; } } From d2aa176871cb6c79effc58d8e1870c1d75cbec28 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Fri, 12 Jan 2018 15:54:50 +0100 Subject: [PATCH 05/33] Better implementation for getValueLengthInRange --- .../chunksTextBuffer/chunksTextBuffer.ts | 56 +++++++++++++++++-- 1 file changed, 52 insertions(+), 4 deletions(-) diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts index f49a5ff3efa..b7601d6a9b9 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts @@ -76,26 +76,51 @@ export class ChunksTextBuffer implements ITextBuffer { } getValueLengthInRange(range: Range, eol: EndOfLinePreference): number { - - // TODO - return this.getValueInRange(range, eol).length; + if (range.isEmpty()) { + return 0; + } + const eolCount = range.endLineNumber - range.startLineNumber; + const result = this._actual.getValueLengthInRange(range); + switch (eol) { + case EndOfLinePreference.TextDefined: + return result; + case EndOfLinePreference.LF: + if (this.getEOL() === '\n') { + return result; + } else { + return result - eolCount; // \r\n => \n + } + case EndOfLinePreference.CRLF: + if (this.getEOL() === '\r\n') { + return result; + } else { + return result + eolCount; // \n => \r\n + } + } + return 0; } + getLineCount(): number { - // TODO: perhaps cache? return this._actual.getLineCount(); } + getLinesContent(): string[] { return this._actual.getLinesContent(); } + getLineContent(lineNumber: number): string { return this._actual.getLineContent(lineNumber); } + getLineCharCode(lineNumber: number, index: number): number { + // TODO return this.getLineContent(lineNumber).charCodeAt(index); } + getLineLength(lineNumber: number): number { return this._actual.getLineLength(lineNumber); } + getLineFirstNonWhitespaceColumn(lineNumber: number): number { throw new Error('TODO'); } @@ -592,6 +617,29 @@ class Buffer { return result; } + public getValueLengthInRange(range: Range): number { + const start = BufferCursorPool.take(); + const end = BufferCursorPool.take(); + + if (!this._getOffsetAt(range.startLineNumber, range.startColumn, start)) { + BufferCursorPool.put(start); + BufferCursorPool.put(end); + throw new Error(`Line not found`); + } + + if (!this._getOffsetAt(range.endLineNumber, range.endColumn, end)) { + BufferCursorPool.put(start); + BufferCursorPool.put(end); + throw new Error(`Line not found`); + } + + const result = end.offset - start.offset; + + BufferCursorPool.put(start); + BufferCursorPool.put(end); + return result; + } + //#region Editing private _mergeAdjacentEdits(edits: OffsetLenEdit[]): OffsetLenEdit[] { From a983217a4d0111275e7a8737a001607ad93b24bc Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Fri, 12 Jan 2018 16:20:44 +0100 Subject: [PATCH 06/33] Implement offset -> (ln;col) conversion --- .../model/chunksTextBuffer/bufferPiece.ts | 17 ++++ .../chunksTextBuffer/chunksTextBuffer.ts | 81 ++++++++++++++++--- 2 files changed, 88 insertions(+), 10 deletions(-) diff --git a/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts b/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts index d682c373eab..cc1a887589d 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts @@ -49,6 +49,23 @@ export class BufferPiece { return this._str.substr(from, length); } + public findLineStartBeforeOffset(offset: number): number { + if (this._lineStarts.length === 0 || offset < this._lineStarts[0]) { + return -1; + } + + // TODO: implement binary search + for (let i = this._lineStarts.length - 1; i >= 0; i--) { + let lineStart = this._lineStarts[i]; + + if (lineStart <= offset) { + return i; + } + } + + return -1; + } + public static deleteLastChar(target: BufferPiece): BufferPiece { const targetCharsLength = target.length(); const targetLineStartsLength = target.newLineCount(); diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts index b7601d6a9b9..1daefc55226 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts @@ -38,14 +38,13 @@ export class ChunksTextBuffer implements ITextBuffer { return ''; } getEOL(): string { - // TODO - return '\n'; + return this._actual.getEOL(); } getOffsetAt(lineNumber: number, column: number): number { - return this._actual.getOffsetAt(lineNumber, column); + return this._actual.convertPositionToOffset(lineNumber, column); } getPositionAt(offset: number): Position { - throw new Error('TODO'); + return this._actual.convertOffsetToPosition(offset); } getRangeAt(offset: number, length: number): Range { throw new Error('TODO'); @@ -582,16 +581,78 @@ class Buffer { return true; } - public getOffsetAt(lineNumber: number, column: number): number { - const offset = BufferCursorPool.take(); + public convertPositionToOffset(lineNumber: number, column: number): number { + const r = BufferCursorPool.take(); - if (!this._getOffsetAt(lineNumber, column, offset)) { - BufferCursorPool.put(offset); + if (!this._getOffsetAt(lineNumber, column, r)) { + BufferCursorPool.put(r); throw new Error(`Position not found`); } - BufferCursorPool.put(offset); - return offset.offset; + const result = r.offset; + + BufferCursorPool.put(r); + return result; + } + + /** + * returns `lineNumber` + */ + private _findLineStartBeforeOffsetInLeaf(offset: number, leafIndex: number, leafStartOffset: number, leafStartNewLineCount: number, result: BufferCursor): number { + const leaf = this._leafs[leafIndex]; + const lineStartIndex = leaf.findLineStartBeforeOffset(offset - leafStartOffset); + const lineStartOffset = leafStartOffset = leaf.lineStartFor(lineStartIndex); + + result.set(lineStartOffset, leafIndex, leafStartOffset, leafStartNewLineCount); + return leafStartNewLineCount + lineStartIndex + 1; + } + + /** + * returns `lineNumber`. + */ + private _findLineStartBeforeOffset(offset: number, location: BufferCursor, result: BufferCursor): number { + + let leafIndex = location.leafIndex; + let leafStartOffset = location.leafStartOffset; + let leafStartNewLineCount = location.leafStartNewLineCount; + while (true) { + const leaf = this._leafs[leafIndex]; + + if (leaf.newLineCount() > 1 && leaf.lineStartFor(0) + leafStartOffset >= offset) { + // must be in this leaf + return this._findLineStartBeforeOffsetInLeaf(offset, leafIndex, leafStartOffset, leafStartNewLineCount, result); + } + + // continue looking in previous leaf + leafIndex--; + + if (leafIndex < 0) { + result.set(0, 0, 0, 0); + return 1; + } + + leafStartOffset -= this._leafs[leafIndex].length(); + leafStartNewLineCount -= this._leafs[leafIndex].newLineCount(); + } + } + + public convertOffsetToPosition(offset: number): Position { + const r = BufferCursorPool.take(); + const lineStart = BufferCursorPool.take(); + + if (!this._findOffset(offset, r)) { + BufferCursorPool.put(r); + BufferCursorPool.put(lineStart); + throw new Error(`Offset not found`); + } + + const lineNumber = this._findLineStartBeforeOffset(offset, r, lineStart); + const column = offset - lineStart.offset + 1; + + BufferCursorPool.put(r); + BufferCursorPool.put(lineStart); + + return new Position(lineNumber, column); } public getValueInRange(range: Range): string { From 612db35849987c3a2f9c22ddd6b964f7f60f941c Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Fri, 12 Jan 2018 16:28:39 +0100 Subject: [PATCH 07/33] Implement more methods --- .../chunksTextBuffer/chunksTextBuffer.ts | 40 ++++++++++++++++--- .../chunksTextBufferBuilder.ts | 4 +- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts index 1daefc55226..1bf8fb01079 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts @@ -13,25 +13,27 @@ import { Range } from 'vs/editor/common/core/range'; export class ChunksTextBuffer implements ITextBuffer { private _actual: Buffer; + private _mightContainRTL: boolean; + private _mightContainNonBasicASCII: boolean; - constructor(pieces: BufferPiece[], _averageChunkSize: number, eol: '\r\n' | '\n') { + constructor(pieces: BufferPiece[], _averageChunkSize: number, eol: '\r\n' | '\n', containsRTL: boolean, isBasicASCII: boolean) { const averageChunkSize = Math.floor(Math.min(65536.0, Math.max(128.0, _averageChunkSize))); const delta = Math.floor(averageChunkSize / 3); const min = averageChunkSize - delta; const max = 2 * min; this._actual = new Buffer(pieces, min, max, eol); + this._mightContainRTL = containsRTL; + this._mightContainNonBasicASCII = !isBasicASCII; } equals(other: ITextBuffer): boolean { throw new Error('TODO'); } mightContainRTL(): boolean { - // TODO - return true; + return this._mightContainRTL; } mightContainNonBasicASCII(): boolean { - // TODO - return true; + return this._mightContainNonBasicASCII; } getBOM(): string { // TODO @@ -47,7 +49,7 @@ export class ChunksTextBuffer implements ITextBuffer { return this._actual.convertOffsetToPosition(offset); } getRangeAt(offset: number, length: number): Range { - throw new Error('TODO'); + return this._actual.convertOffsetLenToRange(offset, length); } getValueInRange(range: Range, eol: EndOfLinePreference): string { if (range.isEmpty()) { @@ -655,6 +657,32 @@ class Buffer { return new Position(lineNumber, column); } + public convertOffsetLenToRange(offset: number, len: number): Range { + const r = BufferCursorPool.take(); + const lineStart = BufferCursorPool.take(); + + if (!this._findOffset(offset, r)) { + BufferCursorPool.put(r); + BufferCursorPool.put(lineStart); + throw new Error(`Offset not found`); + } + const startLineNumber = this._findLineStartBeforeOffset(offset, r, lineStart); + const startColumn = offset - lineStart.offset + 1; + + if (!this._findOffset(offset + len, r)) { + BufferCursorPool.put(r); + BufferCursorPool.put(lineStart); + throw new Error(`Offset not found`); + } + const endLineNumber = this._findLineStartBeforeOffset(offset + len, r, lineStart); + const endColumn = offset + len - lineStart.offset + 1; + + BufferCursorPool.put(r); + BufferCursorPool.put(lineStart); + + return new Range(startLineNumber, startColumn, endLineNumber, endColumn); + } + public getValueInRange(range: Range): string { const start = BufferCursorPool.take(); const end = BufferCursorPool.take(); diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts index fd3d525e368..1a16096fbbd 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts @@ -16,7 +16,9 @@ export class TextBufferFactory implements ITextBufferFactory { private readonly _pieces: BufferPiece[], private readonly _averageChunkSize: number, private readonly _totalCRCount: number, - private readonly _totalEOLCount: number + private readonly _totalEOLCount: number, + private readonly _containsRTL: boolean, + private readonly _isBasicASCII: boolean, ) { } From c609af00ced16cf62142f57b4d32c9d9d10f3ee2 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Mon, 15 Jan 2018 16:32:40 +0100 Subject: [PATCH 08/33] Move RawContentChange events creation to TextModel --- src/vs/editor/common/model.ts | 3 +- .../model/linesTextBuffer/linesTextBuffer.ts | 24 ++---- src/vs/editor/common/model/textModel.ts | 74 +++++++++++++++++-- .../common/viewModel/splitLinesCollection.ts | 1 + src/vs/editor/test/common/model/model.test.ts | 3 - 5 files changed, 78 insertions(+), 27 deletions(-) diff --git a/src/vs/editor/common/model.ts b/src/vs/editor/common/model.ts index 9f25ed1118f..9d58b81cc05 100644 --- a/src/vs/editor/common/model.ts +++ b/src/vs/editor/common/model.ts @@ -12,7 +12,7 @@ import { IDisposable } from 'vs/base/common/lifecycle'; import { Position, IPosition } from 'vs/editor/common/core/position'; import { Range, IRange } from 'vs/editor/common/core/range'; import { Selection } from 'vs/editor/common/core/selection'; -import { ModelRawContentChangedEvent, IModelContentChangedEvent, IModelDecorationsChangedEvent, IModelLanguageChangedEvent, IModelOptionsChangedEvent, IModelLanguageConfigurationChangedEvent, IModelTokensChangedEvent, IModelContentChange, ModelRawChange } from 'vs/editor/common/model/textModelEvents'; +import { ModelRawContentChangedEvent, IModelContentChangedEvent, IModelDecorationsChangedEvent, IModelLanguageChangedEvent, IModelOptionsChangedEvent, IModelLanguageConfigurationChangedEvent, IModelTokensChangedEvent, IModelContentChange } from 'vs/editor/common/model/textModelEvents'; import { ThemeColor } from 'vs/platform/theme/common/themeService'; /** @@ -1098,7 +1098,6 @@ export class ApplyEditsResult { constructor( public readonly reverseEdits: IIdentifiedSingleEditOperation[], - public readonly rawChanges: ModelRawChange[], public readonly changes: IInternalModelContentChange[], public readonly trimAutoWhitespaceLineNumbers: number[] ) { } diff --git a/src/vs/editor/common/model/linesTextBuffer/linesTextBuffer.ts b/src/vs/editor/common/model/linesTextBuffer/linesTextBuffer.ts index 09cafe2246c..1cbd9fe7c92 100644 --- a/src/vs/editor/common/model/linesTextBuffer/linesTextBuffer.ts +++ b/src/vs/editor/common/model/linesTextBuffer/linesTextBuffer.ts @@ -9,7 +9,6 @@ import { Position } from 'vs/editor/common/core/position'; import * as strings from 'vs/base/common/strings'; import * as arrays from 'vs/base/common/arrays'; import { PrefixSumComputer } from 'vs/editor/common/viewModel/prefixSumComputer'; -import { ModelRawChange, ModelRawLineChanged, ModelRawLinesDeleted, ModelRawLinesInserted } from 'vs/editor/common/model/textModelEvents'; import { ISingleEditOperationIdentifier, IIdentifiedSingleEditOperation, EndOfLinePreference, ITextBuffer, ApplyEditsResult, IInternalModelContentChange } from 'vs/editor/common/model'; export interface IValidatedEditOperation { @@ -252,7 +251,7 @@ export class LinesTextBuffer implements ITextBuffer { public applyEdits(rawOperations: IIdentifiedSingleEditOperation[], recordTrimAutoWhitespace: boolean): ApplyEditsResult { if (rawOperations.length === 0) { - return new ApplyEditsResult([], [], [], []); + return new ApplyEditsResult([], [], []); } let mightContainRTL = this._mightContainRTL; @@ -341,7 +340,7 @@ export class LinesTextBuffer implements ITextBuffer { this._mightContainRTL = mightContainRTL; this._mightContainNonBasicASCII = mightContainNonBasicASCII; - const [rawContentChanges, contentChanges] = this._doApplyEdits(operations); + const contentChanges = this._doApplyEdits(operations); let trimAutoWhitespaceLineNumbers: number[] = null; if (recordTrimAutoWhitespace && newTrimAutoWhitespaceCandidates.length > 0) { @@ -369,7 +368,6 @@ export class LinesTextBuffer implements ITextBuffer { return new ApplyEditsResult( reverseOperations, - rawContentChanges, contentChanges, trimAutoWhitespaceLineNumbers ); @@ -451,18 +449,16 @@ export class LinesTextBuffer implements ITextBuffer { }; } - private _setLineContent(lineNumber: number, content: string, rawContentChanges: ModelRawChange[]): void { + private _setLineContent(lineNumber: number, content: string): void { this._lines[lineNumber - 1] = content; this._lineStarts.changeValue(lineNumber - 1, content.length + this._EOL.length); - rawContentChanges.push(new ModelRawLineChanged(lineNumber, content)); } - private _doApplyEdits(operations: IValidatedEditOperation[]): [ModelRawChange[], IInternalModelContentChange[]] { + private _doApplyEdits(operations: IValidatedEditOperation[]): IInternalModelContentChange[] { // Sort operations descending operations.sort(LinesTextBuffer._sortOpsDescending); - let rawContentChanges: ModelRawChange[] = []; let contentChanges: IInternalModelContentChange[] = []; for (let i = 0, len = operations.length; i < len; i++) { @@ -497,7 +493,7 @@ export class LinesTextBuffer implements ITextBuffer { ); } - this._setLineContent(editLineNumber, editText, rawContentChanges); + this._setLineContent(editLineNumber, editText); } if (editingLinesCnt < deletingLinesCnt) { @@ -507,12 +503,10 @@ export class LinesTextBuffer implements ITextBuffer { const endLineRemains = this._lines[endLineNumber - 1].substring(endColumn - 1); // Reconstruct first line - this._setLineContent(spliceStartLineNumber, this._lines[spliceStartLineNumber - 1] + endLineRemains, rawContentChanges); + this._setLineContent(spliceStartLineNumber, this._lines[spliceStartLineNumber - 1] + endLineRemains); this._lines.splice(spliceStartLineNumber, endLineNumber - spliceStartLineNumber); this._lineStarts.removeValues(spliceStartLineNumber, endLineNumber - spliceStartLineNumber); - - rawContentChanges.push(new ModelRawLinesDeleted(spliceStartLineNumber + 1, endLineNumber)); } if (editingLinesCnt < insertingLinesCnt) { @@ -527,7 +521,7 @@ export class LinesTextBuffer implements ITextBuffer { // Split last line const leftoverLine = this._lines[spliceLineNumber - 1].substring(spliceColumn - 1); - this._setLineContent(spliceLineNumber, this._lines[spliceLineNumber - 1].substring(0, spliceColumn - 1), rawContentChanges); + this._setLineContent(spliceLineNumber, this._lines[spliceLineNumber - 1].substring(0, spliceColumn - 1)); // Lines in the middle let newLines: string[] = new Array(insertingLinesCnt - editingLinesCnt); @@ -540,8 +534,6 @@ export class LinesTextBuffer implements ITextBuffer { newLinesLengths[newLines.length - 1] += leftoverLine.length; this._lines = arrays.arrayInsert(this._lines, startLineNumber + editingLinesCnt, newLines); this._lineStarts.insertValues(startLineNumber + editingLinesCnt, newLinesLengths); - - rawContentChanges.push(new ModelRawLinesInserted(spliceLineNumber + 1, startLineNumber + insertingLinesCnt, newLines)); } const contentChangeRange = new Range(startLineNumber, startColumn, endLineNumber, endColumn); @@ -556,7 +548,7 @@ export class LinesTextBuffer implements ITextBuffer { }); } - return [rawContentChanges, contentChanges]; + return contentChanges; } /** diff --git a/src/vs/editor/common/model/textModel.ts b/src/vs/editor/common/model/textModel.ts index 701c7274e71..023a1adb307 100644 --- a/src/vs/editor/common/model/textModel.ts +++ b/src/vs/editor/common/model/textModel.ts @@ -11,7 +11,7 @@ import { LanguageIdentifier, TokenizationRegistry, LanguageId } from 'vs/editor/ import { EditStack } from 'vs/editor/common/model/editStack'; import { Range, IRange } from 'vs/editor/common/core/range'; import { Selection } from 'vs/editor/common/core/selection'; -import { ModelRawContentChangedEvent, IModelDecorationsChangedEvent, IModelLanguageChangedEvent, IModelLanguageConfigurationChangedEvent, IModelTokensChangedEvent, IModelOptionsChangedEvent, IModelContentChangedEvent, InternalModelContentChangeEvent, ModelRawFlush, ModelRawEOLChanged } from 'vs/editor/common/model/textModelEvents'; +import { ModelRawContentChangedEvent, IModelDecorationsChangedEvent, IModelLanguageChangedEvent, IModelLanguageConfigurationChangedEvent, IModelTokensChangedEvent, IModelOptionsChangedEvent, IModelContentChangedEvent, InternalModelContentChangeEvent, ModelRawFlush, ModelRawEOLChanged, ModelRawChange, ModelRawLineChanged, ModelRawLinesDeleted, ModelRawLinesInserted } from 'vs/editor/common/model/textModelEvents'; import { onUnexpectedError } from 'vs/base/common/errors'; import { IMarkdownString } from 'vs/base/common/htmlContent'; import * as strings from 'vs/base/common/strings'; @@ -1038,21 +1038,83 @@ export class TextModel extends Disposable implements model.ITextModel { } } + private static _eolCount(text: string): number { + let eolCount = 0; + for (let i = 0, len = text.length; i < len; i++) { + const chr = text.charCodeAt(i); + + if (chr === CharCode.CarriageReturn) { + eolCount++; + if (i + 1 < len && text.charCodeAt(i + 1) === CharCode.LineFeed) { + // \r\n... case + i++; // skip \n + } else { + // \r... case + } + } else if (chr === CharCode.LineFeed) { + eolCount++; + } + } + return eolCount; + } + private _applyEdits(rawOperations: model.IIdentifiedSingleEditOperation[]): model.IIdentifiedSingleEditOperation[] { for (let i = 0, len = rawOperations.length; i < len; i++) { rawOperations[i].range = this.validateRange(rawOperations[i].range); } + + const oldLineCount = this._buffer.getLineCount(); const result = this._buffer.applyEdits(rawOperations, this._options.trimAutoWhitespace); - const rawContentChanges = result.rawChanges; + const newLineCount = this._buffer.getLineCount(); + const contentChanges = result.changes; this._trimAutoWhitespaceLines = result.trimAutoWhitespaceLineNumbers; - if (rawContentChanges.length !== 0 || contentChanges.length !== 0) { + if (contentChanges.length !== 0) { + let rawContentChanges: ModelRawChange[] = []; + + let lineCount = oldLineCount; for (let i = 0, len = contentChanges.length; i < len; i++) { - const contentChange = contentChanges[i]; - this._tokens.applyEdits(contentChange.range, contentChange.lines); + const change = contentChanges[i]; + this._tokens.applyEdits(change.range, change.lines); this._onDidChangeDecorations.fire(); - this._decorationsTree.acceptReplace(contentChange.rangeOffset, contentChange.rangeLength, contentChange.text.length, contentChange.forceMoveMarkers); + this._decorationsTree.acceptReplace(change.rangeOffset, change.rangeLength, change.text.length, change.forceMoveMarkers); + + const startLineNumber = change.range.startLineNumber; + const endLineNumber = change.range.endLineNumber; + + const deletingLinesCnt = endLineNumber - startLineNumber; + const insertingLinesCnt = TextModel._eolCount(change.text); + const editingLinesCnt = Math.min(deletingLinesCnt, insertingLinesCnt); + + const changeLineCountDelta = (insertingLinesCnt - deletingLinesCnt); + + for (let j = editingLinesCnt; j >= 0; j--) { + const editLineNumber = startLineNumber + j; + const currentEditLineNumber = newLineCount - lineCount - changeLineCountDelta + editLineNumber; + rawContentChanges.push(new ModelRawLineChanged(editLineNumber, this.getLineContent(currentEditLineNumber))); + } + + if (editingLinesCnt < deletingLinesCnt) { + // Must delete some lines + const spliceStartLineNumber = startLineNumber + editingLinesCnt; + rawContentChanges.push(new ModelRawLinesDeleted(spliceStartLineNumber + 1, endLineNumber)); + } + + if (editingLinesCnt < insertingLinesCnt) { + // Must insert some lines + const spliceLineNumber = startLineNumber + editingLinesCnt; + const cnt = insertingLinesCnt - editingLinesCnt; + const fromLineNumber = newLineCount - lineCount - cnt + spliceLineNumber + 1; + let newLines: string[] = []; + for (let i = 0; i < cnt; i++) { + let lineNumber = fromLineNumber + i; + newLines[lineNumber - fromLineNumber] = this.getLineContent(lineNumber); + } + rawContentChanges.push(new ModelRawLinesInserted(spliceLineNumber + 1, startLineNumber + insertingLinesCnt, newLines)); + } + + lineCount += changeLineCountDelta; } this._increaseVersionId(); diff --git a/src/vs/editor/common/viewModel/splitLinesCollection.ts b/src/vs/editor/common/viewModel/splitLinesCollection.ts index 9855eda6939..24009cea46a 100644 --- a/src/vs/editor/common/viewModel/splitLinesCollection.ts +++ b/src/vs/editor/common/viewModel/splitLinesCollection.ts @@ -403,6 +403,7 @@ export class SplitLinesCollection implements IViewModelLinesCollection { insertPrefixSumValues[i] = outputLineCount; } + // TODO@Alex: use arrays.arrayInsert this.lines = this.lines.slice(0, fromLineNumber - 1).concat(insertLines).concat(this.lines.slice(fromLineNumber - 1)); this.prefixSumComputer.insertValues(fromLineNumber - 1, insertPrefixSumValues); diff --git a/src/vs/editor/test/common/model/model.test.ts b/src/vs/editor/test/common/model/model.test.ts index 92b7e5b714a..e610caa3ddf 100644 --- a/src/vs/editor/test/common/model/model.test.ts +++ b/src/vs/editor/test/common/model/model.test.ts @@ -129,7 +129,6 @@ suite('Editor Model - Model', () => { thisModel.applyEdits([EditOperation.insert(new Position(1, 3), ' new line\nNo longer')]); assert.deepEqual(e, new ModelRawContentChangedEvent( [ - new ModelRawLineChanged(1, 'My new line First Line'), new ModelRawLineChanged(1, 'My new line'), new ModelRawLinesInserted(2, 2, ['No longer First Line']), ], @@ -245,7 +244,6 @@ suite('Editor Model - Model', () => { thisModel.applyEdits([EditOperation.delete(new Range(1, 4, 2, 6))]); assert.deepEqual(e, new ModelRawContentChangedEvent( [ - new ModelRawLineChanged(1, 'My '), new ModelRawLineChanged(1, 'My Second Line'), new ModelRawLinesDeleted(2, 2), ], @@ -266,7 +264,6 @@ suite('Editor Model - Model', () => { thisModel.applyEdits([EditOperation.delete(new Range(1, 4, 3, 5))]); assert.deepEqual(e, new ModelRawContentChangedEvent( [ - new ModelRawLineChanged(1, 'My '), new ModelRawLineChanged(1, 'My Third Line'), new ModelRawLinesDeleted(2, 3), ], From da3b3b61d6751a9c75445b02216d1b2aef5c2938 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Mon, 15 Jan 2018 17:21:48 +0100 Subject: [PATCH 09/33] Hook up editing --- .../model/chunksTextBuffer/bufferPiece.ts | 6 +- .../chunksTextBuffer/chunksTextBuffer.ts | 334 +++++++++++++++++- .../chunksTextBufferBuilder.ts | 6 +- 3 files changed, 334 insertions(+), 12 deletions(-) diff --git a/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts b/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts index cc1a887589d..6672bff60f3 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts @@ -82,7 +82,7 @@ export class BufferPiece { newLineStarts.set(targetLineStarts); // TODO: does this work correctly? return new BufferPiece( - target._str.substring(0, targetCharsLength - 1), + target._str.substr(0, targetCharsLength - 1), newLineStarts ); } @@ -139,7 +139,7 @@ export class BufferPiece { return; } - let pieces: string[]; + let pieces: string[] = new Array(2 * editsSize + 1); let originalFromIndex = 0; let piecesTextLength = 0; for (let i = 0; i < editsSize; i++) { @@ -155,7 +155,7 @@ export class BufferPiece { } // maintain the chars that survive to the right of the last edit - let text = target._str.substring(originalFromIndex, originalCharsLength - originalFromIndex); + let text = target._str.substr(originalFromIndex, originalCharsLength - originalFromIndex); pieces[2 * editsSize] = text; piecesTextLength += text.length; diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts index 1bf8fb01079..1c365f48aa9 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts @@ -5,10 +5,22 @@ 'use strict'; import { CharCode } from 'vs/base/common/charCode'; -import { ITextBuffer, EndOfLinePreference, IIdentifiedSingleEditOperation, ApplyEditsResult } from 'vs/editor/common/model'; +import { ITextBuffer, EndOfLinePreference, IIdentifiedSingleEditOperation, ApplyEditsResult, ISingleEditOperationIdentifier, IInternalModelContentChange } from 'vs/editor/common/model'; import { BufferPiece, LeafOffsetLenEdit } from 'vs/editor/common/model/chunksTextBuffer/bufferPiece'; import { Position } from 'vs/editor/common/core/position'; import { Range } from 'vs/editor/common/core/range'; +import * as strings from 'vs/base/common/strings'; + +export interface IValidatedEditOperation { + sortIndex: number; + identifier: ISingleEditOperationIdentifier; + range: Range; + rangeOffset: number; + rangeLength: number; + lines: string[]; + forceMoveMarkers: boolean; + isAutoWhitespaceEdit: boolean; +} export class ChunksTextBuffer implements ITextBuffer { @@ -131,8 +143,318 @@ export class ChunksTextBuffer implements ITextBuffer { setEOL(newEOL: string): void { throw new Error('TODO'); } + + private static _sortOpsAscending(a: IValidatedEditOperation, b: IValidatedEditOperation): number { + let r = Range.compareRangesUsingEnds(a.range, b.range); + if (r === 0) { + return a.sortIndex - b.sortIndex; + } + return r; + } + + private static _sortOpsDescending(a: IValidatedEditOperation, b: IValidatedEditOperation): number { + let r = Range.compareRangesUsingEnds(a.range, b.range); + if (r === 0) { + return b.sortIndex - a.sortIndex; + } + return -r; + } + applyEdits(rawOperations: IIdentifiedSingleEditOperation[], recordTrimAutoWhitespace: boolean): ApplyEditsResult { - throw new Error('TODO'); + if (rawOperations.length === 0) { + return new ApplyEditsResult([], [], []); + } + + let mightContainRTL = this._mightContainRTL; + let mightContainNonBasicASCII = this._mightContainNonBasicASCII; + let canReduceOperations = true; + + let operations: IValidatedEditOperation[] = []; + for (let i = 0; i < rawOperations.length; i++) { + let op = rawOperations[i]; + if (canReduceOperations && op._isTracked) { + canReduceOperations = false; + } + let validatedRange = op.range; + if (!mightContainRTL && op.text) { + // check if the new inserted text contains RTL + mightContainRTL = strings.containsRTL(op.text); + } + if (!mightContainNonBasicASCII && op.text) { + mightContainNonBasicASCII = !strings.isBasicASCII(op.text); + } + operations[i] = { + sortIndex: i, + identifier: op.identifier || null, + range: validatedRange, + rangeOffset: this.getOffsetAt(validatedRange.startLineNumber, validatedRange.startColumn), + rangeLength: this.getValueLengthInRange(validatedRange, EndOfLinePreference.TextDefined), + lines: op.text ? op.text.split(/\r\n|\r|\n/) : null, + forceMoveMarkers: op.forceMoveMarkers || false, + isAutoWhitespaceEdit: op.isAutoWhitespaceEdit || false + }; + } + + // Sort operations ascending + operations.sort(ChunksTextBuffer._sortOpsAscending); + + for (let i = 0, count = operations.length - 1; i < count; i++) { + let rangeEnd = operations[i].range.getEndPosition(); + let nextRangeStart = operations[i + 1].range.getStartPosition(); + + if (nextRangeStart.isBefore(rangeEnd)) { + // overlapping ranges + throw new Error('Overlapping ranges are not allowed!'); + } + } + + if (canReduceOperations) { + operations = this._reduceOperations(operations); + } + + // Delta encode operations + let reverseRanges = ChunksTextBuffer._getInverseEditRanges(operations); + let newTrimAutoWhitespaceCandidates: { lineNumber: number, oldContent: string }[] = []; + + for (let i = 0; i < operations.length; i++) { + let op = operations[i]; + let reverseRange = reverseRanges[i]; + + if (recordTrimAutoWhitespace && op.isAutoWhitespaceEdit && op.range.isEmpty()) { + // Record already the future line numbers that might be auto whitespace removal candidates on next edit + for (let lineNumber = reverseRange.startLineNumber; lineNumber <= reverseRange.endLineNumber; lineNumber++) { + let currentLineContent = ''; + if (lineNumber === reverseRange.startLineNumber) { + currentLineContent = this.getLineContent(op.range.startLineNumber); + if (strings.firstNonWhitespaceIndex(currentLineContent) !== -1) { + continue; + } + } + newTrimAutoWhitespaceCandidates.push({ lineNumber: lineNumber, oldContent: currentLineContent }); + } + } + } + + let reverseOperations: IIdentifiedSingleEditOperation[] = []; + for (let i = 0; i < operations.length; i++) { + let op = operations[i]; + let reverseRange = reverseRanges[i]; + + reverseOperations[i] = { + identifier: op.identifier, + range: reverseRange, + text: this.getValueInRange(op.range, EndOfLinePreference.TextDefined), + forceMoveMarkers: op.forceMoveMarkers + }; + } + + this._mightContainRTL = mightContainRTL; + this._mightContainNonBasicASCII = mightContainNonBasicASCII; + + const contentChanges = this._doApplyEdits(operations); + + let trimAutoWhitespaceLineNumbers: number[] = null; + if (recordTrimAutoWhitespace && newTrimAutoWhitespaceCandidates.length > 0) { + // sort line numbers auto whitespace removal candidates for next edit descending + newTrimAutoWhitespaceCandidates.sort((a, b) => b.lineNumber - a.lineNumber); + + trimAutoWhitespaceLineNumbers = []; + for (let i = 0, len = newTrimAutoWhitespaceCandidates.length; i < len; i++) { + let lineNumber = newTrimAutoWhitespaceCandidates[i].lineNumber; + if (i > 0 && newTrimAutoWhitespaceCandidates[i - 1].lineNumber === lineNumber) { + // Do not have the same line number twice + continue; + } + + let prevContent = newTrimAutoWhitespaceCandidates[i].oldContent; + let lineContent = this.getLineContent(lineNumber); + + if (lineContent.length === 0 || lineContent === prevContent || strings.firstNonWhitespaceIndex(lineContent) !== -1) { + continue; + } + + trimAutoWhitespaceLineNumbers.push(lineNumber); + } + } + + return new ApplyEditsResult( + reverseOperations, + contentChanges, + trimAutoWhitespaceLineNumbers + ); + } + + /** + * Transform operations such that they represent the same logic edit, + * but that they also do not cause OOM crashes. + */ + private _reduceOperations(operations: IValidatedEditOperation[]): IValidatedEditOperation[] { + if (operations.length < 1000) { + // We know from empirical testing that a thousand edits work fine regardless of their shape. + return operations; + } + + // At one point, due to how events are emitted and how each operation is handled, + // some operations can trigger a high ammount of temporary string allocations, + // that will immediately get edited again. + // e.g. a formatter inserting ridiculous ammounts of \n on a model with a single line + // Therefore, the strategy is to collapse all the operations into a huge single edit operation + return [this._toSingleEditOperation(operations)]; + } + + _toSingleEditOperation(operations: IValidatedEditOperation[]): IValidatedEditOperation { + let forceMoveMarkers = false, + firstEditRange = operations[0].range, + lastEditRange = operations[operations.length - 1].range, + entireEditRange = new Range(firstEditRange.startLineNumber, firstEditRange.startColumn, lastEditRange.endLineNumber, lastEditRange.endColumn), + lastEndLineNumber = firstEditRange.startLineNumber, + lastEndColumn = firstEditRange.startColumn, + result: string[] = []; + + for (let i = 0, len = operations.length; i < len; i++) { + let operation = operations[i], + range = operation.range; + + forceMoveMarkers = forceMoveMarkers || operation.forceMoveMarkers; + + // (1) -- Push old text + for (let lineNumber = lastEndLineNumber; lineNumber < range.startLineNumber; lineNumber++) { + if (lineNumber === lastEndLineNumber) { + result.push(this.getLineContent(lineNumber).substring(lastEndColumn - 1)); + } else { + result.push('\n'); + result.push(this.getLineContent(lineNumber)); + } + } + + if (range.startLineNumber === lastEndLineNumber) { + result.push(this.getLineContent(range.startLineNumber).substring(lastEndColumn - 1, range.startColumn - 1)); + } else { + result.push('\n'); + result.push(this.getLineContent(range.startLineNumber).substring(0, range.startColumn - 1)); + } + + // (2) -- Push new text + if (operation.lines) { + for (let j = 0, lenJ = operation.lines.length; j < lenJ; j++) { + if (j !== 0) { + result.push('\n'); + } + result.push(operation.lines[j]); + } + } + + lastEndLineNumber = operation.range.endLineNumber; + lastEndColumn = operation.range.endColumn; + } + + return { + sortIndex: 0, + identifier: operations[0].identifier, + range: entireEditRange, + rangeOffset: this.getOffsetAt(entireEditRange.startLineNumber, entireEditRange.startColumn), + rangeLength: this.getValueLengthInRange(entireEditRange, EndOfLinePreference.TextDefined), + lines: result.join('').split('\n'), + forceMoveMarkers: forceMoveMarkers, + isAutoWhitespaceEdit: false + }; + } + + private _doApplyEdits(operations: IValidatedEditOperation[]): IInternalModelContentChange[] { + + // Sort operations descending + operations.sort(ChunksTextBuffer._sortOpsDescending); + + let contentChanges: IInternalModelContentChange[] = []; + let edits: OffsetLenEdit[] = []; + + for (let i = 0, len = operations.length; i < len; i++) { + const op = operations[i]; + + const text = (op.lines ? op.lines.join(this.getEOL()) : ''); + edits[i] = new OffsetLenEdit(op.sortIndex, op.rangeOffset, op.rangeLength, text); + + const startLineNumber = op.range.startLineNumber; + const startColumn = op.range.startColumn; + const endLineNumber = op.range.endLineNumber; + const endColumn = op.range.endColumn; + + if (startLineNumber === endLineNumber && startColumn === endColumn && (!op.lines || op.lines.length === 0)) { + // no-op + continue; + } + + contentChanges.push({ + range: op.range, + rangeLength: op.rangeLength, + text: text, + lines: op.lines, + rangeOffset: op.rangeOffset, + forceMoveMarkers: op.forceMoveMarkers + }); + } + + this._actual.replaceOffsetLen(edits); + + return contentChanges; + } + + /** + * Assumes `operations` are validated and sorted ascending + */ + public static _getInverseEditRanges(operations: IValidatedEditOperation[]): Range[] { + let result: Range[] = []; + + let prevOpEndLineNumber: number; + let prevOpEndColumn: number; + let prevOp: IValidatedEditOperation = null; + for (let i = 0, len = operations.length; i < len; i++) { + let op = operations[i]; + + let startLineNumber: number; + let startColumn: number; + + if (prevOp) { + if (prevOp.range.endLineNumber === op.range.startLineNumber) { + startLineNumber = prevOpEndLineNumber; + startColumn = prevOpEndColumn + (op.range.startColumn - prevOp.range.endColumn); + } else { + startLineNumber = prevOpEndLineNumber + (op.range.startLineNumber - prevOp.range.endLineNumber); + startColumn = op.range.startColumn; + } + } else { + startLineNumber = op.range.startLineNumber; + startColumn = op.range.startColumn; + } + + let resultRange: Range; + + if (op.lines && op.lines.length > 0) { + // the operation inserts something + let lineCount = op.lines.length; + let firstLine = op.lines[0]; + let lastLine = op.lines[lineCount - 1]; + + if (lineCount === 1) { + // single line insert + resultRange = new Range(startLineNumber, startColumn, startLineNumber, startColumn + firstLine.length); + } else { + // multi line insert + resultRange = new Range(startLineNumber, startColumn, startLineNumber + lineCount - 1, lastLine.length + 1); + } + } else { + // There is nothing to insert + resultRange = new Range(startLineNumber, startColumn, startLineNumber, startColumn); + } + + prevOpEndLineNumber = resultRange.endLineNumber; + prevOpEndColumn = resultRange.endColumn; + + result.push(resultRange); + prevOp = op; + } + + return result; } } @@ -792,8 +1114,8 @@ class Buffer { text = text + '\n'; this._findOffsetCloseAfter(edit.offset + edit.length + 1, tmp, tmp2); - startLeafIndex = tmp2.leafIndex; - startInnerOffset = tmp2.offset - tmp2.leafStartOffset; + endLeafIndex = tmp2.leafIndex; + endInnerOffset = tmp2.offset - tmp2.leafStartOffset; // this._findOffset(edit.offset + edit.length + 1, tmp); // endLeafIndex = tmp.leafIndex; // endInnerOffset = tmp.offset - tmp.leafStartOffset; @@ -870,8 +1192,8 @@ class Buffer { const edits = this._resolveEdits(_edits); let accumulatedLeafIndex = 0; - let accumulatedLeafEdits: LeafOffsetLenEdit[]; - let replacements: LeafReplacement[]; + let accumulatedLeafEdits: LeafOffsetLenEdit[] = []; + let replacements: LeafReplacement[] = []; for (let i = 0, len = edits.length; i < len; i++) { const edit = edits[i]; diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts index 1a16096fbbd..f27fa634ebe 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts @@ -45,7 +45,7 @@ export class TextBufferFactory implements ITextBufferFactory { // TODO console.warn(`mixed line endings not handled correctly at this time!`); } - return new ChunksTextBuffer(this._pieces, this._averageChunkSize, this._getEOL(defaultEOL)); + return new ChunksTextBuffer(this._pieces, this._averageChunkSize, this._getEOL(defaultEOL), this._containsRTL, this._isBasicASCII); } public getFirstLineText(lengthLimit: number): string { @@ -98,7 +98,7 @@ export class ChunksTextBufferBuilder implements ITextBufferBuilder { const lastChar = chunk.charCodeAt(chunk.length - 1); if (lastChar === CharCode.CarriageReturn || (lastChar >= 0xd800 && lastChar <= 0xdbff)) { // last character is \r or a high surrogate => keep it back - this._acceptChunk1(chunk.substring(0, chunk.length - 1), false); + this._acceptChunk1(chunk.substr(0, chunk.length - 1), false); this._hasPreviousChar = true; this._previousChar = lastChar; } else { @@ -139,7 +139,7 @@ export class ChunksTextBufferBuilder implements ITextBufferBuilder { public finish(): TextBufferFactory { this._finish(); console.log(`${this.totalCRCount}, ${this.totalEOLCount}`); - return new TextBufferFactory(this._rawPieces, this._averageChunkSize, this.totalCRCount, this.totalEOLCount); + return new TextBufferFactory(this._rawPieces, this._averageChunkSize, this.totalCRCount, this.totalEOLCount, this.containsRTL, this.isBasicASCII); } private _finish(): void { From 21fe74802522831a179797fba4032c9e428491fa Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Mon, 15 Jan 2018 17:22:04 +0100 Subject: [PATCH 10/33] Normalize eol --- .../model/chunksTextBuffer/bufferPiece.ts | 4 ++++ .../chunksTextBufferBuilder.ts | 20 +++++++------------ 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts b/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts index 6672bff60f3..ec8b1a5ec4e 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts @@ -66,6 +66,10 @@ export class BufferPiece { return -1; } + public static normalizeEOL(target: BufferPiece, eol: '\r\n' | '\n'): BufferPiece { + return new BufferPiece(target._str.replace(/\r\n|\r|\n/g, eol)); + } + public static deleteLastChar(target: BufferPiece): BufferPiece { const targetCharsLength = target.length(); const targetLineStartsLength = target.newLineCount(); diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts index f27fa634ebe..87968b9c534 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts @@ -41,11 +41,15 @@ export class TextBufferFactory implements ITextBufferFactory { } public create(defaultEOL: DefaultEndOfLine): ITextBuffer { + const eol = this._getEOL(defaultEOL); + let pieces = this._pieces; if (this._totalCRCount > 0 && this._totalCRCount !== this._totalEOLCount) { - // TODO - console.warn(`mixed line endings not handled correctly at this time!`); + // Normalize pieces + for (let i = 0, len = pieces.length; i < len; i++) { + pieces[i] = BufferPiece.normalizeEOL(pieces[i], eol); + } } - return new ChunksTextBuffer(this._pieces, this._averageChunkSize, this._getEOL(defaultEOL), this._containsRTL, this._isBasicASCII); + return new ChunksTextBuffer(pieces, this._averageChunkSize, eol, this._containsRTL, this._isBasicASCII); } public getFirstLineText(lengthLimit: number): string { @@ -78,16 +82,6 @@ export class ChunksTextBufferBuilder implements ITextBufferBuilder { this.isBasicASCII = true; } - // private _updateCRCount(chunk: string): void { - // // Count how many \r are present in chunk to determine the majority EOL sequence - // let chunkCarriageReturnCnt = 0; - // let lastCarriageReturnIndex = -1; - // while ((lastCarriageReturnIndex = chunk.indexOf('\r', lastCarriageReturnIndex + 1)) !== -1) { - // chunkCarriageReturnCnt++; - // } - // this.totalCRCount += chunkCarriageReturnCnt; - // } - public acceptChunk(chunk: string): void { if (chunk.length === 0) { return; From a59c44284f8c0d6e0eeb7c5280eba6a61ae20cda Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Mon, 15 Jan 2018 17:25:52 +0100 Subject: [PATCH 11/33] Fix getLineLength for last line --- .../common/model/chunksTextBuffer/chunksTextBuffer.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts index 1c365f48aa9..34060b9498c 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts @@ -824,7 +824,13 @@ class Buffer { throw new Error(`Line not found`); } - const result = end.offset - start.offset - this._eolLength; + let result: number; + if (lineNumber === this.getLineCount()) { + // last line is not trailed by an eol + result = end.offset - start.offset; + } else { + result = end.offset - start.offset - this._eolLength; + } BufferCursorPool.put(start); BufferCursorPool.put(end); From 5d85f6d50a0f265cebc8ece5723a39709b1cc062 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Mon, 15 Jan 2018 17:33:57 +0100 Subject: [PATCH 12/33] Better EOL normalization --- .../model/chunksTextBuffer/bufferPiece.ts | 12 ++++-- .../chunksTextBufferBuilder.ts | 38 +++++++++++-------- 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts b/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts index ec8b1a5ec4e..ded929bb31a 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts @@ -222,29 +222,33 @@ export function createUint32Array(arr: number[]): Uint32Array { export class LineStarts { constructor( public readonly lineStarts: number[], - public readonly carriageReturnCnt: number + public readonly cr: number, + public readonly lf: number, + public readonly crlf: number ) { } } export function createLineStarts(str: string): LineStarts { let r: number[] = [], rLength = 0; - let carriageReturnCnt = 0; + let cr = 0, lf = 0, crlf = 0; for (let i = 0, len = str.length; i < len; i++) { const chr = str.charCodeAt(i); if (chr === CharCode.CarriageReturn) { - carriageReturnCnt++; if (i + 1 < len && str.charCodeAt(i + 1) === CharCode.LineFeed) { // \r\n... case + crlf++; r[rLength++] = i + 2; i++; // skip \n } else { + cr++; // \r... case r[rLength++] = i + 1; } } else if (chr === CharCode.LineFeed) { + lf++; r[rLength++] = i + 1; } } - return new LineStarts(r, carriageReturnCnt); + return new LineStarts(r, cr, lf, crlf); } diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts index 87968b9c534..384eb608a4b 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts @@ -15,8 +15,9 @@ export class TextBufferFactory implements ITextBufferFactory { constructor( private readonly _pieces: BufferPiece[], private readonly _averageChunkSize: number, - private readonly _totalCRCount: number, - private readonly _totalEOLCount: number, + private readonly _cr: number, + private readonly _lf: number, + private readonly _crlf: number, private readonly _containsRTL: boolean, private readonly _isBasicASCII: boolean, ) { @@ -28,11 +29,13 @@ export class TextBufferFactory implements ITextBufferFactory { * Otherwise returns '\n'. More lines end with '\n'. */ private _getEOL(defaultEOL: DefaultEndOfLine): '\r\n' | '\n' { - if (this._totalEOLCount === 0) { + const totalEOLCount = this._cr + this._lf + this._crlf; + const totalCRCount = this._cr + this._crlf; + if (totalEOLCount === 0) { // This is an empty file or a file with precisely one line return (defaultEOL === DefaultEndOfLine.LF ? '\n' : '\r\n'); } - if (this._totalCRCount > this._totalEOLCount / 2) { + if (totalCRCount > totalEOLCount / 2) { // More than half of the file contains \r\n ending lines return '\r\n'; } @@ -43,7 +46,11 @@ export class TextBufferFactory implements ITextBufferFactory { public create(defaultEOL: DefaultEndOfLine): ITextBuffer { const eol = this._getEOL(defaultEOL); let pieces = this._pieces; - if (this._totalCRCount > 0 && this._totalCRCount !== this._totalEOLCount) { + + if ( + (eol === '\r\n' && (this._cr > 0 || this._lf > 0)) + || (eol === '\n' && (this._cr > 0 || this._crlf > 0)) + ) { // Normalize pieces for (let i = 0, len = pieces.length; i < len; i++) { pieces[i] = BufferPiece.normalizeEOL(pieces[i], eol); @@ -65,8 +72,9 @@ export class ChunksTextBufferBuilder implements ITextBufferBuilder { private _previousChar: number; private _averageChunkSize: number; - private totalCRCount: number; - private totalEOLCount: number; + private cr: number; + private lf: number; + private crlf: number; private containsRTL: boolean; private isBasicASCII: boolean; @@ -76,8 +84,9 @@ export class ChunksTextBufferBuilder implements ITextBufferBuilder { this._previousChar = 0; this._averageChunkSize = 0; - this.totalCRCount = 0; - this.totalEOLCount = 0; + this.cr = 0; + this.lf = 0; + this.crlf = 0; this.containsRTL = false; this.isBasicASCII = true; } @@ -126,14 +135,14 @@ export class ChunksTextBufferBuilder implements ITextBufferBuilder { const lineStarts = createLineStarts(chunk); this._rawPieces.push(new BufferPiece(chunk, createUint32Array(lineStarts.lineStarts))); - this.totalCRCount += lineStarts.carriageReturnCnt; - this.totalEOLCount += lineStarts.lineStarts.length; + this.cr += lineStarts.cr; + this.lf += lineStarts.lf; + this.crlf += lineStarts.crlf; } public finish(): TextBufferFactory { this._finish(); - console.log(`${this.totalCRCount}, ${this.totalEOLCount}`); - return new TextBufferFactory(this._rawPieces, this._averageChunkSize, this.totalCRCount, this.totalEOLCount, this.containsRTL, this.isBasicASCII); + return new TextBufferFactory(this._rawPieces, this._averageChunkSize, this.cr, this.lf, this.crlf, this.containsRTL, this.isBasicASCII); } private _finish(): void { @@ -152,8 +161,7 @@ export class ChunksTextBufferBuilder implements ITextBufferBuilder { const newLastPiece = BufferPiece.join(lastPiece, tmp); this._rawPieces[this._rawPieces.length - 1] = newLastPiece; if (this._previousChar === CharCode.CarriageReturn) { - this.totalCRCount++; - this.totalEOLCount++; + this.cr++; } } } From 91efcf9cf679233fb7e26b5301fd38a638aa9a4b Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Mon, 15 Jan 2018 17:43:43 +0100 Subject: [PATCH 13/33] Fix getLineContent for last line --- .../common/model/chunksTextBuffer/chunksTextBuffer.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts index 34060b9498c..16a31292cb3 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts @@ -807,7 +807,13 @@ class Buffer { throw new Error(`Line not found`); } - const result = this.extractString(start, end.offset - start.offset - this._eolLength); + let result: string; + if (lineNumber === this.getLineCount()) { + // last line is not trailed by an eol + result = this.extractString(start, end.offset - start.offset); + } else { + result = this.extractString(start, end.offset - start.offset - this._eolLength); + } BufferCursorPool.put(start); BufferCursorPool.put(end); From 5cf50d5bedeb727e6578e699627b2df576df2fad Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Mon, 15 Jan 2018 17:50:53 +0100 Subject: [PATCH 14/33] Correct sorting of edits before applying changes --- .../common/model/chunksTextBuffer/chunksTextBuffer.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts index 16a31292cb3..7c987dae12c 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts @@ -1199,7 +1199,16 @@ class Buffer { return prevLeaf; } + private static _compareEdits(a: OffsetLenEdit, b: OffsetLenEdit): number { + if (a.offset === b.offset) { + return (a.initialIndex - b.initialIndex); + } + return a.offset - b.offset; + } + public replaceOffsetLen(_edits: OffsetLenEdit[]): void { + _edits.sort(Buffer._compareEdits); + const initialLeafLength = this._leafs.length; const edits = this._resolveEdits(_edits); From f3eff0f8cbb5d5a8edc82d683c726b5a28590cc2 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Tue, 16 Jan 2018 09:29:14 +0100 Subject: [PATCH 15/33] Implement getLineFirstNonWhitespaceColumn --- .../model/chunksTextBuffer/bufferPiece.ts | 14 ++++++ .../chunksTextBuffer/chunksTextBuffer.ts | 43 ++++++++++++++++++- 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts b/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts index ded929bb31a..91066193848 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts @@ -66,6 +66,20 @@ export class BufferPiece { return -1; } + public findLineFirstNonWhitespaceIndexInLeaf(searchStartOffset: number): number { + for (let i = searchStartOffset, len = this._str.length; i < len; i++) { + const chCode = this._str.charCodeAt(i); + if (chCode === CharCode.CarriageReturn || chCode === CharCode.LineFeed) { + // Reached EOL + return -1; + } + if (chCode !== CharCode.Space && chCode !== CharCode.Tab) { + return i; + } + } + return -1; + } + public static normalizeEOL(target: BufferPiece, eol: '\r\n' | '\n'): BufferPiece { return new BufferPiece(target._str.replace(/\r\n|\r|\n/g, eol)); } diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts index 7c987dae12c..6e253301c52 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts @@ -135,7 +135,11 @@ export class ChunksTextBuffer implements ITextBuffer { } getLineFirstNonWhitespaceColumn(lineNumber: number): number { - throw new Error('TODO'); + const result = this._actual.getLineFirstNonWhitespaceIndex(lineNumber); + if (result === -1) { + return 0; + } + return result + 1; } getLineLastNonWhitespaceColumn(lineNumber: number): number { throw new Error('TODO'); @@ -793,6 +797,10 @@ class Buffer { return true; } + public getLength(): number { + return this._nodes.length[1]; + } + public getLineCount(): number { return this._nodes.newLineCount[1] + 1; } @@ -843,6 +851,39 @@ class Buffer { return result; } + public getLineFirstNonWhitespaceIndex(lineNumber: number): number { + const start = BufferCursorPool.take(); + + if (!this._findLineStart(lineNumber, start)) { + BufferCursorPool.put(start); + throw new Error(`Line not found`); + } + + let leafIndex = start.leafIndex; + let searchStartOffset = start.offset - start.leafStartOffset; + BufferCursorPool.put(start); + + const leafsCount = this._leafs.length; + let totalDelta = 0; + while (true) { + const leaf = this._leafs[leafIndex]; + + const leafResult = leaf.findLineFirstNonWhitespaceIndexInLeaf(searchStartOffset); + if (leafResult !== -1) { + return (leafResult - searchStartOffset) + totalDelta; + } + + leafIndex++; + + if (leafIndex >= leafsCount) { + return -1; + } + + totalDelta += (leaf.length() - searchStartOffset); + searchStartOffset = 0; + } + } + public getLinesContent(): string[] { let result: string[] = new Array(this.getLineCount()); let resultIndex = 0; From 4632cd1cdcf01f1bd02d9903287191fcb5f25705 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Tue, 16 Jan 2018 09:56:27 +0100 Subject: [PATCH 16/33] Fix issue in offset -> (ln;col) conversion --- .../common/model/chunksTextBuffer/chunksTextBuffer.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts index 6e253301c52..5a349902425 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts @@ -978,10 +978,10 @@ class Buffer { private _findLineStartBeforeOffsetInLeaf(offset: number, leafIndex: number, leafStartOffset: number, leafStartNewLineCount: number, result: BufferCursor): number { const leaf = this._leafs[leafIndex]; const lineStartIndex = leaf.findLineStartBeforeOffset(offset - leafStartOffset); - const lineStartOffset = leafStartOffset = leaf.lineStartFor(lineStartIndex); + const lineStartOffset = leafStartOffset + leaf.lineStartFor(lineStartIndex); result.set(lineStartOffset, leafIndex, leafStartOffset, leafStartNewLineCount); - return leafStartNewLineCount + lineStartIndex + 1; + return leafStartNewLineCount + lineStartIndex + 2; } /** @@ -995,7 +995,7 @@ class Buffer { while (true) { const leaf = this._leafs[leafIndex]; - if (leaf.newLineCount() > 1 && leaf.lineStartFor(0) + leafStartOffset >= offset) { + if (leaf.newLineCount() > 1 && leaf.lineStartFor(0) + leafStartOffset <= offset) { // must be in this leaf return this._findLineStartBeforeOffsetInLeaf(offset, leafIndex, leafStartOffset, leafStartNewLineCount, result); } From 1ccc3e0bbbf8493443d2e147f08e1c455e182c22 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Tue, 16 Jan 2018 10:14:29 +0100 Subject: [PATCH 17/33] Implement ChunksTextBuffer.equals --- .../chunksTextBuffer/chunksTextBuffer.ts | 55 ++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts index 5a349902425..dbde6393671 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts @@ -39,7 +39,10 @@ export class ChunksTextBuffer implements ITextBuffer { } equals(other: ITextBuffer): boolean { - throw new Error('TODO'); + if (!(other instanceof ChunksTextBuffer)) { + return false; + } + return this._actual.equals(other._actual); } mightContainRTL(): boolean { return this._mightContainRTL; @@ -587,6 +590,56 @@ class Buffer { this._rebuildNodes(); } + equals(other: Buffer): boolean { + return Buffer.equals(this, other); + } + + private static equals(a: Buffer, b: Buffer): boolean { + const aLength = a.getLength(); + const bLength = b.getLength(); + if (aLength !== bLength) { + return false; + } + if (a.getLineCount() !== b.getLineCount()) { + return false; + } + + let remaining = aLength; + let aLeafIndex = 0, aLeaf = a._leafs[aLeafIndex], aLeafLength = aLeaf.length(), aLeafRemaining = aLeafLength; + let bLeafIndex = 0, bLeaf = b._leafs[bLeafIndex], bLeafLength = bLeaf.length(), bLeafRemaining = bLeafLength; + + while (remaining > 0) { + let consuming = Math.min(aLeafRemaining, bLeafRemaining); + + let aStr = aLeaf.substr(aLeafLength - aLeafRemaining, consuming); + let bStr = bLeaf.substr(bLeafLength - bLeafRemaining, consuming); + + if (aStr !== bStr) { + return false; + } + + remaining -= consuming; + aLeafRemaining -= consuming; + bLeafRemaining -= consuming; + + if (aLeafRemaining === 0) { + aLeafIndex++; + aLeaf = a._leafs[aLeafIndex]; + aLeafLength = aLeaf.length(); + aLeafRemaining = aLeafLength; + } + + if (bLeafRemaining === 0) { + bLeafIndex++; + bLeaf = b._leafs[bLeafIndex]; + bLeafLength = bLeaf.length(); + bLeafRemaining = bLeafLength; + } + } + + return true; + } + public getEOL(): string { return this._eol; } From 26359a948193b5d786d36990647de9a12c0bd8a7 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Tue, 16 Jan 2018 10:46:03 +0100 Subject: [PATCH 18/33] Fix some (ln;col) -> offset conversion --- src/vs/editor/common/model.ts | 1 + .../chunksTextBuffer/chunksTextBuffer.ts | 45 +++++-------------- .../model/linesTextBuffer/linesTextBuffer.ts | 4 ++ src/vs/editor/common/model/textModel.ts | 3 +- 4 files changed, 19 insertions(+), 34 deletions(-) diff --git a/src/vs/editor/common/model.ts b/src/vs/editor/common/model.ts index 9d58b81cc05..3ce343c7411 100644 --- a/src/vs/editor/common/model.ts +++ b/src/vs/editor/common/model.ts @@ -1079,6 +1079,7 @@ export interface ITextBuffer { getValueInRange(range: Range, eol: EndOfLinePreference): string; getValueLengthInRange(range: Range, eol: EndOfLinePreference): number; + getLength(): number; getLineCount(): number; getLinesContent(): string[]; getLineContent(lineNumber: number): string; diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts index dbde6393671..2ab37067f60 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts @@ -116,6 +116,10 @@ export class ChunksTextBuffer implements ITextBuffer { return 0; } + public getLength(): number { + return this._actual.getLength(); + } + getLineCount(): number { return this._actual.getLineCount(); } @@ -1014,12 +1018,12 @@ class Buffer { public convertPositionToOffset(lineNumber: number, column: number): number { const r = BufferCursorPool.take(); - if (!this._getOffsetAt(lineNumber, column, r)) { + if (!this._findLineStart(lineNumber, r)) { BufferCursorPool.put(r); throw new Error(`Position not found`); } - const result = r.offset; + const result = r.offset + column - 1; BufferCursorPool.put(r); return result; @@ -1048,7 +1052,7 @@ class Buffer { while (true) { const leaf = this._leafs[leafIndex]; - if (leaf.newLineCount() > 1 && leaf.lineStartFor(0) + leafStartOffset <= offset) { + if (leaf.newLineCount() >= 1 && leaf.lineStartFor(0) + leafStartOffset <= offset) { // must be in this leaf return this._findLineStartBeforeOffsetInLeaf(offset, leafIndex, leafStartOffset, leafStartNewLineCount, result); } @@ -1113,48 +1117,23 @@ class Buffer { public getValueInRange(range: Range): string { const start = BufferCursorPool.take(); - const end = BufferCursorPool.take(); if (!this._getOffsetAt(range.startLineNumber, range.startColumn, start)) { BufferCursorPool.put(start); - BufferCursorPool.put(end); throw new Error(`Line not found`); } - if (!this._getOffsetAt(range.endLineNumber, range.endColumn, end)) { - BufferCursorPool.put(start); - BufferCursorPool.put(end); - throw new Error(`Line not found`); - } - - const result = this.extractString(start, end.offset - start.offset); + const endOffset = this.convertPositionToOffset(range.endLineNumber, range.endColumn); + const result = this.extractString(start, endOffset - start.offset); BufferCursorPool.put(start); - BufferCursorPool.put(end); return result; } public getValueLengthInRange(range: Range): number { - const start = BufferCursorPool.take(); - const end = BufferCursorPool.take(); - - if (!this._getOffsetAt(range.startLineNumber, range.startColumn, start)) { - BufferCursorPool.put(start); - BufferCursorPool.put(end); - throw new Error(`Line not found`); - } - - if (!this._getOffsetAt(range.endLineNumber, range.endColumn, end)) { - BufferCursorPool.put(start); - BufferCursorPool.put(end); - throw new Error(`Line not found`); - } - - const result = end.offset - start.offset; - - BufferCursorPool.put(start); - BufferCursorPool.put(end); - return result; + const startOffset = this.convertPositionToOffset(range.startLineNumber, range.startColumn); + const endOffset = this.convertPositionToOffset(range.endLineNumber, range.endColumn); + return endOffset - startOffset; } //#region Editing diff --git a/src/vs/editor/common/model/linesTextBuffer/linesTextBuffer.ts b/src/vs/editor/common/model/linesTextBuffer/linesTextBuffer.ts index 1cbd9fe7c92..73ac79ad7e5 100644 --- a/src/vs/editor/common/model/linesTextBuffer/linesTextBuffer.ts +++ b/src/vs/editor/common/model/linesTextBuffer/linesTextBuffer.ts @@ -190,6 +190,10 @@ export class LinesTextBuffer implements ITextBuffer { return endOffset - startOffset; } + public getLength(): number { + return this._lineStarts.getTotalValue(); + } + public getLineCount(): number { return this._lines.length; } diff --git a/src/vs/editor/common/model/textModel.ts b/src/vs/editor/common/model/textModel.ts index 023a1adb307..bac60124491 100644 --- a/src/vs/editor/common/model/textModel.ts +++ b/src/vs/editor/common/model/textModel.ts @@ -895,7 +895,8 @@ export class TextModel extends Disposable implements model.ITextModel { public modifyPosition(rawPosition: IPosition, offset: number): Position { this._assertNotDisposed(); - return this.getPositionAt(this.getOffsetAt(rawPosition) + offset); + let candidate = this.getOffsetAt(rawPosition) + offset; + return this.getPositionAt(Math.min(this._buffer.getLength(), Math.max(0, candidate))); } public getFullModelRange(): Range { From de218362e82e46f82ffe3c5152bb3b2910bf697c Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Tue, 16 Jan 2018 11:04:21 +0100 Subject: [PATCH 19/33] Better edits sorting --- .../editor/common/model/chunksTextBuffer/chunksTextBuffer.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts index 2ab37067f60..ad85b50c12c 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts @@ -1274,7 +1274,10 @@ class Buffer { private static _compareEdits(a: OffsetLenEdit, b: OffsetLenEdit): number { if (a.offset === b.offset) { - return (a.initialIndex - b.initialIndex); + if (a.length === b.length) { + return (a.initialIndex - b.initialIndex); + } + return (a.length - b.length); } return a.offset - b.offset; } From cad270088c035ab8df3101311ad7f99a274db217 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Tue, 16 Jan 2018 11:10:48 +0100 Subject: [PATCH 20/33] Implement ChunksTextBuffer.setEOL --- src/vs/editor/common/model.ts | 2 +- .../chunksTextBuffer/chunksTextBuffer.ts | 19 +++++++++++++++++-- .../model/linesTextBuffer/linesTextBuffer.ts | 2 +- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/vs/editor/common/model.ts b/src/vs/editor/common/model.ts index 3ce343c7411..6c208408ba3 100644 --- a/src/vs/editor/common/model.ts +++ b/src/vs/editor/common/model.ts @@ -1088,7 +1088,7 @@ export interface ITextBuffer { getLineFirstNonWhitespaceColumn(lineNumber: number): number; getLineLastNonWhitespaceColumn(lineNumber: number): number; - setEOL(newEOL: string): void; + setEOL(newEOL: '\r\n' | '\n'): void; applyEdits(rawOperations: IIdentifiedSingleEditOperation[], recordTrimAutoWhitespace: boolean): ApplyEditsResult; } diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts index ad85b50c12c..505489ac30d 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts @@ -151,8 +151,12 @@ export class ChunksTextBuffer implements ITextBuffer { getLineLastNonWhitespaceColumn(lineNumber: number): number { throw new Error('TODO'); } - setEOL(newEOL: string): void { - throw new Error('TODO'); + setEOL(newEOL: '\r\n' | '\n'): void { + if (this.getEOL() === newEOL) { + // nothing to do... + return; + } + this._actual.setEOL(newEOL); } private static _sortOpsAscending(a: IValidatedEditOperation, b: IValidatedEditOperation): number { @@ -1365,6 +1369,17 @@ class Buffer { this._rebuildNodes(); } + public setEOL(newEOL: '\r\n' | '\n'): void { + let leafs: BufferPiece[] = []; + for (let i = 0, len = this._leafs.length; i < len; i++) { + leafs[i] = BufferPiece.normalizeEOL(this._leafs[i], newEOL); + } + this._leafs = leafs; + this._rebuildNodes(); + this._eol = newEOL; + this._eolLength = this._eol.length; + } + //#endregion private IS_NODE(i: number): boolean { diff --git a/src/vs/editor/common/model/linesTextBuffer/linesTextBuffer.ts b/src/vs/editor/common/model/linesTextBuffer/linesTextBuffer.ts index 73ac79ad7e5..d1ebb3ec17d 100644 --- a/src/vs/editor/common/model/linesTextBuffer/linesTextBuffer.ts +++ b/src/vs/editor/common/model/linesTextBuffer/linesTextBuffer.ts @@ -232,7 +232,7 @@ export class LinesTextBuffer implements ITextBuffer { //#region Editing - public setEOL(newEOL: string): void { + public setEOL(newEOL: '\r\n' | '\n'): void { this._EOL = newEOL; this._constructLineStarts(); } From 74d70805fd18cbf2b3dacebf6b5b143c81fa82d4 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Tue, 16 Jan 2018 11:24:57 +0100 Subject: [PATCH 21/33] Fix edge case when searching for last offset in file --- src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts index 505489ac30d..75e2960ee3f 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts @@ -757,7 +757,7 @@ class Buffer { while (true) { const leaf = this._leafs[leafIndex]; - if (innerOffset < leaf.length()) { + if (innerOffset < leaf.length() || (innerOffset === leaf.length() && leafIndex + 1 === leafsCount)) { result.set(offset, leafIndex, leafStartOffset, leafStartNewLineCount); return true; } From 5db6bdddd8add87bd572ae7afaf3282b1fcc00d3 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Tue, 16 Jan 2018 11:30:29 +0100 Subject: [PATCH 22/33] Fix NPE in Buffer.equals --- .../chunksTextBuffer/chunksTextBuffer.ts | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts index 75e2960ee3f..7677656adc8 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts @@ -613,23 +613,10 @@ class Buffer { } let remaining = aLength; - let aLeafIndex = 0, aLeaf = a._leafs[aLeafIndex], aLeafLength = aLeaf.length(), aLeafRemaining = aLeafLength; - let bLeafIndex = 0, bLeaf = b._leafs[bLeafIndex], bLeafLength = bLeaf.length(), bLeafRemaining = bLeafLength; + let aLeafIndex = -1, aLeaf = null, aLeafLength = 0, aLeafRemaining = 0; + let bLeafIndex = -1, bLeaf = null, bLeafLength = 0, bLeafRemaining = 0; while (remaining > 0) { - let consuming = Math.min(aLeafRemaining, bLeafRemaining); - - let aStr = aLeaf.substr(aLeafLength - aLeafRemaining, consuming); - let bStr = bLeaf.substr(bLeafLength - bLeafRemaining, consuming); - - if (aStr !== bStr) { - return false; - } - - remaining -= consuming; - aLeafRemaining -= consuming; - bLeafRemaining -= consuming; - if (aLeafRemaining === 0) { aLeafIndex++; aLeaf = a._leafs[aLeafIndex]; @@ -643,6 +630,19 @@ class Buffer { bLeafLength = bLeaf.length(); bLeafRemaining = bLeafLength; } + + let consuming = Math.min(aLeafRemaining, bLeafRemaining); + + let aStr = aLeaf.substr(aLeafLength - aLeafRemaining, consuming); + let bStr = bLeaf.substr(bLeafLength - bLeafRemaining, consuming); + + if (aStr !== bStr) { + return false; + } + + remaining -= consuming; + aLeafRemaining -= consuming; + bLeafRemaining -= consuming; } return true; From af160c40ccc98eb056707934cb857ef4d7cd2d9d Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Tue, 16 Jan 2018 11:38:25 +0100 Subject: [PATCH 23/33] Validate offset before passing it down --- src/vs/editor/common/model/textModel.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/vs/editor/common/model/textModel.ts b/src/vs/editor/common/model/textModel.ts index bac60124491..472868a177f 100644 --- a/src/vs/editor/common/model/textModel.ts +++ b/src/vs/editor/common/model/textModel.ts @@ -635,8 +635,9 @@ export class TextModel extends Disposable implements model.ITextModel { return this._buffer.getOffsetAt(position.lineNumber, position.column); } - public getPositionAt(offset: number): Position { + public getPositionAt(rawOffset: number): Position { this._assertNotDisposed(); + let offset = (Math.min(this._buffer.getLength(), Math.max(0, rawOffset))); return this._buffer.getPositionAt(offset); } From 01afb2549467ab6e0af979a2ab95983755b1ff1f Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Tue, 16 Jan 2018 12:18:08 +0100 Subject: [PATCH 24/33] Optimize buffer creation --- .../model/chunksTextBuffer/bufferPiece.ts | 45 ++++++++++++++++--- .../chunksTextBufferBuilder.ts | 23 +++++----- 2 files changed, 53 insertions(+), 15 deletions(-) diff --git a/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts b/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts index 91066193848..78c9915d8b8 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts @@ -23,7 +23,7 @@ export class BufferPiece { constructor(str: string, lineStarts: Uint32Array = null) { this._str = str; if (lineStarts === null) { - this._lineStarts = createUint32Array(createLineStarts(str).lineStarts); + this._lineStarts = createLineStartsFast(str); } else { this._lineStarts = lineStarts; } @@ -235,16 +235,41 @@ export function createUint32Array(arr: number[]): Uint32Array { export class LineStarts { constructor( - public readonly lineStarts: number[], + public readonly lineStarts: Uint32Array, public readonly cr: number, public readonly lf: number, - public readonly crlf: number + public readonly crlf: number, + public readonly isBasicASCII: boolean ) { } } -export function createLineStarts(str: string): LineStarts { +export function createLineStartsFast(str: string): Uint32Array { let r: number[] = [], rLength = 0; + for (let i = 0, len = str.length; i < len; i++) { + const chr = str.charCodeAt(i); + + if (chr === CharCode.CarriageReturn) { + if (i + 1 < len && str.charCodeAt(i + 1) === CharCode.LineFeed) { + // \r\n... case + r[rLength++] = i + 2; + i++; // skip \n + } else { + // \r... case + r[rLength++] = i + 1; + } + } else if (chr === CharCode.LineFeed) { + r[rLength++] = i + 1; + } + } + return createUint32Array(r); +} + +export function createLineStarts(r: number[], str: string): LineStarts { + r.length = 0; + + let rLength = 0; let cr = 0, lf = 0, crlf = 0; + let isBasicASCII = true; for (let i = 0, len = str.length; i < len; i++) { const chr = str.charCodeAt(i); @@ -262,7 +287,17 @@ export function createLineStarts(str: string): LineStarts { } else if (chr === CharCode.LineFeed) { lf++; r[rLength++] = i + 1; + } else { + if (isBasicASCII) { + if (chr !== CharCode.Tab && (chr < 32 || chr > 126)) { + isBasicASCII = false; + } + } } } - return new LineStarts(r, cr, lf, crlf); + + const result = new LineStarts(createUint32Array(r), cr, lf, crlf, isBasicASCII); + r.length = 0; + + return result; } diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts index 384eb608a4b..e1268de0dd5 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts @@ -6,7 +6,7 @@ import * as strings from 'vs/base/common/strings'; import { ITextBufferBuilder, ITextBufferFactory, ITextBuffer, DefaultEndOfLine } from 'vs/editor/common/model'; -import { BufferPiece, createLineStarts, createUint32Array } from 'vs/editor/common/model/chunksTextBuffer/bufferPiece'; +import { BufferPiece, createLineStarts } from 'vs/editor/common/model/chunksTextBuffer/bufferPiece'; import { ChunksTextBuffer } from 'vs/editor/common/model/chunksTextBuffer/chunksTextBuffer'; import { CharCode } from 'vs/base/common/charCode'; @@ -71,6 +71,7 @@ export class ChunksTextBufferBuilder implements ITextBufferBuilder { private _hasPreviousChar: boolean; private _previousChar: number; private _averageChunkSize: number; + private _tmpLineStarts: number[]; private cr: number; private lf: number; @@ -83,6 +84,7 @@ export class ChunksTextBufferBuilder implements ITextBufferBuilder { this._hasPreviousChar = false; this._previousChar = 0; this._averageChunkSize = 0; + this._tmpLineStarts = []; this.cr = 0; this.lf = 0; @@ -109,13 +111,6 @@ export class ChunksTextBufferBuilder implements ITextBufferBuilder { this._hasPreviousChar = false; this._previousChar = lastChar; } - - if (!this.containsRTL) { - this.containsRTL = strings.containsRTL(chunk); - } - if (this.isBasicASCII) { - this.isBasicASCII = strings.isBasicASCII(chunk); - } } private _acceptChunk1(chunk: string, allowEmptyStrings: boolean): void { @@ -132,12 +127,20 @@ export class ChunksTextBufferBuilder implements ITextBufferBuilder { } private _acceptChunk2(chunk: string): void { - const lineStarts = createLineStarts(chunk); + const lineStarts = createLineStarts(this._tmpLineStarts, chunk); - this._rawPieces.push(new BufferPiece(chunk, createUint32Array(lineStarts.lineStarts))); + this._rawPieces.push(new BufferPiece(chunk, lineStarts.lineStarts)); this.cr += lineStarts.cr; this.lf += lineStarts.lf; this.crlf += lineStarts.crlf; + + if (this.isBasicASCII) { + this.isBasicASCII = lineStarts.isBasicASCII; + } + if (!this.isBasicASCII && !this.containsRTL) { + // No need to check if is basic ASCII + this.containsRTL = strings.containsRTL(chunk); + } } public finish(): TextBufferFactory { From fa82eeaef5ef8c069472a40001819b7688e9c973 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Tue, 16 Jan 2018 12:24:56 +0100 Subject: [PATCH 25/33] Folding is not supported for huge files --- src/vs/editor/contrib/folding/folding.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/vs/editor/contrib/folding/folding.ts b/src/vs/editor/contrib/folding/folding.ts index 283801a2d60..21e8d35f4ce 100644 --- a/src/vs/editor/contrib/folding/folding.ts +++ b/src/vs/editor/contrib/folding/folding.ts @@ -133,7 +133,8 @@ export class FoldingController implements IEditorContribution { this.localToDispose = dispose(this.localToDispose); let model = this.editor.getModel(); - if (!this._isEnabled || !model) { + if (!this._isEnabled || !model || model.isTooLargeForTokenization()) { + // huge files get no view model, so they cannot support hidden areas return; } From f678d17b03ca9d50f5bb2cbdf7cb52e2191012f5 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Tue, 16 Jan 2018 12:29:00 +0100 Subject: [PATCH 26/33] Optimize away looping through lines for huge files --- src/vs/editor/common/model/textModel.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/vs/editor/common/model/textModel.ts b/src/vs/editor/common/model/textModel.ts index 472868a177f..d7a914bb843 100644 --- a/src/vs/editor/common/model/textModel.ts +++ b/src/vs/editor/common/model/textModel.ts @@ -493,6 +493,10 @@ export class TextModel extends Disposable implements model.ITextModel { public isDominatedByLongLines(): boolean { this._assertNotDisposed(); + if (this.isTooLargeForTokenization()) { + // Cannot word wrap huge files anyways, so it doesn't really matter + return false; + } let smallLineCharCount = 0; let longLineCharCount = 0; From 728f825f04647ab9b886d8842cf62cde238d739e Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Wed, 17 Jan 2018 09:28:16 +0100 Subject: [PATCH 27/33] Add more TextModel tests --- .../test/common/model/textModel.test.ts | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/src/vs/editor/test/common/model/textModel.test.ts b/src/vs/editor/test/common/model/textModel.test.ts index 323ca9886b3..0994bbe7c32 100644 --- a/src/vs/editor/test/common/model/textModel.test.ts +++ b/src/vs/editor/test/common/model/textModel.test.ts @@ -763,6 +763,66 @@ suite('Editor Model - TextModel', () => { model.dispose(); }); + + test('getLineFirstNonWhitespaceColumn', () => { + let model = TextModel.createFromString([ + 'asd', + ' asd', + '\tasd', + ' asd', + '\t\tasd', + ' ', + ' ', + '\t', + '\t\t', + ' \tasd', + '', + '' + ].join('\n')); + + assert.equal(model.getLineFirstNonWhitespaceColumn(1), 1, '1'); + assert.equal(model.getLineFirstNonWhitespaceColumn(2), 2, '2'); + assert.equal(model.getLineFirstNonWhitespaceColumn(3), 2, '3'); + assert.equal(model.getLineFirstNonWhitespaceColumn(4), 3, '4'); + assert.equal(model.getLineFirstNonWhitespaceColumn(5), 3, '5'); + assert.equal(model.getLineFirstNonWhitespaceColumn(6), 0, '6'); + assert.equal(model.getLineFirstNonWhitespaceColumn(7), 0, '7'); + assert.equal(model.getLineFirstNonWhitespaceColumn(8), 0, '8'); + assert.equal(model.getLineFirstNonWhitespaceColumn(9), 0, '9'); + assert.equal(model.getLineFirstNonWhitespaceColumn(10), 4, '10'); + assert.equal(model.getLineFirstNonWhitespaceColumn(11), 0, '11'); + assert.equal(model.getLineFirstNonWhitespaceColumn(12), 0, '12'); + }); + + test('getLineLastNonWhitespaceColumn', () => { + let model = TextModel.createFromString([ + 'asd', + 'asd ', + 'asd\t', + 'asd ', + 'asd\t\t', + ' ', + ' ', + '\t', + '\t\t', + 'asd \t', + '', + '' + ].join('\n')); + + assert.equal(model.getLineLastNonWhitespaceColumn(1), 4, '1'); + assert.equal(model.getLineLastNonWhitespaceColumn(2), 4, '2'); + assert.equal(model.getLineLastNonWhitespaceColumn(3), 4, '3'); + assert.equal(model.getLineLastNonWhitespaceColumn(4), 4, '4'); + assert.equal(model.getLineLastNonWhitespaceColumn(5), 4, '5'); + assert.equal(model.getLineLastNonWhitespaceColumn(6), 0, '6'); + assert.equal(model.getLineLastNonWhitespaceColumn(7), 0, '7'); + assert.equal(model.getLineLastNonWhitespaceColumn(8), 0, '8'); + assert.equal(model.getLineLastNonWhitespaceColumn(9), 0, '9'); + assert.equal(model.getLineLastNonWhitespaceColumn(10), 4, '10'); + assert.equal(model.getLineLastNonWhitespaceColumn(11), 0, '11'); + assert.equal(model.getLineLastNonWhitespaceColumn(12), 0, '12'); + }); }); suite('TextModel.mightContainRTL', () => { From 4ad47b013d35a8510ece6bd854af769f120bfe63 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Wed, 17 Jan 2018 09:29:15 +0100 Subject: [PATCH 28/33] Implement ChunksTextBuffer.getLineLastNonWhitespaceColumn --- .../model/chunksTextBuffer/bufferPiece.ts | 18 +++++- .../chunksTextBuffer/chunksTextBuffer.ts | 58 ++++++++++++++++++- 2 files changed, 72 insertions(+), 4 deletions(-) diff --git a/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts b/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts index 78c9915d8b8..f91a6157d1c 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts @@ -66,12 +66,26 @@ export class BufferPiece { return -1; } - public findLineFirstNonWhitespaceIndexInLeaf(searchStartOffset: number): number { + public findLineFirstNonWhitespaceIndex(searchStartOffset: number): number { for (let i = searchStartOffset, len = this._str.length; i < len; i++) { const chCode = this._str.charCodeAt(i); if (chCode === CharCode.CarriageReturn || chCode === CharCode.LineFeed) { // Reached EOL - return -1; + return -2; + } + if (chCode !== CharCode.Space && chCode !== CharCode.Tab) { + return i; + } + } + return -1; + } + + public findLineLastNonWhitespaceIndex(searchStartOffset: number): number { + for (let i = searchStartOffset - 1; i >= 0; i--) { + const chCode = this._str.charCodeAt(i); + if (chCode === CharCode.CarriageReturn || chCode === CharCode.LineFeed) { + // Reached EOL + return -2; } if (chCode !== CharCode.Space && chCode !== CharCode.Tab) { return i; diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts index 7677656adc8..33d31749722 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts @@ -149,7 +149,11 @@ export class ChunksTextBuffer implements ITextBuffer { return result + 1; } getLineLastNonWhitespaceColumn(lineNumber: number): number { - throw new Error('TODO'); + const result = this._actual.getLineLastNonWhitespaceIndex(lineNumber); + if (result === -1) { + return 0; + } + return result + 1; } setEOL(newEOL: '\r\n' | '\n'): void { if (this.getEOL() === newEOL) { @@ -929,7 +933,11 @@ class Buffer { while (true) { const leaf = this._leafs[leafIndex]; - const leafResult = leaf.findLineFirstNonWhitespaceIndexInLeaf(searchStartOffset); + const leafResult = leaf.findLineFirstNonWhitespaceIndex(searchStartOffset); + if (leafResult === -2) { + // reached EOL + return -1; + } if (leafResult !== -1) { return (leafResult - searchStartOffset) + totalDelta; } @@ -945,6 +953,52 @@ class Buffer { } } + public getLineLastNonWhitespaceIndex(lineNumber: number): number { + const start = BufferCursorPool.take(); + const end = BufferCursorPool.take(); + + if (!this._findLineStart(lineNumber, start)) { + BufferCursorPool.put(start); + BufferCursorPool.put(end); + throw new Error(`Line not found`); + } + + this._findLineEnd(start, lineNumber, end); + + const startOffset = start.offset; + const endOffset = end.offset; + let leafIndex = end.leafIndex; + let searchStartOffset = end.offset - end.leafStartOffset - this._eolLength; + + BufferCursorPool.put(start); + BufferCursorPool.put(end); + + let totalDelta = 0; + while (true) { + const leaf = this._leafs[leafIndex]; + + const leafResult = leaf.findLineLastNonWhitespaceIndex(searchStartOffset); + if (leafResult === -2) { + // reached EOL + return -1; + } + if (leafResult !== -1) { + const delta = (searchStartOffset - 1 - leafResult); + const absoluteOffset = (endOffset - this._eolLength) - delta - totalDelta; + return absoluteOffset - startOffset; + } + + leafIndex--; + + if (leafIndex < 0) { + return -1; + } + + totalDelta += searchStartOffset; + searchStartOffset = leaf.length(); + } + } + public getLinesContent(): string[] { let result: string[] = new Array(this.getLineCount()); let resultIndex = 0; From 5422c4bd0a5fa79c340ea1d0941d8be7ec66de19 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Wed, 17 Jan 2018 09:32:20 +0100 Subject: [PATCH 29/33] Handle BOM (if present at the start of the first piece) --- .../model/chunksTextBuffer/chunksTextBuffer.ts | 7 ++++--- .../chunksTextBuffer/chunksTextBufferBuilder.ts | 14 ++++++++++++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts index 33d31749722..13a8933380e 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts @@ -24,11 +24,13 @@ export interface IValidatedEditOperation { export class ChunksTextBuffer implements ITextBuffer { + private _BOM: string; private _actual: Buffer; private _mightContainRTL: boolean; private _mightContainNonBasicASCII: boolean; - constructor(pieces: BufferPiece[], _averageChunkSize: number, eol: '\r\n' | '\n', containsRTL: boolean, isBasicASCII: boolean) { + constructor(pieces: BufferPiece[], _averageChunkSize: number, BOM: string, eol: '\r\n' | '\n', containsRTL: boolean, isBasicASCII: boolean) { + this._BOM = BOM; const averageChunkSize = Math.floor(Math.min(65536.0, Math.max(128.0, _averageChunkSize))); const delta = Math.floor(averageChunkSize / 3); const min = averageChunkSize - delta; @@ -51,8 +53,7 @@ export class ChunksTextBuffer implements ITextBuffer { return this._mightContainNonBasicASCII; } getBOM(): string { - // TODO - return ''; + return this._BOM; } getEOL(): string { return this._actual.getEOL(); diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts index e1268de0dd5..8be31da92c0 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts @@ -15,6 +15,7 @@ export class TextBufferFactory implements ITextBufferFactory { constructor( private readonly _pieces: BufferPiece[], private readonly _averageChunkSize: number, + private readonly _BOM: string, private readonly _cr: number, private readonly _lf: number, private readonly _crlf: number, @@ -56,7 +57,7 @@ export class TextBufferFactory implements ITextBufferFactory { pieces[i] = BufferPiece.normalizeEOL(pieces[i], eol); } } - return new ChunksTextBuffer(pieces, this._averageChunkSize, eol, this._containsRTL, this._isBasicASCII); + return new ChunksTextBuffer(pieces, this._averageChunkSize, this._BOM, eol, this._containsRTL, this._isBasicASCII); } public getFirstLineText(lengthLimit: number): string { @@ -73,6 +74,7 @@ export class ChunksTextBufferBuilder implements ITextBufferBuilder { private _averageChunkSize: number; private _tmpLineStarts: number[]; + private BOM: string; private cr: number; private lf: number; private crlf: number; @@ -86,6 +88,7 @@ export class ChunksTextBufferBuilder implements ITextBufferBuilder { this._averageChunkSize = 0; this._tmpLineStarts = []; + this.BOM = ''; this.cr = 0; this.lf = 0; this.crlf = 0; @@ -98,6 +101,13 @@ export class ChunksTextBufferBuilder implements ITextBufferBuilder { return; } + if (this._rawPieces.length === 0) { + if (strings.startsWithUTF8BOM(chunk)) { + this.BOM = strings.UTF8_BOM_CHARACTER; + chunk = chunk.substr(1); + } + } + this._averageChunkSize = (this._averageChunkSize * this._rawPieces.length + chunk.length) / (this._rawPieces.length + 1); const lastChar = chunk.charCodeAt(chunk.length - 1); @@ -145,7 +155,7 @@ export class ChunksTextBufferBuilder implements ITextBufferBuilder { public finish(): TextBufferFactory { this._finish(); - return new TextBufferFactory(this._rawPieces, this._averageChunkSize, this.cr, this.lf, this.crlf, this.containsRTL, this.isBasicASCII); + return new TextBufferFactory(this._rawPieces, this._averageChunkSize, this.BOM, this.cr, this.lf, this.crlf, this.containsRTL, this.isBasicASCII); } private _finish(): void { From 53669f642eb17ce82a530569f30d3cb82a2c8fb8 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Wed, 17 Jan 2018 11:10:58 +0100 Subject: [PATCH 30/33] Improved implementations --- .../model/chunksTextBuffer/bufferPiece.ts | 20 +++--- .../chunksTextBufferBuilder.ts | 9 ++- .../chunksTextBuffer/bufferPiece.test.ts | 61 +++++++++++++++++++ 3 files changed, 81 insertions(+), 9 deletions(-) create mode 100644 src/vs/editor/test/common/model/chunksTextBuffer/bufferPiece.test.ts diff --git a/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts b/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts index f91a6157d1c..53063409aea 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts @@ -54,16 +54,22 @@ export class BufferPiece { return -1; } - // TODO: implement binary search - for (let i = this._lineStarts.length - 1; i >= 0; i--) { - let lineStart = this._lineStarts[i]; + let low = 0, high = this._lineStarts.length - 1; - if (lineStart <= offset) { - return i; + while (low < high) { + let mid = low + Math.ceil((high - low) / 2); + let lineStart = this._lineStarts[mid]; + + if (offset === lineStart) { + return mid; + } else if (offset < lineStart) { + high = mid - 1; + } else { + low = mid; } } - return -1; + return low; } public findLineFirstNonWhitespaceIndex(searchStartOffset: number): number { @@ -111,7 +117,7 @@ export class BufferPiece { } let newLineStarts = new Uint32Array(newLineStartsLength); - newLineStarts.set(targetLineStarts); // TODO: does this work correctly? + newLineStarts.set(targetLineStarts); return new BufferPiece( target._str.substr(0, targetCharsLength - 1), diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts index 8be31da92c0..12eaf167b28 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts @@ -61,8 +61,13 @@ export class TextBufferFactory implements ITextBufferFactory { } public getFirstLineText(lengthLimit: number): string { - console.log(`TODO`); - return ''; + const firstPiece = this._pieces[0]; + if (firstPiece.newLineCount() === 0) { + return firstPiece.substr(0, lengthLimit); + } + + const firstEOLOffset = firstPiece.lineStartFor(0); + return firstPiece.substr(0, Math.min(lengthLimit, firstEOLOffset)); } } diff --git a/src/vs/editor/test/common/model/chunksTextBuffer/bufferPiece.test.ts b/src/vs/editor/test/common/model/chunksTextBuffer/bufferPiece.test.ts new file mode 100644 index 00000000000..a6cc8f6f22d --- /dev/null +++ b/src/vs/editor/test/common/model/chunksTextBuffer/bufferPiece.test.ts @@ -0,0 +1,61 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +'use strict'; + +import * as assert from 'assert'; +import { BufferPiece } from 'vs/editor/common/model/chunksTextBuffer/bufferPiece'; + +suite('BufferPiece', () => { + test('findLineStartBeforeOffset', () => { + let piece = new BufferPiece([ + 'Line1\r\n', + 'l2\n', + 'another\r', + 'and\r\n', + 'finally\n', + 'last' + ].join('')); + + assert.equal(piece.length(), 35); + assert.deepEqual(piece.findLineStartBeforeOffset(0), -1); + assert.deepEqual(piece.findLineStartBeforeOffset(1), -1); + assert.deepEqual(piece.findLineStartBeforeOffset(2), -1); + assert.deepEqual(piece.findLineStartBeforeOffset(3), -1); + assert.deepEqual(piece.findLineStartBeforeOffset(4), -1); + assert.deepEqual(piece.findLineStartBeforeOffset(5), -1); + assert.deepEqual(piece.findLineStartBeforeOffset(6), -1); + assert.deepEqual(piece.findLineStartBeforeOffset(7), 0); + assert.deepEqual(piece.findLineStartBeforeOffset(8), 0); + assert.deepEqual(piece.findLineStartBeforeOffset(9), 0); + assert.deepEqual(piece.findLineStartBeforeOffset(10), 1); + assert.deepEqual(piece.findLineStartBeforeOffset(11), 1); + assert.deepEqual(piece.findLineStartBeforeOffset(12), 1); + assert.deepEqual(piece.findLineStartBeforeOffset(13), 1); + assert.deepEqual(piece.findLineStartBeforeOffset(14), 1); + assert.deepEqual(piece.findLineStartBeforeOffset(15), 1); + assert.deepEqual(piece.findLineStartBeforeOffset(16), 1); + assert.deepEqual(piece.findLineStartBeforeOffset(17), 1); + assert.deepEqual(piece.findLineStartBeforeOffset(18), 2); + assert.deepEqual(piece.findLineStartBeforeOffset(19), 2); + assert.deepEqual(piece.findLineStartBeforeOffset(20), 2); + assert.deepEqual(piece.findLineStartBeforeOffset(21), 2); + assert.deepEqual(piece.findLineStartBeforeOffset(22), 2); + assert.deepEqual(piece.findLineStartBeforeOffset(23), 3); + assert.deepEqual(piece.findLineStartBeforeOffset(24), 3); + assert.deepEqual(piece.findLineStartBeforeOffset(25), 3); + assert.deepEqual(piece.findLineStartBeforeOffset(26), 3); + assert.deepEqual(piece.findLineStartBeforeOffset(27), 3); + assert.deepEqual(piece.findLineStartBeforeOffset(28), 3); + assert.deepEqual(piece.findLineStartBeforeOffset(29), 3); + assert.deepEqual(piece.findLineStartBeforeOffset(30), 3); + assert.deepEqual(piece.findLineStartBeforeOffset(31), 4); + assert.deepEqual(piece.findLineStartBeforeOffset(32), 4); + assert.deepEqual(piece.findLineStartBeforeOffset(33), 4); + assert.deepEqual(piece.findLineStartBeforeOffset(34), 4); + assert.deepEqual(piece.findLineStartBeforeOffset(35), 4); + assert.deepEqual(piece.findLineStartBeforeOffset(36), 4); + }); +}); From 4d3ca2a56b3a74b5a13f30edeaed007a114ed840 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Wed, 17 Jan 2018 15:26:00 +0100 Subject: [PATCH 31/33] Implement more efficient getLineCharCode --- .../chunksTextBuffer/chunksTextBuffer.ts | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts index 13a8933380e..b4c26664a75 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts @@ -134,8 +134,7 @@ export class ChunksTextBuffer implements ITextBuffer { } getLineCharCode(lineNumber: number, index: number): number { - // TODO - return this.getLineContent(lineNumber).charCodeAt(index); + return this._actual.getLineCharCode(lineNumber, index); } getLineLength(lineNumber: number): number { @@ -894,6 +893,23 @@ class Buffer { return result; } + public getLineCharCode(lineNumber: number, index: number): number { + const start = BufferCursorPool.take(); + + if (!this._findLineStart(lineNumber, start)) { + BufferCursorPool.put(start); + throw new Error(`Line not found`); + } + + const tmp = BufferCursorPool.take(); + this._findOffsetCloseAfter(start.offset + index, start, tmp); + const result = this._leafs[tmp.leafIndex].charCodeAt(tmp.offset - tmp.leafStartNewLineCount); + BufferCursorPool.put(tmp); + + BufferCursorPool.put(start); + return result; + } + public getLineLength(lineNumber: number): number { const start = BufferCursorPool.take(); const end = BufferCursorPool.take(); From 2209a63eb77910b0ca619f529772ac8e5e73a623 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Wed, 17 Jan 2018 15:49:12 +0100 Subject: [PATCH 32/33] Fix bad merge --- src/vs/editor/common/model/linesTextBuffer/linesTextBuffer.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/vs/editor/common/model/linesTextBuffer/linesTextBuffer.ts b/src/vs/editor/common/model/linesTextBuffer/linesTextBuffer.ts index 52386d0ab8d..af20bfb5ade 100644 --- a/src/vs/editor/common/model/linesTextBuffer/linesTextBuffer.ts +++ b/src/vs/editor/common/model/linesTextBuffer/linesTextBuffer.ts @@ -190,10 +190,6 @@ export class LinesTextBuffer implements ITextBuffer { return endOffset - startOffset; } - public getLength(): number { - return this._lineStarts.getTotalValue(); - } - public getLineCount(): number { return this._lines.length; } From 2ffaacf56bedd72b6a4d7626a58586a48e8771c1 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Thu, 18 Jan 2018 17:55:18 +0100 Subject: [PATCH 33/33] Implement snapshots; prepare for merging to master --- .../chunksTextBuffer/chunksTextBuffer.ts | 40 +++++++++++++++++++ src/vs/editor/common/model/textModel.ts | 2 +- .../test/common/model/textModel.test.ts | 12 +++--- 3 files changed, 48 insertions(+), 6 deletions(-) diff --git a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts index b4c26664a75..d089cd00af0 100644 --- a/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts +++ b/src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts @@ -10,6 +10,7 @@ import { BufferPiece, LeafOffsetLenEdit } from 'vs/editor/common/model/chunksTex import { Position } from 'vs/editor/common/core/position'; import { Range } from 'vs/editor/common/core/range'; import * as strings from 'vs/base/common/strings'; +import { ITextSnapshot } from 'vs/platform/files/common/files'; export interface IValidatedEditOperation { sortIndex: number; @@ -92,6 +93,10 @@ export class ChunksTextBuffer implements ITextBuffer { return null; } + public createSnapshot(preserveBOM: boolean): ITextSnapshot { + return this._actual.createSnapshot(preserveBOM ? this._BOM : ''); + } + getValueLengthInRange(range: Range, eol: EndOfLinePreference): number { if (range.isEmpty()) { return 0; @@ -566,6 +571,37 @@ const BufferCursorPool = new class { } }; +class BufferSnapshot implements ITextSnapshot { + + private readonly _pieces: BufferPiece[]; + private readonly _piecesLength: number; + private readonly _BOM: string; + private _piecesIndex: number; + + constructor(pieces: BufferPiece[], BOM: string) { + this._pieces = pieces; + this._piecesLength = this._pieces.length; + this._BOM = BOM; + this._piecesIndex = 0; + } + + public read(): string { + if (this._piecesIndex >= this._piecesLength) { + return null; + } + + let result: string = null; + if (this._piecesIndex === 0) { + result = this._BOM + this._pieces[this._piecesIndex].text; + } else { + result = this._pieces[this._piecesIndex].text; + } + + this._piecesIndex++; + return result; + } +} + class Buffer { private _minLeafLength: number; @@ -1205,6 +1241,10 @@ class Buffer { return result; } + public createSnapshot(BOM: string): ITextSnapshot { + return new BufferSnapshot(this._leafs, BOM); + } + public getValueLengthInRange(range: Range): number { const startOffset = this.convertPositionToOffset(range.startLineNumber, range.startColumn); const endOffset = this.convertPositionToOffset(range.endLineNumber, range.endColumn); diff --git a/src/vs/editor/common/model/textModel.ts b/src/vs/editor/common/model/textModel.ts index 88777784b31..6506c95e3b0 100644 --- a/src/vs/editor/common/model/textModel.ts +++ b/src/vs/editor/common/model/textModel.ts @@ -37,7 +37,7 @@ import { LinesTextBufferBuilder } from 'vs/editor/common/model/linesTextBuffer/l import { ChunksTextBufferBuilder } from 'vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder'; // Here is the master switch for the text buffer implementation: -const USE_CHUNKS_TEXT_BUFFER = true; +const USE_CHUNKS_TEXT_BUFFER = false; function createTextBufferBuilder() { if (USE_CHUNKS_TEXT_BUFFER) { return new ChunksTextBufferBuilder(); diff --git a/src/vs/editor/test/common/model/textModel.test.ts b/src/vs/editor/test/common/model/textModel.test.ts index be2fd30bd13..a99bde0f438 100644 --- a/src/vs/editor/test/common/model/textModel.test.ts +++ b/src/vs/editor/test/common/model/textModel.test.ts @@ -891,16 +891,18 @@ suite('TextModel.createSnapshot', () => { let snapshot = model.createSnapshot(); let actual = ''; - // 70999 length => 2 read calls are necessary + // 70999 length => at most 2 read calls are necessary let tmp1 = snapshot.read(); assert.ok(tmp1); actual += tmp1; let tmp2 = snapshot.read(); - assert.ok(tmp2); - actual += tmp2; - - assert.equal(snapshot.read(), null); + if (tmp2 === null) { + // all good + } else { + actual += tmp2; + assert.equal(snapshot.read(), null); + } assert.equal(actual, text);