From c2dafddbae5dc7f9b31c2f60617abd8dcc9f7896 Mon Sep 17 00:00:00 2001 From: Matt Bierner Date: Wed, 4 Nov 2020 12:13:28 -0800 Subject: [PATCH] Add splitLines helper function (#109869) * Add splitLines helper function I noticed there are a lot of places in our codebase what split strings using a hardcoded `/\r\n|\r|\n/` regular expression. This change extracts that to a new `strings.splitLines` helper * Update snippetSession.ts --- src/vs/base/common/strings.ts | 4 ++++ .../common/controller/cursorTypeOperations.ts | 2 +- src/vs/editor/common/model/mirrorTextModel.ts | 3 ++- src/vs/editor/common/modes/textToHtmlTokenizer.ts | 2 +- .../editor/contrib/gotoError/gotoErrorWidget.ts | 3 ++- src/vs/editor/contrib/snippet/snippetVariables.ts | 4 ++-- src/vs/editor/standalone/browser/colorizer.ts | 2 +- .../editor/standalone/browser/standaloneEditor.ts | 3 ++- .../linesTextBuffer/textBufferAutoTestUtils.ts | 7 ++++--- .../pieceTreeTextBuffer.test.ts | 15 ++++++++------- .../api/common/extHostDocumentContentProviders.ts | 3 ++- .../browser/parts/editor/editorStatus.ts | 4 ++-- .../contrib/markers/browser/markersModel.ts | 3 ++- .../contrib/output/common/outputLinkComputer.ts | 2 +- .../tags/electron-browser/workspaceTagsService.ts | 5 +++-- .../themes/browser/themes.test.contribution.ts | 3 ++- 16 files changed, 39 insertions(+), 26 deletions(-) diff --git a/src/vs/base/common/strings.ts b/src/vs/base/common/strings.ts index 9e35fd6cb9c..f571440f7d0 100644 --- a/src/vs/base/common/strings.ts +++ b/src/vs/base/common/strings.ts @@ -208,6 +208,10 @@ export function regExpFlags(regexp: RegExp): string { + ((regexp as any /* standalone editor compilation */).unicode ? 'u' : ''); } +export function splitLines(str: string): string[] { + return str.split(/\r\n|\r|\n/); +} + /** * Returns first index of the string that is not whitespace. * If string is empty or contains only whitespaces, returns -1 diff --git a/src/vs/editor/common/controller/cursorTypeOperations.ts b/src/vs/editor/common/controller/cursorTypeOperations.ts index d383ffc63cd..cf538a83386 100644 --- a/src/vs/editor/common/controller/cursorTypeOperations.ts +++ b/src/vs/editor/common/controller/cursorTypeOperations.ts @@ -128,7 +128,7 @@ export class TypeOperations { if (text.charCodeAt(text.length - 1) === CharCode.CarriageReturn) { text = text.substr(0, text.length - 1); } - let lines = text.split(/\r\n|\r|\n/); + let lines = strings.splitLines(text); if (lines.length === selections.length) { return lines; } diff --git a/src/vs/editor/common/model/mirrorTextModel.ts b/src/vs/editor/common/model/mirrorTextModel.ts index 6a1272552b0..9e14ea63640 100644 --- a/src/vs/editor/common/model/mirrorTextModel.ts +++ b/src/vs/editor/common/model/mirrorTextModel.ts @@ -3,6 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ +import { splitLines } from 'vs/base/common/strings'; import { URI } from 'vs/base/common/uri'; import { Position } from 'vs/editor/common/core/position'; import { IRange } from 'vs/editor/common/core/range'; @@ -131,7 +132,7 @@ export class MirrorTextModel { // Nothing to insert return; } - let insertLines = insertText.split(/\r\n|\r|\n/); + let insertLines = splitLines(insertText); if (insertLines.length === 1) { // Inserting text on one line this._setLineText(position.lineNumber - 1, diff --git a/src/vs/editor/common/modes/textToHtmlTokenizer.ts b/src/vs/editor/common/modes/textToHtmlTokenizer.ts index dd64e79ded6..c486664163e 100644 --- a/src/vs/editor/common/modes/textToHtmlTokenizer.ts +++ b/src/vs/editor/common/modes/textToHtmlTokenizer.ts @@ -101,7 +101,7 @@ export function tokenizeLineToHTML(text: string, viewLineTokens: IViewLineTokens function _tokenizeToString(text: string, tokenizationSupport: IReducedTokenizationSupport): string { let result = `
`; - let lines = text.split(/\r\n|\r|\n/); + let lines = strings.splitLines(text); let currentState = tokenizationSupport.getInitialState(); for (let i = 0, len = lines.length; i < len; i++) { let line = lines[i]; diff --git a/src/vs/editor/contrib/gotoError/gotoErrorWidget.ts b/src/vs/editor/contrib/gotoError/gotoErrorWidget.ts index 66bdc31f06c..93800f9f33f 100644 --- a/src/vs/editor/contrib/gotoError/gotoErrorWidget.ts +++ b/src/vs/editor/contrib/gotoError/gotoErrorWidget.ts @@ -30,6 +30,7 @@ import { MenuId, IMenuService } from 'vs/platform/actions/common/actions'; import { IContextKeyService } from 'vs/platform/contextkey/common/contextkey'; import { createAndFillInActionBarActions } from 'vs/platform/actions/browser/menuEntryActionViewItem'; import { IInstantiationService } from 'vs/platform/instantiation/common/instantiation'; +import { splitLines } from 'vs/base/common/strings'; class MessageWidget { @@ -102,7 +103,7 @@ class MessageWidget { } } - const lines = message.split(/\r\n|\r|\n/g); + const lines = splitLines(message); this._lines = lines.length; this._longestLineLength = 0; for (const line of lines) { diff --git a/src/vs/editor/contrib/snippet/snippetVariables.ts b/src/vs/editor/contrib/snippet/snippetVariables.ts index cd5d505bcc9..bdfe96e7c65 100644 --- a/src/vs/editor/contrib/snippet/snippetVariables.ts +++ b/src/vs/editor/contrib/snippet/snippetVariables.ts @@ -10,7 +10,7 @@ import { ITextModel } from 'vs/editor/common/model'; import { Selection } from 'vs/editor/common/core/selection'; import { VariableResolver, Variable, Text } from 'vs/editor/contrib/snippet/snippetParser'; import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry'; -import { getLeadingWhitespace, commonPrefixLength, isFalsyOrWhitespace } from 'vs/base/common/strings'; +import { getLeadingWhitespace, commonPrefixLength, isFalsyOrWhitespace, splitLines } from 'vs/base/common/strings'; import { IWorkspaceContextService } from 'vs/platform/workspace/common/workspace'; import { isSingleFolderWorkspaceIdentifier, toWorkspaceIdentifier, WORKSPACE_EXTENSION, IWorkspaceIdentifier } from 'vs/platform/workspaces/common/workspaces'; import { ILabelService } from 'vs/platform/label/common/label'; @@ -111,7 +111,7 @@ export class SelectionBasedVariableResolver implements VariableResolver { return false; } if (marker instanceof Text) { - varLeadingWhitespace = getLeadingWhitespace(marker.value.split(/\r\n|\r|\n/).pop()!); + varLeadingWhitespace = getLeadingWhitespace(splitLines(marker.value).pop()!); } return true; }); diff --git a/src/vs/editor/standalone/browser/colorizer.ts b/src/vs/editor/standalone/browser/colorizer.ts index fdde96abcf0..a45712fa85d 100644 --- a/src/vs/editor/standalone/browser/colorizer.ts +++ b/src/vs/editor/standalone/browser/colorizer.ts @@ -54,7 +54,7 @@ export class Colorizer { if (strings.startsWithUTF8BOM(text)) { text = text.substr(1); } - let lines = text.split(/\r\n|\r|\n/); + let lines = strings.splitLines(text); let language = modeService.getModeId(mimeType); if (!language) { return Promise.resolve(_fakeColorize(lines, tabSize)); diff --git a/src/vs/editor/standalone/browser/standaloneEditor.ts b/src/vs/editor/standalone/browser/standaloneEditor.ts index fb2aaac0277..6d5fc9c32e4 100644 --- a/src/vs/editor/standalone/browser/standaloneEditor.ts +++ b/src/vs/editor/standalone/browser/standaloneEditor.ts @@ -41,6 +41,7 @@ import { clearAllFontInfos } from 'vs/editor/browser/config/configuration'; import { IEditorProgressService } from 'vs/platform/progress/common/progress'; import { IClipboardService } from 'vs/platform/clipboard/common/clipboardService'; import { StandaloneThemeServiceImpl } from 'vs/editor/standalone/browser/standaloneThemeServiceImpl'; +import { splitLines } from 'vs/base/common/strings'; type Omit = Pick>; @@ -288,7 +289,7 @@ export function tokenize(text: string, languageId: string): Token[][] { modeService.triggerMode(languageId); let tokenizationSupport = getSafeTokenizationSupport(languageId); - let lines = text.split(/\r\n|\r|\n/); + let lines = splitLines(text); let result: Token[][] = []; let state = tokenizationSupport.getInitialState(); for (let i = 0, len = lines.length; i < len; i++) { diff --git a/src/vs/editor/test/common/model/linesTextBuffer/textBufferAutoTestUtils.ts b/src/vs/editor/test/common/model/linesTextBuffer/textBufferAutoTestUtils.ts index 49841dabc05..4f87bf6388e 100644 --- a/src/vs/editor/test/common/model/linesTextBuffer/textBufferAutoTestUtils.ts +++ b/src/vs/editor/test/common/model/linesTextBuffer/textBufferAutoTestUtils.ts @@ -4,6 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { CharCode } from 'vs/base/common/charCode'; +import { splitLines } from 'vs/base/common/strings'; import { Range } from 'vs/editor/common/core/range'; import { DefaultEndOfLine, ITextBuffer, ITextBufferBuilder, ValidAnnotatedEditOperation } from 'vs/editor/common/model'; @@ -34,7 +35,7 @@ export function getRandomString(minLength: number, maxLength: number): string { export function generateRandomEdits(chunks: string[], editCnt: number): ValidAnnotatedEditOperation[] { let lines: string[] = []; for (const chunk of chunks) { - let newLines = chunk.split(/\r\n|\r|\n/); + let newLines = splitLines(chunk); if (lines.length === 0) { lines.push(...newLines); } else { @@ -64,7 +65,7 @@ export function generateRandomEdits(chunks: string[], editCnt: number): ValidAnn export function generateSequentialInserts(chunks: string[], editCnt: number): ValidAnnotatedEditOperation[] { let lines: string[] = []; for (const chunk of chunks) { - let newLines = chunk.split(/\r\n|\r|\n/); + let newLines = splitLines(chunk); if (lines.length === 0) { lines.push(...newLines); } else { @@ -96,7 +97,7 @@ export function generateSequentialInserts(chunks: string[], editCnt: number): Va export function generateRandomReplaces(chunks: string[], editCnt: number, searchStringLen: number, replaceStringLen: number): ValidAnnotatedEditOperation[] { let lines: string[] = []; for (const chunk of chunks) { - let newLines = chunk.split(/\r\n|\r|\n/); + let newLines = splitLines(chunk); if (lines.length === 0) { lines.push(...newLines); } else { diff --git a/src/vs/editor/test/common/model/pieceTreeTextBuffer/pieceTreeTextBuffer.test.ts b/src/vs/editor/test/common/model/pieceTreeTextBuffer/pieceTreeTextBuffer.test.ts index ddec1faccd1..019b23f4f3f 100644 --- a/src/vs/editor/test/common/model/pieceTreeTextBuffer/pieceTreeTextBuffer.test.ts +++ b/src/vs/editor/test/common/model/pieceTreeTextBuffer/pieceTreeTextBuffer.test.ts @@ -14,6 +14,7 @@ import { PieceTreeTextBufferBuilder } from 'vs/editor/common/model/pieceTreeText import { NodeColor, SENTINEL, TreeNode } from 'vs/editor/common/model/pieceTreeTextBuffer/rbTreeBase'; import { createTextModel } from 'vs/editor/test/common/editorTestUtils'; import { SearchData } from 'vs/editor/common/model/textModelSearch'; +import { splitLines } from 'vs/base/common/strings'; const alphabet = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ\r\n'; @@ -75,7 +76,7 @@ function trimLineFeed(text: string): string { //#region Assertion function testLinesContent(str: string, pieceTable: PieceTreeBase) { - let lines = str.split(/\r\n|\r|\n/); + let lines = splitLines(str); assert.equal(pieceTable.getLineCount(), lines.length); assert.equal(pieceTable.getLinesRawContent(), str); for (let i = 0; i < lines.length; i++) { @@ -997,7 +998,7 @@ suite('CRLF', () => { pieceTable.delete(2, 3); str = str.substring(0, 2) + str.substring(2 + 3); - let lines = str.split(/\r\n|\r|\n/); + let lines = splitLines(str); assert.equal(pieceTable.getLineCount(), lines.length); assertTreeInvariants(pieceTable); }); @@ -1012,7 +1013,7 @@ suite('CRLF', () => { pieceTable.delete(4, 1); str = str.substring(0, 4) + str.substring(4 + 1); - let lines = str.split(/\r\n|\r|\n/); + let lines = splitLines(str); assert.equal(pieceTable.getLineCount(), lines.length); assertTreeInvariants(pieceTable); }); @@ -1033,7 +1034,7 @@ suite('CRLF', () => { pieceTable.insert(3, '\r\r\r\n'); str = str.substring(0, 3) + '\r\r\r\n' + str.substring(3); - let lines = str.split(/\r\n|\r|\n/); + let lines = splitLines(str); assert.equal(pieceTable.getLineCount(), lines.length); assertTreeInvariants(pieceTable); }); @@ -1205,7 +1206,7 @@ suite('centralized lineStarts with CRLF', () => { pieceTable.delete(2, 3); str = str.substring(0, 2) + str.substring(2 + 3); - let lines = str.split(/\r\n|\r|\n/); + let lines = splitLines(str); assert.equal(pieceTable.getLineCount(), lines.length); assertTreeInvariants(pieceTable); }); @@ -1218,7 +1219,7 @@ suite('centralized lineStarts with CRLF', () => { pieceTable.delete(4, 1); str = str.substring(0, 4) + str.substring(4 + 1); - let lines = str.split(/\r\n|\r|\n/); + let lines = splitLines(str); assert.equal(pieceTable.getLineCount(), lines.length); assertTreeInvariants(pieceTable); }); @@ -1238,7 +1239,7 @@ suite('centralized lineStarts with CRLF', () => { pieceTable.insert(3, '\r\r\r\n'); str = str.substring(0, 3) + '\r\r\r\n' + str.substring(3); - let lines = str.split(/\r\n|\r|\n/); + let lines = splitLines(str); assert.equal(pieceTable.getLineCount(), lines.length); assertTreeInvariants(pieceTable); }); diff --git a/src/vs/workbench/api/common/extHostDocumentContentProviders.ts b/src/vs/workbench/api/common/extHostDocumentContentProviders.ts index eba5e784576..e4d2b49a4e1 100644 --- a/src/vs/workbench/api/common/extHostDocumentContentProviders.ts +++ b/src/vs/workbench/api/common/extHostDocumentContentProviders.ts @@ -13,6 +13,7 @@ import { ExtHostDocumentsAndEditors } from './extHostDocumentsAndEditors'; import { Schemas } from 'vs/base/common/network'; import { ILogService } from 'vs/platform/log/common/log'; import { CancellationToken } from 'vs/base/common/cancellation'; +import { splitLines } from 'vs/base/common/strings'; export class ExtHostDocumentContentProvider implements ExtHostDocumentContentProvidersShape { @@ -61,7 +62,7 @@ export class ExtHostDocumentContentProvider implements ExtHostDocumentContentPro } // create lines and compare - const lines = value.split(/\r\n|\r|\n/); + const lines = splitLines(value); // broadcast event when content changed if (!document.equalLines(lines)) { diff --git a/src/vs/workbench/browser/parts/editor/editorStatus.ts b/src/vs/workbench/browser/parts/editor/editorStatus.ts index bfa6f87dc73..29206ee3f6d 100644 --- a/src/vs/workbench/browser/parts/editor/editorStatus.ts +++ b/src/vs/workbench/browser/parts/editor/editorStatus.ts @@ -6,7 +6,7 @@ import 'vs/css!./media/editorstatus'; import * as nls from 'vs/nls'; import { runAtThisOrScheduleAtNextAnimationFrame } from 'vs/base/browser/dom'; -import { format, compare } from 'vs/base/common/strings'; +import { format, compare, splitLines } from 'vs/base/common/strings'; import { extname, basename, isEqual } from 'vs/base/common/resources'; import { areFunctions, withNullAsUndefined, withUndefinedAsNull } from 'vs/base/common/types'; import { URI } from 'vs/base/common/uri'; @@ -918,7 +918,7 @@ class ShowCurrentMarkerInStatusbarContribution extends Disposable { this.currentMarker = this.getMarker(); if (this.hasToUpdateStatus(previousMarker, this.currentMarker)) { if (this.currentMarker) { - const line = this.currentMarker.message.split(/\r\n|\r|\n/g)[0]; + const line = splitLines(this.currentMarker.message)[0]; const text = `${this.getType(this.currentMarker)} ${line}`; if (!this.statusBarEntryAccessor.value) { this.statusBarEntryAccessor.value = this.statusbarService.addEntry({ text: '', ariaLabel: '' }, 'statusbar.currentProblem', nls.localize('currentProblem', "Current Problem"), StatusbarAlignment.LEFT); diff --git a/src/vs/workbench/contrib/markers/browser/markersModel.ts b/src/vs/workbench/contrib/markers/browser/markersModel.ts index f8016d3d663..f2544a4023e 100644 --- a/src/vs/workbench/contrib/markers/browser/markersModel.ts +++ b/src/vs/workbench/contrib/markers/browser/markersModel.ts @@ -12,6 +12,7 @@ import { ResourceMap } from 'vs/base/common/map'; import { Emitter, Event } from 'vs/base/common/event'; import { Hasher } from 'vs/base/common/hash'; import { withUndefinedAsNull } from 'vs/base/common/types'; +import { splitLines } from 'vs/base/common/strings'; export function compareMarkersByUri(a: IMarker, b: IMarker) { @@ -95,7 +96,7 @@ export class Marker { private _lines: string[] | undefined; get lines(): string[] { if (!this._lines) { - this._lines = this.marker.message.split(/\r\n|\r|\n/g); + this._lines = splitLines(this.marker.message); } return this._lines; } diff --git a/src/vs/workbench/contrib/output/common/outputLinkComputer.ts b/src/vs/workbench/contrib/output/common/outputLinkComputer.ts index e532dbc50c2..d9297827ba6 100644 --- a/src/vs/workbench/contrib/output/common/outputLinkComputer.ts +++ b/src/vs/workbench/contrib/output/common/outputLinkComputer.ts @@ -56,7 +56,7 @@ export class OutputLinkComputer { } const links: ILink[] = []; - const lines = model.getValue().split(/\r\n|\r|\n/); + const lines = strings.splitLines(model.getValue()); // For each workspace root patterns for (const [folderUri, folderPatterns] of this.patterns) { diff --git a/src/vs/workbench/contrib/tags/electron-browser/workspaceTagsService.ts b/src/vs/workbench/contrib/tags/electron-browser/workspaceTagsService.ts index 35b943f8750..dffea760b9a 100644 --- a/src/vs/workbench/contrib/tags/electron-browser/workspaceTagsService.ts +++ b/src/vs/workbench/contrib/tags/electron-browser/workspaceTagsService.ts @@ -14,6 +14,7 @@ import { registerSingleton } from 'vs/platform/instantiation/common/extensions'; import { IWorkspaceTagsService, Tags } from 'vs/workbench/contrib/tags/common/workspaceTags'; import { getHashedRemotesFromConfig } from 'vs/workbench/contrib/tags/electron-browser/workspaceTags'; import { IProductService } from 'vs/platform/product/common/productService'; +import { splitLines } from 'vs/base/common/strings'; const MetaModulesToLookFor = [ // Azure packages @@ -412,7 +413,7 @@ export class WorkspaceTagsService implements IWorkspaceTagsService { } const requirementsTxtPromises = getFilePromises('requirements.txt', this.fileService, this.textFileService, content => { - const dependencies: string[] = content.value.split(/\r\n|\r|\n/); + const dependencies: string[] = splitLines(content.value); for (let dependency of dependencies) { // Dependencies in requirements.txt can have 3 formats: `foo==3.1, foo>=3.1, foo` const format1 = dependency.split('=='); @@ -423,7 +424,7 @@ export class WorkspaceTagsService implements IWorkspaceTagsService { }); const pipfilePromises = getFilePromises('pipfile', this.fileService, this.textFileService, content => { - let dependencies: string[] = content.value.split(/\r\n|\r|\n/); + let dependencies: string[] = splitLines(content.value); // We're only interested in the '[packages]' section of the Pipfile dependencies = dependencies.slice(dependencies.indexOf('[packages]') + 1); diff --git a/src/vs/workbench/contrib/themes/browser/themes.test.contribution.ts b/src/vs/workbench/contrib/themes/browser/themes.test.contribution.ts index 0eb622bbef6..d97e3c08705 100644 --- a/src/vs/workbench/contrib/themes/browser/themes.test.contribution.ts +++ b/src/vs/workbench/contrib/themes/browser/themes.test.contribution.ts @@ -18,6 +18,7 @@ import { Color } from 'vs/base/common/color'; import { IFileService } from 'vs/platform/files/common/files'; import { basename } from 'vs/base/common/resources'; import { Schemas } from 'vs/base/common/network'; +import { splitLines } from 'vs/base/common/strings'; interface IToken { c: string; @@ -220,7 +221,7 @@ class Snapper { if (!grammar) { return []; } - let lines = content.split(/\r\n|\r|\n/); + let lines = splitLines(content); let result = this._tokenize(grammar, lines); return this._getThemesResult(grammar, lines).then((themesResult) => {