Microsoft/vscode
1
0
Fork 0
mirror of https://github.com/microsoft/vscode.git synced 2026-04-25 11:08:51 +01:00
Code Issues Packages Projects Releases Wiki Activity

API: Finalise text document encoding (fix #241449) (#246016)

Browse Source 
* API: Finalise text document encoding (fix #241449)

* address feedback
This commit is contained in:
Benjamin Pasero
2025-04-10 20:28:21 +02:00
committed by GitHub
parent 25f2519859
commit f0ee501c7f
10 changed files with 212 additions and 228 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
extensions/vscode-api-tests/src/singlefolder-tests/workspace.test.ts
View File

@@ -1373,7 +1373,7 @@ suite('vscode API - workspace', () => {
let err;
try {
await vscode.workspace.decode(new Uint8Array([0, 0, 0, 0]), doc1.uri);
await vscode.workspace.decode(new Uint8Array([0, 0, 0, 0]), { uri: doc1.uri });
} catch (e) {
err = e;
}
@@ -1403,18 +1403,18 @@ suite('vscode API - workspace', () => {
const uri = root.with({ path: posix.join(root.path, 'file.txt') });
// without setting
assert.strictEqual(await vscode.workspace.decode(Buffer.from('Hello World'), uri), 'Hello World');
assert.strictEqual(await vscode.workspace.decode(Buffer.from('Hellö Wörld'), uri), 'Hellö Wörld');
assert.strictEqual(await vscode.workspace.decode(new Uint8Array([0xEF, 0xBB, 0xBF, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100]), uri), 'Hello World'); // UTF-8 with BOM
assert.strictEqual(await vscode.workspace.decode(new Uint8Array([0xFE, 0xFF, 0, 72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 100]), uri), 'Hello World'); // UTF-16 BE with BOM
assert.strictEqual(await vscode.workspace.decode(new Uint8Array([0xFF, 0xFE, 72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 100, 0]), uri), 'Hello World'); // UTF-16 LE with BOM
assert.strictEqual(await vscode.workspace.decode(new Uint8Array([0, 72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 100]), uri), 'Hello World');
assert.strictEqual(await vscode.workspace.decode(new Uint8Array([72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 100, 0]), uri), 'Hello World');
assert.strictEqual(await vscode.workspace.decode(Buffer.from('Hello World'), { uri }), 'Hello World');
assert.strictEqual(await vscode.workspace.decode(Buffer.from('Hellö Wörld'), { uri }), 'Hellö Wörld');
assert.strictEqual(await vscode.workspace.decode(new Uint8Array([0xEF, 0xBB, 0xBF, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100]), { uri }), 'Hello World'); // UTF-8 with BOM
assert.strictEqual(await vscode.workspace.decode(new Uint8Array([0xFE, 0xFF, 0, 72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 100]), { uri }), 'Hello World'); // UTF-16 BE with BOM
assert.strictEqual(await vscode.workspace.decode(new Uint8Array([0xFF, 0xFE, 72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 100, 0]), { uri }), 'Hello World'); // UTF-16 LE with BOM
assert.strictEqual(await vscode.workspace.decode(new Uint8Array([0, 72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 100]), { uri }), 'Hello World');
assert.strictEqual(await vscode.workspace.decode(new Uint8Array([72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 100, 0]), { uri }), 'Hello World');
// with auto-guess encoding
try {
await vscode.workspace.getConfiguration('files', uri).update('autoGuessEncoding', true, vscode.ConfigurationTarget.Global);
assert.strictEqual(await vscode.workspace.decode(new Uint8Array([72, 101, 108, 108, 0xF6, 32, 87, 0xF6, 114, 108, 100]), uri), 'Hellö Wörld');
assert.strictEqual(await vscode.workspace.decode(new Uint8Array([72, 101, 108, 108, 0xF6, 32, 87, 0xF6, 114, 108, 100]), { uri }), 'Hellö Wörld');
} finally {
await vscode.workspace.getConfiguration('files', uri).update('autoGuessEncoding', false, vscode.ConfigurationTarget.Global);
}
@@ -1422,19 +1422,19 @@ suite('vscode API - workspace', () => {
// with encoding setting
try {
await vscode.workspace.getConfiguration('files', uri).update('encoding', 'windows1252', vscode.ConfigurationTarget.Global);
assert.strictEqual(await vscode.workspace.decode(new Uint8Array([72, 101, 108, 108, 0xF6, 32, 87, 0xF6, 114, 108, 100]), uri), 'Hellö Wörld');
assert.strictEqual(await vscode.workspace.decode(new Uint8Array([72, 101, 108, 108, 0xF6, 32, 87, 0xF6, 114, 108, 100]), { uri }), 'Hellö Wörld');
} finally {
await vscode.workspace.getConfiguration('files', uri).update('encoding', 'utf8', vscode.ConfigurationTarget.Global);
}
// with encoding provided
assert.strictEqual(await vscode.workspace.decode(new Uint8Array([72, 101, 108, 108, 0xF6, 32, 87, 0xF6, 114, 108, 100]), uri, { encoding: 'windows1252' }), 'Hellö Wörld');
assert.strictEqual(await vscode.workspace.decode(Buffer.from('Hello World'), uri, { encoding: 'foobar123' }), 'Hello World');
assert.strictEqual(await vscode.workspace.decode(new Uint8Array([72, 101, 108, 108, 0xF6, 32, 87, 0xF6, 114, 108, 100]), { encoding: 'windows1252' }), 'Hellö Wörld');
assert.strictEqual(await vscode.workspace.decode(Buffer.from('Hello World'), { encoding: 'foobar123' }), 'Hello World');
// binary
let err;
try {
await vscode.workspace.decode(new Uint8Array([0, 0, 0, 0]), uri);
await vscode.workspace.decode(new Uint8Array([0, 0, 0, 0]), { uri });
} catch (e) {
err = e;
}
@@ -1445,32 +1445,32 @@ suite('vscode API - workspace', () => {
const uri = root.with({ path: posix.join(root.path, 'file.txt') });
// without setting
assert.strictEqual((await vscode.workspace.encode('Hello World', uri)).toString(), 'Hello World');
assert.strictEqual((await vscode.workspace.encode('Hello World', { uri })).toString(), 'Hello World');
// with encoding setting
try {
await vscode.workspace.getConfiguration('files', uri).update('encoding', 'utf8bom', vscode.ConfigurationTarget.Global);
assert.ok(equalsUint8Array(await vscode.workspace.encode('Hello World', uri), new Uint8Array([0xEF, 0xBB, 0xBF, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100])));
assert.ok(equalsUint8Array(await vscode.workspace.encode('Hello World', { uri }), new Uint8Array([0xEF, 0xBB, 0xBF, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100])));
await vscode.workspace.getConfiguration('files', uri).update('encoding', 'utf16le', vscode.ConfigurationTarget.Global);
assert.ok(equalsUint8Array(await vscode.workspace.encode('Hello World', uri), new Uint8Array([0xFF, 0xFE, 72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 100, 0])));
assert.ok(equalsUint8Array(await vscode.workspace.encode('Hello World', { uri }), new Uint8Array([0xFF, 0xFE, 72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 100, 0])));
await vscode.workspace.getConfiguration('files', uri).update('encoding', 'utf16be', vscode.ConfigurationTarget.Global);
assert.ok(equalsUint8Array(await vscode.workspace.encode('Hello World', uri), new Uint8Array([0xFE, 0xFF, 0, 72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 100])));
assert.ok(equalsUint8Array(await vscode.workspace.encode('Hello World', { uri }), new Uint8Array([0xFE, 0xFF, 0, 72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 100])));
await vscode.workspace.getConfiguration('files', uri).update('encoding', 'cp1252', vscode.ConfigurationTarget.Global);
assert.ok(equalsUint8Array(await vscode.workspace.encode('Hellö Wörld', uri), new Uint8Array([72, 101, 108, 108, 0xF6, 32, 87, 0xF6, 114, 108, 100])));
assert.ok(equalsUint8Array(await vscode.workspace.encode('Hellö Wörld', { uri }), new Uint8Array([72, 101, 108, 108, 0xF6, 32, 87, 0xF6, 114, 108, 100])));
} finally {
await vscode.workspace.getConfiguration('files', uri).update('encoding', 'utf8', vscode.ConfigurationTarget.Global);
}
// with encoding provided
assert.ok(equalsUint8Array(await vscode.workspace.encode('Hello World', uri, { encoding: 'utf8' }), new Uint8Array([72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100])));
assert.ok(equalsUint8Array(await vscode.workspace.encode('Hello World', uri, { encoding: 'utf8bom' }), new Uint8Array([0xEF, 0xBB, 0xBF, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100])));
assert.ok(equalsUint8Array(await vscode.workspace.encode('Hello World', uri, { encoding: 'utf16le' }), new Uint8Array([0xFF, 0xFE, 72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 100, 0])));
assert.ok(equalsUint8Array(await vscode.workspace.encode('Hello World', uri, { encoding: 'utf16be' }), new Uint8Array([0xFE, 0xFF, 0, 72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 100])));
assert.ok(equalsUint8Array(await vscode.workspace.encode('Hellö Wörld', uri, { encoding: 'cp1252' }), new Uint8Array([72, 101, 108, 108, 0xF6, 32, 87, 0xF6, 114, 108, 100])));
assert.ok(equalsUint8Array(await vscode.workspace.encode('Hello World', uri, { encoding: 'foobar123' }), new Uint8Array([72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100])));
assert.ok(equalsUint8Array(await vscode.workspace.encode('Hello World', { encoding: 'utf8' }), new Uint8Array([72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100])));
assert.ok(equalsUint8Array(await vscode.workspace.encode('Hello World', { encoding: 'utf8bom' }), new Uint8Array([0xEF, 0xBB, 0xBF, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100])));
assert.ok(equalsUint8Array(await vscode.workspace.encode('Hello World', { encoding: 'utf16le' }), new Uint8Array([0xFF, 0xFE, 72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 100, 0])));
assert.ok(equalsUint8Array(await vscode.workspace.encode('Hello World', { encoding: 'utf16be' }), new Uint8Array([0xFE, 0xFF, 0, 72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 100])));
assert.ok(equalsUint8Array(await vscode.workspace.encode('Hellö Wörld', { encoding: 'cp1252' }), new Uint8Array([72, 101, 108, 108, 0xF6, 32, 87, 0xF6, 114, 108, 100])));
assert.ok(equalsUint8Array(await vscode.workspace.encode('Hello World', { encoding: 'foobar123' }), new Uint8Array([72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100])));
});
function equalsUint8Array(a: Uint8Array, b: Uint8Array): boolean {
@@ -1497,7 +1497,7 @@ suite('vscode API - workspace', () => {
const text = doc.getText();
assert.strictEqual(text, originalText);
const buf = await vscode.workspace.encode(text, uri, { encoding: 'windows1252' });
const buf = await vscode.workspace.encode(text, { encoding: 'windows1252' });
await vscode.workspace.fs.writeFile(uri, buf);
doc = await vscode.workspace.openTextDocument(uri, { encoding: 'windows1252' });
@@ -1516,10 +1516,10 @@ suite('vscode API - workspace', () => {
doc = await vscode.workspace.openTextDocument(uri, { encoding: 'utf8bom' });
assert.strictEqual(doc.encoding, 'utf8bom');
const decoded = await vscode.workspace.decode(new Uint8Array(buffer), uri, { encoding: 'utf8bom' });
const decoded = await vscode.workspace.decode(new Uint8Array(buffer), { encoding: 'utf8bom' });
assert.strictEqual(decoded, 'Hello World');
const encoded = await vscode.workspace.encode('Hello World', uri, { encoding: 'utf8bom' });
const encoded = await vscode.workspace.encode('Hello World', { encoding: 'utf8bom' });
assert.ok(equalsUint8Array(encoded, new Uint8Array(buffer)));
});
});