diff --git a/build/gulpfile.vscode.js b/build/gulpfile.vscode.js index 326fac6bc24..5ed1e395d49 100644 --- a/build/gulpfile.vscode.js +++ b/build/gulpfile.vscode.js @@ -32,6 +32,7 @@ const i18n = require('./lib/i18n'); const deps = require('./dependencies'); const getElectronVersion = require('./lib/electron').getElectronVersion; const createAsar = require('./lib/asar').createAsar; +const minimist = require('minimist'); const productionDependencies = deps.getProductionDependencies(path.dirname(__dirname)); // @ts-ignore @@ -478,7 +479,7 @@ gulp.task('vscode-translations-push', ['optimize-vscode'], function () { ).pipe(i18n.pushXlfFiles(apiHostname, apiName, apiToken)); }); -gulp.task('vscode-translations-push-test', ['optimize-vscode'], function () { +gulp.task('vscode-translations-export', ['optimize-vscode'], function () { const pathToMetadata = './out-vscode/nls.metadata.json'; const pathToExtensions = './extensions/*'; const pathToSetup = 'build/win32/**/{Default.isl,messages.en.isl}'; @@ -487,20 +488,26 @@ gulp.task('vscode-translations-push-test', ['optimize-vscode'], function () { gulp.src(pathToMetadata).pipe(i18n.createXlfFilesForCoreBundle()), gulp.src(pathToSetup).pipe(i18n.createXlfFilesForIsl()), gulp.src(pathToExtensions).pipe(i18n.createXlfFilesForExtensions()) - ).pipe(i18n.findObsoleteResources(apiHostname, apiName, apiToken) - ).pipe(vfs.dest('../vscode-transifex-input')); + ).pipe(vfs.dest('../vscode-translations-export')); }); gulp.task('vscode-translations-pull', function () { return es.merge([...i18n.defaultLanguages, ...i18n.extraLanguages].map(language => { let includeDefault = !!innoSetupConfig[language.id].defaultInfo; - return i18n.pullSetupXlfFiles(apiHostname, apiName, apiToken, language, includeDefault).pipe(vfs.dest(`../vscode-localization/${language.id}/setup`)); + return i18n.pullSetupXlfFiles(apiHostname, apiName, apiToken, language, includeDefault).pipe(vfs.dest(`../vscode-translations-import/${language.id}/setup`)); })); }); gulp.task('vscode-translations-import', function () { + var options = minimist(process.argv.slice(2), { + string: 'location', + default: { + location: '../vscode-translations-import' + } + }); return es.merge([...i18n.defaultLanguages, ...i18n.extraLanguages].map(language => { - return gulp.src(`../vscode-localization/${language.id}/setup/*/*.xlf`) + let id = language.transifexId || language.id; + return gulp.src(`${options.location}/${id}/setup/*/*.xlf`) .pipe(i18n.prepareIslFiles(language, innoSetupConfig[language.id])) .pipe(vfs.dest(`./build/win32/i18n`)); })); diff --git a/build/lib/i18n.js b/build/lib/i18n.js index e026eaacb3e..1fc6b060e9c 100644 --- a/build/lib/i18n.js +++ b/build/lib/i18n.js @@ -37,7 +37,7 @@ exports.extraLanguages = [ { id: 'tr', folderName: 'trk' } ]; // non built-in extensions also that are transifex and need to be part of the language packs -const externalExtensionsWithTranslations = { +exports.externalExtensionsWithTranslations = { 'vscode-chrome-debug': 'msjsdiag.debugger-for-chrome', 'vscode-node-debug': 'ms-vscode.node-debug', 'vscode-node-debug2': 'ms-vscode.node-debug2' @@ -229,12 +229,15 @@ XLF.parse = function (xlfString) { if (!unit.target) { return; // No translation available } - const val = unit.target.toString(); + let val = unit.target[0]; + if (typeof val !== 'string') { + val = val._; + } if (key && val) { messages[key] = decodeEntities(val); } else { - reject(new Error(`XLF parsing error: XLIFF file does not contain full localization data. ID or target translation for one of the trans-unit nodes is not present.`)); + reject(new Error(`XLF parsing error: XLIFF file ${originalFilePath} does not contain full localization data. ID or target translation for one of the trans-unit nodes is not present.`)); } }); files.push({ messages: messages, originalFilePath: originalFilePath, language: language.toLowerCase() }); @@ -743,7 +746,7 @@ function getAllResources(project, apiHostname, username, password) { } function findObsoleteResources(apiHostname, username, password) { let resourcesByProject = Object.create(null); - resourcesByProject[extensionsProject] = [].concat(externalExtensionsWithTranslations); // clone + resourcesByProject[extensionsProject] = [].concat(exports.externalExtensionsWithTranslations); // clone return event_stream_1.through(function (file) { const project = path.dirname(file.relative); const fileName = path.basename(file.path); @@ -1019,17 +1022,18 @@ function createI18nFile(originalFilePath, messages) { } const i18nPackVersion = "1.0.0"; function pullI18nPackFiles(apiHostname, username, password, language, resultingTranslationPaths) { - return pullCoreAndExtensionsXlfFiles(apiHostname, username, password, language, externalExtensionsWithTranslations) - .pipe(prepareI18nPackFiles(externalExtensionsWithTranslations, resultingTranslationPaths, language.id === 'ps')); + return pullCoreAndExtensionsXlfFiles(apiHostname, username, password, language, exports.externalExtensionsWithTranslations) + .pipe(prepareI18nPackFiles(exports.externalExtensionsWithTranslations, resultingTranslationPaths, language.id === 'ps')); } exports.pullI18nPackFiles = pullI18nPackFiles; function prepareI18nPackFiles(externalExtensions, resultingTranslationPaths, pseudo = false) { let parsePromises = []; let mainPack = { version: i18nPackVersion, contents: {} }; let extensionsPacks = {}; + let errors = []; return event_stream_1.through(function (xlf) { - let project = path.dirname(xlf.path); - let resource = path.basename(xlf.path, '.xlf'); + let project = path.dirname(xlf.relative); + let resource = path.basename(xlf.relative, '.xlf'); let contents = xlf.contents.toString(); let parsePromise = pseudo ? XLF.parsePseudo(contents) : XLF.parse(contents); parsePromises.push(parsePromise); @@ -1055,10 +1059,15 @@ function prepareI18nPackFiles(externalExtensions, resultingTranslationPaths, pse mainPack.contents[path.substr(firstSlash + 1)] = file.messages; } }); + }).catch(reason => { + errors.push(reason); }); }, function () { Promise.all(parsePromises) .then(() => { + if (errors.length > 0) { + throw errors; + } const translatedMainFile = createI18nFile('./main', mainPack); resultingTranslationPaths.push({ id: 'vscode', resourceName: 'main.i18n.json' }); this.queue(translatedMainFile); @@ -1075,7 +1084,9 @@ function prepareI18nPackFiles(externalExtensions, resultingTranslationPaths, pse } this.queue(null); }) - .catch(reason => { throw new Error(reason); }); + .catch((reason) => { + this.emit('error', reason); + }); }); } exports.prepareI18nPackFiles = prepareI18nPackFiles; @@ -1093,11 +1104,15 @@ function prepareIslFiles(language, innoSetupConfig) { let translatedFile = createIslFile(file.originalFilePath, file.messages, language, innoSetupConfig); stream.queue(translatedFile); }); + }).catch(reason => { + this.emit('error', reason); }); }, function () { Promise.all(parsePromises) .then(() => { this.queue(null); }) - .catch(reason => { throw new Error(reason); }); + .catch(reason => { + this.emit('error', reason); + }); }); } exports.prepareIslFiles = prepareIslFiles; diff --git a/build/lib/i18n.ts b/build/lib/i18n.ts index 26e512d95b6..81ba0c0939e 100644 --- a/build/lib/i18n.ts +++ b/build/lib/i18n.ts @@ -57,7 +57,7 @@ export const extraLanguages: Language[] = [ ]; // non built-in extensions also that are transifex and need to be part of the language packs -const externalExtensionsWithTranslations = { +export const externalExtensionsWithTranslations = { 'vscode-chrome-debug': 'msjsdiag.debugger-for-chrome', 'vscode-node-debug': 'ms-vscode.node-debug', 'vscode-node-debug2': 'ms-vscode.node-debug2' @@ -325,11 +325,14 @@ export class XLF { return; // No translation available } - const val = unit.target.toString(); + let val = unit.target[0]; + if (typeof val !== 'string') { + val = val._; + } if (key && val) { messages[key] = decodeEntities(val); } else { - reject(new Error(`XLF parsing error: XLIFF file does not contain full localization data. ID or target translation for one of the trans-unit nodes is not present.`)); + reject(new Error(`XLF parsing error: XLIFF file ${originalFilePath} does not contain full localization data. ID or target translation for one of the trans-unit nodes is not present.`)); } }); files.push({ messages: messages, originalFilePath: originalFilePath, language: language.toLowerCase() }); @@ -1181,9 +1184,10 @@ export function prepareI18nPackFiles(externalExtensions: Map, resultingT let parsePromises: Promise[] = []; let mainPack: I18nPack = { version: i18nPackVersion, contents: {} }; let extensionsPacks: Map = {}; + let errors: any[] = []; return through(function (this: ThroughStream, xlf: File) { - let project = path.dirname(xlf.path); - let resource = path.basename(xlf.path, '.xlf'); + let project = path.dirname(xlf.relative); + let resource = path.basename(xlf.relative, '.xlf'); let contents = xlf.contents.toString(); let parsePromise = pseudo ? XLF.parsePseudo(contents) : XLF.parse(contents); parsePromises.push(parsePromise); @@ -1210,10 +1214,15 @@ export function prepareI18nPackFiles(externalExtensions: Map, resultingT } }); } - ); + ).catch(reason => { + errors.push(reason); + }); }, function () { Promise.all(parsePromises) .then(() => { + if (errors.length > 0) { + throw errors; + } const translatedMainFile = createI18nFile('./main', mainPack); resultingTranslationPaths.push({ id: 'vscode', resourceName: 'main.i18n.json' }); @@ -1232,7 +1241,9 @@ export function prepareI18nPackFiles(externalExtensions: Map, resultingT } this.queue(null); }) - .catch(reason => { throw new Error(reason); }); + .catch((reason) => { + this.emit('error', reason); + }); }); } @@ -1253,11 +1264,15 @@ export function prepareIslFiles(language: Language, innoSetupConfig: InnoSetup): stream.queue(translatedFile); }); } - ); + ).catch(reason => { + this.emit('error', reason); + }); }, function () { Promise.all(parsePromises) .then(() => { this.queue(null); }) - .catch(reason => { throw new Error(reason); }); + .catch(reason => { + this.emit('error', reason); + }); }); } diff --git a/build/npm/update-localization-extension.js b/build/npm/update-localization-extension.js index eebf63ed330..50dcaca3eb6 100644 --- a/build/npm/update-localization-extension.js +++ b/build/npm/update-localization-extension.js @@ -9,16 +9,31 @@ let i18n = require("../lib/i18n"); let fs = require("fs"); let path = require("path"); + +let gulp = require('gulp'); let vfs = require("vinyl-fs"); let rimraf = require('rimraf'); +let minimist = require('minimist'); -function update(idOrPath) { +function update(options) { + let idOrPath = options._; if (!idOrPath) { throw new Error('Argument must be the location of the localization extension.'); } + let transifex = options.transifex; + let location = options.location; + if (transifex === true && location !== undefined) { + throw new Error('Either --transifex or --location can be specified, but not both.'); + } + if (!transifex && !location) { + transifex = true; + } + if (location !== undefined && !fs.existsSync(location)) { + throw new Error(`${location} doesn't exist.`); + } let locExtFolder = idOrPath; if (/^\w{2}(-\w+)?$/.test(idOrPath)) { - locExtFolder = '../vscode-language-pack-' + idOrPath; + locExtFolder = path.join('..', 'vscode-loc', 'i18n', `vscode-language-pack-${idOrPath}`); } let locExtStat = fs.statSync(locExtFolder); if (!locExtStat || !locExtStat.isDirectory) { @@ -54,21 +69,64 @@ function update(idOrPath) { rimraf.sync(translationDataFolder); } - console.log('Downloading translations for \'' + languageId + '\' to \'' + translationDataFolder + '\'...'); - const translationPaths = []; - i18n.pullI18nPackFiles(server, userName, apiToken, { id: languageId }, translationPaths) - .pipe(vfs.dest(translationDataFolder)).on('end', function () { - localization.translations = []; - for (let tp of translationPaths) { - localization.translations.push({ id: tp.id, path: `./translations/${tp.resourceName}`}); - } - fs.writeFileSync(path.join(locExtFolder, 'package.json'), JSON.stringify(packageJSON, null, '\t')); - }); - + if (transifex) { + console.log(`Downloading translations for ${languageId} to '${translationDataFolder}' ...`); + let translationPaths = []; + i18n.pullI18nPackFiles(server, userName, apiToken, { id: languageId }, translationPaths) + .on('error', (error) => { + console.log(`Error occured while importing translations:`); + translationPaths = undefined; + if (Array.isArray(error)) { + error.forEach(console.log); + } else if (error) { + console.log(error); + } else { + console.log('Unknown error'); + } + }) + .pipe(vfs.dest(translationDataFolder)) + .on('end', function () { + if (translationPaths !== undefined) { + localization.translations = []; + for (let tp of translationPaths) { + localization.translations.push({ id: tp.id, path: `./translations/${tp.resourceName}`}); + } + fs.writeFileSync(path.join(locExtFolder, 'package.json'), JSON.stringify(packageJSON, null, '\t')); + } + }); + } else { + console.log(`Importing translations for ${languageId} form '${location}' to '${translationDataFolder}' ...`); + let translationPaths = []; + gulp.src(path.join(location, languageId, '**', '*.xlf')) + .pipe(i18n.prepareI18nPackFiles(i18n.externalExtensionsWithTranslations, translationPaths, languageId === 'ps')) + .on('error', (error) => { + console.log(`Error occured while importing translations:`); + translationPaths = undefined; + if (Array.isArray(error)) { + error.forEach(console.log); + } else if (error) { + console.log(error); + } else { + console.log('Unknown error'); + } + }) + .pipe(vfs.dest(translationDataFolder)) + .on('end', function () { + if (translationPaths !== undefined) { + localization.translations = []; + for (let tp of translationPaths) { + localization.translations.push({ id: tp.id, path: `./translations/${tp.resourceName}`}); + } + fs.writeFileSync(path.join(locExtFolder, 'package.json'), JSON.stringify(packageJSON, null, '\t')); + } + }); + } }); - - } if (path.basename(process.argv[1]) === 'update-localization-extension.js') { - update(process.argv[2]); + var options = minimist(process.argv.slice(2), { + boolean: 'transifex', + string: 'location' + }); + update(options); }