mirror of
https://github.com/Microsoft/vscode
synced 2024-09-19 18:48:00 +00:00
Implemented transifex push and pull for translations together with json->xlf->json parsing.
This commit is contained in:
Michel Kaporin
parent
bbd5a558ec
commit
bb1e7117f1
|
|
@ -29,6 +29,7 @@ const packageJson = require('../package.json');
|
|||
const product = require('../product.json');
|
||||
const shrinkwrap = require('../npm-shrinkwrap.json');
|
||||
const crypto = require('crypto');
|
||||
const i18n = require('./lib/i18n');
|
||||
|
||||
const dependencies = Object.keys(shrinkwrap.dependencies)
|
||||
.concat(Array.isArray(product.extraNodeModules) ? product.extraNodeModules : []); // additional dependencies from our product configuration
|
||||
|
|
@ -339,6 +340,38 @@ gulp.task('vscode-linux-ia32-min', ['minify-vscode', 'clean-vscode-linux-ia32'],
|
|||
gulp.task('vscode-linux-x64-min', ['minify-vscode', 'clean-vscode-linux-x64'], packageTask('linux', 'x64', { minified: true }));
|
||||
gulp.task('vscode-linux-arm-min', ['minify-vscode', 'clean-vscode-linux-arm'], packageTask('linux', 'arm', { minified: true }));
|
||||
|
||||
const apiUrl = process.env.TRANSIFEX_API_URL;
|
||||
const apiName = process.env.TRANSIFEX_API_NAME;
|
||||
const apiToken = process.env.TRANSIFEX_API_TOKEN;
|
||||
|
||||
gulp.task('vscode-translations-update', function() {
|
||||
const pathToMetadata = './out-vscode/nls.metadata.json';
|
||||
const pathToExtensions = './extensions/**/*.nls.json';
|
||||
const pathToSetup = 'build/win32/**/{Default.isl,messages.en.isl}';
|
||||
|
||||
gulp.src(pathToMetadata)
|
||||
.pipe(i18n.prepareXlfFiles())
|
||||
.pipe(i18n.pushXlfFiles(apiUrl, apiName, apiToken));
|
||||
|
||||
gulp.src(pathToSetup)
|
||||
.pipe(i18n.prepareXlfFiles())
|
||||
.pipe(i18n.pushXlfFiles(apiUrl, apiName, apiToken));
|
||||
|
||||
return gulp.src(pathToExtensions)
|
||||
.pipe(i18n.prepareXlfFiles('vscode-extensions'))
|
||||
.pipe(i18n.pushXlfFiles(apiUrl, apiName, apiToken));
|
||||
});
|
||||
|
||||
gulp.task('vscode-translations-pull', function() {
|
||||
i18n.pullXlfFiles('vscode-editor-workbench', apiUrl, apiName, apiToken)
|
||||
.pipe(i18n.prepareJsonFiles())
|
||||
.pipe(vfs.dest('./i18n'));
|
||||
|
||||
return i18n.pullXlfFiles('vscode-extensions', apiUrl, apiName, apiToken)
|
||||
.pipe(i18n.prepareJsonFiles())
|
||||
.pipe(vfs.dest('./i18n'));
|
||||
});
|
||||
|
||||
// Sourcemaps
|
||||
|
||||
gulp.task('upload-vscode-sourcemaps', ['minify-vscode'], () => {
|
||||
|
|
|
|||
|
|
@ -10,6 +10,13 @@ var event_stream_1 = require("event-stream");
|
|||
var File = require("vinyl");
|
||||
var Is = require("is");
|
||||
var util = require('gulp-util');
|
||||
var xml2js = require("xml2js");
|
||||
var glob = require("glob");
|
||||
var util = require('gulp-util');
|
||||
var request = require('request');
|
||||
var es = require('event-stream');
|
||||
var iconv = require('iconv-lite');
|
||||
|
||||
function log(message) {
|
||||
var rest = [];
|
||||
for (var _i = 1; _i < arguments.length; _i++) {
|
||||
|
|
@ -37,6 +44,174 @@ var BundledFormat;
|
|||
}
|
||||
BundledFormat.is = is;
|
||||
})(BundledFormat || (BundledFormat = {}));
|
||||
|
||||
var PackageJsonFormat;
|
||||
(function (PackageJsonFormat) {
|
||||
function is(value) {
|
||||
if (Is.undef(value) || !Is.object(value)) {
|
||||
return false;
|
||||
}
|
||||
return Object.keys(value).every(function (key) {
|
||||
var element = value[key];
|
||||
return Is.string(element) || (Is.object(element) && Is.defined(element.message) && Is.defined(element.comment));
|
||||
});
|
||||
}
|
||||
PackageJsonFormat.is = is;
|
||||
})(PackageJsonFormat || (PackageJsonFormat = {}));
|
||||
var ModuleJsonFormat;
|
||||
(function (ModuleJsonFormat) {
|
||||
function is(value) {
|
||||
var candidate = value;
|
||||
return Is.defined(candidate)
|
||||
&& Is.array(candidate.messages) && candidate.messages.every(function (message) { return Is.string(message); })
|
||||
&& Is.array(candidate.keys) && candidate.keys.every(function (key) { return Is.string(key) || LocalizeInfo.is(key); });
|
||||
}
|
||||
ModuleJsonFormat.is = is;
|
||||
})(ModuleJsonFormat || (ModuleJsonFormat = {}));
|
||||
var Line = (function () {
|
||||
function Line(indent) {
|
||||
if (indent === void 0) { indent = 0; }
|
||||
this.indent = indent;
|
||||
this.buffer = [];
|
||||
if (indent > 0) {
|
||||
this.buffer.push(new Array(indent + 1).join(' '));
|
||||
}
|
||||
}
|
||||
Line.prototype.append = function (value) {
|
||||
this.buffer.push(value);
|
||||
return this;
|
||||
};
|
||||
Line.prototype.toString = function () {
|
||||
return this.buffer.join('');
|
||||
};
|
||||
return Line;
|
||||
}());
|
||||
exports.Line = Line;
|
||||
var TextModel = (function () {
|
||||
function TextModel(contents) {
|
||||
this._lines = contents.split(/\r\n|\r|\n/);
|
||||
}
|
||||
Object.defineProperty(TextModel.prototype, "lines", {
|
||||
get: function () {
|
||||
return this._lines;
|
||||
},
|
||||
enumerable: true,
|
||||
configurable: true
|
||||
});
|
||||
return TextModel;
|
||||
}());
|
||||
var XLF = (function () {
|
||||
function XLF(project) {
|
||||
this.project = project;
|
||||
this.buffer = [];
|
||||
this.files = Object.create(null);
|
||||
}
|
||||
XLF.prototype.toString = function () {
|
||||
this.appendHeader();
|
||||
for (var file in this.files) {
|
||||
this.appendNewLine("<file original=\"" + file + "\" source-language=\"en\" datatype=\"plaintext\"><body>", 2);
|
||||
for (var _i = 0, _a = this.files[file]; _i < _a.length; _i++) {
|
||||
var item = _a[_i];
|
||||
this.addStringItem(item);
|
||||
}
|
||||
this.appendNewLine('</body></file>', 2);
|
||||
}
|
||||
this.appendFooter();
|
||||
return this.buffer.join('\r\n');
|
||||
};
|
||||
XLF.prototype.addFile = function (original, keys, messages) {
|
||||
this.files[original] = [];
|
||||
var existingKeys = [];
|
||||
for (var _i = 0, keys_1 = keys; _i < keys_1.length; _i++) {
|
||||
var key = keys_1[_i];
|
||||
// Ignore duplicate keys because Transifex does not populate those with translated values.
|
||||
if (existingKeys.indexOf(key) !== -1) {
|
||||
continue;
|
||||
}
|
||||
existingKeys.push(key);
|
||||
var message = encodeEntities(messages[keys.indexOf(key)]);
|
||||
var comment = undefined;
|
||||
// Check if the message contains description (if so, it becomes an object type in JSON)
|
||||
if (Is.string(key)) {
|
||||
this.files[original].push({ id: key, message: message, comment: comment });
|
||||
}
|
||||
else {
|
||||
if (key['comment'] && key['comment'].length > 0) {
|
||||
comment = key['comment'].map(function (comment) { return encodeEntities(comment); }).join('\r\n');
|
||||
}
|
||||
this.files[original].push({ id: key['key'], message: message, comment: comment });
|
||||
}
|
||||
}
|
||||
};
|
||||
XLF.prototype.addStringItem = function (item) {
|
||||
if (!item.id || !item.message) {
|
||||
//throw new Error('No item ID or value specified.');
|
||||
}
|
||||
this.appendNewLine("<trans-unit id=\"" + item.id + "\">", 4);
|
||||
this.appendNewLine("<source xml:lang=\"en\">" + encodeEntities(item.message) + "</source>", 6);
|
||||
if (item.comment) {
|
||||
this.appendNewLine("<note>" + item.comment + "</note>", 6);
|
||||
}
|
||||
this.appendNewLine('</trans-unit>', 4);
|
||||
};
|
||||
XLF.prototype.appendHeader = function () {
|
||||
this.appendNewLine('<?xml version="1.0" encoding="utf-8"?>', 0);
|
||||
this.appendNewLine('<xliff version="1.2" xmlns="urn:oasis:names:tc:xliff:document:1.2">', 0);
|
||||
};
|
||||
XLF.prototype.appendFooter = function () {
|
||||
this.appendNewLine('</xliff>', 0);
|
||||
};
|
||||
XLF.prototype.appendNewLine = function (content, indent) {
|
||||
var line = new Line(indent);
|
||||
line.append(content);
|
||||
this.buffer.push(line.toString());
|
||||
};
|
||||
return XLF;
|
||||
}());
|
||||
XLF.parse = function (xlfString) {
|
||||
return new Promise(function (resolve, reject) {
|
||||
var parser = new xml2js.Parser();
|
||||
var files = [];
|
||||
parser.parseString(xlfString, function (err, result) {
|
||||
if (err) {
|
||||
reject("Failed to parse XLIFF string. " + err);
|
||||
}
|
||||
var fileNodes = result['xliff']['file'];
|
||||
if (!fileNodes) {
|
||||
reject('XLIFF file does not contain "xliff" or "file" node(s) required for parsing.');
|
||||
}
|
||||
fileNodes.forEach(function (file) {
|
||||
var originalFilePath = file.$.original;
|
||||
if (!originalFilePath) {
|
||||
reject('XLIFF file node does not contain original attribute to determine the original location of the resource file.');
|
||||
}
|
||||
var language = file.$['target-language'].toLowerCase();
|
||||
if (!language) {
|
||||
reject('XLIFF file node does not contain target-language attribute to determine translated language.');
|
||||
}
|
||||
var messages = {};
|
||||
var transUnits = file.body[0]['trans-unit'];
|
||||
transUnits.forEach(function (unit) {
|
||||
var key = unit.$.id;
|
||||
if (!unit.target) {
|
||||
return; // No translation available
|
||||
}
|
||||
var val = unit.target.toString();
|
||||
if (key && val) {
|
||||
messages[key] = decodeEntities(val);
|
||||
}
|
||||
else {
|
||||
reject('XLIFF file does not contain full localization data. ID or target translation for one of the trans-unit nodes is not present.');
|
||||
}
|
||||
});
|
||||
files.push({ messages: messages, originalFilePath: originalFilePath, language: language });
|
||||
});
|
||||
resolve(files);
|
||||
});
|
||||
});
|
||||
};
|
||||
exports.XLF = XLF;
|
||||
|
||||
var vscodeLanguages = [
|
||||
'chs',
|
||||
'cht',
|
||||
|
|
@ -68,6 +243,28 @@ var iso639_3_to_2 = {
|
|||
'sve': 'sv-se',
|
||||
'trk': 'tr'
|
||||
};
|
||||
|
||||
var iso639_2_to_3 = {
|
||||
'zh-cn': 'chs',
|
||||
'zh-tw': 'cht',
|
||||
'cs-cz': 'csy',
|
||||
'de': 'deu',
|
||||
'en': 'enu',
|
||||
'es': 'esn',
|
||||
'fr': 'fra',
|
||||
'hu': 'hun',
|
||||
'it': 'ita',
|
||||
'ja': 'jpn',
|
||||
'ko': 'kor',
|
||||
'nl': 'nld',
|
||||
'pl': 'plk',
|
||||
'pt-br': 'ptb',
|
||||
'pt': 'ptg',
|
||||
'ru': 'rus',
|
||||
'sv-se': 'sve',
|
||||
'tr': 'trk'
|
||||
};
|
||||
|
||||
function sortLanguages(directoryNames) {
|
||||
return directoryNames.map(function (dirName) {
|
||||
var lower = dirName.toLowerCase();
|
||||
|
|
@ -288,3 +485,472 @@ function processNlsFiles(opts) {
|
|||
});
|
||||
}
|
||||
exports.processNlsFiles = processNlsFiles;
|
||||
|
||||
function prepareXlfFiles(projectName, extensionName) {
|
||||
return event_stream_1.through(function (file) {
|
||||
if (!file.isBuffer()) {
|
||||
log('Error', "Failed to read component file: " + file.relative);
|
||||
}
|
||||
var extension = path.extname(file.path);
|
||||
if (extension === '.json') {
|
||||
var json = JSON.parse(file.contents.toString('utf8'));
|
||||
if (BundledFormat.is(json)) {
|
||||
importBundleJson(file, json, this);
|
||||
}
|
||||
else if (PackageJsonFormat.is(json) || ModuleJsonFormat.is(json)) {
|
||||
importModuleOrPackageJson(file, json, projectName, this, extensionName);
|
||||
}
|
||||
else {
|
||||
log('Error', 'JSON format cannot be deduced.');
|
||||
}
|
||||
}
|
||||
else if (extension === '.isl') {
|
||||
importIsl(file, this);
|
||||
}
|
||||
});
|
||||
}
|
||||
exports.prepareXlfFiles = prepareXlfFiles;
|
||||
function getResource(sourceFile) {
|
||||
var editorProject = 'vscode-editor', workbenchProject = 'vscode-workbench';
|
||||
var resource;
|
||||
if (sourceFile.startsWith('vs/platform')) {
|
||||
return { name: 'vs/platform', project: editorProject };
|
||||
}
|
||||
else if (sourceFile.startsWith('vs/editor/contrib')) {
|
||||
return { name: 'vs/editor/contrib', project: editorProject };
|
||||
}
|
||||
else if (sourceFile.startsWith('vs/editor')) {
|
||||
return { name: 'vs/editor', project: editorProject };
|
||||
}
|
||||
else if (sourceFile.startsWith('vs/base')) {
|
||||
return { name: 'vs/base', project: editorProject };
|
||||
}
|
||||
else if (sourceFile.startsWith('vs/code')) {
|
||||
return { name: 'vs/code', project: workbenchProject };
|
||||
}
|
||||
else if (sourceFile.startsWith('vs/workbench/parts')) {
|
||||
resource = sourceFile.split('/', 4).join('/');
|
||||
return { name: resource, project: workbenchProject };
|
||||
}
|
||||
else if (sourceFile.startsWith('vs/workbench/services')) {
|
||||
resource = sourceFile.split('/', 4).join('/');
|
||||
return { name: resource, project: workbenchProject };
|
||||
}
|
||||
else if (sourceFile.startsWith('vs/workbench')) {
|
||||
return { name: 'vs/workbench', project: workbenchProject };
|
||||
}
|
||||
throw new Error("Could not identify the XLF bundle for " + sourceFile);
|
||||
}
|
||||
function importBundleJson(file, json, stream) {
|
||||
var transifexEditorXlfs = Object.create(null);
|
||||
for (var source in json.keys) {
|
||||
var projectResource = getResource(source);
|
||||
var resource = projectResource.name;
|
||||
var project = projectResource.project;
|
||||
var keys = json.keys[source];
|
||||
var messages = json.messages[source];
|
||||
if (keys.length !== messages.length) {
|
||||
log('Error:', "There is a mismatch between keys and messages in " + file.relative);
|
||||
}
|
||||
var xlf = transifexEditorXlfs[resource] ? transifexEditorXlfs[resource] : transifexEditorXlfs[resource] = new XLF(project);
|
||||
xlf.addFile(source, keys, messages);
|
||||
}
|
||||
for (var resource in transifexEditorXlfs) {
|
||||
var newFilePath = transifexEditorXlfs[resource].project + "/" + resource.replace(/\//g, '_') + ".xlf";
|
||||
var xlfFile = new File({ path: newFilePath, contents: new Buffer(transifexEditorXlfs[resource].toString(), 'utf-8') });
|
||||
stream.emit('data', xlfFile);
|
||||
}
|
||||
}
|
||||
// Keeps existing XLF instances and a state of how many files were already processed for faster file emission
|
||||
var extensions = Object.create(null);
|
||||
function importModuleOrPackageJson(file, json, projectName, stream, extensionName) {
|
||||
if (ModuleJsonFormat.is(json) && json['keys'].length !== json['messages'].length) {
|
||||
log('Error:', "There is a mismatch between keys and messages in " + file.relative);
|
||||
}
|
||||
// Prepare the source path for <original/> attribute in XLF & extract messages from JSON
|
||||
var formattedSourcePath = file.relative.replace(/\\/g, '/');
|
||||
var messages = Object.keys(json).map(function (key) { return json[key].toString(); });
|
||||
// Stores the amount of localization files to be transformed to XLF before the emission
|
||||
var localizationFilesCount, originalFilePath;
|
||||
// If preparing XLF for external extension, then use different glob pattern and source path
|
||||
if (extensionName) {
|
||||
localizationFilesCount = glob.sync('**/*.nls.json').length;
|
||||
originalFilePath = "" + formattedSourcePath.substr(0, formattedSourcePath.length - '.nls.json'.length);
|
||||
}
|
||||
else {
|
||||
// Used for vscode/extensions folder
|
||||
extensionName = formattedSourcePath.split('/')[0];
|
||||
localizationFilesCount = glob.sync("./extensions/" + extensionName + "/**/*.nls.json").length;
|
||||
originalFilePath = "extensions/" + formattedSourcePath.substr(0, formattedSourcePath.length - '.nls.json'.length);
|
||||
}
|
||||
var extension = extensions[extensionName] ?
|
||||
extensions[extensionName] : extensions[extensionName] = { xlf: new XLF(projectName), processed: 0 };
|
||||
if (ModuleJsonFormat.is(json)) {
|
||||
extension.xlf.addFile(originalFilePath, json['keys'], json['messages']);
|
||||
}
|
||||
else {
|
||||
extension.xlf.addFile(originalFilePath, Object.keys(json), messages);
|
||||
}
|
||||
// Check if XLF is populated with file nodes to emit it
|
||||
if (++extensions[extensionName].processed === localizationFilesCount) {
|
||||
var newFilePath = path.join(projectName, extensionName + '.xlf');
|
||||
var xlfFile = new File({ path: newFilePath, contents: new Buffer(extension.xlf.toString(), 'utf-8') });
|
||||
stream.emit('data', xlfFile);
|
||||
}
|
||||
}
|
||||
var islXlf, islProcessed = 0;
|
||||
function importIsl(file, stream) {
|
||||
var islFiles = ['Default.isl', 'messages.en.isl'];
|
||||
var projectName = 'vscode-workbench';
|
||||
var xlf = islXlf ? islXlf : islXlf = new XLF(projectName), keys = [], messages = [];
|
||||
var model = new TextModel(file.contents.toString());
|
||||
var inMessageSection = false;
|
||||
model.lines.forEach(function (line) {
|
||||
if (line.length === 0) {
|
||||
return;
|
||||
}
|
||||
var firstChar = line.charAt(0);
|
||||
switch (firstChar) {
|
||||
case ';':
|
||||
// Comment line;
|
||||
return;
|
||||
case '[':
|
||||
inMessageSection = '[Messages]' === line || '[CustomMessages]' === line;
|
||||
return;
|
||||
}
|
||||
if (!inMessageSection) {
|
||||
return;
|
||||
}
|
||||
var sections = line.split('=');
|
||||
if (sections.length !== 2) {
|
||||
log('Error:', "Badly formatted message found: " + line);
|
||||
}
|
||||
else {
|
||||
var key = sections[0];
|
||||
var value = sections[1];
|
||||
if (key.length > 0 && value.length > 0) {
|
||||
keys.push(key);
|
||||
messages.push(value);
|
||||
}
|
||||
}
|
||||
});
|
||||
var originalPath = file.path.substring(file.cwd.length + 1, file.path.split('.')[0].length).replace(/\\/g, '/');
|
||||
xlf.addFile(originalPath, keys, messages);
|
||||
// Emit only upon all ISL files combined into single XLF instance
|
||||
if (++islProcessed === islFiles.length) {
|
||||
var newFilePath = path.join(projectName, 'setup.xlf');
|
||||
var xlfFile = new File({ path: newFilePath, contents: new Buffer(xlf.toString(), 'utf-8') });
|
||||
stream.emit('data', xlfFile);
|
||||
}
|
||||
}
|
||||
function pushXlfFiles(apiUrl, username, password) {
|
||||
return event_stream_1.through(function (file) {
|
||||
var project = path.dirname(file.relative);
|
||||
var fileName = path.basename(file.path);
|
||||
var slug = fileName.substr(0, fileName.length - '.xlf'.length);
|
||||
var credentials = {
|
||||
'user': username,
|
||||
'password': password
|
||||
};
|
||||
// Check if resource already exists, if not, then create it.
|
||||
tryGetResource(project, slug, apiUrl, credentials).then(function (exists) {
|
||||
if (exists) {
|
||||
updateResource(project, slug, file, apiUrl, credentials);
|
||||
}
|
||||
else {
|
||||
createResource(project, slug, file, apiUrl, credentials);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
exports.pushXlfFiles = pushXlfFiles;
|
||||
function tryGetResource(project, slug, apiUrl, credentials) {
|
||||
return new Promise(function (resolve, reject) {
|
||||
var url = apiUrl + "/project/" + project + "/resource/" + slug + "/?details";
|
||||
request.get(url, { 'auth': credentials }).on('response', function (response) {
|
||||
if (response.statusCode === 404) {
|
||||
resolve(false);
|
||||
}
|
||||
else if (response.statusCode === 200) {
|
||||
resolve(true);
|
||||
}
|
||||
else {
|
||||
reject("Failed to query resource " + slug + ". Response: " + response.statusCode + " " + response.statusMessage);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
function createResource(project, slug, xlfFile, apiUrl, credentials) {
|
||||
var url = apiUrl + "/project/" + project + "/resources";
|
||||
var options = {
|
||||
'body': {
|
||||
'content': xlfFile.contents.toString(),
|
||||
'name': slug,
|
||||
'slug': slug,
|
||||
'i18n_type': 'XLIFF'
|
||||
},
|
||||
'json': true,
|
||||
'auth': credentials
|
||||
};
|
||||
request.post(url, options, function (err, res) {
|
||||
if (err) {
|
||||
log('Error:', "Failed to create Transifex " + project + "/" + slug + ": " + err);
|
||||
}
|
||||
if (res.statusCode === 201) {
|
||||
log("Resource " + project + "/" + slug + " successfully created on Transifex.");
|
||||
}
|
||||
else {
|
||||
log('Error:', "Something went wrong creating " + slug + " in " + project + ". " + res.statusCode);
|
||||
}
|
||||
});
|
||||
}
|
||||
/**
|
||||
* The following link provides information about how Transifex handles updates of a resource file:
|
||||
* https://dev.befoolish.co/tx-docs/public/projects/updating-content#what-happens-when-you-update-files
|
||||
*/
|
||||
function updateResource(project, slug, xlfFile, apiUrl, credentials) {
|
||||
var url = apiUrl + "/project/" + project + "/resource/" + slug + "/content";
|
||||
var options = {
|
||||
'body': { 'content': xlfFile.contents.toString() },
|
||||
'json': true,
|
||||
'auth': credentials
|
||||
};
|
||||
request.put(url, options, function (err, res, body) {
|
||||
if (err) {
|
||||
log('Error:', "Failed to update Transifex " + project + "/" + slug + ": " + err);
|
||||
}
|
||||
if (res.statusCode === 200) {
|
||||
log("Resource " + project + "/" + slug + " successfully updated on Transifex. Strings added: " + body['strings_added'] + ", updated: " + body['strings_updated'] + ", deleted: " + body['strings_delete']);
|
||||
}
|
||||
else {
|
||||
log('Error:', "Something went wrong updating " + slug + " in " + project + ". " + res.statusCode);
|
||||
}
|
||||
});
|
||||
}
|
||||
function getMetadataResources(pathToMetadata) {
|
||||
var metadata = fs.readFileSync(pathToMetadata).toString('utf8');
|
||||
var json = JSON.parse(metadata);
|
||||
var slugs = [];
|
||||
var _loop_1 = function (source) {
|
||||
var projectResource = getResource(source);
|
||||
if (!slugs.find(function (slug) { return slug.name === projectResource.name && slug.project === projectResource.project; })) {
|
||||
slugs.push(projectResource);
|
||||
}
|
||||
};
|
||||
for (var source in json['keys']) {
|
||||
_loop_1(source);
|
||||
}
|
||||
return slugs;
|
||||
}
|
||||
function obtainProjectResources(projectName) {
|
||||
var resources;
|
||||
if (projectName === 'vscode-editor-workbench') {
|
||||
resources = getMetadataResources('./out-vscode/nls.metadata.json');
|
||||
resources.push({ name: 'setup', project: 'vscode-workbench' });
|
||||
}
|
||||
else if (projectName === 'vscode-extensions') {
|
||||
var extensionsToLocalize = glob.sync('./extensions/**/*.nls.json').map(function (extension) { return extension.split('/')[2]; });
|
||||
var resourcesToPull_1 = [];
|
||||
resources = [];
|
||||
extensionsToLocalize.forEach(function (extension) {
|
||||
if (resourcesToPull_1.indexOf(extension) === -1) {
|
||||
resourcesToPull_1.push(extension);
|
||||
resources.push({ name: extension, project: projectName });
|
||||
}
|
||||
});
|
||||
}
|
||||
return resources;
|
||||
}
|
||||
function pullXlfFiles(projectName, apiUrl, username, password, resources) {
|
||||
if (!resources) {
|
||||
resources = obtainProjectResources(projectName);
|
||||
}
|
||||
if (!resources) {
|
||||
throw new Error('Transifex projects and resources must be defined to be able to pull translations from Transifex.');
|
||||
}
|
||||
var credentials = {
|
||||
'auth': {
|
||||
'user': username,
|
||||
'password': password
|
||||
}
|
||||
};
|
||||
var expectedTranslationsCount = vscodeLanguages.length * resources.length;
|
||||
var translationsRetrieved = 0, called = false;
|
||||
return es.readable(function (count, callback) {
|
||||
// Mark end of stream when all resources were retrieved
|
||||
if (translationsRetrieved === expectedTranslationsCount) {
|
||||
return this.emit('end');
|
||||
}
|
||||
if (!called) {
|
||||
called = true;
|
||||
var stream_1 = this;
|
||||
vscodeLanguages.map(function (language) {
|
||||
resources.map(function (resource) {
|
||||
var slug = resource.name.replace(/\//g, '_');
|
||||
var project = resource.project;
|
||||
var iso639 = iso639_3_to_2[language];
|
||||
var url = apiUrl + "/project/" + project + "/resource/" + slug + "/translation/" + iso639 + "?file&mode=onlyreviewed";
|
||||
var xlfBuffer = '', responseCode;
|
||||
request.get(url, credentials)
|
||||
.on('response', function (response) {
|
||||
responseCode = response.statusCode;
|
||||
})
|
||||
.on('data', function (data) { return xlfBuffer += data; })
|
||||
.on('end', function () {
|
||||
if (responseCode === 200) {
|
||||
stream_1.emit('data', new File({ contents: new Buffer(xlfBuffer) }));
|
||||
}
|
||||
else {
|
||||
log('Error:', slug + " in " + project + " returned no data. Response code: " + responseCode + ".");
|
||||
}
|
||||
translationsRetrieved++;
|
||||
})
|
||||
.on('error', function (error) {
|
||||
log('Error:', "Failed to query resource " + slug + " with the following error: " + error);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
callback();
|
||||
});
|
||||
}
|
||||
exports.pullXlfFiles = pullXlfFiles;
|
||||
function prepareJsonFiles() {
|
||||
return event_stream_1.through(function (xlf) {
|
||||
var stream = this;
|
||||
XLF.parse(xlf.contents.toString()).then(function (resolvedFiles) {
|
||||
resolvedFiles.forEach(function (file) {
|
||||
var messages = file.messages, translatedFile;
|
||||
// ISL file path always starts with 'build/'
|
||||
if (file.originalFilePath.startsWith('build/')) {
|
||||
var defaultLanguages = { 'zh-cn': true, 'zh-tw': true, 'ko': true };
|
||||
if (path.basename(file.originalFilePath) === 'Default' && !defaultLanguages[file.language]) {
|
||||
return;
|
||||
}
|
||||
translatedFile = createIslFile('..', file.originalFilePath, messages, iso639_2_to_3[file.language]);
|
||||
}
|
||||
else {
|
||||
translatedFile = createI18nFile(iso639_2_to_3[file.language], file.originalFilePath, messages);
|
||||
}
|
||||
stream.emit('data', translatedFile);
|
||||
});
|
||||
}, function (rejectReason) {
|
||||
log('Error:', rejectReason);
|
||||
});
|
||||
});
|
||||
}
|
||||
exports.prepareJsonFiles = prepareJsonFiles;
|
||||
function createI18nFile(base, originalFilePath, messages) {
|
||||
var content = [
|
||||
'/*---------------------------------------------------------------------------------------------',
|
||||
' * Copyright (c) Microsoft Corporation. All rights reserved.',
|
||||
' * Licensed under the MIT License. See License.txt in the project root for license information.',
|
||||
' *--------------------------------------------------------------------------------------------*/',
|
||||
'// Do not edit this file. It is machine generated.'
|
||||
].join('\n') + '\n' + JSON.stringify(messages, null, '\t').replace(/\r\n/g, '\n');
|
||||
return new File({
|
||||
path: path.join(base, originalFilePath + '.i18n.json'),
|
||||
contents: new Buffer(content, 'utf8')
|
||||
});
|
||||
}
|
||||
exports.createI18nFile = createI18nFile;
|
||||
var languageNames = {
|
||||
'chs': 'Simplified Chinese',
|
||||
'cht': 'Traditional Chinese',
|
||||
'kor': 'Korean'
|
||||
};
|
||||
var languageIds = {
|
||||
'chs': '$0804',
|
||||
'cht': '$0404',
|
||||
'kor': '$0412'
|
||||
};
|
||||
var encodings = {
|
||||
'chs': 'CP936',
|
||||
'cht': 'CP950',
|
||||
'jpn': 'CP932',
|
||||
'kor': 'CP949',
|
||||
'deu': 'CP1252',
|
||||
'fra': 'CP1252',
|
||||
'esn': 'CP1252',
|
||||
'rus': 'CP1251',
|
||||
'ita': 'CP1252'
|
||||
};
|
||||
function createIslFile(base, originalFilePath, messages, language) {
|
||||
var content = [];
|
||||
var originalContent;
|
||||
if (path.basename(originalFilePath) === 'Default') {
|
||||
originalContent = new TextModel(fs.readFileSync(originalFilePath + '.isl', 'utf8'));
|
||||
}
|
||||
else {
|
||||
originalContent = new TextModel(fs.readFileSync(originalFilePath + '.en.isl', 'utf8'));
|
||||
}
|
||||
originalContent.lines.forEach(function (line) {
|
||||
if (line.length > 0) {
|
||||
var firstChar = line.charAt(0);
|
||||
if (firstChar === '[' || firstChar === ';') {
|
||||
if (line === '; *** Inno Setup version 5.5.3+ English messages ***') {
|
||||
content.push("; *** Inno Setup version 5.5.3+ " + languageNames[language] + " messages ***");
|
||||
}
|
||||
else {
|
||||
content.push(line);
|
||||
}
|
||||
}
|
||||
else {
|
||||
var sections = line.split('=');
|
||||
var key = sections[0];
|
||||
var translated = line;
|
||||
if (key) {
|
||||
if (key === 'LanguageName') {
|
||||
translated = key + "=" + languageNames[language];
|
||||
}
|
||||
else if (key === 'LanguageID') {
|
||||
translated = key + "=" + languageIds[language];
|
||||
}
|
||||
else if (key === 'LanguageCodePage') {
|
||||
translated = key + "=" + encodings[language].substr(2);
|
||||
}
|
||||
else {
|
||||
var translatedMessage = messages[key];
|
||||
if (translatedMessage) {
|
||||
translated = key + "=" + translatedMessage;
|
||||
}
|
||||
}
|
||||
}
|
||||
content.push(translated);
|
||||
}
|
||||
}
|
||||
});
|
||||
var tag = iso639_3_to_2[language];
|
||||
var basename = path.basename(originalFilePath);
|
||||
var filePath = path.join(base, path.dirname(originalFilePath), basename) + "." + tag + ".isl";
|
||||
return new File({
|
||||
path: filePath,
|
||||
contents: iconv.encode(new Buffer(content.join('\r\n'), 'utf8'), encodings[language])
|
||||
});
|
||||
}
|
||||
exports.createIslFile = createIslFile;
|
||||
function encodeEntities(value) {
|
||||
var result = [];
|
||||
for (var i = 0; i < value.length; i++) {
|
||||
var ch = value[i];
|
||||
switch (ch) {
|
||||
case '<':
|
||||
result.push('<');
|
||||
break;
|
||||
case '>':
|
||||
result.push('>');
|
||||
break;
|
||||
case '&':
|
||||
result.push('&');
|
||||
break;
|
||||
default:
|
||||
result.push(ch);
|
||||
}
|
||||
}
|
||||
return result.join('');
|
||||
}
|
||||
function decodeEntities(value) {
|
||||
return value.replace(/</g, '<').replace(/>/g, '>').replace(/&/g, '&');
|
||||
}
|
||||
exports.decodeEntities = decodeEntities;
|
||||
//# sourceMappingURL=i18n.js.map
|
||||
|
|
|
|||
|
|
@ -10,8 +10,13 @@ import { through } from 'event-stream';
|
|||
import { ThroughStream } from 'through';
|
||||
import File = require('vinyl');
|
||||
import * as Is from 'is';
|
||||
import xml2js = require('xml2js');
|
||||
import * as glob from 'glob';
|
||||
|
||||
var util = require('gulp-util');
|
||||
const request = require('request');
|
||||
const es = require('event-stream');
|
||||
var iconv = require('iconv-lite');
|
||||
|
||||
function log(message: any, ...rest: any[]): void {
|
||||
util.log(util.colors.green('[i18n]'), message, ...rest);
|
||||
|
|
@ -21,6 +26,17 @@ interface Map<V> {
|
|||
[key: string]: V;
|
||||
}
|
||||
|
||||
interface Item {
|
||||
id: string;
|
||||
message: string;
|
||||
comment: string;
|
||||
}
|
||||
|
||||
interface Resource {
|
||||
name: string;
|
||||
project: string;
|
||||
}
|
||||
|
||||
interface LocalizeInfo {
|
||||
key: string;
|
||||
comment: string[];
|
||||
|
|
@ -52,6 +68,205 @@ module BundledFormat {
|
|||
}
|
||||
}
|
||||
|
||||
interface ValueFormat {
|
||||
message: string;
|
||||
comment: string[];
|
||||
}
|
||||
|
||||
interface PackageJsonFormat {
|
||||
[key: string]: string | ValueFormat;
|
||||
}
|
||||
|
||||
module PackageJsonFormat {
|
||||
export function is(value: any): value is PackageJsonFormat {
|
||||
if (Is.undef(value) || !Is.object(value)) {
|
||||
return false;
|
||||
}
|
||||
return Object.keys(value).every(key => {
|
||||
let element = value[key];
|
||||
return Is.string(element) || (Is.object(element) && Is.defined(element.message) && Is.defined(element.comment));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
interface ModuleJsonFormat {
|
||||
messages: string[];
|
||||
keys: (string | LocalizeInfo)[];
|
||||
}
|
||||
|
||||
module ModuleJsonFormat {
|
||||
export function is(value: any): value is ModuleJsonFormat {
|
||||
let candidate = value as ModuleJsonFormat;
|
||||
return Is.defined(candidate)
|
||||
&& Is.array(candidate.messages) && candidate.messages.every(message => Is.string(message))
|
||||
&& Is.array(candidate.keys) && candidate.keys.every(key => Is.string(key) || LocalizeInfo.is(key));
|
||||
}
|
||||
}
|
||||
|
||||
export class Line {
|
||||
private buffer: string[] = [];
|
||||
|
||||
constructor(private indent: number = 0) {
|
||||
if (indent > 0) {
|
||||
this.buffer.push(new Array(indent + 1).join(' '));
|
||||
}
|
||||
}
|
||||
|
||||
public append(value: string): Line {
|
||||
this.buffer.push(value);
|
||||
return this;
|
||||
}
|
||||
|
||||
public toString(): string {
|
||||
return this.buffer.join('');
|
||||
}
|
||||
}
|
||||
|
||||
class TextModel {
|
||||
private _lines: string[];
|
||||
|
||||
constructor(contents: string) {
|
||||
this._lines = contents.split(/\r\n|\r|\n/);
|
||||
}
|
||||
|
||||
public get lines(): string[] {
|
||||
return this._lines;
|
||||
}
|
||||
}
|
||||
|
||||
export class XLF {
|
||||
private buffer: string[];
|
||||
private files: Map<Item[]>;
|
||||
|
||||
constructor(public project: string) {
|
||||
this.buffer = [];
|
||||
this.files = Object.create(null);
|
||||
}
|
||||
|
||||
public toString(): string {
|
||||
this.appendHeader();
|
||||
|
||||
for (let file in this.files) {
|
||||
this.appendNewLine(`<file original="${file}" source-language="en" datatype="plaintext"><body>`, 2);
|
||||
for (let item of this.files[file]) {
|
||||
this.addStringItem(item);
|
||||
}
|
||||
this.appendNewLine('</body></file>', 2);
|
||||
}
|
||||
|
||||
this.appendFooter();
|
||||
return this.buffer.join('\r\n');
|
||||
}
|
||||
|
||||
public addFile(original: string, keys: any[], messages: string[]) {
|
||||
this.files[original] = [];
|
||||
let existingKeys = [];
|
||||
|
||||
for (let key of keys) {
|
||||
// Ignore duplicate keys because Transifex does not populate those with translated values.
|
||||
if (existingKeys.indexOf(key) !== -1) {
|
||||
continue;
|
||||
}
|
||||
existingKeys.push(key);
|
||||
|
||||
let message: string = encodeEntities(messages[keys.indexOf(key)]);
|
||||
let comment: string = undefined;
|
||||
|
||||
// Check if the message contains description (if so, it becomes an object type in JSON)
|
||||
if (Is.string(key)) {
|
||||
this.files[original].push({ id: key, message: message, comment: comment });
|
||||
} else {
|
||||
if (key['comment'] && key['comment'].length > 0) {
|
||||
comment = key['comment'].map(comment => encodeEntities(comment)).join('\r\n');
|
||||
}
|
||||
|
||||
this.files[original].push({ id: key['key'], message: message, comment: comment });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private addStringItem(item: Item): void {
|
||||
if (!item.id || !item.message) {
|
||||
//throw new Error('No item ID or value specified.');
|
||||
}
|
||||
|
||||
this.appendNewLine(`<trans-unit id="${item.id}">`, 4);
|
||||
this.appendNewLine(`<source xml:lang="en">${encodeEntities(item.message)}</source>`, 6);
|
||||
|
||||
if (item.comment) {
|
||||
this.appendNewLine(`<note>${item.comment}</note>`, 6);
|
||||
}
|
||||
|
||||
this.appendNewLine('</trans-unit>', 4);
|
||||
}
|
||||
|
||||
private appendHeader(): void {
|
||||
this.appendNewLine('<?xml version="1.0" encoding="utf-8"?>', 0);
|
||||
this.appendNewLine('<xliff version="1.2" xmlns="urn:oasis:names:tc:xliff:document:1.2">', 0);
|
||||
}
|
||||
|
||||
private appendFooter(): void {
|
||||
this.appendNewLine('</xliff>', 0);
|
||||
}
|
||||
|
||||
private appendNewLine(content: string, indent?: number): void {
|
||||
let line = new Line(indent);
|
||||
line.append(content);
|
||||
this.buffer.push(line.toString());
|
||||
}
|
||||
|
||||
static parse = function(xlfString: string) : Promise<{ messages: Map<string>, originalFilePath: string, language: string }[]> {
|
||||
return new Promise((resolve, reject) => {
|
||||
let parser = new xml2js.Parser();
|
||||
|
||||
let files: { messages: Map<string>, originalFilePath: string, language: string }[] = [];
|
||||
|
||||
parser.parseString(xlfString, function(err, result) {
|
||||
if (err) {
|
||||
reject(`Failed to parse XLIFF string. ${err}`);
|
||||
}
|
||||
|
||||
const fileNodes: any[] = result['xliff']['file'];
|
||||
if (!fileNodes) {
|
||||
reject('XLIFF file does not contain "xliff" or "file" node(s) required for parsing.');
|
||||
}
|
||||
|
||||
fileNodes.forEach((file) => {
|
||||
const originalFilePath = file.$.original;
|
||||
if (!originalFilePath) {
|
||||
reject('XLIFF file node does not contain original attribute to determine the original location of the resource file.');
|
||||
}
|
||||
const language = file.$['target-language'].toLowerCase();
|
||||
if (!language) {
|
||||
reject('XLIFF file node does not contain target-language attribute to determine translated language.');
|
||||
}
|
||||
|
||||
let messages: Map<string> = {};
|
||||
const transUnits = file.body[0]['trans-unit'];
|
||||
|
||||
transUnits.forEach(unit => {
|
||||
const key = unit.$.id;
|
||||
if (!unit.target) {
|
||||
return; // No translation available
|
||||
}
|
||||
|
||||
const val = unit.target.toString();
|
||||
if (key && val) {
|
||||
messages[key] = decodeEntities(val);
|
||||
} else {
|
||||
reject('XLIFF file does not contain full localization data. ID or target translation for one of the trans-unit nodes is not present.');
|
||||
}
|
||||
});
|
||||
|
||||
files.push({ messages: messages, originalFilePath: originalFilePath, language: language });
|
||||
});
|
||||
|
||||
resolve(files);
|
||||
});
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
const vscodeLanguages: string[] = [
|
||||
'chs',
|
||||
'cht',
|
||||
|
|
@ -85,6 +300,26 @@ const iso639_3_to_2: Map<string> = {
|
|||
'trk': 'tr'
|
||||
};
|
||||
|
||||
const iso639_2_to_3: Map<string> = {
|
||||
'zh-cn': 'chs',
|
||||
'zh-tw': 'cht',
|
||||
'cs-cz': 'csy',
|
||||
'de': 'deu',
|
||||
'en': 'enu',
|
||||
'es': 'esn',
|
||||
'fr': 'fra',
|
||||
'hu': 'hun',
|
||||
'it': 'ita',
|
||||
'ja': 'jpn',
|
||||
'ko': 'kor',
|
||||
'nl': 'nld',
|
||||
'pl': 'plk',
|
||||
'pt-br': 'ptb',
|
||||
'pt': 'ptg',
|
||||
'ru': 'rus',
|
||||
'sv-se': 'sve',
|
||||
'tr': 'trk'
|
||||
};
|
||||
interface IDirectoryInfo {
|
||||
name: string;
|
||||
iso639_2: string;
|
||||
|
|
@ -138,7 +373,7 @@ function stripComments(content: string): string {
|
|||
}
|
||||
});
|
||||
return result;
|
||||
};
|
||||
}
|
||||
|
||||
function escapeCharacters(value:string):string {
|
||||
var result:string[] = [];
|
||||
|
|
@ -308,4 +543,509 @@ export function processNlsFiles(opts:{fileHeader:string;}): ThroughStream {
|
|||
}
|
||||
this.emit('data', file);
|
||||
});
|
||||
}
|
||||
|
||||
export function prepareXlfFiles(projectName?: string, extensionName?: string): ThroughStream {
|
||||
return through(
|
||||
function (file: File) {
|
||||
if (!file.isBuffer()) {
|
||||
log('Error', `Failed to read component file: ${file.relative}`);
|
||||
}
|
||||
|
||||
const extension = path.extname(file.path);
|
||||
if (extension === '.json') {
|
||||
const json = JSON.parse((<Buffer>file.contents).toString('utf8'));
|
||||
|
||||
if (BundledFormat.is(json)) {
|
||||
importBundleJson(file, json, this);
|
||||
} else if (PackageJsonFormat.is(json) || ModuleJsonFormat.is(json)) {
|
||||
importModuleOrPackageJson(file, json, projectName, this, extensionName);
|
||||
} else {
|
||||
log('Error', 'JSON format cannot be deduced.');
|
||||
}
|
||||
} else if (extension === '.isl') {
|
||||
importIsl(file, this);
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
function getResource(sourceFile: string): Resource {
|
||||
const editorProject: string = 'vscode-editor',
|
||||
workbenchProject: string = 'vscode-workbench';
|
||||
let resource: string;
|
||||
|
||||
if (sourceFile.startsWith('vs/platform')) {
|
||||
return { name: 'vs/platform', project: editorProject };
|
||||
} else if (sourceFile.startsWith('vs/editor/contrib')) {
|
||||
return { name: 'vs/editor/contrib', project: editorProject };
|
||||
} else if (sourceFile.startsWith('vs/editor')) {
|
||||
return { name: 'vs/editor', project: editorProject };
|
||||
} else if (sourceFile.startsWith('vs/base')) {
|
||||
return { name: 'vs/base', project: editorProject };
|
||||
} else if (sourceFile.startsWith('vs/code')) {
|
||||
return { name: 'vs/code', project: workbenchProject };
|
||||
} else if (sourceFile.startsWith('vs/workbench/parts')) {
|
||||
resource = sourceFile.split('/', 4).join('/');
|
||||
return { name: resource, project: workbenchProject };
|
||||
} else if (sourceFile.startsWith('vs/workbench/services')) {
|
||||
resource = sourceFile.split('/', 4).join('/');
|
||||
return { name: resource, project: workbenchProject };
|
||||
} else if (sourceFile.startsWith('vs/workbench')) {
|
||||
return { name: 'vs/workbench', project: workbenchProject };
|
||||
}
|
||||
|
||||
throw new Error (`Could not identify the XLF bundle for ${sourceFile}`);
|
||||
}
|
||||
|
||||
|
||||
function importBundleJson(file: File, json: BundledFormat, stream: ThroughStream): void {
|
||||
let transifexEditorXlfs: Map<XLF> = Object.create(null);
|
||||
|
||||
for (let source in json.keys) {
|
||||
const projectResource = getResource(source);
|
||||
const resource = projectResource.name;
|
||||
const project = projectResource.project;
|
||||
|
||||
const keys = json.keys[source];
|
||||
const messages = json.messages[source];
|
||||
if (keys.length !== messages.length) {
|
||||
log('Error:', `There is a mismatch between keys and messages in ${file.relative}`);
|
||||
}
|
||||
|
||||
let xlf = transifexEditorXlfs[resource] ? transifexEditorXlfs[resource] : transifexEditorXlfs[resource] = new XLF(project);
|
||||
xlf.addFile(source, keys, messages);
|
||||
}
|
||||
|
||||
for (let resource in transifexEditorXlfs) {
|
||||
const newFilePath = `${transifexEditorXlfs[resource].project}/${resource.replace(/\//g, '_')}.xlf`;
|
||||
const xlfFile = new File({ path: newFilePath, contents: new Buffer(transifexEditorXlfs[resource].toString(), 'utf-8')});
|
||||
stream.emit('data', xlfFile);
|
||||
}
|
||||
}
|
||||
|
||||
// Keeps existing XLF instances and a state of how many files were already processed for faster file emission
|
||||
var extensions: Map<{ xlf: XLF, processed: number }> = Object.create(null);
|
||||
function importModuleOrPackageJson(file: File, json: ModuleJsonFormat | PackageJsonFormat, projectName: string, stream: ThroughStream, extensionName?: string): void {
|
||||
if (ModuleJsonFormat.is(json) && json['keys'].length !== json['messages'].length) {
|
||||
log('Error:', `There is a mismatch between keys and messages in ${file.relative}`);
|
||||
}
|
||||
|
||||
// Prepare the source path for <original/> attribute in XLF & extract messages from JSON
|
||||
const formattedSourcePath = file.relative.replace(/\\/g, '/');
|
||||
const messages = Object.keys(json).map((key) => json[key].toString());
|
||||
|
||||
// Stores the amount of localization files to be transformed to XLF before the emission
|
||||
let localizationFilesCount,
|
||||
originalFilePath;
|
||||
// If preparing XLF for external extension, then use different glob pattern and source path
|
||||
if (extensionName) {
|
||||
localizationFilesCount = glob.sync('**/*.nls.json').length;
|
||||
originalFilePath = `${formattedSourcePath.substr(0, formattedSourcePath.length - '.nls.json'.length)}`;
|
||||
} else {
|
||||
// Used for vscode/extensions folder
|
||||
extensionName = formattedSourcePath.split('/')[0];
|
||||
localizationFilesCount = glob.sync(`./extensions/${extensionName}/**/*.nls.json`).length;
|
||||
originalFilePath = `extensions/${formattedSourcePath.substr(0, formattedSourcePath.length - '.nls.json'.length)}`;
|
||||
}
|
||||
|
||||
let extension = extensions[extensionName] ?
|
||||
extensions[extensionName] : extensions[extensionName] = { xlf: new XLF(projectName), processed: 0 };
|
||||
|
||||
if (ModuleJsonFormat.is(json)) {
|
||||
extension.xlf.addFile(originalFilePath, json['keys'], json['messages']);
|
||||
} else {
|
||||
extension.xlf.addFile(originalFilePath, Object.keys(json), messages);
|
||||
}
|
||||
|
||||
// Check if XLF is populated with file nodes to emit it
|
||||
if (++extensions[extensionName].processed === localizationFilesCount) {
|
||||
const newFilePath = path.join(projectName, extensionName + '.xlf');
|
||||
const xlfFile = new File({ path: newFilePath, contents: new Buffer(extension.xlf.toString(), 'utf-8')});
|
||||
stream.emit('data', xlfFile);
|
||||
}
|
||||
}
|
||||
|
||||
var islXlf: XLF,
|
||||
islProcessed: number = 0;
|
||||
|
||||
function importIsl(file: File, stream: ThroughStream) {
|
||||
const islFiles = ['Default.isl', 'messages.en.isl'];
|
||||
const projectName = 'vscode-workbench';
|
||||
|
||||
let xlf = islXlf ? islXlf : islXlf = new XLF(projectName),
|
||||
keys: string[] = [],
|
||||
messages: string[] = [];
|
||||
|
||||
let model = new TextModel(file.contents.toString());
|
||||
let inMessageSection = false;
|
||||
model.lines.forEach(line => {
|
||||
if (line.length === 0) {
|
||||
return;
|
||||
}
|
||||
let firstChar = line.charAt(0);
|
||||
switch (firstChar) {
|
||||
case ';':
|
||||
// Comment line;
|
||||
return;
|
||||
case '[':
|
||||
inMessageSection = '[Messages]' === line || '[CustomMessages]' === line;
|
||||
return;
|
||||
}
|
||||
if (!inMessageSection) {
|
||||
return;
|
||||
}
|
||||
let sections: string[] = line.split('=');
|
||||
if (sections.length !== 2) {
|
||||
log('Error:', `Badly formatted message found: ${line}`);
|
||||
} else {
|
||||
let key = sections[0];
|
||||
let value = sections[1];
|
||||
if (key.length > 0 && value.length > 0) {
|
||||
keys.push(key);
|
||||
messages.push(value);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const originalPath = file.path.substring(file.cwd.length+1, file.path.split('.')[0].length).replace(/\\/g, '/');
|
||||
xlf.addFile(originalPath, keys, messages);
|
||||
|
||||
// Emit only upon all ISL files combined into single XLF instance
|
||||
if (++islProcessed === islFiles.length) {
|
||||
const newFilePath = path.join(projectName, 'setup.xlf');
|
||||
const xlfFile = new File({ path: newFilePath, contents: new Buffer(xlf.toString(), 'utf-8')});
|
||||
stream.emit('data', xlfFile);
|
||||
}
|
||||
}
|
||||
|
||||
export function pushXlfFiles(apiUrl: string, username: string, password: string): ThroughStream {
|
||||
return through(function(file: File) {
|
||||
const project = path.dirname(file.relative);
|
||||
const fileName = path.basename(file.path);
|
||||
const slug = fileName.substr(0, fileName.length - '.xlf'.length);
|
||||
const credentials = {
|
||||
'user': username,
|
||||
'password': password
|
||||
};
|
||||
|
||||
// Check if resource already exists, if not, then create it.
|
||||
tryGetResource(project, slug, apiUrl, credentials).then(exists => {
|
||||
if (exists) {
|
||||
updateResource(project, slug, file, apiUrl, credentials);
|
||||
} else {
|
||||
createResource(project, slug, file, apiUrl, credentials);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function tryGetResource(project: string, slug: string, apiUrl: string, credentials: any): Promise<boolean> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const url = `${apiUrl}/project/${project}/resource/${slug}/?details`;
|
||||
request.get(url, { 'auth': credentials }).on('response', function (response) {
|
||||
if (response.statusCode === 404) {
|
||||
resolve(false);
|
||||
} else if (response.statusCode === 200) {
|
||||
resolve(true);
|
||||
} else {
|
||||
reject(`Failed to query resource ${slug}. Response: ${response.statusCode} ${response.statusMessage}`);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function createResource(project: string, slug: string, xlfFile: File, apiUrl:string, credentials: any): void {
|
||||
const url = `${apiUrl}/project/${project}/resources`;
|
||||
const options = {
|
||||
'body': {
|
||||
'content': xlfFile.contents.toString(),
|
||||
'name': slug,
|
||||
'slug': slug,
|
||||
'i18n_type': 'XLIFF'
|
||||
},
|
||||
'json': true,
|
||||
'auth': credentials
|
||||
};
|
||||
|
||||
request.post(url, options, function(err, res) {
|
||||
if (err) {
|
||||
log('Error:', `Failed to create Transifex ${project}/${slug}: ${err}`);
|
||||
}
|
||||
|
||||
if (res.statusCode === 201) {
|
||||
log(`Resource ${project}/${slug} successfully created on Transifex.`);
|
||||
} else {
|
||||
log('Error:', `Something went wrong creating ${slug} in ${project}. ${res.statusCode}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* The following link provides information about how Transifex handles updates of a resource file:
|
||||
* https://dev.befoolish.co/tx-docs/public/projects/updating-content#what-happens-when-you-update-files
|
||||
*/
|
||||
function updateResource(project: string, slug: string, xlfFile: File, apiUrl: string, credentials: any) : void {
|
||||
const url = `${apiUrl}/project/${project}/resource/${slug}/content`;
|
||||
const options = {
|
||||
'body': { 'content': xlfFile.contents.toString() },
|
||||
'json': true,
|
||||
'auth': credentials
|
||||
};
|
||||
|
||||
request.put(url, options, function(err, res, body) {
|
||||
if (err) {
|
||||
log('Error:', `Failed to update Transifex ${project}/${slug}: ${err}`);
|
||||
}
|
||||
|
||||
if (res.statusCode === 200) {
|
||||
log(`Resource ${project}/${slug} successfully updated on Transifex. Strings added: ${body['strings_added']}, updated: ${body['strings_updated']}, deleted: ${body['strings_delete']}`);
|
||||
} else {
|
||||
log('Error:', `Something went wrong updating ${slug} in ${project}. ${res.statusCode}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function getMetadataResources(pathToMetadata: string) : Resource[] {
|
||||
const metadata = fs.readFileSync(pathToMetadata).toString('utf8');
|
||||
const json = JSON.parse(metadata);
|
||||
let slugs = [];
|
||||
|
||||
for (let source in json['keys']) {
|
||||
let projectResource = getResource(source);
|
||||
if (!slugs.find(slug => slug.name === projectResource.name && slug.project === projectResource.project)) {
|
||||
slugs.push(projectResource);
|
||||
}
|
||||
}
|
||||
|
||||
return slugs;
|
||||
}
|
||||
|
||||
function obtainProjectResources(projectName: string): Resource[] {
|
||||
let resources: Resource[];
|
||||
|
||||
if (projectName === 'vscode-editor-workbench') {
|
||||
resources = getMetadataResources('./out-vscode/nls.metadata.json');
|
||||
resources.push({ name: 'setup', project: 'vscode-workbench' });
|
||||
} else if (projectName === 'vscode-extensions') {
|
||||
let extensionsToLocalize: string[] = glob.sync('./extensions/**/*.nls.json').map(extension => extension.split('/')[2]);
|
||||
let resourcesToPull: string[] = [];
|
||||
resources = [];
|
||||
|
||||
extensionsToLocalize.forEach(extension => {
|
||||
if (resourcesToPull.indexOf(extension) === -1) { // remove duplicate elements returned by glob
|
||||
resourcesToPull.push(extension);
|
||||
resources.push({ name: extension, project: projectName });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return resources;
|
||||
}
|
||||
|
||||
export function pullXlfFiles(projectName: string, apiUrl: string, username: string, password: string, resources?: Resource[]): NodeJS.ReadableStream {
|
||||
if (!resources) {
|
||||
resources = obtainProjectResources(projectName);
|
||||
}
|
||||
if (!resources) {
|
||||
throw new Error('Transifex projects and resources must be defined to be able to pull translations from Transifex.');
|
||||
}
|
||||
|
||||
const credentials = {
|
||||
'auth': {
|
||||
'user': username,
|
||||
'password': password
|
||||
}
|
||||
};
|
||||
let expectedTranslationsCount = vscodeLanguages.length * resources.length;
|
||||
let translationsRetrieved = 0, called = false;
|
||||
|
||||
return es.readable(function(count, callback) {
|
||||
// Mark end of stream when all resources were retrieved
|
||||
if (translationsRetrieved === expectedTranslationsCount) {
|
||||
return this.emit('end');
|
||||
}
|
||||
|
||||
if (!called) {
|
||||
called = true;
|
||||
const stream = this;
|
||||
|
||||
vscodeLanguages.map(function(language) {
|
||||
resources.map(function (resource) {
|
||||
const slug = resource.name.replace(/\//g, '_');
|
||||
const project = resource.project;
|
||||
const iso639 = iso639_3_to_2[language];
|
||||
const url = `${apiUrl}/project/${project}/resource/${slug}/translation/${iso639}?file&mode=onlyreviewed`;
|
||||
|
||||
let xlfBuffer: string = '', responseCode: number;
|
||||
request.get(url, credentials)
|
||||
.on('response', (response) => {
|
||||
responseCode = response.statusCode;
|
||||
})
|
||||
.on('data', (data) => xlfBuffer += data)
|
||||
.on('end', () => {
|
||||
if (responseCode === 200) {
|
||||
stream.emit('data', new File({ contents: new Buffer(xlfBuffer) }));
|
||||
} else {
|
||||
log('Error:', `${slug} in ${project} returned no data. Response code: ${responseCode}.`);
|
||||
}
|
||||
translationsRetrieved++;
|
||||
})
|
||||
.on('error', (error) => {
|
||||
log('Error:', `Failed to query resource ${slug} with the following error: ${error}`);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
callback();
|
||||
});
|
||||
}
|
||||
|
||||
export function prepareJsonFiles(): ThroughStream {
|
||||
return through(function(xlf: File) {
|
||||
let stream = this;
|
||||
|
||||
XLF.parse(xlf.contents.toString()).then(
|
||||
function(resolvedFiles) {
|
||||
resolvedFiles.forEach(file => {
|
||||
let messages = file.messages, translatedFile;
|
||||
|
||||
// ISL file path always starts with 'build/'
|
||||
if (file.originalFilePath.startsWith('build/')) {
|
||||
const defaultLanguages = { 'zh-cn': true, 'zh-tw': true, 'ko': true };
|
||||
if (path.basename(file.originalFilePath) === 'Default' && !defaultLanguages[file.language]) {
|
||||
return;
|
||||
}
|
||||
|
||||
translatedFile = createIslFile('..', file.originalFilePath, messages, iso639_2_to_3[file.language]);
|
||||
} else {
|
||||
translatedFile = createI18nFile(iso639_2_to_3[file.language], file.originalFilePath, messages);
|
||||
}
|
||||
|
||||
stream.emit('data', translatedFile);
|
||||
});
|
||||
},
|
||||
function(rejectReason) {
|
||||
log('Error:', rejectReason);
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
export function createI18nFile(base: string, originalFilePath: string, messages: Map<string>): File {
|
||||
let content = [
|
||||
'/*---------------------------------------------------------------------------------------------',
|
||||
' * Copyright (c) Microsoft Corporation. All rights reserved.',
|
||||
' * Licensed under the MIT License. See License.txt in the project root for license information.',
|
||||
' *--------------------------------------------------------------------------------------------*/',
|
||||
'// Do not edit this file. It is machine generated.'
|
||||
].join('\n') + '\n' + JSON.stringify(messages, null, '\t').replace(/\r\n/g, '\n');
|
||||
|
||||
return new File({
|
||||
path: path.join(base, originalFilePath + '.i18n.json'),
|
||||
contents: new Buffer(content, 'utf8')
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
const languageNames: Map<string> = {
|
||||
'chs': 'Simplified Chinese',
|
||||
'cht': 'Traditional Chinese',
|
||||
'kor': 'Korean'
|
||||
};
|
||||
|
||||
const languageIds: Map<string> = {
|
||||
'chs': '$0804',
|
||||
'cht': '$0404',
|
||||
'kor': '$0412'
|
||||
};
|
||||
|
||||
const encodings: Map<string> = {
|
||||
'chs': 'CP936',
|
||||
'cht': 'CP950',
|
||||
'jpn': 'CP932',
|
||||
'kor': 'CP949',
|
||||
'deu': 'CP1252',
|
||||
'fra': 'CP1252',
|
||||
'esn': 'CP1252',
|
||||
'rus': 'CP1251',
|
||||
'ita': 'CP1252'
|
||||
};
|
||||
|
||||
export function createIslFile(base: string, originalFilePath: string, messages: Map<string>, language: string): File {
|
||||
let content: string[] = [];
|
||||
let originalContent: TextModel;
|
||||
if (path.basename(originalFilePath) === 'Default') {
|
||||
originalContent = new TextModel(fs.readFileSync(originalFilePath + '.isl', 'utf8'));
|
||||
} else {
|
||||
originalContent = new TextModel(fs.readFileSync(originalFilePath + '.en.isl', 'utf8'));
|
||||
}
|
||||
|
||||
originalContent.lines.forEach(line => {
|
||||
if (line.length > 0) {
|
||||
let firstChar = line.charAt(0);
|
||||
if (firstChar === '[' || firstChar === ';') {
|
||||
if (line === '; *** Inno Setup version 5.5.3+ English messages ***') {
|
||||
content.push(`; *** Inno Setup version 5.5.3+ ${languageNames[language]} messages ***`);
|
||||
} else {
|
||||
content.push(line);
|
||||
}
|
||||
} else {
|
||||
let sections: string[] = line.split('=');
|
||||
let key = sections[0];
|
||||
let translated = line;
|
||||
if (key) {
|
||||
if (key === 'LanguageName') {
|
||||
translated = `${key}=${languageNames[language]}`;
|
||||
} else if (key === 'LanguageID') {
|
||||
translated = `${key}=${languageIds[language]}`;
|
||||
} else if (key === 'LanguageCodePage') {
|
||||
translated = `${key}=${encodings[language].substr(2)}`;
|
||||
} else {
|
||||
let translatedMessage = messages[key];
|
||||
if (translatedMessage) {
|
||||
translated = `${key}=${translatedMessage}`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
content.push(translated);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let tag = iso639_3_to_2[language];
|
||||
let basename = path.basename(originalFilePath);
|
||||
let filePath = `${path.join(base, path.dirname(originalFilePath), basename)}.${tag}.isl`;
|
||||
|
||||
return new File({
|
||||
path: filePath,
|
||||
contents: iconv.encode(new Buffer(content.join('\r\n'), 'utf8'), encodings[language])
|
||||
});
|
||||
}
|
||||
|
||||
function encodeEntities(value: string): string {
|
||||
var result: string[] = [];
|
||||
for (var i = 0; i < value.length; i++) {
|
||||
var ch = value[i];
|
||||
switch (ch) {
|
||||
case '<':
|
||||
result.push('<');
|
||||
break;
|
||||
case '>':
|
||||
result.push('>');
|
||||
break;
|
||||
case '&':
|
||||
result.push('&');
|
||||
break;
|
||||
default:
|
||||
result.push(ch);
|
||||
}
|
||||
}
|
||||
return result.join('');
|
||||
}
|
||||
|
||||
export function decodeEntities(value:string): string {
|
||||
return value.replace(/</g, '<').replace(/>/g, '>').replace(/&/g, '&');
|
||||
}
|
||||
|
|
@ -403,4 +403,4 @@
|
|||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue