2016-03-08 10:26:06 +00:00
|
|
|
/*---------------------------------------------------------------------------------------------
|
|
|
|
|
* Copyright (c) Microsoft Corporation. All rights reserved.
|
|
|
|
|
* Licensed under the MIT License. See License.txt in the project root for license information.
|
|
|
|
|
*--------------------------------------------------------------------------------------------*/
|
|
|
|
|
|
|
|
|
|
import * as path from 'path';
|
|
|
|
|
import * as fs from 'fs';
|
|
|
|
|
|
2017-03-20 14:01:18 +00:00
|
|
|
import { through, readable } from 'event-stream';
|
2016-03-08 10:26:06 +00:00
|
|
|
import { ThroughStream } from 'through';
|
|
|
|
|
import File = require('vinyl');
|
|
|
|
|
import * as Is from 'is';
|
2017-03-20 14:01:18 +00:00
|
|
|
import * as xml2js from 'xml2js';
|
2017-03-02 20:53:02 +00:00
|
|
|
import * as glob from 'glob';
|
2017-06-30 09:42:52 +00:00
|
|
|
import * as https from 'https';
|
2016-03-08 10:26:06 +00:00
|
|
|
|
|
|
|
|
var util = require('gulp-util');
|
2017-04-13 17:58:13 +00:00
|
|
|
var iconv = require('iconv-lite');
|
2016-07-19 09:37:20 +00:00
|
|
|
|
2016-03-08 10:26:06 +00:00
|
|
|
function log(message: any, ...rest: any[]): void {
|
2016-07-19 09:37:20 +00:00
|
|
|
util.log(util.colors.green('[i18n]'), message, ...rest);
|
2016-03-08 10:26:06 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
interface Map<V> {
|
|
|
|
|
[key: string]: V;
|
|
|
|
|
}
|
|
|
|
|
|
2017-03-02 20:53:02 +00:00
|
|
|
interface Item {
|
|
|
|
|
id: string;
|
|
|
|
|
message: string;
|
|
|
|
|
comment: string;
|
|
|
|
|
}
|
|
|
|
|
|
2017-04-05 14:22:22 +00:00
|
|
|
export interface Resource {
|
2017-03-02 20:53:02 +00:00
|
|
|
name: string;
|
|
|
|
|
project: string;
|
|
|
|
|
}
|
|
|
|
|
|
2017-04-13 16:19:51 +00:00
|
|
|
interface ParsedXLF {
|
|
|
|
|
messages: Map<string>;
|
|
|
|
|
originalFilePath: string;
|
|
|
|
|
language: string;
|
|
|
|
|
}
|
|
|
|
|
|
2016-03-08 10:26:06 +00:00
|
|
|
interface LocalizeInfo {
|
|
|
|
|
key: string;
|
|
|
|
|
comment: string[];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
module LocalizeInfo {
|
|
|
|
|
export function is(value: any): value is LocalizeInfo {
|
|
|
|
|
let candidate = value as LocalizeInfo;
|
|
|
|
|
return Is.defined(candidate) && Is.string(candidate.key) && (Is.undef(candidate.comment) || (Is.array(candidate.comment) && candidate.comment.every(element => Is.string(element))));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
interface BundledFormat {
|
|
|
|
|
keys: Map<(string | LocalizeInfo)[]>;
|
|
|
|
|
messages: Map<string[]>;
|
|
|
|
|
bundles: Map<string[]>;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
module BundledFormat {
|
|
|
|
|
export function is(value: any): value is BundledFormat {
|
|
|
|
|
if (Is.undef(value)) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
let candidate = value as BundledFormat;
|
|
|
|
|
let length = Object.keys(value).length;
|
|
|
|
|
|
|
|
|
|
return length === 3 && Is.defined(candidate.keys) && Is.defined(candidate.messages) && Is.defined(candidate.bundles);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2017-03-02 20:53:02 +00:00
|
|
|
interface ValueFormat {
|
|
|
|
|
message: string;
|
|
|
|
|
comment: string[];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
interface PackageJsonFormat {
|
|
|
|
|
[key: string]: string | ValueFormat;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
module PackageJsonFormat {
|
|
|
|
|
export function is(value: any): value is PackageJsonFormat {
|
|
|
|
|
if (Is.undef(value) || !Is.object(value)) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
return Object.keys(value).every(key => {
|
|
|
|
|
let element = value[key];
|
|
|
|
|
return Is.string(element) || (Is.object(element) && Is.defined(element.message) && Is.defined(element.comment));
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
interface ModuleJsonFormat {
|
|
|
|
|
messages: string[];
|
|
|
|
|
keys: (string | LocalizeInfo)[];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
module ModuleJsonFormat {
|
|
|
|
|
export function is(value: any): value is ModuleJsonFormat {
|
|
|
|
|
let candidate = value as ModuleJsonFormat;
|
|
|
|
|
return Is.defined(candidate)
|
|
|
|
|
&& Is.array(candidate.messages) && candidate.messages.every(message => Is.string(message))
|
|
|
|
|
&& Is.array(candidate.keys) && candidate.keys.every(key => Is.string(key) || LocalizeInfo.is(key));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export class Line {
|
|
|
|
|
private buffer: string[] = [];
|
|
|
|
|
|
|
|
|
|
constructor(private indent: number = 0) {
|
|
|
|
|
if (indent > 0) {
|
|
|
|
|
this.buffer.push(new Array(indent + 1).join(' '));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public append(value: string): Line {
|
|
|
|
|
this.buffer.push(value);
|
|
|
|
|
return this;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public toString(): string {
|
|
|
|
|
return this.buffer.join('');
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
class TextModel {
|
|
|
|
|
private _lines: string[];
|
|
|
|
|
|
|
|
|
|
constructor(contents: string) {
|
|
|
|
|
this._lines = contents.split(/\r\n|\r|\n/);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public get lines(): string[] {
|
|
|
|
|
return this._lines;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export class XLF {
|
2017-05-15 13:59:14 +00:00
|
|
|
private buffer: string[];
|
2017-03-02 20:53:02 +00:00
|
|
|
private files: Map<Item[]>;
|
|
|
|
|
|
2017-05-15 13:59:14 +00:00
|
|
|
constructor(public project: string) {
|
|
|
|
|
this.buffer = [];
|
2017-03-02 20:53:02 +00:00
|
|
|
this.files = Object.create(null);
|
|
|
|
|
}
|
|
|
|
|
|
2017-05-15 13:59:14 +00:00
|
|
|
public toString(): string {
|
|
|
|
|
this.appendHeader();
|
2017-03-02 20:53:02 +00:00
|
|
|
|
|
|
|
|
for (let file in this.files) {
|
|
|
|
|
this.appendNewLine(`<file original="${file}" source-language="en" datatype="plaintext"><body>`, 2);
|
|
|
|
|
for (let item of this.files[file]) {
|
|
|
|
|
this.addStringItem(item);
|
|
|
|
|
}
|
2017-05-15 13:59:14 +00:00
|
|
|
this.appendNewLine('</body></file>', 2);
|
2017-03-02 20:53:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
this.appendFooter();
|
|
|
|
|
return this.buffer.join('\r\n');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public addFile(original: string, keys: any[], messages: string[]) {
|
|
|
|
|
this.files[original] = [];
|
|
|
|
|
let existingKeys = [];
|
|
|
|
|
|
|
|
|
|
for (let key of keys) {
|
|
|
|
|
// Ignore duplicate keys because Transifex does not populate those with translated values.
|
|
|
|
|
if (existingKeys.indexOf(key) !== -1) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
existingKeys.push(key);
|
|
|
|
|
|
|
|
|
|
let message: string = encodeEntities(messages[keys.indexOf(key)]);
|
|
|
|
|
let comment: string = undefined;
|
|
|
|
|
|
|
|
|
|
// Check if the message contains description (if so, it becomes an object type in JSON)
|
|
|
|
|
if (Is.string(key)) {
|
|
|
|
|
this.files[original].push({ id: key, message: message, comment: comment });
|
|
|
|
|
} else {
|
|
|
|
|
if (key['comment'] && key['comment'].length > 0) {
|
|
|
|
|
comment = key['comment'].map(comment => encodeEntities(comment)).join('\r\n');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
this.files[original].push({ id: key['key'], message: message, comment: comment });
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2017-05-15 13:59:14 +00:00
|
|
|
private addStringItem(item: Item): void {
|
|
|
|
|
if (!item.id || !item.message) {
|
|
|
|
|
throw new Error('No item ID or value specified.');
|
|
|
|
|
}
|
2017-03-02 20:53:02 +00:00
|
|
|
|
2017-05-15 13:59:14 +00:00
|
|
|
this.appendNewLine(`<trans-unit id="${item.id}">`, 4);
|
|
|
|
|
this.appendNewLine(`<source xml:lang="en">${item.message}</source>`, 6);
|
2017-03-02 20:53:02 +00:00
|
|
|
|
2017-05-15 13:59:14 +00:00
|
|
|
if (item.comment) {
|
|
|
|
|
this.appendNewLine(`<note>${item.comment}</note>`, 6);
|
|
|
|
|
}
|
2017-03-02 20:53:02 +00:00
|
|
|
|
2017-05-15 13:59:14 +00:00
|
|
|
this.appendNewLine('</trans-unit>', 4);
|
2017-03-02 20:53:02 +00:00
|
|
|
}
|
|
|
|
|
|
2017-05-15 13:59:14 +00:00
|
|
|
private appendHeader(): void {
|
|
|
|
|
this.appendNewLine('<?xml version="1.0" encoding="utf-8"?>', 0);
|
|
|
|
|
this.appendNewLine('<xliff version="1.2" xmlns="urn:oasis:names:tc:xliff:document:1.2">', 0);
|
2017-03-02 20:53:02 +00:00
|
|
|
}
|
|
|
|
|
|
2017-05-15 13:59:14 +00:00
|
|
|
private appendFooter(): void {
|
|
|
|
|
this.appendNewLine('</xliff>', 0);
|
|
|
|
|
}
|
2017-03-02 20:53:02 +00:00
|
|
|
|
2017-05-15 13:59:14 +00:00
|
|
|
private appendNewLine(content: string, indent?: number): void {
|
|
|
|
|
let line = new Line(indent);
|
|
|
|
|
line.append(content);
|
|
|
|
|
this.buffer.push(line.toString());
|
|
|
|
|
}
|
2017-03-02 20:53:02 +00:00
|
|
|
|
2017-05-15 13:59:14 +00:00
|
|
|
static parse = function (xlfString: string): Promise<ParsedXLF[]> {
|
2017-03-02 20:53:02 +00:00
|
|
|
return new Promise((resolve, reject) => {
|
|
|
|
|
let parser = new xml2js.Parser();
|
|
|
|
|
|
|
|
|
|
let files: { messages: Map<string>, originalFilePath: string, language: string }[] = [];
|
|
|
|
|
|
2017-05-15 13:59:14 +00:00
|
|
|
parser.parseString(xlfString, function (err, result) {
|
2017-03-02 20:53:02 +00:00
|
|
|
if (err) {
|
|
|
|
|
reject(`Failed to parse XLIFF string. ${err}`);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const fileNodes: any[] = result['xliff']['file'];
|
|
|
|
|
if (!fileNodes) {
|
|
|
|
|
reject('XLIFF file does not contain "xliff" or "file" node(s) required for parsing.');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fileNodes.forEach((file) => {
|
|
|
|
|
const originalFilePath = file.$.original;
|
|
|
|
|
if (!originalFilePath) {
|
|
|
|
|
reject('XLIFF file node does not contain original attribute to determine the original location of the resource file.');
|
|
|
|
|
}
|
|
|
|
|
const language = file.$['target-language'].toLowerCase();
|
|
|
|
|
if (!language) {
|
|
|
|
|
reject('XLIFF file node does not contain target-language attribute to determine translated language.');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
let messages: Map<string> = {};
|
|
|
|
|
const transUnits = file.body[0]['trans-unit'];
|
|
|
|
|
|
|
|
|
|
transUnits.forEach(unit => {
|
|
|
|
|
const key = unit.$.id;
|
|
|
|
|
if (!unit.target) {
|
|
|
|
|
return; // No translation available
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const val = unit.target.toString();
|
|
|
|
|
if (key && val) {
|
|
|
|
|
messages[key] = decodeEntities(val);
|
|
|
|
|
} else {
|
|
|
|
|
reject('XLIFF file does not contain full localization data. ID or target translation for one of the trans-unit nodes is not present.');
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
files.push({ messages: messages, originalFilePath: originalFilePath, language: language });
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
resolve(files);
|
|
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
2016-03-08 10:26:06 +00:00
|
|
|
const iso639_3_to_2: Map<string> = {
|
2017-04-06 14:37:52 +00:00
|
|
|
'chs': 'zh-cn',
|
|
|
|
|
'cht': 'zh-tw',
|
2016-03-22 11:50:39 +00:00
|
|
|
'csy': 'cs-cz',
|
2016-03-08 10:26:06 +00:00
|
|
|
'deu': 'de',
|
|
|
|
|
'enu': 'en',
|
|
|
|
|
'esn': 'es',
|
|
|
|
|
'fra': 'fr',
|
|
|
|
|
'hun': 'hu',
|
|
|
|
|
'ita': 'it',
|
|
|
|
|
'jpn': 'ja',
|
|
|
|
|
'kor': 'ko',
|
|
|
|
|
'nld': 'nl',
|
|
|
|
|
'plk': 'pl',
|
2016-03-22 11:50:39 +00:00
|
|
|
'ptb': 'pt-br',
|
2016-03-08 10:26:06 +00:00
|
|
|
'ptg': 'pt',
|
|
|
|
|
'rus': 'ru',
|
2016-03-22 11:50:39 +00:00
|
|
|
'sve': 'sv-se',
|
2016-03-08 10:26:06 +00:00
|
|
|
'trk': 'tr'
|
|
|
|
|
};
|
|
|
|
|
|
2017-04-06 14:37:52 +00:00
|
|
|
/**
|
|
|
|
|
* Used to map Transifex to VS Code language code representation.
|
|
|
|
|
*/
|
2017-03-02 20:53:02 +00:00
|
|
|
const iso639_2_to_3: Map<string> = {
|
2017-04-06 07:45:42 +00:00
|
|
|
'zh-hans': 'chs',
|
|
|
|
|
'zh-hant': 'cht',
|
2017-03-02 20:53:02 +00:00
|
|
|
'cs-cz': 'csy',
|
|
|
|
|
'de': 'deu',
|
|
|
|
|
'en': 'enu',
|
|
|
|
|
'es': 'esn',
|
|
|
|
|
'fr': 'fra',
|
|
|
|
|
'hu': 'hun',
|
|
|
|
|
'it': 'ita',
|
|
|
|
|
'ja': 'jpn',
|
|
|
|
|
'ko': 'kor',
|
|
|
|
|
'nl': 'nld',
|
|
|
|
|
'pl': 'plk',
|
|
|
|
|
'pt-br': 'ptb',
|
|
|
|
|
'pt': 'ptg',
|
|
|
|
|
'ru': 'rus',
|
|
|
|
|
'sv-se': 'sve',
|
|
|
|
|
'tr': 'trk'
|
|
|
|
|
};
|
2017-04-06 14:37:52 +00:00
|
|
|
|
2016-03-08 10:26:06 +00:00
|
|
|
interface IDirectoryInfo {
|
|
|
|
|
name: string;
|
|
|
|
|
iso639_2: string;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function sortLanguages(directoryNames: string[]): IDirectoryInfo[] {
|
|
|
|
|
return directoryNames.map((dirName) => {
|
|
|
|
|
var lower = dirName.toLowerCase();
|
|
|
|
|
return {
|
|
|
|
|
name: lower,
|
|
|
|
|
iso639_2: iso639_3_to_2[lower]
|
|
|
|
|
};
|
|
|
|
|
}).sort((a: IDirectoryInfo, b: IDirectoryInfo): number => {
|
|
|
|
|
if (!a.iso639_2 && !b.iso639_2) {
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
if (!a.iso639_2) {
|
|
|
|
|
return -1;
|
|
|
|
|
}
|
|
|
|
|
if (!b.iso639_2) {
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
return a.iso639_2 < b.iso639_2 ? -1 : (a.iso639_2 > b.iso639_2 ? 1 : 0);
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function stripComments(content: string): string {
|
|
|
|
|
/**
|
|
|
|
|
* First capturing group matches double quoted string
|
|
|
|
|
* Second matches single quotes string
|
|
|
|
|
* Third matches block comments
|
|
|
|
|
* Fourth matches line comments
|
|
|
|
|
*/
|
|
|
|
|
var regexp: RegExp = /("(?:[^\\\"]*(?:\\.)?)*")|('(?:[^\\\']*(?:\\.)?)*')|(\/\*(?:\r?\n|.)*?\*\/)|(\/{2,}.*?(?:(?:\r?\n)|$))/g;
|
|
|
|
|
let result = content.replace(regexp, (match, m1, m2, m3, m4) => {
|
|
|
|
|
// Only one of m1, m2, m3, m4 matches
|
|
|
|
|
if (m3) {
|
|
|
|
|
// A block comment. Replace with nothing
|
|
|
|
|
return '';
|
|
|
|
|
} else if (m4) {
|
|
|
|
|
// A line comment. If it ends in \r?\n then keep it.
|
|
|
|
|
let length = m4.length;
|
|
|
|
|
if (length > 2 && m4[length - 1] === '\n') {
|
2017-05-15 13:59:14 +00:00
|
|
|
return m4[length - 2] === '\r' ? '\r\n' : '\n';
|
2016-03-08 10:26:06 +00:00
|
|
|
} else {
|
|
|
|
|
return '';
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
// We match a string
|
|
|
|
|
return match;
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
return result;
|
2017-03-02 20:53:02 +00:00
|
|
|
}
|
2016-03-08 10:26:06 +00:00
|
|
|
|
2017-05-15 13:59:14 +00:00
|
|
|
function escapeCharacters(value: string): string {
|
|
|
|
|
var result: string[] = [];
|
2016-03-08 10:26:06 +00:00
|
|
|
for (var i = 0; i < value.length; i++) {
|
|
|
|
|
var ch = value.charAt(i);
|
2017-05-15 13:59:14 +00:00
|
|
|
switch (ch) {
|
2016-03-08 10:26:06 +00:00
|
|
|
case '\'':
|
|
|
|
|
result.push('\\\'');
|
|
|
|
|
break;
|
|
|
|
|
case '"':
|
|
|
|
|
result.push('\\"');
|
|
|
|
|
break;
|
|
|
|
|
case '\\':
|
|
|
|
|
result.push('\\\\');
|
|
|
|
|
break;
|
|
|
|
|
case '\n':
|
|
|
|
|
result.push('\\n');
|
|
|
|
|
break;
|
|
|
|
|
case '\r':
|
|
|
|
|
result.push('\\r');
|
|
|
|
|
break;
|
|
|
|
|
case '\t':
|
|
|
|
|
result.push('\\t');
|
|
|
|
|
break;
|
|
|
|
|
case '\b':
|
|
|
|
|
result.push('\\b');
|
|
|
|
|
break;
|
|
|
|
|
case '\f':
|
|
|
|
|
result.push('\\f');
|
|
|
|
|
break;
|
|
|
|
|
default:
|
|
|
|
|
result.push(ch);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return result.join('');
|
|
|
|
|
}
|
|
|
|
|
|
2017-06-01 10:54:18 +00:00
|
|
|
function processCoreBundleFormat(fileHeader: string, languages: string[], json: BundledFormat, emitter: any) {
|
2016-03-08 10:26:06 +00:00
|
|
|
let keysSection = json.keys;
|
|
|
|
|
let messageSection = json.messages;
|
|
|
|
|
let bundleSection = json.bundles;
|
|
|
|
|
|
|
|
|
|
let statistics: Map<number> = Object.create(null);
|
|
|
|
|
|
|
|
|
|
let total: number = 0;
|
|
|
|
|
let defaultMessages: Map<Map<string>> = Object.create(null);
|
|
|
|
|
let modules = Object.keys(keysSection);
|
|
|
|
|
modules.forEach((module) => {
|
|
|
|
|
let keys = keysSection[module];
|
|
|
|
|
let messages = messageSection[module];
|
|
|
|
|
if (!messages || keys.length !== messages.length) {
|
|
|
|
|
emitter.emit('error', `Message for module ${module} corrupted. Mismatch in number of keys and messages.`);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
let messageMap: Map<string> = Object.create(null);
|
|
|
|
|
defaultMessages[module] = messageMap;
|
|
|
|
|
keys.map((key, i) => {
|
|
|
|
|
total++;
|
|
|
|
|
if (Is.string(key)) {
|
|
|
|
|
messageMap[key] = messages[i];
|
|
|
|
|
} else {
|
|
|
|
|
messageMap[key.key] = messages[i];
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
let languageDirectory = path.join(__dirname, '..', '..', 'i18n');
|
2017-06-01 10:54:18 +00:00
|
|
|
let languageDirs;
|
2017-06-01 13:13:31 +00:00
|
|
|
if (languages) {
|
2017-06-01 10:54:18 +00:00
|
|
|
languageDirs = sortLanguages(languages);
|
|
|
|
|
} else {
|
|
|
|
|
languageDirs = sortLanguages(fs.readdirSync(languageDirectory).filter((item) => fs.statSync(path.join(languageDirectory, item)).isDirectory()));
|
|
|
|
|
}
|
|
|
|
|
languageDirs.forEach((language) => {
|
2016-03-08 10:26:06 +00:00
|
|
|
if (!language.iso639_2) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
2016-07-19 09:37:20 +00:00
|
|
|
if (process.env['VSCODE_BUILD_VERBOSE']) {
|
|
|
|
|
log(`Generating nls bundles for: ${language.iso639_2}`);
|
|
|
|
|
}
|
|
|
|
|
|
2016-03-08 10:26:06 +00:00
|
|
|
statistics[language.iso639_2] = 0;
|
|
|
|
|
let localizedModules: Map<string[]> = Object.create(null);
|
|
|
|
|
let cwd = path.join(languageDirectory, language.name, 'src');
|
|
|
|
|
modules.forEach((module) => {
|
|
|
|
|
let order = keysSection[module];
|
|
|
|
|
let i18nFile = path.join(cwd, module) + '.i18n.json';
|
|
|
|
|
let messages: Map<string> = null;
|
|
|
|
|
if (fs.existsSync(i18nFile)) {
|
|
|
|
|
let content = stripComments(fs.readFileSync(i18nFile, 'utf8'));
|
|
|
|
|
messages = JSON.parse(content);
|
|
|
|
|
} else {
|
2016-07-19 09:37:20 +00:00
|
|
|
if (process.env['VSCODE_BUILD_VERBOSE']) {
|
|
|
|
|
log(`No localized messages found for module ${module}. Using default messages.`);
|
|
|
|
|
}
|
2016-03-08 10:26:06 +00:00
|
|
|
messages = defaultMessages[module];
|
|
|
|
|
statistics[language.iso639_2] = statistics[language.iso639_2] + Object.keys(messages).length;
|
|
|
|
|
}
|
|
|
|
|
let localizedMessages: string[] = [];
|
|
|
|
|
order.forEach((keyInfo) => {
|
|
|
|
|
let key: string = null;
|
|
|
|
|
if (Is.string(keyInfo)) {
|
|
|
|
|
key = keyInfo;
|
|
|
|
|
} else {
|
|
|
|
|
key = keyInfo.key;
|
|
|
|
|
}
|
|
|
|
|
let message: string = messages[key];
|
|
|
|
|
if (!message) {
|
2016-07-19 09:37:20 +00:00
|
|
|
if (process.env['VSCODE_BUILD_VERBOSE']) {
|
|
|
|
|
log(`No localized message found for key ${key} in module ${module}. Using default message.`);
|
|
|
|
|
}
|
2016-03-08 10:26:06 +00:00
|
|
|
message = defaultMessages[module][key];
|
|
|
|
|
statistics[language.iso639_2] = statistics[language.iso639_2] + 1;
|
|
|
|
|
}
|
|
|
|
|
localizedMessages.push(message);
|
|
|
|
|
});
|
|
|
|
|
localizedModules[module] = localizedMessages;
|
|
|
|
|
});
|
|
|
|
|
Object.keys(bundleSection).forEach((bundle) => {
|
|
|
|
|
let modules = bundleSection[bundle];
|
|
|
|
|
let contents: string[] = [
|
2016-05-31 14:29:52 +00:00
|
|
|
fileHeader,
|
2016-03-08 10:26:06 +00:00
|
|
|
`define("${bundle}.nls.${language.iso639_2}", {`
|
|
|
|
|
];
|
|
|
|
|
modules.forEach((module, index) => {
|
|
|
|
|
contents.push(`\t"${module}": [`);
|
|
|
|
|
let messages = localizedModules[module];
|
|
|
|
|
if (!messages) {
|
|
|
|
|
emitter.emit('error', `Didn't find messages for module ${module}.`);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
messages.forEach((message, index) => {
|
2017-05-15 13:59:14 +00:00
|
|
|
contents.push(`\t\t"${escapeCharacters(message)}${index < messages.length ? '",' : '"'}`);
|
2016-03-08 10:26:06 +00:00
|
|
|
});
|
|
|
|
|
contents.push(index < modules.length - 1 ? '\t],' : '\t]');
|
|
|
|
|
});
|
|
|
|
|
contents.push('});');
|
2017-05-15 13:59:14 +00:00
|
|
|
emitter.emit('data', new File({ path: bundle + '.nls.' + language.iso639_2 + '.js', contents: new Buffer(contents.join('\n'), 'utf-8') }));
|
2016-03-08 10:26:06 +00:00
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
Object.keys(statistics).forEach(key => {
|
|
|
|
|
let value = statistics[key];
|
2016-07-19 09:37:20 +00:00
|
|
|
log(`${key} has ${value} untranslated strings.`);
|
2016-03-08 10:26:06 +00:00
|
|
|
});
|
2017-06-01 12:29:56 +00:00
|
|
|
languageDirs.forEach(dir => {
|
|
|
|
|
const language = dir.name;
|
2016-03-08 10:26:06 +00:00
|
|
|
let iso639_2 = iso639_3_to_2[language];
|
|
|
|
|
if (!iso639_2) {
|
|
|
|
|
log(`\tCouldn't find iso639 2 mapping for language ${language}. Using default language instead.`);
|
|
|
|
|
} else {
|
|
|
|
|
let stats = statistics[iso639_2];
|
|
|
|
|
if (Is.undef(stats)) {
|
2016-07-19 09:37:20 +00:00
|
|
|
log(`\tNo translations found for language ${language}. Using default language instead.`);
|
2016-03-08 10:26:06 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
2017-06-01 10:54:18 +00:00
|
|
|
export function processNlsFiles(opts: { fileHeader: string; languages: string[] }): ThroughStream {
|
2017-05-15 13:59:14 +00:00
|
|
|
return through(function (file: File) {
|
2016-03-08 10:26:06 +00:00
|
|
|
let fileName = path.basename(file.path);
|
|
|
|
|
if (fileName === 'nls.metadata.json') {
|
|
|
|
|
let json = null;
|
|
|
|
|
if (file.isBuffer()) {
|
2016-10-20 10:39:08 +00:00
|
|
|
json = JSON.parse((<Buffer>file.contents).toString('utf8'));
|
2016-03-08 10:26:06 +00:00
|
|
|
} else {
|
2016-07-19 09:37:20 +00:00
|
|
|
this.emit('error', `Failed to read component file: ${file.relative}`);
|
2016-03-08 10:26:06 +00:00
|
|
|
}
|
|
|
|
|
if (BundledFormat.is(json)) {
|
2017-06-01 10:54:18 +00:00
|
|
|
processCoreBundleFormat(opts.fileHeader, opts.languages, json, this);
|
2016-03-08 10:26:06 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
this.emit('data', file);
|
|
|
|
|
});
|
2017-03-02 20:53:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function prepareXlfFiles(projectName?: string, extensionName?: string): ThroughStream {
|
|
|
|
|
return through(
|
|
|
|
|
function (file: File) {
|
|
|
|
|
if (!file.isBuffer()) {
|
2017-04-06 07:45:42 +00:00
|
|
|
throw new Error(`Failed to read component file: ${file.relative}`);
|
2017-03-02 20:53:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const extension = path.extname(file.path);
|
|
|
|
|
if (extension === '.json') {
|
|
|
|
|
const json = JSON.parse((<Buffer>file.contents).toString('utf8'));
|
|
|
|
|
|
|
|
|
|
if (BundledFormat.is(json)) {
|
|
|
|
|
importBundleJson(file, json, this);
|
|
|
|
|
} else if (PackageJsonFormat.is(json) || ModuleJsonFormat.is(json)) {
|
|
|
|
|
importModuleOrPackageJson(file, json, projectName, this, extensionName);
|
|
|
|
|
} else {
|
2017-04-06 07:45:42 +00:00
|
|
|
throw new Error(`JSON format cannot be deduced for ${file.relative}.`);
|
2017-03-02 20:53:02 +00:00
|
|
|
}
|
|
|
|
|
} else if (extension === '.isl') {
|
|
|
|
|
importIsl(file, this);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
2017-04-05 13:58:07 +00:00
|
|
|
const editorProject: string = 'vscode-editor',
|
2017-04-05 16:12:24 +00:00
|
|
|
workbenchProject: string = 'vscode-workbench',
|
2017-04-06 07:45:42 +00:00
|
|
|
extensionsProject: string = 'vscode-extensions',
|
2017-04-05 16:12:24 +00:00
|
|
|
setupProject: string = 'vscode-setup';
|
2017-04-05 13:58:07 +00:00
|
|
|
|
2017-04-05 14:22:22 +00:00
|
|
|
export function getResource(sourceFile: string): Resource {
|
2017-03-02 20:53:02 +00:00
|
|
|
let resource: string;
|
|
|
|
|
|
2017-05-15 13:59:14 +00:00
|
|
|
if (/^vs\/platform/.test(sourceFile)) {
|
2017-03-02 20:53:02 +00:00
|
|
|
return { name: 'vs/platform', project: editorProject };
|
2017-05-15 13:59:14 +00:00
|
|
|
} else if (/^vs\/editor\/contrib/.test(sourceFile)) {
|
2017-03-02 20:53:02 +00:00
|
|
|
return { name: 'vs/editor/contrib', project: editorProject };
|
2017-05-15 13:59:14 +00:00
|
|
|
} else if (/^vs\/editor/.test(sourceFile)) {
|
2017-03-02 20:53:02 +00:00
|
|
|
return { name: 'vs/editor', project: editorProject };
|
2017-05-15 13:59:14 +00:00
|
|
|
} else if (/^vs\/base/.test(sourceFile)) {
|
2017-03-02 20:53:02 +00:00
|
|
|
return { name: 'vs/base', project: editorProject };
|
2017-05-15 13:59:14 +00:00
|
|
|
} else if (/^vs\/code/.test(sourceFile)) {
|
2017-03-02 20:53:02 +00:00
|
|
|
return { name: 'vs/code', project: workbenchProject };
|
2017-05-15 13:59:14 +00:00
|
|
|
} else if (/^vs\/workbench\/parts/.test(sourceFile)) {
|
2017-03-02 20:53:02 +00:00
|
|
|
resource = sourceFile.split('/', 4).join('/');
|
|
|
|
|
return { name: resource, project: workbenchProject };
|
2017-05-15 13:59:14 +00:00
|
|
|
} else if (/^vs\/workbench\/services/.test(sourceFile)) {
|
2017-03-02 20:53:02 +00:00
|
|
|
resource = sourceFile.split('/', 4).join('/');
|
|
|
|
|
return { name: resource, project: workbenchProject };
|
2017-05-15 13:59:14 +00:00
|
|
|
} else if (/^vs\/workbench/.test(sourceFile)) {
|
2017-03-02 20:53:02 +00:00
|
|
|
return { name: 'vs/workbench', project: workbenchProject };
|
|
|
|
|
}
|
|
|
|
|
|
2017-05-15 13:59:14 +00:00
|
|
|
throw new Error(`Could not identify the XLF bundle for ${sourceFile}`);
|
2017-03-02 20:53:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
function importBundleJson(file: File, json: BundledFormat, stream: ThroughStream): void {
|
2017-03-17 16:24:18 +00:00
|
|
|
let bundleXlfs: Map<XLF> = Object.create(null);
|
2017-03-02 20:53:02 +00:00
|
|
|
|
|
|
|
|
for (let source in json.keys) {
|
|
|
|
|
const projectResource = getResource(source);
|
|
|
|
|
const resource = projectResource.name;
|
|
|
|
|
const project = projectResource.project;
|
|
|
|
|
|
|
|
|
|
const keys = json.keys[source];
|
|
|
|
|
const messages = json.messages[source];
|
|
|
|
|
if (keys.length !== messages.length) {
|
2017-04-05 09:58:31 +00:00
|
|
|
throw new Error(`There is a mismatch between keys and messages in ${file.relative}`);
|
2017-03-02 20:53:02 +00:00
|
|
|
}
|
|
|
|
|
|
2017-03-17 16:24:18 +00:00
|
|
|
let xlf = bundleXlfs[resource] ? bundleXlfs[resource] : bundleXlfs[resource] = new XLF(project);
|
2017-03-31 13:24:26 +00:00
|
|
|
xlf.addFile('src/' + source, keys, messages);
|
2017-03-02 20:53:02 +00:00
|
|
|
}
|
|
|
|
|
|
2017-03-17 16:24:18 +00:00
|
|
|
for (let resource in bundleXlfs) {
|
|
|
|
|
const newFilePath = `${bundleXlfs[resource].project}/${resource.replace(/\//g, '_')}.xlf`;
|
2017-05-15 13:59:14 +00:00
|
|
|
const xlfFile = new File({ path: newFilePath, contents: new Buffer(bundleXlfs[resource].toString(), 'utf-8') });
|
2017-03-02 20:53:02 +00:00
|
|
|
stream.emit('data', xlfFile);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Keeps existing XLF instances and a state of how many files were already processed for faster file emission
|
|
|
|
|
var extensions: Map<{ xlf: XLF, processed: number }> = Object.create(null);
|
|
|
|
|
function importModuleOrPackageJson(file: File, json: ModuleJsonFormat | PackageJsonFormat, projectName: string, stream: ThroughStream, extensionName?: string): void {
|
|
|
|
|
if (ModuleJsonFormat.is(json) && json['keys'].length !== json['messages'].length) {
|
2017-04-05 09:58:31 +00:00
|
|
|
throw new Error(`There is a mismatch between keys and messages in ${file.relative}`);
|
2017-03-02 20:53:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Prepare the source path for <original/> attribute in XLF & extract messages from JSON
|
|
|
|
|
const formattedSourcePath = file.relative.replace(/\\/g, '/');
|
|
|
|
|
const messages = Object.keys(json).map((key) => json[key].toString());
|
|
|
|
|
|
|
|
|
|
// Stores the amount of localization files to be transformed to XLF before the emission
|
|
|
|
|
let localizationFilesCount,
|
|
|
|
|
originalFilePath;
|
|
|
|
|
// If preparing XLF for external extension, then use different glob pattern and source path
|
|
|
|
|
if (extensionName) {
|
|
|
|
|
localizationFilesCount = glob.sync('**/*.nls.json').length;
|
|
|
|
|
originalFilePath = `${formattedSourcePath.substr(0, formattedSourcePath.length - '.nls.json'.length)}`;
|
|
|
|
|
} else {
|
|
|
|
|
// Used for vscode/extensions folder
|
|
|
|
|
extensionName = formattedSourcePath.split('/')[0];
|
|
|
|
|
localizationFilesCount = glob.sync(`./extensions/${extensionName}/**/*.nls.json`).length;
|
|
|
|
|
originalFilePath = `extensions/${formattedSourcePath.substr(0, formattedSourcePath.length - '.nls.json'.length)}`;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
let extension = extensions[extensionName] ?
|
|
|
|
|
extensions[extensionName] : extensions[extensionName] = { xlf: new XLF(projectName), processed: 0 };
|
|
|
|
|
|
2017-06-30 09:42:52 +00:00
|
|
|
// .nls.json can come with empty array of keys and messages, check for it
|
|
|
|
|
if (ModuleJsonFormat.is(json) && json.keys.length !== 0) {
|
|
|
|
|
extension.xlf.addFile(originalFilePath, json.keys, json.messages);
|
|
|
|
|
} else if (PackageJsonFormat.is(json) && Object.keys(json).length !== 0) {
|
2017-03-02 20:53:02 +00:00
|
|
|
extension.xlf.addFile(originalFilePath, Object.keys(json), messages);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Check if XLF is populated with file nodes to emit it
|
|
|
|
|
if (++extensions[extensionName].processed === localizationFilesCount) {
|
|
|
|
|
const newFilePath = path.join(projectName, extensionName + '.xlf');
|
2017-05-15 13:59:14 +00:00
|
|
|
const xlfFile = new File({ path: newFilePath, contents: new Buffer(extension.xlf.toString(), 'utf-8') });
|
2017-03-02 20:53:02 +00:00
|
|
|
stream.emit('data', xlfFile);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function importIsl(file: File, stream: ThroughStream) {
|
2017-04-05 16:12:24 +00:00
|
|
|
let projectName: string,
|
|
|
|
|
resourceFile: string;
|
|
|
|
|
if (path.basename(file.path) === 'Default.isl') {
|
|
|
|
|
projectName = setupProject;
|
|
|
|
|
resourceFile = 'setup_default.xlf';
|
|
|
|
|
} else {
|
|
|
|
|
projectName = workbenchProject;
|
|
|
|
|
resourceFile = 'setup_messages.xlf';
|
|
|
|
|
}
|
2017-03-02 20:53:02 +00:00
|
|
|
|
2017-04-05 16:12:24 +00:00
|
|
|
let xlf = new XLF(projectName),
|
2017-03-02 20:53:02 +00:00
|
|
|
keys: string[] = [],
|
|
|
|
|
messages: string[] = [];
|
|
|
|
|
|
|
|
|
|
let model = new TextModel(file.contents.toString());
|
|
|
|
|
let inMessageSection = false;
|
|
|
|
|
model.lines.forEach(line => {
|
|
|
|
|
if (line.length === 0) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
let firstChar = line.charAt(0);
|
|
|
|
|
switch (firstChar) {
|
|
|
|
|
case ';':
|
|
|
|
|
// Comment line;
|
|
|
|
|
return;
|
|
|
|
|
case '[':
|
|
|
|
|
inMessageSection = '[Messages]' === line || '[CustomMessages]' === line;
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
if (!inMessageSection) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
let sections: string[] = line.split('=');
|
|
|
|
|
if (sections.length !== 2) {
|
2017-04-05 09:58:31 +00:00
|
|
|
throw new Error(`Badly formatted message found: ${line}`);
|
2017-03-02 20:53:02 +00:00
|
|
|
} else {
|
|
|
|
|
let key = sections[0];
|
|
|
|
|
let value = sections[1];
|
|
|
|
|
if (key.length > 0 && value.length > 0) {
|
|
|
|
|
keys.push(key);
|
|
|
|
|
messages.push(value);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
2017-05-15 13:59:14 +00:00
|
|
|
const originalPath = file.path.substring(file.cwd.length + 1, file.path.split('.')[0].length).replace(/\\/g, '/');
|
2017-03-02 20:53:02 +00:00
|
|
|
xlf.addFile(originalPath, keys, messages);
|
|
|
|
|
|
|
|
|
|
// Emit only upon all ISL files combined into single XLF instance
|
2017-04-05 16:12:24 +00:00
|
|
|
const newFilePath = path.join(projectName, resourceFile);
|
2017-05-15 13:59:14 +00:00
|
|
|
const xlfFile = new File({ path: newFilePath, contents: new Buffer(xlf.toString(), 'utf-8') });
|
2017-04-05 16:12:24 +00:00
|
|
|
stream.emit('data', xlfFile);
|
2017-03-02 20:53:02 +00:00
|
|
|
}
|
|
|
|
|
|
2017-03-20 14:01:18 +00:00
|
|
|
export function pushXlfFiles(apiHostname: string, username: string, password: string): ThroughStream {
|
2017-03-17 16:24:18 +00:00
|
|
|
let tryGetPromises = [];
|
|
|
|
|
let updateCreatePromises = [];
|
|
|
|
|
|
2017-05-15 13:59:14 +00:00
|
|
|
return through(function (file: File) {
|
2017-03-02 20:53:02 +00:00
|
|
|
const project = path.dirname(file.relative);
|
|
|
|
|
const fileName = path.basename(file.path);
|
|
|
|
|
const slug = fileName.substr(0, fileName.length - '.xlf'.length);
|
2017-03-20 14:01:18 +00:00
|
|
|
const credentials = `${username}:${password}`;
|
2017-03-02 20:53:02 +00:00
|
|
|
|
|
|
|
|
// Check if resource already exists, if not, then create it.
|
2017-03-20 14:01:18 +00:00
|
|
|
let promise = tryGetResource(project, slug, apiHostname, credentials);
|
2017-03-17 16:24:18 +00:00
|
|
|
tryGetPromises.push(promise);
|
|
|
|
|
promise.then(exists => {
|
2017-03-02 20:53:02 +00:00
|
|
|
if (exists) {
|
2017-03-20 14:01:18 +00:00
|
|
|
promise = updateResource(project, slug, file, apiHostname, credentials);
|
2017-03-02 20:53:02 +00:00
|
|
|
} else {
|
2017-03-20 14:01:18 +00:00
|
|
|
promise = createResource(project, slug, file, apiHostname, credentials);
|
2017-03-02 20:53:02 +00:00
|
|
|
}
|
2017-03-17 16:24:18 +00:00
|
|
|
updateCreatePromises.push(promise);
|
|
|
|
|
});
|
|
|
|
|
|
2017-05-15 13:59:14 +00:00
|
|
|
}, function () {
|
2017-03-17 16:24:18 +00:00
|
|
|
// End the pipe only after all the communication with Transifex API happened
|
|
|
|
|
Promise.all(tryGetPromises).then(() => {
|
|
|
|
|
Promise.all(updateCreatePromises).then(() => {
|
|
|
|
|
this.emit('end');
|
2017-04-05 09:58:31 +00:00
|
|
|
}).catch((reason) => { throw new Error(reason); });
|
|
|
|
|
}).catch((reason) => { throw new Error(reason); });
|
2017-03-02 20:53:02 +00:00
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
2017-03-20 14:01:18 +00:00
|
|
|
function tryGetResource(project: string, slug: string, apiHostname: string, credentials: string): Promise<boolean> {
|
2017-03-02 20:53:02 +00:00
|
|
|
return new Promise((resolve, reject) => {
|
2017-03-20 14:01:18 +00:00
|
|
|
const options = {
|
|
|
|
|
hostname: apiHostname,
|
|
|
|
|
path: `/api/2/project/${project}/resource/${slug}/?details`,
|
|
|
|
|
auth: credentials,
|
|
|
|
|
method: 'GET'
|
|
|
|
|
};
|
|
|
|
|
|
2017-06-30 09:42:52 +00:00
|
|
|
const request = https.request(options, (response) => {
|
2017-03-02 20:53:02 +00:00
|
|
|
if (response.statusCode === 404) {
|
|
|
|
|
resolve(false);
|
|
|
|
|
} else if (response.statusCode === 200) {
|
|
|
|
|
resolve(true);
|
|
|
|
|
} else {
|
2017-03-20 14:01:18 +00:00
|
|
|
reject(`Failed to query resource ${project}/${slug}. Response: ${response.statusCode} ${response.statusMessage}`);
|
2017-03-02 20:53:02 +00:00
|
|
|
}
|
2017-04-05 09:58:31 +00:00
|
|
|
});
|
|
|
|
|
request.on('error', (err) => {
|
2017-03-20 14:01:18 +00:00
|
|
|
reject(`Failed to get ${project}/${slug} on Transifex: ${err}`);
|
2017-03-02 20:53:02 +00:00
|
|
|
});
|
2017-03-20 14:01:18 +00:00
|
|
|
|
|
|
|
|
request.end();
|
2017-03-02 20:53:02 +00:00
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
2017-03-20 14:01:18 +00:00
|
|
|
function createResource(project: string, slug: string, xlfFile: File, apiHostname: string, credentials: any): Promise<any> {
|
|
|
|
|
return new Promise((resolve, reject) => {
|
|
|
|
|
const data = JSON.stringify({
|
|
|
|
|
'content': xlfFile.contents.toString(),
|
|
|
|
|
'name': slug,
|
|
|
|
|
'slug': slug,
|
|
|
|
|
'i18n_type': 'XLIFF'
|
|
|
|
|
});
|
2017-03-17 16:24:18 +00:00
|
|
|
const options = {
|
2017-03-20 14:01:18 +00:00
|
|
|
hostname: apiHostname,
|
|
|
|
|
path: `/api/2/project/${project}/resources`,
|
|
|
|
|
headers: {
|
|
|
|
|
'Content-Type': 'application/json',
|
|
|
|
|
'Content-Length': Buffer.byteLength(data)
|
2017-03-17 16:24:18 +00:00
|
|
|
},
|
2017-03-20 14:01:18 +00:00
|
|
|
auth: credentials,
|
|
|
|
|
method: 'POST'
|
2017-03-17 16:24:18 +00:00
|
|
|
};
|
2017-03-02 20:53:02 +00:00
|
|
|
|
2017-06-30 09:42:52 +00:00
|
|
|
let request = https.request(options, (res) => {
|
2017-03-17 16:24:18 +00:00
|
|
|
if (res.statusCode === 201) {
|
|
|
|
|
log(`Resource ${project}/${slug} successfully created on Transifex.`);
|
|
|
|
|
} else {
|
2017-03-20 14:01:18 +00:00
|
|
|
reject(`Something went wrong in the request creating ${slug} in ${project}. ${res.statusCode}`);
|
2017-03-17 16:24:18 +00:00
|
|
|
}
|
2017-04-05 09:58:31 +00:00
|
|
|
});
|
|
|
|
|
request.on('error', (err) => {
|
2017-03-20 14:01:18 +00:00
|
|
|
reject(`Failed to create ${project}/${slug} on Transifex: ${err}`);
|
2017-03-17 16:24:18 +00:00
|
|
|
});
|
2017-03-20 14:01:18 +00:00
|
|
|
|
|
|
|
|
request.write(data);
|
|
|
|
|
request.end();
|
2017-03-02 20:53:02 +00:00
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* The following link provides information about how Transifex handles updates of a resource file:
|
|
|
|
|
* https://dev.befoolish.co/tx-docs/public/projects/updating-content#what-happens-when-you-update-files
|
|
|
|
|
*/
|
2017-05-15 13:59:14 +00:00
|
|
|
function updateResource(project: string, slug: string, xlfFile: File, apiHostname: string, credentials: string): Promise<any> {
|
2017-03-20 14:01:18 +00:00
|
|
|
return new Promise((resolve, reject) => {
|
|
|
|
|
const data = JSON.stringify({ content: xlfFile.contents.toString() });
|
2017-03-17 16:24:18 +00:00
|
|
|
const options = {
|
2017-03-20 14:01:18 +00:00
|
|
|
hostname: apiHostname,
|
|
|
|
|
path: `/api/2/project/${project}/resource/${slug}/content`,
|
|
|
|
|
headers: {
|
|
|
|
|
'Content-Type': 'application/json',
|
|
|
|
|
'Content-Length': Buffer.byteLength(data)
|
|
|
|
|
},
|
|
|
|
|
auth: credentials,
|
|
|
|
|
method: 'PUT'
|
2017-03-17 16:24:18 +00:00
|
|
|
};
|
2017-03-02 20:53:02 +00:00
|
|
|
|
2017-06-30 09:42:52 +00:00
|
|
|
let request = https.request(options, (res) => {
|
2017-03-17 16:24:18 +00:00
|
|
|
if (res.statusCode === 200) {
|
2017-03-20 14:01:18 +00:00
|
|
|
res.setEncoding('utf8');
|
|
|
|
|
|
|
|
|
|
let responseBuffer: string = '';
|
|
|
|
|
res.on('data', function (chunk) {
|
|
|
|
|
responseBuffer += chunk;
|
|
|
|
|
});
|
|
|
|
|
res.on('end', () => {
|
|
|
|
|
const response = JSON.parse(responseBuffer);
|
|
|
|
|
log(`Resource ${project}/${slug} successfully updated on Transifex. Strings added: ${response.strings_added}, updated: ${response.strings_added}, deleted: ${response.strings_added}`);
|
|
|
|
|
resolve();
|
|
|
|
|
});
|
2017-03-17 16:24:18 +00:00
|
|
|
} else {
|
2017-03-20 14:01:18 +00:00
|
|
|
reject(`Something went wrong in the request updating ${slug} in ${project}. ${res.statusCode}`);
|
2017-03-17 16:24:18 +00:00
|
|
|
}
|
2017-04-05 09:58:31 +00:00
|
|
|
});
|
|
|
|
|
request.on('error', (err) => {
|
2017-03-20 14:01:18 +00:00
|
|
|
reject(`Failed to update ${project}/${slug} on Transifex: ${err}`);
|
2017-03-17 16:24:18 +00:00
|
|
|
});
|
2017-03-20 14:01:18 +00:00
|
|
|
|
|
|
|
|
request.write(data);
|
|
|
|
|
request.end();
|
2017-03-02 20:53:02 +00:00
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function obtainProjectResources(projectName: string): Resource[] {
|
2017-04-05 16:12:24 +00:00
|
|
|
let resources: Resource[] = [];
|
2017-03-02 20:53:02 +00:00
|
|
|
|
2017-04-06 07:45:42 +00:00
|
|
|
if (projectName === editorProject) {
|
2017-06-09 12:43:30 +00:00
|
|
|
const json = fs.readFileSync('./build/lib/i18n.resources.json', 'utf8');
|
|
|
|
|
resources = JSON.parse(json).editor;
|
2017-04-06 07:45:42 +00:00
|
|
|
} else if (projectName === workbenchProject) {
|
2017-06-09 12:43:30 +00:00
|
|
|
const json = fs.readFileSync('./build/lib/i18n.resources.json', 'utf8');
|
|
|
|
|
resources = JSON.parse(json).workbench;
|
2017-04-06 07:45:42 +00:00
|
|
|
} else if (projectName === extensionsProject) {
|
2017-03-02 20:53:02 +00:00
|
|
|
let extensionsToLocalize: string[] = glob.sync('./extensions/**/*.nls.json').map(extension => extension.split('/')[2]);
|
|
|
|
|
let resourcesToPull: string[] = [];
|
|
|
|
|
|
|
|
|
|
extensionsToLocalize.forEach(extension => {
|
|
|
|
|
if (resourcesToPull.indexOf(extension) === -1) { // remove duplicate elements returned by glob
|
|
|
|
|
resourcesToPull.push(extension);
|
|
|
|
|
resources.push({ name: extension, project: projectName });
|
|
|
|
|
}
|
|
|
|
|
});
|
2017-04-06 07:45:42 +00:00
|
|
|
} else if (projectName === setupProject) {
|
2017-04-05 16:12:24 +00:00
|
|
|
resources.push({ name: 'setup_default', project: setupProject });
|
2017-03-02 20:53:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return resources;
|
|
|
|
|
}
|
|
|
|
|
|
2017-04-05 16:12:24 +00:00
|
|
|
export function pullXlfFiles(projectName: string, apiHostname: string, username: string, password: string, languages: string[], resources?: Resource[]): NodeJS.ReadableStream {
|
2017-03-02 20:53:02 +00:00
|
|
|
if (!resources) {
|
|
|
|
|
resources = obtainProjectResources(projectName);
|
|
|
|
|
}
|
|
|
|
|
if (!resources) {
|
|
|
|
|
throw new Error('Transifex projects and resources must be defined to be able to pull translations from Transifex.');
|
|
|
|
|
}
|
|
|
|
|
|
2017-03-20 14:01:18 +00:00
|
|
|
const credentials = `${username}:${password}`;
|
2017-04-05 16:12:24 +00:00
|
|
|
let expectedTranslationsCount = languages.length * resources.length;
|
2017-03-02 20:53:02 +00:00
|
|
|
let translationsRetrieved = 0, called = false;
|
|
|
|
|
|
2017-05-15 13:59:14 +00:00
|
|
|
return readable(function (count, callback) {
|
2017-03-02 20:53:02 +00:00
|
|
|
// Mark end of stream when all resources were retrieved
|
|
|
|
|
if (translationsRetrieved === expectedTranslationsCount) {
|
|
|
|
|
return this.emit('end');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (!called) {
|
|
|
|
|
called = true;
|
|
|
|
|
const stream = this;
|
|
|
|
|
|
2017-04-05 16:12:24 +00:00
|
|
|
// Retrieve XLF files from main projects
|
2017-05-15 13:59:14 +00:00
|
|
|
languages.map(function (language) {
|
|
|
|
|
resources.map(function (resource) {
|
2017-04-05 16:12:24 +00:00
|
|
|
retrieveResource(language, resource, apiHostname, credentials).then((file: File) => {
|
|
|
|
|
stream.emit('data', file);
|
|
|
|
|
translationsRetrieved++;
|
|
|
|
|
}).catch(error => { throw new Error(error); });
|
2017-03-02 20:53:02 +00:00
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
callback();
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
2017-04-05 16:12:24 +00:00
|
|
|
function retrieveResource(language: string, resource: Resource, apiHostname, credentials): Promise<File> {
|
|
|
|
|
return new Promise<File>((resolve, reject) => {
|
|
|
|
|
const slug = resource.name.replace(/\//g, '_');
|
|
|
|
|
const project = resource.project;
|
2017-04-06 14:37:52 +00:00
|
|
|
const iso639 = language.toLowerCase();
|
2017-04-05 16:12:24 +00:00
|
|
|
const options = {
|
|
|
|
|
hostname: apiHostname,
|
|
|
|
|
path: `/api/2/project/${project}/resource/${slug}/translation/${iso639}?file&mode=onlyreviewed`,
|
|
|
|
|
auth: credentials,
|
|
|
|
|
method: 'GET'
|
|
|
|
|
};
|
|
|
|
|
|
2017-06-30 09:42:52 +00:00
|
|
|
let request = https.request(options, (res) => {
|
2017-05-15 13:59:14 +00:00
|
|
|
let xlfBuffer: Buffer[] = [];
|
|
|
|
|
res.on('data', (chunk: Buffer) => xlfBuffer.push(chunk));
|
|
|
|
|
res.on('end', () => {
|
|
|
|
|
if (res.statusCode === 200) {
|
|
|
|
|
resolve(new File({ contents: Buffer.concat(xlfBuffer), path: `${project}/${iso639_2_to_3[language]}/${slug}.xlf` }));
|
|
|
|
|
}
|
|
|
|
|
reject(`${slug} in ${project} returned no data. Response code: ${res.statusCode}.`);
|
|
|
|
|
});
|
2017-04-05 16:12:24 +00:00
|
|
|
});
|
|
|
|
|
request.on('error', (err) => {
|
|
|
|
|
reject(`Failed to query resource ${slug} with the following error: ${err}`);
|
|
|
|
|
});
|
|
|
|
|
request.end();
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
2017-03-02 20:53:02 +00:00
|
|
|
export function prepareJsonFiles(): ThroughStream {
|
2017-04-13 16:19:51 +00:00
|
|
|
let parsePromises: Promise<ParsedXLF[]>[] = [];
|
|
|
|
|
|
2017-05-15 13:59:14 +00:00
|
|
|
return through(function (xlf: File) {
|
2017-03-02 20:53:02 +00:00
|
|
|
let stream = this;
|
2017-04-13 16:19:51 +00:00
|
|
|
let parsePromise = XLF.parse(xlf.contents.toString());
|
|
|
|
|
parsePromises.push(parsePromise);
|
|
|
|
|
parsePromise.then(
|
2017-05-15 13:59:14 +00:00
|
|
|
function (resolvedFiles) {
|
2017-03-02 20:53:02 +00:00
|
|
|
resolvedFiles.forEach(file => {
|
|
|
|
|
let messages = file.messages, translatedFile;
|
|
|
|
|
|
|
|
|
|
// ISL file path always starts with 'build/'
|
2017-05-15 13:59:14 +00:00
|
|
|
if (/^build\//.test(file.originalFilePath)) {
|
2017-04-06 14:37:52 +00:00
|
|
|
const defaultLanguages = { 'zh-hans': true, 'zh-hant': true, 'ko': true };
|
2017-03-02 20:53:02 +00:00
|
|
|
if (path.basename(file.originalFilePath) === 'Default' && !defaultLanguages[file.language]) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
translatedFile = createIslFile('..', file.originalFilePath, messages, iso639_2_to_3[file.language]);
|
|
|
|
|
} else {
|
|
|
|
|
translatedFile = createI18nFile(iso639_2_to_3[file.language], file.originalFilePath, messages);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
stream.emit('data', translatedFile);
|
|
|
|
|
});
|
|
|
|
|
},
|
2017-05-15 13:59:14 +00:00
|
|
|
function (rejectReason) {
|
2017-04-05 09:58:31 +00:00
|
|
|
throw new Error(`XLF parsing error: ${rejectReason}`);
|
2017-03-02 20:53:02 +00:00
|
|
|
}
|
|
|
|
|
);
|
2017-05-15 13:59:14 +00:00
|
|
|
}, function () {
|
2017-04-13 16:19:51 +00:00
|
|
|
Promise.all(parsePromises)
|
|
|
|
|
.then(() => { this.emit('end'); })
|
|
|
|
|
.catch(reason => { throw new Error(reason); });
|
2017-03-02 20:53:02 +00:00
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
2017-04-13 16:19:51 +00:00
|
|
|
function createI18nFile(base: string, originalFilePath: string, messages: Map<string>): File {
|
2017-03-02 20:53:02 +00:00
|
|
|
let content = [
|
|
|
|
|
'/*---------------------------------------------------------------------------------------------',
|
|
|
|
|
' * Copyright (c) Microsoft Corporation. All rights reserved.',
|
|
|
|
|
' * Licensed under the MIT License. See License.txt in the project root for license information.',
|
|
|
|
|
' *--------------------------------------------------------------------------------------------*/',
|
|
|
|
|
'// Do not edit this file. It is machine generated.'
|
|
|
|
|
].join('\n') + '\n' + JSON.stringify(messages, null, '\t').replace(/\r\n/g, '\n');
|
|
|
|
|
|
|
|
|
|
return new File({
|
|
|
|
|
path: path.join(base, originalFilePath + '.i18n.json'),
|
|
|
|
|
contents: new Buffer(content, 'utf8')
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
const languageNames: Map<string> = {
|
|
|
|
|
'chs': 'Simplified Chinese',
|
|
|
|
|
'cht': 'Traditional Chinese',
|
|
|
|
|
'kor': 'Korean'
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
const languageIds: Map<string> = {
|
|
|
|
|
'chs': '$0804',
|
|
|
|
|
'cht': '$0404',
|
|
|
|
|
'kor': '$0412'
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
const encodings: Map<string> = {
|
|
|
|
|
'chs': 'CP936',
|
|
|
|
|
'cht': 'CP950',
|
|
|
|
|
'jpn': 'CP932',
|
|
|
|
|
'kor': 'CP949',
|
|
|
|
|
'deu': 'CP1252',
|
|
|
|
|
'fra': 'CP1252',
|
|
|
|
|
'esn': 'CP1252',
|
|
|
|
|
'rus': 'CP1251',
|
2017-05-24 08:06:44 +00:00
|
|
|
'ita': 'CP1252',
|
2017-06-09 14:15:52 +00:00
|
|
|
'ptb': 'CP1252',
|
|
|
|
|
'hun': 'CP1250',
|
|
|
|
|
'trk': 'CP1254'
|
2017-03-02 20:53:02 +00:00
|
|
|
};
|
|
|
|
|
|
2017-04-13 16:19:51 +00:00
|
|
|
function createIslFile(base: string, originalFilePath: string, messages: Map<string>, language: string): File {
|
2017-03-02 20:53:02 +00:00
|
|
|
let content: string[] = [];
|
|
|
|
|
let originalContent: TextModel;
|
|
|
|
|
if (path.basename(originalFilePath) === 'Default') {
|
|
|
|
|
originalContent = new TextModel(fs.readFileSync(originalFilePath + '.isl', 'utf8'));
|
|
|
|
|
} else {
|
|
|
|
|
originalContent = new TextModel(fs.readFileSync(originalFilePath + '.en.isl', 'utf8'));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
originalContent.lines.forEach(line => {
|
|
|
|
|
if (line.length > 0) {
|
|
|
|
|
let firstChar = line.charAt(0);
|
|
|
|
|
if (firstChar === '[' || firstChar === ';') {
|
|
|
|
|
if (line === '; *** Inno Setup version 5.5.3+ English messages ***') {
|
|
|
|
|
content.push(`; *** Inno Setup version 5.5.3+ ${languageNames[language]} messages ***`);
|
|
|
|
|
} else {
|
|
|
|
|
content.push(line);
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
let sections: string[] = line.split('=');
|
|
|
|
|
let key = sections[0];
|
|
|
|
|
let translated = line;
|
|
|
|
|
if (key) {
|
|
|
|
|
if (key === 'LanguageName') {
|
|
|
|
|
translated = `${key}=${languageNames[language]}`;
|
|
|
|
|
} else if (key === 'LanguageID') {
|
|
|
|
|
translated = `${key}=${languageIds[language]}`;
|
|
|
|
|
} else if (key === 'LanguageCodePage') {
|
|
|
|
|
translated = `${key}=${encodings[language].substr(2)}`;
|
|
|
|
|
} else {
|
|
|
|
|
let translatedMessage = messages[key];
|
|
|
|
|
if (translatedMessage) {
|
|
|
|
|
translated = `${key}=${translatedMessage}`;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
content.push(translated);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
2017-04-13 16:19:51 +00:00
|
|
|
const tag = iso639_3_to_2[language];
|
|
|
|
|
const basename = path.basename(originalFilePath);
|
|
|
|
|
const filePath = `${path.join(base, path.dirname(originalFilePath), basename)}.${tag}.isl`;
|
2017-03-02 20:53:02 +00:00
|
|
|
|
|
|
|
|
return new File({
|
|
|
|
|
path: filePath,
|
2017-04-13 17:58:13 +00:00
|
|
|
contents: iconv.encode(new Buffer(content.join('\r\n'), 'utf8'), encodings[language])
|
2017-03-02 20:53:02 +00:00
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function encodeEntities(value: string): string {
|
|
|
|
|
var result: string[] = [];
|
|
|
|
|
for (var i = 0; i < value.length; i++) {
|
|
|
|
|
var ch = value[i];
|
|
|
|
|
switch (ch) {
|
|
|
|
|
case '<':
|
|
|
|
|
result.push('<');
|
|
|
|
|
break;
|
|
|
|
|
case '>':
|
|
|
|
|
result.push('>');
|
|
|
|
|
break;
|
|
|
|
|
case '&':
|
|
|
|
|
result.push('&');
|
|
|
|
|
break;
|
|
|
|
|
default:
|
|
|
|
|
result.push(ch);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return result.join('');
|
|
|
|
|
}
|
|
|
|
|
|
2017-05-15 13:59:14 +00:00
|
|
|
function decodeEntities(value: string): string {
|
2017-03-02 20:53:02 +00:00
|
|
|
return value.replace(/</g, '<').replace(/>/g, '>').replace(/&/g, '&');
|
2016-03-08 10:26:06 +00:00
|
|
|
}
|
