2016-03-08 10:26:06 +00:00
/ * - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
* Copyright ( c ) Microsoft Corporation . All rights reserved .
* Licensed under the MIT License . See License . txt in the project root for license information .
* -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- * /
import * as path from 'path' ;
import * as fs from 'fs' ;
2018-01-25 20:13:24 +00:00
import { through , readable , ThroughStream } from 'event-stream' ;
2016-03-08 10:26:06 +00:00
import File = require ( 'vinyl' ) ;
import * as Is from 'is' ;
2017-03-20 14:01:18 +00:00
import * as xml2js from 'xml2js' ;
2017-03-02 20:53:02 +00:00
import * as glob from 'glob' ;
2017-06-30 09:42:52 +00:00
import * as https from 'https' ;
2018-01-25 20:13:24 +00:00
import * as gulp from 'gulp' ;
2016-03-08 10:26:06 +00:00
var util = require ( 'gulp-util' ) ;
2017-04-13 17:58:13 +00:00
var iconv = require ( 'iconv-lite' ) ;
2016-07-19 09:37:20 +00:00
2018-01-26 09:00:11 +00:00
const NUMBER_OF_CONCURRENT_DOWNLOADS = 4 ;
2018-01-16 09:57:27 +00:00
2016-03-08 10:26:06 +00:00
function log ( message : any , . . . rest : any [ ] ) : void {
2016-07-19 09:37:20 +00:00
util . log ( util . colors . green ( '[i18n]' ) , message , . . . rest ) ;
2016-03-08 10:26:06 +00:00
}
2018-01-25 20:13:24 +00:00
export interface Language {
id : string ; // laguage id, e.g. zh-tw, de
transifexId? : string ; // language id used in transifex, e.g zh-hant, de (optional, if not set, the id is used)
folderName? : string ; // language specific folder name, e.g. cht, deu (optional, if not set, the id is used)
}
export interface InnoSetup {
codePage : string ; //code page for encoding (http://www.jrsoftware.org/ishelp/index.php?topic=langoptionssection)
defaultInfo ? : {
name : string ; // inno setup language name
id : string ; // locale identifier (https://msdn.microsoft.com/en-us/library/dd318693.aspx)
} ;
}
export const defaultLanguages : Language [ ] = [
{ id : 'zh-tw' , folderName : 'cht' , transifexId : 'zh-hant' } ,
{ id : 'zh-cn' , folderName : 'chs' , transifexId : 'zh-hans' } ,
{ id : 'ja' , folderName : 'jpn' } ,
{ id : 'ko' , folderName : 'kor' } ,
{ id : 'de' , folderName : 'deu' } ,
{ id : 'fr' , folderName : 'fra' } ,
{ id : 'es' , folderName : 'esn' } ,
{ id : 'ru' , folderName : 'rus' } ,
{ id : 'it' , folderName : 'ita' }
] ;
// languages requested by the community to non-stable builds
export const extraLanguages : Language [ ] = [
{ id : 'pt-br' , folderName : 'ptb' } ,
{ id : 'hu' , folderName : 'hun' } ,
{ id : 'tr' , folderName : 'trk' }
] ;
2018-01-26 09:00:11 +00:00
// non built-in extensions also that are transifex and need to be part of the language packs
2018-01-29 15:35:01 +00:00
const externalExtensionsWithTranslations = {
'vscode-chrome-debug' : 'msjsdiag.debugger-for-chrome' ,
'vscode-node-debug' : 'ms-vscode.node-debug' ,
'vscode-node-debug2' : 'ms-vscode.node-debug2'
} ;
2018-01-26 09:00:11 +00:00
2016-03-08 10:26:06 +00:00
interface Map < V > {
[ key : string ] : V ;
}
2017-03-02 20:53:02 +00:00
interface Item {
id : string ;
message : string ;
comment : string ;
}
2017-04-05 14:22:22 +00:00
export interface Resource {
2017-03-02 20:53:02 +00:00
name : string ;
project : string ;
}
2017-04-13 16:19:51 +00:00
interface ParsedXLF {
messages : Map < string > ;
originalFilePath : string ;
language : string ;
}
2016-03-08 10:26:06 +00:00
interface LocalizeInfo {
key : string ;
comment : string [ ] ;
}
module LocalizeInfo {
export function is ( value : any ) : value is LocalizeInfo {
let candidate = value as LocalizeInfo ;
return Is . defined ( candidate ) && Is . string ( candidate . key ) && ( Is . undef ( candidate . comment ) || ( Is . array ( candidate . comment ) && candidate . comment . every ( element = > Is . string ( element ) ) ) ) ;
}
}
interface BundledFormat {
keys : Map < ( string | LocalizeInfo ) [ ] > ;
messages : Map < string [ ] > ;
bundles : Map < string [ ] > ;
}
module BundledFormat {
export function is ( value : any ) : value is BundledFormat {
if ( Is . undef ( value ) ) {
return false ;
}
let candidate = value as BundledFormat ;
let length = Object . keys ( value ) . length ;
return length === 3 && Is . defined ( candidate . keys ) && Is . defined ( candidate . messages ) && Is . defined ( candidate . bundles ) ;
}
}
2017-03-02 20:53:02 +00:00
interface ValueFormat {
message : string ;
comment : string [ ] ;
}
interface PackageJsonFormat {
[ key : string ] : string | ValueFormat ;
}
module PackageJsonFormat {
export function is ( value : any ) : value is PackageJsonFormat {
if ( Is . undef ( value ) || ! Is . object ( value ) ) {
return false ;
}
return Object . keys ( value ) . every ( key = > {
let element = value [ key ] ;
return Is . string ( element ) || ( Is . object ( element ) && Is . defined ( element . message ) && Is . defined ( element . comment ) ) ;
} ) ;
}
}
interface ModuleJsonFormat {
messages : string [ ] ;
keys : ( string | LocalizeInfo ) [ ] ;
}
module ModuleJsonFormat {
export function is ( value : any ) : value is ModuleJsonFormat {
let candidate = value as ModuleJsonFormat ;
return Is . defined ( candidate )
&& Is . array ( candidate . messages ) && candidate . messages . every ( message = > Is . string ( message ) )
&& Is . array ( candidate . keys ) && candidate . keys . every ( key = > Is . string ( key ) || LocalizeInfo . is ( key ) ) ;
}
}
2018-01-25 20:13:24 +00:00
interface BundledExtensionHeaderFormat {
id : string ;
type : string ;
hash : string ;
outDir : string ;
}
interface BundledExtensionFormat {
[ key : string ] : {
messages : string [ ] ;
keys : ( string | LocalizeInfo ) [ ] ;
} ;
}
2017-03-02 20:53:02 +00:00
export class Line {
private buffer : string [ ] = [ ] ;
constructor ( private indent : number = 0 ) {
if ( indent > 0 ) {
this . buffer . push ( new Array ( indent + 1 ) . join ( ' ' ) ) ;
}
}
public append ( value : string ) : Line {
this . buffer . push ( value ) ;
return this ;
}
public toString ( ) : string {
return this . buffer . join ( '' ) ;
}
}
class TextModel {
private _lines : string [ ] ;
constructor ( contents : string ) {
this . _lines = contents . split ( /\r\n|\r|\n/ ) ;
}
public get lines ( ) : string [ ] {
return this . _lines ;
}
}
export class XLF {
2017-05-15 13:59:14 +00:00
private buffer : string [ ] ;
2017-03-02 20:53:02 +00:00
private files : Map < Item [ ] > ;
2018-01-26 10:16:48 +00:00
public numberOfMessages : number ;
2017-03-02 20:53:02 +00:00
2017-05-15 13:59:14 +00:00
constructor ( public project : string ) {
this . buffer = [ ] ;
2017-03-02 20:53:02 +00:00
this . files = Object . create ( null ) ;
2018-01-26 10:16:48 +00:00
this . numberOfMessages = 0 ;
2017-03-02 20:53:02 +00:00
}
2017-05-15 13:59:14 +00:00
public toString ( ) : string {
this . appendHeader ( ) ;
2017-03-02 20:53:02 +00:00
for ( let file in this . files ) {
this . appendNewLine ( ` <file original=" ${ file } " source-language="en" datatype="plaintext"><body> ` , 2 ) ;
for ( let item of this . files [ file ] ) {
this . addStringItem ( item ) ;
}
2017-05-15 13:59:14 +00:00
this . appendNewLine ( '</body></file>' , 2 ) ;
2017-03-02 20:53:02 +00:00
}
this . appendFooter ( ) ;
return this . buffer . join ( '\r\n' ) ;
}
2018-01-25 20:13:24 +00:00
public addFile ( original : string , keys : ( string | LocalizeInfo ) [ ] , messages : string [ ] ) {
2018-01-31 10:37:58 +00:00
if ( keys . length === 0 ) {
console . log ( 'No keys in ' + original ) ;
return ;
}
2018-01-25 20:13:24 +00:00
if ( keys . length !== messages . length ) {
throw new Error ( ` Unmatching keys( ${ keys . length } ) and messages( ${ messages . length } ). ` ) ;
}
2018-01-26 10:16:48 +00:00
this . numberOfMessages += keys . length ;
2017-03-02 20:53:02 +00:00
this . files [ original ] = [ ] ;
2018-01-25 20:13:24 +00:00
let existingKeys = new Set < string > ( ) ;
for ( let i = 0 ; i < keys . length ; i ++ ) {
let key = keys [ i ] ;
let realKey : string ;
let comment : string ;
2017-03-02 20:53:02 +00:00
if ( Is . string ( key ) ) {
2018-01-25 20:13:24 +00:00
realKey = key ;
comment = undefined ;
} else if ( LocalizeInfo . is ( key ) ) {
realKey = key . key ;
if ( key . comment && key . comment . length > 0 ) {
comment = key . comment . map ( comment = > encodeEntities ( comment ) ) . join ( '\r\n' ) ;
2017-03-02 20:53:02 +00:00
}
}
2018-01-25 20:13:24 +00:00
if ( ! realKey || existingKeys . has ( realKey ) ) {
continue ;
}
existingKeys . add ( realKey ) ;
let message : string = encodeEntities ( messages [ i ] ) ;
this . files [ original ] . push ( { id : realKey , message : message , comment : comment } ) ;
2017-03-02 20:53:02 +00:00
}
}
2017-05-15 13:59:14 +00:00
private addStringItem ( item : Item ) : void {
if ( ! item . id || ! item . message ) {
2017-12-04 21:13:55 +00:00
throw new Error ( ` No item ID or value specified: ${ JSON . stringify ( item ) } ` ) ;
2017-05-15 13:59:14 +00:00
}
2017-03-02 20:53:02 +00:00
2017-05-15 13:59:14 +00:00
this . appendNewLine ( ` <trans-unit id=" ${ item . id } "> ` , 4 ) ;
this . appendNewLine ( ` <source xml:lang="en"> ${ item . message } </source> ` , 6 ) ;
2017-03-02 20:53:02 +00:00
2017-05-15 13:59:14 +00:00
if ( item . comment ) {
this . appendNewLine ( ` <note> ${ item . comment } </note> ` , 6 ) ;
}
2017-03-02 20:53:02 +00:00
2017-05-15 13:59:14 +00:00
this . appendNewLine ( '</trans-unit>' , 4 ) ;
2017-03-02 20:53:02 +00:00
}
2017-05-15 13:59:14 +00:00
private appendHeader ( ) : void {
this . appendNewLine ( '<?xml version="1.0" encoding="utf-8"?>' , 0 ) ;
this . appendNewLine ( '<xliff version="1.2" xmlns="urn:oasis:names:tc:xliff:document:1.2">' , 0 ) ;
2017-03-02 20:53:02 +00:00
}
2017-05-15 13:59:14 +00:00
private appendFooter ( ) : void {
this . appendNewLine ( '</xliff>' , 0 ) ;
}
2017-03-02 20:53:02 +00:00
2017-05-15 13:59:14 +00:00
private appendNewLine ( content : string , indent? : number ) : void {
let line = new Line ( indent ) ;
line . append ( content ) ;
this . buffer . push ( line . toString ( ) ) ;
}
2017-03-02 20:53:02 +00:00
2018-01-29 17:00:16 +00:00
static parsePseudo = function ( xlfString : string ) : Promise < ParsedXLF [ ] > {
return new Promise ( ( resolve , reject ) = > {
let parser = new xml2js . Parser ( ) ;
let files : { messages : Map < string > , originalFilePath : string , language : string } [ ] = [ ] ;
parser . parseString ( xlfString , function ( err , result ) {
const fileNodes : any [ ] = result [ 'xliff' ] [ 'file' ] ;
2018-01-31 10:37:58 +00:00
fileNodes . forEach ( file = > {
2018-01-29 17:00:16 +00:00
const originalFilePath = file . $ . original ;
2018-01-31 10:37:58 +00:00
const messages : Map < string > = { } ;
2018-01-29 17:00:16 +00:00
const transUnits = file . body [ 0 ] [ 'trans-unit' ] ;
2018-01-31 10:37:58 +00:00
if ( transUnits ) {
transUnits . forEach ( unit = > {
const key = unit . $ . id ;
const val = pseudify ( unit . source [ 0 ] [ '_' ] . toString ( ) ) ;
if ( key && val ) {
messages [ key ] = decodeEntities ( val ) ;
}
} ) ;
files . push ( { messages : messages , originalFilePath : originalFilePath , language : 'ps' } ) ;
}
2018-01-29 17:00:16 +00:00
} ) ;
2018-01-30 08:57:51 +00:00
resolve ( files ) ;
2018-01-29 17:00:16 +00:00
} ) ;
} ) ;
} ;
2017-05-15 13:59:14 +00:00
static parse = function ( xlfString : string ) : Promise < ParsedXLF [ ] > {
2017-03-02 20:53:02 +00:00
return new Promise ( ( resolve , reject ) = > {
let parser = new xml2js . Parser ( ) ;
let files : { messages : Map < string > , originalFilePath : string , language : string } [ ] = [ ] ;
2017-05-15 13:59:14 +00:00
parser . parseString ( xlfString , function ( err , result ) {
2017-03-02 20:53:02 +00:00
if ( err ) {
2018-01-25 20:13:24 +00:00
reject ( new Error ( ` XLF parsing error: Failed to parse XLIFF string. ${ err } ` ) ) ;
2017-03-02 20:53:02 +00:00
}
const fileNodes : any [ ] = result [ 'xliff' ] [ 'file' ] ;
if ( ! fileNodes ) {
2018-01-25 20:13:24 +00:00
reject ( new Error ( ` XLF parsing error: XLIFF file does not contain "xliff" or "file" node(s) required for parsing. ` ) ) ;
2017-03-02 20:53:02 +00:00
}
fileNodes . forEach ( ( file ) = > {
const originalFilePath = file . $ . original ;
if ( ! originalFilePath ) {
2018-01-25 20:13:24 +00:00
reject ( new Error ( ` XLF parsing error: XLIFF file node does not contain original attribute to determine the original location of the resource file. ` ) ) ;
2017-03-02 20:53:02 +00:00
}
2018-01-29 17:00:16 +00:00
let language = file . $ [ 'target-language' ] ;
2017-03-02 20:53:02 +00:00
if ( ! language ) {
2018-01-25 20:13:24 +00:00
reject ( new Error ( ` XLF parsing error: XLIFF file node does not contain target-language attribute to determine translated language. ` ) ) ;
2017-03-02 20:53:02 +00:00
}
2018-01-31 10:37:58 +00:00
const messages : Map < string > = { } ;
2017-03-02 20:53:02 +00:00
const transUnits = file . body [ 0 ] [ 'trans-unit' ] ;
2018-01-31 10:37:58 +00:00
if ( transUnits ) {
transUnits . forEach ( unit = > {
const key = unit . $ . id ;
if ( ! unit . target ) {
return ; // No translation available
}
const val = unit . target . toString ( ) ;
if ( key && val ) {
messages [ key ] = decodeEntities ( val ) ;
} else {
reject ( new Error ( ` XLF parsing error: XLIFF file does not contain full localization data. ID or target translation for one of the trans-unit nodes is not present. ` ) ) ;
}
} ) ;
files . push ( { messages : messages , originalFilePath : originalFilePath , language : language.toLowerCase ( ) } ) ;
}
2017-03-02 20:53:02 +00:00
} ) ;
resolve ( files ) ;
} ) ;
} ) ;
} ;
}
2018-01-16 09:57:27 +00:00
export interface ITask < T > {
( ) : T ;
}
interface ILimitedTaskFactory < T > {
factory : ITask < Promise < T > > ;
c : ( value? : T | Thenable < T > ) = > void ;
e : ( error? : any ) = > void ;
}
export class Limiter < T > {
private runningPromises : number ;
private outstandingPromises : ILimitedTaskFactory < any > [ ] ;
constructor ( private maxDegreeOfParalellism : number ) {
this . outstandingPromises = [ ] ;
this . runningPromises = 0 ;
}
queue ( factory : ITask < Promise < T > > ) : Promise < T > {
return new Promise < T > ( ( c , e ) = > {
2018-01-25 20:13:24 +00:00
this . outstandingPromises . push ( { factory , c , e } ) ;
2018-01-16 09:57:27 +00:00
this . consume ( ) ;
} ) ;
}
private consume ( ) : void {
while ( this . outstandingPromises . length && this . runningPromises < this . maxDegreeOfParalellism ) {
const iLimitedTask = this . outstandingPromises . shift ( ) ;
this . runningPromises ++ ;
const promise = iLimitedTask . factory ( ) ;
promise . then ( iLimitedTask . c ) . catch ( iLimitedTask . e ) ;
promise . then ( ( ) = > this . consumed ( ) ) . catch ( ( ) = > this . consumed ( ) ) ;
}
}
private consumed ( ) : void {
this . runningPromises -- ;
this . consume ( ) ;
}
}
2018-01-25 20:13:24 +00:00
function sortLanguages ( languages : Language [ ] ) : Language [ ] {
return languages . sort ( ( a : Language , b : Language ) : number = > {
return a . id < b . id ? - 1 : ( a . id > b . id ? 1 : 0 ) ;
2016-03-08 10:26:06 +00:00
} ) ;
}
function stripComments ( content : string ) : string {
/ * *
* First capturing group matches double quoted string
* Second matches single quotes string
* Third matches block comments
* Fourth matches line comments
* /
var regexp : RegExp = /("(?:[^\\\"]*(?:\\.)?)*")|('(?:[^\\\']*(?:\\.)?)*')|(\/\*(?:\r?\n|.)*?\*\/)|(\/{2,}.*?(?:(?:\r?\n)|$))/g ;
let result = content . replace ( regexp , ( match , m1 , m2 , m3 , m4 ) = > {
// Only one of m1, m2, m3, m4 matches
if ( m3 ) {
// A block comment. Replace with nothing
return '' ;
} else if ( m4 ) {
// A line comment. If it ends in \r?\n then keep it.
let length = m4 . length ;
if ( length > 2 && m4 [ length - 1 ] === '\n' ) {
2017-05-15 13:59:14 +00:00
return m4 [ length - 2 ] === '\r' ? '\r\n' : '\n' ;
2016-03-08 10:26:06 +00:00
} else {
return '' ;
}
} else {
// We match a string
return match ;
}
} ) ;
return result ;
2017-03-02 20:53:02 +00:00
}
2016-03-08 10:26:06 +00:00
2017-05-15 13:59:14 +00:00
function escapeCharacters ( value : string ) : string {
var result : string [ ] = [ ] ;
2016-03-08 10:26:06 +00:00
for ( var i = 0 ; i < value . length ; i ++ ) {
var ch = value . charAt ( i ) ;
2017-05-15 13:59:14 +00:00
switch ( ch ) {
2016-03-08 10:26:06 +00:00
case '\'' :
result . push ( '\\\'' ) ;
break ;
case '"' :
result . push ( '\\"' ) ;
break ;
case '\\' :
result . push ( '\\\\' ) ;
break ;
case '\n' :
result . push ( '\\n' ) ;
break ;
case '\r' :
result . push ( '\\r' ) ;
break ;
case '\t' :
result . push ( '\\t' ) ;
break ;
case '\b' :
result . push ( '\\b' ) ;
break ;
case '\f' :
result . push ( '\\f' ) ;
break ;
default :
result . push ( ch ) ;
}
}
return result . join ( '' ) ;
}
2018-01-25 20:13:24 +00:00
function processCoreBundleFormat ( fileHeader : string , languages : Language [ ] , json : BundledFormat , emitter : ThroughStream ) {
2016-03-08 10:26:06 +00:00
let keysSection = json . keys ;
let messageSection = json . messages ;
let bundleSection = json . bundles ;
let statistics : Map < number > = Object . create ( null ) ;
let total : number = 0 ;
let defaultMessages : Map < Map < string > > = Object . create ( null ) ;
let module s = Object . keys ( keysSection ) ;
module s.forEach ( ( module ) = > {
let keys = keysSection [ module ] ;
let messages = messageSection [ module ] ;
if ( ! messages || keys . length !== messages . length ) {
emitter . emit ( 'error' , ` Message for module ${ module } corrupted. Mismatch in number of keys and messages. ` ) ;
return ;
}
let messageMap : Map < string > = Object . create ( null ) ;
defaultMessages [ module ] = messageMap ;
keys . map ( ( key , i ) = > {
total ++ ;
2018-01-25 20:13:24 +00:00
if ( typeof key === 'string' ) {
2016-03-08 10:26:06 +00:00
messageMap [ key ] = messages [ i ] ;
} else {
messageMap [ key . key ] = messages [ i ] ;
}
} ) ;
} ) ;
let languageDirectory = path . join ( __dirname , '..' , '..' , 'i18n' ) ;
2018-01-25 20:13:24 +00:00
let sortedLanguages = sortLanguages ( languages ) ;
sortedLanguages . forEach ( ( language ) = > {
2016-07-19 09:37:20 +00:00
if ( process . env [ 'VSCODE_BUILD_VERBOSE' ] ) {
2018-01-25 20:13:24 +00:00
log ( ` Generating nls bundles for: ${ language . id } ` ) ;
2016-07-19 09:37:20 +00:00
}
2018-01-25 20:13:24 +00:00
statistics [ language . id ] = 0 ;
2016-03-08 10:26:06 +00:00
let localizedModules : Map < string [ ] > = Object . create ( null ) ;
2018-01-25 20:13:24 +00:00
let languageFolderName = language . folderName || language . id ;
let cwd = path . join ( languageDirectory , languageFolderName , 'src' ) ;
2016-03-08 10:26:06 +00:00
module s.forEach ( ( module ) = > {
let order = keysSection [ module ] ;
let i18nFile = path . join ( cwd , module ) + '.i18n.json' ;
let messages : Map < string > = null ;
if ( fs . existsSync ( i18nFile ) ) {
let content = stripComments ( fs . readFileSync ( i18nFile , 'utf8' ) ) ;
messages = JSON . parse ( content ) ;
} else {
2016-07-19 09:37:20 +00:00
if ( process . env [ 'VSCODE_BUILD_VERBOSE' ] ) {
log ( ` No localized messages found for module ${ module } . Using default messages. ` ) ;
}
2016-03-08 10:26:06 +00:00
messages = defaultMessages [ module ] ;
2018-01-25 20:13:24 +00:00
statistics [ language . id ] = statistics [ language . id ] + Object . keys ( messages ) . length ;
2016-03-08 10:26:06 +00:00
}
let localizedMessages : string [ ] = [ ] ;
order . forEach ( ( keyInfo ) = > {
let key : string = null ;
2018-01-25 20:13:24 +00:00
if ( typeof keyInfo === 'string' ) {
2016-03-08 10:26:06 +00:00
key = keyInfo ;
} else {
key = keyInfo . key ;
}
let message : string = messages [ key ] ;
if ( ! message ) {
2016-07-19 09:37:20 +00:00
if ( process . env [ 'VSCODE_BUILD_VERBOSE' ] ) {
log ( ` No localized message found for key ${ key } in module ${ module } . Using default message. ` ) ;
}
2016-03-08 10:26:06 +00:00
message = defaultMessages [ module ] [ key ] ;
2018-01-25 20:13:24 +00:00
statistics [ language . id ] = statistics [ language . id ] + 1 ;
2016-03-08 10:26:06 +00:00
}
localizedMessages . push ( message ) ;
} ) ;
localizedModules [ module ] = localizedMessages ;
} ) ;
Object . keys ( bundleSection ) . forEach ( ( bundle ) = > {
let module s = bundleSection [ bundle ] ;
let contents : string [ ] = [
2016-05-31 14:29:52 +00:00
fileHeader ,
2018-01-25 20:13:24 +00:00
` define(" ${ bundle } .nls. ${ language . id } ", { `
2016-03-08 10:26:06 +00:00
] ;
module s.forEach ( ( module , index ) = > {
contents . push ( ` \ t" ${ module } ": [ ` ) ;
let messages = localizedModules [ module ] ;
if ( ! messages ) {
emitter . emit ( 'error' , ` Didn't find messages for module ${ module } . ` ) ;
return ;
}
messages . forEach ( ( message , index ) = > {
2017-05-15 13:59:14 +00:00
contents . push ( ` \ t \ t" ${ escapeCharacters ( message ) } ${ index < messages . length ? '",' : '"' } ` ) ;
2016-03-08 10:26:06 +00:00
} ) ;
contents . push ( index < module s.length - 1 ? '\t],' : '\t]' ) ;
} ) ;
contents . push ( '});' ) ;
2018-01-25 20:13:24 +00:00
emitter . queue ( new File ( { path : bundle + '.nls.' + language . id + '.js' , contents : new Buffer ( contents . join ( '\n' ) , 'utf-8' ) } ) ) ;
2016-03-08 10:26:06 +00:00
} ) ;
} ) ;
Object . keys ( statistics ) . forEach ( key = > {
let value = statistics [ key ] ;
2016-07-19 09:37:20 +00:00
log ( ` ${ key } has ${ value } untranslated strings. ` ) ;
2016-03-08 10:26:06 +00:00
} ) ;
2018-01-25 20:13:24 +00:00
sortedLanguages . forEach ( language = > {
let stats = statistics [ language . id ] ;
if ( Is . undef ( stats ) ) {
log ( ` \ tNo translations found for language ${ language . id } . Using default language instead. ` ) ;
2016-03-08 10:26:06 +00:00
}
} ) ;
}
2018-01-25 20:13:24 +00:00
export function processNlsFiles ( opts : { fileHeader : string ; languages : Language [ ] } ) : ThroughStream {
return through ( function ( this : ThroughStream , file : File ) {
2016-03-08 10:26:06 +00:00
let fileName = path . basename ( file . path ) ;
if ( fileName === 'nls.metadata.json' ) {
let json = null ;
if ( file . isBuffer ( ) ) {
2016-10-20 10:39:08 +00:00
json = JSON . parse ( ( < Buffer > file . contents ) . toString ( 'utf8' ) ) ;
2016-03-08 10:26:06 +00:00
} else {
2016-07-19 09:37:20 +00:00
this . emit ( 'error' , ` Failed to read component file: ${ file . relative } ` ) ;
2018-01-25 20:13:24 +00:00
return ;
2016-03-08 10:26:06 +00:00
}
if ( BundledFormat . is ( json ) ) {
2017-06-01 10:54:18 +00:00
processCoreBundleFormat ( opts . fileHeader , opts . languages , json , this ) ;
2016-03-08 10:26:06 +00:00
}
}
2018-01-25 20:13:24 +00:00
this . queue ( file ) ;
2016-03-08 10:26:06 +00:00
} ) ;
2017-03-02 20:53:02 +00:00
}
2017-04-05 13:58:07 +00:00
const editorProject : string = 'vscode-editor' ,
2017-04-05 16:12:24 +00:00
workbenchProject : string = 'vscode-workbench' ,
2017-04-06 07:45:42 +00:00
extensionsProject : string = 'vscode-extensions' ,
2017-04-05 16:12:24 +00:00
setupProject : string = 'vscode-setup' ;
2017-04-05 13:58:07 +00:00
2017-04-05 14:22:22 +00:00
export function getResource ( sourceFile : string ) : Resource {
2017-03-02 20:53:02 +00:00
let resource : string ;
2017-05-15 13:59:14 +00:00
if ( /^vs\/platform/ . test ( sourceFile ) ) {
2017-03-02 20:53:02 +00:00
return { name : 'vs/platform' , project : editorProject } ;
2017-05-15 13:59:14 +00:00
} else if ( /^vs\/editor\/contrib/ . test ( sourceFile ) ) {
2017-03-02 20:53:02 +00:00
return { name : 'vs/editor/contrib' , project : editorProject } ;
2017-05-15 13:59:14 +00:00
} else if ( /^vs\/editor/ . test ( sourceFile ) ) {
2017-03-02 20:53:02 +00:00
return { name : 'vs/editor' , project : editorProject } ;
2017-05-15 13:59:14 +00:00
} else if ( /^vs\/base/ . test ( sourceFile ) ) {
2017-03-02 20:53:02 +00:00
return { name : 'vs/base' , project : editorProject } ;
2017-05-15 13:59:14 +00:00
} else if ( /^vs\/code/ . test ( sourceFile ) ) {
2017-03-02 20:53:02 +00:00
return { name : 'vs/code' , project : workbenchProject } ;
2017-05-15 13:59:14 +00:00
} else if ( /^vs\/workbench\/parts/ . test ( sourceFile ) ) {
2017-03-02 20:53:02 +00:00
resource = sourceFile . split ( '/' , 4 ) . join ( '/' ) ;
return { name : resource , project : workbenchProject } ;
2017-05-15 13:59:14 +00:00
} else if ( /^vs\/workbench\/services/ . test ( sourceFile ) ) {
2017-03-02 20:53:02 +00:00
resource = sourceFile . split ( '/' , 4 ) . join ( '/' ) ;
return { name : resource , project : workbenchProject } ;
2017-05-15 13:59:14 +00:00
} else if ( /^vs\/workbench/ . test ( sourceFile ) ) {
2017-03-02 20:53:02 +00:00
return { name : 'vs/workbench' , project : workbenchProject } ;
}
2017-05-15 13:59:14 +00:00
throw new Error ( ` Could not identify the XLF bundle for ${ sourceFile } ` ) ;
2017-03-02 20:53:02 +00:00
}
2018-01-25 20:13:24 +00:00
export function createXlfFilesForCoreBundle ( ) : ThroughStream {
return through ( function ( this : ThroughStream , file : File ) {
const basename = path . basename ( file . path ) ;
if ( basename === 'nls.metadata.json' ) {
if ( file . isBuffer ( ) ) {
const xlfs : Map < XLF > = Object . create ( null ) ;
const json : BundledFormat = JSON . parse ( ( file . contents as Buffer ) . toString ( 'utf8' ) ) ;
for ( let coreModule in json . keys ) {
const projectResource = getResource ( coreModule ) ;
const resource = projectResource . name ;
const project = projectResource . project ;
const keys = json . keys [ coreModule ] ;
const messages = json . messages [ coreModule ] ;
if ( keys . length !== messages . length ) {
this . emit ( 'error' , ` There is a mismatch between keys and messages in ${ file . relative } for module ${ coreModule } ` ) ;
return ;
} else {
let xlf = xlfs [ resource ] ;
if ( ! xlf ) {
xlf = new XLF ( project ) ;
xlfs [ resource ] = xlf ;
}
xlf . addFile ( ` src/ ${ coreModule } ` , keys , messages ) ;
}
}
for ( let resource in xlfs ) {
const xlf = xlfs [ resource ] ;
const filePath = ` ${ xlf . project } / ${ resource . replace ( /\//g , '_' ) } .xlf ` ;
const xlfFile = new File ( {
path : filePath ,
contents : new Buffer ( xlf . toString ( ) , 'utf8' )
} ) ;
this . queue ( xlfFile ) ;
}
} else {
this . emit ( 'error' , new Error ( ` File ${ file . relative } is not using a buffer content ` ) ) ;
return ;
}
} else {
this . emit ( 'error' , new Error ( ` File ${ file . relative } is not a core meta data file. ` ) ) ;
return ;
2017-03-02 20:53:02 +00:00
}
2018-01-25 20:13:24 +00:00
} ) ;
2017-03-02 20:53:02 +00:00
}
2018-01-25 20:13:24 +00:00
export function createXlfFilesForExtensions ( ) : ThroughStream {
let counter : number = 0 ;
let folderStreamEnded : boolean = false ;
let folderStreamEndEmitted : boolean = false ;
return through ( function ( this : ThroughStream , extensionFolder : File ) {
const folderStream = this ;
const stat = fs . statSync ( extensionFolder . path ) ;
if ( ! stat . isDirectory ( ) ) {
2017-03-02 20:53:02 +00:00
return ;
}
2018-01-25 20:13:24 +00:00
let extensionName = path . basename ( extensionFolder . path ) ;
if ( extensionName === 'node_modules' ) {
2017-03-02 20:53:02 +00:00
return ;
}
2018-01-25 20:13:24 +00:00
counter ++ ;
let _xlf : XLF ;
function getXlf() {
if ( ! _xlf ) {
_xlf = new XLF ( extensionsProject ) ;
}
return _xlf ;
}
gulp . src ( [ ` ./extensions/ ${ extensionName } /package.nls.json ` , ` ./extensions/ ${ extensionName } /**/nls.metadata.json ` ] ) . pipe ( through ( function ( file : File ) {
if ( file . isBuffer ( ) ) {
const buffer : Buffer = file . contents as Buffer ;
const basename = path . basename ( file . path ) ;
if ( basename === 'package.nls.json' ) {
const json : PackageJsonFormat = JSON . parse ( buffer . toString ( 'utf8' ) ) ;
const keys = Object . keys ( json ) ;
const messages = keys . map ( ( key ) = > {
const value = json [ key ] ;
if ( Is . string ( value ) ) {
return value ;
} else if ( value ) {
return value . message ;
} else {
return ` Unknown message for key: ${ key } ` ;
}
} ) ;
getXlf ( ) . addFile ( ` extensions/ ${ extensionName } /package ` , keys , messages ) ;
} else if ( basename === 'nls.metadata.json' ) {
const json : BundledExtensionFormat = JSON . parse ( buffer . toString ( 'utf8' ) ) ;
const relPath = path . relative ( ` ./extensions/ ${ extensionName } ` , path . dirname ( file . path ) ) ;
for ( let file in json ) {
const fileContent = json [ file ] ;
getXlf ( ) . addFile ( ` extensions/ ${ extensionName } / ${ relPath } / ${ file } ` , fileContent . keys , fileContent . messages ) ;
}
} else {
this . emit ( 'error' , new Error ( ` ${ file . path } is not a valid extension nls file ` ) ) ;
return ;
}
}
} , function ( ) {
if ( _xlf ) {
let xlfFile = new File ( {
path : path.join ( extensionsProject , extensionName + '.xlf' ) ,
contents : new Buffer ( _xlf . toString ( ) , 'utf8' )
} ) ;
folderStream . queue ( xlfFile ) ;
}
this . queue ( null ) ;
counter -- ;
if ( counter === 0 && folderStreamEnded && ! folderStreamEndEmitted ) {
folderStreamEndEmitted = true ;
folderStream . queue ( null ) ;
2017-03-02 20:53:02 +00:00
}
2018-01-25 20:13:24 +00:00
} ) ) ;
} , function ( ) {
folderStreamEnded = true ;
if ( counter === 0 ) {
folderStreamEndEmitted = true ;
this . queue ( null ) ;
2017-03-02 20:53:02 +00:00
}
} ) ;
2018-01-25 20:13:24 +00:00
}
export function createXlfFilesForIsl ( ) : ThroughStream {
return through ( function ( this : ThroughStream , file : File ) {
let projectName : string ,
resourceFile : string ;
if ( path . basename ( file . path ) === 'Default.isl' ) {
projectName = setupProject ;
resourceFile = 'setup_default.xlf' ;
} else {
projectName = workbenchProject ;
resourceFile = 'setup_messages.xlf' ;
}
let xlf = new XLF ( projectName ) ,
keys : string [ ] = [ ] ,
messages : string [ ] = [ ] ;
let model = new TextModel ( file . contents . toString ( ) ) ;
let inMessageSection = false ;
model . lines . forEach ( line = > {
if ( line . length === 0 ) {
return ;
}
let firstChar = line . charAt ( 0 ) ;
switch ( firstChar ) {
case ';' :
// Comment line;
return ;
case '[' :
inMessageSection = '[Messages]' === line || '[CustomMessages]' === line ;
return ;
}
if ( ! inMessageSection ) {
return ;
}
let sections : string [ ] = line . split ( '=' ) ;
if ( sections . length !== 2 ) {
throw new Error ( ` Badly formatted message found: ${ line } ` ) ;
} else {
let key = sections [ 0 ] ;
let value = sections [ 1 ] ;
if ( key . length > 0 && value . length > 0 ) {
keys . push ( key ) ;
messages . push ( value ) ;
}
}
} ) ;
2017-03-02 20:53:02 +00:00
2018-01-25 20:13:24 +00:00
const originalPath = file . path . substring ( file . cwd . length + 1 , file . path . split ( '.' ) [ 0 ] . length ) . replace ( /\\/g , '/' ) ;
xlf . addFile ( originalPath , keys , messages ) ;
2017-03-02 20:53:02 +00:00
2018-01-25 20:13:24 +00:00
// Emit only upon all ISL files combined into single XLF instance
const newFilePath = path . join ( projectName , resourceFile ) ;
const xlfFile = new File ( { path : newFilePath , contents : new Buffer ( xlf . toString ( ) , 'utf-8' ) } ) ;
this . queue ( xlfFile ) ;
} ) ;
2017-03-02 20:53:02 +00:00
}
2017-03-20 14:01:18 +00:00
export function pushXlfFiles ( apiHostname : string , username : string , password : string ) : ThroughStream {
2017-03-17 16:24:18 +00:00
let tryGetPromises = [ ] ;
let updateCreatePromises = [ ] ;
2018-01-25 20:13:24 +00:00
return through ( function ( this : ThroughStream , file : File ) {
2017-03-02 20:53:02 +00:00
const project = path . dirname ( file . relative ) ;
const fileName = path . basename ( file . path ) ;
const slug = fileName . substr ( 0 , fileName . length - '.xlf' . length ) ;
2017-03-20 14:01:18 +00:00
const credentials = ` ${ username } : ${ password } ` ;
2017-03-02 20:53:02 +00:00
// Check if resource already exists, if not, then create it.
2017-03-20 14:01:18 +00:00
let promise = tryGetResource ( project , slug , apiHostname , credentials ) ;
2017-03-17 16:24:18 +00:00
tryGetPromises . push ( promise ) ;
promise . then ( exists = > {
2017-03-02 20:53:02 +00:00
if ( exists ) {
2017-03-20 14:01:18 +00:00
promise = updateResource ( project , slug , file , apiHostname , credentials ) ;
2017-03-02 20:53:02 +00:00
} else {
2017-03-20 14:01:18 +00:00
promise = createResource ( project , slug , file , apiHostname , credentials ) ;
2017-03-02 20:53:02 +00:00
}
2017-03-17 16:24:18 +00:00
updateCreatePromises . push ( promise ) ;
} ) ;
2017-05-15 13:59:14 +00:00
} , function ( ) {
2017-03-17 16:24:18 +00:00
// End the pipe only after all the communication with Transifex API happened
Promise . all ( tryGetPromises ) . then ( ( ) = > {
Promise . all ( updateCreatePromises ) . then ( ( ) = > {
2018-01-25 20:13:24 +00:00
this . queue ( null ) ;
2017-04-05 09:58:31 +00:00
} ) . catch ( ( reason ) = > { throw new Error ( reason ) ; } ) ;
} ) . catch ( ( reason ) = > { throw new Error ( reason ) ; } ) ;
2017-03-02 20:53:02 +00:00
} ) ;
}
2018-01-26 09:00:11 +00:00
function getAllResources ( project : string , apiHostname : string , username : string , password : string ) : Promise < string [ ] > {
return new Promise ( ( resolve , reject ) = > {
const credentials = ` ${ username } : ${ password } ` ;
const options = {
hostname : apiHostname ,
path : ` /api/2/project/ ${ project } /resources ` ,
auth : credentials ,
method : 'GET'
} ;
const request = https . request ( options , ( res ) = > {
let buffer : Buffer [ ] = [ ] ;
res . on ( 'data' , ( chunk : Buffer ) = > buffer . push ( chunk ) ) ;
res . on ( 'end' , ( ) = > {
if ( res . statusCode === 200 ) {
let json = JSON . parse ( Buffer . concat ( buffer ) . toString ( ) ) ;
if ( Array . isArray ( json ) ) {
resolve ( json . map ( o = > o . slug ) ) ;
return ;
}
reject ( ` Unexpected data format. Response code: ${ res . statusCode } . ` ) ;
} else {
reject ( ` No resources in ${ project } returned no data. Response code: ${ res . statusCode } . ` ) ;
}
} ) ;
} ) ;
request . on ( 'error' , ( err ) = > {
reject ( ` Failed to query resources in ${ project } with the following error: ${ err } . ${ options . path } ` ) ;
} ) ;
request . end ( ) ;
} ) ;
}
export function findObsoleteResources ( apiHostname : string , username : string , password : string ) : ThroughStream {
let resourcesByProject : Map < string [ ] > = Object . create ( null ) ;
resourcesByProject [ extensionsProject ] = [ ] . concat ( externalExtensionsWithTranslations ) ; // clone
return through ( function ( this : ThroughStream , file : File ) {
const project = path . dirname ( file . relative ) ;
const fileName = path . basename ( file . path ) ;
const slug = fileName . substr ( 0 , fileName . length - '.xlf' . length ) ;
let slugs = resourcesByProject [ project ] ;
if ( ! slugs ) {
resourcesByProject [ project ] = slugs = [ ] ;
}
slugs . push ( slug ) ;
this . push ( file ) ;
} , function ( ) {
2018-01-26 10:16:48 +00:00
const json = JSON . parse ( fs . readFileSync ( './build/lib/i18n.resources.json' , 'utf8' ) ) ;
let i18Resources = [ . . . json . editor , . . . json . workbench ] . map ( ( r : Resource ) = > r . project + '/' + r . name . replace ( /\//g , '_' ) ) ;
let extractedResources = [ ] ;
for ( let project of [ workbenchProject , editorProject ] ) {
for ( let resource of resourcesByProject [ project ] ) {
if ( resource !== 'setup_messages' ) {
extractedResources . push ( project + '/' + resource ) ;
}
}
}
if ( i18Resources . length !== extractedResources . length ) {
console . log ( ` [i18n] Obsolete resources in file 'build/lib/i18n.resources.json': JSON.stringify( ${ i18Resources . filter ( p = > extractedResources . indexOf ( p ) === - 1 ) } ) ` ) ;
console . log ( ` [i18n] Missing resources in file 'build/lib/i18n.resources.json': JSON.stringify( ${ extractedResources . filter ( p = > i18Resources . indexOf ( p ) === - 1 ) } ) ` ) ;
}
2018-01-26 09:00:11 +00:00
let promises = [ ] ;
for ( let project in resourcesByProject ) {
promises . push (
getAllResources ( project , apiHostname , username , password ) . then ( resources = > {
let expectedResources = resourcesByProject [ project ] ;
let unusedResources = resources . filter ( resource = > resource && expectedResources . indexOf ( resource ) === - 1 ) ;
if ( unusedResources . length ) {
console . log ( ` [transifex] Obsolete resources in project ' ${ project } ': ${ unusedResources . join ( ', ' ) } ` ) ;
}
} )
) ;
}
return Promise . all ( promises ) . then ( _ = > {
this . push ( null ) ;
} ) . catch ( ( reason ) = > { throw new Error ( reason ) ; } ) ;
} ) ;
}
2017-03-20 14:01:18 +00:00
function tryGetResource ( project : string , slug : string , apiHostname : string , credentials : string ) : Promise < boolean > {
2017-03-02 20:53:02 +00:00
return new Promise ( ( resolve , reject ) = > {
2017-03-20 14:01:18 +00:00
const options = {
hostname : apiHostname ,
path : ` /api/2/project/ ${ project } /resource/ ${ slug } /?details ` ,
auth : credentials ,
method : 'GET'
} ;
2017-06-30 09:42:52 +00:00
const request = https . request ( options , ( response ) = > {
2017-03-02 20:53:02 +00:00
if ( response . statusCode === 404 ) {
resolve ( false ) ;
} else if ( response . statusCode === 200 ) {
resolve ( true ) ;
} else {
2017-03-20 14:01:18 +00:00
reject ( ` Failed to query resource ${ project } / ${ slug } . Response: ${ response . statusCode } ${ response . statusMessage } ` ) ;
2017-03-02 20:53:02 +00:00
}
2017-04-05 09:58:31 +00:00
} ) ;
request . on ( 'error' , ( err ) = > {
2017-03-20 14:01:18 +00:00
reject ( ` Failed to get ${ project } / ${ slug } on Transifex: ${ err } ` ) ;
2017-03-02 20:53:02 +00:00
} ) ;
2017-03-20 14:01:18 +00:00
request . end ( ) ;
2017-03-02 20:53:02 +00:00
} ) ;
}
2017-03-20 14:01:18 +00:00
function createResource ( project : string , slug : string , xlfFile : File , apiHostname : string , credentials : any ) : Promise < any > {
return new Promise ( ( resolve , reject ) = > {
const data = JSON . stringify ( {
'content' : xlfFile . contents . toString ( ) ,
'name' : slug ,
'slug' : slug ,
'i18n_type' : 'XLIFF'
} ) ;
2017-03-17 16:24:18 +00:00
const options = {
2017-03-20 14:01:18 +00:00
hostname : apiHostname ,
path : ` /api/2/project/ ${ project } /resources ` ,
headers : {
'Content-Type' : 'application/json' ,
'Content-Length' : Buffer . byteLength ( data )
2017-03-17 16:24:18 +00:00
} ,
2017-03-20 14:01:18 +00:00
auth : credentials ,
method : 'POST'
2017-03-17 16:24:18 +00:00
} ;
2017-03-02 20:53:02 +00:00
2017-06-30 09:42:52 +00:00
let request = https . request ( options , ( res ) = > {
2017-03-17 16:24:18 +00:00
if ( res . statusCode === 201 ) {
log ( ` Resource ${ project } / ${ slug } successfully created on Transifex. ` ) ;
} else {
2017-03-20 14:01:18 +00:00
reject ( ` Something went wrong in the request creating ${ slug } in ${ project } . ${ res . statusCode } ` ) ;
2017-03-17 16:24:18 +00:00
}
2017-04-05 09:58:31 +00:00
} ) ;
request . on ( 'error' , ( err ) = > {
2017-03-20 14:01:18 +00:00
reject ( ` Failed to create ${ project } / ${ slug } on Transifex: ${ err } ` ) ;
2017-03-17 16:24:18 +00:00
} ) ;
2017-03-20 14:01:18 +00:00
request . write ( data ) ;
request . end ( ) ;
2017-03-02 20:53:02 +00:00
} ) ;
}
/ * *
* The following link provides information about how Transifex handles updates of a resource file :
* https : //dev.befoolish.co/tx-docs/public/projects/updating-content#what-happens-when-you-update-files
* /
2017-05-15 13:59:14 +00:00
function updateResource ( project : string , slug : string , xlfFile : File , apiHostname : string , credentials : string ) : Promise < any > {
2017-03-20 14:01:18 +00:00
return new Promise ( ( resolve , reject ) = > {
const data = JSON . stringify ( { content : xlfFile.contents.toString ( ) } ) ;
2017-03-17 16:24:18 +00:00
const options = {
2017-03-20 14:01:18 +00:00
hostname : apiHostname ,
path : ` /api/2/project/ ${ project } /resource/ ${ slug } /content ` ,
headers : {
'Content-Type' : 'application/json' ,
'Content-Length' : Buffer . byteLength ( data )
} ,
auth : credentials ,
method : 'PUT'
2017-03-17 16:24:18 +00:00
} ;
2017-03-02 20:53:02 +00:00
2017-06-30 09:42:52 +00:00
let request = https . request ( options , ( res ) = > {
2017-03-17 16:24:18 +00:00
if ( res . statusCode === 200 ) {
2017-03-20 14:01:18 +00:00
res . setEncoding ( 'utf8' ) ;
let responseBuffer : string = '' ;
res . on ( 'data' , function ( chunk ) {
responseBuffer += chunk ;
} ) ;
res . on ( 'end' , ( ) = > {
const response = JSON . parse ( responseBuffer ) ;
log ( ` Resource ${ project } / ${ slug } successfully updated on Transifex. Strings added: ${ response . strings_added } , updated: ${ response . strings_added } , deleted: ${ response . strings_added } ` ) ;
resolve ( ) ;
} ) ;
2017-03-17 16:24:18 +00:00
} else {
2017-03-20 14:01:18 +00:00
reject ( ` Something went wrong in the request updating ${ slug } in ${ project } . ${ res . statusCode } ` ) ;
2017-03-17 16:24:18 +00:00
}
2017-04-05 09:58:31 +00:00
} ) ;
request . on ( 'error' , ( err ) = > {
2017-03-20 14:01:18 +00:00
reject ( ` Failed to update ${ project } / ${ slug } on Transifex: ${ err } ` ) ;
2017-03-17 16:24:18 +00:00
} ) ;
2017-03-20 14:01:18 +00:00
request . write ( data ) ;
request . end ( ) ;
2017-03-02 20:53:02 +00:00
} ) ;
}
2018-01-25 20:13:24 +00:00
// cache resources
2018-01-26 09:00:11 +00:00
let _coreAndExtensionResources : Resource [ ] ;
2018-01-25 20:13:24 +00:00
2018-01-29 15:35:01 +00:00
export function pullCoreAndExtensionsXlfFiles ( apiHostname : string , username : string , password : string , language : Language , externalExtensions? : Map < string > ) : NodeJS . ReadableStream {
2018-01-26 09:00:11 +00:00
if ( ! _coreAndExtensionResources ) {
_coreAndExtensionResources = [ ] ;
2018-01-25 20:13:24 +00:00
// editor and workbench
const json = JSON . parse ( fs . readFileSync ( './build/lib/i18n.resources.json' , 'utf8' ) ) ;
2018-01-26 09:00:11 +00:00
_coreAndExtensionResources . push ( . . . json . editor ) ;
_coreAndExtensionResources . push ( . . . json . workbench ) ;
2018-01-25 20:13:24 +00:00
// extensions
let extensionsToLocalize = Object . create ( null ) ;
glob . sync ( './extensions/**/*.nls.json' , ) . forEach ( extension = > extensionsToLocalize [ extension . split ( '/' ) [ 2 ] ] = true ) ;
glob . sync ( './extensions/*/node_modules/vscode-nls' , ) . forEach ( extension = > extensionsToLocalize [ extension . split ( '/' ) [ 2 ] ] = true ) ;
Object . keys ( extensionsToLocalize ) . forEach ( extension = > {
2018-01-26 09:00:11 +00:00
_coreAndExtensionResources . push ( { name : extension , project : extensionsProject } ) ;
2017-03-02 20:53:02 +00:00
} ) ;
2018-01-29 15:35:01 +00:00
if ( externalExtensions ) {
for ( let resourceName in externalExtensions ) {
_coreAndExtensionResources . push ( { name : resourceName , project : extensionsProject } ) ;
}
}
2017-03-02 20:53:02 +00:00
}
2018-01-26 09:00:11 +00:00
return pullXlfFiles ( apiHostname , username , password , language , _coreAndExtensionResources ) ;
2017-03-02 20:53:02 +00:00
}
2018-01-25 20:13:24 +00:00
export function pullSetupXlfFiles ( apiHostname : string , username : string , password : string , language : Language , includeDefault : boolean ) : NodeJS . ReadableStream {
2018-01-26 09:00:11 +00:00
let setupResources = [ { name : 'setup_messages' , project : workbenchProject } ] ;
2018-01-25 20:13:24 +00:00
if ( includeDefault ) {
2018-01-26 09:00:11 +00:00
setupResources . push ( { name : 'setup_default' , project : setupProject } ) ;
2017-03-02 20:53:02 +00:00
}
2018-01-25 20:13:24 +00:00
return pullXlfFiles ( apiHostname , username , password , language , setupResources ) ;
}
2017-03-02 20:53:02 +00:00
2018-01-25 20:13:24 +00:00
function pullXlfFiles ( apiHostname : string , username : string , password : string , language : Language , resources : Resource [ ] ) : NodeJS . ReadableStream {
2017-03-20 14:01:18 +00:00
const credentials = ` ${ username } : ${ password } ` ;
2018-01-25 20:13:24 +00:00
let expectedTranslationsCount = resources . length ;
2017-03-02 20:53:02 +00:00
let translationsRetrieved = 0 , called = false ;
2017-05-15 13:59:14 +00:00
return readable ( function ( count , callback ) {
2017-03-02 20:53:02 +00:00
// Mark end of stream when all resources were retrieved
if ( translationsRetrieved === expectedTranslationsCount ) {
return this . emit ( 'end' ) ;
}
if ( ! called ) {
called = true ;
const stream = this ;
2018-01-25 20:13:24 +00:00
resources . map ( function ( resource ) {
retrieveResource ( language , resource , apiHostname , credentials ) . then ( ( file : File ) = > {
if ( file ) {
2017-04-05 16:12:24 +00:00
stream . emit ( 'data' , file ) ;
2018-01-25 20:13:24 +00:00
}
translationsRetrieved ++ ;
} ) . catch ( error = > { throw new Error ( error ) ; } ) ;
2017-03-02 20:53:02 +00:00
} ) ;
}
callback ( ) ;
} ) ;
}
2018-01-16 09:57:27 +00:00
const limiter = new Limiter < File > ( NUMBER_OF_CONCURRENT_DOWNLOADS ) ;
2017-03-02 20:53:02 +00:00
2018-01-25 20:13:24 +00:00
function retrieveResource ( language : Language , resource : Resource , apiHostname , credentials ) : Promise < File > {
2018-01-16 09:57:27 +00:00
return limiter . queue ( ( ) = > new Promise < File > ( ( resolve , reject ) = > {
2017-04-05 16:12:24 +00:00
const slug = resource . name . replace ( /\//g , '_' ) ;
const project = resource . project ;
2018-01-29 17:00:16 +00:00
let transifexLanguageId = language . id === 'ps' ? 'en' : language . transifexId || language . id ;
2017-04-05 16:12:24 +00:00
const options = {
hostname : apiHostname ,
2018-01-25 20:13:24 +00:00
path : ` /api/2/project/ ${ project } /resource/ ${ slug } /translation/ ${ transifexLanguageId } ?file&mode=onlyreviewed ` ,
2017-04-05 16:12:24 +00:00
auth : credentials ,
2018-01-16 09:57:27 +00:00
port : 443 ,
2017-04-05 16:12:24 +00:00
method : 'GET'
} ;
2018-01-29 15:35:01 +00:00
console . log ( '[transifex] Fetching ' + options . path ) ;
2017-04-05 16:12:24 +00:00
2017-06-30 09:42:52 +00:00
let request = https . request ( options , ( res ) = > {
2017-05-15 13:59:14 +00:00
let xlfBuffer : Buffer [ ] = [ ] ;
res . on ( 'data' , ( chunk : Buffer ) = > xlfBuffer . push ( chunk ) ) ;
res . on ( 'end' , ( ) = > {
if ( res . statusCode === 200 ) {
2018-01-25 20:13:24 +00:00
resolve ( new File ( { contents : Buffer.concat ( xlfBuffer ) , path : ` ${ project } / ${ slug } .xlf ` } ) ) ;
} else if ( res . statusCode === 404 ) {
2018-01-29 15:35:01 +00:00
console . log ( ` [transifex] ${ slug } in ${ project } returned no data. ` ) ;
2018-01-25 20:13:24 +00:00
resolve ( null ) ;
} else {
reject ( ` ${ slug } in ${ project } returned no data. Response code: ${ res . statusCode } . ` ) ;
2017-05-15 13:59:14 +00:00
}
} ) ;
2017-04-05 16:12:24 +00:00
} ) ;
request . on ( 'error' , ( err ) = > {
2018-01-16 09:57:27 +00:00
reject ( ` Failed to query resource ${ slug } with the following error: ${ err } . ${ options . path } ` ) ;
2017-04-05 16:12:24 +00:00
} ) ;
request . end ( ) ;
2018-01-16 09:57:27 +00:00
} ) ) ;
2017-04-05 16:12:24 +00:00
}
2018-01-25 20:13:24 +00:00
export function prepareI18nFiles ( ) : ThroughStream {
2017-04-13 16:19:51 +00:00
let parsePromises : Promise < ParsedXLF [ ] > [ ] = [ ] ;
2018-01-25 20:13:24 +00:00
return through ( function ( this : ThroughStream , xlf : File ) {
2017-03-02 20:53:02 +00:00
let stream = this ;
2017-04-13 16:19:51 +00:00
let parsePromise = XLF . parse ( xlf . contents . toString ( ) ) ;
parsePromises . push ( parsePromise ) ;
parsePromise . then (
2018-01-25 20:13:24 +00:00
resolvedFiles = > {
2017-03-02 20:53:02 +00:00
resolvedFiles . forEach ( file = > {
2018-01-25 20:13:24 +00:00
let translatedFile = createI18nFile ( file . originalFilePath , file . messages ) ;
stream . queue ( translatedFile ) ;
} ) ;
}
) ;
} , function ( ) {
Promise . all ( parsePromises )
. then ( ( ) = > { this . queue ( null ) ; } )
. catch ( reason = > { throw new Error ( reason ) ; } ) ;
} ) ;
}
2017-03-02 20:53:02 +00:00
2018-01-25 20:13:24 +00:00
function createI18nFile ( originalFilePath : string , messages : any ) : File {
let result = Object . create ( null ) ;
result [ '' ] = [
'--------------------------------------------------------------------------------------------' ,
'Copyright (c) Microsoft Corporation. All rights reserved.' ,
'Licensed under the MIT License. See License.txt in the project root for license information.' ,
'--------------------------------------------------------------------------------------------' ,
'Do not edit this file. It is machine generated.'
] ;
for ( let key of Object . keys ( messages ) ) {
result [ key ] = messages [ key ] ;
}
let content = JSON . stringify ( result , null , '\t' ) . replace ( /\r\n/g , '\n' ) ;
return new File ( {
path : path.join ( originalFilePath + '.i18n.json' ) ,
contents : new Buffer ( content , 'utf8' )
} ) ;
}
2017-03-02 20:53:02 +00:00
2018-01-25 20:13:24 +00:00
interface I18nPack {
version : string ;
contents : {
[ path : string ] : Map < string > ;
} ;
}
const i18nPackVersion = "1.0.0" ;
2018-01-29 15:35:01 +00:00
export interface TranslationPath {
id : string ;
resourceName : string ;
}
export function pullI18nPackFiles ( apiHostname : string , username : string , password : string , language : Language , resultingTranslationPaths : TranslationPath [ ] ) : NodeJS . ReadableStream {
return pullCoreAndExtensionsXlfFiles ( apiHostname , username , password , language , externalExtensionsWithTranslations )
2018-01-29 17:00:16 +00:00
. pipe ( prepareI18nPackFiles ( externalExtensionsWithTranslations , resultingTranslationPaths , language . id === 'ps' ) ) ;
2018-01-25 20:13:24 +00:00
}
2018-01-29 17:00:16 +00:00
export function prepareI18nPackFiles ( externalExtensions : Map < string > , resultingTranslationPaths : TranslationPath [ ] , pseudo = false ) : NodeJS . ReadWriteStream {
2018-01-25 20:13:24 +00:00
let parsePromises : Promise < ParsedXLF [ ] > [ ] = [ ] ;
let mainPack : I18nPack = { version : i18nPackVersion , contents : { } } ;
let extensionsPacks : Map < I18nPack > = { } ;
return through ( function ( this : ThroughStream , xlf : File ) {
let stream = this ;
2018-01-26 16:27:41 +00:00
let project = path . dirname ( xlf . path ) ;
let resource = path . basename ( xlf . path , '.xlf' ) ;
2018-01-29 17:00:16 +00:00
let contents = xlf . contents . toString ( ) ;
let parsePromise = pseudo ? XLF . parsePseudo ( contents ) : XLF . parse ( contents ) ;
2018-01-25 20:13:24 +00:00
parsePromises . push ( parsePromise ) ;
parsePromise . then (
resolvedFiles = > {
resolvedFiles . forEach ( file = > {
const path = file . originalFilePath ;
const firstSlash = path . indexOf ( '/' ) ;
2018-01-26 16:27:41 +00:00
if ( project === extensionsProject ) {
let extPack = extensionsPacks [ resource ] ;
if ( ! extPack ) {
extPack = extensionsPacks [ resource ] = { version : i18nPackVersion , contents : { } } ;
2018-01-25 20:13:24 +00:00
}
2018-01-29 15:35:01 +00:00
const externalId = externalExtensions [ resource ] ;
if ( ! externalId ) { // internal extension: remove 'extensions/extensionId/' segnent
const secondSlash = path . indexOf ( '/' , firstSlash + 1 ) ;
extPack . contents [ path . substr ( secondSlash + 1 ) ] = file . messages ;
} else {
extPack . contents [ path ] = file . messages ;
}
2017-03-02 20:53:02 +00:00
} else {
2018-01-26 16:27:41 +00:00
mainPack . contents [ path . substr ( firstSlash + 1 ) ] = file . messages ;
2017-03-02 20:53:02 +00:00
}
} ) ;
}
) ;
2017-05-15 13:59:14 +00:00
} , function ( ) {
2017-04-13 16:19:51 +00:00
Promise . all ( parsePromises )
2018-01-25 20:13:24 +00:00
. then ( ( ) = > {
const translatedMainFile = createI18nFile ( './main' , mainPack ) ;
2018-01-29 15:35:01 +00:00
resultingTranslationPaths . push ( { id : 'vscode' , resourceName : 'main.i18n.json' } ) ;
2018-01-25 20:13:24 +00:00
this . queue ( translatedMainFile ) ;
for ( let extension in extensionsPacks ) {
const translatedExtFile = createI18nFile ( ` ./extensions/ ${ extension } ` , extensionsPacks [ extension ] ) ;
this . queue ( translatedExtFile ) ;
2018-01-29 15:35:01 +00:00
const externalExtensionId = externalExtensions [ extension ] ;
if ( externalExtensionId ) {
resultingTranslationPaths . push ( { id : externalExtensionId , resourceName : ` extensions/ ${ extension } .i18n.json ` } ) ;
} else {
resultingTranslationPaths . push ( { id : ` vscode. ${ extension } ` , resourceName : ` extensions/ ${ extension } .i18n.json ` } ) ;
}
2018-01-25 20:13:24 +00:00
}
this . queue ( null ) ;
} )
2017-04-13 16:19:51 +00:00
. catch ( reason = > { throw new Error ( reason ) ; } ) ;
2017-03-02 20:53:02 +00:00
} ) ;
}
2018-01-25 20:13:24 +00:00
export function prepareIslFiles ( language : Language , innoSetupConfig : InnoSetup ) : ThroughStream {
let parsePromises : Promise < ParsedXLF [ ] > [ ] = [ ] ;
2017-03-02 20:53:02 +00:00
2018-01-25 20:13:24 +00:00
return through ( function ( this : ThroughStream , xlf : File ) {
let stream = this ;
let parsePromise = XLF . parse ( xlf . contents . toString ( ) ) ;
parsePromises . push ( parsePromise ) ;
parsePromise . then (
resolvedFiles = > {
resolvedFiles . forEach ( file = > {
if ( path . basename ( file . originalFilePath ) === 'Default' && ! innoSetupConfig . defaultInfo ) {
return ;
}
let translatedFile = createIslFile ( file . originalFilePath , file . messages , language , innoSetupConfig ) ;
stream . queue ( translatedFile ) ;
} ) ;
}
) ;
} , function ( ) {
Promise . all ( parsePromises )
. then ( ( ) = > { this . queue ( null ) ; } )
. catch ( reason = > { throw new Error ( reason ) ; } ) ;
2017-03-02 20:53:02 +00:00
} ) ;
}
2018-01-25 20:13:24 +00:00
function createIslFile ( originalFilePath : string , messages : Map < string > , language : Language , innoSetup : InnoSetup ) : File {
2017-03-02 20:53:02 +00:00
let content : string [ ] = [ ] ;
let originalContent : TextModel ;
if ( path . basename ( originalFilePath ) === 'Default' ) {
originalContent = new TextModel ( fs . readFileSync ( originalFilePath + '.isl' , 'utf8' ) ) ;
} else {
originalContent = new TextModel ( fs . readFileSync ( originalFilePath + '.en.isl' , 'utf8' ) ) ;
}
originalContent . lines . forEach ( line = > {
if ( line . length > 0 ) {
let firstChar = line . charAt ( 0 ) ;
if ( firstChar === '[' || firstChar === ';' ) {
if ( line === '; *** Inno Setup version 5.5.3+ English messages ***' ) {
2018-01-25 20:13:24 +00:00
content . push ( ` ; *** Inno Setup version 5.5.3+ ${ innoSetup . defaultInfo . name } messages *** ` ) ;
2017-03-02 20:53:02 +00:00
} else {
content . push ( line ) ;
}
} else {
let sections : string [ ] = line . split ( '=' ) ;
let key = sections [ 0 ] ;
let translated = line ;
if ( key ) {
if ( key === 'LanguageName' ) {
2018-01-25 20:13:24 +00:00
translated = ` ${ key } = ${ innoSetup . defaultInfo . name } ` ;
2017-03-02 20:53:02 +00:00
} else if ( key === 'LanguageID' ) {
2018-01-25 20:13:24 +00:00
translated = ` ${ key } = ${ innoSetup . defaultInfo . id } ` ;
2017-03-02 20:53:02 +00:00
} else if ( key === 'LanguageCodePage' ) {
2018-01-25 20:13:24 +00:00
translated = ` ${ key } = ${ innoSetup . codePage . substr ( 2 ) } ` ;
2017-03-02 20:53:02 +00:00
} else {
let translatedMessage = messages [ key ] ;
if ( translatedMessage ) {
translated = ` ${ key } = ${ translatedMessage } ` ;
}
}
}
content . push ( translated ) ;
}
}
} ) ;
2017-04-13 16:19:51 +00:00
const basename = path . basename ( originalFilePath ) ;
2018-01-25 20:13:24 +00:00
const filePath = ` ${ basename } . ${ language . id } .isl ` ;
2017-03-02 20:53:02 +00:00
return new File ( {
path : filePath ,
2018-01-25 20:13:24 +00:00
contents : iconv.encode ( new Buffer ( content . join ( '\r\n' ) , 'utf8' ) , innoSetup . codePage )
2017-03-02 20:53:02 +00:00
} ) ;
}
function encodeEntities ( value : string ) : string {
var result : string [ ] = [ ] ;
for ( var i = 0 ; i < value . length ; i ++ ) {
var ch = value [ i ] ;
switch ( ch ) {
case '<' :
result . push ( '<' ) ;
break ;
case '>' :
result . push ( '>' ) ;
break ;
case '&' :
result . push ( '&' ) ;
break ;
default :
result . push ( ch ) ;
}
}
return result . join ( '' ) ;
}
2017-05-15 13:59:14 +00:00
function decodeEntities ( value : string ) : string {
2017-03-02 20:53:02 +00:00
return value . replace ( /</g , '<' ) . replace ( />/g , '>' ) . replace ( /&/g , '&' ) ;
2018-01-29 17:00:16 +00:00
}
function pseudify ( message : string ) {
return '\uFF3B' + message . replace ( /[aouei]/g , '$&$&' ) + '\uFF3D' ;
2016-03-08 10:26:06 +00:00
}