consume swc as libaray and wire up into the existing transpiler logic

This commit is contained in:
Johannes 2022-09-07 10:46:27 +02:00
parent d7fafc7ada
commit 87e54a0759
No known key found for this signature in database
GPG Key ID: 6DEF802A22264FCA
7 changed files with 273 additions and 140 deletions

View File

@ -20,7 +20,7 @@ gulp.task(compileApiProposalNamesTask);
gulp.task(watchApiProposalNamesTask);
// SWC Client Transpile
const transpileClientSWCTask = task.define('transpile-client-swc', task.series(util.rimraf('out'), util.buildWebNodePaths('out'), transpileClientSWC('src', 'out')));
const transpileClientSWCTask = task.define('transpile-client-swc', task.series(util.rimraf('out'), util.buildWebNodePaths('out'), transpileTask('src', 'out', true)));
gulp.task(transpileClientSWCTask);
// Transpile only

View File

@ -4,7 +4,7 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.watchApiProposalNamesTask = exports.compileApiProposalNamesTask = exports.watchTask = exports.compileTask = exports.transpileTask = exports.transpileClientSWC = void 0;
exports.watchApiProposalNamesTask = exports.compileApiProposalNamesTask = exports.watchTask = exports.compileTask = exports.transpileTask = void 0;
const es = require("event-stream");
const fs = require("fs");
const gulp = require("gulp");
@ -18,28 +18,7 @@ const ansiColors = require("ansi-colors");
const os = require("os");
const File = require("vinyl");
const task = require("./task");
const swc_1 = require("./swc");
const watch = require('./watch');
// --- SWC: transpile -------------------------------------
function transpileClientSWC(src, out) {
return function () {
// run SWC sync and put files straight onto the disk
const swcPromise = (0, swc_1.createSwcClientStream)().exec();
// copy none TS resources, like CSS, images, onto the disk
const bom = require('gulp-bom');
const utf8Filter = util.filter(data => /(\/|\\)test(\/|\\).*utf8/.test(data.path));
const tsFilter = util.filter(data => !/\.ts$/.test(data.path));
const srcStream = gulp.src(`${src}/**`, { base: `${src}` });
const copyStream = srcStream
.pipe(utf8Filter)
.pipe(bom()) // this is required to preserve BOM in test files that loose it otherwise
.pipe(utf8Filter.restore)
.pipe(tsFilter);
const copyPromise = util.streamToPromise(copyStream.pipe(gulp.dest(out)));
return Promise.all([swcPromise, copyPromise]);
};
}
exports.transpileClientSWC = transpileClientSWC;
// --- gulp-tsb: compile and transpile --------------------------------
const reporter = (0, reporter_1.createReporter)();
function getTypeScriptCompilerOptions(src) {
@ -64,7 +43,11 @@ function createCompile(src, build, emitError, transpileOnly) {
if (!build) {
overrideOptions.inlineSourceMap = true;
}
const compilation = tsb.create(projectPath, overrideOptions, { verbose: false, transpileOnly }, err => reporter(err));
const compilation = tsb.create(projectPath, overrideOptions, {
verbose: false,
transpileOnly: Boolean(transpileOnly),
transpileWithSwc: typeof transpileOnly !== 'boolean' && transpileOnly.swc
}, err => reporter(err));
function pipeline(token) {
const bom = require('gulp-bom');
const utf8Filter = util.filter(data => /(\/|\\)test(\/|\\).*utf8/.test(data.path));
@ -95,9 +78,9 @@ function createCompile(src, build, emitError, transpileOnly) {
};
return pipeline;
}
function transpileTask(src, out) {
function transpileTask(src, out, swc) {
return function () {
const transpile = createCompile(src, false, true, true);
const transpile = createCompile(src, false, true, { swc });
const srcPipe = gulp.src(`${src}/**`, { base: `${src}` });
return srcPipe
.pipe(transpile())

View File

@ -17,38 +17,9 @@ import * as os from 'os';
import ts = require('typescript');
import * as File from 'vinyl';
import * as task from './task';
import { createSwcClientStream } from './swc';
const watch = require('./watch');
// --- SWC: transpile -------------------------------------
export function transpileClientSWC(src: string, out: string) {
return function () {
// run SWC sync and put files straight onto the disk
const swcPromise = createSwcClientStream().exec();
// copy none TS resources, like CSS, images, onto the disk
const bom = require('gulp-bom') as typeof import('gulp-bom');
const utf8Filter = util.filter(data => /(\/|\\)test(\/|\\).*utf8/.test(data.path));
const tsFilter = util.filter(data => !/\.ts$/.test(data.path));
const srcStream = gulp.src(`${src}/**`, { base: `${src}` });
const copyStream = srcStream
.pipe(utf8Filter)
.pipe(bom()) // this is required to preserve BOM in test files that loose it otherwise
.pipe(utf8Filter.restore)
.pipe(tsFilter);
const copyPromise = util.streamToPromise(copyStream.pipe(gulp.dest(out)));
return Promise.all([swcPromise, copyPromise]);
};
}
// --- gulp-tsb: compile and transpile --------------------------------
const reporter = createReporter();
@ -68,7 +39,7 @@ function getTypeScriptCompilerOptions(src: string): ts.CompilerOptions {
return options;
}
function createCompile(src: string, build: boolean, emitError: boolean, transpileOnly: boolean) {
function createCompile(src: string, build: boolean, emitError: boolean, transpileOnly: boolean | { swc: boolean }) {
const tsb = require('./tsb') as typeof import('./tsb');
const sourcemaps = require('gulp-sourcemaps') as typeof import('gulp-sourcemaps');
@ -79,7 +50,11 @@ function createCompile(src: string, build: boolean, emitError: boolean, transpil
overrideOptions.inlineSourceMap = true;
}
const compilation = tsb.create(projectPath, overrideOptions, { verbose: false, transpileOnly }, err => reporter(err));
const compilation = tsb.create(projectPath, overrideOptions, {
verbose: false,
transpileOnly: Boolean(transpileOnly),
transpileWithSwc: typeof transpileOnly !== 'boolean' && transpileOnly.swc
}, err => reporter(err));
function pipeline(token?: util.ICancellationToken) {
const bom = require('gulp-bom') as typeof import('gulp-bom');
@ -115,11 +90,11 @@ function createCompile(src: string, build: boolean, emitError: boolean, transpil
return pipeline;
}
export function transpileTask(src: string, out: string): () => NodeJS.ReadWriteStream {
export function transpileTask(src: string, out: string, swc: boolean): () => NodeJS.ReadWriteStream {
return function () {
const transpile = createCompile(src, false, true, true);
const transpile = createCompile(src, false, true, { swc });
const srcPipe = gulp.src(`${src}/**`, { base: `${src}` });
return srcPipe

View File

@ -92,7 +92,9 @@ function create(projectPath, existingOptions, config, onError = _defaultOnError)
}
let result;
if (config.transpileOnly) {
const transpiler = new transpiler_1.Transpiler(logFn, printDiagnostic, projectPath, cmdLine);
const transpiler = !config.transpileWithSwc
? new transpiler_1.TscTranspiler(logFn, printDiagnostic, projectPath, cmdLine)
: new transpiler_1.SwcTranspiler(logFn, printDiagnostic, projectPath, cmdLine);
result = (() => createTranspileStream(transpiler));
}
else {

View File

@ -13,7 +13,7 @@ import { strings } from './utils';
import { readFileSync, statSync } from 'fs';
import * as log from 'fancy-log';
import colors = require('ansi-colors');
import { Transpiler } from './transpiler';
import { ITranspiler, SwcTranspiler, TscTranspiler } from './transpiler';
export interface IncrementalCompiler {
(token?: any): Readable & Writable;
@ -36,7 +36,7 @@ const _defaultOnError = (err: string) => console.log(JSON.stringify(err, null, 4
export function create(
projectPath: string,
existingOptions: Partial<ts.CompilerOptions>,
config: { verbose?: boolean; transpileOnly?: boolean; transpileOnlyIncludesDts?: boolean },
config: { verbose?: boolean; transpileOnly?: boolean; transpileOnlyIncludesDts?: boolean; transpileWithSwc?: boolean },
onError: (message: string) => void = _defaultOnError
): IncrementalCompiler {
@ -95,7 +95,7 @@ export function create(
}
// TRANSPILE ONLY stream doing just TS to JS conversion
function createTranspileStream(transpiler: Transpiler): Readable & Writable {
function createTranspileStream(transpiler: ITranspiler): Readable & Writable {
return through(function (this: through.ThroughStream & { queue(a: any): void }, file: Vinyl) {
// give the file to the compiler
if (file.isStream()) {
@ -126,7 +126,9 @@ export function create(
let result: IncrementalCompiler;
if (config.transpileOnly) {
const transpiler = new Transpiler(logFn, printDiagnostic, projectPath, cmdLine);
const transpiler = !config.transpileWithSwc
? new TscTranspiler(logFn, printDiagnostic, projectPath, cmdLine)
: new SwcTranspiler(logFn, printDiagnostic, projectPath, cmdLine);
result = <any>(() => createTranspileStream(transpiler));
} else {
const _builder = builder.createTypeScriptBuilder({ logFn }, projectPath, cmdLine);

View File

@ -3,8 +3,10 @@
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
var _a;
Object.defineProperty(exports, "__esModule", { value: true });
exports.Transpiler = void 0;
exports.SwcTranspiler = exports.TscTranspiler = void 0;
const swc = require("@swc/core");
const ts = require("typescript");
const threads = require("node:worker_threads");
const Vinyl = require("vinyl");
@ -36,6 +38,35 @@ if (!threads.isMainThread) {
threads.parentPort.postMessage(res);
});
}
class OutputFileNameOracle {
constructor(cmdLine, configFilePath) {
this.getOutputFileName = (file) => {
try {
// windows: path-sep normalizing
file = ts.normalizePath(file);
if (!cmdLine.options.configFilePath) {
// this is needed for the INTERNAL getOutputFileNames-call below...
cmdLine.options.configFilePath = configFilePath;
}
const isDts = file.endsWith('.d.ts');
if (isDts) {
file = file.slice(0, -5) + '.ts';
cmdLine.fileNames.push(file);
}
const outfile = ts.getOutputFileNames(cmdLine, file, true)[0];
if (isDts) {
cmdLine.fileNames.pop();
}
return outfile;
}
catch (err) {
console.error(file, cmdLine.fileNames);
console.error(err);
throw new err;
}
};
}
}
class TranspileWorker {
constructor(outFileFn) {
this.id = TranspileWorker.pool++;
@ -112,39 +143,15 @@ class TranspileWorker {
}
}
TranspileWorker.pool = 1;
class Transpiler {
class TscTranspiler {
constructor(logFn, _onError, configFilePath, _cmdLine) {
this._onError = _onError;
this._cmdLine = _cmdLine;
this._workerPool = [];
this._queue = [];
this._allJobs = [];
logFn('Transpile', `will use ${Transpiler.P} transpile worker`);
this._getOutputFileName = (file) => {
try {
// windows: path-sep normalizing
file = ts.normalizePath(file);
if (!_cmdLine.options.configFilePath) {
// this is needed for the INTERNAL getOutputFileNames-call below...
_cmdLine.options.configFilePath = configFilePath;
}
const isDts = file.endsWith('.d.ts');
if (isDts) {
file = file.slice(0, -5) + '.ts';
_cmdLine.fileNames.push(file);
}
const outfile = ts.getOutputFileNames(_cmdLine, file, true)[0];
if (isDts) {
_cmdLine.fileNames.pop();
}
return outfile;
}
catch (err) {
console.error(file, _cmdLine.fileNames);
console.error(err);
throw new err;
}
};
logFn('Transpile', `will use ${TscTranspiler.P} transpile worker`);
this._outputFileNames = new OutputFileNameOracle(_cmdLine, configFilePath);
}
async join() {
// wait for all penindg jobs
@ -161,7 +168,7 @@ class Transpiler {
return;
}
const newLen = this._queue.push(file);
if (newLen > Transpiler.P ** 2) {
if (newLen > TscTranspiler.P ** 2) {
this._consumeQueue();
}
}
@ -172,8 +179,8 @@ class Transpiler {
}
// kinda LAZYily create workers
if (this._workerPool.length === 0) {
for (let i = 0; i < Transpiler.P; i++) {
this._workerPool.push(new TranspileWorker(file => this._getOutputFileName(file)));
for (let i = 0; i < TscTranspiler.P; i++) {
this._workerPool.push(new TranspileWorker(file => this._outputFileNames.getOutputFileName(file)));
}
}
const freeWorker = this._workerPool.filter(w => !w.isBusy);
@ -187,7 +194,7 @@ class Transpiler {
}
const job = new Promise(resolve => {
const consume = () => {
const files = this._queue.splice(0, Transpiler.P);
const files = this._queue.splice(0, TscTranspiler.P);
if (files.length === 0) {
// DONE
resolve(undefined);
@ -210,11 +217,77 @@ class Transpiler {
}
}
}
exports.Transpiler = Transpiler;
Transpiler.P = Math.floor((0, node_os_1.cpus)().length * .5);
exports.TscTranspiler = TscTranspiler;
TscTranspiler.P = Math.floor((0, node_os_1.cpus)().length * .5);
function _isDefaultEmpty(src) {
return src
.replace('"use strict";', '')
.replace(/\/\*[\s\S]*?\*\/|([^\\:]|^)\/\/.*$/gm, '$1')
.trim().length === 0;
}
class SwcTranspiler {
constructor(_logFn, _onError, configFilePath, _cmdLine) {
this._logFn = _logFn;
this._onError = _onError;
this._cmdLine = _cmdLine;
this._jobs = [];
_logFn('Transpile', `will use SWC to transpile source files`);
this._outputFileNames = new OutputFileNameOracle(_cmdLine, configFilePath);
}
async join() {
const jobs = this._jobs.slice();
this._jobs.length = 0;
await Promise.allSettled(jobs);
}
transpile(file) {
if (this._cmdLine.options.noEmit) {
// not doing ANYTHING here
return;
}
const tsSrc = String(file.contents);
const isAmd = /\n(import|export)/m.test(tsSrc);
const t1 = Date.now();
this._jobs.push(swc.transform(tsSrc, isAmd ? SwcTranspiler._swcrcAmd : SwcTranspiler._swcrcEsm).then(output => {
const outBase = this._cmdLine.options.outDir ?? file.base;
const outPath = this._outputFileNames.getOutputFileName(file.path);
this.onOutfile(new Vinyl({
path: outPath,
base: outBase,
contents: Buffer.from(output.code),
}));
this._logFn('Transpile', `swc took ${Date.now() - t1}ms for ${file.path}`);
}).catch(err => {
this._onError(err);
}));
}
}
exports.SwcTranspiler = SwcTranspiler;
_a = SwcTranspiler;
// --- .swcrc
SwcTranspiler._swcrcAmd = {
exclude: '\.js$',
jsc: {
parser: {
syntax: 'typescript',
tsx: false,
decorators: true
},
target: 'es2020',
loose: false,
minify: {
compress: false,
mangle: false
}
},
module: {
type: 'amd',
noInterop: true
},
minify: false,
};
SwcTranspiler._swcrcEsm = {
..._a._swcrcAmd,
module: {
type: 'es6'
}
};

View File

@ -3,6 +3,7 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import * as swc from '@swc/core';
import * as ts from 'typescript';
import * as threads from 'node:worker_threads';
import * as Vinyl from 'vinyl';
@ -48,6 +49,47 @@ if (!threads.isMainThread) {
});
}
class OutputFileNameOracle {
readonly getOutputFileName: (name: string) => string;
constructor(cmdLine: ts.ParsedCommandLine, configFilePath: string) {
// very complicated logic to re-use TS internal functions to know the output path
// given a TS input path and its config
type InternalTsApi = typeof ts & {
normalizePath(path: string): string;
getOutputFileNames(commandLine: ts.ParsedCommandLine, inputFileName: string, ignoreCase: boolean): readonly string[];
};
this.getOutputFileName = (file) => {
try {
// windows: path-sep normalizing
file = (<InternalTsApi>ts).normalizePath(file);
if (!cmdLine.options.configFilePath) {
// this is needed for the INTERNAL getOutputFileNames-call below...
cmdLine.options.configFilePath = configFilePath;
}
const isDts = file.endsWith('.d.ts');
if (isDts) {
file = file.slice(0, -5) + '.ts';
cmdLine.fileNames.push(file);
}
const outfile = (<InternalTsApi>ts).getOutputFileNames(cmdLine, file, true)[0];
if (isDts) {
cmdLine.fileNames.pop();
}
return outfile;
} catch (err) {
console.error(file, cmdLine.fileNames);
console.error(err);
throw new err;
}
};
}
}
class TranspileWorker {
private static pool = 1;
@ -141,12 +183,18 @@ class TranspileWorker {
}
}
export interface ITranspiler {
onOutfile?: (file: Vinyl) => void;
join(): Promise<void>;
transpile(file: Vinyl): void;
}
export class Transpiler {
export class TscTranspiler implements ITranspiler {
static P = Math.floor(cpus().length * .5);
private readonly _getOutputFileName: (name: string) => string;
private readonly _outputFileNames: OutputFileNameOracle;
public onOutfile?: (file: Vinyl) => void;
@ -160,42 +208,8 @@ export class Transpiler {
configFilePath: string,
private readonly _cmdLine: ts.ParsedCommandLine
) {
logFn('Transpile', `will use ${Transpiler.P} transpile worker`);
// very complicated logic to re-use TS internal functions to know the output path
// given a TS input path and its config
type InternalTsApi = typeof ts & {
normalizePath(path: string): string;
getOutputFileNames(commandLine: ts.ParsedCommandLine, inputFileName: string, ignoreCase: boolean): readonly string[];
};
this._getOutputFileName = (file) => {
try {
// windows: path-sep normalizing
file = (<InternalTsApi>ts).normalizePath(file);
if (!_cmdLine.options.configFilePath) {
// this is needed for the INTERNAL getOutputFileNames-call below...
_cmdLine.options.configFilePath = configFilePath;
}
const isDts = file.endsWith('.d.ts');
if (isDts) {
file = file.slice(0, -5) + '.ts';
_cmdLine.fileNames.push(file);
}
const outfile = (<InternalTsApi>ts).getOutputFileNames(_cmdLine, file, true)[0];
if (isDts) {
_cmdLine.fileNames.pop();
}
return outfile;
} catch (err) {
console.error(file, _cmdLine.fileNames);
console.error(err);
throw new err;
}
};
logFn('Transpile', `will use ${TscTranspiler.P} transpile worker`);
this._outputFileNames = new OutputFileNameOracle(_cmdLine, configFilePath);
}
async join() {
@ -218,7 +232,7 @@ export class Transpiler {
}
const newLen = this._queue.push(file);
if (newLen > Transpiler.P ** 2) {
if (newLen > TscTranspiler.P ** 2) {
this._consumeQueue();
}
}
@ -232,8 +246,8 @@ export class Transpiler {
// kinda LAZYily create workers
if (this._workerPool.length === 0) {
for (let i = 0; i < Transpiler.P; i++) {
this._workerPool.push(new TranspileWorker(file => this._getOutputFileName(file)));
for (let i = 0; i < TscTranspiler.P; i++) {
this._workerPool.push(new TranspileWorker(file => this._outputFileNames.getOutputFileName(file)));
}
}
@ -251,7 +265,7 @@ export class Transpiler {
const job = new Promise(resolve => {
const consume = () => {
const files = this._queue.splice(0, Transpiler.P);
const files = this._queue.splice(0, TscTranspiler.P);
if (files.length === 0) {
// DONE
resolve(undefined);
@ -283,3 +297,87 @@ function _isDefaultEmpty(src: string): boolean {
.replace(/\/\*[\s\S]*?\*\/|([^\\:]|^)\/\/.*$/gm, '$1')
.trim().length === 0;
}
export class SwcTranspiler implements ITranspiler {
onOutfile?: ((file: Vinyl) => void) | undefined;
private readonly _outputFileNames: OutputFileNameOracle;
private _jobs: Promise<any>[] = [];
constructor(
private readonly _logFn: (topic: string, message: string) => void,
private readonly _onError: (err: any) => void,
configFilePath: string,
private readonly _cmdLine: ts.ParsedCommandLine
) {
_logFn('Transpile', `will use SWC to transpile source files`);
this._outputFileNames = new OutputFileNameOracle(_cmdLine, configFilePath);
}
async join(): Promise<void> {
const jobs = this._jobs.slice();
this._jobs.length = 0;
await Promise.allSettled(jobs);
}
transpile(file: Vinyl): void {
if (this._cmdLine.options.noEmit) {
// not doing ANYTHING here
return;
}
const tsSrc = String(file.contents);
const isAmd = /\n(import|export)/m.test(tsSrc);
const t1 = Date.now();
this._jobs.push(swc.transform(tsSrc, isAmd ? SwcTranspiler._swcrcAmd : SwcTranspiler._swcrcEsm).then(output => {
const outBase = this._cmdLine.options.outDir ?? file.base;
const outPath = this._outputFileNames.getOutputFileName(file.path);
this.onOutfile!(new Vinyl({
path: outPath,
base: outBase,
contents: Buffer.from(output.code),
}));
this._logFn('Transpile', `swc took ${Date.now() - t1}ms for ${file.path}`);
}).catch(err => {
this._onError(err);
}));
}
// --- .swcrc
private static readonly _swcrcAmd: swc.Options = {
exclude: '\.js$',
jsc: {
parser: {
syntax: 'typescript',
tsx: false,
decorators: true
},
target: 'es2020',
loose: false,
minify: {
compress: false,
mangle: false
}
},
module: {
type: 'amd',
noInterop: true
},
minify: false,
};
private static readonly _swcrcEsm: swc.Options = {
...this._swcrcAmd,
module: {
type: 'es6'
}
};
}