perf: move jupyter esm out of main snapshot (#21163)

Towards https://github.com/denoland/deno/issues/21136
This commit is contained in:
Divy Srivastava 2023-11-14 13:06:00 -08:00 committed by GitHub
parent d342c0df71
commit ab0c637425
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 383 additions and 368 deletions

View file

@ -206,7 +206,7 @@ pub struct ReplFlags {
pub is_default_command: bool,
}
#[derive(Clone, Debug, Eq, PartialEq)]
#[derive(Clone, Debug, Eq, PartialEq, Default)]
pub struct RunFlags {
pub script: String,
pub watch: Option<WatchFlagsWithPaths>,
@ -311,6 +311,10 @@ impl DenoSubcommand {
pub fn is_run(&self) -> bool {
matches!(self, Self::Run(_))
}
pub fn is_test_or_jupyter(&self) -> bool {
matches!(self, Self::Test(_) | Self::Jupyter(_))
}
}
impl Default for DenoSubcommand {

View file

@ -332,7 +332,6 @@ deno_core::extension!(
esm = [
dir "js",
"40_testing.js",
"40_jupyter.js",
"99_main.js"
],
customizer = |ext: &mut deno_core::Extension| {

View file

@ -634,6 +634,7 @@ impl CliFactory {
Ok(CliMainWorkerFactory::new(
StorageKeyResolver::from_options(&self.options),
self.options.sub_command().clone(),
npm_resolver.clone(),
node_resolver.clone(),
self.blob_store().clone(),

View file

@ -1,4 +1,5 @@
// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
// deno-lint-ignore-file
/*
* @module mod
@ -35,184 +36,215 @@
* }, { raw: true });
* ```
*/
{
const internals = Deno[Deno.internal];
const core = internals.core;
const core = globalThis.Deno.core;
const $display = Symbol.for("Jupyter.display");
const internals = globalThis.__bootstrap.internals;
/** Escape copied from https://deno.land/std@0.192.0/html/entities.ts */
const rawToEntityEntries = [
["&", "&amp;"],
["<", "&lt;"],
[">", "&gt;"],
['"', "&quot;"],
["'", "&#39;"],
];
const $display = Symbol.for("Jupyter.display");
const rawToEntity = new Map(rawToEntityEntries);
/** Escape copied from https://deno.land/std@0.192.0/html/entities.ts */
const rawToEntityEntries = [
["&", "&amp;"],
["<", "&lt;"],
[">", "&gt;"],
['"', "&quot;"],
["'", "&#39;"],
];
const rawRe = new RegExp(`[${[...rawToEntity.keys()].join("")}]`, "g");
const rawToEntity = new Map(rawToEntityEntries);
const rawRe = new RegExp(`[${[...rawToEntity.keys()].join("")}]`, "g");
function escapeHTML(str) {
return str.replaceAll(
rawRe,
(m) => rawToEntity.has(m) ? rawToEntity.get(m) : m,
);
}
/** Duck typing our way to common visualization and tabular libraries */
/** Vegalite */
function isVegaLike(obj) {
return obj !== null && typeof obj === "object" && "toSpec" in obj;
}
function extractVega(obj) {
const spec = obj.toSpec();
if (!("$schema" in spec)) {
return null;
function escapeHTML(str) {
return str.replaceAll(
rawRe,
(m) => rawToEntity.has(m) ? rawToEntity.get(m) : m,
);
}
if (typeof spec !== "object") {
return null;
}
let mediaType = "application/vnd.vega.v5+json";
if (spec.$schema === "https://vega.github.io/schema/vega-lite/v4.json") {
mediaType = "application/vnd.vegalite.v4+json";
} else if (
spec.$schema === "https://vega.github.io/schema/vega-lite/v5.json"
) {
mediaType = "application/vnd.vegalite.v5+json";
}
return {
[mediaType]: spec,
};
}
/** Polars */
function isDataFrameLike(obj) {
const isObject = obj !== null && typeof obj === "object";
if (!isObject) {
return false;
}
const df = obj;
return df.schema !== void 0 && typeof df.schema === "object" &&
df.head !== void 0 && typeof df.head === "function" &&
df.toRecords !== void 0 && typeof df.toRecords === "function";
}
/**
* Map Polars DataType to JSON Schema data types.
* @param dataType - The Polars DataType.
* @returns The corresponding JSON Schema data type.
*/
function mapPolarsTypeToJSONSchema(colType) {
const typeMapping = {
Null: "null",
Bool: "boolean",
Int8: "integer",
Int16: "integer",
Int32: "integer",
Int64: "integer",
UInt8: "integer",
UInt16: "integer",
UInt32: "integer",
UInt64: "integer",
Float32: "number",
Float64: "number",
Date: "string",
Datetime: "string",
Utf8: "string",
Categorical: "string",
List: "array",
Struct: "object",
};
// These colTypes are weird. When you console.dir or console.log them
// they show a `DataType` field, however you can't access it directly until you
// convert it to JSON
const dataType = colType.toJSON()["DataType"];
return typeMapping[dataType] || "string";
}
function extractDataFrame(df) {
const fields = [];
const schema = {
fields,
};
let data = [];
// Convert DataFrame schema to Tabular DataResource schema
for (const [colName, colType] of Object.entries(df.schema)) {
const dataType = mapPolarsTypeToJSONSchema(colType);
schema.fields.push({
name: colName,
type: dataType,
});
/** Duck typing our way to common visualization and tabular libraries */
/** Vegalite */
function isVegaLike(obj) {
return obj !== null && typeof obj === "object" && "toSpec" in obj;
}
function extractVega(obj) {
const spec = obj.toSpec();
if (!("$schema" in spec)) {
return null;
}
if (typeof spec !== "object") {
return null;
}
let mediaType = "application/vnd.vega.v5+json";
if (spec.$schema === "https://vega.github.io/schema/vega-lite/v4.json") {
mediaType = "application/vnd.vegalite.v4+json";
} else if (
spec.$schema === "https://vega.github.io/schema/vega-lite/v5.json"
) {
mediaType = "application/vnd.vegalite.v5+json";
}
return {
[mediaType]: spec,
};
}
/** Polars */
function isDataFrameLike(obj) {
const isObject = obj !== null && typeof obj === "object";
if (!isObject) {
return false;
}
const df = obj;
return df.schema !== void 0 && typeof df.schema === "object" &&
df.head !== void 0 && typeof df.head === "function" &&
df.toRecords !== void 0 && typeof df.toRecords === "function";
}
/**
* Map Polars DataType to JSON Schema data types.
* @param dataType - The Polars DataType.
* @returns The corresponding JSON Schema data type.
*/
function mapPolarsTypeToJSONSchema(colType) {
const typeMapping = {
Null: "null",
Bool: "boolean",
Int8: "integer",
Int16: "integer",
Int32: "integer",
Int64: "integer",
UInt8: "integer",
UInt16: "integer",
UInt32: "integer",
UInt64: "integer",
Float32: "number",
Float64: "number",
Date: "string",
Datetime: "string",
Utf8: "string",
Categorical: "string",
List: "array",
Struct: "object",
};
// These colTypes are weird. When you console.dir or console.log them
// they show a `DataType` field, however you can't access it directly until you
// convert it to JSON
const dataType = colType.toJSON()["DataType"];
return typeMapping[dataType] || "string";
}
// Convert DataFrame data to row-oriented JSON
//
// TODO(rgbkrk): Determine how to get the polars format max rows
// Since pl.setTblRows just sets env var POLARS_FMT_MAX_ROWS,
// we probably just have to pick a number for now.
//
data = df.head(50).toRecords();
let htmlTable = "<table>";
htmlTable += "<thead><tr>";
schema.fields.forEach((field) => {
htmlTable += `<th>${escapeHTML(String(field.name))}</th>`;
});
htmlTable += "</tr></thead>";
htmlTable += "<tbody>";
df.head(10).toRecords().forEach((row) => {
htmlTable += "<tr>";
function extractDataFrame(df) {
const fields = [];
const schema = {
fields,
};
let data = [];
// Convert DataFrame schema to Tabular DataResource schema
for (const [colName, colType] of Object.entries(df.schema)) {
const dataType = mapPolarsTypeToJSONSchema(colType);
schema.fields.push({
name: colName,
type: dataType,
});
}
// Convert DataFrame data to row-oriented JSON
//
// TODO(rgbkrk): Determine how to get the polars format max rows
// Since pl.setTblRows just sets env var POLARS_FMT_MAX_ROWS,
// we probably just have to pick a number for now.
//
data = df.head(50).toRecords();
let htmlTable = "<table>";
htmlTable += "<thead><tr>";
schema.fields.forEach((field) => {
htmlTable += `<td>${escapeHTML(String(row[field.name]))}</td>`;
htmlTable += `<th>${escapeHTML(String(field.name))}</th>`;
});
htmlTable += "</tr>";
});
htmlTable += "</tbody></table>";
return {
"application/vnd.dataresource+json": { data, schema },
"text/html": htmlTable,
};
}
/** Canvas */
function isCanvasLike(obj) {
return obj !== null && typeof obj === "object" && "toDataURL" in obj;
}
/** Possible HTML and SVG Elements */
function isSVGElementLike(obj) {
return obj !== null && typeof obj === "object" && "outerHTML" in obj &&
typeof obj.outerHTML === "string" && obj.outerHTML.startsWith("<svg");
}
function isHTMLElementLike(obj) {
return obj !== null && typeof obj === "object" && "outerHTML" in obj &&
typeof obj.outerHTML === "string";
}
/** Check to see if an object already contains a `Symbol.for("Jupyter.display") */
function hasDisplaySymbol(obj) {
return obj !== null && typeof obj === "object" && $display in obj &&
typeof obj[$display] === "function";
}
function makeDisplayable(obj) {
return {
[$display]: () => obj,
};
}
/**
* Format an object for displaying in Deno
*
* @param obj - The object to be displayed
* @returns MediaBundle
*/
async function format(obj) {
if (hasDisplaySymbol(obj)) {
return await obj[$display]();
htmlTable += "</tr></thead>";
htmlTable += "<tbody>";
df.head(10).toRecords().forEach((row) => {
htmlTable += "<tr>";
schema.fields.forEach((field) => {
htmlTable += `<td>${escapeHTML(String(row[field.name]))}</td>`;
});
htmlTable += "</tr>";
});
htmlTable += "</tbody></table>";
return {
"application/vnd.dataresource+json": { data, schema },
"text/html": htmlTable,
};
}
if (typeof obj !== "object") {
/** Canvas */
function isCanvasLike(obj) {
return obj !== null && typeof obj === "object" && "toDataURL" in obj;
}
/** Possible HTML and SVG Elements */
function isSVGElementLike(obj) {
return obj !== null && typeof obj === "object" && "outerHTML" in obj &&
typeof obj.outerHTML === "string" && obj.outerHTML.startsWith("<svg");
}
function isHTMLElementLike(obj) {
return obj !== null && typeof obj === "object" && "outerHTML" in obj &&
typeof obj.outerHTML === "string";
}
/** Check to see if an object already contains a `Symbol.for("Jupyter.display") */
function hasDisplaySymbol(obj) {
return obj !== null && typeof obj === "object" && $display in obj &&
typeof obj[$display] === "function";
}
function makeDisplayable(obj) {
return {
[$display]: () => obj,
};
}
/**
* Format an object for displaying in Deno
*
* @param obj - The object to be displayed
* @returns MediaBundle
*/
async function format(obj) {
if (hasDisplaySymbol(obj)) {
return await obj[$display]();
}
if (typeof obj !== "object") {
return {
"text/plain": Deno[Deno.internal].inspectArgs(["%o", obj], {
colors: !Deno.noColor,
}),
};
}
if (isCanvasLike(obj)) {
const dataURL = obj.toDataURL();
const parts = dataURL.split(",");
const mime = parts[0].split(":")[1].split(";")[0];
const data = parts[1];
return {
[mime]: data,
};
}
if (isVegaLike(obj)) {
return extractVega(obj);
}
if (isDataFrameLike(obj)) {
return extractDataFrame(obj);
}
if (isSVGElementLike(obj)) {
return {
"image/svg+xml": obj.outerHTML,
};
}
if (isHTMLElementLike(obj)) {
return {
"text/html": obj.outerHTML,
};
}
return {
"text/plain": Deno[Deno.internal].inspectArgs(["%o", obj], {
colors: !Deno.noColor,
@ -220,211 +252,180 @@ async function format(obj) {
};
}
if (isCanvasLike(obj)) {
const dataURL = obj.toDataURL();
const parts = dataURL.split(",");
const mime = parts[0].split(":")[1].split(";")[0];
const data = parts[1];
return {
[mime]: data,
/**
* This function creates a tagged template function for a given media type.
* The tagged template function takes a template string and returns a displayable object.
*
* @param mediatype - The media type for the tagged template function.
* @returns A function that takes a template string and returns a displayable object.
*/
function createTaggedTemplateDisplayable(mediatype) {
return (strings, ...values) => {
const payload = strings.reduce(
(acc, string, i) =>
acc + string + (values[i] !== undefined ? values[i] : ""),
"",
);
return makeDisplayable({ [mediatype]: payload });
};
}
if (isVegaLike(obj)) {
return extractVega(obj);
}
if (isDataFrameLike(obj)) {
return extractDataFrame(obj);
}
if (isSVGElementLike(obj)) {
return {
"image/svg+xml": obj.outerHTML,
};
}
if (isHTMLElementLike(obj)) {
return {
"text/html": obj.outerHTML,
};
}
return {
"text/plain": Deno[Deno.internal].inspectArgs(["%o", obj], {
colors: !Deno.noColor,
}),
};
}
/**
* This function creates a tagged template function for a given media type.
* The tagged template function takes a template string and returns a displayable object.
*
* @param mediatype - The media type for the tagged template function.
* @returns A function that takes a template string and returns a displayable object.
*/
function createTaggedTemplateDisplayable(mediatype) {
return (strings, ...values) => {
const payload = strings.reduce(
(acc, string, i) =>
acc + string + (values[i] !== undefined ? values[i] : ""),
"",
);
return makeDisplayable({ [mediatype]: payload });
};
}
/**
* Show Markdown in Jupyter frontends with a tagged template function.
*
* Takes a template string and returns a displayable object for Jupyter frontends.
*
* @example
* Create a Markdown view.
*
* ```typescript
* md`# Notebooks in TypeScript via Deno ![Deno logo](https://github.com/denoland.png?size=32)
*
* * TypeScript ${Deno.version.typescript}
* * V8 ${Deno.version.v8}
* * Deno ${Deno.version.deno}
*
* Interactive compute with Jupyter _built into Deno_!
* `
* ```
*/
const md = createTaggedTemplateDisplayable("text/markdown");
/**
* Show HTML in Jupyter frontends with a tagged template function.
*
* Takes a template string and returns a displayable object for Jupyter frontends.
*
* @example
* Create an HTML view.
* ```typescript
* html`<h1>Hello, world!</h1>`
* ```
*/
const html = createTaggedTemplateDisplayable("text/html");
/**
* SVG Tagged Template Function.
*
* Takes a template string and returns a displayable object for Jupyter frontends.
*
* Example usage:
*
* svg`<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100">
* <circle cx="50" cy="50" r="40" stroke="green" stroke-width="4" fill="yellow" />
* </svg>`
*/
const svg = createTaggedTemplateDisplayable("image/svg+xml");
function isMediaBundle(obj) {
if (obj == null || typeof obj !== "object" || Array.isArray(obj)) {
return false;
}
for (const key in obj) {
if (typeof key !== "string") {
return false;
}
}
return true;
}
async function formatInner(obj, raw) {
if (raw && isMediaBundle(obj)) {
return obj;
} else {
return await format(obj);
}
}
internals.jupyter = { formatInner };
function enableJupyter() {
const {
op_jupyter_broadcast,
} = core.ensureFastOps();
async function broadcast(
msgType,
content,
{ metadata = {}, buffers = [] } = {},
) {
await op_jupyter_broadcast(msgType, content, metadata, buffers);
}
async function broadcastResult(executionCount, result) {
try {
if (result === undefined) {
return;
}
const data = await format(result);
await broadcast("execute_result", {
execution_count: executionCount,
data,
metadata: {},
});
} catch (err) {
if (err instanceof Error) {
const stack = err.stack || "";
await broadcast("error", {
ename: err.name,
evalue: err.message,
traceback: stack.split("\n"),
});
} else if (typeof err == "string") {
await broadcast("error", {
ename: "Error",
evalue: err,
traceback: [],
});
} else {
await broadcast("error", {
ename: "Error",
evalue:
"An error occurred while formatting a result, but it could not be identified",
traceback: [],
});
}
}
}
internals.jupyter.broadcastResult = broadcastResult;
/**
* Display function for Jupyter Deno Kernel.
* Mimics the behavior of IPython's `display(obj, raw=True)` function to allow
* asynchronous displaying of objects in Jupyter.
* Show Markdown in Jupyter frontends with a tagged template function.
*
* @param obj - The object to be displayed
* @param options - Display options
* Takes a template string and returns a displayable object for Jupyter frontends.
*
* @example
* Create a Markdown view.
*
* ```typescript
* md`# Notebooks in TypeScript via Deno ![Deno logo](https://github.com/denoland.png?size=32)
*
* * TypeScript ${Deno.version.typescript}
* * V8 ${Deno.version.v8}
* * Deno ${Deno.version.deno}
*
* Interactive compute with Jupyter _built into Deno_!
* `
* ```
*/
async function display(obj, options = { raw: false, update: false }) {
const bundle = await formatInner(obj, options.raw);
let messageType = "display_data";
if (options.update) {
messageType = "update_display_data";
const md = createTaggedTemplateDisplayable("text/markdown");
/**
* Show HTML in Jupyter frontends with a tagged template function.
*
* Takes a template string and returns a displayable object for Jupyter frontends.
*
* @example
* Create an HTML view.
* ```typescript
* html`<h1>Hello, world!</h1>`
* ```
*/
const html = createTaggedTemplateDisplayable("text/html");
/**
* SVG Tagged Template Function.
*
* Takes a template string and returns a displayable object for Jupyter frontends.
*
* Example usage:
*
* svg`<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100">
* <circle cx="50" cy="50" r="40" stroke="green" stroke-width="4" fill="yellow" />
* </svg>`
*/
const svg = createTaggedTemplateDisplayable("image/svg+xml");
function isMediaBundle(obj) {
if (obj == null || typeof obj !== "object" || Array.isArray(obj)) {
return false;
}
let transient = {};
if (options.display_id) {
transient = { display_id: options.display_id };
for (const key in obj) {
if (typeof key !== "string") {
return false;
}
}
await broadcast(messageType, {
data: bundle,
metadata: {},
transient,
});
return;
return true;
}
globalThis.Deno.jupyter = {
broadcast,
display,
format,
md,
html,
svg,
$display,
};
}
async function formatInner(obj, raw) {
if (raw && isMediaBundle(obj)) {
return obj;
} else {
return await format(obj);
}
}
internals.enableJupyter = enableJupyter;
internals.jupyter = { formatInner };
function enableJupyter() {
const {
op_jupyter_broadcast,
} = core.ensureFastOps();
async function broadcast(
msgType,
content,
{ metadata = {}, buffers = [] } = {},
) {
await op_jupyter_broadcast(msgType, content, metadata, buffers);
}
async function broadcastResult(executionCount, result) {
try {
if (result === undefined) {
return;
}
const data = await format(result);
await broadcast("execute_result", {
execution_count: executionCount,
data,
metadata: {},
});
} catch (err) {
if (err instanceof Error) {
const stack = err.stack || "";
await broadcast("error", {
ename: err.name,
evalue: err.message,
traceback: stack.split("\n"),
});
} else if (typeof err == "string") {
await broadcast("error", {
ename: "Error",
evalue: err,
traceback: [],
});
} else {
await broadcast("error", {
ename: "Error",
evalue:
"An error occurred while formatting a result, but it could not be identified",
traceback: [],
});
}
}
}
internals.jupyter.broadcastResult = broadcastResult;
/**
* Display function for Jupyter Deno Kernel.
* Mimics the behavior of IPython's `display(obj, raw=True)` function to allow
* asynchronous displaying of objects in Jupyter.
*
* @param obj - The object to be displayed
* @param options - Display options
*/
async function display(obj, options = { raw: false, update: false }) {
const bundle = await formatInner(obj, options.raw);
let messageType = "display_data";
if (options.update) {
messageType = "update_display_data";
}
let transient = {};
if (options.display_id) {
transient = { display_id: options.display_id };
}
await broadcast(messageType, {
data: bundle,
metadata: {},
transient,
});
return;
}
globalThis.Deno.jupyter = {
broadcast,
display,
format,
md,
html,
svg,
$display,
};
}
internals.enableJupyter = enableJupyter;
}

View file

@ -1,4 +1,3 @@
// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
import "ext:cli/40_testing.js";
import "ext:cli/40_jupyter.js";
import "ext:cli/runtime/js/99_main.js";

View file

@ -22,7 +22,6 @@ deno_core::extension!(cli,
esm = [
dir "js",
"40_testing.js",
"40_jupyter.js",
"99_main.js"
],
customizer = |ext: &mut deno_core::Extension| {

View file

@ -438,6 +438,7 @@ pub async fn run(
});
let worker_factory = CliMainWorkerFactory::new(
StorageKeyResolver::empty(),
crate::args::DenoSubcommand::Run(Default::default()),
npm_resolver,
node_resolver,
Default::default(),

View file

@ -46,6 +46,7 @@ use deno_semver::package::PackageReqReference;
use tokio::select;
use crate::args::package_json::PackageJsonDeps;
use crate::args::DenoSubcommand;
use crate::args::StorageKeyResolver;
use crate::emit::Emitter;
use crate::errors;
@ -107,6 +108,7 @@ pub struct CliMainWorkerOptions {
struct SharedWorkerState {
options: CliMainWorkerOptions,
subcommand: DenoSubcommand,
storage_key_resolver: StorageKeyResolver,
npm_resolver: Arc<dyn CliNpmResolver>,
node_resolver: Arc<NodeResolver>,
@ -372,6 +374,7 @@ impl CliMainWorkerFactory {
#[allow(clippy::too_many_arguments)]
pub fn new(
storage_key_resolver: StorageKeyResolver,
subcommand: DenoSubcommand,
npm_resolver: Arc<dyn CliNpmResolver>,
node_resolver: Arc<NodeResolver>,
blob_store: Arc<BlobStore>,
@ -388,6 +391,7 @@ impl CliMainWorkerFactory {
Self {
shared: Arc::new(SharedWorkerState {
options,
subcommand,
storage_key_resolver,
npm_resolver,
node_resolver,
@ -600,12 +604,19 @@ impl CliMainWorkerFactory {
skip_op_registration: shared.options.skip_op_registration,
};
let worker = MainWorker::bootstrap_from_options(
let mut worker = MainWorker::bootstrap_from_options(
main_module.clone(),
permissions,
options,
);
if self.shared.subcommand.is_test_or_jupyter() {
worker.js_runtime.execute_script_static(
"40_jupyter.js",
include_str!("js/40_jupyter.js"),
)?;
}
Ok(CliMainWorker {
main_module,
is_main_cjs,