Raw syntax definition loading (#1655)

Cargo.lock

@@ -2059,6 +2059,7 @@ dependencies = [
  "serde_json",
  "thiserror",
  "walkdir",
+ "yaml-rust",
 ]
 
 [[package]]

NOTICE

@@ -1,5 +1,27 @@
 Licenses for third party components used by this project can be found below.
 
+================================================================================
+The 0BSD License applies to:
+
+* The S-Expression sublime-syntax in `assets/files/SExpressions.sublime-syntax`
+  which is adapted from the S-Expression syntax definition in the Sublime Text
+  package `S-Expressions` (https://github.com/whitequark/Sublime-S-Expressions)
+
+BSD Zero Clause License (0BSD)
+
+Permission to use, copy, modify, and/or distribute this software for
+any purpose with or without fee is hereby granted.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
+AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
+OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+================================================================================
+
 ================================================================================
 The MIT License applies to:
 

assets/files/SExpressions.sublime-syntax

@@ -0,0 +1,73 @@
+%YAML 1.2
+---
+name: S Expressions
+file_extensions: ["sexp"]
+scope: source.sexpr
+
+contexts:
+  main:
+    - match: '(;+).*$'
+      scope: comment.line.sexpr
+      captures:
+        1: punctuation.definition.comment.sexpr
+    - match: '#;'
+      scope: punctuation.definition.comment.sexpr
+      push: comment
+    - match: '#\|'
+      scope: punctuation.definition.comment.sexpr
+      push: block_comment
+
+    - match: '"'
+      scope: punctuation.definition.string.begin.sexpr
+      push: string_unquote
+    - match: '\d+\.\d+'
+      scope: constant.numeric.float.sexpr
+    - match: '\d+'
+      scope: constant.numeric.integer.sexpr
+    - match: '\w+'
+      scope: constant.other.sexpr
+    - match: '\('
+      scope: punctuation.section.parens.begin.sexpr
+      push: main_rparen
+    - match: '\)'
+      scope: invalid.illegal.stray-paren-end
+
+  string_unquote:
+    - meta_scope: string.quoted.double.sexpr
+    - match: '""'
+      scope: constant.character.escape.sexpr
+    - match: '"'
+      scope: punctuation.definition.string.end.sexpr
+      pop: true
+
+  main_rparen:
+    - match: '\)'
+      scope: punctuation.section.parens.end.sexpr
+      pop: true
+    - include: main
+
+  comment:
+    - meta_scope: comment.block.sexpr
+    - match: '\('
+      set: comment_rparen
+
+  comment_lparen:
+    - meta_scope: comment.block.sexpr
+    - match: '\('
+      push: comment_rparen
+    - match: '\)'
+      scope: invalid.illegal.stray-paren-end
+
+  comment_rparen:
+    - meta_scope: comment.block.sexpr
+    - match: '\)'
+      pop: true
+    - include: comment
+
+  block_comment:
+    - meta_scope: comment.block.sexpr
+    - match: '#\|'
+      push: block_comment
+    - match: '\|#'
+      scope: punctuation.definition.comment.sexpr
+      pop: true

crates/typst-library/Cargo.toml

@@ -39,7 +39,7 @@ rustybuzz = "0.7"
 serde_json = "1"
 serde_yaml = "0.8"
 smallvec = "1.10"
-syntect = { version = "5", default-features = false, features = ["parsing", "regex-fancy"] }
+syntect = { version = "5", default-features = false, features = ["parsing", "regex-fancy", "yaml-load"] }
 time = { version = "0.3.20", features = ["formatting"] }
 toml = { version = "0.7.3", default-features = false, features = ["parse"] }
 tracing = "0.1.37"

crates/typst-library/src/layout/par.rs

@@ -119,11 +119,11 @@ pub struct ParElem {
 }
 
 impl Construct for ParElem {
-    fn construct(_: &mut Vm, args: &mut Args) -> SourceResult<Content> {
+    fn construct(vm: &mut Vm, args: &mut Args) -> SourceResult<Content> {
         // The paragraph constructor is special: It doesn't create a paragraph
         // element. Instead, it just ensures that the passed content lives in a
         // separate paragraph and styles it.
-        let styles = Self::set(args)?;
+        let styles = Self::set(vm, args)?;
         let body = args.expect::<Content>("body")?;
         Ok(Content::sequence([
             ParbreakElem::new().pack(),

crates/typst-library/src/text/mod.rs

@@ -520,11 +520,11 @@ impl TextElem {
 }
 
 impl Construct for TextElem {
-    fn construct(_: &mut Vm, args: &mut Args) -> SourceResult<Content> {
+    fn construct(vm: &mut Vm, args: &mut Args) -> SourceResult<Content> {
         // The text constructor is special: It doesn't create a text element.
         // Instead, it leaves the passed argument structurally unchanged, but
         // styles all text in it.
-        let styles = Self::set(args)?;
+        let styles = Self::set(vm, args)?;
         let body = args.expect::<Content>("body")?;
         Ok(body.styled_with_map(styles))
     }

crates/typst-library/src/text/raw.rs

@@ -1,6 +1,13 @@
+use std::hash::Hash;
+use std::sync::Arc;
+
 use once_cell::sync::Lazy;
+use once_cell::unsync::Lazy as UnsyncLazy;
 use syntect::highlighting as synt;
+use syntect::parsing::{SyntaxDefinition, SyntaxSet, SyntaxSetBuilder};
+use typst::diag::FileError;
 use typst::syntax::{self, LinkedNode};
+use typst::util::Bytes;
 
 use super::{
     FontFamily, FontList, Hyphenate, LinebreakElem, SmartQuoteElem, TextElem, TextSize,
@@ -126,6 +133,33 @@ pub struct RawElem {
     /// ````
     #[default(HorizontalAlign(GenAlign::Start))]
     pub align: HorizontalAlign,
+
+    /// One or multiple additional syntax definitions to load. The syntax
+    /// definitions should be in the `sublime-syntax` file format.
+    ///
+    /// ````example
+    /// #set raw(syntaxes: "SExpressions.sublime-syntax")
+    ///
+    /// ```sexp
+    /// (defun factorial (x)
+    ///   (if (zerop x)
+    ///     ; with a comment
+    ///     1
+    ///     (* x (factorial (- x 1)))))
+    /// ```
+    /// ````
+    #[parse(
+        let (syntaxes, data) = parse_syntaxes(vm, args)?;
+        syntaxes
+    )]
+    #[fold]
+    pub syntaxes: SyntaxPaths,
+
+    /// The raw file buffers.
+    #[internal]
+    #[parse(data)]
+    #[fold]
+    pub data: Vec<Bytes>,
 }
 
 impl RawElem {
@@ -163,6 +197,9 @@ impl Show for RawElem {
             .map(to_typst)
             .map_or(Color::BLACK, Color::from);
 
+        let extra_syntaxes =
+            UnsyncLazy::new(|| load(&self.syntaxes(styles), &self.data(styles)).unwrap());
+
         let mut realized = if matches!(lang.as_deref(), Some("typ" | "typst" | "typc")) {
             let root = match lang.as_deref() {
                 Some("typc") => syntax::parse_code(&text),
@@ -181,9 +218,16 @@ impl Show for RawElem {
             );
 
             Content::sequence(seq)
-        } else if let Some(syntax) =
-            lang.and_then(|token| SYNTAXES.find_syntax_by_token(&token))
-        {
+        } else if let Some((syntax_set, syntax)) = lang.and_then(|token| {
+            SYNTAXES
+                .find_syntax_by_token(&token)
+                .map(|syntax| (&*SYNTAXES, syntax))
+                .or_else(|| {
+                    extra_syntaxes
+                        .find_syntax_by_token(&token)
+                        .map(|syntax| (&**extra_syntaxes, syntax))
+                })
+        }) {
             let mut seq = vec![];
             let mut highlighter = syntect::easy::HighlightLines::new(syntax, &THEME);
             for (i, line) in text.lines().enumerate() {
@@ -192,7 +236,7 @@ impl Show for RawElem {
                 }
 
                 for (style, piece) in
-                    highlighter.highlight_line(line, &SYNTAXES).into_iter().flatten()
+                    highlighter.highlight_line(line, syntax_set).into_iter().flatten()
                 {
                     seq.push(styled(piece, foreground.into(), style));
                 }
@@ -319,6 +363,71 @@ fn to_syn(RgbaColor { r, g, b, a }: RgbaColor) -> synt::Color {
     synt::Color { r, g, b, a }
 }
 
+/// A list of bibliography file paths.
+#[derive(Debug, Default, Clone, Hash)]
+pub struct SyntaxPaths(Vec<EcoString>);
+
+cast! {
+    SyntaxPaths,
+    self => self.0.into_value(),
+    v: EcoString => Self(vec![v]),
+    v: Array => Self(v.into_iter().map(Value::cast).collect::<StrResult<_>>()?),
+}
+
+impl Fold for SyntaxPaths {
+    type Output = Self;
+
+    fn fold(mut self, outer: Self::Output) -> Self::Output {
+        self.0.extend(outer.0);
+        self
+    }
+}
+
+/// Load a syntax set from a list of syntax file paths.
+#[comemo::memoize]
+fn load(paths: &SyntaxPaths, bytes: &[Bytes]) -> StrResult<Arc<SyntaxSet>> {
+    let mut out = SyntaxSetBuilder::new();
+
+    // We might have multiple sublime-syntax/yaml files
+    for (path, bytes) in paths.0.iter().zip(bytes.iter()) {
+        let src = std::str::from_utf8(bytes).map_err(|_| FileError::InvalidUtf8)?;
+        out.add(
+            SyntaxDefinition::load_from_str(src, false, None)
+                .map_err(|e| eco_format!("failed to parse syntax file `{path}`: {e}"))?,
+        );
+    }
+
+    Ok(Arc::new(out.build()))
+}
+
+/// Function to parse the syntaxes argument.
+/// Much nicer than having it be part of the `element` macro.
+fn parse_syntaxes(
+    vm: &mut Vm,
+    args: &mut Args,
+) -> SourceResult<(Option<SyntaxPaths>, Option<Vec<Bytes>>)> {
+    let Some(Spanned { v: paths, span }) =
+        args.named::<Spanned<SyntaxPaths>>("syntaxes")?
+    else {
+        return Ok((None, None));
+    };
+
+    // Load syntax files.
+    let data = paths
+        .0
+        .iter()
+        .map(|path| {
+            let id = vm.location().join(path).at(span)?;
+            vm.world().file(id).at(span)
+        })
+        .collect::<SourceResult<Vec<Bytes>>>()?;
+
+    // Check that parsing works.
+    let _ = load(&paths, &data).at(span)?;
+
+    Ok((Some(paths), Some(data)))
+}
+
 /// The syntect syntax definitions.
 ///
 /// Code for syntax set generation is below. The `syntaxes` directory is from

crates/typst-macros/src/element.rs

@@ -523,6 +523,7 @@ fn create_set_impl(element: &Elem) -> TokenStream {
     quote! {
         impl ::typst::model::Set for #ident {
             fn set(
+                vm: &mut Vm,
                 args: &mut ::typst::eval::Args,
             ) -> ::typst::diag::SourceResult<::typst::model::Styles> {
                 let mut styles = ::typst::model::Styles::new();

crates/typst/src/eval/mod.rs

@@ -1471,7 +1471,7 @@ impl Eval for ast::SetRule {
             })
             .at(target.span())?;
         let args = self.args().eval(vm)?;
-        Ok(target.set(args)?.spanned(self.span()))
+        Ok(target.set(vm, args)?.spanned(self.span()))
     }
 }
 

crates/typst/src/model/content.rs

@@ -7,8 +7,8 @@ use comemo::Prehashed;
 use ecow::{eco_format, EcoString, EcoVec};
 
 use super::{
-    element, Behave, Behaviour, ElemFunc, Element, Fold, Guard, Label, Locatable,
-    Location, Recipe, Selector, Style, Styles, Synthesize,
+    element, Behave, Behaviour, ElemFunc, Element, Guard, Label, Locatable, Location,
+    Recipe, Selector, Style, Styles, Synthesize,
 };
 use crate::diag::{SourceResult, StrResult};
 use crate::doc::Meta;
@@ -588,15 +588,6 @@ impl Behave for MetaElem {
     }
 }
 
-impl Fold for Vec<Meta> {
-    type Output = Self;
-
-    fn fold(mut self, outer: Self::Output) -> Self::Output {
-        self.extend(outer);
-        self
-    }
-}
-
 /// Tries to extract the plain-text representation of the element.
 pub trait PlainText {
     /// Write this element's plain text into the given buffer.

crates/typst/src/model/element.rs

@@ -32,7 +32,7 @@ pub trait Construct {
 /// An element's set rule.
 pub trait Set {
     /// Parse relevant arguments into style properties for this element.
-    fn set(args: &mut Args) -> SourceResult<Styles>;
+    fn set(vm: &mut Vm, args: &mut Args) -> SourceResult<Styles>;
 }
 
 /// An element's function.
@@ -80,8 +80,8 @@ impl ElemFunc {
     }
 
     /// Execute the set rule for the element and return the resulting style map.
-    pub fn set(self, mut args: Args) -> SourceResult<Styles> {
-        let styles = (self.0.set)(&mut args)?;
+    pub fn set(self, vm: &mut Vm, mut args: Args) -> SourceResult<Styles> {
+        let styles = (self.0.set)(vm, &mut args)?;
         args.finish()?;
         Ok(styles)
     }
@@ -128,7 +128,7 @@ pub struct NativeElemFunc {
     /// The element's constructor.
     pub construct: fn(&mut Vm, &mut Args) -> SourceResult<Content>,
     /// The element's set rule.
-    pub set: fn(&mut Args) -> SourceResult<Styles>,
+    pub set: fn(&mut Vm, &mut Args) -> SourceResult<Styles>,
     /// Details about the function.
     pub info: Lazy<FuncInfo>,
 }

crates/typst/src/model/styles.rs

@@ -748,3 +748,12 @@ where
         self.map(|inner| inner.fold(outer.unwrap_or_default()))
     }
 }
+
+impl<T> Fold for Vec<T> {
+    type Output = Vec<T>;
+
+    fn fold(mut self, outer: Self::Output) -> Self::Output {
+        self.extend(outer);
+        self
+    }
+}

tests/typ/text/raw-syntaxes.typ

@@ -0,0 +1,14 @@
+// Test code highlighting with custom syntaxes.
+
+---
+#set page(width: 180pt)
+#set text(6pt)
+#set raw(syntaxes: "/files/SExpressions.sublime-syntax")
+
+```sexp
+(defun factorial (x)
+  (if (zerop x)
+    ; with a comment
+    1
+    (* x (factorial (- x 1)))))
+```