development/deno
development/deno
1
0
Fork 0
mirror of https://github.com/denoland/deno synced 2024-06-30 23:46:19 +00:00
Code Issues Projects Releases Wiki Activity

refactor(ext): remove use of brotli::ffi (#24214)

Browse Source
This commit is contained in:
ud2 2024-06-20 23:14:24 +08:00 committed by GitHub
parent 6ab143335a
commit 88e3f465d3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 181 additions and 294 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
Cargo.lock generated
View File

@ -275,11 +275,11 @@ dependencies = [
[[package]]
name = "async-compression"
version = "0.4.8"
version = "0.4.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07dbbf24db18d609b1462965249abdf49129ccad073ec257da372adc83259c60"
checksum = "cd066d0b4ef8ecb03a55319dc13aa6910616d0f44008a045bb1835af830abff5"
dependencies = [
"brotli 4.0.0",
"brotli",
"flate2",
"futures-core",
"memchr",
@ -506,41 +506,20 @@ dependencies = [
[[package]]
name = "brotli"
version = "3.5.0"
version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d640d25bc63c50fb1f0b545ffd80207d2e10a4c965530809b40ba3386825c391"
checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
"brotli-decompressor 2.5.1",
]
[[package]]
name = "brotli"
version = "4.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "125740193d7fee5cc63ab9e16c2fdc4e07c74ba755cc53b327d6ea029e9fc569"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
"brotli-decompressor 3.0.0",
"brotli-decompressor",
]
[[package]]
name = "brotli-decompressor"
version = "2.5.1"
version = "4.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
]
[[package]]
name = "brotli-decompressor"
version = "3.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "65622a320492e09b5e0ac436b14c54ff68199bac392d0e89a6832c4518eea525"
checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
@ -1515,7 +1494,7 @@ dependencies = [
"async-trait",
"base64 0.21.7",
"bencher",
"brotli 3.5.0",
"brotli",
"bytes",
"cache_control",
"deno_core",
@ -1674,7 +1653,7 @@ dependencies = [
"aead-gcm-stream",
"aes",
"async-trait",
"brotli 3.5.0",
"brotli",
"bytes",
"cbc",
"const-oid",

2
Cargo.toml
View File

@ -90,7 +90,7 @@ async-trait = "0.1.73"
base32 = "=0.4.0"
base64 = "0.21.4"
bencher = "0.1"
brotli = "3.3.4"
brotli = "6.0.0"
bytes = "1.4.0"
cache_control = "=0.2.0"
cbc = { version = "=0.1.2", features = ["alloc"] }

2
ext/http/Cargo.toml
View File

@ -24,7 +24,7 @@ harness = false
async-compression = { version = "0.4", features = ["tokio", "brotli", "gzip"] }
async-trait.workspace = true
base64.workspace = true
brotli = "3.3.4"
brotli.workspace = true
bytes.workspace = true
cache_control.workspace = true
deno_core.workspace = true

135
ext/http/response_body.rs
View File

@ -3,8 +3,10 @@ use std::io::Write;
use std::pin::Pin;
use std::rc::Rc;
use brotli::enc::encode::BrotliEncoderOperation;
use brotli::enc::encode::BrotliEncoderParameter;
use brotli::ffi::compressor::BrotliEncoderState;
use brotli::enc::encode::BrotliEncoderStateStruct;
use brotli::writer::StandardAlloc;
use bytes::Bytes;
use bytes::BytesMut;
use deno_core::error::AnyError;
@ -448,58 +450,24 @@ enum BrotliState {
EndOfStream,
}
struct BrotliEncoderStateWrapper {
stm: *mut BrotliEncoderState,
}
#[pin_project]
pub struct BrotliResponseStream {
state: BrotliState,
stm: BrotliEncoderStateWrapper,
current_cursor: usize,
output_written_so_far: usize,
stm: BrotliEncoderStateStruct<StandardAlloc>,
#[pin]
underlying: ResponseStream,
}
impl Drop for BrotliEncoderStateWrapper {
fn drop(&mut self) {
// SAFETY: since we are dropping, we can be sure that this instance will not
// be used again.
unsafe {
brotli::ffi::compressor::BrotliEncoderDestroyInstance(self.stm);
}
}
}
impl BrotliResponseStream {
pub fn new(underlying: ResponseStream) -> Self {
// SAFETY: creating an FFI instance should be OK with these args.
let stm = unsafe {
let stm = brotli::ffi::compressor::BrotliEncoderCreateInstance(
None,
None,
std::ptr::null_mut(),
);
// Quality level 6 is based on google's nginx default value for on-the-fly compression
// https://github.com/google/ngx_brotli#brotli_comp_level
// lgwin 22 is equivalent to brotli window size of (2**22)-16 bytes (~4MB)
brotli::ffi::compressor::BrotliEncoderSetParameter(
stm,
BrotliEncoderParameter::BROTLI_PARAM_QUALITY,
6,
);
brotli::ffi::compressor::BrotliEncoderSetParameter(
stm,
BrotliEncoderParameter::BROTLI_PARAM_LGWIN,
22,
);
BrotliEncoderStateWrapper { stm }
};
let mut stm = BrotliEncoderStateStruct::new(StandardAlloc::default());
// Quality level 6 is based on google's nginx default value for on-the-fly compression
// https://github.com/google/ngx_brotli#brotli_comp_level
// lgwin 22 is equivalent to brotli window size of (2**22)-16 bytes (~4MB)
stm.set_parameter(BrotliEncoderParameter::BROTLI_PARAM_QUALITY, 6);
stm.set_parameter(BrotliEncoderParameter::BROTLI_PARAM_LGWIN, 22);
Self {
stm,
output_written_so_far: 0,
current_cursor: 0,
state: BrotliState::Streaming,
underlying,
}
@ -546,71 +514,46 @@ impl PollFrame for BrotliResponseStream {
let res = match frame {
ResponseStreamResult::NonEmptyBuf(buf) => {
let mut output_written = 0;
let mut total_output_written = 0;
let mut input_size = buf.len();
let input_buffer = buf.as_ref();
let mut len = max_compressed_size(input_size);
let mut output_buffer = vec![0u8; len];
let mut ob_ptr = output_buffer.as_mut_ptr();
let mut output_buffer = vec![0; max_compressed_size(buf.len())];
let mut output_offset = 0;
// SAFETY: these are okay arguments to these FFI calls.
unsafe {
brotli::ffi::compressor::BrotliEncoderCompressStream(
this.stm.stm,
brotli::ffi::compressor::BrotliEncoderOperation::BROTLI_OPERATION_PROCESS,
&mut input_size,
&input_buffer.as_ptr() as *const *const u8 as *mut *const u8,
&mut len,
&mut ob_ptr,
&mut output_written,
);
total_output_written += output_written;
output_written = 0;
this.stm.compress_stream(
BrotliEncoderOperation::BROTLI_OPERATION_FLUSH,
&mut buf.len(),
&buf,
&mut 0,
&mut output_buffer.len(),
&mut output_buffer,
&mut output_offset,
&mut None,
&mut |_, _, _, _| (),
);
brotli::ffi::compressor::BrotliEncoderCompressStream(
this.stm.stm,
brotli::ffi::compressor::BrotliEncoderOperation::BROTLI_OPERATION_FLUSH,
&mut input_size,
&input_buffer.as_ptr() as *const *const u8 as *mut *const u8,
&mut len,
&mut ob_ptr,
&mut output_written,
);
total_output_written += output_written;
};
output_buffer
.truncate(total_output_written - this.output_written_so_far);
this.output_written_so_far = total_output_written;
output_buffer.truncate(output_offset);
ResponseStreamResult::NonEmptyBuf(BufView::from(output_buffer))
}
ResponseStreamResult::EndOfStream => {
let mut len = 1024usize;
let mut output_buffer = vec![0u8; len];
let mut input_size = 0;
let mut output_written = 0;
let ob_ptr = output_buffer.as_mut_ptr();
let mut output_buffer = vec![0; 1024];
let mut output_offset = 0;
// SAFETY: these are okay arguments to these FFI calls.
unsafe {
brotli::ffi::compressor::BrotliEncoderCompressStream(
this.stm.stm,
brotli::ffi::compressor::BrotliEncoderOperation::BROTLI_OPERATION_FINISH,
&mut input_size,
std::ptr::null_mut(),
&mut len,
&ob_ptr as *const *mut u8 as *mut *mut u8,
&mut output_written,
);
};
this.stm.compress_stream(
BrotliEncoderOperation::BROTLI_OPERATION_FINISH,
&mut 0,
&[],
&mut 0,
&mut output_buffer.len(),
&mut output_buffer,
&mut output_offset,
&mut None,
&mut |_, _, _, _| (),
);
if output_written == 0 {
if output_offset == 0 {
this.state = BrotliState::EndOfStream;
ResponseStreamResult::EndOfStream
} else {
this.state = BrotliState::Flushing;
output_buffer.truncate(output_written - this.output_written_so_far);
output_buffer.truncate(output_offset);
ResponseStreamResult::NonEmptyBuf(BufView::from(output_buffer))
}
}

295
ext/node/ops/zlib/brotli.rs
View File

@ -1,9 +1,13 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use brotli::enc::backward_references::BrotliEncoderMode;
use brotli::enc::encode::BrotliEncoderCompress;
use brotli::enc::encode::BrotliEncoderOperation;
use brotli::enc::encode::BrotliEncoderParameter;
use brotli::ffi::compressor::*;
use brotli::ffi::decompressor::ffi::interface::BrotliDecoderResult;
use brotli::ffi::decompressor::ffi::BrotliDecoderState;
use brotli::ffi::decompressor::*;
use brotli::enc::encode::BrotliEncoderStateStruct;
use brotli::writer::StandardAlloc;
use brotli::BrotliDecompressStream;
use brotli::BrotliResult;
use brotli::BrotliState;
use brotli::Decompressor;
use deno_core::error::type_error;
use deno_core::error::AnyError;
@ -12,14 +16,20 @@ use deno_core::JsBuffer;
use deno_core::OpState;
use deno_core::Resource;
use deno_core::ToJsBuffer;
use std::cell::RefCell;
use std::io::Read;
fn encoder_mode(mode: u32) -> Result<BrotliEncoderMode, AnyError> {
if mode > 6 {
return Err(type_error("Invalid encoder mode"));
}
// SAFETY: mode is a valid discriminant for BrotliEncoderMode
unsafe { Ok(std::mem::transmute::<u32, BrotliEncoderMode>(mode)) }
Ok(match mode {
0 => BrotliEncoderMode::BROTLI_MODE_GENERIC,
1 => BrotliEncoderMode::BROTLI_MODE_TEXT,
2 => BrotliEncoderMode::BROTLI_MODE_FONT,
3 => BrotliEncoderMode::BROTLI_FORCE_LSB_PRIOR,
4 => BrotliEncoderMode::BROTLI_FORCE_MSB_PRIOR,
5 => BrotliEncoderMode::BROTLI_FORCE_UTF8_PRIOR,
6 => BrotliEncoderMode::BROTLI_FORCE_SIGNED_PRIOR,
_ => return Err(type_error("Invalid encoder mode")),
})
}
#[op2(fast)]
@ -31,24 +41,22 @@ pub fn op_brotli_compress(
#[smi] lgwin: i32,
#[smi] mode: u32,
) -> Result<usize, AnyError> {
let in_buffer = buffer.as_ptr();
let in_size = buffer.len();
let out_buffer = out.as_mut_ptr();
let mode = encoder_mode(mode)?;
let mut out_size = out.len();
// SAFETY: in_size and in_buffer, out_size and out_buffer are valid for this call.
if unsafe {
BrotliEncoderCompress(
quality,
lgwin,
encoder_mode(mode)?,
in_size,
in_buffer,
&mut out_size as *mut usize,
out_buffer,
)
} != 1
{
let result = BrotliEncoderCompress(
StandardAlloc::default(),
&mut StandardAlloc::default(),
quality,
lgwin,
mode,
buffer.len(),
buffer,
&mut out_size,
out,
&mut |_, _, _, _| (),
);
if result != 1 {
return Err(type_error("Failed to compress"));
}
@ -80,28 +88,25 @@ pub async fn op_brotli_compress_async(
#[smi] lgwin: i32,
#[smi] mode: u32,
) -> Result<ToJsBuffer, AnyError> {
let mode = encoder_mode(mode)?;
tokio::task::spawn_blocking(move || {
let in_buffer = input.as_ptr();
let in_size = input.len();
let mut out = vec![0u8; max_compressed_size(in_size)];
let out_buffer = out.as_mut_ptr();
let input = &*input;
let mut out = vec![0u8; max_compressed_size(input.len())];
let mut out_size = out.len();
// SAFETY: in_size and in_buffer, out_size and out_buffer
// are valid for this call.
if unsafe {
BrotliEncoderCompress(
quality,
lgwin,
encoder_mode(mode)?,
in_size,
in_buffer,
&mut out_size as *mut usize,
out_buffer,
)
} != 1
{
let result = BrotliEncoderCompress(
StandardAlloc::default(),
&mut StandardAlloc::default(),
quality,
lgwin,
mode,
input.len(),
input,
&mut out_size,
&mut out,
&mut |_, _, _, _| (),
);
if result != 1 {
return Err(type_error("Failed to compress"));
}
@ -112,38 +117,26 @@ pub async fn op_brotli_compress_async(
}
struct BrotliCompressCtx {
inst: *mut BrotliEncoderState,
inst: RefCell<BrotliEncoderStateStruct<StandardAlloc>>,
}
impl Resource for BrotliCompressCtx {}
impl Drop for BrotliCompressCtx {
fn drop(&mut self) {
// SAFETY: `self.inst` is the current brotli encoder instance.
// It is not used after the following call.
unsafe { BrotliEncoderDestroyInstance(self.inst) };
}
}
#[op2]
#[smi]
pub fn op_create_brotli_compress(
state: &mut OpState,
#[serde] params: Vec<(u8, i32)>,
) -> u32 {
let inst =
// SAFETY: Creates a brotli encoder instance for default allocators.
unsafe { BrotliEncoderCreateInstance(None, None, std::ptr::null_mut()) };
let mut inst = BrotliEncoderStateStruct::new(StandardAlloc::default());
for (key, value) in params {
// SAFETY: `key` can range from 0-255.
// Any valid u32 can be used for the `value`.
unsafe {
BrotliEncoderSetParameter(inst, encoder_param(key), value as u32);
}
inst.set_parameter(encoder_param(key), value as u32);
}
state.resource_table.add(BrotliCompressCtx { inst })
state.resource_table.add(BrotliCompressCtx {
inst: RefCell::new(inst),
})
}
fn encoder_param(param: u8) -> BrotliEncoderParameter {
@ -160,30 +153,25 @@ pub fn op_brotli_compress_stream(
#[buffer] output: &mut [u8],
) -> Result<usize, AnyError> {
let ctx = state.resource_table.get::<BrotliCompressCtx>(rid)?;
let mut inst = ctx.inst.borrow_mut();
let mut output_offset = 0;
// SAFETY: TODO(littledivy)
unsafe {
let mut available_in = input.len();
let mut next_in = input.as_ptr();
let mut available_out = output.len();
let mut next_out = output.as_mut_ptr();
if BrotliEncoderCompressStream(
ctx.inst,
BrotliEncoderOperation::BROTLI_OPERATION_PROCESS,
&mut available_in,
&mut next_in,
&mut available_out,
&mut next_out,
std::ptr::null_mut(),
) != 1
{
return Err(type_error("Failed to compress"));
}
// On progress, next_out is advanced and available_out is reduced.
Ok(output.len() - available_out)
let result = inst.compress_stream(
BrotliEncoderOperation::BROTLI_OPERATION_PROCESS,
&mut input.len(),
input,
&mut 0,
&mut output.len(),
output,
&mut output_offset,
&mut None,
&mut |_, _, _, _| (),
);
if !result {
return Err(type_error("Failed to compress"));
}
Ok(output_offset)
}
#[op2(fast)]
@ -194,29 +182,25 @@ pub fn op_brotli_compress_stream_end(
#[buffer] output: &mut [u8],
) -> Result<usize, AnyError> {
let ctx = state.resource_table.get::<BrotliCompressCtx>(rid)?;
let mut inst = ctx.inst.borrow_mut();
let mut output_offset = 0;
// SAFETY: TODO(littledivy)
unsafe {
let mut available_out = output.len();
let mut next_out = output.as_mut_ptr();
let mut total_out = 0;
if BrotliEncoderCompressStream(
ctx.inst,
BrotliEncoderOperation::BROTLI_OPERATION_FINISH,
&mut 0,
std::ptr::null_mut(),
&mut available_out,
&mut next_out,
&mut total_out,
) != 1
{
return Err(type_error("Failed to compress"));
}
// On finish, next_out is advanced and available_out is reduced.
Ok(output.len() - available_out)
let result = inst.compress_stream(
BrotliEncoderOperation::BROTLI_OPERATION_FINISH,
&mut 0,
&[],
&mut 0,
&mut output.len(),
output,
&mut output_offset,
&mut None,
&mut |_, _, _, _| (),
);
if !result {
return Err(type_error("Failed to compress"));
}
Ok(output_offset)
}
fn brotli_decompress(buffer: &[u8]) -> Result<ToJsBuffer, AnyError> {
@ -243,25 +227,22 @@ pub async fn op_brotli_decompress_async(
}
struct BrotliDecompressCtx {
inst: *mut BrotliDecoderState,
inst: RefCell<BrotliState<StandardAlloc, StandardAlloc, StandardAlloc>>,
}
impl Resource for BrotliDecompressCtx {}
impl Drop for BrotliDecompressCtx {
fn drop(&mut self) {
// SAFETY: TODO(littledivy)
unsafe { CBrotliDecoderDestroyInstance(self.inst) };
}
}
#[op2(fast)]
#[smi]
pub fn op_create_brotli_decompress(state: &mut OpState) -> u32 {
let inst =
// SAFETY: TODO(littledivy)
unsafe { CBrotliDecoderCreateInstance(None, None, std::ptr::null_mut()) };
state.resource_table.add(BrotliDecompressCtx { inst })
let inst = BrotliState::new(
StandardAlloc::default(),
StandardAlloc::default(),
StandardAlloc::default(),
);
state.resource_table.add(BrotliDecompressCtx {
inst: RefCell::new(inst),
})
}
#[op2(fast)]
@ -273,32 +254,24 @@ pub fn op_brotli_decompress_stream(
#[buffer] output: &mut [u8],
) -> Result<usize, AnyError> {
let ctx = state.resource_table.get::<BrotliDecompressCtx>(rid)?;
let mut inst = ctx.inst.borrow_mut();
let mut output_offset = 0;
// SAFETY: TODO(littledivy)
unsafe {
let mut available_in = input.len();
let mut next_in = input.as_ptr();
let mut available_out = output.len();
let mut next_out = output.as_mut_ptr();
if matches!(
CBrotliDecoderDecompressStream(
ctx.inst,
&mut available_in,
&mut next_in,
&mut available_out,
&mut next_out,
std::ptr::null_mut(),
),
BrotliDecoderResult::BROTLI_DECODER_RESULT_ERROR
) {
let ec = CBrotliDecoderGetErrorCode(ctx.inst) as i32;
return Err(type_error(format!("Failed to decompress, error {ec}")));
}
// On progress, next_out is advanced and available_out is reduced.
Ok(output.len() - available_out)
let result = BrotliDecompressStream(
&mut input.len(),
&mut 0,
input,
&mut output.len(),
&mut output_offset,
output,
&mut 0,
&mut inst,
);
if matches!(result, BrotliResult::ResultFailure) {
return Err(type_error("Failed to decompress"));
}
Ok(output_offset)
}
#[op2(fast)]
@ -309,30 +282,22 @@ pub fn op_brotli_decompress_stream_end(
#[buffer] output: &mut [u8],
) -> Result<usize, AnyError> {
let ctx = state.resource_table.get::<BrotliDecompressCtx>(rid)?;
let mut inst = ctx.inst.borrow_mut();
let mut output_offset = 0;
// SAFETY: TODO(littledivy)
unsafe {
let mut available_out = output.len();
let mut next_out = output.as_mut_ptr();
let mut available_in = 0;
let mut next_in = [];
let mut total_out = 0;
if matches!(
CBrotliDecoderDecompressStream(
ctx.inst,
&mut available_in,
next_in.as_mut_ptr(),
&mut available_out,
&mut next_out,
&mut total_out,
),
BrotliDecoderResult::BROTLI_DECODER_RESULT_ERROR
) {
return Err(type_error("Failed to decompress"));
}
// On finish, next_out is advanced and available_out is reduced.
Ok(output.len() - available_out)
let result = BrotliDecompressStream(
&mut 0,
&mut 0,
&[],
&mut output.len(),
&mut output_offset,
output,
&mut 0,
&mut inst,
);
if matches!(result, BrotliResult::ResultFailure) {
return Err(type_error("Failed to decompress"));
}
Ok(output_offset)
}