utilities/pueue
utilities/pueue
1
0
Fork 0
mirror of https://github.com/nukesor/pueue synced 2024-10-02 22:13:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add new logic to limit remote log output

Browse source
This commit is contained in:
Arne Beer 2021-01-15 21:44:39 +01:00
parent 1065811a88
commit 198aeeee53
5 changed files with 75 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
CHANGELOG.md
View file

@ -12,6 +12,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
This is used to only show the last X lines of each task's stdout and stderr.
- Add the `--full` flag to the `log` subcommand.
This is used to show the whole logfile of each task's stdout and stderr.
- Add the `--successful-only` flag to the `clean` subcommand.
This let's keep you all important logs of failed tasks, while freeing up some screen space.
### Changed

4
client/cli.rs
View file

@ -262,7 +262,7 @@ pub enum SubCommand {
/// Display the log output of finished tasks.
/// Prints either all logs or only the logs of specified tasks.
///
/// When looking at multiple logs, only the last 20 lines will be shown
/// When looking at multiple logs, only the last few lines will be shown
Log {
/// View the task output of these specific tasks.
task_ids: Vec<usize>,
@ -276,7 +276,7 @@ pub enum SubCommand {
#[clap(short, long, conflicts_with = "full")]
lines: Option<usize>,
/// Show the whole std_out and std_err output.
/// Show the whole stdout and stderr output.
/// This is the default if only a single task is being looked at.
#[clap(short, long)]
full: bool,

33
client/display/log.rs
View file

@ -1,14 +1,12 @@
use std::{collections::BTreeMap, io::BufReader};
use std::{
fs::File,
io::{self, Stdout},
};
use std::collections::BTreeMap;
use std::fs::File;
use std::io::{self, Stdout};
use anyhow::Result;
use comfy_table::*;
use snap::read::FrameDecoder;
use pueue::log::get_log_file_handles;
use pueue::log::{get_log_file_handles, read_last_lines};
use pueue::network::message::TaskLogMessage;
use pueue::settings::Settings;
use pueue::task::{TaskResult, TaskStatus};
@ -143,7 +141,7 @@ pub fn print_log(message: &mut TaskLogMessage, settings: &Settings, lines: Optio
/// The daemon didn't send any log output, thereby we didn't request any.
/// If that's the case, read the log files from the local pueue directory
pub fn print_local_log(task_id: usize, settings: &Settings, lines: Option<usize>) {
let (mut stdout_log, mut stderr_log) =
let (mut stdout_file, mut stderr_file) =
match get_log_file_handles(task_id, &settings.shared.pueue_directory) {
Ok((stdout, stderr)) => (stdout, stderr),
Err(err) => {
@ -157,14 +155,14 @@ pub fn print_local_log(task_id: usize, settings: &Settings, lines: Option<usize>
print_local_file(
&mut stdout,
&mut stdout_log,
&mut stdout_file,
&lines,
style_text("stdout:", Some(Color::Green), Some(Attribute::Bold)),
);
print_local_file(
&mut stdout,
&mut stderr_log,
&mut stderr_file,
&lines,
style_text("stderr:", Some(Color::Red), Some(Attribute::Bold)),
);
@ -178,22 +176,7 @@ pub fn print_local_file(stdout: &mut Stdout, file: &mut File, lines: &Option<usi
println!("\n{}", text);
// Only print the last lines if requested
if let Some(lines) = lines {
// TODO: This is super imperformant, but works as long as we don't use the last
// 1000 lines. It would be cleaner to seek to the beginning of the requested
// position and simply stream the content to stdout.
let last_lines: Vec<String> = rev_lines::RevLines::new(BufReader::new(file))
.expect("Failed to read last lines of file")
.take(*lines)
.collect();
println!(
"{}",
last_lines
.into_iter()
.rev()
.collect::<Vec<String>>()
.join("\n")
);
println!("{}", read_last_lines(file, *lines));
return;
}

23
daemon/network/message_handler/log.rs
View file

@ -15,6 +15,22 @@ pub fn get_log(message: LogRequestMessage, state: &SharedState) -> Message {
message.task_ids
};
// Determine, whether we should draw everything or only a part of the log output.
// None implicates that all lines are printed
let lines = if message.full {
None
} else if let Some(lines) = message.lines {
Some(lines)
} else {
// By default only some lines are shown per task, if multiple tasks exist.
// For a single task, the whole log output is shown.
if task_ids.len() > 1 {
Some(15)
} else {
None
}
};
let mut tasks = BTreeMap::new();
for task_id in task_ids.iter() {
if let Some(task) = state.tasks.get(task_id) {
@ -22,8 +38,11 @@ pub fn get_log(message: LogRequestMessage, state: &SharedState) -> Message {
// This isn't as efficient as sending the raw compressed data directly,
// but it's a lot more convenient for now.
let (stdout, stderr) = if message.send_logs {
match read_and_compress_log_files(*task_id, &state.settings.shared.pueue_directory)
{
match read_and_compress_log_files(
*task_id,
&state.settings.shared.pueue_directory,
lines,
) {
Ok((stdout, stderr)) => (Some(stdout), Some(stderr)),
Err(err) => {
// Fail early if there's some problem with getting the log output

49
shared/log.rs
View file

@ -1,5 +1,5 @@
use std::fs::{read_dir, remove_file, File};
use std::io;
use std::io::{self, BufReader, Cursor};
use std::path::PathBuf;
use anyhow::{bail, Result};
@ -51,8 +51,12 @@ pub fn clean_log_handles(task_id: usize, path: &PathBuf) {
/// Return stdout and stderr of a finished process.
/// Task output is compressed using snap to save some memory and bandwidth.
pub fn read_and_compress_log_files(task_id: usize, path: &PathBuf) -> Result<(Vec<u8>, Vec<u8>)> {
let (mut stdout_handle, mut stderr_handle) = match get_log_file_handles(task_id, path) {
pub fn read_and_compress_log_files(
task_id: usize,
path: &PathBuf,
lines: Option<usize>,
) -> Result<(Vec<u8>, Vec<u8>)> {
let (mut stdout_file, mut stderr_file) = match get_log_file_handles(task_id, path) {
Ok((stdout, stderr)) => (stdout, stderr),
Err(err) => {
bail!("Error while opening the output files: {}", err);
@ -61,12 +65,25 @@ pub fn read_and_compress_log_files(task_id: usize, path: &PathBuf) -> Result<(Ve
let mut stdout = Vec::new();
let mut stderr = Vec::new();
{
// Compress log input and pipe it into the base64 encoder
if let Some(lines) = lines {
// Get the last few lines of both files
let stdout_bytes = read_last_lines(&mut stdout_file, lines).into_bytes();
let stderr_bytes = read_last_lines(&mut stderr_file, lines).into_bytes();
let mut stdout_cursor = Cursor::new(stdout_bytes);
let mut stderr_cursor = Cursor::new(stderr_bytes);
// Compress the partial log input and pipe it into the snappy compressor
let mut stdout_compressor = FrameEncoder::new(&mut stdout);
io::copy(&mut stdout_handle, &mut stdout_compressor)?;
io::copy(&mut stdout_cursor, &mut stdout_compressor)?;
let mut stderr_compressor = FrameEncoder::new(&mut stderr);
io::copy(&mut stderr_handle, &mut stderr_compressor)?;
io::copy(&mut stderr_cursor, &mut stderr_compressor)?;
} else {
// Compress the full log input and pipe it into the snappy compressor
let mut stdout_compressor = FrameEncoder::new(&mut stdout);
io::copy(&mut stdout_file, &mut stdout_compressor)?;
let mut stderr_compressor = FrameEncoder::new(&mut stderr);
io::copy(&mut stderr_file, &mut stderr_compressor)?;
}
Ok((stdout, stderr))
@ -86,3 +103,21 @@ pub fn reset_task_log_directory(path: &PathBuf) {
}
}
}
/// Read the last `amount` lines of a file to a string.
///
/// TODO: This is super imperformant, but works as long as we don't use the last
/// 1000 lines. It would be cleaner to seek to the beginning of the requested
/// position and simply stream the content.
pub fn read_last_lines(file: &mut File, amount: usize) -> String {
let last_lines: Vec<String> = rev_lines::RevLines::new(BufReader::new(file))
.expect("Failed to read last lines of file")
.take(amount)
.collect();
last_lines
.into_iter()
.rev()
.collect::<Vec<String>>()
.join("\n")
}