refactor: Simplify shutdown code

This commit is contained in:
Arne Beer 2021-06-15 21:50:01 +02:00
parent dcf3169406
commit 68388c84de
No known key found for this signature in database
GPG key ID: CC9408F679023B65
8 changed files with 153 additions and 103 deletions

161
Cargo.lock generated
View file

@ -38,6 +38,20 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"
[[package]]
name = "assert_cmd"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a88b6bd5df287567ffdf4ddf4d33060048e1068308e5f62d81c6f9824a045a48"
dependencies = [
"bstr",
"doc-comment",
"predicates",
"predicates-core",
"predicates-tree",
"wait-timeout",
]
[[package]]
name = "async-attributes"
version = "1.1.2"
@ -280,6 +294,17 @@ dependencies = [
"once_cell",
]
[[package]]
name = "bstr"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90682c8d613ad3373e66de8c6411e0ae2ab2571e879d2efbf73558cc66f21279"
dependencies = [
"lazy_static",
"memchr",
"regex-automata",
]
[[package]]
name = "bumpalo"
version = "3.7.0"
@ -407,8 +432,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c93d79ba722818d1a6aedfbe2cf4889330c856d0c6772951efbbf3dd283c070a"
dependencies = [
"crossterm",
"strum 0.21.0",
"strum_macros 0.21.1",
"strum",
"strum_macros",
]
[[package]]
@ -527,6 +552,12 @@ version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e25ea47919b1560c4e3b7fe0aaab9becf5b84a10325ddf7db0f0ba5e1026499"
[[package]]
name = "difference"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198"
[[package]]
name = "digest"
version = "0.8.1"
@ -577,6 +608,12 @@ dependencies = [
"winapi",
]
[[package]]
name = "doc-comment"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
[[package]]
name = "dtoa"
version = "0.4.8"
@ -711,9 +748,9 @@ checksum = "62aca2aba2d62b4a7f5b33f3712cb1b0692779a56fb510499d5c0aa594daeaf3"
[[package]]
name = "handlebars"
version = "4.0.0"
version = "4.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28f0fe89affef47e2c9729030a8f6e79df34cb66b8a44ecf91dad43f31150559"
checksum = "2060119114dd8a8bc87facce6384751af8280a7adc8e203c023c95cbb11f5663"
dependencies = [
"log",
"pest",
@ -879,9 +916,9 @@ dependencies = [
[[package]]
name = "mio"
version = "0.7.11"
version = "0.7.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf80d3e903b34e0bd7282b218398aec54e082c840d9baf8339e0080a0c542956"
checksum = "8c2bdb6314ec10835cd3293dd268473a835c02b7b352e788be788b3c6ca6bb16"
dependencies = [
"libc",
"log",
@ -1108,14 +1145,14 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]]
name = "polling"
version = "2.0.3"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fc12d774e799ee9ebae13f4076ca003b40d18a11ac0f3641e6f899618580b7b"
checksum = "92341d779fa34ea8437ef4d82d440d5e1ce3f3ff7f824aa64424cd481f9a1f25"
dependencies = [
"cfg-if",
"libc",
"log",
"wepoll-sys",
"wepoll-ffi",
"winapi",
]
@ -1125,6 +1162,32 @@ version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"
[[package]]
name = "predicates"
version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f49cfaf7fdaa3bfacc6fa3e7054e65148878354a5cfddcf661df4c851f8021df"
dependencies = [
"difference",
"predicates-core",
]
[[package]]
name = "predicates-core"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57e35a3326b75e49aa85f5dc6ec15b41108cf5aee58eabb1f274dd18b73c2451"
[[package]]
name = "predicates-tree"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15f553275e5721409451eb85e15fd9a860a6e5ab4496eb215987502b5f5391f2"
dependencies = [
"predicates-core",
"treeline",
]
[[package]]
name = "pretty_assertions"
version = "0.7.2"
@ -1189,6 +1252,7 @@ name = "pueue"
version = "1.0.0-rc.1"
dependencies = [
"anyhow",
"assert_cmd",
"better-panic",
"chrono",
"chrono-english",
@ -1221,9 +1285,9 @@ dependencies = [
[[package]]
name = "pueue-lib"
version = "0.13.1"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acdc14318becc7bf3b5ea6a6d1a077af94e0e55295efb466b38a731d66dcb975"
checksum = "a1488038dfc56ca945de78532cf25531e3b3704a936e76cfe34b4f8f907e0df3"
dependencies = [
"async-std",
"async-tls",
@ -1233,7 +1297,7 @@ dependencies = [
"config",
"dirs",
"log",
"rand 0.8.3",
"rand 0.8.4",
"rcgen",
"rev_lines",
"rustls",
@ -1244,8 +1308,8 @@ dependencies = [
"serde_yaml",
"shellexpand",
"snap",
"strum 0.20.0",
"strum_macros 0.20.1",
"strum",
"strum_macros",
"thiserror",
"whoami",
]
@ -1280,13 +1344,13 @@ dependencies = [
[[package]]
name = "rand"
version = "0.8.3"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e"
checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8"
dependencies = [
"libc",
"rand_chacha",
"rand_core 0.6.2",
"rand_core 0.6.3",
"rand_hc",
]
@ -1297,7 +1361,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
dependencies = [
"ppv-lite86",
"rand_core 0.6.2",
"rand_core 0.6.3",
]
[[package]]
@ -1317,20 +1381,20 @@ checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc"
[[package]]
name = "rand_core"
version = "0.6.2"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7"
checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"
dependencies = [
"getrandom",
]
[[package]]
name = "rand_hc"
version = "0.3.0"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73"
checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7"
dependencies = [
"rand_core 0.6.2",
"rand_core 0.6.3",
]
[[package]]
@ -1382,6 +1446,12 @@ dependencies = [
"regex-syntax",
]
[[package]]
name = "regex-automata"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
[[package]]
name = "regex-syntax"
version = "0.6.25"
@ -1668,30 +1738,12 @@ version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
[[package]]
name = "strum"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7318c509b5ba57f18533982607f24070a55d353e90d4cae30c467cdb2ad5ac5c"
[[package]]
name = "strum"
version = "0.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aaf86bbcfd1fa9670b7a129f64fc0c9fcbbfe4f1bc4210e9e98fe71ffc12cde2"
[[package]]
name = "strum_macros"
version = "0.20.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee8bc6b87a5112aeeab1f4a9f7ab634fe6cbefc4850006df31267f4cfb9e3149"
dependencies = [
"heck",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "strum_macros"
version = "0.21.1"
@ -1733,7 +1785,7 @@ checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22"
dependencies = [
"cfg-if",
"libc",
"rand 0.8.3",
"rand 0.8.4",
"redox_syscall",
"remove_dir_all",
"winapi",
@ -1798,9 +1850,9 @@ dependencies = [
[[package]]
name = "tokio"
version = "1.6.1"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a38d31d7831c6ed7aad00aa4c12d9375fd225a6dd77da1d25b707346319a975"
checksum = "c79ba603c337335df6ba6dd6afc38c38a7d5e1b0c871678439ea973cd62a118e"
dependencies = [
"autocfg",
"bytes",
@ -1821,6 +1873,12 @@ dependencies = [
"syn",
]
[[package]]
name = "treeline"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41"
[[package]]
name = "typenum"
version = "1.13.0"
@ -1879,6 +1937,15 @@ version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe"
[[package]]
name = "wait-timeout"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6"
dependencies = [
"libc",
]
[[package]]
name = "waker-fn"
version = "1.1.0"
@ -1987,10 +2054,10 @@ dependencies = [
]
[[package]]
name = "wepoll-sys"
version = "3.0.1"
name = "wepoll-ffi"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fcb14dea929042224824779fbc82d9fab8d2e6d3cbc0ac404de8edf489e77ff"
checksum = "d743fdedc5c64377b5fc2bc036b01c7fd642205a0d96356034ae3404d49eb7fb"
dependencies = [
"cc",
]

View file

@ -30,7 +30,7 @@ name = "pueued"
path = "daemon/main.rs"
[dependencies]
pueue-lib = "0.13.1"
pueue-lib = "0.14.0"
#pueue-lib = { git = "https://github.com/Nukesor/pueue-lib", branch = "master" }
#pueue-lib = { path = "../../libraries/pueue-lib" }
@ -74,6 +74,7 @@ procfs = { version = "0.9", default-features = false }
version_check = "0.9"
[dev-dependencies]
assert_cmd = "1"
anyhow = "1"
better-panic = "0.2"
pretty_assertions = "0.7"

View file

@ -477,7 +477,7 @@ impl Client {
};
Ok(Message::Reset(message))
}
SubCommand::Shutdown => Ok(Message::DaemonShutdown),
SubCommand::Shutdown => Ok(Message::DaemonShutdown(Shutdown::Graceful)),
SubCommand::Parallel {
parallel_tasks,
group,

View file

@ -7,7 +7,7 @@ use crossbeam_channel::{unbounded, Sender};
use log::warn;
use pueue_lib::network::certificate::create_certificates;
use pueue_lib::network::message::Message;
use pueue_lib::network::message::{Message, Shutdown};
use pueue_lib::network::protocol::socket_cleanup;
use pueue_lib::network::secret::init_shared_secret;
use pueue_lib::settings::Settings;
@ -145,7 +145,7 @@ fn setup_signal_panic_handling(settings: &Settings, sender: &Sender<Message>) ->
ctrlc::set_handler(move || {
// Notify the task handler
sender_clone
.send(Message::DaemonShutdown)
.send(Message::DaemonShutdown(Shutdown::Emergency))
.expect("Failed to send Message to TaskHandler on Shutdown");
})?;

View file

@ -2,7 +2,6 @@ use crossbeam_channel::Sender;
use std::fmt::Display;
use pueue_lib::network::message::*;
use pueue_lib::network::protocol::socket_cleanup;
use pueue_lib::state::SharedState;
use crate::network::response_helper::*;
@ -45,7 +44,7 @@ pub fn handle_message(message: Message, sender: &Sender<Message>, state: &Shared
Message::Stash(task_ids) => stash::stash(task_ids, state),
Message::Switch(message) => switch::switch(message, state),
Message::Status => get_status(state),
Message::DaemonShutdown => shutdown(sender, state),
Message::DaemonShutdown(shutdown_type) => shutdown(sender, shutdown_type),
_ => create_failure_message("Not implemented yet"),
}
}
@ -71,18 +70,11 @@ fn get_status(state: &SharedState) -> Message {
/// Next, the DaemonShutdown Message will be forwarded to the TaskHandler.
/// The TaskHandler then gracefully shuts down all child processes
/// and exits with std::proces::exit(0).
fn shutdown(sender: &Sender<Message>, state: &SharedState) -> Message {
// Do some socket cleanup (unix socket).
{
let state = state.lock().unwrap();
if let Err(error) = socket_cleanup(&state.settings.shared) {
println!("Failed to cleanup socket after shutdown.");
println!("{}", error);
};
}
fn shutdown(sender: &Sender<Message>, shutdown_type: Shutdown) -> Message {
// Notify the task handler.
sender.send(Message::DaemonShutdown).expect(SENDER_ERR);
sender
.send(Message::DaemonShutdown(shutdown_type))
.expect(SENDER_ERR);
create_success_message("Daemon is shutting down")
}

View file

@ -4,7 +4,7 @@ use log::info;
use pueue_lib::network::message::*;
use crate::task_handler::{Shutdown, TaskHandler};
use crate::task_handler::TaskHandler;
mod kill;
mod pause;
@ -46,8 +46,8 @@ impl TaskHandler {
),
Message::Send(message) => self.send(message.task_id, message.input),
Message::Reset(message) => self.reset(message.children),
Message::DaemonShutdown => {
self.initiate_shutdown(Shutdown::Graceful);
Message::DaemonShutdown(shutdown) => {
self.initiate_shutdown(shutdown);
}
_ => info!("Received unhandled message {:?}", message),
}

View file

@ -18,6 +18,7 @@ use pueue_lib::network::protocol::socket_cleanup;
use pueue_lib::state::{GroupStatus, SharedState, State};
use pueue_lib::task::{Task, TaskResult, TaskStatus};
use crate::pid::cleanup_pid_file;
use crate::platform::process_helper::*;
mod callback;
@ -68,14 +69,6 @@ pub struct TaskHandler {
callback_log_lines: usize,
}
/// Which type of shutdown we're dealing with.
pub enum Shutdown {
/// Emergency is most likely a system unix signal or a CTRL+C in a terminal.
Emergency,
/// Graceful is user initiated and expected.
Graceful,
}
/// Pueue directly interacts with processes.
/// Since these interactions can vary depending on the current platform, this enum is introduced.
/// The intend is to keep any platform specific code out of the top level code.
@ -146,14 +139,11 @@ impl TaskHandler {
}
}
/// Initiate shutdown, if we're supposed to and didn't already do so.
/// Initiate shutdown, which includes killing all children and pausing all groups.
/// We don't have to pause any groups, as no new tasks will be spawned during shutdown anyway.
/// Any groups with queued tasks, will be automatically paused on state-restoration.
fn initiate_shutdown(&mut self, shutdown: Shutdown) {
self.shutdown = Some(shutdown);
info!("Pausing groups and killing all children due to shutdown.");
{
let mut state = self.state.lock().unwrap();
state.set_status_for_all_groups(GroupStatus::Paused);
}
self.kill(vec![], String::new(), true, false, None);
}
@ -177,7 +167,7 @@ impl TaskHandler {
}
// Cleanup the pid file
if let Err(error) = crate::pid::cleanup_pid_file(&self.pueue_directory) {
if let Err(error) = cleanup_pid_file(&self.pueue_directory) {
println!("Failed to cleanup pid during shutdown.");
println!("{}", error);
}

View file

@ -1,40 +1,40 @@
use assert_cmd::prelude::*;
use std::process::{Command, Stdio};
use anyhow::{Context, Result};
use nix::sys::signal::{kill, Signal};
use nix::unistd::Pid;
mod helper;
use helper::*;
#[cfg(target_os = "linux")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[test]
/// Spin up the daemon and send a SIGTERM shortly afterwards.
/// This should trigger the graceful shutdown and kill the process.
async fn test_ctrlc() -> Result<()> {
fn test_ctrlc() -> Result<()> {
let (_settings, tempdir) = helper::base_setup()?;
// Same as boot_daemon, but starts the daemon in non test mode
tokio::spawn(run_and_handle_error(
tempdir.path().clone().to_path_buf(),
false,
));
let mut child = Command::cargo_bin("pueued")?
.arg("--config")
.arg(tempdir.path().join("pueue.yml").to_str().unwrap())
.arg("-vvv")
.stdout(Stdio::piped())
.spawn()?;
let pid = get_pid(tempdir.path())?;
// Send SIGTERM signal to process via nix
use nix::sys::signal;
let nix_pid = nix::unistd::Pid::from_raw(pid);
signal::kill(nix_pid, signal::Signal::SIGTERM).context("Failed to send SIGTERM to pid")?;
let nix_pid = Pid::from_raw(pid);
kill(nix_pid, Signal::SIGTERM).context("Failed to send SIGTERM to pid")?;
// Sleep for 500ms and give the daemon time to shut down
helper::sleep_ms(500);
// Get all processes and make sure the process with our pid no longer exists
// However, since the daemon shuts down gracefully on SIGTERM, it'll exit the test.
// This is why the following code will never be reached or rather, if it will be reached, it'll
// fail.
let processes = procfs::process::all_processes().context("Failed to get all processes")?;
assert!(processes
.iter()
.filter(|process| process.pid == pid)
.collect::<Vec<_>>()
.is_empty());
let result = child.try_wait();
assert!(matches!(result, Ok(Some(_))));
let code = result.unwrap().unwrap();
assert!(matches!(code.code(), Some(1)));
Ok(())
}