From cb3b2d2cbfb795713f604f874fb65ee09ff9edfd Mon Sep 17 00:00:00 2001 From: sagie gur ari Date: Sun, 28 Jun 2020 18:14:46 +0000 Subject: [PATCH] New ftp_get and ftp_get_in_memory commands --- CHANGELOG.md | 2 + docs/sdk.md | 68 +++++++++++++ .../src/sdk/std/net/ftp/get/help.md | 26 +++++ duckscript_sdk/src/sdk/std/net/ftp/get/mod.rs | 97 +++++++++++++++++++ .../src/sdk/std/net/ftp/get/mod_test.rs | 12 +++ .../src/sdk/std/net/ftp/get_in_memory/help.md | 26 +++++ .../src/sdk/std/net/ftp/get_in_memory/mod.rs | 80 +++++++++++++++ .../sdk/std/net/ftp/get_in_memory/mod_test.rs | 12 +++ duckscript_sdk/src/sdk/std/net/ftp/mod.rs | 47 ++++++++- test/std/net/ftp/ftp_get_in_memory_test.ds | 14 +++ test/std/net/ftp/ftp_get_test.ds | 16 +++ 11 files changed, 396 insertions(+), 4 deletions(-) create mode 100644 duckscript_sdk/src/sdk/std/net/ftp/get/help.md create mode 100755 duckscript_sdk/src/sdk/std/net/ftp/get/mod.rs create mode 100644 duckscript_sdk/src/sdk/std/net/ftp/get/mod_test.rs create mode 100644 duckscript_sdk/src/sdk/std/net/ftp/get_in_memory/help.md create mode 100755 duckscript_sdk/src/sdk/std/net/ftp/get_in_memory/mod.rs create mode 100644 duckscript_sdk/src/sdk/std/net/ftp/get_in_memory/mod_test.rs create mode 100644 test/std/net/ftp/ftp_get_in_memory_test.ds create mode 100644 test/std/net/ftp/ftp_get_test.ds diff --git a/CHANGELOG.md b/CHANGELOG.md index ed1de3d..0f2b95c 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ### v0.5.1 +* New ftp_get command. +* New ftp_get_in_memory command. * New ftp_list command. * New ftp_nlst command. diff --git a/docs/sdk.md b/docs/sdk.md index 6fde2e0..6c88e84 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -111,6 +111,8 @@ * [std::net::Hostname (hostname)](#std__net__Hostname) * [std::net::HttpClient (http_client)](#std__net__HttpClient) * [std::net::WGet (wget)](#std__net__WGet) +* [std::net::ftp::Get (ftp_get)](#std__net__ftp__Get) +* [std::net::ftp::GetInMemory (ftp_get_in_memory)](#std__net__ftp__GetInMemory) * [std::net::ftp::List (ftp_list)](#std__net__ftp__List) * [std::net::ftp::NLst (ftp_nlst)](#std__net__ftp__NLst) * [std::process::Execute (exec)](#std__process__Execute) @@ -4143,6 +4145,72 @@ http_client --method "${scope::wget::method}" --output-file "${scope::wget::file #### Aliases: wget + +## std::net::ftp::Get +```sh +result = ftp_get --host [--port 21] [--username ] [--password ] [--path ] [--type ] --remote-file --local-file +``` + +Invokes the FTP GET command from the given connection and file details. + +#### Parameters + +* --host - The host name or IP to connect to +* --port - Optional port number to use (by default 21) +* --username - Optional user name used to login (if not user or password provided, no login operation will be invoked) +* --password - Optional password used to login (if not user or password provided, no login operation will be invoked) +* --path - Optional path on the remote server to invoke operation on +* --type - Optional setting of the transfer type as A (ascii) I (image, binary) +* --remote-file - The remote file to download +* --local-file - The target local file name + +#### Return Value + +true if operation was completed. + +#### Examples + +```sh +ftp_get --host myhost --username someuser --password 12345 --remote-file README.md --local-file README.md +``` + + +#### Aliases: +ftp_get + + +## std::net::ftp::GetInMemory +```sh +handle = ftp_get_in_memory --host [--port 21] [--username ] [--password ] [--path ] [--type ] --remote-file +``` + +Invokes the FTP GET command from the given connection and file details. + +#### Parameters + +* --host - The host name or IP to connect to +* --port - Optional port number to use (by default 21) +* --username - Optional user name used to login (if not user or password provided, no login operation will be invoked) +* --password - Optional password used to login (if not user or password provided, no login operation will be invoked) +* --path - Optional path on the remote server to invoke operation on +* --type - Optional setting of the transfer type as A (ascii) I (image, binary) +* --remote-file - The remote file to download + +#### Return Value + +The binary data handle. + +#### Examples + +```sh +handle = ftp_get_in_memory --host myhost --username someuser --password 12345 --remote-file README.md +text = bytes_to_string ${handle} +``` + + +#### Aliases: +ftp_get_in_memory + ## std::net::ftp::List ```sh diff --git a/duckscript_sdk/src/sdk/std/net/ftp/get/help.md b/duckscript_sdk/src/sdk/std/net/ftp/get/help.md new file mode 100644 index 0000000..3ff8248 --- /dev/null +++ b/duckscript_sdk/src/sdk/std/net/ftp/get/help.md @@ -0,0 +1,26 @@ +```sh +result = ftp_get --host [--port 21] [--username ] [--password ] [--path ] [--type ] --remote-file --local-file +``` + +Invokes the FTP GET command from the given connection and file details. + +#### Parameters + +* --host - The host name or IP to connect to +* --port - Optional port number to use (by default 21) +* --username - Optional user name used to login (if not user or password provided, no login operation will be invoked) +* --password - Optional password used to login (if not user or password provided, no login operation will be invoked) +* --path - Optional path on the remote server to invoke operation on +* --type - Optional setting of the transfer type as A (ascii) I (image, binary) +* --remote-file - The remote file to download +* --local-file - The target local file name + +#### Return Value + +true if operation was completed. + +#### Examples + +```sh +ftp_get --host myhost --username someuser --password 12345 --remote-file README.md --local-file README.md +``` diff --git a/duckscript_sdk/src/sdk/std/net/ftp/get/mod.rs b/duckscript_sdk/src/sdk/std/net/ftp/get/mod.rs new file mode 100755 index 0000000..d4217e1 --- /dev/null +++ b/duckscript_sdk/src/sdk/std/net/ftp/get/mod.rs @@ -0,0 +1,97 @@ +use crate::sdk::std::net::ftp::{validate_and_run_with_connection, Options}; +use crate::utils::io::create_empty_file; +use crate::utils::pckg; +use duckscript::types::command::{Command, CommandResult}; +use ftp::{FtpError, FtpStream}; +use std::fs::OpenOptions; +use std::io::{BufWriter, Error, Read, Write}; + +#[cfg(test)] +#[path = "./mod_test.rs"] +mod mod_test; + +fn write_file(reader: &mut dyn Read, target_file: &str) -> Result<(), Error> { + let mut file = OpenOptions::new().append(true).open(target_file)?; + { + let mut writer = BufWriter::new(&mut file); + + let mut buffer = [0; 10240]; + loop { + let read_size = reader.read(&mut buffer)?; + if read_size > 0 { + writer.write(&buffer[0..read_size])?; + } else { + break; + } + } + + writer.flush()?; + } + file.sync_all()?; + + Ok(()) +} + +#[derive(Clone)] +pub(crate) struct CommandImpl { + package: String, +} + +impl Command for CommandImpl { + fn name(&self) -> String { + pckg::concat(&self.package, "Get") + } + + fn aliases(&self) -> Vec { + vec!["ftp_get".to_string()] + } + + fn help(&self) -> String { + include_str!("help.md").to_string() + } + + fn clone_and_box(&self) -> Box { + Box::new((*self).clone()) + } + + fn run(&self, arguments: Vec) -> CommandResult { + validate_and_run_with_connection( + &arguments, + &|options: &Options| -> Result<(), String> { + if options.remote_file.is_none() { + Err("Missing remote file name".to_string()) + } else if options.local_file.is_none() { + Err("Missing local file name.".to_string()) + } else { + Ok(()) + } + }, + &mut |options: &Options, ftp_stream: &mut FtpStream| -> CommandResult { + let options_clone = options.clone(); + let remote_file = options_clone.remote_file.unwrap(); + let local_file = options_clone.local_file.unwrap(); + + match create_empty_file(&local_file) { + Ok(_) => { + match ftp_stream.retr(&remote_file, |reader| { + match write_file(reader, &local_file) { + Ok(_) => Ok(()), + Err(error) => Err(FtpError::InvalidResponse(error.to_string())), + } + }) { + Ok(_) => CommandResult::Continue(Some(true.to_string())), + Err(error) => CommandResult::Error(error.to_string()), + } + } + Err(error) => CommandResult::Error(error), + } + }, + ) + } +} + +pub(crate) fn create(package: &str) -> Box { + Box::new(CommandImpl { + package: package.to_string(), + }) +} diff --git a/duckscript_sdk/src/sdk/std/net/ftp/get/mod_test.rs b/duckscript_sdk/src/sdk/std/net/ftp/get/mod_test.rs new file mode 100644 index 0000000..f99df3a --- /dev/null +++ b/duckscript_sdk/src/sdk/std/net/ftp/get/mod_test.rs @@ -0,0 +1,12 @@ +use super::*; +use crate::test; + +#[test] +fn common_functions() { + test::test_common_command_functions(create("")); +} + +#[test] +fn run_no_args() { + test::run_script_and_error(vec![create("")], "out = ftp_get", "out"); +} diff --git a/duckscript_sdk/src/sdk/std/net/ftp/get_in_memory/help.md b/duckscript_sdk/src/sdk/std/net/ftp/get_in_memory/help.md new file mode 100644 index 0000000..3be2fa4 --- /dev/null +++ b/duckscript_sdk/src/sdk/std/net/ftp/get_in_memory/help.md @@ -0,0 +1,26 @@ +```sh +handle = ftp_get_in_memory --host [--port 21] [--username ] [--password ] [--path ] [--type ] --remote-file +``` + +Invokes the FTP GET command from the given connection and file details. + +#### Parameters + +* --host - The host name or IP to connect to +* --port - Optional port number to use (by default 21) +* --username - Optional user name used to login (if not user or password provided, no login operation will be invoked) +* --password - Optional password used to login (if not user or password provided, no login operation will be invoked) +* --path - Optional path on the remote server to invoke operation on +* --type - Optional setting of the transfer type as A (ascii) I (image, binary) +* --remote-file - The remote file to download + +#### Return Value + +The binary data handle. + +#### Examples + +```sh +handle = ftp_get_in_memory --host myhost --username someuser --password 12345 --remote-file README.md +text = bytes_to_string ${handle} +``` diff --git a/duckscript_sdk/src/sdk/std/net/ftp/get_in_memory/mod.rs b/duckscript_sdk/src/sdk/std/net/ftp/get_in_memory/mod.rs new file mode 100755 index 0000000..413b729 --- /dev/null +++ b/duckscript_sdk/src/sdk/std/net/ftp/get_in_memory/mod.rs @@ -0,0 +1,80 @@ +use crate::sdk::std::net::ftp::{validate_and_run_with_connection, Options}; +use crate::utils::pckg; +use crate::utils::state::put_handle; +use duckscript::types::command::{Command, CommandResult, Commands}; +use duckscript::types::instruction::Instruction; +use duckscript::types::runtime::StateValue; +use ftp::FtpStream; +use std::collections::HashMap; + +#[cfg(test)] +#[path = "./mod_test.rs"] +mod mod_test; + +#[derive(Clone)] +pub(crate) struct CommandImpl { + package: String, +} + +impl Command for CommandImpl { + fn name(&self) -> String { + pckg::concat(&self.package, "GetInMemory") + } + + fn aliases(&self) -> Vec { + vec!["ftp_get_in_memory".to_string()] + } + + fn help(&self) -> String { + include_str!("help.md").to_string() + } + + fn clone_and_box(&self) -> Box { + Box::new((*self).clone()) + } + + fn requires_context(&self) -> bool { + true + } + + fn run_with_context( + &self, + arguments: Vec, + state: &mut HashMap, + _variables: &mut HashMap, + _output_variable: Option, + _instructions: &Vec, + _commands: &mut Commands, + _line: usize, + ) -> CommandResult { + validate_and_run_with_connection( + &arguments, + &|options: &Options| -> Result<(), String> { + if options.remote_file.is_none() { + Err("Missing remote file name".to_string()) + } else { + Ok(()) + } + }, + &mut |options: &Options, ftp_stream: &mut FtpStream| -> CommandResult { + let options_clone = options.clone(); + let remote_file = options_clone.remote_file.unwrap(); + + match ftp_stream.simple_retr(&remote_file) { + Ok(binary) => { + let key = put_handle(state, StateValue::ByteArray(binary.into_inner())); + + CommandResult::Continue(Some(key)) + } + Err(error) => CommandResult::Error(error.to_string()), + } + }, + ) + } +} + +pub(crate) fn create(package: &str) -> Box { + Box::new(CommandImpl { + package: package.to_string(), + }) +} diff --git a/duckscript_sdk/src/sdk/std/net/ftp/get_in_memory/mod_test.rs b/duckscript_sdk/src/sdk/std/net/ftp/get_in_memory/mod_test.rs new file mode 100644 index 0000000..3946e97 --- /dev/null +++ b/duckscript_sdk/src/sdk/std/net/ftp/get_in_memory/mod_test.rs @@ -0,0 +1,12 @@ +use super::*; +use crate::test; + +#[test] +fn common_functions() { + test::test_common_command_functions(create("")); +} + +#[test] +fn run_no_args() { + test::run_script_and_error(vec![create("")], "out = ftp_get_in_memory", "out"); +} diff --git a/duckscript_sdk/src/sdk/std/net/ftp/mod.rs b/duckscript_sdk/src/sdk/std/net/ftp/mod.rs index 34309f8..68f6ec1 100644 --- a/duckscript_sdk/src/sdk/std/net/ftp/mod.rs +++ b/duckscript_sdk/src/sdk/std/net/ftp/mod.rs @@ -1,3 +1,5 @@ +mod get; +mod get_in_memory; mod list; mod nlst; @@ -12,6 +14,8 @@ static PACKAGE: &str = "ftp"; pub(crate) fn load(commands: &mut Commands, parent: &str) -> Result<(), ScriptError> { let package = pckg::concat(parent, PACKAGE); + commands.set(get::create(&package))?; + commands.set(get_in_memory::create(&package))?; commands.set(list::create(&package))?; commands.set(nlst::create(&package))?; @@ -34,6 +38,8 @@ pub(crate) struct Options { password: Option, path: Option, transfer_type: Option, + remote_file: Option, + local_file: Option, } impl Options { @@ -45,6 +51,8 @@ impl Options { password: None, path: None, transfer_type: None, + remote_file: None, + local_file: None, } } } @@ -57,17 +65,34 @@ enum LookingFor { Password, Path, TransferType, + RemoteFile, + LocalFile, } pub(crate) fn run_with_connection( arguments: &Vec, func: &mut FnMut(&Options, &mut FtpStream) -> CommandResult, +) -> CommandResult { + validate_and_run_with_connection( + arguments, + &|_options: &Options| -> Result<(), String> { Ok(()) }, + func, + ) +} + +pub(crate) fn validate_and_run_with_connection( + arguments: &Vec, + validate_input: &Fn(&Options) -> Result<(), String>, + func: &mut FnMut(&Options, &mut FtpStream) -> CommandResult, ) -> CommandResult { match parse_common_options(&arguments) { - Ok(options) => run_in_ftp_connection_context( - &options, - &mut |ftp_stream: &mut FtpStream| -> CommandResult { func(&options, ftp_stream) }, - ), + Ok(options) => match validate_input(&options) { + Ok(_) => run_in_ftp_connection_context( + &options, + &mut |ftp_stream: &mut FtpStream| -> CommandResult { func(&options, ftp_stream) }, + ), + Err(error) => CommandResult::Error(error), + }, Err(error) => CommandResult::Error(error), } } @@ -85,6 +110,8 @@ fn parse_common_options(arguments: &Vec) -> Result { "--password" => looking_for = LookingFor::Password, "--path" => looking_for = LookingFor::Path, "--type" => looking_for = LookingFor::TransferType, + "--remote-file" => looking_for = LookingFor::RemoteFile, + "--local-file" => looking_for = LookingFor::LocalFile, _ => (), }, LookingFor::Host => { @@ -135,6 +162,18 @@ fn parse_common_options(arguments: &Vec) -> Result { looking_for = LookingFor::Flag; } } + LookingFor::RemoteFile => { + if !argument.is_empty() { + options.remote_file = Some(argument.to_string()); + } + looking_for = LookingFor::Flag; + } + LookingFor::LocalFile => { + if !argument.is_empty() { + options.local_file = Some(argument.to_string()); + } + looking_for = LookingFor::Flag; + } } } diff --git a/test/std/net/ftp/ftp_get_in_memory_test.ds b/test/std/net/ftp/ftp_get_in_memory_test.ds new file mode 100644 index 0000000..42456ef --- /dev/null +++ b/test/std/net/ftp/ftp_get_in_memory_test.ds @@ -0,0 +1,14 @@ + +fn test_valid + handle = ftp_get_in_memory --host test.rebex.net --username demo --password password --remote-file readme.txt + + text = bytes_to_string ${handle} + release ${handle} + + empty = is_empty ${text} + assert_false ${empty} + + found = contains ${text} Welcome + assert ${found} +end + diff --git a/test/std/net/ftp/ftp_get_test.ds b/test/std/net/ftp/ftp_get_test.ds new file mode 100644 index 0000000..9dd628c --- /dev/null +++ b/test/std/net/ftp/ftp_get_test.ds @@ -0,0 +1,16 @@ + +fn test_valid + filename = set ./target/_duckscript_test/net/ftp/get/readme.txt + result = ftp_get --host test.rebex.net --username demo --password password --remote-file readme.txt --local-file ${filename} + + assert ${result} + + text = readfile ${filename} + + empty = is_empty ${text} + assert_false ${empty} + + found = contains ${text} Welcome + assert ${found} +end +