Ma/mscp
1
0
Fork 0
mirror of https://github.com/upa/mscp.git synced 2026-02-13 00:24:42 +08:00
Code Issues Packages Projects Releases 1 Wiki Activity

add mscp_ssh_opts and change -C optarg

Browse Source
This commit is contained in:
Ryo Nakamura
2023-03-03 16:50:06 +09:00
parent a8af79f9cf
commit 363296f499
8 changed files with 120 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
src/main.c
View File

@@ -17,10 +17,10 @@
void usage(bool print_help) {
printf("mscp v" VERSION ": copy files over multiple ssh connections\n"
"\n"
"Usage: mscp [vqDCHdNh] [-n nr_conns] [-m coremask]\n"
"Usage: mscp [vqDHdNh] [-n nr_conns] [-m coremask]\n"
" [-s min_chunk_sz] [-S max_chunk_sz] [-a nr_ahead] [-b buf_sz]\n"
" [-l login_name] [-p port] [-i identity_file]\n"
" [-c cipher_spec] [-M hmac_spec] source ... target\n"
" [-c cipher_spec] [-M hmac_spec] [-C compress] source ... target\n"
"\n");
if (!print_help)
@@ -45,10 +45,10 @@ void usage(bool print_help) {
" -i IDENTITY identity file for public key authentication\n"
" -c CIPHER cipher spec\n"
" -M HMAC hmac spec\n"
" -C enable compression on libssh\n"
" -C COMPRESS enable compression: yes, no, zlib, zlib@openssh.com\n"
" -H disable hostkey check\n"
" -d increment ssh debug output level\n"
" -N disable tcp nodelay (default on)\n"
" -N enable Nagle's algorithm (default disabled)\n"
" -h print this help\n"
"\n");
}
@@ -162,15 +162,17 @@ free_target_out:
int main(int argc, char **argv)
{
struct mscp_ssh_opts s;
struct mscp_opts o;
struct mscp *m;
struct target *t;
int ch, n, i;
char *remote;
memset(&s, 0, sizeof(s));
memset(&o, 0, sizeof(o));
while ((ch = getopt(argc, argv, "n:m:s:S:a:b:vqDrl:p:i:c:M:CHdNh")) != -1) {
while ((ch = getopt(argc, argv, "n:m:s:S:a:b:vqDrl:p:i:c:M:C:HdNh")) != -1) {
switch (ch) {
case 'n':
o.nr_threads = atoi(optarg);
@@ -207,51 +209,55 @@ int main(int argc, char **argv)
/* for compatibility with scp */
break;
case 'l':
if (strlen(optarg) > MSCP_MAX_LOGIN_NAME - 1) {
if (strlen(optarg) > MSCP_SSH_MAX_LOGIN_NAME - 1) {
pr_err("too long login name: %s\n", optarg);
return -1;
}
strncpy(o.ssh_login_name, optarg, MSCP_MAX_LOGIN_NAME - 1);
strncpy(s.login_name, optarg, MSCP_SSH_MAX_LOGIN_NAME - 1);
break;
case 'p':
if (strlen(optarg) > MSCP_MAX_PORT_STR - 1) {
if (strlen(optarg) > MSCP_SSH_MAX_PORT_STR - 1) {
pr_err("too long port string: %s\n", optarg);
return -1;
}
strncpy(o.ssh_port, optarg, MSCP_MAX_PORT_STR);
strncpy(s.port, optarg, MSCP_SSH_MAX_PORT_STR);
break;
case 'i':
if (strlen(optarg) > MSCP_MAX_IDENTITY_PATH - 1) {
if (strlen(optarg) > MSCP_SSH_MAX_IDENTITY_PATH - 1) {
pr_err("too long identity path: %s\n", optarg);
return -1;
}
strncpy(o.ssh_identity, optarg, MSCP_MAX_IDENTITY_PATH);
strncpy(s.identity, optarg, MSCP_SSH_MAX_IDENTITY_PATH);
break;
case 'c':
if (strlen(optarg) > MSCP_MAX_CIPHER_STR - 1) {
if (strlen(optarg) > MSCP_SSH_MAX_CIPHER_STR - 1) {
pr_err("too long cipher string: %s\n", optarg);
return -1;
}
strncpy(o.ssh_cipher_spec, optarg, MSCP_MAX_CIPHER_STR);
strncpy(s.cipher, optarg, MSCP_SSH_MAX_CIPHER_STR);
break;
case 'M':
if (strlen(optarg) > MSCP_MAX_HMACP_STR - 1) {
if (strlen(optarg) > MSCP_SSH_MAX_HMAC_STR - 1) {
pr_err("too long hmac string: %s\n", optarg);
return -1;
}
strncpy(o.ssh_hmac_spec, optarg, MSCP_MAX_HMACP_STR);
strncpy(s.hmac, optarg, MSCP_SSH_MAX_HMAC_STR);
break;
case 'C':
o.ssh_compress_level++;
if (strlen(optarg) > MSCP_SSH_MAX_COMP_STR - 1) {
pr_err("too long compress string: %s\n", optarg);
return -1;
}
strncpy(s.compress, optarg, MSCP_SSH_MAX_COMP_STR);
break;
case 'H':
o.ssh_no_hostkey_check = true;
s.no_hostkey_check = true;
break;
case 'd':
o.ssh_debug_level++;
s.debug_level++;
break;
case 'N':
o.ssh_disable_tcp_nodely = true;
s.enable_nagle = true;
break;
case 'h':
usage(true);
@@ -282,7 +288,7 @@ int main(int argc, char **argv)
remote = t[i - 1].remote;
}
if ((m = mscp_init(remote, &o)) == NULL)
if ((m = mscp_init(remote, &o, &s)) == NULL)
return -1;
if (mscp_connect(m) < 0)

47
src/mscp.c
View File

@@ -18,7 +18,7 @@
struct mscp {
char *remote; /* remote host (and uername) */
struct mscp_opts *opts;
struct ssh_opts ssh_opts;
struct mscp_ssh_opts *ssh_opts;
int *cores; /* usable cpu cores by COREMASK */
int nr_cores; /* length of array of cores */
@@ -187,7 +187,8 @@ static int validate_and_set_defaut_params(struct mscp_opts *o)
return 0;
}
struct mscp *mscp_init(const char *remote_host, struct mscp_opts *opts)
struct mscp *mscp_init(const char *remote_host,
struct mscp_opts *o, struct mscp_ssh_opts *s)
{
struct mscp *m;
int n;
@@ -198,7 +199,7 @@ struct mscp *mscp_init(const char *remote_host, struct mscp_opts *opts)
return NULL;
}
if (validate_and_set_defaut_params(opts) < 0)
if (validate_and_set_defaut_params(o) < 0)
goto free_out;
memset(m, 0, sizeof(*m));
@@ -212,33 +213,19 @@ struct mscp *mscp_init(const char *remote_host, struct mscp_opts *opts)
goto free_out;
}
if (strlen(opts->coremask) > 0) {
if (expand_coremask(opts->coremask, &m->cores, &m->nr_cores) < 0)
if (strlen(o->coremask) > 0) {
if (expand_coremask(o->coremask, &m->cores, &m->nr_cores) < 0)
goto free_out;
pprint(2, "usable cpu cores:");
pprint(1, "usable cpu cores:");
for (n = 0; n < m->nr_cores; n++)
pprint(2, " %d", m->cores[n]);
pprint(2, "\n");
pprint(1, "\n");
}
m->opts = opts;
if (non_null_string(opts->ssh_login_name))
m->ssh_opts.login_name = opts->ssh_login_name;
if (non_null_string(opts->ssh_port))
m->ssh_opts.port = opts->ssh_port;
if (non_null_string(opts->ssh_identity))
m->ssh_opts.identity = opts->ssh_identity;
if (non_null_string(opts->ssh_cipher_spec))
m->ssh_opts.cipher = opts->ssh_cipher_spec;
if (non_null_string(opts->ssh_hmac_spec))
m->ssh_opts.hmac = opts->ssh_hmac_spec;
m->opts = o;
m->ssh_opts = s;
m->ssh_opts.compress = opts->ssh_compress_level;
m->ssh_opts.debuglevel = opts->ssh_debug_level;
m->ssh_opts.no_hostkey_check = opts->ssh_no_hostkey_check;
m->ssh_opts.nodelay = opts->ssh_disable_tcp_nodely;
pprint_set_level(opts->verbose_level);
pprint_set_level(o->verbose_level);
return m;
@@ -249,7 +236,7 @@ free_out:
int mscp_connect(struct mscp *m)
{
m->first = ssh_init_sftp_session(m->remote, &m->ssh_opts);
m->first = ssh_init_sftp_session(m->remote, m->ssh_opts);
if (!m->first)
return -1;
@@ -385,7 +372,7 @@ int mscp_start(struct mscp *m)
m_local = m;
if ((n = list_count(&m->chunk_list)) < m->opts->nr_threads) {
pprint2("we have only %d chunk(s). "
pprint1("we have only %d chunk(s). "
"set number of connections to %d\n", n, n);
m->opts->nr_threads = n;
}
@@ -406,8 +393,8 @@ int mscp_start(struct mscp *m)
m->first = NULL;
}
else {
pprint3("connecting to %s for a copy thread...\n", m->remote);
t->sftp = ssh_init_sftp_session(m->remote, &m->ssh_opts);
pprint2("connecting to %s for a copy thread...\n", m->remote);
t->sftp = ssh_init_sftp_session(m->remote, m->ssh_opts);
if (!t->sftp)
return -1;
}
@@ -678,7 +665,7 @@ static void print_progress_bar(double percent, char *suffix)
" %3d%% ", (int)floor(percent));
}
pprint1("%s%s", buf, suffix);
pprint0("%s%s", buf, suffix);
}
static void print_progress(struct timeval *b, struct timeval *a,
@@ -748,7 +735,7 @@ static void mscp_stat_handler(int signum)
} else {
/* called from mscp_stat_final. calculate progress from the beginning */
print_progress(&s.start, &s.after, s.total, 0, s.done);
pprint(1, "\n"); /* this is final output. */
pprint(0, "\n"); /* this is final output. */
}
}

42
src/mscp.h
View File

@@ -8,11 +8,6 @@
#define MSCP_DIRECTION_R2L 2
#define MSCP_MAX_COREMASK_STR 64
#define MSCP_MAX_LOGIN_NAME 64
#define MSCP_MAX_PORT_STR 32
#define MSCP_MAX_IDENTITY_PATH PATH_MAX
#define MSCP_MAX_CIPHER_STR 32
#define MSCP_MAX_HMACP_STR 32
struct mscp_opts {
/* mscp options */
@@ -29,22 +24,39 @@ struct mscp_opts {
bool quiet;
bool dryrun;
};
#define MSCP_SSH_MAX_LOGIN_NAME 64
#define MSCP_SSH_MAX_PORT_STR 32
#define MSCP_SSH_MAX_IDENTITY_PATH PATH_MAX
#define MSCP_SSH_MAX_CIPHER_STR 32
#define MSCP_SSH_MAX_HMAC_STR 32
#define MSCP_SSH_MAX_COMP_STR 32 /* yes, no, zlib, zlib@openssh.com, none */
#define MSCP_SSH_MAX_PASSWORD 128
#define MSCP_SSH_MAX_PASSPHRASE 128
struct mscp_ssh_opts {
/* ssh options */
char ssh_login_name[MSCP_MAX_LOGIN_NAME];
char ssh_port[MSCP_MAX_PORT_STR];
char ssh_identity[MSCP_MAX_IDENTITY_PATH];
char ssh_cipher_spec[MSCP_MAX_CIPHER_STR];
char ssh_hmac_spec[MSCP_MAX_HMACP_STR];
int ssh_debug_level;
int ssh_compress_level;
bool ssh_no_hostkey_check;
bool ssh_disable_tcp_nodely;
char login_name[MSCP_SSH_MAX_LOGIN_NAME];
char port[MSCP_SSH_MAX_PORT_STR];
char identity[MSCP_SSH_MAX_IDENTITY_PATH];
char cipher[MSCP_SSH_MAX_CIPHER_STR];
char hmac[MSCP_SSH_MAX_HMAC_STR];
char compress[MSCP_SSH_MAX_COMP_STR];
char password[MSCP_SSH_MAX_PASSWORD];
char passphrase[MSCP_SSH_MAX_PASSPHRASE];
int debug_level;
bool no_hostkey_check;
bool enable_nagle;
};
struct mscp;
/* initialize and return a mscp instance with option validation */
struct mscp *mscp_init(const char *remote_host, struct mscp_opts *opts);
struct mscp *mscp_init(const char *remote_host,
struct mscp_opts *o, struct mscp_ssh_opts *s);
/* establish the first SFTP session. mscp_prepare() and mscp_start()
* requires mscp_connect() beforehand */

2
src/pprint.c
View File

@@ -2,7 +2,7 @@
#include <stdarg.h>
#include <pthread.h>
static int pprint_level = 1;
static int pprint_level = 0;
static pthread_mutex_t pprint_lock = PTHREAD_MUTEX_INITIALIZER;

2
src/pprint.h
View File

@@ -11,10 +11,10 @@
void pprint_set_level(int level);
void pprint(int level, const char *fmt, ...);
#define pprint0(fmt, ...) pprint(0, "\r\033[K" fmt, ##__VA_ARGS__)
#define pprint1(fmt, ...) pprint(1, "\r\033[K" fmt, ##__VA_ARGS__)
#define pprint2(fmt, ...) pprint(2, "\r\033[K" fmt, ##__VA_ARGS__)
#define pprint3(fmt, ...) pprint(3, "\r\033[K" fmt, ##__VA_ARGS__)
#define pprint4(fmt, ...) pprint(4, "\r\033[K" fmt, ##__VA_ARGS__)
#endif /* _PPRRINT_H_ */

78
src/ssh.c
View File

@@ -11,29 +11,31 @@
static int ssh_verify_known_hosts(ssh_session session);
static int ssh_set_opts(ssh_session ssh, struct ssh_opts *opts)
{
ssh_set_log_level(opts->debuglevel);
#define is_specified(s) (strlen(s) > 0)
if (opts->login_name &&
static int ssh_set_opts(ssh_session ssh, struct mscp_ssh_opts *opts)
{
ssh_set_log_level(opts->debug_level);
if (is_specified(opts->login_name) &&
ssh_options_set(ssh, SSH_OPTIONS_USER, opts->login_name) < 0) {
pr_err("failed to set login name\n");
return -1;
}
if (opts->port &&
if (is_specified(opts->port) &&
ssh_options_set(ssh, SSH_OPTIONS_PORT_STR, opts->port) < 0) {
pr_err("failed to set port number\n");
return -1;
}
if (opts->identity &&
if (is_specified(opts->identity) &&
ssh_options_set(ssh, SSH_OPTIONS_IDENTITY, opts->identity) < 0) {
pr_err("failed to set identity\n");
return -1;
}
if (opts->cipher) {
if (is_specified(opts->cipher)) {
if (ssh_options_set(ssh, SSH_OPTIONS_CIPHERS_C_S, opts->cipher) < 0) {
pr_err("failed to set cipher for client to server\n");
return -1;
@@ -44,7 +46,7 @@ static int ssh_set_opts(ssh_session ssh, struct ssh_opts *opts)
}
}
if (opts->hmac) {
if (is_specified(opts->hmac)) {
if (ssh_options_set(ssh, SSH_OPTIONS_HMAC_C_S, opts->hmac) < 0) {
pr_err("failed to set hmac for client to server\n");
return -1;
@@ -55,22 +57,25 @@ static int ssh_set_opts(ssh_session ssh, struct ssh_opts *opts)
}
}
if (opts->compress &&
ssh_options_set(ssh, SSH_OPTIONS_COMPRESSION, "yes") < 0) {
if (is_specified(opts->compress) &&
ssh_options_set(ssh, SSH_OPTIONS_COMPRESSION, opts->compress) < 0) {
pr_err("failed to enable ssh compression\n");
return -1;
}
if (opts->nodelay &&
ssh_options_set(ssh, SSH_OPTIONS_NODELAY, &opts->nodelay) < 0) {
pr_err("failed to set nodelay\n");
return -1;
/* if NOT specified to enable Nagle's algorithm, disable it (set TCP_NODELAY) */
if (!opts->enable_nagle) {
int v = 1;
if (ssh_options_set(ssh, SSH_OPTIONS_NODELAY, &v) < 0) {
pr_err("failed to set TCP_NODELAY\n");
return -1;
}
}
return 0;
}
static int ssh_authenticate(ssh_session ssh, struct ssh_opts *opts)
static int ssh_authenticate(ssh_session ssh, struct mscp_ssh_opts *opts)
{
int auth_bit_mask;
int ret;
@@ -86,21 +91,16 @@ static int ssh_authenticate(ssh_session ssh, struct ssh_opts *opts)
ssh_userauth_none(ssh, NULL) == SSH_AUTH_SUCCESS)
return 0;
if (auth_bit_mask & SSH_AUTH_METHOD_PUBLICKEY &&
ssh_userauth_publickey_auto(ssh, NULL, opts->passphrase) == SSH_AUTH_SUCCESS)
return 0;
if (auth_bit_mask & SSH_AUTH_METHOD_PUBLICKEY) {
char *p = is_specified(opts->passphrase) ? opts->passphrase : NULL;
if (ssh_userauth_publickey_auto(ssh, NULL, p) == SSH_AUTH_SUCCESS)
return 0;
}
if (auth_bit_mask & SSH_AUTH_METHOD_PASSWORD) {
if (!opts->password) {
opts->password = malloc(PASSWORD_BUF_SZ);
if (!opts->password) {
pr_err("malloc: %s\n", strerrno());
return -1;
}
memset(opts->password, 0, PASSWORD_BUF_SZ);
if (ssh_getpass("Password: ", opts->password, PASSWORD_BUF_SZ,
0, 0) < 0) {
if (!is_specified(opts->password)) {
if (ssh_getpass("Password: ", opts->password,
MSCP_SSH_MAX_PASSWORD, 0, 0) < 0) {
return -1;
}
}
@@ -115,30 +115,22 @@ static int ssh_authenticate(ssh_session ssh, struct ssh_opts *opts)
static int ssh_cache_passphrase(const char *prompt, char *buf, size_t len, int echo,
int verify, void *userdata)
{
struct ssh_opts *opts = userdata;
struct mscp_ssh_opts *opts = userdata;
/* This function is called on the first time for importing
* priv key file with passphrase. It is not called on the
* second time or after because cached passphrase is passed
* to ssh_userauth_publickey_auto(). */
if (opts->passphrase) {
/* passphrase is cached, but this function is called.
* maybe it was an invalid passphrase? */
free(opts->passphrase);
opts->passphrase = NULL;
}
if (ssh_getpass("Passphrase: ", buf, len, echo, verify) < 0)
return -1;
/* cache the passphrase */
opts->passphrase = malloc(len);
if (!opts->passphrase) {
pr_err("malloc: %s\n", strerrno());
return -1;
if (strlen(buf) > MSCP_SSH_MAX_PASSPHRASE - 1) {
pr_warn("sorry, passphrase is too long to cache...\n");
return 0;
}
memcpy(opts->passphrase, buf, len);
strncpy(opts->passphrase, buf, MSCP_SSH_MAX_PASSPHRASE);
return 0;
}
@@ -148,7 +140,7 @@ static struct ssh_callbacks_struct cb = {
.userdata = NULL,
};
static ssh_session ssh_init_session(const char *sshdst, struct ssh_opts *opts)
static ssh_session ssh_init_session(const char *sshdst, struct mscp_ssh_opts *opts)
{
ssh_session ssh = ssh_new();
@@ -187,7 +179,7 @@ free_out:
return NULL;
}
sftp_session ssh_init_sftp_session(const char *sshdst, struct ssh_opts *opts)
sftp_session ssh_init_sftp_session(const char *sshdst, struct mscp_ssh_opts *opts)
{
sftp_session sftp;
ssh_session ssh = ssh_init_session(sshdst, opts);

19
src/ssh.h
View File

@@ -5,27 +5,12 @@
#include "libssh/libssh.h"
#include "libssh/sftp.h"
struct ssh_opts {
char *login_name; /* -l */
char *port; /* -p */
char *identity; /* -i */
char *cipher; /* -c */
char *hmac; /* -M */
int compress; /* -C */
int nodelay; /* -N */
int debuglevel; /* -v */
bool no_hostkey_check; /* -H */
#define PASSWORD_BUF_SZ 128
char *password; /* password for password auth */
char *passphrase; /* passphrase for private key */
};
#include <mscp.h>
/* ssh_init_sftp_session() creates sftp_session. sshdst accpets
* user@hostname and hostname notations (by libssh).
*/
sftp_session ssh_init_sftp_session(const char *sshdst, struct ssh_opts *opts);
sftp_session ssh_init_sftp_session(const char *sshdst, struct mscp_ssh_opts *opts);
void ssh_sftp_close(sftp_session sftp);
#define sftp_ssh(sftp) (sftp)->session

11
test/test_e2e.py
View File

@@ -182,3 +182,14 @@ def test_override_dst_having_larger_size(mscp, src_prefix, dst_prefix):
assert check_same_md5sum(src, dst)
src.cleanup()
dst.cleanup()
compressions = ["yes", "no", "none"]
@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
@pytest.mark.parametrize("compress", compressions)
def test_compression(mscp, src_prefix, dst_prefix, compress):
src = File("src", size = 1024 * 1024).make()
dst = File("dst", size = 1024 * 1024 * 2).make()
run2ok([mscp, "-H", "-C", compress, src_prefix + src.path, dst_prefix + "dst"])
assert check_same_md5sum(src, dst)
src.cleanup()
dst.cleanup()