Ma/mscp
1
0
Fork 0
mirror of https://github.com/upa/mscp.git synced 2026-02-13 00:24:42 +08:00
Code Issues Packages Projects Releases 1 Wiki Activity

fix when copy multiple sources and various tiny fixes

Browse Source 
* when copying multiple sources, target must be directory
* add multi-src copy test and parametrize src/dst prefixes
* cleanup REAMDE (s/sessions/connections/g)
* make error output in copy functions simple
This commit is contained in:
Ryo Nakamura
2022-11-18 14:42:23 +09:00
parent b8d58b1fba
commit 5e7aa774ca
6 changed files with 98 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
src/file.c
View File

@@ -319,13 +319,16 @@ static int file_fill_recursive(struct list_head *file_list,
int file_fill(sftp_session sftp, struct list_head *file_list, char **src_array, int cnt,
char *dst)
{
bool dst_is_remote, dst_is_dir, dst_dir_no_exist, dst_should_dir;
bool dst_is_remote, dst_is_dir, dst_dir_no_exist, dst_should_dir, dst_must_dir;
char *dst_path, *src_path;
int n, ret;
dst_path = file_find_path(dst);
dst_path = *dst_path == '\0' ? "." : dst_path;
dst_is_remote = file_find_hostname(dst) ? true : false;
dst_must_dir = cnt > 1 ? true : false;
pr_warn("dst_must_dir list_count %d %d\n", list_count(file_list), dst_must_dir);
if (file_is_directory(dst_path, dst_is_remote ? sftp : NULL, false) > 0)
dst_is_dir = true;
else
@@ -342,7 +345,8 @@ int file_fill(sftp_session sftp, struct list_head *file_list, char **src_array,
ret = file_fill_recursive(file_list, dst_is_remote, sftp,
src_path, "", dst_path,
dst_should_dir | dst_is_dir, dst_dir_no_exist);
dst_should_dir | dst_must_dir | dst_is_dir,
dst_dir_no_exist);
if (ret < 0)
return ret;
}
@@ -622,9 +626,9 @@ static sftp_file chunk_open_remote(const char *path, int flags, mode_t mode, siz
/*
* TODO: handle case when read returns 0 (EOF).
*/
static int chunk_copy_internal_local_to_remote(struct chunk *c, int fd, sftp_file sf,
size_t sftp_buf_sz, size_t io_buf_sz,
size_t *counter)
static int _chunk_copy_local_to_remote(struct chunk *c, int fd, sftp_file sf,
size_t sftp_buf_sz, size_t io_buf_sz,
size_t *counter)
{
ssize_t read_bytes, write_bytes, remaind;
char buf[io_buf_sz];
@@ -633,14 +637,13 @@ static int chunk_copy_internal_local_to_remote(struct chunk *c, int fd, sftp_fil
read_bytes = read(fd, buf, min(remaind, io_buf_sz));
if (read_bytes < 0) {
pr_err("failed to read %s: %s\n", c->f->src_path, strerrno());
pr_err("read: %s\n", strerrno());
return -1;
}
write_bytes = sftp_write2(sf, buf, read_bytes, sftp_buf_sz);
if (write_bytes < 0) {
pr_err("failed to write to %s: SFTP error code %d\n",
c->f->dst_path, sftp_get_error(sf->sftp));
pr_err("sftp_write: %d\n", sftp_get_error(sf->sftp));
return -1;
}
@@ -659,9 +662,8 @@ static int chunk_copy_internal_local_to_remote(struct chunk *c, int fd, sftp_fil
#define XFER_BUF_SIZE 16384
#ifdef ASYNC_WRITE
static int chunk_copy_internal_local_to_remote_async(struct chunk *c, int fd,
sftp_file sf, int nr_ahead,
size_t *counter)
static int _chunk_copy_local_to_remote_async(struct chunk *c, int fd,
sftp_file sf, int nr_ahead, size_t *counter)
{
ssize_t read_bytes, remaind, thrown;
char buf[XFER_BUF_SIZE];
@@ -679,13 +681,12 @@ static int chunk_copy_internal_local_to_remote_async(struct chunk *c, int fd,
reqs[idx].len = min(thrown, sizeof(buf));
read_bytes = read(fd, buf, reqs[idx].len);
if (read_bytes < 0) {
pr_err("read from %s failed: %s\n", c->f->src_path, strerrno());
pr_err("read: %s\n", strerrno());
return -1;
}
ret = sftp_async_write(sf, buf, reqs[idx].len, &reqs[idx].id);
if (ret < 0) {
pr_err("sftp_async_write for %s failed: %d\n",
c->f->dst_path, sftp_get_error(sf->sftp));
pr_err("sftp_async_write: %d\n", sftp_get_error(sf->sftp));
return -1;
}
thrown -= reqs[idx].len;
@@ -694,8 +695,7 @@ static int chunk_copy_internal_local_to_remote_async(struct chunk *c, int fd,
for (idx = 0; remaind > 0; idx = (idx + 1) % nr_ahead) {
ret = sftp_async_write_end(sf, reqs[idx].id, 1);
if (ret != SSH_OK) {
pr_err("sftp_async_write_end failed for %s: %d\n",
c->f->dst_path, sftp_get_error(sf->sftp));
pr_err("sftp_async_write_end: %d\n", sftp_get_error(sf->sftp));
return -1;
}
@@ -711,14 +711,13 @@ static int chunk_copy_internal_local_to_remote_async(struct chunk *c, int fd,
reqs[idx].len = min(thrown, sizeof(buf));
read_bytes = read(fd, buf, reqs[idx].len);
if (read_bytes < 0) {
pr_err("read from %s failed: %s\n", c->f->src_path, strerrno());
pr_err("read: %s\n", strerrno());
return -1;
}
ret = sftp_async_write(sf, buf, reqs[idx].len, &reqs[idx].id);
if (ret < 0) {
pr_err("sftp_async_write for %s failed: %d\n",
c->f->dst_path, sftp_get_error(sf->sftp));
pr_err("sftp_async_write: %d\n", sftp_get_error(sf->sftp));
return -1;
}
thrown -= reqs[idx].len;
@@ -735,8 +734,8 @@ static int chunk_copy_internal_local_to_remote_async(struct chunk *c, int fd,
}
#endif
static int chunk_copy_internal_remote_to_local(struct chunk *c, int fd, sftp_file sf,
int nr_ahead, size_t *counter)
static int _chunk_copy_remote_to_local(struct chunk *c, int fd, sftp_file sf,
int nr_ahead, size_t *counter)
{
ssize_t read_bytes, write_bytes, remaind, thrown;
char buf[XFER_BUF_SIZE];
@@ -763,7 +762,7 @@ static int chunk_copy_internal_remote_to_local(struct chunk *c, int fd, sftp_fil
reqs[idx].len = min(thrown, sizeof(buf));
reqs[idx].id = sftp_async_read_begin(sf, reqs[idx].len);
if (reqs[idx].id < 0) {
pr_err("sftp_async_read_begin failed: %d\n",
pr_err("sftp_async_read_begin: %d\n",
sftp_get_error(sf->sftp));
return -1;
}
@@ -773,7 +772,7 @@ static int chunk_copy_internal_remote_to_local(struct chunk *c, int fd, sftp_fil
for (idx = 0; remaind > 0; idx = (idx + 1) % nr_ahead) {
read_bytes = sftp_async_read(sf, buf, reqs[idx].len, reqs[idx].id);
if (read_bytes == SSH_ERROR) {
pr_err("sftp_async_read failed: %d\n", sftp_get_error(sf->sftp));
pr_err("sftp_async_read: %d\n", sftp_get_error(sf->sftp));
return -1;
}
@@ -785,12 +784,12 @@ static int chunk_copy_internal_remote_to_local(struct chunk *c, int fd, sftp_fil
write_bytes = write(fd, buf, read_bytes);
if (write_bytes < 0) {
pr_err("write to %s failed: %s\n", c->f->dst_path, strerrno());
pr_err("write: %s\n", strerrno());
return -1;
}
if (write_bytes < read_bytes) {
pr_err("failed to write full bytes to %s\n", c->f->dst_path);
pr_err("failed to write full bytes\n");
return -1;
}
@@ -834,10 +833,10 @@ static int chunk_copy_local_to_remote(struct chunk *c, sftp_session sftp,
}
#ifndef ASYNC_WRITE
ret = chunk_copy_internal_local_to_remote(c, fd, sf, sftp_buf_sz, io_buf_sz,
ret = _chunk_copy_local_to_remote(c, fd, sf, sftp_buf_sz, io_buf_sz,
counter);
#else
ret = chunk_copy_internal_local_to_remote_async(c, fd, sf, nr_ahead, counter);
ret = _chunk_copy_local_to_remote_async(c, fd, sf, nr_ahead, counter);
#endif
if (ret < 0)
goto out;
@@ -882,7 +881,7 @@ static int chunk_copy_remote_to_local(struct chunk *c, sftp_session sftp,
goto out;
}
ret = chunk_copy_internal_remote_to_local(c, fd, sf, nr_ahead, counter);
ret = _chunk_copy_remote_to_local(c, fd, sf, nr_ahead, counter);
if (ret< 0)
goto out;

15
src/list.h
View File

@@ -514,4 +514,19 @@ static inline void hlist_add_after(struct hlist_node *n,
pos = n)
/**
* list_count - return length of list
* @head the head for your list.
*/
static inline int list_count(struct list_head *head)
{
int n = 0;
struct list_head *p;
list_for_each(p, head) n++;
return n;
}
#endif

21
src/main.c
View File

@@ -74,21 +74,14 @@ void stop_copy_threads(int sig)
}
}
int list_count(struct list_head *head)
{
int n = 0;
struct list_head *p;
list_for_each(p, head) n++;
return n;
}
void usage(bool print_help) {
printf("mscp v" VERSION ": copy files over multiple ssh connections\n"
"\n"
"Usage: mscp [vqDCHdh] [-n nr_conns] [-m coremask]\n"
" [-s min_chunk_sz] [-S max_chunk_sz]\n"
" [-b sftp_buf_sz] [-B io_buf_sz] [-a nr_ahead]\n"
" [-s min_chunk_sz] [-S max_chunk_sz] [-a nr_ahead]\n"
#ifndef ASYNC_WRITE
" [-b sftp_buf_sz] [-B io_buf_sz] \n"
#endif
" [-l login_name] [-p port] [-i identity_file]\n"
" [-c cipher_spec] source ... target\n"
"\n");
@@ -101,12 +94,14 @@ void usage(bool print_help) {
" -s MIN_CHUNK_SIZE min chunk size (default: 64MB)\n"
" -S MAX_CHUNK_SIZE max chunk size (default: filesize / nr_conn)\n"
"\n"
" -a NR_AHEAD number of inflight SFTP commands (default: 16)\n"
#ifndef ASYNC_WRITE
" -b SFTP_BUF_SIZE buf size for sftp_read/write (default 131072B)\n"
" -B IO_BUF_SIZE buf size for read/write (default 131072B)\n"
" Note that the default value is derived from\n"
" qemu/block/ssh.c. need investigation...\n"
" -b and -B affect only local to remote copy\n"
" -a NR_AHEAD number of inflight SFTP read commands (default 16)\n"
#endif
"\n"
" -v increment verbose output level\n"
" -q disable output\n"
@@ -114,7 +109,7 @@ void usage(bool print_help) {
"\n"
" -l LOGIN_NAME login name\n"
" -p PORT port number\n"
" -i IDENTITY identity file for publickey authentication\n"
" -i IDENTITY identity file for public key authentication\n"
" -c CIPHER cipher spec, see `ssh -Q cipher`\n"
" -C enable compression on libssh\n"
" -H disable hostkey check\n"

14
src/rename-logic.py
View File

@@ -32,13 +32,17 @@ def recursive(src, rel_path, dst, dst_should_dir, replace_dir_name):
recursive(next_src, next_rel_path, dst, dst_should_dir, False)
def fill_dst(src, dst):
dst_should_dir = isdir(src) | isdir(dst)
replace_dir_name = not isdir(dst)
recursive(src, "", dst, dst_should_dir, replace_dir_name)
def fill_dst(srclist, dst):
dst_must_dir = len(srclist) > 1
for src in srclist:
dst_should_dir = isdir(src) | isdir(dst)
replace_dir_name = not isdir(dst)
recursive(src, "", dst, dst_should_dir | dst_must_dir, replace_dir_name)
def main():
fill_dst(sys.argv[1], sys.argv[2])
if (len(sys.argv) < 2):
print("usage: {} source ... target".format(sys.argv[0]))
fill_dst(sys.argv[1:len(sys.argv) - 1], sys.argv[len(sys.argv) - 1])
main()