Ma/mscp
1
0
Fork 0
mirror of https://github.com/upa/mscp.git synced 2026-02-16 02:14:42 +08:00
Code Issues Packages Projects Releases 1 Wiki Activity

fix buf size in remote to local copy.

Browse Source 
Too large buffer size for sftp_async_read causes unfinished copy:
sftp_async_read returns 0 althrough data remains.
This commit is contained in:
Ryo Nakamura
2022-11-12 17:30:51 +09:00
parent 71d827d613
commit 756e0759f9
3 changed files with 39 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
docker/README.md
View File

@@ -14,11 +14,11 @@ docker build -t mscp-centos:8 -f docker/centos-8.Dockerfile .
Test `mscp` in the containers.
```console
docker run --init --rm mscp-ubuntu:20.04 /build/mscp/scripts/test-in-container.sh
docker run --init --rm mscp-ubuntu:20.04 /mscp/scripts/test-in-container.sh
docker run --init --rm mscp-ubuntu:22.04 /build/mscp/scripts/test-in-container.sh
docker run --init --rm mscp-ubuntu:22.04 /mscp/scripts/test-in-container.sh
docker run --init --rm mscp-centos:8 /build/mscp/scripts/test-in-container.sh
docker run --init --rm mscp-centos:8 /mscp/scripts/test-in-container.sh
```
Retrieve deb/rpm packages.

46
src/file.c
View File

@@ -654,13 +654,26 @@ static int chunk_copy_internal_local_to_remote(struct chunk *c, int fd, sftp_fil
}
static int chunk_copy_internal_remote_to_local(struct chunk *c, int fd, sftp_file sf,
size_t sftp_buf_sz, size_t *counter)
size_t *counter)
{
#define AHEAD 8
#define AHEAD 8
#define XFER_BUF_SIZE 16384
ssize_t read_bytes, write_bytes, remaind, thrown;
char buf[sftp_buf_sz];
int reqs[AHEAD];
char buf[XFER_BUF_SIZE];
int idx;
struct {
int id;
size_t len;
} reqs[AHEAD];
/* TODO: sftp_buf_sz has no effect on remote to local copy. we
* always use 16384 byte buffer pointed by
* https://api.libssh.org/stable/libssh_tutor_sftp.html. The
* larget read length from sftp_async_read is 65536 byte.
* Read sizes larget than 65536 cause a situation where data
* remainds but sftp_async_read returns 0.
*/
if (c->len == 0)
return 0;
@@ -668,26 +681,30 @@ static int chunk_copy_internal_remote_to_local(struct chunk *c, int fd, sftp_fil
remaind = thrown = c->len;
for (idx = 0; idx < AHEAD && thrown > 0; idx++) {
reqs[idx] = sftp_async_read_begin(sf, min(thrown, sizeof(buf)));
if (reqs[idx] < 0) {
reqs[idx].len = min(thrown, sizeof(buf));
reqs[idx].id = sftp_async_read_begin(sf, reqs[idx].len);
if (reqs[idx].id < 0) {
pr_err("sftp_async_read_begin failed: %d\n",
sftp_get_error(sf->sftp));
return -1;
}
thrown -= min(thrown, sizeof(buf));
thrown -= reqs[idx].len;
}
idx = 0;
do {
read_bytes = sftp_async_read(sf, buf, sizeof(buf), reqs[idx]);
for (idx = 0; remaind > 0;) {
read_bytes = sftp_async_read(sf, buf, reqs[idx].len, reqs[idx].id);
if (read_bytes == SSH_ERROR) {
pr_err("sftp_async_read failed: %d\n", sftp_get_error(sf->sftp));
return -1;
}
remaind -= read_bytes;
reqs[idx] = sftp_async_read_begin(sf, min(remaind, sizeof(buf)));
idx = (idx + 1) % AHEAD;
if (remaind > 0) {
reqs[idx].len = min(remaind, sizeof(buf));
reqs[idx].id = sftp_async_read_begin(sf, reqs[idx].len);
}
write_bytes = write(fd, buf, read_bytes);
if (write_bytes < 0) {
@@ -701,7 +718,8 @@ static int chunk_copy_internal_remote_to_local(struct chunk *c, int fd, sftp_fil
}
*counter += write_bytes;
} while (remaind > 0);
idx = (idx + 1) % AHEAD;
}
return 0;
}
@@ -777,7 +795,7 @@ static int chunk_copy_remote_to_local(struct chunk *c, sftp_session sftp,
goto out;
}
ret = chunk_copy_internal_remote_to_local(c, fd, sf, sftp_buf_sz, counter);
ret = chunk_copy_internal_remote_to_local(c, fd, sf, counter);
if (ret< 0)
goto out;

5
src/main.c
View File

@@ -96,10 +96,13 @@ void usage(bool print_help) {
printf(" -n NR_CONNECTIONS number of connections (default: half of # of cpu cores)\n"
" -s MIN_CHUNK_SIZE min chunk size (default: 64MB)\n"
" -S MAX_CHUNK_SIZE max chunk size (default: filesize / nr_conn)\n"
"\n"
" -b SFTP_BUF_SIZE buf size for sftp_read/write (default 131072B)\n"
" -B IO_BUF_SIZE buf size for read/write (default 131072B)\n"
" Note that this value is derived from\n"
" Note that the default value is derived from\n"
" qemu/block/ssh.c. need investigation...\n"
" -b and -B affect only local to remote copy\n"
"\n"
" -v increment verbose output level\n"
" -q disable output\n"
" -D dry run\n"