2 Commits

Author SHA1 Message Date
Ryo Nakamura
16d3b5b772 fix github actions to obtain libssh version dynamically 2025-08-11 15:44:23 +09:00
Ryo Nakamura
27646fc71d bump libssh version to 0.11.2 2025-08-11 15:10:33 +09:00
25 changed files with 235 additions and 482 deletions

View File

@@ -1 +0,0 @@
build

View File

@@ -10,6 +10,34 @@ env:
BUILD_TYPE: Release
jobs:
build-and-release:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
submodules: true
- name: apply the patch to libssh
run: |
git -C libssh fetch --all --tags --prune
patch -d libssh -p1 < patch/$(git -C libssh describe).patch
# TODO: just building docker does not require packages. fix CMakeLists
- name: install build dependency
run: sudo ./scripts/install-build-deps.sh
- name: Configure Cmake
run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
- name: Build single binary mscp
run: make -C ${{github.workspace}}/build build-single-binary
- name: Release
uses: softprops/action-gh-release@v1
with:
files: |
${{github.workspace}}/build/mscp.linux.x86_64.static
source-release:
runs-on: ubuntu-latest
steps:

View File

@@ -22,7 +22,7 @@ jobs:
- rocky-8.9
- rocky-9.3
- almalinux-9.3
- alpine-3.22
- alpine-3.19
- arch-base
steps:
- uses: actions/checkout@v4

1
.gitignore vendored
View File

@@ -3,6 +3,5 @@ html
compile_commands.json
CMakeUserPresets.json
.*.swp
.cache
include/mscp_version.h

View File

@@ -168,7 +168,7 @@ enable_testing()
#
# When edit DIST_IDS and DIST_VERS, also edit .github/workflows/test.yaml
list(APPEND DIST_IDS ubuntu ubuntu ubuntu rocky rocky almalinux alpine arch)
list(APPEND DIST_VERS 20.04 22.04 24.04 8.9 9.3 9.3 3.22 base)
list(APPEND DIST_VERS 20.04 22.04 24.04 8.9 9.3 9.3 3.19 base)
list(LENGTH DIST_IDS _DIST_LISTLEN)
math(EXPR DIST_LISTLEN "${_DIST_LISTLEN} - 1")
@@ -208,16 +208,6 @@ foreach(x RANGE ${DIST_LISTLEN})
--add-host=ip6-localhost:::1
${DOCKER_IMAGE} /mscp/scripts/test-in-container.sh)
add_custom_target(docker-run-${DOCKER_INDEX}
COMMENT "Start ${DOCKER_IMAGE} container"
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
COMMAND
${CE} run --init --rm --privileged
--sysctl net.ipv6.conf.all.disable_ipv6=0
--add-host=ip6-localhost:::1
-it
${DOCKER_IMAGE} /mscp/scripts/test-in-container.sh bash)
list(APPEND DOCKER_BUILDS docker-build-${DOCKER_INDEX})
list(APPEND DOCKER_BUILDS_NO_CACHE docker-build-${DOCKER_INDEX}-no-cache)
list(APPEND DOCKER_TESTS docker-test-${DOCKER_INDEX})
@@ -288,9 +278,9 @@ add_custom_target(build-single-binary
COMMENT "Build mscp as a single binary in alpine conatiner"
WORKING_DIRECTORY ${mscp_SOURCE_DIR}
BYPRODUCTS ${CMAKE_BINARY_DIR}/${SINGLEBINARYFILE}
DEPENDS docker-build-alpine-3.22
DEPENDS docker-build-alpine-3.19
COMMAND
${CE} run --rm -v ${CMAKE_BINARY_DIR}:/out mscp-alpine:3.22
${CE} run --rm -v ${CMAKE_BINARY_DIR}:/out mscp-alpine:3.19
cp /mscp/build/mscp /out/${SINGLEBINARYFILE})

View File

@@ -1,4 +1,4 @@
FROM alpine:3.22
FROM alpine:3.19
# do not use REQUIREDPKGS build argument because
# this Dockerfile compiles mscp with conan,so we do not need
@@ -19,10 +19,6 @@ RUN ssh-keygen -A \
&& ssh-keygen -f /root/.ssh/id_rsa -N "" \
&& cat /root/.ssh/id_rsa.pub > /root/.ssh/authorized_keys
# disable PerSourcePenaltie, which would distrub test:
# https://undeadly.org/cgi?action=article;sid=20240607042157
RUN echo "PerSourcePenalties=no" > /etc/ssh/sshd_config.d/90-mscp-test.conf
# create test user
RUN addgroup -S test \
&& adduser -S test -G test \

View File

@@ -61,6 +61,12 @@ sudo dnf copr enable upaaa/mscp
sudo dnf install mscp
```
- Single binary `mscp` for x86_64 (not optimal performance)
```console
wget https://github.com/upa/mscp/releases/latest/download/mscp.linux.x86_64.static -O /usr/local/bin/mscp
chmod 755 /usr/local/bin/mscp
```
## Build
@@ -80,7 +86,7 @@ cd mscp
# prepare patched libssh
git submodule update --init
patch -d libssh -p1 < patch/$(git -C libssh describe).patch
patch -d libssh -p1 < patch/$(git --git-dir=./libssh/.git describe).patch
# install build dependency
bash ./scripts/install-build-deps.sh

View File

@@ -1 +1 @@
0.2.4
0.2.1

30
debian/changelog vendored
View File

@@ -1,32 +1,4 @@
mscp (0.2.4) UNRELEASED; urgency=medium
* fix password auth does not work (issue #36)
* add Debian and Devuan to the install dep script (pr #38)
* fallback to default terminal size on no tty (pr #39)
-- Ryo Nakamura <upa@haeena.net> Sat, 08 Nov 2025 19:23:07 +0900
mscp (0.2.3) unstable; urgency=medium
* bump libssh version to 0.11.2 (#25)
* adopt new AIO read API of libssh
* fix path handling when remote dst path has suffix '/' (#24)
* fix remote path handling including '~' (partially)
* try pubkey auth first to avoid PerSourcePenalties
* remove the single-binary version of mscp from release
-- Ryo Nakamura <upa@haeena.net> Tue, 12 Aug 2025 18:11:47 +0900
mscp (0.2.2) unstable; urgency=medium
* bump cmake_minimum_version on libssh (#32)
* fix quiet mode (#30)
* use openssl@3 on macOS (#29)
* add archlinux support (#28)
-- Ryo Nakamura <upa@haeena.net> Wed, 16 Apr 2025 17:01:17 +0900
mscp (0.2.1) unstable; urgency=medium
mscp (0.2.1) UNRELEASED; urgency=medium
* fix broken checkpoint files when copying multiple files (#16)
* fix broken password inputs for resume transfers (#17 and #18)

View File

@@ -41,9 +41,6 @@ mscp \- copy files over multiple SSH connections
.BI \-L \ LIMIT_BITRATE\c
]
[\c
.BI \-B \ BIND_ADDR\c
]
[\c
.BI \-l \ LOGIN_NAME\c
]
[\c
@@ -223,13 +220,6 @@ delivered over SSH. Changing this value is not recommended at present.
Limits the bitrate, specified with k (K), m (M), and g (G), e.g., 100m
indicates 100 Mbps.
.TP
.B \-B \fIBIND_ADDR\fR
Specifies a local IP address to bind. When multiple -B options are
specified, SSH connections will be bound to the addresses in a round
robin manner. This feature enables using multiple network interfaces
having differnt IP addresses.
.TP
.B \-4
Uses IPv4 addresses only.
@@ -445,18 +435,6 @@ will be faster than the default chacha20-poly1305 cipher, particularly
on hosts that support AES-NI.
.PP
Use multiple network interfaces having different IP addresses:
.nf
$ mscp -n 9 -B 10.0.0.10 -B 10.0.0.11 -B 10.0.0.12 many-large-files 192.168.0.1:dst/
.fi
Out of the 9 SSH connections, three of them are bound to each of
10.0.0.10, 10.0.0.11, and 10.0.0.12 in a round robin manner.
.SH "SEE ALSO"
.BR scp (1),
.BR ssh (1),

View File

@@ -2,7 +2,7 @@
MSCP
====
:Date: v0.2.4-6-gcd61363
:Date: v0.2.1
NAME
====
@@ -16,11 +16,10 @@ SYNOPSIS
*COREMASK* ] [ **-u** *MAX_STARTUPS* ] [ **-I** *INTERVAL* ] [ **-W**
*CHECKPOINT* ] [ **-R** *CHECKPOINT* ] [ **-s** *MIN_CHUNK_SIZE* ] [
**-S** *MAX_CHUNK_SIZE* ] [ **-a** *NR_AHEAD* ] [ **-b** *BUF_SIZE* ] [
**-L** *LIMIT_BITRATE* ] [ **-B** *BIND_ADDR* ] [ **-l** *LOGIN_NAME* ]
[ **-P** *PORT* ] [ **-F** *SSH_CONFIG* ] [ **-o** *SSH_OPTION* ] [
**-i** *IDENTITY* ] [ **-J** *DESTINATION* ] [ **-c** *CIPHER* ] [
**-M** *HMAC* ] [ **-C** *COMPRESS* ] [ **-g** *CONGESTION* ] *source
... target*
**-L** *LIMIT_BITRATE* ] [ **-l** *LOGIN_NAME* ] [ **-P** *PORT* ] [
**-F** *SSH_CONFIG* ] [ **-o** *SSH_OPTION* ] [ **-i** *IDENTITY* ] [
**-J** *DESTINATION* ] [ **-c** *CIPHER* ] [ **-M** *HMAC* ] [ **-C**
*COMPRESS* ] [ **-g** *CONGESTION* ] *source ... target*
DESCRIPTION
===========
@@ -121,12 +120,6 @@ OPTIONS
Limits the bitrate, specified with k (K), m (M), and g (G), e.g.,
100m indicates 100 Mbps.
**-B BIND_ADDR**
Specifies a local IP address to bind. When multiple -B options are
specified, SSH connections will be bound to the addresses in a round
robin manner. This feature enables using multiple network interfaces
having differnt IP addresses.
**-4**
Uses IPv4 addresses only.
@@ -276,15 +269,6 @@ SFTP WRITE/READ commands, and **-c aes128-gcm@openssh.com** will be
faster than the default chacha20-poly1305 cipher, particularly on hosts
that support AES-NI.
Use multiple network interfaces having different IP addresses:
::
$ mscp -n 9 -B 10.0.0.10 -B 10.0.0.11 -B 10.0.0.12 many-large-files 192.168.0.1:dst/
Out of the 9 SSH connections, three of them are bound to each of
10.0.0.10, 10.0.0.11, and 10.0.0.12 in a round robin manner.
SEE ALSO
========

View File

@@ -28,8 +28,6 @@
#include <stdbool.h>
#include <limits.h>
#include <stdlib.h>
#define MSCP_DIRECTION_L2R 1 /** Indicates local to remote copy */
#define MSCP_DIRECTION_R2L 2 /** Indicates remote to local copy */
@@ -61,7 +59,6 @@ struct mscp_ssh_opts {
/* ssh options */
char *login_name; /** ssh username */
char *port; /** ssh port */
char **bind_addrs; /** addresses to bind, terminated by NULL */
int ai_family; /** address family */
char *config; /** path to ssh_config, default ~/.ssh/config*/
char **options; /** array of ssh_config options, terminated by NULL */

View File

@@ -37,7 +37,7 @@ index d22178e7..2d6aa0a7 100644
int ssh_buffer_validate_length(struct ssh_buffer_struct *buffer, size_t len);
diff --git a/include/libssh/libssh.h b/include/libssh/libssh.h
index 3bddb019..1d5d7761 100644
index 3bddb019..c6b01c1c 100644
--- a/include/libssh/libssh.h
+++ b/include/libssh/libssh.h
@@ -373,6 +373,7 @@ enum ssh_options_e {
@@ -64,7 +64,7 @@ index 3bddb019..1d5d7761 100644
LIBSSH_API void ssh_buffer_free(ssh_buffer buffer);
#define SSH_BUFFER_FREE(x) \
do { if ((x) != NULL) { ssh_buffer_free(x); x = NULL; } } while(0)
@@ -886,6 +889,12 @@ LIBSSH_API void *ssh_buffer_get(ssh_buffer buffer);
@@ -886,6 +889,11 @@ LIBSSH_API void *ssh_buffer_get(ssh_buffer buffer);
LIBSSH_API uint32_t ssh_buffer_get_len(ssh_buffer buffer);
LIBSSH_API int ssh_session_set_disconnect_message(ssh_session session, const char *message);
@@ -72,7 +72,6 @@ index 3bddb019..1d5d7761 100644
+
+LIBSSH_API const char **ssh_ciphers(void);
+LIBSSH_API const char **ssh_hmacs(void);
+LIBSSH_API void ssh_use_openssh_proxy_jumps(int);
+
#ifndef LIBSSH_LEGACY_0_4
#include "libssh/legacy.h"
@@ -288,7 +287,7 @@ index 2cb64037..51f4c87e 100644
rc = connect(s, itr->ai_addr, itr->ai_addrlen);
if (rc == -1) {
diff --git a/src/misc.c b/src/misc.c
index 774211fb..74e57959 100644
index 774211fb..ae62ddfe 100644
--- a/src/misc.c
+++ b/src/misc.c
@@ -71,6 +71,8 @@
@@ -343,37 +342,6 @@ index 774211fb..74e57959 100644
/**
* @brief Write the requested number of bytes to a local file.
*
@@ -2227,6 +2265,17 @@ ssh_proxyjumps_free(struct ssh_list *proxy_jump_list)
}
}
+static bool force_openssh_proxy_jumps;
+
+/**
+ * @breif set use openssh proxy jumps without the OPENSSH_PROXYJUMP env var
+ */
+void
+ssh_use_openssh_proxy_jumps(int v)
+{
+ force_openssh_proxy_jumps = (v > 0);
+}
+
/**
* @brief Check if libssh proxy jumps is enabled
*
@@ -2241,7 +2290,12 @@ ssh_libssh_proxy_jumps(void)
{
const char *t = getenv("OPENSSH_PROXYJUMP");
+ if (force_openssh_proxy_jumps)
+ return false;
+
return !(t != NULL && t[0] == '1');
}
+
+
/** @} */
diff --git a/src/options.c b/src/options.c
index 785296dd..a82d4d81 100644
--- a/src/options.c

View File

@@ -38,15 +38,6 @@ make -C build install DESTDIR=%{buildroot}
%changelog
* Sat Nov 08 2025 Ryo Nakamura <upa@haeena.net> - 0.2.4-1
- RPM release for v0.2.4
* Tue Aug 12 2025 Ryo Nakamura <upa@haeena.net> - 0.2.3-1
- RPM release for v0.2.3
* Wed Apr 16 2025 Ryo Nakamura <upa@haeena.net> - 0.2.2-1
- RPM release for v0.2.2
* Sat May 11 2024 Ryo Nakamura <upa@haeena.net> - 0.2.1-1
- RPM release for v0.2.1

View File

@@ -47,7 +47,7 @@ case $platform in
cmd="brew install"
pkgs="openssl@3"
;;
Linux-ubuntu* | Linux-debian* | Linux-devuan*)
Linux-ubuntu*)
cmd="apt-get install --no-install-recommends -y"
pkgs="gcc make cmake zlib1g-dev libssl-dev libkrb5-dev"
;;

View File

@@ -28,11 +28,5 @@ for port in 22 8022; do
ssh-keyscan -p $port ::1 >> ${HOME}/.ssh/known_hosts
done
if [ $# -gt 0 ]; then
# command arguments are passed, exec them
"$@"
else
# no arguments passed, run the test
python3 -m pytest -v ../test
fi
# Run test
python3 -m pytest -v ../test

View File

@@ -50,6 +50,8 @@ int bwlimit_init(struct bwlimit *bw, uint64_t bps, uint64_t win)
#define timespeccmp(a, b, expr) \
((a.tv_sec * 1000000000 + a.tv_nsec) expr(b.tv_sec * 1000000000 + b.tv_nsec))
#include <stdio.h>
int bwlimit_wait(struct bwlimit *bw, size_t nr_bytes)
{
struct timespec now, end, rq, rm;

View File

@@ -9,10 +9,10 @@
#include <semaphore.h>
struct bwlimit {
sem_t *sem; /* semaphore */
size_t bps; /* limit bit-rate (bps) */
size_t win; /* window size (msec) */
size_t amt; /* amount of bytes can be sent in a window */
sem_t *sem; /* semaphore */
uint64_t bps; /* limit bit-rate (bps) */
uint64_t win; /* window size (msec) */
size_t amt; /* amount of bytes can be sent in a window */
ssize_t credit; /* remaining bytes can be sent in a window */
struct timespec wstart, wend; /* window start time and end time */

View File

@@ -101,6 +101,7 @@ MDIR *mscp_opendir_wrapped(const char *path)
void mscp_closedir(MDIR *md)
{
int ret;
if (md->remote)
sftp_closedir(md->remote);
else

View File

@@ -28,7 +28,7 @@ void usage(bool print_help)
"Usage: mscp [-46vqDpdNh] [-n nr_conns] [-m coremask] [-u max_startups]\n"
" [-I interval] [-W checkpoint] [-R checkpoint]\n"
" [-s min_chunk_sz] [-S max_chunk_sz] [-a nr_ahead]\n"
" [-b buf_sz] [-L limit_bitrate] [-B bind_addr]\n"
" [-b buf_sz] [-L limit_bitrate]\n"
" [-l login_name] [-P port] [-F ssh_config] [-o ssh_option]\n"
" [-i identity_file] [-J destination] [-c cipher_spec] [-M hmac_spec]\n"
" [-C compress] [-g congestion]\n"
@@ -51,8 +51,7 @@ void usage(bool print_help)
" -S MAX_CHUNK_SIZE max chunk size (default: filesize/nr_conn/4)\n"
" -a NR_AHEAD number of inflight SFTP commands (default: 32)\n"
" -b BUF_SZ buffer size for i/o and transfer\n"
" -L LIMIT_BITRATE limit the bitrate, n[KMG] (default: 0, no limit)\n"
" -B BIND_ADDR bind address (accept multiple times)\n"
" -L LIMIT_BITRATE Limit the bitrate, n[KMG] (default: 0, no limit)\n"
"\n"
" -4 use IPv4\n"
" -6 use IPv6\n"
@@ -118,6 +117,8 @@ char *split_user_host_path(const char *s, char **userp, char **hostp, char **pat
return NULL;
}
user = NULL;
host = NULL;
path = tmp;
for (cp = tmp; *cp; cp++) {
if (*cp == '@' && (cp > tmp) && *(cp - 1) != '\\' && user == NULL) {
@@ -204,12 +205,13 @@ struct target *validate_targets(char **arg, int len)
*/
struct target *t, *t0;
int n, nslash;
int n;
if ((t = calloc(len, sizeof(struct target))) == NULL) {
pr_err("calloc: %s", strerrno());
return NULL;
}
memset(t, 0, len * sizeof(struct target));
/* split remote:path into remote and path */
for (n = 0; n < len; n++) {
@@ -221,33 +223,9 @@ struct target *validate_targets(char **arg, int len)
}
}
/* expand remote path, e.g., empty dst path and '~' */
for (n = 0; n < len; n++) {
if (!t[n].host)
continue;
/* this target is a remote path. check the path and
* expand it. this part is derived from
* openssh-portal prepare_remote_path() function.
*/
char *path = t[n].path;
if (*path == '\0' || strcmp(path, "~") == 0)
t[n].path = strdup(".");
else if (strncmp(path, "~/", 2) == 0) {
if ((nslash = strspn(path + 2, "/")) == strlen(path + 2))
t[n].path = strdup(".");
else
t[n].path = strdup(path + 2 + nslash);
}
if (!t[n].path) {
pr_err("strdup failed: %s", strerrno());
goto free_target_out;
}
}
/* check all user@host are identical. t[len - 1] is the
* destination, so we need to check t[0] to t[len - 2] having
* the identical remote notation */
/* check all user@host are identical. t[len - 1] is destination,
* so we need to check t[0] to t[len - 2] having the identical
* remote notation */
t0 = &t[0];
for (n = 1; n < len - 1; n++) {
if (compare_remote(t0, &t[n]) != 0)
@@ -307,7 +285,7 @@ long atol_with_unit(char *value, bool i)
{
/* value must be "\d+[kKmMgG]?" */
char *u = value + (strlen(value) - 1);
char *u = value + (strlen(optarg) - 1);
long k = i ? 1024 : 1000;
long factor = 1;
long v;
@@ -357,18 +335,18 @@ int main(int argc, char **argv)
struct mscp_ssh_opts s;
struct mscp_opts o;
struct target *t;
int pipe_fd[2];
int ch, n, i, ret;
int direction = 0;
char *remote = NULL, *checkpoint_save = NULL, *checkpoint_load = NULL;
bool quiet = false, dryrun = false, resume = false;
int nr_baddrs = 0;
int nr_options = 0;
memset(&s, 0, sizeof(s));
memset(&o, 0, sizeof(o));
o.severity = MSCP_SEVERITY_WARN;
#define mscpopts "n:m:u:I:W:R:s:S:a:b:L:B:46vqDrl:P:F:o:i:J:c:M:C:g:pdNh"
#define mscpopts "n:m:u:I:W:R:s:S:a:b:L:46vqDrl:P:F:o:i:J:c:M:C:g:pdNh"
while ((ch = getopt(argc, argv, mscpopts)) != -1) {
switch (ch) {
case 'n':
@@ -409,17 +387,6 @@ int main(int argc, char **argv)
case 'L':
o.bitrate = atol_with_unit(optarg, false);
break;
case 'B':
nr_baddrs++;
s.bind_addrs = realloc(s.bind_addrs,
sizeof(char *) * (nr_baddrs + 1));
if (!s.bind_addrs) {
pr_err("realloc: %s", strerrno());
return 1;
}
s.bind_addrs[nr_baddrs - 1] = optarg;
s.bind_addrs[nr_baddrs] = NULL;
break;
case '4':
s.ai_family = AF_INET;
break;
@@ -706,13 +673,8 @@ void print_progress_bar(double percent, char *suffix)
buf[0] = '\0';
if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws) < 0 || ws.ws_col == 0) {
// fallback to default
ws.ws_col = 80;
ws.ws_row = 24;
}
if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws) < 0)
return; /* XXX */
bar_width = min(sizeof(buf), ws.ws_col) - strlen(suffix) - 7;
memset(buf, 0, sizeof(buf));
@@ -790,6 +752,8 @@ struct xfer_stat x;
void print_stat(bool final)
{
struct mscp_stats s;
char buf[8192];
int timeout;
gettimeofday(&x.after, NULL);
if (calculate_timedelta(&x.before, &x.after) > 1 || final) {
@@ -805,6 +769,9 @@ void print_stat(bool final)
void *print_stat_thread(void *arg)
{
struct mscp_stats s;
char buf[8192];
memset(&x, 0, sizeof(x));
gettimeofday(&x.start, NULL);
x.before = x.start;

View File

@@ -78,7 +78,7 @@ struct mscp {
static int expand_coremask(const char *coremask, int **cores, int *nr_cores)
{
int n, *core_list, nr_usable, nr_all;
int n, *core_list, core_list_len = 0, nr_usable, nr_all;
char c[2] = { 'x', '\0' };
const char *_coremask;
long v, needle;
@@ -273,13 +273,7 @@ struct mscp *mscp_init(struct mscp_opts *o, struct mscp_ssh_opts *s)
priv_set_errv("bwlimit_init: %s", strerrno());
goto free_out;
}
/* workaround: set libssh using openssh proxyjump
* https://gitlab.com/libssh/libssh-mirror/-/issues/319 */
ssh_use_openssh_proxy_jumps(1);
/* call ssh_init() because libssh is statically linked */
ssh_init();
pr_notice("bitrate limit: %lu bps", o->bitrate);
return m;
@@ -328,12 +322,10 @@ int mscp_set_dst_path(struct mscp *m, const char *dst_path)
return -1;
}
if (!non_null_string(dst_path)) {
priv_set_errv("empty dst path");
return -1;
}
strncpy(m->dst_path, dst_path, PATH_MAX);
if (!non_null_string(dst_path))
strncpy(m->dst_path, ".", 1);
else
strncpy(m->dst_path, dst_path, PATH_MAX);
return 0;
}
@@ -401,9 +393,6 @@ void *mscp_scan_thread(void *arg)
if (pool_size(m->src_pool) > 1)
a.dst_path_should_dir = true;
if (m->dst_path[strlen(m->dst_path) - 1] == '/')
a.dst_path_should_dir = true;
if (mscp_stat(m->dst_path, &ds, dst_sftp) == 0) {
if (S_ISDIR(ds.st_mode))
a.dst_path_is_dir = true;
@@ -562,9 +551,6 @@ int mscp_start(struct mscp *m)
m->opts->nr_threads = n;
}
pr_notice("threads: %d",m->opts->nr_threads);
pr_notice("bwlimit: %ld bps", m->bw.bps);
for (n = 0; n < m->opts->nr_threads; n++) {
t = mscp_copy_thread_spawn(m, n);
if (!t)

View File

@@ -410,45 +410,46 @@ static int copy_chunk_l2r(struct chunk *c, int fd, sftp_file sf, int nr_ahead, i
return 0;
}
static int copy_chunk_r2l(struct chunk *c, sftp_file sf, int fd,
int nr_ahead, int buf_sz,
struct bwlimit *bw, size_t *counter)
static int copy_chunk_r2l(struct chunk *c, sftp_file sf, int fd, int nr_ahead, int buf_sz,
struct bwlimit *bw, size_t *counter)
{
ssize_t read_bytes, write_bytes, remain, thrown, len, requested;
sftp_aio reqs[nr_ahead];
ssize_t read_bytes, write_bytes, remaind, thrown;
char buf[buf_sz];
int i;
int idx;
struct {
int id;
ssize_t len;
} reqs[nr_ahead];
if (c->len == 0)
return 0;
remain = thrown = c->len;
remaind = thrown = c->len;
for (i = 0; i < nr_ahead && thrown > 0; i++) {
len = min(thrown, sizeof(buf));
requested = sftp_aio_begin_read(sf, len, &reqs[i]);
if (requested == SSH_ERROR) {
priv_set_errv("sftp_aio_begin_read: %d",
for (idx = 0; idx < nr_ahead && thrown > 0; idx++) {
reqs[idx].len = min(thrown, sizeof(buf));
reqs[idx].id = sftp_async_read_begin(sf, reqs[idx].len);
if (reqs[idx].id < 0) {
priv_set_errv("sftp_async_read_begin: %d",
sftp_get_error(sf->sftp));
return -1;
}
thrown -= requested;
bwlimit_wait(bw, requested);
thrown -= reqs[idx].len;
bwlimit_wait(bw, reqs[idx].len);
}
for (i = 0; remain > 0; i = (i + 1) % nr_ahead) {
read_bytes = sftp_aio_wait_read(&reqs[i], buf, sizeof(buf));
for (idx = 0; remaind > 0; idx = (idx + 1) % nr_ahead) {
read_bytes = sftp_async_read(sf, buf, reqs[idx].len, reqs[idx].id);
if (read_bytes == SSH_ERROR) {
priv_set_errv("sftp_aio_wait_read: %d",
sftp_get_error(sf->sftp));
priv_set_errv("sftp_async_read: %d", sftp_get_error(sf->sftp));
return -1;
}
if (thrown > 0) {
len = min(thrown, sizeof(buf));
requested = sftp_aio_begin_read(sf, len, &reqs[i]);
thrown -= requested;
bwlimit_wait(bw, requested);
reqs[idx].len = min(thrown, sizeof(buf));
reqs[idx].id = sftp_async_read_begin(sf, reqs[idx].len);
thrown -= reqs[idx].len;
bwlimit_wait(bw, reqs[idx].len);
}
write_bytes = write(fd, buf, read_bytes);
@@ -463,13 +464,13 @@ static int copy_chunk_r2l(struct chunk *c, sftp_file sf, int fd,
}
*counter += write_bytes;
remain -= write_bytes;
remaind -= read_bytes;
}
if (remain < 0) {
priv_set_errv("invalid remain bytes %ld. last async_read bytes %ld. "
if (remaind < 0) {
priv_set_errv("invalid remaind bytes %ld. last async_read bytes %ld. "
"last write bytes %ld",
remain, read_bytes, write_bytes);
remaind, read_bytes, write_bytes);
return -1;
}

View File

@@ -9,8 +9,7 @@ void set_print_severity(int serverity)
{
if (serverity < 0)
__print_severity = -1; /* no print */
else
__print_severity = serverity;
__print_severity = serverity;
}
int get_print_severity()

View File

@@ -6,7 +6,6 @@
#include <ssh.h>
#include <mscp.h>
#include <atomic.h>
#include <strerrno.h>
#include "libssh/callbacks.h"
@@ -15,9 +14,6 @@
static int ssh_verify_known_hosts(ssh_session session);
static int ssh_authenticate_kbdint(ssh_session session);
static lock bind_addr_lock = PTHREAD_MUTEX_INITIALIZER;
static int bind_addr_idx;
static int ssh_set_opts(ssh_session ssh, struct mscp_ssh_opts *opts)
{
ssh_set_log_level(opts->debug_level);
@@ -33,21 +29,6 @@ static int ssh_set_opts(ssh_session ssh, struct mscp_ssh_opts *opts)
return -1;
}
if (opts->bind_addrs) {
/* ssh_set_opts can be called from multiple threads,
* thus protect bind_addr_idx with a lock. */
char *bind_addr;
LOCK_ACQUIRE(&bind_addr_lock);
bind_addr = opts->bind_addrs[bind_addr_idx];
bind_addr_idx = (opts->bind_addrs[bind_addr_idx + 1] == NULL ?
0 : bind_addr_idx + 1);
LOCK_RELEASE();
if (ssh_options_set(ssh, SSH_OPTIONS_BINDADDR, bind_addr) < 0) {
priv_set_errv("failed to set bind address %s", bind_addr);
return -1;
}
}
if (opts->ai_family &&
ssh_options_set(ssh, SSH_OPTIONS_AI_FAMILY, &opts->ai_family) < 0) {
priv_set_errv("failed to set address family");
@@ -133,32 +114,27 @@ static int ssh_set_opts(ssh_session ssh, struct mscp_ssh_opts *opts)
static int ssh_authenticate(ssh_session ssh, struct mscp_ssh_opts *opts)
{
static int auth_bit_mask;
int auth_bit_mask;
int ret;
if (auth_bit_mask == 0) {
/* the first authentication attempt. try none auth to
* get available auth methods. */
if (ssh_userauth_none(ssh, NULL) == SSH_AUTH_SUCCESS)
return 0;
/* none method */
ret = ssh_userauth_none(ssh, NULL);
if (ret == SSH_AUTH_SUCCESS)
return 0;
/* save auth_bit_mask for further authentications.
* when an authentication succeeds, auth_bit_mask is
* overwritten with the suceeded authentication method
* to avoid authentication failures by other methods.
*/
auth_bit_mask = ssh_userauth_list(ssh, NULL);
}
auth_bit_mask = ssh_userauth_list(ssh, NULL);
if (auth_bit_mask & SSH_AUTH_METHOD_NONE &&
ssh_userauth_none(ssh, NULL) == SSH_AUTH_SUCCESS)
return 0;
auth_bit_mask = ssh_userauth_list(ssh, NULL);
if (auth_bit_mask & SSH_AUTH_METHOD_PUBLICKEY) {
char *p = opts->passphrase ? opts->passphrase : NULL;
if (ssh_userauth_publickey_auto(ssh, NULL, p) == SSH_AUTH_SUCCESS) {
auth_bit_mask = SSH_AUTH_METHOD_PUBLICKEY;
return 0;
}
}
char *p = opts->passphrase ? opts->passphrase : NULL;
if (ssh_userauth_publickey_auto(ssh, NULL, p) == SSH_AUTH_SUCCESS)
return 0;
}
auth_bit_mask = ssh_userauth_list(ssh, NULL);
if (auth_bit_mask & SSH_AUTH_METHOD_PASSWORD) {
if (!opts->password) {
char buf[128] = {};
@@ -172,17 +148,14 @@ static int ssh_authenticate(ssh_session ssh, struct mscp_ssh_opts *opts)
}
}
if (ssh_userauth_password(ssh, NULL, opts->password) == SSH_AUTH_SUCCESS) {
auth_bit_mask = SSH_AUTH_METHOD_PASSWORD;
return 0;
}
if (ssh_userauth_password(ssh, NULL, opts->password) == SSH_AUTH_SUCCESS)
return 0;
}
auth_bit_mask = ssh_userauth_list(ssh, NULL);
if (auth_bit_mask & SSH_AUTH_METHOD_INTERACTIVE) {
if (ssh_authenticate_kbdint(ssh) == SSH_AUTH_SUCCESS) {
auth_bit_mask = SSH_AUTH_METHOD_INTERACTIVE;
return 0;
}
if (ssh_authenticate_kbdint(ssh) == SSH_AUTH_SUCCESS)
return 0;
}
return -1;

View File

@@ -9,71 +9,28 @@ import getpass
import datetime
import time
import os
import re
import shutil
from subprocess import check_output, CalledProcessError, STDOUT
from subprocess import check_call, CalledProcessError
from util import File, check_same_md5sum
def run2ok(args, env = None) -> str:
def run2ok(args, env = None, quiet = False):
cmd = list(map(str, args))
print("cmd: {}".format(" ".join(cmd)))
out = check_output(cmd, env = env, stderr = STDOUT).decode()
print(out)
return out
if not quiet:
print("cmd: {}".format(" ".join(cmd)))
check_call(cmd, env = env)
def run2ng(args, env = None, timeout = None):
def run2ng(args, env = None, timeout = None, quiet = False):
if timeout:
args = ["timeout", "-s", "INT", timeout] + args
cmd = list(map(str, args))
print("cmd: {}".format(" ".join(cmd)))
with pytest.raises(CalledProcessError) as execinfo:
check_output(cmd, env = env, stderr = STDOUT)
out = execinfo.value.stdout.decode()
print(out)
return out
if not quiet:
print("cmd: {}".format(" ".join(cmd)))
with pytest.raises(CalledProcessError):
check_call(cmd, env = env)
@pytest.fixture(autouse=True)
def cleanup_files():
"""
Cleanup files having the following `prefixes` or matching `paths`.
"""
yield
prefixes = [
"src", "dst",
"non_existent_dstdir",
]
paths = [
"/mscp-test-src", "/tmp/mscp-test-src",
"{}/src".format(os.environ["HOME"]),
"{}/dst".format(os.environ["HOME"]),
"/tmp/mscp_test_ssh_config",
"/home/test/dst",
"/home/test/src",
"checkpoint",
]
def remove(path):
print(f"cleanup remove: {fname}")
if os.path.isdir(path):
shutil.rmtree(path)
else:
os.remove(path)
for fname in os.listdir(os.getcwd()):
for prefix in prefixes:
if fname.startswith(prefix):
remove(fname)
break
for path in paths:
if os.path.exists(path):
remove(path)
""" usage test """
@@ -115,7 +72,8 @@ def test_single_copy(mscp, src_prefix, dst_prefix, src, dst):
src.make()
run2ok([mscp, "-vvv", src_prefix + src.path, dst_prefix + dst.path])
assert check_same_md5sum(src, dst)
src.cleanup()
dst.cleanup()
@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
def test_failed_to_copy_nonexistent_file(mscp, src_prefix, dst_prefix):
@@ -136,6 +94,10 @@ def test_double_copy(mscp, src_prefix, dst_prefix, s1, s2, d1, d2):
run2ok([mscp, "-vvv", src_prefix + s1.path, src_prefix + s2.path, dst_prefix + "dst"])
assert check_same_md5sum(s1, d1)
assert check_same_md5sum(s2, d2)
s1.cleanup()
s2.cleanup()
d1.cleanup()
d2.cleanup()
remote_v6_prefix = "[::1]:{}/".format(os.getcwd())
@@ -151,6 +113,10 @@ def test_double_copy_with_ipv6_notation(mscp, src_prefix, dst_prefix, s1, s2, d1
src_prefix + s1.path, src_prefix + s2.path, dst_prefix + "dst"])
assert check_same_md5sum(s1, d1)
assert check_same_md5sum(s2, d2)
s1.cleanup()
s2.cleanup()
d1.cleanup()
d2.cleanup()
remote_user_v6_prefix = "{}@[::1]:{}/".format(getpass.getuser(), os.getcwd())
@@ -167,6 +133,11 @@ def test_double_copy_with_user_and_ipv6_notation(mscp, src_prefix, dst_prefix,
src_prefix + s1.path, src_prefix + s2.path, dst_prefix + "dst"])
assert check_same_md5sum(s1, d1)
assert check_same_md5sum(s2, d2)
s1.cleanup()
s2.cleanup()
d1.cleanup()
d2.cleanup()
param_dir_copy = [
@@ -206,6 +177,10 @@ def test_dir_copy(mscp, src_prefix, dst_prefix, src_dir, dst_dir, src, dst, twic
for sf, df in zip(src, twice):
assert check_same_md5sum(sf, df)
for sf, df, tf in zip(src, dst, twice):
sf.cleanup()
df.cleanup()
tf.cleanup()
param_dir_copy_single = [
@@ -221,7 +196,8 @@ def test_dir_copy_single(mscp, src_prefix, dst_prefix, src_dir, dst_dir, src, ds
os.mkdir(dst_dir)
run2ok([mscp, "-vvv", src_prefix + src_dir, dst_prefix + dst_dir])
assert check_same_md5sum(src, dst)
src.cleanup()
dst.cleanup()
@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
def test_override_single_file(mscp, src_prefix, dst_prefix):
@@ -232,6 +208,8 @@ def test_override_single_file(mscp, src_prefix, dst_prefix):
run2ok([mscp, "-vvv", src_prefix + src.path, dst_prefix + dst.path])
assert check_same_md5sum(src, dst)
src.cleanup()
dst.cleanup()
absolute_remote_prefix = "localhost:"
param_absolute_remote_prefix = [
@@ -245,55 +223,8 @@ def test_copy_file_under_root_to_dir(mscp, src_prefix, dst_prefix):
run2ok([mscp, "-vvv", src_prefix + src.path,
dst_prefix + os.path.dirname(dst.path)])
assert check_same_md5sum(src, dst)
@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
def test_dst_has_suffix_slash(mscp, src_prefix, dst_prefix):
"""
if dst path has suffix '/' like "dir/" and does not exist,
mscp should create dir/ and put dir/src-file-name.
"""
dstdir = "non_existent_dstdir/"
src = File("src", size = 1024 * 1024).make()
dst = File(f"{dstdir}/src")
run2ok([mscp, "-vvv", src_prefix + src.path,
dst_prefix + dstdir])
assert check_same_md5sum(src, dst)
param_tilde_paths = [
("src", "localhost:~/dst"),
("localhost:~/src", "dst"),
]
@pytest.mark.parametrize("src_path, dst_path", param_tilde_paths)
def test_remote_path_contains_tilde(mscp, src_path, dst_path):
"""
if remote path contains '~' as prefix, it should be expanded as '.'.
Note that `~user` notation is not supported yet.
"""
def extract_and_expand(path):
path = path if not ':' in path else path[path.index(':')+1:]
return path.replace('~', os.environ["HOME"])
src_f_path = extract_and_expand(src_path)
dst_f_path = extract_and_expand(dst_path)
src = File(src_f_path, size = 1024 * 1024).make()
dst = File(dst_f_path)
run2ok([mscp, "-vvv", src_path, dst_path])
assert check_same_md5sum(src, dst)
def test_remote_path_contains_tilde2(mscp):
src = File("src", size = 1024 * 1024).make()
dst = File(f"{os.environ['HOME']}/src")
run2ok([mscp, "-vvv", src.path, f"localhost:~"])
assert check_same_md5sum(src, dst)
src.cleanup()
dst.cleanup(preserve_dir = True)
@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
@@ -304,6 +235,9 @@ def test_min_chunk(mscp, src_prefix, dst_prefix):
run2ok([mscp, "-vvv", "-s", 32768, src_prefix + src.path, dst_prefix + dst.path])
assert check_same_md5sum(src, dst)
src.cleanup()
dst.cleanup()
def is_alpine():
if os.path.exists("/etc/os-release"):
@@ -338,7 +272,8 @@ def test_glob_src_path(mscp, src_prefix, dst_prefix,
run2ok([mscp, "-vvv", src_prefix + src_glob_path, dst_prefix + dst_path])
for src, dst in zip(srcs, dsts):
assert check_same_md5sum(src, dst)
src.cleanup()
dst.cleanup()
@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
def test_thread_affinity(mscp, src_prefix, dst_prefix):
@@ -349,6 +284,8 @@ def test_thread_affinity(mscp, src_prefix, dst_prefix):
src_prefix + src.path, dst_prefix + dst.path])
assert check_same_md5sum(src, dst)
src.cleanup()
dst.cleanup()
@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
def test_cannot_override_file_with_dir(mscp, src_prefix, dst_prefix):
@@ -357,6 +294,8 @@ def test_cannot_override_file_with_dir(mscp, src_prefix, dst_prefix):
run2ng([mscp, "-vvv", src_prefix + src.path, dst_prefix + "dst/src"])
src.cleanup()
dst.cleanup()
@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
def test_transfer_zero_bytes(mscp, src_prefix, dst_prefix):
@@ -364,6 +303,8 @@ def test_transfer_zero_bytes(mscp, src_prefix, dst_prefix):
dst = File("dst")
run2ok([mscp, "-vvv", src_prefix + src.path, dst_prefix + "dst"])
assert os.path.exists("dst")
src.cleanup()
dst.cleanup()
@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
def test_override_dst_having_larger_size(mscp, src_prefix, dst_prefix):
@@ -371,6 +312,8 @@ def test_override_dst_having_larger_size(mscp, src_prefix, dst_prefix):
dst = File("dst", size = 1024 * 1024 * 2).make()
run2ok([mscp, "-vvv", src_prefix + src.path, dst_prefix + "dst"])
assert check_same_md5sum(src, dst)
src.cleanup()
dst.cleanup()
@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
def test_dont_truncate_dst(mscp, src_prefix, dst_prefix):
@@ -379,7 +322,7 @@ def test_dont_truncate_dst(mscp, src_prefix, dst_prefix):
run2ok([mscp, "-vvv", src_prefix + f.path, dst_prefix + f.path])
md5_after = f.md5sum()
assert md5_before == md5_after
f.cleanup()
@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
def test_copy_readonly_file(mscp, src_prefix, dst_prefix):
@@ -393,6 +336,8 @@ def test_copy_readonly_file(mscp, src_prefix, dst_prefix):
dst = File("dst")
run2ok([mscp, "-vvv", src_prefix + src.path, dst_prefix + dst.path])
assert check_same_md5sum(src, dst)
src.cleanup()
dst.cleanup()
@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
def test_dont_make_conns_more_than_chunks(mscp, src_prefix, dst_prefix):
@@ -409,7 +354,8 @@ def test_dont_make_conns_more_than_chunks(mscp, src_prefix, dst_prefix):
end = time.time()
for s, d in zip(srcs, dsts):
assert check_same_md5sum(s, d)
shutil.rmtree("src")
shutil.rmtree("dst")
assert((end - start) < 10)
@@ -423,51 +369,24 @@ def test_bwlimit(mscp, src_prefix, dst_prefix):
run2ok([mscp, "-vvv", "-L", "100m", src_prefix + "src", dst_prefix + "dst"])
end = datetime.datetime.now().timestamp()
assert check_same_md5sum(src, dst)
src.cleanup()
dst.cleanup()
assert end - start > 7
@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
def test_bind_ng(mscp, src_prefix, dst_prefix):
"""Bind to invalid address should fail."""
out = run2ng([mscp, "-vvv", "-ddd", "-B", "192.168.10.10",
src_prefix + "src", dst_prefix + "dst"])
assert ("Cannot assign requested address" in out or
"Address not available" in out)
@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
def test_bind_ok(mscp, src_prefix, dst_prefix):
"""Bind to multiple lo addresses and check all of them used """
srcs = [ File(f"src-{x}", size = 128).make() for x in range(10) ]
src_paths = [ src_prefix + src.path for src in srcs ]
out = run2ok([mscp, "-v", "-ddd",
"-B", "127.0.0.10", "-B", "127.0.0.20", "-B", "127.0.0.30",
] +
src_paths + [dst_prefix + "dst/"])
for src in srcs:
dst = File(f"dst/{src.path}")
assert check_same_md5sum(src, dst)
assert "127.0.0.10" in out
assert "127.0.0.20" in out
assert "127.0.0.30" in out
@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
@pytest.mark.parametrize("src, dst", param_single_copy)
def test_set_port_ng(mscp, src_prefix, dst_prefix, src, dst):
src.make()
run2ng([mscp, "-vvv", "-P", 21, src_prefix + src.path, dst_prefix + dst.path])
src.cleanup()
@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
@pytest.mark.parametrize("src, dst", param_single_copy)
def test_set_port_ok(mscp, src_prefix, dst_prefix, src, dst):
src.make()
run2ok([mscp, "-vvv", "-P", 8022, src_prefix + src.path, dst_prefix + dst.path])
src.cleanup()
def test_v4only(mscp):
src = File("src", size = 1024).make()
@@ -475,6 +394,8 @@ def test_v4only(mscp):
dst_prefix = "localhost:{}/".format(os.getcwd())
run2ok([mscp, "-vvv", "-4", src.path, dst_prefix + dst.path])
assert check_same_md5sum(src, dst)
src.cleanup()
dst.cleanup()
def test_v6only(mscp):
src = File("src", size = 1024).make()
@@ -482,26 +403,34 @@ def test_v6only(mscp):
dst_prefix = "ip6-localhost:{}/".format(os.getcwd())
run2ok([mscp, "-vvv", "-6", src.path, dst_prefix + dst.path])
assert check_same_md5sum(src, dst)
src.cleanup()
dst.cleanup()
def test_v4_to_v6_should_fail(mscp):
src = File("src", size = 1024).make()
dst = File("dst")
dst_prefix = "[::1]:{}/".format(os.getcwd())
run2ng([mscp, "-vvv", "-4", src.path, dst_prefix + dst.path])
src.cleanup()
def test_v6_to_v4_should_fail(mscp):
src = File("src", size = 1024).make()
dst = File("dst")
dst_prefix = "127.0.0.1:{}/".format(os.getcwd())
run2ng([mscp, "-vvv", "-6", src.path, dst_prefix + dst.path])
src.cleanup()
def test_quiet_mode(mscp):
def test_quiet_mode(capsys, mscp):
src = File("src", size = 1024).make()
dst = File("dst")
dst_prefix = "127.0.0.1:{}/".format(os.getcwd())
out = run2ok([mscp, "-vvv", "-q", src.path, dst_prefix + dst.path])
run2ok([mscp, "-vvv", "-q", src.path, dst_prefix + dst.path], quiet=True)
assert check_same_md5sum(src, dst)
assert not out
src.cleanup()
dst.cleanup()
captured = capsys.readouterr()
assert not captured.out
assert not captured.err
@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
def test_set_conn_interval(mscp, src_prefix, dst_prefix):
@@ -514,6 +443,8 @@ def test_set_conn_interval(mscp, src_prefix, dst_prefix):
for src, dst in zip(srcs, dsts):
assert check_same_md5sum(src, dst)
src.cleanup()
dst.cleanup()
compressions = ["yes", "no", "none"]
@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
@@ -523,6 +454,8 @@ def test_compression(mscp, src_prefix, dst_prefix, compress):
dst = File("dst", size = 1024 * 1024 * 2).make()
run2ok([mscp, "-vvv", "-C", compress, src_prefix + src.path, dst_prefix + "dst"])
assert check_same_md5sum(src, dst)
src.cleanup()
dst.cleanup()
@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
def test_ccalgo(mscp, src_prefix, dst_prefix):
@@ -559,7 +492,8 @@ def test_config_ok(mscp, src_prefix, dst_prefix):
os.remove(config)
assert check_same_md5sum(src, dst)
src.cleanup()
dst.cleanup()
@pytest.mark.parametrize("src_prefix, dst_prefix", param_testhost_prefix)
def test_config_ng(mscp, src_prefix, dst_prefix):
@@ -573,6 +507,8 @@ def test_config_ng(mscp, src_prefix, dst_prefix):
src_prefix + src.path, dst_prefix + "dst"])
os.remove(config)
src.cleanup()
dst.cleanup()
param_valid_option_ok = [
@@ -589,6 +525,8 @@ def test_inline_option_ok(mscp, src_prefix, dst_prefix, option):
run2ok([mscp, "-vvv"] + option +
[src_prefix + src.path, dst_prefix + dst.path])
assert check_same_md5sum(src, dst)
src.cleanup()
dst.cleanup()
param_valid_option_ng = [
@@ -603,6 +541,7 @@ def test_inline_option_ng(mscp, src_prefix, dst_prefix, option):
dst = File("dst")
run2ng([mscp, "-vvv"] + option +
[src_prefix + src.path, dst_prefix + dst.path])
src.cleanup()
@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
@@ -615,6 +554,8 @@ def test_porxyjump_ok(mscp, src_prefix, dst_prefix):
"-J", "localhost:8022",
src_prefix + src.path, dst_prefix + dst.path])
assert check_same_md5sum(src, dst)
src.cleanup()
dst.cleanup()
@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
@@ -626,6 +567,7 @@ def test_porxyjump_ng(mscp, src_prefix, dst_prefix):
run2ng([mscp, "-n", 4, "-s", 1024 * 1024, "-vvv",
"-J", "invaliduser@localhost:8022",
src_prefix + src.path, dst_prefix + dst.path])
src.cleanup()
# username test assumes that this test runs inside a container, see Dockerfiles
def test_specify_passphrase_via_env(mscp):
@@ -636,6 +578,8 @@ def test_specify_passphrase_via_env(mscp):
run2ok([mscp, "-vvv", "-l", "test", "-i", "/home/test/.ssh/id_rsa_test",
src.path, "localhost:" + dst.path], env = env)
assert check_same_md5sum(src, dst)
src.cleanup()
dst.cleanup()
def test_specify_invalid_passphrase_via_env(mscp):
src = File(os.getcwd() + "/src", size = 1024).make()
@@ -644,6 +588,7 @@ def test_specify_invalid_passphrase_via_env(mscp):
env["MSCP_SSH_AUTH_PASSPHRASE"] = "invalid-keypassphrase"
run2ng([mscp, "-vvv", "-l", "test", "-i", "/home/test/.ssh/id_rsa_test",
src.path, "localhost:" + dst.path], env = env)
src.cleanup()
def test_specify_password_via_env(mscp):
src = File(os.getcwd() + "/src", size = 1024).make()
@@ -653,6 +598,8 @@ def test_specify_password_via_env(mscp):
run2ok([mscp, "-vvv", "-l", "test",
src.path, "localhost:" + dst.path], env = env)
assert check_same_md5sum(src, dst)
src.cleanup()
dst.cleanup()
def test_specify_invalid_password_via_env(mscp):
src = File(os.getcwd() + "/src", size = 1024).make()
@@ -661,44 +608,7 @@ def test_specify_invalid_password_via_env(mscp):
env["MSCP_SSH_AUTH_PASSWORD"] = "invalid-userpassword"
run2ng([mscp, "-vvv", "-l", "test",
src.path, "localhost:" + dst.path], env = env)
@pytest.fixture
def move_pubkey_temporally():
"""
mv ~/.ssh/id_* to id_rsa.bak before test, and move it back after test.
"""
sshdir = os.path.join(os.environ["HOME"], ".ssh")
# move pubkeys to /tmp
moved = []
for fname in os.listdir(sshdir):
if re.match(r"^id_[a-z0-9]+$", fname):
moved.append(fname)
shutil.move(f"{sshdir}/{fname}", f"/tmp/{fname}")
yield
# move back the keys
for fname in moved:
shutil.move(f"/tmp/{fname}", f"{sshdir}/{fname}")
@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
def test_passwordauth_without_pubkey(move_pubkey_temporally,
mscp, src_prefix, dst_prefix):
"""
make sure password auth works (by removing public keys)
"""
src = File(os.getcwd() + "/src", size = 1024).make()
dst = File("/home/test/dst")
env = os.environ
env["MSCP_SSH_AUTH_PASSWORD"] = "userpassword"
run2ok([mscp, "-vvv", "-l", "test",
src.path, "localhost:" + dst.path], env = env)
assert check_same_md5sum(src, dst)
src.cleanup()
@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
def test_10k_files(mscp, src_prefix, dst_prefix):
@@ -710,6 +620,8 @@ def test_10k_files(mscp, src_prefix, dst_prefix):
run2ok([mscp, "-v", src_prefix + "src", dst_prefix + "dst"])
for s, d in zip(srcs, dsts):
assert check_same_md5sum(s, d)
shutil.rmtree("src")
shutil.rmtree("dst")
@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
def test_checkpoint_dump_and_resume(mscp, src_prefix, dst_prefix):
@@ -724,6 +636,10 @@ def test_checkpoint_dump_and_resume(mscp, src_prefix, dst_prefix):
run2ok([mscp, "-vvv", "-R", "checkpoint"])
assert check_same_md5sum(src1, dst1)
assert check_same_md5sum(src2, dst2)
src1.cleanup()
src2.cleanup()
dst1.cleanup()
dst2.cleanup()
os.remove("checkpoint")
@pytest.mark.parametrize("timeout", [ 1, 2, 3, 4, 5 ])
@@ -742,6 +658,10 @@ def test_checkpoint_interrupt_large_file(mscp, timeout, src_prefix, dst_prefix):
run2ok([mscp, "-vv", "-R", "checkpoint"])
assert check_same_md5sum(src1, dst1)
assert check_same_md5sum(src2, dst2)
src1.cleanup()
src2.cleanup()
dst1.cleanup()
dst2.cleanup()
os.remove("checkpoint")
@pytest.mark.parametrize("timeout", [ 1, 2, 3, 4, 5 ])
@@ -767,6 +687,8 @@ def test_checkpoint_interrupt_many_files(mscp, timeout, src_prefix, dst_prefix):
for src, dst in files:
assert check_same_md5sum(src, dst)
src.cleanup()
dst.cleanup()
os.remove("checkpoint")