bump version to 0.2.2

because openssl@1.1 has been disabled since 2024/10/24.

.github/workflows/build-macos.yml

@@ -35,7 +35,7 @@ jobs:
     - name: Configure CMake
       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
-      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DOPENSSL_ROOT_DIR=${{steps.brew-prefix.outputs.HOMEBREW_PREFIX}}/opt/openssl@1.1
+      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DOPENSSL_ROOT_DIR=${{steps.brew-prefix.outputs.HOMEBREW_PREFIX}}/opt/openssl@3
 
     - name: Build
       # Build your program with the given configuration

.github/workflows/test.yml

@@ -12,6 +12,18 @@ env:
 jobs:
   test:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        index: # see DIST_IDS and DIST_VERS lists in CMakeLists.txt
+        - ubuntu-20.04
+        - ubuntu-22.04
+        - ubuntu-24.04
+        - rocky-8.9
+        - rocky-9.3
+        - almalinux-9.3
+        - alpine-3.19
+        - arch-base
     steps:
     - uses: actions/checkout@v4
       with:
@@ -20,7 +32,7 @@ jobs:
     - name: patch to libssh
       run: patch -d libssh -p1 < patch/libssh-0.10.6-2-g6f1b1e76.patch
 
-    # TODO: just building docker does not require libssh. fix CMakeLists
+    # TODO: just building docker images does not require libssh. fix CMakeLists
     - name: install build dependency
       run: |
         sudo apt-get update
@@ -30,7 +42,7 @@ jobs:
       run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
 
     - name: Build Containers
-      run: make -C ${{github.workspace}}/build docker-build-all
+      run: make -C ${{github.workspace}}/build docker-build-${{ matrix.index }}
 
     - name: Run Test
-      run: make -C ${{github.workspace}}/build docker-test-all
+      run: make -C ${{github.workspace}}/build docker-test-${{ matrix.index }}

CMakeLists.txt

@@ -165,8 +165,10 @@ enable_testing()
 # Custom targets to build and test mscp in docker containers.
 # foreach(IN ZIP_LISTS) (cmake >= 3.17) can shorten the following lists.
 # However, ubuntu 20.04 has cmake 3.16.3. So this is a roundabout trick.
-list(APPEND DIST_IDS	ubuntu	ubuntu	ubuntu	rocky	rocky	almalinux	alpine)
-list(APPEND DIST_VERS	20.04	22.04	24.04 	8.9	9.3	9.3		3.19)
+#
+# When edit DIST_IDS and DIST_VERS, also edit .github/workflows/test.yaml
+list(APPEND DIST_IDS  ubuntu ubuntu ubuntu rocky rocky almalinux alpine arch)
+list(APPEND DIST_VERS  20.04  22.04  24.04   8.9   9.3       9.3   3.19 base)
 
 list(LENGTH DIST_IDS _DIST_LISTLEN)
 math(EXPR DIST_LISTLEN "${_DIST_LISTLEN} - 1")

Dockerfile/arch-base.Dockerfile

@@ -0,0 +1,36 @@
+FROM archlinux:base
+
+ARG REQUIREDPKGS
+
+# install pyest and openssh for test
+RUN set -ex && pacman -Syy && pacman --noconfirm -S ${REQUIREDPKGS} openssh python-pytest
+
+RUN mkdir /var/run/sshd        \
+        && ssh-keygen -A        \
+        && ssh-keygen -f /root/.ssh/id_rsa -N ""                \
+        && cat /root/.ssh/id_rsa.pub > /root/.ssh/authorized_keys
+
+# disable PerSourcePenaltie, which would distrub test:
+# https://undeadly.org/cgi?action=article;sid=20240607042157
+RUN echo "PerSourcePenalties=no" > /etc/ssh/sshd_config.d/90-mscp-test.conf
+
+# create test user
+RUN useradd -m -d /home/test test       \
+        && echo "test:userpassword" | chpasswd \
+        && mkdir -p /home/test/.ssh     \
+        && ssh-keygen -f /home/test/.ssh/id_rsa_test -N "keypassphrase" \
+        && cat /home/test/.ssh/id_rsa_test.pub >> /home/test/.ssh/authorized_keys \
+        && chown -R test:test /home/test \
+        && chown -R test:test /home/test/.ssh
+
+ARG mscpdir="/mscp"
+
+COPY . ${mscpdir}
+
+# build
+RUN cd ${mscpdir}                       \
+        && rm -rf build                 \
+        && cmake -B build               \
+        && cd ${mscpdir}/build          \
+        && make -j 2                    \
+        && make install

README.md

@@ -42,7 +42,11 @@ Paper:
 - macOS
 
 ```console
+# Homebrew
 brew install upa/tap/mscp
+
+# MacPorts
+sudo port install mscp
 ```
 
 - Ubuntu
@@ -92,7 +96,7 @@ mkdir build && cd build
 cmake ..
 
 # in macOS, you may need OPENSSL_ROOT_DIR for cmake:
-# cmake .. -DOPENSSL_ROOT_DIR=$(brew --prefix)/opt/openssl@1.1
+# cmake .. -DOPENSSL_ROOT_DIR=$(brew --prefix)/opt/openssl@3
 
 # build
 make

VERSION

@@ -1 +1 @@
-0.2.0
+0.2.2

debian/changelog

@@ -1,4 +1,25 @@
-mscp (0.2.0) UNRELEASED; urgency=medium
+mscp (0.2.2) UNRELEASED; urgency=medium
+
+  * bump cmake_minimum_version on libssh (#32)
+  * fix quiet mode (#30)
+  * use openssl@3 on macOS (#29)
+  * add archlinux support (#28)
+
+ -- Ryo Nakamura <upa@haeena.net>  Wed, 16 Apr 2025 17:01:17 +0900
+
+mscp (0.2.1) unstable; urgency=medium
+
+  * fix broken checkpoint files when copying multiple files (#16)
+  * fix broken password inputs for resume transfers (#17 and #18)
+  * add support [kKmMgG] units for -s, -S, and -b options (#20)
+  * change the default min chunk size to 16MB (#20)
+  * change the default max chunk size to filesize / (nr_conn * 4) (#20)
+  * -s and -S accept integers other than multiples of page sizes (#20)
+  * help now shows available ciphers and HMACs (#20)
+
+ -- Ryo Nakamura <upa@haeena.net>  Sat, 11 May 2024 14:49:52 +0900
+
+mscp (0.2.0) unstable; urgency=medium
 
   * add -J DESTINATION option for ProxyJump (#15)
   * add -o SSH_OPTION option

doc/mscp.1.in

@@ -130,7 +130,8 @@ pinned to any cores.
 
 .TP
 .B \-u \fIMAX_STARTUPS\fR
-Specifies the number of concurrent outgoing SSH connections.
+Specifies the number of concurrent unauthenticated SSH connection
+attempts.
 .B sshd
 limits the number of simultaneous SSH connection attempts by
 .I MaxStartups
@@ -192,12 +193,16 @@ and remove the checkpoint if it returns 0.
 Specifies the minimum chunk size.
 .B mscp
 divides a single file into chunks and copies the chunks in
-parallel. The default value is 67108864 (64MB).
+parallel. The default value is 16M bytes.
 
 .TP
 .B \-S \fIMAX_CHUNK_SIZE\fR
 Specifies the maximum chunk size. The default is file size divided by
-the number of connections.
+the number of connections and devided by 4. If the calculated value
+is smarller than the
+.B MIN_CHUNK_SIZE
+value,
+MIN_CHUNK_SIZE is used.
 
 .TP
 .B \-a \fINR_AHEAD\fR
@@ -289,6 +294,10 @@ connection to the jump host described by
 .TP
 .B \-c \fICIPHER\fR
 Selects the cipher to use for encrypting the data transfer. See
+.B mscp -h
+or
+.B Ciphers
+in
 .UR https://\:www\:.libssh\:.org/\:features/
 libssh features
 .UE .
@@ -296,6 +305,10 @@ libssh features
 .TP
 .B \-M \fIHMAC\fR
 Specifies MAC hash algorithms. See
+.B mscp -h
+or
+.B MAC hashes
+in
 .UR https://\:www\:.libssh\:.org/\:features/
 libssh features
 .UE .
@@ -311,6 +324,9 @@ libssh features
 .TP
 .B \-g \fICONGESTION\fR
 Specifies the TCP congestion control algorithm to use (Linux only).
+See
+.B sysctl net.ipv4.tcp_allowed_congestion_control
+for available values.
 
 .TP
 .B \-p

doc/mscp.rst

@@ -2,7 +2,7 @@
 MSCP
 ====
 
-:Date: v0.1.5-18-ge47d5b7
+:Date: v0.2.1
 
 NAME
 ====
@@ -60,10 +60,10 @@ OPTIONS
    pinned to any cores.
 
 **-u MAX_STARTUPS**
-   Specifies the number of concurrent outgoing SSH connections. **sshd**
-   limits the number of simultaneous SSH connection attempts by
-   *MaxStartups* in *sshd_config.* The default *MaxStartups* is 10;
-   thus, we set the default MAX_STARTUPS 8.
+   Specifies the number of concurrent unauthenticated SSH connection
+   attempts. **sshd** limits the number of simultaneous SSH connection
+   attempts by *MaxStartups* in *sshd_config.* The default *MaxStartups*
+   is 10; thus, we set the default MAX_STARTUPS 8.
 
 **-I INTERVAL**
    Specifies the interval (in seconds) between SSH connection attempts.
@@ -97,12 +97,14 @@ OPTIONS
 
 **-s MIN_CHUNK_SIZE**
    Specifies the minimum chunk size. **mscp** divides a single file into
-   chunks and copies the chunks in parallel. The default value is
-   67108864 (64MB).
+   chunks and copies the chunks in parallel. The default value is 16M
+   bytes.
 
 **-S MAX_CHUNK_SIZE**
    Specifies the maximum chunk size. The default is file size divided by
-   the number of connections.
+   the number of connections and devided by 4. If the calculated value
+   is smarller than the **MIN_CHUNK_SIZE** value, MIN_CHUNK_SIZE is
+   used.
 
 **-a NR_AHEAD**
    Specifies the number of inflight SFTP commands. The default value is
@@ -166,11 +168,12 @@ OPTIONS
 
 **-c CIPHER**
    Selects the cipher to use for encrypting the data transfer. See
-   `libssh features <https://www.libssh.org/features/>`__.
+   **mscp -h** or **Ciphers** in `libssh
+   features <https://www.libssh.org/features/>`__.
 
 **-M HMAC**
-   Specifies MAC hash algorithms. See `libssh
-   features <https://www.libssh.org/features/>`__.
+   Specifies MAC hash algorithms. See **mscp -h** or **MAC hashes** in
+   `libssh features <https://www.libssh.org/features/>`__.
 
 **-C COMPRESS**
    Enables compression: yes, no, zlib, zlib@openssh.com. The default is
@@ -178,6 +181,8 @@ OPTIONS
 
 **-g CONGESTION**
    Specifies the TCP congestion control algorithm to use (Linux only).
+   See **sysctl net.ipv4.tcp_allowed_congestion_control** for available
+   values.
 
 **-p**
    Preserves modification times and access times (file mode bits are

include/mscp.h

@@ -294,4 +294,15 @@ enum {
 };
 
 
+/**
+ * @brief Return available ciphers.
+ */
+const char **mscp_ssh_ciphers(void);
+
+/**
+ * @brief Return available hmacs.
+ */
+ const char **mscp_ssh_hmacs(void);
+
+
 #endif /* _MSCP_H_ */

patch/libssh-0.10.6-2-g6f1b1e76.patch

@@ -1,3 +1,13 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index a64b7708..c6344a5a 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -1,4 +1,4 @@
+-cmake_minimum_required(VERSION 3.3.0)
++cmake_minimum_required(VERSION 3.13.0)
+ cmake_policy(SET CMP0048 NEW)
+ 
+ # Specify search path for CMake modules to be loaded by include()
 diff --git a/ConfigureChecks.cmake b/ConfigureChecks.cmake
 index 9de10225..0f3d20ed 100644
 --- a/ConfigureChecks.cmake
@@ -37,7 +47,7 @@ index 1fce7b76..b64d1455 100644
  int ssh_buffer_validate_length(struct ssh_buffer_struct *buffer, size_t len);
  
 diff --git a/include/libssh/libssh.h b/include/libssh/libssh.h
-index 669a0a96..da5b4099 100644
+index 669a0a96..26b20f3f 100644
 --- a/include/libssh/libssh.h
 +++ b/include/libssh/libssh.h
 @@ -368,6 +368,7 @@ enum ssh_options_e {
@@ -64,11 +74,14 @@ index 669a0a96..da5b4099 100644
  LIBSSH_API void ssh_buffer_free(ssh_buffer buffer);
  #define SSH_BUFFER_FREE(x) \
      do { if ((x) != NULL) { ssh_buffer_free(x); x = NULL; } } while(0)
-@@ -843,6 +846,8 @@ LIBSSH_API void *ssh_buffer_get(ssh_buffer buffer);
+@@ -843,6 +846,11 @@ LIBSSH_API void *ssh_buffer_get(ssh_buffer buffer);
  LIBSSH_API uint32_t ssh_buffer_get_len(ssh_buffer buffer);
  LIBSSH_API int ssh_session_set_disconnect_message(ssh_session session, const char *message);
  
 +typedef ssize_t (*ssh_add_func) (void *ptr, size_t max_bytes, void *userdata);
++
++LIBSSH_API const char **ssh_ciphers(void);
++LIBSSH_API const char **ssh_hmacs(void);
 +
  #ifndef LIBSSH_LEGACY_0_4
  #include "libssh/legacy.h"
@@ -299,6 +312,60 @@ index 15cae644..02ef43b4 100644
          errno = 0;
          rc = connect(s, itr->ai_addr, itr->ai_addrlen);
          if (rc == -1 && (errno != 0) && (errno != EINPROGRESS)) {
+diff --git a/src/misc.c b/src/misc.c
+index 7081f12a..e3879fe4 100644
+--- a/src/misc.c
++++ b/src/misc.c
+@@ -71,6 +71,8 @@
+ #include "libssh/priv.h"
+ #include "libssh/misc.h"
+ #include "libssh/session.h"
++#include "libssh/wrapper.h"
++#include "libssh/crypto.h"
+ 
+ #ifdef HAVE_LIBGCRYPT
+ #define GCRYPT_STRING "/gnutls"
+@@ -2074,4 +2076,40 @@ int ssh_check_hostname_syntax(const char *hostname)
+     return SSH_OK;
+ }
+ 
++/**
++ * @brief Return supported cipher names
++ * @return	The list of cipher names.
++ */
++const char **ssh_ciphers(void)
++{
++     struct ssh_cipher_struct *tab=ssh_get_ciphertab();
++     static const char *ciphers[32];
++     int n;
++
++     memset(ciphers, 0, sizeof(*ciphers));
++
++     for (n = 0; tab[n].name != NULL; n++) {
++	  ciphers[n] = tab[n].name;
++     }
++     return ciphers;
++}
++
++/**
++ * @brief Return supported hmac names
++ * @return	The list of hmac names.
++ */
++const char **ssh_hmacs(void)
++{
++     struct ssh_hmac_struct *tab=ssh_get_hmactab();
++     static const char *hmacs[32];
++     int n;
++
++     memset(hmacs, 0, sizeof(*hmacs));
++
++     for (n = 0; tab[n].name != NULL; n++) {
++	  hmacs[n] = tab[n].name;
++     }
++     return hmacs;
++}
++
+ /** @} */
 diff --git a/src/options.c b/src/options.c
 index b3ecffe1..8de24ed6 100644
 --- a/src/options.c
@@ -392,10 +459,10 @@ index 8c509699..307388e5 100644
      session->opts.flags = SSH_OPT_FLAG_PASSWORD_AUTH |
                            SSH_OPT_FLAG_PUBKEY_AUTH |
 diff --git a/src/sftp.c b/src/sftp.c
-index e01012a8..3b86c3c6 100644
+index e01012a8..702623a0 100644
 --- a/src/sftp.c
 +++ b/src/sftp.c
-@@ -2228,6 +2228,135 @@ ssize_t sftp_write(sftp_file file, const void *buf, size_t count) {
+@@ -2228,6 +2228,132 @@ ssize_t sftp_write(sftp_file file, const void *buf, size_t count) {
    return -1; /* not reached */
  }
  
@@ -434,8 +501,7 @@ index e01012a8..3b86c3c6 100644
 +
 +  buffer = ssh_buffer_new_size(buf_sz, HEADROOM);
 +  if (buffer == NULL) {
-+    ssh_set_error(sftp->session, SSH_FATAL,
-+		  "ssh_buffer_new_size failed: Out of Memory");
++    ssh_set_error_oom(sftp->session);
 +    return -1;
 +  }
 +
@@ -449,16 +515,14 @@ index e01012a8..3b86c3c6 100644
 +                       count); /* len of datastring */
 +
 +  if (rc != SSH_OK){
-+    ssh_set_error(sftp->session, SSH_FATAL,
-+		  "ssh_buffer_pack failed: Out of Memory");
++    ssh_set_error_oom(sftp->session);
 +    ssh_buffer_free(buffer);
 +    return SSH_ERROR;
 +  }
 +
 +  actual = ssh_buffer_add_func(buffer, f, count, userdata);
 +  if (actual < 0){
-+    ssh_set_error(sftp->session, SSH_FATAL,
-+		  "ssh_buffer_add_func failed: %s", strerror(errno));
++    ssh_set_error_oom(sftp->session);
 +    ssh_buffer_free(buffer);
 +    return SSH_ERROR;
 +  }

rpm/mscp.spec.in

@@ -38,6 +38,12 @@ make -C build install DESTDIR=%{buildroot}
 
 
 %changelog
+* Wed Apr 16 2025 Ryo Nakamura <upa@haeena.net> - 0.2.2-1
+- RPM release for v0.2.2
+
+* Sat May 11 2024 Ryo Nakamura <upa@haeena.net> - 0.2.1-1
+- RPM release for v0.2.1
+
 * Mon Apr 15 2024 Ryo Nakamura <upa@haeena.net> - 0.2.0-1
 - RPM release for v0.2.0
 

scripts/install-build-deps.sh

@@ -45,7 +45,7 @@ done
 case $platform in
 	Darwin)
 		cmd="brew install"
-		pkgs="openssl@1.1"
+		pkgs="openssl@3"
 		;;
 	Linux-ubuntu*)
 		cmd="apt-get install --no-install-recommends -y"
@@ -55,6 +55,10 @@ case $platform in
 		cmd="yum install -y"
 		pkgs="gcc make cmake zlib-devel openssl-devel rpm-build"
 		;;
+	Linux-arch*)
+		cmd="pacman --no-confirm -S"
+		pkgs="gcc make cmake"
+		;;
 	FreeBSD-freebsd)
 		cmd="pkg install"
 		pkgs="cmake"

scripts/test-in-container.sh

@@ -17,7 +17,7 @@ sed -i -e 's/AllowTcpForwarding no/AllowTcpForwarding yes/' /etc/ssh/sshd_config
 
 # Run sshd
 if [ ! -e /var/run/sshd.pid ]; then
-	/usr/sbin/sshd
+	/usr/sbin/sshd -E /tmp/sshd.log
 	sleep 1
 fi
 

src/checkpoint.c

@@ -276,7 +276,7 @@ int checkpoint_save(const char *pathname, int dir, const char *user, const char
 	pool_for_each(path_pool, p, i) {
 		if (p->state == FILE_STATE_DONE)
 			continue;
-		if (checkpoint_write_path(fd, p, i) < 0)
+		if (checkpoint_write_path(fd, p, nr_paths) < 0)
 			return -1;
 		nr_paths++;
 	}
@@ -349,7 +349,8 @@ static int checkpoint_load_path(struct checkpoint_obj_hdr *hdr, pool *path_pool)
 		return -1;
 	}
 
-	pr_info("checkpoint:file: %s -> %s", p->path, p->dst_path);
+	pr_info("checkpoint:file: idx=%u %s -> %s", ntohl(path->idx),
+		p->path, p->dst_path);
 
 	return 0;
 }
@@ -375,7 +376,8 @@ static int checkpoint_load_chunk(struct checkpoint_obj_hdr *hdr, pool *path_pool
 		return -1;
 	}
 
-	pr_debug("checkpoint:chunk: %s 0x%lx-0x%lx", p->path, c->off, c->off + c->len);
+	pr_debug("checkpoint:chunk: idx=%u %s 0x%lx-0x%lx", ntohl(chunk->idx),
+		 p->path, c->off, c->off + c->len);
 
 	return 0;
 }

src/main.c

@@ -2,6 +2,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <stdarg.h>
+#include <fcntl.h>
 #include <unistd.h>
 #include <limits.h>
 #include <math.h>
@@ -40,14 +41,14 @@ void usage(bool print_help)
 	printf("    -n NR_CONNECTIONS  number of connections "
 	       "(default: floor(log(cores)*2)+1)\n"
 	       "    -m COREMASK        hex value to specify cores where threads pinned\n"
-	       "    -u MAX_STARTUPS    number of concurrent SSH connection attempts "
+	       "    -u MAX_STARTUPS    number of concurrent unauthed SSH attempts "
 	       "(default: 8)\n"
 	       "    -I INTERVAL        interval between SSH connection attempts (default: 0)\n"
 	       "    -W CHECKPOINT      write states to the checkpoint if transfer fails\n"
 	       "    -R CHECKPOINT      resume transferring from the checkpoint\n"
 	       "\n"
-	       "    -s MIN_CHUNK_SIZE  min chunk size (default: 64MB)\n"
-	       "    -S MAX_CHUNK_SIZE  max chunk size (default: filesize/nr_conn)\n"
+	       "    -s MIN_CHUNK_SIZE  min chunk size (default: 16M bytes)\n"
+	       "    -S MAX_CHUNK_SIZE  max chunk size (default: filesize/nr_conn/4)\n"
 	       "    -a NR_AHEAD        number of inflight SFTP commands (default: 32)\n"
 	       "    -b BUF_SZ          buffer size for i/o and transfer\n"
 	       "    -L LIMIT_BITRATE   Limit the bitrate, n[KMG] (default: 0, no limit)\n"
@@ -75,6 +76,26 @@ void usage(bool print_help)
 	       "    -N                 enable Nagle's algorithm (default disabled)\n"
 	       "    -h                 print this help\n"
 	       "\n");
+
+	const char **ciphers = mscp_ssh_ciphers();
+	const char **hmacs = mscp_ssh_hmacs();
+	int n;
+
+	printf("Available ciphers: ");
+	for (n = 0; ciphers[n] != NULL; n++) {
+		printf("%s", ciphers[n]);
+		if (ciphers[n + 1])
+			printf(", ");
+	}
+	printf("\n\n");
+
+	printf("Available hmacs: ");
+	for (n = 0; hmacs[n] != NULL; n++) {
+		printf("%s", hmacs[n]);
+		if (hmacs[n + 1])
+			printf(", ");
+	}
+	printf("\n\n");
 }
 
 char *strip_brackets(char *s)
@@ -260,6 +281,55 @@ void print_cli(const char *fmt, ...)
 
 void print_stat(bool final);
 
+long atol_with_unit(char *value, bool i)
+{
+	/* value must be "\d+[kKmMgG]?" */
+
+	char *u = value + (strlen(optarg) - 1);
+	long k = i ? 1024 : 1000;
+	long factor = 1;
+	long v;
+
+	switch (*u) {
+	case 'k':
+	case 'K':
+		factor = k;
+		*u = '\0';
+		break;
+	case 'm':
+	case 'M':
+		factor = k * k;
+		*u = '\0';
+		break;
+	case 'g':
+	case 'G':
+		factor = k * k * k;
+		*u = '\0';
+		break;
+	}
+
+	v = atol(value);
+	return v * factor;
+}
+
+int to_dev_null(int fd)
+{
+	int nfd = open("/dev/null", O_WRONLY);
+	if (nfd < 0) {
+		pr_err("open /dev/null: %s", strerrno());
+		return -1;
+	}
+
+	if (dup2(nfd, fd) < 0) {
+		pr_err("dup2: %s", strerrno());
+		return -1;
+	}
+
+	close(nfd);
+
+	return 0;
+}
+
 int main(int argc, char **argv)
 {
 	struct mscp_ssh_opts s;
@@ -269,10 +339,8 @@ int main(int argc, char **argv)
 	int ch, n, i, ret;
 	int direction = 0;
 	char *remote = NULL, *checkpoint_save = NULL, *checkpoint_load = NULL;
-	bool dryrun = false, resume = false;
+	bool quiet = false, dryrun = false, resume = false;
 	int nr_options = 0;
-	size_t factor = 1;
-	char *unit;
 
 	memset(&s, 0, sizeof(s));
 	memset(&o, 0, sizeof(o));
@@ -305,36 +373,19 @@ int main(int argc, char **argv)
 			resume = true;
 			break;
 		case 's':
-			o.min_chunk_sz = atoi(optarg);
+			o.min_chunk_sz = atol_with_unit(optarg, true);
 			break;
 		case 'S':
-			o.max_chunk_sz = atoi(optarg);
+			o.max_chunk_sz = atol_with_unit(optarg, true);
 			break;
 		case 'a':
 			o.nr_ahead = atoi(optarg);
 			break;
 		case 'b':
-			o.buf_sz = atoi(optarg);
+			o.buf_sz = atol_with_unit(optarg, true);
 			break;
 		case 'L':
-			factor = 1;
-			unit = optarg + (strlen(optarg) - 1);
-			if (*unit == 'k' || *unit == 'K') {
-				factor = 1000;
-				*unit = '\0';
-			} else if (*unit == 'm' || *unit == 'M') {
-				factor = 1000000;
-				*unit = '\0';
-			} else if (*unit == 'g' || *unit == 'G') {
-				factor = 1000000000;
-				*unit = '\0';
-			}
-			o.bitrate = atol(optarg);
-			if (o.bitrate == 0) {
-				pr_err("invalid bitrate: %s", optarg);
-				return 1;
-			}
-			o.bitrate *= factor;
+			o.bitrate = atol_with_unit(optarg, false);
 			break;
 		case '4':
 			s.ai_family = AF_INET;
@@ -346,7 +397,7 @@ int main(int argc, char **argv)
 			o.severity++;
 			break;
 		case 'q':
-			o.severity = MSCP_SEVERITY_NONE;
+			quiet = true;
 			break;
 		case 'D':
 			dryrun = true;
@@ -409,6 +460,9 @@ int main(int argc, char **argv)
 		}
 	}
 
+	if (quiet)
+		to_dev_null(STDOUT_FILENO);
+
 	s.password = getenv(ENV_SSH_AUTH_PASSWORD);
 	s.passphrase = getenv(ENV_SSH_AUTH_PASSPHRASE);
 
@@ -487,6 +541,18 @@ int main(int argc, char **argv)
 			pr_err("mscp_checkpoint_load: %s", priv_get_err());
 			return -1;
 		}
+
+		if (dryrun)
+			goto out;
+
+		/* create the first ssh connection to get password or
+		 * passphrase. The sftp session over it will be not
+		 * used for resume transfer in actuality. ToDo:
+		 * connectin managemnet should be improved. */
+		if (mscp_connect(m) < 0) {
+			pr_err("mscp_connect: %s", priv_get_err());
+			return -1;
+		}
 	}
 
 	if (dryrun) {

src/mscp.c

@@ -62,7 +62,7 @@ struct mscp {
 	struct mscp_thread scan; /* mscp_thread for mscp_scan_thread() */
 };
 
-#define DEFAULT_MIN_CHUNK_SZ (64 << 20) /* 64MB */
+#define DEFAULT_MIN_CHUNK_SZ (16 << 20) /* 16MB */
 #define DEFAULT_NR_AHEAD 32
 #define DEFAULT_BUF_SZ 16384
 /* XXX: we use 16384 byte buffer pointed by
@@ -158,23 +158,8 @@ static int validate_and_set_defaut_params(struct mscp_opts *o)
 
 	if (o->min_chunk_sz == 0)
 		o->min_chunk_sz = DEFAULT_MIN_CHUNK_SZ;
-	else {
-		if (o->min_chunk_sz < getpagesize() ||
-		    o->min_chunk_sz % getpagesize() != 0) {
-			priv_set_errv("min chunk size must be "
-				      "larget than and multiple of page size %d: %lu",
-				      getpagesize(), o->min_chunk_sz);
-			return -1;
-		}
-	}
 
 	if (o->max_chunk_sz) {
-		if (o->max_chunk_sz < getpagesize() ||
-		    o->max_chunk_sz % getpagesize() != 0) {
-			priv_set_errv("min chunk size must be larget than and "
-				      "multiple of page size %d: %lu",
-				      getpagesize(), o->max_chunk_sz);
-		}
 		if (o->min_chunk_sz > o->max_chunk_sz) {
 			priv_set_errv("smaller max chunk size than "
 				      "min chunk size: %lu < %lu",
@@ -345,18 +330,10 @@ int mscp_set_dst_path(struct mscp *m, const char *dst_path)
 	return 0;
 }
 
-static int get_page_mask(void)
+static size_t get_page_mask(void)
 {
-	long page_sz = sysconf(_SC_PAGESIZE);
-	size_t page_mask = 0;
-	int n;
-
-	for (n = 0; page_sz > 0; page_sz >>= 1, n++) {
-		page_mask <<= 1;
-		page_mask |= 1;
-	}
-
-	return page_mask >> 1;
+	size_t page_sz = sysconf(_SC_PAGESIZE);
+	return ~(page_sz - 1);
 }
 
 static void mscp_stop_copy_thread(struct mscp *m)

src/path.c

@@ -102,13 +102,10 @@ static int resolve_chunk(struct path *p, size_t size, struct path_resolve_args *
 	size_t chunk_sz, off, len;
 	size_t remaind;
 
-	if (size <= a->min_chunk_sz)
-		chunk_sz = size;
-	else if (a->max_chunk_sz)
+	if (a->max_chunk_sz)
 		chunk_sz = a->max_chunk_sz;
 	else {
-		chunk_sz = (size - (size % a->nr_conn)) / a->nr_conn;
-		chunk_sz &= ~a->chunk_align; /* align with page_sz */
+		chunk_sz = (size / (a->nr_conn * 4)) & a->chunk_align;
 		if (chunk_sz <= a->min_chunk_sz)
 			chunk_sz = a->min_chunk_sz;
 	}

src/ssh.c

@@ -407,3 +407,13 @@ void ssh_sftp_close(sftp_session sftp)
 	ssh_disconnect(ssh);
 	ssh_free(ssh);
 }
+
+const char **mscp_ssh_ciphers(void)
+{
+	return ssh_ciphers();
+}
+
+const char **mscp_ssh_hmacs(void)
+{
+	return ssh_hmacs();
+}

test/test_e2e.py

@@ -15,17 +15,19 @@ from subprocess import check_call, CalledProcessError
 from util import File, check_same_md5sum
 
 
-def run2ok(args, env = None):
+def run2ok(args, env = None, quiet = False):
     cmd = list(map(str, args))
-    print("cmd: {}".format(" ".join(cmd)))
+    if not quiet:
+        print("cmd: {}".format(" ".join(cmd)))
     check_call(cmd, env = env)
 
-def run2ng(args, env = None, timeout = None):
+def run2ng(args, env = None, timeout = None, quiet = False):
     if timeout:
         args = ["timeout", "-s", "INT", timeout] + args
     cmd = list(map(str, args))
-    print("cmd: {}".format(" ".join(cmd)))
-    with pytest.raises(CalledProcessError) as e:
+    if not quiet:
+        print("cmd: {}".format(" ".join(cmd)))
+    with pytest.raises(CalledProcessError):
         check_call(cmd, env = env)
 
 
@@ -418,6 +420,18 @@ def test_v6_to_v4_should_fail(mscp):
     run2ng([mscp, "-vvv", "-6", src.path, dst_prefix + dst.path])
     src.cleanup()
 
+def test_quiet_mode(capsys, mscp):
+    src = File("src", size = 1024).make()
+    dst = File("dst")
+    dst_prefix = "127.0.0.1:{}/".format(os.getcwd())
+    run2ok([mscp, "-vvv", "-q", src.path, dst_prefix + dst.path], quiet=True)
+    assert check_same_md5sum(src, dst)
+    src.cleanup()
+    dst.cleanup()
+    captured = capsys.readouterr()
+    assert not captured.out
+    assert not captured.err
+
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 def test_set_conn_interval(mscp, src_prefix, dst_prefix):
     srcs = []
@@ -628,9 +642,9 @@ def test_checkpoint_dump_and_resume(mscp, src_prefix, dst_prefix):
     dst2.cleanup()
     os.remove("checkpoint")
 
-@pytest.mark.parametrize("timeout", [1, 2, 3, 4, 5, 6])
+@pytest.mark.parametrize("timeout", [ 1, 2, 3, 4, 5 ])
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
-def test_checkpoint_interrupt_and_resume(mscp, timeout, src_prefix, dst_prefix):
+def test_checkpoint_interrupt_large_file(mscp, timeout, src_prefix, dst_prefix):
     """Copy two 100MB files with 200Mbps -> 4 sec + 4 sec """
     src1 = File("src1", size = 100 * 1024 * 1024).make()
     src2 = File("src2", size = 100 * 1024 * 1024).make()
@@ -650,3 +664,31 @@ def test_checkpoint_interrupt_and_resume(mscp, timeout, src_prefix, dst_prefix):
     dst2.cleanup()
     os.remove("checkpoint")
 
+@pytest.mark.parametrize("timeout", [ 1, 2, 3, 4, 5 ])
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_checkpoint_interrupt_many_files(mscp, timeout, src_prefix, dst_prefix):
+    """Copy 100 1-MB files with 4 connections, and interrupt and
+    resume the transfer
+    """
+
+    files = []
+    for x in range(100):
+        files.append((
+            File("src/{:03d}".format(x), size = 1024 * 1024).make(),
+            File("dst/{:03d}".format(x))
+        ))
+
+    run2ng([mscp, "-vv", "-W", "checkpoint", "-L", "80m", "-n", 4,
+            src_prefix + "src",  dst_prefix + "dst"],
+           timeout = timeout)
+    assert os.path.exists("checkpoint")
+
+    run2ok([mscp, "-vv", "-R", "checkpoint"])
+
+    for src, dst in files:
+        assert check_same_md5sum(src, dst)
+        src.cleanup()
+        dst.cleanup()
+
+    os.remove("checkpoint")
+