bump version to 0.1.3

See https://cloudlinux.zendesk.com/hc/en-us/articles/12225072530204-yum-update-error-Error-GPG-check-FAILED

.github/workflows/build-macos.yml

@@ -30,7 +30,7 @@ jobs:
       run: echo "HOMEBREW_PREFIX=$(brew --prefix)" >> $GITHUB_OUTPUT
 
     - name: patch to libssh
-      run: patch -d libssh -p1 < patch/libssh-0.10.4.patch
+      run: patch -d libssh -p1 < patch/libssh-0.10.6-2-g6f1b1e76.patch
 
     - name: Configure CMake
       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.

.github/workflows/build-ubuntu.yml

@@ -23,10 +23,12 @@ jobs:
         submodules: true
 
     - name: install build dependency
-      run: sudo ./scripts/install-build-deps.sh
+      run: |
+        sudo apt-get update
+        sudo ./scripts/install-build-deps.sh
 
     - name: patch to libssh
-      run: patch -d libssh -p1 < patch/libssh-0.10.4.patch
+      run: patch -d libssh -p1 < patch/libssh-0.10.6-2-g6f1b1e76.patch
 
     - name: Configure CMake
       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.

.github/workflows/codeql.yml

@@ -43,10 +43,12 @@ jobs:
         submodules: true
 
     - name: install build dependency
-      run: sudo ./scripts/install-build-deps.sh
+      run: |
+        sudo apt-get update
+        sudo ./scripts/install-build-deps.sh
 
     - name: patch to libssh
-      run: patch -d libssh -p1 < patch/libssh-0.10.4.patch
+      run: patch -d libssh -p1 < patch/libssh-0.10.6-2-g6f1b1e76.patch
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/release.yml

@@ -18,7 +18,7 @@ jobs:
           submodules: true
 
       - name: patch to libssh
-        run: patch -d libssh -p1 < patch/libssh-0.10.4.patch
+        run: patch -d libssh -p1 < patch/libssh-0.10.6-2-g6f1b1e76.patch
 
       # TODO: just building docker does not require packages. fix CMakeLists
       - name: install build dependency
@@ -58,7 +58,7 @@ jobs:
           submodules: true
 
       - name: patch to libssh
-        run: patch -d libssh -p1 < patch/libssh-0.10.4.patch
+        run: patch -d libssh -p1 < patch/libssh-0.10.6-2-g6f1b1e76.patch
 
       - name: Set variables
         run: |

.github/workflows/test.yml

@@ -18,11 +18,13 @@ jobs:
         submodules: true
 
     - name: patch to libssh
-      run: patch -d libssh -p1 < patch/libssh-0.10.4.patch
+      run: patch -d libssh -p1 < patch/libssh-0.10.6-2-g6f1b1e76.patch
 
     # TODO: just building docker does not require libssh. fix CMakeLists
     - name: install build dependency
-      run: sudo ./scripts/install-build-deps.sh
+      run: |
+        sudo apt-get update
+        sudo ./scripts/install-build-deps.sh
 
     - name: configure CMake
       run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}

CMakeLists.txt

@@ -52,6 +52,8 @@ if (BUILD_STATIC)
 endif()
 
 
+option(INSTALL_EXECUTABLE_ONLY OFF) # do not install libmscp
+
 
 # add libssh static library
 set(CMAKE_POLICY_DEFAULT_CMP0077 NEW)
@@ -104,7 +106,9 @@ set_target_properties(mscp-shared
 	OUTPUT_NAME	mscp
 	PUBLIC_HEADER	${mscp_SOURCE_DIR}/include/mscp.h)
 
-install(TARGETS mscp-shared)
+if(!INSTALL_EXECUTABLE_ONLY)
+	install(TARGETS mscp-shared)
+endif()
 
 
 # libmscp.a
@@ -117,7 +121,9 @@ set_target_properties(mscp-static
 	PROPERTIES
 	OUTPUT_NAME	mscp)
 
-install(TARGETS mscp-static)
+if(!INSTALL_EXECUTABLE_ONLY)
+	install(TARGETS mscp-static)
+endif()
 
 
 # mscp executable
@@ -136,6 +142,21 @@ target_compile_options(mscp PRIVATE ${MSCP_COMPILE_OPTS})
 install(TARGETS mscp RUNTIME DESTINATION bin)
 
 
+# mscp manpage and document
+configure_file(
+	${mscp_SOURCE_DIR}/doc/mscp.1.in
+	${PROJECT_BINARY_DIR}/mscp.1)
+
+add_custom_target(update-mscp-rst
+	COMMENT "Update doc/mscp.rst from mscp.1.in"
+	WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
+	COMMAND
+	pandoc -s -f man mscp.1 -t rst -o ${PROJECT_SOURCE_DIR}/doc/mscp.rst)
+
+install(FILES ${PROJECT_BINARY_DIR}/mscp.1
+	DESTINATION ${CMAKE_INSTALL_MANDIR}/man1)
+
+
 # Test
 add_test(NAME	pytest
 	COMMAND	python3 -m pytest -v
@@ -221,7 +242,8 @@ foreach(x RANGE ${DIST_LISTLEN})
 		COMMENT "Test mscp in ${DOCKER_IMAGE} container"
 		WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
 		COMMAND
-		docker run --init --rm ${DOCKER_IMAGE} /mscp/scripts/test-in-container.sh)
+		docker run --init --rm --sysctl net.ipv6.conf.all.disable_ipv6=0
+		${DOCKER_IMAGE} /mscp/scripts/test-in-container.sh)
 
 	add_custom_target(docker-pkg-${DOCKER_INDEX}
 		COMMENT "Retrieve mscp package from ${DOCKER_IMAGE} container"

README.md

@@ -3,17 +3,20 @@
 [![build on ubuntu](https://github.com/upa/mscp/actions/workflows/build-ubuntu.yml/badge.svg)](https://github.com/upa/mscp/actions/workflows/build-ubuntu.yml) [![build on macOS](https://github.com/upa/mscp/actions/workflows/build-macos.yml/badge.svg)](https://github.com/upa/mscp/actions/workflows/build-macos.yml) [![test](https://github.com/upa/mscp/actions/workflows/test.yml/badge.svg)](https://github.com/upa/mscp/actions/workflows/test.yml)
 
 
-`mscp`, a variant of `scp`, copies files over multiple ssh (SFTP)
-connections. Multiple threads and connections in mscp transfer (1)
-multiple files simultaneously and (2) a large file in parallel. It
-would shorten the waiting time for transferring a lot of/large files
-over networks.
+`mscp`, a variant of `scp`, copies files over multiple SSH (SFTP)
+connections by multiple threads. It enables transferring (1) multiple
+files simultaneously and (2) a large file in parallel, reducing the
+transfer time for a lot of/large files over networks.
 
-You can use `mscp` like `scp`, for example, `mscp
-user@example.com:srcfile /tmp/dstfile`. Remote hosts only need to run
-standard `sshd` supporting the SFTP subsystem (e.g. openssh-server),
-and you need to be able to ssh to the hosts as usual. `mscp` does not
-require anything else.
+You can use `mscp` like `scp`, for example:
+
+```shell-session
+$ mscp user@example.com:srcfile /tmp/dstfile
+```
+
+Remote hosts only need to run standard `sshd` supporting the SFTP
+subsystem (e.g. openssh-server), and you need to be able to ssh to the
+hosts as usual. `mscp` does not require anything else.
 
 
 https://github.com/upa/mscp/assets/184632/19230f57-be7f-4ef0-98dd-cb4c460f570d
@@ -26,7 +29,8 @@ Differences from `scp` on usage:
 - `-r` option is not needed to transfer directories.
 - and any other differences I have not implemented and noticed.
 
-
+Paper:
+- Ryo Nakamura and Yohei Kuga. 2023. Multi-threaded scp: Easy and Fast File Transfer over SSH. In Practice and Experience in Advanced Research Computing (PEARC '23). Association for Computing Machinery, New York, NY, USA, 320–323. https://doi.org/10.1145/3569951.3597582
 
 ## Install
 
@@ -36,16 +40,10 @@ Differences from `scp` on usage:
 brew install upa/tap/mscp
 ```
 
-- Ubuntu 22.04
+- Ubuntu
 ```console
-wget https://github.com/upa/mscp/releases/latest/download/mscp_ubuntu-22.04-x86_64.deb
-apt-get install -f ./mscp_ubuntu-22.04-x86_64.deb
-```
-
-- Ubuntu 20.04
-```console
-wget https://github.com/upa/mscp/releases/latest/download/mscp_ubuntu-20.04-x86_64.deb
-apt-get install -f ./mscp_ubuntu-20.04-x86_64.deb
+sudo add-apt-repository ppa:upaa/mscp
+sudo apt-get install mscp
 ```
 
 - Rocky 8.8
@@ -58,7 +56,7 @@ yum install https://github.com/upa/mscp/releases/latest/download/mscp_rocky-8.8-
 yum install https://github.com/upa/mscp/releases/latest/download/mscp_almalinux-8.8-x86_64.rpm
 ```
 
-- Linux with single binary `mscp` (x86_64 only)
+- Linux with single binary `mscp` (x86_64 only, and not optimal performance)
 ```console
 wget https://github.com/upa/mscp/releases/latest/download/mscp.linux.x86.static -O /usr/local/bin/mscp
 chmod 755 /usr/local/bin/mscp
@@ -67,7 +65,7 @@ chmod 755 /usr/local/bin/mscp
 
 ## Build
 
-mscp depends on a patched [libssh](https://www.libssh.org/).  The
+mscp depends on a patched [libssh](https://www.libssh.org/). The
 patch introduces asynchronous SFTP Write, which is derived from
 https://github.com/limes-datentechnik-gmbh/libssh (see [Re: SFTP Write
 async](https://archive.libssh.org/libssh/2020-06/0000004.html)).
@@ -81,7 +79,7 @@ cd mscp
 
 # prepare patched libssh
 git submodule update --init
-patch -d libssh -p1 < patch/libssh-0.10.4.patch
+patch -d libssh -p1 < patch/$(git --git-dir=./libssh/.git describe).patch
 
 # install build dependency
 bash ./scripts/install-build-deps.sh
@@ -99,105 +97,12 @@ make
 # install the mscp binary to CMAKE_INSTALL_PREFIX/bin (usually /usr/local/bin)
 make install
 ```
+
 Source tar balls (`mscp-X.X.X.tar.gz`, not `Source code`) in
 [Releases page](https://github.com/upa/mscp/releases) contains the patched version
 of libssh. So you can start from cmake with it.
 
-## Run
 
-- Usage
+## Documentation
 
-```console
-$ mscp
-mscp v0.0.8: copy files over multiple ssh connections
-
-Usage: mscp [vqDHdNh] [-n nr_conns] [-m coremask] [-u max_startups]
-            [-s min_chunk_sz] [-S max_chunk_sz] [-a nr_ahead] [-b buf_sz]
-            [-l login_name] [-p port] [-i identity_file]
-            [-c cipher_spec] [-M hmac_spec] [-C compress] source ... target
-```
-
-- Example: copy a 15GB file on memory over a 100Gbps link
-  - Two Intel Xeon Gold 6130 machines directly connected with Intel E810 100Gbps NICs.
-  - Default `openssh-server` runs on the remote host.
-
-```console
-$ mscp /var/ram/test.img 10.0.0.1:/var/ram/
-[======================================] 100%   15GB/15GB    1.7GB/s  00:00 ETA
-```
-
-```console
-# with some optimizations. top speed reaches 3.0GB/s.
-$ mscp -n 5 -m 0x1f -c aes128-gcm@openssh.com /var/ram/test.img 10.0.0.1:/var/ram/
-[======================================] 100%   15GB/15GB    2.4GB/s  00:00 ETA
-```
-
-- `-v` option increments verbose output level.
-
-```console
-$ mscp test 10.0.0.1:
-[=======================================] 100%   49B /49B   198.8B/s   00:00 ETA
-```
-
-```console
-$ mscp -vv test 10.0.0.1:
-file: test/test1 -> ./test/test1
-file: test/testdir/asdf -> ./test/testdir/asdf
-file: test/testdir/qwer -> ./test/testdir/qwer
-file: test/test2 -> ./test/test2
-we have only 4 chunk(s). set number of connections to 4
-connecting to localhost for a copy thread...
-connecting to localhost for a copy thread...
-connecting to localhost for a copy thread...
-copy start: test/test1
-copy start: test/test2
-copy start: test/testdir/asdf
-copy start: test/testdir/qwer
-copy done: test/test1
-copy done: test/test2
-copy done: test/testdir/qwer
-copy done: test/testdir/asdf
-[=======================================] 100%   49B /49B   198.1B/s   00:00 ETA
-```
-
-- Full usage
-
-```console
-$ mscp -h
-mscp v0.0.9-11-g5802679: copy files over multiple ssh connections
-
-Usage: mscp [vqDHdNh] [-n nr_conns] [-m coremask] [-u max_startups]
-            [-s min_chunk_sz] [-S max_chunk_sz] [-a nr_ahead] [-b buf_sz]
-            [-l login_name] [-p port] [-F ssh_config] [-i identity_file]
-            [-c cipher_spec] [-M hmac_spec] [-C compress] source ... target
-
-    -n NR_CONNECTIONS  number of connections (default: floor(log(cores)*2)+1)
-    -m COREMASK        hex value to specify cores where threads pinned
-    -u MAX_STARTUPS    number of concurrent outgoing connections (default: 8)
-    -s MIN_CHUNK_SIZE  min chunk size (default: 64MB)
-    -S MAX_CHUNK_SIZE  max chunk size (default: filesize/nr_conn)
-
-    -a NR_AHEAD        number of inflight SFTP commands (default: 32)
-    -b BUF_SZ          buffer size for i/o and transfer
-
-    -v                 increment verbose output level
-    -q                 disable output
-    -D                 dry run. check copy destinations with -vvv
-    -r                 no effect
-
-    -l LOGIN_NAME      login name
-    -p PORT            port number
-    -F CONFIG          path to user ssh config (default ~/.ssh/config)
-    -i IDENTITY        identity file for public key authentication
-    -c CIPHER          cipher spec
-    -M HMAC            hmac spec
-    -C COMPRESS        enable compression: yes, no, zlib, zlib@openssh.com
-    -H                 disable hostkey check
-    -d                 increment ssh debug output level
-    -N                 enable Nagle's algorithm (default disabled)
-    -h                 print this help
-```
-
-
-Note: mscp is still under development, and the author is not
-responsible for any accidents due to mscp.
+[manpage](/doc/mscp.rst) is available.

VERSION

@@ -1 +1 @@
-0.1.1
+0.1.3

debian/.gitignore

@@ -0,0 +1,11 @@
+
+/.debhelper/
+/*/
+!/tests/
+!/source
+
+/*.log
+/*.substvars
+/*.debhelper
+/files
+/mscp

debian/changelog

@@ -0,0 +1,18 @@
+mscp (0.1.3) UNRELEASED; urgency=medium
+
+  * add -I option for inserting intervals between SSH attempts (issue #7)
+  * add -P option, equivalent to -p (just for compatibility)
+  * update libssh to 0.10.6-2-g6f1b1e76 for security fixes
+  * cleanup warning messages when scanning source files fails
+  * fix wrong destination paths for source paths under '/' (issue #8)
+  * reduce memory footprint for coping many (e.g., over 100k) files (issue #8)
+  * add SPDX-License-Identifer to the source files
+  * add manpage
+
+ -- Ryo Nakamura <upa@haeena.net>  Fri, 12 Jan 2024 22:20:24 +0900
+
+mscp (0.1.2) unstable; urgency=medium
+
+  * Initial release for debian packaging
+
+ -- Ryo Nakamura <upa@haeena.net>  Sun, 10 Dec 2023 21:51:49 +0900

debian/compat

@@ -0,0 +1 @@
+10

debian/control

@@ -0,0 +1,17 @@
+Source: mscp
+Section: net
+Priority: optional
+Maintainer: Ryo Nakamura <upa@haeena.net>
+Build-Depends: debhelper (>= 10~), cmake, zlib1g-dev, libssl-dev, libkrb5-dev
+Homepage: https://github.com/upa/mscp
+Standards-Version: 4.5.0.3
+
+
+Package: mscp
+Architecture: linux-any
+Depends: ${misc:Depends}, ${shlibs:Depends}
+Description: mscp, fast file transfer over multiple SSH connections
+ mscp transfers files over multiple SSH connections. Multiple threads
+ and connections in mscp transfer (1) multiple files simultaneously
+ and (2) a large file in parallel. It would shorten the waiting time
+ for transferring a lot of/large files over networks.

debian/copyright

@@ -0,0 +1,12 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Source: https://github.com/upa/mscp
+Upstream-Name: mscp
+Upstream-Contact: Ryo Nakamura <upa@haeena.net>
+
+Files: *
+Copyright: Ryo Nakamura
+License: GPL-3
+
+Files: src/list.h
+Copyright: kazutomo@mcs.anl.gov
+License: GPL

debian/rules

@@ -0,0 +1,13 @@
+#!/usr/bin/make -f
+
+%:
+	dh $@
+
+
+override_dh_auto_configure:
+	dh_auto_configure -- \
+		-DINSTALL_EXECUTABLE_ONLY=ON
+
+override_dh_auto_test:
+
+

debian/source/format

@@ -0,0 +1 @@
+3.0 (native)

doc/README.md

@@ -0,0 +1,11 @@
+
+# Document
+
+The base file of documents is `mscp.1.in`. The manpage of mscp and
+`doc/mscp.rst` are generated from `mscp.1.in`.
+
+When `mscp.1.in` is changed, update `doc/mscp.rst` by:
+
+1. `cd build`
+2. `cmake ..`
+3. `make update-mscp-rst`

doc/mscp.1.in

@@ -0,0 +1,316 @@
+.TH MSCP 1 "@MSCP_BUILD_VERSION@" "mscp" "User Commands"
+
+.SH NAME
+mscp \- copy files over multiple SSH connections
+
+.SH SYNOPSIS
+
+.B mscp
+.RB [ \-vqDHdNh ]
+[\c
+.BI \-n \ NR_CONNECTIONS\c
+]
+[\c
+.BI \-m \ COREMASK\c
+]
+[\c
+.BI \-u \ MAX_STARTUPS\c
+]
+[\c
+.BI \-I \ INTERVAL\c
+]
+[\c
+.BI \-s \ MIN_CHUNK_SIZE\c
+]
+[\c
+.BI \-S \ MAX_CHUNK_SIZE\c
+]
+[\c
+.BI \-a \ NR_AHEAD\c
+]
+[\c
+.BI \-b \ BUF_SIZE\c
+]
+[\c
+.BI \-l \ LOGIN_NAME\c
+]
+[\c
+.BR \-p |\c
+.BI \-P \ PORT\c
+]
+[\c
+.BI \-F \ CONFIG\c
+]
+[\c
+.BI \-i \ IDENTITY\c
+]
+[\c
+.BI \-c \ CIPHER\c
+]
+[\c
+.BI \-M \ HMAC\c
+]
+[\c
+.BI \-C \ COMPRESS\c
+]
+.I source ... target
+
+.SH DESCRIPTION
+
+.PP
+.B mscp
+copies files over multiple SSH (SFTP) connections by multiple
+threads. It enables transferring (1) multiple files simultaneously and
+(2) a large file in parallel, reducing the transfer time for a lot
+of/large files over networks.
+
+.PP
+The usage of
+.B mscp
+imitates the
+.B scp
+command of
+.I OpenSSH,
+for example:
+
+.nf
+    $ mscp srcfile user@example.com:dstfile
+.fi
+
+Remote hosts only need to run standard
+.B sshd
+supporting the SFTP subsystem, and users need to be able to
+.B ssh
+to the hosts as usual.
+.B mscp
+does not require anything else.
+
+.PP
+.B mscp
+uses
+.UR https://\:www\:.libssh\:.org
+libssh
+.UE
+as its SSH implementation. Thus, supported SSH features, for example,
+authentication, encryption, and various options in ssh_config, follow
+what
+.I libssh
+supports.
+
+.SH OPTIONS
+.TP
+.B \-n \fINR_CONNECTIONS\fR
+Specifies the number of SSH connections. The default value is
+calculated from the number of CPU cores on the host with the following
+formula: floor(log(nr_cores)*2)+1.
+
+.TP
+.B \-m \fICOREMASK\fR
+Configures CPU cores to be used by the hexadecimal bitmask.  All CPU
+cores are used by default.
+
+.TP
+.B \-u \fIMAX_STARTUPS\fR
+Specifies the number of concurrent outgoing SSH connections.
+.B sshd
+limits the number of simultaneous SSH connection attempts by
+.I MaxStartups
+in
+.I sshd_config.
+The default
+.I MaxStartups
+is 10; thus, we set the default MAX_STARTUPS 8.
+
+.TP
+.B \-I \fIINTERVAL\fR
+Specifies the interval (in seconds) between SSH connection
+attempts. Some firewall products treat SSH connection attempts from a
+single source IP address for a short period as a brute force attack.
+This option inserts intervals between the attempts to avoid being
+determined as an attack. The default value is 0.
+
+.TP
+.B \-s \fIMIN_CHUNK_SIZE\fR
+Specifies the minimum chunk size.
+.B mscp
+divides a file into chunks and copies the chunks in parallel.
+
+.TP
+.B \-S \fIMAX_CHUNK_SIZE\fR
+Specifies the maximum chunk size. The default is file size divided by
+the number of connections.
+
+.TP
+.B \-a \fINR_AHEAD\fR
+Specifies the number of inflight SFTP commands. The default value is
+32.
+
+.TP
+.B \-b \fIBUF_SIZE\fR
+Specifies the buffer size for I/O and transfer over SFTP. The default
+value is 16384. Note that the SSH specification restricts buffer size
+delivered over SSH. Changing this value is not recommended at present.
+
+.TP
+.B \-v
+Increments the verbose output level.
+
+.TP
+.B \-q
+Quiet mode: turns off all outputs.
+
+.TP
+.B \-D
+Dry-run mode: it scans source files to be copied, calculates chunks,
+and resolves destination file paths. Dry-run mode with
+.B -vv
+option enables confirming files to be copied and their destination
+paths.
+
+.TP
+.B \-r
+No effect.
+.B mscp
+copies recursively if a source path is a directory. This option exists
+for just compatibility.
+
+.TP
+.B \-l \fILOGIN_NAME\fR
+Specifies the username to log in on the remote machine as with
+.I ssh(1).
+
+.TP
+.B \-p,\-P \fIPORT\fR
+Specifies the port number to connect to on the remote machine as with
+ssh(1) and scp(1).
+
+.TP
+.B \-F \fICONFIG\fR
+Specifies an alternative per-user ssh configuration file. Note that
+acceptable options in the configuration file are what
+.I libssh
+supports.
+
+.TP
+.B \-i \fIIDENTITY\fR
+Specifies the identity file for public key authentication.
+
+.TP
+.B \-c \fICIPHER\fR
+Selects the cipher to use for encrypting the data transfer. See
+.UR https://\:www\:.libssh\:.org/\:features/
+libssh features
+.UE .
+
+.TP
+.B \-M \fIHMAC\fR
+Specifies MAC hash algorithms. See
+.UR https://\:www\:.libssh\:.org/\:features/
+libssh features
+.UE .
+
+.TP
+.B \-C \fICOMPRESS\fR
+Enables compression: yes, no, zlib, zlib@openssh.com. The default is
+none. See
+.UR https://\:www\:.libssh\:.org/\:features/
+libssh features
+.UE .
+
+.TP
+.B \-H
+Disables hostkey checking.
+
+.TP
+.B \-d
+Increments the ssh debug output level.
+
+.TP
+.B \-N
+Enables Nagle's algorithm. It is disabled by default.
+
+.TP
+.B \-h
+Prints help.
+
+.SH EXIT STATUS
+Exit status is 0 on success,  and >0 if an error occurs.
+
+.SH NOTES
+
+.PP
+.B mscp
+uses glob(3) for globbing pathnames, including matching patterns for
+local and remote paths. However, globbing on the
+.I remote
+side does not work with musl libc (used in Alpine Linux and the
+single-binary version of mscp) because musl libc does not support
+GLOB_ALTDIRFUNC.
+
+.PP
+.B mscp
+does not support remote-to-remote copy, which
+.B scp
+supports.
+
+.SH EXAMPLES
+
+.PP
+Copy a local file to a remote host with different name:
+
+.nf
+    $ mscp ~/src-file 10.0.0.1:copied-file
+.fi
+
+.PP
+Copy a local file and a directory to /tmp at a remote host:
+
+.nf
+    $ mscp ~/src-file dir1 10.0.0.1:/tmp
+.fi
+
+.PP
+In a long fat network, following options might improve performance:
+
+.nf
+    $ mscp -n 64 -m 0xffff -a 64 -c aes128-gcm@openssh.com src 10.0.0.1:
+.fi
+
+.B -n
+increases the number of SSH connections than default,
+.B -m
+pins threads to specific CPU cores,
+.B -a
+increases asynchronous inflight SFTP WRITE/READ commands, and
+.B -c aes128-gcm@openssh.com
+will be faster than the default chacha20-poly1305 cipher, particularly
+on hosts that support AES-NI.
+
+
+
+.SH "SEE ALSO"
+.BR scp (1),
+.BR ssh (1),
+.BR sshd (8).
+
+.SH "PAPER REFERENCE"
+
+
+Ryo Nakamura and Yohei Kuga. 2023. Multi-threaded scp: Easy and Fast
+File Transfer over SSH. In Practice and Experience in Advanced
+Research Computing (PEARC '23). Association for Computing Machinery,
+New York, NY, USA, 320–323.
+.UR https://\:doi\:.org/\:10.1145/\:3569951.3597582
+DOI
+.UE .
+
+
+.SH CONTACT INFROMATION
+.PP
+For pathces, bug reports, or feature requests, please open an issue on
+.UR https://\:github\:.com/\:upa/\:mscp
+GitHub
+.UE .
+
+.SH AUTHORS
+Ryo Nakamura <upa@haeena.net>

doc/mscp.rst

@@ -0,0 +1,210 @@
+====
+MSCP
+====
+
+:Date:   v0.1.2-14-g24617d2
+
+NAME
+====
+
+mscp - copy files over multiple SSH connections
+
+SYNOPSIS
+========
+
+**mscp** [**-vqDHdNh**] [ **-n**\ *NR_CONNECTIONS* ] [
+**-m**\ *COREMASK* ] [ **-u**\ *MAX_STARTUPS* ] [ **-I**\ *INTERVAL* ] [
+**-s**\ *MIN_CHUNK_SIZE* ] [ **-S**\ *MAX_CHUNK_SIZE* ] [
+**-a**\ *NR_AHEAD* ] [ **-b**\ *BUF_SIZE* ] [ **-l**\ *LOGIN_NAME* ] [
+**-p**\ \| **-P**\ *PORT* ] [ **-F**\ *CONFIG* ] [ **-i**\ *IDENTITY* ]
+[ **-c**\ *CIPHER* ] [ **-M**\ *HMAC* ] [ **-C**\ *COMPRESS* ] *source
+... target*
+
+DESCRIPTION
+===========
+
+**mscp** copies files over multiple SSH (SFTP) connections by multiple
+threads. It enables transferring (1) multiple files simultaneously and
+(2) a large file in parallel, reducing the transfer time for a lot
+of/large files over networks.
+
+The usage of **mscp** imitates the **scp** command of *OpenSSH,* for
+example:
+
+::
+
+       $ mscp srcfile user@example.com:dstfile
+
+Remote hosts only need to run standard **sshd** supporting the SFTP
+subsystem, and users need to be able to **ssh** to the hosts as usual.
+**mscp** does not require anything else.
+
+**mscp** uses `libssh <https://www.libssh.org>`__ as its SSH
+implementation. Thus, supported SSH features, for example,
+authentication, encryption, and various options in ssh_config, follow
+what *libssh* supports.
+
+OPTIONS
+=======
+
+**-n NR_CONNECTIONS**
+   Specifies the number of SSH connections. The default value is
+   calculated from the number of CPU cores on the host with the
+   following formula: floor(log(nr_cores)*2)+1.
+
+**-m COREMASK**
+   Configures CPU cores to be used by the hexadecimal bitmask. All CPU
+   cores are used by default.
+
+**-u MAX_STARTUPS**
+   Specifies the number of concurrent outgoing SSH connections. **sshd**
+   limits the number of simultaneous SSH connection attempts by
+   *MaxStartups* in *sshd_config.* The default *MaxStartups* is 10;
+   thus, we set the default MAX_STARTUPS 8.
+
+**-I INTERVAL**
+   Specifies the interval (in seconds) between SSH connection attempts.
+   Some firewall products treat SSH connection attempts from a single
+   source IP address for a short period as a brute force attack. This
+   option inserts intervals between the attempts to avoid being
+   determined as an attack. The default value is 0.
+
+**-s MIN_CHUNK_SIZE**
+   Specifies the minimum chunk size. **mscp** divides a file into chunks
+   and copies the chunks in parallel.
+
+**-S MAX_CHUNK_SIZE**
+   Specifies the maximum chunk size. The default is file size divided by
+   the number of connections.
+
+**-a NR_AHEAD**
+   Specifies the number of inflight SFTP commands. The default value is
+   32.
+
+**-b BUF_SIZE**
+   Specifies the buffer size for I/O and transfer over SFTP. The default
+   value is 16384. Note that the SSH specification restricts buffer size
+   delivered over SSH. Changing this value is not recommended at
+   present.
+
+**-v**
+   Increments the verbose output level.
+
+**-q**
+   Quiet mode: turns off all outputs.
+
+**-D**
+   Dry-run mode: it scans source files to be copied, calculates chunks,
+   and resolves destination file paths. Dry-run mode with **-vv** option
+   enables confirming files to be copied and their destination paths.
+
+**-r**
+   No effect. **mscp** copies recursively if a source path is a
+   directory. This option exists for just compatibility.
+
+**-l LOGIN_NAME**
+   Specifies the username to log in on the remote machine as with
+   *ssh(1).*
+
+**-p,-P PORT**
+   Specifies the port number to connect to on the remote machine as with
+   ssh(1) and scp(1).
+
+**-F CONFIG**
+   Specifies an alternative per-user ssh configuration file. Note that
+   acceptable options in the configuration file are what *libssh*
+   supports.
+
+**-i IDENTITY**
+   Specifies the identity file for public key authentication.
+
+**-c CIPHER**
+   Selects the cipher to use for encrypting the data transfer. See
+   `libssh features <https://www.libssh.org/features/>`__.
+
+**-M HMAC**
+   Specifies MAC hash algorithms. See `libssh
+   features <https://www.libssh.org/features/>`__.
+
+**-C COMPRESS**
+   Enables compression: yes, no, zlib, zlib@openssh.com. The default is
+   none. See `libssh features <https://www.libssh.org/features/>`__.
+
+**-H**
+   Disables hostkey checking.
+
+**-d**
+   Increments the ssh debug output level.
+
+**-N**
+   Enables Nagle's algorithm. It is disabled by default.
+
+**-h**
+   Prints help.
+
+EXIT STATUS
+===========
+
+Exit status is 0 on success, and >0 if an error occurs.
+
+NOTES
+=====
+
+**mscp** uses glob(3) for globbing pathnames, including matching
+patterns for local and remote paths. However, globbing on the *remote*
+side does not work with musl libc (used in Alpine Linux and the
+single-binary version of mscp) because musl libc does not support
+GLOB_ALTDIRFUNC.
+
+**mscp** does not support remote-to-remote copy, which **scp** supports.
+
+EXAMPLES
+========
+
+Copy a local file to a remote host with different name:
+
+::
+
+       $ mscp ~/src-file 10.0.0.1:copied-file
+
+Copy a local file and a directory to /tmp at a remote host:
+
+::
+
+       $ mscp ~/src-file dir1 10.0.0.1:/tmp
+
+In a long fat network, following options might improve performance:
+
+::
+
+       $ mscp -n 64 -m 0xffff -a 64 -c aes128-gcm@openssh.com src 10.0.0.1:
+
+**-n** increases the number of SSH connections than default, **-m** pins
+threads to specific CPU cores, **-a** increases asynchronous inflight
+SFTP WRITE/READ commands, and **-c aes128-gcm@openssh.com** will be
+faster than the default chacha20-poly1305 cipher, particularly on hosts
+that support AES-NI.
+
+SEE ALSO
+========
+
+**scp**\ (1), **ssh**\ (1), **sshd**\ (8).
+
+PAPER REFERENCE
+===============
+
+Ryo Nakamura and Yohei Kuga. 2023. Multi-threaded scp: Easy and Fast
+File Transfer over SSH. In Practice and Experience in Advanced Research
+Computing (PEARC '23). Association for Computing Machinery, New York,
+NY, USA, 320–323. `DOI <https://doi.org/10.1145/3569951.3597582>`__.
+
+CONTACT INFROMATION
+===================
+
+For pathces, bug reports, or feature requests, please open an issue on
+`GitHub <https://github.com/upa/mscp>`__.
+
+AUTHORS
+=======
+
+Ryo Nakamura <upa@haeena.net>

docker/almalinux-8.8.Dockerfile

@@ -1,7 +1,9 @@
 FROM almalinux:8.8
 
 # install pytest, sshd for test, and rpm-build
-RUN set -ex && yum -y install \
+RUN set -ex && \
+	rpm --import https://repo.almalinux.org/almalinux/RPM-GPG-KEY-AlmaLinux && \
+	yum -y install \
 	python3 python3-pip python3-devel openssh openssh-server openssh-clients rpm-build
 
 RUN python3 -m pip install pytest
@@ -32,5 +34,6 @@ RUN cd ${mscpdir}			\
 
 # install mscp python module
 RUN cd ${mscpdir}	\
-	&& python3 setup.py install --user
+	&& python3 pysetup.py install --user \
+	&& ldconfig
 

docker/alpine-3.17.Dockerfile

@@ -37,5 +37,4 @@ RUN cd ${mscpdir}							\
 
 # install mscp python module
 RUN cd ${mscpdir}	\
-	&& python3 setup.py install --user
-
+	&& python3 pysetup.py install --user

docker/rocky-8.8.Dockerfile

@@ -31,5 +31,6 @@ RUN cd ${mscpdir}			\
 
 # install mscp python module
 RUN cd ${mscpdir}	\
-	&& python3 setup.py install --user
+	&& python3 pysetup.py install --user \
+	&& ldconfig
 

docker/ubuntu-20.04.Dockerfile

@@ -36,4 +36,5 @@ RUN cd ${mscpdir}			\
 
 # install mscp python module
 RUN cd ${mscpdir}       \
-        && python3 setup.py install --user
+	&& python3 pysetup.py install --user \
+	&& ldconfig

docker/ubuntu-22.04.Dockerfile

@@ -35,5 +35,6 @@ RUN cd ${mscpdir}			\
 
 # install mscp python module
 RUN cd ${mscpdir}	\
-	&& python3 setup.py install --user
+	&& python3 pysetup.py install --user \
+	&& ldconfig
 

include/mscp.h

@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
 #ifndef _MSCP_H_
 #define _MSCP_H_
 
@@ -43,7 +44,8 @@ struct mscp_opts {
 	size_t	max_chunk_sz;	/** maximum chunk size (default file size/nr_threads) */
 	size_t	buf_sz;		/** buffer size, default 16k. */
 	char	coremask[MSCP_MAX_COREMASK_STR]; /** hex to specifiy usable cpu cores */
-	int	max_startups;	/* sshd MaxStartups concurrent connections */
+	int	max_startups;	/** sshd MaxStartups concurrent connections */
+	int     interval;	/** interval between SSH connection attempts */
 
 	int	severity; 	/** messaging severity. set MSCP_SERVERITY_* */
 	int	msg_fd;		/** fd to output message. default STDOUT (0),
@@ -56,6 +58,7 @@ struct mscp_opts {
 #define MSCP_SSH_MAX_CIPHER_STR		32
 #define MSCP_SSH_MAX_HMAC_STR		32
 #define MSCP_SSH_MAX_COMP_STR		32 /* yes, no, zlib, zlib@openssh.com, none */
+#define MSCP_SSH_MAX_CCALGO_STR		16
 #define MSCP_SSH_MAX_PASSWORD		128
 #define MSCP_SSH_MAX_PASSPHRASE		128
 
@@ -72,6 +75,7 @@ struct mscp_ssh_opts {
 	char	cipher[MSCP_SSH_MAX_CIPHER_STR];	/** cipher spec */
 	char	hmac[MSCP_SSH_MAX_HMAC_STR];		/** hmacp spec */
 	char	compress[MSCP_SSH_MAX_COMP_STR];	/** yes, no, zlib@openssh.com */
+	char	ccalgo[MSCP_SSH_MAX_CCALGO_STR];	/** TCP cc algorithm */
 
 	char	password[MSCP_SSH_MAX_PASSWORD];	/** password auth passowrd */
 	char	passphrase[MSCP_SSH_MAX_PASSPHRASE];	/** passphrase for private key */

include/mscp_version.h.in

@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
 #ifndef _MSCP_VERSION_H_
 #define _MSCP_VERSION_H_
 

patch/README.md

@@ -1,13 +1,5 @@
 
-Patch(es) in this directory introduces `sftp_async_write()` and
+Patches in this directory introduces `sftp_async_write()` and
 `sftp_async_write_end()` to libssh. Those implementations are derived
 from https://github.com/limes-datentechnik-gmbh/libssh. See [Re: SFTP
 Write async](https://archive.libssh.org/libssh/2020-06/0000004.html).
-
-```console
-git clone https://git.libssh.org/projects/libssh.git/ --depth=1 -b libssh-0.10.4
-cd libssh
-git apply ../pathc/libssh-0.10.4.patch
-
-# then build libssh
-```

patch/libssh-0.10.4.patch

@@ -1,3 +1,28 @@
+diff --git a/ConfigureChecks.cmake b/ConfigureChecks.cmake
+index 7103f303..c64eb39d 100644
+--- a/ConfigureChecks.cmake
++++ b/ConfigureChecks.cmake
+@@ -258,6 +258,7 @@ if (UNIX)
+     check_library_exists(util forkpty "" HAVE_LIBUTIL)
+     check_function_exists(cfmakeraw HAVE_CFMAKERAW)
+     check_function_exists(__strtoull HAVE___STRTOULL)
++    check_symbol_exists(TCP_CONGESTION "netinet/tcp.h" HAVE_TCP_CONGESTION)
+ endif (UNIX)
+ 
+ set(LIBSSH_REQUIRED_LIBRARIES ${_REQUIRED_LIBRARIES} CACHE INTERNAL "libssh required system libraries")
+diff --git a/config.h.cmake b/config.h.cmake
+index 1357615b..1e915ead 100644
+--- a/config.h.cmake
++++ b/config.h.cmake
+@@ -237,6 +237,8 @@
+ 
+ #cmakedefine HAVE_GCC_BOUNDED_ATTRIBUTE 1
+ 
++#cmakedefine HAVE_TCP_CONGESTION 1
++
+ /* Define to 1 if you want to enable GSSAPI */
+ #cmakedefine WITH_GSSAPI 1
+ 
 diff --git a/include/libssh/buffer.h b/include/libssh/buffer.h
 index a55a1b40..e34e075c 100644
 --- a/include/libssh/buffer.h
@@ -12,10 +37,18 @@ index a55a1b40..e34e075c 100644
  int ssh_buffer_validate_length(struct ssh_buffer_struct *buffer, size_t len);
  
 diff --git a/include/libssh/libssh.h b/include/libssh/libssh.h
-index 7857a77b..3eef7a16 100644
+index 7857a77b..6b4d481c 100644
 --- a/include/libssh/libssh.h
 +++ b/include/libssh/libssh.h
-@@ -833,6 +833,7 @@ LIBSSH_API const char* ssh_get_hmac_in(ssh_session session);
+@@ -402,6 +402,7 @@ enum ssh_options_e {
+   SSH_OPTIONS_GSSAPI_AUTH,
+   SSH_OPTIONS_GLOBAL_KNOWNHOSTS,
+   SSH_OPTIONS_NODELAY,
++  SSH_OPTIONS_CCALGO,
+   SSH_OPTIONS_PUBLICKEY_ACCEPTED_TYPES,
+   SSH_OPTIONS_PROCESS_CONFIG,
+   SSH_OPTIONS_REKEY_DATA,
+@@ -833,6 +834,7 @@ LIBSSH_API const char* ssh_get_hmac_in(ssh_session session);
  LIBSSH_API const char* ssh_get_hmac_out(ssh_session session);
  
  LIBSSH_API ssh_buffer ssh_buffer_new(void);
@@ -23,7 +56,7 @@ index 7857a77b..3eef7a16 100644
  LIBSSH_API void ssh_buffer_free(ssh_buffer buffer);
  #define SSH_BUFFER_FREE(x) \
      do { if ((x) != NULL) { ssh_buffer_free(x); x = NULL; } } while(0)
-@@ -843,6 +844,8 @@ LIBSSH_API void *ssh_buffer_get(ssh_buffer buffer);
+@@ -843,6 +845,8 @@ LIBSSH_API void *ssh_buffer_get(ssh_buffer buffer);
  LIBSSH_API uint32_t ssh_buffer_get_len(ssh_buffer buffer);
  LIBSSH_API int ssh_session_set_disconnect_message(ssh_session session, const char *message);
  
@@ -32,6 +65,18 @@ index 7857a77b..3eef7a16 100644
  #ifndef LIBSSH_LEGACY_0_4
  #include "libssh/legacy.h"
  #endif
+diff --git a/include/libssh/session.h b/include/libssh/session.h
+index d3e5787c..15183d1b 100644
+--- a/include/libssh/session.h
++++ b/include/libssh/session.h
+@@ -232,6 +232,7 @@ struct ssh_session_struct {
+         int gss_delegate_creds;
+         int flags;
+         int nodelay;
++        char *ccalgo;
+         bool config_processed;
+         uint8_t options_seen[SOC_MAX];
+         uint64_t rekey_data;
 diff --git a/include/libssh/sftp.h b/include/libssh/sftp.h
 index c855df8a..0fcdb9b8 100644
 --- a/include/libssh/sftp.h
@@ -158,6 +203,106 @@ index e0068015..cc0caf35 100644
  /**
   * @brief Ensure the buffer has at least a certain preallocated size.
   *
+diff --git a/src/connect.c b/src/connect.c
+index 57e37e63..c02397d5 100644
+--- a/src/connect.c
++++ b/src/connect.c
+@@ -156,6 +156,20 @@ static int set_tcp_nodelay(socket_t socket)
+                       sizeof(opt));
+ }
+ 
++static int set_tcp_ccalgo(socket_t socket, const char *ccalgo)
++{
++#ifdef HAVE_TCP_CONGESTION
++	return setsockopt(socket,
++			  IPPROTO_TCP,
++			  TCP_CONGESTION,
++			  (void *)ccalgo,
++			  strlen(ccalgo));
++#else
++	errno = ENOTSUP;
++	return -1;
++#endif
++}
++
+ /**
+  * @internal
+  *
+@@ -256,6 +270,18 @@ socket_t ssh_connect_host_nonblocking(ssh_session session, const char *host,
+             }
+         }
+ 
++        if (session->opts.ccalgo) {
++	     rc = set_tcp_ccalgo(s, session->opts.ccalgo);
++	     if (rc < 0) {
++		 ssh_set_error(session, SSH_FATAL,
++			       "Failed to set TCP_CONGESTION on socket: %s",
++			       ssh_strerror(errno, err_msg, SSH_ERRNO_MSG_MAX));
++		 ssh_connect_socket_close(s);
++		 s = -1;
++		 continue;
++	     }
++	}
++
+         errno = 0;
+         rc = connect(s, itr->ai_addr, itr->ai_addrlen);
+         if (rc == -1 && (errno != 0) && (errno != EINPROGRESS)) {
+diff --git a/src/options.c b/src/options.c
+index 49aaefa2..9f7360c3 100644
+--- a/src/options.c
++++ b/src/options.c
+@@ -210,6 +210,7 @@ int ssh_options_copy(ssh_session src, ssh_session *dest)
+     new->opts.gss_delegate_creds    = src->opts.gss_delegate_creds;
+     new->opts.flags                 = src->opts.flags;
+     new->opts.nodelay               = src->opts.nodelay;
++    new->opts.ccalgo                = src->opts.ccalgo;
+     new->opts.config_processed      = src->opts.config_processed;
+     new->common.log_verbosity       = src->common.log_verbosity;
+     new->common.callbacks           = src->common.callbacks;
+@@ -450,6 +451,10 @@ int ssh_options_set_algo(ssh_session session,
+  *                Set it to disable Nagle's Algorithm (TCP_NODELAY) on the
+  *                session socket. (int, 0=false)
+  *
++ *              - SSH_OPTIONS_CCALGO
++ *                Set it to specify TCP congestion control algorithm on the
++ *                session socket (Linux only). (int, 0=false)
++ *
+  *              - SSH_OPTIONS_PROCESS_CONFIG
+  *                Set it to false to disable automatic processing of per-user
+  *                and system-wide OpenSSH configuration files. LibSSH
+@@ -1013,6 +1018,20 @@ int ssh_options_set(ssh_session session, enum ssh_options_e type,
+                 session->opts.nodelay = (*x & 0xff) > 0 ? 1 : 0;
+             }
+             break;
++	case SSH_OPTIONS_CCALGO:
++            v = value;
++            if (v == NULL || v[0] == '\0') {
++                ssh_set_error_invalid(session);
++                return -1;
++            } else {
++                SAFE_FREE(session->opts.ccalgo);
++                session->opts.ccalgo = strdup(v);
++                if (session->opts.ccalgo == NULL) {
++                    ssh_set_error_oom(session);
++                    return -1;
++                }
++            }
++            break;
+         case SSH_OPTIONS_PROCESS_CONFIG:
+             if (value == NULL) {
+                 ssh_set_error_invalid(session);
+diff --git a/src/session.c b/src/session.c
+index 6025c133..6b197526 100644
+--- a/src/session.c
++++ b/src/session.c
+@@ -108,6 +108,7 @@ ssh_session ssh_new(void)
+     session->opts.fd = -1;
+     session->opts.compressionlevel = 7;
+     session->opts.nodelay = 0;
++    session->opts.ccalgo = NULL;
+ 
+     session->opts.flags = SSH_OPT_FLAG_PASSWORD_AUTH |
+                           SSH_OPT_FLAG_PUBKEY_AUTH |
 diff --git a/src/sftp.c b/src/sftp.c
 index e01012a8..702623a0 100644
 --- a/src/sftp.c

patch/libssh-0.10.6-2-g6f1b1e76.patch

@@ -0,0 +1,442 @@
+diff --git a/ConfigureChecks.cmake b/ConfigureChecks.cmake
+index 9de10225..0f3d20ed 100644
+--- a/ConfigureChecks.cmake
++++ b/ConfigureChecks.cmake
+@@ -258,6 +258,7 @@ if (UNIX)
+     check_library_exists(util forkpty "" HAVE_LIBUTIL)
+     check_function_exists(cfmakeraw HAVE_CFMAKERAW)
+     check_function_exists(__strtoull HAVE___STRTOULL)
++    check_symbol_exists(TCP_CONGESTION "netinet/tcp.h" HAVE_TCP_CONGESTION)
+ endif (UNIX)
+ 
+ set(LIBSSH_REQUIRED_LIBRARIES ${_REQUIRED_LIBRARIES} CACHE INTERNAL "libssh required system libraries")
+diff --git a/config.h.cmake b/config.h.cmake
+index cc83734d..f74cd03b 100644
+--- a/config.h.cmake
++++ b/config.h.cmake
+@@ -237,6 +237,8 @@
+ 
+ #cmakedefine HAVE_GCC_BOUNDED_ATTRIBUTE 1
+ 
++#cmakedefine HAVE_TCP_CONGESTION 1
++
+ /* Define to 1 if you want to enable GSSAPI */
+ #cmakedefine WITH_GSSAPI 1
+ 
+diff --git a/include/libssh/buffer.h b/include/libssh/buffer.h
+index 1fce7b76..b64d1455 100644
+--- a/include/libssh/buffer.h
++++ b/include/libssh/buffer.h
+@@ -37,6 +37,8 @@ int ssh_buffer_add_u8(ssh_buffer buffer, uint8_t data);
+ int ssh_buffer_add_u16(ssh_buffer buffer, uint16_t data);
+ int ssh_buffer_add_u32(ssh_buffer buffer, uint32_t data);
+ int ssh_buffer_add_u64(ssh_buffer buffer, uint64_t data);
++ssize_t ssh_buffer_add_func(ssh_buffer buffer, ssh_add_func f, size_t max_bytes,
++			    void *userdata);
+ 
+ int ssh_buffer_validate_length(struct ssh_buffer_struct *buffer, size_t len);
+ 
+diff --git a/include/libssh/libssh.h b/include/libssh/libssh.h
+index 669a0a96..b6a93ac7 100644
+--- a/include/libssh/libssh.h
++++ b/include/libssh/libssh.h
+@@ -402,6 +402,7 @@ enum ssh_options_e {
+   SSH_OPTIONS_GSSAPI_AUTH,
+   SSH_OPTIONS_GLOBAL_KNOWNHOSTS,
+   SSH_OPTIONS_NODELAY,
++  SSH_OPTIONS_CCALGO,
+   SSH_OPTIONS_PUBLICKEY_ACCEPTED_TYPES,
+   SSH_OPTIONS_PROCESS_CONFIG,
+   SSH_OPTIONS_REKEY_DATA,
+@@ -833,6 +834,7 @@ LIBSSH_API const char* ssh_get_hmac_in(ssh_session session);
+ LIBSSH_API const char* ssh_get_hmac_out(ssh_session session);
+ 
+ LIBSSH_API ssh_buffer ssh_buffer_new(void);
++LIBSSH_API ssh_buffer ssh_buffer_new_size(uint32_t size, uint32_t headroom);
+ LIBSSH_API void ssh_buffer_free(ssh_buffer buffer);
+ #define SSH_BUFFER_FREE(x) \
+     do { if ((x) != NULL) { ssh_buffer_free(x); x = NULL; } } while(0)
+@@ -843,6 +845,8 @@ LIBSSH_API void *ssh_buffer_get(ssh_buffer buffer);
+ LIBSSH_API uint32_t ssh_buffer_get_len(ssh_buffer buffer);
+ LIBSSH_API int ssh_session_set_disconnect_message(ssh_session session, const char *message);
+ 
++typedef ssize_t (*ssh_add_func) (void *ptr, size_t max_bytes, void *userdata);
++
+ #ifndef LIBSSH_LEGACY_0_4
+ #include "libssh/legacy.h"
+ #endif
+diff --git a/include/libssh/session.h b/include/libssh/session.h
+index 97936195..e4a7f80c 100644
+--- a/include/libssh/session.h
++++ b/include/libssh/session.h
+@@ -258,6 +258,7 @@ struct ssh_session_struct {
+         int flags;
+         int exp_flags;
+         int nodelay;
++        char *ccalgo;
+         bool config_processed;
+         uint8_t options_seen[SOC_MAX];
+         uint64_t rekey_data;
+diff --git a/include/libssh/sftp.h b/include/libssh/sftp.h
+index c713466e..e27fe326 100644
+--- a/include/libssh/sftp.h
++++ b/include/libssh/sftp.h
+@@ -565,6 +565,10 @@ LIBSSH_API int sftp_async_read(sftp_file file, void *data, uint32_t len, uint32_
+  */
+ LIBSSH_API ssize_t sftp_write(sftp_file file, const void *buf, size_t count);
+ 
++LIBSSH_API ssize_t sftp_async_write(sftp_file file, ssh_add_func f, size_t count,
++				    void *userdata, uint32_t* id);
++LIBSSH_API int sftp_async_write_end(sftp_file file, uint32_t id, int blocking);
++
+ /**
+  * @brief Seek to a specific location in a file.
+  *
+diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
+index 807313b5..86487087 100644
+--- a/src/CMakeLists.txt
++++ b/src/CMakeLists.txt
+@@ -448,6 +448,11 @@ if (BUILD_STATIC_LIB)
+   if (WIN32)
+     target_compile_definitions(ssh-static PUBLIC "LIBSSH_STATIC")
+   endif (WIN32)
++
++  install(TARGETS ssh-static
++	  EXPORT libssh-config
++	  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
++	  COMPONENT libraries)
+ endif (BUILD_STATIC_LIB)
+ 
+ message(STATUS "Threads_FOUND=${Threads_FOUND}")
+diff --git a/src/buffer.c b/src/buffer.c
+index 8991e006..e0414801 100644
+--- a/src/buffer.c
++++ b/src/buffer.c
+@@ -142,6 +142,40 @@ struct ssh_buffer_struct *ssh_buffer_new(void)
+     return buf;
+ }
+ 
++/**
++ * @brief Create a new SSH buffer with a specified size and headroom.
++ *
++ * @param[in] len       length for newly initialized SSH buffer.
++ * @param[in] headroom  length for headroom
++ * @return A newly initialized SSH buffer, NULL on error.
++ */
++struct ssh_buffer_struct *ssh_buffer_new_size(uint32_t len, uint32_t headroom)
++{
++    struct ssh_buffer_struct *buf = NULL;
++    int rc;
++
++    if (len < headroom)
++	    return NULL;
++
++    buf = calloc(1, sizeof(struct ssh_buffer_struct));
++    if (buf == NULL) {
++        return NULL;
++    }
++
++    rc = ssh_buffer_allocate_size(buf, len);
++    if (rc != 0) {
++        SAFE_FREE(buf);
++        return NULL;
++    }
++
++    buf->pos += headroom;
++    buf->used += headroom;
++
++    buffer_verify(buf);
++
++    return buf;
++}
++
+ /**
+  * @brief Deallocate a SSH buffer.
+  *
+@@ -329,6 +363,49 @@ int ssh_buffer_add_data(struct ssh_buffer_struct *buffer, const void *data, uint
+     return 0;
+ }
+ 
++/**
++ * @brief Add data at the tail of a buffer by an external function
++ *
++ * @param[in]  buffer    The buffer to add data.
++ *
++ * @param[in]  f         function that adds data to the buffer.
++ *
++ * @param[in]  max_bytes The maximum length of the data to add.
++ *
++ * @return               actual bytes added on success, < 0 on error.
++ */
++ssize_t ssh_buffer_add_func(struct ssh_buffer_struct *buffer, ssh_add_func f,
++			    size_t max_bytes, void *userdata)
++{
++    ssize_t actual;
++
++    if (buffer == NULL) {
++        return -1;
++    }
++
++    buffer_verify(buffer);
++
++    if (buffer->used + max_bytes < max_bytes) {
++        return -1;
++    }
++
++    if (buffer->allocated < (buffer->used + max_bytes)) {
++        if (buffer->pos > 0) {
++            buffer_shift(buffer);
++        }
++        if (realloc_buffer(buffer, buffer->used + max_bytes) < 0) {
++            return -1;
++        }
++    }
++
++    if ((actual = f(buffer->data + buffer->used, max_bytes, userdata)) < 0)
++      return -1;
++
++    buffer->used += actual;
++    buffer_verify(buffer);
++    return actual;
++}
++
+ /**
+  * @brief Ensure the buffer has at least a certain preallocated size.
+  *
+diff --git a/src/connect.c b/src/connect.c
+index 15cae644..e7520f40 100644
+--- a/src/connect.c
++++ b/src/connect.c
+@@ -156,6 +156,20 @@ static int set_tcp_nodelay(socket_t socket)
+                       sizeof(opt));
+ }
+ 
++static int set_tcp_ccalgo(socket_t socket, const char *ccalgo)
++{
++#ifdef HAVE_TCP_CONGESTION
++	return setsockopt(socket,
++			  IPPROTO_TCP,
++			  TCP_CONGESTION,
++			  (void *)ccalgo,
++			  strlen(ccalgo));
++#else
++	errno = ENOTSUP;
++	return -1;
++#endif
++}
++
+ /**
+  * @internal
+  *
+@@ -256,6 +270,18 @@ socket_t ssh_connect_host_nonblocking(ssh_session session, const char *host,
+             }
+         }
+ 
++        if (session->opts.ccalgo) {
++	     rc = set_tcp_ccalgo(s, session->opts.ccalgo);
++	     if (rc < 0) {
++		 ssh_set_error(session, SSH_FATAL,
++			       "Failed to set TCP_CONGESTION on socket: %s",
++			       ssh_strerror(errno, err_msg, SSH_ERRNO_MSG_MAX));
++		 ssh_connect_socket_close(s);
++		 s = -1;
++		 continue;
++	     }
++	}
++
+         errno = 0;
+         rc = connect(s, itr->ai_addr, itr->ai_addrlen);
+         if (rc == -1 && (errno != 0) && (errno != EINPROGRESS)) {
+diff --git a/src/options.c b/src/options.c
+index b3ecffe1..fb966fa1 100644
+--- a/src/options.c
++++ b/src/options.c
+@@ -217,6 +217,7 @@ int ssh_options_copy(ssh_session src, ssh_session *dest)
+     new->opts.gss_delegate_creds    = src->opts.gss_delegate_creds;
+     new->opts.flags                 = src->opts.flags;
+     new->opts.nodelay               = src->opts.nodelay;
++    new->opts.ccalgo                = src->opts.ccalgo;
+     new->opts.config_processed      = src->opts.config_processed;
+     new->common.log_verbosity       = src->common.log_verbosity;
+     new->common.callbacks           = src->common.callbacks;
+@@ -458,6 +459,10 @@ int ssh_options_set_algo(ssh_session session,
+  *                Set it to disable Nagle's Algorithm (TCP_NODELAY) on the
+  *                session socket. (int, 0=false)
+  *
++ *              - SSH_OPTIONS_CCALGO
++ *                Set it to specify TCP congestion control algorithm on the
++ *                session socket (Linux only). (int, 0=false)
++ *
+  *              - SSH_OPTIONS_PROCESS_CONFIG
+  *                Set it to false to disable automatic processing of per-user
+  *                and system-wide OpenSSH configuration files. LibSSH
+@@ -1017,6 +1022,20 @@ int ssh_options_set(ssh_session session, enum ssh_options_e type,
+                 session->opts.nodelay = (*x & 0xff) > 0 ? 1 : 0;
+             }
+             break;
++	case SSH_OPTIONS_CCALGO:
++            v = value;
++            if (v == NULL || v[0] == '\0') {
++                ssh_set_error_invalid(session);
++                return -1;
++            } else {
++                SAFE_FREE(session->opts.ccalgo);
++                session->opts.ccalgo = strdup(v);
++                if (session->opts.ccalgo == NULL) {
++                    ssh_set_error_oom(session);
++                    return -1;
++                }
++            }
++            break;
+         case SSH_OPTIONS_PROCESS_CONFIG:
+             if (value == NULL) {
+                 ssh_set_error_invalid(session);
+diff --git a/src/session.c b/src/session.c
+index 8c509699..88602b6a 100644
+--- a/src/session.c
++++ b/src/session.c
+@@ -108,6 +108,7 @@ ssh_session ssh_new(void)
+     session->opts.fd = -1;
+     session->opts.compressionlevel = 7;
+     session->opts.nodelay = 0;
++    session->opts.ccalgo = NULL;
+ 
+     session->opts.flags = SSH_OPT_FLAG_PASSWORD_AUTH |
+                           SSH_OPT_FLAG_PUBKEY_AUTH |
+diff --git a/src/sftp.c b/src/sftp.c
+index e01012a8..702623a0 100644
+--- a/src/sftp.c
++++ b/src/sftp.c
+@@ -2228,6 +2228,132 @@ ssize_t sftp_write(sftp_file file, const void *buf, size_t count) {
+   return -1; /* not reached */
+ }
+ 
++/*
++ * sftp_async_write is based on and sftp_async_write_end is copied from
++ * https://github.com/limes-datentechnik-gmbh/libssh
++ *
++ * sftp_async_write has some optimizations:
++ * - use ssh_buffer_new_size() to reduce realoc_buffer.
++ * - use ssh_buffer_add_func() to avoid memcpy from read buffer to ssh buffer.
++ */
++ssize_t sftp_async_write(sftp_file file, ssh_add_func f, size_t count, void *userdata,
++			 uint32_t* id) {
++  sftp_session sftp = file->sftp;
++  ssh_buffer buffer;
++  uint32_t buf_sz;
++  ssize_t actual;
++  int len;
++  int packetlen;
++  int rc;
++
++#define HEADROOM 16
++  /* sftp_packet_write() prepends a 5-bytes (uint32_t length and
++   * 1-byte type) header to the head of the payload by
++   * ssh_buffer_prepend_data(). Inserting headroom by
++   * ssh_buffer_new_size() eliminates memcpy for prepending the
++   * header.
++   */
++
++  buf_sz = (HEADROOM + /* for header */
++	    sizeof(uint32_t) + /* id */
++	    ssh_string_len(file->handle) + 4 + /* file->handle */
++	    sizeof(uint64_t) + /* file->offset */
++	    sizeof(uint32_t) + /* count */
++	    count); /* datastring */
++
++  buffer = ssh_buffer_new_size(buf_sz, HEADROOM);
++  if (buffer == NULL) {
++    ssh_set_error_oom(sftp->session);
++    return -1;
++  }
++
++  *id = sftp_get_new_id(file->sftp);
++
++  rc = ssh_buffer_pack(buffer,
++                       "dSqd",
++                       *id,
++                       file->handle,
++                       file->offset,
++                       count); /* len of datastring */
++
++  if (rc != SSH_OK){
++    ssh_set_error_oom(sftp->session);
++    ssh_buffer_free(buffer);
++    return SSH_ERROR;
++  }
++
++  actual = ssh_buffer_add_func(buffer, f, count, userdata);
++  if (actual < 0){
++    ssh_set_error_oom(sftp->session);
++    ssh_buffer_free(buffer);
++    return SSH_ERROR;
++  }
++
++  packetlen=ssh_buffer_get_len(buffer)+5;
++  len = sftp_packet_write(file->sftp, SSH_FXP_WRITE, buffer);
++  ssh_buffer_free(buffer);
++  if (len < 0) {
++    return SSH_ERROR;
++  } else  if (len != packetlen) {
++    ssh_set_error(sftp->session, SSH_FATAL,
++      "Could only send %d of %d bytes to remote host!", len, packetlen);
++    SSH_LOG(SSH_LOG_PACKET,
++        "Could not write as much data as expected");
++    return SSH_ERROR;
++  }
++
++  file->offset += actual;
++
++  return actual;
++}
++
++int sftp_async_write_end(sftp_file file, uint32_t id, int blocking) {
++  sftp_session sftp = file->sftp;
++  sftp_message msg = NULL;
++  sftp_status_message status;
++
++  msg = sftp_dequeue(sftp, id);
++  while (msg == NULL) {
++    if (!blocking && ssh_channel_poll(sftp->channel, 0) == 0) {
++      /* we cannot block */
++      return SSH_AGAIN;
++    }
++    if (sftp_read_and_dispatch(sftp) < 0) {
++      /* something nasty has happened */
++      return SSH_ERROR;
++    }
++    msg = sftp_dequeue(sftp, id);
++  }
++
++  switch (msg->packet_type) {
++    case SSH_FXP_STATUS:
++      status = parse_status_msg(msg);
++      sftp_message_free(msg);
++      if (status == NULL) {
++        return SSH_ERROR;
++      }
++      sftp_set_error(sftp, status->status);
++      switch (status->status) {
++        case SSH_FX_OK:
++          status_msg_free(status);
++          return SSH_OK;
++        default:
++          break;
++      }
++      ssh_set_error(sftp->session, SSH_REQUEST_DENIED,
++          "SFTP server: %s", status->errormsg);
++      status_msg_free(status);
++      return SSH_ERROR;
++    default:
++      ssh_set_error(sftp->session, SSH_FATAL,
++          "Received message %d during write!", msg->packet_type);
++      sftp_message_free(msg);
++      return SSH_ERROR;
++  }
++
++  return SSH_ERROR; /* not reached */
++}
++
+ /* Seek to a specific location in a file. */
+ int sftp_seek(sftp_file file, uint32_t new_offset) {
+   if (file == NULL) {

patch/libssh-0.10.6.patch

@@ -0,0 +1,442 @@
+diff --git a/ConfigureChecks.cmake b/ConfigureChecks.cmake
+index 9de10225..0f3d20ed 100644
+--- a/ConfigureChecks.cmake
++++ b/ConfigureChecks.cmake
+@@ -258,6 +258,7 @@ if (UNIX)
+     check_library_exists(util forkpty "" HAVE_LIBUTIL)
+     check_function_exists(cfmakeraw HAVE_CFMAKERAW)
+     check_function_exists(__strtoull HAVE___STRTOULL)
++    check_symbol_exists(TCP_CONGESTION "netinet/tcp.h" HAVE_TCP_CONGESTION)
+ endif (UNIX)
+ 
+ set(LIBSSH_REQUIRED_LIBRARIES ${_REQUIRED_LIBRARIES} CACHE INTERNAL "libssh required system libraries")
+diff --git a/config.h.cmake b/config.h.cmake
+index cc83734d..f74cd03b 100644
+--- a/config.h.cmake
++++ b/config.h.cmake
+@@ -237,6 +237,8 @@
+ 
+ #cmakedefine HAVE_GCC_BOUNDED_ATTRIBUTE 1
+ 
++#cmakedefine HAVE_TCP_CONGESTION 1
++
+ /* Define to 1 if you want to enable GSSAPI */
+ #cmakedefine WITH_GSSAPI 1
+ 
+diff --git a/include/libssh/buffer.h b/include/libssh/buffer.h
+index 1fce7b76..b64d1455 100644
+--- a/include/libssh/buffer.h
++++ b/include/libssh/buffer.h
+@@ -37,6 +37,8 @@ int ssh_buffer_add_u8(ssh_buffer buffer, uint8_t data);
+ int ssh_buffer_add_u16(ssh_buffer buffer, uint16_t data);
+ int ssh_buffer_add_u32(ssh_buffer buffer, uint32_t data);
+ int ssh_buffer_add_u64(ssh_buffer buffer, uint64_t data);
++ssize_t ssh_buffer_add_func(ssh_buffer buffer, ssh_add_func f, size_t max_bytes,
++			    void *userdata);
+ 
+ int ssh_buffer_validate_length(struct ssh_buffer_struct *buffer, size_t len);
+ 
+diff --git a/include/libssh/libssh.h b/include/libssh/libssh.h
+index 669a0a96..b6a93ac7 100644
+--- a/include/libssh/libssh.h
++++ b/include/libssh/libssh.h
+@@ -402,6 +402,7 @@ enum ssh_options_e {
+   SSH_OPTIONS_GSSAPI_AUTH,
+   SSH_OPTIONS_GLOBAL_KNOWNHOSTS,
+   SSH_OPTIONS_NODELAY,
++  SSH_OPTIONS_CCALGO,
+   SSH_OPTIONS_PUBLICKEY_ACCEPTED_TYPES,
+   SSH_OPTIONS_PROCESS_CONFIG,
+   SSH_OPTIONS_REKEY_DATA,
+@@ -833,6 +834,7 @@ LIBSSH_API const char* ssh_get_hmac_in(ssh_session session);
+ LIBSSH_API const char* ssh_get_hmac_out(ssh_session session);
+ 
+ LIBSSH_API ssh_buffer ssh_buffer_new(void);
++LIBSSH_API ssh_buffer ssh_buffer_new_size(uint32_t size, uint32_t headroom);
+ LIBSSH_API void ssh_buffer_free(ssh_buffer buffer);
+ #define SSH_BUFFER_FREE(x) \
+     do { if ((x) != NULL) { ssh_buffer_free(x); x = NULL; } } while(0)
+@@ -843,6 +845,8 @@ LIBSSH_API void *ssh_buffer_get(ssh_buffer buffer);
+ LIBSSH_API uint32_t ssh_buffer_get_len(ssh_buffer buffer);
+ LIBSSH_API int ssh_session_set_disconnect_message(ssh_session session, const char *message);
+ 
++typedef ssize_t (*ssh_add_func) (void *ptr, size_t max_bytes, void *userdata);
++
+ #ifndef LIBSSH_LEGACY_0_4
+ #include "libssh/legacy.h"
+ #endif
+diff --git a/include/libssh/session.h b/include/libssh/session.h
+index 97936195..e4a7f80c 100644
+--- a/include/libssh/session.h
++++ b/include/libssh/session.h
+@@ -258,6 +258,7 @@ struct ssh_session_struct {
+         int flags;
+         int exp_flags;
+         int nodelay;
++        char *ccalgo;
+         bool config_processed;
+         uint8_t options_seen[SOC_MAX];
+         uint64_t rekey_data;
+diff --git a/include/libssh/sftp.h b/include/libssh/sftp.h
+index c713466e..e27fe326 100644
+--- a/include/libssh/sftp.h
++++ b/include/libssh/sftp.h
+@@ -565,6 +565,10 @@ LIBSSH_API int sftp_async_read(sftp_file file, void *data, uint32_t len, uint32_
+  */
+ LIBSSH_API ssize_t sftp_write(sftp_file file, const void *buf, size_t count);
+ 
++LIBSSH_API ssize_t sftp_async_write(sftp_file file, ssh_add_func f, size_t count,
++				    void *userdata, uint32_t* id);
++LIBSSH_API int sftp_async_write_end(sftp_file file, uint32_t id, int blocking);
++
+ /**
+  * @brief Seek to a specific location in a file.
+  *
+diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
+index 807313b5..86487087 100644
+--- a/src/CMakeLists.txt
++++ b/src/CMakeLists.txt
+@@ -448,6 +448,11 @@ if (BUILD_STATIC_LIB)
+   if (WIN32)
+     target_compile_definitions(ssh-static PUBLIC "LIBSSH_STATIC")
+   endif (WIN32)
++
++  install(TARGETS ssh-static
++	  EXPORT libssh-config
++	  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
++	  COMPONENT libraries)
+ endif (BUILD_STATIC_LIB)
+ 
+ message(STATUS "Threads_FOUND=${Threads_FOUND}")
+diff --git a/src/buffer.c b/src/buffer.c
+index 8991e006..e0414801 100644
+--- a/src/buffer.c
++++ b/src/buffer.c
+@@ -142,6 +142,40 @@ struct ssh_buffer_struct *ssh_buffer_new(void)
+     return buf;
+ }
+ 
++/**
++ * @brief Create a new SSH buffer with a specified size and headroom.
++ *
++ * @param[in] len       length for newly initialized SSH buffer.
++ * @param[in] headroom  length for headroom
++ * @return A newly initialized SSH buffer, NULL on error.
++ */
++struct ssh_buffer_struct *ssh_buffer_new_size(uint32_t len, uint32_t headroom)
++{
++    struct ssh_buffer_struct *buf = NULL;
++    int rc;
++
++    if (len < headroom)
++	    return NULL;
++
++    buf = calloc(1, sizeof(struct ssh_buffer_struct));
++    if (buf == NULL) {
++        return NULL;
++    }
++
++    rc = ssh_buffer_allocate_size(buf, len);
++    if (rc != 0) {
++        SAFE_FREE(buf);
++        return NULL;
++    }
++
++    buf->pos += headroom;
++    buf->used += headroom;
++
++    buffer_verify(buf);
++
++    return buf;
++}
++
+ /**
+  * @brief Deallocate a SSH buffer.
+  *
+@@ -329,6 +363,49 @@ int ssh_buffer_add_data(struct ssh_buffer_struct *buffer, const void *data, uint
+     return 0;
+ }
+ 
++/**
++ * @brief Add data at the tail of a buffer by an external function
++ *
++ * @param[in]  buffer    The buffer to add data.
++ *
++ * @param[in]  f         function that adds data to the buffer.
++ *
++ * @param[in]  max_bytes The maximum length of the data to add.
++ *
++ * @return               actual bytes added on success, < 0 on error.
++ */
++ssize_t ssh_buffer_add_func(struct ssh_buffer_struct *buffer, ssh_add_func f,
++			    size_t max_bytes, void *userdata)
++{
++    ssize_t actual;
++
++    if (buffer == NULL) {
++        return -1;
++    }
++
++    buffer_verify(buffer);
++
++    if (buffer->used + max_bytes < max_bytes) {
++        return -1;
++    }
++
++    if (buffer->allocated < (buffer->used + max_bytes)) {
++        if (buffer->pos > 0) {
++            buffer_shift(buffer);
++        }
++        if (realloc_buffer(buffer, buffer->used + max_bytes) < 0) {
++            return -1;
++        }
++    }
++
++    if ((actual = f(buffer->data + buffer->used, max_bytes, userdata)) < 0)
++      return -1;
++
++    buffer->used += actual;
++    buffer_verify(buffer);
++    return actual;
++}
++
+ /**
+  * @brief Ensure the buffer has at least a certain preallocated size.
+  *
+diff --git a/src/connect.c b/src/connect.c
+index 15cae644..e7520f40 100644
+--- a/src/connect.c
++++ b/src/connect.c
+@@ -156,6 +156,20 @@ static int set_tcp_nodelay(socket_t socket)
+                       sizeof(opt));
+ }
+ 
++static int set_tcp_ccalgo(socket_t socket, const char *ccalgo)
++{
++#ifdef HAVE_TCP_CONGESTION
++	return setsockopt(socket,
++			  IPPROTO_TCP,
++			  TCP_CONGESTION,
++			  (void *)ccalgo,
++			  strlen(ccalgo));
++#else
++	errno = ENOTSUP;
++	return -1;
++#endif
++}
++
+ /**
+  * @internal
+  *
+@@ -256,6 +270,18 @@ socket_t ssh_connect_host_nonblocking(ssh_session session, const char *host,
+             }
+         }
+ 
++        if (session->opts.ccalgo) {
++	     rc = set_tcp_ccalgo(s, session->opts.ccalgo);
++	     if (rc < 0) {
++		 ssh_set_error(session, SSH_FATAL,
++			       "Failed to set TCP_CONGESTION on socket: %s",
++			       ssh_strerror(errno, err_msg, SSH_ERRNO_MSG_MAX));
++		 ssh_connect_socket_close(s);
++		 s = -1;
++		 continue;
++	     }
++	}
++
+         errno = 0;
+         rc = connect(s, itr->ai_addr, itr->ai_addrlen);
+         if (rc == -1 && (errno != 0) && (errno != EINPROGRESS)) {
+diff --git a/src/options.c b/src/options.c
+index 38511455..a183605d 100644
+--- a/src/options.c
++++ b/src/options.c
+@@ -217,6 +217,7 @@ int ssh_options_copy(ssh_session src, ssh_session *dest)
+     new->opts.gss_delegate_creds    = src->opts.gss_delegate_creds;
+     new->opts.flags                 = src->opts.flags;
+     new->opts.nodelay               = src->opts.nodelay;
++    new->opts.ccalgo                = src->opts.ccalgo;
+     new->opts.config_processed      = src->opts.config_processed;
+     new->common.log_verbosity       = src->common.log_verbosity;
+     new->common.callbacks           = src->common.callbacks;
+@@ -458,6 +459,10 @@ int ssh_options_set_algo(ssh_session session,
+  *                Set it to disable Nagle's Algorithm (TCP_NODELAY) on the
+  *                session socket. (int, 0=false)
+  *
++ *              - SSH_OPTIONS_CCALGO
++ *                Set it to specify TCP congestion control algorithm on the
++ *                session socket (Linux only). (int, 0=false)
++ *
+  *              - SSH_OPTIONS_PROCESS_CONFIG
+  *                Set it to false to disable automatic processing of per-user
+  *                and system-wide OpenSSH configuration files. LibSSH
+@@ -1023,6 +1028,20 @@ int ssh_options_set(ssh_session session, enum ssh_options_e type,
+                 session->opts.nodelay = (*x & 0xff) > 0 ? 1 : 0;
+             }
+             break;
++	case SSH_OPTIONS_CCALGO:
++            v = value;
++            if (v == NULL || v[0] == '\0') {
++                ssh_set_error_invalid(session);
++                return -1;
++            } else {
++                SAFE_FREE(session->opts.ccalgo);
++                session->opts.ccalgo = strdup(v);
++                if (session->opts.ccalgo == NULL) {
++                    ssh_set_error_oom(session);
++                    return -1;
++                }
++            }
++            break;
+         case SSH_OPTIONS_PROCESS_CONFIG:
+             if (value == NULL) {
+                 ssh_set_error_invalid(session);
+diff --git a/src/session.c b/src/session.c
+index 8c509699..88602b6a 100644
+--- a/src/session.c
++++ b/src/session.c
+@@ -108,6 +108,7 @@ ssh_session ssh_new(void)
+     session->opts.fd = -1;
+     session->opts.compressionlevel = 7;
+     session->opts.nodelay = 0;
++    session->opts.ccalgo = NULL;
+ 
+     session->opts.flags = SSH_OPT_FLAG_PASSWORD_AUTH |
+                           SSH_OPT_FLAG_PUBKEY_AUTH |
+diff --git a/src/sftp.c b/src/sftp.c
+index e01012a8..702623a0 100644
+--- a/src/sftp.c
++++ b/src/sftp.c
+@@ -2228,6 +2228,132 @@ ssize_t sftp_write(sftp_file file, const void *buf, size_t count) {
+   return -1; /* not reached */
+ }
+ 
++/*
++ * sftp_async_write is based on and sftp_async_write_end is copied from
++ * https://github.com/limes-datentechnik-gmbh/libssh
++ *
++ * sftp_async_write has some optimizations:
++ * - use ssh_buffer_new_size() to reduce realoc_buffer.
++ * - use ssh_buffer_add_func() to avoid memcpy from read buffer to ssh buffer.
++ */
++ssize_t sftp_async_write(sftp_file file, ssh_add_func f, size_t count, void *userdata,
++			 uint32_t* id) {
++  sftp_session sftp = file->sftp;
++  ssh_buffer buffer;
++  uint32_t buf_sz;
++  ssize_t actual;
++  int len;
++  int packetlen;
++  int rc;
++
++#define HEADROOM 16
++  /* sftp_packet_write() prepends a 5-bytes (uint32_t length and
++   * 1-byte type) header to the head of the payload by
++   * ssh_buffer_prepend_data(). Inserting headroom by
++   * ssh_buffer_new_size() eliminates memcpy for prepending the
++   * header.
++   */
++
++  buf_sz = (HEADROOM + /* for header */
++	    sizeof(uint32_t) + /* id */
++	    ssh_string_len(file->handle) + 4 + /* file->handle */
++	    sizeof(uint64_t) + /* file->offset */
++	    sizeof(uint32_t) + /* count */
++	    count); /* datastring */
++
++  buffer = ssh_buffer_new_size(buf_sz, HEADROOM);
++  if (buffer == NULL) {
++    ssh_set_error_oom(sftp->session);
++    return -1;
++  }
++
++  *id = sftp_get_new_id(file->sftp);
++
++  rc = ssh_buffer_pack(buffer,
++                       "dSqd",
++                       *id,
++                       file->handle,
++                       file->offset,
++                       count); /* len of datastring */
++
++  if (rc != SSH_OK){
++    ssh_set_error_oom(sftp->session);
++    ssh_buffer_free(buffer);
++    return SSH_ERROR;
++  }
++
++  actual = ssh_buffer_add_func(buffer, f, count, userdata);
++  if (actual < 0){
++    ssh_set_error_oom(sftp->session);
++    ssh_buffer_free(buffer);
++    return SSH_ERROR;
++  }
++
++  packetlen=ssh_buffer_get_len(buffer)+5;
++  len = sftp_packet_write(file->sftp, SSH_FXP_WRITE, buffer);
++  ssh_buffer_free(buffer);
++  if (len < 0) {
++    return SSH_ERROR;
++  } else  if (len != packetlen) {
++    ssh_set_error(sftp->session, SSH_FATAL,
++      "Could only send %d of %d bytes to remote host!", len, packetlen);
++    SSH_LOG(SSH_LOG_PACKET,
++        "Could not write as much data as expected");
++    return SSH_ERROR;
++  }
++
++  file->offset += actual;
++
++  return actual;
++}
++
++int sftp_async_write_end(sftp_file file, uint32_t id, int blocking) {
++  sftp_session sftp = file->sftp;
++  sftp_message msg = NULL;
++  sftp_status_message status;
++
++  msg = sftp_dequeue(sftp, id);
++  while (msg == NULL) {
++    if (!blocking && ssh_channel_poll(sftp->channel, 0) == 0) {
++      /* we cannot block */
++      return SSH_AGAIN;
++    }
++    if (sftp_read_and_dispatch(sftp) < 0) {
++      /* something nasty has happened */
++      return SSH_ERROR;
++    }
++    msg = sftp_dequeue(sftp, id);
++  }
++
++  switch (msg->packet_type) {
++    case SSH_FXP_STATUS:
++      status = parse_status_msg(msg);
++      sftp_message_free(msg);
++      if (status == NULL) {
++        return SSH_ERROR;
++      }
++      sftp_set_error(sftp, status->status);
++      switch (status->status) {
++        case SSH_FX_OK:
++          status_msg_free(status);
++          return SSH_OK;
++        default:
++          break;
++      }
++      ssh_set_error(sftp->session, SSH_REQUEST_DENIED,
++          "SFTP server: %s", status->errormsg);
++      status_msg_free(status);
++      return SSH_ERROR;
++    default:
++      ssh_set_error(sftp->session, SSH_FATAL,
++          "Received message %d during write!", msg->packet_type);
++      sftp_message_free(msg);
++      return SSH_ERROR;
++  }
++
++  return SSH_ERROR; /* not reached */
++}
++
+ /* Seek to a specific location in a file. */
+ int sftp_seek(sftp_file file, uint32_t new_offset) {
+   if (file == NULL) {

src/atomic.h

@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
 #ifndef _ATOMIC_H_
 #define _ATOMIC_H_
 

src/fileops.c

@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
 #include <stdlib.h>
 #include <string.h>
 #include <errno.h>

src/fileops.h

@@ -1,4 +1,4 @@
-
+/* SPDX-License-Identifier: GPL-3.0-only */
 #include <dirent.h>
 #include <sys/stat.h>
 #include <glob.h>

src/main.c

@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
 #include <stdio.h>
 #include <stdlib.h>
 #include <stdarg.h>
@@ -18,10 +19,12 @@
 void usage(bool print_help) {
 	printf("mscp " MSCP_BUILD_VERSION ": copy files over multiple ssh connections\n"
 	       "\n"
-	       "Usage: mscp [vqDHdNh] [-n nr_conns] [-m coremask] [-u max_startups]\n"
+	       "Usage: mscp [vqDHdNh] [-n nr_conns] [-m coremask]\n"
+	       "            [-u max_startups] [-I interval]\n"
 	       "            [-s min_chunk_sz] [-S max_chunk_sz] [-a nr_ahead] [-b buf_sz]\n"
 	       "            [-l login_name] [-p port] [-F ssh_config] [-i identity_file]\n"
-	       "            [-c cipher_spec] [-M hmac_spec] [-C compress] source ... target\n"
+	       "            [-c cipher_spec] [-M hmac_spec] [-C compress] [-g congestion]\n"
+	       "            source ... target\n"
 	       "\n");
 
 	if (!print_help)
@@ -30,11 +33,12 @@ void usage(bool print_help) {
 	printf("    -n NR_CONNECTIONS  number of connections "
 	       "(default: floor(log(cores)*2)+1)\n"
 	       "    -m COREMASK        hex value to specify cores where threads pinned\n"
-	       "    -u MAX_STARTUPS    number of concurrent outgoing connections "
+	       "    -u MAX_STARTUPS    number of concurrent SSH connection attempts "
 	       "(default: 8)\n"
+	       "    -I INTERVAL        interval between SSH connection attempts (default: 0)\n"
+	       "\n"
 	       "    -s MIN_CHUNK_SIZE  min chunk size (default: 64MB)\n"
 	       "    -S MAX_CHUNK_SIZE  max chunk size (default: filesize/nr_conn)\n"
-	       "\n"
 	       "    -a NR_AHEAD        number of inflight SFTP commands (default: 32)\n"
 	       "    -b BUF_SZ          buffer size for i/o and transfer\n"
 	       "\n"
@@ -44,13 +48,14 @@ void usage(bool print_help) {
 	       "    -r                 no effect\n"
 	       "\n"
 	       "    -l LOGIN_NAME      login name\n"
-	       "    -p PORT            port number\n"
+	       "    -p/-P PORT         port number\n"
 	       "    -F CONFIG          path to user ssh config (default ~/.ssh/config)\n"
 	       "    -i IDENTITY        identity file for public key authentication\n"
 	       "    -c CIPHER          cipher spec\n"
 	       "    -M HMAC            hmac spec\n"
 	       "    -C COMPRESS        enable compression: "
 	       "yes, no, zlib, zlib@openssh.com\n"
+	       "    -g CONGESTION      specify TCP congestion control algorithm\n"
 	       "    -H                 disable hostkey check\n"
 	       "    -d                 increment ssh debug output level\n"
 	       "    -N                 enable Nagle's algorithm (default disabled)\n"
@@ -58,58 +63,113 @@ void usage(bool print_help) {
 	       "\n");
 }
 
-char *split_remote_and_path(const char *string, char **remote, char **path)
+char *strip_brackets(char *s)
 {
-	char *s, *p;
-
-	/* split user@host:path into user@host, and path.
-	 * return value is strdup()ed memory (for free()).
-	 */
-
-	if (!(s = strdup(string))) {
-		fprintf(stderr, "strdup: %s\n", strerror(errno));
-		return NULL;
+	if (s[0] == '[' && s[strlen(s) - 1] == ']') {
+		s[strlen(s) - 1] = '\0';
+		return s + 1;
 	}
-
-	if ((p = strchr(s, ':'))) {
-		if (p == s || ((p > s) && *(p - 1) == '\\')) {
-			/* first byte is colon, or escaped colon. no user@host here  */
-			goto no_remote;
-		} else {
-			/* we found ':', so this is remote:path notation. split it */
-			*p = '\0';
-			*remote = s;
-			*path = p + 1;
-			return s;
-		}
-	}
-
-no_remote:
-	*remote = NULL;
-	*path = s;
 	return s;
 }
 
+char *split_user_host_path(const char *s, char **userp, char **hostp, char **pathp)
+{
+	char *tmp, *cp, *user = NULL, *host = NULL, *path = NULL;
+	bool inbrackets = false;
+
+	if (!(tmp = strdup(s))) {
+		fprintf(stderr, "stdrup: %s\n", strerror(errno));
+		return NULL;
+	}
+
+	user = NULL;
+	host = NULL;
+	path = tmp;
+	for (cp = tmp; *cp; cp++) {
+		if (*cp == '@' && (cp > tmp) && *(cp - 1) != '\\' && user == NULL) {
+			/* cp is non-escaped '@', so this '@' is the
+			 * delimitater between username and host. */
+			*cp = '\0';
+			user = tmp;
+			host = cp + 1;
+		}
+		if (*cp == '[')
+			inbrackets = true;
+		if (*cp == ']')
+			inbrackets = false;
+		if (*cp == ':' && (cp > tmp) && *(cp - 1) != '\\') {
+			if (!inbrackets) {
+				/* cp is non-escaped ':' and not in
+				 * brackets for IPv6 address
+				 * notation. So, this ':' is the
+				 * delimitater between host and
+				 * path. */
+				*cp = '\0';
+				host = host == NULL ? tmp : host;
+				path = cp + 1;
+				break;
+			}
+		}
+	}
+	*userp = user;
+	*hostp = host ? strip_brackets(host) : NULL;
+	*pathp = path;
+	return tmp;
+}
+
 struct target {
-	char *remote;
+	char *copy;
+	char *user;
+	char *host;
 	char *path;
 };
 
+int compare_remote(struct target *a, struct target *b)
+{
+	/* return 0 if a and b have the identical user@host, otherwise 1 */
+	int alen, blen;
+
+	if (a->user) {
+		if (!b->user)
+			return 1;
+		alen = strlen(a->user);
+		blen = strlen(b->user);
+		if (alen != blen)
+			return 1;
+		if (strncmp(a->user, b->user, alen) != 0)
+			return 1;
+	} else if (b->user)
+		return 1;
+
+	if (a->host) {
+		if (!b->host)
+			return 1;
+		alen = strlen(a->host);
+		blen = strlen(b->host);
+		if (alen != blen)
+			return 1;
+		if (strncmp(a->host, b->host, alen) != 0)
+			return 1;
+	} else if (b->host)
+		return 1;
+
+	return 0;
+}
+
 struct target *validate_targets(char **arg, int len)
 {
 	/* arg is array of source ... destination.
 	 * There are two cases:
 	 *
-	 * 1. remote:path remote:path ... path, remote to local copy
-	 * 2. path path ... remote:path, local to remote copy.
+	 * 1. user@host:path host:path ... path, remote to local copy
+	 * 2. path path ... host:path, local to remote copy.
 	 *
-	 * This function split (remote:)path args into struct target,
+	 * This function split user@remote:path args into struct target,
 	 * and validate all remotes are identical (mscp does not support
 	 * remote to remote copy).
 	 */
 
-	struct target *t;
-	char *r;
+	struct target *t, *t0;
 	int n;
 
 	if ((t = calloc(len, sizeof(struct target))) == NULL) {
@@ -120,33 +180,28 @@ struct target *validate_targets(char **arg, int len)
 
 	/* split remote:path into remote and path */
 	for (n = 0; n < len; n++) {
-		if (split_remote_and_path(arg[n], &t[n].remote, &t[n].path) == NULL)
+		t[n].copy = split_user_host_path(arg[n], &t[n].user,
+					      &t[n].host, &t[n].path);
+		if (!t[n].copy)
 			goto free_target_out;
 	}
 
-	/* check all remote are identical. t[len - 1] is destination,
+	/* check all user@host are identical. t[len - 1] is destination,
 	 * so we need to check t[0] to t[len - 2] having the identical
-	 * remote */
-	r = t[0].remote;
+	 * remote notation */
+	t0 = &t[0];
 	for (n = 1; n < len - 1; n++) {
-		if (!r && t[n].remote) {
+		if (compare_remote(t0, &t[n]) != 0)
 			goto invalid_remotes;
-		}
-		if (r) {
-			if (!t[n].remote ||
-			    strlen(r) != strlen(t[n].remote) ||
-			    strcmp(r, t[n].remote) != 0)
-				goto invalid_remotes;
-		}
 	}
 
 	/* check inconsistent remote position in args */
-	if (t[0].remote == NULL && t[len - 1].remote == NULL) {
+	if (t[0].host == NULL && t[len - 1].host == NULL) {
 		fprintf(stderr, "no remote host given\n");
 		goto free_split_out;
 	}
 
-	if (t[0].remote != NULL && t[len - 1].remote != NULL) {
+	if (t[0].host != NULL && t[len - 1].host != NULL) {
 		fprintf(stderr, "no local path given\n");
 		goto free_split_out;
 	}
@@ -154,11 +209,11 @@ struct target *validate_targets(char **arg, int len)
 	return t;
 
 invalid_remotes:
-	fprintf(stderr, "specified remote host invalid\n");
+	fprintf(stderr, "invalid remote host notation\n");
 
 free_split_out:
 	for (n = 0; n < len; n++)
-		t[n].remote ? free(t[n].remote) : free(t[n].path);
+		if (t[n].copy) free(t[n].copy);
 
 free_target_out:
 	free(t);
@@ -171,8 +226,6 @@ pthread_t tid_stat = 0;
 
 void sigint_handler(int sig)
 {
-	if (tid_stat)
-		pthread_cancel(tid_stat);
 	mscp_stop(m);
 }
 
@@ -202,7 +255,8 @@ int main(int argc, char **argv)
 	memset(&o, 0, sizeof(o));
 	o.severity = MSCP_SEVERITY_WARN;
 
-	while ((ch = getopt(argc, argv, "n:m:u:s:S:a:b:vqDrl:p:i:F:c:M:C:HdNh")) != -1) {
+	while ((ch = getopt(argc, argv,
+			    "n:m:u:I:s:S:a:b:vqDrl:P:p:i:F:c:M:C:g:HdNh")) != -1) {
 		switch (ch) {
 		case 'n':
 			o.nr_threads = atoi(optarg);
@@ -218,6 +272,9 @@ int main(int argc, char **argv)
 		case 'u':
 			o.max_startups = atoi(optarg);
 			break;
+		case 'I':
+			o.interval = atoi(optarg);
+			break;
 		case 's':
 			o.min_chunk_sz = atoi(optarg);
 			break;
@@ -249,6 +306,8 @@ int main(int argc, char **argv)
 			}
 			strncpy(s.login_name, optarg, MSCP_SSH_MAX_LOGIN_NAME - 1);
 			break;
+		case 'P':
+			/* fallthough for compatibility with scp */
 		case 'p':
 			if (strlen(optarg) > MSCP_SSH_MAX_PORT_STR - 1) {
 				fprintf(stderr, "long port string: %s\n", optarg);
@@ -287,6 +346,13 @@ int main(int argc, char **argv)
 			}
 			strncpy(s.compress, optarg, MSCP_SSH_MAX_COMP_STR);
 			break;
+		case 'g':
+			if (strlen(optarg) > MSCP_SSH_MAX_CCALGO_STR - 1) {
+				fprintf(stderr, "long ccalgo string: %s\n", optarg);
+				return -1;
+			}
+			strncpy(s.ccalgo, optarg, MSCP_SSH_MAX_CCALGO_STR);
+			break;
 		case 'H':
 			s.no_hostkey_check = true;
 			break;
@@ -315,14 +381,19 @@ int main(int argc, char **argv)
 	if ((t = validate_targets(argv + optind, i)) == NULL)
 		return -1;
 
-	if (t[0].remote) {
+	if (t[0].host) {
 		/* copy remote to local */
 		direction = MSCP_DIRECTION_R2L;
-		remote = t[0].remote;
+		remote = t[0].host;
+		if (t[0].user != NULL && s.login_name[0] == '\0')
+			strncpy(s.login_name, t[0].user, MSCP_SSH_MAX_LOGIN_NAME - 1);
 	} else {
 		/* copy local to remote */
 		direction = MSCP_DIRECTION_L2R;
-		remote = t[i - 1].remote;
+		remote = t[i - 1].host;
+		if (t[i - 1].user != NULL && s.login_name[0] == '\0')
+			strncpy(s.login_name, t[i - 1].user,
+				MSCP_SSH_MAX_LOGIN_NAME - 1);
 	}
 
 	if (!dryrun) {
@@ -419,23 +490,43 @@ double calculate_bps(size_t diff, struct timeval *b, struct timeval *a)
         return (double)diff / calculate_timedelta(b, a);
 }
 
+
 char *calculate_eta(size_t remain, size_t diff, struct timeval *b, struct timeval *a,
 		    bool final)
 {
         static char buf[16];
-        double elapsed = calculate_timedelta(b, a);
-        double eta;
 
-        if (diff == 0)
+#define bps_window_size 16
+	static double bps_window[bps_window_size];
+	static size_t sum, idx, count;
+        double elapsed = calculate_timedelta(b, a);
+	double bps = diff / elapsed;
+	double avg, eta;
+
+	/* early return when diff == 0 (stalled) or final output */
+        if (diff == 0) {
                 snprintf(buf, sizeof(buf), "--:-- ETA");
-	else if (final) {
+		return buf;
+	}
+	if (final) {
 		snprintf(buf, sizeof(buf), "%02d:%02d    ",
 			 (int)(floor(elapsed / 60)), (int)round(elapsed) % 60);
-	} else {
-                eta = remain / (diff / elapsed);
-                snprintf(buf, sizeof(buf), "%02d:%02d ETA",
-                         (int)floor(eta / 60), (int)round(eta) % 60);
-        }
+		return buf;
+	}
+
+	/* drop the old bps value and add the recent one */
+	sum -= bps_window[idx];
+	bps_window[idx] = bps;
+	sum += bps_window[idx];
+	idx = (idx + 1) % bps_window_size;
+	count++;
+
+	/* calcuate ETA from avg of recent bps values */
+	avg = sum / min(count, bps_window_size);
+	eta = remain / avg;
+	snprintf(buf, sizeof(buf), "%02d:%02d ETA",
+		 (int)floor(eta / 60), (int)round(eta) % 60);
+
         return buf;
 }
 
@@ -482,7 +573,7 @@ void print_progress(struct timeval *b, struct timeval *a,
         char *byte_units[] = { "B ", "KB", "MB", "GB", "TB", "PB" };
         char suffix[128];
         int bps_u, byte_tu, byte_du;
-        size_t total_round, done_round;
+        double total_round, done_round;
         int percent;
         double bps;
 
@@ -505,11 +596,11 @@ void print_progress(struct timeval *b, struct timeval *a,
         percent = floor(((double)(done) / (double)total) * 100);
 
         done_round = done;
-        for (byte_du = 0; done_round > 1000 && byte_du < array_size(byte_units) - 1;
+        for (byte_du = 0; done_round > 1024 && byte_du < array_size(byte_units) - 1;
              byte_du++)
                 done_round /= 1024;
 
-        snprintf(suffix, sizeof(suffix), "%4lu%s/%lu%s %6.1f%s  %s",
+        snprintf(suffix, sizeof(suffix), "%4.1lf%s/%.1lf%s %6.1f%s  %s",
                  done_round, byte_units[byte_du], total_round, byte_units[byte_tu],
                  bps, bps_units[bps_u],
 		 calculate_eta(total - done, done - last, b, a, final));
@@ -526,17 +617,42 @@ struct xfer_stat {
 };
 struct xfer_stat x;
 
-void print_stat_thread_cleanup(void *arg)
+void print_stat(bool final)
 {
+	struct pollfd pfd = { .fd = msg_fd, .events = POLLIN };
 	struct mscp_stats s;
+	char buf[8192];
+	int timeout;
+
+	if (poll(&pfd, 1, !final ? 100 : 0) < 0) {
+		fprintf(stderr, "poll: %s\n", strerror(errno));
+		return;
+	}
+
+	if (pfd.revents & POLLIN) {
+		memset(buf, 0, sizeof(buf));
+		if (read(msg_fd, buf, sizeof(buf)) < 0) {
+			fprintf(stderr, "read: %s\n", strerror(errno));
+			return;
+		}
+		print_cli("\r\033[K" "%s", buf);
+	}
 
 	gettimeofday(&x.after, NULL);
-	mscp_get_stats(m, &s);
-	x.total = s.total;
-	x.done = s.done;
+	if (calculate_timedelta(&x.before, &x.after) > 1 || final) {
+		mscp_get_stats(m, &s);
+		x.total = s.total;
+		x.done = s.done;
+		print_progress(!final ? &x.before : &x.start, &x.after,
+			       x.total, !final ? x.last : 0, x.done, final);
+		x.before = x.after;
+		x.last = x.done;
+	}
+}
 
-	/* print progress from the beginning */
-	print_progress(&x.start, &x.after, x.total, 0, x.done, true);
+void print_stat_thread_cleanup(void *arg)
+{
+	print_stat(true);
 	print_cli("\n"); /* final output */
 }
 
@@ -555,31 +671,7 @@ void *print_stat_thread(void *arg)
         pthread_cleanup_push(print_stat_thread_cleanup, NULL);
 
 	while (true) {
-		if (poll(&pfd, 1, 100) < 0) {
-			fprintf(stderr, "poll: %s\n", strerror(errno));
-			return NULL;
-		}
-
-		if (pfd.revents & POLLIN) {
-			memset(buf, 0, sizeof(buf));
-			if (read(msg_fd, buf, sizeof(buf)) < 0) {
-				fprintf(stderr, "read: %s\n", strerror(errno));
-				return NULL;
-			}
-			print_cli("\r\033[K" "%s", buf);
-		}
-
-		gettimeofday(&x.after, NULL);
-		if (calculate_timedelta(&x.before, &x.after) > 1) {
-			mscp_get_stats(m, &s);
-			x.total = s.total;
-			x.done = s.done;
-
-			print_progress(&x.before, &x.after, x.total, x.last, x.done,
-				       false);
-			x.before = x.after;
-			x.last = x.done;
-		}
+		print_stat(false);
 	}
 
 	pthread_cleanup_pop(1);

src/message.c

@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
 #include <stdio.h>
 #include <stdarg.h>
 #include <string.h>

src/message.h

@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
 #ifndef _MESSAGE_H_
 #define _MESSAGE_H_
 

src/mscp.c

@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
 #include <stdbool.h>
 #include <unistd.h>
 #include <math.h>
@@ -80,8 +81,6 @@ struct src {
 #define non_null_string(s) (s[0] != '\0')
 
 
-
-
 static int expand_coremask(const char *coremask, int **cores, int *nr_cores)
 {
         int n, *core_list, core_list_len = 0, nr_usable, nr_all;
@@ -203,6 +202,11 @@ static int validate_and_set_defaut_params(struct mscp_opts *o)
 		return -1;
 	}
 
+	if (o->interval > 0) {
+		/* when the interval is set, establish SSH connections sequentially. */
+		o->max_startups = 1;
+	}
+
 	if (o->msg_fd == 0)
 		o->msg_fd = STDOUT_FILENO;
 
@@ -554,6 +558,8 @@ int mscp_start(struct mscp *m)
 int mscp_join(struct mscp *m)
 {
 	struct mscp_thread *t;
+	struct path *p;
+	size_t done = 0, nr_copied = 0, nr_tobe_copied = 0;
 	int n, ret = 0;
 
 	/* waiting for scan thread joins... */
@@ -563,6 +569,7 @@ int mscp_join(struct mscp *m)
 	RWLOCK_READ_ACQUIRE(&m->thread_rwlock);
 	list_for_each_entry(t, &m->thread_list, list) {
 		pthread_join(t->tid, NULL);
+		done += t->done;
 		if (t->ret < 0)
 			ret = t->ret;
 		if (t->sftp) {
@@ -577,10 +584,36 @@ int mscp_join(struct mscp *m)
 		m->first = NULL;
 	}
 
+	/* count up number of transferred files */
+	list_for_each_entry(p, &m->path_list, list) {
+		nr_tobe_copied++;
+		if (p->state == FILE_STATE_DONE) {
+			nr_copied++;
+		}
+	}
+
+	mpr_notice(m->msg_fp, "%lu/%lu bytes copied for %lu/%lu files\n",
+		   done, m->total_bytes, nr_copied, nr_tobe_copied);
+
 	return ret;
 }
 
-/* copy thread related functions */
+/* copy thread-related functions */
+
+static void wait_for_interval(int interval)
+{
+	_Atomic static long next;
+	struct timeval t;
+	long now;
+
+	gettimeofday(&t, NULL);
+	now = t.tv_sec * 1000000 + t.tv_usec;
+
+	if (next - now > 0)
+		usleep(next - now);
+
+	next = now + interval * 1000000;
+}
 
 static void mscp_copy_thread_cleanup(void *arg)
 {
@@ -590,16 +623,17 @@ static void mscp_copy_thread_cleanup(void *arg)
 
 void *mscp_copy_thread(void *arg)
 {
-        sftp_session src_sftp, dst_sftp;
-        struct mscp_thread *t = arg;
+	sftp_session src_sftp, dst_sftp;
+	struct mscp_thread *t = arg;
 	struct mscp *m = t->m;
-        struct chunk *c;
+	struct chunk *c;
+	bool nomore;
+
+        pthread_cleanup_push(mscp_copy_thread_cleanup, t);
 
         if (t->cpu > -1) {
-                if (set_thread_affinity(pthread_self(), t->cpu) < 0) {
-			t->ret = -1;
-                        return NULL;
-		}
+                if (set_thread_affinity(pthread_self(), t->cpu) < 0)
+			goto err_out;
         }
 
 	if (sem_wait(m->sem) < 0) {
@@ -608,9 +642,12 @@ void *mscp_copy_thread(void *arg)
 		goto err_out;
 	}
 
-	mpr_notice(m->msg_fp, "connecting to %s for a copy thread[%d]...\n",
-		   m->remote, t->id);
-	t->sftp = ssh_init_sftp_session(m->remote, m->ssh_opts);
+	if (!(nomore = chunk_pool_is_empty(&m->cp))) {
+		if (m->opts->interval > 0)
+			wait_for_interval(m->opts->interval);
+		mpr_notice(m->msg_fp, "thread:%d connecting to %s\n", t->id, m->remote);
+		t->sftp = ssh_init_sftp_session(m->remote, m->ssh_opts);
+	}
 
 	if (sem_post(m->sem) < 0) {
 		mscp_set_error("sem_post: %s", strerrno());
@@ -618,8 +655,13 @@ void *mscp_copy_thread(void *arg)
 		goto err_out;
 	}
 
+	if (nomore) {
+		mpr_notice(m->msg_fp, "thread:%d no more connections needed\n", t->id);
+		goto out;
+	}
+
 	if (!t->sftp) {
-		mpr_err(m->msg_fp, "copy thread[%d]: %s\n", t->id, mscp_get_error());
+		mpr_err(m->msg_fp, "thread:%d: %s\n", t->id, mscp_get_error());
 		goto err_out;
 	}
 
@@ -636,10 +678,6 @@ void *mscp_copy_thread(void *arg)
                 return NULL; /* not reached */
         }
 
-        pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
-        pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL);
-        pthread_cleanup_push(mscp_copy_thread_cleanup, t);
-
         while (1) {
                 c = chunk_pool_pop(&m->cp);
 		if (c == CHUNK_POP_WAIT) {
@@ -659,8 +697,8 @@ void *mscp_copy_thread(void *arg)
         pthread_cleanup_pop(1);
 
         if (t->ret < 0)
-		mpr_err(m->msg_fp, "copy failed: chunk %s 0x%010lx-0x%010lx\n",
-			c->p->path, c->off, c->off + c->len);
+		mpr_err(m->msg_fp, "thread:%d copy failed: %s 0x%010lx-0x%010lx\n",
+			t->id, c->p->path, c->off, c->off + c->len);
 
         return NULL;
 
@@ -668,12 +706,16 @@ err_out:
 	t->finished = true;
 	t->ret = -1;
 	return NULL;
+out:
+	t->finished = true;
+	t->ret = 0;
+	return NULL;
 }
 
 
-/* cleanup related functions */
+/* cleanup-related functions */
 
-static void free_src(struct list_head *list)
+static void list_free_src(struct list_head *list)
 {
 	struct src *s;
 	s = list_entry(list, typeof(*s), list);
@@ -681,21 +723,14 @@ static void free_src(struct list_head *list)
 	free(s);
 }
 
-static void free_path(struct list_head *list)
+static void list_free_path(struct list_head *list)
 {
 	struct path *p;
 	p = list_entry(list, typeof(*p), list);
-	free(p);
+	free_path(p);
 }
 
-static void free_chunk(struct list_head *list)
-{
-	struct chunk *c;
-	c = list_entry(list, typeof(*c), list);
-	free(c);
-}
-
-static void free_thread(struct list_head *list)
+static void list_free_thread(struct list_head *list)
 {
 	struct mscp_thread *t;
 	t = list_entry(list, typeof(*t), list);
@@ -709,17 +744,17 @@ void mscp_cleanup(struct mscp *m)
 		m->first = NULL;
 	}
 
-	list_free_f(&m->src_list, free_src);
+	list_free_f(&m->src_list, list_free_src);
 	INIT_LIST_HEAD(&m->src_list);
 
-	list_free_f(&m->path_list, free_path);
+	list_free_f(&m->path_list, list_free_path);
 	INIT_LIST_HEAD(&m->path_list);
 
 	chunk_pool_release(&m->cp);
 	chunk_pool_init(&m->cp);
 
 	RWLOCK_WRITE_ACQUIRE(&m->thread_rwlock);
-	list_free_f(&m->thread_list, free_thread);
+	list_free_f(&m->thread_list, list_free_thread);
 	RWLOCK_RELEASE();
 }
 
@@ -737,19 +772,20 @@ void mscp_free(struct mscp *m)
 
 void mscp_get_stats(struct mscp *m, struct mscp_stats *s)
 {
+	int nr_finished = 0, nr_threads = 0;
 	struct mscp_thread *t;
-	bool finished = true;
 
 	s->total = m->total_bytes;
 	s->done = 0;
 
 	RWLOCK_READ_ACQUIRE(&m->thread_rwlock);
 	list_for_each_entry(t, &m->thread_list, list) {
+		nr_threads++;
 		s->done += t->done;
-		if (!t->finished)
-			finished = false;
+		if (t->finished)
+			nr_finished++;
 	}
 	RWLOCK_RELEASE();
 
-	s->finished = finished;
+	s->finished = nr_threads > 0 ? (nr_finished == nr_threads) : false;
 }

src/path.c

@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
 #include <string.h>
 #include <unistd.h>
 #include <dirent.h>
@@ -13,7 +14,6 @@
 #include <path.h>
 #include <message.h>
 
-
 /* chunk pool operations */
 #define CHUNK_POOL_STATE_FILLING	0
 #define CHUNK_POOL_STATE_FILLED		1
@@ -49,6 +49,10 @@ size_t chunk_pool_size(struct chunk_pool *cp)
 	return cp->count;
 }
 
+bool chunk_pool_is_empty(struct chunk_pool *cp)
+{
+	return list_empty(&cp->list);
+}
 
 struct chunk *chunk_pool_pop(struct chunk_pool *cp)
 {
@@ -68,7 +72,7 @@ struct chunk *chunk_pool_pop(struct chunk_pool *cp)
 	}
 	LOCK_RELEASE();
 
-	/* return CHUNK_POP_WAIT would be very rare case, because it
+	/* return CHUNK_POP_WAIT would be a rare case, because it
 	 * means copying over SSH is faster than traversing
 	 * local/remote file paths.
 	 */
@@ -89,53 +93,66 @@ void chunk_pool_release(struct chunk_pool *cp)
 }
 
 /* paths of copy source resoltion */
-static int resolve_dst_path(const char *src_file_path, char *dst_file_path,
-			    struct path_resolve_args *a)
+static char *resolve_dst_path(const char *src_file_path, struct path_resolve_args *a)
 {
-        char copy[PATH_MAX];
+        char copy[PATH_MAX + 1], dst_file_path[PATH_MAX + 1];
         char *prefix;
         int offset;
+	int ret;
 
-        strncpy(copy, a->src_path, PATH_MAX - 1);
+        strncpy(copy, a->src_path, PATH_MAX);
         prefix = dirname(copy);
         if (!prefix) {
                 mscp_set_error("dirname: %s", strerrno());
-                return -1;
+                return NULL;
         }
-        if (strlen(prefix) == 1 && prefix[0] == '.')
-                offset = 0;
-        else
-                offset = strlen(prefix) + 1;
+
+	offset = strlen(prefix) + 1;
+	if (strlen(prefix) == 1) { /* corner cases */
+		switch (prefix[0]) {
+		case '.':
+			offset = 0;
+			break;
+		case '/':
+			offset = 1;
+			break;
+		}
+	}
 
         if (!a->src_path_is_dir && !a->dst_path_is_dir) {
                 /* src path is file. dst path is (1) file, or (2) does not exist.
                  * In the second case, we need to put src under the dst.
                  */
                 if (a->dst_path_should_dir)
-                        snprintf(dst_file_path, PATH_MAX - 1, "%s/%s",
-                                 a->dst_path, a->src_path + offset);
+                        ret = snprintf(dst_file_path, PATH_MAX, "%s/%s",
+				       a->dst_path, a->src_path + offset);
                 else
-                        strncpy(dst_file_path, a->dst_path, PATH_MAX - 1);
+                        ret = snprintf(dst_file_path, PATH_MAX, "%s", a->dst_path);
         }
 
         /* src is file, and dst is dir */
         if (!a->src_path_is_dir && a->dst_path_is_dir)
-                snprintf(dst_file_path, PATH_MAX - 1, "%s/%s",
-			 a->dst_path, a->src_path + offset);
+                ret = snprintf(dst_file_path, PATH_MAX, "%s/%s",
+			       a->dst_path, a->src_path + offset);
 
         /* both are directory */
         if (a->src_path_is_dir && a->dst_path_is_dir)
-                snprintf(dst_file_path, PATH_MAX - 1, "%s/%s",
-			 a->dst_path, src_file_path + offset);
+                ret = snprintf(dst_file_path, PATH_MAX, "%s/%s",
+			       a->dst_path, src_file_path + offset);
 
         /* dst path does not exist. change dir name to dst_path */
         if (a->src_path_is_dir && !a->dst_path_is_dir)
-                snprintf(dst_file_path, PATH_MAX - 1, "%s/%s",
-                         a->dst_path, src_file_path + strlen(a->src_path) + 1);
+                ret = snprintf(dst_file_path, PATH_MAX, "%s/%s",
+			       a->dst_path, src_file_path + strlen(a->src_path) + 1);
+
+	if (ret >= PATH_MAX) {
+		mpr_warn(a->msg_fp, "Too long path: %s\n", dst_file_path);
+		return NULL;
+	}
 
         mpr_debug(a->msg_fp, "file: %s -> %s\n", src_file_path, dst_file_path);
 
-        return 0;
+        return strndup(dst_file_path, PATH_MAX);
 }
 
 /* chunk preparation */
@@ -191,6 +208,15 @@ static int resolve_chunk(struct path *p, struct path_resolve_args *a)
         return 0;
 }
 
+void free_path(struct path *p)
+{
+	if (p->path)
+		free(p->path);
+	if (p->dst_path)
+		free(p->dst_path);
+	free(p);
+}
+
 static int append_path(sftp_session sftp, const char *path, struct stat st,
 		       struct list_head *path_list, struct path_resolve_args *a)
 {
@@ -203,13 +229,16 @@ static int append_path(sftp_session sftp, const char *path, struct stat st,
 
 	memset(p, 0, sizeof(*p));
 	INIT_LIST_HEAD(&p->list);
-	strncpy(p->path, path, PATH_MAX - 1);
+	p->path = strndup(path, PATH_MAX);
+	if (!p->path)
+		goto free_out;
 	p->size = st.st_size;
 	p->mode = st.st_mode;
 	p->state = FILE_STATE_INIT;
 	lock_init(&p->lock);
 
-	if (resolve_dst_path(p->path, p->dst_path, a) < 0)
+	p->dst_path = resolve_dst_path(p->path, a);
+	if (!p->dst_path)
 		goto free_out;
 
 	if (resolve_chunk(p, a) < 0)
@@ -222,7 +251,7 @@ static int append_path(sftp_session sftp, const char *path, struct stat st,
 	return 0;
 
 free_out:
-	free(p);
+	free_path(p);
 	return -1;
 }
 
@@ -239,14 +268,16 @@ static bool check_path_should_skip(const char *path)
 static int walk_path_recursive(sftp_session sftp, const char *path,
 			       struct list_head *path_list, struct path_resolve_args *a)
 {
-	char next_path[PATH_MAX];
+	char next_path[PATH_MAX + 1];
 	struct dirent *e;
 	struct stat st;
 	MDIR *d;
 	int ret;
 
-	if (mscp_stat(path, &st, sftp) < 0)
+	if (mscp_stat(path, &st, sftp) < 0) {
+		mpr_warn(a->msg_fp, "%s: %s\n",  strerrno(), path);
 		return -1;
+	}
 
 	if (S_ISREG(st.st_mode)) {
 		/* this path is regular file. it is to be copied */
@@ -256,22 +287,25 @@ static int walk_path_recursive(sftp_session sftp, const char *path,
 	if (!S_ISDIR(st.st_mode))
 		return 0; /* not a regular file and not a directory, skip it. */
 
-	/* ok, this path is directory. walk it. */
-	if (!(d = mscp_opendir(path, sftp)))
+	/* ok, this path is a directory. walk through it. */
+	if (!(d = mscp_opendir(path, sftp))) {
+		mpr_warn(a->msg_fp, "%s: %s\n", strerrno(), path);
 		return -1;
+	}
 	
 	for (e = mscp_readdir(d); e; e = mscp_readdir(d)) {
 		if (check_path_should_skip(e->d_name))
 			continue;
 		
-		if (strlen(path) + 1 + strlen(e->d_name) > PATH_MAX) {
-			mscp_set_error("too long path: %s/%s", path, e->d_name);
-			return -1;
+		ret = snprintf(next_path, PATH_MAX, "%s/%s", path, e->d_name);
+		if (ret >= PATH_MAX) {
+			mpr_warn(a->msg_fp, "Too long path: %s/%s\n", path, e->d_name);
+			continue;
 		}
-		snprintf(next_path, sizeof(next_path), "%s/%s", path, e->d_name);
-		ret = walk_path_recursive(sftp, next_path, path_list, a);
-		if (ret < 0)
-			return ret;
+
+		walk_path_recursive(sftp, next_path, path_list, a);
+		/* do not stop even when walk_path_recursive returns
+		 * -1 due to an unreadable file. go to a next file. */
 	}
 
 	mscp_closedir(d);

src/path.h

@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
 #ifndef _PATH_H_
 #define _PATH_H_
 
@@ -14,11 +15,11 @@
 struct path {
 	struct list_head	list;	/* mscp->path_list */
 
-	char	path[PATH_MAX];		/* file path */
-	size_t	size;			/* size of file on this path */
-	mode_t	mode;			/* permission */
+	char	*path;		/* file path */
+	size_t	size;		/* size of file on this path */
+	mode_t	mode;		/* permission */
 
-	char	dst_path[PATH_MAX];	/* copy dst path */
+	char	*dst_path;	/* copy dst path */
 
 	int	state;
 	lock	lock;
@@ -62,6 +63,9 @@ bool chunk_pool_is_filled(struct chunk_pool *cp);
 /* return number of chunks in the pool */
 size_t chunk_pool_size(struct chunk_pool *cp);
 
+/* return true if chunk pool is empty (all chunks are already poped) */
+bool chunk_pool_is_empty(struct chunk_pool *cp);
+
 /* free chunks in the chunk_pool */
 void chunk_pool_release(struct chunk_pool *cp);
 
@@ -90,6 +94,9 @@ struct path_resolve_args {
 int walk_src_path(sftp_session src_sftp, const char *src_path,
 		  struct list_head *path_list, struct path_resolve_args *a);
 
+/* free struct path */
+void free_path(struct path *p);
+
 /* copy a chunk. either src_sftp or dst_sftp is not null, and another is null */
 int copy_chunk(FILE *msg_fp, struct chunk *c,
 	       sftp_session src_sftp, sftp_session dst_sftp,

src/platform.c

@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
 #ifdef __APPLE__
 #include <stdlib.h>
 #include <sys/types.h>

src/platform.h

@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
 #ifndef _PLATFORM_H_
 #define _PLATFORM_H_
 

src/pymscp.c

@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
 #define PY_SSIZE_T_CLEAN
 #include <Python.h>
 #include <errno.h>
@@ -97,6 +98,7 @@ static PyObject *wrap_mscp_init(PyObject *self, PyObject *args, PyObject *kw)
 		"coremask",	/* const char * */
 
 		"max_startups",	/* int */
+		"interval",	/* int */
 		"severity",	/* int, MSCP_SERVERITY_* */
 		"msg_fd",	/* int */
 
@@ -109,6 +111,7 @@ static PyObject *wrap_mscp_init(PyObject *self, PyObject *args, PyObject *kw)
 		"cipher",	/* const char * */
 		"hmac",		/* const char * */
 		"compress",	/* const char * */
+		"ccalgo",       /* const char * */
 		"password",	/* const char * */
 		"passphrase",	/* const char * */
 
@@ -117,10 +120,10 @@ static PyObject *wrap_mscp_init(PyObject *self, PyObject *args, PyObject *kw)
 		"enable_nagle",		/* bool */
 		NULL,
 	};
-	const char *fmt = "si" "|" "ii" "kkk" "s" "iii" "ssss" "sssss" "ipp";
+	const char *fmt = "si" "|" "ii" "kkk" "s" "iiii" "ssss" "ssssss" "ipp";
 	char *coremask = NULL;
 	char *login_name = NULL, *port = NULL, *config = NULL, *identity = NULL;
-	char *cipher = NULL, *hmac = NULL, *compress = NULL;
+	char *cipher = NULL, *hmac = NULL, *compress = NULL, *ccalgo = NULL;
 	char *password = NULL, *passphrase = NULL;
 
 	struct instance *i;
@@ -145,6 +148,7 @@ static PyObject *wrap_mscp_init(PyObject *self, PyObject *args, PyObject *kw)
 					  &i->mo.buf_sz,
 					  &coremask,
 					  &i->mo.max_startups,
+					  &i->mo.interval,
 					  &i->mo.severity,
 					  &i->mo.msg_fd,
 					  &login_name,
@@ -154,6 +158,7 @@ static PyObject *wrap_mscp_init(PyObject *self, PyObject *args, PyObject *kw)
 					  &cipher,
 					  &hmac,
 					  &compress,
+					  &ccalgo,
 					  &password,
 					  &passphrase,
 					  &i->so.debug_level,
@@ -179,6 +184,8 @@ static PyObject *wrap_mscp_init(PyObject *self, PyObject *args, PyObject *kw)
 		strncpy(i->so.hmac, hmac, MSCP_SSH_MAX_HMAC_STR - 1);
 	if (compress)
 		strncpy(i->so.compress, compress, MSCP_SSH_MAX_COMP_STR - 1);
+	if (ccalgo)
+		strncpy(i->so.ccalgo, ccalgo, MSCP_SSH_MAX_CCALGO_STR - 1);
 	if (password)
 		strncpy(i->so.password, password, MSCP_SSH_MAX_PASSWORD - 1);
 	if (passphrase)

src/rename-logic.py

@@ -1,48 +0,0 @@
-#!/usr/bin/env python3
-
-from os.path import dirname, basename, isfile, isdir, exists
-from os import listdir
-import sys
-
-"""
-
-This file simply implements the src_path to dst_path conversion logic
-just for test. file_fill() and file_fill_recursive() in file.c
-implements this logic.
-
-"""
-
-
-def recursive(src, rel_path, dst, dst_should_dir, replace_dir_name):
-    
-    if isfile(src):
-        if dst_should_dir:
-            print("{} => {}/{}{}".format(src, dst, rel_path, basename(src)))
-        else:
-            print("{} => {}{}".format(src, rel_path, dst))
-        return
-        
-    # src is directory
-    for f in listdir(src):
-        next_src = "{}/{}".format(src, f)
-        if replace_dir_name and dst_should_dir:
-            next_rel_path = ""
-        else:
-            next_rel_path = "{}{}/".format(rel_path, basename(src))
-        recursive(next_src, next_rel_path, dst, dst_should_dir, False)
-
-
-def fill_dst(srclist, dst):
-    dst_must_dir = len(srclist) > 1
-    for src in srclist:
-        dst_should_dir = isdir(src) | isdir(dst)
-        replace_dir_name = not isdir(dst)
-        recursive(src, "", dst, dst_should_dir | dst_must_dir, replace_dir_name)
-
-
-def main():
-    if (len(sys.argv) < 2):
-        print("usage: {} source ... target".format(sys.argv[0]))
-    fill_dst(sys.argv[1:len(sys.argv) - 1], sys.argv[len(sys.argv) - 1])
-
-main()

src/ssh.c

@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
 #include <stdio.h>
 #include <string.h>
 #include <unistd.h>
@@ -64,6 +65,12 @@ static int ssh_set_opts(ssh_session ssh, struct mscp_ssh_opts *opts)
 		return -1;
 	}
 
+	if (is_specified(opts->ccalgo) &&
+	    ssh_options_set(ssh, SSH_OPTIONS_CCALGO, opts->ccalgo) < 0) {
+		mscp_set_error("failed to set cclago");
+		return -1;
+	}
+
 	/* if NOT specified to enable Nagle's algorithm, disable it (set TCP_NODELAY) */
 	if (!opts->enable_nagle) {
 		int v = 1;

src/ssh.h

@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
 #ifndef _SSH_H_
 #define _SSH_H_
 

src/util.h

@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
 #ifndef _UTIL_H_
 #define _UTIL_H_
 

test/test_e2e.py

@@ -3,7 +3,9 @@
 test_e2e.py: End-to-End test for mscp executable.
 """
 
+import platform
 import pytest
+import getpass
 import os
 
 from subprocess import check_call, CalledProcessError, PIPE
@@ -89,6 +91,47 @@ def test_double_copy(mscp, src_prefix, dst_prefix, s1, s2, d1, d2):
     d1.cleanup()
     d2.cleanup()
 
+
+remote_v6_prefix = "[::1]:{}/".format(os.getcwd())
+param_remote_v6_prefix = [
+    ("", remote_v6_prefix), (remote_v6_prefix, "")
+]
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_v6_prefix)
+@pytest.mark.parametrize("s1, s2, d1, d2", param_double_copy)
+def test_double_copy_with_ipv6_notation(mscp, src_prefix, dst_prefix, s1, s2, d1, d2):
+    s1.make()
+    s2.make()
+    run2ok([mscp, "-H", "-vvv",
+            src_prefix + s1.path, src_prefix + s2.path, dst_prefix + "dst"])
+    assert check_same_md5sum(s1, d1)
+    assert check_same_md5sum(s2, d2)
+    s1.cleanup()
+    s2.cleanup()
+    d1.cleanup()
+    d2.cleanup()
+
+
+remote_user_v6_prefix = "{}@[::1]:{}/".format(getpass.getuser(), os.getcwd())
+param_remote_user_v6_prefix = [
+    ("", remote_user_v6_prefix), (remote_user_v6_prefix, "")
+]
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_user_v6_prefix)
+@pytest.mark.parametrize("s1, s2, d1, d2", param_double_copy)
+def test_double_copy_with_user_and_ipv6_notation(mscp, src_prefix, dst_prefix,
+                                                 s1, s2, d1, d2):
+    s1.make()
+    s2.make()
+    run2ok([mscp, "-H", "-vvv",
+            src_prefix + s1.path, src_prefix + s2.path, dst_prefix + "dst"])
+    assert check_same_md5sum(s1, d1)
+    assert check_same_md5sum(s2, d2)
+    s1.cleanup()
+    s2.cleanup()
+    d1.cleanup()
+    d2.cleanup()
+
+
+
 param_dir_copy = [
     ( "src_dir", "dst_dir",
         [ File("src_dir/t1", size = 64),
@@ -143,7 +186,7 @@ param_dir_copy_single = [
 def test_dir_copy_single(mscp, src_prefix, dst_prefix, src_dir, dst_dir, src, dst):
     src.make()
     os.mkdir(dst_dir)
-    run2ok(["mscp", "-H", "-vvv", src_prefix + src_dir, dst_prefix + dst_dir])
+    run2ok([mscp, "-H", "-vvv", src_prefix + src_dir, dst_prefix + dst_dir])
     assert check_same_md5sum(src, dst)
     src.cleanup()
     dst.cleanup()
@@ -160,6 +203,22 @@ def test_override_single_file(mscp, src_prefix, dst_prefix):
     src.cleanup()
     dst.cleanup()
 
+absolute_remote_prefix = "localhost:"
+param_absolute_remote_prefix = [
+    ("", absolute_remote_prefix), (absolute_remote_prefix, "")
+]
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_absolute_remote_prefix)
+def test_copy_file_under_root_to_dir(mscp, src_prefix, dst_prefix):
+    src = File("/mscp-test-src", size = 1024).make()
+    dst = File("/tmp/mscp-test-src")
+
+    run2ok([mscp, "-H", "-vvv", src_prefix + src.path,
+            dst_prefix + os.path.dirname(dst.path)])
+    assert check_same_md5sum(src, dst)
+    src.cleanup()
+    dst.cleanup(preserve_dir = True)
+
+
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 def test_min_chunk(mscp, src_prefix, dst_prefix):
     src = File("src", size = 16 * 1024).make()
@@ -257,6 +316,28 @@ def test_dont_truncate_dst(mscp, src_prefix, dst_prefix):
     assert md5_before == md5_after
     f.cleanup()
 
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+@pytest.mark.parametrize("src, dst", param_single_copy)
+def test_set_port(mscp, src_prefix, dst_prefix, src, dst):
+    src.make()
+    run2ng([mscp, "-H", "-vvv", "-p", 21, src_prefix + src.path, dst_prefix + dst.path])
+    run2ng([mscp, "-H", "-vvv", "-P", 21, src_prefix + src.path, dst_prefix + dst.path])
+    src.cleanup()
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_set_conn_interval(mscp, src_prefix, dst_prefix):
+    srcs = []
+    dsts = []
+    for x in range(500):
+        srcs.append(File("src/file{}".format(x), size = 128).make())
+        dsts.append(File("dst/file{}".format(x)))
+    run2ok([mscp, "-H", "-vvv", "-I", 1, src_prefix + "src", dst_prefix + "dst"])
+
+    for src, dst in zip(srcs, dsts):
+        assert check_same_md5sum(src, dst)
+        src.cleanup()
+        dst.cleanup()
+
 compressions = ["yes", "no", "none"]
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 @pytest.mark.parametrize("compress", compressions)
@@ -268,6 +349,21 @@ def test_compression(mscp, src_prefix, dst_prefix, compress):
     src.cleanup()
     dst.cleanup()
 
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_ccalgo(mscp, src_prefix, dst_prefix):
+    src = File("src", size = 1024 * 1024).make()
+    dst = File("dst").make()
+    if platform.system() == "Darwin":
+        # Darwin does not support TCP_CONGESTION
+        algo = "cubic"
+        run = run2ng
+    elif platform.system() == "Linux":
+        # Linux supports TCP_CONGESTION
+        with open("/proc/sys/net/ipv4/tcp_allowed_congestion_control", "r") as f:
+            algo = f.read().strip().split().pop()
+        run = run2ok
+    run([mscp, "-H", "-vvv", "-g", algo, src_prefix + src.path, dst_prefix + "dst"])
+
 
 testhost = "mscptestlocalhost"
 testhost_prefix = "{}:{}/".format(testhost, os.getcwd()) # use current dir

test/test_python.py

@@ -121,6 +121,7 @@ param_invalid_kwargs = [
     { "cipher": "invalid" },
     { "hmac": "invalid"},
     { "compress": "invalid"},
+    { "ccalgo": "invalid"},
 ]
 
 @pytest.mark.parametrize("kw", param_invalid_kwargs)

test/util.py

@@ -46,8 +46,10 @@ class File():
         with open(self.path, "wb") as f:
             f.write(os.urandom(self.size))
 
-    def cleanup(self):
+    def cleanup(self, preserve_dir = False):
         os.remove(self.path)
+        if preserve_dir:
+            return
         tmp = os.path.dirname(self.path)
         while tmp and not tmp in [".", "/"]:
             if len(os.listdir(tmp)) == 0: