adjust mscp.rst for v0.2.4

This fixes #36, password auth does not occur.

.clang-format

@@ -689,6 +689,8 @@ ForEachMacros:
   - 'xbc_node_for_each_key_value'
   - 'xbc_node_for_each_subkey'
   - 'zorro_for_each_dev'
+  - 'pool_iter_for_each'
+  - 'pool_for_each'
 
 IncludeBlocks: Preserve
 IncludeCategories:

.dockerignore

@@ -0,0 +1 @@
+build

.github/workflows/build-freebsd.yml

@@ -15,13 +15,17 @@ jobs:
       with:
         submodules: true
 
+    - name: apply the patch to libssh
+      run: |
+        git -C libssh fetch --all --tags --prune
+        patch -d libssh -p1 < patch/$(git -C libssh describe).patch
+
     - name: Build in FreeBSD
       uses: vmactions/freebsd-vm@v1
       with:
         prepare: |
           pkg install -y git cmake
         run: |
-          patch -d libssh -p1 < patch/libssh-0.10.6-2-g6f1b1e76.patch
           cmake -B build -DCMAKE_BUILD_TYPE=Release
           cmake --build build
           build/mscp -h

.github/workflows/build-macos.yml

@@ -18,10 +18,15 @@ jobs:
     runs-on: macos-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
 
+    - name: apply the patch to libssh
+      run: |
+        git -C libssh fetch --all --tags --prune
+        patch -d libssh -p1 < patch/$(git -C libssh describe).patch
+
     - name: install build dependency
       run: ./scripts/install-build-deps.sh
 
@@ -29,13 +34,10 @@ jobs:
       id: brew-prefix
       run: echo "HOMEBREW_PREFIX=$(brew --prefix)" >> $GITHUB_OUTPUT
 
-    - name: patch to libssh
-      run: patch -d libssh -p1 < patch/libssh-0.10.6-2-g6f1b1e76.patch
-
     - name: Configure CMake
       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
-      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DOPENSSL_ROOT_DIR=${{steps.brew-prefix.outputs.HOMEBREW_PREFIX}}/opt/openssl@1.1
+      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DOPENSSL_ROOT_DIR=${{steps.brew-prefix.outputs.HOMEBREW_PREFIX}}/opt/openssl@3
 
     - name: Build
       # Build your program with the given configuration

.github/workflows/build-ubuntu.yml

@@ -18,18 +18,20 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
 
+    - name: apply the patch to libssh
+      run: |
+        git -C libssh fetch --all --tags --prune
+        patch -d libssh -p1 < patch/$(git -C libssh describe).patch
+
     - name: install build dependency
       run: |
         sudo apt-get update
         sudo ./scripts/install-build-deps.sh
 
-    - name: patch to libssh
-      run: patch -d libssh -p1 < patch/libssh-0.10.6-2-g6f1b1e76.patch
-
     - name: Configure CMake
       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type

.github/workflows/codeql.yml

@@ -38,21 +38,23 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         submodules: true
 
+    - name: apply the patch to libssh
+      run: |
+        git -C libssh fetch --all --tags --prune
+        patch -d libssh -p1 < patch/$(git -C libssh describe).patch
+
     - name: install build dependency
       run: |
         sudo apt-get update
         sudo ./scripts/install-build-deps.sh
 
-    - name: patch to libssh
-      run: patch -d libssh -p1 < patch/libssh-0.10.6-2-g6f1b1e76.patch
-
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -65,7 +67,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, Go, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -78,6 +80,6 @@ jobs:
     #   ./location_of_script_within_repo/buildscript.sh
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3
       with:
         category: "/language:${{matrix.language}}"

.github/workflows/release.yml

@@ -10,47 +10,17 @@ env:
     BUILD_TYPE: Release
 
 jobs:
-  build-and-release:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          submodules: true
-
-      - name: patch to libssh
-        run: patch -d libssh -p1 < patch/libssh-0.10.6-2-g6f1b1e76.patch
-
-      # TODO: just building docker does not require packages. fix CMakeLists
-      - name: install build dependency
-        run: sudo ./scripts/install-build-deps.sh
-
-      - name: Configure Cmake
-        run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
-
-      - name: Build Containers
-        run: make -C ${{github.workspace}}/build docker-build-all
-
-      - name: Test
-        run: make -C ${{github.workspace}}/build docker-test-all
-
-      - name: Build single binary mscp
-        run: make -C ${{github.workspace}}/build build-single-binary
-
-      - name: Release
-        uses: softprops/action-gh-release@v1
-        with:
-          files: |
-            ${{github.workspace}}/build/mscp.linux.x86_64.static
-
   source-release:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: true
 
-      - name: patch to libssh
-        run: patch -d libssh -p1 < patch/libssh-0.10.6-2-g6f1b1e76.patch
+      - name: apply the patch to libssh
+        run: |
+          git -C libssh fetch --all --tags --prune
+          patch -d libssh -p1 < patch/$(git -C libssh describe).patch
 
       - name: Set variables
         run: |

.github/workflows/test.yml

@@ -12,15 +12,29 @@ env:
 jobs:
   test:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        index: # see DIST_IDS and DIST_VERS lists in CMakeLists.txt
+        - ubuntu-20.04
+        - ubuntu-22.04
+        - ubuntu-24.04
+        - rocky-8.9
+        - rocky-9.3
+        - almalinux-9.3
+        - alpine-3.22
+        - arch-base
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
 
-    - name: patch to libssh
-      run: patch -d libssh -p1 < patch/libssh-0.10.6-2-g6f1b1e76.patch
+    - name: apply the patch to libssh
+      run: |
+        git -C libssh fetch --all --tags --prune
+        patch -d libssh -p1 < patch/$(git -C libssh describe).patch
 
-    # TODO: just building docker does not require libssh. fix CMakeLists
+    # TODO: just building docker images does not require libssh. fix CMakeLists
     - name: install build dependency
       run: |
         sudo apt-get update
@@ -30,7 +44,7 @@ jobs:
       run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
 
     - name: Build Containers
-      run: make -C ${{github.workspace}}/build docker-build-all
+      run: make -C ${{github.workspace}}/build docker-build-${{ matrix.index }}
 
     - name: Run Test
-      run: make -C ${{github.workspace}}/build docker-test-all
+      run: make -C ${{github.workspace}}/build docker-test-${{ matrix.index }}

.gitignore

@@ -3,5 +3,6 @@ html
 compile_commands.json
 CMakeUserPresets.json
 .*.swp
+.cache
 
 include/mscp_version.h

CMakeLists.txt

@@ -51,6 +51,13 @@ if (BUILD_STATIC)
 	endif()
 endif()
 
+option(USE_PODMAN OFF) # use podman instread of docker
+if(USE_PODMAN)
+	message(STATUS "Use podman instead of docker")
+	set(CE podman) # CE means Container Engine
+else()
+	set(CE docker)
+endif()
 
 
 # add libssh static library
@@ -84,11 +91,14 @@ endif()
 
 
 # Symbol check
-check_symbol_exists(strlcat string.h HAVE_STRLCAT)
+check_symbol_exists(htonll	arpa/inet.h	HAVE_HTONLL)
+check_symbol_exists(ntohll	arpa/inet.h	HAVE_NTOHLL)
+check_symbol_exists(strlcat	string.h	HAVE_STRLCAT)
 if (NOT HAVE_STRLCAT)
 	list(APPEND OPENBSD_COMPAT_SRC src/openbsd-compat/strlcat.c)
 endif()
 
+
 # generate config.h in build dir
 configure_file(
 	${mscp_SOURCE_DIR}/include/config.h.in
@@ -98,8 +108,8 @@ list(APPEND MSCP_BUILD_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/include)
 
 # libmscp.a
 set(LIBMSCP_SRC
-	src/mscp.c src/ssh.c src/fileops.c src/path.c src/platform.c
-	src/print.c src/strerrno.c
+	src/mscp.c src/ssh.c src/fileops.c src/path.c src/checkpoint.c
+	src/bwlimit.c src/platform.c src/print.c src/pool.c src/strerrno.c
 	${OPENBSD_COMPAT_SRC})
 add_library(mscp-static STATIC ${LIBMSCP_SRC})
 target_include_directories(mscp-static
@@ -155,42 +165,59 @@ enable_testing()
 # Custom targets to build and test mscp in docker containers.
 # foreach(IN ZIP_LISTS) (cmake >= 3.17) can shorten the following lists.
 # However, ubuntu 20.04 has cmake 3.16.3. So this is a roundabout trick.
-list(APPEND DIST_NAMES	ubuntu	ubuntu	rocky	rocky	almalinux	alpine)
-list(APPEND DIST_VERS	20.04	22.04	8.9	9.3	9.3		3.19)
-list(APPEND DIST_PKGS	deb	deb	rpm	rpm	rpm		static)
+#
+# When edit DIST_IDS and DIST_VERS, also edit .github/workflows/test.yaml
+list(APPEND DIST_IDS  ubuntu ubuntu ubuntu rocky rocky almalinux alpine arch)
+list(APPEND DIST_VERS  20.04  22.04  24.04   8.9   9.3       9.3   3.22 base)
 
-list(LENGTH DIST_NAMES _DIST_LISTLEN)
+list(LENGTH DIST_IDS _DIST_LISTLEN)
 math(EXPR DIST_LISTLEN "${_DIST_LISTLEN} - 1")
 
 foreach(x RANGE ${DIST_LISTLEN})
-	list(GET DIST_NAMES	${x} DIST_NAME)
+	list(GET DIST_IDS	${x} DIST_ID)
 	list(GET DIST_VERS	${x} DIST_VER)
-	list(GET DIST_PKGS	${x} DIST_PKG)
 
-	set(DOCKER_IMAGE mscp-${DIST_NAME}:${DIST_VER})
-	set(DOCKER_INDEX ${DIST_NAME}-${DIST_VER})
-	set(PKG_FILE_NAME
-		mscp_${DIST_NAME}-${DIST_VER}-${ARCH}.${DIST_PKG})
+	set(DOCKER_IMAGE mscp-${DIST_ID}:${DIST_VER})
+	set(DOCKER_INDEX ${DIST_ID}-${DIST_VER})
+	execute_process(
+		COMMAND ${CMAKE_SOURCE_DIR}/scripts/install-build-deps.sh
+		--dont-install --platform Linux-${DIST_ID}
+		OUTPUT_VARIABLE REQUIREDPKGS
+		OUTPUT_STRIP_TRAILING_WHITESPACE)
 
 	add_custom_target(docker-build-${DOCKER_INDEX}
 		COMMENT "Build mscp in ${DOCKER_IMAGE} container"
 		WORKING_DIRECTORY ${mscp_SOURCE_DIR}
 		COMMAND
-		docker build -t ${DOCKER_IMAGE} -f Dockerfile/${DOCKER_INDEX}.Dockerfile .)
+		${CE} build --build-arg REQUIREDPKGS=${REQUIREDPKGS}
+		-t ${DOCKER_IMAGE} -f Dockerfile/${DOCKER_INDEX}.Dockerfile .)
 
 	add_custom_target(docker-build-${DOCKER_INDEX}-no-cache
 		COMMENT "Build mscp in ${DOCKER_IMAGE} container"
 		WORKING_DIRECTORY ${mscp_SOURCE_DIR}
 		COMMAND
-		docker build --no-cache -t ${DOCKER_IMAGE} -f Dockerfile/${DOCKER_INDEX}.Dockerfile .)
+		${CE} build --build-arg REQUIREDPKGS=${REQUIREDPKGS} --no-cache
+		-t ${DOCKER_IMAGE} -f Dockerfile/${DOCKER_INDEX}.Dockerfile .)
 
 	add_custom_target(docker-test-${DOCKER_INDEX}
 		COMMENT "Test mscp in ${DOCKER_IMAGE} container"
 		WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
 		COMMAND
-		docker run --init --rm --sysctl net.ipv6.conf.all.disable_ipv6=0
+		${CE} run --init --rm --privileged
+		--sysctl net.ipv6.conf.all.disable_ipv6=0
+		--add-host=ip6-localhost:::1
 		${DOCKER_IMAGE} /mscp/scripts/test-in-container.sh)
 
+	add_custom_target(docker-run-${DOCKER_INDEX}
+		COMMENT "Start ${DOCKER_IMAGE} container"
+		WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+		COMMAND
+		${CE} run --init --rm --privileged
+		--sysctl net.ipv6.conf.all.disable_ipv6=0
+		--add-host=ip6-localhost:::1
+		-it
+		${DOCKER_IMAGE} /mscp/scripts/test-in-container.sh bash)
+
 	list(APPEND DOCKER_BUILDS		docker-build-${DOCKER_INDEX})
 	list(APPEND DOCKER_BUILDS_NO_CACHE	docker-build-${DOCKER_INDEX}-no-cache)
 	list(APPEND DOCKER_TESTS		docker-test-${DOCKER_INDEX})
@@ -204,14 +231,21 @@ add_custom_target(docker-test-all		DEPENDS ${DOCKER_TESTS})
 ### debuild-related definitions
 
 set(DEBBUILDCONTAINER mscp-build-deb)
+execute_process(
+	COMMAND ${CMAKE_SOURCE_DIR}/scripts/install-build-deps.sh
+	--dont-install --platform Linux-ubuntu
+	OUTPUT_VARIABLE REQUIREDPKGS_DEB
+	OUTPUT_STRIP_TRAILING_WHITESPACE)
+
 add_custom_target(build-deb
 	COMMENT "build mscp deb files inside a container"
 	WORKING_DIRECTORY ${mscp_SOURCE_DIR}
 	BYPRODUCTS ${CMAKE_BINARY_DIR}/debbuild
 	COMMAND
-	docker build -t ${DEBBUILDCONTAINER} -f Dockerfile/build-deb.Dockerfile .
+	${CE} build --build-arg REQUIREDPKGS=${REQUIREDPKGS_DEB}
+	-t ${DEBBUILDCONTAINER} -f Dockerfile/build-deb.Dockerfile .
 	COMMAND
-	docker run --rm -v ${CMAKE_BINARY_DIR}:/out ${DEBBUILDCONTAINER}
+	${CE} run --rm -v ${CMAKE_BINARY_DIR}:/out ${DEBBUILDCONTAINER}
 	cp -r /debbuild /out/)
 
 
@@ -222,23 +256,29 @@ configure_file(
 	${mscp_SOURCE_DIR}/rpm/mscp.spec.in
 	${mscp_SOURCE_DIR}/rpm/mscp.spec
 	@ONLY)
-configure_file(
-	${mscp_SOURCE_DIR}/Dockerfile/build-srpm.Dockerfile.in
-	${mscp_SOURCE_DIR}/Dockerfile/build-srpm.Dockerfile
-	@ONLY)
+#configure_file(
+#	${mscp_SOURCE_DIR}/Dockerfile/build-srpm.Dockerfile.in
+#	${mscp_SOURCE_DIR}/Dockerfile/build-srpm.Dockerfile
+#	@ONLY)
 
 # Custom target to build mscp as a src.rpm in docker.
 set(RPMBUILDCONTAINER mscp-build-srpm)
-set(SRPMFILE mscp-${MSCP_VERSION}-1.el9.src.rpm)
+execute_process(
+	COMMAND ${CMAKE_SOURCE_DIR}/scripts/install-build-deps.sh
+	--dont-install --platform Linux-rocky
+	OUTPUT_VARIABLE REQUIREDPKGS_RPM
+	OUTPUT_STRIP_TRAILING_WHITESPACE)
+
 add_custom_target(build-srpm
 	COMMENT "Build mscp src.rpm inside a container"
 	WORKING_DIRECTORY ${mscp_SOURCE_DIR}
-	BYPRODUCTS ${CMAKE_BINARY_DIR}/${SRPMFILE}
 	COMMAND
-	docker build -t ${RPMBUILDCONTAINER} -f Dockerfile/build-srpm.Dockerfile .
+	${CE} build --build-arg REQUIREDPKGS=${REQUIREDPKGS_RPM}
+	--build-arg MSCP_VERSION=${MSCP_VERSION}
+	-t ${RPMBUILDCONTAINER} -f Dockerfile/build-srpm.Dockerfile .
 	COMMAND
-	docker run --rm -v ${CMAKE_BINARY_DIR}:/out ${RPMBUILDCONTAINER}
-	cp /root/rpmbuild/SRPMS/${SRPMFILE} /out/)
+	${CE} run --rm -v ${CMAKE_BINARY_DIR}:/out ${RPMBUILDCONTAINER}
+	bash -c "cp /root/rpmbuild/SRPMS/mscp-*.src.rpm /out/")
 
 ### single-binary-build-related definitions
 
@@ -248,9 +288,9 @@ add_custom_target(build-single-binary
 	COMMENT "Build mscp as a single binary in alpine conatiner"
 	WORKING_DIRECTORY ${mscp_SOURCE_DIR}
 	BYPRODUCTS ${CMAKE_BINARY_DIR}/${SINGLEBINARYFILE}
-	DEPENDS docker-build-alpine-3.19
+	DEPENDS docker-build-alpine-3.22
 	COMMAND
-	docker run --rm -v ${CMAKE_BINARY_DIR}:/out mscp-alpine:3.19
+	${CE} run --rm -v ${CMAKE_BINARY_DIR}:/out mscp-alpine:3.22
 	cp /mscp/build/mscp /out/${SINGLEBINARYFILE})
 
 

Dockerfile/.gitignore

@@ -1,3 +0,0 @@
-
-# generated by cmake
-build-srpm.Dockerfile

Dockerfile/README.md

@@ -1,19 +1,7 @@
 
 Dockerfiles for building and testing mscp.
 
-Build container:
-
-```
-docker build -t mscp-DIST:VER -f docker/DIST-VER.Dockerfile .
-```
-
-Run test:
-
-```
-docker run --init --rm mscp-DST:VER /mscp/scripts/test-in-container.sh
-```
-
-Custom targets to build and test mscp in the containers are provided
-via `cmake`.  See `make docker-*` targets. `make docker-build-all`
-builds all container images, and `make docker-test-all` runs the test
-in all container images.
+cmake provides custom targets to build and test mscp in the containers
+See `make docker-*` targets. `make docker-build-all` builds all
+container images, and `make docker-test-all` runs the test in all
+container images.

Dockerfile/almalinux-9.3.Dockerfile

@@ -1,8 +1,11 @@
 FROM almalinux:9.3
 
+ARG REQUIREDPKGS
+
 # install pytest, sshd for test, and rpm-build
 RUN set -ex && yum -y install \
-	python3 python3-pip python3-devel openssh openssh-server openssh-clients rpm-build
+	${REQUIREDPKGS} python3 python3-pip python3-devel \
+	openssh openssh-server openssh-clients rpm-build
 
 RUN python3 -m pip install pytest
 
@@ -28,9 +31,6 @@ ARG mscpdir="/mscp"
 
 COPY . ${mscpdir}
 
-# install build dependency
-RUN ${mscpdir}/scripts/install-build-deps.sh
-
 # build
 RUN cd ${mscpdir}			\
         && rm -rf build			\

Dockerfile/alpine-3.22.Dockerfile

@@ -1,10 +1,15 @@
-FROM alpine:3.19
+FROM alpine:3.22
+
+# do not use REQUIREDPKGS build argument because
+# this Dockerfile compiles mscp with conan,so we do not need
+# libssl-dev and zlib-dev
 
 # Build mscp with conan to create single binary mscp
 
 RUN apk add --no-cache \
-	gcc make cmake python3 py3-pip perl linux-headers libc-dev	\
-	openssh bash python3-dev py3-pytest g++
+	gcc make cmake libc-dev \
+	linux-headers openssh bash perl \
+	python3 py3-pip python3-dev py3-pytest g++
 
 RUN pip3 install --break-system-packages conan
 
@@ -14,6 +19,10 @@ RUN ssh-keygen -A \
 	&& ssh-keygen -f /root/.ssh/id_rsa -N ""                \
 	&& cat /root/.ssh/id_rsa.pub > /root/.ssh/authorized_keys
 
+# disable PerSourcePenaltie, which would distrub test:
+# https://undeadly.org/cgi?action=article;sid=20240607042157
+RUN echo "PerSourcePenalties=no" > /etc/ssh/sshd_config.d/90-mscp-test.conf
+
 # create test user
 RUN addgroup -S test \
 	&& adduser -S test -G test \

Dockerfile/arch-base.Dockerfile

@@ -0,0 +1,36 @@
+FROM archlinux:base
+
+ARG REQUIREDPKGS
+
+# install pyest and openssh for test
+RUN set -ex && pacman -Syy && pacman --noconfirm -S ${REQUIREDPKGS} openssh python-pytest
+
+RUN mkdir /var/run/sshd        \
+        && ssh-keygen -A        \
+        && ssh-keygen -f /root/.ssh/id_rsa -N ""                \
+        && cat /root/.ssh/id_rsa.pub > /root/.ssh/authorized_keys
+
+# disable PerSourcePenaltie, which would distrub test:
+# https://undeadly.org/cgi?action=article;sid=20240607042157
+RUN echo "PerSourcePenalties=no" > /etc/ssh/sshd_config.d/90-mscp-test.conf
+
+# create test user
+RUN useradd -m -d /home/test test       \
+        && echo "test:userpassword" | chpasswd \
+        && mkdir -p /home/test/.ssh     \
+        && ssh-keygen -f /home/test/.ssh/id_rsa_test -N "keypassphrase" \
+        && cat /home/test/.ssh/id_rsa_test.pub >> /home/test/.ssh/authorized_keys \
+        && chown -R test:test /home/test \
+        && chown -R test:test /home/test/.ssh
+
+ARG mscpdir="/mscp"
+
+COPY . ${mscpdir}
+
+# build
+RUN cd ${mscpdir}                       \
+        && rm -rf build                 \
+        && cmake -B build               \
+        && cd ${mscpdir}/build          \
+        && make -j 2                    \
+        && make install

Dockerfile/build-deb.Dockerfile

@@ -1,16 +1,16 @@
 FROM ubuntu:22.04
 
+ARG REQUIREDPKGS
+
 ARG DEBIAN_FRONTEND=noninteractive
 RUN set -ex && apt-get update && apt-get install -y --no-install-recommends \
-	ca-certificates build-essential devscripts debhelper gcc make cmake
+	${REQUIREDPKGS} ca-certificates \
+	build-essential devscripts debhelper gcc make cmake
 
 ARG mscpdir="/debbuild/mscp"
 
 COPY . ${mscpdir}
 
-# install build dependency
-RUN ${mscpdir}/scripts/install-build-deps.sh
-
 # build
 RUN cd ${mscpdir} 	\
 	&& debuild -us -uc -S \

Dockerfile/build-srpm.Dockerfile

@@ -1,16 +1,16 @@
 FROM rockylinux:9
 
-# install pytest, sshd for test, and rpm-build
-RUN set -ex && yum -y install rpm-build rpmdevtools
+ARG REQUIREDPKGS
+ARG MSCP_VERSION
 
-ARG mscpdir="/mscp-@MSCP_VERSION@"
-ARG mscptgz="mscp-@MSCP_VERSION@.tar.gz"
+# install pytest, sshd for test, and rpm-build
+RUN set -ex && yum -y install ${REQUIREDPKGS} rpm-build rpmdevtools
+
+ARG mscpdir="/mscp-${MSCP_VERSION}"
+ARG mscptgz="mscp-${MSCP_VERSION}.tar.gz"
 
 COPY . ${mscpdir}
 
-# install build dependency
-RUN ${mscpdir}/scripts/install-build-deps.sh
-
 # prepare rpmbuild
 RUN rpmdev-setuptree \
 	&& rm -rf ${mscpdir}/build	\

Dockerfile/rocky-8.9.Dockerfile

@@ -1,8 +1,12 @@
 FROM rockylinux:8.9
 
+ARG REQUIREDPKGS
+
 # install pytest, sshd for test, and rpm-build
 RUN set -ex && yum -y install \
-	python3 python3-pip python3-devel openssh openssh-server openssh-clients rpm-build
+	${REQUIREDPKGS}	\
+	python3 python3-pip python3-devel \
+	openssh openssh-server openssh-clients rpm-build
 
 RUN python3 -m pip install pytest
 
@@ -29,9 +33,6 @@ ARG mscpdir="/mscp"
 
 COPY . ${mscpdir}
 
-# install build dependency
-RUN ${mscpdir}/scripts/install-build-deps.sh
-
 # build
 RUN cd ${mscpdir}			\
         && rm -rf build			\

Dockerfile/rocky-9.3.Dockerfile

@@ -1,8 +1,12 @@
 FROM rockylinux:9.3
 
+ARG REQUIREDPKGS
+
 # install pytest, sshd for test, and rpm-build
 RUN set -ex && yum -y install \
-	python3 python3-pip python3-devel openssh openssh-server openssh-clients rpm-build
+	${REQUIREDPKGS}	\
+	python3 python3-pip python3-devel \
+	openssh openssh-server openssh-clients rpm-build
 
 RUN python3 -m pip install pytest
 
@@ -28,9 +32,6 @@ ARG mscpdir="/mscp"
 
 COPY . ${mscpdir}
 
-# install build dependency
-RUN ${mscpdir}/scripts/install-build-deps.sh
-
 # build
 RUN cd ${mscpdir}			\
         && rm -rf build			\

Dockerfile/ubuntu-20.04.Dockerfile

@@ -1,16 +1,13 @@
 FROM ubuntu:20.04
 
+ARG REQUIREDPKGS
+
 ARG DEBIAN_FRONTEND=noninteractive
 RUN set -ex && apt-get update && apt-get install -y --no-install-recommends \
-	ca-certificates
-
-# install pytest, and sshd for test
-RUN apt-get install -y --no-install-recommends  \
-        python3 python3-pip python3-dev openssh-server
+	${REQUIREDPKGS} ca-certificates python3 python3-pip python3-dev openssh-server
 
 RUN python3 -m pip install pytest
 
-
 # preparation for sshd
 RUN mkdir /var/run/sshd        \
 	&& ssh-keygen -A	\
@@ -31,10 +28,6 @@ ARG mscpdir="/mscp"
 
 COPY . ${mscpdir}
 
-# install build dependency
-RUN ${mscpdir}/scripts/install-build-deps.sh
-
-
 # build
 RUN cd ${mscpdir}			\
 	&& rm -rf build			\

Dockerfile/ubuntu-22.04.Dockerfile

@@ -1,16 +1,13 @@
 FROM ubuntu:22.04
 
+ARG REQUIREDPKGS
+
 ARG DEBIAN_FRONTEND=noninteractive
 RUN set -ex && apt-get update && apt-get install -y --no-install-recommends \
-	ca-certificates
-
-# install pytest, and sshd for test
-RUN apt-get install -y --no-install-recommends  \
-        python3 python3-pip python3-dev openssh-server
+	${REQUIREDPKGS} ca-certificates python3 python3-pip python3-dev openssh-server
 
 RUN python3 -m pip install pytest
 
-
 # preparation for sshd
 RUN mkdir /var/run/sshd        \
 	&& ssh-keygen -A	\
@@ -31,10 +28,6 @@ ARG mscpdir="/mscp"
 
 COPY . ${mscpdir}
 
-# install build dependency
-RUN ${mscpdir}/scripts/install-build-deps.sh
-
-
 # build
 RUN cd ${mscpdir}			\
 	&& rm -rf build			\

Dockerfile/ubuntu-24.04.Dockerfile

@@ -0,0 +1,37 @@
+FROM ubuntu:24.04
+
+ARG REQUIREDPKGS
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN set -ex && apt-get update && apt-get install -y --no-install-recommends \
+	${REQUIREDPKGS} ca-certificates openssh-server vim-tiny \
+	python3 python3-pip python3-dev python3-pytest
+
+
+# preparation for sshd
+RUN mkdir /var/run/sshd        \
+	&& ssh-keygen -A	\
+	&& ssh-keygen -f /root/.ssh/id_rsa -N ""                \
+	&& cat /root/.ssh/id_rsa.pub > /root/.ssh/authorized_keys
+
+# create test user
+RUN useradd -m -d /home/test test	\
+	&& echo "test:userpassword" | chpasswd \
+	&& mkdir -p /home/test/.ssh	\
+	&& ssh-keygen -f /home/test/.ssh/id_rsa_test -N "keypassphrase"	\
+	&& cat /home/test/.ssh/id_rsa_test.pub >> /home/test/.ssh/authorized_keys \
+	&& chown -R test:test /home/test \
+	&& chown -R test:test /home/test/.ssh
+
+
+ARG mscpdir="/mscp"
+
+COPY . ${mscpdir}
+
+# build
+RUN cd ${mscpdir}			\
+	&& rm -rf build			\
+	&& cmake -B build		\
+	&& cd ${mscpdir}/build		\
+	&& make	-j 2			\
+	&& make install

README.md

@@ -15,7 +15,7 @@ transfer time for a lot of/large files over networks.
 You can use `mscp` like `scp`, for example:
 
 ```shell-session
-$ mscp user@example.com:srcfile /tmp/dstfile
+$ mscp srcfile user@example.com:dstfile
 ```
 
 Remote hosts only need to run standard `sshd` supporting the SFTP
@@ -27,10 +27,11 @@ https://github.com/upa/mscp/assets/184632/19230f57-be7f-4ef0-98dd-cb4c460f570d
 
 --------------------------------------------------------------------
 
-Differences from `scp` on usage:
+Major differences from `scp` on usage:
 
 - Remote-to-remote copy is not supported.
 - `-r` option is not needed to transfer directories.
+- Checkpointing for resuming failed transfer is supported.
 - and any other differences I have not implemented and noticed.
 
 Paper:
@@ -41,7 +42,11 @@ Paper:
 - macOS
 
 ```console
+# Homebrew
 brew install upa/tap/mscp
+
+# MacPorts
+sudo port install mscp
 ```
 
 - Ubuntu
@@ -56,12 +61,6 @@ sudo dnf copr enable upaaa/mscp
 sudo dnf install mscp
 ```
 
-- Single binary `mscp` for x86_64 (not optimal performance)
-```console
-wget https://github.com/upa/mscp/releases/latest/download/mscp.linux.x86.static -O /usr/local/bin/mscp
-chmod 755 /usr/local/bin/mscp
-```
-
 
 ## Build
 
@@ -81,7 +80,7 @@ cd mscp
 
 # prepare patched libssh
 git submodule update --init
-patch -d libssh -p1 < patch/$(git --git-dir=./libssh/.git describe).patch
+patch -d libssh -p1 < patch/$(git -C libssh describe).patch
 
 # install build dependency
 bash ./scripts/install-build-deps.sh
@@ -91,7 +90,7 @@ mkdir build && cd build
 cmake ..
 
 # in macOS, you may need OPENSSL_ROOT_DIR for cmake:
-# cmake .. -DOPENSSL_ROOT_DIR=$(brew --prefix)/opt/openssl@1.1
+# cmake .. -DOPENSSL_ROOT_DIR=$(brew --prefix)/opt/openssl@3
 
 # build
 make
@@ -101,10 +100,10 @@ make install
 ```
 
 Source tar balls (`mscp-X.X.X.tar.gz`, not `Source code`) in
-[Releases page](https://github.com/upa/mscp/releases) contains the patched version
+[Releases page](https://github.com/upa/mscp/releases) contain the patched version
 of libssh. So you can start from cmake with it.
 
 
 ## Documentation
 
-[manpage](/doc/mscp.rst) is available.
+[manpage](/doc/mscp.rst) is available.

VERSION

@@ -1 +1 @@
-0.1.4
+0.2.4

debian/changelog

@@ -1,4 +1,68 @@
-mscp (0.1.4) UNRELEASED; urgency=medium
+mscp (0.2.4) UNRELEASED; urgency=medium
+
+  * fix password auth does not work (issue #36)
+  * add Debian and Devuan to the install dep script (pr #38)
+  * fallback to default terminal size on no tty (pr #39)
+
+ -- Ryo Nakamura <upa@haeena.net>  Sat, 08 Nov 2025 19:23:07 +0900
+
+mscp (0.2.3) unstable; urgency=medium
+
+  * bump libssh version to 0.11.2 (#25)
+  * adopt new AIO read API of libssh
+  * fix path handling when remote dst path has suffix '/' (#24)
+  * fix remote path handling including '~' (partially)
+  * try pubkey auth first to avoid PerSourcePenalties
+  * remove the single-binary version of mscp from release
+
+ -- Ryo Nakamura <upa@haeena.net>  Tue, 12 Aug 2025 18:11:47 +0900
+
+mscp (0.2.2) unstable; urgency=medium
+
+  * bump cmake_minimum_version on libssh (#32)
+  * fix quiet mode (#30)
+  * use openssl@3 on macOS (#29)
+  * add archlinux support (#28)
+
+ -- Ryo Nakamura <upa@haeena.net>  Wed, 16 Apr 2025 17:01:17 +0900
+
+mscp (0.2.1) unstable; urgency=medium
+
+  * fix broken checkpoint files when copying multiple files (#16)
+  * fix broken password inputs for resume transfers (#17 and #18)
+  * add support [kKmMgG] units for -s, -S, and -b options (#20)
+  * change the default min chunk size to 16MB (#20)
+  * change the default max chunk size to filesize / (nr_conn * 4) (#20)
+  * -s and -S accept integers other than multiples of page sizes (#20)
+  * help now shows available ciphers and HMACs (#20)
+
+ -- Ryo Nakamura <upa@haeena.net>  Sat, 11 May 2024 14:49:52 +0900
+
+mscp (0.2.0) unstable; urgency=medium
+
+  * add -J DESTINATION option for ProxyJump (#15)
+  * add -o SSH_OPTION option
+  * add -L LIMIT_BITRATE option (#14)
+  * add keyboard interactive authentication support. Also, we have manually
+    tested that mscp works with cisco DUO MFA (#2)
+  * remove -H disable host key check option (ssh_config or -o option can do
+    this instead)
+  * fix copying files of permission r--r--r--
+  * update github actions: checkout v3 to v4, and codeql from v2 to v3
+  * add ubuntu 24.04 test
+
+ -- Ryo Nakamura <upa@haeena.net>  Mon, 15 Apr 2024 00:05:20 +0900
+
+mscp (0.1.5) unstable; urgency=medium
+
+  * add support for resuming failed transfer (#5 and #10)
+  * remove the list structure derived from the linux kernel and refactoring
+    for this change.
+  * add and fix test cases (changing port number and number of connections)
+
+ -- Ryo Nakamura <upa@haeena.net>  Thu, 14 Mar 2024 12:51:23 +0900
+
+mscp (0.1.4) unstable; urgency=medium
 
   * add a test for builds on FreeBSD
   * updat container images for tests

doc/RELEASE.md

@@ -22,10 +22,12 @@ mscp_0.1.4_source.build  mscp_0.1.4_source.changes
 
 ### To publush mscp in launchpad PPA:
 
-1. write changes in `debian/changelog` at main branch (the date command needed here is `date -R`)
+1. write changes in `debian/changelog` at main branch (the date
+   command needed here is `date -R`)
 2. switch to `ppa-focal` or `ppa-jammy` branch
-3. reblase to the `main` branch and modify `debian/changes`:
-   * change `UNRELEASED` to the release name (`focal` or `jammy`).
+3. rebase to the `main` branch and modify `debian/changes`:
+   * change `mscp (X.X.X) UNRELEASED;` to `mscp (X.X.X-1~RELEASENAME) RELEASENAME;`
+	 where `RELEASENAME` is `focal` or `jammy`.
 4. run `make build-deb` at the build directory and `cd debbuild`
 5. sign the files with `debsign -k [GPGKEYID] mscp_X.X.X~X_source.changes`
 5. upload the files with `dput ppa:upaa/mscp mscp_X.X.X~X_source.changes`
@@ -52,7 +54,8 @@ mscp-0.1.3-1.el9.src.rpm
 
 ### To publish mscp in COPR:
 
-1. update `changelog` in `rpm/mscp.spec` (the date command needed here is `date "+%a %b %d %Y"`)
+1. update `rpm/mscp.spec.in`, the `changelog` section (the date
+   command needed here is `date "+%a %b %d %Y"`)
 2. run `make build-srpm`
 3. download `mscp-X.X.X-1.yyy.src.rpm`
 4. upload the src.rpm to Build page at COPR.

doc/mscp.1.in

@@ -6,7 +6,7 @@ mscp \- copy files over multiple SSH connections
 .SH SYNOPSIS
 
 .B mscp
-.RB [ \-46vqDpHdNh ]
+.RB [ \-46vqDpdNh ]
 [\c
 .BI \-n \ NR_CONNECTIONS\c
 ]
@@ -20,6 +20,12 @@ mscp \- copy files over multiple SSH connections
 .BI \-I \ INTERVAL\c
 ]
 [\c
+.BI \-W \ CHECKPOINT\c
+]
+[\c
+.BI \-R \ CHECKPOINT\c
+]
+[\c
 .BI \-s \ MIN_CHUNK_SIZE\c
 ]
 [\c
@@ -32,18 +38,27 @@ mscp \- copy files over multiple SSH connections
 .BI \-b \ BUF_SIZE\c
 ]
 [\c
+.BI \-L \ LIMIT_BITRATE\c
+]
+[\c
 .BI \-l \ LOGIN_NAME\c
 ]
 [\c
 .BI \-P \ PORT\c
 ]
 [\c
-.BI \-F \ CONFIG\c
+.BI \-F \ SSH_CONFIG\c
+]
+[\c
+.BI \-o \ SSH_OPTION\c
 ]
 [\c
 .BI \-i \ IDENTITY\c
 ]
 [\c
+.BI \-J \ DESTINATION\c
+]
+[\c
 .BI \-c \ CIPHER\c
 ]
 [\c
@@ -69,7 +84,7 @@ of/large files over networks.
 .PP
 The usage of
 .B mscp
-imitates the
+follows the
 .B scp
 command of
 .I OpenSSH,
@@ -108,12 +123,15 @@ formula: floor(log(nr_cores)*2)+1.
 
 .TP
 .B \-m \fICOREMASK\fR
-Configures CPU cores to be used by the hexadecimal bitmask.  All CPU
-cores are used by default.
+Configures CPU cores to be used by the hexadecimal bitmask. For
+example, -m 0x25 pins threads onto CPU cores 0, 2, and 5. The default
+value is not specified: all CPU cores are used and no threads are
+pinned to any cores.
 
 .TP
 .B \-u \fIMAX_STARTUPS\fR
-Specifies the number of concurrent outgoing SSH connections.
+Specifies the number of concurrent unauthenticated SSH connection
+attempts.
 .B sshd
 limits the number of simultaneous SSH connection attempts by
 .I MaxStartups
@@ -131,16 +149,60 @@ single source IP address for a short period as a brute force attack.
 This option inserts intervals between the attempts to avoid being
 determined as an attack. The default value is 0.
 
+.TP
+.B \-W \fICHECKPOINT\fR
+Specifies a checkpoint file to save the state of a failed
+transfer. When transferring fails due to, for example, connection
+disruption or user interrupt,
+.B mscp
+writes the information about the remaining files and chunks to the
+specified checkpoint file.
+.B \-W
+option with
+.B \-D
+(dry-run mode) only writes a checkpoint file and exits.
+
+
+.TP
+.B \-R \fICHECKPOINT\fR
+Specifies a checkpoint file to resume a transfer. When a checkpoint
+file is passed,
+.B mscp
+reads the checkpoint to load a remote host, copy direction, and files
+and their chunks to be transferred. Namely,
+.B mscp
+can resume a past failed transfer from the checkpoint. Resuming with a
+checkpoint does not require
+.I source ... target
+arguments. Other SSH connection options, such as port number and
+config file, should be specified as with the failed run. In addition,
+checkpoint files have file paths as relative paths. Thus, you must run
+.B mscp
+in the same working directory as the failed run. You can see the
+contents of a checkpoint file with the
+.B mscp \-vv \-D \-R CHECKPOINT
+command (Dry-run mode).  Note that the checkpoint file is not
+automatically removed after the resumed transfer ends
+successfully. Users should check the return value of
+.B mscp
+and remove the checkpoint if it returns 0.
+
+
 .TP
 .B \-s \fIMIN_CHUNK_SIZE\fR
 Specifies the minimum chunk size.
 .B mscp
-divides a file into chunks and copies the chunks in parallel.
+divides a single file into chunks and copies the chunks in
+parallel. The default value is 16M bytes.
 
 .TP
 .B \-S \fIMAX_CHUNK_SIZE\fR
 Specifies the maximum chunk size. The default is file size divided by
-the number of connections.
+the number of connections and devided by 4. If the calculated value
+is smarller than the
+.B MIN_CHUNK_SIZE
+value,
+MIN_CHUNK_SIZE is used.
 
 .TP
 .B \-a \fINR_AHEAD\fR
@@ -153,6 +215,11 @@ Specifies the buffer size for I/O and transfer over SFTP. The default
 value is 16384. Note that the SSH specification restricts buffer size
 delivered over SSH. Changing this value is not recommended at present.
 
+.TP
+.B \-L \fILIMIT_BITRATE\fR
+Limits the bitrate, specified with k (K), m (M), and g (G), e.g., 100m
+indicates 100 Mbps.
+
 .TP
 .B \-4
 Uses IPv4 addresses only.
@@ -172,10 +239,9 @@ Quiet mode: turns off all outputs.
 .TP
 .B \-D
 Dry-run mode: it scans source files to be copied, calculates chunks,
-and resolves destination file paths. Dry-run mode with
+resolves destination file paths, and exits. Dry-run mode with
 .B -vv
-option enables confirming files to be copied and their destination
-paths.
+option can confirm files to be copied and their destination paths.
 
 .TP
 .B \-r
@@ -192,22 +258,46 @@ Specifies the username to log in on the remote machine as with
 .TP
 .B \-P \fIPORT\fR
 Specifies the port number to connect to on the remote machine as with
-ssh(1) and scp(1).
+.I scp(1).
 
 .TP
-.B \-F \fICONFIG\fR
+.B \-F \fISSH_CONFIG\fR
 Specifies an alternative per-user ssh configuration file. Note that
 acceptable options in the configuration file are what
 .I libssh
 supports.
 
+.TP
+.B \-o \fISSH_OPTION\fR
+Specifies ssh options in the format used in ssh_config. Note that
+acceptable options are what
+.I libssh
+supports.
+
 .TP
 .B \-i \fIIDENTITY\fR
 Specifies the identity file for public key authentication.
 
+.TP
+.B \-J \fIDESTINATION\fR
+A shortcut to define a
+.B ProxyJump
+configuration directive. Each SFTP session of
+.B mscp
+connects to the target host by first making an
+.B ssh
+connection to the jump host described by
+.I destination.
+
+
+
 .TP
 .B \-c \fICIPHER\fR
 Selects the cipher to use for encrypting the data transfer. See
+.B mscp -h
+or
+.B Ciphers
+in
 .UR https://\:www\:.libssh\:.org/\:features/
 libssh features
 .UE .
@@ -215,6 +305,10 @@ libssh features
 .TP
 .B \-M \fIHMAC\fR
 Specifies MAC hash algorithms. See
+.B mscp -h
+or
+.B MAC hashes
+in
 .UR https://\:www\:.libssh\:.org/\:features/
 libssh features
 .UE .
@@ -230,16 +324,15 @@ libssh features
 .TP
 .B \-g \fICONGESTION\fR
 Specifies the TCP congestion control algorithm to use (Linux only).
+See
+.B sysctl net.ipv4.tcp_allowed_congestion_control
+for available values.
 
 .TP
 .B \-p
 Preserves modification times and access times (file mode bits are
 preserved by default).
 
-.TP
-.B \-H
-Disables hostkey checking.
-
 .TP
 .B \-d
 Increments the ssh debug output level.
@@ -306,6 +399,24 @@ Copy a local file and a directory to /tmp at a remote host:
     $ mscp ~/src-file dir1 10.0.0.1:/tmp
 .fi
 
+.PP
+Save a checkpoint if transfer fails:
+
+.nf
+    $ mscp -W mscp.checkpoint many-large-files 10.0.0.1:dst/
+.fi
+
+.PP
+Check the remaining files and chunks, and resume the failed transfer:
+
+.nf
+    # Dump the content of a checkpoint and exit (dry-run mode)
+    $ mscp -vv -D -R mscp.checkpoint
+
+    # resume transferring from the checkpoint
+    $ mscp -R mscp.checkpoint
+.fi
+
 .PP
 In a long fat network, following options might improve performance:
 
@@ -324,7 +435,6 @@ will be faster than the default chacha20-poly1305 cipher, particularly
 on hosts that support AES-NI.
 
 
-
 .SH "SEE ALSO"
 .BR scp (1),
 .BR ssh (1),
@@ -342,9 +452,9 @@ DOI
 .UE .
 
 
-.SH CONTACT INFROMATION
+.SH CONTACT INFORMATION
 .PP
-For pathces, bug reports, or feature requests, please open an issue on
+For patches, bug reports, or feature requests, please open an issue on
 .UR https://\:github\:.com/\:upa/\:mscp
 GitHub
 .UE .

doc/mscp.rst

@@ -2,7 +2,7 @@
 MSCP
 ====
 
-:Date:   v0.1.3-23-ga9c59f7
+:Date: v0.2.4
 
 NAME
 ====
@@ -12,13 +12,14 @@ mscp - copy files over multiple SSH connections
 SYNOPSIS
 ========
 
-**mscp** [**-46vqDpHdNh**] [ **-n**\ *NR_CONNECTIONS* ] [
-**-m**\ *COREMASK* ] [ **-u**\ *MAX_STARTUPS* ] [ **-I**\ *INTERVAL* ] [
-**-s**\ *MIN_CHUNK_SIZE* ] [ **-S**\ *MAX_CHUNK_SIZE* ] [
-**-a**\ *NR_AHEAD* ] [ **-b**\ *BUF_SIZE* ] [ **-l**\ *LOGIN_NAME* ] [
-**-P**\ *PORT* ] [ **-F**\ *CONFIG* ] [ **-i**\ *IDENTITY* ] [
-**-c**\ *CIPHER* ] [ **-M**\ *HMAC* ] [ **-C**\ *COMPRESS* ] *source ...
-target*
+**mscp** [**-46vqDpdNh**] [ **-n** *NR_CONNECTIONS* ] [ **-m**
+*COREMASK* ] [ **-u** *MAX_STARTUPS* ] [ **-I** *INTERVAL* ] [ **-W**
+*CHECKPOINT* ] [ **-R** *CHECKPOINT* ] [ **-s** *MIN_CHUNK_SIZE* ] [
+**-S** *MAX_CHUNK_SIZE* ] [ **-a** *NR_AHEAD* ] [ **-b** *BUF_SIZE* ] [
+**-L** *LIMIT_BITRATE* ] [ **-l** *LOGIN_NAME* ] [ **-P** *PORT* ] [
+**-F** *SSH_CONFIG* ] [ **-o** *SSH_OPTION* ] [ **-i** *IDENTITY* ] [
+**-J** *DESTINATION* ] [ **-c** *CIPHER* ] [ **-M** *HMAC* ] [ **-C**
+*COMPRESS* ] [ **-g** *CONGESTION* ] *source ... target*
 
 DESCRIPTION
 ===========
@@ -28,7 +29,7 @@ threads. It enables transferring (1) multiple files simultaneously and
 (2) a large file in parallel, reducing the transfer time for a lot
 of/large files over networks.
 
-The usage of **mscp** imitates the **scp** command of *OpenSSH,* for
+The usage of **mscp** follows the **scp** command of *OpenSSH,* for
 example:
 
 ::
@@ -53,14 +54,16 @@ OPTIONS
    following formula: floor(log(nr_cores)*2)+1.
 
 **-m COREMASK**
-   Configures CPU cores to be used by the hexadecimal bitmask. All CPU
-   cores are used by default.
+   Configures CPU cores to be used by the hexadecimal bitmask. For
+   example, -m 0x25 pins threads onto CPU cores 0, 2, and 5. The default
+   value is not specified: all CPU cores are used and no threads are
+   pinned to any cores.
 
 **-u MAX_STARTUPS**
-   Specifies the number of concurrent outgoing SSH connections. **sshd**
-   limits the number of simultaneous SSH connection attempts by
-   *MaxStartups* in *sshd_config.* The default *MaxStartups* is 10;
-   thus, we set the default MAX_STARTUPS 8.
+   Specifies the number of concurrent unauthenticated SSH connection
+   attempts. **sshd** limits the number of simultaneous SSH connection
+   attempts by *MaxStartups* in *sshd_config.* The default *MaxStartups*
+   is 10; thus, we set the default MAX_STARTUPS 8.
 
 **-I INTERVAL**
    Specifies the interval (in seconds) between SSH connection attempts.
@@ -69,13 +72,39 @@ OPTIONS
    option inserts intervals between the attempts to avoid being
    determined as an attack. The default value is 0.
 
+**-W CHECKPOINT**
+   Specifies a checkpoint file to save the state of a failed transfer.
+   When transferring fails due to, for example, connection disruption or
+   user interrupt, **mscp** writes the information about the remaining
+   files and chunks to the specified checkpoint file. **-W** option with
+   **-D** (dry-run mode) only writes a checkpoint file and exits.
+
+**-R CHECKPOINT**
+   Specifies a checkpoint file to resume a transfer. When a checkpoint
+   file is passed, **mscp** reads the checkpoint to load a remote host,
+   copy direction, and files and their chunks to be transferred. Namely,
+   **mscp** can resume a past failed transfer from the checkpoint.
+   Resuming with a checkpoint does not require *source ... target*
+   arguments. Other SSH connection options, such as port number and
+   config file, should be specified as with the failed run. In addition,
+   checkpoint files have file paths as relative paths. Thus, you must
+   run **mscp** in the same working directory as the failed run. You can
+   see the contents of a checkpoint file with the **mscp -vv -D -R
+   CHECKPOINT** command (Dry-run mode). Note that the checkpoint file is
+   not automatically removed after the resumed transfer ends
+   successfully. Users should check the return value of **mscp** and
+   remove the checkpoint if it returns 0.
+
 **-s MIN_CHUNK_SIZE**
-   Specifies the minimum chunk size. **mscp** divides a file into chunks
-   and copies the chunks in parallel.
+   Specifies the minimum chunk size. **mscp** divides a single file into
+   chunks and copies the chunks in parallel. The default value is 16M
+   bytes.
 
 **-S MAX_CHUNK_SIZE**
    Specifies the maximum chunk size. The default is file size divided by
-   the number of connections.
+   the number of connections and devided by 4. If the calculated value
+   is smarller than the **MIN_CHUNK_SIZE** value, MIN_CHUNK_SIZE is
+   used.
 
 **-a NR_AHEAD**
    Specifies the number of inflight SFTP commands. The default value is
@@ -87,6 +116,10 @@ OPTIONS
    delivered over SSH. Changing this value is not recommended at
    present.
 
+**-L LIMIT_BITRATE**
+   Limits the bitrate, specified with k (K), m (M), and g (G), e.g.,
+   100m indicates 100 Mbps.
+
 **-4**
    Uses IPv4 addresses only.
 
@@ -101,8 +134,8 @@ OPTIONS
 
 **-D**
    Dry-run mode: it scans source files to be copied, calculates chunks,
-   and resolves destination file paths. Dry-run mode with **-vv** option
-   enables confirming files to be copied and their destination paths.
+   resolves destination file paths, and exits. Dry-run mode with **-vv**
+   option can confirm files to be copied and their destination paths.
 
 **-r**
    No effect. **mscp** copies recursively if a source path is a
@@ -114,35 +147,47 @@ OPTIONS
 
 **-P PORT**
    Specifies the port number to connect to on the remote machine as with
-   ssh(1) and scp(1).
+   *scp(1).*
 
-**-F CONFIG**
+**-F SSH_CONFIG**
    Specifies an alternative per-user ssh configuration file. Note that
    acceptable options in the configuration file are what *libssh*
    supports.
 
+**-o SSH_OPTION**
+   Specifies ssh options in the format used in ssh_config. Note that
+   acceptable options are what *libssh* supports.
+
 **-i IDENTITY**
    Specifies the identity file for public key authentication.
 
+**-J DESTINATION**
+   A shortcut to define a **ProxyJump** configuration directive. Each
+   SFTP session of **mscp** connects to the target host by first making
+   an **ssh** connection to the jump host described by *destination.*
+
 **-c CIPHER**
    Selects the cipher to use for encrypting the data transfer. See
-   `libssh features <https://www.libssh.org/features/>`__.
+   **mscp -h** or **Ciphers** in `libssh
+   features <https://www.libssh.org/features/>`__.
 
 **-M HMAC**
-   Specifies MAC hash algorithms. See `libssh
-   features <https://www.libssh.org/features/>`__.
+   Specifies MAC hash algorithms. See **mscp -h** or **MAC hashes** in
+   `libssh features <https://www.libssh.org/features/>`__.
 
 **-C COMPRESS**
    Enables compression: yes, no, zlib, zlib@openssh.com. The default is
    none. See `libssh features <https://www.libssh.org/features/>`__.
 
+**-g CONGESTION**
+   Specifies the TCP congestion control algorithm to use (Linux only).
+   See **sysctl net.ipv4.tcp_allowed_congestion_control** for available
+   values.
+
 **-p**
    Preserves modification times and access times (file mode bits are
    preserved by default).
 
-**-H**
-   Disables hostkey checking.
-
 **-d**
    Increments the ssh debug output level.
 
@@ -196,6 +241,22 @@ Copy a local file and a directory to /tmp at a remote host:
 
        $ mscp ~/src-file dir1 10.0.0.1:/tmp
 
+Save a checkpoint if transfer fails:
+
+::
+
+       $ mscp -W mscp.checkpoint many-large-files 10.0.0.1:dst/
+
+Check the remaining files and chunks, and resume the failed transfer:
+
+::
+
+       # Dump the content of a checkpoint and exit (dry-run mode)
+       $ mscp -vv -D -R mscp.checkpoint
+
+       # resume transferring from the checkpoint
+       $ mscp -R mscp.checkpoint
+
 In a long fat network, following options might improve performance:
 
 ::
@@ -221,10 +282,10 @@ File Transfer over SSH. In Practice and Experience in Advanced Research
 Computing (PEARC '23). Association for Computing Machinery, New York,
 NY, USA, 320–323. `DOI <https://doi.org/10.1145/3569951.3597582>`__.
 
-CONTACT INFROMATION
+CONTACT INFORMATION
 ===================
 
-For pathces, bug reports, or feature requests, please open an issue on
+For patches, bug reports, or feature requests, please open an issue on
 `GitHub <https://github.com/upa/mscp>`__.
 
 AUTHORS

include/config.h.in

@@ -9,4 +9,10 @@
 /* Define to 1 if you have the strlcat function. */
 #cmakedefine HAVE_STRLCAT 1
 
+/* Define to 1 if you have the htonll function. */
+#cmakedefine HAVE_HTONLL 1
+
+/* Define to 1 if you have the ntohll function. */
+#cmakedefine HAVE_NTOHLL 1
+
 #endif /* _CONFIG_H_ */

include/mscp.h

@@ -16,17 +16,19 @@
  * libmscp is follows:
  *
  * 1. create mscp instance with mscp_init()
- * 2. connect to remote host with mscp_connect()
- * 3. add path to source files with mscp_add_src_path()
- * 4. set path to destination with mscp_set_dst_path()
- * 5. start to scan source files with mscp_scan()
- * 6. start copy with mscp_start()
- * 7. wait for copy finished with mscp_join()
- * 8. cleanup mscp instance with mscp_cleanup() and mscp_free()
+ * 2. set remote host and copy direction with mscp_set_remote()
+ * 3. connect to remote host with mscp_connect()
+ * 4. add path to source files with mscp_add_src_path()
+ * 5. set path to destination with mscp_set_dst_path()
+ * 6. start to scan source files with mscp_scan()
+ * 7. start copy with mscp_start()
+ * 8. wait for copy finished with mscp_join()
+ * 9. cleanup mscp instance with mscp_cleanup() and mscp_free()
  */
 
 #include <stdbool.h>
 #include <limits.h>
+#include <stdlib.h>
 
 #define MSCP_DIRECTION_L2R	1	/** Indicates local to remote copy */
 #define MSCP_DIRECTION_R2L	2	/** Indicates remote to local copy */
@@ -41,11 +43,11 @@ struct mscp_opts {
 	size_t	min_chunk_sz;	/** minimum chunk size (default 64MB) */
 	size_t	max_chunk_sz;	/** maximum chunk size (default file size/nr_threads) */
 	size_t	buf_sz;		/** buffer size, default 16k. */
-	char	*coremask; /** hex to specifiy usable cpu cores */
+	size_t	bitrate;	/** bits-per-seconds to limit bandwidth */
+	char	*coremask;	/** hex to specifiy usable cpu cores */
 	int	max_startups;	/** sshd MaxStartups concurrent connections */
 	int     interval;	/** interval between SSH connection attempts */
 	bool	preserve_ts;	/** preserve file timestamps */
-
 	int	severity; 	/** messaging severity. set MSCP_SERVERITY_* */
 };
 
@@ -60,7 +62,9 @@ struct mscp_ssh_opts {
 	char	*port;		/** ssh port */
 	int	ai_family;	/** address family */
 	char	*config;	/** path to ssh_config, default ~/.ssh/config*/
+	char	**options;	/** array of ssh_config options, terminated by NULL */
 	char	*identity;	/** path to private key */
+	char	*proxyjump;	/** ProxyJump configuration directive (shortcut) */
 	char	*cipher;	/** cipher spec */
 	char	*hmac;		/** hmacp spec */
 	char	*compress;	/** yes, no, zlib@openssh.com */
@@ -70,7 +74,6 @@ struct mscp_ssh_opts {
 	char	*passphrase;	/** passphrase for private key */
 
 	int	debug_level;		/** inclirement libssh debug output level */
-	bool	no_hostkey_check;	/** do not check host keys */
 	bool	enable_nagle;		/** enable Nagle's algorithm if true */
 };
 
@@ -92,7 +95,6 @@ struct mscp_ssh_opts {
 struct mscp_stats {
 	size_t total;	/** total bytes to be transferred */
 	size_t done;	/** total bytes transferred */
-	bool finished;	/** true when all copy threads finished */
 };
 
 
@@ -102,15 +104,22 @@ struct mscp;
 /**
  * @brief Creates a new mscp instance.
  *
- * @param remote_host	remote host for file transer.
- * @param direction	copy direction, `MSCP_DIRECTION_L2R` or `MSCP_DIRECTION_R2L`
  * @param o		options for configuring mscp.
  * @param s		options for configuring ssh connections.
  *
  * @retrun 		A new mscp instance or NULL on error.
  */
-struct mscp *mscp_init(const char *remote_host, int direction,
-		       struct mscp_opts *o, struct mscp_ssh_opts *s);
+struct mscp *mscp_init(struct mscp_opts *o, struct mscp_ssh_opts *s);
+
+/**
+ * @brief Set remote host and copy direction.
+ *
+ * @param remote_host	remote host for file transer.
+ * @param direction	copy direction, `MSCP_DIRECTION_L2R` or `MSCP_DIRECTION_R2L`
+ *
+ * @return              0 on success, < 0 if an error occured.
+ */
+int mscp_set_remote(struct mscp *m, const char *remote_host, int direction);
 
 /**
  * @brief Connect the first SSH connection. mscp_connect connects to
@@ -128,9 +137,7 @@ int mscp_connect(struct mscp *m);
 
 /**
  * @brief Add a source file path to be copied. The path indicates
- * either a file or directory. The path can be `user@host:path`
- * notation.  In this case, `dst_path` for mscp_set_dst_path() must
- * not contain remote host notation.
+ * either a file or directory.
  *
  * @param m		mscp instance.
  * @param src_path	source file path to be copied.
@@ -141,9 +148,7 @@ int mscp_add_src_path(struct mscp *m, const char *src_path);
 
 /**
  * @brief Set the destination file path. The path indicates either a
- * file, directory, or nonexistent path. The path can be
- * `user@host:path` notation.  In this case, all source paths appended
- * by mscp_set_src_path() must not contain remote host notation.
+ * file, directory, or nonexistent path.
  *
  * @param m		mscp instance.
  * @param dst_path	destination path to which source files copied.
@@ -167,15 +172,50 @@ int mscp_set_dst_path(struct mscp *m, const char *dst_path);
 int mscp_scan(struct mscp *m);
 
 /**
- * @brief Join scna thread invoked by mscp_scan(). mscp_join()
- * involves this, so that mscp_scan_join() should be called when
- * mscp_scan() is called by mscp_start() is not.
+ * @brief Join scan thread invoked by mscp_scan() if it
+ * runs. mscp_join() involves mscp_can_join(). Thus, there is no need
+ * to call this function alone.
  *
  * @param m	mscp instance.
  * @return	0 on success, < 0 if an error occured.
  */
 int mscp_scan_join(struct mscp *m);
 
+/**
+ * @brief get information about remote host and copy direction from a
+ * checkpoint file specified by *pathname. This functions returns
+ * remote host name to *renote, and the copy direction into *dir.
+ * Thus, you can call mscp_init with those values.
+ *
+ * @param pathname	path to a checkpoint file.
+ * @param remote	char buffer to which remote hostname is stored.
+ * @param len		length of *remote.
+ * @param dir		int to which the copy direction is stored.
+ */
+int mscp_checkpoint_get_remote(const char *pathname, char *remote, size_t len, int *dir);
+
+/**
+ * @brief load information about untransferred files and chunks at the
+ * last transfer . mscp_checkpoint_load() loads files and associated
+ * chunks from the checkpoint file pointed by pathname. If you call
+ * mscp_checkpoint_load(), do not call mscp_scan().
+ *
+ * @param m		mscp instance.
+ * @param pathname	path to a checkpoint file.
+ * @return		0 on success, < 0 if an error occured.
+ */
+int mscp_checkpoint_load(struct mscp *m, const char *pathname);
+
+/**
+ * @brief save information about untransferred files and chunks to a
+ * checkpoint file.
+ *
+ * @param m		mscp instance.
+ * @param pathname	path to a checkpoint file.
+ * @return		0 on success, < 0 if an error occured.
+ */
+int mscp_checkpoint_save(struct mscp *m, const char *pathname);
+
 /**
  * @brief Start to copy files. mscp_start() returns immediately. You
  * can get statistics via mscp_get_stats() or messages via pipe set by
@@ -255,4 +295,15 @@ enum {
 };
 
 
+/**
+ * @brief Return available ciphers.
+ */
+const char **mscp_ssh_ciphers(void);
+
+/**
+ * @brief Return available hmacs.
+ */
+ const char **mscp_ssh_hmacs(void);
+
+
 #endif /* _MSCP_H_ */

patch/README.md

@@ -1,5 +1,6 @@
 
-Patches in this directory introduces `sftp_async_write()` and
-`sftp_async_write_end()` to libssh. Those implementations are derived
-from https://github.com/limes-datentechnik-gmbh/libssh. See [Re: SFTP
-Write async](https://archive.libssh.org/libssh/2020-06/0000004.html).
+Patches in this directory introduce enhancements for libssh including
+`sftp_async_write()` and `sftp_async_write_end()`, derived from
+https://github.com/limes-datentechnik-gmbh/libssh. See [Re: SFTP Write
+async](https://archive.libssh.org/libssh/2020-06/0000004.html).
+

patch/libssh-0.10.6-2-g6f1b1e76.patch

@@ -1,3 +1,13 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index a64b7708..c6344a5a 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -1,4 +1,4 @@
+-cmake_minimum_required(VERSION 3.3.0)
++cmake_minimum_required(VERSION 3.13.0)
+ cmake_policy(SET CMP0048 NEW)
+ 
+ # Specify search path for CMake modules to be loaded by include()
 diff --git a/ConfigureChecks.cmake b/ConfigureChecks.cmake
 index 9de10225..0f3d20ed 100644
 --- a/ConfigureChecks.cmake
@@ -37,7 +47,7 @@ index 1fce7b76..b64d1455 100644
  int ssh_buffer_validate_length(struct ssh_buffer_struct *buffer, size_t len);
  
 diff --git a/include/libssh/libssh.h b/include/libssh/libssh.h
-index 669a0a96..da5b4099 100644
+index 669a0a96..26b20f3f 100644
 --- a/include/libssh/libssh.h
 +++ b/include/libssh/libssh.h
 @@ -368,6 +368,7 @@ enum ssh_options_e {
@@ -64,11 +74,14 @@ index 669a0a96..da5b4099 100644
  LIBSSH_API void ssh_buffer_free(ssh_buffer buffer);
  #define SSH_BUFFER_FREE(x) \
      do { if ((x) != NULL) { ssh_buffer_free(x); x = NULL; } } while(0)
-@@ -843,6 +846,8 @@ LIBSSH_API void *ssh_buffer_get(ssh_buffer buffer);
+@@ -843,6 +846,11 @@ LIBSSH_API void *ssh_buffer_get(ssh_buffer buffer);
  LIBSSH_API uint32_t ssh_buffer_get_len(ssh_buffer buffer);
  LIBSSH_API int ssh_session_set_disconnect_message(ssh_session session, const char *message);
  
 +typedef ssize_t (*ssh_add_func) (void *ptr, size_t max_bytes, void *userdata);
++
++LIBSSH_API const char **ssh_ciphers(void);
++LIBSSH_API const char **ssh_hmacs(void);
 +
  #ifndef LIBSSH_LEGACY_0_4
  #include "libssh/legacy.h"
@@ -299,6 +312,60 @@ index 15cae644..02ef43b4 100644
          errno = 0;
          rc = connect(s, itr->ai_addr, itr->ai_addrlen);
          if (rc == -1 && (errno != 0) && (errno != EINPROGRESS)) {
+diff --git a/src/misc.c b/src/misc.c
+index 7081f12a..e3879fe4 100644
+--- a/src/misc.c
++++ b/src/misc.c
+@@ -71,6 +71,8 @@
+ #include "libssh/priv.h"
+ #include "libssh/misc.h"
+ #include "libssh/session.h"
++#include "libssh/wrapper.h"
++#include "libssh/crypto.h"
+ 
+ #ifdef HAVE_LIBGCRYPT
+ #define GCRYPT_STRING "/gnutls"
+@@ -2074,4 +2076,40 @@ int ssh_check_hostname_syntax(const char *hostname)
+     return SSH_OK;
+ }
+ 
++/**
++ * @brief Return supported cipher names
++ * @return	The list of cipher names.
++ */
++const char **ssh_ciphers(void)
++{
++     struct ssh_cipher_struct *tab=ssh_get_ciphertab();
++     static const char *ciphers[32];
++     int n;
++
++     memset(ciphers, 0, sizeof(*ciphers));
++
++     for (n = 0; tab[n].name != NULL; n++) {
++	  ciphers[n] = tab[n].name;
++     }
++     return ciphers;
++}
++
++/**
++ * @brief Return supported hmac names
++ * @return	The list of hmac names.
++ */
++const char **ssh_hmacs(void)
++{
++     struct ssh_hmac_struct *tab=ssh_get_hmactab();
++     static const char *hmacs[32];
++     int n;
++
++     memset(hmacs, 0, sizeof(*hmacs));
++
++     for (n = 0; tab[n].name != NULL; n++) {
++	  hmacs[n] = tab[n].name;
++     }
++     return hmacs;
++}
++
+ /** @} */
 diff --git a/src/options.c b/src/options.c
 index b3ecffe1..8de24ed6 100644
 --- a/src/options.c

patch/libssh-0.11.2.patch

@@ -0,0 +1,605 @@
+diff --git a/ConfigureChecks.cmake b/ConfigureChecks.cmake
+index 8765dc6e..766e7d16 100644
+--- a/ConfigureChecks.cmake
++++ b/ConfigureChecks.cmake
+@@ -209,6 +209,7 @@ if (UNIX)
+     check_library_exists(util forkpty "" HAVE_LIBUTIL)
+     check_function_exists(cfmakeraw HAVE_CFMAKERAW)
+     check_function_exists(__strtoull HAVE___STRTOULL)
++    check_symbol_exists(TCP_CONGESTION "netinet/tcp.h" HAVE_TCP_CONGESTION)
+ endif (UNIX)
+ 
+ set(LIBSSH_REQUIRED_LIBRARIES ${_REQUIRED_LIBRARIES} CACHE INTERNAL "libssh required system libraries")
+diff --git a/config.h.cmake b/config.h.cmake
+index 8dce5273..ef534762 100644
+--- a/config.h.cmake
++++ b/config.h.cmake
+@@ -219,6 +219,8 @@
+ 
+ #cmakedefine HAVE_GCC_BOUNDED_ATTRIBUTE 1
+ 
++#cmakedefine HAVE_TCP_CONGESTION 1
++
+ /* Define to 1 if you want to enable GSSAPI */
+ #cmakedefine WITH_GSSAPI 1
+ 
+diff --git a/include/libssh/buffer.h b/include/libssh/buffer.h
+index d22178e7..2d6aa0a7 100644
+--- a/include/libssh/buffer.h
++++ b/include/libssh/buffer.h
+@@ -37,6 +37,8 @@ int ssh_buffer_add_u8(ssh_buffer buffer, uint8_t data);
+ int ssh_buffer_add_u16(ssh_buffer buffer, uint16_t data);
+ int ssh_buffer_add_u32(ssh_buffer buffer, uint32_t data);
+ int ssh_buffer_add_u64(ssh_buffer buffer, uint64_t data);
++ssize_t ssh_buffer_add_func(ssh_buffer buffer, ssh_add_func f, size_t max_bytes,
++			    void *userdata);
+ 
+ int ssh_buffer_validate_length(struct ssh_buffer_struct *buffer, size_t len);
+ 
+diff --git a/include/libssh/libssh.h b/include/libssh/libssh.h
+index 3bddb019..1d5d7761 100644
+--- a/include/libssh/libssh.h
++++ b/include/libssh/libssh.h
+@@ -373,6 +373,7 @@ enum ssh_options_e {
+     SSH_OPTIONS_HOST,
+     SSH_OPTIONS_PORT,
+     SSH_OPTIONS_PORT_STR,
++    SSH_OPTIONS_AI_FAMILY,
+     SSH_OPTIONS_FD,
+     SSH_OPTIONS_USER,
+     SSH_OPTIONS_SSH_DIR,
+@@ -407,6 +408,7 @@ enum ssh_options_e {
+     SSH_OPTIONS_GSSAPI_AUTH,
+     SSH_OPTIONS_GLOBAL_KNOWNHOSTS,
+     SSH_OPTIONS_NODELAY,
++    SSH_OPTIONS_CCALGO,
+     SSH_OPTIONS_PUBLICKEY_ACCEPTED_TYPES,
+     SSH_OPTIONS_PROCESS_CONFIG,
+     SSH_OPTIONS_REKEY_DATA,
+@@ -876,6 +878,7 @@ LIBSSH_API const char* ssh_get_hmac_in(ssh_session session);
+ LIBSSH_API const char* ssh_get_hmac_out(ssh_session session);
+ 
+ LIBSSH_API ssh_buffer ssh_buffer_new(void);
++LIBSSH_API ssh_buffer ssh_buffer_new_size(uint32_t size, uint32_t headroom);
+ LIBSSH_API void ssh_buffer_free(ssh_buffer buffer);
+ #define SSH_BUFFER_FREE(x) \
+     do { if ((x) != NULL) { ssh_buffer_free(x); x = NULL; } } while(0)
+@@ -886,6 +889,12 @@ LIBSSH_API void *ssh_buffer_get(ssh_buffer buffer);
+ LIBSSH_API uint32_t ssh_buffer_get_len(ssh_buffer buffer);
+ LIBSSH_API int ssh_session_set_disconnect_message(ssh_session session, const char *message);
+ 
++typedef ssize_t (*ssh_add_func) (void *ptr, size_t max_bytes, void *userdata);
++
++LIBSSH_API const char **ssh_ciphers(void);
++LIBSSH_API const char **ssh_hmacs(void);
++LIBSSH_API void ssh_use_openssh_proxy_jumps(int);
++
+ #ifndef LIBSSH_LEGACY_0_4
+ #include "libssh/legacy.h"
+ #endif
+diff --git a/include/libssh/session.h b/include/libssh/session.h
+index aed94072..327cf4fe 100644
+--- a/include/libssh/session.h
++++ b/include/libssh/session.h
+@@ -255,6 +255,7 @@ struct ssh_session_struct {
+         unsigned long timeout; /* seconds */
+         unsigned long timeout_usec;
+         uint16_t port;
++        int  ai_family;
+         socket_t fd;
+         int StrictHostKeyChecking;
+         char compressionlevel;
+@@ -264,6 +265,7 @@ struct ssh_session_struct {
+         int flags;
+         int exp_flags;
+         int nodelay;
++        char *ccalgo;
+         bool config_processed;
+         uint8_t options_seen[SOC_MAX];
+         uint64_t rekey_data;
+diff --git a/include/libssh/sftp.h b/include/libssh/sftp.h
+index cf4458c3..1a864795 100644
+--- a/include/libssh/sftp.h
++++ b/include/libssh/sftp.h
+@@ -569,6 +569,10 @@ SSH_DEPRECATED LIBSSH_API int sftp_async_read(sftp_file file,
+                                               uint32_t len,
+                                               uint32_t id);
+ 
++LIBSSH_API ssize_t sftp_async_write(sftp_file file, ssh_add_func f, size_t count,
++				    void *userdata, uint32_t* id);
++LIBSSH_API int sftp_async_write_end(sftp_file file, uint32_t id, int blocking);
++
+ /**
+  * @brief Write to a file using an opened sftp file handle.
+  *
+diff --git a/src/buffer.c b/src/buffer.c
+index 449fa941..f49e8af6 100644
+--- a/src/buffer.c
++++ b/src/buffer.c
+@@ -142,6 +142,40 @@ struct ssh_buffer_struct *ssh_buffer_new(void)
+     return buf;
+ }
+ 
++/**
++ * @brief Create a new SSH buffer with a specified size and headroom.
++ *
++ * @param[in] len       length for newly initialized SSH buffer.
++ * @param[in] headroom  length for headroom
++ * @return A newly initialized SSH buffer, NULL on error.
++ */
++struct ssh_buffer_struct *ssh_buffer_new_size(uint32_t len, uint32_t headroom)
++{
++    struct ssh_buffer_struct *buf = NULL;
++    int rc;
++
++    if (len < headroom)
++	    return NULL;
++
++    buf = calloc(1, sizeof(struct ssh_buffer_struct));
++    if (buf == NULL) {
++        return NULL;
++    }
++
++    rc = ssh_buffer_allocate_size(buf, len);
++    if (rc != 0) {
++        SAFE_FREE(buf);
++        return NULL;
++    }
++
++    buf->pos += headroom;
++    buf->used += headroom;
++
++    buffer_verify(buf);
++
++    return buf;
++}
++
+ /**
+  * @brief Deallocate a SSH buffer.
+  *
+@@ -329,6 +363,49 @@ int ssh_buffer_add_data(struct ssh_buffer_struct *buffer, const void *data, uint
+     return 0;
+ }
+ 
++/**
++ * @brief Add data at the tail of a buffer by an external function
++ *
++ * @param[in]  buffer    The buffer to add data.
++ *
++ * @param[in]  f         function that adds data to the buffer.
++ *
++ * @param[in]  max_bytes The maximum length of the data to add.
++ *
++ * @return               actual bytes added on success, < 0 on error.
++ */
++ssize_t ssh_buffer_add_func(struct ssh_buffer_struct *buffer, ssh_add_func f,
++			    size_t max_bytes, void *userdata)
++{
++    ssize_t actual;
++
++    if (buffer == NULL) {
++        return -1;
++    }
++
++    buffer_verify(buffer);
++
++    if (buffer->used + max_bytes < max_bytes) {
++        return -1;
++    }
++
++    if (buffer->allocated < (buffer->used + max_bytes)) {
++        if (buffer->pos > 0) {
++            buffer_shift(buffer);
++        }
++        if (realloc_buffer(buffer, buffer->used + max_bytes) < 0) {
++            return -1;
++        }
++    }
++
++    if ((actual = f(buffer->data + buffer->used, max_bytes, userdata)) < 0)
++      return -1;
++
++    buffer->used += actual;
++    buffer_verify(buffer);
++    return actual;
++}
++
+ /**
+  * @brief Ensure the buffer has at least a certain preallocated size.
+  *
+diff --git a/src/connect.c b/src/connect.c
+index 2cb64037..51f4c87e 100644
+--- a/src/connect.c
++++ b/src/connect.c
+@@ -109,7 +109,7 @@ static int ssh_connect_socket_close(socket_t s)
+ #endif
+ }
+ 
+-static int getai(const char *host, int port, struct addrinfo **ai)
++static int getai(const char *host, int port, int ai_family, struct addrinfo **ai)
+ {
+     const char *service = NULL;
+     struct addrinfo hints;
+@@ -118,7 +118,7 @@ static int getai(const char *host, int port, struct addrinfo **ai)
+     ZERO_STRUCT(hints);
+ 
+     hints.ai_protocol = IPPROTO_TCP;
+-    hints.ai_family = PF_UNSPEC;
++    hints.ai_family = ai_family > 0 ? ai_family : PF_UNSPEC;
+     hints.ai_socktype = SOCK_STREAM;
+ 
+     if (port == 0) {
+@@ -151,6 +151,20 @@ static int set_tcp_nodelay(socket_t socket)
+                       sizeof(opt));
+ }
+ 
++static int set_tcp_ccalgo(socket_t socket, const char *ccalgo)
++{
++#ifdef HAVE_TCP_CONGESTION
++	return setsockopt(socket,
++			  IPPROTO_TCP,
++			  TCP_CONGESTION,
++			  (void *)ccalgo,
++			  strlen(ccalgo));
++#else
++	errno = ENOTSUP;
++	return -1;
++#endif
++}
++
+ /**
+  * @internal
+  *
+@@ -168,7 +182,7 @@ socket_t ssh_connect_host_nonblocking(ssh_session session, const char *host,
+     struct addrinfo *ai = NULL;
+     struct addrinfo *itr = NULL;
+ 
+-    rc = getai(host, port, &ai);
++    rc = getai(host, port, session->opts.ai_family, &ai);
+     if (rc != 0) {
+         ssh_set_error(session, SSH_FATAL,
+                       "Failed to resolve hostname %s (%s)",
+@@ -194,7 +208,7 @@ socket_t ssh_connect_host_nonblocking(ssh_session session, const char *host,
+ 
+             SSH_LOG(SSH_LOG_PACKET, "Resolving %s", bind_addr);
+ 
+-            rc = getai(bind_addr, 0, &bind_ai);
++            rc = getai(bind_addr, 0, session->opts.ai_family, &bind_ai);
+             if (rc != 0) {
+                 ssh_set_error(session, SSH_FATAL,
+                               "Failed to resolve bind address %s (%s)",
+@@ -251,6 +265,18 @@ socket_t ssh_connect_host_nonblocking(ssh_session session, const char *host,
+             }
+         }
+ 
++        if (session->opts.ccalgo) {
++	     rc = set_tcp_ccalgo(s, session->opts.ccalgo);
++	     if (rc < 0) {
++		 ssh_set_error(session, SSH_FATAL,
++			       "Failed to set TCP_CONGESTION on socket: %s",
++			       ssh_strerror(errno, err_msg, SSH_ERRNO_MSG_MAX));
++		 ssh_connect_socket_close(s);
++		 s = -1;
++		 continue;
++	     }
++	}
++
+         errno = 0;
+         rc = connect(s, itr->ai_addr, itr->ai_addrlen);
+         if (rc == -1) {
+diff --git a/src/misc.c b/src/misc.c
+index 774211fb..74e57959 100644
+--- a/src/misc.c
++++ b/src/misc.c
+@@ -71,6 +71,8 @@
+ #include "libssh/priv.h"
+ #include "libssh/misc.h"
+ #include "libssh/session.h"
++#include "libssh/wrapper.h"
++#include "libssh/crypto.h"
+ 
+ #ifdef HAVE_LIBGCRYPT
+ #define GCRYPT_STRING "/gcrypt"
+@@ -2054,6 +2056,42 @@ ssize_t ssh_readn(int fd, void *buf, size_t nbytes)
+     return total_bytes_read;
+ }
+ 
++/**
++ * @brief Return supported cipher names
++ * @return	The list of cipher names.
++ */
++const char **ssh_ciphers(void)
++{
++     struct ssh_cipher_struct *tab=ssh_get_ciphertab();
++     static const char *ciphers[32];
++     int n;
++
++     memset(ciphers, 0, sizeof(*ciphers));
++
++     for (n = 0; tab[n].name != NULL; n++) {
++	  ciphers[n] = tab[n].name;
++     }
++     return ciphers;
++}
++
++/**
++ * @brief Return supported hmac names
++ * @return	The list of hmac names.
++ */
++const char **ssh_hmacs(void)
++{
++     struct ssh_hmac_struct *tab=ssh_get_hmactab();
++     static const char *hmacs[32];
++     int n;
++
++     memset(hmacs, 0, sizeof(*hmacs));
++
++     for (n = 0; tab[n].name != NULL; n++) {
++	  hmacs[n] = tab[n].name;
++     }
++     return hmacs;
++}
++
+ /**
+  * @brief Write the requested number of bytes to a local file.
+  *
+@@ -2227,6 +2265,17 @@ ssh_proxyjumps_free(struct ssh_list *proxy_jump_list)
+     }
+ }
+ 
++static bool force_openssh_proxy_jumps;
++
++/**
++ * @breif set use openssh proxy jumps without the OPENSSH_PROXYJUMP env var
++ */
++void
++ssh_use_openssh_proxy_jumps(int v)
++{
++    force_openssh_proxy_jumps = (v > 0);
++}
++
+ /**
+  * @brief Check if libssh proxy jumps is enabled
+  *
+@@ -2241,7 +2290,12 @@ ssh_libssh_proxy_jumps(void)
+ {
+     const char *t = getenv("OPENSSH_PROXYJUMP");
+ 
++    if (force_openssh_proxy_jumps)
++	    return false;
++
+     return !(t != NULL && t[0] == '1');
+ }
+ 
++
++
+ /** @} */
+diff --git a/src/options.c b/src/options.c
+index 785296dd..a82d4d81 100644
+--- a/src/options.c
++++ b/src/options.c
+@@ -251,6 +251,7 @@ int ssh_options_copy(ssh_session src, ssh_session *dest)
+     new->opts.gss_delegate_creds    = src->opts.gss_delegate_creds;
+     new->opts.flags                 = src->opts.flags;
+     new->opts.nodelay               = src->opts.nodelay;
++    new->opts.ccalgo                = src->opts.ccalgo;
+     new->opts.config_processed      = src->opts.config_processed;
+     new->opts.control_master        = src->opts.control_master;
+     new->common.log_verbosity       = src->common.log_verbosity;
+@@ -326,6 +327,9 @@ int ssh_options_set_algo(ssh_session session,
+  *              - SSH_OPTIONS_PORT_STR:
+  *                The port to connect to (const char *).
+  *
++ *              - SSH_OPTIONS_AI_FAMILY:
++ *                The address family for connecting (int *).
++ *
+  *              - SSH_OPTIONS_FD:
+  *                The file descriptor to use (socket_t).\n
+  *                \n
+@@ -571,6 +575,10 @@ int ssh_options_set_algo(ssh_session session,
+  *                Set it to disable Nagle's Algorithm (TCP_NODELAY) on the
+  *                session socket. (int, 0=false)
+  *
++ *              - SSH_OPTIONS_CCALGO
++ *                Set it to specify TCP congestion control algorithm on the
++ *                session socket (Linux only). (int, 0=false)
++ *
+  *              - SSH_OPTIONS_PROCESS_CONFIG
+  *                Set it to false to disable automatic processing of per-user
+  *                and system-wide OpenSSH configuration files. LibSSH
+@@ -727,6 +735,21 @@ int ssh_options_set(ssh_session session, enum ssh_options_e type,
+                 session->opts.port = i & 0xffffU;
+             }
+             break;
++        case SSH_OPTIONS_AI_FAMILY:
++            if (value == NULL) {
++                session->opts.ai_family = 0;
++                ssh_set_error_invalid(session);
++                return -1;
++            } else {
++                int *x = (int *) value;
++                if (*x < 0) {
++		    session->opts.ai_family = 0;
++                    ssh_set_error_invalid(session);
++                    return -1;
++                }
++                session->opts.ai_family = *x;
++            }
++            break;
+         case SSH_OPTIONS_FD:
+             if (value == NULL) {
+                 session->opts.fd = SSH_INVALID_SOCKET;
+@@ -1241,6 +1264,20 @@ int ssh_options_set(ssh_session session, enum ssh_options_e type,
+                 session->opts.nodelay = (*x & 0xff) > 0 ? 1 : 0;
+             }
+             break;
++	case SSH_OPTIONS_CCALGO:
++            v = value;
++            if (v == NULL || v[0] == '\0') {
++                ssh_set_error_invalid(session);
++                return -1;
++            } else {
++                SAFE_FREE(session->opts.ccalgo);
++                session->opts.ccalgo = strdup(v);
++                if (session->opts.ccalgo == NULL) {
++                    ssh_set_error_oom(session);
++                    return -1;
++                }
++            }
++            break;
+         case SSH_OPTIONS_PROCESS_CONFIG:
+             if (value == NULL) {
+                 ssh_set_error_invalid(session);
+diff --git a/src/session.c b/src/session.c
+index 9fd5d946..ed9f908e 100644
+--- a/src/session.c
++++ b/src/session.c
+@@ -107,9 +107,11 @@ ssh_session ssh_new(void)
+     /* OPTIONS */
+     session->opts.StrictHostKeyChecking = 1;
+     session->opts.port = 22;
++    session->opts.ai_family = 0;
+     session->opts.fd = -1;
+     session->opts.compressionlevel = 7;
+     session->opts.nodelay = 0;
++    session->opts.ccalgo = NULL;
+     session->opts.identities_only = false;
+     session->opts.control_master = SSH_CONTROL_MASTER_NO;
+ 
+diff --git a/src/sftp.c b/src/sftp.c
+index 37b4133b..12b6d296 100644
+--- a/src/sftp.c
++++ b/src/sftp.c
+@@ -1488,6 +1488,132 @@ ssize_t sftp_write(sftp_file file, const void *buf, size_t count) {
+   return -1; /* not reached */
+ }
+ 
++/*
++ * sftp_async_write is based on and sftp_async_write_end is copied from
++ * https://github.com/limes-datentechnik-gmbh/libssh
++ *
++ * sftp_async_write has some optimizations:
++ * - use ssh_buffer_new_size() to reduce realoc_buffer.
++ * - use ssh_buffer_add_func() to avoid memcpy from read buffer to ssh buffer.
++ */
++ssize_t sftp_async_write(sftp_file file, ssh_add_func f, size_t count, void *userdata,
++			 uint32_t* id) {
++  sftp_session sftp = file->sftp;
++  ssh_buffer buffer;
++  uint32_t buf_sz;
++  ssize_t actual;
++  int len;
++  int packetlen;
++  int rc;
++
++#define HEADROOM 16
++  /* sftp_packet_write() prepends a 5-bytes (uint32_t length and
++   * 1-byte type) header to the head of the payload by
++   * ssh_buffer_prepend_data(). Inserting headroom by
++   * ssh_buffer_new_size() eliminates memcpy for prepending the
++   * header.
++   */
++
++  buf_sz = (HEADROOM + /* for header */
++	    sizeof(uint32_t) + /* id */
++	    ssh_string_len(file->handle) + 4 + /* file->handle */
++	    sizeof(uint64_t) + /* file->offset */
++	    sizeof(uint32_t) + /* count */
++	    count); /* datastring */
++
++  buffer = ssh_buffer_new_size(buf_sz, HEADROOM);
++  if (buffer == NULL) {
++    ssh_set_error_oom(sftp->session);
++    return -1;
++  }
++
++  *id = sftp_get_new_id(file->sftp);
++
++  rc = ssh_buffer_pack(buffer,
++                       "dSqd",
++                       *id,
++                       file->handle,
++                       file->offset,
++                       count); /* len of datastring */
++
++  if (rc != SSH_OK){
++    ssh_set_error_oom(sftp->session);
++    ssh_buffer_free(buffer);
++    return SSH_ERROR;
++  }
++
++  actual = ssh_buffer_add_func(buffer, f, count, userdata);
++  if (actual < 0){
++    ssh_set_error_oom(sftp->session);
++    ssh_buffer_free(buffer);
++    return SSH_ERROR;
++  }
++
++  packetlen=ssh_buffer_get_len(buffer)+5;
++  len = sftp_packet_write(file->sftp, SSH_FXP_WRITE, buffer);
++  ssh_buffer_free(buffer);
++  if (len < 0) {
++    return SSH_ERROR;
++  } else  if (len != packetlen) {
++    ssh_set_error(sftp->session, SSH_FATAL,
++      "Could only send %d of %d bytes to remote host!", len, packetlen);
++    SSH_LOG(SSH_LOG_PACKET,
++        "Could not write as much data as expected");
++    return SSH_ERROR;
++  }
++
++  file->offset += actual;
++
++  return actual;
++}
++
++int sftp_async_write_end(sftp_file file, uint32_t id, int blocking) {
++  sftp_session sftp = file->sftp;
++  sftp_message msg = NULL;
++  sftp_status_message status;
++
++  msg = sftp_dequeue(sftp, id);
++  while (msg == NULL) {
++    if (!blocking && ssh_channel_poll(sftp->channel, 0) == 0) {
++      /* we cannot block */
++      return SSH_AGAIN;
++    }
++    if (sftp_read_and_dispatch(sftp) < 0) {
++      /* something nasty has happened */
++      return SSH_ERROR;
++    }
++    msg = sftp_dequeue(sftp, id);
++  }
++
++  switch (msg->packet_type) {
++    case SSH_FXP_STATUS:
++      status = parse_status_msg(msg);
++      sftp_message_free(msg);
++      if (status == NULL) {
++        return SSH_ERROR;
++      }
++      sftp_set_error(sftp, status->status);
++      switch (status->status) {
++        case SSH_FX_OK:
++          status_msg_free(status);
++          return SSH_OK;
++        default:
++          break;
++      }
++      ssh_set_error(sftp->session, SSH_REQUEST_DENIED,
++          "SFTP server: %s", status->errormsg);
++      status_msg_free(status);
++      return SSH_ERROR;
++    default:
++      ssh_set_error(sftp->session, SSH_FATAL,
++          "Received message %d during write!", msg->packet_type);
++      sftp_message_free(msg);
++      return SSH_ERROR;
++  }
++
++  return SSH_ERROR; /* not reached */
++}
++
+ /* Seek to a specific location in a file. */
+ int sftp_seek(sftp_file file, uint32_t new_offset) {
+   if (file == NULL) {

rpm/mscp.spec.in

@@ -38,8 +38,26 @@ make -C build install DESTDIR=%{buildroot}
 
 
 %changelog
-* Wed Feb 07 2024 Ryo nakamura <upa@haeena.net> - 0.1.4-0
+* Sat Nov 08 2025 Ryo Nakamura <upa@haeena.net> - 0.2.4-1
+- RPM release for v0.2.4
+
+* Tue Aug 12 2025 Ryo Nakamura <upa@haeena.net> - 0.2.3-1
+- RPM release for v0.2.3
+
+* Wed Apr 16 2025 Ryo Nakamura <upa@haeena.net> - 0.2.2-1
+- RPM release for v0.2.2
+
+* Sat May 11 2024 Ryo Nakamura <upa@haeena.net> - 0.2.1-1
+- RPM release for v0.2.1
+
+* Mon Apr 15 2024 Ryo Nakamura <upa@haeena.net> - 0.2.0-1
+- RPM release for v0.2.0
+
+* Thu Mar 14 2024 Ryo Nakamura <upa@haeena.net> - 0.1.5-0
+- RPM release for v0.1.5
+
+* Wed Feb 07 2024 Ryo Nakamura <upa@haeena.net> - 0.1.4-0
 - RPM release for v0.1.4
 
-* Sat Feb 03 2024 Ryo nakamura <upa@haeena.net> - 0.1.3-0
+* Sat Feb 03 2024 Ryo Nakamura <upa@haeena.net> - 0.1.3-0
 - Initial release for rpm packaging

scripts/install-build-deps.sh

@@ -3,34 +3,75 @@
 # Install build dpenedencies.
 
 set -e
-set -u
+#set -u
+
+function print_help() {
+	echo "$0 [options]"
+	echo "    --dont-install         Print required packages."
+	echo "    --platform [PLATFORM]  PLATFORM is Kernel-ID, e.g., Linux-ubuntu."
+	echo "                           Automatically detected if not specified."
+}
 
 platform=$(uname -s)
+doinstall=1
 
 if [ -e /etc/os-release ]; then
 	source /etc/os-release
 	platform=${platform}-${ID}
 fi
 
-set -x
+while getopts h-: opt; do
+        optarg="${!OPTIND}"
+        [[ "$opt" = - ]] && opt="-$OPTARG"
+	case "-${opt}" in
+		--dont-install)
+			doinstall=0
+			;;
+		--platform)
+			platform=$optarg
+			shift
+			;;
+		-h)
+			print_help
+			exit 0
+			;;
+		*)
+			print_help
+			exit 1
+			;;
+	esac
+done
 
 case $platform in
 	Darwin)
-		brew install openssl@1.1
+		cmd="brew install"
+		pkgs="openssl@3"
 		;;
-	Linux-ubuntu*)
-		apt-get install --no-install-recommends -y \
-			gcc make cmake zlib1g-dev libssl-dev libkrb5-dev
+	Linux-ubuntu* | Linux-debian* | Linux-devuan*)
+		cmd="apt-get install --no-install-recommends -y"
+		pkgs="gcc make cmake zlib1g-dev libssl-dev libkrb5-dev"
 		;;
 	Linux-centos* | Linux-rhel* | Linux-rocky* | Linux-almalinux)
-		yum install -y \
-			gcc make cmake zlib-devel openssl-devel rpm-build
+		cmd="yum install -y"
+		pkgs="gcc make cmake zlib-devel openssl-devel rpm-build"
+		;;
+	Linux-arch*)
+		cmd="pacman --no-confirm -S"
+		pkgs="gcc make cmake"
 		;;
 	FreeBSD-freebsd)
-		pkg install cmake
+		cmd="pkg install"
+		pkgs="cmake"
 		;;
 	*)
 		echo "unsupported platform: $platform"
 		exit 1
 		;;
 esac
+
+if [ $doinstall -gt 0 ]; then
+	echo do "$cmd $pkgs"
+	$cmd $pkgs
+else
+	echo $pkgs
+fi

scripts/test-in-container.sh

@@ -8,15 +8,31 @@ cd $script_dir
 
 set -x
 
+# sshd Linsten on 22 and 8022
+echo "Port 22" >> /etc/ssh/sshd_config
+echo "Port 8022" >> /etc/ssh/sshd_config
+
+## Alpine default sshd disables TcpForwarding, which is required for proxyjump test
+sed -i -e 's/AllowTcpForwarding no/AllowTcpForwarding yes/' /etc/ssh/sshd_config
+
 # Run sshd
 if [ ! -e /var/run/sshd.pid ]; then
-	/usr/sbin/sshd
+	/usr/sbin/sshd -E /tmp/sshd.log
 	sleep 1
 fi
 
-ssh-keyscan localhost >> ${HOME}/.ssh/known_hosts
-ssh-keyscan 127.0.0.1 >> ${HOME}/.ssh/known_hosts
-ssh-keyscan ::1 >> ${HOME}/.ssh/known_hosts
+for port in 22 8022; do
+	ssh-keyscan -p $port localhost >> ${HOME}/.ssh/known_hosts
+	ssh-keyscan -p $port ip6-localhost >> ${HOME}/.ssh/known_hosts
+	ssh-keyscan -p $port 127.0.0.1 >> ${HOME}/.ssh/known_hosts
+	ssh-keyscan -p $port ::1 >> ${HOME}/.ssh/known_hosts
+done
 
-# Run test
-python3 -m pytest ../test -v
+
+if [ $# -gt 0 ]; then
+	# command arguments are passed, exec them
+	"$@"
+else
+	# no arguments passed, run the test
+	python3 -m pytest -v ../test
+fi

src/bwlimit.c

@@ -0,0 +1,92 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
+#include <errno.h>
+
+#include <bwlimit.h>
+#include <platform.h>
+
+#define timespeczerorize(ts)    \
+	do {                    \
+		ts.tv_sec = 0;  \
+		ts.tv_nsec = 0; \
+	} while (0)
+
+int bwlimit_init(struct bwlimit *bw, uint64_t bps, uint64_t win)
+{
+	if (!(bw->sem = sem_create(1)))
+		return -1;
+
+	bw->bps = bps;
+	bw->win = win; /* msec window */
+	bw->amt = (double)bps / 8 / 1000 * win; /* bytes in a window (msec) */
+	bw->credit = bw->amt;
+	timespeczerorize(bw->wstart);
+	timespeczerorize(bw->wend);
+
+	return 0;
+}
+
+#define timespecisset(ts) ((ts).tv_sec || (ts).tv_nsec)
+
+#define timespecmsadd(a, msec, r)                                \
+	do {                                                     \
+		(r).tv_sec = (a).tv_sec;                         \
+		(r).tv_nsec = (a).tv_nsec + (msec * 1000000);    \
+		if ((r).tv_nsec > 1000000000) {                  \
+			(r).tv_sec += (r.tv_nsec) / 1000000000L; \
+			(r).tv_nsec = (r.tv_nsec) % 1000000000L; \
+		}                                                \
+	} while (0)
+
+#define timespecsub(a, b, r)                             \
+	do {                                             \
+		(r).tv_sec = (a).tv_sec - (b).tv_sec;    \
+		(r).tv_nsec = (a).tv_nsec - (b).tv_nsec; \
+		if ((r).tv_nsec < 0) {                   \
+			(r).tv_sec -= 1;                 \
+			(r).tv_nsec += 1000000000;       \
+		}                                        \
+	} while (0)
+
+#define timespeccmp(a, b, expr) \
+	((a.tv_sec * 1000000000 + a.tv_nsec) expr(b.tv_sec * 1000000000 + b.tv_nsec))
+
+int bwlimit_wait(struct bwlimit *bw, size_t nr_bytes)
+{
+	struct timespec now, end, rq, rm;
+
+	if (bw->bps == 0)
+		return 0; /* no bandwidth limit */
+
+	if (sem_wait(bw->sem) < 0)
+		return -1;
+
+	clock_gettime(CLOCK_MONOTONIC, &now);
+
+	if (!timespecisset(bw->wstart)) {
+		bw->wstart = now;
+		timespecmsadd(bw->wstart, bw->win, bw->wend);
+	}
+
+	bw->credit -= nr_bytes;
+
+	if (bw->credit < 0) {
+		/* no more credit on this window. sleep until the end
+		 * of this window + additional time for the remaining
+		 * bytes. */
+		uint64_t addition = (double)(bw->credit * -1) / (bw->bps / 8);
+		timespecmsadd(bw->wend, addition * 1000, end);
+		if (timespeccmp(end, now, >)) {
+			timespecsub(end, now, rq);
+			while (nanosleep(&rq, &rm) == -1) {
+				if (errno != EINTR)
+					break;
+				rq = rm;
+			}
+		}
+		bw->credit = bw->amt;
+		timespeczerorize(bw->wstart);
+	}
+
+	sem_post(bw->sem);
+	return 0;
+}

src/bwlimit.h

@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
+#ifndef _BWLIMIT_H_
+#define _BWLIMIT_H_
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <sys/types.h>
+#include <time.h>
+#include <semaphore.h>
+
+struct bwlimit {
+	sem_t	*sem;	/* semaphore */
+	size_t	bps;	/* limit bit-rate (bps) */
+	size_t	win;	/* window size (msec) */
+	size_t	amt;	/* amount of bytes can be sent in a window */
+
+	ssize_t		credit;	/* remaining bytes can be sent in a window */
+	struct timespec wstart, wend; /* window start time and end time */
+};
+
+int bwlimit_init(struct bwlimit *bw, uint64_t bps, uint64_t win);
+/* if bps is 0, it means that bwlimit is not active. If so,
+ * bwlimit_wait() returns immediately. */
+
+int bwlimit_wait(struct bwlimit *bw, size_t nr_bytes);
+
+
+#endif /* _BWLIMIT_H_ */

src/checkpoint.c

@@ -0,0 +1,505 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
+#include <fcntl.h>
+#include <sys/uio.h>
+#include <arpa/inet.h>
+
+#include <path.h>
+#include <print.h>
+#include <platform.h>
+#include <strerrno.h>
+#include <openbsd-compat/openbsd-compat.h>
+
+#include <checkpoint.h>
+
+#define MSCP_CHECKPOINT_MAGIC 0x7063736dUL /* mscp in ascii */
+#define MSCP_CHECKPOINT_VERSION 0x1
+
+/**
+ * mscp checkpoint file format. All values are network byte order.
+ *
+ * The file starts with the File header:
+ * 0                   1                   2                   3
+ *  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+ * +---------------------------------------------------------------+
+ * |                          Magic Code                           |
+ * +---------------+-----------------------------------------------+
+ * |     Version   |
+ * +---------------+
+ *
+ * Magic code: 0x7063736dUL
+ *
+ * Version: 1.
+ *
+ *
+ * Each object in a checkpoint always starts with an object header:
+ * +---------------+---------------+-------------------------------+
+ * |     Type      |      rsv      |             Length            |
+ * +---------------+---------------+-------------------------------+
+ *
+ * Type: 0x0A (meta), 0x0B (path), or 0x0C (chunk)
+ *
+ * Rsv: reserved
+ *
+ * Length: Length of this object including the object header.
+ *
+ *
+ * Meta object provides generaic information for the failed copy:
+ * +---------------+---------------+-------------------------------+
+ * |     Type      |      rsv      |             Length            |
+ * +---------------+---------------+-------------------------------+
+ * |  Direction    | Remote string ...
+ * +---------------+------------------
+ *
+ * Direction: 1 (Local-to-Remote copy) or 2 (Remote-to-Local copy)
+ *
+ * Remote string: Remote host, e.g., user@hostname and IP address,
+ * string including '\0'.
+ *
+ *
+ * Path object represnts a file with sourcen and destination paths:
+ * +---------------+---------------+-------------------------------+
+ * |     Type      |      rsv      |             Length            |
+ * +---------------+---------------+-------------------------------+
+ * |                             Index                             |
+ * +-------------------------------+-------------------------------+
+ * |         Source offset         |      Destination offset       |
+ * +-------------------------------+-------------------------------+
+ * //                                                             //
+ * //                     Source path string                      //
+ * //                                                             //
+ * +---------------------------------------------------------------+
+ * //                                                             //
+ * //                   Destination path string                   //
+ * //                                                             //
+ * +---------------------------------------------------------------+
+ *
+ * Index: 32-bit unsigned int indicating this path (used by chunks)
+ *
+ * Source offset: Offset of the Source path string from the head of
+ * this object. It is identical to the end of the Destination offset
+ * filed.
+ *
+ * Destination offset: Offset of the Destnation path string from the
+ * head of this object. It also indicates the end of the Source path
+ * string.
+ *
+ * Source path string: String of copy source path (including '\0').
+ *
+ * Destination path string: string of copy destination path (including
+ * '\0').
+ *
+ *
+ * Chunk object represents a chunk associated with a path object:
+ * +---------------+---------------+-------------------------------+
+ * |     Type      |      rsv      |             Length            |
+ * +---------------+---------------+-------------------------------+
+ * |                             Index                             |
+ * +---------------------------------------------------------------+
+ * |                             Chunk                             |
+ * |                             offset                            |
+ * +---------------------------------------------------------------+
+ * |                             Chunk                             |
+ * |                             length                            |
+ * +---------------------------------------------------------------+
+ *
+ * Index: 32 bit unsigned int indicating the index of a path object
+ * this chunk associated with.
+ *
+ * Chunk offset: 64 bit unsigned int indicating the offset of this
+ * chunk from the head of the associating a file.
+ *
+ * Chunk length: 64 bit unsigned int indicating the length (bytes) of
+ * this chunk.
+ */
+
+enum {
+	OBJ_TYPE_META = 0x0A,
+	OBJ_TYPE_PATH = 0x0B,
+	OBJ_TYPE_CHUNK = 0x0C,
+};
+
+struct checkpoint_file_hdr {
+	uint32_t magic;
+	uint8_t version;
+} __attribute__((packed));
+
+struct checkpoint_obj_hdr {
+	uint8_t type;
+	uint8_t rsv;
+	uint16_t len; /* length of an object including this hdr */
+} __attribute__((packed));
+
+struct checkpoint_obj_meta {
+	struct checkpoint_obj_hdr hdr;
+	uint8_t direction; /* L2R or R2L */
+
+	char remote[0];
+} __attribute__((packed));
+
+struct checkpoint_obj_path {
+	struct checkpoint_obj_hdr hdr;
+
+	uint32_t idx;
+	uint16_t src_off; /* offset to the src path string (including
+			   * \0) from the head of this object. */
+	uint16_t dst_off; /* offset to the dst path string (including
+			   * \0) from the head of this object */
+} __attribute__((packed));
+
+#define obj_path_src(o) ((char *)(o) + ntohs(o->src_off))
+#define obj_path_dst(o) ((char *)(o) + ntohs(o->dst_off))
+
+#define obj_path_src_len(o) (ntohs(o->dst_off) - ntohs(o->src_off))
+#define obj_path_dst_len(o) (ntohs(o->hdr.len) - ntohs(o->dst_off))
+
+#define obj_path_validate(o)				\
+	((ntohs(o->hdr.len) > ntohs(o->dst_off)) &&	\
+	 (ntohs(o->dst_off) > ntohs(o->src_off)) &&	\
+	 (obj_path_src_len(o) < PATH_MAX) &&		\
+	 (obj_path_dst_len(o) < PATH_MAX))
+
+struct checkpoint_obj_chunk {
+	struct checkpoint_obj_hdr hdr;
+
+	uint32_t idx; /* index indicating associating path */
+	uint64_t off;
+	uint64_t len;
+} __attribute__((packed));
+
+#define CHECKPOINT_OBJ_MAXLEN (sizeof(struct checkpoint_obj_path) + PATH_MAX * 2)
+
+static int checkpoint_write_path(int fd, struct path *p, unsigned int idx)
+{
+	char buf[CHECKPOINT_OBJ_MAXLEN];
+	struct checkpoint_obj_path *path = (struct checkpoint_obj_path *)buf;
+	size_t src_len, dst_len;
+	struct iovec iov[3];
+
+	p->data = idx; /* save idx to be pointed by chunks */
+
+	src_len = strlen(p->path) + 1;
+	dst_len = strlen(p->dst_path) + 1;
+
+	memset(buf, 0, sizeof(buf));
+	path->hdr.type = OBJ_TYPE_PATH;
+	path->hdr.len = htons(sizeof(*path) + src_len + dst_len);
+
+	path->idx = htonl(idx);
+	path->src_off = htons(sizeof(*path));
+	path->dst_off = htons(sizeof(*path) + src_len);
+
+	iov[0].iov_base = path;
+	iov[0].iov_len = sizeof(*path);
+	iov[1].iov_base = p->path;
+	iov[1].iov_len = src_len;
+	iov[2].iov_base = p->dst_path;
+	iov[2].iov_len = dst_len;
+
+	if (writev(fd, iov, 3) < 0) {
+		priv_set_errv("writev: %s", strerrno());
+		return -1;
+	}
+	return 0;
+}
+
+static int checkpoint_write_chunk(int fd, struct chunk *c)
+{
+	struct checkpoint_obj_chunk chunk;
+
+	memset(&chunk, 0, sizeof(chunk));
+	chunk.hdr.type = OBJ_TYPE_CHUNK;
+	chunk.hdr.len = htons(sizeof(chunk));
+
+	chunk.idx = htonl(c->p->data); /* index stored by checkpoint_write_path */
+	chunk.off = htonll(c->off);
+	chunk.len = htonll(c->len);
+
+	if (write(fd, &chunk, sizeof(chunk)) < 0) {
+		priv_set_errv("writev: %s", strerrno());
+		return -1;
+	}
+	return 0;
+}
+
+int checkpoint_save(const char *pathname, int dir, const char *user, const char *remote,
+		    pool *path_pool, pool *chunk_pool)
+{
+	struct checkpoint_file_hdr hdr;
+	struct checkpoint_obj_meta meta;
+	struct iovec iov[3];
+	struct chunk *c;
+	struct path *p;
+	char buf[1024];
+	unsigned int i, nr_paths, nr_chunks;
+	int fd, ret;
+
+	fd = open(pathname, O_WRONLY | O_CREAT | O_TRUNC,
+		  S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH);
+	if (fd < 0) {
+		priv_set_errv("open: %s: %s", pathname, strerrno());
+		return -1;
+	}
+
+	/* write file hdr */
+	hdr.magic = htonl(MSCP_CHECKPOINT_MAGIC);
+	hdr.version = MSCP_CHECKPOINT_VERSION;
+
+	/* write meta */
+	if (user)
+		ret = snprintf(buf, sizeof(buf), "%s@%s", user, remote);
+	else
+		ret = snprintf(buf, sizeof(buf), "%s", remote);
+	if (ret >= sizeof(buf)) {
+		priv_set_errv("too long username and/or remote");
+		return -1;
+	}
+
+	memset(&meta, 0, sizeof(meta));
+	meta.hdr.type = OBJ_TYPE_META;
+	meta.hdr.len = htons(sizeof(meta) + strlen(buf) + 1);
+	meta.direction = dir;
+
+	iov[0].iov_base = &hdr;
+	iov[0].iov_len = sizeof(hdr);
+	iov[1].iov_base = &meta;
+	iov[1].iov_len = sizeof(meta);
+	iov[2].iov_base = buf;
+	iov[2].iov_len = strlen(buf) + 1;
+
+	if (writev(fd, iov, 3) < 0) {
+		priv_set_errv("writev: %s", strerrno());
+		return -1;
+	}
+
+	/* write paths */
+	nr_paths = 0;
+	pool_for_each(path_pool, p, i) {
+		if (p->state == FILE_STATE_DONE)
+			continue;
+		if (checkpoint_write_path(fd, p, nr_paths) < 0)
+			return -1;
+		nr_paths++;
+	}
+
+	/* write chunks */
+	nr_chunks = 0;
+	pool_for_each(chunk_pool, c, i) {
+		if (c->state == CHUNK_STATE_DONE)
+			continue;
+		if (checkpoint_write_chunk(fd, c) < 0)
+			return -1;
+		nr_chunks++;
+	}
+
+	pr_notice("checkpoint: %u paths and %u chunks saved", nr_paths, nr_chunks);
+
+	return 0;
+}
+
+static int checkpoint_load_meta(struct checkpoint_obj_hdr *hdr, char *remote, size_t len,
+				int *dir)
+{
+	struct checkpoint_obj_meta *meta = (struct checkpoint_obj_meta *)hdr;
+
+	if (len < ntohs(hdr->len) - sizeof(*meta)) {
+		priv_set_errv("too short buffer");
+		return -1;
+	}
+	snprintf(remote, len, "%s", meta->remote);
+	*dir = meta->direction;
+
+	pr_notice("checkpoint: remote=%s direction=%s", meta->remote,
+		  meta->direction == MSCP_DIRECTION_L2R ? "local-to-remote" :
+		  meta->direction == MSCP_DIRECTION_R2L ? "remote-to-local" :
+							  "invalid");
+
+	return 0;
+}
+
+static int checkpoint_load_path(struct checkpoint_obj_hdr *hdr, pool *path_pool)
+{
+	struct checkpoint_obj_path *path = (struct checkpoint_obj_path *)hdr;
+	struct path *p;
+	char *s, *d;
+
+	if (!obj_path_validate(path)) {
+		priv_set_errv("invalid path object");
+		return -1;
+	}
+
+	if (!(s = strndup(obj_path_src(path), obj_path_src_len(path)))) {
+		priv_set_errv("strdup: %s", strerrno());
+		return -1;
+	}
+
+	if (!(d = strndup(obj_path_dst(path), obj_path_dst_len(path)))) {
+		priv_set_errv("strdup: %s", strerrno());
+		free(s);
+		return -1;
+	}
+
+	if (!(p = alloc_path(s, d))) {
+		free(s);
+		free(d);
+		return -1;
+	}
+
+	if (pool_push(path_pool, p) < 0) {
+		priv_set_errv("pool_push: %s", strerrno());
+		return -1;
+	}
+
+	pr_info("checkpoint:file: idx=%u %s -> %s", ntohl(path->idx),
+		p->path, p->dst_path);
+
+	return 0;
+}
+
+static int checkpoint_load_chunk(struct checkpoint_obj_hdr *hdr, pool *path_pool,
+				 pool *chunk_pool)
+{
+	struct checkpoint_obj_chunk *chunk = (struct checkpoint_obj_chunk *)hdr;
+	struct chunk *c;
+	struct path *p;
+
+	if (!(p = pool_get(path_pool, ntohl(chunk->idx)))) {
+		/* we assumes all paths are already loaded in the order */
+		priv_set_errv("path index %u not found", ntohl(chunk->idx));
+		return -1;
+	}
+
+	if (!(c = alloc_chunk(p, ntohll(chunk->off), ntohll(chunk->len))))
+		return -1;
+
+	if (pool_push(chunk_pool, c) < 0) {
+		priv_set_errv("pool_push: %s", strerrno());
+		return -1;
+	}
+
+	pr_debug("checkpoint:chunk: idx=%u %s 0x%lx-0x%lx", ntohl(chunk->idx),
+		 p->path, c->off, c->off + c->len);
+
+	return 0;
+}
+
+static int checkpoint_read_obj(int fd, void *buf, size_t count)
+{
+	struct checkpoint_obj_hdr *hdr = (struct checkpoint_obj_hdr *)buf;
+	ssize_t ret, objlen, objbuflen;
+
+	memset(buf, 0, count);
+
+	if (count < sizeof(*hdr)) {
+		priv_set_errv("too short buffer");
+		return -1;
+	}
+
+	ret = read(fd, hdr, sizeof(*hdr));
+	if (ret == 0)
+		return 0; /* no more objects */
+	if (ret < 0)
+		return -1;
+
+	objlen = ntohs(hdr->len) - sizeof(*hdr);
+	objbuflen = count - sizeof(*hdr);
+	if (objbuflen < objlen) {
+		priv_set_errv("too short buffer");
+		return -1;
+	}
+
+	ret = read(fd, buf + sizeof(*hdr), objlen);
+	if (ret < objlen) {
+		priv_set_errv("checkpoint truncated");
+		return -1;
+	}
+
+	return 1;
+}
+
+static int checkpoint_read_file_hdr(int fd)
+{
+	struct checkpoint_file_hdr hdr;
+	ssize_t ret;
+
+	ret = read(fd, &hdr, sizeof(hdr));
+	if (ret < 0) {
+		priv_set_errv("read: %s", strerrno());
+		return -1;
+	}
+	if (ret < sizeof(hdr)) {
+		priv_set_errv("checkpoint truncated");
+		return -1;
+	}
+
+	if (ntohl(hdr.magic) != MSCP_CHECKPOINT_MAGIC) {
+		priv_set_errv("checkpoint: invalid megic code");
+		return -1;
+	}
+
+	if (hdr.version != MSCP_CHECKPOINT_VERSION) {
+		priv_set_errv("checkpoint: unknown version %u", hdr.version);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int checkpoint_load(const char *pathname, char *remote, size_t len, int *dir,
+			   pool *path_pool, pool *chunk_pool)
+{
+	char buf[CHECKPOINT_OBJ_MAXLEN];
+	struct checkpoint_obj_hdr *hdr;
+	int fd, ret;
+
+	if ((fd = open(pathname, O_RDONLY)) < 0) {
+		priv_set_errv("open: %s: %s", pathname, strerrno());
+		return -1;
+	}
+
+	if (checkpoint_read_file_hdr(fd) < 0)
+		return -1;
+
+	hdr = (struct checkpoint_obj_hdr *)buf;
+	while ((ret = checkpoint_read_obj(fd, buf, sizeof(buf))) > 0) {
+		switch (hdr->type) {
+		case OBJ_TYPE_META:
+			if (!remote || !dir)
+				break;
+			if (checkpoint_load_meta(hdr, remote, len, dir) < 0)
+				return -1;
+			if (!path_pool || !chunk_pool)
+				goto out;
+			break;
+		case OBJ_TYPE_PATH:
+			if (!path_pool)
+				break;
+			if (checkpoint_load_path(hdr, path_pool) < 0)
+				return -1;
+			break;
+		case OBJ_TYPE_CHUNK:
+			if (!path_pool)
+				break;
+			if (checkpoint_load_chunk(hdr, path_pool, chunk_pool) < 0)
+				return -1;
+			break;
+		default:
+			priv_set_errv("unknown obj type %u", hdr->type);
+			return -1;
+		}
+	}
+
+out:
+	close(fd);
+
+	return 0;
+}
+
+int checkpoint_load_remote(const char *pathname, char *remote, size_t len, int *dir)
+{
+	return checkpoint_load(pathname, remote, len, dir, NULL, NULL);
+}
+
+int checkpoint_load_paths(const char *pathname, pool *path_pool, pool *chunk_pool)
+{
+	return checkpoint_load(pathname, NULL, 0, NULL, path_pool, chunk_pool);
+}

src/checkpoint.h

@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
+#ifndef _CHECKPOINT_H_
+#define _CHECKPOINT_H_
+
+#include <pool.h>
+
+/* checkpoint_save() stores states to a checkponint file (pathname) */
+int checkpoint_save(const char *pathname, int dir, const char *user, const char *remote,
+		    pool *path_pool, pool *chunk_pool);
+
+/* checkpoint_load_meta() reads a checkpoint file (pathname) and returns
+ * remote host string to *remote and transfer direction to *dir.
+ */
+int checkpoint_load_remote(const char *pathname, char *remote, size_t len, int *dir);
+
+/* checkpoint_load_paths() reads a checkpoint file (pathname) and
+ * fills path_pool and chunk_pool.
+ */
+int checkpoint_load_paths(const char *pathname, pool *path_pool, pool *chunk_pool);
+
+#endif /* _CHECKPOINT_H_ */

src/fileops.c

@@ -101,7 +101,6 @@ MDIR *mscp_opendir_wrapped(const char *path)
 
 void mscp_closedir(MDIR *md)
 {
-	int ret;
 	if (md->remote)
 		sftp_closedir(md->remote);
 	else
@@ -323,12 +322,14 @@ int mscp_setstat(const char *path, struct stat *st, bool preserve_ts, sftp_sessi
 		ret = sftp_setstat(sftp, path, &attr);
 		sftp_err_to_errno(sftp);
 	} else {
-		if ((ret = chmod(path, st->st_mode)) < 0)
-			return ret;
 		if ((ret = truncate(path, st->st_size)) < 0)
 			return ret;
-		if (preserve_ts)
-			ret = setutimes(path, st->st_atim, st->st_mtim);
+		if (preserve_ts) {
+			if ((ret = setutimes(path, st->st_atim, st->st_mtim)) < 0)
+				return ret;
+		}
+		if ((ret = chmod(path, st->st_mode)) < 0)
+			return ret;
 	}
 
 	return ret;

src/list.h

@@ -1,573 +0,0 @@
-/**
- * 
- * I grub it from linux kernel source code and fix it for user space
- * program. Of course, this is a GPL licensed header file.
- *
- * Here is a recipe to cook list.h for user space program
- *
- * 1. copy list.h from linux/include/list.h
- * 2. remove 
- *     - #ifdef __KERNE__ and its #endif
- *     - all #include line
- *     - prefetch() and rcu related functions
- * 3. add macro offsetof() and container_of
- *
- * - kazutomo@mcs.anl.gov
- */
-#ifndef _LINUX_LIST_H
-#define _LINUX_LIST_H
-
-/**
- * @name from other kernel headers
- */
-/*@{*/
-
-/**
- * Get offset of a member
- */
-#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
-
-/**
- * Casts a member of a structure out to the containing structure
- * @param ptr        the pointer to the member.
- * @param type       the type of the container struct this is embedded in.
- * @param member     the name of the member within the struct.
- *
- */
-#define container_of(ptr, type, member) ({                      \
-        const typeof( ((type *)0)->member ) *__mptr = (ptr);    \
-        (type *)( (char *)__mptr - offsetof(type,member) );})
-/*@}*/
-
-
-/*
- * These are non-NULL pointers that will result in page faults
- * under normal circumstances, used to verify that nobody uses
- * non-initialized list entries.
- */
-#define LIST_POISON1  ((void *) 0x00100100)
-#define LIST_POISON2  ((void *) 0x00200200)
-
-/**
- * Simple doubly linked list implementation.
- *
- * Some of the internal functions ("__xxx") are useful when
- * manipulating whole lists rather than single entries, as
- * sometimes we already know the next/prev entries and we can
- * generate better code by using them directly rather than
- * using the generic single-entry routines.
- */
-struct list_head {
-	struct list_head *next, *prev;
-};
-
-#define LIST_HEAD_INIT(name) { &(name), &(name) }
-
-#define LIST_HEAD(name) \
-	struct list_head name = LIST_HEAD_INIT(name)
-
-#define INIT_LIST_HEAD(ptr) do { \
-	(ptr)->next = (ptr); (ptr)->prev = (ptr); \
-} while (0)
-
-/*
- * Insert a new entry between two known consecutive entries.
- *
- * This is only for internal list manipulation where we know
- * the prev/next entries already!
- */
-static inline void __list_add(struct list_head *new,
-			      struct list_head *prev,
-			      struct list_head *next)
-{
-	next->prev = new;
-	new->next = next;
-	new->prev = prev;
-	prev->next = new;
-}
-
-/**
- * list_add - add a new entry
- * @new: new entry to be added
- * @head: list head to add it after
- *
- * Insert a new entry after the specified head.
- * This is good for implementing stacks.
- */
-static inline void list_add(struct list_head *new, struct list_head *head)
-{
-	__list_add(new, head, head->next);
-}
-
-/**
- * list_add_tail - add a new entry
- * @new: new entry to be added
- * @head: list head to add it before
- *
- * Insert a new entry before the specified head.
- * This is useful for implementing queues.
- */
-static inline void list_add_tail(struct list_head *new, struct list_head *head)
-{
-	__list_add(new, head->prev, head);
-}
-
-
-/*
- * Delete a list entry by making the prev/next entries
- * point to each other.
- *
- * This is only for internal list manipulation where we know
- * the prev/next entries already!
- */
-static inline void __list_del(struct list_head * prev, struct list_head * next)
-{
-	next->prev = prev;
-	prev->next = next;
-}
-
-/**
- * list_del - deletes entry from list.
- * @entry: the element to delete from the list.
- * Note: list_empty on entry does not return true after this, the entry is
- * in an undefined state.
- */
-static inline void list_del(struct list_head *entry)
-{
-	__list_del(entry->prev, entry->next);
-	entry->next = LIST_POISON1;
-	entry->prev = LIST_POISON2;
-}
-
-
-
-/**
- * list_del_init - deletes entry from list and reinitialize it.
- * @entry: the element to delete from the list.
- */
-static inline void list_del_init(struct list_head *entry)
-{
-	__list_del(entry->prev, entry->next);
-	INIT_LIST_HEAD(entry);
-}
-
-/**
- * list_move - delete from one list and add as another's head
- * @list: the entry to move
- * @head: the head that will precede our entry
- */
-static inline void list_move(struct list_head *list, struct list_head *head)
-{
-        __list_del(list->prev, list->next);
-        list_add(list, head);
-}
-
-/**
- * list_move_tail - delete from one list and add as another's tail
- * @list: the entry to move
- * @head: the head that will follow our entry
- */
-static inline void list_move_tail(struct list_head *list,
-				  struct list_head *head)
-{
-        __list_del(list->prev, list->next);
-        list_add_tail(list, head);
-}
-
-/**
- * list_empty - tests whether a list is empty
- * @head: the list to test.
- */
-static inline int list_empty(const struct list_head *head)
-{
-	return head->next == head;
-}
-
-static inline void __list_splice(struct list_head *list,
-				 struct list_head *head)
-{
-	struct list_head *first = list->next;
-	struct list_head *last = list->prev;
-	struct list_head *at = head->next;
-
-	first->prev = head;
-	head->next = first;
-
-	last->next = at;
-	at->prev = last;
-}
-
-/**
- * list_splice - join two lists
- * @list: the new list to add.
- * @head: the place to add it in the first list.
- */
-static inline void list_splice(struct list_head *list, struct list_head *head)
-{
-	if (!list_empty(list))
-		__list_splice(list, head);
-}
-
-static inline void __list_splice_tail(struct list_head *list,
-				      struct list_head *head)
-{
-	struct list_head *first = list->next;
-	struct list_head *last = list->prev;
-	struct list_head *at = head->prev;
-
-	first->prev = at;
-	at->next = first;
-
-	last->next = head;
-	at->prev = last;
-}
-
-/**
- * list_splice_tail - join two lists
- * @list: the new list to add.
- * @head: the place to add it in the first list.
- */
-static inline void list_splice_tail(struct list_head *list, struct list_head *head)
-{
-	if (!list_empty(list))
-		__list_splice_tail(list, head);
-}
-
-
-/**
- * list_splice_init - join two lists and reinitialise the emptied list.
- * @list: the new list to add.
- * @head: the place to add it in the first list.
- *
- * The list at @list is reinitialised
- */
-static inline void list_splice_init(struct list_head *list,
-				    struct list_head *head)
-{
-	if (!list_empty(list)) {
-		__list_splice(list, head);
-		INIT_LIST_HEAD(list);
-	}
-}
-
-/**
- * list_entry - get the struct for this entry
- * @ptr:	the &struct list_head pointer.
- * @type:	the type of the struct this is embedded in.
- * @member:	the name of the list_struct within the struct.
- */
-#define list_entry(ptr, type, member) \
-	container_of(ptr, type, member)
-
-/**
- * list_for_each	-	iterate over a list
- * @pos:	the &struct list_head to use as a loop counter.
- * @head:	the head for your list.
- */
-
-#define list_for_each(pos, head) \
-  for (pos = (head)->next; pos != (head);	\
-       pos = pos->next)
-
-/**
- * __list_for_each	-	iterate over a list
- * @pos:	the &struct list_head to use as a loop counter.
- * @head:	the head for your list.
- *
- * This variant differs from list_for_each() in that it's the
- * simplest possible list iteration code, no prefetching is done.
- * Use this for code that knows the list to be very short (empty
- * or 1 entry) most of the time.
- */
-#define __list_for_each(pos, head) \
-	for (pos = (head)->next; pos != (head); pos = pos->next)
-
-/**
- * list_for_each_prev	-	iterate over a list backwards
- * @pos:	the &struct list_head to use as a loop counter.
- * @head:	the head for your list.
- */
-#define list_for_each_prev(pos, head) \
-	for (pos = (head)->prev; prefetch(pos->prev), pos != (head); \
-        	pos = pos->prev)
-
-/**
- * list_for_each_safe	-	iterate over a list safe against removal of list entry
- * @pos:	the &struct list_head to use as a loop counter.
- * @n:		another &struct list_head to use as temporary storage
- * @head:	the head for your list.
- */
-#define list_for_each_safe(pos, n, head) \
-	for (pos = (head)->next, n = pos->next; pos != (head); \
-		pos = n, n = pos->next)
-
-/**
- * list_for_each_entry	-	iterate over list of given type
- * @pos:	the type * to use as a loop counter.
- * @head:	the head for your list.
- * @member:	the name of the list_struct within the struct.
- */
-#define list_for_each_entry(pos, head, member)				\
-	for (pos = list_entry((head)->next, typeof(*pos), member);	\
-	     &pos->member != (head);					\
-	     pos = list_entry(pos->member.next, typeof(*pos), member))
-
-/**
- * list_for_each_entry_reverse - iterate backwards over list of given type.
- * @pos:	the type * to use as a loop counter.
- * @head:	the head for your list.
- * @member:	the name of the list_struct within the struct.
- */
-#define list_for_each_entry_reverse(pos, head, member)			\
-	for (pos = list_entry((head)->prev, typeof(*pos), member);	\
-	     &pos->member != (head); 	\
-	     pos = list_entry(pos->member.prev, typeof(*pos), member))
-
-/**
- * list_prepare_entry - prepare a pos entry for use as a start point in
- *			list_for_each_entry_continue
- * @pos:	the type * to use as a start point
- * @head:	the head of the list
- * @member:	the name of the list_struct within the struct.
- */
-#define list_prepare_entry(pos, head, member) \
-	((pos) ? : list_entry(head, typeof(*pos), member))
-
-/**
- * list_for_each_entry_continue -	iterate over list of given type
- *			continuing after existing point
- * @pos:	the type * to use as a loop counter.
- * @head:	the head for your list.
- * @member:	the name of the list_struct within the struct.
- */
-#define list_for_each_entry_continue(pos, head, member) 		\
-	for (pos = list_entry(pos->member.next, typeof(*pos), member);	\
-	     &pos->member != (head);	\
-	     pos = list_entry(pos->member.next, typeof(*pos), member))
-
-/**
- * list_for_each_entry_safe - iterate over list of given type safe against removal of list entry
- * @pos:	the type * to use as a loop counter.
- * @n:		another type * to use as temporary storage
- * @head:	the head for your list.
- * @member:	the name of the list_struct within the struct.
- */
-#define list_for_each_entry_safe(pos, n, head, member)			\
-	for (pos = list_entry((head)->next, typeof(*pos), member),	\
-		n = list_entry(pos->member.next, typeof(*pos), member);	\
-	     &pos->member != (head); 					\
-	     pos = n, n = list_entry(n->member.next, typeof(*n), member))
-
-/**
- * list_for_each_entry_safe_continue -	iterate over list of given type
- *			continuing after existing point safe against removal of list entry
- * @pos:	the type * to use as a loop counter.
- * @n:		another type * to use as temporary storage
- * @head:	the head for your list.
- * @member:	the name of the list_struct within the struct.
- */
-#define list_for_each_entry_safe_continue(pos, n, head, member) 		\
-	for (pos = list_entry(pos->member.next, typeof(*pos), member), 		\
-		n = list_entry(pos->member.next, typeof(*pos), member);		\
-	     &pos->member != (head);						\
-	     pos = n, n = list_entry(n->member.next, typeof(*n), member))
-
-/**
- * list_for_each_entry_safe_reverse - iterate backwards over list of given type safe against
- *				      removal of list entry
- * @pos:	the type * to use as a loop counter.
- * @n:		another type * to use as temporary storage
- * @head:	the head for your list.
- * @member:	the name of the list_struct within the struct.
- */
-#define list_for_each_entry_safe_reverse(pos, n, head, member)		\
-	for (pos = list_entry((head)->prev, typeof(*pos), member),	\
-		n = list_entry(pos->member.prev, typeof(*pos), member);	\
-	     &pos->member != (head); 					\
-	     pos = n, n = list_entry(n->member.prev, typeof(*n), member))
-
-
-
-
-/*
- * Double linked lists with a single pointer list head.
- * Mostly useful for hash tables where the two pointer list head is
- * too wasteful.
- * You lose the ability to access the tail in O(1).
- */
-
-struct hlist_head {
-	struct hlist_node *first;
-};
-
-struct hlist_node {
-	struct hlist_node *next, **pprev;
-};
-
-#define HLIST_HEAD_INIT { .first = NULL }
-#define HLIST_HEAD(name) struct hlist_head name = {  .first = NULL }
-#define INIT_HLIST_HEAD(ptr) ((ptr)->first = NULL)
-#define INIT_HLIST_NODE(ptr) ((ptr)->next = NULL, (ptr)->pprev = NULL)
-
-static inline int hlist_unhashed(const struct hlist_node *h)
-{
-	return !h->pprev;
-}
-
-static inline int hlist_empty(const struct hlist_head *h)
-{
-	return !h->first;
-}
-
-static inline void __hlist_del(struct hlist_node *n)
-{
-	struct hlist_node *next = n->next;
-	struct hlist_node **pprev = n->pprev;
-	*pprev = next;
-	if (next)
-		next->pprev = pprev;
-}
-
-static inline void hlist_del(struct hlist_node *n)
-{
-	__hlist_del(n);
-	n->next = LIST_POISON1;
-	n->pprev = LIST_POISON2;
-}
-
-
-static inline void hlist_del_init(struct hlist_node *n)
-{
-	if (n->pprev)  {
-		__hlist_del(n);
-		INIT_HLIST_NODE(n);
-	}
-}
-
-static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h)
-{
-	struct hlist_node *first = h->first;
-	n->next = first;
-	if (first)
-		first->pprev = &n->next;
-	h->first = n;
-	n->pprev = &h->first;
-}
-
-
-
-/* next must be != NULL */
-static inline void hlist_add_before(struct hlist_node *n,
-					struct hlist_node *next)
-{
-	n->pprev = next->pprev;
-	n->next = next;
-	next->pprev = &n->next;
-	*(n->pprev) = n;
-}
-
-static inline void hlist_add_after(struct hlist_node *n,
-					struct hlist_node *next)
-{
-	next->next = n->next;
-	n->next = next;
-	next->pprev = &n->next;
-
-	if(next->next)
-		next->next->pprev  = &next->next;
-}
-
-
-
-#define hlist_entry(ptr, type, member) container_of(ptr,type,member)
-
-#define hlist_for_each(pos, head) \
-	for (pos = (head)->first; pos && ({ prefetch(pos->next); 1; }); \
-	     pos = pos->next)
-
-#define hlist_for_each_safe(pos, n, head) \
-	for (pos = (head)->first; pos && ({ n = pos->next; 1; }); \
-	     pos = n)
-
-/**
- * hlist_for_each_entry	- iterate over list of given type
- * @tpos:	the type * to use as a loop counter.
- * @pos:	the &struct hlist_node to use as a loop counter.
- * @head:	the head for your list.
- * @member:	the name of the hlist_node within the struct.
- */
-#define hlist_for_each_entry(tpos, pos, head, member)			 \
-	for (pos = (head)->first;					 \
-	     pos && ({ prefetch(pos->next); 1;}) &&			 \
-		({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
-	     pos = pos->next)
-
-/**
- * hlist_for_each_entry_continue - iterate over a hlist continuing after existing point
- * @tpos:	the type * to use as a loop counter.
- * @pos:	the &struct hlist_node to use as a loop counter.
- * @member:	the name of the hlist_node within the struct.
- */
-#define hlist_for_each_entry_continue(tpos, pos, member)		 \
-	for (pos = (pos)->next;						 \
-	     pos && ({ prefetch(pos->next); 1;}) &&			 \
-		({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
-	     pos = pos->next)
-
-/**
- * hlist_for_each_entry_from - iterate over a hlist continuing from existing point
- * @tpos:	the type * to use as a loop counter.
- * @pos:	the &struct hlist_node to use as a loop counter.
- * @member:	the name of the hlist_node within the struct.
- */
-#define hlist_for_each_entry_from(tpos, pos, member)			 \
-	for (; pos && ({ prefetch(pos->next); 1;}) &&			 \
-		({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
-	     pos = pos->next)
-
-/**
- * hlist_for_each_entry_safe - iterate over list of given type safe against removal of list entry
- * @tpos:	the type * to use as a loop counter.
- * @pos:	the &struct hlist_node to use as a loop counter.
- * @n:		another &struct hlist_node to use as temporary storage
- * @head:	the head for your list.
- * @member:	the name of the hlist_node within the struct.
- */
-#define hlist_for_each_entry_safe(tpos, pos, n, head, member) 		 \
-	for (pos = (head)->first;					 \
-	     pos && ({ n = pos->next; 1; }) && 				 \
-		({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
-	     pos = n)
-
-
-/**
- * list_count - return length of list
- * @head        the head for your list.
- */
-static inline int list_count(struct list_head *head)
-{
-	int n = 0;
-	struct list_head *p;
-
-	list_for_each(p, head) n++;
-	return n;
-}
-
-
-/**
- * list_free_f - free items in a list with a function
- * @head	the heaf for your list.
- * @f           function that releases an item in the list.
- */
-static inline void list_free_f(struct list_head *head, void (*f)(struct list_head *))
-{
-	struct list_head *p, *n;
-
-        list_for_each_safe(p, n, head) {
-                list_del(p);
-                f(p);
-        }
-}
-
-#endif
-

src/main.c

@@ -2,6 +2,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <stdarg.h>
+#include <fcntl.h>
 #include <unistd.h>
 #include <limits.h>
 #include <math.h>
@@ -18,17 +19,19 @@
 #include <strerrno.h>
 #include <print.h>
 
-#include "config.h"
+#include <config.h>
 
 void usage(bool print_help)
 {
 	printf("mscp " MSCP_BUILD_VERSION ": copy files over multiple SSH connections\n"
 	       "\n"
-	       "Usage: mscp [-46vqDpHdNh] [-n nr_conns] [-m coremask]\n"
-	       "            [-u max_startups] [-I interval]\n"
-	       "            [-s min_chunk_sz] [-S max_chunk_sz] [-a nr_ahead] [-b buf_sz]\n"
-	       "            [-l login_name] [-P port] [-F ssh_config] [-i identity_file]\n"
-	       "            [-c cipher_spec] [-M hmac_spec] [-C compress] [-g congestion]\n"
+	       "Usage: mscp [-46vqDpdNh] [-n nr_conns] [-m coremask] [-u max_startups]\n"
+	       "            [-I interval] [-W checkpoint] [-R checkpoint]\n"
+	       "            [-s min_chunk_sz] [-S max_chunk_sz] [-a nr_ahead]\n"
+	       "            [-b buf_sz] [-L limit_bitrate]\n"
+	       "            [-l login_name] [-P port] [-F ssh_config] [-o ssh_option]\n"
+	       "            [-i identity_file] [-J destination] [-c cipher_spec] [-M hmac_spec]\n"
+	       "            [-C compress] [-g congestion]\n"
 	       "            source ... target\n"
 	       "\n");
 
@@ -38,14 +41,17 @@ void usage(bool print_help)
 	printf("    -n NR_CONNECTIONS  number of connections "
 	       "(default: floor(log(cores)*2)+1)\n"
 	       "    -m COREMASK        hex value to specify cores where threads pinned\n"
-	       "    -u MAX_STARTUPS    number of concurrent SSH connection attempts "
+	       "    -u MAX_STARTUPS    number of concurrent unauthed SSH attempts "
 	       "(default: 8)\n"
 	       "    -I INTERVAL        interval between SSH connection attempts (default: 0)\n"
+	       "    -W CHECKPOINT      write states to the checkpoint if transfer fails\n"
+	       "    -R CHECKPOINT      resume transferring from the checkpoint\n"
 	       "\n"
-	       "    -s MIN_CHUNK_SIZE  min chunk size (default: 64MB)\n"
-	       "    -S MAX_CHUNK_SIZE  max chunk size (default: filesize/nr_conn)\n"
+	       "    -s MIN_CHUNK_SIZE  min chunk size (default: 16M bytes)\n"
+	       "    -S MAX_CHUNK_SIZE  max chunk size (default: filesize/nr_conn/4)\n"
 	       "    -a NR_AHEAD        number of inflight SFTP commands (default: 32)\n"
 	       "    -b BUF_SZ          buffer size for i/o and transfer\n"
+	       "    -L LIMIT_BITRATE   Limit the bitrate, n[KMG] (default: 0, no limit)\n"
 	       "\n"
 	       "    -4                 use IPv4\n"
 	       "    -6                 use IPv6\n"
@@ -56,19 +62,40 @@ void usage(bool print_help)
 	       "\n"
 	       "    -l LOGIN_NAME      login name\n"
 	       "    -P PORT            port number\n"
-	       "    -F CONFIG          path to user ssh config (default ~/.ssh/config)\n"
+	       "    -F SSH_CONFIG      path to user ssh config (default ~/.ssh/config)\n"
+	       "    -o SSH_OPTION      ssh_config option\n"
 	       "    -i IDENTITY        identity file for public key authentication\n"
+	       "    -J DESTINATION     ProxyJump destination\n"
 	       "    -c CIPHER          cipher spec\n"
 	       "    -M HMAC            hmac spec\n"
 	       "    -C COMPRESS        enable compression: "
 	       "yes, no, zlib, zlib@openssh.com\n"
 	       "    -g CONGESTION      specify TCP congestion control algorithm\n"
 	       "    -p                 preserve timestamps of files\n"
-	       "    -H                 disable hostkey check\n"
 	       "    -d                 increment ssh debug output level\n"
 	       "    -N                 enable Nagle's algorithm (default disabled)\n"
 	       "    -h                 print this help\n"
 	       "\n");
+
+	const char **ciphers = mscp_ssh_ciphers();
+	const char **hmacs = mscp_ssh_hmacs();
+	int n;
+
+	printf("Available ciphers: ");
+	for (n = 0; ciphers[n] != NULL; n++) {
+		printf("%s", ciphers[n]);
+		if (ciphers[n + 1])
+			printf(", ");
+	}
+	printf("\n\n");
+
+	printf("Available hmacs: ");
+	for (n = 0; hmacs[n] != NULL; n++) {
+		printf("%s", hmacs[n]);
+		if (hmacs[n + 1])
+			printf(", ");
+	}
+	printf("\n\n");
 }
 
 char *strip_brackets(char *s)
@@ -90,8 +117,6 @@ char *split_user_host_path(const char *s, char **userp, char **hostp, char **pat
 		return NULL;
 	}
 
-	user = NULL;
-	host = NULL;
 	path = tmp;
 	for (cp = tmp; *cp; cp++) {
 		if (*cp == '@' && (cp > tmp) && *(cp - 1) != '\\' && user == NULL) {
@@ -178,13 +203,12 @@ struct target *validate_targets(char **arg, int len)
 	 */
 
 	struct target *t, *t0;
-	int n;
+	int n, nslash;
 
 	if ((t = calloc(len, sizeof(struct target))) == NULL) {
 		pr_err("calloc: %s", strerrno());
 		return NULL;
 	}
-	memset(t, 0, len * sizeof(struct target));
 
 	/* split remote:path into remote and path */
 	for (n = 0; n < len; n++) {
@@ -196,9 +220,33 @@ struct target *validate_targets(char **arg, int len)
 		}
 	}
 
-	/* check all user@host are identical. t[len - 1] is destination,
-	 * so we need to check t[0] to t[len - 2] having the identical
-	 * remote notation */
+	/* expand remote path, e.g., empty dst path and '~' */
+	for (n = 0; n < len; n++) {
+		if (!t[n].host)
+			continue;
+
+		/* this target is a remote path. check the path and
+		 * expand it. this part is derived from
+		 * openssh-portal prepare_remote_path() function.
+		 */
+		char *path = t[n].path;
+		if (*path == '\0' || strcmp(path, "~") == 0)
+			t[n].path = strdup(".");
+		else if (strncmp(path, "~/", 2) == 0) {
+			if ((nslash = strspn(path + 2, "/")) == strlen(path + 2))
+				t[n].path = strdup(".");
+			else
+				t[n].path = strdup(path + 2 + nslash);
+		}
+		if (!t[n].path) {
+			pr_err("strdup failed: %s", strerrno());
+			goto free_target_out;
+		}
+	}
+
+	/* check all user@host are identical. t[len - 1] is the
+	 * destination, so we need to check t[0] to t[len - 2] having
+	 * the identical remote notation */
 	t0 = &t[0];
 	for (n = 1; n < len - 1; n++) {
 		if (compare_remote(t0, &t[n]) != 0)
@@ -233,12 +281,12 @@ free_target_out:
 
 struct mscp *m = NULL;
 pthread_t tid_stat = 0;
+bool interrupted = false;
 
 void sigint_handler(int sig)
 {
+	interrupted = true;
 	mscp_stop(m);
-	if (tid_stat > 0)
-		pthread_cancel(tid_stat);
 }
 
 void *print_stat_thread(void *arg);
@@ -252,22 +300,73 @@ void print_cli(const char *fmt, ...)
 	va_end(va);
 }
 
+void print_stat(bool final);
+
+long atol_with_unit(char *value, bool i)
+{
+	/* value must be "\d+[kKmMgG]?" */
+
+	char *u = value + (strlen(value) - 1);
+	long k = i ? 1024 : 1000;
+	long factor = 1;
+	long v;
+
+	switch (*u) {
+	case 'k':
+	case 'K':
+		factor = k;
+		*u = '\0';
+		break;
+	case 'm':
+	case 'M':
+		factor = k * k;
+		*u = '\0';
+		break;
+	case 'g':
+	case 'G':
+		factor = k * k * k;
+		*u = '\0';
+		break;
+	}
+
+	v = atol(value);
+	return v * factor;
+}
+
+int to_dev_null(int fd)
+{
+	int nfd = open("/dev/null", O_WRONLY);
+	if (nfd < 0) {
+		pr_err("open /dev/null: %s", strerrno());
+		return -1;
+	}
+
+	if (dup2(nfd, fd) < 0) {
+		pr_err("dup2: %s", strerrno());
+		return -1;
+	}
+
+	close(nfd);
+
+	return 0;
+}
+
 int main(int argc, char **argv)
 {
 	struct mscp_ssh_opts s;
 	struct mscp_opts o;
 	struct target *t;
-	int pipe_fd[2];
 	int ch, n, i, ret;
 	int direction = 0;
-	char *remote;
-	bool dryrun = false;
+	char *remote = NULL, *checkpoint_save = NULL, *checkpoint_load = NULL;
+	bool quiet = false, dryrun = false, resume = false;
+	int nr_options = 0;
 
 	memset(&s, 0, sizeof(s));
 	memset(&o, 0, sizeof(o));
 	o.severity = MSCP_SEVERITY_WARN;
 
-#define mscpopts "n:m:u:I:s:S:a:b:46vqDrl:P:i:F:c:M:C:g:pHdNh"
+#define mscpopts "n:m:u:I:W:R:s:S:a:b:L:46vqDrl:P:F:o:i:J:c:M:C:g:pdNh"
 	while ((ch = getopt(argc, argv, mscpopts)) != -1) {
 		switch (ch) {
 		case 'n':
@@ -286,17 +385,27 @@ int main(int argc, char **argv)
 		case 'I':
 			o.interval = atoi(optarg);
 			break;
+		case 'W':
+			checkpoint_save = optarg;
+			break;
+		case 'R':
+			checkpoint_load = optarg;
+			resume = true;
+			break;
 		case 's':
-			o.min_chunk_sz = atoi(optarg);
+			o.min_chunk_sz = atol_with_unit(optarg, true);
 			break;
 		case 'S':
-			o.max_chunk_sz = atoi(optarg);
+			o.max_chunk_sz = atol_with_unit(optarg, true);
 			break;
 		case 'a':
 			o.nr_ahead = atoi(optarg);
 			break;
 		case 'b':
-			o.buf_sz = atoi(optarg);
+			o.buf_sz = atol_with_unit(optarg, true);
+			break;
+		case 'L':
+			o.bitrate = atol_with_unit(optarg, false);
 			break;
 		case '4':
 			s.ai_family = AF_INET;
@@ -308,7 +417,7 @@ int main(int argc, char **argv)
 			o.severity++;
 			break;
 		case 'q':
-			o.severity = MSCP_SEVERITY_NONE;
+			quiet = true;
 			break;
 		case 'D':
 			dryrun = true;
@@ -325,9 +434,22 @@ int main(int argc, char **argv)
 		case 'F':
 			s.config = optarg;
 			break;
+		case 'o':
+			nr_options++;
+			s.options = realloc(s.options, sizeof(char *) * (nr_options + 1));
+			if (!s.options) {
+				pr_err("realloc: %s", strerrno());
+				return 1;
+			}
+			s.options[nr_options - 1] = optarg;
+			s.options[nr_options] = NULL;
+			break;
 		case 'i':
 			s.identity = optarg;
 			break;
+		case 'J':
+			s.proxyjump = optarg;
+			break;
 		case 'c':
 			s.cipher = optarg;
 			break;
@@ -343,9 +465,6 @@ int main(int argc, char **argv)
 		case 'p':
 			o.preserve_ts = true;
 			break;
-		case 'H':
-			s.no_hostkey_check = true;
-			break;
 		case 'd':
 			s.debug_level++;
 			break;
@@ -361,56 +480,99 @@ int main(int argc, char **argv)
 		}
 	}
 
+	if (quiet)
+		to_dev_null(STDOUT_FILENO);
+
 	s.password = getenv(ENV_SSH_AUTH_PASSWORD);
 	s.passphrase = getenv(ENV_SSH_AUTH_PASSPHRASE);
 
-	if (argc - optind < 2) {
-		/* mscp needs at lease 2 (src and target) argument */
-		usage(false);
-		return 1;
-	}
-	i = argc - optind;
-
-	if ((t = validate_targets(argv + optind, i)) == NULL)
-		return -1;
-
-	if (t[0].host) {
-		/* copy remote to local */
-		direction = MSCP_DIRECTION_R2L;
-		remote = t[0].host;
-		s.login_name = s.login_name ? s.login_name : t[0].user;
-	} else {
-		/* copy local to remote */
-		direction = MSCP_DIRECTION_L2R;
-		remote = t[i - 1].host;
-		s.login_name = s.login_name ? s.login_name : t[i - 1].user;
-	}
-
-	if ((m = mscp_init(remote, direction, &o, &s)) == NULL) {
+	if ((m = mscp_init(&o, &s)) == NULL) {
 		pr_err("mscp_init: %s", priv_get_err());
 		return -1;
 	}
 
-	if (mscp_connect(m) < 0) {
-		pr_err("mscp_connect: %s", priv_get_err());
-		return -1;
-	}
+	if (!resume) {
+		/* normal transfer (not resume) */
+		if (argc - optind < 2) {
+			/* mscp needs at lease 2 (src and target) argument */
+			usage(false);
+			return 1;
+		}
+		i = argc - optind;
 
-	for (n = 0; n < i - 1; n++) {
-		if (mscp_add_src_path(m, t[n].path) < 0) {
-			pr_err("mscp_add_src_path: %s", priv_get_err());
+		if ((t = validate_targets(argv + optind, i)) == NULL)
+			return -1;
+
+		if (t[0].host) {
+			/* copy remote to local */
+			direction = MSCP_DIRECTION_R2L;
+			remote = t[0].host;
+			s.login_name = s.login_name ? s.login_name : t[0].user;
+		} else {
+			/* copy local to remote */
+			direction = MSCP_DIRECTION_L2R;
+			remote = t[i - 1].host;
+			s.login_name = s.login_name ? s.login_name : t[i - 1].user;
+		}
+
+		if (mscp_set_remote(m, remote, direction) < 0) {
+			pr_err("mscp_set_remote: %s", priv_get_err());
 			return -1;
 		}
-	}
 
-	if (mscp_set_dst_path(m, t[i - 1].path) < 0) {
-		pr_err("mscp_set_dst_path: %s", priv_get_err());
-		return -1;
-	}
+		if (mscp_connect(m) < 0) {
+			pr_err("mscp_connect: %s", priv_get_err());
+			return -1;
+		}
 
-	if (mscp_scan(m) < 0) {
-		pr_err("mscp_scan: %s", priv_get_err());
-		return -1;
+		for (n = 0; n < i - 1; n++) {
+			if (mscp_add_src_path(m, t[n].path) < 0) {
+				pr_err("mscp_add_src_path: %s", priv_get_err());
+				return -1;
+			}
+		}
+
+		if (mscp_set_dst_path(m, t[i - 1].path) < 0) {
+			pr_err("mscp_set_dst_path: %s", priv_get_err());
+			return -1;
+		}
+
+		/* start to scan source files and resolve their destination paths */
+		if (mscp_scan(m) < 0) {
+			pr_err("mscp_scan: %s", priv_get_err());
+			return -1;
+		}
+	} else {
+		/* resume a transfer from the specified checkpoint */
+		char r[512];
+		int d;
+		if (mscp_checkpoint_get_remote(checkpoint_load, r, sizeof(r), &d) < 0) {
+			pr_err("mscp_checkpoint_get_remote: %s", priv_get_err());
+			return -1;
+		}
+
+		if (mscp_set_remote(m, r, d) < 0) {
+			pr_err("mscp_set_remote: %s", priv_get_err());
+			return -1;
+		}
+
+		/* load paths and chunks to be transferred from checkpoint */
+		if (mscp_checkpoint_load(m, checkpoint_load) < 0) {
+			pr_err("mscp_checkpoint_load: %s", priv_get_err());
+			return -1;
+		}
+
+		if (dryrun)
+			goto out;
+
+		/* create the first ssh connection to get password or
+		 * passphrase. The sftp session over it will be not
+		 * used for resume transfer in actuality. ToDo:
+		 * connectin managemnet should be improved. */
+		if (mscp_connect(m) < 0) {
+			pr_err("mscp_connect: %s", priv_get_err());
+			return -1;
+		}
 	}
 
 	if (dryrun) {
@@ -437,7 +599,20 @@ int main(int argc, char **argv)
 	pthread_cancel(tid_stat);
 	pthread_join(tid_stat, NULL);
 
+	print_stat(true);
+	print_cli("\n"); /* final output */
 out:
+	if (interrupted)
+		ret = 1;
+
+	if ((dryrun || ret != 0) && checkpoint_save) {
+		print_cli("save checkpoint to %s\n", checkpoint_save);
+		if (mscp_checkpoint_save(m, checkpoint_save) < 0) {
+			pr_err("mscp_checkpoint_save: %s", priv_get_err());
+			return -1;
+		}
+	}
+
 	mscp_cleanup(m);
 	mscp_free(m);
 
@@ -518,8 +693,13 @@ void print_progress_bar(double percent, char *suffix)
 
 	buf[0] = '\0';
 
-	if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws) < 0)
-		return; /* XXX */
+	if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws) < 0 || ws.ws_col == 0) {
+
+		// fallback to default
+		ws.ws_col = 80;
+		ws.ws_row = 24;
+	}
+
 	bar_width = min(sizeof(buf), ws.ws_col) - strlen(suffix) - 7;
 
 	memset(buf, 0, sizeof(buf));
@@ -597,8 +777,6 @@ struct xfer_stat x;
 void print_stat(bool final)
 {
 	struct mscp_stats s;
-	char buf[8192];
-	int timeout;
 
 	gettimeofday(&x.after, NULL);
 	if (calculate_timedelta(&x.before, &x.after) > 1 || final) {
@@ -612,30 +790,16 @@ void print_stat(bool final)
 	}
 }
 
-void print_stat_thread_cleanup(void *arg)
-{
-	print_stat(true);
-	print_cli("\n"); /* final output */
-}
-
 void *print_stat_thread(void *arg)
 {
-	struct mscp_stats s;
-	char buf[8192];
-
 	memset(&x, 0, sizeof(x));
 	gettimeofday(&x.start, NULL);
 	x.before = x.start;
 
-	pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
-	pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL);
-	pthread_cleanup_push(print_stat_thread_cleanup, NULL);
-
 	while (true) {
 		print_stat(false);
 		sleep(1);
 	}
 
-	pthread_cleanup_pop(1);
 	return NULL;
 }

src/mscp.c

@@ -6,66 +6,63 @@
 #include <semaphore.h>
 #include <sys/time.h>
 
-#include <list.h>
+#include <pool.h>
 #include <minmax.h>
 #include <ssh.h>
 #include <path.h>
+#include <checkpoint.h>
 #include <fileops.h>
 #include <atomic.h>
 #include <platform.h>
 #include <print.h>
 #include <strerrno.h>
 #include <mscp.h>
+#include <bwlimit.h>
 
 #include <openbsd-compat/openbsd-compat.h>
 
+struct mscp_thread {
+	struct mscp *m;
+	sftp_session sftp;
+
+	/* attributes used by copy threads */
+	size_t copied_bytes;
+	int id;
+	int cpu;
+
+	/* thread-specific values */
+	pthread_t tid;
+	int ret;
+};
+
 struct mscp {
 	char *remote; /* remote host (and uername) */
 	int direction; /* copy direction */
+	char dst_path[PATH_MAX];
+
 	struct mscp_opts *opts;
 	struct mscp_ssh_opts *ssh_opts;
 
 	int *cores; /* usable cpu cores by COREMASK */
 	int nr_cores; /* length of array of cores */
 
-	sem_t *sem; /* semaphore for concurrent 
-						 * connecting ssh sessions */
+	sem_t *sem; /* semaphore for concurrent  connecting ssh sessions */
 
 	sftp_session first; /* first sftp session */
 
-	char dst_path[PATH_MAX];
-	struct list_head src_list;
-	struct list_head path_list;
-	struct chunk_pool cp;
+	pool *src_pool, *path_pool, *chunk_pool, *thread_pool;
 
-	pthread_t tid_scan; /* tid for scan thread */
-	int ret_scan; /* return code from scan thread */
+	size_t total_bytes; /* total_bytes to be copied */
+	bool chunk_pool_ready;
+#define chunk_pool_is_ready(m) ((m)->chunk_pool_ready)
+#define chunk_pool_set_ready(m, b) ((m)->chunk_pool_ready = b)
 
-	size_t total_bytes; /* total bytes to be transferred */
+	struct bwlimit bw; /* bandwidth limit mechanism */
 
-	struct list_head thread_list;
-	rwlock thread_rwlock;
+	struct mscp_thread scan; /* mscp_thread for mscp_scan_thread() */
 };
 
-struct mscp_thread {
-	struct list_head list; /* mscp->thread_list */
-
-	struct mscp *m;
-	int id;
-	sftp_session sftp;
-	pthread_t tid;
-	int cpu;
-	size_t done;
-	bool finished;
-	int ret;
-};
-
-struct src {
-	struct list_head list; /* mscp->src_list */
-	char *path;
-};
-
-#define DEFAULT_MIN_CHUNK_SZ (64 << 20) /* 64MB */
+#define DEFAULT_MIN_CHUNK_SZ (16 << 20) /* 16MB */
 #define DEFAULT_NR_AHEAD 32
 #define DEFAULT_BUF_SZ 16384
 /* XXX: we use 16384 byte buffer pointed by
@@ -81,7 +78,7 @@ struct src {
 
 static int expand_coremask(const char *coremask, int **cores, int *nr_cores)
 {
-	int n, *core_list, core_list_len = 0, nr_usable, nr_all;
+	int n, *core_list, nr_usable, nr_all;
 	char c[2] = { 'x', '\0' };
 	const char *_coremask;
 	long v, needle;
@@ -161,23 +158,8 @@ static int validate_and_set_defaut_params(struct mscp_opts *o)
 
 	if (o->min_chunk_sz == 0)
 		o->min_chunk_sz = DEFAULT_MIN_CHUNK_SZ;
-	else {
-		if (o->min_chunk_sz < getpagesize() ||
-		    o->min_chunk_sz % getpagesize() != 0) {
-			priv_set_errv("min chunk size must be "
-				      "larget than and multiple of page size %d: %lu",
-				      getpagesize(), o->min_chunk_sz);
-			return -1;
-		}
-	}
 
 	if (o->max_chunk_sz) {
-		if (o->max_chunk_sz < getpagesize() ||
-		    o->max_chunk_sz % getpagesize() != 0) {
-			priv_set_errv("min chunk size must be larget than and "
-				      "multiple of page size %d: %lu",
-				      getpagesize(), o->max_chunk_sz);
-		}
 		if (o->min_chunk_sz > o->max_chunk_sz) {
 			priv_set_errv("smaller max chunk size than "
 				      "min chunk size: %lu < %lu",
@@ -208,54 +190,72 @@ static int validate_and_set_defaut_params(struct mscp_opts *o)
 	return 0;
 }
 
-struct mscp *mscp_init(const char *remote_host, int direction, struct mscp_opts *o,
-		       struct mscp_ssh_opts *s)
+int mscp_set_remote(struct mscp *m, const char *remote_host, int direction)
 {
-	struct mscp *m;
-	int n;
-
 	if (!remote_host) {
 		priv_set_errv("empty remote host");
-		return NULL;
+		return -1;
 	}
 
 	if (!(direction == MSCP_DIRECTION_L2R || direction == MSCP_DIRECTION_R2L)) {
 		priv_set_errv("invalid copy direction: %d", direction);
-		return NULL;
+		return -1;
 	}
 
+	if (!(m->remote = strdup(remote_host))) {
+		priv_set_errv("strdup: %s", strerrno());
+		return -1;
+	}
+	m->direction = direction;
+
+	return 0;
+}
+
+struct mscp *mscp_init(struct mscp_opts *o, struct mscp_ssh_opts *s)
+{
+	struct mscp *m;
+	int n;
+
 	set_print_severity(o->severity);
 
 	if (validate_and_set_defaut_params(o) < 0) {
 		return NULL;
 	}
 
-	m = malloc(sizeof(*m));
-	if (!m) {
+	if (!(m = malloc(sizeof(*m)))) {
 		priv_set_errv("malloc: %s", strerrno());
 		return NULL;
 	}
-
 	memset(m, 0, sizeof(*m));
-	INIT_LIST_HEAD(&m->src_list);
-	INIT_LIST_HEAD(&m->path_list);
-	chunk_pool_init(&m->cp);
+	m->opts = o;
+	m->ssh_opts = s;
+	chunk_pool_set_ready(m, false);
 
-	INIT_LIST_HEAD(&m->thread_list);
-	rwlock_init(&m->thread_rwlock);
+	if (!(m->src_pool = pool_new())) {
+		priv_set_errv("pool_new: %s", strerrno());
+		goto free_out;
+	}
+
+	if (!(m->path_pool = pool_new())) {
+		priv_set_errv("pool_new: %s", strerrno());
+		goto free_out;
+	}
+
+	if (!(m->chunk_pool = pool_new())) {
+		priv_set_errv("pool_new: %s", strerrno());
+		goto free_out;
+	}
+
+	if (!(m->thread_pool = pool_new())) {
+		priv_set_errv("pool_new: %s", strerrno());
+		goto free_out;
+	}
 
 	if ((m->sem = sem_create(o->max_startups)) == NULL) {
 		priv_set_errv("sem_create: %s", strerrno());
 		goto free_out;
 	}
 
-	m->remote = strdup(remote_host);
-	if (!m->remote) {
-		priv_set_errv("strdup: %s", strerrno());
-		goto free_out;
-	}
-	m->direction = direction;
-
 	if (o->coremask) {
 		if (expand_coremask(o->coremask, &m->cores, &m->nr_cores) < 0)
 			goto free_out;
@@ -269,12 +269,31 @@ struct mscp *mscp_init(const char *remote_host, int direction, struct mscp_opts
 		pr_notice("usable cpu cores:%s", b);
 	}
 
-	m->opts = o;
-	m->ssh_opts = s;
+	if (bwlimit_init(&m->bw, o->bitrate, 100) < 0) { /* 100ms window (hardcoded) */
+		priv_set_errv("bwlimit_init: %s", strerrno());
+		goto free_out;
+	}
+
+	/* workaround: set libssh using openssh proxyjump
+	 * https://gitlab.com/libssh/libssh-mirror/-/issues/319 */
+	ssh_use_openssh_proxy_jumps(1);
+
+	/* call ssh_init() because libssh is statically linked */
+	ssh_init();
 
 	return m;
 
 free_out:
+	if (m->src_pool)
+		pool_free(m->src_pool);
+	if (m->path_pool)
+		pool_free(m->path_pool);
+	if (m->chunk_pool)
+		pool_free(m->chunk_pool);
+	if (m->thread_pool)
+		pool_free(m->thread_pool);
+	if (m->remote)
+		free(m->remote);
 	free(m);
 	return NULL;
 }
@@ -290,23 +309,15 @@ int mscp_connect(struct mscp *m)
 
 int mscp_add_src_path(struct mscp *m, const char *src_path)
 {
-	struct src *s;
-
-	s = malloc(sizeof(*s));
+	char *s = strdup(src_path);
 	if (!s) {
-		priv_set_errv("malloc: %s", strerrno());
+		priv_set_errv("strdup: %s", strerrno());
 		return -1;
 	}
-
-	memset(s, 0, sizeof(*s));
-	s->path = strdup(src_path);
-	if (!s->path) {
-		priv_set_errv("malloc: %s", strerrno());
-		free(s);
+	if (pool_push(m->src_pool, s) < 0) {
+		priv_set_errv("pool_push: %s", strerrno());
 		return -1;
 	}
-
-	list_add_tail(&s->list, &m->src_list);
 	return 0;
 }
 
@@ -317,44 +328,38 @@ int mscp_set_dst_path(struct mscp *m, const char *dst_path)
 		return -1;
 	}
 
-	if (!non_null_string(dst_path))
-		strncpy(m->dst_path, ".", 1);
-	else
-		strncpy(m->dst_path, dst_path, PATH_MAX);
+	if (!non_null_string(dst_path)) {
+		priv_set_errv("empty dst path");
+		return -1;
+	}
+
+	strncpy(m->dst_path, dst_path, PATH_MAX);
 
 	return 0;
 }
 
-static int get_page_mask(void)
+static size_t get_page_mask(void)
 {
-	long page_sz = sysconf(_SC_PAGESIZE);
-	size_t page_mask = 0;
-	int n;
-
-	for (n = 0; page_sz > 0; page_sz >>= 1, n++) {
-		page_mask <<= 1;
-		page_mask |= 1;
-	}
-
-	return page_mask >> 1;
+	size_t page_sz = sysconf(_SC_PAGESIZE);
+	return ~(page_sz - 1);
 }
 
 static void mscp_stop_copy_thread(struct mscp *m)
 {
 	struct mscp_thread *t;
-
-	RWLOCK_READ_ACQUIRE(&m->thread_rwlock);
-	list_for_each_entry(t, &m->thread_list, list) {
-		if (!t->finished)
+	unsigned int idx;
+	pool_lock(m->thread_pool);
+	pool_for_each(m->thread_pool, t, idx) {
+		if (t->tid)
 			pthread_cancel(t->tid);
 	}
-	RWLOCK_RELEASE();
+	pool_unlock(m->thread_pool);
 }
 
 static void mscp_stop_scan_thread(struct mscp *m)
 {
-	if (m->tid_scan)
-		pthread_cancel(m->tid_scan);
+	if (m->scan.tid)
+		pthread_cancel(m->scan.tid);
 }
 
 void mscp_stop(struct mscp *m)
@@ -365,25 +370,23 @@ void mscp_stop(struct mscp *m)
 
 void *mscp_scan_thread(void *arg)
 {
-	struct mscp *m = arg;
+	struct mscp_thread *t = arg;
+	struct mscp *m = t->m;
 	sftp_session src_sftp = NULL, dst_sftp = NULL;
 	struct path_resolve_args a;
-	struct list_head tmp;
 	struct path *p;
-	struct src *s;
 	struct stat ss, ds;
+	char *src_path;
 	glob_t pglob;
 	int n;
 
-	m->ret_scan = 0;
-
 	switch (m->direction) {
 	case MSCP_DIRECTION_L2R:
 		src_sftp = NULL;
-		dst_sftp = m->first;
+		dst_sftp = t->sftp;
 		break;
 	case MSCP_DIRECTION_R2L:
-		src_sftp = m->first;
+		src_sftp = t->sftp;
 		dst_sftp = NULL;
 		break;
 	default:
@@ -395,7 +398,10 @@ void *mscp_scan_thread(void *arg)
 	memset(&a, 0, sizeof(a));
 	a.total_bytes = &m->total_bytes;
 
-	if (list_count(&m->src_list) > 1)
+	if (pool_size(m->src_pool) > 1)
+		a.dst_path_should_dir = true;
+
+	if (m->dst_path[strlen(m->dst_path) - 1] == '/')
 		a.dst_path_should_dir = true;
 
 	if (mscp_stat(m->dst_path, &ds, dst_sftp) == 0) {
@@ -403,7 +409,8 @@ void *mscp_scan_thread(void *arg)
 			a.dst_path_is_dir = true;
 	}
 
-	a.cp = &m->cp;
+	a.path_pool = m->path_pool;
+	a.chunk_pool = m->chunk_pool;
 	a.nr_conn = m->opts->nr_threads;
 	a.min_chunk_sz = m->opts->min_chunk_sz;
 	a.max_chunk_sz = m->opts->max_chunk_sz;
@@ -411,55 +418,56 @@ void *mscp_scan_thread(void *arg)
 
 	pr_info("start to walk source path(s)");
 
-	/* walk a src_path recusively, and resolve path->dst_path for each src */
-	list_for_each_entry(s, &m->src_list, list) {
+	/* walk each src_path recusively, and resolve path->dst_path for each src */
+	pool_iter_for_each(m->src_pool, src_path) {
 		memset(&pglob, 0, sizeof(pglob));
-		if (mscp_glob(s->path, GLOB_NOCHECK, &pglob, src_sftp) < 0) {
+		if (mscp_glob(src_path, GLOB_NOCHECK, &pglob, src_sftp) < 0) {
 			pr_err("mscp_glob: %s", strerrno());
 			goto err_out;
 		}
 
 		for (n = 0; n < pglob.gl_pathc; n++) {
 			if (mscp_stat(pglob.gl_pathv[n], &ss, src_sftp) < 0) {
-				pr_err("stat: %s %s", s->path, strerrno());
+				pr_err("stat: %s %s", src_path, strerrno());
 				goto err_out;
 			}
 
 			if (!a.dst_path_should_dir && pglob.gl_pathc > 1)
-				a.dst_path_should_dir = true; /* we have over 1 src */
+				a.dst_path_should_dir = true; /* we have over 1 srces */
 
 			/* set path specific args */
 			a.src_path = pglob.gl_pathv[n];
 			a.dst_path = m->dst_path;
 			a.src_path_is_dir = S_ISDIR(ss.st_mode);
 
-			INIT_LIST_HEAD(&tmp);
-			if (walk_src_path(src_sftp, pglob.gl_pathv[n], &tmp, &a) < 0)
+			if (walk_src_path(src_sftp, pglob.gl_pathv[n], &a) < 0)
 				goto err_out;
-
-			list_splice_tail(&tmp, m->path_list.prev);
 		}
 		mscp_globfree(&pglob);
 	}
 
 	pr_info("walk source path(s) done");
-	chunk_pool_set_filled(&m->cp);
-	m->ret_scan = 0;
+	t->ret = 0;
+	chunk_pool_set_ready(m, true);
 	return NULL;
 
 err_out:
-	chunk_pool_set_filled(&m->cp);
-	m->ret_scan = -1;
+	t->ret = -1;
+	chunk_pool_set_ready(m, true);
 	return NULL;
 }
 
 int mscp_scan(struct mscp *m)
 {
-	int ret = pthread_create(&m->tid_scan, NULL, mscp_scan_thread, m);
-	if (ret < 0) {
+	struct mscp_thread *t = &m->scan;
+	int ret;
+
+	memset(t, 0, sizeof(*t));
+	t->m = m;
+	t->sftp = m->first;
+
+	if ((ret = pthread_create(&t->tid, NULL, mscp_scan_thread, t)) < 0) {
 		priv_set_err("pthread_create: %d", ret);
-		m->tid_scan = 0;
-		mscp_stop(m);
 		return -1;
 	}
 
@@ -468,8 +476,7 @@ int mscp_scan(struct mscp *m)
 	 * finished. If the number of chunks are smaller than
 	 * nr_threads, we adjust nr_threads to the number of chunks.
 	 */
-	while (!chunk_pool_is_filled(&m->cp) &&
-	       chunk_pool_size(&m->cp) < m->opts->nr_threads)
+	while (!chunk_pool_is_ready(m) && pool_size(m->chunk_pool) < m->opts->nr_threads)
 		usleep(100);
 
 	return 0;
@@ -477,14 +484,45 @@ int mscp_scan(struct mscp *m)
 
 int mscp_scan_join(struct mscp *m)
 {
-	if (m->tid_scan) {
-		pthread_join(m->tid_scan, NULL);
-		m->tid_scan = 0;
-		return m->ret_scan;
+	struct mscp_thread *t = &m->scan;
+	if (t->tid) {
+		pthread_join(t->tid, NULL);
+		t->tid = 0;
+		return t->ret;
 	}
 	return 0;
 }
 
+int mscp_checkpoint_get_remote(const char *pathname, char *remote, size_t len, int *dir)
+{
+	return checkpoint_load_remote(pathname, remote, len, dir);
+}
+
+int mscp_checkpoint_load(struct mscp *m, const char *pathname)
+{
+	struct chunk *c;
+	unsigned int i;
+
+	if (checkpoint_load_paths(pathname, m->path_pool, m->chunk_pool) < 0)
+		return -1;
+
+	/* totaling up bytes to be transferred and set chunk_pool is
+	 * ready instead of the mscp_scan thread */
+	m->total_bytes = 0;
+	pool_for_each(m->chunk_pool, c, i) {
+		m->total_bytes += c->len;
+	}
+	chunk_pool_set_ready(m, true);
+
+	return 0;
+}
+
+int mscp_checkpoint_save(struct mscp *m, const char *pathname)
+{
+	return checkpoint_save(pathname, m->direction, m->ssh_opts->login_name, m->remote,
+			       m->path_pool, m->chunk_pool);
+}
+
 static void *mscp_copy_thread(void *arg);
 
 static struct mscp_thread *mscp_copy_thread_spawn(struct mscp *m, int id)
@@ -492,9 +530,8 @@ static struct mscp_thread *mscp_copy_thread_spawn(struct mscp *m, int id)
 	struct mscp_thread *t;
 	int ret;
 
-	t = malloc(sizeof(*t));
-	if (!t) {
-		priv_set_errv("malloc: %s,", strerrno());
+	if (!(t = malloc(sizeof(*t)))) {
+		priv_set_errv("malloc: %s", strerrno());
 		return NULL;
 	}
 
@@ -506,8 +543,7 @@ static struct mscp_thread *mscp_copy_thread_spawn(struct mscp *m, int id)
 	else
 		t->cpu = m->cores[id % m->nr_cores];
 
-	ret = pthread_create(&t->tid, NULL, mscp_copy_thread, t);
-	if (ret < 0) {
+	if ((ret = pthread_create(&t->tid, NULL, mscp_copy_thread, t)) < 0) {
 		priv_set_errv("pthread_create: %d", ret);
 		free(t);
 		return NULL;
@@ -521,19 +557,22 @@ int mscp_start(struct mscp *m)
 	struct mscp_thread *t;
 	int n, ret = 0;
 
-	if ((n = chunk_pool_size(&m->cp)) < m->opts->nr_threads) {
+	if ((n = pool_size(m->chunk_pool)) < m->opts->nr_threads) {
 		pr_notice("we have %d chunk(s), set number of connections to %d", n, n);
 		m->opts->nr_threads = n;
 	}
 
+	pr_notice("threads: %d",m->opts->nr_threads);
+	pr_notice("bwlimit: %ld bps", m->bw.bps);
+
 	for (n = 0; n < m->opts->nr_threads; n++) {
 		t = mscp_copy_thread_spawn(m, n);
 		if (!t)
 			break;
-
-		RWLOCK_WRITE_ACQUIRE(&m->thread_rwlock);
-		list_add_tail(&t->list, &m->thread_list);
-		RWLOCK_RELEASE();
+		if (pool_push_lock(m->thread_pool, t) < 0) {
+			priv_set_errv("pool_push_lock: %s", strerrno());
+			break;
+		}
 	}
 
 	return n;
@@ -543,17 +582,20 @@ int mscp_join(struct mscp *m)
 {
 	struct mscp_thread *t;
 	struct path *p;
-	size_t done = 0, nr_copied = 0, nr_tobe_copied = 0;
+	unsigned int idx;
+	size_t total_copied_bytes = 0, nr_copied = 0, nr_tobe_copied = 0;
 	int n, ret = 0;
 
 	/* waiting for scan thread joins... */
 	ret = mscp_scan_join(m);
 
 	/* waiting for copy threads join... */
-	RWLOCK_READ_ACQUIRE(&m->thread_rwlock);
-	list_for_each_entry(t, &m->thread_list, list) {
+	pool_for_each(m->thread_pool, t, idx) {
 		pthread_join(t->tid, NULL);
-		done += t->done;
+	}
+
+	pool_for_each(m->thread_pool, t, idx) {
+		total_copied_bytes += t->copied_bytes;
 		if (t->ret != 0)
 			ret = t->ret;
 		if (t->sftp) {
@@ -561,23 +603,22 @@ int mscp_join(struct mscp *m)
 			t->sftp = NULL;
 		}
 	}
-	RWLOCK_RELEASE();
-
-	if (m->first) {
-		ssh_sftp_close(m->first);
-		m->first = NULL;
-	}
 
 	/* count up number of transferred files */
-	list_for_each_entry(p, &m->path_list, list) {
+	pool_iter_for_each(m->path_pool, p) {
 		nr_tobe_copied++;
 		if (p->state == FILE_STATE_DONE) {
 			nr_copied++;
 		}
 	}
 
-	pr_notice("%lu/%lu bytes copied for %lu/%lu files", done, m->total_bytes,
-		  nr_copied, nr_tobe_copied);
+	if (m->first) {
+		ssh_sftp_close(m->first);
+		m->first = NULL;
+	}
+
+	pr_notice("%lu/%lu bytes copied for %lu/%lu files", total_copied_bytes,
+		  m->total_bytes, nr_copied, nr_tobe_copied);
 
 	return ret;
 }
@@ -599,25 +640,17 @@ static void wait_for_interval(int interval)
 	next = now + interval * 1000000;
 }
 
-static void mscp_copy_thread_cleanup(void *arg)
-{
-	struct mscp_thread *t = arg;
-	t->finished = true;
-}
-
 void *mscp_copy_thread(void *arg)
 {
 	sftp_session src_sftp, dst_sftp;
 	struct mscp_thread *t = arg;
 	struct mscp *m = t->m;
 	struct chunk *c;
-	bool nomore;
+	bool next_chunk_exist;
 
 	/* when error occurs, each thread prints error messages
 	 * immediately with pr_* functions. */
 
-	pthread_cleanup_push(mscp_copy_thread_cleanup, t);
-
 	if (t->cpu > -1) {
 		if (set_thread_affinity(pthread_self(), t->cpu) < 0) {
 			pr_err("set_thread_affinity: %s", priv_get_err());
@@ -631,7 +664,7 @@ void *mscp_copy_thread(void *arg)
 		goto err_out;
 	}
 
-	if (!(nomore = chunk_pool_is_empty(&m->cp))) {
+	if ((next_chunk_exist = pool_iter_has_next_lock(m->chunk_pool))) {
 		if (m->opts->interval > 0)
 			wait_for_interval(m->opts->interval);
 		pr_notice("thread[%d]: connecting to %s", t->id, m->remote);
@@ -643,7 +676,7 @@ void *mscp_copy_thread(void *arg)
 		goto err_out;
 	}
 
-	if (nomore) {
+	if (!next_chunk_exist) {
 		pr_notice("thread[%d]: no more connections needed", t->id);
 		goto out;
 	}
@@ -668,23 +701,22 @@ void *mscp_copy_thread(void *arg)
 	}
 
 	while (1) {
-		c = chunk_pool_pop(&m->cp);
-		if (c == CHUNK_POP_WAIT) {
-			usleep(100); /* XXX: hard code */
-			continue;
+		c = pool_iter_next_lock(m->chunk_pool);
+		if (c == NULL) {
+			if (!chunk_pool_is_ready(m)) {
+				/* a new chunk will be added. wait for it. */
+				usleep(100);
+				continue;
+			}
+			break; /* no more chunks */
 		}
 
-		if (!c)
-			break; /* no more chunks */
-
 		if ((t->ret = copy_chunk(c, src_sftp, dst_sftp, m->opts->nr_ahead,
-					 m->opts->buf_sz, m->opts->preserve_ts,
-					 &t->done)) < 0)
+					 m->opts->buf_sz, m->opts->preserve_ts, &m->bw,
+					 &t->copied_bytes)) < 0)
 			break;
 	}
 
-	pthread_cleanup_pop(1);
-
 	if (t->ret < 0) {
 		pr_err("thread[%d]: copy failed: %s -> %s, 0x%010lx-0x%010lx, %s", t->id,
 		       c->p->path, c->p->dst_path, c->off, c->off + c->len,
@@ -694,39 +726,15 @@ void *mscp_copy_thread(void *arg)
 	return NULL;
 
 err_out:
-	t->finished = true;
 	t->ret = -1;
 	return NULL;
 out:
-	t->finished = true;
 	t->ret = 0;
 	return NULL;
 }
 
 /* cleanup-related functions */
 
-static void list_free_src(struct list_head *list)
-{
-	struct src *s;
-	s = list_entry(list, typeof(*s), list);
-	free(s->path);
-	free(s);
-}
-
-static void list_free_path(struct list_head *list)
-{
-	struct path *p;
-	p = list_entry(list, typeof(*p), list);
-	free_path(p);
-}
-
-static void list_free_thread(struct list_head *list)
-{
-	struct mscp_thread *t;
-	t = list_entry(list, typeof(*t), list);
-	free(t);
-}
-
 void mscp_cleanup(struct mscp *m)
 {
 	if (m->first) {
@@ -734,23 +742,17 @@ void mscp_cleanup(struct mscp *m)
 		m->first = NULL;
 	}
 
-	list_free_f(&m->src_list, list_free_src);
-	INIT_LIST_HEAD(&m->src_list);
-
-	list_free_f(&m->path_list, list_free_path);
-	INIT_LIST_HEAD(&m->path_list);
-
-	chunk_pool_release(&m->cp);
-	chunk_pool_init(&m->cp);
-
-	RWLOCK_WRITE_ACQUIRE(&m->thread_rwlock);
-	list_free_f(&m->thread_list, list_free_thread);
-	RWLOCK_RELEASE();
+	pool_zeroize(m->src_pool, free);
+	pool_zeroize(m->path_pool, (pool_map_f)free_path);
+	pool_zeroize(m->chunk_pool, free);
+	pool_zeroize(m->thread_pool, free);
 }
 
 void mscp_free(struct mscp *m)
 {
-	mscp_cleanup(m);
+	pool_destroy(m->src_pool, free);
+	pool_destroy(m->path_pool, (pool_map_f)free_path);
+
 	if (m->remote)
 		free(m->remote);
 	if (m->cores)
@@ -762,20 +764,13 @@ void mscp_free(struct mscp *m)
 
 void mscp_get_stats(struct mscp *m, struct mscp_stats *s)
 {
-	int nr_finished = 0, nr_threads = 0;
 	struct mscp_thread *t;
+	unsigned int idx;
 
 	s->total = m->total_bytes;
 	s->done = 0;
 
-	RWLOCK_READ_ACQUIRE(&m->thread_rwlock);
-	list_for_each_entry(t, &m->thread_list, list) {
-		nr_threads++;
-		s->done += t->done;
-		if (t->finished)
-			nr_finished++;
+	pool_for_each(m->thread_pool, t, idx) {
+		s->done += t->copied_bytes;
 	}
-	RWLOCK_RELEASE();
-
-	s->finished = nr_threads > 0 ? (nr_finished == nr_threads) : false;
 }

src/path.c

@@ -9,90 +9,11 @@
 #include <ssh.h>
 #include <minmax.h>
 #include <fileops.h>
-#include <list.h>
 #include <atomic.h>
 #include <path.h>
 #include <strerrno.h>
 #include <print.h>
 
-/* chunk pool operations */
-#define CHUNK_POOL_STATE_FILLING 0
-#define CHUNK_POOL_STATE_FILLED 1
-
-void chunk_pool_init(struct chunk_pool *cp)
-{
-	memset(cp, 0, sizeof(*cp));
-	INIT_LIST_HEAD(&cp->list);
-	lock_init(&cp->lock);
-	cp->state = CHUNK_POOL_STATE_FILLING;
-}
-
-static void chunk_pool_add(struct chunk_pool *cp, struct chunk *c)
-{
-	LOCK_ACQUIRE(&cp->lock);
-	list_add_tail(&c->list, &cp->list);
-	cp->count += 1;
-	LOCK_RELEASE();
-}
-
-void chunk_pool_set_filled(struct chunk_pool *cp)
-{
-	cp->state = CHUNK_POOL_STATE_FILLED;
-}
-
-bool chunk_pool_is_filled(struct chunk_pool *cp)
-{
-	return (cp->state == CHUNK_POOL_STATE_FILLED);
-}
-
-size_t chunk_pool_size(struct chunk_pool *cp)
-{
-	return cp->count;
-}
-
-bool chunk_pool_is_empty(struct chunk_pool *cp)
-{
-	return list_empty(&cp->list);
-}
-
-struct chunk *chunk_pool_pop(struct chunk_pool *cp)
-{
-	struct list_head *first;
-	struct chunk *c = NULL;
-
-	LOCK_ACQUIRE(&cp->lock);
-	first = cp->list.next;
-	if (list_empty(&cp->list)) {
-		if (!chunk_pool_is_filled(cp))
-			c = CHUNK_POP_WAIT;
-		else
-			c = NULL; /* no more chunks */
-	} else {
-		c = list_entry(first, struct chunk, list);
-		list_del(first);
-	}
-	LOCK_RELEASE();
-
-	/* return CHUNK_POP_WAIT would be a rare case, because it
-	 * means copying over SSH is faster than traversing
-	 * local/remote file paths.
-	 */
-
-	return c;
-}
-
-static void chunk_free(struct list_head *list)
-{
-	struct chunk *c;
-	c = list_entry(list, typeof(*c), list);
-	free(c);
-}
-
-void chunk_pool_release(struct chunk_pool *cp)
-{
-	list_free_f(&cp->list, chunk_free);
-}
-
 /* paths of copy source resoltion */
 static char *resolve_dst_path(const char *src_file_path, struct path_resolve_args *a)
 {
@@ -157,7 +78,7 @@ static char *resolve_dst_path(const char *src_file_path, struct path_resolve_arg
 }
 
 /* chunk preparation */
-static struct chunk *alloc_chunk(struct path *p)
+struct chunk *alloc_chunk(struct path *p, size_t off, size_t len)
 {
 	struct chunk *c;
 
@@ -168,43 +89,45 @@ static struct chunk *alloc_chunk(struct path *p)
 	memset(c, 0, sizeof(*c));
 
 	c->p = p;
-	c->off = 0;
-	c->len = 0;
+	c->off = off;
+	c->len = len;
+	c->state = CHUNK_STATE_INIT;
 	refcnt_inc(&p->refcnt);
 	return c;
 }
 
-static int resolve_chunk(struct path *p, struct path_resolve_args *a)
+static int resolve_chunk(struct path *p, size_t size, struct path_resolve_args *a)
 {
 	struct chunk *c;
-	size_t chunk_sz;
-	size_t size;
+	size_t chunk_sz, off, len;
+	size_t remaind;
 
-	if (p->size <= a->min_chunk_sz)
-		chunk_sz = p->size;
-	else if (a->max_chunk_sz)
+	if (a->max_chunk_sz)
 		chunk_sz = a->max_chunk_sz;
 	else {
-		chunk_sz = (p->size - (p->size % a->nr_conn)) / a->nr_conn;
-		chunk_sz &= ~a->chunk_align; /* align with page_sz */
+		chunk_sz = (size / (a->nr_conn * 4)) & a->chunk_align;
 		if (chunk_sz <= a->min_chunk_sz)
 			chunk_sz = a->min_chunk_sz;
 	}
 
-	/* for (size = f->size; size > 0;) does not create a file
-         * (chunk) when file size is 0. This do {} while (size > 0)
-         * creates just open/close a 0-byte file.
+	/* for (size = size; size > 0;) does not create a file (chunk)
+         * when file size is 0. This do {} while (remaind > 0) creates
+         * just open/close a 0-byte file.
          */
-	size = p->size;
+	remaind = size;
 	do {
-		c = alloc_chunk(p);
+		off = size - remaind;
+		len = remaind < chunk_sz ? remaind : chunk_sz;
+		c = alloc_chunk(p, off, len);
 		if (!c)
 			return -1;
-		c->off = p->size - size;
-		c->len = size < chunk_sz ? size : chunk_sz;
-		size -= c->len;
-		chunk_pool_add(a->cp, c);
-	} while (size > 0);
+
+		remaind -= len;
+		if (pool_push_lock(a->chunk_pool, c) < 0) {
+			pr_err("pool_push_lock: %s", strerrno());
+			return -1;
+		}
+	} while (remaind > 0);
 
 	return 0;
 }
@@ -218,38 +141,54 @@ void free_path(struct path *p)
 	free(p);
 }
 
-static int append_path(sftp_session sftp, const char *path, struct stat st,
-		       struct list_head *path_list, struct path_resolve_args *a)
+struct path *alloc_path(char *path, char *dst_path)
 {
 	struct path *p;
 
 	if (!(p = malloc(sizeof(*p)))) {
-		priv_set_errv("failed to allocate memory: %s", strerrno());
+		pr_err("malloc: %s", strerrno());
+		return NULL;
+	}
+	memset(p, 0, sizeof(*p));
+
+	p->path = path;
+	p->dst_path = dst_path;
+	p->state = FILE_STATE_INIT;
+	lock_init(&p->lock);
+	p->data = 0;
+
+	return p;
+}
+
+static int append_path(sftp_session sftp, const char *path, struct stat st,
+		       struct path_resolve_args *a)
+{
+	struct path *p;
+	char *src, *dst;
+
+	if (!(src = strdup(path))) {
+		pr_err("strdup: %s", strerrno());
 		return -1;
 	}
 
-	memset(p, 0, sizeof(*p));
-	INIT_LIST_HEAD(&p->list);
-	p->path = strndup(path, PATH_MAX);
-	if (!p->path) {
-		pr_err("strndup: %s", strerrno());
-		goto free_out;
+	if (!(dst = resolve_dst_path(src, a))) {
+		free(src);
+		return -1;
 	}
-	p->size = st.st_size;
-	p->mode = st.st_mode;
-	p->state = FILE_STATE_INIT;
-	lock_init(&p->lock);
 
-	p->dst_path = resolve_dst_path(p->path, a);
-	if (!p->dst_path)
-		goto free_out;
+	if (!(p = alloc_path(src, dst)))
+		return -1;
 
-	if (resolve_chunk(p, a) < 0)
+	if (resolve_chunk(p, st.st_size, a) < 0)
 		return -1; /* XXX: do not free path becuase chunk(s)
 			    * was added to chunk pool already */
 
-	list_add_tail(&p->list, path_list);
-	*a->total_bytes += p->size;
+	if (pool_push_lock(a->path_pool, p) < 0) {
+		pr_err("pool_push: %s", strerrno());
+		goto free_out;
+	}
+
+	*a->total_bytes += st.st_size;
 
 	return 0;
 
@@ -269,7 +208,7 @@ static bool check_path_should_skip(const char *path)
 }
 
 static int walk_path_recursive(sftp_session sftp, const char *path,
-			       struct list_head *path_list, struct path_resolve_args *a)
+			       struct path_resolve_args *a)
 {
 	char next_path[PATH_MAX + 1];
 	struct dirent *e;
@@ -284,7 +223,7 @@ static int walk_path_recursive(sftp_session sftp, const char *path,
 
 	if (S_ISREG(st.st_mode)) {
 		/* this path is regular file. it is to be copied */
-		return append_path(sftp, path, st, path_list, a);
+		return append_path(sftp, path, st, a);
 	}
 
 	if (!S_ISDIR(st.st_mode))
@@ -306,7 +245,7 @@ static int walk_path_recursive(sftp_session sftp, const char *path,
 			continue;
 		}
 
-		walk_path_recursive(sftp, next_path, path_list, a);
+		walk_path_recursive(sftp, next_path, a);
 		/* do not stop even when walk_path_recursive returns
 		 * -1 due to an unreadable file. go to a next
 		 * file. Thus, do not pass error messages via
@@ -321,9 +260,9 @@ static int walk_path_recursive(sftp_session sftp, const char *path,
 }
 
 int walk_src_path(sftp_session src_sftp, const char *src_path,
-		  struct list_head *path_list, struct path_resolve_args *a)
+		  struct path_resolve_args *a)
 {
-	return walk_path_recursive(src_sftp, src_path, path_list, a);
+	return walk_path_recursive(src_sftp, src_path, a);
 }
 
 /* based on
@@ -369,7 +308,7 @@ next:
 	 * end. see https://bugzilla.mindrot.org/show_bug.cgi?id=3431 */
 	f = mscp_open(p->dst_path, O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR, sftp);
 	if (!f) {
-		priv_set_errv("mscp_open %s: %s\n", p->dst_path, strerrno());
+		priv_set_errv("mscp_open %s: %s", p->dst_path, strerrno());
 		return -1;
 	}
 
@@ -406,7 +345,7 @@ static ssize_t read_to_buf(void *ptr, size_t len, void *userdata)
 }
 
 static int copy_chunk_l2r(struct chunk *c, int fd, sftp_file sf, int nr_ahead, int buf_sz,
-			  size_t *counter)
+			  struct bwlimit *bw, size_t *counter)
 {
 	ssize_t read_bytes, remaind, thrown;
 	int idx, ret;
@@ -424,11 +363,12 @@ static int copy_chunk_l2r(struct chunk *c, int fd, sftp_file sf, int nr_ahead, i
 		reqs[idx].len = sftp_async_write(sf, read_to_buf, reqs[idx].len, &fd,
 						 &reqs[idx].id);
 		if (reqs[idx].len < 0) {
-			priv_set_errv("sftp_async_write: %s or %s",
-				      sftp_get_ssh_error(sf->sftp), strerrno());
+			priv_set_errv("sftp_async_write: %s",
+				      sftp_get_ssh_error(sf->sftp));
 			return -1;
 		}
 		thrown -= reqs[idx].len;
+		bwlimit_wait(bw, reqs[idx].len);
 	}
 
 	for (idx = 0; remaind > 0; idx = (idx + 1) % nr_ahead) {
@@ -452,11 +392,12 @@ static int copy_chunk_l2r(struct chunk *c, int fd, sftp_file sf, int nr_ahead, i
 		reqs[idx].len = sftp_async_write(sf, read_to_buf, reqs[idx].len, &fd,
 						 &reqs[idx].id);
 		if (reqs[idx].len < 0) {
-			priv_set_errv("sftp_async_write: %s or %s",
-				      sftp_get_ssh_error(sf->sftp), strerrno());
+			priv_set_errv("sftp_async_write: %s",
+				      sftp_get_ssh_error(sf->sftp));
 			return -1;
 		}
 		thrown -= reqs[idx].len;
+		bwlimit_wait(bw, reqs[idx].len);
 	}
 
 	if (remaind < 0) {
@@ -469,44 +410,45 @@ static int copy_chunk_l2r(struct chunk *c, int fd, sftp_file sf, int nr_ahead, i
 	return 0;
 }
 
-static int copy_chunk_r2l(struct chunk *c, sftp_file sf, int fd, int nr_ahead, int buf_sz,
-			  size_t *counter)
+static int copy_chunk_r2l(struct chunk *c, sftp_file sf, int fd,
+			   int nr_ahead, int buf_sz,
+			   struct bwlimit *bw, size_t *counter)
 {
-	ssize_t read_bytes, write_bytes, remaind, thrown;
+	ssize_t read_bytes, write_bytes, remain, thrown, len, requested;
+	sftp_aio reqs[nr_ahead];
 	char buf[buf_sz];
-	int idx;
-	struct {
-		int id;
-		ssize_t len;
-	} reqs[nr_ahead];
+	int i;
 
 	if (c->len == 0)
 		return 0;
 
-	remaind = thrown = c->len;
+	remain = thrown = c->len;
 
-	for (idx = 0; idx < nr_ahead && thrown > 0; idx++) {
-		reqs[idx].len = min(thrown, sizeof(buf));
-		reqs[idx].id = sftp_async_read_begin(sf, reqs[idx].len);
-		if (reqs[idx].id < 0) {
-			priv_set_errv("sftp_async_read_begin: %d",
+	for (i = 0; i < nr_ahead && thrown > 0; i++) {
+		len = min(thrown, sizeof(buf));
+		requested = sftp_aio_begin_read(sf, len, &reqs[i]);
+		if (requested == SSH_ERROR) {
+			priv_set_errv("sftp_aio_begin_read: %d",
 				      sftp_get_error(sf->sftp));
 			return -1;
 		}
-		thrown -= reqs[idx].len;
+		thrown -= requested;
+		bwlimit_wait(bw, requested);
 	}
 
-	for (idx = 0; remaind > 0; idx = (idx + 1) % nr_ahead) {
-		read_bytes = sftp_async_read(sf, buf, reqs[idx].len, reqs[idx].id);
+	for (i = 0; remain > 0; i = (i + 1) % nr_ahead) {
+		read_bytes = sftp_aio_wait_read(&reqs[i], buf, sizeof(buf));
 		if (read_bytes == SSH_ERROR) {
-			priv_set_errv("sftp_async_read: %d", sftp_get_error(sf->sftp));
+			priv_set_errv("sftp_aio_wait_read: %d",
+				      sftp_get_error(sf->sftp));
 			return -1;
 		}
 
 		if (thrown > 0) {
-			reqs[idx].len = min(thrown, sizeof(buf));
-			reqs[idx].id = sftp_async_read_begin(sf, reqs[idx].len);
-			thrown -= reqs[idx].len;
+			len = min(thrown, sizeof(buf));
+			requested = sftp_aio_begin_read(sf, len, &reqs[i]);
+			thrown -= requested;
+			bwlimit_wait(bw, requested);
 		}
 
 		write_bytes = write(fd, buf, read_bytes);
@@ -521,13 +463,13 @@ static int copy_chunk_r2l(struct chunk *c, sftp_file sf, int fd, int nr_ahead, i
 		}
 
 		*counter += write_bytes;
-		remaind -= read_bytes;
+		remain -= write_bytes;
 	}
 
-	if (remaind < 0) {
-		priv_set_errv("invalid remaind bytes %ld. last async_read bytes %ld. "
+	if (remain < 0) {
+		priv_set_errv("invalid remain bytes %ld. last async_read bytes %ld. "
 			      "last write bytes %ld",
-			      remaind, read_bytes, write_bytes);
+			      remain, read_bytes, write_bytes);
 		return -1;
 	}
 
@@ -535,19 +477,22 @@ static int copy_chunk_r2l(struct chunk *c, sftp_file sf, int fd, int nr_ahead, i
 }
 
 static int _copy_chunk(struct chunk *c, mf *s, mf *d, int nr_ahead, int buf_sz,
-		       size_t *counter)
+		       struct bwlimit *bw, size_t *counter)
 {
 	if (s->local && d->remote) /* local to remote copy */
-		return copy_chunk_l2r(c, s->local, d->remote, nr_ahead, buf_sz, counter);
+		return copy_chunk_l2r(c, s->local, d->remote, nr_ahead, buf_sz, bw,
+				      counter);
 	else if (s->remote && d->local) /* remote to local copy */
-		return copy_chunk_r2l(c, s->remote, d->local, nr_ahead, buf_sz, counter);
+		return copy_chunk_r2l(c, s->remote, d->local, nr_ahead, buf_sz, bw,
+				      counter);
 
 	assert(false);
 	return -1; /* not reached */
 }
 
 int copy_chunk(struct chunk *c, sftp_session src_sftp, sftp_session dst_sftp,
-	       int nr_ahead, int buf_sz, bool preserve_ts, size_t *counter)
+	       int nr_ahead, int buf_sz, bool preserve_ts, struct bwlimit *bw,
+	       size_t *counter)
 {
 	mode_t mode;
 	int flags;
@@ -562,8 +507,7 @@ int copy_chunk(struct chunk *c, sftp_session src_sftp, sftp_session dst_sftp,
 	/* open src */
 	flags = O_RDONLY;
 	mode = S_IRUSR;
-	s = mscp_open(c->p->path, flags, mode, src_sftp);
-	if (!s) {
+	if (!(s = mscp_open(c->p->path, flags, mode, src_sftp))) {
 		priv_set_errv("mscp_open: %s: %s", c->p->path, strerrno());
 		return -1;
 	}
@@ -575,8 +519,7 @@ int copy_chunk(struct chunk *c, sftp_session src_sftp, sftp_session dst_sftp,
 	/* open dst */
 	flags = O_WRONLY;
 	mode = S_IRUSR | S_IWUSR;
-	d = mscp_open(c->p->dst_path, flags, mode, dst_sftp);
-	if (!d) {
+	if (!(d = mscp_open(c->p->dst_path, flags, mode, dst_sftp))) {
 		mscp_close(s);
 		priv_set_errv("mscp_open: %s: %s", c->p->dst_path, strerrno());
 		return -1;
@@ -586,9 +529,10 @@ int copy_chunk(struct chunk *c, sftp_session src_sftp, sftp_session dst_sftp,
 		return -1;
 	}
 
+	c->state = CHUNK_STATE_COPING;
 	pr_debug("copy chunk start: %s 0x%lx-0x%lx", c->p->path, c->off, c->off + c->len);
 
-	ret = _copy_chunk(c, s, d, nr_ahead, buf_sz, counter);
+	ret = _copy_chunk(c, s, d, nr_ahead, buf_sz, bw, counter);
 
 	pr_debug("copy chunk done: %s 0x%lx-0x%lx", c->p->path, c->off, c->off + c->len);
 
@@ -613,5 +557,8 @@ int copy_chunk(struct chunk *c, sftp_session src_sftp, sftp_session dst_sftp,
 		pr_info("copy done: %s", c->p->path);
 	}
 
+	if (ret == 0)
+		c->state = CHUNK_STATE_DONE;
+
 	return ret;
 }

src/path.h

@@ -6,66 +6,38 @@
 #include <fcntl.h>
 #include <dirent.h>
 #include <sys/stat.h>
-
-#include <list.h>
+#include <pool.h>
 #include <atomic.h>
 #include <ssh.h>
+#include <bwlimit.h>
 
 struct path {
-	struct list_head list; /* mscp->path_list */
-
 	char *path; /* file path */
-	size_t size; /* size of file on this path */
-	mode_t mode; /* permission */
-
 	char *dst_path; /* copy dst path */
 
-	int state;
+	refcnt refcnt; /* number of associated chunks */
 	lock lock;
-	refcnt refcnt;
-};
+	int state;
 #define FILE_STATE_INIT 0
 #define FILE_STATE_OPENED 1
 #define FILE_STATE_DONE 2
 
-struct chunk {
-	struct list_head list; /* chunk_pool->list */
+	uint64_t data; /* used by other components, i.e., checkpoint */
+};
 
+struct path *alloc_path(char *path, char *dst_path);
+
+struct chunk {
 	struct path *p;
 	size_t off; /* offset of this chunk on the file on path p */
 	size_t len; /* length of this chunk */
-	size_t done; /* copied bytes for this chunk by a thread */
-};
-
-struct chunk_pool {
-	struct list_head list; /* list of struct chunk */
-	size_t count;
-	lock lock;
 	int state;
+#define CHUNK_STATE_INIT 0
+#define CHUNK_STATE_COPING 1
+#define CHUNK_STATE_DONE 2
 };
 
-/* initialize chunk pool */
-void chunk_pool_init(struct chunk_pool *cp);
-
-/* acquire a chunk from pool. return value is NULL indicates no more
- * chunk, GET_CHUNK_WAIT means caller should waits until a chunk is
- * added, or pointer to chunk.
- */
-struct chunk *chunk_pool_pop(struct chunk_pool *cp);
-#define CHUNK_POP_WAIT ((void *)-1)
-
-/* set and check fillingchunks to this pool has finished */
-void chunk_pool_set_filled(struct chunk_pool *cp);
-bool chunk_pool_is_filled(struct chunk_pool *cp);
-
-/* return number of chunks in the pool */
-size_t chunk_pool_size(struct chunk_pool *cp);
-
-/* return true if chunk pool is empty (all chunks are already poped) */
-bool chunk_pool_is_empty(struct chunk_pool *cp);
-
-/* free chunks in the chunk_pool */
-void chunk_pool_release(struct chunk_pool *cp);
+struct chunk *alloc_chunk(struct path *p, size_t off, size_t len);
 
 struct path_resolve_args {
 	size_t *total_bytes;
@@ -78,25 +50,24 @@ struct path_resolve_args {
 	bool dst_path_should_dir;
 
 	/* args to resolve chunks for a path */
-	struct chunk_pool *cp;
+	pool *path_pool;
+	pool *chunk_pool;
 	int nr_conn;
 	size_t min_chunk_sz;
 	size_t max_chunk_sz;
 	size_t chunk_align;
 };
 
-/* recursivly walk through src_path and fill path_list for each file */
+/* walk src_path recursivly and fill a->path_pool with found files */
 int walk_src_path(sftp_session src_sftp, const char *src_path,
-		  struct list_head *path_list, struct path_resolve_args *a);
+		  struct path_resolve_args *a);
 
 /* free struct path */
 void free_path(struct path *p);
 
 /* copy a chunk. either src_sftp or dst_sftp is not null, and another is null */
 int copy_chunk(struct chunk *c, sftp_session src_sftp, sftp_session dst_sftp,
-	       int nr_ahead, int buf_sz, bool preserve_ts, size_t *counter);
-
-/* just print contents. just for debugging */
-void path_dump(struct list_head *path_list);
+	       int nr_ahead, int buf_sz, bool preserve_ts, struct bwlimit *bw,
+	       size_t *counter);
 
 #endif /* _PATH_H_ */

src/platform.c

@@ -22,6 +22,7 @@
 #error unsupported platform
 #endif
 
+#include <config.h>
 #include <platform.h>
 #include <strerrno.h>
 #include <print.h>

src/platform.h

@@ -2,8 +2,11 @@
 #ifndef _PLATFORM_H_
 #define _PLATFORM_H_
 
+#include <config.h>
+
 #include <pthread.h>
 #include <semaphore.h>
+#include <stdint.h>
 
 int nr_cpus(void);
 int set_thread_affinity(pthread_t tid, int core);
@@ -20,4 +23,25 @@ int setutimes(const char *path, struct timespec atime, struct timespec mtime);
 sem_t *sem_create(int value);
 int sem_release(sem_t *sem);
 
+#ifdef HAVE_HTONLL
+#include <arpa/inet.h> /* Apple has htonll and ntohll in arpa/inet.h */
+#endif
+
+/* copied from libssh: libssh/include/libssh/priv.h */
+#ifndef HAVE_HTONLL
+#ifdef WORDS_BIGENDIAN
+#define htonll(x) (x)
+#else
+#define htonll(x) (((uint64_t)htonl((x)&0xFFFFFFFF) << 32) | htonl((x) >> 32))
+#endif
+#endif
+
+#ifndef HAVE_NTOHLL
+#ifdef WORDS_BIGENDIAN
+#define ntohll(x) (x)
+#else
+#define ntohll(x) (((uint64_t)ntohl((x)&0xFFFFFFFF) << 32) | ntohl((x) >> 32))
+#endif
+#endif
+
 #endif /* _PLATFORM_H_ */

src/pool.c

@@ -0,0 +1,123 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
+#include <string.h>
+#include <stdlib.h>
+#include <pool.h>
+
+#define DEFAULT_START_SIZE 16
+
+pool *pool_new(void)
+{
+	pool *p;
+	p = malloc(sizeof(*p));
+	if (!p)
+		return NULL;
+	memset(p, 0, sizeof(*p));
+
+	p->array = calloc(DEFAULT_START_SIZE, sizeof(void *));
+	if (!p->array) {
+		free(p);
+		return NULL;
+	}
+
+	p->len = DEFAULT_START_SIZE;
+	p->num = 0;
+	lock_init(&p->lock);
+	return p;
+}
+
+void pool_free(pool *p)
+{
+	if (p->array) {
+		free(p->array);
+		p->array = NULL;
+	}
+	free(p);
+}
+
+void pool_zeroize(pool *p, pool_map_f f)
+{
+	void *v;
+	pool_iter_for_each(p, v) {
+		f(v);
+	}
+	p->num = 0;
+}
+
+void pool_destroy(pool *p, pool_map_f f)
+{
+	pool_zeroize(p, f);
+	pool_free(p);
+}
+
+int pool_push(pool *p, void *v)
+{
+	if (p->num == p->len) {
+		/* expand array */
+		size_t newlen = p->len * 2;
+		void *new = realloc(p->array, newlen * sizeof(void *));
+		if (new == NULL)
+			return -1;
+		p->len = newlen;
+		p->array = new;
+	}
+	p->array[p->num] = v;
+	__sync_synchronize();
+	p->num++;
+	return 0;
+}
+
+int pool_push_lock(pool *p, void *v)
+{
+	int ret = -1;
+	pool_lock(p);
+	ret = pool_push(p, v);
+	pool_unlock(p);
+	return ret;
+}
+
+void *pool_pop(pool *p)
+{
+	return p->num == 0 ? NULL : p->array[--p->num];
+}
+
+void *pool_pop_lock(pool *p)
+{
+	void *v;
+	pool_lock(p);
+	v = pool_pop(p);
+	pool_unlock(p);
+	return v;
+}
+
+void *pool_get(pool *p, unsigned int idx)
+{
+	return p->num <= idx ? NULL : p->array[idx];
+}
+
+void *pool_iter_next(pool *p)
+{
+	if (p->num <= p->idx)
+		return NULL;
+
+	void *v = p->array[p->idx];
+	p->idx++;
+	return v;
+}
+
+void *pool_iter_next_lock(pool *p)
+{
+	void *v = NULL;
+	pool_lock(p);
+	v = pool_iter_next(p);
+	pool_unlock(p);
+	return v;
+}
+
+bool pool_iter_has_next_lock(pool *p)
+{
+	bool next_exist;
+	pool_lock(p);
+	next_exist = (p->idx < p->num);
+	pool_unlock(p);
+	return next_exist;
+}

src/pool.h

@@ -0,0 +1,94 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
+#ifndef _POOL_H_
+#define _POOL_H_
+
+#include <stdbool.h>
+#include <stddef.h>
+
+#include <atomic.h>
+
+/* A pool like a stack with an iterator walking from the bottom to the
+ * top. The memory foot print for a pool never shrinks. Thus this is
+ * not suitable for long-term uses. */
+
+struct pool_struct {
+	void **array;
+	size_t len; /* length of array */
+	size_t num; /* number of items in the array */
+	size_t idx; /* index used dy iter */
+	lock lock;
+};
+
+typedef struct pool_struct pool;
+
+/* allocate a new pool */
+pool *pool_new(void);
+
+/* func type applied to each item in a pool */
+typedef void (*pool_map_f)(void *v);
+
+/* apply f, which free an item, to all items and set num to 0 */
+void pool_zeroize(pool *p, pool_map_f f);
+
+/* free pool->array and pool */
+void pool_free(pool *p);
+
+/* free pool->array and pool after applying f to all items in p->array */
+void pool_destroy(pool *p, pool_map_f f);
+
+#define pool_lock(p) LOCK_ACQUIRE(&(p->lock))
+#define pool_unlock(p) LOCK_RELEASE()
+
+/*
+ * pool_push() pushes *v to pool *p. pool_push_lock() does this while
+ * locking *p.
+ */
+int pool_push(pool *p, void *v);
+int pool_push_lock(pool *p, void *v);
+
+/*
+ * pool_pop() pops the last *v pushed to *p. pool_pop_lock() does this
+ * while locking *p.
+ */
+void *pool_pop(pool *p);
+void *pool_pop_lock(pool *p);
+
+/* pool_get() returns value indexed by idx */
+void *pool_get(pool *p, unsigned int idx);
+
+#define pool_size(p) ((p)->num)
+#define pool_is_empty(p) (pool_size(p) == 0)
+
+/*
+ * pool->idx indicates next *v in an iteration. This has two
+ * use-cases.
+ *
+ * (1) A simple list: just a single thread has a pool, and the thread
+ * can call pool_iter_for_each() for the pool (not thread safe).
+ *
+ * (2) A thread-safe queue: one thread initializes the iterator for a
+ * pool by pool_iter_init(). Then, multiple threads get a next *v
+ * concurrently by pool_iter_next_lock(), which means dequeuing. At
+ * this time, other thread can add new *v by pool_push_lock(), which
+ * means enqueuing. During this, other threads must not intercept the
+ * pool by pool_iter_* functions.
+ */
+
+#define pool_iter_init(p) (p->idx = 0)
+void *pool_iter_next(pool *p);
+void *pool_iter_next_lock(pool *p);
+
+/* pool_iter_has_next_lock() returns true if pool_iter_next(_lock)
+ * function will retrun a next value, otherwise false, which means
+ * there is no more values in this iteration. */
+bool pool_iter_has_next_lock(pool *p);
+
+#define pool_iter_for_each(p, v) \
+	pool_iter_init(p);       \
+	for (v = pool_iter_next(p); v != NULL; v = pool_iter_next(p))
+
+#define pool_for_each(p, v, idx) \
+	idx = 0;                 \
+	for (v = pool_get(p, idx); v != NULL; v = pool_get(p, ++idx))
+
+#endif /* _POOL_H_ */

src/print.c

@@ -9,7 +9,8 @@ void set_print_severity(int serverity)
 {
 	if (serverity < 0)
 		__print_severity = -1; /* no print */
-	__print_severity = serverity;
+        else
+                __print_severity = serverity;
 }
 
 int get_print_severity()

src/ssh.c

@@ -4,13 +4,15 @@
 #include <unistd.h>
 #include <stdlib.h>
 
-#include "libssh/callbacks.h"
-
 #include <ssh.h>
 #include <mscp.h>
 #include <strerrno.h>
 
+#include "libssh/callbacks.h"
+#include "libssh/options.h"
+
 static int ssh_verify_known_hosts(ssh_session session);
+static int ssh_authenticate_kbdint(ssh_session session);
 
 static int ssh_set_opts(ssh_session ssh, struct mscp_ssh_opts *opts)
 {
@@ -86,30 +88,50 @@ static int ssh_set_opts(ssh_session ssh, struct mscp_ssh_opts *opts)
 		return -1;
 	}
 
+	if (opts->proxyjump) {
+		char buf[256];
+		memset(buf, 0, sizeof(buf));
+		snprintf(buf, sizeof(buf), "proxyjump=%s", opts->proxyjump);
+		if (ssh_config_parse_string(ssh, buf) != SSH_OK) {
+			priv_set_errv("failed to set ssh option: %s", buf);
+			return -1;
+		}
+	}
+
+	if (opts->options) {
+		int n;
+		for (n = 0; opts->options[n]; n++) {
+			if (ssh_config_parse_string(ssh, opts->options[n]) != SSH_OK) {
+				priv_set_errv("failed to set ssh option: %s",
+					      opts->options[n]);
+				return -1;
+			}
+		}
+	}
+
 	return 0;
 }
 
 static int ssh_authenticate(ssh_session ssh, struct mscp_ssh_opts *opts)
 {
-	int auth_bit_mask;
+	static int auth_bit_mask;
 	int ret;
 
-	/* none method */
-	ret = ssh_userauth_none(ssh, NULL);
-	if (ret == SSH_AUTH_SUCCESS)
-		return 0;
+        if (auth_bit_mask == 0) {
+                /* the first authentication attempt. try none auth to
+                 * get available auth methods. */
+                if (ssh_userauth_none(ssh, NULL) == SSH_AUTH_SUCCESS)
+                        return 0;
 
-	auth_bit_mask = ssh_userauth_list(ssh, NULL);
-
-	if (auth_bit_mask & SSH_AUTH_METHOD_NONE &&
-	    ssh_userauth_none(ssh, NULL) == SSH_AUTH_SUCCESS)
-		return 0;
+                /* save auth_bit_mask for further authentications */
+                auth_bit_mask = ssh_userauth_list(ssh, NULL);
+        }
 
 	if (auth_bit_mask & SSH_AUTH_METHOD_PUBLICKEY) {
-		char *p = opts->passphrase ? opts->passphrase : NULL;
-		if (ssh_userauth_publickey_auto(ssh, NULL, p) == SSH_AUTH_SUCCESS)
-			return 0;
-	}
+                char *p = opts->passphrase ? opts->passphrase : NULL;
+                if (ssh_userauth_publickey_auto(ssh, NULL, p) == SSH_AUTH_SUCCESS)
+                        return 0;
+        }
 
 	if (auth_bit_mask & SSH_AUTH_METHOD_PASSWORD) {
 		if (!opts->password) {
@@ -128,6 +150,12 @@ static int ssh_authenticate(ssh_session ssh, struct mscp_ssh_opts *opts)
 			return 0;
 	}
 
+	auth_bit_mask = ssh_userauth_list(ssh, NULL);
+	if (auth_bit_mask & SSH_AUTH_METHOD_INTERACTIVE) {
+		if (ssh_authenticate_kbdint(ssh) == SSH_AUTH_SUCCESS)
+			return 0;
+	}
+
 	return -1;
 }
 
@@ -191,7 +219,7 @@ static ssh_session ssh_init_session(const char *sshdst, struct mscp_ssh_opts *op
 		goto disconnect_out;
 	}
 
-	if (!opts->no_hostkey_check && ssh_verify_known_hosts(ssh) != 0) {
+	if (ssh_verify_known_hosts(ssh) != 0) {
 		priv_set_errv("ssh_veriy_known_hosts failed");
 		goto disconnect_out;
 	}
@@ -319,6 +347,54 @@ static int ssh_verify_known_hosts(ssh_session session)
 	return 0;
 }
 
+static int ssh_authenticate_kbdint(ssh_session ssh)
+{
+	/* Copied and bit modified from
+	 * https://api.libssh.org/stable/libssh_tutor_authentication.html */
+	int rc;
+
+	rc = ssh_userauth_kbdint(ssh, NULL, NULL);
+	while (rc == SSH_AUTH_INFO) {
+		const char *name, *instruction;
+		int nprompts, iprompt;
+
+		name = ssh_userauth_kbdint_getname(ssh);
+		instruction = ssh_userauth_kbdint_getinstruction(ssh);
+		nprompts = ssh_userauth_kbdint_getnprompts(ssh);
+
+		if (strlen(name) > 0)
+			printf("%s\n", name);
+		if (strlen(instruction) > 0)
+			printf("%s\n", instruction);
+		for (iprompt = 0; iprompt < nprompts; iprompt++) {
+			const char *prompt;
+			char echo;
+
+			prompt = ssh_userauth_kbdint_getprompt(ssh, iprompt, &echo);
+			if (echo) {
+				char buf[128], *ptr;
+
+				printf("%s", prompt);
+				if (fgets(buf, sizeof(buf), stdin) == NULL)
+					return SSH_AUTH_ERROR;
+				buf[sizeof(buf) - 1] = '\0';
+				if ((ptr = strchr(buf, '\n')) != NULL)
+					*ptr = '\0';
+				if (ssh_userauth_kbdint_setanswer(ssh, iprompt, buf) < 0)
+					return SSH_AUTH_ERROR;
+				memset(buf, 0, strlen(buf));
+			} else {
+				char *ptr;
+				ptr = getpass(prompt);
+				if (ssh_userauth_kbdint_setanswer(ssh, iprompt, ptr) < 0)
+					return SSH_AUTH_ERROR;
+			}
+		}
+		rc = ssh_userauth_kbdint(ssh, NULL, NULL);
+	}
+	return rc;
+}
+
 void ssh_sftp_close(sftp_session sftp)
 {
 	ssh_session ssh = sftp_ssh(sftp);
@@ -329,3 +405,13 @@ void ssh_sftp_close(sftp_session sftp)
 	ssh_disconnect(ssh);
 	ssh_free(ssh);
 }
+
+const char **mscp_ssh_ciphers(void)
+{
+	return ssh_ciphers();
+}
+
+const char **mscp_ssh_hmacs(void)
+{
+	return ssh_hmacs();
+}

test/test_e2e.py

@@ -6,24 +6,72 @@ test_e2e.py: End-to-End test for mscp executable.
 import platform
 import pytest
 import getpass
+import datetime
+import time
 import os
+import re
+import shutil
 
-from subprocess import check_call, CalledProcessError, PIPE
+from subprocess import check_call, CalledProcessError
 from util import File, check_same_md5sum
 
 
-def run2ok(args, env = None):
+def run2ok(args, env = None, quiet = False):
     cmd = list(map(str, args))
-    print("cmd: {}".format(" ".join(cmd)))
+    if not quiet:
+        print("cmd: {}".format(" ".join(cmd)))
     check_call(cmd, env = env)
 
-def run2ng(args, env = None):
+def run2ng(args, env = None, timeout = None, quiet = False):
+    if timeout:
+        args = ["timeout", "-s", "INT", timeout] + args
     cmd = list(map(str, args))
-    print("cmd: {}".format(" ".join(cmd)))
-    with pytest.raises(CalledProcessError) as e:
+    if not quiet:
+        print("cmd: {}".format(" ".join(cmd)))
+    with pytest.raises(CalledProcessError):
         check_call(cmd, env = env)
 
 
+@pytest.fixture(autouse=True)
+def cleanup_files():
+    """
+    Cleanup files having the following `prefixes` or matching `paths`.
+    """
+
+    yield
+
+    prefixes = [
+        "src", "dst",
+        "non_existent_dstdir",
+    ]
+    paths = [
+        "/mscp-test-src", "/tmp/mscp-test-src",
+        "{}/src".format(os.environ["HOME"]),
+        "{}/dst".format(os.environ["HOME"]),
+        "/tmp/mscp_test_ssh_config",
+        "/home/test/dst",
+        "/home/test/src",
+        "checkpoint",
+    ]
+
+    def remove(path):
+        print(f"cleanup remove: {fname}")
+        if os.path.isdir(path):
+            shutil.rmtree(path)
+        else:
+            os.remove(path)
+
+    for fname in os.listdir(os.getcwd()):
+        for prefix in prefixes:
+            if fname.startswith(prefix):
+                remove(fname)
+                break
+
+    for path in paths:
+        if os.path.exists(path):
+            remove(path)
+
+
 """ usage test """
 
 def test_usage(mscp):
@@ -62,16 +110,15 @@ param_single_copy = [
 @pytest.mark.parametrize("src, dst", param_single_copy)
 def test_single_copy(mscp, src_prefix, dst_prefix, src, dst):
     src.make()
-    run2ok([mscp, "-H", "-vvv", src_prefix + src.path, dst_prefix + dst.path])
+    run2ok([mscp, "-vvv", src_prefix + src.path, dst_prefix + dst.path])
     assert check_same_md5sum(src, dst)
-    src.cleanup()
-    dst.cleanup()
+
 
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 def test_failed_to_copy_nonexistent_file(mscp, src_prefix, dst_prefix):
     src = "nonexistent_src"
     dst = "nonexistent_dst"
-    run2ng([mscp, "-H", "-vvv", src_prefix + src, dst_prefix + dst])
+    run2ng([mscp, "-vvv", src_prefix + src, dst_prefix + dst])
 
 param_double_copy = [
     (File("src1", size = 1024 * 1024), File("src2", size = 1024 * 1024),
@@ -83,13 +130,9 @@ param_double_copy = [
 def test_double_copy(mscp, src_prefix, dst_prefix, s1, s2, d1, d2):
     s1.make()
     s2.make()
-    run2ok([mscp, "-H", "-vvv", src_prefix + s1.path, src_prefix + s2.path, dst_prefix + "dst"])
+    run2ok([mscp, "-vvv", src_prefix + s1.path, src_prefix + s2.path, dst_prefix + "dst"])
     assert check_same_md5sum(s1, d1)
     assert check_same_md5sum(s2, d2)
-    s1.cleanup()
-    s2.cleanup()
-    d1.cleanup()
-    d2.cleanup()
 
 
 remote_v6_prefix = "[::1]:{}/".format(os.getcwd())
@@ -101,14 +144,10 @@ param_remote_v6_prefix = [
 def test_double_copy_with_ipv6_notation(mscp, src_prefix, dst_prefix, s1, s2, d1, d2):
     s1.make()
     s2.make()
-    run2ok([mscp, "-H", "-vvv",
+    run2ok([mscp, "-vvv",
             src_prefix + s1.path, src_prefix + s2.path, dst_prefix + "dst"])
     assert check_same_md5sum(s1, d1)
     assert check_same_md5sum(s2, d2)
-    s1.cleanup()
-    s2.cleanup()
-    d1.cleanup()
-    d2.cleanup()
 
 
 remote_user_v6_prefix = "{}@[::1]:{}/".format(getpass.getuser(), os.getcwd())
@@ -121,15 +160,10 @@ def test_double_copy_with_user_and_ipv6_notation(mscp, src_prefix, dst_prefix,
                                                  s1, s2, d1, d2):
     s1.make()
     s2.make()
-    run2ok([mscp, "-H", "-vvv",
+    run2ok([mscp, "-vvv",
             src_prefix + s1.path, src_prefix + s2.path, dst_prefix + "dst"])
     assert check_same_md5sum(s1, d1)
     assert check_same_md5sum(s2, d2)
-    s1.cleanup()
-    s2.cleanup()
-    d1.cleanup()
-    d2.cleanup()
-
 
 
 param_dir_copy = [
@@ -161,18 +195,14 @@ def test_dir_copy(mscp, src_prefix, dst_prefix, src_dir, dst_dir, src, dst, twic
     for f in src:
         f.make()
 
-    run2ok([mscp, "-H", "-vvv", src_prefix + src_dir, dst_prefix + dst_dir])
+    run2ok([mscp, "-vvv", src_prefix + src_dir, dst_prefix + dst_dir])
     for sf, df in zip(src, dst):
         assert check_same_md5sum(sf, df)
 
-    run2ok([mscp, "-H", "-vvv", src_prefix + src_dir, dst_prefix + dst_dir])
+    run2ok([mscp, "-vvv", src_prefix + src_dir, dst_prefix + dst_dir])
     for sf, df in zip(src, twice):
         assert check_same_md5sum(sf, df)
 
-    for sf, df, tf in zip(src, dst, twice):
-        sf.cleanup()
-        df.cleanup()
-        tf.cleanup()
 
 
 param_dir_copy_single = [
@@ -186,10 +216,9 @@ param_dir_copy_single = [
 def test_dir_copy_single(mscp, src_prefix, dst_prefix, src_dir, dst_dir, src, dst):
     src.make()
     os.mkdir(dst_dir)
-    run2ok([mscp, "-H", "-vvv", src_prefix + src_dir, dst_prefix + dst_dir])
+    run2ok([mscp, "-vvv", src_prefix + src_dir, dst_prefix + dst_dir])
     assert check_same_md5sum(src, dst)
-    src.cleanup()
-    dst.cleanup()
+
 
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 def test_override_single_file(mscp, src_prefix, dst_prefix):
@@ -197,11 +226,9 @@ def test_override_single_file(mscp, src_prefix, dst_prefix):
     dst = File("dst", size = 128).make()
     assert not check_same_md5sum(src, dst)
 
-    run2ok([mscp, "-H", "-vvv", src_prefix + src.path, dst_prefix + dst.path])
+    run2ok([mscp, "-vvv", src_prefix + src.path, dst_prefix + dst.path])
     assert check_same_md5sum(src, dst)
 
-    src.cleanup()
-    dst.cleanup()
 
 absolute_remote_prefix = "localhost:"
 param_absolute_remote_prefix = [
@@ -212,11 +239,58 @@ def test_copy_file_under_root_to_dir(mscp, src_prefix, dst_prefix):
     src = File("/mscp-test-src", size = 1024).make()
     dst = File("/tmp/mscp-test-src")
 
-    run2ok([mscp, "-H", "-vvv", src_prefix + src.path,
+    run2ok([mscp, "-vvv", src_prefix + src.path,
             dst_prefix + os.path.dirname(dst.path)])
     assert check_same_md5sum(src, dst)
-    src.cleanup()
-    dst.cleanup(preserve_dir = True)
+
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_dst_has_suffix_slash(mscp, src_prefix, dst_prefix):
+    """
+    if dst path has suffix '/' like "dir/" and does not exist,
+    mscp should create dir/ and put dir/src-file-name.
+    """
+    dstdir = "non_existent_dstdir/"
+
+    src = File("src", size = 1024 * 1024).make()
+    dst = File(f"{dstdir}/src")
+
+    run2ok([mscp, "-vvv", src_prefix + src.path,
+            dst_prefix + dstdir])
+
+    assert check_same_md5sum(src, dst)
+
+
+param_tilde_paths = [
+    ("src", "localhost:~/dst"),
+    ("localhost:~/src", "dst"),
+]
+@pytest.mark.parametrize("src_path, dst_path", param_tilde_paths)
+def test_remote_path_contains_tilde(mscp, src_path, dst_path):
+    """
+    if remote path contains '~' as prefix, it should be expanded as '.'.
+    Note that `~user` notation is not supported yet.
+    """
+    def extract_and_expand(path):
+        path = path if not ':' in path else path[path.index(':')+1:]
+        return path.replace('~', os.environ["HOME"])
+
+    src_f_path = extract_and_expand(src_path)
+    dst_f_path = extract_and_expand(dst_path)
+
+    src = File(src_f_path, size = 1024 * 1024).make()
+    dst = File(dst_f_path)
+
+    run2ok([mscp, "-vvv", src_path, dst_path])
+    assert check_same_md5sum(src, dst)
+
+
+def test_remote_path_contains_tilde2(mscp):
+    src = File("src", size = 1024 * 1024).make()
+    dst = File(f"{os.environ['HOME']}/src")
+
+    run2ok([mscp, "-vvv", src.path, f"localhost:~"])
+    assert check_same_md5sum(src, dst)
 
 
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
@@ -224,12 +298,9 @@ def test_min_chunk(mscp, src_prefix, dst_prefix):
     src = File("src", size = 16 * 1024).make()
     dst = File("dst")
 
-    run2ok([mscp, "-H", "-vvv", "-s", 32768, src_prefix + src.path, dst_prefix + dst.path])
+    run2ok([mscp, "-vvv", "-s", 32768, src_prefix + src.path, dst_prefix + dst.path])
     assert check_same_md5sum(src, dst)
 
-    src.cleanup()
-    dst.cleanup()
-
 
 def is_alpine():
     if os.path.exists("/etc/os-release"):
@@ -261,100 +332,146 @@ def test_glob_src_path(mscp, src_prefix, dst_prefix,
     for src in srcs:
         src.make(size = 1024 * 1024)
 
-    run2ok([mscp, "-H", "-vvv", src_prefix + src_glob_path, dst_prefix + dst_path])
+    run2ok([mscp, "-vvv", src_prefix + src_glob_path, dst_prefix + dst_path])
     for src, dst in zip(srcs, dsts):
         assert check_same_md5sum(src, dst)
-        src.cleanup()
-        dst.cleanup()
+
 
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 def test_thread_affinity(mscp, src_prefix, dst_prefix):
     src = File("src", size = 64 * 1024).make()
     dst = File("dst")
 
-    run2ok([mscp, "-H", "-vvv", "-n", 4, "-m", "0x01",
+    run2ok([mscp, "-vvv", "-n", 4, "-m", "0x01",
             src_prefix + src.path, dst_prefix + dst.path])
     assert check_same_md5sum(src, dst)
 
-    src.cleanup()
-    dst.cleanup()
 
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 def test_cannot_override_file_with_dir(mscp, src_prefix, dst_prefix):
     src = File("src", size = 128).make()
     dst = File("dst").make()
 
-    run2ng([mscp, "-H", "-vvv", src_prefix + src.path, dst_prefix + "dst/src"])
+    run2ng([mscp, "-vvv", src_prefix + src.path, dst_prefix + "dst/src"])
 
-    src.cleanup()
-    dst.cleanup()
 
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 def test_transfer_zero_bytes(mscp, src_prefix, dst_prefix):
     src = File("src", size = 0).make()
     dst = File("dst")
-    run2ok([mscp, "-H", "-vvv", src_prefix + src.path, dst_prefix + "dst"])
+    run2ok([mscp, "-vvv", src_prefix + src.path, dst_prefix + "dst"])
     assert os.path.exists("dst")
-    src.cleanup()
-    dst.cleanup()
 
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 def test_override_dst_having_larger_size(mscp, src_prefix, dst_prefix):
     src = File("src", size = 1024 * 1024).make()
     dst = File("dst", size = 1024 * 1024 * 2).make()
-    run2ok([mscp, "-H", "-vvv", src_prefix + src.path, dst_prefix + "dst"])
+    run2ok([mscp, "-vvv", src_prefix + src.path, dst_prefix + "dst"])
     assert check_same_md5sum(src, dst)
-    src.cleanup()
-    dst.cleanup()
 
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 def test_dont_truncate_dst(mscp, src_prefix, dst_prefix):
     f = File("srcanddst", size = 1024 * 1024 * 128).make()
     md5_before = f.md5sum()
-    run2ok([mscp, "-H", "-vvv", src_prefix + f.path, dst_prefix + f.path])
+    run2ok([mscp, "-vvv", src_prefix + f.path, dst_prefix + f.path])
     md5_after = f.md5sum()
     assert md5_before == md5_after
-    f.cleanup()
+
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_copy_readonly_file(mscp, src_prefix, dst_prefix):
+    """When a source file permission is r--r--r--, if chmod(r--r--r--)
+    runs first on the remote side, following truncate() and setutime()
+    fail due to permission deneid. So, run chmod() after truncate()
+    and setutime()
+
+    """
+    src = File("src", size = 1024 * 1024 * 128, perm = 0o444).make()
+    dst = File("dst")
+    run2ok([mscp, "-vvv", src_prefix + src.path, dst_prefix + dst.path])
+    assert check_same_md5sum(src, dst)
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_dont_make_conns_more_than_chunks(mscp, src_prefix, dst_prefix):
+    # copy 100 files with -n 20 -I 1 options. if mscp creates 20 SSH
+    # connections although all files have been copied, it is error.
+    srcs = []
+    dsts = []
+    for n in range(100):
+        srcs.append(File("src/src-{:06d}".format(n), size=1024).make())
+        dsts.append(File("dst/src-{:06d}".format(n)))
+    start = time.time()
+    run2ok([mscp, "-v", "-n", "20", "-I", "1",
+            src_prefix + "src", dst_prefix + "dst"])
+    end = time.time()
+    for s, d in zip(srcs, dsts):
+        assert check_same_md5sum(s, d)
+
+    assert((end - start) < 10)
+
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_bwlimit(mscp, src_prefix, dst_prefix):
+    """Copy 100MB file with 100Mbps bitrate, this requires 8 seconds."""
+    src = File("src", size = 100 * 1024 * 1024).make()
+    dst = File("dst")
+
+    start = datetime.datetime.now().timestamp()
+    run2ok([mscp, "-vvv", "-L", "100m", src_prefix + "src", dst_prefix + "dst"])
+    end = datetime.datetime.now().timestamp()
+    assert check_same_md5sum(src, dst)
+    assert end - start > 7
+
 
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 @pytest.mark.parametrize("src, dst", param_single_copy)
-def test_set_port(mscp, src_prefix, dst_prefix, src, dst):
+def test_set_port_ng(mscp, src_prefix, dst_prefix, src, dst):
     src.make()
-    run2ng([mscp, "-H", "-vvv", "-p", 21, src_prefix + src.path, dst_prefix + dst.path])
-    run2ng([mscp, "-H", "-vvv", "-P", 21, src_prefix + src.path, dst_prefix + dst.path])
-    src.cleanup()
+    run2ng([mscp, "-vvv", "-P", 21, src_prefix + src.path, dst_prefix + dst.path])
+
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+@pytest.mark.parametrize("src, dst", param_single_copy)
+def test_set_port_ok(mscp, src_prefix, dst_prefix, src, dst):
+    src.make()
+    run2ok([mscp, "-vvv", "-P", 8022, src_prefix + src.path, dst_prefix + dst.path])
 
 def test_v4only(mscp):
     src = File("src", size = 1024).make()
     dst = File("dst")
     dst_prefix = "localhost:{}/".format(os.getcwd())
-    run2ok([mscp, "-H", "-vvv", "-4", src.path, dst_prefix + dst.path])
+    run2ok([mscp, "-vvv", "-4", src.path, dst_prefix + dst.path])
     assert check_same_md5sum(src, dst)
-    src.cleanup()
-    dst.cleanup()
 
 def test_v6only(mscp):
     src = File("src", size = 1024).make()
     dst = File("dst")
-    dst_prefix = "localhost:{}/".format(os.getcwd())
-    run2ok([mscp, "-H", "-vvv", "-6", src.path, dst_prefix + dst.path])
+    dst_prefix = "ip6-localhost:{}/".format(os.getcwd())
+    run2ok([mscp, "-vvv", "-6", src.path, dst_prefix + dst.path])
     assert check_same_md5sum(src, dst)
-    src.cleanup()
-    dst.cleanup()
 
 def test_v4_to_v6_should_fail(mscp):
     src = File("src", size = 1024).make()
     dst = File("dst")
     dst_prefix = "[::1]:{}/".format(os.getcwd())
-    run2ng([mscp, "-H", "-vvv", "-4", src.path, dst_prefix + dst.path])
-    src.cleanup()
+    run2ng([mscp, "-vvv", "-4", src.path, dst_prefix + dst.path])
 
 def test_v6_to_v4_should_fail(mscp):
     src = File("src", size = 1024).make()
     dst = File("dst")
     dst_prefix = "127.0.0.1:{}/".format(os.getcwd())
-    run2ng([mscp, "-H", "-vvv", "-6", src.path, dst_prefix + dst.path])
-    src.cleanup()
+    run2ng([mscp, "-vvv", "-6", src.path, dst_prefix + dst.path])
+
+def test_quiet_mode(capsys, mscp):
+    src = File("src", size = 1024).make()
+    dst = File("dst")
+    dst_prefix = "127.0.0.1:{}/".format(os.getcwd())
+    run2ok([mscp, "-vvv", "-q", src.path, dst_prefix + dst.path], quiet=True)
+    assert check_same_md5sum(src, dst)
+
+    captured = capsys.readouterr()
+    assert not captured.out
+    assert not captured.err
 
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 def test_set_conn_interval(mscp, src_prefix, dst_prefix):
@@ -363,12 +480,10 @@ def test_set_conn_interval(mscp, src_prefix, dst_prefix):
     for x in range(500):
         srcs.append(File("src/file{}".format(x), size = 128).make())
         dsts.append(File("dst/file{}".format(x)))
-    run2ok([mscp, "-H", "-vvv", "-I", 1, src_prefix + "src", dst_prefix + "dst"])
+    run2ok([mscp, "-vvv", "-I", 1, src_prefix + "src", dst_prefix + "dst"])
 
     for src, dst in zip(srcs, dsts):
         assert check_same_md5sum(src, dst)
-        src.cleanup()
-        dst.cleanup()
 
 compressions = ["yes", "no", "none"]
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
@@ -376,10 +491,8 @@ compressions = ["yes", "no", "none"]
 def test_compression(mscp, src_prefix, dst_prefix, compress):
     src = File("src", size = 1024 * 1024).make()
     dst = File("dst", size = 1024 * 1024 * 2).make()
-    run2ok([mscp, "-H", "-vvv", "-C", compress, src_prefix + src.path, dst_prefix + "dst"])
+    run2ok([mscp, "-vvv", "-C", compress, src_prefix + src.path, dst_prefix + "dst"])
     assert check_same_md5sum(src, dst)
-    src.cleanup()
-    dst.cleanup()
 
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 def test_ccalgo(mscp, src_prefix, dst_prefix):
@@ -394,7 +507,7 @@ def test_ccalgo(mscp, src_prefix, dst_prefix):
         with open("/proc/sys/net/ipv4/tcp_allowed_congestion_control", "r") as f:
             algo = f.read().strip().split().pop()
         run = run2ok
-    run([mscp, "-H", "-vvv", "-g", algo, src_prefix + src.path, dst_prefix + "dst"])
+    run([mscp, "-vvv", "-g", algo, src_prefix + src.path, dst_prefix + "dst"])
 
 
 testhost = "mscptestlocalhost"
@@ -411,13 +524,12 @@ def test_config_ok(mscp, src_prefix, dst_prefix):
 
     src = File("src", size = 1024 * 1024).make()
     dst = File("dst", size = 1024 * 1024 * 2).make()
-    run2ok([mscp, "-H", "-vvv", "-F", config,
+    run2ok([mscp, "-vvv", "-F", config,
             src_prefix + src.path, dst_prefix + "dst"])
 
     os.remove(config)
     assert check_same_md5sum(src, dst)
-    src.cleanup()
-    dst.cleanup()
+
 
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_testhost_prefix)
 def test_config_ng(mscp, src_prefix, dst_prefix):
@@ -427,12 +539,63 @@ def test_config_ng(mscp, src_prefix, dst_prefix):
 
     src = File("src", size = 1024 * 1024).make()
     dst = File("dst", size = 1024 * 1024 * 2).make()
-    run2ng([mscp, "-H", "-vvv", "-F", config,
+    run2ng([mscp, "-vvv", "-F", config,
             src_prefix + src.path, dst_prefix + "dst"])
 
     os.remove(config)
-    src.cleanup()
-    dst.cleanup()
+
+
+param_valid_option_ok = [
+    [ "-o", "Port=8022" ],
+    [ "-o", "Port=8022", "-o", "User=root" ],
+    [ "-o", "unknown-option-is-silently-ignored" ],
+]
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+@pytest.mark.parametrize("option", param_valid_option_ok)
+def test_inline_option_ok(mscp, src_prefix, dst_prefix, option):
+    """ change port number with -o option. it should be ok. """
+    src = File("src", size = 1024 * 1024).make()
+    dst = File("dst")
+    run2ok([mscp, "-vvv"] + option +
+           [src_prefix + src.path, dst_prefix + dst.path])
+    assert check_same_md5sum(src, dst)
+
+
+param_valid_option_ng = [
+    [ "-o", "Port=8023" ],
+    [ "-o", "User=invaliduser" ],
+]
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+@pytest.mark.parametrize("option", param_valid_option_ng)
+def test_inline_option_ng(mscp, src_prefix, dst_prefix, option):
+    """ change port number with -o option. it should be ng. """
+    src = File("src", size = 1024 * 1024).make()
+    dst = File("dst")
+    run2ng([mscp, "-vvv"] + option +
+           [src_prefix + src.path, dst_prefix + dst.path])
+
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_porxyjump_ok(mscp, src_prefix, dst_prefix):
+    """ test -J proxyjump option"""
+    src = File("src", size = 10 * 1024 * 1024).make()
+    dst = File("dst")
+    # use small min-chunk-size to use multiple connections
+    run2ok([mscp, "-n", 4, "-s", 1024 * 1024, "-vvv",
+            "-J", "localhost:8022",
+            src_prefix + src.path, dst_prefix + dst.path])
+    assert check_same_md5sum(src, dst)
+
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_porxyjump_ng(mscp, src_prefix, dst_prefix):
+    """ test -J proxyjump option, invalid jump node causes fail"""
+    src = File("src", size = 10 * 1024 * 1024).make()
+    dst = File("dst")
+    # use small min-chunk-size to use multiple connections
+    run2ng([mscp, "-n", 4, "-s", 1024 * 1024, "-vvv",
+            "-J", "invaliduser@localhost:8022",
+            src_prefix + src.path, dst_prefix + dst.path])
 
 # username test assumes that this test runs inside a container, see Dockerfiles
 def test_specify_passphrase_via_env(mscp):
@@ -440,38 +603,140 @@ def test_specify_passphrase_via_env(mscp):
     dst = File("/home/test/dst")
     env = os.environ
     env["MSCP_SSH_AUTH_PASSPHRASE"]  = "keypassphrase"
-    run2ok([mscp, "-H", "-vvv", "-l", "test", "-i", "/home/test/.ssh/id_rsa_test",
+    run2ok([mscp, "-vvv", "-l", "test", "-i", "/home/test/.ssh/id_rsa_test",
             src.path, "localhost:" + dst.path], env = env)
     assert check_same_md5sum(src, dst)
-    src.cleanup()
-    dst.cleanup()
 
 def test_specify_invalid_passphrase_via_env(mscp):
     src = File(os.getcwd() + "/src", size = 1024).make()
     dst = File("/home/test/dst")
     env = os.environ
     env["MSCP_SSH_AUTH_PASSPHRASE"]  = "invalid-keypassphrase"
-    run2ng([mscp, "-H", "-vvv", "-l", "test", "-i", "/home/test/.ssh/id_rsa_test",
+    run2ng([mscp, "-vvv", "-l", "test", "-i", "/home/test/.ssh/id_rsa_test",
             src.path, "localhost:" + dst.path], env = env)
-    src.cleanup()
 
 def test_specify_password_via_env(mscp):
     src = File(os.getcwd() + "/src", size = 1024).make()
     dst = File("/home/test/dst")
     env = os.environ
     env["MSCP_SSH_AUTH_PASSWORD"]  = "userpassword"
-    run2ok([mscp, "-H", "-vvv", "-l", "test",
+    run2ok([mscp, "-vvv", "-l", "test",
             src.path, "localhost:" + dst.path], env = env)
     assert check_same_md5sum(src, dst)
-    src.cleanup()
-    dst.cleanup()
 
 def test_specify_invalid_password_via_env(mscp):
     src = File(os.getcwd() + "/src", size = 1024).make()
     dst = File("/home/test/dst")
     env = os.environ
     env["MSCP_SSH_AUTH_PASSWORD"]  = "invalid-userpassword"
-    run2ng([mscp, "-H", "-vvv", "-l", "test",
+    run2ng([mscp, "-vvv", "-l", "test",
             src.path, "localhost:" + dst.path], env = env)
-    src.cleanup()
+
+
+@pytest.fixture
+def move_pubkey_temporally():
+    """
+    mv ~/.ssh/id_* to id_rsa.bak before test, and move it back after test.
+    """
+
+    sshdir = os.path.join(os.environ["HOME"], ".ssh")
+
+    # move pubkeys to /tmp
+    moved = []
+    for fname in os.listdir(sshdir):
+        if re.match(r"^id_[a-z0-9]+$", fname):
+            moved.append(fname)
+            shutil.move(f"{sshdir}/{fname}", f"/tmp/{fname}")
+
+    yield
+
+    # move back the keys
+    for fname in moved:
+        shutil.move(f"/tmp/{fname}", f"{sshdir}/{fname}")
+
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_passwordauth_without_pubkey(move_pubkey_temporally,
+                                     mscp, src_prefix, dst_prefix):
+    """
+    make sure password auth works (by removing public keys)
+    """
+    src = File(os.getcwd() + "/src", size = 1024).make()
+    dst = File("/home/test/dst")
+    env = os.environ
+    env["MSCP_SSH_AUTH_PASSWORD"]  = "userpassword"
+    run2ok([mscp, "-vvv", "-l", "test",
+            src.path, "localhost:" + dst.path], env = env)
+    assert check_same_md5sum(src, dst)
+
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_10k_files(mscp, src_prefix, dst_prefix):
+    srcs = []
+    dsts = []
+    for n in range(10000):
+        srcs.append(File("src/src-{:06d}".format(n), size=1024).make())
+        dsts.append(File("dst/src-{:06d}".format(n)))
+    run2ok([mscp, "-v", src_prefix + "src", dst_prefix + "dst"])
+    for s, d in zip(srcs, dsts):
+        assert check_same_md5sum(s, d)
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_checkpoint_dump_and_resume(mscp, src_prefix, dst_prefix):
+    src1 = File("src1", size = 64 * 1024 * 1024).make()
+    src2 = File("src2", size = 64 * 1024 * 1024).make()
+    dst1 = File("dst/src1")
+    dst2 = File("dst/src2")
+    run2ok([mscp, "-vvv", "-W", "checkpoint", "-D",
+            src_prefix + "src1", src_prefix + "src2", dst_prefix + "dst"])
+    assert os.path.exists("checkpoint")
+
+    run2ok([mscp, "-vvv", "-R", "checkpoint"])
+    assert check_same_md5sum(src1, dst1)
+    assert check_same_md5sum(src2, dst2)
+    os.remove("checkpoint")
+
+@pytest.mark.parametrize("timeout", [ 1, 2, 3, 4, 5 ])
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_checkpoint_interrupt_large_file(mscp, timeout, src_prefix, dst_prefix):
+    """Copy two 100MB files with 200Mbps -> 4 sec + 4 sec """
+    src1 = File("src1", size = 100 * 1024 * 1024).make()
+    src2 = File("src2", size = 100 * 1024 * 1024).make()
+    dst1 = File("dst/src1")
+    dst2 = File("dst/src2")
+    run2ng([mscp, "-vv", "-W", "checkpoint", "-L", "200m",
+            src_prefix + "src1", src_prefix + "src2", dst_prefix + "dst"],
+           timeout = timeout)
+    assert os.path.exists("checkpoint")
+
+    run2ok([mscp, "-vv", "-R", "checkpoint"])
+    assert check_same_md5sum(src1, dst1)
+    assert check_same_md5sum(src2, dst2)
+    os.remove("checkpoint")
+
+@pytest.mark.parametrize("timeout", [ 1, 2, 3, 4, 5 ])
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_checkpoint_interrupt_many_files(mscp, timeout, src_prefix, dst_prefix):
+    """Copy 100 1-MB files with 4 connections, and interrupt and
+    resume the transfer
+    """
+
+    files = []
+    for x in range(100):
+        files.append((
+            File("src/{:03d}".format(x), size = 1024 * 1024).make(),
+            File("dst/{:03d}".format(x))
+        ))
+
+    run2ng([mscp, "-vv", "-W", "checkpoint", "-L", "80m", "-n", 4,
+            src_prefix + "src",  dst_prefix + "dst"],
+           timeout = timeout)
+    assert os.path.exists("checkpoint")
+
+    run2ok([mscp, "-vv", "-R", "checkpoint"])
+
+    for src, dst in files:
+        assert check_same_md5sum(src, dst)
+
+    os.remove("checkpoint")