add workflow_dispatch to release.yml

Default source tar balls in github releases do not include
submodules. source-release job uploads mscp tar ball with patched
libssh.

.github/workflows/build-macos.yml

@@ -19,20 +19,27 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
+      with:
+        submodules: true
 
-      # libssh and cmake are already installed on the macos-latest runner, 2022/11/5
-      #    - name: install build dependency
-      #run: |
-      #  brew update
-      #  brew install libssh
-      #  brew install cmake
+    - name: install build dependency
+      run: ./scripts/install-build-deps.sh
+
+    - name: save homebrew prefix
+      id: brew-prefix
+      run: echo "HOMEBREW_PREFIX=$(brew --prefix)" >> $GITHUB_OUTPUT
+
+    - name: patch to libssh
+      run: patch -d libssh -p1 < patch/libssh-0.10.4.patch
 
     - name: Configure CMake
       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
-      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
+      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DOPENSSL_ROOT_DIR=${{steps.brew-prefix.outputs.HOMEBREW_PREFIX}}/opt/openssl@1.1
 
     - name: Build
       # Build your program with the given configuration
       run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}
 
+    - name: Run
+      run: ${{github.workspace}}/build/mscp -h

.github/workflows/build-ubuntu.yml

@@ -19,9 +19,15 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
+      with:
+        submodules: true
+
     - name: install build dependency
       run: sudo ./scripts/install-build-deps.sh
 
+    - name: patch to libssh
+      run: patch -d libssh -p1 < patch/libssh-0.10.4.patch
+
     - name: Configure CMake
       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
@@ -31,3 +37,5 @@ jobs:
       # Build your program with the given configuration
       run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}
 
+    - name: Run
+      run: ${{github.workspace}}/build/mscp -h

.github/workflows/codeql.yml

@@ -39,10 +39,15 @@ jobs:
     steps:
     - name: Checkout repository
       uses: actions/checkout@v3
+      with:
+        submodules: true
 
-    - name: Install build dependency
+    - name: install build dependency
       run: sudo ./scripts/install-build-deps.sh
 
+    - name: patch to libssh
+      run: patch -d libssh -p1 < patch/libssh-0.10.4.patch
+
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v2

.github/workflows/release.yml

@@ -4,6 +4,7 @@ on:
   push:
     tags:
       - "v*.*.*"
+  workflow_dispatch:
 
 env:
     BUILD_TYPE: Release
@@ -13,21 +14,32 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: true
+
+      - name: patch to libssh
+        run: patch -d libssh -p1 < patch/libssh-0.10.4.patch
+
+      # TODO: just building docker does not require packages. fix CMakeLists
+      - name: install build dependency
+        run: sudo ./scripts/install-build-deps.sh
 
       - name: Set variables
         run: |
           VER=$(cat VERSION)
           echo "VERSION=$VER" >> $GITHUB_ENV
 
-      # TODO: docker build does not require libssh. fix CMakeLists
-      - name: install build dependency
-        run: sudo ./scripts/install-build-deps.sh
-
       - name: Configure Cmake
-        run: cmake -B ${{github.workspace}}/build -DBUILD_PKG=1 -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
+        run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
 
-      - name: Build Packages
-        run: make -C ${{github.workspace}}/build package-all-in-docker
+      - name: Build Containers
+        run: make -C ${{github.workspace}}/build docker-build-all
+
+      - name: Test
+        run: make -C ${{github.workspace}}/build docker-test-all
+
+      - name: Retrieve packages from containers
+        run: make -C ${{github.workspace}}/build docker-pkg-all
 
       - name: Release
         uses: softprops/action-gh-release@v1
@@ -36,3 +48,31 @@ jobs:
             ${{github.workspace}}/build/mscp_${{env.VERSION}}-ubuntu-20.04-x86_64.deb
             ${{github.workspace}}/build/mscp_${{env.VERSION}}-ubuntu-22.04-x86_64.deb
             ${{github.workspace}}/build/mscp_${{env.VERSION}}-centos-8-x86_64.rpm
+            ${{github.workspace}}/build/mscp_${{env.VERSION}}-rocky-8.6-x86_64.rpm
+
+  source-release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+
+      - name: patch to libssh
+        run: patch -d libssh -p1 < patch/libssh-0.10.4.patch
+
+      - name: Set variables
+        run: |
+          VER=$(cat VERSION)
+          echo "VERSION=$VER" >> $GITHUB_ENV
+
+      - name: archive
+        run: |
+          cd ..
+          cp -r mscp mscp-${{env.VERSION}}
+          tar cvf mscp-${{env.VERSION}}.tar.gz --exclude-vcs mscp-${{env.VERSION}}
+
+      - name: Release
+        uses: softprops/action-gh-release@v1
+        with:
+          files: |
+            ${{github.workspace}}/../mscp-${{env.VERSION}}.tar.gz

.github/workflows/test.yml

@@ -14,12 +14,21 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
-    - name: Build ubuntu 22.04 container
-      run: docker build --rm -t mscp-ubuntu:22.04 -f docker/ubuntu-22.04.Dockerfile .
-    - name: Run test on ubuntu 22.04
-      run: docker run --init --rm mscp-ubuntu:22.04 /mscp/scripts/test-in-container.sh
+      with:
+        submodules: true
 
-    - name: Build centos 8 container
-      run: docker build --rm -t mscp-centos:8 -f docker/centos-8.Dockerfile .
-    - name: Run test on centos 8
-      run: docker run --init --rm mscp-centos:8 /mscp/scripts/test-in-container.sh
+    - name: patch to libssh
+      run: patch -d libssh -p1 < patch/libssh-0.10.4.patch
+
+    # TODO: just building docker does not require libssh. fix CMakeLists
+    - name: install build dependency
+      run: sudo ./scripts/install-build-deps.sh
+
+    - name: configure CMake
+      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
+
+    - name: Build Containers
+      run: make -C ${{github.workspace}}/build docker-build-all
+
+    - name: Run Test
+      run: make -C ${{github.workspace}}/build docker-test-all

.gitignore

@@ -1,2 +1,3 @@
 build
+compile_commands.json
 .*.swp

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "libssh"]
+	path = libssh
+	url = https://git.libssh.org/projects/libssh.git
+	ignore = dirty

CMakeLists.txt

@@ -7,22 +7,48 @@ project(mscp
 	LANGUAGES C)
 
 set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -DDEBUG")
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/modules)
+
+
+# add libssh static library
+add_subdirectory(libssh EXCLUDE_FROM_ALL)
 
 if(APPLE)
 	list(APPEND CMAKE_PREFIX_PATH /usr/local) # intel mac homebrew prefix
 	list(APPEND CMAKE_PREFIX_PATH /opt/homebrew) # arm mac homebrew prefix
 endif() # APPLE
 
-find_package(libssh REQUIRED)
 
+# mscp executable
 add_executable(mscp src/main.c src/platform.c src/ssh.c src/file.c src/pprint.c)
-target_include_directories(mscp PUBLIC ./src)
-target_link_libraries(mscp ssh pthread m)
+
+set(MSCP_LINK_LIBS m pthread)
+set(MSCP_LINK_DIRS "")
+set(MSCP_COMPILE_OPTS "")
+set(MSCP_INCLUDE_DIRS ${mscp_SOURCE_DIR}/src)
+
+list(APPEND MSCP_COMPILE_OPTS -iquote ${CMAKE_CURRENT_BINARY_DIR}/libssh/include)
+list(APPEND MSCP_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/libssh/include)
+list(APPEND MSCP_LINK_LIBS ssh-static)
+
+find_package(GSSAPI)
+list(APPEND MSCP_LINK_LIBS ${GSSAPI_LIBRARIES})
+
+find_package(OpenSSL)
+list(APPEND MSCP_LINK_LIBS ${OPENSSL_LIBRARIES})
+
+find_package(ZLIB)
+list(APPEND MSCP_LINK_LIBS ${ZLIB_LIBRARIES})
+
+target_compile_definitions(mscp PUBLIC ASYNC_WRITE=1)
+
+target_include_directories(mscp PRIVATE ${MSCP_INCLUDE_DIRS})
+target_link_directories(mscp PRIVATE ${MSCP_LINK_DIRS})
+target_link_libraries(mscp PRIVATE ${MSCP_LINK_LIBS})
+target_compile_options(mscp PRIVATE ${MSCP_COMPILE_OPTS})
 target_compile_definitions(mscp PUBLIC _VERSION="${PROJECT_VERSION}")
 
-install(TARGETS mscp
-	RUNTIME		DESTINATION bin
-)
+install(TARGETS mscp RUNTIME DESTINATION bin)
 
 
 
@@ -35,82 +61,90 @@ add_test(NAME	pytest
 enable_testing()
 
 
-# Build Packages
-if(BUILD_PKG)
 
-	# CPACK Rules
-	set(CPACK_SET_DESTDIR		true)
-	set(CPACK_PROJECT_NAME		${PROJECT_NAME})
-	set(CPACK_PROJECT_VERSION	${PROJECT_VERSION})
-	set(CPACK_PACKAGE_CONTACT	"Ryo Nakamura <upa@haeena.net>")
-	set(CPACK_PACKAGE_DESCRIPTION
-		"mscp, copy files over multiple ssh connections")
+# CPACK Rules
+set(CPACK_SET_DESTDIR		true)
+set(CPACK_PROJECT_NAME		${PROJECT_NAME})
+set(CPACK_PROJECT_VERSION	${PROJECT_VERSION})
+set(CPACK_PACKAGE_CONTACT	"Ryo Nakamura <upa@haeena.net>")
+set(CPACK_PACKAGE_DESCRIPTION
+	"mscp, copy files over multiple ssh connections")
 
-	# on linux
-	if(UNIX AND NOT APPLE)
-		execute_process(COMMAND
-			bash "-c" "cat /etc/os-release|grep '^ID='|cut -d '=' -f 2|tr -d '\"'"
-			OUTPUT_VARIABLE DIST_NAME OUTPUT_STRIP_TRAILING_WHITESPACE)
-		execute_process(COMMAND
-			bash "-c" "cat /etc/os-release|grep '^VERSION_ID='|cut -d '=' -f 2|tr -d '\"'"
-			OUTPUT_VARIABLE DIST_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE)
-		execute_process(COMMAND uname -p
-			OUTPUT_VARIABLE ARCH OUTPUT_STRIP_TRAILING_WHITESPACE)
-		set(PACKAGE_FILE_NAME
-			${PROJECT_NAME}_${PROJECT_VERSION}-${DIST_NAME}-${DIST_VERSION}-${ARCH})
+execute_process(COMMAND uname -m
+	OUTPUT_VARIABLE ARCH OUTPUT_STRIP_TRAILING_WHITESPACE)
 
-		set(CPACK_DEBIAN_FILE_NAME		${PACKAGE_FILE_NAME}.deb)
-		set(CPACK_DEBIAN_PACKAGE_DEPENDS	"libssh-4")
-		set(CPACK_DEBIAN_PACKAGE_HOMEPAGE	"https://github.com/upa/mscp")
+if(UNIX AND NOT APPLE) # on linux
+	execute_process(COMMAND
+		bash "-c" "cat /etc/os-release|grep '^ID='|cut -d '=' -f 2|tr -d '\"'"
+		OUTPUT_VARIABLE DIST_NAME OUTPUT_STRIP_TRAILING_WHITESPACE)
+	execute_process(COMMAND
+		bash "-c" "cat /etc/os-release|grep '^VERSION_ID='|cut -d '=' -f 2|tr -d '\"'"
+		OUTPUT_VARIABLE DIST_VER OUTPUT_STRIP_TRAILING_WHITESPACE)
+	execute_process(COMMAND
+		bash "-c" "${mscp_SOURCE_DIR}/scripts/print-install-deps.sh ${DIST_NAME}-${DIST_VER}"
+		OUTPUT_VARIABLE DIST_DEP OUTPUT_STRIP_TRAILING_WHITESPACE)
 
-		set(CPACK_RPM_FILE_NAME			${PACKAGE_FILE_NAME}.rpm)
-		set(CPACK_RPM_PACKAGE_REQUIRES		"libssh")
-		set(CPACK_RPM_PACKAGE_HOMEPAGE		"https://github.com/upa/mscp")
-		set(CPACK_RPM_PACKAGE_DESCRIPTION	${CPACK_PACKAGE_DESCRIPTION})
+	set(PACKAGE_FILE_NAME
+		${PROJECT_NAME}_${PROJECT_VERSION}-${DIST_NAME}-${DIST_VER}-${ARCH})
 
-	endif() # on linux
+	set(CPACK_DEBIAN_FILE_NAME		${PACKAGE_FILE_NAME}.deb)
+	set(CPACK_DEBIAN_PACKAGE_DEPENDS	${DIST_DEP})
+	set(CPACK_DEBIAN_PACKAGE_HOMEPAGE	"https://github.com/upa/mscp")
 
-	include(CPack)
+	set(CPACK_RPM_FILE_NAME			${PACKAGE_FILE_NAME}.rpm)
+	set(CPACK_RPM_PACKAGE_REQUIRES		${DIST_DEP})
+	set(CPACK_RPM_PACKAGE_HOMEPAGE		"https://github.com/upa/mscp")
+	set(CPACK_RPM_PACKAGE_DESCRIPTION	${CPACK_PACKAGE_DESCRIPTION})
+endif() # on linux
+
+include(CPack)
 
 
-	# Custom commands to build mscp in docker containers
 
-	# Ubuntu 20.04
-	add_custom_target(package-ubuntu-20.04-in-docker
-		COMMENT "Build mscp in ubuntu 20.04 docker container"
-		WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-		COMMAND docker build -t mscp-ubuntu:20.04 -f docker/ubuntu-20.04.Dockerfile .
-		COMMAND docker run --init --rm mscp-ubuntu:20.04
-			/mscp/scripts/test-in-container.sh
-		COMMAND docker run --rm -v ${CMAKE_BINARY_DIR}:/out mscp-ubuntu:20.04
-			cp /mscp/build/mscp_${PROJECT_VERSION}-ubuntu-20.04-${ARCH}.deb /out/)
+# Custom targets to build and test mscp in docker containers.
+# foreach(IN ZIP_LISTS) (cmake >= 3.17) can shorten the following lists.
+# However, ubuntu 20.04 has cmake 3.16.3. So this is a roundabout trick.
+list(APPEND DIST_NAMES	ubuntu	ubuntu	centos	rocky)
+list(APPEND DIST_VERS	20.04	22.04	8	8.6)
+list(APPEND DIST_PKGS	deb	deb	rpm	rpm)
 
-	# Ubuntu 22.04
-	add_custom_target(package-ubuntu-22.04-in-docker
-		COMMENT "Build mscp in ubuntu 22.04 docker container"
-		WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-		COMMAND docker build -t mscp-ubuntu:22.04 -f docker/ubuntu-22.04.Dockerfile .
-		COMMAND docker run --init --rm mscp-ubuntu:22.04
-			/mscp/scripts/test-in-container.sh
-		COMMAND docker run --rm -v ${CMAKE_BINARY_DIR}:/out mscp-ubuntu:22.04
-		cp /mscp/build/mscp_${PROJECT_VERSION}-ubuntu-22.04-${ARCH}.deb /out/)
+list(LENGTH DIST_NAMES _DIST_LISTLEN)
+math(EXPR DIST_LISTLEN "${_DIST_LISTLEN} - 1")
 
-	# CentOS 8
-	add_custom_target(package-centos-8-in-docker
-		COMMENT "Build mscp in centos 8 docker container"
-		WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-		COMMAND docker build -t mscp-centos:8 -f docker/centos-8.Dockerfile .
-		COMMAND docker run --init --rm mscp-centos:8
-			/mscp/scripts/test-in-container.sh
-		COMMAND docker run --rm -v ${CMAKE_BINARY_DIR}:/out mscp-centos:8
-		cp /mscp/build/mscp_${PROJECT_VERSION}-centos-8-${ARCH}.rpm /out/)
+foreach(x RANGE ${DIST_LISTLEN})
+	list(GET DIST_NAMES	${x} DIST_NAME)
+	list(GET DIST_VERS	${x} DIST_VER)
+	list(GET DIST_PKGS	${x} DIST_PKG)
 
-	# build on all conatiners
-	add_custom_target(package-all-in-docker
-		COMMENT "Build mscp in all docker containers"
-		DEPENDS package-ubuntu-20.04-in-docker
-		DEPENDS package-ubuntu-22.04-in-docker
-		DEPENDS package-centos-8-in-docker)
+	set(DOCKER_IMAGE mscp-${DIST_NAME}:${DIST_VER})
+	set(DOCKER_INDEX ${DIST_NAME}-${DIST_VER})
+	set(PKG_FILE_NAME
+		mscp_${PROJECT_VERSION}-${DIST_NAME}-${DIST_VER}-${ARCH}.${DIST_PKG})
 
-endif() # BUILD_PKG
+	add_custom_target(docker-build-${DOCKER_INDEX}
+		COMMENT "Build mscp in ${DOCKER_IMAGE} container"
+		WORKING_DIRECTORY ${mscp_SOURCE_DIR}
+		COMMAND
+		docker build -t ${DOCKER_IMAGE} -f docker/${DOCKER_INDEX}.Dockerfile .)
 
+	add_custom_target(docker-test-${DOCKER_INDEX}
+		COMMENT "Test mscp in ${DOCKER_IMAGE} container"
+		WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+		COMMAND
+		docker run --init --rm ${DOCKER_IMAGE} /mscp/scripts/test-in-container.sh)
+
+	add_custom_target(docker-pkg-${DOCKER_INDEX}
+		COMMENT "Retrieve mscp package from ${DOCKER_IMAGE} container"
+		WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+		COMMAND
+		docker run --rm -v ${CMAKE_BINARY_DIR}:/out ${DOCKER_IMAGE}
+		cp /mscp/build/${PKG_FILE_NAME} /out/)
+
+	list(APPEND DOCKER_BUILDS	docker-build-${DOCKER_INDEX})
+	list(APPEND DOCKER_TESTS	docker-test-${DOCKER_INDEX})
+	list(APPEND DOCKER_PKGS		docker-pkg-${DOCKER_INDEX})
+endforeach()
+
+add_custom_target(docker-build-all	DEPENDS ${DOCKER_BUILDS})
+add_custom_target(docker-test-all	DEPENDS ${DOCKER_TESTS})
+add_custom_target(docker-pkg-all	DEPENDS ${DOCKER_PKGS})

README.md

@@ -3,53 +3,70 @@
 [![build on ubuntu](https://github.com/upa/mscp/actions/workflows/build-ubuntu.yml/badge.svg)](https://github.com/upa/mscp/actions/workflows/build-ubuntu.yml) [![build on macOS](https://github.com/upa/mscp/actions/workflows/build-macos.yml/badge.svg)](https://github.com/upa/mscp/actions/workflows/build-macos.yml) [![test](https://github.com/upa/mscp/actions/workflows/test.yml/badge.svg)](https://github.com/upa/mscp/actions/workflows/test.yml)
 
 
-`mscp`, a variant of `scp`, copies files over multiple ssh (sftp)
-sessions. Multiple threads in mscp transfer (1) multiple files
-simultaneously and (2) a large file in parallel. It may shorten the
-waiting time for transferring a lot of/large files over networks.
+`mscp`, a variant of `scp`, copies files over multiple ssh (SFTP)
+connections. Multiple threads and connections in mscp transfer (1)
+multiple files simultaneously and (2) a large file in parallel. It
+would shorten the waiting time for transferring a lot of/large files
+over networks.
 
-You can use `mscp` like `scp`, e.g., `mscp example.com:srcfile
-/tmp/dstfile`. Remote hosts only need to run `sshd` supporting the
-SFTP subsystem, and you need to be able to ssh to the hosts (as
-usual).
+You can use `mscp` like `scp`, for example, `mscp
+user@example.com:srcfile /tmp/dstfile`. Remote hosts only need to run
+standard `sshd` supporting the SFTP subsystem, and you need to be able
+to ssh to the hosts (as usual). `mscp` does not require anything else.
 
-Differences from `scp` are:
+
+Differences from `scp`:
 
 - remote glob on remote shell expansion is not supported.
 - remote to remote copy is not supported.
 - `-r` option is not needed.
-- and any other differences I have not noticed and implemented...
+- and any other differences I have not implemented and noticed...
+
+
+## Install
+
+- homebrew
+
+```console
+brew install upa/tap/mscp
+```
+
+- Linux
+
+Download a package for your environment from [Releases
+page](https://github.com/upa/mscp/releases).
+
 
 ## Build
 
-mscp depends on [libssh](https://www.libssh.org/).
+mscp depends on a patched [libssh](https://www.libssh.org/).  The
+patch introduces asynchronous SFTP Write, which is derived from
+https://github.com/limes-datentechnik-gmbh/libssh (see [Re: SFTP Write
+async](https://archive.libssh.org/libssh/2020-06/0000004.html)).
 
-- macOS
-
-```console
-brew install libssh
-```
-
-- ubuntu
-
-```console
-sudo apt-get install libssh-dev
-```
-
-- rhel
-
-```console
-sudo yum install libssh-devel
-```
-
-Clone and build this repositoy.
+Currently macOS and Linux (Ubuntu, CentOS, Rocky) are supported.
 
 ```console
+# 1. clone this repository
 git clone https://github.com/upa/mscp.git
 cd mscp
 
-mkdir build && cd build
-cmake .. && make
+# 2. prepare patched libssh
+git submodule update --init
+patch -d libssh -p1 < patch/libssh-0.10.4.patch
+
+# 3. install build dependency
+bash ./scripts/install-build-deps.sh
+
+# 4. configure mscp
+mkdir build && mv build
+cmake ..
+
+## in macOS, you may need OPENSSL_ROOT_DIR for cmake:
+cmake .. -DOPENSSL_ROOT_DIR=$(brew --prefix)/opt/openssl@1.1
+
+# build
+make
 
 # install the mscp binary to CMAKE_INSTALL_PREFIX/bin (usually /usr/local/bin)
 make install
@@ -59,50 +76,29 @@ make install
 
 - Usage
 
-```shell-session
+```console
 $ mscp
-mscp v0.0.0: copy files over multiple ssh connections
+mscp v0.0.2: copy files over multiple ssh connections
 
-Usage: mscp [vqDCHdh] [-n nr_conns]
-            [-s min_chunk_sz] [-S max_chunk_sz]
-            [-b sftp_buf_sz] [-B io_buf_sz]
+Usage: mscp [vqDCHdh] [-n nr_conns] [-m coremask]
+            [-s min_chunk_sz] [-S max_chunk_sz] [-a nr_ahead]
             [-l login_name] [-p port] [-i identity_file]
             [-c cipher_spec] source ... target
-
-    -n NR_CONNECTIONS  number of connections (default: half of # of cpu cores)
-    -s MIN_CHUNK_SIZE  min chunk size (default: 64MB)
-    -S MAX_CHUNK_SIZE  max chunk size (default: filesize / nr_conn)
-    -b SFTP_BUF_SIZE   buf size for sftp_read/write (default 131072B)
-    -B IO_BUF_SIZE     buf size for read/write (default 131072B)
-                       Note that this value is derived from
-                       qemu/block/ssh.c. need investigation...
-    -v                 increment verbose output level
-    -q                 disable output
-    -D                 dry run
-
-    -l LOGIN_NAME      login name
-    -p PORT            port number
-    -i IDENTITY        identity file for publickey authentication
-    -c CIPHER          cipher spec, see `ssh -Q cipher`
-    -C                 enable compression on libssh
-    -H                 disable hostkey check
-    -d                 increment ssh debug output level
-    -h                 print this help
 ```
 
 - Example: copy an 8GB file on tmpfs over a 100Gbps link
   - Two Intel Xeon Gold 6130 machines directly connected with Intel E810 100Gbps NICs.
 
-```shell-session
+```console
 $ mscp /tmp/test.img 10.0.0.1:/tmp/
-[===============================================================] 100% 8GB/8GB 3.02GB/s 
+[=====================================================] 100% 8GB/8GB 3.02GB/s 
 ```
 
-- `-v` options increment verbose output level.
+- `-v` option increments verbose output level.
 
-```shell-session
+```console
 $ mscp test 10.0.0.1:
-[===============================================================] 100% 13B/13B 2.41KB/s 
+[=====================================================] 100% 13B/13B 2.41KB/s 
 
 $ mscp -v test 10.0.0.1:
 file test/test.txt (local) -> ./test/test.txt (remote) 9B
@@ -114,7 +110,7 @@ copy start: test/test2/2.txt
 copy done: test/1.txt
 copy done: test/test2/2.txt
 copy done: test/test.txt
-[===============================================================] 100% 13B/13B 2.51KB/s 
+[=====================================================] 100% 13B/13B 2.51KB/s 
 
 $ mscp -vv -n 4 test 10.0.0.1:
 connecting to 10.0.0.1 for checking destinations...
@@ -131,8 +127,43 @@ copy start: test/test2/2.txt
 copy done: test/test.txt
 copy done: test/test2/2.txt
 copy done: test/1.txt
-[===============================================================] 100% 13B/13B 3.27KB/s
+[=====================================================] 100% 13B/13B 3.27KB/s
 ```
 
+- Full usage
+
+```console
+$ mscp -h
+mscp v0.0.2: copy files over multiple ssh connections
+
+Usage: mscp [vqDCHdh] [-n nr_conns] [-m coremask]
+            [-s min_chunk_sz] [-S max_chunk_sz] [-a nr_ahead]
+            [-l login_name] [-p port] [-i identity_file]
+            [-c cipher_spec] source ... target
+
+    -n NR_CONNECTIONS  number of connections (default: half of # of cpu cores)
+    -m COREMASK        hex value to specify cores where threads pinned
+    -s MIN_CHUNK_SIZE  min chunk size (default: 64MB)
+    -S MAX_CHUNK_SIZE  max chunk size (default: filesize / nr_conn)
+
+    -a NR_AHEAD        number of inflight SFTP commands (default: 16)
+
+    -v                 increment verbose output level
+    -q                 disable output
+    -D                 dry run
+
+    -l LOGIN_NAME      login name
+    -p PORT            port number
+    -i IDENTITY        identity file for public key authentication
+    -c CIPHER          cipher spec, see `ssh -Q cipher`
+    -C                 enable compression on libssh
+    -H                 disable hostkey check
+    -d                 increment ssh debug output level
+    -h                 print this help
+```
+
+
+
+
 Note: mscp is still under development, and the author is not
-responsible for any accidents on mscp.
+responsible for any accidents due to mscp.

VERSION

@@ -1 +1 @@
-0.0.0
+0.0.2

cmake/modules/FindGSSAPI.cmake

@@ -0,0 +1,325 @@
+# - Try to find GSSAPI
+# Once done this will define
+#
+#  KRB5_CONFIG - Path to krb5-config
+#  GSSAPI_ROOT_DIR - Set this variable to the root installation of GSSAPI
+#
+# Read-Only variables:
+#  GSSAPI_FLAVOR_MIT - set to TURE if MIT Kerberos has been found
+#  GSSAPI_FLAVOR_HEIMDAL - set to TRUE if Heimdal Keberos has been found
+#  GSSAPI_FOUND - system has GSSAPI
+#  GSSAPI_INCLUDE_DIR - the GSSAPI include directory
+#  GSSAPI_LIBRARIES - Link these to use GSSAPI
+#  GSSAPI_DEFINITIONS - Compiler switches required for using GSSAPI
+#
+#=============================================================================
+#  Copyright (c) 2013 Andreas Schneider <asn@cryptomilk.org>
+#
+#  Distributed under the OSI-approved BSD License (the "License");
+#  see accompanying file Copyright.txt for details.
+#
+#  This software is distributed WITHOUT ANY WARRANTY; without even the
+#  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#  See the License for more information.
+#=============================================================================
+#
+
+find_path(GSSAPI_ROOT_DIR
+    NAMES
+        include/gssapi.h
+        include/gssapi/gssapi.h
+    HINTS
+        ${_GSSAPI_ROOT_HINTS}
+    PATHS
+        ${_GSSAPI_ROOT_PATHS}
+)
+mark_as_advanced(GSSAPI_ROOT_DIR)
+
+if (UNIX)
+    find_program(KRB5_CONFIG
+        NAMES
+            krb5-config
+        PATHS
+            ${GSSAPI_ROOT_DIR}/bin
+            /opt/local/bin)
+    mark_as_advanced(KRB5_CONFIG)
+
+    if (KRB5_CONFIG)
+        # Check if we have MIT KRB5
+        execute_process(
+            COMMAND
+                ${KRB5_CONFIG} --vendor
+            RESULT_VARIABLE
+                _GSSAPI_VENDOR_RESULT
+            OUTPUT_VARIABLE
+                _GSSAPI_VENDOR_STRING)
+
+        if ((_GSSAPI_VENDOR_STRING MATCHES ".*Massachusetts.*") OR (_GSSAPI_VENDOR_STRING
+                        MATCHES ".*MITKerberosShim.*"))
+            set(GSSAPI_FLAVOR_MIT TRUE)
+        else()
+            execute_process(
+                COMMAND
+                    ${KRB5_CONFIG} --libs gssapi
+                RESULT_VARIABLE
+                    _GSSAPI_LIBS_RESULT
+                OUTPUT_VARIABLE
+                    _GSSAPI_LIBS_STRING)
+
+            if (_GSSAPI_LIBS_STRING MATCHES ".*roken.*")
+                set(GSSAPI_FLAVOR_HEIMDAL TRUE)
+            endif()
+        endif()
+
+        # Get the include dir
+        execute_process(
+            COMMAND
+                ${KRB5_CONFIG} --cflags gssapi
+            RESULT_VARIABLE
+                _GSSAPI_INCLUDE_RESULT
+            OUTPUT_VARIABLE
+                _GSSAPI_INCLUDE_STRING)
+        string(REGEX REPLACE "(\r?\n)+$" "" _GSSAPI_INCLUDE_STRING "${_GSSAPI_INCLUDE_STRING}")
+        string(REGEX REPLACE " *-I" "" _GSSAPI_INCLUDEDIR "${_GSSAPI_INCLUDE_STRING}")
+    endif()
+
+    if (NOT GSSAPI_FLAVOR_MIT AND NOT GSSAPI_FLAVOR_HEIMDAL)
+        # Check for HEIMDAL
+        find_package(PkgConfig)
+        if (PKG_CONFIG_FOUND)
+            pkg_check_modules(_GSSAPI heimdal-gssapi)
+        endif (PKG_CONFIG_FOUND)
+
+        if (_GSSAPI_FOUND)
+            set(GSSAPI_FLAVOR_HEIMDAL TRUE)
+        else()
+            find_path(_GSSAPI_ROKEN
+                NAMES
+                    roken.h
+                PATHS
+                    ${GSSAPI_ROOT_DIR}/include
+                    ${_GSSAPI_INCLUDEDIR})
+            if (_GSSAPI_ROKEN)
+                set(GSSAPI_FLAVOR_HEIMDAL TRUE)
+            endif()
+        endif ()
+    endif()
+endif (UNIX)
+
+find_path(GSSAPI_INCLUDE_DIR
+    NAMES
+        gssapi.h
+        gssapi/gssapi.h
+    PATHS
+        ${GSSAPI_ROOT_DIR}/include
+        ${_GSSAPI_INCLUDEDIR}
+)
+
+if (GSSAPI_FLAVOR_MIT)
+    find_library(GSSAPI_LIBRARY
+        NAMES
+            gssapi_krb5
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(KRB5_LIBRARY
+        NAMES
+            krb5
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(K5CRYPTO_LIBRARY
+        NAMES
+            k5crypto
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(COM_ERR_LIBRARY
+        NAMES
+            com_err
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    if (GSSAPI_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${GSSAPI_LIBRARY}
+        )
+    endif (GSSAPI_LIBRARY)
+
+    if (KRB5_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${KRB5_LIBRARY}
+        )
+    endif (KRB5_LIBRARY)
+
+    if (K5CRYPTO_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${K5CRYPTO_LIBRARY}
+        )
+    endif (K5CRYPTO_LIBRARY)
+
+    if (COM_ERR_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${COM_ERR_LIBRARY}
+        )
+    endif (COM_ERR_LIBRARY)
+endif (GSSAPI_FLAVOR_MIT)
+
+if (GSSAPI_FLAVOR_HEIMDAL)
+    find_library(GSSAPI_LIBRARY
+        NAMES
+            gssapi
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(KRB5_LIBRARY
+        NAMES
+            krb5
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(HCRYPTO_LIBRARY
+        NAMES
+            hcrypto
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(COM_ERR_LIBRARY
+        NAMES
+            com_err
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(HEIMNTLM_LIBRARY
+        NAMES
+            heimntlm
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(HX509_LIBRARY
+        NAMES
+            hx509
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(ASN1_LIBRARY
+        NAMES
+            asn1
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(WIND_LIBRARY
+        NAMES
+            wind
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(ROKEN_LIBRARY
+        NAMES
+            roken
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    if (GSSAPI_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${GSSAPI_LIBRARY}
+        )
+    endif (GSSAPI_LIBRARY)
+
+    if (KRB5_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${KRB5_LIBRARY}
+        )
+    endif (KRB5_LIBRARY)
+
+    if (HCRYPTO_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${HCRYPTO_LIBRARY}
+        )
+    endif (HCRYPTO_LIBRARY)
+
+    if (COM_ERR_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${COM_ERR_LIBRARY}
+        )
+    endif (COM_ERR_LIBRARY)
+
+    if (HEIMNTLM_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${HEIMNTLM_LIBRARY}
+        )
+    endif (HEIMNTLM_LIBRARY)
+
+    if (HX509_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${HX509_LIBRARY}
+        )
+    endif (HX509_LIBRARY)
+
+    if (ASN1_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${ASN1_LIBRARY}
+        )
+    endif (ASN1_LIBRARY)
+
+    if (WIND_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${WIND_LIBRARY}
+        )
+    endif (WIND_LIBRARY)
+
+    if (ROKEN_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${WIND_LIBRARY}
+        )
+    endif (ROKEN_LIBRARY)
+endif (GSSAPI_FLAVOR_HEIMDAL)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(GSSAPI DEFAULT_MSG GSSAPI_LIBRARIES GSSAPI_INCLUDE_DIR)
+
+if (GSSAPI_INCLUDE_DIRS AND GSSAPI_LIBRARIES)
+    set(GSSAPI_FOUND TRUE)
+endif (GSSAPI_INCLUDE_DIRS AND GSSAPI_LIBRARIES)
+
+# show the GSSAPI_INCLUDE_DIRS and GSSAPI_LIBRARIES variables only in the advanced view
+mark_as_advanced(GSSAPI_INCLUDE_DIRS GSSAPI_LIBRARIES)

docker/README.md

@@ -6,19 +6,23 @@ cd ..
 
 docker build -t mscp-ubuntu:20.04 -f docker/ubuntu-20.04.Dockerfile .
 
-docker build -t mscp-ubuntu:22.04 -f docker/Dockerfile-ubuntu-22.04 .
+docker build -t mscp-ubuntu:22.04 -f docker/ubuntu-22.04.Dockerfile .
 
-docker build -t mscp-centos:8 -f docker/Dockerfile-centos-8 .
+docker build -t mscp-centos:8 -f docker/centos-8.Dockerfile .
+
+docker build -t mscp-rocky:8.6 -f docker/rocky-8.6.Dockerfile .
 ```
 
 Test `mscp` in the containers.
 
 ```console
-docker run --init --rm mscp-ubuntu:20.04 /build/mscp/scripts/test-in-container.sh
+docker run --init --rm mscp-ubuntu:20.04 /mscp/scripts/test-in-container.sh
 
-docker run --init --rm mscp-ubuntu:22.04 /build/mscp/scripts/test-in-container.sh
+docker run --init --rm mscp-ubuntu:22.04 /mscp/scripts/test-in-container.sh
 
-docker run --init --rm mscp-centos:8 /build/mscp/scripts/test-in-container.sh
+docker run --init --rm mscp-centos:8 /mscp/scripts/test-in-container.sh
+
+docker run --init --rm mscp-rocky:8.6 /mscp/scripts/test-in-container.sh
 ```
 
 Retrieve deb/rpm packages.
@@ -32,6 +36,9 @@ docker run --rm -v (pwd):/out mscp-ubuntu:22.04 \
 
 docker run --rm -v (pwd):/out mscp-centos:8 \
        cp /mscp/build/mscp_0.0.0-centos-8-x86_64.rpm /out/
+
+docker run --rm -v (pwd):/out mscp-rocky:8.6 \
+       cp /mscp/build/mscp_0.0.0-rocky-8.6-x86_64.rpm /out/
 ```
 
 I don't know whether these are good way.

docker/centos-8.Dockerfile

@@ -28,8 +28,9 @@ RUN ${mscpdir}/scripts/install-build-deps.sh
 # build
 RUN cd ${mscpdir}			\
         && rm -rf build			\
-        && cmake -B build -DBUILD_PKG=1 \
+        && cmake -B build 		\
         && cd ${mscpdir}/build          \
 	&& make				\
-	&& cpack -G RPM CPackConfig.cmake
+	&& cpack -G RPM CPackConfig.cmake \
+	&& rpm -iv *.rpm
 

docker/rocky-8.6.Dockerfile

@@ -0,0 +1,31 @@
+FROM rockylinux:8.6
+
+ARG mscpdir="/mscp"
+
+COPY . ${mscpdir}
+
+# install numpy and pytest, sshd for test, and rpm-build
+RUN set -ex && yum -y install \
+	python3 python3-pip openssh openssh-server openssh-clients rpm-build
+
+RUN python3 -m pip install numpy pytest
+
+
+# preparation for sshd
+RUN  mkdir /var/run/sshd        \
+	&& ssh-keygen -A	\
+        && ssh-keygen -f /root/.ssh/id_rsa -N ""                \
+        && mv /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
+
+# install build dependency
+RUN ${mscpdir}/scripts/install-build-deps.sh
+
+# build
+RUN cd ${mscpdir}			\
+        && rm -rf build			\
+        && cmake -B build		\
+        && cd ${mscpdir}/build          \
+	&& make				\
+	&& cpack -G RPM CPackConfig.cmake \
+	&& rpm -iv *.rpm
+

docker/ubuntu-20.04.Dockerfile

@@ -28,7 +28,8 @@ RUN ${mscpdir}/scripts/install-build-deps.sh
 # build
 RUN cd ${mscpdir}			\
 	&& rm -rf build			\
-	&& cmake -B build -DBUILD_PKG=1	\
+	&& cmake -B build		\
 	&& cd ${mscpdir}/build		\
 	&& make				\
-	&& cpack -G DEB CPackConfig.cmake 
+	&& cpack -G DEB CPackConfig.cmake \
+	&& dpkg -i *.deb

docker/ubuntu-22.04.Dockerfile

@@ -28,7 +28,8 @@ RUN ${mscpdir}/scripts/install-build-deps.sh
 # build
 RUN cd ${mscpdir}			\
 	&& rm -rf build			\
-	&& cmake -B build -DBUILD_PKG=1	\
+	&& cmake -B build		\
 	&& cd ${mscpdir}/build		\
 	&& make				\
-	&& cpack -G DEB CPackConfig.cmake 
+	&& cpack -G DEB CPackConfig.cmake \
+	&& dpkg -i *.deb

patch/README.md

@@ -0,0 +1,13 @@
+
+Patch(es) in this directory introduces `sftp_async_write()` and
+`sftp_async_write_end()` to libssh. Those implementations are derived
+from https://github.com/limes-datentechnik-gmbh/libssh. See [Re: SFTP
+Write async](https://archive.libssh.org/libssh/2020-06/0000004.html).
+
+```console
+git clone https://git.libssh.org/projects/libssh.git/ --depth=1 -b libssh-0.10.4
+cd libssh
+git apply ../pathc/libssh-0.10.4.patch
+
+# then build libssh
+```

patch/libssh-0.10.4.patch

@@ -0,0 +1,166 @@
+diff --git a/DefineOptions.cmake b/DefineOptions.cmake
+index 068db988..6db0fc0f 100644
+--- a/DefineOptions.cmake
++++ b/DefineOptions.cmake
+@@ -17,7 +17,7 @@ option(UNIT_TESTING "Build with unit tests" OFF)
+ option(CLIENT_TESTING "Build with client tests; requires openssh" OFF)
+ option(SERVER_TESTING "Build with server tests; requires openssh and dropbear" OFF)
+ option(WITH_BENCHMARKS "Build benchmarks tools" OFF)
+-option(WITH_EXAMPLES "Build examples" ON)
++option(WITH_EXAMPLES "Build examples" OFF)
+ option(WITH_NACL "Build with libnacl (curve25519)" ON)
+ option(WITH_SYMBOL_VERSIONING "Build with symbol versioning" ON)
+ option(WITH_ABI_BREAK "Allow ABI break" OFF)
+@@ -25,6 +25,7 @@ option(WITH_GEX "Enable DH Group exchange mechanisms" ON)
+ option(WITH_INSECURE_NONE "Enable insecure none cipher and MAC algorithms (not suitable for production!)" OFF)
+ option(FUZZ_TESTING "Build with fuzzer for the server and client (automatically enables none cipher!)" OFF)
+ option(PICKY_DEVELOPER "Build with picky developer flags" OFF)
++option(WITH_STATIC_LIB "Build static library" ON)
+ 
+ if (WITH_ZLIB)
+     set(WITH_LIBZ ON)
+@@ -60,3 +61,7 @@ endif (NOT GLOBAL_CLIENT_CONFIG)
+ if (FUZZ_TESTING)
+   set(WITH_INSECURE_NONE ON)
+ endif (FUZZ_TESTING)
++
++if (WITH_STATIC_LIB)
++  set(BUILD_STATIC_LIB ON)
++endif()
+diff --git a/include/libssh/sftp.h b/include/libssh/sftp.h
+index c855df8a..1fd1710a 100644
+--- a/include/libssh/sftp.h
++++ b/include/libssh/sftp.h
+@@ -565,6 +565,9 @@ LIBSSH_API int sftp_async_read(sftp_file file, void *data, uint32_t len, uint32_
+  */
+ LIBSSH_API ssize_t sftp_write(sftp_file file, const void *buf, size_t count);
+ 
++LIBSSH_API int sftp_async_write(sftp_file file, const void *buf, size_t count, uint32_t* id);
++LIBSSH_API int sftp_async_write_end(sftp_file file, uint32_t id, int blocking);
++
+ /**
+  * @brief Seek to a specific location in a file.
+  *
+diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
+index c090fef7..e2f86309 100644
+--- a/src/CMakeLists.txt
++++ b/src/CMakeLists.txt
+@@ -435,6 +435,11 @@ if (BUILD_STATIC_LIB)
+   if (WIN32)
+     target_compile_definitions(ssh-static PUBLIC "LIBSSH_STATIC")
+   endif (WIN32)
++
++  install(TARGETS ssh-static
++	  EXPORT libssh-config
++	  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
++	  COMPONENT libraries)
+ endif (BUILD_STATIC_LIB)
+ 
+ message(STATUS "Threads_FOUND=${Threads_FOUND}")
+diff --git a/src/sftp.c b/src/sftp.c
+index e01012a8..7b5dc249 100644
+--- a/src/sftp.c
++++ b/src/sftp.c
+@@ -2228,6 +2228,102 @@ ssize_t sftp_write(sftp_file file, const void *buf, size_t count) {
+   return -1; /* not reached */
+ }
+ 
++/*
++ * sftp_async_write and sftp_async_write_end are copied from
++ * https://github.com/limes-datentechnik-gmbh/libssh
++ */
++int sftp_async_write(sftp_file file, const void *buf, size_t count, uint32_t* id) {
++  sftp_session sftp = file->sftp;
++  ssh_buffer buffer;
++  int len;
++  int packetlen;
++  int rc;
++
++  buffer = ssh_buffer_new();
++  if (buffer == NULL) {
++    ssh_set_error_oom(sftp->session);
++    return -1;
++  }
++
++  *id = sftp_get_new_id(file->sftp);
++
++  rc = ssh_buffer_pack(buffer,
++                       "dSqdP",
++                       *id,
++                       file->handle,
++                       file->offset,
++                       count, /* len of datastring */
++                       (size_t)count, buf);
++  if (rc != SSH_OK){
++    ssh_set_error_oom(sftp->session);
++    ssh_buffer_free(buffer);
++    return SSH_ERROR;
++  }
++  packetlen=ssh_buffer_get_len(buffer)+5;
++  len = sftp_packet_write(file->sftp, SSH_FXP_WRITE, buffer);
++  ssh_buffer_free(buffer);
++  if (len < 0) {
++    return SSH_ERROR;
++  } else  if (len != packetlen) {
++    ssh_set_error(sftp->session, SSH_FATAL,
++      "Could only send %d of %d bytes to remote host!", len, packetlen);
++    SSH_LOG(SSH_LOG_PACKET,
++        "Could not write as much data as expected");
++    return SSH_ERROR;
++  }
++
++  file->offset += count;
++
++  return SSH_OK;
++}
++
++int sftp_async_write_end(sftp_file file, uint32_t id, int blocking) {
++  sftp_session sftp = file->sftp;
++  sftp_message msg = NULL;
++  sftp_status_message status;
++
++  msg = sftp_dequeue(sftp, id);
++  while (msg == NULL) {
++    if (!blocking && ssh_channel_poll(sftp->channel, 0) == 0) {
++      /* we cannot block */
++      return SSH_AGAIN;
++    }
++    if (sftp_read_and_dispatch(sftp) < 0) {
++      /* something nasty has happened */
++      return SSH_ERROR;
++    }
++    msg = sftp_dequeue(sftp, id);
++  }
++
++  switch (msg->packet_type) {
++    case SSH_FXP_STATUS:
++      status = parse_status_msg(msg);
++      sftp_message_free(msg);
++      if (status == NULL) {
++        return SSH_ERROR;
++      }
++      sftp_set_error(sftp, status->status);
++      switch (status->status) {
++        case SSH_FX_OK:
++          status_msg_free(status);
++          return SSH_OK;
++        default:
++          break;
++      }
++      ssh_set_error(sftp->session, SSH_REQUEST_DENIED,
++          "SFTP server: %s", status->errormsg);
++      status_msg_free(status);
++      return SSH_ERROR;
++    default:
++      ssh_set_error(sftp->session, SSH_FATAL,
++          "Received message %d during write!", msg->packet_type);
++      sftp_message_free(msg);
++      return SSH_ERROR;
++  }
++
++  return SSH_ERROR; /* not reached */
++}
++
+ /* Seek to a specific location in a file. */
+ int sftp_seek(sftp_file file, uint32_t new_offset) {
+   if (file == NULL) {

scripts/install-build-deps.sh

@@ -1,18 +1,30 @@
-#!/bin/bash -e
+#!/bin/bash -eu
 #
 # Install build dpenedencies.
 
-source /etc/os-release
+platform=$(uname -s)
+
+if [ -e /etc/os-release ]; then
+	source /etc/os-release
+	platform=${platform}-${ID}
+fi
+
 set -x
 
-case $ID in
-	ubuntu*)
-		apt-get install -y gcc make cmake libssh-dev
+case $platform in
+	Darwin)
+		brew install openssl@1.1
 		;;
-	centos* | rhel*)
-		dnf install -y gcc make cmake libssh-devel rpm-build
+	Linux-ubuntu*)
+		apt-get install -y \
+			gcc make cmake zlib1g-dev libssl-dev libkrb5-dev
+		;;
+	Linux-centos* | Linux-rhel* | Linux-rocky*)
+		yum install -y \
+			gcc make cmake zlib-devel openssl-devel rpm-build
 		;;
 	*)
-		echo "unsupported dependency install: $ID"
+		echo "unsupported platform: $platform"
 		exit 1
+		;;
 esac

scripts/print-install-deps.sh

@@ -0,0 +1,23 @@
+#!/bin/bash -e
+#
+# Print install dpenedencies on Linux. CMake runs this script to obtain deps for CPACK.
+# mscp dependes on packages on which libssh depends.
+
+source /etc/os-release
+
+release=$1
+
+case $release in
+	ubuntu-20.04*)
+		echo "libc6 (>= 2.27), libgssapi-krb5-2 (>= 1.17), libssl1.1 (>= 1.1.1), zlib1g (>= 1:1.1.4)"
+		;;
+	ubuntu-22.04*)
+		echo "libc6 (>= 2.33), libgssapi-krb5-2 (>= 1.17), libssl3 (>= 3.0.0~~alpha1), zlib1g (>= 1:1.1.4)"
+		;;
+	centos* | rhel* | rocky*)
+		echo "glibc crypto-policies krb5-libs openssl-libs libcom_err"
+		;;
+	*)
+		echo "unsupported install dependency: $release"
+		exit 1
+esac

scripts/test-in-container.sh

@@ -1,31 +1,13 @@
 #!/bin/bash -e
 #
-# Run this script in docker containers. This script installs mscp from built package
-# and run test for mscp in the installed path.
+# Run this script in mscp docker containers.
+# This script runs end-to-end test with installed mscp.
 
-source /etc/os-release
 script_dir=$(cd $(dirname ${0}) && pwd)
 cd $script_dir
-project_version=$(cat ../VERSION)
-arch=$(uname -p)
 
 set -x
 
-# install package
-case $ID in
-        ubuntu*)
-		pkg=mscp_${project_version}-${ID}-${VERSION_ID}-${arch}.deb
-		dpkg -i ../build/$pkg
-                ;;
-        centos* | rhel*)
-		pkg=mscp_${project_version}-${ID}-${VERSION_ID}-${arch}.rpm
-		rpm -iv ../build/$pkg
-                ;;
-        *)
-                echo "unsupported test platform: $ID"
-                exit 1
-esac
-
 # Run sshd
 if [ ! -e /var/run/sshd.pid ]; then
 	/usr/sbin/sshd

src/file.c

@@ -319,13 +319,15 @@ static int file_fill_recursive(struct list_head *file_list,
 int file_fill(sftp_session sftp, struct list_head *file_list, char **src_array, int cnt,
 	      char *dst)
 {
-	bool dst_is_remote, dst_is_dir, dst_dir_no_exist, dst_should_dir;
+	bool dst_is_remote, dst_is_dir, dst_dir_no_exist, dst_should_dir, dst_must_dir;
 	char *dst_path, *src_path;
 	int n, ret;
 
 	dst_path = file_find_path(dst);
 	dst_path = *dst_path == '\0' ? "." : dst_path;
 	dst_is_remote = file_find_hostname(dst) ? true : false;
+	dst_must_dir = cnt > 1 ? true : false;
+
 	if (file_is_directory(dst_path, dst_is_remote ? sftp : NULL, false) > 0)
 		dst_is_dir = true;
 	else
@@ -342,7 +344,8 @@ int file_fill(sftp_session sftp, struct list_head *file_list, char **src_array,
 
 		ret = file_fill_recursive(file_list, dst_is_remote, sftp,
 					  src_path, "", dst_path,
-					  dst_should_dir | dst_is_dir, dst_dir_no_exist);
+					  dst_should_dir | dst_must_dir | dst_is_dir,
+					  dst_dir_no_exist);
 		if (ret < 0)
 			return ret;
 	}
@@ -364,6 +367,7 @@ static int file_dst_prepare(struct file *f, sftp_session sftp)
 
 	pr_debug("prepare for %s\n", path);
 
+	/* mkdir -p */
 	for (p = strchr(path + 1, '/'); p; p = strchr(p + 1, '/')) {
 		*p = '\0';
 
@@ -395,6 +399,30 @@ static int file_dst_prepare(struct file *f, sftp_session sftp)
 		*p = '/';
 	}
 
+	/* open file with O_TRUNC to set file size 0 */
+	mode = O_WRONLY|O_CREAT|O_TRUNC;
+	if (sftp) {
+		sftp_file sf;
+		if ((sf = sftp_open(sftp, f->dst_path, mode, S_IRUSR|S_IWUSR)) == NULL) {
+			pr_err("sftp_open: %s\n", sftp_get_ssh_error(sftp));
+			return -1;
+		}
+		if (sftp_close(sf) < 0) {
+			pr_err("sftp_close: %s\n", sftp_get_ssh_error(sftp));
+			return -1;
+		}
+	} else {
+		int fd;
+		if ((fd = open(f->dst_path, mode, S_IRUSR|S_IWUSR)) < 0) {
+			pr_err("open: %s\n", strerrno());
+			return -1;
+		}
+		if (close(fd) < 0) {
+			pr_err("close: %s\n", strerrno());
+			return -1;
+		}
+	}
+
 	return 0;
 }
 
@@ -619,39 +647,27 @@ static sftp_file chunk_open_remote(const char *path, int flags, mode_t mode, siz
 	return sf;
 }
 
-static int chunk_copy_internal(struct chunk *c, int fd, sftp_file sf,
-			       size_t sftp_buf_sz, size_t io_buf_sz,
-			       bool reverse, size_t *counter)
+/*
+ * TODO: handle case when read returns 0 (EOF).
+ */
+static int _chunk_copy_local_to_remote(struct chunk *c, int fd, sftp_file sf,
+				       size_t sftp_buf_sz, size_t io_buf_sz,
+				       size_t *counter)
 {
-	size_t remaind, read_bytes, write_bytes;
+	ssize_t read_bytes, write_bytes, remaind;
 	char buf[io_buf_sz];
 
-	/* if reverse is false, copy fd->sf (local to remote).
-	 * if reverse is true, copy sf->fd (remote to local)
-	 */
-
 	for (remaind = c->len; remaind > 0;) {
 
-		if (!reverse)
-			read_bytes = read(fd, buf, min(remaind, io_buf_sz));
-		else
-			read_bytes = sftp_read2(sf, buf, min(remaind, io_buf_sz),
-						sftp_buf_sz);
-
+		read_bytes = read(fd, buf, min(remaind, io_buf_sz));
 		if (read_bytes < 0) {
-			pr_err("failed to read %s: %s\n", c->f->dst_path,
-			       !reverse ? strerrno() : sftp_get_ssh_error(sf->sftp));
+			pr_err("read: %s\n", strerrno());
 			return -1;
 		}
 
-		if (!reverse)
-			write_bytes = sftp_write2(sf, buf, read_bytes, sftp_buf_sz);
-		else
-			write_bytes = write(fd, buf, read_bytes);
-
+		write_bytes = sftp_write2(sf, buf, read_bytes, sftp_buf_sz);
 		if (write_bytes < 0) {
-			pr_err("failed to write %s: %s\n", c->f->dst_path,
-			       !reverse ? strerrno() : sftp_get_ssh_error(sf->sftp));
+			pr_err("sftp_write: %d\n", sftp_get_error(sf->sftp));
 			return -1;
 		}
 
@@ -667,9 +683,157 @@ static int chunk_copy_internal(struct chunk *c, int fd, sftp_file sf,
 	return 0;
 }
 
+#define XFER_BUF_SIZE	16384
+
+#ifdef ASYNC_WRITE
+static int _chunk_copy_local_to_remote_async(struct chunk *c, int fd,
+					     sftp_file sf, int nr_ahead, size_t *counter)
+{
+	ssize_t read_bytes, remaind, thrown;
+	char buf[XFER_BUF_SIZE];
+	int idx, ret;
+	struct {
+		uint32_t id;
+		ssize_t  len;
+	} reqs[nr_ahead];
+
+	if (c->len == 0)
+		return 0;
+
+	remaind = thrown = c->len;
+	for (idx = 0; idx < nr_ahead && thrown > 0; idx++) {
+		reqs[idx].len = min(thrown, sizeof(buf));
+		read_bytes = read(fd, buf, reqs[idx].len);
+		if (read_bytes < 0) {
+			pr_err("read: %s\n", strerrno());
+			return -1;
+		}
+		ret = sftp_async_write(sf, buf, reqs[idx].len, &reqs[idx].id);
+		if (ret < 0) {
+			pr_err("sftp_async_write: %d\n", sftp_get_error(sf->sftp));
+			return -1;
+		}
+		thrown -= reqs[idx].len;
+	}
+
+	for (idx = 0; remaind > 0; idx = (idx + 1) % nr_ahead) {
+		ret = sftp_async_write_end(sf, reqs[idx].id, 1);
+		if (ret != SSH_OK) {
+			pr_err("sftp_async_write_end: %d\n", sftp_get_error(sf->sftp));
+			return -1;
+		}
+
+		*counter += reqs[idx].len;
+		remaind -= reqs[idx].len;
+
+		if (remaind <= 0)
+			break;
+
+		if (thrown <= 0)
+			continue;
+
+		reqs[idx].len = min(thrown, sizeof(buf));
+		read_bytes = read(fd, buf, reqs[idx].len);
+		if (read_bytes < 0) {
+			pr_err("read: %s\n", strerrno());
+			return -1;
+		}
+
+		ret = sftp_async_write(sf, buf, reqs[idx].len, &reqs[idx].id);
+		if (ret < 0) {
+			pr_err("sftp_async_write: %d\n", sftp_get_error(sf->sftp));
+			return -1;
+		}
+		thrown -= reqs[idx].len;
+	}
+
+	if (remaind < 0) {
+		pr_err("invalid remaind bytes %ld. last async_write_end bytes %lu. "
+		       "last read bytes %ld\n",
+		       remaind, reqs[idx].len, read_bytes);
+		return -1;
+	}
+
+	return 0;
+}
+#endif
+
+static int _chunk_copy_remote_to_local(struct chunk *c, int fd, sftp_file sf,
+				       int nr_ahead, size_t *counter)
+{
+	ssize_t read_bytes, write_bytes, remaind, thrown;
+	char buf[XFER_BUF_SIZE];
+	int idx;
+	struct {
+		int id;
+		ssize_t len;
+	} reqs[nr_ahead];
+
+	/* TODO: sftp_buf_sz has no effect on remote to local copy. we
+	 * always use 16384 byte buffer pointed by
+	 * https://api.libssh.org/stable/libssh_tutor_sftp.html. The
+	 * larget read length from sftp_async_read is 65536 byte.
+	 * Read sizes larget than 65536 cause a situation where data
+	 * remainds but sftp_async_read returns 0.
+	 */
+
+	if (c->len == 0)
+		return 0;
+
+	remaind = thrown = c->len;
+
+	for (idx = 0; idx < nr_ahead && thrown > 0; idx++) {
+		reqs[idx].len = min(thrown, sizeof(buf));
+		reqs[idx].id = sftp_async_read_begin(sf, reqs[idx].len);
+		if (reqs[idx].id < 0) {
+			pr_err("sftp_async_read_begin: %d\n",
+			       sftp_get_error(sf->sftp));
+			return -1;
+		}
+		thrown -= reqs[idx].len;
+	}
+
+	for (idx = 0; remaind > 0; idx = (idx + 1) % nr_ahead) {
+		read_bytes = sftp_async_read(sf, buf, reqs[idx].len, reqs[idx].id);
+		if (read_bytes == SSH_ERROR) {
+			pr_err("sftp_async_read: %d\n", sftp_get_error(sf->sftp));
+			return -1;
+		}
+
+		if (thrown > 0) {
+			reqs[idx].len = min(thrown, sizeof(buf));
+			reqs[idx].id = sftp_async_read_begin(sf, reqs[idx].len);
+			thrown -= reqs[idx].len;
+		}
+
+		write_bytes = write(fd, buf, read_bytes);
+		if (write_bytes < 0) {
+			pr_err("write: %s\n", strerrno());
+			return -1;
+		}
+
+		if (write_bytes < read_bytes) {
+			pr_err("failed to write full bytes\n");
+			return -1;
+		}
+
+		*counter += write_bytes;
+		remaind -= read_bytes;
+	}
+
+	if (remaind < 0) {
+		pr_err("invalid remaind bytes %ld. last async_read bytes %ld. "
+		       "last write bytes %ld\n",
+		       remaind, read_bytes, write_bytes);
+		return -1;
+	}
+
+	return 0;
+}
+
 static int chunk_copy_local_to_remote(struct chunk *c, sftp_session sftp,
 				      size_t sftp_buf_sz, size_t io_buf_sz,
-				      size_t *counter)
+				      int nr_ahead, size_t *counter)
 {
 	struct file *f = c->f;
 	sftp_file sf = NULL;
@@ -685,14 +849,19 @@ static int chunk_copy_local_to_remote(struct chunk *c, sftp_session sftp,
 		goto out;
 	}
 
-	flags = O_WRONLY|O_CREAT;
+	flags = O_WRONLY;
 	mode = S_IRUSR|S_IWUSR;
 	if (!(sf = chunk_open_remote(f->dst_path, flags, mode, c->off, sftp))) {
 		ret = -1;
 		goto out;
 	}
 
-	ret = chunk_copy_internal(c, fd, sf, sftp_buf_sz, io_buf_sz, false, counter);
+#ifndef ASYNC_WRITE
+	ret = _chunk_copy_local_to_remote(c, fd, sf, sftp_buf_sz, io_buf_sz,
+						  counter);
+#else
+	ret = _chunk_copy_local_to_remote_async(c, fd, sf, nr_ahead, counter);
+#endif
 	if (ret < 0)
 		goto out;
 
@@ -713,8 +882,7 @@ out:
 }
 
 static int chunk_copy_remote_to_local(struct chunk *c, sftp_session sftp,
-				      size_t sftp_buf_sz, size_t io_buf_sz,
-				      size_t *counter)
+				      int nr_ahead, size_t *counter)
 {
 	struct file *f = c->f;
 	sftp_file sf = NULL;
@@ -723,7 +891,7 @@ static int chunk_copy_remote_to_local(struct chunk *c, sftp_session sftp,
 	int fd = 0;
 	int ret = 0;
 
-	flags = O_WRONLY|O_CREAT;
+	flags = O_WRONLY;
 	mode = S_IRUSR|S_IWUSR;
 	if ((fd = chunk_open_local(f->dst_path, flags, mode, c->off)) < 0) {
 		ret = -1;
@@ -737,7 +905,7 @@ static int chunk_copy_remote_to_local(struct chunk *c, sftp_session sftp,
 		goto out;
 	}
 
-	ret = chunk_copy_internal(c, fd, sf, sftp_buf_sz, io_buf_sz, true, counter);
+	ret = _chunk_copy_remote_to_local(c, fd, sf, nr_ahead, counter);
 	if (ret< 0)
 		goto out;
 
@@ -753,7 +921,7 @@ out:
 
 
 int chunk_copy(struct chunk *c, sftp_session sftp, size_t sftp_buf_sz, size_t io_buf_sz,
-	       size_t *counter)
+	       int nr_ahead, size_t *counter)
 {
 	struct file *f = c->f;
 	int ret = 0;
@@ -767,11 +935,10 @@ int chunk_copy(struct chunk *c, sftp_session sftp, size_t sftp_buf_sz, size_t io
 
 
 	if (f->dst_is_remote)
-		ret = chunk_copy_local_to_remote(c, sftp,
-						 sftp_buf_sz, io_buf_sz, counter);
+		ret = chunk_copy_local_to_remote(c, sftp, sftp_buf_sz, io_buf_sz,
+						 nr_ahead, counter);
 	else
-		ret = chunk_copy_remote_to_local(c, sftp,
-						 sftp_buf_sz, io_buf_sz, counter);
+		ret = chunk_copy_remote_to_local(c, sftp, nr_ahead, counter);
 
 	if (ret < 0)
 		return ret;

src/file.h

@@ -3,8 +3,8 @@
 
 #include <limits.h>
 #include <pthread.h>
-#include <libssh/libssh.h>
-#include <libssh/sftp.h>
+#include "libssh/libssh.h"
+#include "libssh/sftp.h"
 
 #include <list.h>
 #include <atomic.h>
@@ -37,7 +37,8 @@ struct file {
  * if the file state of the chunk is INIT:
  *     acquire the file lock
  * *         if file state is INIT:
- *             create destination file and directory if necessary
+ *             create directory if necessary
+ *             open file with O_TRUNC and close.
  *             set file state OPENED.
  *             // only the first thread in the lock open the destination file
  *     release the file lock
@@ -71,8 +72,8 @@ int chunk_fill(struct list_head *file_list, struct list_head *chunk_list,
 
 struct chunk *chunk_acquire(struct list_head *chunk_list);
 int chunk_prepare(struct chunk *c, sftp_session sftp);
-int chunk_copy(struct chunk *c, sftp_session sftp,
-	       size_t sftp_buf_sz, size_t io_buf_sz, size_t *counter);
+int chunk_copy(struct chunk *c, sftp_session sftp, size_t sftp_buf_sz, size_t io_buf_sz,
+	       int nr_ahead, size_t *counter);
 
 #ifdef DEBUG
 void file_dump(struct list_head *file_list);

src/list.h

@@ -514,4 +514,19 @@ static inline void hlist_add_after(struct hlist_node *n,
 	     pos = n)
 
 
+/**
+ * list_count - return length of list
+ * @head        the head for your list.
+ */
+static inline int list_count(struct list_head *head)
+{
+	int n = 0;
+	struct list_head *p;
+
+	list_for_each(p, head) n++;
+	return n;
+}
+
+
 #endif
+

src/main.c

@@ -26,6 +26,7 @@
 #define DEFAULT_SFTP_BUF_SZ     131072          /* derived from qemu/block/ssh.c */
 #define DEFAULT_IO_BUF_SZ       DEFAULT_SFTP_BUF_SZ
 /* XXX: need to investigate max buf size for sftp_read/sftp_write */
+#define DEFAULT_NR_AHEAD	16
 
 struct mscp {
 	char                    *host;  /* remote host (and username) */
@@ -33,12 +34,13 @@ struct mscp {
 	sftp_session            ctrl;   /* control sftp session */
 
 	struct list_head        file_list;
-	struct list_head        chunk_list;     /* stack of chunks */
-	lock                    chunk_lock;  /* lock for chunk list */
+	struct list_head        chunk_list;	/* stack of chunks */
+	lock                    chunk_lock;	/* lock for chunk list */
 
 	char    *target;
 
 	int     sftp_buf_sz, io_buf_sz;
+	int	nr_ahead;	/* # of ahead read command for remote to local copy */
 
 	struct timeval start;   /* timestamp of starting copy */
 };
@@ -48,6 +50,7 @@ struct mscp_thread {
 	sftp_session    sftp;
 
 	pthread_t       tid;
+	int		cpu;
 	size_t          done;           /* copied bytes */
 	bool            finished;
 	int		ret;
@@ -66,25 +69,19 @@ void stop_copy_threads(int sig)
 
 	pr("stopping...\n");
 	for (n = 0; n < nr_threads; n++) {
-		pthread_cancel(threads[n].tid);
+		if (!threads[n].finished)
+			pthread_cancel(threads[n].tid);
 	}
 }
 
-int list_count(struct list_head *head)
-{
-	int n = 0;
-	struct list_head *p;
-
-	list_for_each(p, head) n++;
-	return n;
-}
-
 void usage(bool print_help) {
 	printf("mscp v" VERSION ": copy files over multiple ssh connections\n"
 	       "\n"
-	       "Usage: mscp [vqDCHdh] [-n nr_conns]\n"
-	       "            [-s min_chunk_sz] [-S max_chunk_sz]\n"
-	       "            [-b sftp_buf_sz] [-B io_buf_sz]\n"
+	       "Usage: mscp [vqDCHdh] [-n nr_conns] [-m coremask]\n"
+	       "            [-s min_chunk_sz] [-S max_chunk_sz] [-a nr_ahead]\n"
+#ifndef ASYNC_WRITE
+	       "            [-b sftp_buf_sz] [-B io_buf_sz] \n"
+#endif
 	       "            [-l login_name] [-p port] [-i identity_file]\n"
 	       "            [-c cipher_spec] source ... target\n"
 	       "\n");
@@ -93,19 +90,26 @@ void usage(bool print_help) {
 		return;
 
 	printf("    -n NR_CONNECTIONS  number of connections (default: half of # of cpu cores)\n"
+	       "    -m COREMASK        hex value to specify cores where threads pinned\n"
 	       "    -s MIN_CHUNK_SIZE  min chunk size (default: 64MB)\n"
 	       "    -S MAX_CHUNK_SIZE  max chunk size (default: filesize / nr_conn)\n"
+	       "\n"
+	       "    -a NR_AHEAD        number of inflight SFTP commands (default: 16)\n"
+#ifndef ASYNC_WRITE
 	       "    -b SFTP_BUF_SIZE   buf size for sftp_read/write (default 131072B)\n"
 	       "    -B IO_BUF_SIZE     buf size for read/write (default 131072B)\n"
-	       "                       Note that this value is derived from\n"
+	       "                       Note that the default value is derived from\n"
 	       "                       qemu/block/ssh.c. need investigation...\n"
+	       "                       -b and -B affect only local to remote copy\n"
+#endif
+	       "\n"
 	       "    -v                 increment verbose output level\n"
 	       "    -q                 disable output\n"
 	       "    -D                 dry run\n"
 	       "\n"
 	       "    -l LOGIN_NAME      login name\n"
 	       "    -p PORT            port number\n"
-	       "    -i IDENTITY        identity file for publickey authentication\n"
+	       "    -i IDENTITY        identity file for public key authentication\n"
 	       "    -c CIPHER          cipher spec, see `ssh -Q cipher`\n"
 	       "    -C                 enable compression on libssh\n"
 	       "    -H                 disable hostkey check\n"
@@ -157,15 +161,75 @@ err_out:
 	return NULL;
 }
 
+int expand_coremask(const char *coremask, int **cores, int *nr_cores)
+{
+	int n, *core_list, core_list_len = 0, nr_usable, nr_all;
+	char c[2] = { 'x', '\0' };
+	const char *_coremask;
+	long v, needle;
+
+	/*
+	 * This function returns array of usable cores in `cores` and
+	 * returns the number of usable cores (array length) through
+	 * nr_cores.
+	 */
+
+	if (strncmp(coremask, "0x", 2) == 0)
+		_coremask = coremask + 2;
+	else
+		_coremask = coremask;
+
+	core_list = realloc(NULL, sizeof(int) * 64);
+	if (!core_list) {
+		pr_err("failed to realloc: %s\n", strerrno());
+		return -1;
+	}
+
+	nr_usable = 0;
+	nr_all = 0;
+	for (n = strlen(_coremask) - 1; n >=0; n--) {
+		c[0] = _coremask[n];
+		v = strtol(c, NULL, 16);
+		if (v == LONG_MIN || v == LONG_MAX) {
+			pr_err("invalid coremask: %s\n", coremask);
+			return -1;
+		}
+
+		for (needle = 0x01; needle < 0x10; needle <<= 1) {
+			nr_all++;
+ 			if (v & needle) {
+				nr_usable++;
+				core_list = realloc(core_list, sizeof(int) * nr_usable);
+				if (!core_list) {
+					pr_err("failed to realloc: %s\n", strerrno());
+					return -1;
+				}
+				core_list[nr_usable - 1] = nr_all - 1;
+			}
+		}
+	}
+
+	if (nr_usable < 1) {
+		pr_err("invalid core mask: %s\n", coremask);
+		return -1;
+	}
+
+	*cores = core_list;
+	*nr_cores = nr_usable;
+	return 0;
+}
+
 int main(int argc, char **argv)
 {
 	struct mscp m;
 	struct ssh_opts opts;
 	int min_chunk_sz = DEFAULT_MIN_CHUNK_SZ;
 	int max_chunk_sz = 0;
+	char *coremask = NULL;;
 	int verbose = 1;
 	bool dryrun = false;
 	int ret = 0, n;
+	int *cores, nr_cores;
 	char ch;
 
 	memset(&opts, 0, sizeof(opts));
@@ -175,11 +239,12 @@ int main(int argc, char **argv)
 	lock_init(&m.chunk_lock);
 	m.sftp_buf_sz = DEFAULT_SFTP_BUF_SZ;
 	m.io_buf_sz = DEFAULT_IO_BUF_SZ;
+	m.nr_ahead = DEFAULT_NR_AHEAD;
 
 	nr_threads = (int)(nr_cpus() / 2);
 	nr_threads = nr_threads == 0 ? 1 : nr_threads;
 
-	while ((ch = getopt(argc, argv, "n:s:S:b:B:vqDl:p:i:c:CHdh")) != -1) {
+	while ((ch = getopt(argc, argv, "n:m:s:S:b:B:a:vqDl:p:i:c:CHdh")) != -1) {
 		switch (ch) {
 		case 'n':
 			nr_threads = atoi(optarg);
@@ -188,6 +253,9 @@ int main(int argc, char **argv)
 				return 1;
 			}
 			break;
+		case 'm':
+			coremask = optarg;
+			break;
 		case 's':
 			min_chunk_sz = atoi(optarg);
 			if (min_chunk_sz < getpagesize()) {
@@ -232,6 +300,13 @@ int main(int argc, char **argv)
 				return -1;
 			}
 			break;
+		case 'a':
+			m.nr_ahead = atoi(optarg);
+			if (m.nr_ahead < 1) {
+				pr_err("invalid number of ahead: %s\n", optarg);
+				return -1;
+			}
+			break;
 		case 'v':
 			verbose++;
 			break;
@@ -273,20 +348,29 @@ int main(int argc, char **argv)
 
 	pprint_set_level(verbose);
 
+	if (argc - optind < 2) {
+		/* mscp needs at lease 2 (src and target) argument */
+		usage(false);
+		return 1;
+	}
+	m.target = argv[argc - 1];
+
 	if (max_chunk_sz > 0 && min_chunk_sz > max_chunk_sz) {
 		pr_err("smaller max chunk size than min chunk size: %d < %d\n",
 		       max_chunk_sz, min_chunk_sz);
 		return 1;
 	}
 
-	if (argc - optind < 2) {
-		/* mscp needs at lease 2 (src and target) argument */
-		usage(false);
-		return 1;
+	/* expand usable cores from coremask */
+	if (coremask) {
+		if (expand_coremask(coremask, &cores, &nr_cores) < 0)
+			return -1;
+		pprint(2, "cpu cores:");
+		for (n = 0; n < nr_cores; n++)
+			pprint(2, " %d", cores[n]);
+		pprint(2, "\n");
 	}
 
-	m.target = argv[argc - 1];
-
 	/* create control session */
 	m.host = find_hostname(optind, argc, argv);
 	if (!m.host) {
@@ -323,13 +407,6 @@ int main(int argc, char **argv)
 	if (dryrun)
 		return 0;
 
-	/* register SIGINT to stop thrads */
-	if (signal(SIGINT, stop_copy_threads) == SIG_ERR) {
-		pr_err("cannot set signal: %s\n", strerrno());
-		ret = 1;
-		goto out;
-	}
-
 	/* prepare thread instances */
 	if ((n = list_count(&m.chunk_list)) < nr_threads) {
 		pprint3("we have only %d chunk(s). set nr_conns to %d\n", n, n);
@@ -342,6 +419,11 @@ int main(int argc, char **argv)
 		struct mscp_thread *t = &threads[n];
 		t->mscp = &m;
 		t->finished = false;
+		if (!coremask)
+			t->cpu = -1;
+		else
+			t->cpu = cores[n % nr_cores];
+
 		pprint3("connecting to %s for a copy thread...\n", m.host);
 		t->sftp = ssh_make_sftp_session(m.host, m.opts);
 		if (!t->sftp) {
@@ -359,6 +441,13 @@ int main(int argc, char **argv)
 		goto join_out;
 	}
 
+	/* register SIGINT to stop threads */
+	if (signal(SIGINT, stop_copy_threads) == SIG_ERR) {
+		pr_err("cannot set signal: %s\n", strerrno());
+		ret = 1;
+		goto out;
+	}
+
 	/* save start time */
 	gettimeofday(&m.start, NULL);
 
@@ -398,9 +487,14 @@ out:
 void mscp_copy_thread_cleanup(void *arg)
 {
 	struct mscp_thread *t = arg;
-	if (t->sftp)
+	if (t->sftp) {
+		/* XXX: sftp_free --> ssh_poll sometimes blocked with
+		 * no responses. So wet nonblocking. */
+		ssh_set_blocking(sftp_ssh(t->sftp), 1);
 		ssh_sftp_close(t->sftp);
+	}
 	t->finished = true;
+	__sync_synchronize();
 }
 
 void *mscp_copy_thread(void *arg)
@@ -410,6 +504,11 @@ void *mscp_copy_thread(void *arg)
 	sftp_session sftp = t->sftp;
 	struct chunk *c;
 
+	if (t->cpu > -1) {
+		if (set_thread_affinity(pthread_self(), t->cpu) < 0)
+			return NULL;
+	}
+
 	pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
 	pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL);
 	pthread_cleanup_push(mscp_copy_thread_cleanup, t);
@@ -425,13 +524,17 @@ void *mscp_copy_thread(void *arg)
 		if ((t->ret = chunk_prepare(c, sftp)) < 0)
 			break;
 
-		if ((t->ret = chunk_copy(c, sftp,
-					 m->sftp_buf_sz, m->io_buf_sz, &t->done)) < 0)
+		if ((t->ret = chunk_copy(c, sftp, m->sftp_buf_sz, m->io_buf_sz,
+					 m->nr_ahead, &t->done)) < 0)
 			break;
 	}
 
 	pthread_cleanup_pop(1);
 
+	if (t->ret < 0)
+		pr_err("copy failed: chunk %s 0x%010lx-0x%010lx\n",
+		       c->f->src_path, c->off, c->off + c->len);
+
 	return NULL;
 }
 
@@ -490,11 +593,11 @@ static void print_progress_bar(double percent, char *suffix)
 static void print_progress(struct timeval *start, struct timeval *end,
 			   size_t total, size_t last, size_t done)
 {
-	char *bps_units[] = { "B/s", "KB/s", "MB/s", "GB/s" };
-	char *byte_units[] = { "B", "KB", "MB", "GB", "TB", "PB" };
+	char *bps_units[] = { "B/s ", "KB/s", "MB/s", "GB/s" };
+	char *byte_units[] = { "B ", "KB", "MB", "GB", "TB", "PB" };
 	char suffix[128];
 	int bps_u, byte_tu, byte_du;
-	size_t total_round;
+	size_t total_round, done_round;
 	int percent;
 	double bps;
 
@@ -515,11 +618,14 @@ static void print_progress(struct timeval *start, struct timeval *end,
 		bps /= 1000;
 
 	percent = floor(((double)(done) / (double)total) * 100);
-	for (byte_du = 0; done > 1000 && byte_du < array_size(byte_units) - 1; byte_du++)
-		done /= 1024;
 
-	snprintf(suffix, sizeof(suffix), "%lu%s/%lu%s %.2f%s ",
-		 done, byte_units[byte_du], total_round, byte_units[byte_tu],
+	done_round = done;
+	for (byte_du = 0; done_round > 1000 && byte_du < array_size(byte_units) - 1;
+	     byte_du++)
+		done_round /= 1024;
+
+	snprintf(suffix, sizeof(suffix), "%lu%s/%lu%s %6.1f%s ",
+		 done_round, byte_units[byte_du], total_round, byte_units[byte_tu],
 		 bps, bps_units[bps_u]);
 
 	print_progress_bar(percent, suffix);
@@ -579,10 +685,10 @@ void *mscp_monitor_thread(void *arg)
 		}
 		gettimeofday(&b, NULL);
 
-		usleep(500000);
+		usleep(1000000);
 
 		for (n = 0; n < nr_threads; n++) {
-			done += threads[n].done;;
+			done += threads[n].done;
 			if (!threads[n].finished)
 				all_done = false;
 		}

src/platform.c

@@ -24,6 +24,13 @@ int nr_cpus()
 
 	return n;
 }
+
+int set_thread_affinity(pthread_t tid, int core)
+{
+	pr_warn("setting thread afinity is not implemented on apple\n");
+	return 0;
+}
+
 #endif
 
 #ifdef linux
@@ -34,5 +41,19 @@ int nr_cpus()
 		return CPU_COUNT(&cpu_set);
 	return -1;
 }
+
+int set_thread_affinity(pthread_t tid, int core)
+{
+	cpu_set_t target_cpu_set;
+	int ret = 0;
+
+	CPU_ZERO(&target_cpu_set);
+	CPU_SET(core, &target_cpu_set);
+	ret = pthread_setaffinity_np(tid, sizeof(target_cpu_set), &target_cpu_set);
+	if (ret < 0)
+		pr_err("failed to set thread/cpu affinity for core %d: %s",
+		       core, strerrno());
+	return ret;
+}
 #endif
 

src/platform.h

@@ -1,6 +1,9 @@
 #ifndef _PLATFORM_H_
 #define _PLATFORM_H_
 
+#include <pthread.h>
+
 int nr_cpus();
+int set_thread_affinity(pthread_t tid, int core);
 
 #endif /* _PLATFORM_H_ */

src/rename-logic.py

@@ -32,13 +32,17 @@ def recursive(src, rel_path, dst, dst_should_dir, replace_dir_name):
         recursive(next_src, next_rel_path, dst, dst_should_dir, False)
 
 
-def fill_dst(src, dst):
-    dst_should_dir = isdir(src) | isdir(dst)
-    replace_dir_name = not isdir(dst)
-    recursive(src, "", dst, dst_should_dir, replace_dir_name)
+def fill_dst(srclist, dst):
+    dst_must_dir = len(srclist) > 1
+    for src in srclist:
+        dst_should_dir = isdir(src) | isdir(dst)
+        replace_dir_name = not isdir(dst)
+        recursive(src, "", dst, dst_should_dir | dst_must_dir, replace_dir_name)
 
 
 def main():
-    fill_dst(sys.argv[1], sys.argv[2])
+    if (len(sys.argv) < 2):
+        print("usage: {} source ... target".format(sys.argv[0]))
+    fill_dst(sys.argv[1:len(sys.argv) - 1], sys.argv[len(sys.argv) - 1])
 
 main()

src/ssh.c

@@ -246,9 +246,9 @@ void ssh_sftp_close(sftp_session sftp)
 }
 
 
-int sftp_write2(sftp_file sf, const void *buf, size_t len, size_t sftp_buf_sz)
+ssize_t sftp_write2(sftp_file sf, const void *buf, size_t len, size_t sftp_buf_sz)
 {
-	int ret, nbytes;
+	ssize_t ret, nbytes;
 
 	for (nbytes = 0; nbytes < len;) {
 		ret = sftp_write(sf, buf + nbytes,
@@ -260,9 +260,9 @@ int sftp_write2(sftp_file sf, const void *buf, size_t len, size_t sftp_buf_sz)
 	return nbytes;
 }
 
-int sftp_read2(sftp_file sf, void *buf, size_t len, size_t sftp_buf_sz)
+ssize_t sftp_read2(sftp_file sf, void *buf, size_t len, size_t sftp_buf_sz)
 {
-	int ret, nbytes;
+	ssize_t ret, nbytes;
 
 	for (nbytes = 0; nbytes < len;) {
 		ret = sftp_read(sf, buf + nbytes,

src/ssh.h

@@ -2,8 +2,8 @@
 #define _SSH_H_
 
 #include <stdbool.h>
-#include <libssh/libssh.h>
-#include <libssh/sftp.h>
+#include "libssh/libssh.h"
+#include "libssh/sftp.h"
 
 
 struct ssh_opts {
@@ -28,7 +28,7 @@ void ssh_sftp_close(sftp_session sftp);
 #define sftp_get_ssh_error(sftp) ssh_get_error(sftp_ssh(sftp))
 
 /* wrapping multiple sftp_read|write */
-int sftp_write2(sftp_file sf, const void *buf, size_t len, size_t sftp_buf_sz);
-int sftp_read2(sftp_file sf, void *buf, size_t len, size_t sftp_buf_sz);
+ssize_t sftp_write2(sftp_file sf, const void *buf, size_t len, size_t sftp_buf_sz);
+ssize_t sftp_read2(sftp_file sf, void *buf, size_t len, size_t sftp_buf_sz);
 
 #endif /* _SSH_H_ */

test/test_e2e.py

@@ -29,6 +29,7 @@ param_invalid_hostnames = [
     (["a:a", "b:b", "c:c"]), (["a:a", "b:b", "c"]), (["a:a", "b", "c:c"]),
     (["a", "b:b", "c:c"])
 ]
+
 @pytest.mark.parametrize("args", param_invalid_hostnames)
 def test_nonidentical_hostnames(mscp, args):
     run2ng([mscp] + args)
@@ -39,6 +40,9 @@ def test_nonidentical_hostnames(mscp, args):
 """ copy test """
 
 remote_prefix = "localhost:{}/".format(os.getcwd()) # use current dir
+param_remote_prefix = [
+    ("", remote_prefix), (remote_prefix, "")
+]
 
 param_single_copy = [
     (File("src", size = 64), File("dst")),
@@ -46,23 +50,33 @@ param_single_copy = [
     (File("src", size = 128 * 1024 * 1024), File("dst")),
 ]
 
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 @pytest.mark.parametrize("src, dst", param_single_copy)
-def test_single_copy_remote2local(mscp, src, dst):
+def test_single_copy(mscp, src_prefix, dst_prefix, src, dst):
     src.make()
-    run2ok([mscp, "-H", remote_prefix + src.path, dst.path])
-    assert check_same_md5sum(src, dst)
-    src.cleanup()
-    dst.cleanup()
-
-@pytest.mark.parametrize("src, dst", param_single_copy)
-def test_single_copy_local2remote(mscp, src, dst):
-    src.make()
-    run2ok([mscp, "-H", src.path, remote_prefix + dst.path])
+    run2ok([mscp, "-H", src_prefix + src.path, dst_prefix + dst.path])
     assert check_same_md5sum(src, dst)
     src.cleanup()
     dst.cleanup()
 
 
+param_double_copy = [
+    (File("src1", size = 1024 * 1024), File("src2", size = 1024 * 1024),
+     File("dst/src1"), File("dst/src2")
+     )
+]
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+@pytest.mark.parametrize("s1, s2, d1, d2", param_double_copy)
+def test_double_copy(mscp, src_prefix, dst_prefix, s1, s2, d1, d2):
+    s1.make()
+    s2.make()
+    run2ok([mscp, "-H", src_prefix + s1.path, src_prefix + s2.path, dst_prefix + "dst"])
+    assert check_same_md5sum(s1, d1)
+    assert check_same_md5sum(s2, d2)
+    s1.cleanup()
+    s2.cleanup()
+    d1.cleanup()
+    d2.cleanup()
 
 param_dir_copy = [
     ( "src_dir", "dst_dir",
@@ -87,16 +101,17 @@ does not exist. If dst_dir exists, scp copies src_dir to
 dst_dir/src_dir. So, this test checks both cases.
 """
 
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 @pytest.mark.parametrize("src_dir, dst_dir, src, dst, twice", param_dir_copy)
-def test_dir_copy_remote2local(mscp, src_dir, dst_dir, src, dst, twice):
+def test_dir_copy(mscp, src_prefix, dst_prefix, src_dir, dst_dir, src, dst, twice):
     for f in src:
         f.make()
 
-    run2ok([mscp, "-H", remote_prefix + src_dir, dst_dir])
+    run2ok([mscp, "-H", src_prefix + src_dir, dst_prefix + dst_dir])
     for sf, df in zip(src, dst):
         assert check_same_md5sum(sf, df)
 
-    run2ok([mscp, "-H", remote_prefix + src_dir, dst_dir])
+    run2ok([mscp, "-H", src_prefix + src_dir, dst_prefix + dst_dir])
     for sf, df in zip(src, twice):
         assert check_same_md5sum(sf, df)
 
@@ -105,28 +120,6 @@ def test_dir_copy_remote2local(mscp, src_dir, dst_dir, src, dst, twice):
         df.cleanup()
         tf.cleanup()
 
-@pytest.mark.parametrize("src_dir, dst_dir, src, dst, twice", param_dir_copy)
-def test_dir_copy_local2remote(mscp, src_dir, dst_dir, src, dst, twice):
-    for f in src:
-        f.make()
-
-    run2ok([mscp, "-H", src_dir, remote_prefix + dst_dir])
-    for sf, df in zip(src, dst):
-        assert check_same_md5sum(sf, df)
-
-    run2ok([mscp, "-H", src_dir, remote_prefix + dst_dir])
-    for sf, df in zip(src, twice):
-        assert check_same_md5sum(sf, df)
-
-    for sf, df, tf in zip(src, dst, twice):
-        sf.cleanup()
-        df.cleanup()
-        tf.cleanup()
-
-
-param_remote_prefix = [
-    ("", remote_prefix), (remote_prefix, "")
-]
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 def test_override_single_file(mscp, src_prefix, dst_prefix):
     src = File("src", size = 128).make()
@@ -150,6 +143,18 @@ def test_min_chunk(mscp, src_prefix, dst_prefix):
     src.cleanup()
     dst.cleanup()
 
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_thread_affinity(mscp, src_prefix, dst_prefix):
+    src = File("src", size = 64 * 1024).make()
+    dst = File("dst")
+
+    run2ok([mscp, "-H", "-n", 4, "-m", "0x01", "-s", 8192, "-S", 65536,
+            src_prefix + src.path, dst_prefix + dst.path])
+    assert check_same_md5sum(src, dst)
+
+    src.cleanup()
+    dst.cleanup()
+
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 def test_cannot_override_file_with_dir(mscp, src_prefix, dst_prefix):
     src = File("src", size = 128).make()
@@ -169,3 +174,11 @@ def test_transfer_zero_bytes(mscp, src_prefix, dst_prefix):
     src.cleanup()
     dst.cleanup()
 
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_override_dst_having_larger_size(mscp, src_prefix, dst_prefix):
+    src = File("src", size = 1024 * 1024).make()
+    dst = File("dst", size = 1024 * 1024 * 2).make()
+    run2ok([mscp, "-H", src_prefix + src.path, dst_prefix + "dst"])
+    assert check_same_md5sum(src, dst)
+    src.cleanup()
+    dst.cleanup()