bump version to 0.0.5 and update README

set default nr_threads to floor(log(cores) * 2) + 1)
This change prevents mscp from establishing too many ssh connections on many-core machines in default.
2026-02-08 22:04:44 +08:00 · 2022-12-11 14:20:13 +09:00 · 2022-12-11 14:01:52 +09:00 · 2022-12-11 13:23:41 +09:00 · 2022-12-10 21:48:24 +09:00 · 2022-12-08 18:01:50 +09:00
34 changed files with 1971 additions and 592 deletions
--- a/.github/workflows/build-macos.yml
+++ b/.github/workflows/build-macos.yml
@@ -19,20 +19,27 @@ jobs:

    steps:
    - uses: actions/checkout@v3
+      with:
+        submodules: true

-      # libssh and cmake are already installed on the macos-latest runner, 2022/11/5
-      #    - name: install build dependency
-      #run: |
-      #  brew update
-      #  brew install libssh
-      #  brew install cmake
+    - name: install build dependency
+      run: ./scripts/install-build-deps.sh
+
+    - name: save homebrew prefix
+      id: brew-prefix
+      run: echo "HOMEBREW_PREFIX=$(brew --prefix)" >> $GITHUB_OUTPUT
+
+    - name: patch to libssh
+      run: patch -d libssh -p1 < patch/libssh-0.10.4.patch

    - name: Configure CMake
      # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
      # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
-      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
+      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DOPENSSL_ROOT_DIR=${{steps.brew-prefix.outputs.HOMEBREW_PREFIX}}/opt/openssl@1.1

    - name: Build
      # Build your program with the given configuration
      run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}

+    - name: Run
+      run: ${{github.workspace}}/build/mscp -h
--- a/.github/workflows/build-ubuntu.yml
+++ b/.github/workflows/build-ubuntu.yml
@@ -19,9 +19,15 @@ jobs:

    steps:
    - uses: actions/checkout@v3
+      with:
+        submodules: true
+
    - name: install build dependency
      run: sudo ./scripts/install-build-deps.sh

+    - name: patch to libssh
+      run: patch -d libssh -p1 < patch/libssh-0.10.4.patch
+
    - name: Configure CMake
      # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
      # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
@@ -31,3 +37,5 @@ jobs:
      # Build your program with the given configuration
      run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}

+    - name: Run
+      run: ${{github.workspace}}/build/mscp -h
--- a/.github/workflows/codeql.yml
+++ b/.github/workflows/codeql.yml
@@ -39,10 +39,15 @@ jobs:
    steps:
    - name: Checkout repository
      uses: actions/checkout@v3
+      with:
+        submodules: true

-    - name: Install build dependency
+    - name: install build dependency
      run: sudo ./scripts/install-build-deps.sh

+    - name: patch to libssh
+      run: patch -d libssh -p1 < patch/libssh-0.10.4.patch
+
    # Initializes the CodeQL tools for scanning.
    - name: Initialize CodeQL
      uses: github/codeql-action/init@v2
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -4,6 +4,7 @@ on:
  push:
    tags:
      - "v*.*.*"
+  workflow_dispatch:

 env:
    BUILD_TYPE: Release
@@ -13,21 +14,32 @@ jobs:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
+        with:
+          submodules: true
+
+      - name: patch to libssh
+        run: patch -d libssh -p1 < patch/libssh-0.10.4.patch
+
+      # TODO: just building docker does not require packages. fix CMakeLists
+      - name: install build dependency
+        run: sudo ./scripts/install-build-deps.sh

      - name: Set variables
        run: |
          VER=$(cat VERSION)
          echo "VERSION=$VER" >> $GITHUB_ENV

-      # TODO: docker build does not require libssh. fix CMakeLists
-      - name: install build dependency
-        run: sudo ./scripts/install-build-deps.sh
-
      - name: Configure Cmake
-        run: cmake -B ${{github.workspace}}/build -DBUILD_PKG=1 -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
+        run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}

-      - name: Build Packages
-        run: make -C ${{github.workspace}}/build package-all-in-docker
+      - name: Build Containers
+        run: make -C ${{github.workspace}}/build docker-build-all
+
+      - name: Test
+        run: make -C ${{github.workspace}}/build docker-test-all
+
+      - name: Retrieve packages from containers
+        run: make -C ${{github.workspace}}/build docker-pkg-all

      - name: Release
        uses: softprops/action-gh-release@v1
@@ -36,3 +48,31 @@ jobs:
            ${{github.workspace}}/build/mscp_${{env.VERSION}}-ubuntu-20.04-x86_64.deb
            ${{github.workspace}}/build/mscp_${{env.VERSION}}-ubuntu-22.04-x86_64.deb
            ${{github.workspace}}/build/mscp_${{env.VERSION}}-centos-8-x86_64.rpm
+            ${{github.workspace}}/build/mscp_${{env.VERSION}}-rocky-8.6-x86_64.rpm
+
+  source-release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+
+      - name: patch to libssh
+        run: patch -d libssh -p1 < patch/libssh-0.10.4.patch
+
+      - name: Set variables
+        run: |
+          VER=$(cat VERSION)
+          echo "VERSION=$VER" >> $GITHUB_ENV
+
+      - name: archive
+        run: |
+          cd ..
+          cp -r mscp mscp-${{env.VERSION}}
+          tar cvf mscp-${{env.VERSION}}.tar.gz --exclude-vcs mscp-${{env.VERSION}}
+
+      - name: Release
+        uses: softprops/action-gh-release@v1
+        with:
+          files: |
+            ${{github.workspace}}/../mscp-${{env.VERSION}}.tar.gz
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -14,12 +14,21 @@ jobs:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3
-    - name: Build ubuntu 22.04 container
-      run: docker build --rm -t mscp-ubuntu:22.04 -f docker/ubuntu-22.04.Dockerfile .
-    - name: Run test on ubuntu 22.04
-      run: docker run --init --rm mscp-ubuntu:22.04 /mscp/scripts/test-in-container.sh
+      with:
+        submodules: true

-    - name: Build centos 8 container
-      run: docker build --rm -t mscp-centos:8 -f docker/centos-8.Dockerfile .
-    - name: Run test on centos 8
-      run: docker run --init --rm mscp-centos:8 /mscp/scripts/test-in-container.sh
+    - name: patch to libssh
+      run: patch -d libssh -p1 < patch/libssh-0.10.4.patch
+
+    # TODO: just building docker does not require libssh. fix CMakeLists
+    - name: install build dependency
+      run: sudo ./scripts/install-build-deps.sh
+
+    - name: configure CMake
+      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
+
+    - name: Build Containers
+      run: make -C ${{github.workspace}}/build docker-build-all
+
+    - name: Run Test
+      run: make -C ${{github.workspace}}/build docker-test-all
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
 build
+compile_commands.json
 .*.swp
--- a/.gitmodules
+++ b/.gitmodules
@@ -0,0 +1,4 @@
+[submodule "libssh"]
+	path = libssh
+	url = https://git.libssh.org/projects/libssh.git
+	ignore = dirty
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -7,22 +7,46 @@ project(mscp
 	LANGUAGES C)

 set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -DDEBUG")
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/modules)
+
+
+# add libssh static library
+add_subdirectory(libssh EXCLUDE_FROM_ALL)

 if(APPLE)
 	list(APPEND CMAKE_PREFIX_PATH /usr/local) # intel mac homebrew prefix
 	list(APPEND CMAKE_PREFIX_PATH /opt/homebrew) # arm mac homebrew prefix
 endif() # APPLE

-find_package(libssh REQUIRED)

+# mscp executable
 add_executable(mscp src/main.c src/platform.c src/ssh.c src/file.c src/pprint.c)
-target_include_directories(mscp PUBLIC ./src)
-target_link_libraries(mscp ssh pthread m)
+
+set(MSCP_LINK_LIBS m pthread)
+set(MSCP_LINK_DIRS "")
+set(MSCP_COMPILE_OPTS "")
+set(MSCP_INCLUDE_DIRS ${mscp_SOURCE_DIR}/src)
+
+list(APPEND MSCP_COMPILE_OPTS -iquote ${CMAKE_CURRENT_BINARY_DIR}/libssh/include)
+list(APPEND MSCP_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/libssh/include)
+list(APPEND MSCP_LINK_LIBS ssh-static)
+
+find_package(GSSAPI)
+list(APPEND MSCP_LINK_LIBS ${GSSAPI_LIBRARIES})
+
+find_package(OpenSSL)
+list(APPEND MSCP_LINK_LIBS ${OPENSSL_LIBRARIES})
+
+find_package(ZLIB)
+list(APPEND MSCP_LINK_LIBS ${ZLIB_LIBRARIES})
+
+target_include_directories(mscp PRIVATE ${MSCP_INCLUDE_DIRS})
+target_link_directories(mscp PRIVATE ${MSCP_LINK_DIRS})
+target_link_libraries(mscp PRIVATE ${MSCP_LINK_LIBS})
+target_compile_options(mscp PRIVATE ${MSCP_COMPILE_OPTS})
 target_compile_definitions(mscp PUBLIC _VERSION="${PROJECT_VERSION}")

-install(TARGETS mscp
-	RUNTIME		DESTINATION bin
-)
+install(TARGETS mscp RUNTIME DESTINATION bin)



@@ -35,82 +59,90 @@ add_test(NAME	pytest
 enable_testing()


-# Build Packages
-if(BUILD_PKG)

-	# CPACK Rules
-	set(CPACK_SET_DESTDIR		true)
-	set(CPACK_PROJECT_NAME		${PROJECT_NAME})
-	set(CPACK_PROJECT_VERSION	${PROJECT_VERSION})
-	set(CPACK_PACKAGE_CONTACT	"Ryo Nakamura <upa@haeena.net>")
-	set(CPACK_PACKAGE_DESCRIPTION
-		"mscp, copy files over multiple ssh connections")
+# CPACK Rules
+set(CPACK_SET_DESTDIR		true)
+set(CPACK_PROJECT_NAME		${PROJECT_NAME})
+set(CPACK_PROJECT_VERSION	${PROJECT_VERSION})
+set(CPACK_PACKAGE_CONTACT	"Ryo Nakamura <upa@haeena.net>")
+set(CPACK_PACKAGE_DESCRIPTION
+	"mscp, copy files over multiple ssh connections")

-	# on linux
-	if(UNIX AND NOT APPLE)
-		execute_process(COMMAND
-			bash "-c" "cat /etc/os-release|grep '^ID='|cut -d '=' -f 2|tr -d '\"'"
-			OUTPUT_VARIABLE DIST_NAME OUTPUT_STRIP_TRAILING_WHITESPACE)
-		execute_process(COMMAND
-			bash "-c" "cat /etc/os-release|grep '^VERSION_ID='|cut -d '=' -f 2|tr -d '\"'"
-			OUTPUT_VARIABLE DIST_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE)
-		execute_process(COMMAND uname -p
-			OUTPUT_VARIABLE ARCH OUTPUT_STRIP_TRAILING_WHITESPACE)
-		set(PACKAGE_FILE_NAME
-			${PROJECT_NAME}_${PROJECT_VERSION}-${DIST_NAME}-${DIST_VERSION}-${ARCH})
+execute_process(COMMAND uname -m
+	OUTPUT_VARIABLE ARCH OUTPUT_STRIP_TRAILING_WHITESPACE)

-		set(CPACK_DEBIAN_FILE_NAME		${PACKAGE_FILE_NAME}.deb)
-		set(CPACK_DEBIAN_PACKAGE_DEPENDS	"libssh-4")
-		set(CPACK_DEBIAN_PACKAGE_HOMEPAGE	"https://github.com/upa/mscp")
+if(UNIX AND NOT APPLE) # on linux
+	execute_process(COMMAND
+		bash "-c" "cat /etc/os-release|grep '^ID='|cut -d '=' -f 2|tr -d '\"'"
+		OUTPUT_VARIABLE DIST_NAME OUTPUT_STRIP_TRAILING_WHITESPACE)
+	execute_process(COMMAND
+		bash "-c" "cat /etc/os-release|grep '^VERSION_ID='|cut -d '=' -f 2|tr -d '\"'"
+		OUTPUT_VARIABLE DIST_VER OUTPUT_STRIP_TRAILING_WHITESPACE)
+	execute_process(COMMAND
+		bash "-c" "${mscp_SOURCE_DIR}/scripts/print-install-deps.sh ${DIST_NAME}-${DIST_VER}"
+		OUTPUT_VARIABLE DIST_DEP OUTPUT_STRIP_TRAILING_WHITESPACE)

-		set(CPACK_RPM_FILE_NAME			${PACKAGE_FILE_NAME}.rpm)
-		set(CPACK_RPM_PACKAGE_REQUIRES		"libssh")
-		set(CPACK_RPM_PACKAGE_HOMEPAGE		"https://github.com/upa/mscp")
-		set(CPACK_RPM_PACKAGE_DESCRIPTION	${CPACK_PACKAGE_DESCRIPTION})
+	set(PACKAGE_FILE_NAME
+		${PROJECT_NAME}_${PROJECT_VERSION}-${DIST_NAME}-${DIST_VER}-${ARCH})

-	endif() # on linux
+	set(CPACK_DEBIAN_FILE_NAME		${PACKAGE_FILE_NAME}.deb)
+	set(CPACK_DEBIAN_PACKAGE_DEPENDS	${DIST_DEP})
+	set(CPACK_DEBIAN_PACKAGE_HOMEPAGE	"https://github.com/upa/mscp")

-	include(CPack)
+	set(CPACK_RPM_FILE_NAME			${PACKAGE_FILE_NAME}.rpm)
+	set(CPACK_RPM_PACKAGE_REQUIRES		${DIST_DEP})
+	set(CPACK_RPM_PACKAGE_HOMEPAGE		"https://github.com/upa/mscp")
+	set(CPACK_RPM_PACKAGE_DESCRIPTION	${CPACK_PACKAGE_DESCRIPTION})
+endif() # on linux
+
+include(CPack)


-	# Custom commands to build mscp in docker containers

-	# Ubuntu 20.04
-	add_custom_target(package-ubuntu-20.04-in-docker
-		COMMENT "Build mscp in ubuntu 20.04 docker container"
-		WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-		COMMAND docker build -t mscp-ubuntu:20.04 -f docker/ubuntu-20.04.Dockerfile .
-		COMMAND docker run --init --rm mscp-ubuntu:20.04
-			/mscp/scripts/test-in-container.sh
-		COMMAND docker run --rm -v ${CMAKE_BINARY_DIR}:/out mscp-ubuntu:20.04
-			cp /mscp/build/mscp_${PROJECT_VERSION}-ubuntu-20.04-${ARCH}.deb /out/)
+# Custom targets to build and test mscp in docker containers.
+# foreach(IN ZIP_LISTS) (cmake >= 3.17) can shorten the following lists.
+# However, ubuntu 20.04 has cmake 3.16.3. So this is a roundabout trick.
+list(APPEND DIST_NAMES	ubuntu	ubuntu	centos	rocky)
+list(APPEND DIST_VERS	20.04	22.04	8	8.6)
+list(APPEND DIST_PKGS	deb	deb	rpm	rpm)

-	# Ubuntu 22.04
-	add_custom_target(package-ubuntu-22.04-in-docker
-		COMMENT "Build mscp in ubuntu 22.04 docker container"
-		WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-		COMMAND docker build -t mscp-ubuntu:22.04 -f docker/ubuntu-22.04.Dockerfile .
-		COMMAND docker run --init --rm mscp-ubuntu:22.04
-			/mscp/scripts/test-in-container.sh
-		COMMAND docker run --rm -v ${CMAKE_BINARY_DIR}:/out mscp-ubuntu:22.04
-		cp /mscp/build/mscp_${PROJECT_VERSION}-ubuntu-22.04-${ARCH}.deb /out/)
+list(LENGTH DIST_NAMES _DIST_LISTLEN)
+math(EXPR DIST_LISTLEN "${_DIST_LISTLEN} - 1")

-	# CentOS 8
-	add_custom_target(package-centos-8-in-docker
-		COMMENT "Build mscp in centos 8 docker container"
-		WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-		COMMAND docker build -t mscp-centos:8 -f docker/centos-8.Dockerfile .
-		COMMAND docker run --init --rm mscp-centos:8
-			/mscp/scripts/test-in-container.sh
-		COMMAND docker run --rm -v ${CMAKE_BINARY_DIR}:/out mscp-centos:8
-		cp /mscp/build/mscp_${PROJECT_VERSION}-centos-8-${ARCH}.rpm /out/)
+foreach(x RANGE ${DIST_LISTLEN})
+	list(GET DIST_NAMES	${x} DIST_NAME)
+	list(GET DIST_VERS	${x} DIST_VER)
+	list(GET DIST_PKGS	${x} DIST_PKG)

-	# build on all conatiners
-	add_custom_target(package-all-in-docker
-		COMMENT "Build mscp in all docker containers"
-		DEPENDS package-ubuntu-20.04-in-docker
-		DEPENDS package-ubuntu-22.04-in-docker
-		DEPENDS package-centos-8-in-docker)
+	set(DOCKER_IMAGE mscp-${DIST_NAME}:${DIST_VER})
+	set(DOCKER_INDEX ${DIST_NAME}-${DIST_VER})
+	set(PKG_FILE_NAME
+		mscp_${PROJECT_VERSION}-${DIST_NAME}-${DIST_VER}-${ARCH}.${DIST_PKG})

-endif() # BUILD_PKG
+	add_custom_target(docker-build-${DOCKER_INDEX}
+		COMMENT "Build mscp in ${DOCKER_IMAGE} container"
+		WORKING_DIRECTORY ${mscp_SOURCE_DIR}
+		COMMAND
+		docker build -t ${DOCKER_IMAGE} -f docker/${DOCKER_INDEX}.Dockerfile .)

+	add_custom_target(docker-test-${DOCKER_INDEX}
+		COMMENT "Test mscp in ${DOCKER_IMAGE} container"
+		WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+		COMMAND
+		docker run --init --rm ${DOCKER_IMAGE} /mscp/scripts/test-in-container.sh)
+
+	add_custom_target(docker-pkg-${DOCKER_INDEX}
+		COMMENT "Retrieve mscp package from ${DOCKER_IMAGE} container"
+		WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+		COMMAND
+		docker run --rm -v ${CMAKE_BINARY_DIR}:/out ${DOCKER_IMAGE}
+		cp /mscp/build/${PKG_FILE_NAME} /out/)
+
+	list(APPEND DOCKER_BUILDS	docker-build-${DOCKER_INDEX})
+	list(APPEND DOCKER_TESTS	docker-test-${DOCKER_INDEX})
+	list(APPEND DOCKER_PKGS		docker-pkg-${DOCKER_INDEX})
+endforeach()
+
+add_custom_target(docker-build-all	DEPENDS ${DOCKER_BUILDS})
+add_custom_target(docker-test-all	DEPENDS ${DOCKER_TESTS})
+add_custom_target(docker-pkg-all	DEPENDS ${DOCKER_PKGS})
--- a/README.md
+++ b/README.md
@@ -3,53 +3,71 @@
 [![build on ubuntu](https://github.com/upa/mscp/actions/workflows/build-ubuntu.yml/badge.svg)](https://github.com/upa/mscp/actions/workflows/build-ubuntu.yml) [![build on macOS](https://github.com/upa/mscp/actions/workflows/build-macos.yml/badge.svg)](https://github.com/upa/mscp/actions/workflows/build-macos.yml) [![test](https://github.com/upa/mscp/actions/workflows/test.yml/badge.svg)](https://github.com/upa/mscp/actions/workflows/test.yml)


-`mscp`, a variant of `scp`, copies files over multiple ssh (sftp)
-sessions. Multiple threads in mscp transfer (1) multiple files
-simultaneously and (2) a large file in parallel. It may shorten the
-waiting time for transferring a lot of/large files over networks.
+`mscp`, a variant of `scp`, copies files over multiple ssh (SFTP)
+connections. Multiple threads and connections in mscp transfer (1)
+multiple files simultaneously and (2) a large file in parallel. It
+would shorten the waiting time for transferring a lot of/large files
+over networks.

-You can use `mscp` like `scp`, e.g., `mscp example.com:srcfile
-/tmp/dstfile`. Remote hosts only need to run `sshd` supporting the
-SFTP subsystem, and you need to be able to ssh to the hosts (as
-usual).
+You can use `mscp` like `scp`, for example, `mscp
+user@example.com:srcfile /tmp/dstfile`. Remote hosts only need to run
+standard `sshd` supporting the SFTP subsystem (e.g. openssh-server),
+and you need to be able to ssh to the hosts as usual. `mscp` does not
+require anything else.

-Differences from `scp` are:
+
+Differences from `scp` on usage:

 - remote glob on remote shell expansion is not supported.
 - remote to remote copy is not supported.
 - `-r` option is not needed.
- and any other differences I have not noticed and implemented...
+- and any other differences I have not implemented and noticed.
+
+
+## Install
+
+- homebrew
+
+```console
+brew install upa/tap/mscp
+```
+
+- Linux
+
+Download a package for your environment from [Releases
+page](https://github.com/upa/mscp/releases).
+

 ## Build

-mscp depends on [libssh](https://www.libssh.org/).
+mscp depends on a patched [libssh](https://www.libssh.org/).  The
+patch introduces asynchronous SFTP Write, which is derived from
+https://github.com/limes-datentechnik-gmbh/libssh (see [Re: SFTP Write
+async](https://archive.libssh.org/libssh/2020-06/0000004.html)).

- macOS
-
-```console
-brew install libssh
-```
-
- ubuntu
-
-```console
-sudo apt-get install libssh-dev
-```
-
- rhel
-
-```console
-sudo yum install libssh-devel
-```
-
-Clone and build this repositoy.
+Currently macOS and Linux (Ubuntu, CentOS, Rocky) are supported.

 ```console
+# clone this repository
 git clone https://github.com/upa/mscp.git
 cd mscp

-mkdir build && cd build
-cmake .. && make
+# prepare patched libssh
+git submodule update --init
+patch -d libssh -p1 < patch/libssh-0.10.4.patch
+
+# install build dependency
+bash ./scripts/install-build-deps.sh
+
+# configure mscp
+mkdir build && mv build
+cmake ..
+
+## in macOS, you may need OPENSSL_ROOT_DIR for cmake:
+cmake .. -DOPENSSL_ROOT_DIR=$(brew --prefix)/opt/openssl@1.1
+
+# build
+make

 # install the mscp binary to CMAKE_INSTALL_PREFIX/bin (usually /usr/local/bin)
 make install
@@ -59,80 +77,96 @@ make install

 - Usage

-```shell-session
+```console
 $ mscp
-mscp v0.0.0: copy files over multiple ssh connections
+mscp v0.0.5: copy files over multiple ssh connections

-Usage: mscp [vqDCHdh] [-n nr_conns]
-            [-s min_chunk_sz] [-S max_chunk_sz]
-            [-b sftp_buf_sz] [-B io_buf_sz]
+Usage: mscp [vqDCHdNh] [-n nr_conns] [-m coremask]
+            [-s min_chunk_sz] [-S max_chunk_sz] [-a nr_ahead] [-b buf_sz]
            [-l login_name] [-p port] [-i identity_file]
-            [-c cipher_spec] source ... target
+            [-c cipher_spec] [-M hmac_spec] source ... target
+```

-    -n NR_CONNECTIONS  number of connections (default: half of # of cpu cores)
+- Example: copy a 15GB file on memory over a 100Gbps link
+  - Two Intel Xeon Gold 6130 machines directly connected with Intel E810 100Gbps NICs.
+  - Default `openssh-server` runs on the remote host.
+
+```console
+$ mscp /var/ram/test.img 10.0.0.1:/var/ram/
+[======================================] 100%   15GB/15GB    1.7GB/s  00:00 ETA
+```
+
+```console
+# with some optimizations. top speed reaches 3.0GB/s.
+$ mscp -n 5 -m 0x1f -c aes128-gcm@openssh.com /var/ram/test.img 10.0.0.1:/var/ram/
+[======================================] 100%   15GB/15GB    2.4GB/s  00:00 ETA
+```
+
+- `-v` option increments verbose output level.
+
+```console
+$ mscp test 10.0.0.1:
+[======================================] 100%   26B /26B     6.3KB/s  00:00 ETA
+```
+
+```console
+$ mscp -vv test 10.0.0.1:
+number of connections: 7
+connecting to 10.0.0.1 for checking destinations...
+file test/testdir/asdf (local) -> ./test/testdir/asdf (remote) 9B
+file test/testdir/qwer (local) -> ./test/testdir/qwer (remote) 5B
+file test/test1 (local) -> ./test/test1 (remote) 6B
+file test/test2 (local) -> ./test/test2 (remote) 6B
+we have only 4 chunk(s). set number of connections to 4
+connecting to 10.0.0.1 for a copy thread...
+connecting to 10.0.0.1 for a copy thread...
+connecting to 10.0.0.1 for a copy thread...
+copy start: test/test1
+copy start: test/test2
+copy done: test/test1
+copy start: test/testdir/asdf
+copy done: test/test2
+copy start: test/testdir/qwer
+copy done: test/testdir/qwer
+copy done: test/testdir/asdf
+[======================================] 100%   26B /26B     5.2KB/s  00:00 ETA
+```
+
+- Full usage
+
+```console
+$ mscp -h
+mscp v0.0.5: copy files over multiple ssh connections
+
+Usage: mscp [vqDCHdNh] [-n nr_conns] [-m coremask]
+            [-s min_chunk_sz] [-S max_chunk_sz] [-a nr_ahead] [-b buf_sz]
+            [-l login_name] [-p port] [-i identity_file]
+            [-c cipher_spec] [-M hmac_spec] source ... target
+
+    -n NR_CONNECTIONS  number of connections (default: floor(log(cores)*2)+1)
+    -m COREMASK        hex value to specify cores where threads pinned
    -s MIN_CHUNK_SIZE  min chunk size (default: 64MB)
-    -S MAX_CHUNK_SIZE  max chunk size (default: filesize / nr_conn)
-    -b SFTP_BUF_SIZE   buf size for sftp_read/write (default 131072B)
-    -B IO_BUF_SIZE     buf size for read/write (default 131072B)
-                       Note that this value is derived from
-                       qemu/block/ssh.c. need investigation...
+    -S MAX_CHUNK_SIZE  max chunk size (default: filesize/nr_conn)
+
+    -a NR_AHEAD        number of inflight SFTP commands (default: 32)
+    -b BUF_SZ          buffer size for i/o and transfer
+
    -v                 increment verbose output level
    -q                 disable output
    -D                 dry run

    -l LOGIN_NAME      login name
    -p PORT            port number
-    -i IDENTITY        identity file for publickey authentication
-    -c CIPHER          cipher spec, see `ssh -Q cipher`
+    -i IDENTITY        identity file for public key authentication
+    -c CIPHER          cipher spec
+    -M HMAC            hmac spec
    -C                 enable compression on libssh
    -H                 disable hostkey check
    -d                 increment ssh debug output level
+    -N                 disable tcp nodelay (default on)
    -h                 print this help
 ```

- Example: copy an 8GB file on tmpfs over a 100Gbps link
-  - Two Intel Xeon Gold 6130 machines directly connected with Intel E810 100Gbps NICs.
-
-```shell-session
-$ mscp /tmp/test.img 10.0.0.1:/tmp/
-[===============================================================] 100% 8GB/8GB 3.02GB/s 
-```
-
- `-v` options increment verbose output level.
-
-```shell-session
-$ mscp test 10.0.0.1:
-[===============================================================] 100% 13B/13B 2.41KB/s 
-
-$ mscp -v test 10.0.0.1:
-file test/test.txt (local) -> ./test/test.txt (remote) 9B
-file test/test2/2.txt (local) -> ./test/test2/2.txt (remote) 2B
-file test/1.txt (local) -> ./test/1.txt (remote) 2B
-copy start: test/test.txt
-copy start: test/1.txt
-copy start: test/test2/2.txt
-copy done: test/1.txt
-copy done: test/test2/2.txt
-copy done: test/test.txt
-[===============================================================] 100% 13B/13B 2.51KB/s 
-
-$ mscp -vv -n 4 test 10.0.0.1:
-connecting to 10.0.0.1 for checking destinations...
-file test/test.txt (local) -> ./test/test.txt (remote) 9B
-file test/test2/2.txt (local) -> ./test/test2/2.txt (remote) 2B
-file test/1.txt (local) -> ./test/1.txt (remote) 2B
-connecting to 10.0.0.1 for a copy thread...
-connecting to 10.0.0.1 for a copy thread...
-connecting to 10.0.0.1 for a copy thread...
-connecting to 10.0.0.1 for a copy thread...
-copy start: test/test.txt
-copy start: test/1.txt
-copy start: test/test2/2.txt
-copy done: test/test.txt
-copy done: test/test2/2.txt
-copy done: test/1.txt
-[===============================================================] 100% 13B/13B 3.27KB/s
-```

 Note: mscp is still under development, and the author is not
-responsible for any accidents on mscp.
+responsible for any accidents due to mscp.
--- a/2
+++ b/2
@@ -1 +1 @@
-0.0.0
+0.0.5
--- a/cmake/modules/FindGSSAPI.cmake
+++ b/cmake/modules/FindGSSAPI.cmake
@@ -0,0 +1,325 @@
+# - Try to find GSSAPI
+# Once done this will define
+#
+#  KRB5_CONFIG - Path to krb5-config
+#  GSSAPI_ROOT_DIR - Set this variable to the root installation of GSSAPI
+#
+# Read-Only variables:
+#  GSSAPI_FLAVOR_MIT - set to TURE if MIT Kerberos has been found
+#  GSSAPI_FLAVOR_HEIMDAL - set to TRUE if Heimdal Keberos has been found
+#  GSSAPI_FOUND - system has GSSAPI
+#  GSSAPI_INCLUDE_DIR - the GSSAPI include directory
+#  GSSAPI_LIBRARIES - Link these to use GSSAPI
+#  GSSAPI_DEFINITIONS - Compiler switches required for using GSSAPI
+#
+#=============================================================================
+#  Copyright (c) 2013 Andreas Schneider <asn@cryptomilk.org>
+#
+#  Distributed under the OSI-approved BSD License (the "License");
+#  see accompanying file Copyright.txt for details.
+#
+#  This software is distributed WITHOUT ANY WARRANTY; without even the
+#  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#  See the License for more information.
+#=============================================================================
+#
+
+find_path(GSSAPI_ROOT_DIR
+    NAMES
+        include/gssapi.h
+        include/gssapi/gssapi.h
+    HINTS
+        ${_GSSAPI_ROOT_HINTS}
+    PATHS
+        ${_GSSAPI_ROOT_PATHS}
+)
+mark_as_advanced(GSSAPI_ROOT_DIR)
+
+if (UNIX)
+    find_program(KRB5_CONFIG
+        NAMES
+            krb5-config
+        PATHS
+            ${GSSAPI_ROOT_DIR}/bin
+            /opt/local/bin)
+    mark_as_advanced(KRB5_CONFIG)
+
+    if (KRB5_CONFIG)
+        # Check if we have MIT KRB5
+        execute_process(
+            COMMAND
+                ${KRB5_CONFIG} --vendor
+            RESULT_VARIABLE
+                _GSSAPI_VENDOR_RESULT
+            OUTPUT_VARIABLE
+                _GSSAPI_VENDOR_STRING)
+
+        if ((_GSSAPI_VENDOR_STRING MATCHES ".*Massachusetts.*") OR (_GSSAPI_VENDOR_STRING
+                        MATCHES ".*MITKerberosShim.*"))
+            set(GSSAPI_FLAVOR_MIT TRUE)
+        else()
+            execute_process(
+                COMMAND
+                    ${KRB5_CONFIG} --libs gssapi
+                RESULT_VARIABLE
+                    _GSSAPI_LIBS_RESULT
+                OUTPUT_VARIABLE
+                    _GSSAPI_LIBS_STRING)
+
+            if (_GSSAPI_LIBS_STRING MATCHES ".*roken.*")
+                set(GSSAPI_FLAVOR_HEIMDAL TRUE)
+            endif()
+        endif()
+
+        # Get the include dir
+        execute_process(
+            COMMAND
+                ${KRB5_CONFIG} --cflags gssapi
+            RESULT_VARIABLE
+                _GSSAPI_INCLUDE_RESULT
+            OUTPUT_VARIABLE
+                _GSSAPI_INCLUDE_STRING)
+        string(REGEX REPLACE "(\r?\n)+$" "" _GSSAPI_INCLUDE_STRING "${_GSSAPI_INCLUDE_STRING}")
+        string(REGEX REPLACE " *-I" "" _GSSAPI_INCLUDEDIR "${_GSSAPI_INCLUDE_STRING}")
+    endif()
+
+    if (NOT GSSAPI_FLAVOR_MIT AND NOT GSSAPI_FLAVOR_HEIMDAL)
+        # Check for HEIMDAL
+        find_package(PkgConfig)
+        if (PKG_CONFIG_FOUND)
+            pkg_check_modules(_GSSAPI heimdal-gssapi)
+        endif (PKG_CONFIG_FOUND)
+
+        if (_GSSAPI_FOUND)
+            set(GSSAPI_FLAVOR_HEIMDAL TRUE)
+        else()
+            find_path(_GSSAPI_ROKEN
+                NAMES
+                    roken.h
+                PATHS
+                    ${GSSAPI_ROOT_DIR}/include
+                    ${_GSSAPI_INCLUDEDIR})
+            if (_GSSAPI_ROKEN)
+                set(GSSAPI_FLAVOR_HEIMDAL TRUE)
+            endif()
+        endif ()
+    endif()
+endif (UNIX)
+
+find_path(GSSAPI_INCLUDE_DIR
+    NAMES
+        gssapi.h
+        gssapi/gssapi.h
+    PATHS
+        ${GSSAPI_ROOT_DIR}/include
+        ${_GSSAPI_INCLUDEDIR}
+)
+
+if (GSSAPI_FLAVOR_MIT)
+    find_library(GSSAPI_LIBRARY
+        NAMES
+            gssapi_krb5
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(KRB5_LIBRARY
+        NAMES
+            krb5
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(K5CRYPTO_LIBRARY
+        NAMES
+            k5crypto
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(COM_ERR_LIBRARY
+        NAMES
+            com_err
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    if (GSSAPI_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${GSSAPI_LIBRARY}
+        )
+    endif (GSSAPI_LIBRARY)
+
+    if (KRB5_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${KRB5_LIBRARY}
+        )
+    endif (KRB5_LIBRARY)
+
+    if (K5CRYPTO_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${K5CRYPTO_LIBRARY}
+        )
+    endif (K5CRYPTO_LIBRARY)
+
+    if (COM_ERR_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${COM_ERR_LIBRARY}
+        )
+    endif (COM_ERR_LIBRARY)
+endif (GSSAPI_FLAVOR_MIT)
+
+if (GSSAPI_FLAVOR_HEIMDAL)
+    find_library(GSSAPI_LIBRARY
+        NAMES
+            gssapi
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(KRB5_LIBRARY
+        NAMES
+            krb5
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(HCRYPTO_LIBRARY
+        NAMES
+            hcrypto
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(COM_ERR_LIBRARY
+        NAMES
+            com_err
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(HEIMNTLM_LIBRARY
+        NAMES
+            heimntlm
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(HX509_LIBRARY
+        NAMES
+            hx509
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(ASN1_LIBRARY
+        NAMES
+            asn1
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(WIND_LIBRARY
+        NAMES
+            wind
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(ROKEN_LIBRARY
+        NAMES
+            roken
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    if (GSSAPI_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${GSSAPI_LIBRARY}
+        )
+    endif (GSSAPI_LIBRARY)
+
+    if (KRB5_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${KRB5_LIBRARY}
+        )
+    endif (KRB5_LIBRARY)
+
+    if (HCRYPTO_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${HCRYPTO_LIBRARY}
+        )
+    endif (HCRYPTO_LIBRARY)
+
+    if (COM_ERR_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${COM_ERR_LIBRARY}
+        )
+    endif (COM_ERR_LIBRARY)
+
+    if (HEIMNTLM_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${HEIMNTLM_LIBRARY}
+        )
+    endif (HEIMNTLM_LIBRARY)
+
+    if (HX509_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${HX509_LIBRARY}
+        )
+    endif (HX509_LIBRARY)
+
+    if (ASN1_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${ASN1_LIBRARY}
+        )
+    endif (ASN1_LIBRARY)
+
+    if (WIND_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${WIND_LIBRARY}
+        )
+    endif (WIND_LIBRARY)
+
+    if (ROKEN_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${WIND_LIBRARY}
+        )
+    endif (ROKEN_LIBRARY)
+endif (GSSAPI_FLAVOR_HEIMDAL)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(GSSAPI DEFAULT_MSG GSSAPI_LIBRARIES GSSAPI_INCLUDE_DIR)
+
+if (GSSAPI_INCLUDE_DIRS AND GSSAPI_LIBRARIES)
+    set(GSSAPI_FOUND TRUE)
+endif (GSSAPI_INCLUDE_DIRS AND GSSAPI_LIBRARIES)
+
+# show the GSSAPI_INCLUDE_DIRS and GSSAPI_LIBRARIES variables only in the advanced view
+mark_as_advanced(GSSAPI_INCLUDE_DIRS GSSAPI_LIBRARIES)
--- a/docker/README.md
+++ b/docker/README.md
@@ -6,19 +6,23 @@ cd ..

 docker build -t mscp-ubuntu:20.04 -f docker/ubuntu-20.04.Dockerfile .

-docker build -t mscp-ubuntu:22.04 -f docker/Dockerfile-ubuntu-22.04 .
+docker build -t mscp-ubuntu:22.04 -f docker/ubuntu-22.04.Dockerfile .

-docker build -t mscp-centos:8 -f docker/Dockerfile-centos-8 .
+docker build -t mscp-centos:8 -f docker/centos-8.Dockerfile .
+
+docker build -t mscp-rocky:8.6 -f docker/rocky-8.6.Dockerfile .
 ```

 Test `mscp` in the containers.

 ```console
-docker run --init --rm mscp-ubuntu:20.04 /build/mscp/scripts/test-in-container.sh
+docker run --init --rm mscp-ubuntu:20.04 /mscp/scripts/test-in-container.sh

-docker run --init --rm mscp-ubuntu:22.04 /build/mscp/scripts/test-in-container.sh
+docker run --init --rm mscp-ubuntu:22.04 /mscp/scripts/test-in-container.sh

-docker run --init --rm mscp-centos:8 /build/mscp/scripts/test-in-container.sh
+docker run --init --rm mscp-centos:8 /mscp/scripts/test-in-container.sh
+
+docker run --init --rm mscp-rocky:8.6 /mscp/scripts/test-in-container.sh
 ```

 Retrieve deb/rpm packages.
@@ -32,6 +36,9 @@ docker run --rm -v (pwd):/out mscp-ubuntu:22.04 \

 docker run --rm -v (pwd):/out mscp-centos:8 \
       cp /mscp/build/mscp_0.0.0-centos-8-x86_64.rpm /out/
+
+docker run --rm -v (pwd):/out mscp-rocky:8.6 \
+       cp /mscp/build/mscp_0.0.0-rocky-8.6-x86_64.rpm /out/
 ```

 I don't know whether these are good way.
--- a/docker/centos-8.Dockerfile
+++ b/docker/centos-8.Dockerfile
@@ -28,8 +28,9 @@ RUN ${mscpdir}/scripts/install-build-deps.sh
 # build
 RUN cd ${mscpdir}			\
        && rm -rf build			\
-        && cmake -B build -DBUILD_PKG=1 \
+        && cmake -B build 		\
        && cd ${mscpdir}/build          \
 	&& make				\
-	&& cpack -G RPM CPackConfig.cmake
+	&& cpack -G RPM CPackConfig.cmake \
+	&& rpm -iv *.rpm

--- a/docker/rocky-8.6.Dockerfile
+++ b/docker/rocky-8.6.Dockerfile
@@ -0,0 +1,31 @@
+FROM rockylinux:8.6
+
+ARG mscpdir="/mscp"
+
+COPY . ${mscpdir}
+
+# install numpy and pytest, sshd for test, and rpm-build
+RUN set -ex && yum -y install \
+	python3 python3-pip openssh openssh-server openssh-clients rpm-build
+
+RUN python3 -m pip install numpy pytest
+
+
+# preparation for sshd
+RUN  mkdir /var/run/sshd        \
+	&& ssh-keygen -A	\
+        && ssh-keygen -f /root/.ssh/id_rsa -N ""                \
+        && mv /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
+
+# install build dependency
+RUN ${mscpdir}/scripts/install-build-deps.sh
+
+# build
+RUN cd ${mscpdir}			\
+        && rm -rf build			\
+        && cmake -B build		\
+        && cd ${mscpdir}/build          \
+	&& make				\
+	&& cpack -G RPM CPackConfig.cmake \
+	&& rpm -iv *.rpm
+
--- a/docker/ubuntu-20.04.Dockerfile
+++ b/docker/ubuntu-20.04.Dockerfile
@@ -28,7 +28,8 @@ RUN ${mscpdir}/scripts/install-build-deps.sh
 # build
 RUN cd ${mscpdir}			\
 	&& rm -rf build			\
-	&& cmake -B build -DBUILD_PKG=1	\
+	&& cmake -B build		\
 	&& cd ${mscpdir}/build		\
 	&& make				\
-	&& cpack -G DEB CPackConfig.cmake 
+	&& cpack -G DEB CPackConfig.cmake \
+	&& dpkg -i *.deb
--- a/docker/ubuntu-22.04.Dockerfile
+++ b/docker/ubuntu-22.04.Dockerfile
@@ -28,7 +28,8 @@ RUN ${mscpdir}/scripts/install-build-deps.sh
 # build
 RUN cd ${mscpdir}			\
 	&& rm -rf build			\
-	&& cmake -B build -DBUILD_PKG=1	\
+	&& cmake -B build		\
 	&& cd ${mscpdir}/build		\
 	&& make				\
-	&& cpack -G DEB CPackConfig.cmake 
+	&& cpack -G DEB CPackConfig.cmake \
+	&& dpkg -i *.deb
--- a/1
+++ b/1
--- a/patch/README.md
+++ b/patch/README.md
@@ -0,0 +1,13 @@
+
+Patch(es) in this directory introduces `sftp_async_write()` and
+`sftp_async_write_end()` to libssh. Those implementations are derived
+from https://github.com/limes-datentechnik-gmbh/libssh. See [Re: SFTP
+Write async](https://archive.libssh.org/libssh/2020-06/0000004.html).
+
+```console
+git clone https://git.libssh.org/projects/libssh.git/ --depth=1 -b libssh-0.10.4
+cd libssh
+git apply ../pathc/libssh-0.10.4.patch
+
+# then build libssh
+```
--- a/patch/libssh-0.10.4.patch
+++ b/patch/libssh-0.10.4.patch
@@ -0,0 +1,342 @@
+diff --git a/DefineOptions.cmake b/DefineOptions.cmake
+index 068db988..5fc3c8fc 100644
+--- a/DefineOptions.cmake
+++ b/DefineOptions.cmake
+@@ -1,7 +1,7 @@
+ option(WITH_GSSAPI "Build with GSSAPI support" ON)
+ option(WITH_ZLIB "Build with ZLIB support" ON)
+ option(WITH_SFTP "Build with SFTP support" ON)
+-option(WITH_SERVER "Build with SSH server support" ON)
+option(WITH_SERVER "Build with SSH server support" OFF)
+ option(WITH_DEBUG_CRYPTO "Build with cryto debug output" OFF)
+ option(WITH_DEBUG_PACKET "Build with packet debug output" OFF)
+ option(WITH_DEBUG_CALLTRACE "Build with calltrace debug output" ON)
+@@ -11,13 +11,13 @@ option(WITH_MBEDTLS "Compile against libmbedtls" OFF)
+ option(WITH_BLOWFISH_CIPHER "Compile with blowfish support" OFF)
+ option(WITH_PCAP "Compile with Pcap generation support" ON)
+ option(WITH_INTERNAL_DOC "Compile doxygen internal documentation" OFF)
+-option(BUILD_SHARED_LIBS "Build shared libraries" ON)
+option(BUILD_SHARED_LIBS "Build shared libraries" OFF)
+ option(WITH_PKCS11_URI "Build with PKCS#11 URI support" OFF)
+ option(UNIT_TESTING "Build with unit tests" OFF)
+ option(CLIENT_TESTING "Build with client tests; requires openssh" OFF)
+ option(SERVER_TESTING "Build with server tests; requires openssh and dropbear" OFF)
+ option(WITH_BENCHMARKS "Build benchmarks tools" OFF)
+-option(WITH_EXAMPLES "Build examples" ON)
+option(WITH_EXAMPLES "Build examples" OFF)
+ option(WITH_NACL "Build with libnacl (curve25519)" ON)
+ option(WITH_SYMBOL_VERSIONING "Build with symbol versioning" ON)
+ option(WITH_ABI_BREAK "Allow ABI break" OFF)
+@@ -25,6 +25,7 @@ option(WITH_GEX "Enable DH Group exchange mechanisms" ON)
+ option(WITH_INSECURE_NONE "Enable insecure none cipher and MAC algorithms (not suitable for production!)" OFF)
+ option(FUZZ_TESTING "Build with fuzzer for the server and client (automatically enables none cipher!)" OFF)
+ option(PICKY_DEVELOPER "Build with picky developer flags" OFF)
+option(WITH_STATIC_LIB "Build static library" ON)
+ 
+ if (WITH_ZLIB)
+     set(WITH_LIBZ ON)
+@@ -60,3 +61,7 @@ endif (NOT GLOBAL_CLIENT_CONFIG)
+ if (FUZZ_TESTING)
+   set(WITH_INSECURE_NONE ON)
+ endif (FUZZ_TESTING)
+
+if (WITH_STATIC_LIB)
+  set(BUILD_STATIC_LIB ON)
+endif()
+diff --git a/include/libssh/buffer.h b/include/libssh/buffer.h
+index a55a1b40..e34e075c 100644
+--- a/include/libssh/buffer.h
+++ b/include/libssh/buffer.h
+@@ -33,6 +33,8 @@ int ssh_buffer_add_u8(ssh_buffer buffer, uint8_t data);
+ int ssh_buffer_add_u16(ssh_buffer buffer, uint16_t data);
+ int ssh_buffer_add_u32(ssh_buffer buffer, uint32_t data);
+ int ssh_buffer_add_u64(ssh_buffer buffer, uint64_t data);
+ssize_t ssh_buffer_add_func(ssh_buffer buffer, ssh_add_func f, size_t max_bytes,
+			    void *userdata);
+ 
+ int ssh_buffer_validate_length(struct ssh_buffer_struct *buffer, size_t len);
+ 
+diff --git a/include/libssh/libssh.h b/include/libssh/libssh.h
+index 7857a77b..3eef7a16 100644
+--- a/include/libssh/libssh.h
+++ b/include/libssh/libssh.h
+@@ -833,6 +833,7 @@ LIBSSH_API const char* ssh_get_hmac_in(ssh_session session);
+ LIBSSH_API const char* ssh_get_hmac_out(ssh_session session);
+ 
+ LIBSSH_API ssh_buffer ssh_buffer_new(void);
+LIBSSH_API ssh_buffer ssh_buffer_new_size(uint32_t size, uint32_t headroom);
+ LIBSSH_API void ssh_buffer_free(ssh_buffer buffer);
+ #define SSH_BUFFER_FREE(x) \
+     do { if ((x) != NULL) { ssh_buffer_free(x); x = NULL; } } while(0)
+@@ -843,6 +844,8 @@ LIBSSH_API void *ssh_buffer_get(ssh_buffer buffer);
+ LIBSSH_API uint32_t ssh_buffer_get_len(ssh_buffer buffer);
+ LIBSSH_API int ssh_session_set_disconnect_message(ssh_session session, const char *message);
+ 
+typedef ssize_t (*ssh_add_func) (void *ptr, size_t max_bytes, void *userdata);
+
+ #ifndef LIBSSH_LEGACY_0_4
+ #include "libssh/legacy.h"
+ #endif
+diff --git a/include/libssh/sftp.h b/include/libssh/sftp.h
+index c855df8a..0fcdb9b8 100644
+--- a/include/libssh/sftp.h
+++ b/include/libssh/sftp.h
+@@ -565,6 +565,10 @@ LIBSSH_API int sftp_async_read(sftp_file file, void *data, uint32_t len, uint32_
+  */
+ LIBSSH_API ssize_t sftp_write(sftp_file file, const void *buf, size_t count);
+ 
+LIBSSH_API ssize_t sftp_async_write(sftp_file file, ssh_add_func f, size_t count,
+				    void *userdata, uint32_t* id);
+LIBSSH_API int sftp_async_write_end(sftp_file file, uint32_t id, int blocking);
+
+ /**
+  * @brief Seek to a specific location in a file.
+  *
+diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
+index c090fef7..e2f86309 100644
+--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
+@@ -435,6 +435,11 @@ if (BUILD_STATIC_LIB)
+   if (WIN32)
+     target_compile_definitions(ssh-static PUBLIC "LIBSSH_STATIC")
+   endif (WIN32)
+
+  install(TARGETS ssh-static
+	  EXPORT libssh-config
+	  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+	  COMPONENT libraries)
+ endif (BUILD_STATIC_LIB)
+ 
+ message(STATUS "Threads_FOUND=${Threads_FOUND}")
+diff --git a/src/buffer.c b/src/buffer.c
+index e0068015..cc0caf35 100644
+--- a/src/buffer.c
+++ b/src/buffer.c
+@@ -141,6 +141,40 @@ struct ssh_buffer_struct *ssh_buffer_new(void)
+     return buf;
+ }
+ 
+/**
+ * @brief Create a new SSH buffer with a specified size and headroom.
+ *
+ * @param[in] len       length for newly initialized SSH buffer.
+ * @param[in] headroom  length for headroom
+ * @return A newly initialized SSH buffer, NULL on error.
+ */
+struct ssh_buffer_struct *ssh_buffer_new_size(uint32_t len, uint32_t headroom)
+{
+    struct ssh_buffer_struct *buf = NULL;
+    int rc;
+
+    if (len < headroom)
+	    return NULL;
+
+    buf = calloc(1, sizeof(struct ssh_buffer_struct));
+    if (buf == NULL) {
+        return NULL;
+    }
+
+    rc = ssh_buffer_allocate_size(buf, len);
+    if (rc != 0) {
+        SAFE_FREE(buf);
+        return NULL;
+    }
+
+    buf->pos += headroom;
+    buf->used += headroom;
+
+    buffer_verify(buf);
+
+    return buf;
+}
+
+ /**
+  * @brief Deallocate a SSH buffer.
+  *
+@@ -328,6 +362,49 @@ int ssh_buffer_add_data(struct ssh_buffer_struct *buffer, const void *data, uint
+     return 0;
+ }
+ 
+/**
+ * @brief Add data at the tail of a buffer by an external function
+ *
+ * @param[in]  buffer    The buffer to add data.
+ *
+ * @param[in]  f         function that adds data to the buffer.
+ *
+ * @param[in]  max_bytes The maximum length of the data to add.
+ *
+ * @return               actual bytes added on success, < 0 on error.
+ */
+ssize_t ssh_buffer_add_func(struct ssh_buffer_struct *buffer, ssh_add_func f,
+			    size_t max_bytes, void *userdata)
+{
+    ssize_t actual;
+
+    if (buffer == NULL) {
+        return -1;
+    }
+
+    buffer_verify(buffer);
+
+    if (buffer->used + max_bytes < max_bytes) {
+        return -1;
+    }
+
+    if (buffer->allocated < (buffer->used + max_bytes)) {
+        if (buffer->pos > 0) {
+            buffer_shift(buffer);
+        }
+        if (realloc_buffer(buffer, buffer->used + max_bytes) < 0) {
+            return -1;
+        }
+    }
+
+    if ((actual = f(buffer->data + buffer->used, max_bytes, userdata)) < 0)
+      return -1;
+
+    buffer->used += actual;
+    buffer_verify(buffer);
+    return actual;
+}
+
+ /**
+  * @brief Ensure the buffer has at least a certain preallocated size.
+  *
+diff --git a/src/sftp.c b/src/sftp.c
+index e01012a8..702623a0 100644
+--- a/src/sftp.c
+++ b/src/sftp.c
+@@ -2228,6 +2228,132 @@ ssize_t sftp_write(sftp_file file, const void *buf, size_t count) {
+   return -1; /* not reached */
+ }
+ 
+/*
+ * sftp_async_write is based on and sftp_async_write_end is copied from
+ * https://github.com/limes-datentechnik-gmbh/libssh
+ *
+ * sftp_async_write has some optimizations:
+ * - use ssh_buffer_new_size() to reduce realoc_buffer.
+ * - use ssh_buffer_add_func() to avoid memcpy from read buffer to ssh buffer.
+ */
+ssize_t sftp_async_write(sftp_file file, ssh_add_func f, size_t count, void *userdata,
+			 uint32_t* id) {
+  sftp_session sftp = file->sftp;
+  ssh_buffer buffer;
+  uint32_t buf_sz;
+  ssize_t actual;
+  int len;
+  int packetlen;
+  int rc;
+
+#define HEADROOM 16
+  /* sftp_packet_write() prepends a 5-bytes (uint32_t length and
+   * 1-byte type) header to the head of the payload by
+   * ssh_buffer_prepend_data(). Inserting headroom by
+   * ssh_buffer_new_size() eliminates memcpy for prepending the
+   * header.
+   */
+
+  buf_sz = (HEADROOM + /* for header */
+	    sizeof(uint32_t) + /* id */
+	    ssh_string_len(file->handle) + 4 + /* file->handle */
+	    sizeof(uint64_t) + /* file->offset */
+	    sizeof(uint32_t) + /* count */
+	    count); /* datastring */
+
+  buffer = ssh_buffer_new_size(buf_sz, HEADROOM);
+  if (buffer == NULL) {
+    ssh_set_error_oom(sftp->session);
+    return -1;
+  }
+
+  *id = sftp_get_new_id(file->sftp);
+
+  rc = ssh_buffer_pack(buffer,
+                       "dSqd",
+                       *id,
+                       file->handle,
+                       file->offset,
+                       count); /* len of datastring */
+
+  if (rc != SSH_OK){
+    ssh_set_error_oom(sftp->session);
+    ssh_buffer_free(buffer);
+    return SSH_ERROR;
+  }
+
+  actual = ssh_buffer_add_func(buffer, f, count, userdata);
+  if (actual < 0){
+    ssh_set_error_oom(sftp->session);
+    ssh_buffer_free(buffer);
+    return SSH_ERROR;
+  }
+
+  packetlen=ssh_buffer_get_len(buffer)+5;
+  len = sftp_packet_write(file->sftp, SSH_FXP_WRITE, buffer);
+  ssh_buffer_free(buffer);
+  if (len < 0) {
+    return SSH_ERROR;
+  } else  if (len != packetlen) {
+    ssh_set_error(sftp->session, SSH_FATAL,
+      "Could only send %d of %d bytes to remote host!", len, packetlen);
+    SSH_LOG(SSH_LOG_PACKET,
+        "Could not write as much data as expected");
+    return SSH_ERROR;
+  }
+
+  file->offset += actual;
+
+  return actual;
+}
+
+int sftp_async_write_end(sftp_file file, uint32_t id, int blocking) {
+  sftp_session sftp = file->sftp;
+  sftp_message msg = NULL;
+  sftp_status_message status;
+
+  msg = sftp_dequeue(sftp, id);
+  while (msg == NULL) {
+    if (!blocking && ssh_channel_poll(sftp->channel, 0) == 0) {
+      /* we cannot block */
+      return SSH_AGAIN;
+    }
+    if (sftp_read_and_dispatch(sftp) < 0) {
+      /* something nasty has happened */
+      return SSH_ERROR;
+    }
+    msg = sftp_dequeue(sftp, id);
+  }
+
+  switch (msg->packet_type) {
+    case SSH_FXP_STATUS:
+      status = parse_status_msg(msg);
+      sftp_message_free(msg);
+      if (status == NULL) {
+        return SSH_ERROR;
+      }
+      sftp_set_error(sftp, status->status);
+      switch (status->status) {
+        case SSH_FX_OK:
+          status_msg_free(status);
+          return SSH_OK;
+        default:
+          break;
+      }
+      ssh_set_error(sftp->session, SSH_REQUEST_DENIED,
+          "SFTP server: %s", status->errormsg);
+      status_msg_free(status);
+      return SSH_ERROR;
+    default:
+      ssh_set_error(sftp->session, SSH_FATAL,
+          "Received message %d during write!", msg->packet_type);
+      sftp_message_free(msg);
+      return SSH_ERROR;
+  }
+
+  return SSH_ERROR; /* not reached */
+}
+
+ /* Seek to a specific location in a file. */
+ int sftp_seek(sftp_file file, uint32_t new_offset) {
+   if (file == NULL) {
--- a/patch/libssh-0.9.6.patch
+++ b/patch/libssh-0.9.6.patch
@@ -0,0 +1,164 @@
+diff --git a/DefineOptions.cmake b/DefineOptions.cmake
+index b82a5018..f1f2ab9d 100644
+--- a/DefineOptions.cmake
+++ b/DefineOptions.cmake
+@@ -15,13 +15,14 @@ option(UNIT_TESTING "Build with unit tests" OFF)
+ option(CLIENT_TESTING "Build with client tests; requires openssh" OFF)
+ option(SERVER_TESTING "Build with server tests; requires openssh and dropbear" OFF)
+ option(WITH_BENCHMARKS "Build benchmarks tools" OFF)
+-option(WITH_EXAMPLES "Build examples" ON)
+option(WITH_EXAMPLES "Build examples" OFF)
+ option(WITH_NACL "Build with libnacl (curve25519)" ON)
+ option(WITH_SYMBOL_VERSIONING "Build with symbol versioning" ON)
+ option(WITH_ABI_BREAK "Allow ABI break" OFF)
+ option(WITH_GEX "Enable DH Group exchange mechanisms" ON)
+ option(FUZZ_TESTING "Build with fuzzer for the server" OFF)
+ option(PICKY_DEVELOPER "Build with picky developer flags" OFF)
+option(WITH_STATIC_LIB "Build static library" ON)
+ 
+ if (WITH_ZLIB)
+     set(WITH_LIBZ ON)
+@@ -53,3 +54,7 @@ endif (NOT GLOBAL_BIND_CONFIG)
+ if (NOT GLOBAL_CLIENT_CONFIG)
+   set(GLOBAL_CLIENT_CONFIG "/etc/ssh/ssh_config")
+ endif (NOT GLOBAL_CLIENT_CONFIG)
+
+if (WITH_STATIC_LIB)
+  set(BUILD_STATIC_LIB ON)
+endif()
+diff --git a/include/libssh/sftp.h b/include/libssh/sftp.h
+index 8c14b21d..95ac1d6b 100644
+--- a/include/libssh/sftp.h
+++ b/include/libssh/sftp.h
+@@ -565,6 +565,9 @@ LIBSSH_API int sftp_async_read(sftp_file file, void *data, uint32_t len, uint32_
+  */
+ LIBSSH_API ssize_t sftp_write(sftp_file file, const void *buf, size_t count);
+ 
+LIBSSH_API int sftp_async_write(sftp_file file, const void *buf, size_t count, uint32_t* id);
+LIBSSH_API int sftp_async_write_end(sftp_file file, uint32_t id, int blocking);
+
+ /**
+  * @brief Seek to a specific location in a file.
+  *
+diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
+index a576cf71..303a1c7f 100644
+--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
+@@ -412,6 +412,10 @@ if (BUILD_STATIC_LIB)
+   if (WIN32)
+     target_compile_definitions(ssh-static PUBLIC "LIBSSH_STATIC")
+   endif (WIN32)
+  install(TARGETS ssh-static
+	EXPORT libssh-config
+	LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+	COMPONENT libraries)
+ endif (BUILD_STATIC_LIB)
+ 
+ message(STATUS "Threads_FOUND=${Threads_FOUND}")
+diff --git a/src/sftp.c b/src/sftp.c
+index a8346040..a4261ec9 100644
+--- a/src/sftp.c
+++ b/src/sftp.c
+@@ -2234,6 +2234,102 @@ ssize_t sftp_write(sftp_file file, const void *buf, size_t count) {
+   return -1; /* not reached */
+ }
+ 
+/*
+ * sftp_async_write and sftp_async_write_end are copied from
+ * https://github.com/limes-datentechnik-gmbh/libssh
+ */
+int sftp_async_write(sftp_file file, const void *buf, size_t count, uint32_t* id) {
+  sftp_session sftp = file->sftp;
+  ssh_buffer buffer;
+  int len;
+  int packetlen;
+  int rc;
+
+  buffer = ssh_buffer_new();
+  if (buffer == NULL) {
+    ssh_set_error_oom(sftp->session);
+    return -1;
+  }
+
+  *id = sftp_get_new_id(file->sftp);
+
+  rc = ssh_buffer_pack(buffer,
+                       "dSqdP",
+                       *id,
+                       file->handle,
+                       file->offset,
+                       count, /* len of datastring */
+                       (size_t)count, buf);
+  if (rc != SSH_OK){
+    ssh_set_error_oom(sftp->session);
+    ssh_buffer_free(buffer);
+    return SSH_ERROR;
+  }
+  packetlen=ssh_buffer_get_len(buffer)+5;
+  len = sftp_packet_write(file->sftp, SSH_FXP_WRITE, buffer);
+  ssh_buffer_free(buffer);
+  if (len < 0) {
+    return SSH_ERROR;
+  } else  if (len != packetlen) {
+    ssh_set_error(sftp->session, SSH_FATAL,
+      "Could only send %d of %d bytes to remote host!", len, packetlen);
+    SSH_LOG(SSH_LOG_PACKET,
+        "Could not write as much data as expected");
+    return SSH_ERROR;
+  }
+
+  file->offset += count;
+
+  return SSH_OK;
+}
+
+int sftp_async_write_end(sftp_file file, uint32_t id, int blocking) {
+  sftp_session sftp = file->sftp;
+  sftp_message msg = NULL;
+  sftp_status_message status;
+
+  msg = sftp_dequeue(sftp, id);
+  while (msg == NULL) {
+    if (!blocking && ssh_channel_poll(sftp->channel, 0) == 0) {
+      /* we cannot block */
+      return SSH_AGAIN;
+    }
+    if (sftp_read_and_dispatch(sftp) < 0) {
+      /* something nasty has happened */
+      return SSH_ERROR;
+    }
+    msg = sftp_dequeue(sftp, id);
+  }
+
+  switch (msg->packet_type) {
+    case SSH_FXP_STATUS:
+      status = parse_status_msg(msg);
+      sftp_message_free(msg);
+      if (status == NULL) {
+        return SSH_ERROR;
+      }
+      sftp_set_error(sftp, status->status);
+      switch (status->status) {
+        case SSH_FX_OK:
+          status_msg_free(status);
+          return SSH_OK;
+        default:
+          break;
+      }
+      ssh_set_error(sftp->session, SSH_REQUEST_DENIED,
+          "SFTP server: %s", status->errormsg);
+      status_msg_free(status);
+      return SSH_ERROR;
+    default:
+      ssh_set_error(sftp->session, SSH_FATAL,
+          "Received message %d during write!", msg->packet_type);
+      sftp_message_free(msg);
+      return SSH_ERROR;
+  }
+
+  return SSH_ERROR; /* not reached */
+}
+
+ /* Seek to a specific location in a file. */
+ int sftp_seek(sftp_file file, uint32_t new_offset) {
+   if (file == NULL) {
--- a/scripts/install-build-deps.sh
+++ b/scripts/install-build-deps.sh
@@ -1,18 +1,30 @@
-#!/bin/bash -e
+#!/bin/bash -eu
 #
 # Install build dpenedencies.

-source /etc/os-release
+platform=$(uname -s)
+
+if [ -e /etc/os-release ]; then
+	source /etc/os-release
+	platform=${platform}-${ID}
+fi
+
 set -x

-case $ID in
-	ubuntu*)
-		apt-get install -y gcc make cmake libssh-dev
+case $platform in
+	Darwin)
+		brew install openssl@1.1
 		;;
-	centos* | rhel*)
-		dnf install -y gcc make cmake libssh-devel rpm-build
+	Linux-ubuntu*)
+		apt-get install -y \
+			gcc make cmake zlib1g-dev libssl-dev libkrb5-dev
+		;;
+	Linux-centos* | Linux-rhel* | Linux-rocky*)
+		yum install -y \
+			gcc make cmake zlib-devel openssl-devel rpm-build
 		;;
 	*)
-		echo "unsupported dependency install: $ID"
+		echo "unsupported platform: $platform"
 		exit 1
+		;;
 esac
--- a/scripts/print-install-deps.sh
+++ b/scripts/print-install-deps.sh
@@ -0,0 +1,23 @@
+#!/bin/bash -e
+#
+# Print install dpenedencies on Linux. CMake runs this script to obtain deps for CPACK.
+# mscp dependes on packages on which libssh depends.
+
+source /etc/os-release
+
+release=$1
+
+case $release in
+	ubuntu-20.04*)
+		echo "libc6 (>= 2.27), libgssapi-krb5-2 (>= 1.17), libssl1.1 (>= 1.1.1), zlib1g (>= 1:1.1.4)"
+		;;
+	ubuntu-22.04*)
+		echo "libc6 (>= 2.33), libgssapi-krb5-2 (>= 1.17), libssl3 (>= 3.0.0~~alpha1), zlib1g (>= 1:1.1.4)"
+		;;
+	centos* | rhel* | rocky*)
+		echo "glibc crypto-policies krb5-libs openssl-libs libcom_err"
+		;;
+	*)
+		echo "unsupported install dependency: $release"
+		exit 1
+esac
--- a/scripts/test-in-container.sh
+++ b/scripts/test-in-container.sh
@@ -1,31 +1,13 @@
 #!/bin/bash -e
 #
-# Run this script in docker containers. This script installs mscp from built package
-# and run test for mscp in the installed path.
+# Run this script in mscp docker containers.
+# This script runs end-to-end test with installed mscp.

-source /etc/os-release
 script_dir=$(cd $(dirname ${0}) && pwd)
 cd $script_dir
-project_version=$(cat ../VERSION)
-arch=$(uname -p)

 set -x

-# install package
-case $ID in
-        ubuntu*)
-		pkg=mscp_${project_version}-${ID}-${VERSION_ID}-${arch}.deb
-		dpkg -i ../build/$pkg
-                ;;
-        centos* | rhel*)
-		pkg=mscp_${project_version}-${ID}-${VERSION_ID}-${arch}.rpm
-		rpm -iv ../build/$pkg
-                ;;
-        *)
-                echo "unsupported test platform: $ID"
-                exit 1
-esac
-
 # Run sshd
 if [ ! -e /var/run/sshd.pid ]; then
 	/usr/sbin/sshd
--- a/src/atomic.h
+++ b/src/atomic.h
@@ -55,4 +55,17 @@ static inline void lock_release(lock *l)
 	}
 }

+static inline void lock_release_via_cleanup(void *l)
+{
+	lock_release(l);
+}
+
+#define LOCK_ACQUIRE_THREAD(l)				\
+		lock_acquire(l);			\
+		pthread_cleanup_push(lock_release_via_cleanup, l)
+
+
+#define LOCK_RELEASE_THREAD(l) \
+	pthread_cleanup_pop(1)
+
 #endif /* _ATOMIC_H_ */
--- a/src/file.c
+++ b/src/file.c
@@ -319,13 +319,15 @@ static int file_fill_recursive(struct list_head *file_list,
 int file_fill(sftp_session sftp, struct list_head *file_list, char **src_array, int cnt,
 	      char *dst)
 {
-	bool dst_is_remote, dst_is_dir, dst_dir_no_exist, dst_should_dir;
+	bool dst_is_remote, dst_is_dir, dst_dir_no_exist, dst_should_dir, dst_must_dir;
 	char *dst_path, *src_path;
 	int n, ret;

 	dst_path = file_find_path(dst);
 	dst_path = *dst_path == '\0' ? "." : dst_path;
 	dst_is_remote = file_find_hostname(dst) ? true : false;
+	dst_must_dir = cnt > 1 ? true : false;
+
 	if (file_is_directory(dst_path, dst_is_remote ? sftp : NULL, false) > 0)
 		dst_is_dir = true;
 	else
@@ -342,7 +344,8 @@ int file_fill(sftp_session sftp, struct list_head *file_list, char **src_array,

 		ret = file_fill_recursive(file_list, dst_is_remote, sftp,
 					  src_path, "", dst_path,
-					  dst_should_dir | dst_is_dir, dst_dir_no_exist);
+					  dst_should_dir | dst_must_dir | dst_is_dir,
+					  dst_dir_no_exist);
 		if (ret < 0)
 			return ret;
 	}
@@ -364,6 +367,7 @@ static int file_dst_prepare(struct file *f, sftp_session sftp)

 	pr_debug("prepare for %s\n", path);

+	/* mkdir -p */
 	for (p = strchr(path + 1, '/'); p; p = strchr(p + 1, '/')) {
 		*p = '\0';

@@ -395,6 +399,30 @@ static int file_dst_prepare(struct file *f, sftp_session sftp)
 		*p = '/';
 	}

+	/* open file with O_TRUNC to set file size 0 */
+	mode = O_WRONLY|O_CREAT|O_TRUNC;
+	if (sftp) {
+		sftp_file sf;
+		if ((sf = sftp_open(sftp, f->dst_path, mode, S_IRUSR|S_IWUSR)) == NULL) {
+			pr_err("sftp_open: %s\n", sftp_get_ssh_error(sftp));
+			return -1;
+		}
+		if (sftp_close(sf) < 0) {
+			pr_err("sftp_close: %s\n", sftp_get_ssh_error(sftp));
+			return -1;
+		}
+	} else {
+		int fd;
+		if ((fd = open(f->dst_path, mode, S_IRUSR|S_IWUSR)) < 0) {
+			pr_err("open: %s\n", strerrno());
+			return -1;
+		}
+		if (close(fd) < 0) {
+			pr_err("close: %s\n", strerrno());
+			return -1;
+		}
+	}
+
 	return 0;
 }

@@ -464,7 +492,7 @@ int chunk_fill(struct list_head *file_list, struct list_head *chunk_list,
 			chunk_sz = max_chunk_sz;
 		else {
 			chunk_sz = (f->size - (f->size % nr_conn)) / nr_conn;
-			chunk_sz &= ~page_mask; /* align in page_sz */
+			chunk_sz &= ~page_mask; /* align with page_sz */
 			if (chunk_sz <= min_chunk_sz)
 				chunk_sz = min_chunk_sz;
 		}
@@ -526,7 +554,7 @@ int chunk_prepare(struct chunk *c, sftp_session sftp)
 	struct file *f = c->f;
 	int ret = 0;

-	lock_acquire(&f->lock); /* XXX: is always acquiring lock per-chunk heavy? */
+	LOCK_ACQUIRE_THREAD(&f->lock);
 	if (f->state == FILE_STATE_INIT) {
 		if (file_dst_prepare(f, f->dst_is_remote ? sftp : NULL) < 0) {
 			ret = -1;
@@ -537,11 +565,11 @@ int chunk_prepare(struct chunk *c, sftp_session sftp)
 	}

 out:
-	lock_release(&f->lock);
+	LOCK_RELEASE_THREAD();
 	return ret;
 }

-static mode_t chunk_get_mode(const char *path, sftp_session sftp)
+static mode_t file_get_mode(const char *path, sftp_session sftp)
 {
 	mode_t mode;

@@ -564,7 +592,7 @@ static mode_t chunk_get_mode(const char *path, sftp_session sftp)
 	return mode;
 }

-static int chunk_set_mode(const char *path, mode_t mode, sftp_session sftp)
+static int file_set_mode(const char *path, mode_t mode, sftp_session sftp)
 {
 	if (sftp) {
 		if (sftp_chmod(sftp, path, mode) < 0) {
@@ -619,57 +647,142 @@ static sftp_file chunk_open_remote(const char *path, int flags, mode_t mode, siz
 	return sf;
 }

-static int chunk_copy_internal(struct chunk *c, int fd, sftp_file sf,
-			       size_t sftp_buf_sz, size_t io_buf_sz,
-			       bool reverse, size_t *counter)
+static ssize_t read_to_buf(void *ptr, size_t len, void *userdata)
 {
-	size_t remaind, read_bytes, write_bytes;
-	char buf[io_buf_sz];
+	int fd = *((int *)userdata);
+	return read(fd, ptr, len);
+}

-	/* if reverse is false, copy fd->sf (local to remote).
-	 * if reverse is true, copy sf->fd (remote to local)
-	 */
+static int chunk_copy_local_to_remote_async(struct chunk *c, int fd, sftp_file sf,
+					    int nr_ahead, int buf_sz, size_t *counter)
+{
+	ssize_t read_bytes, remaind, thrown;
+	int idx, ret;
+	struct {
+		uint32_t id;
+		ssize_t  len;
+	} reqs[nr_ahead];

-	for (remaind = c->len; remaind > 0;) {
+	if (c->len == 0)
+		return 0;

-		if (!reverse)
-			read_bytes = read(fd, buf, min(remaind, io_buf_sz));
-		else
-			read_bytes = sftp_read2(sf, buf, min(remaind, io_buf_sz),
-						sftp_buf_sz);
+	remaind = thrown = c->len;
+	for (idx = 0; idx < nr_ahead && thrown > 0; idx++) {
+		reqs[idx].len = min(thrown, buf_sz);
+		reqs[idx].len = sftp_async_write(sf, read_to_buf, reqs[idx].len, &fd,
+						 &reqs[idx].id);
+		if (reqs[idx].len < 0) {
+			pr_err("sftp_async_write: %d or %s\n",
+			       sftp_get_error(sf->sftp), strerrno());
+			return -1;
+		}
+		thrown -= reqs[idx].len;
+	}

-		if (read_bytes < 0) {
-			pr_err("failed to read %s: %s\n", c->f->dst_path,
-			       !reverse ? strerrno() : sftp_get_ssh_error(sf->sftp));
+	for (idx = 0; remaind > 0; idx = (idx + 1) % nr_ahead) {
+		ret = sftp_async_write_end(sf, reqs[idx].id, 1);
+		if (ret != SSH_OK) {
+			pr_err("sftp_async_write_end: %d\n", sftp_get_error(sf->sftp));
 			return -1;
 		}

-		if (!reverse)
-			write_bytes = sftp_write2(sf, buf, read_bytes, sftp_buf_sz);
-		else
-			write_bytes = write(fd, buf, read_bytes);
+		*counter += reqs[idx].len;
+		remaind -= reqs[idx].len;

+		if (remaind <= 0)
+			break;
+
+		if (thrown <= 0)
+			continue;
+
+		reqs[idx].len = min(thrown, buf_sz);
+		reqs[idx].len = sftp_async_write(sf, read_to_buf, reqs[idx].len, &fd,
+						 &reqs[idx].id);
+		if (reqs[idx].len < 0) {
+			pr_err("sftp_async_write: %d or %s\n",
+			       sftp_get_error(sf->sftp), strerrno());
+			return -1;
+		}
+		thrown -= reqs[idx].len;
+	}
+
+	if (remaind < 0) {
+		pr_err("invalid remaind bytes %ld. last async_write_end bytes %lu.",
+		       remaind, reqs[idx].len);
+		return -1;
+	}
+
+	return 0;
+}
+
+
+static int chunk_copy_remote_to_local_async(struct chunk *c, int fd, sftp_file sf,
+					    int nr_ahead, int buf_sz, size_t *counter)
+{
+	ssize_t read_bytes, write_bytes, remaind, thrown;
+	char buf[buf_sz];
+	int idx;
+	struct {
+		int id;
+		ssize_t len;
+	} reqs[nr_ahead];
+
+	if (c->len == 0)
+		return 0;
+
+	remaind = thrown = c->len;
+
+	for (idx = 0; idx < nr_ahead && thrown > 0; idx++) {
+		reqs[idx].len = min(thrown, sizeof(buf));
+		reqs[idx].id = sftp_async_read_begin(sf, reqs[idx].len);
+		if (reqs[idx].id < 0) {
+			pr_err("sftp_async_read_begin: %d\n",
+			       sftp_get_error(sf->sftp));
+			return -1;
+		}
+		thrown -= reqs[idx].len;
+	}
+
+	for (idx = 0; remaind > 0; idx = (idx + 1) % nr_ahead) {
+		read_bytes = sftp_async_read(sf, buf, reqs[idx].len, reqs[idx].id);
+		if (read_bytes == SSH_ERROR) {
+			pr_err("sftp_async_read: %d\n", sftp_get_error(sf->sftp));
+			return -1;
+		}
+
+		if (thrown > 0) {
+			reqs[idx].len = min(thrown, sizeof(buf));
+			reqs[idx].id = sftp_async_read_begin(sf, reqs[idx].len);
+			thrown -= reqs[idx].len;
+		}
+
+		write_bytes = write(fd, buf, read_bytes);
 		if (write_bytes < 0) {
-			pr_err("failed to write %s: %s\n", c->f->dst_path,
-			       !reverse ? strerrno() : sftp_get_ssh_error(sf->sftp));
+			pr_err("write: %s\n", strerrno());
 			return -1;
 		}

 		if (write_bytes < read_bytes) {
-			pr_err("failed to write full bytes to %s\n", c->f->dst_path);
+			pr_err("failed to write full bytes\n");
 			return -1;
 		}

 		*counter += write_bytes;
-		remaind -= write_bytes;
+		remaind -= read_bytes;
+	}
+
+	if (remaind < 0) {
+		pr_err("invalid remaind bytes %ld. last async_read bytes %ld. "
+		       "last write bytes %ld\n",
+		       remaind, read_bytes, write_bytes);
+		return -1;
 	}

 	return 0;
 }

 static int chunk_copy_local_to_remote(struct chunk *c, sftp_session sftp,
-				      size_t sftp_buf_sz, size_t io_buf_sz,
-				      size_t *counter)
+				      int nr_ahead, int buf_sz, size_t *counter)
 {
 	struct file *f = c->f;
 	sftp_file sf = NULL;
@@ -685,25 +798,16 @@ static int chunk_copy_local_to_remote(struct chunk *c, sftp_session sftp,
 		goto out;
 	}

-	flags = O_WRONLY|O_CREAT;
+	flags = O_WRONLY;
 	mode = S_IRUSR|S_IWUSR;
 	if (!(sf = chunk_open_remote(f->dst_path, flags, mode, c->off, sftp))) {
 		ret = -1;
 		goto out;
 	}

-	ret = chunk_copy_internal(c, fd, sf, sftp_buf_sz, io_buf_sz, false, counter);
+	ret = chunk_copy_local_to_remote_async(c, fd, sf, nr_ahead, buf_sz, counter);
 	if (ret < 0)
 		goto out;
-
-	if ((mode = chunk_get_mode(f->src_path, NULL)) < 0) {
-		ret = -1;
-		goto out;
-	}
-	if (chunk_set_mode(f->dst_path, mode, sftp) < 0) {
-		ret = -1;
-	}
-
 out:
 	if (fd > 0)
 		close(fd);
@@ -713,8 +817,7 @@ out:
 }

 static int chunk_copy_remote_to_local(struct chunk *c, sftp_session sftp,
-				      size_t sftp_buf_sz, size_t io_buf_sz,
-				      size_t *counter)
+				      int nr_ahead, int buf_sz, size_t *counter)
 {
 	struct file *f = c->f;
 	sftp_file sf = NULL;
@@ -723,7 +826,7 @@ static int chunk_copy_remote_to_local(struct chunk *c, sftp_session sftp,
 	int fd = 0;
 	int ret = 0;

-	flags = O_WRONLY|O_CREAT;
+	flags = O_WRONLY;
 	mode = S_IRUSR|S_IWUSR;
 	if ((fd = chunk_open_local(f->dst_path, flags, mode, c->off)) < 0) {
 		ret = -1;
@@ -737,7 +840,7 @@ static int chunk_copy_remote_to_local(struct chunk *c, sftp_session sftp,
 		goto out;
 	}

-	ret = chunk_copy_internal(c, fd, sf, sftp_buf_sz, io_buf_sz, true, counter);
+	ret = chunk_copy_remote_to_local_async(c, fd, sf, nr_ahead, buf_sz, counter);
 	if (ret< 0)
 		goto out;

@@ -750,44 +853,51 @@ out:
 	return ret;
 }

+static int file_cleanup(struct file *f, sftp_session sftp)
+{
+	sftp_session s, d;
+	mode_t mode;

+	if (f->dst_is_remote) {
+		s = NULL;
+		d = sftp;
+	} else {
+		s = sftp;
+		d = NULL;
+	}

-int chunk_copy(struct chunk *c, sftp_session sftp, size_t sftp_buf_sz, size_t io_buf_sz,
+	if ((mode = file_get_mode(f->src_path, s)) < 0)
+		return -1;
+	if (file_set_mode(f->dst_path, mode, d) < 0)
+		return -1;
+	return 0;
+}
+
+int chunk_copy(struct chunk *c, sftp_session sftp, int nr_ahead, int buf_sz,
 	       size_t *counter)
 {
 	struct file *f = c->f;
 	int ret = 0;

-	pr_debug("copy %s %s -> %s %s off=0x%010lx\n",
-		 f->src_path, strloc(f->src_is_remote),
-		 f->dst_path, strloc(f->dst_is_remote), c->off);
-
 	pprint4("copy start: chunk %s 0x%010lx-0x%010lx %luB\n",
 		c->f->src_path, c->off, c->off + c->len, c->len);

-
 	if (f->dst_is_remote)
-		ret = chunk_copy_local_to_remote(c, sftp,
-						 sftp_buf_sz, io_buf_sz, counter);
+		ret = chunk_copy_local_to_remote(c, sftp, nr_ahead, buf_sz, counter);
 	else
-		ret = chunk_copy_remote_to_local(c, sftp,
-						 sftp_buf_sz, io_buf_sz, counter);
+		ret = chunk_copy_remote_to_local(c, sftp, nr_ahead, buf_sz, counter);

 	if (ret < 0)
 		return ret;

-	pr_debug("done %s %s -> %s %s off=0x%010lx\n",
-		 f->src_path, strloc(f->src_is_remote),
-		 f->dst_path, strloc(f->dst_is_remote), c->off);
-
 	pprint4("copy done: chunk %s 0x%010lx-0x%010lx %luB\n",
 		c->f->src_path, c->off, c->off + c->len, c->len);

 	if (refcnt_dec(&f->refcnt) == 0) {
 		f->state = FILE_STATE_DONE;
 		pprint2("copy done: %s\n", f->src_path);
+		ret = file_cleanup(f, sftp);
 	}

-
 	return ret;
 }
--- a/src/file.h
+++ b/src/file.h
@@ -3,8 +3,8 @@

 #include <limits.h>
 #include <pthread.h>
-#include <libssh/libssh.h>
-#include <libssh/sftp.h>
+#include "libssh/libssh.h"
+#include "libssh/sftp.h"

 #include <list.h>
 #include <atomic.h>
@@ -37,7 +37,8 @@ struct file {
 * if the file state of the chunk is INIT:
 *     acquire the file lock
 * *         if file state is INIT:
- *             create destination file and directory if necessary
+ *             create directory if necessary
+ *             open file with O_TRUNC and close.
 *             set file state OPENED.
 *             // only the first thread in the lock open the destination file
 *     release the file lock
@@ -71,8 +72,9 @@ int chunk_fill(struct list_head *file_list, struct list_head *chunk_list,

 struct chunk *chunk_acquire(struct list_head *chunk_list);
 int chunk_prepare(struct chunk *c, sftp_session sftp);
-int chunk_copy(struct chunk *c, sftp_session sftp,
-	       size_t sftp_buf_sz, size_t io_buf_sz, size_t *counter);
+int chunk_copy(struct chunk *c, sftp_session sftp, int nr_ahead, int buf_sz,
+	       size_t *counter);
+

 #ifdef DEBUG
 void file_dump(struct list_head *file_list);
--- a/src/list.h
+++ b/src/list.h
@@ -514,4 +514,19 @@ static inline void hlist_add_after(struct hlist_node *n,
 	     pos = n)


+/**
+ * list_count - return length of list
+ * @head        the head for your list.
+ */
+static inline int list_count(struct list_head *head)
+{
+	int n = 0;
+	struct list_head *p;
+
+	list_for_each(p, head) n++;
+	return n;
+}
+
+
 #endif
+
--- a/src/main.c
+++ b/src/main.c
@@ -23,93 +23,95 @@
 #endif

 #define DEFAULT_MIN_CHUNK_SZ    (64 << 20)      /* 64MB */
-#define DEFAULT_SFTP_BUF_SZ     131072          /* derived from qemu/block/ssh.c */
-#define DEFAULT_IO_BUF_SZ       DEFAULT_SFTP_BUF_SZ
-/* XXX: need to investigate max buf size for sftp_read/sftp_write */
+#define DEFAULT_NR_AHEAD	32
+#define DEFAULT_BUF_SZ		16384
+/* XXX: we use 16384 byte buffer pointed by
+ * https://api.libssh.org/stable/libssh_tutor_sftp.html. The larget
+ * read length from sftp_async_read is 65536 byte. Read sizes larger
+ * than 65536 cause a situation where data remainds but
+ * sftp_async_read returns 0.
+ */

-struct mscp {
-	char                    *host;  /* remote host (and username) */
-	struct ssh_opts         *opts;  /* ssh parameters */
-	sftp_session            ctrl;   /* control sftp session */

-	struct list_head        file_list;
-	struct list_head        chunk_list;     /* stack of chunks */
-	lock                    chunk_lock;  /* lock for chunk list */
-
-	char    *target;
-
-	int     sftp_buf_sz, io_buf_sz;
-
-	struct timeval start;   /* timestamp of starting copy */
-};

 struct mscp_thread {
-	struct mscp     *mscp;
 	sftp_session    sftp;

 	pthread_t       tid;
+	int		cpu;
 	size_t          done;           /* copied bytes */
 	bool            finished;
 	int		ret;
 };

-void *mscp_copy_thread(void *arg);
-void *mscp_monitor_thread(void *arg);
+struct mscp {
+	char                    *host;  /* remote host (and username) */
+	struct ssh_opts         *opts;  /* ssh parameters */
+
+	struct list_head        file_list;
+	struct list_head        chunk_list;	/* stack of chunks */
+	lock                    chunk_lock;	/* lock for chunk list */
+
+	char    *target;
+
+	int	nr_threads;	/* number of threads */
+	int	buf_sz;		/* i/o buf size */
+	int	nr_ahead;	/* # of ahead read command for remote to local copy */
+
+	struct mscp_thread *threads;
+} m;
+
+void *mscp_copy_thread(void *arg);
+int mscp_stat_init();
+void mscp_stat_final();
+

-pthread_t mtid;
-struct mscp_thread *threads;
-int nr_threads;

 void stop_copy_threads(int sig)
 {
 	int n;

 	pr("stopping...\n");
-	for (n = 0; n < nr_threads; n++) {
-		pthread_cancel(threads[n].tid);
+	for (n = 0; n < m.nr_threads; n++) {
+		if (m.threads[n].tid && !m.threads[n].finished)
+			pthread_cancel(m.threads[n].tid);
 	}
 }

-int list_count(struct list_head *head)
-{
-	int n = 0;
-	struct list_head *p;
-
-	list_for_each(p, head) n++;
-	return n;
-}
-
 void usage(bool print_help) {
 	printf("mscp v" VERSION ": copy files over multiple ssh connections\n"
 	       "\n"
-	       "Usage: mscp [vqDCHdh] [-n nr_conns]\n"
-	       "            [-s min_chunk_sz] [-S max_chunk_sz]\n"
-	       "            [-b sftp_buf_sz] [-B io_buf_sz]\n"
+	       "Usage: mscp [vqDCHdNh] [-n nr_conns] [-m coremask]\n"
+	       "            [-s min_chunk_sz] [-S max_chunk_sz] [-a nr_ahead] [-b buf_sz]\n"
 	       "            [-l login_name] [-p port] [-i identity_file]\n"
-	       "            [-c cipher_spec] source ... target\n"
+	       "            [-c cipher_spec] [-M hmac_spec] source ... target\n"
 	       "\n");

 	if (!print_help)
 		return;

-	printf("    -n NR_CONNECTIONS  number of connections (default: half of # of cpu cores)\n"
+	printf("    -n NR_CONNECTIONS  number of connections "
+	       "(default: floor(log(cores)*2)+1)\n"
+	       "    -m COREMASK        hex value to specify cores where threads pinned\n"
 	       "    -s MIN_CHUNK_SIZE  min chunk size (default: 64MB)\n"
-	       "    -S MAX_CHUNK_SIZE  max chunk size (default: filesize / nr_conn)\n"
-	       "    -b SFTP_BUF_SIZE   buf size for sftp_read/write (default 131072B)\n"
-	       "    -B IO_BUF_SIZE     buf size for read/write (default 131072B)\n"
-	       "                       Note that this value is derived from\n"
-	       "                       qemu/block/ssh.c. need investigation...\n"
+	       "    -S MAX_CHUNK_SIZE  max chunk size (default: filesize/nr_conn)\n"
+	       "\n"
+	       "    -a NR_AHEAD        number of inflight SFTP commands (default: 32)\n"
+	       "    -b BUF_SZ          buffer size for i/o and transfer\n"
+	       "\n"
 	       "    -v                 increment verbose output level\n"
 	       "    -q                 disable output\n"
 	       "    -D                 dry run\n"
 	       "\n"
 	       "    -l LOGIN_NAME      login name\n"
 	       "    -p PORT            port number\n"
-	       "    -i IDENTITY        identity file for publickey authentication\n"
-	       "    -c CIPHER          cipher spec, see `ssh -Q cipher`\n"
+	       "    -i IDENTITY        identity file for public key authentication\n"
+	       "    -c CIPHER          cipher spec\n"
+	       "    -M HMAC            hmac spec\n"
 	       "    -C                 enable compression on libssh\n"
 	       "    -H                 disable hostkey check\n"
 	       "    -d                 increment ssh debug output level\n"
+	       "    -N                 disable tcp nodelay (default on)\n"
 	       "    -h                 print this help\n"
 	       "\n");
 }
@@ -157,37 +159,107 @@ err_out:
 	return NULL;
 }

+int expand_coremask(const char *coremask, int **cores, int *nr_cores)
+{
+	int n, *core_list, core_list_len = 0, nr_usable, nr_all;
+	char c[2] = { 'x', '\0' };
+	const char *_coremask;
+	long v, needle;
+	int ncores = nr_cpus();
+
+	/*
+	 * This function returns array of usable cores in `cores` and
+	 * returns the number of usable cores (array length) through
+	 * nr_cores.
+	 */
+
+	if (strncmp(coremask, "0x", 2) == 0)
+		_coremask = coremask + 2;
+	else
+		_coremask = coremask;
+
+	core_list = realloc(NULL, sizeof(int) * 64);
+	if (!core_list) {
+		pr_err("failed to realloc: %s\n", strerrno());
+		return -1;
+	}
+
+	nr_usable = 0;
+	nr_all = 0;
+	for (n = strlen(_coremask) - 1; n >=0; n--) {
+		c[0] = _coremask[n];
+		v = strtol(c, NULL, 16);
+		if (v == LONG_MIN || v == LONG_MAX) {
+			pr_err("invalid coremask: %s\n", coremask);
+			return -1;
+		}
+
+		for (needle = 0x01; needle < 0x10; needle <<= 1) {
+			nr_all++;
+			if (nr_all > ncores)
+				break; /* too long coremask */
+ 			if (v & needle) {
+				nr_usable++;
+				core_list = realloc(core_list, sizeof(int) * nr_usable);
+				if (!core_list) {
+					pr_err("failed to realloc: %s\n", strerrno());
+					return -1;
+				}
+				core_list[nr_usable - 1] = nr_all - 1;
+			}
+		}
+	}
+
+	if (nr_usable < 1) {
+		pr_err("invalid core mask: %s\n", coremask);
+		return -1;
+	}
+
+	*cores = core_list;
+	*nr_cores = nr_usable;
+	return 0;
+}
+
+int default_nr_threads()
+{
+	return (int)(floor(log(nr_cpus()) * 2) + 1);
+}
+
 int main(int argc, char **argv)
 {
-	struct mscp m;
 	struct ssh_opts opts;
+	sftp_session ctrl;
 	int min_chunk_sz = DEFAULT_MIN_CHUNK_SZ;
 	int max_chunk_sz = 0;
+	char *coremask = NULL;;
 	int verbose = 1;
 	bool dryrun = false;
 	int ret = 0, n;
+	int *cores, nr_cores;
 	char ch;

 	memset(&opts, 0, sizeof(opts));
+	opts.nodelay = 1;
 	memset(&m, 0, sizeof(m));
 	INIT_LIST_HEAD(&m.file_list);
 	INIT_LIST_HEAD(&m.chunk_list);
 	lock_init(&m.chunk_lock);
-	m.sftp_buf_sz = DEFAULT_SFTP_BUF_SZ;
-	m.io_buf_sz = DEFAULT_IO_BUF_SZ;
+	m.nr_ahead = DEFAULT_NR_AHEAD;
+	m.buf_sz = DEFAULT_BUF_SZ;
+	m.nr_threads = default_nr_threads();

-	nr_threads = (int)(nr_cpus() / 2);
-	nr_threads = nr_threads == 0 ? 1 : nr_threads;
-
-	while ((ch = getopt(argc, argv, "n:s:S:b:B:vqDl:p:i:c:CHdh")) != -1) {
+	while ((ch = getopt(argc, argv, "n:m:s:S:a:b:vqDl:p:i:c:M:CHdNh")) != -1) {
 		switch (ch) {
 		case 'n':
-			nr_threads = atoi(optarg);
-			if (nr_threads < 1) {
+			m.nr_threads = atoi(optarg);
+			if (m.nr_threads < 1) {
 				pr_err("invalid number of connections: %s\n", optarg);
 				return 1;
 			}
 			break;
+		case 'm':
+			coremask = optarg;
+			break;
 		case 's':
 			min_chunk_sz = atoi(optarg);
 			if (min_chunk_sz < getpagesize()) {
@@ -218,16 +290,16 @@ int main(int argc, char **argv)
 				return -1;
 			}
 			break;
-		case 'b':
-			m.sftp_buf_sz = atoi(optarg);
-			if (m.sftp_buf_sz < 1) {
-				pr_err("invalid buffer size: %s\n", optarg);
+		case 'a':
+			m.nr_ahead = atoi(optarg);
+			if (m.nr_ahead < 1) {
+				pr_err("invalid number of ahead: %s\n", optarg);
 				return -1;
 			}
 			break;
-		case 'B':
-			m.io_buf_sz = atoi(optarg);
-			if (m.io_buf_sz < 1) {
+		case 'b':
+			m.buf_sz = atoi(optarg);
+			if (m.buf_sz < 1) {
 				pr_err("invalid buffer size: %s\n", optarg);
 				return -1;
 			}
@@ -253,6 +325,9 @@ int main(int argc, char **argv)
 		case 'c':
 			opts.cipher = optarg;
 			break;
+		case 'M':
+			opts.hmac = optarg;
+			break;
 		case 'C':
 			opts.compress++;
 			break;
@@ -262,6 +337,9 @@ int main(int argc, char **argv)
 		case 'd':
 			opts.debuglevel++;
 			break;
+		case 'N':
+			opts.nodelay = 0;
+			break;
 		case 'h':
 			usage(true);
 			return 0;
@@ -273,19 +351,29 @@ int main(int argc, char **argv)

 	pprint_set_level(verbose);

+	if (argc - optind < 2) {
+		/* mscp needs at lease 2 (src and target) argument */
+		usage(false);
+		return 1;
+	}
+	m.target = argv[argc - 1];
+
 	if (max_chunk_sz > 0 && min_chunk_sz > max_chunk_sz) {
 		pr_err("smaller max chunk size than min chunk size: %d < %d\n",
 		       max_chunk_sz, min_chunk_sz);
 		return 1;
 	}

-	if (argc - optind < 2) {
-		/* mscp needs at lease 2 (src and target) argument */
-		usage(false);
-		return 1;
+	/* expand usable cores from coremask */
+	if (coremask) {
+		if (expand_coremask(coremask, &cores, &nr_cores) < 0)
+			return -1;
+		pprint(2, "cpu cores:");
+		for (n = 0; n < nr_cores; n++)
+			pprint(2, " %d", cores[n]);
+		pprint(2, "\n");
 	}
-
-	m.target = argv[argc - 1];
+	pprint2("number of connections: %d\n", m.nr_threads);

 	/* create control session */
 	m.host = find_hostname(optind, argc, argv);
@@ -294,15 +382,14 @@ int main(int argc, char **argv)
 		return 1;
 	}
 	pprint3("connecting to %s for checking destinations...\n", m.host);
-	m.ctrl = ssh_make_sftp_session(m.host, &opts);
-	if (!m.ctrl)
+	ctrl = ssh_init_sftp_session(m.host, &opts);
+	if (!ctrl)
 		return 1;
 	m.opts = &opts; /* save ssh-able ssh_opts */


 	/* fill file list */
-	ret = file_fill(m.ctrl, &m.file_list, &argv[optind], argc - optind - 1,
-			m.target);
+	ret = file_fill(ctrl, &m.file_list, &argv[optind], argc - optind - 1, m.target);
 	if (ret < 0)
 		goto out;

@@ -312,7 +399,7 @@ int main(int argc, char **argv)

 	/* fill chunk list */
 	ret = chunk_fill(&m.file_list, &m.chunk_list,
-			 nr_threads, min_chunk_sz, max_chunk_sz);
+			 m.nr_threads, min_chunk_sz, max_chunk_sz);
 	if (ret < 0)
 		goto out;

@@ -320,51 +407,50 @@ int main(int argc, char **argv)
 	chunk_dump(&m.chunk_list);
 #endif

-	if (dryrun)
+	if (dryrun) {
+		ssh_sftp_close(ctrl);
 		return 0;
+	}

-	/* register SIGINT to stop thrads */
-	if (signal(SIGINT, stop_copy_threads) == SIG_ERR) {
-		pr_err("cannot set signal: %s\n", strerrno());
+	/* prepare thread instances */
+	if ((n = list_count(&m.chunk_list)) < m.nr_threads) {
+		pprint2("we have only %d chunk(s). "
+			"set number of connections to %d\n", n, n);
+		m.nr_threads = n;
+	}
+
+	m.threads = calloc(m.nr_threads, sizeof(struct mscp_thread));
+	memset(m.threads, 0, m.nr_threads * sizeof(struct mscp_thread));
+	for (n = 0; n < m.nr_threads; n++) {
+		struct mscp_thread *t = &m.threads[n];
+		t->finished = false;
+		if (!coremask)
+			t->cpu = -1;
+		else
+			t->cpu = cores[n % nr_cores];
+
+		if (n == 0) {
+			t->sftp = ctrl; /* reuse ctrl sftp session */
+			ctrl = NULL;
+		} else {
+			pprint3("connecting to %s for a copy thread...\n", m.host);
+			t->sftp = ssh_init_sftp_session(m.host, m.opts);
+		}
+		if (!t->sftp) {
+			ret = 1;
+			goto out;
+		}
+	}
+
+	/* init mscp stat for printing progress bar */
+	if (mscp_stat_init() < 0) {
 		ret = 1;
 		goto out;
 	}

-	/* prepare thread instances */
-	if ((n = list_count(&m.chunk_list)) < nr_threads) {
-		pprint3("we have only %d chunk(s). set nr_conns to %d\n", n, n);
-		nr_threads = n;
-	}
-
-	threads = calloc(nr_threads, sizeof(struct mscp_thread));
-	memset(threads, 0, nr_threads * sizeof(struct mscp_thread));
-	for (n = 0; n < nr_threads; n++) {
-		struct mscp_thread *t = &threads[n];
-		t->mscp = &m;
-		t->finished = false;
-		pprint3("connecting to %s for a copy thread...\n", m.host);
-		t->sftp = ssh_make_sftp_session(m.host, m.opts);
-		if (!t->sftp) {
-			ret = 1;
-			goto join_out;
-		}
-	}
-
-	/* spawn count thread */
-	ret = pthread_create(&mtid, NULL, mscp_monitor_thread, &m);
-	if (ret < 0) {
-		pr_err("pthread_create error: %d\n", ret);
-		stop_copy_threads(0);
-		ret = 1;
-		goto join_out;
-	}
-
-	/* save start time */
-	gettimeofday(&m.start, NULL);
-
-	/* spawn threads */
-	for (n = 0; n < nr_threads; n++) {
-		struct mscp_thread *t = &threads[n];
+	/* spawn copy threads */
+	for (n = 0; n < m.nr_threads; n++) {
+		struct mscp_thread *t = &m.threads[n];
 		ret = pthread_create(&t->tid, NULL, mscp_copy_thread, t);
 		if (ret < 0) {
 			pr_err("pthread_create error: %d\n", ret);
@@ -374,23 +460,37 @@ int main(int argc, char **argv)
 		}
 	}

-join_out:
-	/* waiting for threads join... */
-	for (n = 0; n < nr_threads; n++)
-		if (threads[n].tid) {
-			pthread_join(threads[n].tid, NULL);
-			if (threads[n].ret < 0)
-				ret = threads[n].ret;
-		}
-
-	if (mtid != 0) {
-		pthread_cancel(mtid);
-		pthread_join(mtid, NULL);
+	/* register SIGINT to stop threads */
+	if (signal(SIGINT, stop_copy_threads) == SIG_ERR) {
+		pr_err("cannot set signal: %s\n", strerrno());
+		ret = 1;
+		goto out;
 	}

+join_out:
+	/* waiting for threads join... */
+	for (n = 0; n < m.nr_threads; n++) {
+		if (m.threads[n].tid) {
+			pthread_join(m.threads[n].tid, NULL);
+			if (m.threads[n].ret < 0)
+				ret = m.threads[n].ret;
+		}
+	}
+
+	/* print final result */
+	mscp_stat_final();
+
 out:
-	if (m.ctrl)
-		ssh_sftp_close(m.ctrl);
+	if (ctrl)
+		ssh_sftp_close(ctrl);
+
+	if (m.threads) {
+		for (n = 0; n < m.nr_threads; n++) {
+			struct mscp_thread *t = &m.threads[n];
+			if (t->sftp)
+				ssh_sftp_close(t->sftp);
+		}
+	}

 	return ret;
 }
@@ -398,26 +498,28 @@ out:
 void mscp_copy_thread_cleanup(void *arg)
 {
 	struct mscp_thread *t = arg;
-	if (t->sftp)
-		ssh_sftp_close(t->sftp);
 	t->finished = true;
 }

 void *mscp_copy_thread(void *arg)
 {
 	struct mscp_thread *t = arg;
-	struct mscp *m = t->mscp;
 	sftp_session sftp = t->sftp;
 	struct chunk *c;

+	if (t->cpu > -1) {
+		if (set_thread_affinity(pthread_self(), t->cpu) < 0)
+			return NULL;
+	}
+
 	pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
 	pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL);
 	pthread_cleanup_push(mscp_copy_thread_cleanup, t);

 	while (1) {
-		lock_acquire(&m->chunk_lock);
-		c = chunk_acquire(&m->chunk_list);
-		lock_release(&m->chunk_lock);
+		LOCK_ACQUIRE_THREAD(&m.chunk_lock);
+		c = chunk_acquire(&m.chunk_list);
+		LOCK_RELEASE_THREAD();

 		if (!c)
 			break; /* no more chunks */
@@ -425,17 +527,23 @@ void *mscp_copy_thread(void *arg)
 		if ((t->ret = chunk_prepare(c, sftp)) < 0)
 			break;

-		if ((t->ret = chunk_copy(c, sftp,
-					 m->sftp_buf_sz, m->io_buf_sz, &t->done)) < 0)
+		if ((t->ret = chunk_copy(c, sftp, m.nr_ahead, m.buf_sz, &t->done)) < 0)
 			break;
 	}

 	pthread_cleanup_pop(1);

+	if (t->ret < 0)
+		pr_err("copy failed: chunk %s 0x%010lx-0x%010lx\n",
+		       c->f->src_path, c->off, c->off + c->len);
+
 	return NULL;
 }

-static double calculate_bps(size_t diff, struct timeval *b, struct timeval *a)
+
+/* progress bar-related functions */
+
+double calculate_timedelta(struct timeval *b, struct timeval *a)
 {
 	double sec, usec;

@@ -448,27 +556,48 @@ static double calculate_bps(size_t diff, struct timeval *b, struct timeval *a)
 	usec = a->tv_usec - b->tv_usec;
 	sec += usec / 1000000;

-	return (double)diff / sec;
+	return sec;
 }

-static void print_progress_bar(double percent, char *suffix)
+double calculate_bps(size_t diff, struct timeval *b, struct timeval *a)
+{
+	return (double)diff / calculate_timedelta(b, a);
+}
+
+char *calculate_eta(size_t remain, size_t diff, struct timeval *b, struct timeval *a)
+{
+	static char buf[16];
+	double elapsed = calculate_timedelta(b, a);
+	double eta;
+
+	if (diff == 0)
+		snprintf(buf, sizeof(buf), "--:-- ETA");
+	else {
+		eta = remain / (diff / elapsed);
+		snprintf(buf, sizeof(buf), "%02d:%02d ETA",
+			 (int)floor(eta / 60), (int)round(eta) % 60);
+	}
+	return buf;
+}
+
+void print_progress_bar(double percent, char *suffix)
 {
 	int n, thresh, bar_width;
 	struct winsize ws;
 	char buf[128];

 	/*
-	 * [=======>   ] XX.X% SUFFIX
+	 * [=======>   ] XX% SUFFIX
 	 */

 	buf[0] = '\0';

 	if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws) < 0)
 		return; /* XXX */
-	bar_width = min(sizeof(buf), ws.ws_col) - strlen(suffix) - 8;
+	bar_width = min(sizeof(buf), ws.ws_col) - strlen(suffix) - 7;

+	memset(buf, 0, sizeof(buf));
 	if (bar_width > 8) {
-		memset(buf, 0, sizeof(buf));
 		thresh = floor(bar_width * (percent / 100)) - 1;

 		for (n = 1; n < bar_width - 1; n++) {
@@ -487,14 +616,14 @@ static void print_progress_bar(double percent, char *suffix)
 	pprint1("%s%s", buf, suffix);
 }

-static void print_progress(struct timeval *start, struct timeval *end,
-			   size_t total, size_t last, size_t done)
+void print_progress(struct timeval *b, struct timeval *a,
+		    size_t total, size_t last, size_t done)
 {
-	char *bps_units[] = { "B/s", "KB/s", "MB/s", "GB/s" };
-	char *byte_units[] = { "B", "KB", "MB", "GB", "TB", "PB" };
+	char *bps_units[] = { "B/s ", "KB/s", "MB/s", "GB/s" };
+	char *byte_units[] = { "B ", "KB", "MB", "GB", "TB", "PB" };
 	char suffix[128];
 	int bps_u, byte_tu, byte_du;
-	size_t total_round;
+	size_t total_round, done_round;
 	int percent;
 	double bps;

@@ -510,91 +639,74 @@ static void print_progress(struct timeval *start, struct timeval *end,
 	     byte_tu++)
 		total_round /= 1024;

-	bps = calculate_bps(done - last, start, end);
+	bps = calculate_bps(done - last, b, a);
 	for (bps_u = 0; bps > 1000 && bps_u < array_size(bps_units); bps_u++)
 		bps /= 1000;

 	percent = floor(((double)(done) / (double)total) * 100);
-	for (byte_du = 0; done > 1000 && byte_du < array_size(byte_units) - 1; byte_du++)
-		done /= 1024;

-	snprintf(suffix, sizeof(suffix), "%lu%s/%lu%s %.2f%s ",
-		 done, byte_units[byte_du], total_round, byte_units[byte_tu],
-		 bps, bps_units[bps_u]);
+	done_round = done;
+	for (byte_du = 0; done_round > 1000 && byte_du < array_size(byte_units) - 1;
+	     byte_du++)
+		done_round /= 1024;
+
+	snprintf(suffix, sizeof(suffix), "%4lu%s/%lu%s %6.1f%s  %s",
+		 done_round, byte_units[byte_du], total_round, byte_units[byte_tu],
+		 bps, bps_units[bps_u], calculate_eta(total - done, done - last, b, a));

 	print_progress_bar(percent, suffix);
 }

-void mscp_monitor_thread_cleanup(void *arg)
+
+struct mscp_stat {
+	struct timeval start, before, after;
+	size_t total;
+	size_t last;
+	size_t done;
+} s;
+
+void mscp_stat_handler(int signum)
 {
-	struct mscp *m = arg;
-	struct timeval end;
-	struct file *f;
-	size_t total, done;
 	int n;

-	total = done = 0;
+	for (s.done = 0, n = 0; n < m.nr_threads; n++)
+		s.done += m.threads[n].done;

-	gettimeofday(&end, NULL);
-
-	/* get total byte to be transferred */
-	list_for_each_entry(f, &m->file_list, list) {
-		total += f->size;
+	gettimeofday(&s.after, NULL);
+	if (signum == SIGALRM) {
+		alarm(1);
+		print_progress(&s.before, &s.after, s.total, s.last, s.done);
+		s.before = s.after;
+		s.last = s.done;
+	} else {
+		/* called from mscp_stat_final. calculate progress from the beginning */
+		print_progress(&s.start, &s.after, s.total, 0, s.done);
 	}
-
-	/* get total byte transferred */
-	for (n = 0; n < nr_threads; n++) {
-		done += threads[n].done;
-	}
-
-	print_progress(&m->start, &end, total, 0, done);
-	fputs("\n", stdout); /* the final ouput. we need \n */
 }

-void *mscp_monitor_thread(void *arg)
+int mscp_stat_init()
 {
-	struct mscp *m = arg;
-	struct timeval a, b;
 	struct file *f;
-	bool all_done;
-	size_t total, done, last;
-	int n;

-	pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
-	pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL);
-	pthread_cleanup_push(mscp_monitor_thread_cleanup, m);
-
-	/* get total byte to be transferred */
-	total = 0;
-	list_for_each_entry(f, &m->file_list, list) {
-		total += f->size;
+	memset(&s, 0, sizeof(s));
+	list_for_each_entry(f, &m.file_list, list) {
+		s.total += f->size;
 	}

-	while (1) {
-		all_done = true;
-		last = done = 0;
-
-		for (n = 0; n < nr_threads; n++) {
-			last += threads[n].done;
-		}
-		gettimeofday(&b, NULL);
-
-		usleep(500000);
-
-		for (n = 0; n < nr_threads; n++) {
-			done += threads[n].done;;
-			if (!threads[n].finished)
-				all_done = false;
-		}
-		gettimeofday(&a, NULL);
-
-		print_progress(&b, &a, total, last, done);
-
-		if (all_done || total == done)
-			break;
+	if (signal(SIGALRM, mscp_stat_handler) == SIG_ERR) {
+		pr_err("signal: %s\n", strerrno());
+		return -1;
 	}

-	pthread_cleanup_pop(1);
+	gettimeofday(&s.start, NULL);
+	s.before = s.start;
+	alarm(1);

-	return NULL;
+	return 0;
+}
+
+void mscp_stat_final()
+{
+	alarm(0);
+	mscp_stat_handler(0);
 }
--- a/src/platform.c
+++ b/src/platform.c
@@ -24,6 +24,13 @@ int nr_cpus()

 	return n;
 }
+
+int set_thread_affinity(pthread_t tid, int core)
+{
+	pr_warn("setting thread afinity is not implemented on apple\n");
+	return 0;
+}
+
 #endif

 #ifdef linux
@@ -34,5 +41,19 @@ int nr_cpus()
 		return CPU_COUNT(&cpu_set);
 	return -1;
 }
+
+int set_thread_affinity(pthread_t tid, int core)
+{
+	cpu_set_t target_cpu_set;
+	int ret = 0;
+
+	CPU_ZERO(&target_cpu_set);
+	CPU_SET(core, &target_cpu_set);
+	ret = pthread_setaffinity_np(tid, sizeof(target_cpu_set), &target_cpu_set);
+	if (ret < 0)
+		pr_err("failed to set thread/cpu affinity for core %d: %s",
+		       core, strerrno());
+	return ret;
+}
 #endif

--- a/src/platform.h
+++ b/src/platform.h
@@ -1,6 +1,9 @@
 #ifndef _PLATFORM_H_
 #define _PLATFORM_H_

+#include <pthread.h>
+
 int nr_cpus();
+int set_thread_affinity(pthread_t tid, int core);

 #endif /* _PLATFORM_H_ */
--- a/src/rename-logic.py
+++ b/src/rename-logic.py
@@ -32,13 +32,17 @@ def recursive(src, rel_path, dst, dst_should_dir, replace_dir_name):
        recursive(next_src, next_rel_path, dst, dst_should_dir, False)


-def fill_dst(src, dst):
-    dst_should_dir = isdir(src) | isdir(dst)
-    replace_dir_name = not isdir(dst)
-    recursive(src, "", dst, dst_should_dir, replace_dir_name)
+def fill_dst(srclist, dst):
+    dst_must_dir = len(srclist) > 1
+    for src in srclist:
+        dst_should_dir = isdir(src) | isdir(dst)
+        replace_dir_name = not isdir(dst)
+        recursive(src, "", dst, dst_should_dir | dst_must_dir, replace_dir_name)


 def main():
-    fill_dst(sys.argv[1], sys.argv[2])
+    if (len(sys.argv) < 2):
+        print("usage: {} source ... target".format(sys.argv[0]))
+    fill_dst(sys.argv[1:len(sys.argv) - 1], sys.argv[len(sys.argv) - 1])

 main()
--- a/src/ssh.c
+++ b/src/ssh.c
@@ -1,6 +1,9 @@
 #include <stdio.h>
 #include <string.h>
 #include <unistd.h>
+#include <stdlib.h>
+
+#include "libssh/callbacks.h"

 #include <ssh.h>
 #include <util.h>
@@ -32,11 +35,23 @@ static int ssh_set_opts(ssh_session ssh, struct ssh_opts *opts)

 	if (opts->cipher) {
 		if (ssh_options_set(ssh, SSH_OPTIONS_CIPHERS_C_S, opts->cipher) < 0) {
-			pr_err("failed to set cipher client to server\n");
+			pr_err("failed to set cipher for client to server\n");
 			return -1;
 		}
 		if (ssh_options_set(ssh, SSH_OPTIONS_CIPHERS_S_C, opts->cipher) < 0) {
-			pr_err("failed to set cipher client to server\n");
+			pr_err("failed to set cipher for server to client\n");
+			return -1;
+		}
+	}
+
+	if (opts->hmac) {
+		pr_warn("%s\n", opts->hmac);
+		if (ssh_options_set(ssh, SSH_OPTIONS_HMAC_C_S, opts->hmac) < 0) {
+			pr_err("failed to set hmac for client to server\n");
+			return -1;
+		}
+		if (ssh_options_set(ssh, SSH_OPTIONS_HMAC_S_C, opts->hmac) < 0) {
+			pr_err("failed to set hmac for server to client\n");
 			return -1;
 		}
 	}
@@ -47,6 +62,12 @@ static int ssh_set_opts(ssh_session ssh, struct ssh_opts *opts)
 		return -1;
 	}

+	if (opts->nodelay &&
+	    ssh_options_set(ssh, SSH_OPTIONS_NODELAY, &opts->nodelay) < 0) {
+		pr_err("failed to set nodelay\n");
+		return -1;
+	}
+
 	return 0;
 }

@@ -63,31 +84,79 @@ static int ssh_authenticate(ssh_session ssh, struct ssh_opts *opts)
 	auth_bit_mask = ssh_userauth_list(ssh, NULL);

 	if (auth_bit_mask & SSH_AUTH_METHOD_NONE &&
-	    ssh_userauth_none(ssh, NULL) == SSH_AUTH_SUCCESS) {
+	    ssh_userauth_none(ssh, NULL) == SSH_AUTH_SUCCESS)
 		return 0;
-	}

 	if (auth_bit_mask & SSH_AUTH_METHOD_PUBLICKEY &&
-	    ssh_userauth_publickey_auto(ssh, NULL, NULL) == SSH_AUTH_SUCCESS) {
+	    ssh_userauth_publickey_auto(ssh, NULL, opts->passphrase) == SSH_AUTH_SUCCESS)
 		return 0;
-	}

 	if (auth_bit_mask & SSH_AUTH_METHOD_PASSWORD) {
 		if (!opts->password) {
-			opts->password = getpass("Password: ");
+			opts->password = malloc(PASSWORD_BUF_SZ);
+			if (!opts->password) {
+				pr_err("malloc: %s\n", strerrno());
+				return -1;
+			}
+			memset(opts->password, 0, PASSWORD_BUF_SZ);
+
+			if (ssh_getpass("Password: ", opts->password, PASSWORD_BUF_SZ,
+					0, 0) < 0) {
+				return -1;
+			}
 		}
+
 		if (ssh_userauth_password(ssh, NULL, opts->password) == SSH_AUTH_SUCCESS)
 			return 0;
 	}

-	pr_err("authentication failure: %s\n", ssh_get_error(ssh));
 	return -1;
 }

-static ssh_session ssh_make_ssh_session(char *sshdst, struct ssh_opts *opts)
+static int ssh_cache_passphrase(const char *prompt, char *buf, size_t len, int echo,
+				int verify, void *userdata)
+{
+	struct ssh_opts *opts = userdata;
+
+	/* This function is called on the first time for importing
+	 * priv key file with passphrase. It is not called on the
+	 * second time or after because cached passphrase is passed
+	 * to ssh_userauth_publickey_auto(). */
+
+	if (opts->passphrase) {
+		/* passphrase is cached, but this function is called.
+		 * maybe it was an invalid passphrase? */
+		free(opts->passphrase);
+		opts->passphrase = NULL;
+	}
+
+	if (ssh_getpass("Passphrase: ", buf, len, echo, verify) < 0)
+		return -1;
+
+	/* cache the passphrase */
+	opts->passphrase = malloc(len);
+	if (!opts->passphrase) {
+		pr_err("malloc: %s\n", strerrno());
+		return -1;
+	}
+	memcpy(opts->passphrase, buf, len);
+
+	return 0;
+}
+
+static struct ssh_callbacks_struct cb = {
+	.auth_function = ssh_cache_passphrase,
+	.userdata = NULL,
+};
+
+static ssh_session ssh_init_session(char *sshdst, struct ssh_opts *opts)
 {
 	ssh_session ssh = ssh_new();

+	ssh_callbacks_init(&cb);
+	cb.userdata = opts;
+	ssh_set_callbacks(ssh, &cb);
+
 	if (ssh_set_opts(ssh, opts) != 0)
 		goto free_out;

@@ -119,10 +188,10 @@ free_out:
 	return NULL;
 }

-sftp_session ssh_make_sftp_session(char *sshdst, struct ssh_opts *opts)
+sftp_session ssh_init_sftp_session(char *sshdst, struct ssh_opts *opts)
 {
 	sftp_session sftp;
-	ssh_session ssh = ssh_make_ssh_session(sshdst, opts);
+	ssh_session ssh = ssh_init_session(sshdst, opts);

 	if (!ssh) {
 		return NULL;
@@ -205,7 +274,7 @@ static int ssh_verify_known_hosts(ssh_session session)

 	case SSH_KNOWN_HOSTS_UNKNOWN:
 		hexa = ssh_get_hexa(hash, hlen);
-		fprintf(stderr,"The server is unknown. Do you trust the host key?\n");
+		fprintf(stderr, "The server is unknown. Do you trust the host key?\n");
 		fprintf(stderr, "Public key hash: %s\n", hexa);
 		fprintf(stderr, "(yes/no): ");
 		ssh_string_free_char(hexa);
@@ -240,36 +309,10 @@ static int ssh_verify_known_hosts(ssh_session session)
 void ssh_sftp_close(sftp_session sftp)
 {
 	ssh_session ssh = sftp_ssh(sftp);
-	sftp_free(sftp);
+	/* XXX: sftp_free is stuck in ssh_poll_ctx_dopoll() when build type is Release.
+	 * skip sftp_free inappropriately...
+	 */
+	//sftp_free(sftp);
 	ssh_disconnect(ssh);
 	ssh_free(ssh);
 }
-
-
-int sftp_write2(sftp_file sf, const void *buf, size_t len, size_t sftp_buf_sz)
-{
-	int ret, nbytes;
-
-	for (nbytes = 0; nbytes < len;) {
-		ret = sftp_write(sf, buf + nbytes,
-				 min(len - nbytes, sftp_buf_sz));
-		if (ret < 0)
-			return ret;
-		nbytes += ret;
-	}
-	return nbytes;
-}
-
-int sftp_read2(sftp_file sf, void *buf, size_t len, size_t sftp_buf_sz)
-{
-	int ret, nbytes;
-
-	for (nbytes = 0; nbytes < len;) {
-		ret = sftp_read(sf, buf + nbytes,
-				min(len - nbytes, sftp_buf_sz));
-		if (ret < 0)
-			return ret;
-		nbytes += ret;
-	}
-	return nbytes;
-}
--- a/src/ssh.h
+++ b/src/ssh.h
@@ -2,8 +2,8 @@
 #define _SSH_H_

 #include <stdbool.h>
-#include <libssh/libssh.h>
-#include <libssh/sftp.h>
+#include "libssh/libssh.h"
+#include "libssh/sftp.h"


 struct ssh_opts {
@@ -11,24 +11,24 @@ struct ssh_opts {
 	char    *port;			/* -p */
 	char    *identity;		/* -i */
 	char    *cipher;		/* -c */
+	char	*hmac;			/* -M */
 	int     compress;		/* -C */
+	int	nodelay;		/* -N */
 	int     debuglevel;		/* -v */
 	bool	no_hostkey_check;	/* -H */

-	char    *password;		/* filled at the first connecting phase */
+#define PASSWORD_BUF_SZ	128
+	char    *password;	/* password for password auth */
+	char	*passphrase;	/* passphrase for private key  */
 };

-/* ssh_make_sftp_session() creates sftp_session. sshdst accpets
+/* ssh_init_sftp_session() creates sftp_session. sshdst accpets
 * user@hostname and hostname notations (by libssh).
 */
-sftp_session ssh_make_sftp_session(char *sshdst, struct ssh_opts *opts);
+sftp_session ssh_init_sftp_session(char *sshdst, struct ssh_opts *opts);
 void ssh_sftp_close(sftp_session sftp);

 #define sftp_ssh(sftp) (sftp)->session
 #define sftp_get_ssh_error(sftp) ssh_get_error(sftp_ssh(sftp))

-/* wrapping multiple sftp_read|write */
-int sftp_write2(sftp_file sf, const void *buf, size_t len, size_t sftp_buf_sz);
-int sftp_read2(sftp_file sf, void *buf, size_t len, size_t sftp_buf_sz);
-
 #endif /* _SSH_H_ */
--- a/test/test_e2e.py
+++ b/test/test_e2e.py
@@ -29,6 +29,7 @@ param_invalid_hostnames = [
    (["a:a", "b:b", "c:c"]), (["a:a", "b:b", "c"]), (["a:a", "b", "c:c"]),
    (["a", "b:b", "c:c"])
 ]
+
@pytest.mark.parametrize("args", param_invalid_hostnames)
 def test_nonidentical_hostnames(mscp, args):
    run2ng([mscp] + args)
@@ -39,6 +40,9 @@ def test_nonidentical_hostnames(mscp, args):
 """ copy test """

 remote_prefix = "localhost:{}/".format(os.getcwd()) # use current dir
+param_remote_prefix = [
+    ("", remote_prefix), (remote_prefix, "")
+]

 param_single_copy = [
    (File("src", size = 64), File("dst")),
@@ -46,23 +50,33 @@ param_single_copy = [
    (File("src", size = 128 * 1024 * 1024), File("dst")),
 ]

+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
@pytest.mark.parametrize("src, dst", param_single_copy)
-def test_single_copy_remote2local(mscp, src, dst):
+def test_single_copy(mscp, src_prefix, dst_prefix, src, dst):
    src.make()
-    run2ok([mscp, "-H", remote_prefix + src.path, dst.path])
-    assert check_same_md5sum(src, dst)
-    src.cleanup()
-    dst.cleanup()
-
-@pytest.mark.parametrize("src, dst", param_single_copy)
-def test_single_copy_local2remote(mscp, src, dst):
-    src.make()
-    run2ok([mscp, "-H", src.path, remote_prefix + dst.path])
+    run2ok([mscp, "-H", src_prefix + src.path, dst_prefix + dst.path])
    assert check_same_md5sum(src, dst)
    src.cleanup()
    dst.cleanup()


+param_double_copy = [
+    (File("src1", size = 1024 * 1024), File("src2", size = 1024 * 1024),
+     File("dst/src1"), File("dst/src2")
+     )
+]
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+@pytest.mark.parametrize("s1, s2, d1, d2", param_double_copy)
+def test_double_copy(mscp, src_prefix, dst_prefix, s1, s2, d1, d2):
+    s1.make()
+    s2.make()
+    run2ok([mscp, "-H", src_prefix + s1.path, src_prefix + s2.path, dst_prefix + "dst"])
+    assert check_same_md5sum(s1, d1)
+    assert check_same_md5sum(s2, d2)
+    s1.cleanup()
+    s2.cleanup()
+    d1.cleanup()
+    d2.cleanup()

 param_dir_copy = [
    ( "src_dir", "dst_dir",
@@ -87,16 +101,17 @@ does not exist. If dst_dir exists, scp copies src_dir to
 dst_dir/src_dir. So, this test checks both cases.
 """

+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
@pytest.mark.parametrize("src_dir, dst_dir, src, dst, twice", param_dir_copy)
-def test_dir_copy_remote2local(mscp, src_dir, dst_dir, src, dst, twice):
+def test_dir_copy(mscp, src_prefix, dst_prefix, src_dir, dst_dir, src, dst, twice):
    for f in src:
        f.make()

-    run2ok([mscp, "-H", remote_prefix + src_dir, dst_dir])
+    run2ok([mscp, "-H", src_prefix + src_dir, dst_prefix + dst_dir])
    for sf, df in zip(src, dst):
        assert check_same_md5sum(sf, df)

-    run2ok([mscp, "-H", remote_prefix + src_dir, dst_dir])
+    run2ok([mscp, "-H", src_prefix + src_dir, dst_prefix + dst_dir])
    for sf, df in zip(src, twice):
        assert check_same_md5sum(sf, df)

@@ -105,28 +120,6 @@ def test_dir_copy_remote2local(mscp, src_dir, dst_dir, src, dst, twice):
        df.cleanup()
        tf.cleanup()

-@pytest.mark.parametrize("src_dir, dst_dir, src, dst, twice", param_dir_copy)
-def test_dir_copy_local2remote(mscp, src_dir, dst_dir, src, dst, twice):
-    for f in src:
-        f.make()
-
-    run2ok([mscp, "-H", src_dir, remote_prefix + dst_dir])
-    for sf, df in zip(src, dst):
-        assert check_same_md5sum(sf, df)
-
-    run2ok([mscp, "-H", src_dir, remote_prefix + dst_dir])
-    for sf, df in zip(src, twice):
-        assert check_same_md5sum(sf, df)
-
-    for sf, df, tf in zip(src, dst, twice):
-        sf.cleanup()
-        df.cleanup()
-        tf.cleanup()
-
-
-param_remote_prefix = [
-    ("", remote_prefix), (remote_prefix, "")
-]
@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 def test_override_single_file(mscp, src_prefix, dst_prefix):
    src = File("src", size = 128).make()
@@ -150,6 +143,18 @@ def test_min_chunk(mscp, src_prefix, dst_prefix):
    src.cleanup()
    dst.cleanup()

+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_thread_affinity(mscp, src_prefix, dst_prefix):
+    src = File("src", size = 64 * 1024).make()
+    dst = File("dst")
+
+    run2ok([mscp, "-H", "-n", 4, "-m", "0x01", "-s", 8192, "-S", 65536,
+            src_prefix + src.path, dst_prefix + dst.path])
+    assert check_same_md5sum(src, dst)
+
+    src.cleanup()
+    dst.cleanup()
+
@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 def test_cannot_override_file_with_dir(mscp, src_prefix, dst_prefix):
    src = File("src", size = 128).make()
@@ -169,3 +174,11 @@ def test_transfer_zero_bytes(mscp, src_prefix, dst_prefix):
    src.cleanup()
    dst.cleanup()

+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_override_dst_having_larger_size(mscp, src_prefix, dst_prefix):
+    src = File("src", size = 1024 * 1024).make()
+    dst = File("dst", size = 1024 * 1024 * 2).make()
+    run2ok([mscp, "-H", src_prefix + src.path, dst_prefix + "dst"])
+    assert check_same_md5sum(src, dst)
+    src.cleanup()
+    dst.cleanup()
Author	SHA1	Message	Date
Ryo Nakamura	a8db569fbd	bump version to 0.0.5 and update README	2022-12-11 14:20:13 +09:00
Ryo Nakamura	3d98451bba	set default nr_threads to floor(log(cores) * 2) + 1) This change prevents mscp from establishing too many ssh connections on many-core machines in default.	2022-12-11 14:01:52 +09:00
Ryo Nakamura	d27db01d8d	use pthread_cleanup to acquire and release lock In chunk_prepare(), if multiple threads wait for acquiring f->lock, and then pthread_cancel() is called, the waiting threads are never canceled because pthread_mutex_lock() is not a cancellation point. So, use pthread_cleanup_push/pop to release the lock.	2022-12-11 13:23:41 +09:00
Ryo Nakamura	45cde99a85	allocate headroom for SFTP header This commit makes ssh_buffer_new_size() can insert headroom. This headroom can eliminate memcpy involved in ssh_buffer_prepend_data() for inserting SFTP common header.	2022-12-10 21:48:24 +09:00
Ryo Nakamura	6ae3f0f9f1	set default NR_AHEAD to 32	2022-12-08 18:01:50 +09:00
Ryo Nakamura	847c80276a	fix final progress output	2022-12-06 20:04:04 +09:00
Ryo Nakamura	c4ea9a1e78	add ssh_buffer_new_size and ssh_buffer_add_func to libssh sftp_async_write() with these functions reduces 1. realloc_buffer by ssh_buffer_new_size() 2. memcpy from read data to ssh buffer by ssh_buffer_add_func()	2022-12-06 15:02:14 +09:00
Ryo Nakamura	289293e812	change prompt for ssh key passphrase	2022-12-05 22:27:53 +09:00
Ryo Nakamura	1441873db6	reuse ctrl sftp session for the first copy thread	2022-12-05 21:47:00 +09:00
Ryo Nakamura	a2caa93d2a	update libssh build options	2022-12-05 19:46:02 +09:00
Ryo Nakamura	e1d14623f4	set TCP_NODELAY by default and introduce -N option to disable it	2022-12-04 21:32:48 +09:00
Ryo Nakamura	3b794ab51b	remove unused code and introduce -b buf_sz option This commit removes ifdef ASYNC_WRITE. So, mscp always depends on the patched libssh.	2022-12-03 20:48:43 +09:00
Ryo Nakamura	50c6781811	little cleanup	2022-12-02 23:35:45 +09:00
Ryo Nakamura	5846c6b6a9	cache passphrase for private key for later connections.	2022-12-02 23:20:23 +09:00
Ryo Nakamura	03a3a6dc4b	add auth callback for input passphrase of privkey	2022-12-02 22:28:56 +09:00
Ryo Nakamura	03b857b51a	add -M hmac option	2022-12-02 21:13:13 +09:00
Ryo Nakamura	d646fc1f89	use sigalrm for printing progress bar	2022-11-28 00:14:05 +09:00
Ryo Nakamura	5188cf6df6	add ETA to progress print	2022-11-27 20:36:24 +09:00
Ryo Nakamura	130e735e65	skip sftp_free() inappropriately	2022-11-27 19:48:13 +09:00
Ryo Nakamura	e3ed4f89d2	update REAMDE	2022-11-27 00:37:07 +09:00
Ryo Nakamura	db1431ed6a	only the last thread changes dst file permission	2022-11-27 00:06:39 +09:00
Ryo Nakamura	bf3ee25bae	add libssh-0.9.6.patch	2022-11-26 23:16:56 +09:00
Ryo Nakamura	8cc964ca8a	fix duplicate error message on ssh auth failed. and fix the final \n with -q	2022-11-26 17:34:24 +09:00
Ryo Nakamura	e0fe88c9c4	update README for v0.0.4	2022-11-26 01:30:25 +09:00
Ryo Nakamura	73cfee29aa	bump version to 0.0.4 mistake for v0.0.3 releasing...	2022-11-26 00:48:24 +09:00
Ryo Nakamura	392ffc0d0e	add workflow_dispatch to release.yml	2022-11-26 00:21:06 +09:00
Ryo Nakamura	612c3c41d4	github: add source-release job Default source tar balls in github releases do not include submodules. source-release job uploads mscp tar ball with patched libssh.	2022-11-26 00:13:49 +09:00
Ryo Nakamura	876a60382c	bump version to 0.0.2	2022-11-24 23:27:02 +09:00
Ryo Nakamura	0f0354f848	fix comments on release.yml	2022-11-24 23:26:04 +09:00
Ryo Nakamura	33d1adcdbb	fix codeql.yml	2022-11-24 23:20:51 +09:00
Ryo Nakamura	adbcb3701e	update README	2022-11-24 23:18:40 +09:00
Ryo Nakamura	5495e5dd61	fix codeql.yml: init submodules and apply patch to libssh	2022-11-24 23:05:45 +09:00
Ryo Nakamura	52fc2a71b0	fix build-macos.yml	2022-11-24 23:03:22 +09:00
Ryo Nakamura	b46a6f15c1	update github workflows to fit patched libssh	2022-11-24 22:59:59 +09:00
Ryo Nakamura	6f9aaeab80	update README: use patch instead of git apply	2022-11-20 18:20:10 +09:00
Ryo Nakamura	fc2d34eaee	remove `sudo`	2022-11-20 18:13:26 +09:00
Ryo Nakamura	e22bc5523b	fix CPACK build dependency	2022-11-20 18:03:56 +09:00
Ryo Nakamura	b6b283f8b5	set WITH_EXAMPLES OFF	2022-11-20 17:04:22 +09:00
Ryo Nakamura	4b5d300fa4	update README for cmkae with libssh	2022-11-20 16:50:00 +09:00
Ryo Nakamura	dc0dd60287	build libssh-static from mscp cmake	2022-11-20 16:14:07 +09:00
Ryo Nakamura	4129a47a3a	add ignore dirty for libssh	2022-11-20 15:38:25 +09:00
Ryo Nakamura	7079ff6542	add libssh 0.10.4 as submodule	2022-11-20 15:37:15 +09:00
Ryo Nakamura	8e266517da	use list(LENGTH) to iterate DIST_ lists	2022-11-20 02:02:43 +09:00
Ryo Nakamura	8395c05d67	fix CMakeLists.txt * install mscp from package at docker build phase. * add docker-*-all custom targets.	2022-11-20 00:37:01 +09:00
Ryo Nakamura	04b7ec3e4b	cleanup docker-related commands in cmake TODO: add bruild procedure for async_write	2022-11-19 23:40:29 +09:00
Ryo Nakamura	e3c9c82bb8	Merge branch 'async-write'	2022-11-18 22:33:13 +09:00
Ryo Nakamura	dca0241824	add O_TRUNC when the first open() for a file	2022-11-18 22:30:34 +09:00
Ryo Nakamura	f4d04b848e	don't stop on macos whem -m coremask is set	2022-11-18 22:24:46 +09:00
Ryo Nakamura	7f9c63fa92	add patch/README.md	2022-11-18 22:06:07 +09:00
Ryo Nakamura	fe8101ed51	add patch/README.md	2022-11-18 22:04:50 +09:00
Ryo Nakamura	2c66652f74	update README	2022-11-18 21:54:42 +09:00
Ryo Nakamura	04ae5ee1dc	remove pr_warn debug	2022-11-18 21:52:58 +09:00
Ryo Nakamura	e1bddb85bd	cleanup CMakeLists.txt * now mscp with async write is compilable on both macos and ubuntu	2022-11-18 21:51:43 +09:00
Ryo Nakamura	71efeaa4ba	fix type of id for sftp_async_write	2022-11-18 20:33:50 +09:00
Ryo Nakamura	9193911e6b	Merge branch 'main' of github.com:upa/sscp	2022-11-18 20:21:11 +09:00
Ryo Nakamura	5e7aa774ca	fix when copy multiple sources and various tiny fixes * when copying multiple sources, target must be directory * add multi-src copy test and parametrize src/dst prefixes * cleanup REAMDE (s/sessions/connections/g) * make error output in copy functions simple	2022-11-18 20:20:19 +09:00
Ryo Nakamura	c92a5f71d4	fix copy multiple files and various tiny fixes * when coping multiple files, target must be directory * add multi-src copy test and parametrize src/dst prefixes * cleanup REAMDE (s/sessions/connections/g) * make error output in copy functions simple	2022-11-18 14:42:23 +09:00
Ryo Nakamura	b8d58b1fba	tiny fix - set ssh nonblocking before closing sftp session - fix pprint for core mask	2022-11-18 13:47:24 +09:00
Ryo Nakamura	5ede4dc122	fix async handling	2022-11-17 23:46:51 +09:00
Ryo Nakamura	2d66f4ca14	fix idx increment (typo) and async write improves copy speed!	2022-11-17 22:09:59 +09:00
Ryo Nakamura	fb2f0b2e45	fix typo	2022-11-17 21:48:39 +09:00
Ryo Nakamura	d448f9eb8a	implement local-to-remote copy with async_write	2022-11-17 21:46:21 +09:00
Ryo Nakamura	a2b4a4c7b3	update README for building mscp with patched libssh	2022-11-17 20:54:24 +09:00
Ryo Nakamura	06c27b96f4	add libssh to .gitignore	2022-11-17 20:42:40 +09:00
Ryo Nakamura	7f1b7ec762	remove libssh git submodule	2022-11-17 20:42:00 +09:00
Ryo Nakamura	4d3c37382c	add libssh as submodule and LIBSSH_PATH to build mscp with static built libssh.	2022-11-17 20:29:39 +09:00
Ryo Nakamura	41da0c5cfe	check invalid coremask	2022-11-16 01:57:27 +09:00
Ryo Nakamura	a69115a4dc	add -m coremask option	2022-11-15 19:57:53 +09:00
Ryo Nakamura	0421172778	bump up version: 0.0.1	2022-11-13 18:23:55 +09:00
Ryo Nakamura	3bd72beb83	Update README.md	2022-11-13 18:17:52 +09:00
Ryo Nakamura	b8e204ae41	update README	2022-11-13 18:14:06 +09:00
Ryo Nakamura	613961b71d	run mscp -h last on ci build	2022-11-13 17:57:50 +09:00
Ryo Nakamura	8719b35694	add rocky 8.6 support	2022-11-13 17:53:46 +09:00
Ryo Nakamura	e9d5ceb462	add memory barrier to notify monitor thread of copy threads finished	2022-11-13 15:33:45 +09:00
Ryo Nakamura	81a7fbd2d8	add -a nr_ahead option	2022-11-13 15:31:12 +09:00
Ryo Nakamura	cfbbae860c	little trick to make progress bar stable	2022-11-12 17:45:08 +09:00
Ryo Nakamura	756e0759f9	fix buf size in remote to local copy. Too large buffer size for sftp_async_read causes unfinished copy: sftp_async_read returns 0 althrough data remains.	2022-11-12 17:30:51 +09:00
Ryo Nakamura	71d827d613	fix some thread handling	2022-11-12 16:11:20 +09:00
Ryo Nakamura	73e884f9c5	use sftp_async_read for remote to local copy	2022-11-12 15:30:01 +09:00
Ryo Nakamura	8eb9e69c1c	fix incorrect ret handling for read/write	2022-11-08 10:06:30 +00:00
Ryo Nakamura	04488f258c	fix docker/README.md	2022-11-07 02:26:35 +09:00
Ryo Nakamura	c6e469ff3e	add install from homebrew tap to README	2022-11-06 20:25:32 +09:00
Ryo Nakamura	e202939f9e	update README for install	2022-11-06 19:39:20 +09:00
@@ -1 +1 @@
 .0.0
 .0.5