bump version to 0.0.6

and fix typo in README

.github/workflows/build-macos.yml

@@ -19,18 +19,23 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
+      with:
+        submodules: true
 
-      # libssh and cmake are already installed on the macos-latest runner, 2022/11/5
+    - name: install build dependency
-      #    - name: install build dependency
+      run: ./scripts/install-build-deps.sh
-      #run: |
+
-      #  brew update
+    - name: save homebrew prefix
-      #  brew install libssh
+      id: brew-prefix
-      #  brew install cmake
+      run: echo "HOMEBREW_PREFIX=$(brew --prefix)" >> $GITHUB_OUTPUT
+
+    - name: patch to libssh
+      run: patch -d libssh -p1 < patch/libssh-0.10.4.patch
 
     - name: Configure CMake
       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
-      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
+      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DOPENSSL_ROOT_DIR=${{steps.brew-prefix.outputs.HOMEBREW_PREFIX}}/opt/openssl@1.1
 
     - name: Build
       # Build your program with the given configuration

.github/workflows/build-ubuntu.yml

@@ -19,9 +19,15 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
+      with:
+        submodules: true
+
     - name: install build dependency
       run: sudo ./scripts/install-build-deps.sh
 
+    - name: patch to libssh
+      run: patch -d libssh -p1 < patch/libssh-0.10.4.patch
+
     - name: Configure CMake
       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type

.github/workflows/codeql.yml

@@ -39,10 +39,15 @@ jobs:
     steps:
     - name: Checkout repository
       uses: actions/checkout@v3
+      with:
+        submodules: true
 
-    - name: Install build dependency
+    - name: install build dependency
       run: sudo ./scripts/install-build-deps.sh
 
+    - name: patch to libssh
+      run: patch -d libssh -p1 < patch/libssh-0.10.4.patch
+
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v2

.github/workflows/release.yml

@@ -4,6 +4,7 @@ on:
   push:
     tags:
       - "v*.*.*"
+  workflow_dispatch:
 
 env:
     BUILD_TYPE: Release
@@ -13,21 +14,32 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: true
+
+      - name: patch to libssh
+        run: patch -d libssh -p1 < patch/libssh-0.10.4.patch
+
+      # TODO: just building docker does not require packages. fix CMakeLists
+      - name: install build dependency
+        run: sudo ./scripts/install-build-deps.sh
 
       - name: Set variables
         run: |
           VER=$(cat VERSION)
           echo "VERSION=$VER" >> $GITHUB_ENV
 
-      # TODO: docker build does not require libssh. fix CMakeLists
-      - name: install build dependency
-        run: sudo ./scripts/install-build-deps.sh
-
       - name: Configure Cmake
-        run: cmake -B ${{github.workspace}}/build -DBUILD_PKG=1 -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
+        run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
 
-      - name: Build Packages
+      - name: Build Containers
-        run: make -C ${{github.workspace}}/build package-all-in-docker
+        run: make -C ${{github.workspace}}/build docker-build-all
+
+      - name: Test
+        run: make -C ${{github.workspace}}/build docker-test-all
+
+      - name: Retrieve packages from containers
+        run: make -C ${{github.workspace}}/build docker-pkg-all
 
       - name: Release
         uses: softprops/action-gh-release@v1
@@ -37,3 +49,30 @@ jobs:
             ${{github.workspace}}/build/mscp_${{env.VERSION}}-ubuntu-22.04-x86_64.deb
             ${{github.workspace}}/build/mscp_${{env.VERSION}}-centos-8-x86_64.rpm
             ${{github.workspace}}/build/mscp_${{env.VERSION}}-rocky-8.6-x86_64.rpm
+
+  source-release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+
+      - name: patch to libssh
+        run: patch -d libssh -p1 < patch/libssh-0.10.4.patch
+
+      - name: Set variables
+        run: |
+          VER=$(cat VERSION)
+          echo "VERSION=$VER" >> $GITHUB_ENV
+
+      - name: archive
+        run: |
+          cd ..
+          cp -r mscp mscp-${{env.VERSION}}
+          tar zcvf mscp-${{env.VERSION}}.tar.gz --exclude-vcs mscp-${{env.VERSION}}
+
+      - name: Release
+        uses: softprops/action-gh-release@v1
+        with:
+          files: |
+            ${{github.workspace}}/../mscp-${{env.VERSION}}.tar.gz

.github/workflows/test.yml

@@ -14,12 +14,21 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
-    - name: Build ubuntu 22.04 container
+      with:
-      run: docker build --rm -t mscp-ubuntu:22.04 -f docker/ubuntu-22.04.Dockerfile .
+        submodules: true
-    - name: Run test on ubuntu 22.04
-      run: docker run --init --rm mscp-ubuntu:22.04 /mscp/scripts/test-in-container.sh
 
-    - name: Build centos 8 container
+    - name: patch to libssh
-      run: docker build --rm -t mscp-centos:8 -f docker/centos-8.Dockerfile .
+      run: patch -d libssh -p1 < patch/libssh-0.10.4.patch
-    - name: Run test on centos 8
+
-      run: docker run --init --rm mscp-centos:8 /mscp/scripts/test-in-container.sh
+    # TODO: just building docker does not require libssh. fix CMakeLists
+    - name: install build dependency
+      run: sudo ./scripts/install-build-deps.sh
+
+    - name: configure CMake
+      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
+
+    - name: Build Containers
+      run: make -C ${{github.workspace}}/build docker-build-all
+
+    - name: Run Test
+      run: make -C ${{github.workspace}}/build docker-test-all

.gitignore

@@ -1,2 +1,3 @@
 build
+compile_commands.json
 .*.swp

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "libssh"]
+	path = libssh
+	url = https://git.libssh.org/projects/libssh.git
+	ignore = dirty

CMakeLists.txt

@@ -7,22 +7,46 @@ project(mscp
 	LANGUAGES C)
 
 set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -DDEBUG")
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/modules)
+
+
+# add libssh static library
+add_subdirectory(libssh EXCLUDE_FROM_ALL)
 
 if(APPLE)
 	list(APPEND CMAKE_PREFIX_PATH /usr/local) # intel mac homebrew prefix
 	list(APPEND CMAKE_PREFIX_PATH /opt/homebrew) # arm mac homebrew prefix
 endif() # APPLE
 
-find_package(libssh REQUIRED)
 
+# mscp executable
 add_executable(mscp src/main.c src/platform.c src/ssh.c src/file.c src/pprint.c)
-target_include_directories(mscp PUBLIC ./src)
+
-target_link_libraries(mscp ssh pthread m)
+set(MSCP_LINK_LIBS m pthread)
+set(MSCP_LINK_DIRS "")
+set(MSCP_COMPILE_OPTS "")
+set(MSCP_INCLUDE_DIRS ${mscp_SOURCE_DIR}/src)
+
+list(APPEND MSCP_COMPILE_OPTS -iquote ${CMAKE_CURRENT_BINARY_DIR}/libssh/include)
+list(APPEND MSCP_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/libssh/include)
+list(APPEND MSCP_LINK_LIBS ssh-static)
+
+find_package(GSSAPI)
+list(APPEND MSCP_LINK_LIBS ${GSSAPI_LIBRARIES})
+
+find_package(OpenSSL)
+list(APPEND MSCP_LINK_LIBS ${OPENSSL_LIBRARIES})
+
+find_package(ZLIB)
+list(APPEND MSCP_LINK_LIBS ${ZLIB_LIBRARIES})
+
+target_include_directories(mscp PRIVATE ${MSCP_INCLUDE_DIRS})
+target_link_directories(mscp PRIVATE ${MSCP_LINK_DIRS})
+target_link_libraries(mscp PRIVATE ${MSCP_LINK_LIBS})
+target_compile_options(mscp PRIVATE ${MSCP_COMPILE_OPTS})
 target_compile_definitions(mscp PUBLIC _VERSION="${PROJECT_VERSION}")
 
-install(TARGETS mscp
+install(TARGETS mscp RUNTIME DESTINATION bin)
-	RUNTIME		DESTINATION bin
-)
 
 
 
@@ -35,93 +59,90 @@ add_test(NAME	pytest
 enable_testing()
 
 
-# Build Packages
-if(BUILD_PKG)
 
-	# CPACK Rules
+# CPACK Rules
-	set(CPACK_SET_DESTDIR		true)
+set(CPACK_SET_DESTDIR		true)
-	set(CPACK_PROJECT_NAME		${PROJECT_NAME})
+set(CPACK_PROJECT_NAME		${PROJECT_NAME})
-	set(CPACK_PROJECT_VERSION	${PROJECT_VERSION})
+set(CPACK_PROJECT_VERSION	${PROJECT_VERSION})
-	set(CPACK_PACKAGE_CONTACT	"Ryo Nakamura <upa@haeena.net>")
+set(CPACK_PACKAGE_CONTACT	"Ryo Nakamura <upa@haeena.net>")
-	set(CPACK_PACKAGE_DESCRIPTION
+set(CPACK_PACKAGE_DESCRIPTION
-		"mscp, copy files over multiple ssh connections")
+	"mscp, copy files over multiple ssh connections")
 
-	# on linux
+execute_process(COMMAND uname -m
-	if(UNIX AND NOT APPLE)
+	OUTPUT_VARIABLE ARCH OUTPUT_STRIP_TRAILING_WHITESPACE)
-		execute_process(COMMAND
-			bash "-c" "cat /etc/os-release|grep '^ID='|cut -d '=' -f 2|tr -d '\"'"
-			OUTPUT_VARIABLE DIST_NAME OUTPUT_STRIP_TRAILING_WHITESPACE)
-		execute_process(COMMAND
-			bash "-c" "cat /etc/os-release|grep '^VERSION_ID='|cut -d '=' -f 2|tr -d '\"'"
-			OUTPUT_VARIABLE DIST_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE)
-		execute_process(COMMAND uname -p
-			OUTPUT_VARIABLE ARCH OUTPUT_STRIP_TRAILING_WHITESPACE)
-		set(PACKAGE_FILE_NAME
-			${PROJECT_NAME}_${PROJECT_VERSION}-${DIST_NAME}-${DIST_VERSION}-${ARCH})
 
-		set(CPACK_DEBIAN_FILE_NAME		${PACKAGE_FILE_NAME}.deb)
+if(UNIX AND NOT APPLE) # on linux
-		set(CPACK_DEBIAN_PACKAGE_DEPENDS	"libssh-4")
+	execute_process(COMMAND
-		set(CPACK_DEBIAN_PACKAGE_HOMEPAGE	"https://github.com/upa/mscp")
+		bash "-c" "cat /etc/os-release|grep '^ID='|cut -d '=' -f 2|tr -d '\"'"
+		OUTPUT_VARIABLE DIST_NAME OUTPUT_STRIP_TRAILING_WHITESPACE)
+	execute_process(COMMAND
+		bash "-c" "cat /etc/os-release|grep '^VERSION_ID='|cut -d '=' -f 2|tr -d '\"'"
+		OUTPUT_VARIABLE DIST_VER OUTPUT_STRIP_TRAILING_WHITESPACE)
+	execute_process(COMMAND
+		bash "-c" "${mscp_SOURCE_DIR}/scripts/print-install-deps.sh ${DIST_NAME}-${DIST_VER}"
+		OUTPUT_VARIABLE DIST_DEP OUTPUT_STRIP_TRAILING_WHITESPACE)
 
-		set(CPACK_RPM_FILE_NAME			${PACKAGE_FILE_NAME}.rpm)
+	set(PACKAGE_FILE_NAME
-		set(CPACK_RPM_PACKAGE_REQUIRES		"libssh")
+		${PROJECT_NAME}_${PROJECT_VERSION}-${DIST_NAME}-${DIST_VER}-${ARCH})
-		set(CPACK_RPM_PACKAGE_HOMEPAGE		"https://github.com/upa/mscp")
-		set(CPACK_RPM_PACKAGE_DESCRIPTION	${CPACK_PACKAGE_DESCRIPTION})
 
-	endif() # on linux
+	set(CPACK_DEBIAN_FILE_NAME		${PACKAGE_FILE_NAME}.deb)
+	set(CPACK_DEBIAN_PACKAGE_DEPENDS	${DIST_DEP})
+	set(CPACK_DEBIAN_PACKAGE_HOMEPAGE	"https://github.com/upa/mscp")
 
-	include(CPack)
+	set(CPACK_RPM_FILE_NAME			${PACKAGE_FILE_NAME}.rpm)
+	set(CPACK_RPM_PACKAGE_REQUIRES		${DIST_DEP})
+	set(CPACK_RPM_PACKAGE_HOMEPAGE		"https://github.com/upa/mscp")
+	set(CPACK_RPM_PACKAGE_DESCRIPTION	${CPACK_PACKAGE_DESCRIPTION})
+endif() # on linux
+
+include(CPack)
 
 
-	# Custom commands to build mscp in docker containers
 
-	# Ubuntu 20.04
+# Custom targets to build and test mscp in docker containers.
-	add_custom_target(package-ubuntu-20.04-in-docker
+# foreach(IN ZIP_LISTS) (cmake >= 3.17) can shorten the following lists.
-		COMMENT "Build mscp in ubuntu 20.04 docker container"
+# However, ubuntu 20.04 has cmake 3.16.3. So this is a roundabout trick.
-		WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+list(APPEND DIST_NAMES	ubuntu	ubuntu	centos	rocky)
-		COMMAND docker build -t mscp-ubuntu:20.04 -f docker/ubuntu-20.04.Dockerfile .
+list(APPEND DIST_VERS	20.04	22.04	8	8.6)
-		COMMAND docker run --init --rm mscp-ubuntu:20.04
+list(APPEND DIST_PKGS	deb	deb	rpm	rpm)
-			/mscp/scripts/test-in-container.sh
-		COMMAND docker run --rm -v ${CMAKE_BINARY_DIR}:/out mscp-ubuntu:20.04
-			cp /mscp/build/mscp_${PROJECT_VERSION}-ubuntu-20.04-${ARCH}.deb /out/)
 
-	# Ubuntu 22.04
+list(LENGTH DIST_NAMES _DIST_LISTLEN)
-	add_custom_target(package-ubuntu-22.04-in-docker
+math(EXPR DIST_LISTLEN "${_DIST_LISTLEN} - 1")
-		COMMENT "Build mscp in ubuntu 22.04 docker container"
-		WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-		COMMAND docker build -t mscp-ubuntu:22.04 -f docker/ubuntu-22.04.Dockerfile .
-		COMMAND docker run --init --rm mscp-ubuntu:22.04
-			/mscp/scripts/test-in-container.sh
-		COMMAND docker run --rm -v ${CMAKE_BINARY_DIR}:/out mscp-ubuntu:22.04
-		cp /mscp/build/mscp_${PROJECT_VERSION}-ubuntu-22.04-${ARCH}.deb /out/)
 
-	# CentOS 8
+foreach(x RANGE ${DIST_LISTLEN})
-	add_custom_target(package-centos-8-in-docker
+	list(GET DIST_NAMES	${x} DIST_NAME)
-		COMMENT "Build mscp in centos 8 docker container"
+	list(GET DIST_VERS	${x} DIST_VER)
-		WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+	list(GET DIST_PKGS	${x} DIST_PKG)
-		COMMAND docker build -t mscp-centos:8 -f docker/centos-8.Dockerfile .
-		COMMAND docker run --init --rm mscp-centos:8
-			/mscp/scripts/test-in-container.sh
-		COMMAND docker run --rm -v ${CMAKE_BINARY_DIR}:/out mscp-centos:8
-		cp /mscp/build/mscp_${PROJECT_VERSION}-centos-8-${ARCH}.rpm /out/)
 
-	# Rocky 8.6
+	set(DOCKER_IMAGE mscp-${DIST_NAME}:${DIST_VER})
-	add_custom_target(package-rocky-8.6-in-docker
+	set(DOCKER_INDEX ${DIST_NAME}-${DIST_VER})
-		COMMENT "Build mscp in rocky 8.6 docker container"
+	set(PKG_FILE_NAME
-		WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+		mscp_${PROJECT_VERSION}-${DIST_NAME}-${DIST_VER}-${ARCH}.${DIST_PKG})
-		COMMAND docker build -t mscp-rocky:8.6 -f docker/rocky-8.6.Dockerfile .
-		COMMAND docker run --init --rm mscp-rocky:8.6
-			/mscp/scripts/test-in-container.sh
-		COMMAND docker run --rm -v ${CMAKE_BINARY_DIR}:/out mscp-rocky:8.6
-		cp /mscp/build/mscp_${PROJECT_VERSION}-rocky-8.6-${ARCH}.rpm /out/)
 
-	# build on all conatiners
+	add_custom_target(docker-build-${DOCKER_INDEX}
-	add_custom_target(package-all-in-docker
+		COMMENT "Build mscp in ${DOCKER_IMAGE} container"
-		COMMENT "Build mscp in all docker containers"
+		WORKING_DIRECTORY ${mscp_SOURCE_DIR}
-		DEPENDS package-ubuntu-20.04-in-docker
+		COMMAND
-		DEPENDS package-ubuntu-22.04-in-docker
+		docker build -t ${DOCKER_IMAGE} -f docker/${DOCKER_INDEX}.Dockerfile .)
-		DEPENDS package-centos-8-in-docker
-		DEPENDS package-rocky-8.6-in-docker)
 
-endif() # BUILD_PKG
+	add_custom_target(docker-test-${DOCKER_INDEX}
+		COMMENT "Test mscp in ${DOCKER_IMAGE} container"
+		WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+		COMMAND
+		docker run --init --rm ${DOCKER_IMAGE} /mscp/scripts/test-in-container.sh)
 
+	add_custom_target(docker-pkg-${DOCKER_INDEX}
+		COMMENT "Retrieve mscp package from ${DOCKER_IMAGE} container"
+		WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+		COMMAND
+		docker run --rm -v ${CMAKE_BINARY_DIR}:/out ${DOCKER_IMAGE}
+		cp /mscp/build/${PKG_FILE_NAME} /out/)
+
+	list(APPEND DOCKER_BUILDS	docker-build-${DOCKER_INDEX})
+	list(APPEND DOCKER_TESTS	docker-test-${DOCKER_INDEX})
+	list(APPEND DOCKER_PKGS		docker-pkg-${DOCKER_INDEX})
+endforeach()
+
+add_custom_target(docker-build-all	DEPENDS ${DOCKER_BUILDS})
+add_custom_target(docker-test-all	DEPENDS ${DOCKER_TESTS})
+add_custom_target(docker-pkg-all	DEPENDS ${DOCKER_PKGS})

README.md

@@ -1,25 +1,29 @@
-# mscp
+# mscp: multi-threaded scp
 
 [![build on ubuntu](https://github.com/upa/mscp/actions/workflows/build-ubuntu.yml/badge.svg)](https://github.com/upa/mscp/actions/workflows/build-ubuntu.yml) [![build on macOS](https://github.com/upa/mscp/actions/workflows/build-macos.yml/badge.svg)](https://github.com/upa/mscp/actions/workflows/build-macos.yml) [![test](https://github.com/upa/mscp/actions/workflows/test.yml/badge.svg)](https://github.com/upa/mscp/actions/workflows/test.yml)
 
 
 `mscp`, a variant of `scp`, copies files over multiple ssh (SFTP)
-sessions. Multiple threads in mscp transfer (1) multiple files
+connections. Multiple threads and connections in mscp transfer (1)
-simultaneously and (2) a large file in parallel. It may shorten the
+multiple files simultaneously and (2) a large file in parallel. It
-waiting time for transferring a lot of/large files over networks.
+would shorten the waiting time for transferring a lot of/large files
+over networks.
 
 You can use `mscp` like `scp`, for example, `mscp
 user@example.com:srcfile /tmp/dstfile`. Remote hosts only need to run
-standard `sshd` supporting the SFTP subsystem, and you need to be able
+standard `sshd` supporting the SFTP subsystem (e.g. openssh-server),
-to ssh to the hosts (as usual). `mscp` does not require anything else.
+and you need to be able to ssh to the hosts as usual. `mscp` does not
+require anything else.
+
+https://user-images.githubusercontent.com/184632/206889149-7cc6178a-6f0f-41e6-855c-d25e15a9abc5.mp4
 
 
-Differences from `scp` are:
+Differences from `scp` on usage:
 
 - remote glob on remote shell expansion is not supported.
 - remote to remote copy is not supported.
 - `-r` option is not needed.
-- and any other differences I have not implemented and noticed...
+- and any other differences I have not implemented and noticed.
 
 
 ## Install
@@ -36,40 +40,43 @@ Download a package for your environment from [Releases
 page](https://github.com/upa/mscp/releases).
 
 
-## Build from source
+## Build
 
-mscp depends on [libssh](https://www.libssh.org/).
+mscp depends on a patched [libssh](https://www.libssh.org/).  The
+patch introduces asynchronous SFTP Write, which is derived from
+https://github.com/limes-datentechnik-gmbh/libssh (see [Re: SFTP Write
+async](https://archive.libssh.org/libssh/2020-06/0000004.html)).
 
-- macOS
+Currently macOS and Linux (Ubuntu, CentOS, Rocky) are supported.
-
-```console
-brew install libssh
-```
-
-- ubuntu
-
-```console
-sudo apt-get install libssh-dev
-```
-
-- rhel
-
-```console
-sudo yum install libssh-devel
-```
-
-Clone and build this repositoy.
 
 ```console
+# clone this repository
 git clone https://github.com/upa/mscp.git
 cd mscp
 
+# prepare patched libssh
+git submodule update --init
+patch -d libssh -p1 < patch/libssh-0.10.4.patch
+
+# install build dependency
+bash ./scripts/install-build-deps.sh
+
+# configure mscp
 mkdir build && cd build
-cmake .. && make
+cmake ..
+
+# in macOS, you may need OPENSSL_ROOT_DIR for cmake:
+# cmake .. -DOPENSSL_ROOT_DIR=$(brew --prefix)/opt/openssl@1.1
+
+# build
+make
 
 # install the mscp binary to CMAKE_INSTALL_PREFIX/bin (usually /usr/local/bin)
 make install
 ```
+Source tar balls (`mscp-X.X.X.tar.gz`, not `Source code`) in
+[Releases page](https://github.com/upa/mscp/releases) contains the patched version
+of libssh. So you can start from cmake with it.
 
 ## Run
 
@@ -77,93 +84,92 @@ make install
 
 ```console
 $ mscp
-mscp v0.0.0: copy files over multiple ssh connections
+mscp v0.0.5: copy files over multiple ssh connections
 
-Usage: mscp [vqDCHdh] [-n nr_conns]
+Usage: mscp [vqDCHdNh] [-n nr_conns] [-m coremask]
-            [-s min_chunk_sz] [-S max_chunk_sz]
+            [-s min_chunk_sz] [-S max_chunk_sz] [-a nr_ahead] [-b buf_sz]
-            [-b sftp_buf_sz] [-B io_buf_sz] [-a nr_ahead]
             [-l login_name] [-p port] [-i identity_file]
-            [-c cipher_spec] source ... target
+            [-c cipher_spec] [-M hmac_spec] source ... target
 ```
 
-- Example: copy an 8GB file on tmpfs over a 100Gbps link
+- Example: copy a 15GB file on memory over a 100Gbps link
   - Two Intel Xeon Gold 6130 machines directly connected with Intel E810 100Gbps NICs.
+  - Default `openssh-server` runs on the remote host.
 
 ```console
-$ mscp /tmp/test.img 10.0.0.1:/tmp/
+$ mscp /var/ram/test.img 10.0.0.1:/var/ram/
-[=====================================================] 100% 8GB/8GB 3.02GB/s 
+[======================================] 100%   15GB/15GB    1.7GB/s  00:00 ETA
+```
+
+```console
+# with some optimizations. top speed reaches 3.0GB/s.
+$ mscp -n 5 -m 0x1f -c aes128-gcm@openssh.com /var/ram/test.img 10.0.0.1:/var/ram/
+[======================================] 100%   15GB/15GB    2.4GB/s  00:00 ETA
 ```
 
 - `-v` option increments verbose output level.
 
 ```console
 $ mscp test 10.0.0.1:
-[=====================================================] 100% 13B/13B 2.41KB/s 
+[======================================] 100%   26B /26B     6.3KB/s  00:00 ETA
+```
 
-$ mscp -v test 10.0.0.1:
+```console
-file test/test.txt (local) -> ./test/test.txt (remote) 9B
+$ mscp -vv test 10.0.0.1:
-file test/test2/2.txt (local) -> ./test/test2/2.txt (remote) 2B
+number of connections: 7
-file test/1.txt (local) -> ./test/1.txt (remote) 2B
-copy start: test/test.txt
-copy start: test/1.txt
-copy start: test/test2/2.txt
-copy done: test/1.txt
-copy done: test/test2/2.txt
-copy done: test/test.txt
-[=====================================================] 100% 13B/13B 2.51KB/s 
-
-$ mscp -vv -n 4 test 10.0.0.1:
 connecting to 10.0.0.1 for checking destinations...
-file test/test.txt (local) -> ./test/test.txt (remote) 9B
+file test/testdir/asdf (local) -> ./test/testdir/asdf (remote) 9B
-file test/test2/2.txt (local) -> ./test/test2/2.txt (remote) 2B
+file test/testdir/qwer (local) -> ./test/testdir/qwer (remote) 5B
-file test/1.txt (local) -> ./test/1.txt (remote) 2B
+file test/test1 (local) -> ./test/test1 (remote) 6B
+file test/test2 (local) -> ./test/test2 (remote) 6B
+we have only 4 chunk(s). set number of connections to 4
 connecting to 10.0.0.1 for a copy thread...
 connecting to 10.0.0.1 for a copy thread...
 connecting to 10.0.0.1 for a copy thread...
-connecting to 10.0.0.1 for a copy thread...
+copy start: test/test1
-copy start: test/test.txt
+copy start: test/test2
-copy start: test/1.txt
+copy done: test/test1
-copy start: test/test2/2.txt
+copy start: test/testdir/asdf
-copy done: test/test.txt
+copy done: test/test2
-copy done: test/test2/2.txt
+copy start: test/testdir/qwer
-copy done: test/1.txt
+copy done: test/testdir/qwer
-[=====================================================] 100% 13B/13B 3.27KB/s
+copy done: test/testdir/asdf
+[======================================] 100%   26B /26B     5.2KB/s  00:00 ETA
 ```
 
 - Full usage
 
 ```console
 $ mscp -h
-mscp v0.0.0: copy files over multiple ssh connections
+mscp v0.0.6: copy files over multiple ssh connections
 
-Usage: mscp [vqDCHdh] [-n nr_conns]
+Usage: mscp [vqDCHdNh] [-n nr_conns] [-m coremask]
-            [-s min_chunk_sz] [-S max_chunk_sz]
+            [-s min_chunk_sz] [-S max_chunk_sz] [-a nr_ahead] [-b buf_sz]
-            [-b sftp_buf_sz] [-B io_buf_sz] [-a nr_ahead]
             [-l login_name] [-p port] [-i identity_file]
-            [-c cipher_spec] source ... target
+            [-c cipher_spec] [-M hmac_spec] source ... target
 
-    -n NR_CONNECTIONS  number of connections (default: half of # of cpu cores)
+    -n NR_CONNECTIONS  number of connections (default: floor(log(cores)*2)+1)
+    -m COREMASK        hex value to specify cores where threads pinned
     -s MIN_CHUNK_SIZE  min chunk size (default: 64MB)
-    -S MAX_CHUNK_SIZE  max chunk size (default: filesize / nr_conn)
+    -S MAX_CHUNK_SIZE  max chunk size (default: filesize/nr_conn)
 
-    -b SFTP_BUF_SIZE   buf size for sftp_read/write (default 131072B)
+    -a NR_AHEAD        number of inflight SFTP commands (default: 32)
-    -B IO_BUF_SIZE     buf size for read/write (default 131072B)
+    -b BUF_SZ          buffer size for i/o and transfer
-                       Note that the default value is derived from
-                       qemu/block/ssh.c. need investigation...
-                       -b and -B affect only local to remote copy
-    -a NR_AHEAD        number of inflight SFTP read commands (default 16)
 
     -v                 increment verbose output level
     -q                 disable output
     -D                 dry run
+    -r                 no effect
 
     -l LOGIN_NAME      login name
     -p PORT            port number
-    -i IDENTITY        identity file for publickey authentication
+    -i IDENTITY        identity file for public key authentication
-    -c CIPHER          cipher spec, see `ssh -Q cipher`
+    -c CIPHER          cipher spec
+    -M HMAC            hmac spec
     -C                 enable compression on libssh
     -H                 disable hostkey check
     -d                 increment ssh debug output level
+    -N                 disable tcp nodelay (default on)
     -h                 print this help
 ```
 

VERSION

@@ -1 +1 @@
-0.0.1
+0.0.6

cmake/modules/FindGSSAPI.cmake

@@ -0,0 +1,325 @@
+# - Try to find GSSAPI
+# Once done this will define
+#
+#  KRB5_CONFIG - Path to krb5-config
+#  GSSAPI_ROOT_DIR - Set this variable to the root installation of GSSAPI
+#
+# Read-Only variables:
+#  GSSAPI_FLAVOR_MIT - set to TURE if MIT Kerberos has been found
+#  GSSAPI_FLAVOR_HEIMDAL - set to TRUE if Heimdal Keberos has been found
+#  GSSAPI_FOUND - system has GSSAPI
+#  GSSAPI_INCLUDE_DIR - the GSSAPI include directory
+#  GSSAPI_LIBRARIES - Link these to use GSSAPI
+#  GSSAPI_DEFINITIONS - Compiler switches required for using GSSAPI
+#
+#=============================================================================
+#  Copyright (c) 2013 Andreas Schneider <asn@cryptomilk.org>
+#
+#  Distributed under the OSI-approved BSD License (the "License");
+#  see accompanying file Copyright.txt for details.
+#
+#  This software is distributed WITHOUT ANY WARRANTY; without even the
+#  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#  See the License for more information.
+#=============================================================================
+#
+
+find_path(GSSAPI_ROOT_DIR
+    NAMES
+        include/gssapi.h
+        include/gssapi/gssapi.h
+    HINTS
+        ${_GSSAPI_ROOT_HINTS}
+    PATHS
+        ${_GSSAPI_ROOT_PATHS}
+)
+mark_as_advanced(GSSAPI_ROOT_DIR)
+
+if (UNIX)
+    find_program(KRB5_CONFIG
+        NAMES
+            krb5-config
+        PATHS
+            ${GSSAPI_ROOT_DIR}/bin
+            /opt/local/bin)
+    mark_as_advanced(KRB5_CONFIG)
+
+    if (KRB5_CONFIG)
+        # Check if we have MIT KRB5
+        execute_process(
+            COMMAND
+                ${KRB5_CONFIG} --vendor
+            RESULT_VARIABLE
+                _GSSAPI_VENDOR_RESULT
+            OUTPUT_VARIABLE
+                _GSSAPI_VENDOR_STRING)
+
+        if ((_GSSAPI_VENDOR_STRING MATCHES ".*Massachusetts.*") OR (_GSSAPI_VENDOR_STRING
+                        MATCHES ".*MITKerberosShim.*"))
+            set(GSSAPI_FLAVOR_MIT TRUE)
+        else()
+            execute_process(
+                COMMAND
+                    ${KRB5_CONFIG} --libs gssapi
+                RESULT_VARIABLE
+                    _GSSAPI_LIBS_RESULT
+                OUTPUT_VARIABLE
+                    _GSSAPI_LIBS_STRING)
+
+            if (_GSSAPI_LIBS_STRING MATCHES ".*roken.*")
+                set(GSSAPI_FLAVOR_HEIMDAL TRUE)
+            endif()
+        endif()
+
+        # Get the include dir
+        execute_process(
+            COMMAND
+                ${KRB5_CONFIG} --cflags gssapi
+            RESULT_VARIABLE
+                _GSSAPI_INCLUDE_RESULT
+            OUTPUT_VARIABLE
+                _GSSAPI_INCLUDE_STRING)
+        string(REGEX REPLACE "(\r?\n)+$" "" _GSSAPI_INCLUDE_STRING "${_GSSAPI_INCLUDE_STRING}")
+        string(REGEX REPLACE " *-I" "" _GSSAPI_INCLUDEDIR "${_GSSAPI_INCLUDE_STRING}")
+    endif()
+
+    if (NOT GSSAPI_FLAVOR_MIT AND NOT GSSAPI_FLAVOR_HEIMDAL)
+        # Check for HEIMDAL
+        find_package(PkgConfig)
+        if (PKG_CONFIG_FOUND)
+            pkg_check_modules(_GSSAPI heimdal-gssapi)
+        endif (PKG_CONFIG_FOUND)
+
+        if (_GSSAPI_FOUND)
+            set(GSSAPI_FLAVOR_HEIMDAL TRUE)
+        else()
+            find_path(_GSSAPI_ROKEN
+                NAMES
+                    roken.h
+                PATHS
+                    ${GSSAPI_ROOT_DIR}/include
+                    ${_GSSAPI_INCLUDEDIR})
+            if (_GSSAPI_ROKEN)
+                set(GSSAPI_FLAVOR_HEIMDAL TRUE)
+            endif()
+        endif ()
+    endif()
+endif (UNIX)
+
+find_path(GSSAPI_INCLUDE_DIR
+    NAMES
+        gssapi.h
+        gssapi/gssapi.h
+    PATHS
+        ${GSSAPI_ROOT_DIR}/include
+        ${_GSSAPI_INCLUDEDIR}
+)
+
+if (GSSAPI_FLAVOR_MIT)
+    find_library(GSSAPI_LIBRARY
+        NAMES
+            gssapi_krb5
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(KRB5_LIBRARY
+        NAMES
+            krb5
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(K5CRYPTO_LIBRARY
+        NAMES
+            k5crypto
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(COM_ERR_LIBRARY
+        NAMES
+            com_err
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    if (GSSAPI_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${GSSAPI_LIBRARY}
+        )
+    endif (GSSAPI_LIBRARY)
+
+    if (KRB5_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${KRB5_LIBRARY}
+        )
+    endif (KRB5_LIBRARY)
+
+    if (K5CRYPTO_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${K5CRYPTO_LIBRARY}
+        )
+    endif (K5CRYPTO_LIBRARY)
+
+    if (COM_ERR_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${COM_ERR_LIBRARY}
+        )
+    endif (COM_ERR_LIBRARY)
+endif (GSSAPI_FLAVOR_MIT)
+
+if (GSSAPI_FLAVOR_HEIMDAL)
+    find_library(GSSAPI_LIBRARY
+        NAMES
+            gssapi
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(KRB5_LIBRARY
+        NAMES
+            krb5
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(HCRYPTO_LIBRARY
+        NAMES
+            hcrypto
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(COM_ERR_LIBRARY
+        NAMES
+            com_err
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(HEIMNTLM_LIBRARY
+        NAMES
+            heimntlm
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(HX509_LIBRARY
+        NAMES
+            hx509
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(ASN1_LIBRARY
+        NAMES
+            asn1
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(WIND_LIBRARY
+        NAMES
+            wind
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    find_library(ROKEN_LIBRARY
+        NAMES
+            roken
+        PATHS
+            ${GSSAPI_ROOT_DIR}/lib
+            ${_GSSAPI_LIBDIR}
+    )
+
+    if (GSSAPI_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${GSSAPI_LIBRARY}
+        )
+    endif (GSSAPI_LIBRARY)
+
+    if (KRB5_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${KRB5_LIBRARY}
+        )
+    endif (KRB5_LIBRARY)
+
+    if (HCRYPTO_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${HCRYPTO_LIBRARY}
+        )
+    endif (HCRYPTO_LIBRARY)
+
+    if (COM_ERR_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${COM_ERR_LIBRARY}
+        )
+    endif (COM_ERR_LIBRARY)
+
+    if (HEIMNTLM_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${HEIMNTLM_LIBRARY}
+        )
+    endif (HEIMNTLM_LIBRARY)
+
+    if (HX509_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${HX509_LIBRARY}
+        )
+    endif (HX509_LIBRARY)
+
+    if (ASN1_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${ASN1_LIBRARY}
+        )
+    endif (ASN1_LIBRARY)
+
+    if (WIND_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${WIND_LIBRARY}
+        )
+    endif (WIND_LIBRARY)
+
+    if (ROKEN_LIBRARY)
+        set(GSSAPI_LIBRARIES
+            ${GSSAPI_LIBRARIES}
+            ${WIND_LIBRARY}
+        )
+    endif (ROKEN_LIBRARY)
+endif (GSSAPI_FLAVOR_HEIMDAL)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(GSSAPI DEFAULT_MSG GSSAPI_LIBRARIES GSSAPI_INCLUDE_DIR)
+
+if (GSSAPI_INCLUDE_DIRS AND GSSAPI_LIBRARIES)
+    set(GSSAPI_FOUND TRUE)
+endif (GSSAPI_INCLUDE_DIRS AND GSSAPI_LIBRARIES)
+
+# show the GSSAPI_INCLUDE_DIRS and GSSAPI_LIBRARIES variables only in the advanced view
+mark_as_advanced(GSSAPI_INCLUDE_DIRS GSSAPI_LIBRARIES)

docker/centos-8.Dockerfile

@@ -28,8 +28,9 @@ RUN ${mscpdir}/scripts/install-build-deps.sh
 # build
 RUN cd ${mscpdir}			\
         && rm -rf build			\
-        && cmake -B build -DBUILD_PKG=1 \
+        && cmake -B build 		\
         && cd ${mscpdir}/build          \
 	&& make				\
-	&& cpack -G RPM CPackConfig.cmake
+	&& cpack -G RPM CPackConfig.cmake \
+	&& rpm -iv *.rpm
 

docker/rocky-8.6.Dockerfile

@@ -5,7 +5,7 @@ ARG mscpdir="/mscp"
 COPY . ${mscpdir}
 
 # install numpy and pytest, sshd for test, and rpm-build
-RUN set -ex && yum -y update && yum -y install \
+RUN set -ex && yum -y install \
 	python3 python3-pip openssh openssh-server openssh-clients rpm-build
 
 RUN python3 -m pip install numpy pytest
@@ -23,8 +23,9 @@ RUN ${mscpdir}/scripts/install-build-deps.sh
 # build
 RUN cd ${mscpdir}			\
         && rm -rf build			\
-        && cmake -B build -DBUILD_PKG=1 \
+        && cmake -B build		\
         && cd ${mscpdir}/build          \
 	&& make				\
-	&& cpack -G RPM CPackConfig.cmake
+	&& cpack -G RPM CPackConfig.cmake \
+	&& rpm -iv *.rpm
 

docker/ubuntu-20.04.Dockerfile

@@ -28,7 +28,8 @@ RUN ${mscpdir}/scripts/install-build-deps.sh
 # build
 RUN cd ${mscpdir}			\
 	&& rm -rf build			\
-	&& cmake -B build -DBUILD_PKG=1	\
+	&& cmake -B build		\
 	&& cd ${mscpdir}/build		\
 	&& make				\
-	&& cpack -G DEB CPackConfig.cmake 
+	&& cpack -G DEB CPackConfig.cmake \
+	&& dpkg -i *.deb

docker/ubuntu-22.04.Dockerfile

@@ -28,7 +28,8 @@ RUN ${mscpdir}/scripts/install-build-deps.sh
 # build
 RUN cd ${mscpdir}			\
 	&& rm -rf build			\
-	&& cmake -B build -DBUILD_PKG=1	\
+	&& cmake -B build		\
 	&& cd ${mscpdir}/build		\
 	&& make				\
-	&& cpack -G DEB CPackConfig.cmake 
+	&& cpack -G DEB CPackConfig.cmake \
+	&& dpkg -i *.deb

patch/README.md

@@ -0,0 +1,13 @@
+
+Patch(es) in this directory introduces `sftp_async_write()` and
+`sftp_async_write_end()` to libssh. Those implementations are derived
+from https://github.com/limes-datentechnik-gmbh/libssh. See [Re: SFTP
+Write async](https://archive.libssh.org/libssh/2020-06/0000004.html).
+
+```console
+git clone https://git.libssh.org/projects/libssh.git/ --depth=1 -b libssh-0.10.4
+cd libssh
+git apply ../pathc/libssh-0.10.4.patch
+
+# then build libssh
+```

patch/libssh-0.10.4.patch

@@ -0,0 +1,342 @@
+diff --git a/DefineOptions.cmake b/DefineOptions.cmake
+index 068db988..5fc3c8fc 100644
+--- a/DefineOptions.cmake
++++ b/DefineOptions.cmake
+@@ -1,7 +1,7 @@
+ option(WITH_GSSAPI "Build with GSSAPI support" ON)
+ option(WITH_ZLIB "Build with ZLIB support" ON)
+ option(WITH_SFTP "Build with SFTP support" ON)
+-option(WITH_SERVER "Build with SSH server support" ON)
++option(WITH_SERVER "Build with SSH server support" OFF)
+ option(WITH_DEBUG_CRYPTO "Build with cryto debug output" OFF)
+ option(WITH_DEBUG_PACKET "Build with packet debug output" OFF)
+ option(WITH_DEBUG_CALLTRACE "Build with calltrace debug output" ON)
+@@ -11,13 +11,13 @@ option(WITH_MBEDTLS "Compile against libmbedtls" OFF)
+ option(WITH_BLOWFISH_CIPHER "Compile with blowfish support" OFF)
+ option(WITH_PCAP "Compile with Pcap generation support" ON)
+ option(WITH_INTERNAL_DOC "Compile doxygen internal documentation" OFF)
+-option(BUILD_SHARED_LIBS "Build shared libraries" ON)
++option(BUILD_SHARED_LIBS "Build shared libraries" OFF)
+ option(WITH_PKCS11_URI "Build with PKCS#11 URI support" OFF)
+ option(UNIT_TESTING "Build with unit tests" OFF)
+ option(CLIENT_TESTING "Build with client tests; requires openssh" OFF)
+ option(SERVER_TESTING "Build with server tests; requires openssh and dropbear" OFF)
+ option(WITH_BENCHMARKS "Build benchmarks tools" OFF)
+-option(WITH_EXAMPLES "Build examples" ON)
++option(WITH_EXAMPLES "Build examples" OFF)
+ option(WITH_NACL "Build with libnacl (curve25519)" ON)
+ option(WITH_SYMBOL_VERSIONING "Build with symbol versioning" ON)
+ option(WITH_ABI_BREAK "Allow ABI break" OFF)
+@@ -25,6 +25,7 @@ option(WITH_GEX "Enable DH Group exchange mechanisms" ON)
+ option(WITH_INSECURE_NONE "Enable insecure none cipher and MAC algorithms (not suitable for production!)" OFF)
+ option(FUZZ_TESTING "Build with fuzzer for the server and client (automatically enables none cipher!)" OFF)
+ option(PICKY_DEVELOPER "Build with picky developer flags" OFF)
++option(WITH_STATIC_LIB "Build static library" ON)
+ 
+ if (WITH_ZLIB)
+     set(WITH_LIBZ ON)
+@@ -60,3 +61,7 @@ endif (NOT GLOBAL_CLIENT_CONFIG)
+ if (FUZZ_TESTING)
+   set(WITH_INSECURE_NONE ON)
+ endif (FUZZ_TESTING)
++
++if (WITH_STATIC_LIB)
++  set(BUILD_STATIC_LIB ON)
++endif()
+diff --git a/include/libssh/buffer.h b/include/libssh/buffer.h
+index a55a1b40..e34e075c 100644
+--- a/include/libssh/buffer.h
++++ b/include/libssh/buffer.h
+@@ -33,6 +33,8 @@ int ssh_buffer_add_u8(ssh_buffer buffer, uint8_t data);
+ int ssh_buffer_add_u16(ssh_buffer buffer, uint16_t data);
+ int ssh_buffer_add_u32(ssh_buffer buffer, uint32_t data);
+ int ssh_buffer_add_u64(ssh_buffer buffer, uint64_t data);
++ssize_t ssh_buffer_add_func(ssh_buffer buffer, ssh_add_func f, size_t max_bytes,
++			    void *userdata);
+ 
+ int ssh_buffer_validate_length(struct ssh_buffer_struct *buffer, size_t len);
+ 
+diff --git a/include/libssh/libssh.h b/include/libssh/libssh.h
+index 7857a77b..3eef7a16 100644
+--- a/include/libssh/libssh.h
++++ b/include/libssh/libssh.h
+@@ -833,6 +833,7 @@ LIBSSH_API const char* ssh_get_hmac_in(ssh_session session);
+ LIBSSH_API const char* ssh_get_hmac_out(ssh_session session);
+ 
+ LIBSSH_API ssh_buffer ssh_buffer_new(void);
++LIBSSH_API ssh_buffer ssh_buffer_new_size(uint32_t size, uint32_t headroom);
+ LIBSSH_API void ssh_buffer_free(ssh_buffer buffer);
+ #define SSH_BUFFER_FREE(x) \
+     do { if ((x) != NULL) { ssh_buffer_free(x); x = NULL; } } while(0)
+@@ -843,6 +844,8 @@ LIBSSH_API void *ssh_buffer_get(ssh_buffer buffer);
+ LIBSSH_API uint32_t ssh_buffer_get_len(ssh_buffer buffer);
+ LIBSSH_API int ssh_session_set_disconnect_message(ssh_session session, const char *message);
+ 
++typedef ssize_t (*ssh_add_func) (void *ptr, size_t max_bytes, void *userdata);
++
+ #ifndef LIBSSH_LEGACY_0_4
+ #include "libssh/legacy.h"
+ #endif
+diff --git a/include/libssh/sftp.h b/include/libssh/sftp.h
+index c855df8a..0fcdb9b8 100644
+--- a/include/libssh/sftp.h
++++ b/include/libssh/sftp.h
+@@ -565,6 +565,10 @@ LIBSSH_API int sftp_async_read(sftp_file file, void *data, uint32_t len, uint32_
+  */
+ LIBSSH_API ssize_t sftp_write(sftp_file file, const void *buf, size_t count);
+ 
++LIBSSH_API ssize_t sftp_async_write(sftp_file file, ssh_add_func f, size_t count,
++				    void *userdata, uint32_t* id);
++LIBSSH_API int sftp_async_write_end(sftp_file file, uint32_t id, int blocking);
++
+ /**
+  * @brief Seek to a specific location in a file.
+  *
+diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
+index c090fef7..e2f86309 100644
+--- a/src/CMakeLists.txt
++++ b/src/CMakeLists.txt
+@@ -435,6 +435,11 @@ if (BUILD_STATIC_LIB)
+   if (WIN32)
+     target_compile_definitions(ssh-static PUBLIC "LIBSSH_STATIC")
+   endif (WIN32)
++
++  install(TARGETS ssh-static
++	  EXPORT libssh-config
++	  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
++	  COMPONENT libraries)
+ endif (BUILD_STATIC_LIB)
+ 
+ message(STATUS "Threads_FOUND=${Threads_FOUND}")
+diff --git a/src/buffer.c b/src/buffer.c
+index e0068015..cc0caf35 100644
+--- a/src/buffer.c
++++ b/src/buffer.c
+@@ -141,6 +141,40 @@ struct ssh_buffer_struct *ssh_buffer_new(void)
+     return buf;
+ }
+ 
++/**
++ * @brief Create a new SSH buffer with a specified size and headroom.
++ *
++ * @param[in] len       length for newly initialized SSH buffer.
++ * @param[in] headroom  length for headroom
++ * @return A newly initialized SSH buffer, NULL on error.
++ */
++struct ssh_buffer_struct *ssh_buffer_new_size(uint32_t len, uint32_t headroom)
++{
++    struct ssh_buffer_struct *buf = NULL;
++    int rc;
++
++    if (len < headroom)
++	    return NULL;
++
++    buf = calloc(1, sizeof(struct ssh_buffer_struct));
++    if (buf == NULL) {
++        return NULL;
++    }
++
++    rc = ssh_buffer_allocate_size(buf, len);
++    if (rc != 0) {
++        SAFE_FREE(buf);
++        return NULL;
++    }
++
++    buf->pos += headroom;
++    buf->used += headroom;
++
++    buffer_verify(buf);
++
++    return buf;
++}
++
+ /**
+  * @brief Deallocate a SSH buffer.
+  *
+@@ -328,6 +362,49 @@ int ssh_buffer_add_data(struct ssh_buffer_struct *buffer, const void *data, uint
+     return 0;
+ }
+ 
++/**
++ * @brief Add data at the tail of a buffer by an external function
++ *
++ * @param[in]  buffer    The buffer to add data.
++ *
++ * @param[in]  f         function that adds data to the buffer.
++ *
++ * @param[in]  max_bytes The maximum length of the data to add.
++ *
++ * @return               actual bytes added on success, < 0 on error.
++ */
++ssize_t ssh_buffer_add_func(struct ssh_buffer_struct *buffer, ssh_add_func f,
++			    size_t max_bytes, void *userdata)
++{
++    ssize_t actual;
++
++    if (buffer == NULL) {
++        return -1;
++    }
++
++    buffer_verify(buffer);
++
++    if (buffer->used + max_bytes < max_bytes) {
++        return -1;
++    }
++
++    if (buffer->allocated < (buffer->used + max_bytes)) {
++        if (buffer->pos > 0) {
++            buffer_shift(buffer);
++        }
++        if (realloc_buffer(buffer, buffer->used + max_bytes) < 0) {
++            return -1;
++        }
++    }
++
++    if ((actual = f(buffer->data + buffer->used, max_bytes, userdata)) < 0)
++      return -1;
++
++    buffer->used += actual;
++    buffer_verify(buffer);
++    return actual;
++}
++
+ /**
+  * @brief Ensure the buffer has at least a certain preallocated size.
+  *
+diff --git a/src/sftp.c b/src/sftp.c
+index e01012a8..702623a0 100644
+--- a/src/sftp.c
++++ b/src/sftp.c
+@@ -2228,6 +2228,132 @@ ssize_t sftp_write(sftp_file file, const void *buf, size_t count) {
+   return -1; /* not reached */
+ }
+ 
++/*
++ * sftp_async_write is based on and sftp_async_write_end is copied from
++ * https://github.com/limes-datentechnik-gmbh/libssh
++ *
++ * sftp_async_write has some optimizations:
++ * - use ssh_buffer_new_size() to reduce realoc_buffer.
++ * - use ssh_buffer_add_func() to avoid memcpy from read buffer to ssh buffer.
++ */
++ssize_t sftp_async_write(sftp_file file, ssh_add_func f, size_t count, void *userdata,
++			 uint32_t* id) {
++  sftp_session sftp = file->sftp;
++  ssh_buffer buffer;
++  uint32_t buf_sz;
++  ssize_t actual;
++  int len;
++  int packetlen;
++  int rc;
++
++#define HEADROOM 16
++  /* sftp_packet_write() prepends a 5-bytes (uint32_t length and
++   * 1-byte type) header to the head of the payload by
++   * ssh_buffer_prepend_data(). Inserting headroom by
++   * ssh_buffer_new_size() eliminates memcpy for prepending the
++   * header.
++   */
++
++  buf_sz = (HEADROOM + /* for header */
++	    sizeof(uint32_t) + /* id */
++	    ssh_string_len(file->handle) + 4 + /* file->handle */
++	    sizeof(uint64_t) + /* file->offset */
++	    sizeof(uint32_t) + /* count */
++	    count); /* datastring */
++
++  buffer = ssh_buffer_new_size(buf_sz, HEADROOM);
++  if (buffer == NULL) {
++    ssh_set_error_oom(sftp->session);
++    return -1;
++  }
++
++  *id = sftp_get_new_id(file->sftp);
++
++  rc = ssh_buffer_pack(buffer,
++                       "dSqd",
++                       *id,
++                       file->handle,
++                       file->offset,
++                       count); /* len of datastring */
++
++  if (rc != SSH_OK){
++    ssh_set_error_oom(sftp->session);
++    ssh_buffer_free(buffer);
++    return SSH_ERROR;
++  }
++
++  actual = ssh_buffer_add_func(buffer, f, count, userdata);
++  if (actual < 0){
++    ssh_set_error_oom(sftp->session);
++    ssh_buffer_free(buffer);
++    return SSH_ERROR;
++  }
++
++  packetlen=ssh_buffer_get_len(buffer)+5;
++  len = sftp_packet_write(file->sftp, SSH_FXP_WRITE, buffer);
++  ssh_buffer_free(buffer);
++  if (len < 0) {
++    return SSH_ERROR;
++  } else  if (len != packetlen) {
++    ssh_set_error(sftp->session, SSH_FATAL,
++      "Could only send %d of %d bytes to remote host!", len, packetlen);
++    SSH_LOG(SSH_LOG_PACKET,
++        "Could not write as much data as expected");
++    return SSH_ERROR;
++  }
++
++  file->offset += actual;
++
++  return actual;
++}
++
++int sftp_async_write_end(sftp_file file, uint32_t id, int blocking) {
++  sftp_session sftp = file->sftp;
++  sftp_message msg = NULL;
++  sftp_status_message status;
++
++  msg = sftp_dequeue(sftp, id);
++  while (msg == NULL) {
++    if (!blocking && ssh_channel_poll(sftp->channel, 0) == 0) {
++      /* we cannot block */
++      return SSH_AGAIN;
++    }
++    if (sftp_read_and_dispatch(sftp) < 0) {
++      /* something nasty has happened */
++      return SSH_ERROR;
++    }
++    msg = sftp_dequeue(sftp, id);
++  }
++
++  switch (msg->packet_type) {
++    case SSH_FXP_STATUS:
++      status = parse_status_msg(msg);
++      sftp_message_free(msg);
++      if (status == NULL) {
++        return SSH_ERROR;
++      }
++      sftp_set_error(sftp, status->status);
++      switch (status->status) {
++        case SSH_FX_OK:
++          status_msg_free(status);
++          return SSH_OK;
++        default:
++          break;
++      }
++      ssh_set_error(sftp->session, SSH_REQUEST_DENIED,
++          "SFTP server: %s", status->errormsg);
++      status_msg_free(status);
++      return SSH_ERROR;
++    default:
++      ssh_set_error(sftp->session, SSH_FATAL,
++          "Received message %d during write!", msg->packet_type);
++      sftp_message_free(msg);
++      return SSH_ERROR;
++  }
++
++  return SSH_ERROR; /* not reached */
++}
++
+ /* Seek to a specific location in a file. */
+ int sftp_seek(sftp_file file, uint32_t new_offset) {
+   if (file == NULL) {

patch/libssh-0.9.6.patch

@@ -0,0 +1,164 @@
+diff --git a/DefineOptions.cmake b/DefineOptions.cmake
+index b82a5018..f1f2ab9d 100644
+--- a/DefineOptions.cmake
++++ b/DefineOptions.cmake
+@@ -15,13 +15,14 @@ option(UNIT_TESTING "Build with unit tests" OFF)
+ option(CLIENT_TESTING "Build with client tests; requires openssh" OFF)
+ option(SERVER_TESTING "Build with server tests; requires openssh and dropbear" OFF)
+ option(WITH_BENCHMARKS "Build benchmarks tools" OFF)
+-option(WITH_EXAMPLES "Build examples" ON)
++option(WITH_EXAMPLES "Build examples" OFF)
+ option(WITH_NACL "Build with libnacl (curve25519)" ON)
+ option(WITH_SYMBOL_VERSIONING "Build with symbol versioning" ON)
+ option(WITH_ABI_BREAK "Allow ABI break" OFF)
+ option(WITH_GEX "Enable DH Group exchange mechanisms" ON)
+ option(FUZZ_TESTING "Build with fuzzer for the server" OFF)
+ option(PICKY_DEVELOPER "Build with picky developer flags" OFF)
++option(WITH_STATIC_LIB "Build static library" ON)
+ 
+ if (WITH_ZLIB)
+     set(WITH_LIBZ ON)
+@@ -53,3 +54,7 @@ endif (NOT GLOBAL_BIND_CONFIG)
+ if (NOT GLOBAL_CLIENT_CONFIG)
+   set(GLOBAL_CLIENT_CONFIG "/etc/ssh/ssh_config")
+ endif (NOT GLOBAL_CLIENT_CONFIG)
++
++if (WITH_STATIC_LIB)
++  set(BUILD_STATIC_LIB ON)
++endif()
+diff --git a/include/libssh/sftp.h b/include/libssh/sftp.h
+index 8c14b21d..95ac1d6b 100644
+--- a/include/libssh/sftp.h
++++ b/include/libssh/sftp.h
+@@ -565,6 +565,9 @@ LIBSSH_API int sftp_async_read(sftp_file file, void *data, uint32_t len, uint32_
+  */
+ LIBSSH_API ssize_t sftp_write(sftp_file file, const void *buf, size_t count);
+ 
++LIBSSH_API int sftp_async_write(sftp_file file, const void *buf, size_t count, uint32_t* id);
++LIBSSH_API int sftp_async_write_end(sftp_file file, uint32_t id, int blocking);
++
+ /**
+  * @brief Seek to a specific location in a file.
+  *
+diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
+index a576cf71..303a1c7f 100644
+--- a/src/CMakeLists.txt
++++ b/src/CMakeLists.txt
+@@ -412,6 +412,10 @@ if (BUILD_STATIC_LIB)
+   if (WIN32)
+     target_compile_definitions(ssh-static PUBLIC "LIBSSH_STATIC")
+   endif (WIN32)
++  install(TARGETS ssh-static
++	EXPORT libssh-config
++	LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
++	COMPONENT libraries)
+ endif (BUILD_STATIC_LIB)
+ 
+ message(STATUS "Threads_FOUND=${Threads_FOUND}")
+diff --git a/src/sftp.c b/src/sftp.c
+index a8346040..a4261ec9 100644
+--- a/src/sftp.c
++++ b/src/sftp.c
+@@ -2234,6 +2234,102 @@ ssize_t sftp_write(sftp_file file, const void *buf, size_t count) {
+   return -1; /* not reached */
+ }
+ 
++/*
++ * sftp_async_write and sftp_async_write_end are copied from
++ * https://github.com/limes-datentechnik-gmbh/libssh
++ */
++int sftp_async_write(sftp_file file, const void *buf, size_t count, uint32_t* id) {
++  sftp_session sftp = file->sftp;
++  ssh_buffer buffer;
++  int len;
++  int packetlen;
++  int rc;
++
++  buffer = ssh_buffer_new();
++  if (buffer == NULL) {
++    ssh_set_error_oom(sftp->session);
++    return -1;
++  }
++
++  *id = sftp_get_new_id(file->sftp);
++
++  rc = ssh_buffer_pack(buffer,
++                       "dSqdP",
++                       *id,
++                       file->handle,
++                       file->offset,
++                       count, /* len of datastring */
++                       (size_t)count, buf);
++  if (rc != SSH_OK){
++    ssh_set_error_oom(sftp->session);
++    ssh_buffer_free(buffer);
++    return SSH_ERROR;
++  }
++  packetlen=ssh_buffer_get_len(buffer)+5;
++  len = sftp_packet_write(file->sftp, SSH_FXP_WRITE, buffer);
++  ssh_buffer_free(buffer);
++  if (len < 0) {
++    return SSH_ERROR;
++  } else  if (len != packetlen) {
++    ssh_set_error(sftp->session, SSH_FATAL,
++      "Could only send %d of %d bytes to remote host!", len, packetlen);
++    SSH_LOG(SSH_LOG_PACKET,
++        "Could not write as much data as expected");
++    return SSH_ERROR;
++  }
++
++  file->offset += count;
++
++  return SSH_OK;
++}
++
++int sftp_async_write_end(sftp_file file, uint32_t id, int blocking) {
++  sftp_session sftp = file->sftp;
++  sftp_message msg = NULL;
++  sftp_status_message status;
++
++  msg = sftp_dequeue(sftp, id);
++  while (msg == NULL) {
++    if (!blocking && ssh_channel_poll(sftp->channel, 0) == 0) {
++      /* we cannot block */
++      return SSH_AGAIN;
++    }
++    if (sftp_read_and_dispatch(sftp) < 0) {
++      /* something nasty has happened */
++      return SSH_ERROR;
++    }
++    msg = sftp_dequeue(sftp, id);
++  }
++
++  switch (msg->packet_type) {
++    case SSH_FXP_STATUS:
++      status = parse_status_msg(msg);
++      sftp_message_free(msg);
++      if (status == NULL) {
++        return SSH_ERROR;
++      }
++      sftp_set_error(sftp, status->status);
++      switch (status->status) {
++        case SSH_FX_OK:
++          status_msg_free(status);
++          return SSH_OK;
++        default:
++          break;
++      }
++      ssh_set_error(sftp->session, SSH_REQUEST_DENIED,
++          "SFTP server: %s", status->errormsg);
++      status_msg_free(status);
++      return SSH_ERROR;
++    default:
++      ssh_set_error(sftp->session, SSH_FATAL,
++          "Received message %d during write!", msg->packet_type);
++      sftp_message_free(msg);
++      return SSH_ERROR;
++  }
++
++  return SSH_ERROR; /* not reached */
++}
++
+ /* Seek to a specific location in a file. */
+ int sftp_seek(sftp_file file, uint32_t new_offset) {
+   if (file == NULL) {

scripts/install-build-deps.sh

@@ -1,18 +1,30 @@
-#!/bin/bash -e
+#!/bin/bash -eu
 #
 # Install build dpenedencies.
 
-source /etc/os-release
+platform=$(uname -s)
+
+if [ -e /etc/os-release ]; then
+	source /etc/os-release
+	platform=${platform}-${ID}
+fi
+
 set -x
 
-case $ID in
+case $platform in
-	ubuntu*)
+	Darwin)
-		apt-get install -y gcc make cmake libssh-dev
+		brew install openssl@1.1
 		;;
-	centos* | rhel* | rocky*)
+	Linux-ubuntu*)
-		yum install -y gcc make cmake libssh-devel rpm-build
+		apt-get install -y \
+			gcc make cmake zlib1g-dev libssl-dev libkrb5-dev
+		;;
+	Linux-centos* | Linux-rhel* | Linux-rocky*)
+		yum install -y \
+			gcc make cmake zlib-devel openssl-devel rpm-build
 		;;
 	*)
-		echo "unsupported dependency install: $ID"
+		echo "unsupported platform: $platform"
 		exit 1
+		;;
 esac

scripts/print-install-deps.sh

@@ -0,0 +1,23 @@
+#!/bin/bash -e
+#
+# Print install dpenedencies on Linux. CMake runs this script to obtain deps for CPACK.
+# mscp dependes on packages on which libssh depends.
+
+source /etc/os-release
+
+release=$1
+
+case $release in
+	ubuntu-20.04*)
+		echo "libc6 (>= 2.27), libgssapi-krb5-2 (>= 1.17), libssl1.1 (>= 1.1.1), zlib1g (>= 1:1.1.4)"
+		;;
+	ubuntu-22.04*)
+		echo "libc6 (>= 2.33), libgssapi-krb5-2 (>= 1.17), libssl3 (>= 3.0.0~~alpha1), zlib1g (>= 1:1.1.4)"
+		;;
+	centos* | rhel* | rocky*)
+		echo "glibc crypto-policies krb5-libs openssl-libs libcom_err"
+		;;
+	*)
+		echo "unsupported install dependency: $release"
+		exit 1
+esac

scripts/test-in-container.sh

@@ -1,31 +1,13 @@
 #!/bin/bash -e
 #
-# Run this script in docker containers. This script installs mscp from built package
+# Run this script in mscp docker containers.
-# and run test for mscp in the installed path.
+# This script runs end-to-end test with installed mscp.
 
-source /etc/os-release
 script_dir=$(cd $(dirname ${0}) && pwd)
 cd $script_dir
-project_version=$(cat ../VERSION)
-arch=$(uname -p)
 
 set -x
 
-# install package
-case $ID in
-        ubuntu*)
-		pkg=mscp_${project_version}-${ID}-${VERSION_ID}-${arch}.deb
-		dpkg -i ../build/$pkg
-                ;;
-        centos* | rhel* | rocky*)
-		pkg=mscp_${project_version}-${ID}-${VERSION_ID}-${arch}.rpm
-		rpm -iv ../build/$pkg
-                ;;
-        *)
-                echo "unsupported test platform: $ID"
-                exit 1
-esac
-
 # Run sshd
 if [ ! -e /var/run/sshd.pid ]; then
 	/usr/sbin/sshd

src/atomic.h

@@ -55,4 +55,17 @@ static inline void lock_release(lock *l)
 	}
 }
 
+static inline void lock_release_via_cleanup(void *l)
+{
+	lock_release(l);
+}
+
+#define LOCK_ACQUIRE_THREAD(l)				\
+		lock_acquire(l);			\
+		pthread_cleanup_push(lock_release_via_cleanup, l)
+
+
+#define LOCK_RELEASE_THREAD(l) \
+	pthread_cleanup_pop(1)
+
 #endif /* _ATOMIC_H_ */

src/file.c

@@ -319,13 +319,15 @@ static int file_fill_recursive(struct list_head *file_list,
 int file_fill(sftp_session sftp, struct list_head *file_list, char **src_array, int cnt,
 	      char *dst)
 {
-	bool dst_is_remote, dst_is_dir, dst_dir_no_exist, dst_should_dir;
+	bool dst_is_remote, dst_is_dir, dst_dir_no_exist, dst_should_dir, dst_must_dir;
 	char *dst_path, *src_path;
 	int n, ret;
 
 	dst_path = file_find_path(dst);
 	dst_path = *dst_path == '\0' ? "." : dst_path;
 	dst_is_remote = file_find_hostname(dst) ? true : false;
+	dst_must_dir = cnt > 1 ? true : false;
+
 	if (file_is_directory(dst_path, dst_is_remote ? sftp : NULL, false) > 0)
 		dst_is_dir = true;
 	else
@@ -342,7 +344,8 @@ int file_fill(sftp_session sftp, struct list_head *file_list, char **src_array,
 
 		ret = file_fill_recursive(file_list, dst_is_remote, sftp,
 					  src_path, "", dst_path,
-					  dst_should_dir | dst_is_dir, dst_dir_no_exist);
+					  dst_should_dir | dst_must_dir | dst_is_dir,
+					  dst_dir_no_exist);
 		if (ret < 0)
 			return ret;
 	}
@@ -364,6 +367,7 @@ static int file_dst_prepare(struct file *f, sftp_session sftp)
 
 	pr_debug("prepare for %s\n", path);
 
+	/* mkdir -p */
 	for (p = strchr(path + 1, '/'); p; p = strchr(p + 1, '/')) {
 		*p = '\0';
 
@@ -395,6 +399,30 @@ static int file_dst_prepare(struct file *f, sftp_session sftp)
 		*p = '/';
 	}
 
+	/* open file with O_TRUNC to set file size 0 */
+	mode = O_WRONLY|O_CREAT|O_TRUNC;
+	if (sftp) {
+		sftp_file sf;
+		if ((sf = sftp_open(sftp, f->dst_path, mode, S_IRUSR|S_IWUSR)) == NULL) {
+			pr_err("sftp_open: %s\n", sftp_get_ssh_error(sftp));
+			return -1;
+		}
+		if (sftp_close(sf) < 0) {
+			pr_err("sftp_close: %s\n", sftp_get_ssh_error(sftp));
+			return -1;
+		}
+	} else {
+		int fd;
+		if ((fd = open(f->dst_path, mode, S_IRUSR|S_IWUSR)) < 0) {
+			pr_err("open: %s\n", strerrno());
+			return -1;
+		}
+		if (close(fd) < 0) {
+			pr_err("close: %s\n", strerrno());
+			return -1;
+		}
+	}
+
 	return 0;
 }
 
@@ -464,7 +492,7 @@ int chunk_fill(struct list_head *file_list, struct list_head *chunk_list,
 			chunk_sz = max_chunk_sz;
 		else {
 			chunk_sz = (f->size - (f->size % nr_conn)) / nr_conn;
-			chunk_sz &= ~page_mask; /* align in page_sz */
+			chunk_sz &= ~page_mask; /* align with page_sz */
 			if (chunk_sz <= min_chunk_sz)
 				chunk_sz = min_chunk_sz;
 		}
@@ -526,7 +554,7 @@ int chunk_prepare(struct chunk *c, sftp_session sftp)
 	struct file *f = c->f;
 	int ret = 0;
 
-	lock_acquire(&f->lock); /* XXX: is always acquiring lock per-chunk heavy? */
+	LOCK_ACQUIRE_THREAD(&f->lock);
 	if (f->state == FILE_STATE_INIT) {
 		if (file_dst_prepare(f, f->dst_is_remote ? sftp : NULL) < 0) {
 			ret = -1;
@@ -537,11 +565,11 @@ int chunk_prepare(struct chunk *c, sftp_session sftp)
 	}
 
 out:
-	lock_release(&f->lock);
+	LOCK_RELEASE_THREAD();
 	return ret;
 }
 
-static mode_t chunk_get_mode(const char *path, sftp_session sftp)
+static mode_t file_get_mode(const char *path, sftp_session sftp)
 {
 	mode_t mode;
 
@@ -564,7 +592,7 @@ static mode_t chunk_get_mode(const char *path, sftp_session sftp)
 	return mode;
 }
 
-static int chunk_set_mode(const char *path, mode_t mode, sftp_session sftp)
+static int file_set_mode(const char *path, mode_t mode, sftp_session sftp)
 {
 	if (sftp) {
 		if (sftp_chmod(sftp, path, mode) < 0) {
@@ -619,61 +647,86 @@ static sftp_file chunk_open_remote(const char *path, int flags, mode_t mode, siz
 	return sf;
 }
 
-static int chunk_copy_internal_local_to_remote(struct chunk *c, int fd, sftp_file sf,
+static ssize_t read_to_buf(void *ptr, size_t len, void *userdata)
-					       size_t sftp_buf_sz, size_t io_buf_sz,
-					       size_t *counter)
 {
-	ssize_t read_bytes, write_bytes, remaind;
+	int fd = *((int *)userdata);
-	char buf[io_buf_sz];
+	return read(fd, ptr, len);
+}
 
-	for (remaind = c->len; remaind > 0;) {
+static int chunk_copy_local_to_remote_async(struct chunk *c, int fd, sftp_file sf,
+					    int nr_ahead, int buf_sz, size_t *counter)
+{
+	ssize_t read_bytes, remaind, thrown;
+	int idx, ret;
+	struct {
+		uint32_t id;
+		ssize_t  len;
+	} reqs[nr_ahead];
 
-		read_bytes = read(fd, buf, min(remaind, io_buf_sz));
+	if (c->len == 0)
-		if (read_bytes < 0) {
+		return 0;
-			pr_err("failed to read %s: %s\n", c->f->src_path, strerrno());
+
+	remaind = thrown = c->len;
+	for (idx = 0; idx < nr_ahead && thrown > 0; idx++) {
+		reqs[idx].len = min(thrown, buf_sz);
+		reqs[idx].len = sftp_async_write(sf, read_to_buf, reqs[idx].len, &fd,
+						 &reqs[idx].id);
+		if (reqs[idx].len < 0) {
+			pr_err("sftp_async_write: %d or %s\n",
+			       sftp_get_error(sf->sftp), strerrno());
+			return -1;
+		}
+		thrown -= reqs[idx].len;
+	}
+
+	for (idx = 0; remaind > 0; idx = (idx + 1) % nr_ahead) {
+		ret = sftp_async_write_end(sf, reqs[idx].id, 1);
+		if (ret != SSH_OK) {
+			pr_err("sftp_async_write_end: %d\n", sftp_get_error(sf->sftp));
 			return -1;
 		}
 
-		write_bytes = sftp_write2(sf, buf, read_bytes, sftp_buf_sz);
+		*counter += reqs[idx].len;
-		if (write_bytes < 0) {
+		remaind -= reqs[idx].len;
-			pr_err("failed to write to %s: SFTP error code %d\n",
+
-			       c->f->dst_path, sftp_get_error(sf->sftp));
+		if (remaind <= 0)
+			break;
+
+		if (thrown <= 0)
+			continue;
+
+		reqs[idx].len = min(thrown, buf_sz);
+		reqs[idx].len = sftp_async_write(sf, read_to_buf, reqs[idx].len, &fd,
+						 &reqs[idx].id);
+		if (reqs[idx].len < 0) {
+			pr_err("sftp_async_write: %d or %s\n",
+			       sftp_get_error(sf->sftp), strerrno());
 			return -1;
 		}
+		thrown -= reqs[idx].len;
+	}
 
-		if (write_bytes < read_bytes) {
+	if (remaind < 0) {
-			pr_err("failed to write full bytes to %s\n", c->f->dst_path);
+		pr_err("invalid remaind bytes %ld. last async_write_end bytes %lu.",
-			return -1;
+		       remaind, reqs[idx].len);
-		}
+		return -1;
-
-		*counter += write_bytes;
-		remaind -= write_bytes;
 	}
 
 	return 0;
 }
 
-static int chunk_copy_internal_remote_to_local(struct chunk *c, int fd, sftp_file sf,
-					       int nr_ahead, size_t *counter)
-{
-#define XFER_BUF_SIZE	16384
 
+static int chunk_copy_remote_to_local_async(struct chunk *c, int fd, sftp_file sf,
+					    int nr_ahead, int buf_sz, size_t *counter)
+{
 	ssize_t read_bytes, write_bytes, remaind, thrown;
-	char buf[XFER_BUF_SIZE];
+	char buf[buf_sz];
 	int idx;
 	struct {
 		int id;
-		size_t len;
+		ssize_t len;
 	} reqs[nr_ahead];
 
-	/* TODO: sftp_buf_sz has no effect on remote to local copy. we
-	 * always use 16384 byte buffer pointed by
-	 * https://api.libssh.org/stable/libssh_tutor_sftp.html. The
-	 * larget read length from sftp_async_read is 65536 byte.
-	 * Read sizes larget than 65536 cause a situation where data
-	 * remainds but sftp_async_read returns 0.
-	 */
-
 	if (c->len == 0)
 		return 0;
 
@@ -683,49 +736,53 @@ static int chunk_copy_internal_remote_to_local(struct chunk *c, int fd, sftp_fil
 		reqs[idx].len = min(thrown, sizeof(buf));
 		reqs[idx].id = sftp_async_read_begin(sf, reqs[idx].len);
 		if (reqs[idx].id < 0) {
-			pr_err("sftp_async_read_begin failed: %d\n",
+			pr_err("sftp_async_read_begin: %d\n",
 			       sftp_get_error(sf->sftp));
 			return -1;
 		}
 		thrown -= reqs[idx].len;
 	}
 
-	for (idx = 0; remaind > 0;) {
+	for (idx = 0; remaind > 0; idx = (idx + 1) % nr_ahead) {
-
 		read_bytes = sftp_async_read(sf, buf, reqs[idx].len, reqs[idx].id);
 		if (read_bytes == SSH_ERROR) {
-			pr_err("sftp_async_read failed: %d\n", sftp_get_error(sf->sftp));
+			pr_err("sftp_async_read: %d\n", sftp_get_error(sf->sftp));
 			return -1;
 		}
 
-		remaind -= read_bytes;
+		if (thrown > 0) {
-
+			reqs[idx].len = min(thrown, sizeof(buf));
-		if (remaind > 0) {
-			reqs[idx].len = min(remaind, sizeof(buf));
 			reqs[idx].id = sftp_async_read_begin(sf, reqs[idx].len);
+			thrown -= reqs[idx].len;
 		}
 
 		write_bytes = write(fd, buf, read_bytes);
 		if (write_bytes < 0) {
-			pr_err("write to %s failed: %s\n", c->f->dst_path, strerrno());
+			pr_err("write: %s\n", strerrno());
 			return -1;
 		}
 
 		if (write_bytes < read_bytes) {
-			pr_err("failed to write full bytes to %s\n", c->f->dst_path);
+			pr_err("failed to write full bytes\n");
 			return -1;
 		}
 
 		*counter += write_bytes;
-		idx = (idx + 1) % nr_ahead;
+		remaind -= read_bytes;
+	}
+
+	if (remaind < 0) {
+		pr_err("invalid remaind bytes %ld. last async_read bytes %ld. "
+		       "last write bytes %ld\n",
+		       remaind, read_bytes, write_bytes);
+		return -1;
 	}
 
 	return 0;
 }
 
 static int chunk_copy_local_to_remote(struct chunk *c, sftp_session sftp,
-				      size_t sftp_buf_sz, size_t io_buf_sz,
+				      int nr_ahead, int buf_sz, size_t *counter)
-				      size_t *counter)
 {
 	struct file *f = c->f;
 	sftp_file sf = NULL;
@@ -741,26 +798,16 @@ static int chunk_copy_local_to_remote(struct chunk *c, sftp_session sftp,
 		goto out;
 	}
 
-	flags = O_WRONLY|O_CREAT;
+	flags = O_WRONLY;
 	mode = S_IRUSR|S_IWUSR;
 	if (!(sf = chunk_open_remote(f->dst_path, flags, mode, c->off, sftp))) {
 		ret = -1;
 		goto out;
 	}
 
-	ret = chunk_copy_internal_local_to_remote(c, fd, sf, sftp_buf_sz, io_buf_sz,
+	ret = chunk_copy_local_to_remote_async(c, fd, sf, nr_ahead, buf_sz, counter);
-						  counter);
 	if (ret < 0)
 		goto out;
-
-	if ((mode = chunk_get_mode(f->src_path, NULL)) < 0) {
-		ret = -1;
-		goto out;
-	}
-	if (chunk_set_mode(f->dst_path, mode, sftp) < 0) {
-		ret = -1;
-	}
-
 out:
 	if (fd > 0)
 		close(fd);
@@ -770,7 +817,7 @@ out:
 }
 
 static int chunk_copy_remote_to_local(struct chunk *c, sftp_session sftp,
-				      int nr_ahead, size_t *counter)
+				      int nr_ahead, int buf_sz, size_t *counter)
 {
 	struct file *f = c->f;
 	sftp_file sf = NULL;
@@ -779,7 +826,7 @@ static int chunk_copy_remote_to_local(struct chunk *c, sftp_session sftp,
 	int fd = 0;
 	int ret = 0;
 
-	flags = O_WRONLY|O_CREAT;
+	flags = O_WRONLY;
 	mode = S_IRUSR|S_IWUSR;
 	if ((fd = chunk_open_local(f->dst_path, flags, mode, c->off)) < 0) {
 		ret = -1;
@@ -793,7 +840,7 @@ static int chunk_copy_remote_to_local(struct chunk *c, sftp_session sftp,
 		goto out;
 	}
 
-	ret = chunk_copy_internal_remote_to_local(c, fd, sf, nr_ahead, counter);
+	ret = chunk_copy_remote_to_local_async(c, fd, sf, nr_ahead, buf_sz, counter);
 	if (ret< 0)
 		goto out;
 
@@ -806,43 +853,51 @@ out:
 	return ret;
 }
 
+static int file_cleanup(struct file *f, sftp_session sftp)
+{
+	sftp_session s, d;
+	mode_t mode;
 
+	if (f->dst_is_remote) {
+		s = NULL;
+		d = sftp;
+	} else {
+		s = sftp;
+		d = NULL;
+	}
 
-int chunk_copy(struct chunk *c, sftp_session sftp, size_t sftp_buf_sz, size_t io_buf_sz,
+	if ((mode = file_get_mode(f->src_path, s)) < 0)
-	       int nr_ahead, size_t *counter)
+		return -1;
+	if (file_set_mode(f->dst_path, mode, d) < 0)
+		return -1;
+	return 0;
+}
+
+int chunk_copy(struct chunk *c, sftp_session sftp, int nr_ahead, int buf_sz,
+	       size_t *counter)
 {
 	struct file *f = c->f;
 	int ret = 0;
 
-	pr_debug("copy %s %s -> %s %s off=0x%010lx\n",
-		 f->src_path, strloc(f->src_is_remote),
-		 f->dst_path, strloc(f->dst_is_remote), c->off);
-
 	pprint4("copy start: chunk %s 0x%010lx-0x%010lx %luB\n",
 		c->f->src_path, c->off, c->off + c->len, c->len);
 
-
 	if (f->dst_is_remote)
-		ret = chunk_copy_local_to_remote(c, sftp,
+		ret = chunk_copy_local_to_remote(c, sftp, nr_ahead, buf_sz, counter);
-						 sftp_buf_sz, io_buf_sz, counter);
 	else
-		ret = chunk_copy_remote_to_local(c, sftp, nr_ahead, counter);
+		ret = chunk_copy_remote_to_local(c, sftp, nr_ahead, buf_sz, counter);
 
 	if (ret < 0)
 		return ret;
 
-	pr_debug("done %s %s -> %s %s off=0x%010lx\n",
-		 f->src_path, strloc(f->src_is_remote),
-		 f->dst_path, strloc(f->dst_is_remote), c->off);
-
 	pprint4("copy done: chunk %s 0x%010lx-0x%010lx %luB\n",
 		c->f->src_path, c->off, c->off + c->len, c->len);
 
 	if (refcnt_dec(&f->refcnt) == 0) {
 		f->state = FILE_STATE_DONE;
 		pprint2("copy done: %s\n", f->src_path);
+		ret = file_cleanup(f, sftp);
 	}
 
-
 	return ret;
 }

src/file.h

@@ -3,8 +3,8 @@
 
 #include <limits.h>
 #include <pthread.h>
-#include <libssh/libssh.h>
+#include "libssh/libssh.h"
-#include <libssh/sftp.h>
+#include "libssh/sftp.h"
 
 #include <list.h>
 #include <atomic.h>
@@ -37,7 +37,8 @@ struct file {
  * if the file state of the chunk is INIT:
  *     acquire the file lock
  * *         if file state is INIT:
- *             create destination file and directory if necessary
+ *             create directory if necessary
+ *             open file with O_TRUNC and close.
  *             set file state OPENED.
  *             // only the first thread in the lock open the destination file
  *     release the file lock
@@ -71,8 +72,9 @@ int chunk_fill(struct list_head *file_list, struct list_head *chunk_list,
 
 struct chunk *chunk_acquire(struct list_head *chunk_list);
 int chunk_prepare(struct chunk *c, sftp_session sftp);
-int chunk_copy(struct chunk *c, sftp_session sftp, size_t sftp_buf_sz, size_t io_buf_sz,
+int chunk_copy(struct chunk *c, sftp_session sftp, int nr_ahead, int buf_sz,
-	       int nr_ahead, size_t *counter);
+	       size_t *counter);
+
 
 #ifdef DEBUG
 void file_dump(struct list_head *file_list);

src/list.h

@@ -514,4 +514,19 @@ static inline void hlist_add_after(struct hlist_node *n,
 	     pos = n)
 
 
+/**
+ * list_count - return length of list
+ * @head        the head for your list.
+ */
+static inline int list_count(struct list_head *head)
+{
+	int n = 0;
+	struct list_head *p;
+
+	list_for_each(p, head) n++;
+	return n;
+}
+
+
 #endif
+

src/main.c

@@ -23,15 +23,30 @@
 #endif
 
 #define DEFAULT_MIN_CHUNK_SZ    (64 << 20)      /* 64MB */
-#define DEFAULT_SFTP_BUF_SZ     131072          /* derived from qemu/block/ssh.c */
+#define DEFAULT_NR_AHEAD	32
-#define DEFAULT_IO_BUF_SZ       DEFAULT_SFTP_BUF_SZ
+#define DEFAULT_BUF_SZ		16384
-/* XXX: need to investigate max buf size for sftp_read/sftp_write */
+/* XXX: we use 16384 byte buffer pointed by
-#define DEFAULT_NR_AHEAD	16
+ * https://api.libssh.org/stable/libssh_tutor_sftp.html. The larget
+ * read length from sftp_async_read is 65536 byte. Read sizes larger
+ * than 65536 cause a situation where data remainds but
+ * sftp_async_read returns 0.
+ */
+
+
+
+struct mscp_thread {
+	sftp_session    sftp;
+
+	pthread_t       tid;
+	int		cpu;
+	size_t          done;           /* copied bytes */
+	bool            finished;
+	int		ret;
+};
 
 struct mscp {
 	char                    *host;  /* remote host (and username) */
 	struct ssh_opts         *opts;  /* ssh parameters */
-	sftp_session            ctrl;   /* control sftp session */
 
 	struct list_head        file_list;
 	struct list_head        chunk_list;	/* stack of chunks */
@@ -39,84 +54,65 @@ struct mscp {
 
 	char    *target;
 
-	int     sftp_buf_sz, io_buf_sz;
+	int	nr_threads;	/* number of threads */
+	int	buf_sz;		/* i/o buf size */
 	int	nr_ahead;	/* # of ahead read command for remote to local copy */
 
-	struct timeval start;   /* timestamp of starting copy */
+	struct mscp_thread *threads;
-};
+} m;
-
-struct mscp_thread {
-	struct mscp     *mscp;
-	sftp_session    sftp;
-
-	pthread_t       tid;
-	size_t          done;           /* copied bytes */
-	bool            finished;
-	int		ret;
-};
 
 void *mscp_copy_thread(void *arg);
-void *mscp_monitor_thread(void *arg);
+int mscp_stat_init();
+void mscp_stat_final();
+
 
-pthread_t mtid;
-struct mscp_thread *threads;
-int nr_threads;
 
 void stop_copy_threads(int sig)
 {
 	int n;
 
 	pr("stopping...\n");
-	for (n = 0; n < nr_threads; n++) {
+	for (n = 0; n < m.nr_threads; n++) {
-		if (!threads[n].finished)
+		if (m.threads[n].tid && !m.threads[n].finished)
-			pthread_cancel(threads[n].tid);
+			pthread_cancel(m.threads[n].tid);
 	}
 }
 
-int list_count(struct list_head *head)
-{
-	int n = 0;
-	struct list_head *p;
-
-	list_for_each(p, head) n++;
-	return n;
-}
-
 void usage(bool print_help) {
 	printf("mscp v" VERSION ": copy files over multiple ssh connections\n"
 	       "\n"
-	       "Usage: mscp [vqDCHdh] [-n nr_conns]\n"
+	       "Usage: mscp [vqDCHdNh] [-n nr_conns] [-m coremask]\n"
-	       "            [-s min_chunk_sz] [-S max_chunk_sz]\n"
+	       "            [-s min_chunk_sz] [-S max_chunk_sz] [-a nr_ahead] [-b buf_sz]\n"
-	       "            [-b sftp_buf_sz] [-B io_buf_sz] [-a nr_ahead]\n"
 	       "            [-l login_name] [-p port] [-i identity_file]\n"
-	       "            [-c cipher_spec] source ... target\n"
+	       "            [-c cipher_spec] [-M hmac_spec] source ... target\n"
 	       "\n");
 
 	if (!print_help)
 		return;
 
-	printf("    -n NR_CONNECTIONS  number of connections (default: half of # of cpu cores)\n"
+	printf("    -n NR_CONNECTIONS  number of connections "
+	       "(default: floor(log(cores)*2)+1)\n"
+	       "    -m COREMASK        hex value to specify cores where threads pinned\n"
 	       "    -s MIN_CHUNK_SIZE  min chunk size (default: 64MB)\n"
-	       "    -S MAX_CHUNK_SIZE  max chunk size (default: filesize / nr_conn)\n"
+	       "    -S MAX_CHUNK_SIZE  max chunk size (default: filesize/nr_conn)\n"
 	       "\n"
-	       "    -b SFTP_BUF_SIZE   buf size for sftp_read/write (default 131072B)\n"
+	       "    -a NR_AHEAD        number of inflight SFTP commands (default: 32)\n"
-	       "    -B IO_BUF_SIZE     buf size for read/write (default 131072B)\n"
+	       "    -b BUF_SZ          buffer size for i/o and transfer\n"
-	       "                       Note that the default value is derived from\n"
-	       "                       qemu/block/ssh.c. need investigation...\n"
-	       "                       -b and -B affect only local to remote copy\n"
-	       "    -a NR_AHEAD        number of inflight SFTP read commands (default 16)\n"
 	       "\n"
 	       "    -v                 increment verbose output level\n"
 	       "    -q                 disable output\n"
 	       "    -D                 dry run\n"
+	       "    -r                 no effect\n"
 	       "\n"
 	       "    -l LOGIN_NAME      login name\n"
 	       "    -p PORT            port number\n"
-	       "    -i IDENTITY        identity file for publickey authentication\n"
+	       "    -i IDENTITY        identity file for public key authentication\n"
-	       "    -c CIPHER          cipher spec, see `ssh -Q cipher`\n"
+	       "    -c CIPHER          cipher spec\n"
+	       "    -M HMAC            hmac spec\n"
 	       "    -C                 enable compression on libssh\n"
 	       "    -H                 disable hostkey check\n"
 	       "    -d                 increment ssh debug output level\n"
+	       "    -N                 disable tcp nodelay (default on)\n"
 	       "    -h                 print this help\n"
 	       "\n");
 }
@@ -164,38 +160,107 @@ err_out:
 	return NULL;
 }
 
+int expand_coremask(const char *coremask, int **cores, int *nr_cores)
+{
+	int n, *core_list, core_list_len = 0, nr_usable, nr_all;
+	char c[2] = { 'x', '\0' };
+	const char *_coremask;
+	long v, needle;
+	int ncores = nr_cpus();
+
+	/*
+	 * This function returns array of usable cores in `cores` and
+	 * returns the number of usable cores (array length) through
+	 * nr_cores.
+	 */
+
+	if (strncmp(coremask, "0x", 2) == 0)
+		_coremask = coremask + 2;
+	else
+		_coremask = coremask;
+
+	core_list = realloc(NULL, sizeof(int) * 64);
+	if (!core_list) {
+		pr_err("failed to realloc: %s\n", strerrno());
+		return -1;
+	}
+
+	nr_usable = 0;
+	nr_all = 0;
+	for (n = strlen(_coremask) - 1; n >=0; n--) {
+		c[0] = _coremask[n];
+		v = strtol(c, NULL, 16);
+		if (v == LONG_MIN || v == LONG_MAX) {
+			pr_err("invalid coremask: %s\n", coremask);
+			return -1;
+		}
+
+		for (needle = 0x01; needle < 0x10; needle <<= 1) {
+			nr_all++;
+			if (nr_all > ncores)
+				break; /* too long coremask */
+ 			if (v & needle) {
+				nr_usable++;
+				core_list = realloc(core_list, sizeof(int) * nr_usable);
+				if (!core_list) {
+					pr_err("failed to realloc: %s\n", strerrno());
+					return -1;
+				}
+				core_list[nr_usable - 1] = nr_all - 1;
+			}
+		}
+	}
+
+	if (nr_usable < 1) {
+		pr_err("invalid core mask: %s\n", coremask);
+		return -1;
+	}
+
+	*cores = core_list;
+	*nr_cores = nr_usable;
+	return 0;
+}
+
+int default_nr_threads()
+{
+	return (int)(floor(log(nr_cpus()) * 2) + 1);
+}
+
 int main(int argc, char **argv)
 {
-	struct mscp m;
 	struct ssh_opts opts;
+	sftp_session ctrl;
 	int min_chunk_sz = DEFAULT_MIN_CHUNK_SZ;
 	int max_chunk_sz = 0;
+	char *coremask = NULL;;
 	int verbose = 1;
 	bool dryrun = false;
 	int ret = 0, n;
+	int *cores, nr_cores;
 	char ch;
 
 	memset(&opts, 0, sizeof(opts));
+	opts.nodelay = 1;
 	memset(&m, 0, sizeof(m));
 	INIT_LIST_HEAD(&m.file_list);
 	INIT_LIST_HEAD(&m.chunk_list);
 	lock_init(&m.chunk_lock);
-	m.sftp_buf_sz = DEFAULT_SFTP_BUF_SZ;
-	m.io_buf_sz = DEFAULT_IO_BUF_SZ;
 	m.nr_ahead = DEFAULT_NR_AHEAD;
+	m.buf_sz = DEFAULT_BUF_SZ;
+	m.nr_threads = default_nr_threads();
 
-	nr_threads = (int)(nr_cpus() / 2);
+	while ((ch = getopt(argc, argv, "n:m:s:S:a:b:vqDrl:p:i:c:M:CHdNh")) != -1) {
-	nr_threads = nr_threads == 0 ? 1 : nr_threads;
-
-	while ((ch = getopt(argc, argv, "n:s:S:b:B:a:vqDl:p:i:c:CHdh")) != -1) {
 		switch (ch) {
 		case 'n':
-			nr_threads = atoi(optarg);
+			m.nr_threads = atoi(optarg);
-			if (nr_threads < 1) {
+			if (m.nr_threads < 1) {
 				pr_err("invalid number of connections: %s\n", optarg);
 				return 1;
 			}
 			break;
+		case 'm':
+			coremask = optarg;
+			break;
 		case 's':
 			min_chunk_sz = atoi(optarg);
 			if (min_chunk_sz < getpagesize()) {
@@ -226,20 +291,6 @@ int main(int argc, char **argv)
 				return -1;
 			}
 			break;
-		case 'b':
-			m.sftp_buf_sz = atoi(optarg);
-			if (m.sftp_buf_sz < 1) {
-				pr_err("invalid buffer size: %s\n", optarg);
-				return -1;
-			}
-			break;
-		case 'B':
-			m.io_buf_sz = atoi(optarg);
-			if (m.io_buf_sz < 1) {
-				pr_err("invalid buffer size: %s\n", optarg);
-				return -1;
-			}
-			break;
 		case 'a':
 			m.nr_ahead = atoi(optarg);
 			if (m.nr_ahead < 1) {
@@ -247,6 +298,13 @@ int main(int argc, char **argv)
 				return -1;
 			}
 			break;
+		case 'b':
+			m.buf_sz = atoi(optarg);
+			if (m.buf_sz < 1) {
+				pr_err("invalid buffer size: %s\n", optarg);
+				return -1;
+			}
+			break;
 		case 'v':
 			verbose++;
 			break;
@@ -256,6 +314,9 @@ int main(int argc, char **argv)
 		case 'D':
 			dryrun = true;
 			break;
+		case 'r':
+			/* for compatibility with scp */
+			break;
 		case 'l':
 			opts.login_name = optarg;
 			break;
@@ -268,6 +329,9 @@ int main(int argc, char **argv)
 		case 'c':
 			opts.cipher = optarg;
 			break;
+		case 'M':
+			opts.hmac = optarg;
+			break;
 		case 'C':
 			opts.compress++;
 			break;
@@ -277,6 +341,9 @@ int main(int argc, char **argv)
 		case 'd':
 			opts.debuglevel++;
 			break;
+		case 'N':
+			opts.nodelay = 0;
+			break;
 		case 'h':
 			usage(true);
 			return 0;
@@ -288,19 +355,29 @@ int main(int argc, char **argv)
 
 	pprint_set_level(verbose);
 
+	if (argc - optind < 2) {
+		/* mscp needs at lease 2 (src and target) argument */
+		usage(false);
+		return 1;
+	}
+	m.target = argv[argc - 1];
+
 	if (max_chunk_sz > 0 && min_chunk_sz > max_chunk_sz) {
 		pr_err("smaller max chunk size than min chunk size: %d < %d\n",
 		       max_chunk_sz, min_chunk_sz);
 		return 1;
 	}
 
-	if (argc - optind < 2) {
+	/* expand usable cores from coremask */
-		/* mscp needs at lease 2 (src and target) argument */
+	if (coremask) {
-		usage(false);
+		if (expand_coremask(coremask, &cores, &nr_cores) < 0)
-		return 1;
+			return -1;
+		pprint(2, "cpu cores:");
+		for (n = 0; n < nr_cores; n++)
+			pprint(2, " %d", cores[n]);
+		pprint(2, "\n");
 	}
-
+	pprint2("number of connections: %d\n", m.nr_threads);
-	m.target = argv[argc - 1];
 
 	/* create control session */
 	m.host = find_hostname(optind, argc, argv);
@@ -309,15 +386,14 @@ int main(int argc, char **argv)
 		return 1;
 	}
 	pprint3("connecting to %s for checking destinations...\n", m.host);
-	m.ctrl = ssh_make_sftp_session(m.host, &opts);
+	ctrl = ssh_init_sftp_session(m.host, &opts);
-	if (!m.ctrl)
+	if (!ctrl)
 		return 1;
 	m.opts = &opts; /* save ssh-able ssh_opts */
 
 
 	/* fill file list */
-	ret = file_fill(m.ctrl, &m.file_list, &argv[optind], argc - optind - 1,
+	ret = file_fill(ctrl, &m.file_list, &argv[optind], argc - optind - 1, m.target);
-			m.target);
 	if (ret < 0)
 		goto out;
 
@@ -327,7 +403,7 @@ int main(int argc, char **argv)
 
 	/* fill chunk list */
 	ret = chunk_fill(&m.file_list, &m.chunk_list,
-			 nr_threads, min_chunk_sz, max_chunk_sz);
+			 m.nr_threads, min_chunk_sz, max_chunk_sz);
 	if (ret < 0)
 		goto out;
 
@@ -335,51 +411,50 @@ int main(int argc, char **argv)
 	chunk_dump(&m.chunk_list);
 #endif
 
-	if (dryrun)
+	if (dryrun) {
+		ssh_sftp_close(ctrl);
 		return 0;
-
-	/* prepare thread instances */
-	if ((n = list_count(&m.chunk_list)) < nr_threads) {
-		pprint3("we have only %d chunk(s). set nr_conns to %d\n", n, n);
-		nr_threads = n;
 	}
 
-	threads = calloc(nr_threads, sizeof(struct mscp_thread));
+	/* prepare thread instances */
-	memset(threads, 0, nr_threads * sizeof(struct mscp_thread));
+	if ((n = list_count(&m.chunk_list)) < m.nr_threads) {
-	for (n = 0; n < nr_threads; n++) {
+		pprint2("we have only %d chunk(s). "
-		struct mscp_thread *t = &threads[n];
+			"set number of connections to %d\n", n, n);
-		t->mscp = &m;
+		m.nr_threads = n;
+	}
+
+	m.threads = calloc(m.nr_threads, sizeof(struct mscp_thread));
+	memset(m.threads, 0, m.nr_threads * sizeof(struct mscp_thread));
+	for (n = 0; n < m.nr_threads; n++) {
+		struct mscp_thread *t = &m.threads[n];
 		t->finished = false;
-		pprint3("connecting to %s for a copy thread...\n", m.host);
+		if (!coremask)
-		t->sftp = ssh_make_sftp_session(m.host, m.opts);
+			t->cpu = -1;
+		else
+			t->cpu = cores[n % nr_cores];
+
+		if (n == 0) {
+			t->sftp = ctrl; /* reuse ctrl sftp session */
+			ctrl = NULL;
+		} else {
+			pprint3("connecting to %s for a copy thread...\n", m.host);
+			t->sftp = ssh_init_sftp_session(m.host, m.opts);
+		}
 		if (!t->sftp) {
 			ret = 1;
-			goto join_out;
+			goto out;
 		}
 	}
 
-	/* spawn count thread */
+	/* init mscp stat for printing progress bar */
-	ret = pthread_create(&mtid, NULL, mscp_monitor_thread, &m);
+	if (mscp_stat_init() < 0) {
-	if (ret < 0) {
-		pr_err("pthread_create error: %d\n", ret);
-		stop_copy_threads(0);
-		ret = 1;
-		goto join_out;
-	}
-
-	/* register SIGINT to stop threads */
-	if (signal(SIGINT, stop_copy_threads) == SIG_ERR) {
-		pr_err("cannot set signal: %s\n", strerrno());
 		ret = 1;
 		goto out;
 	}
 
-	/* save start time */
+	/* spawn copy threads */
-	gettimeofday(&m.start, NULL);
+	for (n = 0; n < m.nr_threads; n++) {
-
+		struct mscp_thread *t = &m.threads[n];
-	/* spawn threads */
-	for (n = 0; n < nr_threads; n++) {
-		struct mscp_thread *t = &threads[n];
 		ret = pthread_create(&t->tid, NULL, mscp_copy_thread, t);
 		if (ret < 0) {
 			pr_err("pthread_create error: %d\n", ret);
@@ -389,23 +464,37 @@ int main(int argc, char **argv)
 		}
 	}
 
-join_out:
+	/* register SIGINT to stop threads */
-	/* waiting for threads join... */
+	if (signal(SIGINT, stop_copy_threads) == SIG_ERR) {
-	for (n = 0; n < nr_threads; n++)
+		pr_err("cannot set signal: %s\n", strerrno());
-		if (threads[n].tid) {
+		ret = 1;
-			pthread_join(threads[n].tid, NULL);
+		goto out;
-			if (threads[n].ret < 0)
-				ret = threads[n].ret;
-		}
-
-	if (mtid != 0) {
-		pthread_cancel(mtid);
-		pthread_join(mtid, NULL);
 	}
 
+join_out:
+	/* waiting for threads join... */
+	for (n = 0; n < m.nr_threads; n++) {
+		if (m.threads[n].tid) {
+			pthread_join(m.threads[n].tid, NULL);
+			if (m.threads[n].ret < 0)
+				ret = m.threads[n].ret;
+		}
+	}
+
+	/* print final result */
+	mscp_stat_final();
+
 out:
-	if (m.ctrl)
+	if (ctrl)
-		ssh_sftp_close(m.ctrl);
+		ssh_sftp_close(ctrl);
+
+	if (m.threads) {
+		for (n = 0; n < m.nr_threads; n++) {
+			struct mscp_thread *t = &m.threads[n];
+			if (t->sftp)
+				ssh_sftp_close(t->sftp);
+		}
+	}
 
 	return ret;
 }
@@ -413,27 +502,28 @@ out:
 void mscp_copy_thread_cleanup(void *arg)
 {
 	struct mscp_thread *t = arg;
-	if (t->sftp)
-		ssh_sftp_close(t->sftp);
 	t->finished = true;
-	__sync_synchronize();
 }
 
 void *mscp_copy_thread(void *arg)
 {
 	struct mscp_thread *t = arg;
-	struct mscp *m = t->mscp;
 	sftp_session sftp = t->sftp;
 	struct chunk *c;
 
+	if (t->cpu > -1) {
+		if (set_thread_affinity(pthread_self(), t->cpu) < 0)
+			return NULL;
+	}
+
 	pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
 	pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL);
 	pthread_cleanup_push(mscp_copy_thread_cleanup, t);
 
 	while (1) {
-		lock_acquire(&m->chunk_lock);
+		LOCK_ACQUIRE_THREAD(&m.chunk_lock);
-		c = chunk_acquire(&m->chunk_list);
+		c = chunk_acquire(&m.chunk_list);
-		lock_release(&m->chunk_lock);
+		LOCK_RELEASE_THREAD();
 
 		if (!c)
 			break; /* no more chunks */
@@ -441,8 +531,7 @@ void *mscp_copy_thread(void *arg)
 		if ((t->ret = chunk_prepare(c, sftp)) < 0)
 			break;
 
-		if ((t->ret = chunk_copy(c, sftp, m->sftp_buf_sz, m->io_buf_sz,
+		if ((t->ret = chunk_copy(c, sftp, m.nr_ahead, m.buf_sz, &t->done)) < 0)
-					 m->nr_ahead, &t->done)) < 0)
 			break;
 	}
 
@@ -455,7 +544,10 @@ void *mscp_copy_thread(void *arg)
 	return NULL;
 }
 
-static double calculate_bps(size_t diff, struct timeval *b, struct timeval *a)
+
+/* progress bar-related functions */
+
+double calculate_timedelta(struct timeval *b, struct timeval *a)
 {
 	double sec, usec;
 
@@ -468,27 +560,48 @@ static double calculate_bps(size_t diff, struct timeval *b, struct timeval *a)
 	usec = a->tv_usec - b->tv_usec;
 	sec += usec / 1000000;
 
-	return (double)diff / sec;
+	return sec;
 }
 
-static void print_progress_bar(double percent, char *suffix)
+double calculate_bps(size_t diff, struct timeval *b, struct timeval *a)
+{
+	return (double)diff / calculate_timedelta(b, a);
+}
+
+char *calculate_eta(size_t remain, size_t diff, struct timeval *b, struct timeval *a)
+{
+	static char buf[16];
+	double elapsed = calculate_timedelta(b, a);
+	double eta;
+
+	if (diff == 0)
+		snprintf(buf, sizeof(buf), "--:-- ETA");
+	else {
+		eta = remain / (diff / elapsed);
+		snprintf(buf, sizeof(buf), "%02d:%02d ETA",
+			 (int)floor(eta / 60), (int)round(eta) % 60);
+	}
+	return buf;
+}
+
+void print_progress_bar(double percent, char *suffix)
 {
 	int n, thresh, bar_width;
 	struct winsize ws;
 	char buf[128];
 
 	/*
-	 * [=======>   ] XX.X% SUFFIX
+	 * [=======>   ] XX% SUFFIX
 	 */
 
 	buf[0] = '\0';
 
 	if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws) < 0)
 		return; /* XXX */
-	bar_width = min(sizeof(buf), ws.ws_col) - strlen(suffix) - 8;
+	bar_width = min(sizeof(buf), ws.ws_col) - strlen(suffix) - 7;
 
+	memset(buf, 0, sizeof(buf));
 	if (bar_width > 8) {
-		memset(buf, 0, sizeof(buf));
 		thresh = floor(bar_width * (percent / 100)) - 1;
 
 		for (n = 1; n < bar_width - 1; n++) {
@@ -507,8 +620,8 @@ static void print_progress_bar(double percent, char *suffix)
 	pprint1("%s%s", buf, suffix);
 }
 
-static void print_progress(struct timeval *start, struct timeval *end,
+void print_progress(struct timeval *b, struct timeval *a,
-			   size_t total, size_t last, size_t done)
+		    size_t total, size_t last, size_t done)
 {
 	char *bps_units[] = { "B/s ", "KB/s", "MB/s", "GB/s" };
 	char *byte_units[] = { "B ", "KB", "MB", "GB", "TB", "PB" };
@@ -530,7 +643,7 @@ static void print_progress(struct timeval *start, struct timeval *end,
 	     byte_tu++)
 		total_round /= 1024;
 
-	bps = calculate_bps(done - last, start, end);
+	bps = calculate_bps(done - last, b, a);
 	for (bps_u = 0; bps > 1000 && bps_u < array_size(bps_units); bps_u++)
 		bps /= 1000;
 
@@ -541,83 +654,64 @@ static void print_progress(struct timeval *start, struct timeval *end,
 	     byte_du++)
 		done_round /= 1024;
 
-	snprintf(suffix, sizeof(suffix), "%lu%s/%lu%s %6.1f%s ",
+	snprintf(suffix, sizeof(suffix), "%4lu%s/%lu%s %6.1f%s  %s",
 		 done_round, byte_units[byte_du], total_round, byte_units[byte_tu],
-		 bps, bps_units[bps_u]);
+		 bps, bps_units[bps_u], calculate_eta(total - done, done - last, b, a));
 
 	print_progress_bar(percent, suffix);
 }
 
-void mscp_monitor_thread_cleanup(void *arg)
+
+struct mscp_stat {
+	struct timeval start, before, after;
+	size_t total;
+	size_t last;
+	size_t done;
+} s;
+
+void mscp_stat_handler(int signum)
 {
-	struct mscp *m = arg;
-	struct timeval end;
-	struct file *f;
-	size_t total, done;
 	int n;
 
-	total = done = 0;
+	for (s.done = 0, n = 0; n < m.nr_threads; n++)
+		s.done += m.threads[n].done;
 
-	gettimeofday(&end, NULL);
+	gettimeofday(&s.after, NULL);
-
+	if (signum == SIGALRM) {
-	/* get total byte to be transferred */
+		alarm(1);
-	list_for_each_entry(f, &m->file_list, list) {
+		print_progress(&s.before, &s.after, s.total, s.last, s.done);
-		total += f->size;
+		s.before = s.after;
+		s.last = s.done;
+	} else {
+		/* called from mscp_stat_final. calculate progress from the beginning */
+		print_progress(&s.start, &s.after, s.total, 0, s.done);
+		pprint(1, "\n"); /* this is final output. */
 	}
-
-	/* get total byte transferred */
-	for (n = 0; n < nr_threads; n++) {
-		done += threads[n].done;
-	}
-
-	print_progress(&m->start, &end, total, 0, done);
-	fputs("\n", stdout); /* the final ouput. we need \n */
 }
 
-void *mscp_monitor_thread(void *arg)
+int mscp_stat_init()
 {
-	struct mscp *m = arg;
-	struct timeval a, b;
 	struct file *f;
-	bool all_done;
-	size_t total, done, last;
-	int n;
 
-	pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
+	memset(&s, 0, sizeof(s));
-	pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL);
+	list_for_each_entry(f, &m.file_list, list) {
-	pthread_cleanup_push(mscp_monitor_thread_cleanup, m);
+		s.total += f->size;
-
-	/* get total byte to be transferred */
-	total = 0;
-	list_for_each_entry(f, &m->file_list, list) {
-		total += f->size;
 	}
 
-	while (1) {
+	if (signal(SIGALRM, mscp_stat_handler) == SIG_ERR) {
-		all_done = true;
+		pr_err("signal: %s\n", strerrno());
-		last = done = 0;
+		return -1;
-
-		for (n = 0; n < nr_threads; n++) {
-			last += threads[n].done;
-		}
-		gettimeofday(&b, NULL);
-
-		usleep(1000000);
-
-		for (n = 0; n < nr_threads; n++) {
-			done += threads[n].done;
-			if (!threads[n].finished)
-				all_done = false;
-		}
-		gettimeofday(&a, NULL);
-
-		print_progress(&b, &a, total, last, done);
-
-		if (all_done || total == done)
-			break;
 	}
 
-	pthread_cleanup_pop(1);
+	gettimeofday(&s.start, NULL);
+	s.before = s.start;
+	alarm(1);
 
-	return NULL;
+	return 0;
+}
+
+void mscp_stat_final()
+{
+	alarm(0);
+	mscp_stat_handler(0);
 }

src/platform.c

@@ -24,6 +24,13 @@ int nr_cpus()
 
 	return n;
 }
+
+int set_thread_affinity(pthread_t tid, int core)
+{
+	pr_warn("setting thread afinity is not implemented on apple\n");
+	return 0;
+}
+
 #endif
 
 #ifdef linux
@@ -34,5 +41,19 @@ int nr_cpus()
 		return CPU_COUNT(&cpu_set);
 	return -1;
 }
+
+int set_thread_affinity(pthread_t tid, int core)
+{
+	cpu_set_t target_cpu_set;
+	int ret = 0;
+
+	CPU_ZERO(&target_cpu_set);
+	CPU_SET(core, &target_cpu_set);
+	ret = pthread_setaffinity_np(tid, sizeof(target_cpu_set), &target_cpu_set);
+	if (ret < 0)
+		pr_err("failed to set thread/cpu affinity for core %d: %s",
+		       core, strerrno());
+	return ret;
+}
 #endif
 

src/platform.h

@@ -1,6 +1,9 @@
 #ifndef _PLATFORM_H_
 #define _PLATFORM_H_
 
+#include <pthread.h>
+
 int nr_cpus();
+int set_thread_affinity(pthread_t tid, int core);
 
 #endif /* _PLATFORM_H_ */

src/rename-logic.py

@@ -32,13 +32,17 @@ def recursive(src, rel_path, dst, dst_should_dir, replace_dir_name):
         recursive(next_src, next_rel_path, dst, dst_should_dir, False)
 
 
-def fill_dst(src, dst):
+def fill_dst(srclist, dst):
-    dst_should_dir = isdir(src) | isdir(dst)
+    dst_must_dir = len(srclist) > 1
-    replace_dir_name = not isdir(dst)
+    for src in srclist:
-    recursive(src, "", dst, dst_should_dir, replace_dir_name)
+        dst_should_dir = isdir(src) | isdir(dst)
+        replace_dir_name = not isdir(dst)
+        recursive(src, "", dst, dst_should_dir | dst_must_dir, replace_dir_name)
 
 
 def main():
-    fill_dst(sys.argv[1], sys.argv[2])
+    if (len(sys.argv) < 2):
+        print("usage: {} source ... target".format(sys.argv[0]))
+    fill_dst(sys.argv[1:len(sys.argv) - 1], sys.argv[len(sys.argv) - 1])
 
 main()

src/ssh.c

@@ -1,6 +1,9 @@
 #include <stdio.h>
 #include <string.h>
 #include <unistd.h>
+#include <stdlib.h>
+
+#include "libssh/callbacks.h"
 
 #include <ssh.h>
 #include <util.h>
@@ -32,11 +35,22 @@ static int ssh_set_opts(ssh_session ssh, struct ssh_opts *opts)
 
 	if (opts->cipher) {
 		if (ssh_options_set(ssh, SSH_OPTIONS_CIPHERS_C_S, opts->cipher) < 0) {
-			pr_err("failed to set cipher client to server\n");
+			pr_err("failed to set cipher for client to server\n");
 			return -1;
 		}
 		if (ssh_options_set(ssh, SSH_OPTIONS_CIPHERS_S_C, opts->cipher) < 0) {
-			pr_err("failed to set cipher client to server\n");
+			pr_err("failed to set cipher for server to client\n");
+			return -1;
+		}
+	}
+
+	if (opts->hmac) {
+		if (ssh_options_set(ssh, SSH_OPTIONS_HMAC_C_S, opts->hmac) < 0) {
+			pr_err("failed to set hmac for client to server\n");
+			return -1;
+		}
+		if (ssh_options_set(ssh, SSH_OPTIONS_HMAC_S_C, opts->hmac) < 0) {
+			pr_err("failed to set hmac for server to client\n");
 			return -1;
 		}
 	}
@@ -47,6 +61,12 @@ static int ssh_set_opts(ssh_session ssh, struct ssh_opts *opts)
 		return -1;
 	}
 
+	if (opts->nodelay &&
+	    ssh_options_set(ssh, SSH_OPTIONS_NODELAY, &opts->nodelay) < 0) {
+		pr_err("failed to set nodelay\n");
+		return -1;
+	}
+
 	return 0;
 }
 
@@ -63,31 +83,79 @@ static int ssh_authenticate(ssh_session ssh, struct ssh_opts *opts)
 	auth_bit_mask = ssh_userauth_list(ssh, NULL);
 
 	if (auth_bit_mask & SSH_AUTH_METHOD_NONE &&
-	    ssh_userauth_none(ssh, NULL) == SSH_AUTH_SUCCESS) {
+	    ssh_userauth_none(ssh, NULL) == SSH_AUTH_SUCCESS)
 		return 0;
-	}
 
 	if (auth_bit_mask & SSH_AUTH_METHOD_PUBLICKEY &&
-	    ssh_userauth_publickey_auto(ssh, NULL, NULL) == SSH_AUTH_SUCCESS) {
+	    ssh_userauth_publickey_auto(ssh, NULL, opts->passphrase) == SSH_AUTH_SUCCESS)
 		return 0;
-	}
 
 	if (auth_bit_mask & SSH_AUTH_METHOD_PASSWORD) {
 		if (!opts->password) {
-			opts->password = getpass("Password: ");
+			opts->password = malloc(PASSWORD_BUF_SZ);
+			if (!opts->password) {
+				pr_err("malloc: %s\n", strerrno());
+				return -1;
+			}
+			memset(opts->password, 0, PASSWORD_BUF_SZ);
+
+			if (ssh_getpass("Password: ", opts->password, PASSWORD_BUF_SZ,
+					0, 0) < 0) {
+				return -1;
+			}
 		}
+
 		if (ssh_userauth_password(ssh, NULL, opts->password) == SSH_AUTH_SUCCESS)
 			return 0;
 	}
 
-	pr_err("authentication failure: %s\n", ssh_get_error(ssh));
 	return -1;
 }
 
-static ssh_session ssh_make_ssh_session(char *sshdst, struct ssh_opts *opts)
+static int ssh_cache_passphrase(const char *prompt, char *buf, size_t len, int echo,
+				int verify, void *userdata)
+{
+	struct ssh_opts *opts = userdata;
+
+	/* This function is called on the first time for importing
+	 * priv key file with passphrase. It is not called on the
+	 * second time or after because cached passphrase is passed
+	 * to ssh_userauth_publickey_auto(). */
+
+	if (opts->passphrase) {
+		/* passphrase is cached, but this function is called.
+		 * maybe it was an invalid passphrase? */
+		free(opts->passphrase);
+		opts->passphrase = NULL;
+	}
+
+	if (ssh_getpass("Passphrase: ", buf, len, echo, verify) < 0)
+		return -1;
+
+	/* cache the passphrase */
+	opts->passphrase = malloc(len);
+	if (!opts->passphrase) {
+		pr_err("malloc: %s\n", strerrno());
+		return -1;
+	}
+	memcpy(opts->passphrase, buf, len);
+
+	return 0;
+}
+
+static struct ssh_callbacks_struct cb = {
+	.auth_function = ssh_cache_passphrase,
+	.userdata = NULL,
+};
+
+static ssh_session ssh_init_session(char *sshdst, struct ssh_opts *opts)
 {
 	ssh_session ssh = ssh_new();
 
+	ssh_callbacks_init(&cb);
+	cb.userdata = opts;
+	ssh_set_callbacks(ssh, &cb);
+
 	if (ssh_set_opts(ssh, opts) != 0)
 		goto free_out;
 
@@ -119,10 +187,10 @@ free_out:
 	return NULL;
 }
 
-sftp_session ssh_make_sftp_session(char *sshdst, struct ssh_opts *opts)
+sftp_session ssh_init_sftp_session(char *sshdst, struct ssh_opts *opts)
 {
 	sftp_session sftp;
-	ssh_session ssh = ssh_make_ssh_session(sshdst, opts);
+	ssh_session ssh = ssh_init_session(sshdst, opts);
 
 	if (!ssh) {
 		return NULL;
@@ -205,7 +273,7 @@ static int ssh_verify_known_hosts(ssh_session session)
 
 	case SSH_KNOWN_HOSTS_UNKNOWN:
 		hexa = ssh_get_hexa(hash, hlen);
-		fprintf(stderr,"The server is unknown. Do you trust the host key?\n");
+		fprintf(stderr, "The server is unknown. Do you trust the host key?\n");
 		fprintf(stderr, "Public key hash: %s\n", hexa);
 		fprintf(stderr, "(yes/no): ");
 		ssh_string_free_char(hexa);
@@ -240,36 +308,10 @@ static int ssh_verify_known_hosts(ssh_session session)
 void ssh_sftp_close(sftp_session sftp)
 {
 	ssh_session ssh = sftp_ssh(sftp);
-	sftp_free(sftp);
+	/* XXX: sftp_free is stuck in ssh_poll_ctx_dopoll() when build type is Release.
+	 * skip sftp_free inappropriately...
+	 */
+	//sftp_free(sftp);
 	ssh_disconnect(ssh);
 	ssh_free(ssh);
 }
-
-
-ssize_t sftp_write2(sftp_file sf, const void *buf, size_t len, size_t sftp_buf_sz)
-{
-	ssize_t ret, nbytes;
-
-	for (nbytes = 0; nbytes < len;) {
-		ret = sftp_write(sf, buf + nbytes,
-				 min(len - nbytes, sftp_buf_sz));
-		if (ret < 0)
-			return ret;
-		nbytes += ret;
-	}
-	return nbytes;
-}
-
-ssize_t sftp_read2(sftp_file sf, void *buf, size_t len, size_t sftp_buf_sz)
-{
-	ssize_t ret, nbytes;
-
-	for (nbytes = 0; nbytes < len;) {
-		ret = sftp_read(sf, buf + nbytes,
-				min(len - nbytes, sftp_buf_sz));
-		if (ret < 0)
-			return ret;
-		nbytes += ret;
-	}
-	return nbytes;
-}

src/ssh.h

@@ -2,8 +2,8 @@
 #define _SSH_H_
 
 #include <stdbool.h>
-#include <libssh/libssh.h>
+#include "libssh/libssh.h"
-#include <libssh/sftp.h>
+#include "libssh/sftp.h"
 
 
 struct ssh_opts {
@@ -11,24 +11,24 @@ struct ssh_opts {
 	char    *port;			/* -p */
 	char    *identity;		/* -i */
 	char    *cipher;		/* -c */
+	char	*hmac;			/* -M */
 	int     compress;		/* -C */
+	int	nodelay;		/* -N */
 	int     debuglevel;		/* -v */
 	bool	no_hostkey_check;	/* -H */
 
-	char    *password;		/* filled at the first connecting phase */
+#define PASSWORD_BUF_SZ	128
+	char    *password;	/* password for password auth */
+	char	*passphrase;	/* passphrase for private key  */
 };
 
-/* ssh_make_sftp_session() creates sftp_session. sshdst accpets
+/* ssh_init_sftp_session() creates sftp_session. sshdst accpets
  * user@hostname and hostname notations (by libssh).
  */
-sftp_session ssh_make_sftp_session(char *sshdst, struct ssh_opts *opts);
+sftp_session ssh_init_sftp_session(char *sshdst, struct ssh_opts *opts);
 void ssh_sftp_close(sftp_session sftp);
 
 #define sftp_ssh(sftp) (sftp)->session
 #define sftp_get_ssh_error(sftp) ssh_get_error(sftp_ssh(sftp))
 
-/* wrapping multiple sftp_read|write */
-ssize_t sftp_write2(sftp_file sf, const void *buf, size_t len, size_t sftp_buf_sz);
-ssize_t sftp_read2(sftp_file sf, void *buf, size_t len, size_t sftp_buf_sz);
-
 #endif /* _SSH_H_ */

test/test_e2e.py

@@ -29,6 +29,7 @@ param_invalid_hostnames = [
     (["a:a", "b:b", "c:c"]), (["a:a", "b:b", "c"]), (["a:a", "b", "c:c"]),
     (["a", "b:b", "c:c"])
 ]
+
 @pytest.mark.parametrize("args", param_invalid_hostnames)
 def test_nonidentical_hostnames(mscp, args):
     run2ng([mscp] + args)
@@ -39,6 +40,9 @@ def test_nonidentical_hostnames(mscp, args):
 """ copy test """
 
 remote_prefix = "localhost:{}/".format(os.getcwd()) # use current dir
+param_remote_prefix = [
+    ("", remote_prefix), (remote_prefix, "")
+]
 
 param_single_copy = [
     (File("src", size = 64), File("dst")),
@@ -46,23 +50,33 @@ param_single_copy = [
     (File("src", size = 128 * 1024 * 1024), File("dst")),
 ]
 
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 @pytest.mark.parametrize("src, dst", param_single_copy)
-def test_single_copy_remote2local(mscp, src, dst):
+def test_single_copy(mscp, src_prefix, dst_prefix, src, dst):
     src.make()
-    run2ok([mscp, "-H", remote_prefix + src.path, dst.path])
+    run2ok([mscp, "-H", src_prefix + src.path, dst_prefix + dst.path])
-    assert check_same_md5sum(src, dst)
-    src.cleanup()
-    dst.cleanup()
-
-@pytest.mark.parametrize("src, dst", param_single_copy)
-def test_single_copy_local2remote(mscp, src, dst):
-    src.make()
-    run2ok([mscp, "-H", src.path, remote_prefix + dst.path])
     assert check_same_md5sum(src, dst)
     src.cleanup()
     dst.cleanup()
 
 
+param_double_copy = [
+    (File("src1", size = 1024 * 1024), File("src2", size = 1024 * 1024),
+     File("dst/src1"), File("dst/src2")
+     )
+]
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+@pytest.mark.parametrize("s1, s2, d1, d2", param_double_copy)
+def test_double_copy(mscp, src_prefix, dst_prefix, s1, s2, d1, d2):
+    s1.make()
+    s2.make()
+    run2ok([mscp, "-H", src_prefix + s1.path, src_prefix + s2.path, dst_prefix + "dst"])
+    assert check_same_md5sum(s1, d1)
+    assert check_same_md5sum(s2, d2)
+    s1.cleanup()
+    s2.cleanup()
+    d1.cleanup()
+    d2.cleanup()
 
 param_dir_copy = [
     ( "src_dir", "dst_dir",
@@ -87,16 +101,17 @@ does not exist. If dst_dir exists, scp copies src_dir to
 dst_dir/src_dir. So, this test checks both cases.
 """
 
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 @pytest.mark.parametrize("src_dir, dst_dir, src, dst, twice", param_dir_copy)
-def test_dir_copy_remote2local(mscp, src_dir, dst_dir, src, dst, twice):
+def test_dir_copy(mscp, src_prefix, dst_prefix, src_dir, dst_dir, src, dst, twice):
     for f in src:
         f.make()
 
-    run2ok([mscp, "-H", remote_prefix + src_dir, dst_dir])
+    run2ok([mscp, "-H", src_prefix + src_dir, dst_prefix + dst_dir])
     for sf, df in zip(src, dst):
         assert check_same_md5sum(sf, df)
 
-    run2ok([mscp, "-H", remote_prefix + src_dir, dst_dir])
+    run2ok([mscp, "-H", src_prefix + src_dir, dst_prefix + dst_dir])
     for sf, df in zip(src, twice):
         assert check_same_md5sum(sf, df)
 
@@ -105,28 +120,6 @@ def test_dir_copy_remote2local(mscp, src_dir, dst_dir, src, dst, twice):
         df.cleanup()
         tf.cleanup()
 
-@pytest.mark.parametrize("src_dir, dst_dir, src, dst, twice", param_dir_copy)
-def test_dir_copy_local2remote(mscp, src_dir, dst_dir, src, dst, twice):
-    for f in src:
-        f.make()
-
-    run2ok([mscp, "-H", src_dir, remote_prefix + dst_dir])
-    for sf, df in zip(src, dst):
-        assert check_same_md5sum(sf, df)
-
-    run2ok([mscp, "-H", src_dir, remote_prefix + dst_dir])
-    for sf, df in zip(src, twice):
-        assert check_same_md5sum(sf, df)
-
-    for sf, df, tf in zip(src, dst, twice):
-        sf.cleanup()
-        df.cleanup()
-        tf.cleanup()
-
-
-param_remote_prefix = [
-    ("", remote_prefix), (remote_prefix, "")
-]
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 def test_override_single_file(mscp, src_prefix, dst_prefix):
     src = File("src", size = 128).make()
@@ -150,6 +143,18 @@ def test_min_chunk(mscp, src_prefix, dst_prefix):
     src.cleanup()
     dst.cleanup()
 
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_thread_affinity(mscp, src_prefix, dst_prefix):
+    src = File("src", size = 64 * 1024).make()
+    dst = File("dst")
+
+    run2ok([mscp, "-H", "-n", 4, "-m", "0x01", "-s", 8192, "-S", 65536,
+            src_prefix + src.path, dst_prefix + dst.path])
+    assert check_same_md5sum(src, dst)
+
+    src.cleanup()
+    dst.cleanup()
+
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 def test_cannot_override_file_with_dir(mscp, src_prefix, dst_prefix):
     src = File("src", size = 128).make()
@@ -169,3 +174,11 @@ def test_transfer_zero_bytes(mscp, src_prefix, dst_prefix):
     src.cleanup()
     dst.cleanup()
 
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_override_dst_having_larger_size(mscp, src_prefix, dst_prefix):
+    src = File("src", size = 1024 * 1024).make()
+    dst = File("dst", size = 1024 * 1024 * 2).make()
+    run2ok([mscp, "-H", src_prefix + src.path, dst_prefix + "dst"])
+    assert check_same_md5sum(src, dst)
+    src.cleanup()
+    dst.cleanup()