bump version to 0.1.2

add --sysctl net.ipv6.conf.all.disable_ipv6=0 for docker run.
docker disables ipv6 on all interfaces inside containers by default, even ::1 on lo. It causes testing mscp with IPv6 fails. Thus, this commit disables disable_ipv6 via the --sysctl option.
2026-02-11 15:34:53 +08:00 · 2023-11-01 19:54:18 +09:00 · 2023-11-01 19:54:18 +09:00 · 2023-11-01 19:54:18 +09:00 · 2023-11-01 19:54:18 +09:00 · 2023-11-01 19:54:18 +09:00
29 changed files with 1450 additions and 489 deletions
--- a/.github/workflows/build-macos.yml
+++ b/.github/workflows/build-macos.yml
@@ -2,9 +2,9 @@ name: build on macOS

 on:
  push:
-    branches: [ "main" ]
+    branches: [ "main", "dev" ]
  pull_request:
-    branches: [ "main" ]
+    branches: [ "main", "dev" ]

 env:
  # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
--- a/.github/workflows/build-ubuntu.yml
+++ b/.github/workflows/build-ubuntu.yml
@@ -2,9 +2,9 @@ name: build on ubuntu

 on:
  push:
-    branches: [ "main" ]
+    branches: [ "main", "dev" ]
  pull_request:
-    branches: [ "main" ]
+    branches: [ "main", "dev" ]

 env:
  # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
--- a/.github/workflows/codeql.yml
+++ b/.github/workflows/codeql.yml
@@ -13,10 +13,10 @@ name: "CodeQL"

 on:
  push:
-    branches: [ "main" ]
+    branches: [ "main", "dev" ]
  pull_request:
    # The branches below must be a subset of the branches above
-    branches: [ "main" ]
+    branches: [ "main", "dev" ]
  schedule:
    - cron: '35 11 * * 5'

--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -2,9 +2,9 @@ name: test

 on:
  push:
-    branches: [ "main" ]
+    branches: [ "main", "dev" ]
  pull_request:
-    branches: [ "main" ]
+    branches: [ "main", "dev" ]

 env:
  BUILD_TYPE: Release
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,8 @@ compile_commands.json
 CMakeUserPresets.json
 .*.swp

+include/mscp_version.h
+
 dist
 *.egg-info
 __pycache__
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -6,6 +6,26 @@ project(mscp
 	VERSION ${MSCP_VERSION}
 	LANGUAGES C)

+
+find_package(Git)
+if (Git_FOUND)
+	# based on https://github.com/nocnokneo/cmake-git-versioning-example
+	execute_process(
+		COMMAND	${GIT_EXECUTABLE} describe --tags --dirty --match "v*"
+		OUTPUT_VARIABLE	GIT_DESCRIBE_VERSION
+		RESULT_VARIABLE	GIT_DESCRIBE_ERROR_CODE
+		OUTPUT_STRIP_TRAILING_WHITESPACE)
+	if(NOT GIT_DESCRIBE_ERROR_CODE)
+		set(MSCP_BUILD_VERSION ${GIT_DESCRIBE_VERSION})
+	endif()
+endif()
+
+if (NOT MSCP_BUILD_VERSION)
+	message(STATUS "Failed to determine version via Git. Use VERSION file instead.")
+	set(MSCP_BUILD_VERSION v${MSCP_VERSION})
+endif()
+
+
 include(GNUInstallDirs)

 set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -DDEBUG")
@@ -62,9 +82,16 @@ if(BUILD_CONAN)
 	list(APPEND MSCP_LINK_LIBS OpenSSL::Crypto)
 endif()

-set(LIBMSCP_SRC src/mscp.c src/ssh.c src/path.c src/platform.c src/message.c)
+
+# generate version header file
+configure_file(
+	${mscp_SOURCE_DIR}/include/mscp_version.h.in
+	${mscp_SOURCE_DIR}/include/mscp_version.h)
+

 # libmscp.so
+set(LIBMSCP_SRC
+	src/mscp.c src/ssh.c src/fileops.c src/path.c src/platform.c src/message.c)
 add_library(mscp-shared SHARED ${LIBMSCP_SRC})
 target_include_directories(mscp-shared
 	PUBLIC	$<BUILD_INTERFACE:${mscp_SOURCE_DIR}/include>
@@ -104,7 +131,6 @@ if (BUILD_STATIC)
 	target_link_options(mscp PRIVATE -static)
 endif()
 target_compile_options(mscp PRIVATE ${MSCP_COMPILE_OPTS})
-target_compile_definitions(mscp PUBLIC _VERSION="${PROJECT_VERSION}")


 install(TARGETS mscp RUNTIME DESTINATION bin)
@@ -185,11 +211,18 @@ foreach(x RANGE ${DIST_LISTLEN})
 		COMMAND
 		docker build -t ${DOCKER_IMAGE} -f docker/${DOCKER_INDEX}.Dockerfile .)

+	add_custom_target(docker-build-${DOCKER_INDEX}-no-cache
+		COMMENT "Build mscp in ${DOCKER_IMAGE} container"
+		WORKING_DIRECTORY ${mscp_SOURCE_DIR}
+		COMMAND
+		docker build --no-cache -t ${DOCKER_IMAGE} -f docker/${DOCKER_INDEX}.Dockerfile .)
+
 	add_custom_target(docker-test-${DOCKER_INDEX}
 		COMMENT "Test mscp in ${DOCKER_IMAGE} container"
 		WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
 		COMMAND
-		docker run --init --rm ${DOCKER_IMAGE} /mscp/scripts/test-in-container.sh)
+		docker run --init --rm --sysctl net.ipv6.conf.all.disable_ipv6=0
+		${DOCKER_IMAGE} /mscp/scripts/test-in-container.sh)

 	add_custom_target(docker-pkg-${DOCKER_INDEX}
 		COMMENT "Retrieve mscp package from ${DOCKER_IMAGE} container"
@@ -198,11 +231,13 @@ foreach(x RANGE ${DIST_LISTLEN})
 		docker run --rm -v ${CMAKE_BINARY_DIR}:/out ${DOCKER_IMAGE}
 		cp /mscp/build/${PKG_FILE_NAME} /out/)

-	list(APPEND DOCKER_BUILDS	docker-build-${DOCKER_INDEX})
-	list(APPEND DOCKER_TESTS	docker-test-${DOCKER_INDEX})
-	list(APPEND DOCKER_PKGS		docker-pkg-${DOCKER_INDEX})
+	list(APPEND DOCKER_BUILDS		docker-build-${DOCKER_INDEX})
+	list(APPEND DOCKER_BUILDS_NO_CACHE	docker-build-${DOCKER_INDEX}-no-cache)
+	list(APPEND DOCKER_TESTS		docker-test-${DOCKER_INDEX})
+	list(APPEND DOCKER_PKGS			docker-pkg-${DOCKER_INDEX})
 endforeach()

-add_custom_target(docker-build-all	DEPENDS ${DOCKER_BUILDS})
-add_custom_target(docker-test-all	DEPENDS ${DOCKER_TESTS})
-add_custom_target(docker-pkg-all	DEPENDS ${DOCKER_PKGS})
+add_custom_target(docker-build-all		DEPENDS ${DOCKER_BUILDS})
+add_custom_target(docker-build-all-no-cache	DEPENDS ${DOCKER_BUILDS_NO_CACHE})
+add_custom_target(docker-test-all		DEPENDS ${DOCKER_TESTS})
+add_custom_target(docker-pkg-all		DEPENDS ${DOCKER_PKGS})
--- a/README.md
+++ b/README.md
@@ -15,18 +15,19 @@ standard `sshd` supporting the SFTP subsystem (e.g. openssh-server),
 and you need to be able to ssh to the hosts as usual. `mscp` does not
 require anything else.

-https://user-images.githubusercontent.com/184632/206889149-7cc6178a-6f0f-41e6-855c-d25e15a9abc5.mp4
+
+https://github.com/upa/mscp/assets/184632/19230f57-be7f-4ef0-98dd-cb4c460f570d

 --------------------------------------------------------------------

 Differences from `scp` on usage:

- remote glob on remote shell expansion is not supported.
- remote to remote copy is not supported.
+- Remote-to-remote copy is not supported.
 - `-r` option is not needed to transfer directories.
 - and any other differences I have not implemented and noticed.

-
+Paper:
+- Ryo Nakamura and Yohei Kuga. 2023. Multi-threaded scp: Easy and Fast File Transfer over SSH. In Practice and Experience in Advanced Research Computing (PEARC '23). Association for Computing Machinery, New York, NY, USA, 320–323. https://doi.org/10.1145/3569951.3597582

 ## Install

@@ -164,11 +165,11 @@ copy done: test/testdir/asdf

 ```console
 $ mscp -h
-mscp v0.0.8: copy files over multiple ssh connections
+mscp v0.0.9-11-g5802679: copy files over multiple ssh connections

 Usage: mscp [vqDHdNh] [-n nr_conns] [-m coremask] [-u max_startups]
            [-s min_chunk_sz] [-S max_chunk_sz] [-a nr_ahead] [-b buf_sz]
-            [-l login_name] [-p port] [-i identity_file]
+            [-l login_name] [-p port] [-F ssh_config] [-i identity_file]
            [-c cipher_spec] [-M hmac_spec] [-C compress] source ... target

    -n NR_CONNECTIONS  number of connections (default: floor(log(cores)*2)+1)
@@ -187,6 +188,7 @@ Usage: mscp [vqDHdNh] [-n nr_conns] [-m coremask] [-u max_startups]

    -l LOGIN_NAME      login name
    -p PORT            port number
+    -F CONFIG          path to user ssh config (default ~/.ssh/config)
    -i IDENTITY        identity file for public key authentication
    -c CIPHER          cipher spec
    -M HMAC            hmac spec
--- a/2
+++ b/2
@@ -1 +1 @@
-0.0.9
+0.1.2
--- a/docker/alpine-3.17.Dockerfile
+++ b/docker/alpine-3.17.Dockerfile
@@ -39,7 +39,3 @@ RUN cd ${mscpdir}							\
 RUN cd ${mscpdir}	\
 	&& python3 setup.py install --user

-# Need Fix: A trick putting libmscp.so to python mscp module dir does not work on alpine,
-# so install libmscp.
-RUN cd ${mscpdir}/build	\
-	&& make install
--- a/examples/.gitignore
+++ b/examples/.gitignore
@@ -0,0 +1,3 @@
+simple-copy-dest
+*.img
+.ipynb_checkpoints
--- a/examples/mscp-example.ipynb
+++ b/examples/mscp-example.ipynb
@@ -0,0 +1,226 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "ccda9e3a-35de-43fc-9b6e-02475c763f6b",
+   "metadata": {},
+   "source": [
+    "# mscp python binding example"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 60,
+   "id": "df04d655-a082-47eb-9a1e-154ebc2a5655",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import glob\n",
+    "import time\n",
+    "import os\n",
+    "\n",
+    "import mscp"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 53,
+   "id": "e9ed4519-c3fd-4639-89a5-1c1cdffd9519",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "this_dir = os.getcwd()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fee75bf8-df40-45f4-81d1-113069c34f13",
+   "metadata": {},
+   "source": [
+    "## Simple copy"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 54,
+   "id": "2b06e6d3-30cc-47be-bd4f-af27eb141c8c",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['../src/ssh.c',\n",
+       " '../src/mscp.c',\n",
+       " '../src/platform.c',\n",
+       " '../src/pymscp.c',\n",
+       " '../src/main.c',\n",
+       " '../src/path.c',\n",
+       " '../src/message.c',\n",
+       " '../src/fileops.c']"
+      ]
+     },
+     "execution_count": 54,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# preparing files to be transferred\n",
+    "c_sources = glob.glob(\"../src/*.c\")\n",
+    "c_sources"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 55,
+   "id": "89bb4558-9472-4d26-9af3-24f426b15edc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# copy files using mscp\n",
+    "dst_dir = this_dir + \"/simple-copy-dest\"\n",
+    "m = mscp.mscp(\"localhost\", mscp.LOCAL2REMOTE)\n",
+    "m.copy(c_sources, dst_dir)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 56,
+   "id": "6daf2c98-8905-4039-b82a-a593df3107fe",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['ssh.c',\n",
+       " 'mscp.c',\n",
+       " 'platform.c',\n",
+       " 'pymscp.c',\n",
+       " 'main.c',\n",
+       " 'path.c',\n",
+       " 'message.c',\n",
+       " 'fileops.c']"
+      ]
+     },
+     "execution_count": 56,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "os.listdir(\"simple-copy-dest\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f4a3869a-878e-43b0-9758-a049eaf8b5bd",
+   "metadata": {},
+   "source": [
+    "## Simple Copy with Python Rich ProgressBar"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 64,
+   "id": "e7cb7cd6-b845-4d26-93ed-aee8ed3983ab",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# make a 256MB file\n",
+    "src = \"example-256MB-src.img\"\n",
+    "with open(src, \"wb\") as f:\n",
+    "    f.seek(128 * 1024 * 1024 -1, 0)\n",
+    "    f.write(b'1')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 69,
+   "id": "878607ed-5c06-4b15-81ac-9845dad0c9c6",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b700e9fc00464969a22a26300404dc35",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Output()"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
+      ],
+      "text/plain": []
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# copy the 256MB file while ploting progress bar using python rich\n",
+    "dst = this_dir + \"/example-256MB-dst.img\"\n",
+    "\n",
+    "kw = {\"nr_threads\": 1, \"nr_ahead\": 1} # slow mscp to watch the progress bar\n",
+    "\n",
+    "m = mscp.mscp(\"localhost\", mscp.LOCAL2REMOTE, **kw)\n",
+    "m.copy(src, dst, nonblock = True)\n",
+    "\n",
+    "# m.stats() returns total bytes to be transferred, bytes transferred (done), and finished (bool).\n",
+    "total, done, finished = m.stats()\n",
+    "with Progress() as progress:\n",
+    "\n",
+    "    task = progress.add_task(f\"[green]Copying {src}\", total = total)\n",
+    "\n",
+    "    while not progress.finished:\n",
+    "        total, done, finished = m.stats()\n",
+    "        progress.update(task, completed = done)\n",
+    "        time.sleep(0.5)\n",
+    "\n",
+    "m.join()\n",
+    "m.cleanup()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
--- a/examples/mscp-python.py
+++ b/examples/mscp-python.py
@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+"""mscp.py
+
+An example python script running mscp
+"""
+
+import argparse
+import time
+import sys
+
+from rich.progress import Progress
+
+import mscp
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-f", "--from", dest = "fr",
+                        metavar = "REMOTE", default = None,
+                        help = "copy a file from this remote host")
+    parser.add_argument("-t", "--to", metavar = "REMOTE", default = None,
+                        help = "copy a file to this remote host")
+    parser.add_argument("source", help = "path to source file to be copied")
+    parser.add_argument("destination", help = "path of copy destination")
+
+    args = parser.parse_args()
+
+    if args.fr and args.to:
+        print("-f and -t are exclusive", file = sys.stderr)
+        sys.exit(1)
+    elif args.fr:
+        d = mscp.REMOTE2LOCAL
+        remote = args.fr
+    elif args.to:
+        d = mscp.LOCAL2REMOTE
+        remote = args.to
+    else:
+        print("-f or -t must be specified", file = sys.stderr)
+        sys.exit(1)
+
+
+    m = mscp.mscp(remote, d)
+    m.connect()
+    m.add_src_path(args.source)
+    m.set_dst_path(args.destination)
+    m.scan()
+    m.start()
+
+    total, done, finished = m.stats()
+    with Progress() as progress:
+
+        task = progress.add_task("[green]Copying...", total = total)
+
+        while not progress.finished:
+            total, done, finished = m.stats()
+            progress.update(task, completed = done)
+            time.sleep(0.5)
+
+    m.join()
+    m.cleanup()
+
+
+if __name__ == "__main__":
+    main()
--- a/include/mscp.h
+++ b/include/mscp.h
@@ -56,6 +56,7 @@ struct mscp_opts {
 #define MSCP_SSH_MAX_CIPHER_STR		32
 #define MSCP_SSH_MAX_HMAC_STR		32
 #define MSCP_SSH_MAX_COMP_STR		32 /* yes, no, zlib, zlib@openssh.com, none */
+#define MSCP_SSH_MAX_CCALGO_STR		16
 #define MSCP_SSH_MAX_PASSWORD		128
 #define MSCP_SSH_MAX_PASSPHRASE		128

@@ -67,10 +68,12 @@ struct mscp_ssh_opts {
 	/* ssh options */
 	char	login_name[MSCP_SSH_MAX_LOGIN_NAME];	/** ssh username */
 	char	port[MSCP_SSH_MAX_PORT_STR];		/** ssh port */
+	char	config[PATH_MAX];	/** path to ssh_config, default ~/.ssh/config*/
 	char	identity[MSCP_SSH_MAX_IDENTITY_PATH];	/** path to private key */
 	char	cipher[MSCP_SSH_MAX_CIPHER_STR];	/** cipher spec */
 	char	hmac[MSCP_SSH_MAX_HMAC_STR];		/** hmacp spec */
 	char	compress[MSCP_SSH_MAX_COMP_STR];	/** yes, no, zlib@openssh.com */
+	char	ccalgo[MSCP_SSH_MAX_CCALGO_STR];	/** TCP cc algorithm */

 	char	password[MSCP_SSH_MAX_PASSWORD];	/** password auth passowrd */
 	char	passphrase[MSCP_SSH_MAX_PASSPHRASE];	/** passphrase for private key */
--- a/include/mscp_version.h.in
+++ b/include/mscp_version.h.in
@@ -0,0 +1,7 @@
+#ifndef _MSCP_VERSION_H_
+#define _MSCP_VERSION_H_
+
+#define MSCP_VERSION		"@MSCP_VERSION@"
+#define MSCP_BUILD_VERSION	"@MSCP_BUILD_VERSION@"
+
+#endif /* _MSCP_VERSION_H_ */
--- a/mscp/mscp.py
+++ b/mscp/mscp.py
@@ -37,7 +37,7 @@ SEVERITY_DEBUG  = pymscp.SEVERITY_DEBUG

 STATE_INIT      = 0
 STATE_CONNECTED = 1
-STATE_SCANNED  = 2
+STATE_SCANNED   = 2
 STATE_RUNNING   = 3
 STATE_STOPPED   = 4
 STATE_JOINED    = 5
@@ -47,7 +47,7 @@ STATE_RELEASED  = 7
 _state_str = {
    STATE_INIT:      "init",
    STATE_CONNECTED: "connected",
-    STATE_SCANNED:  "scanned",
+    STATE_SCANNED:   "scanned",
    STATE_RUNNING:   "running",
    STATE_STOPPED:   "stopped",
    STATE_JOINED:    "joined",
@@ -115,6 +115,8 @@ class mscp:
        pymscp.mscp_set_dst_path(m = self.m, dst_path = dst_path);

    def scan(self):
+        if self.state == STATE_SCANNED:
+            return
        if self.state != STATE_CONNECTED:
            raise RuntimeError("invalid mscp state: {}".format(self.__state2str()))
        if not self.src_paths:
@@ -139,6 +141,8 @@ class mscp:
        self.state = STATE_STOPPED

    def join(self):
+        if self.state == STATE_JOINED:
+            return
        if not (self.state == STATE_RUNNING or self.state == STATE_STOPPED):
            raise RuntimeError("invalid mscp state: {}".format(self.__state2str()))
        pymscp.mscp_join(m = self.m)
--- a/patch/libssh-0.10.4.patch
+++ b/patch/libssh-0.10.4.patch
@@ -1,3 +1,28 @@
+diff --git a/ConfigureChecks.cmake b/ConfigureChecks.cmake
+index 7103f303..c64eb39d 100644
+--- a/ConfigureChecks.cmake
+++ b/ConfigureChecks.cmake
+@@ -258,6 +258,7 @@ if (UNIX)
+     check_library_exists(util forkpty "" HAVE_LIBUTIL)
+     check_function_exists(cfmakeraw HAVE_CFMAKERAW)
+     check_function_exists(__strtoull HAVE___STRTOULL)
+    check_symbol_exists(TCP_CONGESTION "netinet/tcp.h" HAVE_TCP_CONGESTION)
+ endif (UNIX)
+ 
+ set(LIBSSH_REQUIRED_LIBRARIES ${_REQUIRED_LIBRARIES} CACHE INTERNAL "libssh required system libraries")
+diff --git a/config.h.cmake b/config.h.cmake
+index 1357615b..1e915ead 100644
+--- a/config.h.cmake
+++ b/config.h.cmake
+@@ -237,6 +237,8 @@
+ 
+ #cmakedefine HAVE_GCC_BOUNDED_ATTRIBUTE 1
+ 
+#cmakedefine HAVE_TCP_CONGESTION 1
+
+ /* Define to 1 if you want to enable GSSAPI */
+ #cmakedefine WITH_GSSAPI 1
+ 
 diff --git a/include/libssh/buffer.h b/include/libssh/buffer.h
 index a55a1b40..e34e075c 100644
 --- a/include/libssh/buffer.h
@@ -12,10 +37,18 @@ index a55a1b40..e34e075c 100644
 int ssh_buffer_validate_length(struct ssh_buffer_struct *buffer, size_t len);
 
 diff --git a/include/libssh/libssh.h b/include/libssh/libssh.h
-index 7857a77b..3eef7a16 100644
+index 7857a77b..6b4d481c 100644
 --- a/include/libssh/libssh.h
 +++ b/include/libssh/libssh.h
-@@ -833,6 +833,7 @@ LIBSSH_API const char* ssh_get_hmac_in(ssh_session session);
+@@ -402,6 +402,7 @@ enum ssh_options_e {
+   SSH_OPTIONS_GSSAPI_AUTH,
+   SSH_OPTIONS_GLOBAL_KNOWNHOSTS,
+   SSH_OPTIONS_NODELAY,
+  SSH_OPTIONS_CCALGO,
+   SSH_OPTIONS_PUBLICKEY_ACCEPTED_TYPES,
+   SSH_OPTIONS_PROCESS_CONFIG,
+   SSH_OPTIONS_REKEY_DATA,
+@@ -833,6 +834,7 @@ LIBSSH_API const char* ssh_get_hmac_in(ssh_session session);
 LIBSSH_API const char* ssh_get_hmac_out(ssh_session session);
 
 LIBSSH_API ssh_buffer ssh_buffer_new(void);
@@ -23,7 +56,7 @@ index 7857a77b..3eef7a16 100644
 LIBSSH_API void ssh_buffer_free(ssh_buffer buffer);
 #define SSH_BUFFER_FREE(x) \
     do { if ((x) != NULL) { ssh_buffer_free(x); x = NULL; } } while(0)
-@@ -843,6 +844,8 @@ LIBSSH_API void *ssh_buffer_get(ssh_buffer buffer);
+@@ -843,6 +845,8 @@ LIBSSH_API void *ssh_buffer_get(ssh_buffer buffer);
 LIBSSH_API uint32_t ssh_buffer_get_len(ssh_buffer buffer);
 LIBSSH_API int ssh_session_set_disconnect_message(ssh_session session, const char *message);
 
@@ -32,6 +65,18 @@ index 7857a77b..3eef7a16 100644
 #ifndef LIBSSH_LEGACY_0_4
 #include "libssh/legacy.h"
 #endif
+diff --git a/include/libssh/session.h b/include/libssh/session.h
+index d3e5787c..15183d1b 100644
+--- a/include/libssh/session.h
+++ b/include/libssh/session.h
+@@ -232,6 +232,7 @@ struct ssh_session_struct {
+         int gss_delegate_creds;
+         int flags;
+         int nodelay;
+        char *ccalgo;
+         bool config_processed;
+         uint8_t options_seen[SOC_MAX];
+         uint64_t rekey_data;
 diff --git a/include/libssh/sftp.h b/include/libssh/sftp.h
 index c855df8a..0fcdb9b8 100644
 --- a/include/libssh/sftp.h
@@ -158,6 +203,106 @@ index e0068015..cc0caf35 100644
 /**
  * @brief Ensure the buffer has at least a certain preallocated size.
  *
+diff --git a/src/connect.c b/src/connect.c
+index 57e37e63..c02397d5 100644
+--- a/src/connect.c
+++ b/src/connect.c
+@@ -156,6 +156,20 @@ static int set_tcp_nodelay(socket_t socket)
+                       sizeof(opt));
+ }
+ 
+static int set_tcp_ccalgo(socket_t socket, const char *ccalgo)
+{
+#ifdef HAVE_TCP_CONGESTION
+	return setsockopt(socket,
+			  IPPROTO_TCP,
+			  TCP_CONGESTION,
+			  (void *)ccalgo,
+			  strlen(ccalgo));
+#else
+	errno = ENOTSUP;
+	return -1;
+#endif
+}
+
+ /**
+  * @internal
+  *
+@@ -256,6 +270,18 @@ socket_t ssh_connect_host_nonblocking(ssh_session session, const char *host,
+             }
+         }
+ 
+        if (session->opts.ccalgo) {
+	     rc = set_tcp_ccalgo(s, session->opts.ccalgo);
+	     if (rc < 0) {
+		 ssh_set_error(session, SSH_FATAL,
+			       "Failed to set TCP_CONGESTION on socket: %s",
+			       ssh_strerror(errno, err_msg, SSH_ERRNO_MSG_MAX));
+		 ssh_connect_socket_close(s);
+		 s = -1;
+		 continue;
+	     }
+	}
+
+         errno = 0;
+         rc = connect(s, itr->ai_addr, itr->ai_addrlen);
+         if (rc == -1 && (errno != 0) && (errno != EINPROGRESS)) {
+diff --git a/src/options.c b/src/options.c
+index 49aaefa2..9f7360c3 100644
+--- a/src/options.c
+++ b/src/options.c
+@@ -210,6 +210,7 @@ int ssh_options_copy(ssh_session src, ssh_session *dest)
+     new->opts.gss_delegate_creds    = src->opts.gss_delegate_creds;
+     new->opts.flags                 = src->opts.flags;
+     new->opts.nodelay               = src->opts.nodelay;
+    new->opts.ccalgo                = src->opts.ccalgo;
+     new->opts.config_processed      = src->opts.config_processed;
+     new->common.log_verbosity       = src->common.log_verbosity;
+     new->common.callbacks           = src->common.callbacks;
+@@ -450,6 +451,10 @@ int ssh_options_set_algo(ssh_session session,
+  *                Set it to disable Nagle's Algorithm (TCP_NODELAY) on the
+  *                session socket. (int, 0=false)
+  *
+ *              - SSH_OPTIONS_CCALGO
+ *                Set it to specify TCP congestion control algorithm on the
+ *                session socket (Linux only). (int, 0=false)
+ *
+  *              - SSH_OPTIONS_PROCESS_CONFIG
+  *                Set it to false to disable automatic processing of per-user
+  *                and system-wide OpenSSH configuration files. LibSSH
+@@ -1013,6 +1018,20 @@ int ssh_options_set(ssh_session session, enum ssh_options_e type,
+                 session->opts.nodelay = (*x & 0xff) > 0 ? 1 : 0;
+             }
+             break;
+	case SSH_OPTIONS_CCALGO:
+            v = value;
+            if (v == NULL || v[0] == '\0') {
+                ssh_set_error_invalid(session);
+                return -1;
+            } else {
+                SAFE_FREE(session->opts.ccalgo);
+                session->opts.ccalgo = strdup(v);
+                if (session->opts.ccalgo == NULL) {
+                    ssh_set_error_oom(session);
+                    return -1;
+                }
+            }
+            break;
+         case SSH_OPTIONS_PROCESS_CONFIG:
+             if (value == NULL) {
+                 ssh_set_error_invalid(session);
+diff --git a/src/session.c b/src/session.c
+index 6025c133..6b197526 100644
+--- a/src/session.c
+++ b/src/session.c
+@@ -108,6 +108,7 @@ ssh_session ssh_new(void)
+     session->opts.fd = -1;
+     session->opts.compressionlevel = 7;
+     session->opts.nodelay = 0;
+    session->opts.ccalgo = NULL;
+ 
+     session->opts.flags = SSH_OPT_FLAG_PASSWORD_AUTH |
+                           SSH_OPT_FLAG_PUBKEY_AUTH |
 diff --git a/src/sftp.c b/src/sftp.c
 index e01012a8..702623a0 100644
 --- a/src/sftp.c
--- a/setup.py
+++ b/setup.py
@@ -11,6 +11,9 @@ if sys.platform == "linux":
 elif sys.platform == "darwin":
    libmscp = "libmscp.dylib"

+data_dir = sys.prefix + "/lib"
+libmscp = "build/" + libmscp
+
 setup(
    name='mscp',
    version = version,
@@ -20,7 +23,7 @@ setup(
    url = "https://github.com/upa/mscp",
    packages = find_packages("mscp"),
    package_dir = {"": "mscp"},
-    data_files = [ ("", ["build/" + libmscp])],
+    data_files = [ (data_dir, [libmscp])],
    py_modules = [ "mscp" ],
    ext_modules = [
        Extension(
--- a/src/fileops.c
+++ b/src/fileops.c
@@ -0,0 +1,366 @@
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <dirent.h>
+
+#include <fileops.h>
+#include <ssh.h>
+#include <message.h>
+
+
+sftp_session __thread tls_sftp;
+/* tls_sftp is used *_wrapped() functions */
+
+void set_tls_sftp_session(sftp_session sftp)
+{
+	tls_sftp = sftp;
+}
+
+static void sftp_err_to_errno(sftp_session sftp)
+{
+	int sftperr = sftp_get_error(sftp);
+
+	switch (sftperr){
+	case SSH_FX_OK:
+	case SSH_FX_EOF:
+		errno = 0;
+		break;
+	case SSH_FX_NO_SUCH_FILE:
+	case SSH_FX_NO_SUCH_PATH:
+		errno = ENOENT;
+		break;
+	case SSH_FX_PERMISSION_DENIED:
+		errno = EACCES;
+		break;
+	case SSH_FX_FAILURE:
+		errno = EINVAL;
+	case SSH_FX_BAD_MESSAGE:
+		errno = EBADMSG;
+	case SSH_FX_NO_CONNECTION:
+		errno = ENOTCONN;
+		break;
+	case SSH_FX_CONNECTION_LOST:
+		errno = ENETRESET;
+		break;
+	case SSH_FX_OP_UNSUPPORTED:
+		errno = EOPNOTSUPP;
+		break;
+	case SSH_FX_INVALID_HANDLE:
+		errno = EBADF;
+		break;
+	case SSH_FX_FILE_ALREADY_EXISTS:
+		errno = EEXIST;
+		break;
+	case SSH_FX_WRITE_PROTECT:
+		errno = EPERM;
+		break;
+	case SSH_FX_NO_MEDIA:
+		errno = ENODEV;
+		break;
+	default:
+		mpr_warn(stderr, "unkown SSH_FX response %d", sftperr);
+	}
+}
+
+
+MDIR *mscp_opendir(const char *path, sftp_session sftp)
+{
+	MDIR *md;
+
+	if (!(md = malloc(sizeof(*md))))
+		return NULL;
+	memset(md, 0, sizeof(*md));
+
+	if (sftp) {
+		md->remote = sftp_opendir(sftp, path);
+		sftp_err_to_errno(sftp);
+		if (!md->remote) {
+			goto free_out;
+		}
+	} else {
+		md->local = opendir(path);
+		if (!md->local) {
+			goto free_out;
+		}
+	}
+
+	return md;
+
+free_out:
+	free(md);
+	return NULL;
+}
+
+MDIR *mscp_opendir_wrapped(const char *path)
+{
+	return mscp_opendir(path, tls_sftp);
+}
+
+void mscp_closedir(MDIR *md)
+{
+	int ret;
+	if (md->remote)
+		sftp_closedir(md->remote);
+	else
+		closedir(md->local);
+
+	free(md);
+}
+
+
+struct dirent __thread tls_dirent;
+/* tls_dirent contains dirent converted from sftp_attributes returned
+ * from sftp_readdir(). This trick is derived from openssh's
+ * fudge_readdir() */
+
+struct dirent *mscp_readdir(MDIR *md)
+{
+	sftp_attributes attr;
+	struct dirent *ret = NULL;
+	static int inum = 1;
+
+	if (md->remote) {
+		attr = sftp_readdir(md->remote->sftp, md->remote);
+		if (!attr) {
+			sftp_err_to_errno(md->remote->sftp);
+			return NULL;
+		}
+
+		memset(&tls_dirent, 0, sizeof(tls_dirent));
+		strncpy(tls_dirent.d_name, attr->name, sizeof(tls_dirent.d_name) - 1);
+		tls_dirent.d_ino = inum++;
+		if (!inum)
+			inum = 1;
+		ret = &tls_dirent;
+		sftp_attributes_free(attr);
+	} else
+		ret = readdir(md->local);
+
+	return ret;
+}
+
+int mscp_mkdir(const char *path, mode_t mode, sftp_session sftp)
+{
+	int ret;
+
+	if (sftp) {
+		ret = sftp_mkdir(sftp, path, mode);
+		sftp_err_to_errno(sftp);
+	} else
+		ret = mkdir(path, mode);
+
+	if (ret < 0 && errno == EEXIST) {
+		ret = 0;
+	}
+
+	return ret;
+}
+
+
+static void sftp_attr_to_stat(sftp_attributes attr, struct stat *st)
+{
+        memset(st, 0, sizeof(*st));
+	st->st_size	= attr->size;
+	st->st_uid	= attr->uid;
+	st->st_gid	= attr->gid;
+	st->st_mode	= attr->permissions;
+
+	switch (attr->type) {
+	case SSH_FILEXFER_TYPE_REGULAR:
+		st->st_mode |= S_IFREG;
+		break;
+	case SSH_FILEXFER_TYPE_DIRECTORY:
+		st->st_mode |= S_IFDIR;
+		break;
+	case SSH_FILEXFER_TYPE_SYMLINK:
+		st->st_mode |= S_IFLNK;
+		break;
+	case SSH_FILEXFER_TYPE_SPECIAL:
+		st->st_mode |= S_IFCHR; /* or block? */
+		break;
+	case SSH_FILEXFER_TYPE_UNKNOWN:
+		st->st_mode |= S_IFIFO; /* really? */
+		break;
+	default:
+		mpr_warn(stderr, "unkown SSH_FILEXFER_TYPE %d", attr->type);
+	}
+
+	/* ToDo: convert atime, ctime, and mtime */
+}
+
+
+int mscp_stat(const char *path, struct stat *st, sftp_session sftp)
+{
+	sftp_attributes attr;
+	int ret = 0;
+
+	if (sftp) {
+		attr = sftp_stat(sftp, path);
+		sftp_err_to_errno(sftp);
+		if (!attr)
+			return -1;
+
+		sftp_attr_to_stat(attr, st);
+		sftp_attributes_free(attr);
+		ret = 0;
+	} else
+		ret = stat(path, st);
+
+	return ret;
+}
+
+int mscp_stat_wrapped(const char *path, struct stat *st)
+{
+	return mscp_stat(path, st, tls_sftp);
+}
+
+int mscp_lstat(const char *path, struct stat *st, sftp_session sftp)
+{
+	sftp_attributes attr;
+	int ret = 0;
+
+	if (sftp) {
+		attr = sftp_lstat(sftp, path);
+		sftp_err_to_errno(sftp);
+		if (!attr)
+			return -1;
+
+		sftp_attr_to_stat(attr, st);
+		sftp_attributes_free(attr);
+		ret = 0;
+	} else
+		ret = lstat(path, st);
+
+	return ret;
+}
+
+int mscp_lstat_wrapped(const char *path, struct stat *st)
+{
+	return mscp_lstat(path, st, tls_sftp);
+}
+
+
+mf *mscp_open(const char *path, int flags, mode_t mode, sftp_session sftp)
+{
+	mf *f;
+
+	f = malloc(sizeof(*f));
+	if (!f)
+		return NULL;
+	memset(f, 0, sizeof(*f));
+
+	if (sftp) {
+		f->remote = sftp_open(sftp, path, flags, mode);
+		if (!f->remote) {
+			sftp_err_to_errno(sftp);
+			goto free_out;
+		}
+	} else {
+		f->local = open(path, flags, mode);
+		if (f->local < 0)
+			goto free_out;
+	}
+
+	return f;
+
+free_out:
+	free(f);
+	return NULL;
+}
+
+void mscp_close(mf *f)
+{
+	if (f->remote)
+		sftp_close(f->remote);
+	if (f->local > 0)
+		close(f->local);
+	free(f);
+}
+
+off_t mscp_lseek(mf *f, off_t off)
+{
+	off_t ret;
+
+	if (f->remote) {
+		ret = sftp_seek64(f->remote, off);
+		sftp_err_to_errno(f->remote->sftp);
+	} else
+		ret = lseek(f->local, off, SEEK_SET);
+
+	return ret;
+}
+
+int mscp_setstat(const char *path, mode_t mode, size_t size, sftp_session sftp)
+{
+	int ret;
+
+	if (sftp) {
+		struct sftp_attributes_struct attr;
+		memset(&attr, 0, sizeof(attr));
+		attr.permissions = mode;
+		attr.size = size;
+		attr.flags = (SSH_FILEXFER_ATTR_PERMISSIONS|SSH_FILEXFER_ATTR_SIZE);
+		ret = sftp_setstat(sftp, path, &attr);
+		sftp_err_to_errno(sftp);
+	} else {
+		if ((ret = chmod(path, mode)) < 0)
+			return ret;
+		if ((ret = truncate(path, size)) < 0)
+			return ret;
+	}
+
+	return ret;
+}
+
+int mscp_glob(const char *pattern, int flags, glob_t *pglob, sftp_session sftp)
+{
+	int ret;
+	if (sftp) {
+#ifndef GLOB_ALTDIRFUNC
+#define GLOB_NOALTDIRMAGIC INT_MAX
+		/* musl does not implement GLOB_ALTDIRFUNC */
+		pglob->gl_pathc = 1;
+		pglob->gl_pathv = malloc(sizeof(char *));
+		pglob->gl_pathv[0] = strdup(pattern);
+		pglob->gl_offs = GLOB_NOALTDIRMAGIC;
+		return 0;
+#else
+		flags |= GLOB_ALTDIRFUNC;
+		set_tls_sftp_session(sftp);
+#ifdef __APPLE__
+		pglob->gl_opendir = (void *(*)(const char *))mscp_opendir_wrapped;
+		pglob->gl_readdir = (struct dirent *(*)(void *))mscp_readdir;
+		pglob->gl_closedir = (void (*)(void *))mscp_closedir;
+		pglob->gl_lstat = mscp_lstat_wrapped;
+		pglob->gl_stat = mscp_stat_wrapped;
+#elif linux
+		pglob->gl_opendir = (void *(*)(const char *))mscp_opendir_wrapped;
+		pglob->gl_readdir = (void *(*)(void *))mscp_readdir;
+		pglob->gl_closedir = (void (*)(void *))mscp_closedir;
+		pglob->gl_lstat = (int (*)(const char *, void *))mscp_lstat_wrapped;
+		pglob->gl_stat = (int (*)(const char *, void *))mscp_stat_wrapped;
+#else
+#error unsupported platform
+#endif
+#endif
+	}
+
+	ret = glob(pattern, flags, NULL, pglob);
+
+	if (sftp)
+		set_tls_sftp_session(NULL);
+	return ret;
+}
+
+void mscp_globfree(glob_t *pglob)
+{
+#ifndef GLOB_ALTDIRFUNC
+	if (pglob->gl_offs == GLOB_NOALTDIRMAGIC) {
+		free(pglob->gl_pathv[0]);
+		free(pglob->gl_pathv);
+		return;
+	}
+#endif
+	globfree(pglob);
+}
--- a/src/fileops.h
+++ b/src/fileops.h
@@ -0,0 +1,57 @@
+
+#include <dirent.h>
+#include <sys/stat.h>
+#include <glob.h>
+
+#include <ssh.h>
+
+void set_tls_sftp_session(sftp_session sftp);
+/* sftp_session set by set_tls_sftp_session is sued in
+ mscp_open_wrapped(), mscp_stat_wrapped(), and
+ mscp_lstat_wrapped(). This _wrapped() functions exist for
+ sftp_glob() */
+
+/* directory operations */
+
+struct mdir_struct {
+	DIR *local;
+	sftp_dir remote;
+};
+typedef struct mdir_struct MDIR;
+
+
+MDIR *mscp_opendir(const char *path, sftp_session sftp);
+MDIR *mscp_opendir_wrapped(const char *path);
+void mscp_closedir(MDIR *md);
+struct dirent *mscp_readdir(MDIR *md);
+
+int mscp_mkdir(const char *path, mode_t mode, sftp_session sftp);
+
+/* stat operations */
+int mscp_stat(const char *path, struct stat *st, sftp_session sftp);
+int mscp_stat_wrapped(const char *path, struct stat *st);
+
+int mscp_lstat(const char *path, struct stat *st, sftp_session sftp);
+int mscp_lstat_wrapped(const char *path, struct stat *st);
+
+
+/* file operations */
+
+struct mf_struct {
+	sftp_file remote;
+	int local;
+};
+typedef struct mf_struct mf;
+
+mf *mscp_open(const char *path, int flags, mode_t mode, sftp_session sftp);
+void mscp_close(mf *f);
+off_t mscp_lseek(mf *f, off_t off);
+
+/* mscp_setstat() involves chmod and truncate. It executes both at
+ * once via a single SFTP command (sftp_setstat()).
+ */
+int mscp_setstat(const char *path, mode_t mode, size_t size, sftp_session sftp);
+
+/* remote glob */
+int mscp_glob(const char *pattern, int flags, glob_t *pglob, sftp_session sftp);
+void mscp_globfree(glob_t *pglob);
--- a/src/main.c
+++ b/src/main.c
@@ -11,23 +11,18 @@
 #include <pthread.h>

 #include <mscp.h>
+#include <mscp_version.h>
 #include <util.h>


-#ifndef _VERSION /* passed through cmake */
-#define VERSION "(unknown)"
-#else
-#define VERSION _VERSION
-#endif
-
-
 void usage(bool print_help) {
-	printf("mscp v" VERSION ": copy files over multiple ssh connections\n"
+	printf("mscp " MSCP_BUILD_VERSION ": copy files over multiple ssh connections\n"
 	       "\n"
 	       "Usage: mscp [vqDHdNh] [-n nr_conns] [-m coremask] [-u max_startups]\n"
 	       "            [-s min_chunk_sz] [-S max_chunk_sz] [-a nr_ahead] [-b buf_sz]\n"
-	       "            [-l login_name] [-p port] [-i identity_file]\n"
-	       "            [-c cipher_spec] [-M hmac_spec] [-C compress] source ... target\n"
+	       "            [-l login_name] [-p port] [-F ssh_config] [-i identity_file]\n"
+	       "            [-c cipher_spec] [-M hmac_spec] [-C compress] [-g congestion]\n"
+	       "            source ... target\n"
 	       "\n");

 	if (!print_help)
@@ -51,11 +46,13 @@ void usage(bool print_help) {
 	       "\n"
 	       "    -l LOGIN_NAME      login name\n"
 	       "    -p PORT            port number\n"
+	       "    -F CONFIG          path to user ssh config (default ~/.ssh/config)\n"
 	       "    -i IDENTITY        identity file for public key authentication\n"
 	       "    -c CIPHER          cipher spec\n"
 	       "    -M HMAC            hmac spec\n"
 	       "    -C COMPRESS        enable compression: "
 	       "yes, no, zlib, zlib@openssh.com\n"
+	       "    -g CONGESTION      specify TCP congestion control algorithm\n"
 	       "    -H                 disable hostkey check\n"
 	       "    -d                 increment ssh debug output level\n"
 	       "    -N                 enable Nagle's algorithm (default disabled)\n"
@@ -63,58 +60,113 @@ void usage(bool print_help) {
 	       "\n");
 }

-char *split_remote_and_path(const char *string, char **remote, char **path)
+char *strip_brackets(char *s)
 {
-	char *s, *p;
-
-	/* split user@host:path into user@host, and path.
-	 * return value is strdup()ed memory (for free()).
-	 */
-
-	if (!(s = strdup(string))) {
-		fprintf(stderr, "strdup: %s\n", strerror(errno));
-		return NULL;
+	if (s[0] == '[' && s[strlen(s) - 1] == ']') {
+		s[strlen(s) - 1] = '\0';
+		return s + 1;
 	}
-
-	if ((p = strchr(s, ':'))) {
-		if (p == s || ((p > s) && *(p - 1) == '\\')) {
-			/* first byte is colon, or escaped colon. no user@host here  */
-			goto no_remote;
-		} else {
-			/* we found ':', so this is remote:path notation. split it */
-			*p = '\0';
-			*remote = s;
-			*path = p + 1;
-			return s;
-		}
-	}
-
-no_remote:
-	*remote = NULL;
-	*path = s;
 	return s;
 }

+char *split_user_host_path(const char *s, char **userp, char **hostp, char **pathp)
+{
+	char *tmp, *cp, *user = NULL, *host = NULL, *path = NULL;
+	bool inbrackets = false;
+
+	if (!(tmp = strdup(s))) {
+		fprintf(stderr, "stdrup: %s\n", strerror(errno));
+		return NULL;
+	}
+
+	user = NULL;
+	host = NULL;
+	path = tmp;
+	for (cp = tmp; *cp; cp++) {
+		if (*cp == '@' && (cp > tmp) && *(cp - 1) != '\\' && user == NULL) {
+			/* cp is non-escaped '@', so this '@' is the
+			 * delimitater between username and host. */
+			*cp = '\0';
+			user = tmp;
+			host = cp + 1;
+		}
+		if (*cp == '[')
+			inbrackets = true;
+		if (*cp == ']')
+			inbrackets = false;
+		if (*cp == ':' && (cp > tmp) && *(cp - 1) != '\\') {
+			if (!inbrackets) {
+				/* cp is non-escaped ':' and not in
+				 * brackets for IPv6 address
+				 * notation. So, this ':' is the
+				 * delimitater between host and
+				 * path. */
+				*cp = '\0';
+				host = host == NULL ? tmp : host;
+				path = cp + 1;
+				break;
+			}
+		}
+	}
+	*userp = user;
+	*hostp = host ? strip_brackets(host) : NULL;
+	*pathp = path;
+	return tmp;
+}
+
 struct target {
-	char *remote;
+	char *copy;
+	char *user;
+	char *host;
 	char *path;
 };

+int compare_remote(struct target *a, struct target *b)
+{
+	/* return 0 if a and b have the identical user@host, otherwise 1 */
+	int alen, blen;
+
+	if (a->user) {
+		if (!b->user)
+			return 1;
+		alen = strlen(a->user);
+		blen = strlen(b->user);
+		if (alen != blen)
+			return 1;
+		if (strncmp(a->user, b->user, alen) != 0)
+			return 1;
+	} else if (b->user)
+		return 1;
+
+	if (a->host) {
+		if (!b->host)
+			return 1;
+		alen = strlen(a->host);
+		blen = strlen(b->host);
+		if (alen != blen)
+			return 1;
+		if (strncmp(a->host, b->host, alen) != 0)
+			return 1;
+	} else if (b->host)
+		return 1;
+
+	return 0;
+}
+
 struct target *validate_targets(char **arg, int len)
 {
 	/* arg is array of source ... destination.
 	 * There are two cases:
 	 *
-	 * 1. remote:path remote:path ... path, remote to local copy
-	 * 2. path path ... remote:path, local to remote copy.
+	 * 1. user@host:path host:path ... path, remote to local copy
+	 * 2. path path ... host:path, local to remote copy.
 	 *
-	 * This function split (remote:)path args into struct target,
+	 * This function split user@remote:path args into struct target,
 	 * and validate all remotes are identical (mscp does not support
 	 * remote to remote copy).
 	 */

-	struct target *t;
-	char *r;
+	struct target *t, *t0;
 	int n;

 	if ((t = calloc(len, sizeof(struct target))) == NULL) {
@@ -125,33 +177,28 @@ struct target *validate_targets(char **arg, int len)

 	/* split remote:path into remote and path */
 	for (n = 0; n < len; n++) {
-		if (split_remote_and_path(arg[n], &t[n].remote, &t[n].path) == NULL)
+		t[n].copy = split_user_host_path(arg[n], &t[n].user,
+					      &t[n].host, &t[n].path);
+		if (!t[n].copy)
 			goto free_target_out;
 	}

-	/* check all remote are identical. t[len - 1] is destination,
+	/* check all user@host are identical. t[len - 1] is destination,
 	 * so we need to check t[0] to t[len - 2] having the identical
-	 * remote */
-	r = t[0].remote;
+	 * remote notation */
+	t0 = &t[0];
 	for (n = 1; n < len - 1; n++) {
-		if (!r && t[n].remote) {
+		if (compare_remote(t0, &t[n]) != 0)
 			goto invalid_remotes;
-		}
-		if (r) {
-			if (!t[n].remote ||
-			    strlen(r) != strlen(t[n].remote) ||
-			    strcmp(r, t[n].remote) != 0)
-				goto invalid_remotes;
-		}
 	}

 	/* check inconsistent remote position in args */
-	if (t[0].remote == NULL && t[len - 1].remote == NULL) {
+	if (t[0].host == NULL && t[len - 1].host == NULL) {
 		fprintf(stderr, "no remote host given\n");
 		goto free_split_out;
 	}

-	if (t[0].remote != NULL && t[len - 1].remote != NULL) {
+	if (t[0].host != NULL && t[len - 1].host != NULL) {
 		fprintf(stderr, "no local path given\n");
 		goto free_split_out;
 	}
@@ -159,11 +206,11 @@ struct target *validate_targets(char **arg, int len)
 	return t;

 invalid_remotes:
-	fprintf(stderr, "specified remote host invalid\n");
+	fprintf(stderr, "invalid remote host notation\n");

 free_split_out:
 	for (n = 0; n < len; n++)
-		t[n].remote ? free(t[n].remote) : free(t[n].path);
+		if (t[n].copy) free(t[n].copy);

 free_target_out:
 	free(t);
@@ -176,8 +223,6 @@ pthread_t tid_stat = 0;

 void sigint_handler(int sig)
 {
-	if (tid_stat)
-		pthread_cancel(tid_stat);
 	mscp_stop(m);
 }

@@ -207,7 +252,8 @@ int main(int argc, char **argv)
 	memset(&o, 0, sizeof(o));
 	o.severity = MSCP_SEVERITY_WARN;

-	while ((ch = getopt(argc, argv, "n:m:u:s:S:a:b:vqDrl:p:i:c:M:C:HdNh")) != -1) {
+	while ((ch = getopt(argc, argv,
+			    "n:m:u:s:S:a:b:vqDrl:p:i:F:c:M:C:g:HdNh")) != -1) {
 		switch (ch) {
 		case 'n':
 			o.nr_threads = atoi(optarg);
@@ -261,6 +307,9 @@ int main(int argc, char **argv)
 			}
 			strncpy(s.port, optarg, MSCP_SSH_MAX_PORT_STR);
 			break;
+		case 'F':
+			strncpy(s.config, optarg, PATH_MAX - 1);
+			break;
 		case 'i':
 			if (strlen(optarg) > MSCP_SSH_MAX_IDENTITY_PATH - 1) {
 				fprintf(stderr, "long identity path: %s\n", optarg);
@@ -289,6 +338,13 @@ int main(int argc, char **argv)
 			}
 			strncpy(s.compress, optarg, MSCP_SSH_MAX_COMP_STR);
 			break;
+		case 'g':
+			if (strlen(optarg) > MSCP_SSH_MAX_CCALGO_STR - 1) {
+				fprintf(stderr, "long ccalgo string: %s\n", optarg);
+				return -1;
+			}
+			strncpy(s.ccalgo, optarg, MSCP_SSH_MAX_CCALGO_STR);
+			break;
 		case 'H':
 			s.no_hostkey_check = true;
 			break;
@@ -317,14 +373,19 @@ int main(int argc, char **argv)
 	if ((t = validate_targets(argv + optind, i)) == NULL)
 		return -1;

-	if (t[0].remote) {
+	if (t[0].host) {
 		/* copy remote to local */
 		direction = MSCP_DIRECTION_R2L;
-		remote = t[0].remote;
+		remote = t[0].host;
+		if (t[0].user != NULL && s.login_name[0] == '\0')
+			strncpy(s.login_name, t[0].user, MSCP_SSH_MAX_LOGIN_NAME - 1);
 	} else {
 		/* copy local to remote */
 		direction = MSCP_DIRECTION_L2R;
-		remote = t[i - 1].remote;
+		remote = t[i - 1].host;
+		if (t[i - 1].user != NULL && s.login_name[0] == '\0')
+			strncpy(s.login_name, t[i - 1].user,
+				MSCP_SSH_MAX_LOGIN_NAME - 1);
 	}

 	if (!dryrun) {
@@ -421,23 +482,43 @@ double calculate_bps(size_t diff, struct timeval *b, struct timeval *a)
        return (double)diff / calculate_timedelta(b, a);
 }

+
 char *calculate_eta(size_t remain, size_t diff, struct timeval *b, struct timeval *a,
 		    bool final)
 {
        static char buf[16];
-        double elapsed = calculate_timedelta(b, a);
-        double eta;

-        if (diff == 0)
+#define bps_window_size 16
+	static double bps_window[bps_window_size];
+	static size_t sum, idx, count;
+        double elapsed = calculate_timedelta(b, a);
+	double bps = diff / elapsed;
+	double avg, eta;
+
+	/* early return when diff == 0 (stalled) or final output */
+        if (diff == 0) {
                snprintf(buf, sizeof(buf), "--:-- ETA");
-	else if (final) {
+		return buf;
+	}
+	if (final) {
 		snprintf(buf, sizeof(buf), "%02d:%02d    ",
 			 (int)(floor(elapsed / 60)), (int)round(elapsed) % 60);
-	} else {
-                eta = remain / (diff / elapsed);
-                snprintf(buf, sizeof(buf), "%02d:%02d ETA",
-                         (int)floor(eta / 60), (int)round(eta) % 60);
-        }
+		return buf;
+	}
+
+	/* drop the old bps value and add the recent one */
+	sum -= bps_window[idx];
+	bps_window[idx] = bps;
+	sum += bps_window[idx];
+	idx = (idx + 1) % bps_window_size;
+	count++;
+
+	/* calcuate ETA from avg of recent bps values */
+	avg = sum / min(count, bps_window_size);
+	eta = remain / avg;
+	snprintf(buf, sizeof(buf), "%02d:%02d ETA",
+		 (int)floor(eta / 60), (int)round(eta) % 60);
+
        return buf;
 }

@@ -484,7 +565,7 @@ void print_progress(struct timeval *b, struct timeval *a,
        char *byte_units[] = { "B ", "KB", "MB", "GB", "TB", "PB" };
        char suffix[128];
        int bps_u, byte_tu, byte_du;
-        size_t total_round, done_round;
+        double total_round, done_round;
        int percent;
        double bps;

@@ -507,11 +588,11 @@ void print_progress(struct timeval *b, struct timeval *a,
        percent = floor(((double)(done) / (double)total) * 100);

        done_round = done;
-        for (byte_du = 0; done_round > 1000 && byte_du < array_size(byte_units) - 1;
+        for (byte_du = 0; done_round > 1024 && byte_du < array_size(byte_units) - 1;
             byte_du++)
                done_round /= 1024;

-        snprintf(suffix, sizeof(suffix), "%4lu%s/%lu%s %6.1f%s  %s",
+        snprintf(suffix, sizeof(suffix), "%4.1lf%s/%.1lf%s %6.1f%s  %s",
                 done_round, byte_units[byte_du], total_round, byte_units[byte_tu],
                 bps, bps_units[bps_u],
 		 calculate_eta(total - done, done - last, b, a, final));
@@ -528,17 +609,42 @@ struct xfer_stat {
 };
 struct xfer_stat x;

-void print_stat_thread_cleanup(void *arg)
+void print_stat(bool final)
 {
+	struct pollfd pfd = { .fd = msg_fd, .events = POLLIN };
 	struct mscp_stats s;
+	char buf[8192];
+	int timeout;
+
+	if (poll(&pfd, 1, !final ? 100 : 0) < 0) {
+		fprintf(stderr, "poll: %s\n", strerror(errno));
+		return;
+	}
+
+	if (pfd.revents & POLLIN) {
+		memset(buf, 0, sizeof(buf));
+		if (read(msg_fd, buf, sizeof(buf)) < 0) {
+			fprintf(stderr, "read: %s\n", strerror(errno));
+			return;
+		}
+		print_cli("\r\033[K" "%s", buf);
+	}

 	gettimeofday(&x.after, NULL);
-	mscp_get_stats(m, &s);
-	x.total = s.total;
-	x.done = s.done;
+	if (calculate_timedelta(&x.before, &x.after) > 1 || final) {
+		mscp_get_stats(m, &s);
+		x.total = s.total;
+		x.done = s.done;
+		print_progress(!final ? &x.before : &x.start, &x.after,
+			       x.total, !final ? x.last : 0, x.done, final);
+		x.before = x.after;
+		x.last = x.done;
+	}
+}

-	/* print progress from the beginning */
-	print_progress(&x.start, &x.after, x.total, 0, x.done, true);
+void print_stat_thread_cleanup(void *arg)
+{
+	print_stat(true);
 	print_cli("\n"); /* final output */
 }

@@ -557,31 +663,7 @@ void *print_stat_thread(void *arg)
        pthread_cleanup_push(print_stat_thread_cleanup, NULL);

 	while (true) {
-		if (poll(&pfd, 1, 100) < 0) {
-			fprintf(stderr, "poll: %s\n", strerror(errno));
-			return NULL;
-		}
-
-		if (pfd.revents & POLLIN) {
-			memset(buf, 0, sizeof(buf));
-			if (read(msg_fd, buf, sizeof(buf)) < 0) {
-				fprintf(stderr, "read: %s\n", strerror(errno));
-				return NULL;
-			}
-			print_cli("\r\033[K" "%s", buf);
-		}
-
-		gettimeofday(&x.after, NULL);
-		if (calculate_timedelta(&x.before, &x.after) > 1) {
-			mscp_get_stats(m, &s);
-			x.total = s.total;
-			x.done = s.done;
-
-			print_progress(&x.before, &x.after, x.total, x.last, x.done,
-				       false);
-			x.before = x.after;
-			x.last = x.done;
-		}
+		print_stat(false);
 	}

 	pthread_cleanup_pop(1);
--- a/src/message.h
+++ b/src/message.h
@@ -2,6 +2,7 @@
 #define _MESSAGE_H_

 #include <libgen.h>
+#include <stdio.h>

 #include <mscp.h>

--- a/src/mscp.c
+++ b/src/mscp.c
@@ -9,6 +9,7 @@
 #include <util.h>       
 #include <ssh.h>                
 #include <path.h>
+#include <fileops.h>
 #include <atomic.h>             
 #include <platform.h>
 #include <message.h>
@@ -376,7 +377,9 @@ void *mscp_scan_thread(void *arg)
 	struct list_head tmp;
 	struct path *p;
 	struct src *s;
-	mstat ss, ds;
+	struct stat ss, ds;
+	glob_t pglob;
+	int n;
 	
 	m->ret_scan = 0;

@@ -403,9 +406,8 @@ void *mscp_scan_thread(void *arg)
 		a.dst_path_should_dir = true;

 	if (mscp_stat(m->dst_path, &ds, dst_sftp) == 0) {
-		if (mstat_is_dir(ds))
+		if (S_ISDIR(ds.st_mode))
 			a.dst_path_is_dir = true;
-		mscp_stat_free(ds);
 	}

 	a.cp = &m->cp;
@@ -418,23 +420,33 @@ void *mscp_scan_thread(void *arg)

 	/* walk a src_path recusively, and resolve path->dst_path for each src */
 	list_for_each_entry(s, &m->src_list, list) {
-		if (mscp_stat(s->path, &ss, src_sftp) < 0) {
-			mscp_set_error("stat: %s", mscp_strerror(src_sftp));
-			mscp_stat_free(ss);
+		memset(&pglob, 0, sizeof(pglob));
+		if (mscp_glob(s->path, GLOB_NOCHECK, &pglob, src_sftp) < 0) {
+			mscp_set_error("mscp_glob: %s", strerrno());
 			goto err_out;
 		}

-		/* set path specific args */
-		a.src_path = s->path;
-		a.dst_path = m->dst_path;
-		a.src_path_is_dir = mstat_is_dir(ss);
-		mscp_stat_free(ss);
+		for (n = 0; n < pglob.gl_pathc; n++) {
+			if (mscp_stat(pglob.gl_pathv[n], &ss, src_sftp) < 0) {
+				mscp_set_error("stat: %s %s", s->path, strerrno());
+				goto err_out;
+			}

-		INIT_LIST_HEAD(&tmp);
-		if (walk_src_path(src_sftp, s->path, &tmp, &a) < 0)
-			goto err_out;
+			if (!a.dst_path_should_dir && pglob.gl_pathc > 1)
+				a.dst_path_should_dir = true; /* we have over 1 src */

-		list_splice_tail(&tmp, m->path_list.prev);
+			/* set path specific args */
+			a.src_path = pglob.gl_pathv[n];
+			a.dst_path = m->dst_path;
+			a.src_path_is_dir = S_ISDIR(ss.st_mode);
+
+			INIT_LIST_HEAD(&tmp);
+			if (walk_src_path(src_sftp, pglob.gl_pathv[n], &tmp, &a) < 0)
+				goto err_out;
+
+			list_splice_tail(&tmp, m->path_list.prev);
+		}
+		mscp_globfree(&pglob);
 	}

 	mpr_info(m->msg_fp, "walk source path(s) done\n");
@@ -542,6 +554,8 @@ int mscp_start(struct mscp *m)
 int mscp_join(struct mscp *m)
 {
 	struct mscp_thread *t;
+	struct path *p;
+	size_t done = 0, nr_copied = 0, nr_tobe_copied = 0;
 	int n, ret = 0;

 	/* waiting for scan thread joins... */
@@ -551,6 +565,7 @@ int mscp_join(struct mscp *m)
 	RWLOCK_READ_ACQUIRE(&m->thread_rwlock);
 	list_for_each_entry(t, &m->thread_list, list) {
 		pthread_join(t->tid, NULL);
+		done += t->done;
 		if (t->ret < 0)
 			ret = t->ret;
 		if (t->sftp) {
@@ -565,6 +580,17 @@ int mscp_join(struct mscp *m)
 		m->first = NULL;
 	}

+	/* count up number of transferred files */
+	list_for_each_entry(p, &m->path_list, list) {
+		nr_tobe_copied++;
+		if (p->state == FILE_STATE_DONE) {
+			nr_copied++;
+		}
+	}
+
+	mpr_notice(m->msg_fp, "%lu/%lu bytes copied for %lu/%lu files\n",
+		   done, m->total_bytes, nr_copied, nr_tobe_copied);
+
 	return ret;
 }

@@ -596,7 +622,7 @@ void *mscp_copy_thread(void *arg)
 		goto err_out;
 	}

-	mpr_notice(m->msg_fp, "connecting to %s for a copy thread[%d]...\n",
+	mpr_notice(m->msg_fp, "connecting to %s for copy thread:%d...\n",
 		   m->remote, t->id);
 	t->sftp = ssh_init_sftp_session(m->remote, m->ssh_opts);

@@ -607,7 +633,7 @@ void *mscp_copy_thread(void *arg)
 	}

 	if (!t->sftp) {
-		mpr_err(m->msg_fp, "copy thread[%d]: %s\n", t->id, mscp_get_error());
+		mpr_err(m->msg_fp, "copy thread:%d: %s\n", t->id, mscp_get_error());
 		goto err_out;
 	}

@@ -647,8 +673,8 @@ void *mscp_copy_thread(void *arg)
        pthread_cleanup_pop(1);

        if (t->ret < 0)
-                mscp_set_error("copy failed: chunk %s 0x%010lx-0x%010lx",
-			       c->p->path, c->off, c->off + c->len);
+		mpr_err(m->msg_fp, "copy failed: chunk %s 0x%010lx-0x%010lx\n",
+			c->p->path, c->off, c->off + c->len);

        return NULL;

--- a/src/path.c
+++ b/src/path.c
@@ -7,6 +7,7 @@

 #include <ssh.h>
 #include <util.h>
+#include <fileops.h>
 #include <list.h>
 #include <atomic.h>
 #include <path.h>
@@ -190,7 +191,7 @@ static int resolve_chunk(struct path *p, struct path_resolve_args *a)
        return 0;
 }

-static int append_path(sftp_session sftp, const char *path, mstat s,
+static int append_path(sftp_session sftp, const char *path, struct stat st,
 		       struct list_head *path_list, struct path_resolve_args *a)
 {
 	struct path *p;
@@ -203,8 +204,8 @@ static int append_path(sftp_session sftp, const char *path, mstat s,
 	memset(p, 0, sizeof(*p));
 	INIT_LIST_HEAD(&p->list);
 	strncpy(p->path, path, PATH_MAX - 1);
-	p->size = mstat_size(s);
-	p->mode = mstat_mode(s);
+	p->size = st.st_size;
+	p->mode = st.st_mode;
 	p->state = FILE_STATE_INIT;
 	lock_init(&p->lock);

@@ -239,48 +240,36 @@ static int walk_path_recursive(sftp_session sftp, const char *path,
 			       struct list_head *path_list, struct path_resolve_args *a)
 {
 	char next_path[PATH_MAX];
-	mdirent *e;
-	mdir *d;
-	mstat s;
+	struct dirent *e;
+	struct stat st;
+	MDIR *d;
 	int ret;

-	if (mscp_stat(path, &s, sftp) < 0)
+	if (mscp_stat(path, &st, sftp) < 0)
 		return -1;

-	if (mstat_is_regular(s)) {
+	if (S_ISREG(st.st_mode)) {
 		/* this path is regular file. it is to be copied */
-		ret = append_path(sftp, path, s, path_list, a);
-		mscp_stat_free(s);
-		return ret;
+		return append_path(sftp, path, st, path_list, a);
 	}

-	if (!mstat_is_dir(s)) {
-		/* not regular file and not directory, skip it. */
-		mscp_stat_free(s);
-		return 0; 
-	}
-
-	mscp_stat_free(s);
-
+	if (!S_ISDIR(st.st_mode))
+		return 0; /* not a regular file and not a directory, skip it. */

 	/* ok, this path is directory. walk it. */
 	if (!(d = mscp_opendir(path, sftp)))
 		return -1;
 	
-	for (e = mscp_readdir(d); !mdirent_is_null(e); e = mscp_readdir(d)) {
-		if (check_path_should_skip(mdirent_name(e))) {
-			mscp_dirent_free(e);
+	for (e = mscp_readdir(d); e; e = mscp_readdir(d)) {
+		if (check_path_should_skip(e->d_name))
 			continue;
-		}
 		
-		if (strlen(path) + 1 + strlen(mdirent_name(e)) > PATH_MAX) {
-			mscp_set_error("too long path: %s/%s", path, mdirent_name(e));
-			mscp_dirent_free(e);
+		if (strlen(path) + 1 + strlen(e->d_name) > PATH_MAX) {
+			mscp_set_error("too long path: %s/%s", path, e->d_name);
 			return -1;
 		}
-		snprintf(next_path, sizeof(next_path), "%s/%s", path, mdirent_name(e));
+		snprintf(next_path, sizeof(next_path), "%s/%s", path, e->d_name);
 		ret = walk_path_recursive(sftp, next_path, path_list, a);
-		mscp_dirent_free(e);
 		if (ret < 0)
 			return ret;
 	}
@@ -314,10 +303,11 @@ static int touch_dst_path(struct path *p, sftp_session sftp)
 {
        /* XXX: should reflect the permission of the original directory? */
        mode_t mode =  S_IRWXU | S_IRWXG | S_IRWXO;
+	struct stat st;
        char path[PATH_MAX];
        char *needle;
        int ret;
-	mfh h;
+	mf *f;

        strncpy(path, p->dst_path, sizeof(path));

@@ -326,22 +316,17 @@ static int touch_dst_path(struct path *p, sftp_session sftp)
        for (needle = strchr(path + 1, '/'); needle; needle = strchr(needle + 1, '/')) {
                *needle = '\0';

-		mstat s;
-		if (mscp_stat(path, &s, sftp) == 0) {
-			if (mstat_is_dir(s)) {
-				mscp_stat_free(s);
+		if (mscp_stat(path, &st, sftp) == 0) {
+			if (S_ISDIR(st.st_mode))
 				goto next; /* directory exists. go deeper */
-			} else {
-				mscp_stat_free(s);
+			else
 				return -1; /* path exists, but not directory. */
-			}
 		}

-		if (mscp_stat_check_err_noent(sftp) == 0) {
+		if (errno == ENOENT) {
 			/* no file on the path. create directory. */
 			if (mscp_mkdir(path, mode, sftp) < 0) {
-				mscp_set_error("mkdir %s: %s", path,
-					       mscp_strerror(sftp));
+				mscp_set_error("mscp_mkdir %s: %s", path, strerrno());
 				return -1;
 			}
 		}
@@ -349,12 +334,15 @@ static int touch_dst_path(struct path *p, sftp_session sftp)
                *needle = '/';
        }

-        /* open file with O_TRUNC to set file size 0 */
-	h = mscp_open(p->dst_path, O_WRONLY|O_CREAT|O_TRUNC, S_IRUSR|S_IWUSR, 0, sftp);
-	if (mscp_open_is_failed(h))
+	/* Do not set O_TRUNC here. Instead, do mscp_setstat() at the
+	 * end. see https://bugzilla.mindrot.org/show_bug.cgi?id=3431 */
+	f = mscp_open(p->dst_path, O_WRONLY|O_CREAT, S_IRUSR|S_IWUSR, sftp);
+	if (!f) {
+		mscp_set_error("mscp_open %s: %s\n", p->dst_path, strerrno());
 		return -1;
+	}

-	mscp_close(h);
+	mscp_close(f);

        return 0;
 }
@@ -518,16 +506,16 @@ static int copy_chunk_r2l(struct chunk *c, sftp_file sf, int fd,
        return 0;
 }

-static int _copy_chunk(struct chunk *c, mfh s, mfh d,
+static int _copy_chunk(struct chunk *c, mf *s, mf *d,
 		       int nr_ahead, int buf_sz, size_t *counter)
 {
-	if (s.fd > 0 && d.sf) /* local to remote copy */
-		return copy_chunk_l2r(c, s.fd, d.sf, nr_ahead, buf_sz, counter);
-	else if (s.sf && d.fd > 0)  /* remote to local copy */
-		return copy_chunk_r2l(c, s.sf, d.fd, nr_ahead, buf_sz, counter);
+	if (s->local && d->remote) /* local to remote copy */
+		return copy_chunk_l2r(c, s->local, d->remote, nr_ahead, buf_sz, counter);
+	else if (s->remote && d->local)  /* remote to local copy */
+		return copy_chunk_r2l(c, s->remote, d->local, nr_ahead, buf_sz, counter);

-	assert(true); /* not reached */
-	return -1;
+	assert(false);
+	return -1; /* not reached */
 }

 int copy_chunk(FILE *msg_fp, struct chunk *c,
@@ -536,7 +524,7 @@ int copy_chunk(FILE *msg_fp, struct chunk *c,
 {
 	mode_t mode;
 	int flags;
-	mfh s, d;
+	mf *s, *d;
 	int ret;

 	assert((src_sftp && !dst_sftp) || (!src_sftp && dst_sftp));
@@ -547,21 +535,33 @@ int copy_chunk(FILE *msg_fp, struct chunk *c,
 	/* open src */
        flags = O_RDONLY;
        mode = S_IRUSR;
-	s = mscp_open(c->p->path, flags, mode, c->off, src_sftp);
-	if (mscp_open_is_failed(s)) {
-		mscp_close(d);
+	s = mscp_open(c->p->path, flags, mode, src_sftp);
+	if (!s) {
+		mscp_set_error("mscp_open: %s: %s", c->p->path, strerrno());
+		return -1;
+	}
+	if (mscp_lseek(s, c->off) < 0) {
+		mscp_set_error("mscp_lseek: %s: %s", c->p->path, strerrno());
 		return -1;
 	}

 	/* open dst */
        flags = O_WRONLY;
        mode = S_IRUSR|S_IWUSR;
-	d = mscp_open(c->p->dst_path, flags, mode, c->off, dst_sftp);
-	if (mscp_open_is_failed(d))
+	d = mscp_open(c->p->dst_path, flags, mode, dst_sftp);
+	if (!d) {
+		mscp_close(s);
+		mscp_set_error("mscp_open: %s: %s", c->p->dst_path, strerrno());
 		return -1;
+	}
+	if (mscp_lseek(d, c->off) < 0) {
+		mscp_set_error("mscp_lseek: %s: %s", c->p->dst_path, strerrno());
+		return -1;
+	}

 	mpr_debug(msg_fp, "copy chunk start: %s 0x%lx-0x%lx\n",
 		  c->p->path, c->off, c->off + c->len);
+
 	ret = _copy_chunk(c, s, d, nr_ahead, buf_sz, counter);

 	mpr_debug(msg_fp, "copy chunk done: %s 0x%lx-0x%lx\n",
@@ -575,7 +575,9 @@ int copy_chunk(FILE *msg_fp, struct chunk *c,

 	if (refcnt_dec(&c->p->refcnt) == 0) {
 		c->p->state = FILE_STATE_DONE;
-		mscp_chmod(c->p->dst_path, c->p->mode, dst_sftp);
+		if (mscp_setstat(c->p->dst_path, c->p->mode, c->p->size, dst_sftp) < 0)
+			mpr_err(msg_fp, "failed to chmod and truncate %s: %s\n",
+				c->p->path, strerrno());
 		mpr_info(msg_fp, "copy done: %s\n", c->p->path);
 	}

--- a/src/path.h
+++ b/src/path.h
@@ -98,253 +98,4 @@ int copy_chunk(FILE *msg_fp, struct chunk *c,
 /* just print contents. just for debugging */
 void path_dump(struct list_head *path_list);

-
-
-/* wrap DIR/dirent and sftp_dir/sftp_attribute. not thread safe */
-struct mscp_dir {
-        DIR *l;
-        sftp_dir r;
-        sftp_session sftp;
-};
-typedef struct mscp_dir mdir;
-
-struct mscp_dirent {
-        struct dirent *l;
-        sftp_attributes r;
-};
-typedef struct mscp_dirent mdirent;
-
-#define mdirent_name(e) ((e->l) ? e->l->d_name : e->r->name)
-#define mdirent_is_dir(e) ((e->l) ?                                     \
-                           (e->l->d_type == DT_DIR) :                   \
-                           (e->r->type == SSH_FILEXFER_TYPE_DIRECTORY))
-#define mdirent_is_null(e) (e->l == NULL && e->r == NULL)
-
-static mdir *mscp_opendir(const char *path, sftp_session sftp)
-{
-        mdir *d;
-
-        if (!(d = malloc(sizeof(*d))))
-                return NULL;
-        memset(d, 0, sizeof(*d));
-
-        d->sftp = sftp;
-
-        if (sftp) {
-                d->r = sftp_opendir(sftp, path);
-                if (!d->r) {
-                        mscp_set_error("sftp_opendir '%s': %s",
-				       path, sftp_get_ssh_error(sftp));
-                        free(d);
-                        return NULL;
-                }
-        } else {
-                d->l = opendir(path);
-                if (!d->l) {
-                        mscp_set_error("opendir '%s': %s", path, strerrno());
-                        free(d);
-                        return NULL;
-                }
-        }
-        return d;
-}
-
-static int mscp_closedir(mdir *d)
-{
-        int ret;
-        if (d->r)
-                ret = sftp_closedir(d->r);
-        else
-                ret = closedir(d->l);
-        free(d);
-        return ret;
-}
-
-static mdirent *mscp_readdir(mdir *d)
-{
-        static mdirent e;
-
-        memset(&e, 0, sizeof(e));
-        if (d->r)
-                e.r = sftp_readdir(d->sftp, d->r);
-        else
-                e.l = readdir(d->l);
-        return &e;
-}
-
-static void mscp_dirent_free(mdirent *e)
-{
-	if (e->r) {
-		sftp_attributes_free(e->r);
-		e->r = NULL;
-	}
-}
-
-/* wrap retriving error */
-static const char *mscp_strerror(sftp_session sftp)
-{
-	if (sftp)
-		return sftp_get_ssh_error(sftp);
-	return strerrno();
-}
-
-/* warp stat/sftp_stat */
-struct mscp_stat {
-        struct stat l;
-        sftp_attributes r;
-};
-typedef struct mscp_stat mstat;
-
-static int mscp_stat(const char *path, mstat *s, sftp_session sftp)
-{
-        memset(s, 0, sizeof(*s));
-
-        if (sftp) {
-                s->r = sftp_stat(sftp, path);
-                if (!s->r) {
-			mscp_set_error("sftp_stat: %s %s",
-				       sftp_get_ssh_error(sftp), path);
-                        return -1;
-		}
-        } else {
-                if (stat(path, &s->l) < 0) {
-			mscp_set_error("stat: %s %s", strerrno(), path);
-                        return -1;
-		}
-        }
-
-        return 0;
-}
-
-static int mscp_stat_check_err_noent(sftp_session sftp)
-{
-	if (sftp) {
-		if (sftp_get_error(sftp) == SSH_FX_NO_SUCH_PATH ||
-		    sftp_get_error(sftp) == SSH_FX_NO_SUCH_FILE)
-			return 0;
-	} else {
-		if (errno == ENOENT)
-			return 0;
-	}
-	return -1;
-}
-
-static void mscp_stat_free(mstat s) {
-        if (s.r)
-                sftp_attributes_free(s.r);
-}
-
-#define mstat_size(s) ((s.r) ? s.r->size : s.l.st_size)
-#define mstat_mode(s) ((s.r) ?                                  \
-                       s.r->permissions :                       \
-                       s.l.st_mode & (S_IRWXU|S_IRWXG|S_IRWXO))
-#define mstat_is_regular(s) ((s.r) ?                                    \
-                             (s.r->type == SSH_FILEXFER_TYPE_REGULAR) : \
-                             S_ISREG(s.l.st_mode))
-#define mstat_is_dir(s) ((s.r) ?                        \
-                         (s.r->type == SSH_FILEXFER_TYPE_DIRECTORY) :   \
-                         S_ISDIR(s.l.st_mode))
-
-/* wrap mkdir */
-static int mscp_mkdir(const char *path, mode_t mode, sftp_session sftp)
-{
-	int ret;
-
-	if (sftp) {
-		ret = sftp_mkdir(sftp, path, mode);
-		if (ret < 0 &&
-		    sftp_get_error(sftp) != SSH_FX_FILE_ALREADY_EXISTS) {
-			mscp_set_error("sftp_mkdir '%s': %s",
-				       path, sftp_get_ssh_error(sftp));
-			return -1;
-		}
-	} else {
-		if (mkdir(path, mode) == -1 && errno != EEXIST) {
-			mscp_set_error("mkdir '%s': %s", path, strerrno());
-			return -1;
-		}
-	}
-
-	return 0;
-}
-
-/* wrap open/sftp_open */
-struct mscp_file_handle {
-	int fd;
-	sftp_file sf;
-};
-typedef struct mscp_file_handle mfh;
-
-static mfh mscp_open(const char *path, int flags, mode_t mode, size_t off,
-		      sftp_session sftp)
-{
-	mfh h;
-
-	h.fd = -1;
-	h.sf = NULL;
-
-	if (sftp) {
-		h.sf = sftp_open(sftp, path, flags, mode);
-		if (!h.sf) {
-			mscp_set_error("sftp_open '%s': %s",
-				       path, sftp_get_ssh_error(sftp));
-			return h;
-		}
-
-		if (sftp_seek64(h.sf, off) < 0) {
-			mscp_set_error("sftp_seek64 '%s': %s",
-				       path, sftp_get_ssh_error(sftp));
-			sftp_close(h.sf);
-			h.sf = NULL;
-			return h;
-		}
-	} else {
-		h.fd = open(path, flags, mode);
-		if (h.fd < 0) {
-			mscp_set_error("open '%s': %s", path, strerrno());
-			return h;
-		}
-		if (lseek(h.fd, off, SEEK_SET) < 0) {
-			mscp_set_error("lseek '%s': %s", path, strerrno());
-			close(h.fd);
-			h.fd = -1;
-			return h;
-		}
-	}
-
-	return h;
-}
-
-#define mscp_open_is_failed(h) (h.fd < 0 && h.sf == NULL)
-
-static void mscp_close(mfh h)
-{
-	if (h.sf)
-		sftp_close(h.sf);
-	if (h.fd > 0)
-		close(h.fd);
-	h.sf = NULL;
-	h.fd = -1;
-}
-
-/* wrap chmod/sftp_chmod */
-
-static int mscp_chmod(const char *path, mode_t mode, sftp_session sftp)
-{
-	if (sftp) {
-		if (sftp_chmod(sftp, path, mode) < 0)  {
-			mscp_set_error("sftp_chmod '%s': %s",
-				       path, sftp_get_ssh_error(sftp));
-			return -1;
-		}
-	} else {
-		if (chmod(path, mode) < 0) {
-			mscp_set_error("chmod '%s': %s", path, strerrno());
-			return -1;
-		}
-	}
-
-	return 0;
-}
-
 #endif /* _PATH_H_ */
--- a/src/pymscp.c
+++ b/src/pymscp.c
@@ -103,11 +103,13 @@ static PyObject *wrap_mscp_init(PyObject *self, PyObject *args, PyObject *kw)
 		/*  mscp_ssh_opts */
 		"login_name",	/* const char * */
 		"port",		/* const char * */
+		"config",	/* const char * */
 		"identity",	/* const char * */

 		"cipher",	/* const char * */
 		"hmac",		/* const char * */
 		"compress",	/* const char * */
+		"ccalgo",       /* const char * */
 		"password",	/* const char * */
 		"passphrase",	/* const char * */

@@ -116,10 +118,10 @@ static PyObject *wrap_mscp_init(PyObject *self, PyObject *args, PyObject *kw)
 		"enable_nagle",		/* bool */
 		NULL,
 	};
-	const char *fmt = "si" "|" "ii" "kkk" "s" "iii" "sss" "sssss" "ipp";
+	const char *fmt = "si" "|" "ii" "kkk" "s" "iii" "ssss" "ssssss" "ipp";
 	char *coremask = NULL;
-	char *login_name = NULL, *port = NULL, *identity = NULL;
-	char *cipher = NULL, *hmac = NULL, *compress = NULL;
+	char *login_name = NULL, *port = NULL, *config = NULL, *identity = NULL;
+	char *cipher = NULL, *hmac = NULL, *compress = NULL, *ccalgo = NULL;
 	char *password = NULL, *passphrase = NULL;

 	struct instance *i;
@@ -148,10 +150,12 @@ static PyObject *wrap_mscp_init(PyObject *self, PyObject *args, PyObject *kw)
 					  &i->mo.msg_fd,
 					  &login_name,
 					  &port,
+					  &config,
 					  &identity,
 					  &cipher,
 					  &hmac,
 					  &compress,
+					  &ccalgo,
 					  &password,
 					  &passphrase,
 					  &i->so.debug_level,
@@ -167,6 +171,8 @@ static PyObject *wrap_mscp_init(PyObject *self, PyObject *args, PyObject *kw)
 		strncpy(i->so.login_name, login_name, MSCP_SSH_MAX_LOGIN_NAME - 1);
 	if (port)
 		strncpy(i->so.port, port, MSCP_SSH_MAX_PORT_STR - 1);
+	if (config)
+		strncpy(i->so.config, config, PATH_MAX - 1);
 	if (identity)
 		strncpy(i->so.identity, identity, MSCP_SSH_MAX_IDENTITY_PATH - 1);
 	if (cipher)
@@ -175,6 +181,8 @@ static PyObject *wrap_mscp_init(PyObject *self, PyObject *args, PyObject *kw)
 		strncpy(i->so.hmac, hmac, MSCP_SSH_MAX_HMAC_STR - 1);
 	if (compress)
 		strncpy(i->so.compress, compress, MSCP_SSH_MAX_COMP_STR - 1);
+	if (ccalgo)
+		strncpy(i->so.ccalgo, ccalgo, MSCP_SSH_MAX_CCALGO_STR - 1);
 	if (password)
 		strncpy(i->so.password, password, MSCP_SSH_MAX_PASSWORD - 1);
 	if (passphrase)
@@ -375,7 +383,7 @@ static PyObject *wrap_mscp_get_stats(PyObject *self, PyObject *args, PyObject *k

 	mscp_get_stats(m, &s);

-	return Py_BuildValue("KKd", s.total, s.done, s.finished);
+	return Py_BuildValue("KKO", s.total, s.done, PyBool_FromLong(s.finished));
 }

 static PyObject *wrap_mscp_cleanup(PyObject *self, PyObject *args, PyObject *kw)
--- a/src/ssh.c
+++ b/src/ssh.c
@@ -64,6 +64,12 @@ static int ssh_set_opts(ssh_session ssh, struct mscp_ssh_opts *opts)
 		return -1;
 	}

+	if (is_specified(opts->ccalgo) &&
+	    ssh_options_set(ssh, SSH_OPTIONS_CCALGO, opts->ccalgo) < 0) {
+		mscp_set_error("failed to set cclago");
+		return -1;
+	}
+
 	/* if NOT specified to enable Nagle's algorithm, disable it (set TCP_NODELAY) */
 	if (!opts->enable_nagle) {
 		int v = 1;
@@ -73,6 +79,12 @@ static int ssh_set_opts(ssh_session ssh, struct mscp_ssh_opts *opts)
 		}
 	}

+	if (is_specified(opts->config) &&
+	    ssh_options_parse_config(ssh, opts->config) < 0) {
+		mscp_set_error("failed to parse ssh_config: %s", opts->config);
+		return -1;
+	}
+
 	return 0;
 }

@@ -149,14 +161,14 @@ static ssh_session ssh_init_session(const char *sshdst, struct mscp_ssh_opts *op
 	cb.userdata = opts;
 	ssh_set_callbacks(ssh, &cb);

-	if (ssh_set_opts(ssh, opts) != 0)
-		goto free_out;
-
 	if (ssh_options_set(ssh, SSH_OPTIONS_HOST, sshdst) != SSH_OK) {
 		mscp_set_error("failed to set destination host");
 		goto free_out;
 	}

+	if (ssh_set_opts(ssh, opts) != 0)
+		goto free_out;
+
 	if (ssh_connect(ssh) != SSH_OK) {
 		mscp_set_error("failed to connect ssh server: %s", ssh_get_error(ssh));
 		goto free_out;
--- a/test/test_e2e.py
+++ b/test/test_e2e.py
@@ -3,7 +3,9 @@
 test_e2e.py: End-to-End test for mscp executable.
 """

+import platform
 import pytest
+import getpass
 import os

 from subprocess import check_call, CalledProcessError, PIPE
@@ -11,11 +13,15 @@ from util import File, check_same_md5sum


 def run2ok(args):
-    check_call(list(map(str, args)))
+    cmd = list(map(str, args))
+    print("cmd: {}".format(" ".join(cmd)))
+    check_call(cmd)

 def run2ng(args):
+    cmd = list(map(str, args))
+    print("cmd: {}".format(" ".join(cmd)))
    with pytest.raises(CalledProcessError) as e:
-        check_call(list(map(str, args)))
+        check_call(cmd)


 """ usage test """
@@ -56,7 +62,7 @@ param_single_copy = [
@pytest.mark.parametrize("src, dst", param_single_copy)
 def test_single_copy(mscp, src_prefix, dst_prefix, src, dst):
    src.make()
-    run2ok([mscp, "-H", src_prefix + src.path, dst_prefix + dst.path])
+    run2ok([mscp, "-H", "-vvv", src_prefix + src.path, dst_prefix + dst.path])
    assert check_same_md5sum(src, dst)
    src.cleanup()
    dst.cleanup()
@@ -65,7 +71,7 @@ def test_single_copy(mscp, src_prefix, dst_prefix, src, dst):
 def test_failed_to_copy_nonexistent_file(mscp, src_prefix, dst_prefix):
    src = "nonexistent_src"
    dst = "nonexistent_dst"
-    run2ng([mscp, "-H", src_prefix + src, dst_prefix + dst])
+    run2ng([mscp, "-H", "-vvv", src_prefix + src, dst_prefix + dst])

 param_double_copy = [
    (File("src1", size = 1024 * 1024), File("src2", size = 1024 * 1024),
@@ -77,7 +83,7 @@ param_double_copy = [
 def test_double_copy(mscp, src_prefix, dst_prefix, s1, s2, d1, d2):
    s1.make()
    s2.make()
-    run2ok([mscp, "-H", src_prefix + s1.path, src_prefix + s2.path, dst_prefix + "dst"])
+    run2ok([mscp, "-H", "-vvv", src_prefix + s1.path, src_prefix + s2.path, dst_prefix + "dst"])
    assert check_same_md5sum(s1, d1)
    assert check_same_md5sum(s2, d2)
    s1.cleanup()
@@ -85,6 +91,47 @@ def test_double_copy(mscp, src_prefix, dst_prefix, s1, s2, d1, d2):
    d1.cleanup()
    d2.cleanup()

+
+remote_v6_prefix = "[::1]:{}/".format(os.getcwd())
+param_remote_v6_prefix = [
+    ("", remote_v6_prefix), (remote_v6_prefix, "")
+]
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_v6_prefix)
+@pytest.mark.parametrize("s1, s2, d1, d2", param_double_copy)
+def test_double_copy_with_ipv6_notation(mscp, src_prefix, dst_prefix, s1, s2, d1, d2):
+    s1.make()
+    s2.make()
+    run2ok([mscp, "-H", "-vvv",
+            src_prefix + s1.path, src_prefix + s2.path, dst_prefix + "dst"])
+    assert check_same_md5sum(s1, d1)
+    assert check_same_md5sum(s2, d2)
+    s1.cleanup()
+    s2.cleanup()
+    d1.cleanup()
+    d2.cleanup()
+
+
+remote_user_v6_prefix = "{}@[::1]:{}/".format(getpass.getuser(), os.getcwd())
+param_remote_user_v6_prefix = [
+    ("", remote_user_v6_prefix), (remote_user_v6_prefix, "")
+]
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_user_v6_prefix)
+@pytest.mark.parametrize("s1, s2, d1, d2", param_double_copy)
+def test_double_copy_with_user_and_ipv6_notation(mscp, src_prefix, dst_prefix,
+                                                 s1, s2, d1, d2):
+    s1.make()
+    s2.make()
+    run2ok([mscp, "-H", "-vvv",
+            src_prefix + s1.path, src_prefix + s2.path, dst_prefix + "dst"])
+    assert check_same_md5sum(s1, d1)
+    assert check_same_md5sum(s2, d2)
+    s1.cleanup()
+    s2.cleanup()
+    d1.cleanup()
+    d2.cleanup()
+
+
+
 param_dir_copy = [
    ( "src_dir", "dst_dir",
        [ File("src_dir/t1", size = 64),
@@ -114,11 +161,11 @@ def test_dir_copy(mscp, src_prefix, dst_prefix, src_dir, dst_dir, src, dst, twic
    for f in src:
        f.make()

-    run2ok([mscp, "-H", src_prefix + src_dir, dst_prefix + dst_dir])
+    run2ok([mscp, "-H", "-vvv", src_prefix + src_dir, dst_prefix + dst_dir])
    for sf, df in zip(src, dst):
        assert check_same_md5sum(sf, df)

-    run2ok([mscp, "-H", src_prefix + src_dir, dst_prefix + dst_dir])
+    run2ok([mscp, "-H", "-vvv", src_prefix + src_dir, dst_prefix + dst_dir])
    for sf, df in zip(src, twice):
        assert check_same_md5sum(sf, df)

@@ -127,13 +174,30 @@ def test_dir_copy(mscp, src_prefix, dst_prefix, src_dir, dst_dir, src, dst, twic
        df.cleanup()
        tf.cleanup()

+
+param_dir_copy_single = [
+    ("src_dir", "dst_dir",
+     File("src_dir/t1", size = 1024 * 1024),
+     File("dst_dir/src_dir/t1"),
+     )
+]
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+@pytest.mark.parametrize("src_dir, dst_dir, src, dst", param_dir_copy_single)
+def test_dir_copy_single(mscp, src_prefix, dst_prefix, src_dir, dst_dir, src, dst):
+    src.make()
+    os.mkdir(dst_dir)
+    run2ok([mscp, "-H", "-vvv", src_prefix + src_dir, dst_prefix + dst_dir])
+    assert check_same_md5sum(src, dst)
+    src.cleanup()
+    dst.cleanup()
+
@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 def test_override_single_file(mscp, src_prefix, dst_prefix):
    src = File("src", size = 128).make()
    dst = File("dst", size = 128).make()
    assert not check_same_md5sum(src, dst)

-    run2ok([mscp, "-H", src_prefix + src.path, dst_prefix + dst.path])
+    run2ok([mscp, "-H", "-vvv", src_prefix + src.path, dst_prefix + dst.path])
    assert check_same_md5sum(src, dst)

    src.cleanup()
@@ -144,18 +208,55 @@ def test_min_chunk(mscp, src_prefix, dst_prefix):
    src = File("src", size = 16 * 1024).make()
    dst = File("dst")

-    run2ok([mscp, "-H", "-s", 32768, src_prefix + src.path, dst_prefix + dst.path])
+    run2ok([mscp, "-H", "-vvv", "-s", 32768, src_prefix + src.path, dst_prefix + dst.path])
    assert check_same_md5sum(src, dst)

    src.cleanup()
    dst.cleanup()

+
+def is_alpine():
+    if os.path.exists("/etc/os-release"):
+        with open("/etc/os-release", "r") as f:
+            for line in f:
+                if line.strip() == "ID=alpine":
+                    return True
+    return False
+
+param_glob_copy = [
+    (
+        "src*", "dstx",
+        [ File("src1"), File("src2"), File("src3") ],
+        [ File("dstx/src1"), File("dstx/src2"), File("dstx/src3") ],
+    ),
+    (
+        "src*", "dstx",
+        [ File("src1/s1"), File("src2/s2"), File("src3/s3") ],
+        [ File("dstx/s1"), File("dstx/s2"), File("dstx/s3") ],
+    )
+]
+
+@pytest.mark.skipif(is_alpine(),
+                    reason = "musl does not implement glob ALTDIRFUNC")
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+@pytest.mark.parametrize("src_glob_path, dst_path, srcs, dsts", param_glob_copy)
+def test_glob_src_path(mscp, src_prefix, dst_prefix,
+                       src_glob_path, dst_path, srcs, dsts):
+    for src in srcs:
+        src.make(size = 1024 * 1024)
+
+    run2ok([mscp, "-H", "-vvv", src_prefix + src_glob_path, dst_prefix + dst_path])
+    for src, dst in zip(srcs, dsts):
+        assert check_same_md5sum(src, dst)
+        src.cleanup()
+        dst.cleanup()
+
@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 def test_thread_affinity(mscp, src_prefix, dst_prefix):
    src = File("src", size = 64 * 1024).make()
    dst = File("dst")

-    run2ok([mscp, "-H", "-n", 4, "-m", "0x01",
+    run2ok([mscp, "-H", "-vvv", "-n", 4, "-m", "0x01",
            src_prefix + src.path, dst_prefix + dst.path])
    assert check_same_md5sum(src, dst)

@@ -167,7 +268,7 @@ def test_cannot_override_file_with_dir(mscp, src_prefix, dst_prefix):
    src = File("src", size = 128).make()
    dst = File("dst").make()

-    run2ng([mscp, "-H", src_prefix + src.path, dst_prefix + "dst/src"])
+    run2ng([mscp, "-H", "-vvv", src_prefix + src.path, dst_prefix + "dst/src"])

    src.cleanup()
    dst.cleanup()
@@ -176,7 +277,7 @@ def test_cannot_override_file_with_dir(mscp, src_prefix, dst_prefix):
 def test_transfer_zero_bytes(mscp, src_prefix, dst_prefix):
    src = File("src", size = 0).make()
    dst = File("dst")
-    run2ok([mscp, "-H", src_prefix + src.path, dst_prefix + "dst"])
+    run2ok([mscp, "-H", "-vvv", src_prefix + src.path, dst_prefix + "dst"])
    assert os.path.exists("dst")
    src.cleanup()
    dst.cleanup()
@@ -185,18 +286,80 @@ def test_transfer_zero_bytes(mscp, src_prefix, dst_prefix):
 def test_override_dst_having_larger_size(mscp, src_prefix, dst_prefix):
    src = File("src", size = 1024 * 1024).make()
    dst = File("dst", size = 1024 * 1024 * 2).make()
-    run2ok([mscp, "-H", src_prefix + src.path, dst_prefix + "dst"])
+    run2ok([mscp, "-H", "-vvv", src_prefix + src.path, dst_prefix + "dst"])
    assert check_same_md5sum(src, dst)
    src.cleanup()
    dst.cleanup()

+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_dont_truncate_dst(mscp, src_prefix, dst_prefix):
+    f = File("srcanddst", size = 1024 * 1024 * 128).make()
+    md5_before = f.md5sum()
+    run2ok([mscp, "-H", "-vvv", src_prefix + f.path, dst_prefix + f.path])
+    md5_after = f.md5sum()
+    assert md5_before == md5_after
+    f.cleanup()
+
 compressions = ["yes", "no", "none"]
@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
@pytest.mark.parametrize("compress", compressions)
 def test_compression(mscp, src_prefix, dst_prefix, compress):
    src = File("src", size = 1024 * 1024).make()
    dst = File("dst", size = 1024 * 1024 * 2).make()
-    run2ok([mscp, "-H", "-C", compress, src_prefix + src.path, dst_prefix + "dst"])
+    run2ok([mscp, "-H", "-vvv", "-C", compress, src_prefix + src.path, dst_prefix + "dst"])
    assert check_same_md5sum(src, dst)
    src.cleanup()
    dst.cleanup()
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_ccalgo(mscp, src_prefix, dst_prefix):
+    src = File("src", size = 1024 * 1024).make()
+    dst = File("dst").make()
+    if platform.system() == "Darwin":
+        # Darwin does not support TCP_CONGESTION
+        algo = "cubic"
+        run = run2ng
+    elif platform.system() == "Linux":
+        # Linux supports TCP_CONGESTION
+        with open("/proc/sys/net/ipv4/tcp_allowed_congestion_control", "r") as f:
+            algo = f.read().strip().split().pop()
+        run = run2ok
+    run([mscp, "-H", "-vvv", "-g", algo, src_prefix + src.path, dst_prefix + "dst"])
+
+
+testhost = "mscptestlocalhost"
+testhost_prefix = "{}:{}/".format(testhost, os.getcwd()) # use current dir
+param_testhost_prefix = [
+    ("", testhost_prefix), (testhost_prefix, "")
+]
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_testhost_prefix)
+def test_config_ok(mscp, src_prefix, dst_prefix):
+    config = "/tmp/mscp_test_ssh_config"
+    with open(config, "w") as f:
+        f.write("host {}\n".format(testhost))
+        f.write("    hostname localhost\n")
+
+    src = File("src", size = 1024 * 1024).make()
+    dst = File("dst", size = 1024 * 1024 * 2).make()
+    run2ok([mscp, "-H", "-vvv", "-F", config,
+            src_prefix + src.path, dst_prefix + "dst"])
+
+    os.remove(config)
+    assert check_same_md5sum(src, dst)
+    src.cleanup()
+    dst.cleanup()
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_testhost_prefix)
+def test_config_ng(mscp, src_prefix, dst_prefix):
+    config = "/tmp/mscp_test_ssh_config"
+    with open(config, "w") as f:
+        f.write("\n") # use empty ssh_config
+
+    src = File("src", size = 1024 * 1024).make()
+    dst = File("dst", size = 1024 * 1024 * 2).make()
+    run2ng([mscp, "-H", "-vvv", "-F", config,
+            src_prefix + src.path, dst_prefix + "dst"])
+
+    os.remove(config)
+    src.cleanup()
+    dst.cleanup()
--- a/test/test_python.py
+++ b/test/test_python.py
@@ -121,6 +121,7 @@ param_invalid_kwargs = [
    { "cipher": "invalid" },
    { "hmac": "invalid"},
    { "compress": "invalid"},
+    { "ccalgo": "invalid"},
 ]

@pytest.mark.parametrize("kw", param_invalid_kwargs)
--- a/test/util.py
+++ b/test/util.py
@@ -22,7 +22,10 @@ class File():
    def __str__(self):
        return self.path

-    def make(self):
+    def make(self, size = None):
+        if size:
+            self.size = size
+
        d = os.path.dirname(self.path)
        if d:
            os.makedirs(d, exist_ok = True)
Author	SHA1	Message	Date
Ryo Nakamura	9611b4d077	bump version to 0.1.2	2023-11-01 19:54:18 +09:00
Ryo Nakamura	2b9061f5f0	add --sysctl net.ipv6.conf.all.disable_ipv6=0 for docker run. docker disables ipv6 on all interfaces inside containers by default, even ::1 on lo. It causes testing mscp with IPv6 fails. Thus, this commit disables disable_ipv6 via the --sysctl option.	2023-11-01 19:54:18 +09:00
Ryo Nakamura	8e590da322	fix parsing user@host:path. This commit fixes issue #6. Now mscp command correctly parses [x::x] IPv6 address notation in hostname.	2023-11-01 19:54:18 +09:00
Ryo Nakamura	b298b2ec35	main: adopt rolling average of recent eight bps values to calculate ETA	2023-11-01 19:54:18 +09:00
Ryo Nakamura	05a7e96759	main: call only mscp_stop() when receives sigint	2023-11-01 19:54:18 +09:00
Ryo Nakamura	139ba12f1a	write total transferred bytes and number of files at the end of output when serverity is notice.	2023-11-01 19:54:18 +09:00
Ryo Nakamura	cfbadebe6d	change msg: thread[%d] to thread:%d	2023-11-01 19:54:18 +09:00
Ryo Nakamura	d7365683a9	print 1st decimal point in the progress bar	2023-11-01 19:54:18 +09:00
Ryo Nakamura	53a560b130	fix test_e2e for ccalgo and tiny fix on test_dir_copy_single	2023-11-01 19:54:18 +09:00
Ryo Nakamura	bf74aa095a	add -g option to specify TCP cc algorithm This commit introduce SSH_OPTIONS_CCALGO option to the libssh patch and add -g CONGESTION option to mscp.	2023-11-01 19:54:18 +09:00
Ryo Nakamura	a88471fc43	Update README.md add link to PEARC'23 paper	2023-09-11 19:56:33 +09:00
Ryo Nakamura	89e50453a8	bump version to 0.1.1	2023-09-08 17:28:36 +09:00
Ryo Nakamura	bc1cf11cc1	enable github actions on the dev branch	2023-09-08 17:20:45 +09:00
Ryo Nakamura	72841ec12d	fix: use off_t for lseek	2023-09-08 17:19:13 +09:00
Ryo Nakamura	19704a7308	Update README.md replace the demo mp4.	2023-09-07 15:38:45 +09:00
Ryo Nakamura	9227938297	bump version to 0.1.0	2023-09-07 15:06:06 +09:00
Ryo Nakamura	ccc4dedf30	fix docker/alpine-3.17: no need to make install because libmscp is installed by data_files.	2023-09-05 21:20:38 +09:00
Ryo Nakamura	49e8e26f2a	add jupyter example ToDo: refactor state handling of pymscp	2023-09-02 17:23:57 +09:00
Ryo Nakamura	11e024c1da	fix libmscp python bindings. - fix libmscp install path by setup.py with data_files - fix return values of mscp_get_stats() - add examples directory for mscp python binding	2023-08-30 21:24:00 +09:00
Ryo Nakamura	5466a8b9e1	setup.py: fix data_files to isntall libmscp to python library path	2023-08-30 20:35:27 +09:00
Ryo Nakamura	13ec652195	fix mscp_opendir, do not use `tls_sftp`, use `sftp` isntead. The fixed issue causes mscp_opendir wrongly calls opendir() for local when tls_sftp is NULL although sftp is not NULL.	2023-08-30 19:09:29 +09:00
Ryo Nakamura	6b45cf7c9c	update README: adjust to the current usage	2023-08-04 16:12:36 +09:00
Ryo Nakamura	58026790d9	fix usage: "none" is not supported for -F	2023-08-04 16:11:29 +09:00
Ryo Nakamura	23d9577bde	introduce git-based versioning MSCP_BUILD_VERSION (`git describe --tags --dirty --match "v*"`) is passed through include/mscp_version.h.in and cmake. When git is failed, use VERSION file instead (for building from source tar balls that excludes .git).	2023-08-04 16:07:37 +09:00
Ryo Nakamura	24c1bc9149	do not set O_TRUNC when opening destination file. It prevents `mscp localhost:hoge ~/hoge` from truncating the source file. See https://bugzilla.mindrot.org/show_bug.cgi?id=3431. https://github.com/upa/mscp/issues/1	2023-08-04 15:06:14 +09:00
Ryo Nakamura	16f2f88cc9	update README: adjust -h output to HEAD	2023-08-04 14:11:58 +09:00
Ryo Nakamura	2773c7b4d6	add test for specifying ssh_config	2023-08-04 14:04:46 +09:00
Ryo Nakamura	518aa42208	add -F ssh_config option	2023-08-04 13:31:10 +09:00
Ryo Nakamura	3b26c7c719	update README: glob is now supported	2023-08-04 01:53:48 +09:00
Ryo Nakamura	fbc817213b	use pseudo glob/globfree for remote-glob when musl musllibc does not implement GLOB_ALTDIRFUNC, so do not call glob for remote sides when libc is musl. test_e2e.py skips test_glob_src_path when running on alpine.	2023-08-03 21:59:54 +09:00
Ryo Nakamura	5a4c043889	cmake: add docker-build no-cache target	2023-08-03 21:58:59 +09:00
Ryo Nakamura	ba6f53d253	add glob for source paths https://github.com/upa/mscp/issues/3	2023-08-03 20:26:13 +09:00
Ryo Nakamura	9f7c135b15	cleanup wrappers for file operations Previously wrapper functions for open(), opendir(), and stat(), etc, are implemneted in path.h, and now they are in fileops.h and fileops.c. This commit is a reparation for remote glob.	2023-08-03 17:07:39 +09:00
@@ -1 +1 @@
 .0.9
 .1.2