bump version to 0.1.0

because libmscp is installed by data_files.

.gitignore

@@ -4,6 +4,8 @@ compile_commands.json
 CMakeUserPresets.json
 .*.swp
 
+include/mscp_version.h
+
 dist
 *.egg-info
 __pycache__

CMakeLists.txt

@@ -6,6 +6,26 @@ project(mscp
 	VERSION ${MSCP_VERSION}
 	LANGUAGES C)
 
+
+find_package(Git)
+if (Git_FOUND)
+	# based on https://github.com/nocnokneo/cmake-git-versioning-example
+	execute_process(
+		COMMAND	${GIT_EXECUTABLE} describe --tags --dirty --match "v*"
+		OUTPUT_VARIABLE	GIT_DESCRIBE_VERSION
+		RESULT_VARIABLE	GIT_DESCRIBE_ERROR_CODE
+		OUTPUT_STRIP_TRAILING_WHITESPACE)
+	if(NOT GIT_DESCRIBE_ERROR_CODE)
+		set(MSCP_BUILD_VERSION ${GIT_DESCRIBE_VERSION})
+	endif()
+endif()
+
+if (NOT MSCP_BUILD_VERSION)
+	message(STATUS "Failed to determine version via Git. Use VERSION file instead.")
+	set(MSCP_BUILD_VERSION v${MSCP_VERSION})
+endif()
+
+
 include(GNUInstallDirs)
 
 set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -DDEBUG")
@@ -62,9 +82,16 @@ if(BUILD_CONAN)
 	list(APPEND MSCP_LINK_LIBS OpenSSL::Crypto)
 endif()
 
-set(LIBMSCP_SRC src/mscp.c src/ssh.c src/path.c src/platform.c src/message.c)
+
+# generate version header file
+configure_file(
+	${mscp_SOURCE_DIR}/include/mscp_version.h.in
+	${mscp_SOURCE_DIR}/include/mscp_version.h)
+
 
 # libmscp.so
+set(LIBMSCP_SRC
+	src/mscp.c src/ssh.c src/fileops.c src/path.c src/platform.c src/message.c)
 add_library(mscp-shared SHARED ${LIBMSCP_SRC})
 target_include_directories(mscp-shared
 	PUBLIC	$<BUILD_INTERFACE:${mscp_SOURCE_DIR}/include>
@@ -104,7 +131,6 @@ if (BUILD_STATIC)
 	target_link_options(mscp PRIVATE -static)
 endif()
 target_compile_options(mscp PRIVATE ${MSCP_COMPILE_OPTS})
-target_compile_definitions(mscp PUBLIC _VERSION="${PROJECT_VERSION}")
 
 
 install(TARGETS mscp RUNTIME DESTINATION bin)
@@ -185,6 +211,12 @@ foreach(x RANGE ${DIST_LISTLEN})
 		COMMAND
 		docker build -t ${DOCKER_IMAGE} -f docker/${DOCKER_INDEX}.Dockerfile .)
 
+	add_custom_target(docker-build-${DOCKER_INDEX}-no-cache
+		COMMENT "Build mscp in ${DOCKER_IMAGE} container"
+		WORKING_DIRECTORY ${mscp_SOURCE_DIR}
+		COMMAND
+		docker build --no-cache -t ${DOCKER_IMAGE} -f docker/${DOCKER_INDEX}.Dockerfile .)
+
 	add_custom_target(docker-test-${DOCKER_INDEX}
 		COMMENT "Test mscp in ${DOCKER_IMAGE} container"
 		WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
@@ -198,11 +230,13 @@ foreach(x RANGE ${DIST_LISTLEN})
 		docker run --rm -v ${CMAKE_BINARY_DIR}:/out ${DOCKER_IMAGE}
 		cp /mscp/build/${PKG_FILE_NAME} /out/)
 
-	list(APPEND DOCKER_BUILDS	docker-build-${DOCKER_INDEX})
-	list(APPEND DOCKER_TESTS	docker-test-${DOCKER_INDEX})
-	list(APPEND DOCKER_PKGS		docker-pkg-${DOCKER_INDEX})
+	list(APPEND DOCKER_BUILDS		docker-build-${DOCKER_INDEX})
+	list(APPEND DOCKER_BUILDS_NO_CACHE	docker-build-${DOCKER_INDEX}-no-cache)
+	list(APPEND DOCKER_TESTS		docker-test-${DOCKER_INDEX})
+	list(APPEND DOCKER_PKGS			docker-pkg-${DOCKER_INDEX})
 endforeach()
 
-add_custom_target(docker-build-all	DEPENDS ${DOCKER_BUILDS})
-add_custom_target(docker-test-all	DEPENDS ${DOCKER_TESTS})
-add_custom_target(docker-pkg-all	DEPENDS ${DOCKER_PKGS})
+add_custom_target(docker-build-all		DEPENDS ${DOCKER_BUILDS})
+add_custom_target(docker-build-all-no-cache	DEPENDS ${DOCKER_BUILDS_NO_CACHE})
+add_custom_target(docker-test-all		DEPENDS ${DOCKER_TESTS})
+add_custom_target(docker-pkg-all		DEPENDS ${DOCKER_PKGS})

README.md

@@ -21,8 +21,7 @@ https://user-images.githubusercontent.com/184632/206889149-7cc6178a-6f0f-41e6-85
 
 Differences from `scp` on usage:
 
-- remote glob on remote shell expansion is not supported.
-- remote to remote copy is not supported.
+- Remote-to-remote copy is not supported.
 - `-r` option is not needed to transfer directories.
 - and any other differences I have not implemented and noticed.
 
@@ -164,11 +163,11 @@ copy done: test/testdir/asdf
 
 ```console
 $ mscp -h
-mscp v0.0.8: copy files over multiple ssh connections
+mscp v0.0.9-11-g5802679: copy files over multiple ssh connections
 
 Usage: mscp [vqDHdNh] [-n nr_conns] [-m coremask] [-u max_startups]
             [-s min_chunk_sz] [-S max_chunk_sz] [-a nr_ahead] [-b buf_sz]
-            [-l login_name] [-p port] [-i identity_file]
+            [-l login_name] [-p port] [-F ssh_config] [-i identity_file]
             [-c cipher_spec] [-M hmac_spec] [-C compress] source ... target
 
     -n NR_CONNECTIONS  number of connections (default: floor(log(cores)*2)+1)
@@ -187,6 +186,7 @@ Usage: mscp [vqDHdNh] [-n nr_conns] [-m coremask] [-u max_startups]
 
     -l LOGIN_NAME      login name
     -p PORT            port number
+    -F CONFIG          path to user ssh config (default ~/.ssh/config)
     -i IDENTITY        identity file for public key authentication
     -c CIPHER          cipher spec
     -M HMAC            hmac spec

VERSION

@@ -1 +1 @@
-0.0.9
+0.1.0

docker/alpine-3.17.Dockerfile

@@ -39,7 +39,3 @@ RUN cd ${mscpdir}							\
 RUN cd ${mscpdir}	\
 	&& python3 setup.py install --user
 
-# Need Fix: A trick putting libmscp.so to python mscp module dir does not work on alpine,
-# so install libmscp.
-RUN cd ${mscpdir}/build	\
-	&& make install

examples/.gitignore

@@ -0,0 +1,3 @@
+simple-copy-dest
+*.img
+.ipynb_checkpoints

examples/mscp-example.ipynb

@@ -0,0 +1,226 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "ccda9e3a-35de-43fc-9b6e-02475c763f6b",
+   "metadata": {},
+   "source": [
+    "# mscp python binding example"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 60,
+   "id": "df04d655-a082-47eb-9a1e-154ebc2a5655",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import glob\n",
+    "import time\n",
+    "import os\n",
+    "\n",
+    "import mscp"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 53,
+   "id": "e9ed4519-c3fd-4639-89a5-1c1cdffd9519",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "this_dir = os.getcwd()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fee75bf8-df40-45f4-81d1-113069c34f13",
+   "metadata": {},
+   "source": [
+    "## Simple copy"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 54,
+   "id": "2b06e6d3-30cc-47be-bd4f-af27eb141c8c",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['../src/ssh.c',\n",
+       " '../src/mscp.c',\n",
+       " '../src/platform.c',\n",
+       " '../src/pymscp.c',\n",
+       " '../src/main.c',\n",
+       " '../src/path.c',\n",
+       " '../src/message.c',\n",
+       " '../src/fileops.c']"
+      ]
+     },
+     "execution_count": 54,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# preparing files to be transferred\n",
+    "c_sources = glob.glob(\"../src/*.c\")\n",
+    "c_sources"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 55,
+   "id": "89bb4558-9472-4d26-9af3-24f426b15edc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# copy files using mscp\n",
+    "dst_dir = this_dir + \"/simple-copy-dest\"\n",
+    "m = mscp.mscp(\"localhost\", mscp.LOCAL2REMOTE)\n",
+    "m.copy(c_sources, dst_dir)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 56,
+   "id": "6daf2c98-8905-4039-b82a-a593df3107fe",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['ssh.c',\n",
+       " 'mscp.c',\n",
+       " 'platform.c',\n",
+       " 'pymscp.c',\n",
+       " 'main.c',\n",
+       " 'path.c',\n",
+       " 'message.c',\n",
+       " 'fileops.c']"
+      ]
+     },
+     "execution_count": 56,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "os.listdir(\"simple-copy-dest\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f4a3869a-878e-43b0-9758-a049eaf8b5bd",
+   "metadata": {},
+   "source": [
+    "## Simple Copy with Python Rich ProgressBar"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 64,
+   "id": "e7cb7cd6-b845-4d26-93ed-aee8ed3983ab",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# make a 256MB file\n",
+    "src = \"example-256MB-src.img\"\n",
+    "with open(src, \"wb\") as f:\n",
+    "    f.seek(128 * 1024 * 1024 -1, 0)\n",
+    "    f.write(b'1')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 69,
+   "id": "878607ed-5c06-4b15-81ac-9845dad0c9c6",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b700e9fc00464969a22a26300404dc35",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Output()"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
+      ],
+      "text/plain": []
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
+       "</pre>\n"
+      ],
+      "text/plain": [
+       "\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# copy the 256MB file while ploting progress bar using python rich\n",
+    "dst = this_dir + \"/example-256MB-dst.img\"\n",
+    "\n",
+    "kw = {\"nr_threads\": 1, \"nr_ahead\": 1} # slow mscp to watch the progress bar\n",
+    "\n",
+    "m = mscp.mscp(\"localhost\", mscp.LOCAL2REMOTE, **kw)\n",
+    "m.copy(src, dst, nonblock = True)\n",
+    "\n",
+    "# m.stats() returns total bytes to be transferred, bytes transferred (done), and finished (bool).\n",
+    "total, done, finished = m.stats()\n",
+    "with Progress() as progress:\n",
+    "\n",
+    "    task = progress.add_task(f\"[green]Copying {src}\", total = total)\n",
+    "\n",
+    "    while not progress.finished:\n",
+    "        total, done, finished = m.stats()\n",
+    "        progress.update(task, completed = done)\n",
+    "        time.sleep(0.5)\n",
+    "\n",
+    "m.join()\n",
+    "m.cleanup()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/mscp-python.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+"""mscp.py
+
+An example python script running mscp
+"""
+
+import argparse
+import time
+import sys
+
+from rich.progress import Progress
+
+import mscp
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-f", "--from", dest = "fr",
+                        metavar = "REMOTE", default = None,
+                        help = "copy a file from this remote host")
+    parser.add_argument("-t", "--to", metavar = "REMOTE", default = None,
+                        help = "copy a file to this remote host")
+    parser.add_argument("source", help = "path to source file to be copied")
+    parser.add_argument("destination", help = "path of copy destination")
+
+    args = parser.parse_args()
+
+    if args.fr and args.to:
+        print("-f and -t are exclusive", file = sys.stderr)
+        sys.exit(1)
+    elif args.fr:
+        d = mscp.REMOTE2LOCAL
+        remote = args.fr
+    elif args.to:
+        d = mscp.LOCAL2REMOTE
+        remote = args.to
+    else:
+        print("-f or -t must be specified", file = sys.stderr)
+        sys.exit(1)
+
+
+    m = mscp.mscp(remote, d)
+    m.connect()
+    m.add_src_path(args.source)
+    m.set_dst_path(args.destination)
+    m.scan()
+    m.start()
+
+    total, done, finished = m.stats()
+    with Progress() as progress:
+
+        task = progress.add_task("[green]Copying...", total = total)
+
+        while not progress.finished:
+            total, done, finished = m.stats()
+            progress.update(task, completed = done)
+            time.sleep(0.5)
+
+    m.join()
+    m.cleanup()
+
+
+if __name__ == "__main__":
+    main()

include/mscp.h

@@ -67,6 +67,7 @@ struct mscp_ssh_opts {
 	/* ssh options */
 	char	login_name[MSCP_SSH_MAX_LOGIN_NAME];	/** ssh username */
 	char	port[MSCP_SSH_MAX_PORT_STR];		/** ssh port */
+	char	config[PATH_MAX];	/** path to ssh_config, default ~/.ssh/config*/
 	char	identity[MSCP_SSH_MAX_IDENTITY_PATH];	/** path to private key */
 	char	cipher[MSCP_SSH_MAX_CIPHER_STR];	/** cipher spec */
 	char	hmac[MSCP_SSH_MAX_HMAC_STR];		/** hmacp spec */

include/mscp_version.h.in

@@ -0,0 +1,7 @@
+#ifndef _MSCP_VERSION_H_
+#define _MSCP_VERSION_H_
+
+#define MSCP_VERSION		"@MSCP_VERSION@"
+#define MSCP_BUILD_VERSION	"@MSCP_BUILD_VERSION@"
+
+#endif /* _MSCP_VERSION_H_ */

mscp/mscp.py

@@ -37,7 +37,7 @@ SEVERITY_DEBUG  = pymscp.SEVERITY_DEBUG
 
 STATE_INIT      = 0
 STATE_CONNECTED = 1
-STATE_SCANNED  = 2
+STATE_SCANNED   = 2
 STATE_RUNNING   = 3
 STATE_STOPPED   = 4
 STATE_JOINED    = 5
@@ -47,7 +47,7 @@ STATE_RELEASED  = 7
 _state_str = {
     STATE_INIT:      "init",
     STATE_CONNECTED: "connected",
-    STATE_SCANNED:  "scanned",
+    STATE_SCANNED:   "scanned",
     STATE_RUNNING:   "running",
     STATE_STOPPED:   "stopped",
     STATE_JOINED:    "joined",
@@ -115,6 +115,8 @@ class mscp:
         pymscp.mscp_set_dst_path(m = self.m, dst_path = dst_path);
 
     def scan(self):
+        if self.state == STATE_SCANNED:
+            return
         if self.state != STATE_CONNECTED:
             raise RuntimeError("invalid mscp state: {}".format(self.__state2str()))
         if not self.src_paths:
@@ -139,6 +141,8 @@ class mscp:
         self.state = STATE_STOPPED
 
     def join(self):
+        if self.state == STATE_JOINED:
+            return
         if not (self.state == STATE_RUNNING or self.state == STATE_STOPPED):
             raise RuntimeError("invalid mscp state: {}".format(self.__state2str()))
         pymscp.mscp_join(m = self.m)

setup.py

@@ -11,6 +11,9 @@ if sys.platform == "linux":
 elif sys.platform == "darwin":
     libmscp = "libmscp.dylib"
 
+data_dir = sys.prefix + "/lib"
+libmscp = "build/" + libmscp
+
 setup(
     name='mscp',
     version = version,
@@ -20,7 +23,7 @@ setup(
     url = "https://github.com/upa/mscp",
     packages = find_packages("mscp"),
     package_dir = {"": "mscp"},
-    data_files = [ ("", ["build/" + libmscp])],
+    data_files = [ (data_dir, [libmscp])],
     py_modules = [ "mscp" ],
     ext_modules = [
         Extension(

src/fileops.c

@@ -0,0 +1,366 @@
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <dirent.h>
+
+#include <fileops.h>
+#include <ssh.h>
+#include <message.h>
+
+
+sftp_session __thread tls_sftp;
+/* tls_sftp is used *_wrapped() functions */
+
+void set_tls_sftp_session(sftp_session sftp)
+{
+	tls_sftp = sftp;
+}
+
+static void sftp_err_to_errno(sftp_session sftp)
+{
+	int sftperr = sftp_get_error(sftp);
+
+	switch (sftperr){
+	case SSH_FX_OK:
+	case SSH_FX_EOF:
+		errno = 0;
+		break;
+	case SSH_FX_NO_SUCH_FILE:
+	case SSH_FX_NO_SUCH_PATH:
+		errno = ENOENT;
+		break;
+	case SSH_FX_PERMISSION_DENIED:
+		errno = EACCES;
+		break;
+	case SSH_FX_FAILURE:
+		errno = EINVAL;
+	case SSH_FX_BAD_MESSAGE:
+		errno = EBADMSG;
+	case SSH_FX_NO_CONNECTION:
+		errno = ENOTCONN;
+		break;
+	case SSH_FX_CONNECTION_LOST:
+		errno = ENETRESET;
+		break;
+	case SSH_FX_OP_UNSUPPORTED:
+		errno = EOPNOTSUPP;
+		break;
+	case SSH_FX_INVALID_HANDLE:
+		errno = EBADF;
+		break;
+	case SSH_FX_FILE_ALREADY_EXISTS:
+		errno = EEXIST;
+		break;
+	case SSH_FX_WRITE_PROTECT:
+		errno = EPERM;
+		break;
+	case SSH_FX_NO_MEDIA:
+		errno = ENODEV;
+		break;
+	default:
+		mpr_warn(stderr, "unkown SSH_FX response %d", sftperr);
+	}
+}
+
+
+MDIR *mscp_opendir(const char *path, sftp_session sftp)
+{
+	MDIR *md;
+
+	if (!(md = malloc(sizeof(*md))))
+		return NULL;
+	memset(md, 0, sizeof(*md));
+
+	if (sftp) {
+		md->remote = sftp_opendir(sftp, path);
+		sftp_err_to_errno(sftp);
+		if (!md->remote) {
+			goto free_out;
+		}
+	} else {
+		md->local = opendir(path);
+		if (!md->local) {
+			goto free_out;
+		}
+	}
+
+	return md;
+
+free_out:
+	free(md);
+	return NULL;
+}
+
+MDIR *mscp_opendir_wrapped(const char *path)
+{
+	return mscp_opendir(path, tls_sftp);
+}
+
+void mscp_closedir(MDIR *md)
+{
+	int ret;
+	if (md->remote)
+		sftp_closedir(md->remote);
+	else
+		closedir(md->local);
+
+	free(md);
+}
+
+
+struct dirent __thread tls_dirent;
+/* tls_dirent contains dirent converted from sftp_attributes returned
+ * from sftp_readdir(). This trick is derived from openssh's
+ * fudge_readdir() */
+
+struct dirent *mscp_readdir(MDIR *md)
+{
+	sftp_attributes attr;
+	struct dirent *ret = NULL;
+	static int inum = 1;
+
+	if (md->remote) {
+		attr = sftp_readdir(md->remote->sftp, md->remote);
+		if (!attr) {
+			sftp_err_to_errno(md->remote->sftp);
+			return NULL;
+		}
+
+		memset(&tls_dirent, 0, sizeof(tls_dirent));
+		strncpy(tls_dirent.d_name, attr->name, sizeof(tls_dirent.d_name) - 1);
+		tls_dirent.d_ino = inum++;
+		if (!inum)
+			inum = 1;
+		ret = &tls_dirent;
+		sftp_attributes_free(attr);
+	} else
+		ret = readdir(md->local);
+
+	return ret;
+}
+
+int mscp_mkdir(const char *path, mode_t mode, sftp_session sftp)
+{
+	int ret;
+
+	if (sftp) {
+		ret = sftp_mkdir(sftp, path, mode);
+		sftp_err_to_errno(sftp);
+	} else
+		ret = mkdir(path, mode);
+
+	if (ret < 0 && errno == EEXIST) {
+		ret = 0;
+	}
+
+	return ret;
+}
+
+
+static void sftp_attr_to_stat(sftp_attributes attr, struct stat *st)
+{
+        memset(st, 0, sizeof(*st));
+	st->st_size	= attr->size;
+	st->st_uid	= attr->uid;
+	st->st_gid	= attr->gid;
+	st->st_mode	= attr->permissions;
+
+	switch (attr->type) {
+	case SSH_FILEXFER_TYPE_REGULAR:
+		st->st_mode |= S_IFREG;
+		break;
+	case SSH_FILEXFER_TYPE_DIRECTORY:
+		st->st_mode |= S_IFDIR;
+		break;
+	case SSH_FILEXFER_TYPE_SYMLINK:
+		st->st_mode |= S_IFLNK;
+		break;
+	case SSH_FILEXFER_TYPE_SPECIAL:
+		st->st_mode |= S_IFCHR; /* or block? */
+		break;
+	case SSH_FILEXFER_TYPE_UNKNOWN:
+		st->st_mode |= S_IFIFO; /* really? */
+		break;
+	default:
+		mpr_warn(stderr, "unkown SSH_FILEXFER_TYPE %d", attr->type);
+	}
+
+	/* ToDo: convert atime, ctime, and mtime */
+}
+
+
+int mscp_stat(const char *path, struct stat *st, sftp_session sftp)
+{
+	sftp_attributes attr;
+	int ret = 0;
+
+	if (sftp) {
+		attr = sftp_stat(sftp, path);
+		sftp_err_to_errno(sftp);
+		if (!attr)
+			return -1;
+
+		sftp_attr_to_stat(attr, st);
+		sftp_attributes_free(attr);
+		ret = 0;
+	} else
+		ret = stat(path, st);
+
+	return ret;
+}
+
+int mscp_stat_wrapped(const char *path, struct stat *st)
+{
+	return mscp_stat(path, st, tls_sftp);
+}
+
+int mscp_lstat(const char *path, struct stat *st, sftp_session sftp)
+{
+	sftp_attributes attr;
+	int ret = 0;
+
+	if (sftp) {
+		attr = sftp_lstat(sftp, path);
+		sftp_err_to_errno(sftp);
+		if (!attr)
+			return -1;
+
+		sftp_attr_to_stat(attr, st);
+		sftp_attributes_free(attr);
+		ret = 0;
+	} else
+		ret = lstat(path, st);
+
+	return ret;
+}
+
+int mscp_lstat_wrapped(const char *path, struct stat *st)
+{
+	return mscp_lstat(path, st, tls_sftp);
+}
+
+
+mf *mscp_open(const char *path, int flags, mode_t mode, sftp_session sftp)
+{
+	mf *f;
+
+	f = malloc(sizeof(*f));
+	if (!f)
+		return NULL;
+	memset(f, 0, sizeof(*f));
+
+	if (sftp) {
+		f->remote = sftp_open(sftp, path, flags, mode);
+		if (!f->remote) {
+			sftp_err_to_errno(sftp);
+			goto free_out;
+		}
+	} else {
+		f->local = open(path, flags, mode);
+		if (f->local < 0)
+			goto free_out;
+	}
+
+	return f;
+
+free_out:
+	free(f);
+	return NULL;
+}
+
+void mscp_close(mf *f)
+{
+	if (f->remote)
+		sftp_close(f->remote);
+	if (f->local > 0)
+		close(f->local);
+	free(f);
+}
+
+int mscp_lseek(mf *f, size_t off)
+{
+	int ret;
+
+	if (f->remote) {
+		ret = sftp_seek64(f->remote, off);
+		sftp_err_to_errno(f->remote->sftp);
+	} else
+		ret = lseek(f->local, off, SEEK_SET);
+
+	return ret;
+}
+
+int mscp_setstat(const char *path, mode_t mode, size_t size, sftp_session sftp)
+{
+	int ret;
+
+	if (sftp) {
+		struct sftp_attributes_struct attr;
+		memset(&attr, 0, sizeof(attr));
+		attr.permissions = mode;
+		attr.size = size;
+		attr.flags = (SSH_FILEXFER_ATTR_PERMISSIONS|SSH_FILEXFER_ATTR_SIZE);
+		ret = sftp_setstat(sftp, path, &attr);
+		sftp_err_to_errno(sftp);
+	} else {
+		if ((ret = chmod(path, mode)) < 0)
+			return ret;
+		if ((ret = truncate(path, size)) < 0)
+			return ret;
+	}
+
+	return ret;
+}
+
+int mscp_glob(const char *pattern, int flags, glob_t *pglob, sftp_session sftp)
+{
+	int ret;
+	if (sftp) {
+#ifndef GLOB_ALTDIRFUNC
+#define GLOB_NOALTDIRMAGIC INT_MAX
+		/* musl does not implement GLOB_ALTDIRFUNC */
+		pglob->gl_pathc = 1;
+		pglob->gl_pathv = malloc(sizeof(char *));
+		pglob->gl_pathv[0] = strdup(pattern);
+		pglob->gl_offs = GLOB_NOALTDIRMAGIC;
+		return 0;
+#else
+		flags |= GLOB_ALTDIRFUNC;
+		set_tls_sftp_session(sftp);
+#ifdef __APPLE__
+		pglob->gl_opendir = (void *(*)(const char *))mscp_opendir_wrapped;
+		pglob->gl_readdir = (struct dirent *(*)(void *))mscp_readdir;
+		pglob->gl_closedir = (void (*)(void *))mscp_closedir;
+		pglob->gl_lstat = mscp_lstat_wrapped;
+		pglob->gl_stat = mscp_stat_wrapped;
+#elif linux
+		pglob->gl_opendir = (void *(*)(const char *))mscp_opendir_wrapped;
+		pglob->gl_readdir = (void *(*)(void *))mscp_readdir;
+		pglob->gl_closedir = (void (*)(void *))mscp_closedir;
+		pglob->gl_lstat = (int (*)(const char *, void *))mscp_lstat_wrapped;
+		pglob->gl_stat = (int (*)(const char *, void *))mscp_stat_wrapped;
+#else
+#error unsupported platform
+#endif
+#endif
+	}
+
+	ret = glob(pattern, flags, NULL, pglob);
+
+	if (sftp)
+		set_tls_sftp_session(NULL);
+	return ret;
+}
+
+void mscp_globfree(glob_t *pglob)
+{
+#ifndef GLOB_ALTDIRFUNC
+	if (pglob->gl_offs == GLOB_NOALTDIRMAGIC) {
+		free(pglob->gl_pathv[0]);
+		free(pglob->gl_pathv);
+		return;
+	}
+#endif
+	globfree(pglob);
+}

src/fileops.h

@@ -0,0 +1,57 @@
+
+#include <dirent.h>
+#include <sys/stat.h>
+#include <glob.h>
+
+#include <ssh.h>
+
+void set_tls_sftp_session(sftp_session sftp);
+/* sftp_session set by set_tls_sftp_session is sued in
+ mscp_open_wrapped(), mscp_stat_wrapped(), and
+ mscp_lstat_wrapped(). This _wrapped() functions exist for
+ sftp_glob() */
+
+/* directory operations */
+
+struct mdir_struct {
+	DIR *local;
+	sftp_dir remote;
+};
+typedef struct mdir_struct MDIR;
+
+
+MDIR *mscp_opendir(const char *path, sftp_session sftp);
+MDIR *mscp_opendir_wrapped(const char *path);
+void mscp_closedir(MDIR *md);
+struct dirent *mscp_readdir(MDIR *md);
+
+int mscp_mkdir(const char *path, mode_t mode, sftp_session sftp);
+
+/* stat operations */
+int mscp_stat(const char *path, struct stat *st, sftp_session sftp);
+int mscp_stat_wrapped(const char *path, struct stat *st);
+
+int mscp_lstat(const char *path, struct stat *st, sftp_session sftp);
+int mscp_lstat_wrapped(const char *path, struct stat *st);
+
+
+/* file operations */
+
+struct mf_struct {
+	sftp_file remote;
+	int local;
+};
+typedef struct mf_struct mf;
+
+mf *mscp_open(const char *path, int flags, mode_t mode, sftp_session sftp);
+void mscp_close(mf *f);
+int mscp_lseek(mf *f, size_t off);
+
+/* mscp_setstat() involves chmod and truncate. It executes both at
+ * once via a single SFTP command (sftp_setstat()).
+ */
+int mscp_setstat(const char *path, mode_t mode, size_t size, sftp_session sftp);
+
+/* remote glob */
+int mscp_glob(const char *pattern, int flags, glob_t *pglob, sftp_session sftp);
+void mscp_globfree(glob_t *pglob);

src/main.c

@@ -11,22 +11,16 @@
 #include <pthread.h>
 
 #include <mscp.h>
+#include <mscp_version.h>
 #include <util.h>
 
 
-#ifndef _VERSION /* passed through cmake */
-#define VERSION "(unknown)"
-#else
-#define VERSION _VERSION
-#endif
-
-
 void usage(bool print_help) {
-	printf("mscp v" VERSION ": copy files over multiple ssh connections\n"
+	printf("mscp " MSCP_BUILD_VERSION ": copy files over multiple ssh connections\n"
 	       "\n"
 	       "Usage: mscp [vqDHdNh] [-n nr_conns] [-m coremask] [-u max_startups]\n"
 	       "            [-s min_chunk_sz] [-S max_chunk_sz] [-a nr_ahead] [-b buf_sz]\n"
-	       "            [-l login_name] [-p port] [-i identity_file]\n"
+	       "            [-l login_name] [-p port] [-F ssh_config] [-i identity_file]\n"
 	       "            [-c cipher_spec] [-M hmac_spec] [-C compress] source ... target\n"
 	       "\n");
 
@@ -51,6 +45,7 @@ void usage(bool print_help) {
 	       "\n"
 	       "    -l LOGIN_NAME      login name\n"
 	       "    -p PORT            port number\n"
+	       "    -F CONFIG          path to user ssh config (default ~/.ssh/config)\n"
 	       "    -i IDENTITY        identity file for public key authentication\n"
 	       "    -c CIPHER          cipher spec\n"
 	       "    -M HMAC            hmac spec\n"
@@ -207,7 +202,7 @@ int main(int argc, char **argv)
 	memset(&o, 0, sizeof(o));
 	o.severity = MSCP_SEVERITY_WARN;
 
-	while ((ch = getopt(argc, argv, "n:m:u:s:S:a:b:vqDrl:p:i:c:M:C:HdNh")) != -1) {
+	while ((ch = getopt(argc, argv, "n:m:u:s:S:a:b:vqDrl:p:i:F:c:M:C:HdNh")) != -1) {
 		switch (ch) {
 		case 'n':
 			o.nr_threads = atoi(optarg);
@@ -261,6 +256,9 @@ int main(int argc, char **argv)
 			}
 			strncpy(s.port, optarg, MSCP_SSH_MAX_PORT_STR);
 			break;
+		case 'F':
+			strncpy(s.config, optarg, PATH_MAX - 1);
+			break;
 		case 'i':
 			if (strlen(optarg) > MSCP_SSH_MAX_IDENTITY_PATH - 1) {
 				fprintf(stderr, "long identity path: %s\n", optarg);

src/message.h

@@ -2,6 +2,7 @@
 #define _MESSAGE_H_
 
 #include <libgen.h>
+#include <stdio.h>
 
 #include <mscp.h>
 

src/mscp.c

@@ -9,6 +9,7 @@
 #include <util.h>       
 #include <ssh.h>                
 #include <path.h>
+#include <fileops.h>
 #include <atomic.h>             
 #include <platform.h>
 #include <message.h>
@@ -376,7 +377,9 @@ void *mscp_scan_thread(void *arg)
 	struct list_head tmp;
 	struct path *p;
 	struct src *s;
-	mstat ss, ds;
+	struct stat ss, ds;
+	glob_t pglob;
+	int n;
 	
 	m->ret_scan = 0;
 
@@ -403,9 +406,8 @@ void *mscp_scan_thread(void *arg)
 		a.dst_path_should_dir = true;
 
 	if (mscp_stat(m->dst_path, &ds, dst_sftp) == 0) {
-		if (mstat_is_dir(ds))
+		if (S_ISDIR(ds.st_mode))
 			a.dst_path_is_dir = true;
-		mscp_stat_free(ds);
 	}
 
 	a.cp = &m->cp;
@@ -418,23 +420,33 @@ void *mscp_scan_thread(void *arg)
 
 	/* walk a src_path recusively, and resolve path->dst_path for each src */
 	list_for_each_entry(s, &m->src_list, list) {
-		if (mscp_stat(s->path, &ss, src_sftp) < 0) {
-			mscp_set_error("stat: %s", mscp_strerror(src_sftp));
-			mscp_stat_free(ss);
+		memset(&pglob, 0, sizeof(pglob));
+		if (mscp_glob(s->path, GLOB_NOCHECK, &pglob, src_sftp) < 0) {
+			mscp_set_error("mscp_glob: %s", strerrno());
 			goto err_out;
 		}
 
-		/* set path specific args */
-		a.src_path = s->path;
-		a.dst_path = m->dst_path;
-		a.src_path_is_dir = mstat_is_dir(ss);
-		mscp_stat_free(ss);
+		for (n = 0; n < pglob.gl_pathc; n++) {
+			if (mscp_stat(pglob.gl_pathv[n], &ss, src_sftp) < 0) {
+				mscp_set_error("stat: %s %s", s->path, strerrno());
+				goto err_out;
+			}
 
-		INIT_LIST_HEAD(&tmp);
-		if (walk_src_path(src_sftp, s->path, &tmp, &a) < 0)
-			goto err_out;
+			if (!a.dst_path_should_dir && pglob.gl_pathc > 1)
+				a.dst_path_should_dir = true; /* we have over 1 src */
 
-		list_splice_tail(&tmp, m->path_list.prev);
+			/* set path specific args */
+			a.src_path = pglob.gl_pathv[n];
+			a.dst_path = m->dst_path;
+			a.src_path_is_dir = S_ISDIR(ss.st_mode);
+
+			INIT_LIST_HEAD(&tmp);
+			if (walk_src_path(src_sftp, pglob.gl_pathv[n], &tmp, &a) < 0)
+				goto err_out;
+
+			list_splice_tail(&tmp, m->path_list.prev);
+		}
+		mscp_globfree(&pglob);
 	}
 
 	mpr_info(m->msg_fp, "walk source path(s) done\n");
@@ -647,8 +659,8 @@ void *mscp_copy_thread(void *arg)
         pthread_cleanup_pop(1);
 
         if (t->ret < 0)
-                mscp_set_error("copy failed: chunk %s 0x%010lx-0x%010lx",
-			       c->p->path, c->off, c->off + c->len);
+		mpr_err(m->msg_fp, "copy failed: chunk %s 0x%010lx-0x%010lx\n",
+			c->p->path, c->off, c->off + c->len);
 
         return NULL;
 

src/path.c

@@ -7,6 +7,7 @@
 
 #include <ssh.h>
 #include <util.h>
+#include <fileops.h>
 #include <list.h>
 #include <atomic.h>
 #include <path.h>
@@ -190,7 +191,7 @@ static int resolve_chunk(struct path *p, struct path_resolve_args *a)
         return 0;
 }
 
-static int append_path(sftp_session sftp, const char *path, mstat s,
+static int append_path(sftp_session sftp, const char *path, struct stat st,
 		       struct list_head *path_list, struct path_resolve_args *a)
 {
 	struct path *p;
@@ -203,8 +204,8 @@ static int append_path(sftp_session sftp, const char *path, mstat s,
 	memset(p, 0, sizeof(*p));
 	INIT_LIST_HEAD(&p->list);
 	strncpy(p->path, path, PATH_MAX - 1);
-	p->size = mstat_size(s);
-	p->mode = mstat_mode(s);
+	p->size = st.st_size;
+	p->mode = st.st_mode;
 	p->state = FILE_STATE_INIT;
 	lock_init(&p->lock);
 
@@ -239,48 +240,36 @@ static int walk_path_recursive(sftp_session sftp, const char *path,
 			       struct list_head *path_list, struct path_resolve_args *a)
 {
 	char next_path[PATH_MAX];
-	mdirent *e;
-	mdir *d;
-	mstat s;
+	struct dirent *e;
+	struct stat st;
+	MDIR *d;
 	int ret;
 
-	if (mscp_stat(path, &s, sftp) < 0)
+	if (mscp_stat(path, &st, sftp) < 0)
 		return -1;
 
-	if (mstat_is_regular(s)) {
+	if (S_ISREG(st.st_mode)) {
 		/* this path is regular file. it is to be copied */
-		ret = append_path(sftp, path, s, path_list, a);
-		mscp_stat_free(s);
-		return ret;
+		return append_path(sftp, path, st, path_list, a);
 	}
 
-	if (!mstat_is_dir(s)) {
-		/* not regular file and not directory, skip it. */
-		mscp_stat_free(s);
-		return 0; 
-	}
-
-	mscp_stat_free(s);
-
+	if (!S_ISDIR(st.st_mode))
+		return 0; /* not a regular file and not a directory, skip it. */
 
 	/* ok, this path is directory. walk it. */
 	if (!(d = mscp_opendir(path, sftp)))
 		return -1;
 	
-	for (e = mscp_readdir(d); !mdirent_is_null(e); e = mscp_readdir(d)) {
-		if (check_path_should_skip(mdirent_name(e))) {
-			mscp_dirent_free(e);
+	for (e = mscp_readdir(d); e; e = mscp_readdir(d)) {
+		if (check_path_should_skip(e->d_name))
 			continue;
-		}
 		
-		if (strlen(path) + 1 + strlen(mdirent_name(e)) > PATH_MAX) {
-			mscp_set_error("too long path: %s/%s", path, mdirent_name(e));
-			mscp_dirent_free(e);
+		if (strlen(path) + 1 + strlen(e->d_name) > PATH_MAX) {
+			mscp_set_error("too long path: %s/%s", path, e->d_name);
 			return -1;
 		}
-		snprintf(next_path, sizeof(next_path), "%s/%s", path, mdirent_name(e));
+		snprintf(next_path, sizeof(next_path), "%s/%s", path, e->d_name);
 		ret = walk_path_recursive(sftp, next_path, path_list, a);
-		mscp_dirent_free(e);
 		if (ret < 0)
 			return ret;
 	}
@@ -314,10 +303,11 @@ static int touch_dst_path(struct path *p, sftp_session sftp)
 {
         /* XXX: should reflect the permission of the original directory? */
         mode_t mode =  S_IRWXU | S_IRWXG | S_IRWXO;
+	struct stat st;
         char path[PATH_MAX];
         char *needle;
         int ret;
-	mfh h;
+	mf *f;
 
         strncpy(path, p->dst_path, sizeof(path));
 
@@ -326,22 +316,17 @@ static int touch_dst_path(struct path *p, sftp_session sftp)
         for (needle = strchr(path + 1, '/'); needle; needle = strchr(needle + 1, '/')) {
                 *needle = '\0';
 
-		mstat s;
-		if (mscp_stat(path, &s, sftp) == 0) {
-			if (mstat_is_dir(s)) {
-				mscp_stat_free(s);
+		if (mscp_stat(path, &st, sftp) == 0) {
+			if (S_ISDIR(st.st_mode))
 				goto next; /* directory exists. go deeper */
-			} else {
-				mscp_stat_free(s);
+			else
 				return -1; /* path exists, but not directory. */
-			}
 		}
 
-		if (mscp_stat_check_err_noent(sftp) == 0) {
+		if (errno == ENOENT) {
 			/* no file on the path. create directory. */
 			if (mscp_mkdir(path, mode, sftp) < 0) {
-				mscp_set_error("mkdir %s: %s", path,
-					       mscp_strerror(sftp));
+				mscp_set_error("mscp_mkdir %s: %s", path, strerrno());
 				return -1;
 			}
 		}
@@ -349,12 +334,15 @@ static int touch_dst_path(struct path *p, sftp_session sftp)
                 *needle = '/';
         }
 
-        /* open file with O_TRUNC to set file size 0 */
-	h = mscp_open(p->dst_path, O_WRONLY|O_CREAT|O_TRUNC, S_IRUSR|S_IWUSR, 0, sftp);
-	if (mscp_open_is_failed(h))
+	/* Do not set O_TRUNC here. Instead, do mscp_setstat() at the
+	 * end. see https://bugzilla.mindrot.org/show_bug.cgi?id=3431 */
+	f = mscp_open(p->dst_path, O_WRONLY|O_CREAT, S_IRUSR|S_IWUSR, sftp);
+	if (!f) {
+		mscp_set_error("mscp_open %s: %s\n", p->dst_path, strerrno());
 		return -1;
+	}
 
-	mscp_close(h);
+	mscp_close(f);
 
         return 0;
 }
@@ -518,16 +506,16 @@ static int copy_chunk_r2l(struct chunk *c, sftp_file sf, int fd,
         return 0;
 }
 
-static int _copy_chunk(struct chunk *c, mfh s, mfh d,
+static int _copy_chunk(struct chunk *c, mf *s, mf *d,
 		       int nr_ahead, int buf_sz, size_t *counter)
 {
-	if (s.fd > 0 && d.sf) /* local to remote copy */
-		return copy_chunk_l2r(c, s.fd, d.sf, nr_ahead, buf_sz, counter);
-	else if (s.sf && d.fd > 0)  /* remote to local copy */
-		return copy_chunk_r2l(c, s.sf, d.fd, nr_ahead, buf_sz, counter);
+	if (s->local && d->remote) /* local to remote copy */
+		return copy_chunk_l2r(c, s->local, d->remote, nr_ahead, buf_sz, counter);
+	else if (s->remote && d->local)  /* remote to local copy */
+		return copy_chunk_r2l(c, s->remote, d->local, nr_ahead, buf_sz, counter);
 
-	assert(true); /* not reached */
-	return -1;
+	assert(false);
+	return -1; /* not reached */
 }
 
 int copy_chunk(FILE *msg_fp, struct chunk *c,
@@ -536,7 +524,7 @@ int copy_chunk(FILE *msg_fp, struct chunk *c,
 {
 	mode_t mode;
 	int flags;
-	mfh s, d;
+	mf *s, *d;
 	int ret;
 
 	assert((src_sftp && !dst_sftp) || (!src_sftp && dst_sftp));
@@ -547,21 +535,33 @@ int copy_chunk(FILE *msg_fp, struct chunk *c,
 	/* open src */
         flags = O_RDONLY;
         mode = S_IRUSR;
-	s = mscp_open(c->p->path, flags, mode, c->off, src_sftp);
-	if (mscp_open_is_failed(s)) {
-		mscp_close(d);
+	s = mscp_open(c->p->path, flags, mode, src_sftp);
+	if (!s) {
+		mscp_set_error("mscp_open: %s: %s", c->p->path, strerrno());
+		return -1;
+	}
+	if (mscp_lseek(s, c->off) < 0) {
+		mscp_set_error("mscp_lseek: %s: %s", c->p->path, strerrno());
 		return -1;
 	}
 
 	/* open dst */
         flags = O_WRONLY;
         mode = S_IRUSR|S_IWUSR;
-	d = mscp_open(c->p->dst_path, flags, mode, c->off, dst_sftp);
-	if (mscp_open_is_failed(d))
+	d = mscp_open(c->p->dst_path, flags, mode, dst_sftp);
+	if (!d) {
+		mscp_close(s);
+		mscp_set_error("mscp_open: %s: %s", c->p->dst_path, strerrno());
 		return -1;
+	}
+	if (mscp_lseek(d, c->off) < 0) {
+		mscp_set_error("mscp_lseek: %s: %s", c->p->dst_path, strerrno());
+		return -1;
+	}
 
 	mpr_debug(msg_fp, "copy chunk start: %s 0x%lx-0x%lx\n",
 		  c->p->path, c->off, c->off + c->len);
+
 	ret = _copy_chunk(c, s, d, nr_ahead, buf_sz, counter);
 
 	mpr_debug(msg_fp, "copy chunk done: %s 0x%lx-0x%lx\n",
@@ -575,7 +575,9 @@ int copy_chunk(FILE *msg_fp, struct chunk *c,
 
 	if (refcnt_dec(&c->p->refcnt) == 0) {
 		c->p->state = FILE_STATE_DONE;
-		mscp_chmod(c->p->dst_path, c->p->mode, dst_sftp);
+		if (mscp_setstat(c->p->dst_path, c->p->mode, c->p->size, dst_sftp) < 0)
+			mpr_err(msg_fp, "failed to chmod and truncate %s: %s\n",
+				c->p->path, strerrno());
 		mpr_info(msg_fp, "copy done: %s\n", c->p->path);
 	}
 

src/path.h

@@ -98,253 +98,4 @@ int copy_chunk(FILE *msg_fp, struct chunk *c,
 /* just print contents. just for debugging */
 void path_dump(struct list_head *path_list);
 
-
-
-/* wrap DIR/dirent and sftp_dir/sftp_attribute. not thread safe */
-struct mscp_dir {
-        DIR *l;
-        sftp_dir r;
-        sftp_session sftp;
-};
-typedef struct mscp_dir mdir;
-
-struct mscp_dirent {
-        struct dirent *l;
-        sftp_attributes r;
-};
-typedef struct mscp_dirent mdirent;
-
-#define mdirent_name(e) ((e->l) ? e->l->d_name : e->r->name)
-#define mdirent_is_dir(e) ((e->l) ?                                     \
-                           (e->l->d_type == DT_DIR) :                   \
-                           (e->r->type == SSH_FILEXFER_TYPE_DIRECTORY))
-#define mdirent_is_null(e) (e->l == NULL && e->r == NULL)
-
-static mdir *mscp_opendir(const char *path, sftp_session sftp)
-{
-        mdir *d;
-
-        if (!(d = malloc(sizeof(*d))))
-                return NULL;
-        memset(d, 0, sizeof(*d));
-
-        d->sftp = sftp;
-
-        if (sftp) {
-                d->r = sftp_opendir(sftp, path);
-                if (!d->r) {
-                        mscp_set_error("sftp_opendir '%s': %s",
-				       path, sftp_get_ssh_error(sftp));
-                        free(d);
-                        return NULL;
-                }
-        } else {
-                d->l = opendir(path);
-                if (!d->l) {
-                        mscp_set_error("opendir '%s': %s", path, strerrno());
-                        free(d);
-                        return NULL;
-                }
-        }
-        return d;
-}
-
-static int mscp_closedir(mdir *d)
-{
-        int ret;
-        if (d->r)
-                ret = sftp_closedir(d->r);
-        else
-                ret = closedir(d->l);
-        free(d);
-        return ret;
-}
-
-static mdirent *mscp_readdir(mdir *d)
-{
-        static mdirent e;
-
-        memset(&e, 0, sizeof(e));
-        if (d->r)
-                e.r = sftp_readdir(d->sftp, d->r);
-        else
-                e.l = readdir(d->l);
-        return &e;
-}
-
-static void mscp_dirent_free(mdirent *e)
-{
-	if (e->r) {
-		sftp_attributes_free(e->r);
-		e->r = NULL;
-	}
-}
-
-/* wrap retriving error */
-static const char *mscp_strerror(sftp_session sftp)
-{
-	if (sftp)
-		return sftp_get_ssh_error(sftp);
-	return strerrno();
-}
-
-/* warp stat/sftp_stat */
-struct mscp_stat {
-        struct stat l;
-        sftp_attributes r;
-};
-typedef struct mscp_stat mstat;
-
-static int mscp_stat(const char *path, mstat *s, sftp_session sftp)
-{
-        memset(s, 0, sizeof(*s));
-
-        if (sftp) {
-                s->r = sftp_stat(sftp, path);
-                if (!s->r) {
-			mscp_set_error("sftp_stat: %s %s",
-				       sftp_get_ssh_error(sftp), path);
-                        return -1;
-		}
-        } else {
-                if (stat(path, &s->l) < 0) {
-			mscp_set_error("stat: %s %s", strerrno(), path);
-                        return -1;
-		}
-        }
-
-        return 0;
-}
-
-static int mscp_stat_check_err_noent(sftp_session sftp)
-{
-	if (sftp) {
-		if (sftp_get_error(sftp) == SSH_FX_NO_SUCH_PATH ||
-		    sftp_get_error(sftp) == SSH_FX_NO_SUCH_FILE)
-			return 0;
-	} else {
-		if (errno == ENOENT)
-			return 0;
-	}
-	return -1;
-}
-
-static void mscp_stat_free(mstat s) {
-        if (s.r)
-                sftp_attributes_free(s.r);
-}
-
-#define mstat_size(s) ((s.r) ? s.r->size : s.l.st_size)
-#define mstat_mode(s) ((s.r) ?                                  \
-                       s.r->permissions :                       \
-                       s.l.st_mode & (S_IRWXU|S_IRWXG|S_IRWXO))
-#define mstat_is_regular(s) ((s.r) ?                                    \
-                             (s.r->type == SSH_FILEXFER_TYPE_REGULAR) : \
-                             S_ISREG(s.l.st_mode))
-#define mstat_is_dir(s) ((s.r) ?                        \
-                         (s.r->type == SSH_FILEXFER_TYPE_DIRECTORY) :   \
-                         S_ISDIR(s.l.st_mode))
-
-/* wrap mkdir */
-static int mscp_mkdir(const char *path, mode_t mode, sftp_session sftp)
-{
-	int ret;
-
-	if (sftp) {
-		ret = sftp_mkdir(sftp, path, mode);
-		if (ret < 0 &&
-		    sftp_get_error(sftp) != SSH_FX_FILE_ALREADY_EXISTS) {
-			mscp_set_error("sftp_mkdir '%s': %s",
-				       path, sftp_get_ssh_error(sftp));
-			return -1;
-		}
-	} else {
-		if (mkdir(path, mode) == -1 && errno != EEXIST) {
-			mscp_set_error("mkdir '%s': %s", path, strerrno());
-			return -1;
-		}
-	}
-
-	return 0;
-}
-
-/* wrap open/sftp_open */
-struct mscp_file_handle {
-	int fd;
-	sftp_file sf;
-};
-typedef struct mscp_file_handle mfh;
-
-static mfh mscp_open(const char *path, int flags, mode_t mode, size_t off,
-		      sftp_session sftp)
-{
-	mfh h;
-
-	h.fd = -1;
-	h.sf = NULL;
-
-	if (sftp) {
-		h.sf = sftp_open(sftp, path, flags, mode);
-		if (!h.sf) {
-			mscp_set_error("sftp_open '%s': %s",
-				       path, sftp_get_ssh_error(sftp));
-			return h;
-		}
-
-		if (sftp_seek64(h.sf, off) < 0) {
-			mscp_set_error("sftp_seek64 '%s': %s",
-				       path, sftp_get_ssh_error(sftp));
-			sftp_close(h.sf);
-			h.sf = NULL;
-			return h;
-		}
-	} else {
-		h.fd = open(path, flags, mode);
-		if (h.fd < 0) {
-			mscp_set_error("open '%s': %s", path, strerrno());
-			return h;
-		}
-		if (lseek(h.fd, off, SEEK_SET) < 0) {
-			mscp_set_error("lseek '%s': %s", path, strerrno());
-			close(h.fd);
-			h.fd = -1;
-			return h;
-		}
-	}
-
-	return h;
-}
-
-#define mscp_open_is_failed(h) (h.fd < 0 && h.sf == NULL)
-
-static void mscp_close(mfh h)
-{
-	if (h.sf)
-		sftp_close(h.sf);
-	if (h.fd > 0)
-		close(h.fd);
-	h.sf = NULL;
-	h.fd = -1;
-}
-
-/* wrap chmod/sftp_chmod */
-
-static int mscp_chmod(const char *path, mode_t mode, sftp_session sftp)
-{
-	if (sftp) {
-		if (sftp_chmod(sftp, path, mode) < 0)  {
-			mscp_set_error("sftp_chmod '%s': %s",
-				       path, sftp_get_ssh_error(sftp));
-			return -1;
-		}
-	} else {
-		if (chmod(path, mode) < 0) {
-			mscp_set_error("chmod '%s': %s", path, strerrno());
-			return -1;
-		}
-	}
-
-	return 0;
-}
-
 #endif /* _PATH_H_ */

src/pymscp.c

@@ -103,6 +103,7 @@ static PyObject *wrap_mscp_init(PyObject *self, PyObject *args, PyObject *kw)
 		/*  mscp_ssh_opts */
 		"login_name",	/* const char * */
 		"port",		/* const char * */
+		"config",	/* const char * */
 		"identity",	/* const char * */
 
 		"cipher",	/* const char * */
@@ -116,9 +117,9 @@ static PyObject *wrap_mscp_init(PyObject *self, PyObject *args, PyObject *kw)
 		"enable_nagle",		/* bool */
 		NULL,
 	};
-	const char *fmt = "si" "|" "ii" "kkk" "s" "iii" "sss" "sssss" "ipp";
+	const char *fmt = "si" "|" "ii" "kkk" "s" "iii" "ssss" "sssss" "ipp";
 	char *coremask = NULL;
-	char *login_name = NULL, *port = NULL, *identity = NULL;
+	char *login_name = NULL, *port = NULL, *config = NULL, *identity = NULL;
 	char *cipher = NULL, *hmac = NULL, *compress = NULL;
 	char *password = NULL, *passphrase = NULL;
 
@@ -148,6 +149,7 @@ static PyObject *wrap_mscp_init(PyObject *self, PyObject *args, PyObject *kw)
 					  &i->mo.msg_fd,
 					  &login_name,
 					  &port,
+					  &config,
 					  &identity,
 					  &cipher,
 					  &hmac,
@@ -167,6 +169,8 @@ static PyObject *wrap_mscp_init(PyObject *self, PyObject *args, PyObject *kw)
 		strncpy(i->so.login_name, login_name, MSCP_SSH_MAX_LOGIN_NAME - 1);
 	if (port)
 		strncpy(i->so.port, port, MSCP_SSH_MAX_PORT_STR - 1);
+	if (config)
+		strncpy(i->so.config, config, PATH_MAX - 1);
 	if (identity)
 		strncpy(i->so.identity, identity, MSCP_SSH_MAX_IDENTITY_PATH - 1);
 	if (cipher)
@@ -375,7 +379,7 @@ static PyObject *wrap_mscp_get_stats(PyObject *self, PyObject *args, PyObject *k
 
 	mscp_get_stats(m, &s);
 
-	return Py_BuildValue("KKd", s.total, s.done, s.finished);
+	return Py_BuildValue("KKO", s.total, s.done, PyBool_FromLong(s.finished));
 }
 
 static PyObject *wrap_mscp_cleanup(PyObject *self, PyObject *args, PyObject *kw)

src/ssh.c

@@ -73,6 +73,12 @@ static int ssh_set_opts(ssh_session ssh, struct mscp_ssh_opts *opts)
 		}
 	}
 
+	if (is_specified(opts->config) &&
+	    ssh_options_parse_config(ssh, opts->config) < 0) {
+		mscp_set_error("failed to parse ssh_config: %s", opts->config);
+		return -1;
+	}
+
 	return 0;
 }
 
@@ -149,14 +155,14 @@ static ssh_session ssh_init_session(const char *sshdst, struct mscp_ssh_opts *op
 	cb.userdata = opts;
 	ssh_set_callbacks(ssh, &cb);
 
-	if (ssh_set_opts(ssh, opts) != 0)
-		goto free_out;
-
 	if (ssh_options_set(ssh, SSH_OPTIONS_HOST, sshdst) != SSH_OK) {
 		mscp_set_error("failed to set destination host");
 		goto free_out;
 	}
 
+	if (ssh_set_opts(ssh, opts) != 0)
+		goto free_out;
+
 	if (ssh_connect(ssh) != SSH_OK) {
 		mscp_set_error("failed to connect ssh server: %s", ssh_get_error(ssh));
 		goto free_out;

test/test_e2e.py

@@ -11,11 +11,15 @@ from util import File, check_same_md5sum
 
 
 def run2ok(args):
-    check_call(list(map(str, args)))
+    cmd = list(map(str, args))
+    print("cmd: {}".format(" ".join(cmd)))
+    check_call(cmd)
 
 def run2ng(args):
+    cmd = list(map(str, args))
+    print("cmd: {}".format(" ".join(cmd)))
     with pytest.raises(CalledProcessError) as e:
-        check_call(list(map(str, args)))
+        check_call(cmd)
 
 
 """ usage test """
@@ -56,7 +60,7 @@ param_single_copy = [
 @pytest.mark.parametrize("src, dst", param_single_copy)
 def test_single_copy(mscp, src_prefix, dst_prefix, src, dst):
     src.make()
-    run2ok([mscp, "-H", src_prefix + src.path, dst_prefix + dst.path])
+    run2ok([mscp, "-H", "-vvv", src_prefix + src.path, dst_prefix + dst.path])
     assert check_same_md5sum(src, dst)
     src.cleanup()
     dst.cleanup()
@@ -65,7 +69,7 @@ def test_single_copy(mscp, src_prefix, dst_prefix, src, dst):
 def test_failed_to_copy_nonexistent_file(mscp, src_prefix, dst_prefix):
     src = "nonexistent_src"
     dst = "nonexistent_dst"
-    run2ng([mscp, "-H", src_prefix + src, dst_prefix + dst])
+    run2ng([mscp, "-H", "-vvv", src_prefix + src, dst_prefix + dst])
 
 param_double_copy = [
     (File("src1", size = 1024 * 1024), File("src2", size = 1024 * 1024),
@@ -77,7 +81,7 @@ param_double_copy = [
 def test_double_copy(mscp, src_prefix, dst_prefix, s1, s2, d1, d2):
     s1.make()
     s2.make()
-    run2ok([mscp, "-H", src_prefix + s1.path, src_prefix + s2.path, dst_prefix + "dst"])
+    run2ok([mscp, "-H", "-vvv", src_prefix + s1.path, src_prefix + s2.path, dst_prefix + "dst"])
     assert check_same_md5sum(s1, d1)
     assert check_same_md5sum(s2, d2)
     s1.cleanup()
@@ -114,11 +118,11 @@ def test_dir_copy(mscp, src_prefix, dst_prefix, src_dir, dst_dir, src, dst, twic
     for f in src:
         f.make()
 
-    run2ok([mscp, "-H", src_prefix + src_dir, dst_prefix + dst_dir])
+    run2ok([mscp, "-H", "-vvv", src_prefix + src_dir, dst_prefix + dst_dir])
     for sf, df in zip(src, dst):
         assert check_same_md5sum(sf, df)
 
-    run2ok([mscp, "-H", src_prefix + src_dir, dst_prefix + dst_dir])
+    run2ok([mscp, "-H", "-vvv", src_prefix + src_dir, dst_prefix + dst_dir])
     for sf, df in zip(src, twice):
         assert check_same_md5sum(sf, df)
 
@@ -127,13 +131,30 @@ def test_dir_copy(mscp, src_prefix, dst_prefix, src_dir, dst_dir, src, dst, twic
         df.cleanup()
         tf.cleanup()
 
+
+param_dir_copy_single = [
+    ("src_dir", "dst_dir",
+     File("src_dir/t1", size = 1024 * 1024),
+     File("dst_dir/src_dir/t1"),
+     )
+]
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+@pytest.mark.parametrize("src_dir, dst_dir, src, dst", param_dir_copy_single)
+def test_dir_copy_single(mscp, src_prefix, dst_prefix, src_dir, dst_dir, src, dst):
+    src.make()
+    os.mkdir(dst_dir)
+    run2ok(["mscp", "-H", "-vvv", src_prefix + src_dir, dst_prefix + dst_dir])
+    assert check_same_md5sum(src, dst)
+    src.cleanup()
+    dst.cleanup()
+
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 def test_override_single_file(mscp, src_prefix, dst_prefix):
     src = File("src", size = 128).make()
     dst = File("dst", size = 128).make()
     assert not check_same_md5sum(src, dst)
 
-    run2ok([mscp, "-H", src_prefix + src.path, dst_prefix + dst.path])
+    run2ok([mscp, "-H", "-vvv", src_prefix + src.path, dst_prefix + dst.path])
     assert check_same_md5sum(src, dst)
 
     src.cleanup()
@@ -144,18 +165,55 @@ def test_min_chunk(mscp, src_prefix, dst_prefix):
     src = File("src", size = 16 * 1024).make()
     dst = File("dst")
 
-    run2ok([mscp, "-H", "-s", 32768, src_prefix + src.path, dst_prefix + dst.path])
+    run2ok([mscp, "-H", "-vvv", "-s", 32768, src_prefix + src.path, dst_prefix + dst.path])
     assert check_same_md5sum(src, dst)
 
     src.cleanup()
     dst.cleanup()
 
+
+def is_alpine():
+    if os.path.exists("/etc/os-release"):
+        with open("/etc/os-release", "r") as f:
+            for line in f:
+                if line.strip() == "ID=alpine":
+                    return True
+    return False
+
+param_glob_copy = [
+    (
+        "src*", "dstx",
+        [ File("src1"), File("src2"), File("src3") ],
+        [ File("dstx/src1"), File("dstx/src2"), File("dstx/src3") ],
+    ),
+    (
+        "src*", "dstx",
+        [ File("src1/s1"), File("src2/s2"), File("src3/s3") ],
+        [ File("dstx/s1"), File("dstx/s2"), File("dstx/s3") ],
+    )
+]
+
+@pytest.mark.skipif(is_alpine(),
+                    reason = "musl does not implement glob ALTDIRFUNC")
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+@pytest.mark.parametrize("src_glob_path, dst_path, srcs, dsts", param_glob_copy)
+def test_glob_src_path(mscp, src_prefix, dst_prefix,
+                       src_glob_path, dst_path, srcs, dsts):
+    for src in srcs:
+        src.make(size = 1024 * 1024)
+
+    run2ok([mscp, "-H", "-vvv", src_prefix + src_glob_path, dst_prefix + dst_path])
+    for src, dst in zip(srcs, dsts):
+        assert check_same_md5sum(src, dst)
+        src.cleanup()
+        dst.cleanup()
+
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 def test_thread_affinity(mscp, src_prefix, dst_prefix):
     src = File("src", size = 64 * 1024).make()
     dst = File("dst")
 
-    run2ok([mscp, "-H", "-n", 4, "-m", "0x01",
+    run2ok([mscp, "-H", "-vvv", "-n", 4, "-m", "0x01",
             src_prefix + src.path, dst_prefix + dst.path])
     assert check_same_md5sum(src, dst)
 
@@ -167,7 +225,7 @@ def test_cannot_override_file_with_dir(mscp, src_prefix, dst_prefix):
     src = File("src", size = 128).make()
     dst = File("dst").make()
 
-    run2ng([mscp, "-H", src_prefix + src.path, dst_prefix + "dst/src"])
+    run2ng([mscp, "-H", "-vvv", src_prefix + src.path, dst_prefix + "dst/src"])
 
     src.cleanup()
     dst.cleanup()
@@ -176,7 +234,7 @@ def test_cannot_override_file_with_dir(mscp, src_prefix, dst_prefix):
 def test_transfer_zero_bytes(mscp, src_prefix, dst_prefix):
     src = File("src", size = 0).make()
     dst = File("dst")
-    run2ok([mscp, "-H", src_prefix + src.path, dst_prefix + "dst"])
+    run2ok([mscp, "-H", "-vvv", src_prefix + src.path, dst_prefix + "dst"])
     assert os.path.exists("dst")
     src.cleanup()
     dst.cleanup()
@@ -185,18 +243,65 @@ def test_transfer_zero_bytes(mscp, src_prefix, dst_prefix):
 def test_override_dst_having_larger_size(mscp, src_prefix, dst_prefix):
     src = File("src", size = 1024 * 1024).make()
     dst = File("dst", size = 1024 * 1024 * 2).make()
-    run2ok([mscp, "-H", src_prefix + src.path, dst_prefix + "dst"])
+    run2ok([mscp, "-H", "-vvv", src_prefix + src.path, dst_prefix + "dst"])
     assert check_same_md5sum(src, dst)
     src.cleanup()
     dst.cleanup()
 
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_dont_truncate_dst(mscp, src_prefix, dst_prefix):
+    f = File("srcanddst", size = 1024 * 1024 * 128).make()
+    md5_before = f.md5sum()
+    run2ok([mscp, "-H", "-vvv", src_prefix + f.path, dst_prefix + f.path])
+    md5_after = f.md5sum()
+    assert md5_before == md5_after
+    f.cleanup()
+
 compressions = ["yes", "no", "none"]
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 @pytest.mark.parametrize("compress", compressions)
 def test_compression(mscp, src_prefix, dst_prefix, compress):
     src = File("src", size = 1024 * 1024).make()
     dst = File("dst", size = 1024 * 1024 * 2).make()
-    run2ok([mscp, "-H", "-C", compress, src_prefix + src.path, dst_prefix + "dst"])
+    run2ok([mscp, "-H", "-vvv", "-C", compress, src_prefix + src.path, dst_prefix + "dst"])
     assert check_same_md5sum(src, dst)
     src.cleanup()
     dst.cleanup()
+
+
+testhost = "mscptestlocalhost"
+testhost_prefix = "{}:{}/".format(testhost, os.getcwd()) # use current dir
+param_testhost_prefix = [
+    ("", testhost_prefix), (testhost_prefix, "")
+]
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_testhost_prefix)
+def test_config_ok(mscp, src_prefix, dst_prefix):
+    config = "/tmp/mscp_test_ssh_config"
+    with open(config, "w") as f:
+        f.write("host {}\n".format(testhost))
+        f.write("    hostname localhost\n")
+
+    src = File("src", size = 1024 * 1024).make()
+    dst = File("dst", size = 1024 * 1024 * 2).make()
+    run2ok([mscp, "-H", "-vvv", "-F", config,
+            src_prefix + src.path, dst_prefix + "dst"])
+
+    os.remove(config)
+    assert check_same_md5sum(src, dst)
+    src.cleanup()
+    dst.cleanup()
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_testhost_prefix)
+def test_config_ng(mscp, src_prefix, dst_prefix):
+    config = "/tmp/mscp_test_ssh_config"
+    with open(config, "w") as f:
+        f.write("\n") # use empty ssh_config
+
+    src = File("src", size = 1024 * 1024).make()
+    dst = File("dst", size = 1024 * 1024 * 2).make()
+    run2ng([mscp, "-H", "-vvv", "-F", config,
+            src_prefix + src.path, dst_prefix + "dst"])
+
+    os.remove(config)
+    src.cleanup()
+    dst.cleanup()

test/util.py

@@ -22,7 +22,10 @@ class File():
     def __str__(self):
         return self.path
 
-    def make(self):
+    def make(self, size = None):
+        if size:
+            self.size = size
+
         d = os.path.dirname(self.path)
         if d:
             os.makedirs(d, exist_ok = True)