bump version to 0.1.5

not to /etc/ssh/sshd_config.d/*.conf because openssh in Rocky Linux in 8
does not support the config.dy directory.

.clang-format

@@ -689,6 +689,8 @@ ForEachMacros:
   - 'xbc_node_for_each_key_value'
   - 'xbc_node_for_each_subkey'
   - 'zorro_for_each_dev'
+  - 'pool_iter_for_each'
+  - 'pool_for_each'
 
 IncludeBlocks: Preserve
 IncludeCategories:

CMakeLists.txt

@@ -51,6 +51,13 @@ if (BUILD_STATIC)
 	endif()
 endif()
 
+option(USE_PODMAN OFF) # use podman instread of docker
+if(USE_PODMAN)
+	message(STATUS "Use podman instead of docker")
+	set(CE podman) # CE means Container Engine
+else()
+	set(CE docker)
+endif()
 
 
 # add libssh static library
@@ -84,11 +91,14 @@ endif()
 
 
 # Symbol check
-check_symbol_exists(strlcat string.h HAVE_STRLCAT)
+check_symbol_exists(htonll	arpa/inet.h	HAVE_HTONLL)
+check_symbol_exists(ntohll	arpa/inet.h	HAVE_NTOHLL)
+check_symbol_exists(strlcat	string.h	HAVE_STRLCAT)
 if (NOT HAVE_STRLCAT)
 	list(APPEND OPENBSD_COMPAT_SRC src/openbsd-compat/strlcat.c)
 endif()
 
+
 # generate config.h in build dir
 configure_file(
 	${mscp_SOURCE_DIR}/include/config.h.in
@@ -98,8 +108,8 @@ list(APPEND MSCP_BUILD_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/include)
 
 # libmscp.a
 set(LIBMSCP_SRC
-	src/mscp.c src/ssh.c src/fileops.c src/path.c src/platform.c
-	src/print.c src/strerrno.c
+	src/mscp.c src/ssh.c src/fileops.c src/path.c src/checkpoint.c
+	src/platform.c src/print.c src/pool.c src/strerrno.c
 	${OPENBSD_COMPAT_SRC})
 add_library(mscp-static STATIC ${LIBMSCP_SRC})
 target_include_directories(mscp-static
@@ -155,40 +165,45 @@ enable_testing()
 # Custom targets to build and test mscp in docker containers.
 # foreach(IN ZIP_LISTS) (cmake >= 3.17) can shorten the following lists.
 # However, ubuntu 20.04 has cmake 3.16.3. So this is a roundabout trick.
-list(APPEND DIST_NAMES	ubuntu	ubuntu	rocky	rocky	almalinux	alpine)
+list(APPEND DIST_IDS	ubuntu	ubuntu	rocky	rocky	almalinux	alpine)
 list(APPEND DIST_VERS	20.04	22.04	8.9	9.3	9.3		3.19)
 list(APPEND DIST_PKGS	deb	deb	rpm	rpm	rpm		static)
 
-list(LENGTH DIST_NAMES _DIST_LISTLEN)
+list(LENGTH DIST_IDS _DIST_LISTLEN)
 math(EXPR DIST_LISTLEN "${_DIST_LISTLEN} - 1")
 
 foreach(x RANGE ${DIST_LISTLEN})
-	list(GET DIST_NAMES	${x} DIST_NAME)
+	list(GET DIST_IDS	${x} DIST_ID)
 	list(GET DIST_VERS	${x} DIST_VER)
 	list(GET DIST_PKGS	${x} DIST_PKG)
 
-	set(DOCKER_IMAGE mscp-${DIST_NAME}:${DIST_VER})
-	set(DOCKER_INDEX ${DIST_NAME}-${DIST_VER})
-	set(PKG_FILE_NAME
-		mscp_${DIST_NAME}-${DIST_VER}-${ARCH}.${DIST_PKG})
+	set(DOCKER_IMAGE mscp-${DIST_ID}:${DIST_VER})
+	set(DOCKER_INDEX ${DIST_ID}-${DIST_VER})
+	execute_process(
+		COMMAND ${CMAKE_SOURCE_DIR}/scripts/install-build-deps.sh
+		--dont-install --platform Linux-${DIST_ID}
+		OUTPUT_VARIABLE REQUIREDPKGS
+		OUTPUT_STRIP_TRAILING_WHITESPACE)
 
 	add_custom_target(docker-build-${DOCKER_INDEX}
 		COMMENT "Build mscp in ${DOCKER_IMAGE} container"
 		WORKING_DIRECTORY ${mscp_SOURCE_DIR}
 		COMMAND
-		docker build -t ${DOCKER_IMAGE} -f Dockerfile/${DOCKER_INDEX}.Dockerfile .)
+		${CE} build --build-arg REQUIREDPKGS=${REQUIREDPKGS}
+		-t ${DOCKER_IMAGE} -f Dockerfile/${DOCKER_INDEX}.Dockerfile .)
 
 	add_custom_target(docker-build-${DOCKER_INDEX}-no-cache
 		COMMENT "Build mscp in ${DOCKER_IMAGE} container"
 		WORKING_DIRECTORY ${mscp_SOURCE_DIR}
 		COMMAND
-		docker build --no-cache -t ${DOCKER_IMAGE} -f Dockerfile/${DOCKER_INDEX}.Dockerfile .)
+		${CE} build --build-arg REQUIREDPKGS=${REQUIREDPKGS} --no-cache
+		-t ${DOCKER_IMAGE} -f Dockerfile/${DOCKER_INDEX}.Dockerfile .)
 
 	add_custom_target(docker-test-${DOCKER_INDEX}
 		COMMENT "Test mscp in ${DOCKER_IMAGE} container"
 		WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
 		COMMAND
-		docker run --init --rm --sysctl net.ipv6.conf.all.disable_ipv6=0
+		${CE} run --init --rm --sysctl net.ipv6.conf.all.disable_ipv6=0
 		${DOCKER_IMAGE} /mscp/scripts/test-in-container.sh)
 
 	list(APPEND DOCKER_BUILDS		docker-build-${DOCKER_INDEX})
@@ -204,14 +219,21 @@ add_custom_target(docker-test-all		DEPENDS ${DOCKER_TESTS})
 ### debuild-related definitions
 
 set(DEBBUILDCONTAINER mscp-build-deb)
+execute_process(
+	COMMAND ${CMAKE_SOURCE_DIR}/scripts/install-build-deps.sh
+	--dont-install --platform Linux-ubuntu
+	OUTPUT_VARIABLE REQUIREDPKGS_DEB
+	OUTPUT_STRIP_TRAILING_WHITESPACE)
+
 add_custom_target(build-deb
 	COMMENT "build mscp deb files inside a container"
 	WORKING_DIRECTORY ${mscp_SOURCE_DIR}
 	BYPRODUCTS ${CMAKE_BINARY_DIR}/debbuild
 	COMMAND
-	docker build -t ${DEBBUILDCONTAINER} -f Dockerfile/build-deb.Dockerfile .
+	${CE} build --build-arg REQUIREDPKGS=${REQUIREDPKGS_DEB}
+	-t ${DEBBUILDCONTAINER} -f Dockerfile/build-deb.Dockerfile .
 	COMMAND
-	docker run --rm -v ${CMAKE_BINARY_DIR}:/out ${DEBBUILDCONTAINER}
+	${CE} run --rm -v ${CMAKE_BINARY_DIR}:/out ${DEBBUILDCONTAINER}
 	cp -r /debbuild /out/)
 
 
@@ -222,22 +244,30 @@ configure_file(
 	${mscp_SOURCE_DIR}/rpm/mscp.spec.in
 	${mscp_SOURCE_DIR}/rpm/mscp.spec
 	@ONLY)
-configure_file(
-	${mscp_SOURCE_DIR}/Dockerfile/build-srpm.Dockerfile.in
-	${mscp_SOURCE_DIR}/Dockerfile/build-srpm.Dockerfile
-	@ONLY)
+#configure_file(
+#	${mscp_SOURCE_DIR}/Dockerfile/build-srpm.Dockerfile.in
+#	${mscp_SOURCE_DIR}/Dockerfile/build-srpm.Dockerfile
+#	@ONLY)
 
 # Custom target to build mscp as a src.rpm in docker.
 set(RPMBUILDCONTAINER mscp-build-srpm)
 set(SRPMFILE mscp-${MSCP_VERSION}-1.el9.src.rpm)
+execute_process(
+	COMMAND ${CMAKE_SOURCE_DIR}/scripts/install-build-deps.sh
+	--dont-install --platform Linux-rocky
+	OUTPUT_VARIABLE REQUIREDPKGS_RPM
+	OUTPUT_STRIP_TRAILING_WHITESPACE)
+
 add_custom_target(build-srpm
 	COMMENT "Build mscp src.rpm inside a container"
 	WORKING_DIRECTORY ${mscp_SOURCE_DIR}
 	BYPRODUCTS ${CMAKE_BINARY_DIR}/${SRPMFILE}
 	COMMAND
-	docker build -t ${RPMBUILDCONTAINER} -f Dockerfile/build-srpm.Dockerfile .
+	${CE} build --build-arg REQUIREDPKGS=${REQUIREDPKGS_RPM}
+	--build-arg MSCP_VERSION=${MSCP_VERSION}
+	-t ${RPMBUILDCONTAINER} -f Dockerfile/build-srpm.Dockerfile .
 	COMMAND
-	docker run --rm -v ${CMAKE_BINARY_DIR}:/out ${RPMBUILDCONTAINER}
+	${CE} run --rm -v ${CMAKE_BINARY_DIR}:/out ${RPMBUILDCONTAINER}
 	cp /root/rpmbuild/SRPMS/${SRPMFILE} /out/)
 
 ### single-binary-build-related definitions
@@ -250,7 +280,7 @@ add_custom_target(build-single-binary
 	BYPRODUCTS ${CMAKE_BINARY_DIR}/${SINGLEBINARYFILE}
 	DEPENDS docker-build-alpine-3.19
 	COMMAND
-	docker run --rm -v ${CMAKE_BINARY_DIR}:/out mscp-alpine:3.19
+	${CE} run --rm -v ${CMAKE_BINARY_DIR}:/out mscp-alpine:3.19
 	cp /mscp/build/mscp /out/${SINGLEBINARYFILE})
 
 

Dockerfile/.gitignore

@@ -1,3 +0,0 @@
-
-# generated by cmake
-build-srpm.Dockerfile

Dockerfile/README.md

@@ -1,19 +1,7 @@
 
 Dockerfiles for building and testing mscp.
 
-Build container:
-
-```
-docker build -t mscp-DIST:VER -f docker/DIST-VER.Dockerfile .
-```
-
-Run test:
-
-```
-docker run --init --rm mscp-DST:VER /mscp/scripts/test-in-container.sh
-```
-
-Custom targets to build and test mscp in the containers are provided
-via `cmake`.  See `make docker-*` targets. `make docker-build-all`
-builds all container images, and `make docker-test-all` runs the test
-in all container images.
+cmake provides custom targets to build and test mscp in the containers
+See `make docker-*` targets. `make docker-build-all` builds all
+container images, and `make docker-test-all` runs the test in all
+container images.

Dockerfile/almalinux-9.3.Dockerfile

@@ -1,8 +1,11 @@
 FROM almalinux:9.3
 
+ARG REQUIREDPKGS
+
 # install pytest, sshd for test, and rpm-build
 RUN set -ex && yum -y install \
-	python3 python3-pip python3-devel openssh openssh-server openssh-clients rpm-build
+	${REQUIREDPKGS} python3 python3-pip python3-devel \
+	openssh openssh-server openssh-clients rpm-build
 
 RUN python3 -m pip install pytest
 
@@ -28,9 +31,6 @@ ARG mscpdir="/mscp"
 
 COPY . ${mscpdir}
 
-# install build dependency
-RUN ${mscpdir}/scripts/install-build-deps.sh
-
 # build
 RUN cd ${mscpdir}			\
         && rm -rf build			\

Dockerfile/alpine-3.19.Dockerfile

@@ -1,10 +1,15 @@
 FROM alpine:3.19
 
+# do not use REQUIREDPKGS build argument because
+# this Dockerfile compiles mscp with conan,so we do not need
+# libssl-dev and zlib-dev
+
 # Build mscp with conan to create single binary mscp
 
 RUN apk add --no-cache \
-	gcc make cmake python3 py3-pip perl linux-headers libc-dev	\
-	openssh bash python3-dev py3-pytest g++
+	gcc make cmake libc-dev \
+	linux-headers openssh bash perl \
+	python3 py3-pip python3-dev py3-pytest g++
 
 RUN pip3 install --break-system-packages conan
 

Dockerfile/build-deb.Dockerfile

@@ -1,16 +1,16 @@
 FROM ubuntu:22.04
 
+ARG REQUIREDPKGS
+
 ARG DEBIAN_FRONTEND=noninteractive
 RUN set -ex && apt-get update && apt-get install -y --no-install-recommends \
-	ca-certificates build-essential devscripts debhelper gcc make cmake
+	${REQUIREDPKGS} ca-certificates \
+	build-essential devscripts debhelper gcc make cmake
 
 ARG mscpdir="/debbuild/mscp"
 
 COPY . ${mscpdir}
 
-# install build dependency
-RUN ${mscpdir}/scripts/install-build-deps.sh
-
 # build
 RUN cd ${mscpdir} 	\
 	&& debuild -us -uc -S \

Dockerfile/build-srpm.Dockerfile

@@ -1,16 +1,16 @@
 FROM rockylinux:9
 
-# install pytest, sshd for test, and rpm-build
-RUN set -ex && yum -y install rpm-build rpmdevtools
+ARG REQUIREDPKGS
+ARG MSCP_VERSION
 
-ARG mscpdir="/mscp-@MSCP_VERSION@"
-ARG mscptgz="mscp-@MSCP_VERSION@.tar.gz"
+# install pytest, sshd for test, and rpm-build
+RUN set -ex && yum -y install ${REQUIREDPKGS} rpm-build rpmdevtools
+
+ARG mscpdir="/mscp-${MSCP_VERSION}"
+ARG mscptgz="mscp-${MSCP_VERSION}.tar.gz"
 
 COPY . ${mscpdir}
 
-# install build dependency
-RUN ${mscpdir}/scripts/install-build-deps.sh
-
 # prepare rpmbuild
 RUN rpmdev-setuptree \
 	&& rm -rf ${mscpdir}/build	\

Dockerfile/rocky-8.9.Dockerfile

@@ -1,8 +1,12 @@
 FROM rockylinux:8.9
 
+ARG REQUIREDPKGS
+
 # install pytest, sshd for test, and rpm-build
 RUN set -ex && yum -y install \
-	python3 python3-pip python3-devel openssh openssh-server openssh-clients rpm-build
+	${REQUIREDPKGS}	\
+	python3 python3-pip python3-devel \
+	openssh openssh-server openssh-clients rpm-build
 
 RUN python3 -m pip install pytest
 
@@ -29,9 +33,6 @@ ARG mscpdir="/mscp"
 
 COPY . ${mscpdir}
 
-# install build dependency
-RUN ${mscpdir}/scripts/install-build-deps.sh
-
 # build
 RUN cd ${mscpdir}			\
         && rm -rf build			\

Dockerfile/rocky-9.3.Dockerfile

@@ -1,8 +1,12 @@
 FROM rockylinux:9.3
 
+ARG REQUIREDPKGS
+
 # install pytest, sshd for test, and rpm-build
 RUN set -ex && yum -y install \
-	python3 python3-pip python3-devel openssh openssh-server openssh-clients rpm-build
+	${REQUIREDPKGS}	\
+	python3 python3-pip python3-devel \
+	openssh openssh-server openssh-clients rpm-build
 
 RUN python3 -m pip install pytest
 
@@ -28,9 +32,6 @@ ARG mscpdir="/mscp"
 
 COPY . ${mscpdir}
 
-# install build dependency
-RUN ${mscpdir}/scripts/install-build-deps.sh
-
 # build
 RUN cd ${mscpdir}			\
         && rm -rf build			\

Dockerfile/ubuntu-20.04.Dockerfile

@@ -1,16 +1,13 @@
 FROM ubuntu:20.04
 
+ARG REQUIREDPKGS
+
 ARG DEBIAN_FRONTEND=noninteractive
 RUN set -ex && apt-get update && apt-get install -y --no-install-recommends \
-	ca-certificates
-
-# install pytest, and sshd for test
-RUN apt-get install -y --no-install-recommends  \
-        python3 python3-pip python3-dev openssh-server
+	${REQUIREDPKGS} ca-certificates python3 python3-pip python3-dev openssh-server
 
 RUN python3 -m pip install pytest
 
-
 # preparation for sshd
 RUN mkdir /var/run/sshd        \
 	&& ssh-keygen -A	\
@@ -31,10 +28,6 @@ ARG mscpdir="/mscp"
 
 COPY . ${mscpdir}
 
-# install build dependency
-RUN ${mscpdir}/scripts/install-build-deps.sh
-
-
 # build
 RUN cd ${mscpdir}			\
 	&& rm -rf build			\

Dockerfile/ubuntu-22.04.Dockerfile

@@ -1,16 +1,13 @@
 FROM ubuntu:22.04
 
+ARG REQUIREDPKGS
+
 ARG DEBIAN_FRONTEND=noninteractive
 RUN set -ex && apt-get update && apt-get install -y --no-install-recommends \
-	ca-certificates
-
-# install pytest, and sshd for test
-RUN apt-get install -y --no-install-recommends  \
-        python3 python3-pip python3-dev openssh-server
+	${REQUIREDPKGS} ca-certificates python3 python3-pip python3-dev openssh-server
 
 RUN python3 -m pip install pytest
 
-
 # preparation for sshd
 RUN mkdir /var/run/sshd        \
 	&& ssh-keygen -A	\
@@ -31,10 +28,6 @@ ARG mscpdir="/mscp"
 
 COPY . ${mscpdir}
 
-# install build dependency
-RUN ${mscpdir}/scripts/install-build-deps.sh
-
-
 # build
 RUN cd ${mscpdir}			\
 	&& rm -rf build			\

README.md

@@ -31,6 +31,7 @@ Differences from `scp` on usage:
 
 - Remote-to-remote copy is not supported.
 - `-r` option is not needed to transfer directories.
+- Checkpointing for resuming failed transfer is supported.
 - and any other differences I have not implemented and noticed.
 
 Paper:
@@ -58,7 +59,7 @@ sudo dnf install mscp
 
 - Single binary `mscp` for x86_64 (not optimal performance)
 ```console
-wget https://github.com/upa/mscp/releases/latest/download/mscp.linux.x86.static -O /usr/local/bin/mscp
+wget https://github.com/upa/mscp/releases/latest/download/mscp.linux.x86_64.static -O /usr/local/bin/mscp
 chmod 755 /usr/local/bin/mscp
 ```
 
@@ -107,4 +108,4 @@ of libssh. So you can start from cmake with it.
 
 ## Documentation
 
-[manpage](/doc/mscp.rst) is available.
+[manpage](/doc/mscp.rst) is available.

VERSION

@@ -1 +1 @@
-0.1.4
+0.1.5

debian/changelog

@@ -1,4 +1,13 @@
-mscp (0.1.4) UNRELEASED; urgency=medium
+mscp (0.1.5) UNRELEASED; urgency=medium
+
+  * add support for resuming failed transfer (#5 and #10)
+  * remove the list structure derived from the linux kernel and refactoring
+    for this change.
+  * add and fix test cases (changing port number and number of connections)
+
+ -- Ryo Nakamura <upa@haeena.net>  Thu, 14 Mar 2024 12:51:23 +0900
+
+mscp (0.1.4) unstable; urgency=medium
 
   * add a test for builds on FreeBSD
   * updat container images for tests

doc/RELEASE.md

@@ -24,7 +24,7 @@ mscp_0.1.4_source.build  mscp_0.1.4_source.changes
 
 1. write changes in `debian/changelog` at main branch (the date command needed here is `date -R`)
 2. switch to `ppa-focal` or `ppa-jammy` branch
-3. reblase to the `main` branch and modify `debian/changes`:
+3. rebase to the `main` branch and modify `debian/changes`:
    * change `UNRELEASED` to the release name (`focal` or `jammy`).
 4. run `make build-deb` at the build directory and `cd debbuild`
 5. sign the files with `debsign -k [GPGKEYID] mscp_X.X.X~X_source.changes`
@@ -52,7 +52,8 @@ mscp-0.1.3-1.el9.src.rpm
 
 ### To publish mscp in COPR:
 
-1. update `changelog` in `rpm/mscp.spec` (the date command needed here is `date "+%a %b %d %Y"`)
+1. update `rpm/mscp.spec.in`, the `changelog` section (the date
+   command needed here is `date "+%a %b %d %Y"`)
 2. run `make build-srpm`
 3. download `mscp-X.X.X-1.yyy.src.rpm`
 4. upload the src.rpm to Build page at COPR.

doc/mscp.1.in

@@ -20,6 +20,12 @@ mscp \- copy files over multiple SSH connections
 .BI \-I \ INTERVAL\c
 ]
 [\c
+.BI \-W \ CHECKPOINT\c
+]
+[\c
+.BI \-R \ CHECKPOINT\c
+]
+[\c
 .BI \-s \ MIN_CHUNK_SIZE\c
 ]
 [\c
@@ -131,6 +137,45 @@ single source IP address for a short period as a brute force attack.
 This option inserts intervals between the attempts to avoid being
 determined as an attack. The default value is 0.
 
+.TP
+.B \-W \fICHECKPOINT\fR
+Specifies a checkpoint file to save the state of a failed
+transfer. When transferring fails due to, for example, connection
+disruption or user interrupt,
+.B mscp
+writes the information about the remaining files and chunks to the
+specified checkpoint file.
+.B \-W
+option with
+.B \-D
+(dry-run mode) only writes a checkpoint file and exits.
+
+
+.TP
+.B \-R \fICHECKPOINT\fR
+Specifies a checkpoint file to resume a transfer. When a checkpoint
+file is passed,
+.B mscp
+reads the checkpoint to load a remote host, copy direction, and files
+and their chunks to be transferred. Namely,
+.B mscp
+can resume a past failed transfer from the checkpoint. Resuming with a
+checkpoint does not require
+.I source ... target
+arguments. Other SSH connection options, such as port number and
+config file, should be specified as with the failed run. In addition,
+checkpoint files have file paths as relative paths. Thus, you must run
+.B mscp
+in the same working directory as the failed run. You can see the
+contents of a checkpoint file with the
+.B mscp \-vv \-D \-R CHECKPOINT
+command (Dry-run mode).  Note that the checkpoint file is not
+automatically removed after the resumed transfer ends
+successfully. Users should check the return value of
+.B mscp
+and remove the checkpoint if it returns 0.
+
+
 .TP
 .B \-s \fIMIN_CHUNK_SIZE\fR
 Specifies the minimum chunk size.
@@ -172,10 +217,9 @@ Quiet mode: turns off all outputs.
 .TP
 .B \-D
 Dry-run mode: it scans source files to be copied, calculates chunks,
-and resolves destination file paths. Dry-run mode with
+resolves destination file paths, and exits. Dry-run mode with
 .B -vv
-option enables confirming files to be copied and their destination
-paths.
+option can confirm files to be copied and their destination paths.
 
 .TP
 .B \-r
@@ -306,6 +350,24 @@ Copy a local file and a directory to /tmp at a remote host:
     $ mscp ~/src-file dir1 10.0.0.1:/tmp
 .fi
 
+.PP
+Save a checkpoint if transfer fails:
+
+.nf
+    $ mscp -W mscp.checkpoint many-large-files 10.0.0.1:dst/
+.fi
+
+.PP
+Check the remaining files and chunks, and resume the failed transfer:
+
+.nf
+    # Dump the content of a checkpoint and exit (dry-run mode)
+    $ mscp -vv -D -R mscp.checkpoint
+
+    # resume transferring from the checkpoint
+    $ mscp -R mscp.checkpoint
+.fi
+
 .PP
 In a long fat network, following options might improve performance:
 
@@ -324,7 +386,6 @@ will be faster than the default chacha20-poly1305 cipher, particularly
 on hosts that support AES-NI.
 
 
-
 .SH "SEE ALSO"
 .BR scp (1),
 .BR ssh (1),
@@ -342,9 +403,9 @@ DOI
 .UE .
 
 
-.SH CONTACT INFROMATION
+.SH CONTACT INFORMATION
 .PP
-For pathces, bug reports, or feature requests, please open an issue on
+For patches, bug reports, or feature requests, please open an issue on
 .UR https://\:github\:.com/\:upa/\:mscp
 GitHub
 .UE .

doc/mscp.rst

@@ -2,7 +2,7 @@
 MSCP
 ====
 
-:Date:   v0.1.3-23-ga9c59f7
+:Date:   v0.1.4-28-g0d248c5
 
 NAME
 ====
@@ -14,11 +14,12 @@ SYNOPSIS
 
 **mscp** [**-46vqDpHdNh**] [ **-n**\ *NR_CONNECTIONS* ] [
 **-m**\ *COREMASK* ] [ **-u**\ *MAX_STARTUPS* ] [ **-I**\ *INTERVAL* ] [
+**-W**\ *CHECKPOINT* ] [ **-R**\ *CHECKPOINT* ] [
 **-s**\ *MIN_CHUNK_SIZE* ] [ **-S**\ *MAX_CHUNK_SIZE* ] [
 **-a**\ *NR_AHEAD* ] [ **-b**\ *BUF_SIZE* ] [ **-l**\ *LOGIN_NAME* ] [
 **-P**\ *PORT* ] [ **-F**\ *CONFIG* ] [ **-i**\ *IDENTITY* ] [
-**-c**\ *CIPHER* ] [ **-M**\ *HMAC* ] [ **-C**\ *COMPRESS* ] *source ...
-target*
+**-c**\ *CIPHER* ] [ **-M**\ *HMAC* ] [ **-C**\ *COMPRESS* ] [
+**-g**\ *CONGESTION* ] *source ... target*
 
 DESCRIPTION
 ===========
@@ -69,6 +70,29 @@ OPTIONS
    option inserts intervals between the attempts to avoid being
    determined as an attack. The default value is 0.
 
+**-W CHECKPOINT**
+   Specifies a checkpoint file to save the state of a failed transfer.
+   When transferring fails due to, for example, connection disruption or
+   user interrupt, **mscp** writes the information about the remaining
+   files and chunks to the specified checkpoint file. **-W** option with
+   **-D** (dry-run mode) only writes a checkpoint file and exits.
+
+**-R CHECKPOINT**
+   Specifies a checkpoint file to resume a transfer. When a checkpoint
+   file is passed, **mscp** reads the checkpoint to load a remote host,
+   copy direction, and files and their chunks to be transferred. Namely,
+   **mscp** can resume a past failed transfer from the checkpoint.
+   Resuming with a checkpoint does not require *source ... target*
+   arguments. Other SSH connection options, such as port number and
+   config file, should be specified as with the failed run. In addition,
+   checkpoint files have file paths as relative paths. Thus, you must
+   run **mscp** in the same working directory as the failed run. You can
+   see the contents of a checkpoint file with the **mscp -vv -D -R
+   CHECKPOINT** command (Dry-run mode). Note that the checkpoint file is
+   not automatically removed after the resumed transfer ends
+   successfully. Users should check the return value of **mscp** and
+   remove the checkpoint if it returns 0.
+
 **-s MIN_CHUNK_SIZE**
    Specifies the minimum chunk size. **mscp** divides a file into chunks
    and copies the chunks in parallel.
@@ -101,8 +125,8 @@ OPTIONS
 
 **-D**
    Dry-run mode: it scans source files to be copied, calculates chunks,
-   and resolves destination file paths. Dry-run mode with **-vv** option
-   enables confirming files to be copied and their destination paths.
+   resolves destination file paths, and exits. Dry-run mode with **-vv**
+   option can confirm files to be copied and their destination paths.
 
 **-r**
    No effect. **mscp** copies recursively if a source path is a
@@ -136,6 +160,9 @@ OPTIONS
    Enables compression: yes, no, zlib, zlib@openssh.com. The default is
    none. See `libssh features <https://www.libssh.org/features/>`__.
 
+**-g CONGESTION**
+   Specifies the TCP congestion control algorithm to use (Linux only).
+
 **-p**
    Preserves modification times and access times (file mode bits are
    preserved by default).
@@ -196,6 +223,22 @@ Copy a local file and a directory to /tmp at a remote host:
 
        $ mscp ~/src-file dir1 10.0.0.1:/tmp
 
+Save a checkpoint if transfer fails:
+
+::
+
+       $ mscp -W mscp.checkpoint many-large-files 10.0.0.1:dst/
+
+Check the remaining files and chunks, and resume the failed transfer:
+
+::
+
+       # Dump the content of a checkpoint and exit (dry-run mode)
+       $ mscp -vv -D -R mscp.checkpoint
+
+       # resume transferring from the checkpoint
+       $ mscp -R mscp.checkpoint
+
 In a long fat network, following options might improve performance:
 
 ::
@@ -221,10 +264,10 @@ File Transfer over SSH. In Practice and Experience in Advanced Research
 Computing (PEARC '23). Association for Computing Machinery, New York,
 NY, USA, 320–323. `DOI <https://doi.org/10.1145/3569951.3597582>`__.
 
-CONTACT INFROMATION
+CONTACT INFORMATION
 ===================
 
-For pathces, bug reports, or feature requests, please open an issue on
+For patches, bug reports, or feature requests, please open an issue on
 `GitHub <https://github.com/upa/mscp>`__.
 
 AUTHORS

include/config.h.in

@@ -9,4 +9,10 @@
 /* Define to 1 if you have the strlcat function. */
 #cmakedefine HAVE_STRLCAT 1
 
+/* Define to 1 if you have the htonll function. */
+#cmakedefine HAVE_HTONLL 1
+
+/* Define to 1 if you have the ntohll function. */
+#cmakedefine HAVE_NTOHLL 1
+
 #endif /* _CONFIG_H_ */

include/mscp.h

@@ -16,13 +16,14 @@
  * libmscp is follows:
  *
  * 1. create mscp instance with mscp_init()
- * 2. connect to remote host with mscp_connect()
- * 3. add path to source files with mscp_add_src_path()
- * 4. set path to destination with mscp_set_dst_path()
- * 5. start to scan source files with mscp_scan()
- * 6. start copy with mscp_start()
- * 7. wait for copy finished with mscp_join()
- * 8. cleanup mscp instance with mscp_cleanup() and mscp_free()
+ * 2. set remote host and copy direction with mscp_set_remote()
+ * 3. connect to remote host with mscp_connect()
+ * 4. add path to source files with mscp_add_src_path()
+ * 5. set path to destination with mscp_set_dst_path()
+ * 6. start to scan source files with mscp_scan()
+ * 7. start copy with mscp_start()
+ * 8. wait for copy finished with mscp_join()
+ * 9. cleanup mscp instance with mscp_cleanup() and mscp_free()
  */
 
 #include <stdbool.h>
@@ -45,7 +46,6 @@ struct mscp_opts {
 	int	max_startups;	/** sshd MaxStartups concurrent connections */
 	int     interval;	/** interval between SSH connection attempts */
 	bool	preserve_ts;	/** preserve file timestamps */
-
 	int	severity; 	/** messaging severity. set MSCP_SERVERITY_* */
 };
 
@@ -92,7 +92,6 @@ struct mscp_ssh_opts {
 struct mscp_stats {
 	size_t total;	/** total bytes to be transferred */
 	size_t done;	/** total bytes transferred */
-	bool finished;	/** true when all copy threads finished */
 };
 
 
@@ -102,15 +101,22 @@ struct mscp;
 /**
  * @brief Creates a new mscp instance.
  *
- * @param remote_host	remote host for file transer.
- * @param direction	copy direction, `MSCP_DIRECTION_L2R` or `MSCP_DIRECTION_R2L`
  * @param o		options for configuring mscp.
  * @param s		options for configuring ssh connections.
  *
  * @retrun 		A new mscp instance or NULL on error.
  */
-struct mscp *mscp_init(const char *remote_host, int direction,
-		       struct mscp_opts *o, struct mscp_ssh_opts *s);
+struct mscp *mscp_init(struct mscp_opts *o, struct mscp_ssh_opts *s);
+
+/**
+ * @brief Set remote host and copy direction.
+ *
+ * @param remote_host	remote host for file transer.
+ * @param direction	copy direction, `MSCP_DIRECTION_L2R` or `MSCP_DIRECTION_R2L`
+ *
+ * @return              0 on success, < 0 if an error occured.
+ */
+int mscp_set_remote(struct mscp *m, const char *remote_host, int direction);
 
 /**
  * @brief Connect the first SSH connection. mscp_connect connects to
@@ -128,9 +134,7 @@ int mscp_connect(struct mscp *m);
 
 /**
  * @brief Add a source file path to be copied. The path indicates
- * either a file or directory. The path can be `user@host:path`
- * notation.  In this case, `dst_path` for mscp_set_dst_path() must
- * not contain remote host notation.
+ * either a file or directory.
  *
  * @param m		mscp instance.
  * @param src_path	source file path to be copied.
@@ -141,9 +145,7 @@ int mscp_add_src_path(struct mscp *m, const char *src_path);
 
 /**
  * @brief Set the destination file path. The path indicates either a
- * file, directory, or nonexistent path. The path can be
- * `user@host:path` notation.  In this case, all source paths appended
- * by mscp_set_src_path() must not contain remote host notation.
+ * file, directory, or nonexistent path.
  *
  * @param m		mscp instance.
  * @param dst_path	destination path to which source files copied.
@@ -167,15 +169,50 @@ int mscp_set_dst_path(struct mscp *m, const char *dst_path);
 int mscp_scan(struct mscp *m);
 
 /**
- * @brief Join scna thread invoked by mscp_scan(). mscp_join()
- * involves this, so that mscp_scan_join() should be called when
- * mscp_scan() is called by mscp_start() is not.
+ * @brief Join scan thread invoked by mscp_scan() if it
+ * runs. mscp_join() involves mscp_can_join(). Thus, there is no need
+ * to call this function alone.
  *
  * @param m	mscp instance.
  * @return	0 on success, < 0 if an error occured.
  */
 int mscp_scan_join(struct mscp *m);
 
+/**
+ * @brief get information about remote host and copy direction from a
+ * checkpoint file specified by *pathname. This functions returns
+ * remote host name to *renote, and the copy direction into *dir.
+ * Thus, you can call mscp_init with those values.
+ *
+ * @param pathname	path to a checkpoint file.
+ * @param remote	char buffer to which remote hostname is stored.
+ * @param len		length of *remote.
+ * @param dir		int to which the copy direction is stored.
+ */
+int mscp_checkpoint_get_remote(const char *pathname, char *remote, size_t len, int *dir);
+
+/**
+ * @brief load information about untransferred files and chunks at the
+ * last transfer . mscp_checkpoint_load() loads files and associated
+ * chunks from the checkpoint file pointed by pathname. If you call
+ * mscp_checkpoint_load(), do not call mscp_scan().
+ *
+ * @param m		mscp instance.
+ * @param pathname	path to a checkpoint file.
+ * @return		0 on success, < 0 if an error occured.
+ */
+int mscp_checkpoint_load(struct mscp *m, const char *pathname);
+
+/**
+ * @brief save information about untransferred files and chunks to a
+ * checkpoint file.
+ *
+ * @param m		mscp instance.
+ * @param pathname	path to a checkpoint file.
+ * @return		0 on success, < 0 if an error occured.
+ */
+int mscp_checkpoint_save(struct mscp *m, const char *pathname);
+
 /**
  * @brief Start to copy files. mscp_start() returns immediately. You
  * can get statistics via mscp_get_stats() or messages via pipe set by

patch/libssh-0.10.6-2-g6f1b1e76.patch

@@ -392,10 +392,10 @@ index 8c509699..307388e5 100644
      session->opts.flags = SSH_OPT_FLAG_PASSWORD_AUTH |
                            SSH_OPT_FLAG_PUBKEY_AUTH |
 diff --git a/src/sftp.c b/src/sftp.c
-index e01012a8..702623a0 100644
+index e01012a8..3b86c3c6 100644
 --- a/src/sftp.c
 +++ b/src/sftp.c
-@@ -2228,6 +2228,132 @@ ssize_t sftp_write(sftp_file file, const void *buf, size_t count) {
+@@ -2228,6 +2228,135 @@ ssize_t sftp_write(sftp_file file, const void *buf, size_t count) {
    return -1; /* not reached */
  }
  
@@ -434,7 +434,8 @@ index e01012a8..702623a0 100644
 +
 +  buffer = ssh_buffer_new_size(buf_sz, HEADROOM);
 +  if (buffer == NULL) {
-+    ssh_set_error_oom(sftp->session);
++    ssh_set_error(sftp->session, SSH_FATAL,
++		  "ssh_buffer_new_size failed: Out of Memory");
 +    return -1;
 +  }
 +
@@ -448,14 +449,16 @@ index e01012a8..702623a0 100644
 +                       count); /* len of datastring */
 +
 +  if (rc != SSH_OK){
-+    ssh_set_error_oom(sftp->session);
++    ssh_set_error(sftp->session, SSH_FATAL,
++		  "ssh_buffer_pack failed: Out of Memory");
 +    ssh_buffer_free(buffer);
 +    return SSH_ERROR;
 +  }
 +
 +  actual = ssh_buffer_add_func(buffer, f, count, userdata);
 +  if (actual < 0){
-+    ssh_set_error_oom(sftp->session);
++    ssh_set_error(sftp->session, SSH_FATAL,
++		  "ssh_buffer_add_func failed: %s", strerror(errno));
 +    ssh_buffer_free(buffer);
 +    return SSH_ERROR;
 +  }

rpm/mscp.spec.in

@@ -38,8 +38,11 @@ make -C build install DESTDIR=%{buildroot}
 
 
 %changelog
-* Wed Feb 07 2024 Ryo nakamura <upa@haeena.net> - 0.1.4-0
+* Thu Mar 14 2024 Ryo Nakamura <upa@haeena.net> - 0.1.5-0
+- RPM release for v0.1.5
+
+* Wed Feb 07 2024 Ryo Nakamura <upa@haeena.net> - 0.1.4-0
 - RPM release for v0.1.4
 
-* Sat Feb 03 2024 Ryo nakamura <upa@haeena.net> - 0.1.3-0
+* Sat Feb 03 2024 Ryo Nakamura <upa@haeena.net> - 0.1.3-0
 - Initial release for rpm packaging

scripts/install-build-deps.sh

@@ -3,34 +3,71 @@
 # Install build dpenedencies.
 
 set -e
-set -u
+#set -u
+
+function print_help() {
+	echo "$0 [options]"
+	echo "    --dont-install         Print required packages."
+	echo "    --platform [PLATFORM]  PLATFORM is Kernel-ID, e.g., Linux-ubuntu."
+	echo "                           Automatically detected if not specified."
+}
 
 platform=$(uname -s)
+doinstall=1
 
 if [ -e /etc/os-release ]; then
 	source /etc/os-release
 	platform=${platform}-${ID}
 fi
 
-set -x
+while getopts h-: opt; do
+        optarg="${!OPTIND}"
+        [[ "$opt" = - ]] && opt="-$OPTARG"
+	case "-${opt}" in
+		--dont-install)
+			doinstall=0
+			;;
+		--platform)
+			platform=$optarg
+			shift
+			;;
+		-h)
+			print_help
+			exit 0
+			;;
+		*)
+			print_help
+			exit 1
+			;;
+	esac
+done
 
 case $platform in
 	Darwin)
-		brew install openssl@1.1
+		cmd="brew install"
+		pkgs="openssl@1.1"
 		;;
 	Linux-ubuntu*)
-		apt-get install --no-install-recommends -y \
-			gcc make cmake zlib1g-dev libssl-dev libkrb5-dev
+		cmd="apt-get install --no-install-recommends -y"
+		pkgs="gcc make cmake zlib1g-dev libssl-dev libkrb5-dev"
 		;;
 	Linux-centos* | Linux-rhel* | Linux-rocky* | Linux-almalinux)
-		yum install -y \
-			gcc make cmake zlib-devel openssl-devel rpm-build
+		cmd="yum install -y"
+		pkgs="gcc make cmake zlib-devel openssl-devel rpm-build"
 		;;
 	FreeBSD-freebsd)
-		pkg install cmake
+		cmd="pkg install"
+		pkgs="cmake"
 		;;
 	*)
 		echo "unsupported platform: $platform"
 		exit 1
 		;;
 esac
+
+if [ $doinstall -gt 0 ]; then
+	echo do "$cmd $pkgs"
+	$cmd $pkgs
+else
+	echo $pkgs
+fi

scripts/test-in-container.sh

@@ -8,6 +8,10 @@ cd $script_dir
 
 set -x
 
+# sshd Linsten on 22 and 8022
+echo "Port 22" >> /etc/ssh/sshd_config
+echo "Port 8022" >> /etc/ssh/sshd_config
+
 # Run sshd
 if [ ! -e /var/run/sshd.pid ]; then
 	/usr/sbin/sshd
@@ -19,4 +23,4 @@ ssh-keyscan 127.0.0.1 >> ${HOME}/.ssh/known_hosts
 ssh-keyscan ::1 >> ${HOME}/.ssh/known_hosts
 
 # Run test
-python3 -m pytest ../test -v
+python3 -m pytest -v ../test 

src/checkpoint.c

@@ -0,0 +1,503 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
+#include <fcntl.h>
+#include <sys/uio.h>
+#include <arpa/inet.h>
+
+#include <path.h>
+#include <print.h>
+#include <platform.h>
+#include <strerrno.h>
+#include <openbsd-compat/openbsd-compat.h>
+
+#include <checkpoint.h>
+
+#define MSCP_CHECKPOINT_MAGIC 0x7063736dUL /* mscp in ascii */
+#define MSCP_CHECKPOINT_VERSION 0x1
+
+/**
+ * mscp checkpoint file format. All values are network byte order.
+ *
+ * The file starts with the File header:
+ * 0                   1                   2                   3
+ *  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+ * +---------------------------------------------------------------+
+ * |                          Magic Code                           |
+ * +---------------+-----------------------------------------------+
+ * |     Version   |
+ * +---------------+
+ *
+ * Magic code: 0x7063736dUL
+ *
+ * Version: 1.
+ *
+ *
+ * Each object in a checkpoint always starts with an object header:
+ * +---------------+---------------+-------------------------------+
+ * |     Type      |      rsv      |             Length            |
+ * +---------------+---------------+-------------------------------+
+ *
+ * Type: 0x0A (meta), 0x0B (path), or 0x0C (chunk)
+ *
+ * Rsv: reserved
+ *
+ * Length: Length of this object including the object header.
+ *
+ *
+ * Meta object provides generaic information for the failed copy:
+ * +---------------+---------------+-------------------------------+
+ * |     Type      |      rsv      |             Length            |
+ * +---------------+---------------+-------------------------------+
+ * |  Direction    | Remote string ...
+ * +---------------+------------------
+ *
+ * Direction: 1 (Local-to-Remote copy) or 2 (Remote-to-Local copy)
+ *
+ * Remote string: Remote host, e.g., user@hostname and IP address,
+ * string including '\0'.
+ *
+ *
+ * Path object represnts a file with sourcen and destination paths:
+ * +---------------+---------------+-------------------------------+
+ * |     Type      |      rsv      |             Length            |
+ * +---------------+---------------+-------------------------------+
+ * |                             Index                             |
+ * +-------------------------------+-------------------------------+
+ * |         Source offset         |      Destination offset       |
+ * +-------------------------------+-------------------------------+
+ * //                                                             //
+ * //                     Source path string                      //
+ * //                                                             //
+ * +---------------------------------------------------------------+
+ * //                                                             //
+ * //                   Destination path string                   //
+ * //                                                             //
+ * +---------------------------------------------------------------+
+ *
+ * Index: 32-bit unsigned int indicating this path (used by chunks)
+ *
+ * Source offset: Offset of the Source path string from the head of
+ * this object. It is identical to the end of the Destination offset
+ * filed.
+ *
+ * Destination offset: Offset of the Destnation path string from the
+ * head of this object. It also indicates the end of the Source path
+ * string.
+ *
+ * Source path string: String of copy source path (including '\0').
+ *
+ * Destination path string: string of copy destination path (including
+ * '\0').
+ *
+ *
+ * Chunk object represents a chunk associated with a path object:
+ * +---------------+---------------+-------------------------------+
+ * |     Type      |      rsv      |             Length            |
+ * +---------------+---------------+-------------------------------+
+ * |                             Index                             |
+ * +---------------------------------------------------------------+
+ * |                             Chunk                             |
+ * |                             offset                            |
+ * +---------------------------------------------------------------+
+ * |                             Chunk                             |
+ * |                             length                            |
+ * +---------------------------------------------------------------+
+ *
+ * Index: 32 bit unsigned int indicating the index of a path object
+ * this chunk associated with.
+ *
+ * Chunk offset: 64 bit unsigned int indicating the offset of this
+ * chunk from the head of the associating a file.
+ *
+ * Chunk length: 64 bit unsigned int indicating the length (bytes) of
+ * this chunk.
+ */
+
+enum {
+	OBJ_TYPE_META = 0x0A,
+	OBJ_TYPE_PATH = 0x0B,
+	OBJ_TYPE_CHUNK = 0x0C,
+};
+
+struct checkpoint_file_hdr {
+	uint32_t magic;
+	uint8_t version;
+} __attribute__((packed));
+
+struct checkpoint_obj_hdr {
+	uint8_t type;
+	uint8_t rsv;
+	uint16_t len; /* length of an object including this hdr */
+} __attribute__((packed));
+
+struct checkpoint_obj_meta {
+	struct checkpoint_obj_hdr hdr;
+	uint8_t direction; /* L2R or R2L */
+
+	char remote[0];
+} __attribute__((packed));
+
+struct checkpoint_obj_path {
+	struct checkpoint_obj_hdr hdr;
+
+	uint32_t idx;
+	uint16_t src_off; /* offset to the src path string (including
+			   * \0) from the head of this object. */
+	uint16_t dst_off; /* offset to the dst path string (including
+			   * \0) from the head of this object */
+} __attribute__((packed));
+
+#define obj_path_src(o) ((char *)(o) + ntohs(o->src_off))
+#define obj_path_dst(o) ((char *)(o) + ntohs(o->dst_off))
+
+#define obj_path_src_len(o) (ntohs(o->dst_off) - ntohs(o->src_off))
+#define obj_path_dst_len(o) (ntohs(o->hdr.len) - ntohs(o->dst_off))
+
+#define obj_path_validate(o)				\
+	((ntohs(o->hdr.len) > ntohs(o->dst_off)) &&	\
+	 (ntohs(o->dst_off) > ntohs(o->src_off)) &&	\
+	 (obj_path_src_len(o) < PATH_MAX) &&		\
+	 (obj_path_dst_len(o) < PATH_MAX))
+
+struct checkpoint_obj_chunk {
+	struct checkpoint_obj_hdr hdr;
+
+	uint32_t idx; /* index indicating associating path */
+	uint64_t off;
+	uint64_t len;
+} __attribute__((packed));
+
+#define CHECKPOINT_OBJ_MAXLEN (sizeof(struct checkpoint_obj_path) + PATH_MAX * 2)
+
+static int checkpoint_write_path(int fd, struct path *p, unsigned int idx)
+{
+	char buf[CHECKPOINT_OBJ_MAXLEN];
+	struct checkpoint_obj_path *path = (struct checkpoint_obj_path *)buf;
+	size_t src_len, dst_len;
+	struct iovec iov[3];
+
+	p->data = idx; /* save idx to be pointed by chunks */
+
+	src_len = strlen(p->path) + 1;
+	dst_len = strlen(p->dst_path) + 1;
+
+	memset(buf, 0, sizeof(buf));
+	path->hdr.type = OBJ_TYPE_PATH;
+	path->hdr.len = htons(sizeof(*path) + src_len + dst_len);
+
+	path->idx = htonl(idx);
+	path->src_off = htons(sizeof(*path));
+	path->dst_off = htons(sizeof(*path) + src_len);
+
+	iov[0].iov_base = path;
+	iov[0].iov_len = sizeof(*path);
+	iov[1].iov_base = p->path;
+	iov[1].iov_len = src_len;
+	iov[2].iov_base = p->dst_path;
+	iov[2].iov_len = dst_len;
+
+	if (writev(fd, iov, 3) < 0) {
+		priv_set_errv("writev: %s", strerrno());
+		return -1;
+	}
+	return 0;
+}
+
+static int checkpoint_write_chunk(int fd, struct chunk *c)
+{
+	struct checkpoint_obj_chunk chunk;
+
+	memset(&chunk, 0, sizeof(chunk));
+	chunk.hdr.type = OBJ_TYPE_CHUNK;
+	chunk.hdr.len = htons(sizeof(chunk));
+
+	chunk.idx = htonl(c->p->data); /* index stored by checkpoint_write_path */
+	chunk.off = htonll(c->off);
+	chunk.len = htonll(c->len);
+
+	if (write(fd, &chunk, sizeof(chunk)) < 0) {
+		priv_set_errv("writev: %s", strerrno());
+		return -1;
+	}
+	return 0;
+}
+
+int checkpoint_save(const char *pathname, int dir, const char *user, const char *remote,
+		    pool *path_pool, pool *chunk_pool)
+{
+	struct checkpoint_file_hdr hdr;
+	struct checkpoint_obj_meta meta;
+	struct iovec iov[3];
+	struct chunk *c;
+	struct path *p;
+	char buf[1024];
+	unsigned int i, nr_paths, nr_chunks;
+	int fd, ret;
+
+	fd = open(pathname, O_WRONLY | O_CREAT | O_TRUNC,
+		  S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH);
+	if (fd < 0) {
+		priv_set_errv("open: %s: %s", pathname, strerrno());
+		return -1;
+	}
+
+	/* write file hdr */
+	hdr.magic = htonl(MSCP_CHECKPOINT_MAGIC);
+	hdr.version = MSCP_CHECKPOINT_VERSION;
+
+	/* write meta */
+	if (user)
+		ret = snprintf(buf, sizeof(buf), "%s@%s", user, remote);
+	else
+		ret = snprintf(buf, sizeof(buf), "%s", remote);
+	if (ret >= sizeof(buf)) {
+		priv_set_errv("too long username and/or remote");
+		return -1;
+	}
+
+	memset(&meta, 0, sizeof(meta));
+	meta.hdr.type = OBJ_TYPE_META;
+	meta.hdr.len = htons(sizeof(meta) + strlen(buf) + 1);
+	meta.direction = dir;
+
+	iov[0].iov_base = &hdr;
+	iov[0].iov_len = sizeof(hdr);
+	iov[1].iov_base = &meta;
+	iov[1].iov_len = sizeof(meta);
+	iov[2].iov_base = buf;
+	iov[2].iov_len = strlen(buf) + 1;
+
+	if (writev(fd, iov, 3) < 0) {
+		priv_set_errv("writev: %s", strerrno());
+		return -1;
+	}
+
+	/* write paths */
+	nr_paths = 0;
+	pool_for_each(path_pool, p, i) {
+		if (p->state == FILE_STATE_DONE)
+			continue;
+		if (checkpoint_write_path(fd, p, i) < 0)
+			return -1;
+		nr_paths++;
+	}
+
+	/* write chunks */
+	nr_chunks = 0;
+	pool_for_each(chunk_pool, c, i) {
+		if (c->state == CHUNK_STATE_DONE)
+			continue;
+		if (checkpoint_write_chunk(fd, c) < 0)
+			return -1;
+		nr_chunks++;
+	}
+
+	pr_notice("checkpoint: %u paths and %u chunks saved", nr_paths, nr_chunks);
+
+	return 0;
+}
+
+static int checkpoint_load_meta(struct checkpoint_obj_hdr *hdr, char *remote, size_t len,
+				int *dir)
+{
+	struct checkpoint_obj_meta *meta = (struct checkpoint_obj_meta *)hdr;
+
+	if (len < ntohs(hdr->len) - sizeof(*meta)) {
+		priv_set_errv("too short buffer");
+		return -1;
+	}
+	snprintf(remote, len, "%s", meta->remote);
+	*dir = meta->direction;
+
+	pr_notice("checkpoint: remote=%s direction=%s", meta->remote,
+		  meta->direction == MSCP_DIRECTION_L2R ? "local-to-remote" :
+		  meta->direction == MSCP_DIRECTION_R2L ? "remote-to-local" :
+							  "invalid");
+
+	return 0;
+}
+
+static int checkpoint_load_path(struct checkpoint_obj_hdr *hdr, pool *path_pool)
+{
+	struct checkpoint_obj_path *path = (struct checkpoint_obj_path *)hdr;
+	struct path *p;
+	char *s, *d;
+
+	if (!obj_path_validate(path)) {
+		priv_set_errv("invalid path object");
+		return -1;
+	}
+
+	if (!(s = strndup(obj_path_src(path), obj_path_src_len(path)))) {
+		priv_set_errv("strdup: %s", strerrno());
+		return -1;
+	}
+
+	if (!(d = strndup(obj_path_dst(path), obj_path_dst_len(path)))) {
+		priv_set_errv("strdup: %s", strerrno());
+		free(s);
+		return -1;
+	}
+
+	if (!(p = alloc_path(s, d))) {
+		free(s);
+		free(d);
+		return -1;
+	}
+
+	if (pool_push(path_pool, p) < 0) {
+		priv_set_errv("pool_push: %s", strerrno());
+		return -1;
+	}
+
+	pr_info("checkpoint:file: %s -> %s", p->path, p->dst_path);
+
+	return 0;
+}
+
+static int checkpoint_load_chunk(struct checkpoint_obj_hdr *hdr, pool *path_pool,
+				 pool *chunk_pool)
+{
+	struct checkpoint_obj_chunk *chunk = (struct checkpoint_obj_chunk *)hdr;
+	struct chunk *c;
+	struct path *p;
+
+	if (!(p = pool_get(path_pool, ntohl(chunk->idx)))) {
+		/* we assumes all paths are already loaded in the order */
+		priv_set_errv("path index %u not found", ntohl(chunk->idx));
+		return -1;
+	}
+
+	if (!(c = alloc_chunk(p, ntohll(chunk->off), ntohll(chunk->len))))
+		return -1;
+
+	if (pool_push(chunk_pool, c) < 0) {
+		priv_set_errv("pool_push: %s", strerrno());
+		return -1;
+	}
+
+	pr_debug("checkpoint:chunk: %s 0x%lx-0x%lx", p->path, c->off, c->off + c->len);
+
+	return 0;
+}
+
+static int checkpoint_read_obj(int fd, void *buf, size_t count)
+{
+	struct checkpoint_obj_hdr *hdr = (struct checkpoint_obj_hdr *)buf;
+	ssize_t ret, objlen, objbuflen;
+
+	memset(buf, 0, count);
+
+	if (count < sizeof(*hdr)) {
+		priv_set_errv("too short buffer");
+		return -1;
+	}
+
+	ret = read(fd, hdr, sizeof(*hdr));
+	if (ret == 0)
+		return 0; /* no more objects */
+	if (ret < 0)
+		return -1;
+
+	objlen = ntohs(hdr->len) - sizeof(*hdr);
+	objbuflen = count - sizeof(*hdr);
+	if (objbuflen < objlen) {
+		priv_set_errv("too short buffer");
+		return -1;
+	}
+
+	ret = read(fd, buf + sizeof(*hdr), objlen);
+	if (ret < objlen) {
+		priv_set_errv("checkpoint truncated");
+		return -1;
+	}
+
+	return 1;
+}
+
+static int checkpoint_read_file_hdr(int fd)
+{
+	struct checkpoint_file_hdr hdr;
+	ssize_t ret;
+
+	ret = read(fd, &hdr, sizeof(hdr));
+	if (ret < 0) {
+		priv_set_errv("read: %s", strerrno());
+		return -1;
+	}
+	if (ret < sizeof(hdr)) {
+		priv_set_errv("checkpoint truncated");
+		return -1;
+	}
+
+	if (ntohl(hdr.magic) != MSCP_CHECKPOINT_MAGIC) {
+		priv_set_errv("checkpoint: invalid megic code");
+		return -1;
+	}
+
+	if (hdr.version != MSCP_CHECKPOINT_VERSION) {
+		priv_set_errv("checkpoint: unknown version %u", hdr.version);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int checkpoint_load(const char *pathname, char *remote, size_t len, int *dir,
+			   pool *path_pool, pool *chunk_pool)
+{
+	char buf[CHECKPOINT_OBJ_MAXLEN];
+	struct checkpoint_obj_hdr *hdr;
+	int fd, ret;
+
+	if ((fd = open(pathname, O_RDONLY)) < 0) {
+		priv_set_errv("open: %s: %s", pathname, strerrno());
+		return -1;
+	}
+
+	if (checkpoint_read_file_hdr(fd) < 0)
+		return -1;
+
+	hdr = (struct checkpoint_obj_hdr *)buf;
+	while ((ret = checkpoint_read_obj(fd, buf, sizeof(buf))) > 0) {
+		switch (hdr->type) {
+		case OBJ_TYPE_META:
+			if (!remote || !dir)
+				break;
+			if (checkpoint_load_meta(hdr, remote, len, dir) < 0)
+				return -1;
+			if (!path_pool || !chunk_pool)
+				goto out;
+			break;
+		case OBJ_TYPE_PATH:
+			if (!path_pool)
+				break;
+			if (checkpoint_load_path(hdr, path_pool) < 0)
+				return -1;
+			break;
+		case OBJ_TYPE_CHUNK:
+			if (!path_pool)
+				break;
+			if (checkpoint_load_chunk(hdr, path_pool, chunk_pool) < 0)
+				return -1;
+			break;
+		default:
+			priv_set_errv("unknown obj type %u", hdr->type);
+			return -1;
+		}
+	}
+
+out:
+	close(fd);
+
+	return 0;
+}
+
+int checkpoint_load_remote(const char *pathname, char *remote, size_t len, int *dir)
+{
+	return checkpoint_load(pathname, remote, len, dir, NULL, NULL);
+}
+
+int checkpoint_load_paths(const char *pathname, pool *path_pool, pool *chunk_pool)
+{
+	return checkpoint_load(pathname, NULL, 0, NULL, path_pool, chunk_pool);
+}

src/checkpoint.h

@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
+#ifndef _CHECKPOINT_H_
+#define _CHECKPOINT_H_
+
+#include <pool.h>
+
+/* checkpoint_save() stores states to a checkponint file (pathname) */
+int checkpoint_save(const char *pathname, int dir, const char *user, const char *remote,
+		    pool *path_pool, pool *chunk_pool);
+
+/* checkpoint_load_meta() reads a checkpoint file (pathname) and returns
+ * remote host string to *remote and transfer direction to *dir.
+ */
+int checkpoint_load_remote(const char *pathname, char *remote, size_t len, int *dir);
+
+/* checkpoint_load_paths() reads a checkpoint file (pathname) and
+ * fills path_pool and chunk_pool.
+ */
+int checkpoint_load_paths(const char *pathname, pool *path_pool, pool *chunk_pool);
+
+#endif /* _CHECKPOINT_H_ */

src/list.h

@@ -1,573 +0,0 @@
-/**
- * 
- * I grub it from linux kernel source code and fix it for user space
- * program. Of course, this is a GPL licensed header file.
- *
- * Here is a recipe to cook list.h for user space program
- *
- * 1. copy list.h from linux/include/list.h
- * 2. remove 
- *     - #ifdef __KERNE__ and its #endif
- *     - all #include line
- *     - prefetch() and rcu related functions
- * 3. add macro offsetof() and container_of
- *
- * - kazutomo@mcs.anl.gov
- */
-#ifndef _LINUX_LIST_H
-#define _LINUX_LIST_H
-
-/**
- * @name from other kernel headers
- */
-/*@{*/
-
-/**
- * Get offset of a member
- */
-#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
-
-/**
- * Casts a member of a structure out to the containing structure
- * @param ptr        the pointer to the member.
- * @param type       the type of the container struct this is embedded in.
- * @param member     the name of the member within the struct.
- *
- */
-#define container_of(ptr, type, member) ({                      \
-        const typeof( ((type *)0)->member ) *__mptr = (ptr);    \
-        (type *)( (char *)__mptr - offsetof(type,member) );})
-/*@}*/
-
-
-/*
- * These are non-NULL pointers that will result in page faults
- * under normal circumstances, used to verify that nobody uses
- * non-initialized list entries.
- */
-#define LIST_POISON1  ((void *) 0x00100100)
-#define LIST_POISON2  ((void *) 0x00200200)
-
-/**
- * Simple doubly linked list implementation.
- *
- * Some of the internal functions ("__xxx") are useful when
- * manipulating whole lists rather than single entries, as
- * sometimes we already know the next/prev entries and we can
- * generate better code by using them directly rather than
- * using the generic single-entry routines.
- */
-struct list_head {
-	struct list_head *next, *prev;
-};
-
-#define LIST_HEAD_INIT(name) { &(name), &(name) }
-
-#define LIST_HEAD(name) \
-	struct list_head name = LIST_HEAD_INIT(name)
-
-#define INIT_LIST_HEAD(ptr) do { \
-	(ptr)->next = (ptr); (ptr)->prev = (ptr); \
-} while (0)
-
-/*
- * Insert a new entry between two known consecutive entries.
- *
- * This is only for internal list manipulation where we know
- * the prev/next entries already!
- */
-static inline void __list_add(struct list_head *new,
-			      struct list_head *prev,
-			      struct list_head *next)
-{
-	next->prev = new;
-	new->next = next;
-	new->prev = prev;
-	prev->next = new;
-}
-
-/**
- * list_add - add a new entry
- * @new: new entry to be added
- * @head: list head to add it after
- *
- * Insert a new entry after the specified head.
- * This is good for implementing stacks.
- */
-static inline void list_add(struct list_head *new, struct list_head *head)
-{
-	__list_add(new, head, head->next);
-}
-
-/**
- * list_add_tail - add a new entry
- * @new: new entry to be added
- * @head: list head to add it before
- *
- * Insert a new entry before the specified head.
- * This is useful for implementing queues.
- */
-static inline void list_add_tail(struct list_head *new, struct list_head *head)
-{
-	__list_add(new, head->prev, head);
-}
-
-
-/*
- * Delete a list entry by making the prev/next entries
- * point to each other.
- *
- * This is only for internal list manipulation where we know
- * the prev/next entries already!
- */
-static inline void __list_del(struct list_head * prev, struct list_head * next)
-{
-	next->prev = prev;
-	prev->next = next;
-}
-
-/**
- * list_del - deletes entry from list.
- * @entry: the element to delete from the list.
- * Note: list_empty on entry does not return true after this, the entry is
- * in an undefined state.
- */
-static inline void list_del(struct list_head *entry)
-{
-	__list_del(entry->prev, entry->next);
-	entry->next = LIST_POISON1;
-	entry->prev = LIST_POISON2;
-}
-
-
-
-/**
- * list_del_init - deletes entry from list and reinitialize it.
- * @entry: the element to delete from the list.
- */
-static inline void list_del_init(struct list_head *entry)
-{
-	__list_del(entry->prev, entry->next);
-	INIT_LIST_HEAD(entry);
-}
-
-/**
- * list_move - delete from one list and add as another's head
- * @list: the entry to move
- * @head: the head that will precede our entry
- */
-static inline void list_move(struct list_head *list, struct list_head *head)
-{
-        __list_del(list->prev, list->next);
-        list_add(list, head);
-}
-
-/**
- * list_move_tail - delete from one list and add as another's tail
- * @list: the entry to move
- * @head: the head that will follow our entry
- */
-static inline void list_move_tail(struct list_head *list,
-				  struct list_head *head)
-{
-        __list_del(list->prev, list->next);
-        list_add_tail(list, head);
-}
-
-/**
- * list_empty - tests whether a list is empty
- * @head: the list to test.
- */
-static inline int list_empty(const struct list_head *head)
-{
-	return head->next == head;
-}
-
-static inline void __list_splice(struct list_head *list,
-				 struct list_head *head)
-{
-	struct list_head *first = list->next;
-	struct list_head *last = list->prev;
-	struct list_head *at = head->next;
-
-	first->prev = head;
-	head->next = first;
-
-	last->next = at;
-	at->prev = last;
-}
-
-/**
- * list_splice - join two lists
- * @list: the new list to add.
- * @head: the place to add it in the first list.
- */
-static inline void list_splice(struct list_head *list, struct list_head *head)
-{
-	if (!list_empty(list))
-		__list_splice(list, head);
-}
-
-static inline void __list_splice_tail(struct list_head *list,
-				      struct list_head *head)
-{
-	struct list_head *first = list->next;
-	struct list_head *last = list->prev;
-	struct list_head *at = head->prev;
-
-	first->prev = at;
-	at->next = first;
-
-	last->next = head;
-	at->prev = last;
-}
-
-/**
- * list_splice_tail - join two lists
- * @list: the new list to add.
- * @head: the place to add it in the first list.
- */
-static inline void list_splice_tail(struct list_head *list, struct list_head *head)
-{
-	if (!list_empty(list))
-		__list_splice_tail(list, head);
-}
-
-
-/**
- * list_splice_init - join two lists and reinitialise the emptied list.
- * @list: the new list to add.
- * @head: the place to add it in the first list.
- *
- * The list at @list is reinitialised
- */
-static inline void list_splice_init(struct list_head *list,
-				    struct list_head *head)
-{
-	if (!list_empty(list)) {
-		__list_splice(list, head);
-		INIT_LIST_HEAD(list);
-	}
-}
-
-/**
- * list_entry - get the struct for this entry
- * @ptr:	the &struct list_head pointer.
- * @type:	the type of the struct this is embedded in.
- * @member:	the name of the list_struct within the struct.
- */
-#define list_entry(ptr, type, member) \
-	container_of(ptr, type, member)
-
-/**
- * list_for_each	-	iterate over a list
- * @pos:	the &struct list_head to use as a loop counter.
- * @head:	the head for your list.
- */
-
-#define list_for_each(pos, head) \
-  for (pos = (head)->next; pos != (head);	\
-       pos = pos->next)
-
-/**
- * __list_for_each	-	iterate over a list
- * @pos:	the &struct list_head to use as a loop counter.
- * @head:	the head for your list.
- *
- * This variant differs from list_for_each() in that it's the
- * simplest possible list iteration code, no prefetching is done.
- * Use this for code that knows the list to be very short (empty
- * or 1 entry) most of the time.
- */
-#define __list_for_each(pos, head) \
-	for (pos = (head)->next; pos != (head); pos = pos->next)
-
-/**
- * list_for_each_prev	-	iterate over a list backwards
- * @pos:	the &struct list_head to use as a loop counter.
- * @head:	the head for your list.
- */
-#define list_for_each_prev(pos, head) \
-	for (pos = (head)->prev; prefetch(pos->prev), pos != (head); \
-        	pos = pos->prev)
-
-/**
- * list_for_each_safe	-	iterate over a list safe against removal of list entry
- * @pos:	the &struct list_head to use as a loop counter.
- * @n:		another &struct list_head to use as temporary storage
- * @head:	the head for your list.
- */
-#define list_for_each_safe(pos, n, head) \
-	for (pos = (head)->next, n = pos->next; pos != (head); \
-		pos = n, n = pos->next)
-
-/**
- * list_for_each_entry	-	iterate over list of given type
- * @pos:	the type * to use as a loop counter.
- * @head:	the head for your list.
- * @member:	the name of the list_struct within the struct.
- */
-#define list_for_each_entry(pos, head, member)				\
-	for (pos = list_entry((head)->next, typeof(*pos), member);	\
-	     &pos->member != (head);					\
-	     pos = list_entry(pos->member.next, typeof(*pos), member))
-
-/**
- * list_for_each_entry_reverse - iterate backwards over list of given type.
- * @pos:	the type * to use as a loop counter.
- * @head:	the head for your list.
- * @member:	the name of the list_struct within the struct.
- */
-#define list_for_each_entry_reverse(pos, head, member)			\
-	for (pos = list_entry((head)->prev, typeof(*pos), member);	\
-	     &pos->member != (head); 	\
-	     pos = list_entry(pos->member.prev, typeof(*pos), member))
-
-/**
- * list_prepare_entry - prepare a pos entry for use as a start point in
- *			list_for_each_entry_continue
- * @pos:	the type * to use as a start point
- * @head:	the head of the list
- * @member:	the name of the list_struct within the struct.
- */
-#define list_prepare_entry(pos, head, member) \
-	((pos) ? : list_entry(head, typeof(*pos), member))
-
-/**
- * list_for_each_entry_continue -	iterate over list of given type
- *			continuing after existing point
- * @pos:	the type * to use as a loop counter.
- * @head:	the head for your list.
- * @member:	the name of the list_struct within the struct.
- */
-#define list_for_each_entry_continue(pos, head, member) 		\
-	for (pos = list_entry(pos->member.next, typeof(*pos), member);	\
-	     &pos->member != (head);	\
-	     pos = list_entry(pos->member.next, typeof(*pos), member))
-
-/**
- * list_for_each_entry_safe - iterate over list of given type safe against removal of list entry
- * @pos:	the type * to use as a loop counter.
- * @n:		another type * to use as temporary storage
- * @head:	the head for your list.
- * @member:	the name of the list_struct within the struct.
- */
-#define list_for_each_entry_safe(pos, n, head, member)			\
-	for (pos = list_entry((head)->next, typeof(*pos), member),	\
-		n = list_entry(pos->member.next, typeof(*pos), member);	\
-	     &pos->member != (head); 					\
-	     pos = n, n = list_entry(n->member.next, typeof(*n), member))
-
-/**
- * list_for_each_entry_safe_continue -	iterate over list of given type
- *			continuing after existing point safe against removal of list entry
- * @pos:	the type * to use as a loop counter.
- * @n:		another type * to use as temporary storage
- * @head:	the head for your list.
- * @member:	the name of the list_struct within the struct.
- */
-#define list_for_each_entry_safe_continue(pos, n, head, member) 		\
-	for (pos = list_entry(pos->member.next, typeof(*pos), member), 		\
-		n = list_entry(pos->member.next, typeof(*pos), member);		\
-	     &pos->member != (head);						\
-	     pos = n, n = list_entry(n->member.next, typeof(*n), member))
-
-/**
- * list_for_each_entry_safe_reverse - iterate backwards over list of given type safe against
- *				      removal of list entry
- * @pos:	the type * to use as a loop counter.
- * @n:		another type * to use as temporary storage
- * @head:	the head for your list.
- * @member:	the name of the list_struct within the struct.
- */
-#define list_for_each_entry_safe_reverse(pos, n, head, member)		\
-	for (pos = list_entry((head)->prev, typeof(*pos), member),	\
-		n = list_entry(pos->member.prev, typeof(*pos), member);	\
-	     &pos->member != (head); 					\
-	     pos = n, n = list_entry(n->member.prev, typeof(*n), member))
-
-
-
-
-/*
- * Double linked lists with a single pointer list head.
- * Mostly useful for hash tables where the two pointer list head is
- * too wasteful.
- * You lose the ability to access the tail in O(1).
- */
-
-struct hlist_head {
-	struct hlist_node *first;
-};
-
-struct hlist_node {
-	struct hlist_node *next, **pprev;
-};
-
-#define HLIST_HEAD_INIT { .first = NULL }
-#define HLIST_HEAD(name) struct hlist_head name = {  .first = NULL }
-#define INIT_HLIST_HEAD(ptr) ((ptr)->first = NULL)
-#define INIT_HLIST_NODE(ptr) ((ptr)->next = NULL, (ptr)->pprev = NULL)
-
-static inline int hlist_unhashed(const struct hlist_node *h)
-{
-	return !h->pprev;
-}
-
-static inline int hlist_empty(const struct hlist_head *h)
-{
-	return !h->first;
-}
-
-static inline void __hlist_del(struct hlist_node *n)
-{
-	struct hlist_node *next = n->next;
-	struct hlist_node **pprev = n->pprev;
-	*pprev = next;
-	if (next)
-		next->pprev = pprev;
-}
-
-static inline void hlist_del(struct hlist_node *n)
-{
-	__hlist_del(n);
-	n->next = LIST_POISON1;
-	n->pprev = LIST_POISON2;
-}
-
-
-static inline void hlist_del_init(struct hlist_node *n)
-{
-	if (n->pprev)  {
-		__hlist_del(n);
-		INIT_HLIST_NODE(n);
-	}
-}
-
-static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h)
-{
-	struct hlist_node *first = h->first;
-	n->next = first;
-	if (first)
-		first->pprev = &n->next;
-	h->first = n;
-	n->pprev = &h->first;
-}
-
-
-
-/* next must be != NULL */
-static inline void hlist_add_before(struct hlist_node *n,
-					struct hlist_node *next)
-{
-	n->pprev = next->pprev;
-	n->next = next;
-	next->pprev = &n->next;
-	*(n->pprev) = n;
-}
-
-static inline void hlist_add_after(struct hlist_node *n,
-					struct hlist_node *next)
-{
-	next->next = n->next;
-	n->next = next;
-	next->pprev = &n->next;
-
-	if(next->next)
-		next->next->pprev  = &next->next;
-}
-
-
-
-#define hlist_entry(ptr, type, member) container_of(ptr,type,member)
-
-#define hlist_for_each(pos, head) \
-	for (pos = (head)->first; pos && ({ prefetch(pos->next); 1; }); \
-	     pos = pos->next)
-
-#define hlist_for_each_safe(pos, n, head) \
-	for (pos = (head)->first; pos && ({ n = pos->next; 1; }); \
-	     pos = n)
-
-/**
- * hlist_for_each_entry	- iterate over list of given type
- * @tpos:	the type * to use as a loop counter.
- * @pos:	the &struct hlist_node to use as a loop counter.
- * @head:	the head for your list.
- * @member:	the name of the hlist_node within the struct.
- */
-#define hlist_for_each_entry(tpos, pos, head, member)			 \
-	for (pos = (head)->first;					 \
-	     pos && ({ prefetch(pos->next); 1;}) &&			 \
-		({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
-	     pos = pos->next)
-
-/**
- * hlist_for_each_entry_continue - iterate over a hlist continuing after existing point
- * @tpos:	the type * to use as a loop counter.
- * @pos:	the &struct hlist_node to use as a loop counter.
- * @member:	the name of the hlist_node within the struct.
- */
-#define hlist_for_each_entry_continue(tpos, pos, member)		 \
-	for (pos = (pos)->next;						 \
-	     pos && ({ prefetch(pos->next); 1;}) &&			 \
-		({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
-	     pos = pos->next)
-
-/**
- * hlist_for_each_entry_from - iterate over a hlist continuing from existing point
- * @tpos:	the type * to use as a loop counter.
- * @pos:	the &struct hlist_node to use as a loop counter.
- * @member:	the name of the hlist_node within the struct.
- */
-#define hlist_for_each_entry_from(tpos, pos, member)			 \
-	for (; pos && ({ prefetch(pos->next); 1;}) &&			 \
-		({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
-	     pos = pos->next)
-
-/**
- * hlist_for_each_entry_safe - iterate over list of given type safe against removal of list entry
- * @tpos:	the type * to use as a loop counter.
- * @pos:	the &struct hlist_node to use as a loop counter.
- * @n:		another &struct hlist_node to use as temporary storage
- * @head:	the head for your list.
- * @member:	the name of the hlist_node within the struct.
- */
-#define hlist_for_each_entry_safe(tpos, pos, n, head, member) 		 \
-	for (pos = (head)->first;					 \
-	     pos && ({ n = pos->next; 1; }) && 				 \
-		({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
-	     pos = n)
-
-
-/**
- * list_count - return length of list
- * @head        the head for your list.
- */
-static inline int list_count(struct list_head *head)
-{
-	int n = 0;
-	struct list_head *p;
-
-	list_for_each(p, head) n++;
-	return n;
-}
-
-
-/**
- * list_free_f - free items in a list with a function
- * @head	the heaf for your list.
- * @f           function that releases an item in the list.
- */
-static inline void list_free_f(struct list_head *head, void (*f)(struct list_head *))
-{
-	struct list_head *p, *n;
-
-        list_for_each_safe(p, n, head) {
-                list_del(p);
-                f(p);
-        }
-}
-
-#endif
-

src/main.c

@@ -18,14 +18,14 @@
 #include <strerrno.h>
 #include <print.h>
 
-#include "config.h"
+#include <config.h>
 
 void usage(bool print_help)
 {
 	printf("mscp " MSCP_BUILD_VERSION ": copy files over multiple SSH connections\n"
 	       "\n"
 	       "Usage: mscp [-46vqDpHdNh] [-n nr_conns] [-m coremask]\n"
-	       "            [-u max_startups] [-I interval]\n"
+	       "            [-u max_startups] [-I interval] [-W checkpoint] [-R checkpoint]\n"
 	       "            [-s min_chunk_sz] [-S max_chunk_sz] [-a nr_ahead] [-b buf_sz]\n"
 	       "            [-l login_name] [-P port] [-F ssh_config] [-i identity_file]\n"
 	       "            [-c cipher_spec] [-M hmac_spec] [-C compress] [-g congestion]\n"
@@ -41,6 +41,8 @@ void usage(bool print_help)
 	       "    -u MAX_STARTUPS    number of concurrent SSH connection attempts "
 	       "(default: 8)\n"
 	       "    -I INTERVAL        interval between SSH connection attempts (default: 0)\n"
+	       "    -W CHECKPOINT      write states to the checkpoint if transfer fails\n"
+	       "    -R CHECKPOINT      resume transferring from the checkpoint\n"
 	       "\n"
 	       "    -s MIN_CHUNK_SIZE  min chunk size (default: 64MB)\n"
 	       "    -S MAX_CHUNK_SIZE  max chunk size (default: filesize/nr_conn)\n"
@@ -233,12 +235,12 @@ free_target_out:
 
 struct mscp *m = NULL;
 pthread_t tid_stat = 0;
+bool interrupted = false;
 
 void sigint_handler(int sig)
 {
+	interrupted = true;
 	mscp_stop(m);
-	if (tid_stat > 0)
-		pthread_cancel(tid_stat);
 }
 
 void *print_stat_thread(void *arg);
@@ -252,6 +254,8 @@ void print_cli(const char *fmt, ...)
 	va_end(va);
 }
 
+void print_stat(bool final);
+
 int main(int argc, char **argv)
 {
 	struct mscp_ssh_opts s;
@@ -260,14 +264,14 @@ int main(int argc, char **argv)
 	int pipe_fd[2];
 	int ch, n, i, ret;
 	int direction = 0;
-	char *remote;
-	bool dryrun = false;
+	char *remote = NULL, *checkpoint_save = NULL, *checkpoint_load = NULL;
+	bool dryrun = false, resume = false;
 
 	memset(&s, 0, sizeof(s));
 	memset(&o, 0, sizeof(o));
 	o.severity = MSCP_SEVERITY_WARN;
 
-#define mscpopts "n:m:u:I:s:S:a:b:46vqDrl:P:i:F:c:M:C:g:pHdNh"
+#define mscpopts "n:m:u:I:W:R:s:S:a:b:46vqDrl:P:i:F:c:M:C:g:pHdNh"
 	while ((ch = getopt(argc, argv, mscpopts)) != -1) {
 		switch (ch) {
 		case 'n':
@@ -286,6 +290,13 @@ int main(int argc, char **argv)
 		case 'I':
 			o.interval = atoi(optarg);
 			break;
+		case 'W':
+			checkpoint_save = optarg;
+			break;
+		case 'R':
+			checkpoint_load = optarg;
+			resume = true;
+			break;
 		case 's':
 			o.min_chunk_sz = atoi(optarg);
 			break;
@@ -364,53 +375,81 @@ int main(int argc, char **argv)
 	s.password = getenv(ENV_SSH_AUTH_PASSWORD);
 	s.passphrase = getenv(ENV_SSH_AUTH_PASSPHRASE);
 
-	if (argc - optind < 2) {
-		/* mscp needs at lease 2 (src and target) argument */
-		usage(false);
-		return 1;
-	}
-	i = argc - optind;
-
-	if ((t = validate_targets(argv + optind, i)) == NULL)
-		return -1;
-
-	if (t[0].host) {
-		/* copy remote to local */
-		direction = MSCP_DIRECTION_R2L;
-		remote = t[0].host;
-		s.login_name = s.login_name ? s.login_name : t[0].user;
-	} else {
-		/* copy local to remote */
-		direction = MSCP_DIRECTION_L2R;
-		remote = t[i - 1].host;
-		s.login_name = s.login_name ? s.login_name : t[i - 1].user;
-	}
-
-	if ((m = mscp_init(remote, direction, &o, &s)) == NULL) {
+	if ((m = mscp_init(&o, &s)) == NULL) {
 		pr_err("mscp_init: %s", priv_get_err());
 		return -1;
 	}
 
-	if (mscp_connect(m) < 0) {
-		pr_err("mscp_connect: %s", priv_get_err());
-		return -1;
-	}
+	if (!resume) {
+		/* normal transfer (not resume) */
+		if (argc - optind < 2) {
+			/* mscp needs at lease 2 (src and target) argument */
+			usage(false);
+			return 1;
+		}
+		i = argc - optind;
 
-	for (n = 0; n < i - 1; n++) {
-		if (mscp_add_src_path(m, t[n].path) < 0) {
-			pr_err("mscp_add_src_path: %s", priv_get_err());
+		if ((t = validate_targets(argv + optind, i)) == NULL)
+			return -1;
+
+		if (t[0].host) {
+			/* copy remote to local */
+			direction = MSCP_DIRECTION_R2L;
+			remote = t[0].host;
+			s.login_name = s.login_name ? s.login_name : t[0].user;
+		} else {
+			/* copy local to remote */
+			direction = MSCP_DIRECTION_L2R;
+			remote = t[i - 1].host;
+			s.login_name = s.login_name ? s.login_name : t[i - 1].user;
+		}
+
+		if (mscp_set_remote(m, remote, direction) < 0) {
+			pr_err("mscp_set_remote: %s", priv_get_err());
 			return -1;
 		}
-	}
 
-	if (mscp_set_dst_path(m, t[i - 1].path) < 0) {
-		pr_err("mscp_set_dst_path: %s", priv_get_err());
-		return -1;
-	}
+		if (mscp_connect(m) < 0) {
+			pr_err("mscp_connect: %s", priv_get_err());
+			return -1;
+		}
 
-	if (mscp_scan(m) < 0) {
-		pr_err("mscp_scan: %s", priv_get_err());
-		return -1;
+		for (n = 0; n < i - 1; n++) {
+			if (mscp_add_src_path(m, t[n].path) < 0) {
+				pr_err("mscp_add_src_path: %s", priv_get_err());
+				return -1;
+			}
+		}
+
+		if (mscp_set_dst_path(m, t[i - 1].path) < 0) {
+			pr_err("mscp_set_dst_path: %s", priv_get_err());
+			return -1;
+		}
+
+		/* start to scan source files and resolve their destination paths */
+		if (mscp_scan(m) < 0) {
+			pr_err("mscp_scan: %s", priv_get_err());
+			return -1;
+		}
+	} else {
+		/* resume a transfer from the specified checkpoint */
+		char r[512];
+		int d;
+		if (mscp_checkpoint_get_remote(checkpoint_load, r, sizeof(r), &d) < 0) {
+			pr_err("mscp_checkpoint_get_remote: %s", priv_get_err());
+			return -1;
+		}
+
+		if (mscp_set_remote(m, r, d) < 0) {
+			pr_err("mscp_set_remote: %s", priv_get_err());
+			return -1;
+		}
+
+		/* load paths and chunks to be transferred from checkpoint */
+		if (mscp_checkpoint_load(m, checkpoint_load) < 0) {
+			pr_err("mscp_checkpoint_load: %s", priv_get_err());
+			return -1;
+		}
 	}
 
 	if (dryrun) {
@@ -437,7 +476,20 @@ int main(int argc, char **argv)
 	pthread_cancel(tid_stat);
 	pthread_join(tid_stat, NULL);
 
+	print_stat(true);
+	print_cli("\n"); /* final output */
 out:
+	if (interrupted)
+		ret = 1;
+
+	if ((dryrun || ret != 0) && checkpoint_save) {
+		print_cli("save checkpoint to %s\n", checkpoint_save);
+		if (mscp_checkpoint_save(m, checkpoint_save) < 0) {
+			pr_err("mscp_checkpoint_save: %s", priv_get_err());
+			return -1;
+		}
+	}
+
 	mscp_cleanup(m);
 	mscp_free(m);
 
@@ -612,12 +664,6 @@ void print_stat(bool final)
 	}
 }
 
-void print_stat_thread_cleanup(void *arg)
-{
-	print_stat(true);
-	print_cli("\n"); /* final output */
-}
-
 void *print_stat_thread(void *arg)
 {
 	struct mscp_stats s;
@@ -627,15 +673,10 @@ void *print_stat_thread(void *arg)
 	gettimeofday(&x.start, NULL);
 	x.before = x.start;
 
-	pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
-	pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL);
-	pthread_cleanup_push(print_stat_thread_cleanup, NULL);
-
 	while (true) {
 		print_stat(false);
 		sleep(1);
 	}
 
-	pthread_cleanup_pop(1);
 	return NULL;
 }

src/mscp.c

@@ -6,10 +6,11 @@
 #include <semaphore.h>
 #include <sys/time.h>
 
-#include <list.h>
+#include <pool.h>
 #include <minmax.h>
 #include <ssh.h>
 #include <path.h>
+#include <checkpoint.h>
 #include <fileops.h>
 #include <atomic.h>
 #include <platform.h>
@@ -19,50 +20,43 @@
 
 #include <openbsd-compat/openbsd-compat.h>
 
+struct mscp_thread {
+	struct mscp *m;
+	sftp_session sftp;
+
+	/* attributes used by copy threads */
+	size_t copied_bytes;
+	int id;
+	int cpu;
+
+	/* thread-specific values */
+	pthread_t tid;
+	int ret;
+};
+
 struct mscp {
 	char *remote; /* remote host (and uername) */
 	int direction; /* copy direction */
+	char dst_path[PATH_MAX];
+
 	struct mscp_opts *opts;
 	struct mscp_ssh_opts *ssh_opts;
 
 	int *cores; /* usable cpu cores by COREMASK */
 	int nr_cores; /* length of array of cores */
 
-	sem_t *sem; /* semaphore for concurrent 
-						 * connecting ssh sessions */
+	sem_t *sem; /* semaphore for concurrent  connecting ssh sessions */
 
 	sftp_session first; /* first sftp session */
 
-	char dst_path[PATH_MAX];
-	struct list_head src_list;
-	struct list_head path_list;
-	struct chunk_pool cp;
+	pool *src_pool, *path_pool, *chunk_pool, *thread_pool;
 
-	pthread_t tid_scan; /* tid for scan thread */
-	int ret_scan; /* return code from scan thread */
+	size_t total_bytes; /* total_bytes to be copied */
+	bool chunk_pool_ready;
+#define chunk_pool_is_ready(m) ((m)->chunk_pool_ready)
+#define chunk_pool_set_ready(m, b) ((m)->chunk_pool_ready = b)
 
-	size_t total_bytes; /* total bytes to be transferred */
-
-	struct list_head thread_list;
-	rwlock thread_rwlock;
-};
-
-struct mscp_thread {
-	struct list_head list; /* mscp->thread_list */
-
-	struct mscp *m;
-	int id;
-	sftp_session sftp;
-	pthread_t tid;
-	int cpu;
-	size_t done;
-	bool finished;
-	int ret;
-};
-
-struct src {
-	struct list_head list; /* mscp->src_list */
-	char *path;
+	struct mscp_thread scan; /* mscp_thread for mscp_scan_thread() */
 };
 
 #define DEFAULT_MIN_CHUNK_SZ (64 << 20) /* 64MB */
@@ -208,54 +202,72 @@ static int validate_and_set_defaut_params(struct mscp_opts *o)
 	return 0;
 }
 
-struct mscp *mscp_init(const char *remote_host, int direction, struct mscp_opts *o,
-		       struct mscp_ssh_opts *s)
+int mscp_set_remote(struct mscp *m, const char *remote_host, int direction)
 {
-	struct mscp *m;
-	int n;
-
 	if (!remote_host) {
 		priv_set_errv("empty remote host");
-		return NULL;
+		return -1;
 	}
 
 	if (!(direction == MSCP_DIRECTION_L2R || direction == MSCP_DIRECTION_R2L)) {
 		priv_set_errv("invalid copy direction: %d", direction);
-		return NULL;
+		return -1;
 	}
 
+	if (!(m->remote = strdup(remote_host))) {
+		priv_set_errv("strdup: %s", strerrno());
+		return -1;
+	}
+	m->direction = direction;
+
+	return 0;
+}
+
+struct mscp *mscp_init(struct mscp_opts *o, struct mscp_ssh_opts *s)
+{
+	struct mscp *m;
+	int n;
+
 	set_print_severity(o->severity);
 
 	if (validate_and_set_defaut_params(o) < 0) {
 		return NULL;
 	}
 
-	m = malloc(sizeof(*m));
-	if (!m) {
+	if (!(m = malloc(sizeof(*m)))) {
 		priv_set_errv("malloc: %s", strerrno());
 		return NULL;
 	}
-
 	memset(m, 0, sizeof(*m));
-	INIT_LIST_HEAD(&m->src_list);
-	INIT_LIST_HEAD(&m->path_list);
-	chunk_pool_init(&m->cp);
+	m->opts = o;
+	m->ssh_opts = s;
+	chunk_pool_set_ready(m, false);
 
-	INIT_LIST_HEAD(&m->thread_list);
-	rwlock_init(&m->thread_rwlock);
+	if (!(m->src_pool = pool_new())) {
+		priv_set_errv("pool_new: %s", strerrno());
+		goto free_out;
+	}
+
+	if (!(m->path_pool = pool_new())) {
+		priv_set_errv("pool_new: %s", strerrno());
+		goto free_out;
+	}
+
+	if (!(m->chunk_pool = pool_new())) {
+		priv_set_errv("pool_new: %s", strerrno());
+		goto free_out;
+	}
+
+	if (!(m->thread_pool = pool_new())) {
+		priv_set_errv("pool_new: %s", strerrno());
+		goto free_out;
+	}
 
 	if ((m->sem = sem_create(o->max_startups)) == NULL) {
 		priv_set_errv("sem_create: %s", strerrno());
 		goto free_out;
 	}
 
-	m->remote = strdup(remote_host);
-	if (!m->remote) {
-		priv_set_errv("strdup: %s", strerrno());
-		goto free_out;
-	}
-	m->direction = direction;
-
 	if (o->coremask) {
 		if (expand_coremask(o->coremask, &m->cores, &m->nr_cores) < 0)
 			goto free_out;
@@ -269,12 +281,19 @@ struct mscp *mscp_init(const char *remote_host, int direction, struct mscp_opts
 		pr_notice("usable cpu cores:%s", b);
 	}
 
-	m->opts = o;
-	m->ssh_opts = s;
-
 	return m;
 
 free_out:
+	if (m->src_pool)
+		pool_free(m->src_pool);
+	if (m->path_pool)
+		pool_free(m->path_pool);
+	if (m->chunk_pool)
+		pool_free(m->chunk_pool);
+	if (m->thread_pool)
+		pool_free(m->thread_pool);
+	if (m->remote)
+		free(m->remote);
 	free(m);
 	return NULL;
 }
@@ -290,23 +309,15 @@ int mscp_connect(struct mscp *m)
 
 int mscp_add_src_path(struct mscp *m, const char *src_path)
 {
-	struct src *s;
-
-	s = malloc(sizeof(*s));
+	char *s = strdup(src_path);
 	if (!s) {
-		priv_set_errv("malloc: %s", strerrno());
+		priv_set_errv("strdup: %s", strerrno());
 		return -1;
 	}
-
-	memset(s, 0, sizeof(*s));
-	s->path = strdup(src_path);
-	if (!s->path) {
-		priv_set_errv("malloc: %s", strerrno());
-		free(s);
+	if (pool_push(m->src_pool, s) < 0) {
+		priv_set_errv("pool_push: %s", strerrno());
 		return -1;
 	}
-
-	list_add_tail(&s->list, &m->src_list);
 	return 0;
 }
 
@@ -342,19 +353,19 @@ static int get_page_mask(void)
 static void mscp_stop_copy_thread(struct mscp *m)
 {
 	struct mscp_thread *t;
-
-	RWLOCK_READ_ACQUIRE(&m->thread_rwlock);
-	list_for_each_entry(t, &m->thread_list, list) {
-		if (!t->finished)
+	unsigned int idx;
+	pool_lock(m->thread_pool);
+	pool_for_each(m->thread_pool, t, idx) {
+		if (t->tid)
 			pthread_cancel(t->tid);
 	}
-	RWLOCK_RELEASE();
+	pool_unlock(m->thread_pool);
 }
 
 static void mscp_stop_scan_thread(struct mscp *m)
 {
-	if (m->tid_scan)
-		pthread_cancel(m->tid_scan);
+	if (m->scan.tid)
+		pthread_cancel(m->scan.tid);
 }
 
 void mscp_stop(struct mscp *m)
@@ -365,25 +376,23 @@ void mscp_stop(struct mscp *m)
 
 void *mscp_scan_thread(void *arg)
 {
-	struct mscp *m = arg;
+	struct mscp_thread *t = arg;
+	struct mscp *m = t->m;
 	sftp_session src_sftp = NULL, dst_sftp = NULL;
 	struct path_resolve_args a;
-	struct list_head tmp;
 	struct path *p;
-	struct src *s;
 	struct stat ss, ds;
+	char *src_path;
 	glob_t pglob;
 	int n;
 
-	m->ret_scan = 0;
-
 	switch (m->direction) {
 	case MSCP_DIRECTION_L2R:
 		src_sftp = NULL;
-		dst_sftp = m->first;
+		dst_sftp = t->sftp;
 		break;
 	case MSCP_DIRECTION_R2L:
-		src_sftp = m->first;
+		src_sftp = t->sftp;
 		dst_sftp = NULL;
 		break;
 	default:
@@ -395,7 +404,7 @@ void *mscp_scan_thread(void *arg)
 	memset(&a, 0, sizeof(a));
 	a.total_bytes = &m->total_bytes;
 
-	if (list_count(&m->src_list) > 1)
+	if (pool_size(m->src_pool) > 1)
 		a.dst_path_should_dir = true;
 
 	if (mscp_stat(m->dst_path, &ds, dst_sftp) == 0) {
@@ -403,7 +412,8 @@ void *mscp_scan_thread(void *arg)
 			a.dst_path_is_dir = true;
 	}
 
-	a.cp = &m->cp;
+	a.path_pool = m->path_pool;
+	a.chunk_pool = m->chunk_pool;
 	a.nr_conn = m->opts->nr_threads;
 	a.min_chunk_sz = m->opts->min_chunk_sz;
 	a.max_chunk_sz = m->opts->max_chunk_sz;
@@ -411,55 +421,56 @@ void *mscp_scan_thread(void *arg)
 
 	pr_info("start to walk source path(s)");
 
-	/* walk a src_path recusively, and resolve path->dst_path for each src */
-	list_for_each_entry(s, &m->src_list, list) {
+	/* walk each src_path recusively, and resolve path->dst_path for each src */
+	pool_iter_for_each(m->src_pool, src_path) {
 		memset(&pglob, 0, sizeof(pglob));
-		if (mscp_glob(s->path, GLOB_NOCHECK, &pglob, src_sftp) < 0) {
+		if (mscp_glob(src_path, GLOB_NOCHECK, &pglob, src_sftp) < 0) {
 			pr_err("mscp_glob: %s", strerrno());
 			goto err_out;
 		}
 
 		for (n = 0; n < pglob.gl_pathc; n++) {
 			if (mscp_stat(pglob.gl_pathv[n], &ss, src_sftp) < 0) {
-				pr_err("stat: %s %s", s->path, strerrno());
+				pr_err("stat: %s %s", src_path, strerrno());
 				goto err_out;
 			}
 
 			if (!a.dst_path_should_dir && pglob.gl_pathc > 1)
-				a.dst_path_should_dir = true; /* we have over 1 src */
+				a.dst_path_should_dir = true; /* we have over 1 srces */
 
 			/* set path specific args */
 			a.src_path = pglob.gl_pathv[n];
 			a.dst_path = m->dst_path;
 			a.src_path_is_dir = S_ISDIR(ss.st_mode);
 
-			INIT_LIST_HEAD(&tmp);
-			if (walk_src_path(src_sftp, pglob.gl_pathv[n], &tmp, &a) < 0)
+			if (walk_src_path(src_sftp, pglob.gl_pathv[n], &a) < 0)
 				goto err_out;
-
-			list_splice_tail(&tmp, m->path_list.prev);
 		}
 		mscp_globfree(&pglob);
 	}
 
 	pr_info("walk source path(s) done");
-	chunk_pool_set_filled(&m->cp);
-	m->ret_scan = 0;
+	t->ret = 0;
+	chunk_pool_set_ready(m, true);
 	return NULL;
 
 err_out:
-	chunk_pool_set_filled(&m->cp);
-	m->ret_scan = -1;
+	t->ret = -1;
+	chunk_pool_set_ready(m, true);
 	return NULL;
 }
 
 int mscp_scan(struct mscp *m)
 {
-	int ret = pthread_create(&m->tid_scan, NULL, mscp_scan_thread, m);
-	if (ret < 0) {
+	struct mscp_thread *t = &m->scan;
+	int ret;
+
+	memset(t, 0, sizeof(*t));
+	t->m = m;
+	t->sftp = m->first;
+
+	if ((ret = pthread_create(&t->tid, NULL, mscp_scan_thread, t)) < 0) {
 		priv_set_err("pthread_create: %d", ret);
-		m->tid_scan = 0;
-		mscp_stop(m);
 		return -1;
 	}
 
@@ -468,8 +479,7 @@ int mscp_scan(struct mscp *m)
 	 * finished. If the number of chunks are smaller than
 	 * nr_threads, we adjust nr_threads to the number of chunks.
 	 */
-	while (!chunk_pool_is_filled(&m->cp) &&
-	       chunk_pool_size(&m->cp) < m->opts->nr_threads)
+	while (!chunk_pool_is_ready(m) && pool_size(m->chunk_pool) < m->opts->nr_threads)
 		usleep(100);
 
 	return 0;
@@ -477,14 +487,45 @@ int mscp_scan(struct mscp *m)
 
 int mscp_scan_join(struct mscp *m)
 {
-	if (m->tid_scan) {
-		pthread_join(m->tid_scan, NULL);
-		m->tid_scan = 0;
-		return m->ret_scan;
+	struct mscp_thread *t = &m->scan;
+	if (t->tid) {
+		pthread_join(t->tid, NULL);
+		t->tid = 0;
+		return t->ret;
 	}
 	return 0;
 }
 
+int mscp_checkpoint_get_remote(const char *pathname, char *remote, size_t len, int *dir)
+{
+	return checkpoint_load_remote(pathname, remote, len, dir);
+}
+
+int mscp_checkpoint_load(struct mscp *m, const char *pathname)
+{
+	struct chunk *c;
+	unsigned int i;
+
+	if (checkpoint_load_paths(pathname, m->path_pool, m->chunk_pool) < 0)
+		return -1;
+
+	/* totaling up bytes to be transferred and set chunk_pool is
+	 * ready instead of the mscp_scan thread */
+	m->total_bytes = 0;
+	pool_for_each(m->chunk_pool, c, i) {
+		m->total_bytes += c->len;
+	}
+	chunk_pool_set_ready(m, true);
+
+	return 0;
+}
+
+int mscp_checkpoint_save(struct mscp *m, const char *pathname)
+{
+	return checkpoint_save(pathname, m->direction, m->ssh_opts->login_name,
+			       m->remote, m->path_pool, m->chunk_pool);
+}
+
 static void *mscp_copy_thread(void *arg);
 
 static struct mscp_thread *mscp_copy_thread_spawn(struct mscp *m, int id)
@@ -492,9 +533,8 @@ static struct mscp_thread *mscp_copy_thread_spawn(struct mscp *m, int id)
 	struct mscp_thread *t;
 	int ret;
 
-	t = malloc(sizeof(*t));
-	if (!t) {
-		priv_set_errv("malloc: %s,", strerrno());
+	if (!(t = malloc(sizeof(*t)))) {
+		priv_set_errv("malloc: %s", strerrno());
 		return NULL;
 	}
 
@@ -506,8 +546,7 @@ static struct mscp_thread *mscp_copy_thread_spawn(struct mscp *m, int id)
 	else
 		t->cpu = m->cores[id % m->nr_cores];
 
-	ret = pthread_create(&t->tid, NULL, mscp_copy_thread, t);
-	if (ret < 0) {
+	if ((ret = pthread_create(&t->tid, NULL, mscp_copy_thread, t)) < 0) {
 		priv_set_errv("pthread_create: %d", ret);
 		free(t);
 		return NULL;
@@ -521,7 +560,7 @@ int mscp_start(struct mscp *m)
 	struct mscp_thread *t;
 	int n, ret = 0;
 
-	if ((n = chunk_pool_size(&m->cp)) < m->opts->nr_threads) {
+	if ((n = pool_size(m->chunk_pool)) < m->opts->nr_threads) {
 		pr_notice("we have %d chunk(s), set number of connections to %d", n, n);
 		m->opts->nr_threads = n;
 	}
@@ -530,10 +569,10 @@ int mscp_start(struct mscp *m)
 		t = mscp_copy_thread_spawn(m, n);
 		if (!t)
 			break;
-
-		RWLOCK_WRITE_ACQUIRE(&m->thread_rwlock);
-		list_add_tail(&t->list, &m->thread_list);
-		RWLOCK_RELEASE();
+		if (pool_push_lock(m->thread_pool, t) < 0) {
+			priv_set_errv("pool_push_lock: %s", strerrno());
+			break;
+		}
 	}
 
 	return n;
@@ -543,17 +582,20 @@ int mscp_join(struct mscp *m)
 {
 	struct mscp_thread *t;
 	struct path *p;
-	size_t done = 0, nr_copied = 0, nr_tobe_copied = 0;
+	unsigned int idx;
+	size_t total_copied_bytes = 0, nr_copied = 0, nr_tobe_copied = 0;
 	int n, ret = 0;
 
 	/* waiting for scan thread joins... */
 	ret = mscp_scan_join(m);
 
 	/* waiting for copy threads join... */
-	RWLOCK_READ_ACQUIRE(&m->thread_rwlock);
-	list_for_each_entry(t, &m->thread_list, list) {
+	pool_for_each(m->thread_pool, t, idx) {
 		pthread_join(t->tid, NULL);
-		done += t->done;
+	}
+
+	pool_for_each(m->thread_pool, t, idx) {
+		total_copied_bytes += t->copied_bytes;
 		if (t->ret != 0)
 			ret = t->ret;
 		if (t->sftp) {
@@ -561,23 +603,22 @@ int mscp_join(struct mscp *m)
 			t->sftp = NULL;
 		}
 	}
-	RWLOCK_RELEASE();
-
-	if (m->first) {
-		ssh_sftp_close(m->first);
-		m->first = NULL;
-	}
 
 	/* count up number of transferred files */
-	list_for_each_entry(p, &m->path_list, list) {
+	pool_iter_for_each(m->path_pool, p) {
 		nr_tobe_copied++;
 		if (p->state == FILE_STATE_DONE) {
 			nr_copied++;
 		}
 	}
 
-	pr_notice("%lu/%lu bytes copied for %lu/%lu files", done, m->total_bytes,
-		  nr_copied, nr_tobe_copied);
+	if (m->first) {
+		ssh_sftp_close(m->first);
+		m->first = NULL;
+	}
+
+	pr_notice("%lu/%lu bytes copied for %lu/%lu files", total_copied_bytes,
+		  m->total_bytes, nr_copied, nr_tobe_copied);
 
 	return ret;
 }
@@ -599,25 +640,17 @@ static void wait_for_interval(int interval)
 	next = now + interval * 1000000;
 }
 
-static void mscp_copy_thread_cleanup(void *arg)
-{
-	struct mscp_thread *t = arg;
-	t->finished = true;
-}
-
 void *mscp_copy_thread(void *arg)
 {
 	sftp_session src_sftp, dst_sftp;
 	struct mscp_thread *t = arg;
 	struct mscp *m = t->m;
 	struct chunk *c;
-	bool nomore;
+	bool next_chunk_exist;
 
 	/* when error occurs, each thread prints error messages
 	 * immediately with pr_* functions. */
 
-	pthread_cleanup_push(mscp_copy_thread_cleanup, t);
-
 	if (t->cpu > -1) {
 		if (set_thread_affinity(pthread_self(), t->cpu) < 0) {
 			pr_err("set_thread_affinity: %s", priv_get_err());
@@ -631,7 +664,7 @@ void *mscp_copy_thread(void *arg)
 		goto err_out;
 	}
 
-	if (!(nomore = chunk_pool_is_empty(&m->cp))) {
+	if ((next_chunk_exist = pool_iter_has_next_lock(m->chunk_pool))) {
 		if (m->opts->interval > 0)
 			wait_for_interval(m->opts->interval);
 		pr_notice("thread[%d]: connecting to %s", t->id, m->remote);
@@ -643,7 +676,7 @@ void *mscp_copy_thread(void *arg)
 		goto err_out;
 	}
 
-	if (nomore) {
+	if (!next_chunk_exist) {
 		pr_notice("thread[%d]: no more connections needed", t->id);
 		goto out;
 	}
@@ -668,23 +701,22 @@ void *mscp_copy_thread(void *arg)
 	}
 
 	while (1) {
-		c = chunk_pool_pop(&m->cp);
-		if (c == CHUNK_POP_WAIT) {
-			usleep(100); /* XXX: hard code */
-			continue;
-		}
-
-		if (!c)
+		c = pool_iter_next_lock(m->chunk_pool);
+		if (c == NULL) {
+			if (!chunk_pool_is_ready(m)) {
+				/* a new chunk will be added. wait for it. */
+				usleep(100);
+				continue;
+			}
 			break; /* no more chunks */
+		}
 
 		if ((t->ret = copy_chunk(c, src_sftp, dst_sftp, m->opts->nr_ahead,
 					 m->opts->buf_sz, m->opts->preserve_ts,
-					 &t->done)) < 0)
+					 &t->copied_bytes)) < 0)
 			break;
 	}
 
-	pthread_cleanup_pop(1);
-
 	if (t->ret < 0) {
 		pr_err("thread[%d]: copy failed: %s -> %s, 0x%010lx-0x%010lx, %s", t->id,
 		       c->p->path, c->p->dst_path, c->off, c->off + c->len,
@@ -694,39 +726,15 @@ void *mscp_copy_thread(void *arg)
 	return NULL;
 
 err_out:
-	t->finished = true;
 	t->ret = -1;
 	return NULL;
 out:
-	t->finished = true;
 	t->ret = 0;
 	return NULL;
 }
 
 /* cleanup-related functions */
 
-static void list_free_src(struct list_head *list)
-{
-	struct src *s;
-	s = list_entry(list, typeof(*s), list);
-	free(s->path);
-	free(s);
-}
-
-static void list_free_path(struct list_head *list)
-{
-	struct path *p;
-	p = list_entry(list, typeof(*p), list);
-	free_path(p);
-}
-
-static void list_free_thread(struct list_head *list)
-{
-	struct mscp_thread *t;
-	t = list_entry(list, typeof(*t), list);
-	free(t);
-}
-
 void mscp_cleanup(struct mscp *m)
 {
 	if (m->first) {
@@ -734,23 +742,17 @@ void mscp_cleanup(struct mscp *m)
 		m->first = NULL;
 	}
 
-	list_free_f(&m->src_list, list_free_src);
-	INIT_LIST_HEAD(&m->src_list);
-
-	list_free_f(&m->path_list, list_free_path);
-	INIT_LIST_HEAD(&m->path_list);
-
-	chunk_pool_release(&m->cp);
-	chunk_pool_init(&m->cp);
-
-	RWLOCK_WRITE_ACQUIRE(&m->thread_rwlock);
-	list_free_f(&m->thread_list, list_free_thread);
-	RWLOCK_RELEASE();
+	pool_zeroize(m->src_pool, free);
+	pool_zeroize(m->path_pool, (pool_map_f)free_path);
+	pool_zeroize(m->chunk_pool, free);
+	pool_zeroize(m->thread_pool, free);
 }
 
 void mscp_free(struct mscp *m)
 {
-	mscp_cleanup(m);
+	pool_destroy(m->src_pool, free);
+	pool_destroy(m->path_pool, (pool_map_f)free_path);
+
 	if (m->remote)
 		free(m->remote);
 	if (m->cores)
@@ -762,20 +764,13 @@ void mscp_free(struct mscp *m)
 
 void mscp_get_stats(struct mscp *m, struct mscp_stats *s)
 {
-	int nr_finished = 0, nr_threads = 0;
 	struct mscp_thread *t;
+	unsigned int idx;
 
 	s->total = m->total_bytes;
 	s->done = 0;
 
-	RWLOCK_READ_ACQUIRE(&m->thread_rwlock);
-	list_for_each_entry(t, &m->thread_list, list) {
-		nr_threads++;
-		s->done += t->done;
-		if (t->finished)
-			nr_finished++;
+	pool_for_each(m->thread_pool, t, idx) {
+		s->done += t->copied_bytes;
 	}
-	RWLOCK_RELEASE();
-
-	s->finished = nr_threads > 0 ? (nr_finished == nr_threads) : false;
 }

src/path.c

@@ -9,90 +9,11 @@
 #include <ssh.h>
 #include <minmax.h>
 #include <fileops.h>
-#include <list.h>
 #include <atomic.h>
 #include <path.h>
 #include <strerrno.h>
 #include <print.h>
 
-/* chunk pool operations */
-#define CHUNK_POOL_STATE_FILLING 0
-#define CHUNK_POOL_STATE_FILLED 1
-
-void chunk_pool_init(struct chunk_pool *cp)
-{
-	memset(cp, 0, sizeof(*cp));
-	INIT_LIST_HEAD(&cp->list);
-	lock_init(&cp->lock);
-	cp->state = CHUNK_POOL_STATE_FILLING;
-}
-
-static void chunk_pool_add(struct chunk_pool *cp, struct chunk *c)
-{
-	LOCK_ACQUIRE(&cp->lock);
-	list_add_tail(&c->list, &cp->list);
-	cp->count += 1;
-	LOCK_RELEASE();
-}
-
-void chunk_pool_set_filled(struct chunk_pool *cp)
-{
-	cp->state = CHUNK_POOL_STATE_FILLED;
-}
-
-bool chunk_pool_is_filled(struct chunk_pool *cp)
-{
-	return (cp->state == CHUNK_POOL_STATE_FILLED);
-}
-
-size_t chunk_pool_size(struct chunk_pool *cp)
-{
-	return cp->count;
-}
-
-bool chunk_pool_is_empty(struct chunk_pool *cp)
-{
-	return list_empty(&cp->list);
-}
-
-struct chunk *chunk_pool_pop(struct chunk_pool *cp)
-{
-	struct list_head *first;
-	struct chunk *c = NULL;
-
-	LOCK_ACQUIRE(&cp->lock);
-	first = cp->list.next;
-	if (list_empty(&cp->list)) {
-		if (!chunk_pool_is_filled(cp))
-			c = CHUNK_POP_WAIT;
-		else
-			c = NULL; /* no more chunks */
-	} else {
-		c = list_entry(first, struct chunk, list);
-		list_del(first);
-	}
-	LOCK_RELEASE();
-
-	/* return CHUNK_POP_WAIT would be a rare case, because it
-	 * means copying over SSH is faster than traversing
-	 * local/remote file paths.
-	 */
-
-	return c;
-}
-
-static void chunk_free(struct list_head *list)
-{
-	struct chunk *c;
-	c = list_entry(list, typeof(*c), list);
-	free(c);
-}
-
-void chunk_pool_release(struct chunk_pool *cp)
-{
-	list_free_f(&cp->list, chunk_free);
-}
-
 /* paths of copy source resoltion */
 static char *resolve_dst_path(const char *src_file_path, struct path_resolve_args *a)
 {
@@ -157,7 +78,7 @@ static char *resolve_dst_path(const char *src_file_path, struct path_resolve_arg
 }
 
 /* chunk preparation */
-static struct chunk *alloc_chunk(struct path *p)
+struct chunk *alloc_chunk(struct path *p, size_t off, size_t len)
 {
 	struct chunk *c;
 
@@ -168,43 +89,48 @@ static struct chunk *alloc_chunk(struct path *p)
 	memset(c, 0, sizeof(*c));
 
 	c->p = p;
-	c->off = 0;
-	c->len = 0;
+	c->off = off;
+	c->len = len;
+	c->state = CHUNK_STATE_INIT;
 	refcnt_inc(&p->refcnt);
 	return c;
 }
 
-static int resolve_chunk(struct path *p, struct path_resolve_args *a)
+static int resolve_chunk(struct path *p, size_t size, struct path_resolve_args *a)
 {
 	struct chunk *c;
-	size_t chunk_sz;
-	size_t size;
+	size_t chunk_sz, off, len;
+	size_t remaind;
 
-	if (p->size <= a->min_chunk_sz)
-		chunk_sz = p->size;
+	if (size <= a->min_chunk_sz)
+		chunk_sz = size;
 	else if (a->max_chunk_sz)
 		chunk_sz = a->max_chunk_sz;
 	else {
-		chunk_sz = (p->size - (p->size % a->nr_conn)) / a->nr_conn;
+		chunk_sz = (size - (size % a->nr_conn)) / a->nr_conn;
 		chunk_sz &= ~a->chunk_align; /* align with page_sz */
 		if (chunk_sz <= a->min_chunk_sz)
 			chunk_sz = a->min_chunk_sz;
 	}
 
-	/* for (size = f->size; size > 0;) does not create a file
-         * (chunk) when file size is 0. This do {} while (size > 0)
-         * creates just open/close a 0-byte file.
+	/* for (size = size; size > 0;) does not create a file (chunk)
+         * when file size is 0. This do {} while (remaind > 0) creates
+         * just open/close a 0-byte file.
          */
-	size = p->size;
+	remaind = size;
 	do {
-		c = alloc_chunk(p);
+		off = size - remaind;
+		len = remaind < chunk_sz ? remaind : chunk_sz;
+		c = alloc_chunk(p, off, len);
 		if (!c)
 			return -1;
-		c->off = p->size - size;
-		c->len = size < chunk_sz ? size : chunk_sz;
-		size -= c->len;
-		chunk_pool_add(a->cp, c);
-	} while (size > 0);
+
+		remaind -= len;
+		if (pool_push_lock(a->chunk_pool, c) < 0) {
+			pr_err("pool_push_lock: %s", strerrno());
+			return -1;
+		}
+	} while (remaind > 0);
 
 	return 0;
 }
@@ -218,38 +144,54 @@ void free_path(struct path *p)
 	free(p);
 }
 
-static int append_path(sftp_session sftp, const char *path, struct stat st,
-		       struct list_head *path_list, struct path_resolve_args *a)
+struct path *alloc_path(char *path, char *dst_path)
 {
 	struct path *p;
 
 	if (!(p = malloc(sizeof(*p)))) {
-		priv_set_errv("failed to allocate memory: %s", strerrno());
+		pr_err("malloc: %s", strerrno());
+		return NULL;
+	}
+	memset(p, 0, sizeof(*p));
+
+	p->path = path;
+	p->dst_path = dst_path;
+	p->state = FILE_STATE_INIT;
+	lock_init(&p->lock);
+	p->data = 0;
+
+	return p;
+}
+
+static int append_path(sftp_session sftp, const char *path, struct stat st,
+		       struct path_resolve_args *a)
+{
+	struct path *p;
+	char *src, *dst;
+
+	if (!(src = strdup(path))) {
+		pr_err("strdup: %s", strerrno());
 		return -1;
 	}
 
-	memset(p, 0, sizeof(*p));
-	INIT_LIST_HEAD(&p->list);
-	p->path = strndup(path, PATH_MAX);
-	if (!p->path) {
-		pr_err("strndup: %s", strerrno());
-		goto free_out;
+	if (!(dst = resolve_dst_path(src, a))) {
+		free(src);
+		return -1;
 	}
-	p->size = st.st_size;
-	p->mode = st.st_mode;
-	p->state = FILE_STATE_INIT;
-	lock_init(&p->lock);
 
-	p->dst_path = resolve_dst_path(p->path, a);
-	if (!p->dst_path)
-		goto free_out;
+	if (!(p = alloc_path(src, dst)))
+		return -1;
 
-	if (resolve_chunk(p, a) < 0)
+	if (resolve_chunk(p, st.st_size, a) < 0)
 		return -1; /* XXX: do not free path becuase chunk(s)
 			    * was added to chunk pool already */
 
-	list_add_tail(&p->list, path_list);
-	*a->total_bytes += p->size;
+	if (pool_push_lock(a->path_pool, p) < 0) {
+		pr_err("pool_push: %s", strerrno());
+		goto free_out;
+	}
+
+	*a->total_bytes += st.st_size;
 
 	return 0;
 
@@ -269,7 +211,7 @@ static bool check_path_should_skip(const char *path)
 }
 
 static int walk_path_recursive(sftp_session sftp, const char *path,
-			       struct list_head *path_list, struct path_resolve_args *a)
+			       struct path_resolve_args *a)
 {
 	char next_path[PATH_MAX + 1];
 	struct dirent *e;
@@ -284,7 +226,7 @@ static int walk_path_recursive(sftp_session sftp, const char *path,
 
 	if (S_ISREG(st.st_mode)) {
 		/* this path is regular file. it is to be copied */
-		return append_path(sftp, path, st, path_list, a);
+		return append_path(sftp, path, st, a);
 	}
 
 	if (!S_ISDIR(st.st_mode))
@@ -306,7 +248,7 @@ static int walk_path_recursive(sftp_session sftp, const char *path,
 			continue;
 		}
 
-		walk_path_recursive(sftp, next_path, path_list, a);
+		walk_path_recursive(sftp, next_path, a);
 		/* do not stop even when walk_path_recursive returns
 		 * -1 due to an unreadable file. go to a next
 		 * file. Thus, do not pass error messages via
@@ -321,9 +263,9 @@ static int walk_path_recursive(sftp_session sftp, const char *path,
 }
 
 int walk_src_path(sftp_session src_sftp, const char *src_path,
-		  struct list_head *path_list, struct path_resolve_args *a)
+		  struct path_resolve_args *a)
 {
-	return walk_path_recursive(src_sftp, src_path, path_list, a);
+	return walk_path_recursive(src_sftp, src_path, a);
 }
 
 /* based on
@@ -369,7 +311,7 @@ next:
 	 * end. see https://bugzilla.mindrot.org/show_bug.cgi?id=3431 */
 	f = mscp_open(p->dst_path, O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR, sftp);
 	if (!f) {
-		priv_set_errv("mscp_open %s: %s\n", p->dst_path, strerrno());
+		priv_set_errv("mscp_open %s: %s", p->dst_path, strerrno());
 		return -1;
 	}
 
@@ -424,8 +366,8 @@ static int copy_chunk_l2r(struct chunk *c, int fd, sftp_file sf, int nr_ahead, i
 		reqs[idx].len = sftp_async_write(sf, read_to_buf, reqs[idx].len, &fd,
 						 &reqs[idx].id);
 		if (reqs[idx].len < 0) {
-			priv_set_errv("sftp_async_write: %s or %s",
-				      sftp_get_ssh_error(sf->sftp), strerrno());
+			priv_set_errv("sftp_async_write: %s",
+				      sftp_get_ssh_error(sf->sftp));
 			return -1;
 		}
 		thrown -= reqs[idx].len;
@@ -452,8 +394,8 @@ static int copy_chunk_l2r(struct chunk *c, int fd, sftp_file sf, int nr_ahead, i
 		reqs[idx].len = sftp_async_write(sf, read_to_buf, reqs[idx].len, &fd,
 						 &reqs[idx].id);
 		if (reqs[idx].len < 0) {
-			priv_set_errv("sftp_async_write: %s or %s",
-				      sftp_get_ssh_error(sf->sftp), strerrno());
+			priv_set_errv("sftp_async_write: %s",
+				      sftp_get_ssh_error(sf->sftp));
 			return -1;
 		}
 		thrown -= reqs[idx].len;
@@ -562,8 +504,7 @@ int copy_chunk(struct chunk *c, sftp_session src_sftp, sftp_session dst_sftp,
 	/* open src */
 	flags = O_RDONLY;
 	mode = S_IRUSR;
-	s = mscp_open(c->p->path, flags, mode, src_sftp);
-	if (!s) {
+	if (!(s = mscp_open(c->p->path, flags, mode, src_sftp))) {
 		priv_set_errv("mscp_open: %s: %s", c->p->path, strerrno());
 		return -1;
 	}
@@ -575,8 +516,7 @@ int copy_chunk(struct chunk *c, sftp_session src_sftp, sftp_session dst_sftp,
 	/* open dst */
 	flags = O_WRONLY;
 	mode = S_IRUSR | S_IWUSR;
-	d = mscp_open(c->p->dst_path, flags, mode, dst_sftp);
-	if (!d) {
+	if (!(d = mscp_open(c->p->dst_path, flags, mode, dst_sftp))) {
 		mscp_close(s);
 		priv_set_errv("mscp_open: %s: %s", c->p->dst_path, strerrno());
 		return -1;
@@ -586,6 +526,7 @@ int copy_chunk(struct chunk *c, sftp_session src_sftp, sftp_session dst_sftp,
 		return -1;
 	}
 
+	c->state = CHUNK_STATE_COPING;
 	pr_debug("copy chunk start: %s 0x%lx-0x%lx", c->p->path, c->off, c->off + c->len);
 
 	ret = _copy_chunk(c, s, d, nr_ahead, buf_sz, counter);
@@ -613,5 +554,8 @@ int copy_chunk(struct chunk *c, sftp_session src_sftp, sftp_session dst_sftp,
 		pr_info("copy done: %s", c->p->path);
 	}
 
+	if (ret == 0)
+		c->state = CHUNK_STATE_DONE;
+
 	return ret;
 }

src/path.h

@@ -6,66 +6,37 @@
 #include <fcntl.h>
 #include <dirent.h>
 #include <sys/stat.h>
-
-#include <list.h>
+#include <pool.h>
 #include <atomic.h>
 #include <ssh.h>
 
 struct path {
-	struct list_head list; /* mscp->path_list */
-
 	char *path; /* file path */
-	size_t size; /* size of file on this path */
-	mode_t mode; /* permission */
-
 	char *dst_path; /* copy dst path */
 
-	int state;
+	refcnt refcnt; /* number of associated chunks */
 	lock lock;
-	refcnt refcnt;
-};
+	int state;
 #define FILE_STATE_INIT 0
 #define FILE_STATE_OPENED 1
 #define FILE_STATE_DONE 2
 
-struct chunk {
-	struct list_head list; /* chunk_pool->list */
+	uint64_t data; /* used by other components, i.e., checkpoint */
+};
 
+struct path *alloc_path(char *path, char *dst_path);
+
+struct chunk {
 	struct path *p;
 	size_t off; /* offset of this chunk on the file on path p */
 	size_t len; /* length of this chunk */
-	size_t done; /* copied bytes for this chunk by a thread */
-};
-
-struct chunk_pool {
-	struct list_head list; /* list of struct chunk */
-	size_t count;
-	lock lock;
 	int state;
+#define CHUNK_STATE_INIT 0
+#define CHUNK_STATE_COPING 1
+#define CHUNK_STATE_DONE 2
 };
 
-/* initialize chunk pool */
-void chunk_pool_init(struct chunk_pool *cp);
-
-/* acquire a chunk from pool. return value is NULL indicates no more
- * chunk, GET_CHUNK_WAIT means caller should waits until a chunk is
- * added, or pointer to chunk.
- */
-struct chunk *chunk_pool_pop(struct chunk_pool *cp);
-#define CHUNK_POP_WAIT ((void *)-1)
-
-/* set and check fillingchunks to this pool has finished */
-void chunk_pool_set_filled(struct chunk_pool *cp);
-bool chunk_pool_is_filled(struct chunk_pool *cp);
-
-/* return number of chunks in the pool */
-size_t chunk_pool_size(struct chunk_pool *cp);
-
-/* return true if chunk pool is empty (all chunks are already poped) */
-bool chunk_pool_is_empty(struct chunk_pool *cp);
-
-/* free chunks in the chunk_pool */
-void chunk_pool_release(struct chunk_pool *cp);
+struct chunk *alloc_chunk(struct path *p, size_t off, size_t len);
 
 struct path_resolve_args {
 	size_t *total_bytes;
@@ -78,16 +49,17 @@ struct path_resolve_args {
 	bool dst_path_should_dir;
 
 	/* args to resolve chunks for a path */
-	struct chunk_pool *cp;
+	pool *path_pool;
+	pool *chunk_pool;
 	int nr_conn;
 	size_t min_chunk_sz;
 	size_t max_chunk_sz;
 	size_t chunk_align;
 };
 
-/* recursivly walk through src_path and fill path_list for each file */
+/* walk src_path recursivly and fill a->path_pool with found files */
 int walk_src_path(sftp_session src_sftp, const char *src_path,
-		  struct list_head *path_list, struct path_resolve_args *a);
+		  struct path_resolve_args *a);
 
 /* free struct path */
 void free_path(struct path *p);
@@ -96,7 +68,4 @@ void free_path(struct path *p);
 int copy_chunk(struct chunk *c, sftp_session src_sftp, sftp_session dst_sftp,
 	       int nr_ahead, int buf_sz, bool preserve_ts, size_t *counter);
 
-/* just print contents. just for debugging */
-void path_dump(struct list_head *path_list);
-
 #endif /* _PATH_H_ */

src/platform.c

@@ -22,6 +22,7 @@
 #error unsupported platform
 #endif
 
+#include <config.h>
 #include <platform.h>
 #include <strerrno.h>
 #include <print.h>

src/platform.h

@@ -4,6 +4,7 @@
 
 #include <pthread.h>
 #include <semaphore.h>
+#include <stdint.h>
 
 int nr_cpus(void);
 int set_thread_affinity(pthread_t tid, int core);
@@ -20,4 +21,25 @@ int setutimes(const char *path, struct timespec atime, struct timespec mtime);
 sem_t *sem_create(int value);
 int sem_release(sem_t *sem);
 
+#ifdef HAVE_HTONLL
+#include <arpa/inet.h> /* Apple has htonll and ntohll in arpa/inet.h */
+#endif
+
+/* copied from libssh: libssh/include/libssh/priv.h*/
+#ifndef HAVE_HTONLL
+#ifdef WORDS_BIGENDIAN
+#define htonll(x) (x)
+#else
+#define htonll(x) (((uint64_t)htonl((x)&0xFFFFFFFF) << 32) | htonl((x) >> 32))
+#endif
+#endif
+
+#ifndef HAVE_NTOHLL
+#ifdef WORDS_BIGENDIAN
+#define ntohll(x) (x)
+#else
+#define ntohll(x) (((uint64_t)ntohl((x)&0xFFFFFFFF) << 32) | ntohl((x) >> 32))
+#endif
+#endif
+
 #endif /* _PLATFORM_H_ */

src/pool.c

@@ -0,0 +1,123 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
+#include <string.h>
+#include <stdlib.h>
+#include <pool.h>
+
+#define DEFAULT_START_SIZE 16
+
+pool *pool_new(void)
+{
+	pool *p;
+	p = malloc(sizeof(*p));
+	if (!p)
+		return NULL;
+	memset(p, 0, sizeof(*p));
+
+	p->array = calloc(DEFAULT_START_SIZE, sizeof(void *));
+	if (!p->array) {
+		free(p);
+		return NULL;
+	}
+
+	p->len = DEFAULT_START_SIZE;
+	p->num = 0;
+	lock_init(&p->lock);
+	return p;
+}
+
+void pool_free(pool *p)
+{
+	if (p->array) {
+		free(p->array);
+		p->array = NULL;
+	}
+	free(p);
+}
+
+void pool_zeroize(pool *p, pool_map_f f)
+{
+	void *v;
+	pool_iter_for_each(p, v) {
+		f(v);
+	}
+	p->num = 0;
+}
+
+void pool_destroy(pool *p, pool_map_f f)
+{
+	pool_zeroize(p, f);
+	pool_free(p);
+}
+
+int pool_push(pool *p, void *v)
+{
+	if (p->num == p->len) {
+		/* expand array */
+		size_t newlen = p->len * 2;
+		void *new = realloc(p->array, newlen * sizeof(void *));
+		if (new == NULL)
+			return -1;
+		p->len = newlen;
+		p->array = new;
+	}
+	p->array[p->num] = v;
+	__sync_synchronize();
+	p->num++;
+	return 0;
+}
+
+int pool_push_lock(pool *p, void *v)
+{
+	int ret = -1;
+	pool_lock(p);
+	ret = pool_push(p, v);
+	pool_unlock(p);
+	return ret;
+}
+
+void *pool_pop(pool *p)
+{
+	return p->num == 0 ? NULL : p->array[--p->num];
+}
+
+void *pool_pop_lock(pool *p)
+{
+	void *v;
+	pool_lock(p);
+	v = pool_pop(p);
+	pool_unlock(p);
+	return v;
+}
+
+void *pool_get(pool *p, unsigned int idx)
+{
+	return p->num <= idx ? NULL : p->array[idx];
+}
+
+void *pool_iter_next(pool *p)
+{
+	if (p->num <= p->idx)
+		return NULL;
+
+	void *v = p->array[p->idx];
+	p->idx++;
+	return v;
+}
+
+void *pool_iter_next_lock(pool *p)
+{
+	void *v = NULL;
+	pool_lock(p);
+	v = pool_iter_next(p);
+	pool_unlock(p);
+	return v;
+}
+
+bool pool_iter_has_next_lock(pool *p)
+{
+	bool next_exist;
+	pool_lock(p);
+	next_exist = (p->idx < p->num);
+	pool_unlock(p);
+	return next_exist;
+}

src/pool.h

@@ -0,0 +1,94 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
+#ifndef _POOL_H_
+#define _POOL_H_
+
+#include <stdbool.h>
+#include <stddef.h>
+
+#include <atomic.h>
+
+/* A pool like a stack with an iterator walking from the bottom to the
+ * top. The memory foot print for a pool never shrinks. Thus this is
+ * not suitable for long-term uses. */
+
+struct pool_struct {
+	void **array;
+	size_t len; /* length of array */
+	size_t num; /* number of items in the array */
+	size_t idx; /* index used dy iter */
+	lock lock;
+};
+
+typedef struct pool_struct pool;
+
+/* allocate a new pool */
+pool *pool_new(void);
+
+/* func type applied to each item in a pool */
+typedef void (*pool_map_f)(void *v);
+
+/* apply f, which free an item, to all items and set num to 0 */
+void pool_zeroize(pool *p, pool_map_f f);
+
+/* free pool->array and pool */
+void pool_free(pool *p);
+
+/* free pool->array and pool after applying f to all items in p->array */
+void pool_destroy(pool *p, pool_map_f f);
+
+#define pool_lock(p) LOCK_ACQUIRE(&(p->lock))
+#define pool_unlock(p) LOCK_RELEASE()
+
+/*
+ * pool_push() pushes *v to pool *p. pool_push_lock() does this while
+ * locking *p.
+ */
+int pool_push(pool *p, void *v);
+int pool_push_lock(pool *p, void *v);
+
+/*
+ * pool_pop() pops the last *v pushed to *p. pool_pop_lock() does this
+ * while locking *p.
+ */
+void *pool_pop(pool *p);
+void *pool_pop_lock(pool *p);
+
+/* pool_get() returns value indexed by idx */
+void *pool_get(pool *p, unsigned int idx);
+
+#define pool_size(p) ((p)->num)
+#define pool_is_empty(p) (pool_size(p) == 0)
+
+/*
+ * pool->idx indicates next *v in an iteration. This has two
+ * use-cases.
+ *
+ * (1) A simple list: just a single thread has a pool, and the thread
+ * can call pool_iter_for_each() for the pool (not thread safe).
+ *
+ * (2) A thread-safe queue: one thread initializes the iterator for a
+ * pool by pool_iter_init(). Then, multiple threads get a next *v
+ * concurrently by pool_iter_next_lock(), which means dequeuing. At
+ * this time, other thread can add new *v by pool_push_lock(), which
+ * means enqueuing. During this, other threads must not intercept the
+ * pool by pool_iter_* functions.
+ */
+
+#define pool_iter_init(p) (p->idx = 0)
+void *pool_iter_next(pool *p);
+void *pool_iter_next_lock(pool *p);
+
+/* pool_iter_has_next_lock() returns true if pool_iter_next(_lock)
+ * function will retrun a next value, otherwise false, which means
+ * there is no more values in this iteration. */
+bool pool_iter_has_next_lock(pool *p);
+
+#define pool_iter_for_each(p, v) \
+	pool_iter_init(p);       \
+	for (v = pool_iter_next(p); v != NULL; v = pool_iter_next(p))
+
+#define pool_for_each(p, v, idx) \
+	idx = 0;                 \
+	for (v = pool_get(p, idx); v != NULL; v = pool_get(p, ++idx))
+
+#endif /* _POOL_H_ */

test/test_e2e.py

@@ -6,9 +6,11 @@ test_e2e.py: End-to-End test for mscp executable.
 import platform
 import pytest
 import getpass
+import time
 import os
+import shutil
 
-from subprocess import check_call, CalledProcessError, PIPE
+from subprocess import check_call, CalledProcessError
 from util import File, check_same_md5sum
 
 
@@ -17,13 +19,16 @@ def run2ok(args, env = None):
     print("cmd: {}".format(" ".join(cmd)))
     check_call(cmd, env = env)
 
-def run2ng(args, env = None):
+def run2ng(args, env = None, timeout = None):
+    if timeout:
+        args = ["timeout", "-s", "INT", timeout] + args
     cmd = list(map(str, args))
     print("cmd: {}".format(" ".join(cmd)))
     with pytest.raises(CalledProcessError) as e:
         check_call(cmd, env = env)
 
 
+
 """ usage test """
 
 def test_usage(mscp):
@@ -316,14 +321,40 @@ def test_dont_truncate_dst(mscp, src_prefix, dst_prefix):
     assert md5_before == md5_after
     f.cleanup()
 
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_dont_make_conns_more_than_chunks(mscp, src_prefix, dst_prefix):
+    # copy 100 files with -n 20 -I 1 options. if mscp creates 20 SSH
+    # connections although all files have been copied, it is error.
+    srcs = []
+    dsts = []
+    for n in range(100):
+        srcs.append(File("src/src-{:06d}".format(n), size=1024).make())
+        dsts.append(File("dst/src-{:06d}".format(n)))
+    start = time.time()
+    run2ok([mscp, "-H", "-v", "-n", "20", "-I", "1",
+            src_prefix + "src", dst_prefix + "dst"])
+    end = time.time()
+    for s, d in zip(srcs, dsts):
+        assert check_same_md5sum(s, d)
+    shutil.rmtree("src")
+    shutil.rmtree("dst")
+    assert((end - start) < 10)
+
+
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 @pytest.mark.parametrize("src, dst", param_single_copy)
-def test_set_port(mscp, src_prefix, dst_prefix, src, dst):
+def test_set_port_ng(mscp, src_prefix, dst_prefix, src, dst):
     src.make()
-    run2ng([mscp, "-H", "-vvv", "-p", 21, src_prefix + src.path, dst_prefix + dst.path])
     run2ng([mscp, "-H", "-vvv", "-P", 21, src_prefix + src.path, dst_prefix + dst.path])
     src.cleanup()
 
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+@pytest.mark.parametrize("src, dst", param_single_copy)
+def test_set_port_ok(mscp, src_prefix, dst_prefix, src, dst):
+    src.make()
+    run2ok([mscp, "-H", "-vvv", "-P", 8022, src_prefix + src.path, dst_prefix + dst.path])
+    src.cleanup()
+
 def test_v4only(mscp):
     src = File("src", size = 1024).make()
     dst = File("dst")
@@ -336,7 +367,7 @@ def test_v4only(mscp):
 def test_v6only(mscp):
     src = File("src", size = 1024).make()
     dst = File("dst")
-    dst_prefix = "localhost:{}/".format(os.getcwd())
+    dst_prefix = "ip6-localhost:{}/".format(os.getcwd())
     run2ok([mscp, "-H", "-vvv", "-6", src.path, dst_prefix + dst.path])
     assert check_same_md5sum(src, dst)
     src.cleanup()
@@ -475,3 +506,56 @@ def test_specify_invalid_password_via_env(mscp):
             src.path, "localhost:" + dst.path], env = env)
     src.cleanup()
 
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_10k_files(mscp, src_prefix, dst_prefix):
+    srcs = []
+    dsts = []
+    for n in range(10000):
+        srcs.append(File("src/src-{:06d}".format(n), size=1024).make())
+        dsts.append(File("dst/src-{:06d}".format(n)))
+    run2ok([mscp, "-H", "-v", src_prefix + "src", dst_prefix + "dst"])
+    for s, d in zip(srcs, dsts):
+        assert check_same_md5sum(s, d)
+    shutil.rmtree("src")
+    shutil.rmtree("dst")
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_checkpoint_dump_and_resume(mscp, src_prefix, dst_prefix):
+    src1 = File("src1", size = 512 * 1024 * 1024).make()
+    src2 = File("src2", size = 512 * 1024 * 1024).make()
+    dst1 = File("dst/src1")
+    dst2 = File("dst/src2")
+    run2ok([mscp, "-H", "-vvv", "-W", "checkpoint", "-D",
+            src_prefix + "src1", src_prefix + "src2", dst_prefix + "dst"])
+    assert os.path.exists("checkpoint")
+
+    run2ok([mscp, "-H", "-vvv", "-R", "checkpoint"])
+    assert check_same_md5sum(src1, dst1)
+    assert check_same_md5sum(src2, dst2)
+    src1.cleanup()
+    src2.cleanup()
+    dst1.cleanup()
+    dst2.cleanup()
+    os.remove("checkpoint")
+
+@pytest.mark.parametrize("timeout", [1,2,3])
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_checkpoint_interrupt_and_resume(mscp, timeout, src_prefix, dst_prefix):
+    src1 = File("src1", size = 1024 * 1024 * 1024).make()
+    src2 = File("src2", size = 1024 * 1024 * 1024).make()
+    dst1 = File("dst/src1")
+    dst2 = File("dst/src2")
+    run2ng([mscp, "-H", "-vv", "-W", "checkpoint",
+            "-n", 1, "-s", 8192, "-S", 16384,
+            src_prefix + "src1", src_prefix + "src2", dst_prefix + "dst"],
+           timeout = timeout)
+    assert os.path.exists("checkpoint")
+
+    run2ok([mscp, "-H", "-vv", "-R", "checkpoint"])
+    assert check_same_md5sum(src1, dst1)
+    assert check_same_md5sum(src2, dst2)
+    src1.cleanup()
+    src2.cleanup()
+    dst1.cleanup()
+    dst2.cleanup()
+    os.remove("checkpoint")