bump version to 0.2.0

With podman 5.0.1 on macOS 14.4.1 with M2 Pro, ssh localhost inside
podman containers fails. I'm not sure its reason, but as a work around,
I added --privileged to podman run.

.github/workflows/build-macos.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: macos-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
 

.github/workflows/build-ubuntu.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
 

.github/workflows/codeql.yml

@@ -38,7 +38,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         submodules: true
 
@@ -52,7 +52,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -65,7 +65,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, Go, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -78,6 +78,6 @@ jobs:
     #   ./location_of_script_within_repo/buildscript.sh
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3
       with:
         category: "/language:${{matrix.language}}"

.github/workflows/release.yml

@@ -13,7 +13,7 @@ jobs:
   build-and-release:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: true
 
@@ -27,12 +27,6 @@ jobs:
       - name: Configure Cmake
         run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
 
-      - name: Build Containers
-        run: make -C ${{github.workspace}}/build docker-build-all
-
-      - name: Test
-        run: make -C ${{github.workspace}}/build docker-test-all
-
       - name: Build single binary mscp
         run: make -C ${{github.workspace}}/build build-single-binary
 
@@ -45,7 +39,7 @@ jobs:
   source-release:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: true
 

.github/workflows/test.yml

@@ -13,7 +13,7 @@ jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
 

CMakeLists.txt

@@ -109,7 +109,7 @@ list(APPEND MSCP_BUILD_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/include)
 # libmscp.a
 set(LIBMSCP_SRC
 	src/mscp.c src/ssh.c src/fileops.c src/path.c src/checkpoint.c
-	src/platform.c src/print.c src/pool.c src/strerrno.c
+	src/bwlimit.c src/platform.c src/print.c src/pool.c src/strerrno.c
 	${OPENBSD_COMPAT_SRC})
 add_library(mscp-static STATIC ${LIBMSCP_SRC})
 target_include_directories(mscp-static
@@ -165,9 +165,8 @@ enable_testing()
 # Custom targets to build and test mscp in docker containers.
 # foreach(IN ZIP_LISTS) (cmake >= 3.17) can shorten the following lists.
 # However, ubuntu 20.04 has cmake 3.16.3. So this is a roundabout trick.
-list(APPEND DIST_IDS	ubuntu	ubuntu	rocky	rocky	almalinux	alpine)
+list(APPEND DIST_IDS	ubuntu	ubuntu	ubuntu	rocky	rocky	almalinux	alpine)
-list(APPEND DIST_VERS	20.04	22.04	8.9	9.3	9.3		3.19)
+list(APPEND DIST_VERS	20.04	22.04	24.04 	8.9	9.3	9.3		3.19)
-list(APPEND DIST_PKGS	deb	deb	rpm	rpm	rpm		static)
 
 list(LENGTH DIST_IDS _DIST_LISTLEN)
 math(EXPR DIST_LISTLEN "${_DIST_LISTLEN} - 1")
@@ -175,7 +174,6 @@ math(EXPR DIST_LISTLEN "${_DIST_LISTLEN} - 1")
 foreach(x RANGE ${DIST_LISTLEN})
 	list(GET DIST_IDS	${x} DIST_ID)
 	list(GET DIST_VERS	${x} DIST_VER)
-	list(GET DIST_PKGS	${x} DIST_PKG)
 
 	set(DOCKER_IMAGE mscp-${DIST_ID}:${DIST_VER})
 	set(DOCKER_INDEX ${DIST_ID}-${DIST_VER})
@@ -203,7 +201,9 @@ foreach(x RANGE ${DIST_LISTLEN})
 		COMMENT "Test mscp in ${DOCKER_IMAGE} container"
 		WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
 		COMMAND
-		${CE} run --init --rm --sysctl net.ipv6.conf.all.disable_ipv6=0
+		${CE} run --init --rm --privileged
+		--sysctl net.ipv6.conf.all.disable_ipv6=0
+		--add-host=ip6-localhost:::1
 		${DOCKER_IMAGE} /mscp/scripts/test-in-container.sh)
 
 	list(APPEND DOCKER_BUILDS		docker-build-${DOCKER_INDEX})
@@ -251,7 +251,6 @@ configure_file(
 
 # Custom target to build mscp as a src.rpm in docker.
 set(RPMBUILDCONTAINER mscp-build-srpm)
-set(SRPMFILE mscp-${MSCP_VERSION}-1.el9.src.rpm)
 execute_process(
 	COMMAND ${CMAKE_SOURCE_DIR}/scripts/install-build-deps.sh
 	--dont-install --platform Linux-rocky
@@ -261,14 +260,13 @@ execute_process(
 add_custom_target(build-srpm
 	COMMENT "Build mscp src.rpm inside a container"
 	WORKING_DIRECTORY ${mscp_SOURCE_DIR}
-	BYPRODUCTS ${CMAKE_BINARY_DIR}/${SRPMFILE}
 	COMMAND
 	${CE} build --build-arg REQUIREDPKGS=${REQUIREDPKGS_RPM}
 	--build-arg MSCP_VERSION=${MSCP_VERSION}
 	-t ${RPMBUILDCONTAINER} -f Dockerfile/build-srpm.Dockerfile .
 	COMMAND
 	${CE} run --rm -v ${CMAKE_BINARY_DIR}:/out ${RPMBUILDCONTAINER}
-	cp /root/rpmbuild/SRPMS/${SRPMFILE} /out/)
+	bash -c "cp /root/rpmbuild/SRPMS/mscp-*.src.rpm /out/")
 
 ### single-binary-build-related definitions
 

Dockerfile/ubuntu-24.04.Dockerfile

@@ -0,0 +1,37 @@
+FROM ubuntu:24.04
+
+ARG REQUIREDPKGS
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN set -ex && apt-get update && apt-get install -y --no-install-recommends \
+	${REQUIREDPKGS} ca-certificates openssh-server vim-tiny \
+	python3 python3-pip python3-dev python3-pytest
+
+
+# preparation for sshd
+RUN mkdir /var/run/sshd        \
+	&& ssh-keygen -A	\
+	&& ssh-keygen -f /root/.ssh/id_rsa -N ""                \
+	&& cat /root/.ssh/id_rsa.pub > /root/.ssh/authorized_keys
+
+# create test user
+RUN useradd -m -d /home/test test	\
+	&& echo "test:userpassword" | chpasswd \
+	&& mkdir -p /home/test/.ssh	\
+	&& ssh-keygen -f /home/test/.ssh/id_rsa_test -N "keypassphrase"	\
+	&& cat /home/test/.ssh/id_rsa_test.pub >> /home/test/.ssh/authorized_keys \
+	&& chown -R test:test /home/test \
+	&& chown -R test:test /home/test/.ssh
+
+
+ARG mscpdir="/mscp"
+
+COPY . ${mscpdir}
+
+# build
+RUN cd ${mscpdir}			\
+	&& rm -rf build			\
+	&& cmake -B build		\
+	&& cd ${mscpdir}/build		\
+	&& make	-j 2			\
+	&& make install

README.md

@@ -15,7 +15,7 @@ transfer time for a lot of/large files over networks.
 You can use `mscp` like `scp`, for example:
 
 ```shell-session
-$ mscp user@example.com:srcfile /tmp/dstfile
+$ mscp srcfile user@example.com:dstfile
 ```
 
 Remote hosts only need to run standard `sshd` supporting the SFTP
@@ -27,7 +27,7 @@ https://github.com/upa/mscp/assets/184632/19230f57-be7f-4ef0-98dd-cb4c460f570d
 
 --------------------------------------------------------------------
 
-Differences from `scp` on usage:
+Major differences from `scp` on usage:
 
 - Remote-to-remote copy is not supported.
 - `-r` option is not needed to transfer directories.
@@ -102,7 +102,7 @@ make install
 ```
 
 Source tar balls (`mscp-X.X.X.tar.gz`, not `Source code`) in
-[Releases page](https://github.com/upa/mscp/releases) contains the patched version
+[Releases page](https://github.com/upa/mscp/releases) contain the patched version
 of libssh. So you can start from cmake with it.
 
 

VERSION

@@ -1 +1 @@
-0.1.5
+0.2.0

debian/changelog

@@ -1,4 +1,19 @@
-mscp (0.1.5) UNRELEASED; urgency=medium
+mscp (0.2.0) UNRELEASED; urgency=medium
+
+  * add -J DESTINATION option for ProxyJump (#15)
+  * add -o SSH_OPTION option
+  * add -L LIMIT_BITRATE option (#14)
+  * add keyboard interactive authentication support. Also, we have manually
+    tested that mscp works with cisco DUO MFA (#2)
+  * remove -H disable host key check option (ssh_config or -o option can do
+    this instead)
+  * fix copying files of permission r--r--r--
+  * update github actions: checkout v3 to v4, and codeql from v2 to v3
+  * add ubuntu 24.04 test
+
+ -- Ryo Nakamura <upa@haeena.net>  Mon, 15 Apr 2024 00:05:20 +0900
+
+mscp (0.1.5) unstable; urgency=medium
 
   * add support for resuming failed transfer (#5 and #10)
   * remove the list structure derived from the linux kernel and refactoring

doc/RELEASE.md

@@ -22,10 +22,12 @@ mscp_0.1.4_source.build  mscp_0.1.4_source.changes
 
 ### To publush mscp in launchpad PPA:
 
-1. write changes in `debian/changelog` at main branch (the date command needed here is `date -R`)
+1. write changes in `debian/changelog` at main branch (the date
+   command needed here is `date -R`)
 2. switch to `ppa-focal` or `ppa-jammy` branch
 3. rebase to the `main` branch and modify `debian/changes`:
-   * change `UNRELEASED` to the release name (`focal` or `jammy`).
+   * change `mscp (X.X.X) UNRELEASED;` to `mscp (X.X.X-1~RELEASENAME) RELEASENAME;`
+	 where `RELEASENAME` is `focal` or `jammy`.
 4. run `make build-deb` at the build directory and `cd debbuild`
 5. sign the files with `debsign -k [GPGKEYID] mscp_X.X.X~X_source.changes`
 5. upload the files with `dput ppa:upaa/mscp mscp_X.X.X~X_source.changes`

doc/mscp.1.in

@@ -6,7 +6,7 @@ mscp \- copy files over multiple SSH connections
 .SH SYNOPSIS
 
 .B mscp
-.RB [ \-46vqDpHdNh ]
+.RB [ \-46vqDpdNh ]
 [\c
 .BI \-n \ NR_CONNECTIONS\c
 ]
@@ -38,18 +38,27 @@ mscp \- copy files over multiple SSH connections
 .BI \-b \ BUF_SIZE\c
 ]
 [\c
+.BI \-L \ LIMIT_BITRATE\c
+]
+[\c
 .BI \-l \ LOGIN_NAME\c
 ]
 [\c
 .BI \-P \ PORT\c
 ]
 [\c
-.BI \-F \ CONFIG\c
+.BI \-F \ SSH_CONFIG\c
+]
+[\c
+.BI \-o \ SSH_OPTION\c
 ]
 [\c
 .BI \-i \ IDENTITY\c
 ]
 [\c
+.BI \-J \ DESTINATION\c
+]
+[\c
 .BI \-c \ CIPHER\c
 ]
 [\c
@@ -75,7 +84,7 @@ of/large files over networks.
 .PP
 The usage of
 .B mscp
-imitates the
+follows the
 .B scp
 command of
 .I OpenSSH,
@@ -114,8 +123,10 @@ formula: floor(log(nr_cores)*2)+1.
 
 .TP
 .B \-m \fICOREMASK\fR
-Configures CPU cores to be used by the hexadecimal bitmask.  All CPU
+Configures CPU cores to be used by the hexadecimal bitmask. For
-cores are used by default.
+example, -m 0x25 pins threads onto CPU cores 0, 2, and 5. The default
+value is not specified: all CPU cores are used and no threads are
+pinned to any cores.
 
 .TP
 .B \-u \fIMAX_STARTUPS\fR
@@ -180,7 +191,8 @@ and remove the checkpoint if it returns 0.
 .B \-s \fIMIN_CHUNK_SIZE\fR
 Specifies the minimum chunk size.
 .B mscp
-divides a file into chunks and copies the chunks in parallel.
+divides a single file into chunks and copies the chunks in
+parallel. The default value is 67108864 (64MB).
 
 .TP
 .B \-S \fIMAX_CHUNK_SIZE\fR
@@ -198,6 +210,11 @@ Specifies the buffer size for I/O and transfer over SFTP. The default
 value is 16384. Note that the SSH specification restricts buffer size
 delivered over SSH. Changing this value is not recommended at present.
 
+.TP
+.B \-L \fILIMIT_BITRATE\fR
+Limits the bitrate, specified with k (K), m (M), and g (G), e.g., 100m
+indicates 100 Mbps.
+
 .TP
 .B \-4
 Uses IPv4 addresses only.
@@ -236,19 +253,39 @@ Specifies the username to log in on the remote machine as with
 .TP
 .B \-P \fIPORT\fR
 Specifies the port number to connect to on the remote machine as with
-ssh(1) and scp(1).
+.I scp(1).
 
 .TP
-.B \-F \fICONFIG\fR
+.B \-F \fISSH_CONFIG\fR
 Specifies an alternative per-user ssh configuration file. Note that
 acceptable options in the configuration file are what
 .I libssh
 supports.
 
+.TP
+.B \-o \fISSH_OPTION\fR
+Specifies ssh options in the format used in ssh_config. Note that
+acceptable options are what
+.I libssh
+supports.
+
 .TP
 .B \-i \fIIDENTITY\fR
 Specifies the identity file for public key authentication.
 
+.TP
+.B \-J \fIDESTINATION\fR
+A shortcut to define a
+.B ProxyJump
+configuration directive. Each SFTP session of
+.B mscp
+connects to the target host by first making an
+.B ssh
+connection to the jump host described by
+.I destination.
+
+
+
 .TP
 .B \-c \fICIPHER\fR
 Selects the cipher to use for encrypting the data transfer. See
@@ -280,10 +317,6 @@ Specifies the TCP congestion control algorithm to use (Linux only).
 Preserves modification times and access times (file mode bits are
 preserved by default).
 
-.TP
-.B \-H
-Disables hostkey checking.
-
 .TP
 .B \-d
 Increments the ssh debug output level.

doc/mscp.rst

@@ -2,7 +2,7 @@
 MSCP
 ====
 
-:Date:   v0.1.4-28-g0d248c5
+:Date: v0.1.5-18-ge47d5b7
 
 NAME
 ====
@@ -12,14 +12,14 @@ mscp - copy files over multiple SSH connections
 SYNOPSIS
 ========
 
-**mscp** [**-46vqDpHdNh**] [ **-n**\ *NR_CONNECTIONS* ] [
+**mscp** [**-46vqDpdNh**] [ **-n** *NR_CONNECTIONS* ] [ **-m**
-**-m**\ *COREMASK* ] [ **-u**\ *MAX_STARTUPS* ] [ **-I**\ *INTERVAL* ] [
+*COREMASK* ] [ **-u** *MAX_STARTUPS* ] [ **-I** *INTERVAL* ] [ **-W**
-**-W**\ *CHECKPOINT* ] [ **-R**\ *CHECKPOINT* ] [
+*CHECKPOINT* ] [ **-R** *CHECKPOINT* ] [ **-s** *MIN_CHUNK_SIZE* ] [
-**-s**\ *MIN_CHUNK_SIZE* ] [ **-S**\ *MAX_CHUNK_SIZE* ] [
+**-S** *MAX_CHUNK_SIZE* ] [ **-a** *NR_AHEAD* ] [ **-b** *BUF_SIZE* ] [
-**-a**\ *NR_AHEAD* ] [ **-b**\ *BUF_SIZE* ] [ **-l**\ *LOGIN_NAME* ] [
+**-L** *LIMIT_BITRATE* ] [ **-l** *LOGIN_NAME* ] [ **-P** *PORT* ] [
-**-P**\ *PORT* ] [ **-F**\ *CONFIG* ] [ **-i**\ *IDENTITY* ] [
+**-F** *SSH_CONFIG* ] [ **-o** *SSH_OPTION* ] [ **-i** *IDENTITY* ] [
-**-c**\ *CIPHER* ] [ **-M**\ *HMAC* ] [ **-C**\ *COMPRESS* ] [
+**-J** *DESTINATION* ] [ **-c** *CIPHER* ] [ **-M** *HMAC* ] [ **-C**
-**-g**\ *CONGESTION* ] *source ... target*
+*COMPRESS* ] [ **-g** *CONGESTION* ] *source ... target*
 
 DESCRIPTION
 ===========
@@ -29,7 +29,7 @@ threads. It enables transferring (1) multiple files simultaneously and
 (2) a large file in parallel, reducing the transfer time for a lot
 of/large files over networks.
 
-The usage of **mscp** imitates the **scp** command of *OpenSSH,* for
+The usage of **mscp** follows the **scp** command of *OpenSSH,* for
 example:
 
 ::
@@ -54,8 +54,10 @@ OPTIONS
    following formula: floor(log(nr_cores)*2)+1.
 
 **-m COREMASK**
-   Configures CPU cores to be used by the hexadecimal bitmask. All CPU
+   Configures CPU cores to be used by the hexadecimal bitmask. For
-   cores are used by default.
+   example, -m 0x25 pins threads onto CPU cores 0, 2, and 5. The default
+   value is not specified: all CPU cores are used and no threads are
+   pinned to any cores.
 
 **-u MAX_STARTUPS**
    Specifies the number of concurrent outgoing SSH connections. **sshd**
@@ -94,8 +96,9 @@ OPTIONS
    remove the checkpoint if it returns 0.
 
 **-s MIN_CHUNK_SIZE**
-   Specifies the minimum chunk size. **mscp** divides a file into chunks
+   Specifies the minimum chunk size. **mscp** divides a single file into
-   and copies the chunks in parallel.
+   chunks and copies the chunks in parallel. The default value is
+   67108864 (64MB).
 
 **-S MAX_CHUNK_SIZE**
    Specifies the maximum chunk size. The default is file size divided by
@@ -111,6 +114,10 @@ OPTIONS
    delivered over SSH. Changing this value is not recommended at
    present.
 
+**-L LIMIT_BITRATE**
+   Limits the bitrate, specified with k (K), m (M), and g (G), e.g.,
+   100m indicates 100 Mbps.
+
 **-4**
    Uses IPv4 addresses only.
 
@@ -138,16 +145,25 @@ OPTIONS
 
 **-P PORT**
    Specifies the port number to connect to on the remote machine as with
-   ssh(1) and scp(1).
+   *scp(1).*
 
-**-F CONFIG**
+**-F SSH_CONFIG**
    Specifies an alternative per-user ssh configuration file. Note that
    acceptable options in the configuration file are what *libssh*
    supports.
 
+**-o SSH_OPTION**
+   Specifies ssh options in the format used in ssh_config. Note that
+   acceptable options are what *libssh* supports.
+
 **-i IDENTITY**
    Specifies the identity file for public key authentication.
 
+**-J DESTINATION**
+   A shortcut to define a **ProxyJump** configuration directive. Each
+   SFTP session of **mscp** connects to the target host by first making
+   an **ssh** connection to the jump host described by *destination.*
+
 **-c CIPHER**
    Selects the cipher to use for encrypting the data transfer. See
    `libssh features <https://www.libssh.org/features/>`__.
@@ -167,9 +183,6 @@ OPTIONS
    Preserves modification times and access times (file mode bits are
    preserved by default).
 
-**-H**
-   Disables hostkey checking.
-
 **-d**
    Increments the ssh debug output level.
 

include/mscp.h

@@ -42,7 +42,8 @@ struct mscp_opts {
 	size_t	min_chunk_sz;	/** minimum chunk size (default 64MB) */
 	size_t	max_chunk_sz;	/** maximum chunk size (default file size/nr_threads) */
 	size_t	buf_sz;		/** buffer size, default 16k. */
-	char	*coremask; /** hex to specifiy usable cpu cores */
+	size_t	bitrate;	/** bits-per-seconds to limit bandwidth */
+	char	*coremask;	/** hex to specifiy usable cpu cores */
 	int	max_startups;	/** sshd MaxStartups concurrent connections */
 	int     interval;	/** interval between SSH connection attempts */
 	bool	preserve_ts;	/** preserve file timestamps */
@@ -60,7 +61,9 @@ struct mscp_ssh_opts {
 	char	*port;		/** ssh port */
 	int	ai_family;	/** address family */
 	char	*config;	/** path to ssh_config, default ~/.ssh/config*/
+	char	**options;	/** array of ssh_config options, terminated by NULL */
 	char	*identity;	/** path to private key */
+	char	*proxyjump;	/** ProxyJump configuration directive (shortcut) */
 	char	*cipher;	/** cipher spec */
 	char	*hmac;		/** hmacp spec */
 	char	*compress;	/** yes, no, zlib@openssh.com */
@@ -70,7 +73,6 @@ struct mscp_ssh_opts {
 	char	*passphrase;	/** passphrase for private key */
 
 	int	debug_level;		/** inclirement libssh debug output level */
-	bool	no_hostkey_check;	/** do not check host keys */
 	bool	enable_nagle;		/** enable Nagle's algorithm if true */
 };
 

rpm/mscp.spec.in

@@ -38,6 +38,9 @@ make -C build install DESTDIR=%{buildroot}
 
 
 %changelog
+* Mon Apr 15 2024 Ryo Nakamura <upa@haeena.net> - 0.2.0-1
+- RPM release for v0.2.0
+
 * Thu Mar 14 2024 Ryo Nakamura <upa@haeena.net> - 0.1.5-0
 - RPM release for v0.1.5
 

scripts/test-in-container.sh

@@ -12,15 +12,21 @@ set -x
 echo "Port 22" >> /etc/ssh/sshd_config
 echo "Port 8022" >> /etc/ssh/sshd_config
 
+## Alpine default sshd disables TcpForwarding, which is required for proxyjump test
+sed -i -e 's/AllowTcpForwarding no/AllowTcpForwarding yes/' /etc/ssh/sshd_config
+
 # Run sshd
 if [ ! -e /var/run/sshd.pid ]; then
 	/usr/sbin/sshd
 	sleep 1
 fi
 
-ssh-keyscan localhost >> ${HOME}/.ssh/known_hosts
+for port in 22 8022; do
-ssh-keyscan 127.0.0.1 >> ${HOME}/.ssh/known_hosts
+	ssh-keyscan -p $port localhost >> ${HOME}/.ssh/known_hosts
-ssh-keyscan ::1 >> ${HOME}/.ssh/known_hosts
+	ssh-keyscan -p $port ip6-localhost >> ${HOME}/.ssh/known_hosts
+	ssh-keyscan -p $port 127.0.0.1 >> ${HOME}/.ssh/known_hosts
+	ssh-keyscan -p $port ::1 >> ${HOME}/.ssh/known_hosts
+done
 
 # Run test
 python3 -m pytest -v ../test 

src/bwlimit.c

@@ -0,0 +1,94 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
+#include <errno.h>
+
+#include <bwlimit.h>
+#include <platform.h>
+
+#define timespeczerorize(ts)    \
+	do {                    \
+		ts.tv_sec = 0;  \
+		ts.tv_nsec = 0; \
+	} while (0)
+
+int bwlimit_init(struct bwlimit *bw, uint64_t bps, uint64_t win)
+{
+	if (!(bw->sem = sem_create(1)))
+		return -1;
+
+	bw->bps = bps;
+	bw->win = win; /* msec window */
+	bw->amt = (double)bps / 8 / 1000 * win; /* bytes in a window (msec) */
+	bw->credit = bw->amt;
+	timespeczerorize(bw->wstart);
+	timespeczerorize(bw->wend);
+
+	return 0;
+}
+
+#define timespecisset(ts) ((ts).tv_sec || (ts).tv_nsec)
+
+#define timespecmsadd(a, msec, r)                                \
+	do {                                                     \
+		(r).tv_sec = (a).tv_sec;                         \
+		(r).tv_nsec = (a).tv_nsec + (msec * 1000000);    \
+		if ((r).tv_nsec > 1000000000) {                  \
+			(r).tv_sec += (r.tv_nsec) / 1000000000L; \
+			(r).tv_nsec = (r.tv_nsec) % 1000000000L; \
+		}                                                \
+	} while (0)
+
+#define timespecsub(a, b, r)                             \
+	do {                                             \
+		(r).tv_sec = (a).tv_sec - (b).tv_sec;    \
+		(r).tv_nsec = (a).tv_nsec - (b).tv_nsec; \
+		if ((r).tv_nsec < 0) {                   \
+			(r).tv_sec -= 1;                 \
+			(r).tv_nsec += 1000000000;       \
+		}                                        \
+	} while (0)
+
+#define timespeccmp(a, b, expr) \
+	((a.tv_sec * 1000000000 + a.tv_nsec) expr(b.tv_sec * 1000000000 + b.tv_nsec))
+
+#include <stdio.h>
+
+int bwlimit_wait(struct bwlimit *bw, size_t nr_bytes)
+{
+	struct timespec now, end, rq, rm;
+
+	if (bw->bps == 0)
+		return 0; /* no bandwidth limit */
+
+	if (sem_wait(bw->sem) < 0)
+		return -1;
+
+	clock_gettime(CLOCK_MONOTONIC, &now);
+
+	if (!timespecisset(bw->wstart)) {
+		bw->wstart = now;
+		timespecmsadd(bw->wstart, bw->win, bw->wend);
+	}
+
+	bw->credit -= nr_bytes;
+
+	if (bw->credit < 0) {
+		/* no more credit on this window. sleep until the end
+		 * of this window + additional time for the remaining
+		 * bytes. */
+		uint64_t addition = (double)(bw->credit * -1) / (bw->bps / 8);
+		timespecmsadd(bw->wend, addition * 1000, end);
+		if (timespeccmp(end, now, >)) {
+			timespecsub(end, now, rq);
+			while (nanosleep(&rq, &rm) == -1) {
+				if (errno != EINTR)
+					break;
+				rq = rm;
+			}
+		}
+		bw->credit = bw->amt;
+		timespeczerorize(bw->wstart);
+	}
+
+	sem_post(bw->sem);
+	return 0;
+}

src/bwlimit.h

@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
+#ifndef _BWLIMIT_H_
+#define _BWLIMIT_H_
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <sys/types.h>
+#include <time.h>
+#include <semaphore.h>
+
+struct bwlimit {
+	sem_t		*sem;	/* semaphore */
+	uint64_t	bps;	/* limit bit-rate (bps) */
+	uint64_t	win;	/* window size (msec) */
+	size_t		amt;	/* amount of bytes can be sent in a window */
+
+	ssize_t		credit;	/* remaining bytes can be sent in a window */
+	struct timespec wstart, wend; /* window start time and end time */
+};
+
+int bwlimit_init(struct bwlimit *bw, uint64_t bps, uint64_t win);
+/* if bps is 0, it means that bwlimit is not active. If so,
+ * bwlimit_wait() returns immediately. */
+
+int bwlimit_wait(struct bwlimit *bw, size_t nr_bytes);
+
+
+#endif /* _BWLIMIT_H_ */

src/fileops.c

@@ -323,12 +323,14 @@ int mscp_setstat(const char *path, struct stat *st, bool preserve_ts, sftp_sessi
 		ret = sftp_setstat(sftp, path, &attr);
 		sftp_err_to_errno(sftp);
 	} else {
-		if ((ret = chmod(path, st->st_mode)) < 0)
-			return ret;
 		if ((ret = truncate(path, st->st_size)) < 0)
 			return ret;
-		if (preserve_ts)
+		if (preserve_ts) {
-			ret = setutimes(path, st->st_atim, st->st_mtim);
+			if ((ret = setutimes(path, st->st_atim, st->st_mtim)) < 0)
+				return ret;
+		}
+		if ((ret = chmod(path, st->st_mode)) < 0)
+			return ret;
 	}
 
 	return ret;

src/main.c

@@ -24,11 +24,13 @@ void usage(bool print_help)
 {
 	printf("mscp " MSCP_BUILD_VERSION ": copy files over multiple SSH connections\n"
 	       "\n"
-	       "Usage: mscp [-46vqDpHdNh] [-n nr_conns] [-m coremask]\n"
+	       "Usage: mscp [-46vqDpdNh] [-n nr_conns] [-m coremask] [-u max_startups]\n"
-	       "            [-u max_startups] [-I interval] [-W checkpoint] [-R checkpoint]\n"
+	       "            [-I interval] [-W checkpoint] [-R checkpoint]\n"
-	       "            [-s min_chunk_sz] [-S max_chunk_sz] [-a nr_ahead] [-b buf_sz]\n"
+	       "            [-s min_chunk_sz] [-S max_chunk_sz] [-a nr_ahead]\n"
-	       "            [-l login_name] [-P port] [-F ssh_config] [-i identity_file]\n"
+	       "            [-b buf_sz] [-L limit_bitrate]\n"
-	       "            [-c cipher_spec] [-M hmac_spec] [-C compress] [-g congestion]\n"
+	       "            [-l login_name] [-P port] [-F ssh_config] [-o ssh_option]\n"
+	       "            [-i identity_file] [-J destination] [-c cipher_spec] [-M hmac_spec]\n"
+	       "            [-C compress] [-g congestion]\n"
 	       "            source ... target\n"
 	       "\n");
 
@@ -48,6 +50,7 @@ void usage(bool print_help)
 	       "    -S MAX_CHUNK_SIZE  max chunk size (default: filesize/nr_conn)\n"
 	       "    -a NR_AHEAD        number of inflight SFTP commands (default: 32)\n"
 	       "    -b BUF_SZ          buffer size for i/o and transfer\n"
+	       "    -L LIMIT_BITRATE   Limit the bitrate, n[KMG] (default: 0, no limit)\n"
 	       "\n"
 	       "    -4                 use IPv4\n"
 	       "    -6                 use IPv6\n"
@@ -58,15 +61,16 @@ void usage(bool print_help)
 	       "\n"
 	       "    -l LOGIN_NAME      login name\n"
 	       "    -P PORT            port number\n"
-	       "    -F CONFIG          path to user ssh config (default ~/.ssh/config)\n"
+	       "    -F SSH_CONFIG      path to user ssh config (default ~/.ssh/config)\n"
+	       "    -o SSH_OPTION      ssh_config option\n"
 	       "    -i IDENTITY        identity file for public key authentication\n"
+	       "    -J DESTINATION     ProxyJump destination\n"
 	       "    -c CIPHER          cipher spec\n"
 	       "    -M HMAC            hmac spec\n"
 	       "    -C COMPRESS        enable compression: "
 	       "yes, no, zlib, zlib@openssh.com\n"
 	       "    -g CONGESTION      specify TCP congestion control algorithm\n"
 	       "    -p                 preserve timestamps of files\n"
-	       "    -H                 disable hostkey check\n"
 	       "    -d                 increment ssh debug output level\n"
 	       "    -N                 enable Nagle's algorithm (default disabled)\n"
 	       "    -h                 print this help\n"
@@ -266,12 +270,15 @@ int main(int argc, char **argv)
 	int direction = 0;
 	char *remote = NULL, *checkpoint_save = NULL, *checkpoint_load = NULL;
 	bool dryrun = false, resume = false;
+	int nr_options = 0;
+	size_t factor = 1;
+	char *unit;
 
 	memset(&s, 0, sizeof(s));
 	memset(&o, 0, sizeof(o));
 	o.severity = MSCP_SEVERITY_WARN;
 
-#define mscpopts "n:m:u:I:W:R:s:S:a:b:46vqDrl:P:i:F:c:M:C:g:pHdNh"
+#define mscpopts "n:m:u:I:W:R:s:S:a:b:L:46vqDrl:P:F:o:i:J:c:M:C:g:pdNh"
 	while ((ch = getopt(argc, argv, mscpopts)) != -1) {
 		switch (ch) {
 		case 'n':
@@ -309,6 +316,26 @@ int main(int argc, char **argv)
 		case 'b':
 			o.buf_sz = atoi(optarg);
 			break;
+		case 'L':
+			factor = 1;
+			unit = optarg + (strlen(optarg) - 1);
+			if (*unit == 'k' || *unit == 'K') {
+				factor = 1000;
+				*unit = '\0';
+			} else if (*unit == 'm' || *unit == 'M') {
+				factor = 1000000;
+				*unit = '\0';
+			} else if (*unit == 'g' || *unit == 'G') {
+				factor = 1000000000;
+				*unit = '\0';
+			}
+			o.bitrate = atol(optarg);
+			if (o.bitrate == 0) {
+				pr_err("invalid bitrate: %s", optarg);
+				return 1;
+			}
+			o.bitrate *= factor;
+			break;
 		case '4':
 			s.ai_family = AF_INET;
 			break;
@@ -336,9 +363,22 @@ int main(int argc, char **argv)
 		case 'F':
 			s.config = optarg;
 			break;
+		case 'o':
+			nr_options++;
+			s.options = realloc(s.options, sizeof(char *) * (nr_options + 1));
+			if (!s.options) {
+				pr_err("realloc: %s", strerrno());
+				return 1;
+			}
+			s.options[nr_options - 1] = optarg;
+			s.options[nr_options] = NULL;
+			break;
 		case 'i':
 			s.identity = optarg;
 			break;
+		case 'J':
+			s.proxyjump = optarg;
+			break;
 		case 'c':
 			s.cipher = optarg;
 			break;
@@ -354,9 +394,6 @@ int main(int argc, char **argv)
 		case 'p':
 			o.preserve_ts = true;
 			break;
-		case 'H':
-			s.no_hostkey_check = true;
-			break;
 		case 'd':
 			s.debug_level++;
 			break;

src/mscp.c

@@ -17,6 +17,7 @@
 #include <print.h>
 #include <strerrno.h>
 #include <mscp.h>
+#include <bwlimit.h>
 
 #include <openbsd-compat/openbsd-compat.h>
 
@@ -56,6 +57,8 @@ struct mscp {
 #define chunk_pool_is_ready(m) ((m)->chunk_pool_ready)
 #define chunk_pool_set_ready(m, b) ((m)->chunk_pool_ready = b)
 
+	struct bwlimit bw; /* bandwidth limit mechanism */
+
 	struct mscp_thread scan; /* mscp_thread for mscp_scan_thread() */
 };
 
@@ -281,6 +284,12 @@ struct mscp *mscp_init(struct mscp_opts *o, struct mscp_ssh_opts *s)
 		pr_notice("usable cpu cores:%s", b);
 	}
 
+	if (bwlimit_init(&m->bw, o->bitrate, 100) < 0) { /* 100ms window (hardcoded) */
+		priv_set_errv("bwlimit_init: %s", strerrno());
+		goto free_out;
+	}
+	pr_notice("bitrate limit: %lu bps", o->bitrate);
+
 	return m;
 
 free_out:
@@ -522,8 +531,8 @@ int mscp_checkpoint_load(struct mscp *m, const char *pathname)
 
 int mscp_checkpoint_save(struct mscp *m, const char *pathname)
 {
-	return checkpoint_save(pathname, m->direction, m->ssh_opts->login_name,
+	return checkpoint_save(pathname, m->direction, m->ssh_opts->login_name, m->remote,
-			       m->remote, m->path_pool, m->chunk_pool);
+			       m->path_pool, m->chunk_pool);
 }
 
 static void *mscp_copy_thread(void *arg);
@@ -712,7 +721,7 @@ void *mscp_copy_thread(void *arg)
 		}
 
 		if ((t->ret = copy_chunk(c, src_sftp, dst_sftp, m->opts->nr_ahead,
-					 m->opts->buf_sz, m->opts->preserve_ts,
+					 m->opts->buf_sz, m->opts->preserve_ts, &m->bw,
 					 &t->copied_bytes)) < 0)
 			break;
 	}

src/path.c

@@ -348,7 +348,7 @@ static ssize_t read_to_buf(void *ptr, size_t len, void *userdata)
 }
 
 static int copy_chunk_l2r(struct chunk *c, int fd, sftp_file sf, int nr_ahead, int buf_sz,
-			  size_t *counter)
+			  struct bwlimit *bw, size_t *counter)
 {
 	ssize_t read_bytes, remaind, thrown;
 	int idx, ret;
@@ -371,6 +371,7 @@ static int copy_chunk_l2r(struct chunk *c, int fd, sftp_file sf, int nr_ahead, i
 			return -1;
 		}
 		thrown -= reqs[idx].len;
+		bwlimit_wait(bw, reqs[idx].len);
 	}
 
 	for (idx = 0; remaind > 0; idx = (idx + 1) % nr_ahead) {
@@ -399,6 +400,7 @@ static int copy_chunk_l2r(struct chunk *c, int fd, sftp_file sf, int nr_ahead, i
 			return -1;
 		}
 		thrown -= reqs[idx].len;
+		bwlimit_wait(bw, reqs[idx].len);
 	}
 
 	if (remaind < 0) {
@@ -412,7 +414,7 @@ static int copy_chunk_l2r(struct chunk *c, int fd, sftp_file sf, int nr_ahead, i
 }
 
 static int copy_chunk_r2l(struct chunk *c, sftp_file sf, int fd, int nr_ahead, int buf_sz,
-			  size_t *counter)
+			  struct bwlimit *bw, size_t *counter)
 {
 	ssize_t read_bytes, write_bytes, remaind, thrown;
 	char buf[buf_sz];
@@ -436,6 +438,7 @@ static int copy_chunk_r2l(struct chunk *c, sftp_file sf, int fd, int nr_ahead, i
 			return -1;
 		}
 		thrown -= reqs[idx].len;
+		bwlimit_wait(bw, reqs[idx].len);
 	}
 
 	for (idx = 0; remaind > 0; idx = (idx + 1) % nr_ahead) {
@@ -449,6 +452,7 @@ static int copy_chunk_r2l(struct chunk *c, sftp_file sf, int fd, int nr_ahead, i
 			reqs[idx].len = min(thrown, sizeof(buf));
 			reqs[idx].id = sftp_async_read_begin(sf, reqs[idx].len);
 			thrown -= reqs[idx].len;
+			bwlimit_wait(bw, reqs[idx].len);
 		}
 
 		write_bytes = write(fd, buf, read_bytes);
@@ -477,19 +481,22 @@ static int copy_chunk_r2l(struct chunk *c, sftp_file sf, int fd, int nr_ahead, i
 }
 
 static int _copy_chunk(struct chunk *c, mf *s, mf *d, int nr_ahead, int buf_sz,
-		       size_t *counter)
+		       struct bwlimit *bw, size_t *counter)
 {
 	if (s->local && d->remote) /* local to remote copy */
-		return copy_chunk_l2r(c, s->local, d->remote, nr_ahead, buf_sz, counter);
+		return copy_chunk_l2r(c, s->local, d->remote, nr_ahead, buf_sz, bw,
+				      counter);
 	else if (s->remote && d->local) /* remote to local copy */
-		return copy_chunk_r2l(c, s->remote, d->local, nr_ahead, buf_sz, counter);
+		return copy_chunk_r2l(c, s->remote, d->local, nr_ahead, buf_sz, bw,
+				      counter);
 
 	assert(false);
 	return -1; /* not reached */
 }
 
 int copy_chunk(struct chunk *c, sftp_session src_sftp, sftp_session dst_sftp,
-	       int nr_ahead, int buf_sz, bool preserve_ts, size_t *counter)
+	       int nr_ahead, int buf_sz, bool preserve_ts, struct bwlimit *bw,
+	       size_t *counter)
 {
 	mode_t mode;
 	int flags;
@@ -529,7 +536,7 @@ int copy_chunk(struct chunk *c, sftp_session src_sftp, sftp_session dst_sftp,
 	c->state = CHUNK_STATE_COPING;
 	pr_debug("copy chunk start: %s 0x%lx-0x%lx", c->p->path, c->off, c->off + c->len);
 
-	ret = _copy_chunk(c, s, d, nr_ahead, buf_sz, counter);
+	ret = _copy_chunk(c, s, d, nr_ahead, buf_sz, bw, counter);
 
 	pr_debug("copy chunk done: %s 0x%lx-0x%lx", c->p->path, c->off, c->off + c->len);
 

src/path.h

@@ -9,6 +9,7 @@
 #include <pool.h>
 #include <atomic.h>
 #include <ssh.h>
+#include <bwlimit.h>
 
 struct path {
 	char *path; /* file path */
@@ -66,6 +67,7 @@ void free_path(struct path *p);
 
 /* copy a chunk. either src_sftp or dst_sftp is not null, and another is null */
 int copy_chunk(struct chunk *c, sftp_session src_sftp, sftp_session dst_sftp,
-	       int nr_ahead, int buf_sz, bool preserve_ts, size_t *counter);
+	       int nr_ahead, int buf_sz, bool preserve_ts, struct bwlimit *bw,
+	       size_t *counter);
 
 #endif /* _PATH_H_ */

src/platform.h

@@ -2,6 +2,8 @@
 #ifndef _PLATFORM_H_
 #define _PLATFORM_H_
 
+#include <config.h>
+
 #include <pthread.h>
 #include <semaphore.h>
 #include <stdint.h>
@@ -25,7 +27,7 @@ int sem_release(sem_t *sem);
 #include <arpa/inet.h> /* Apple has htonll and ntohll in arpa/inet.h */
 #endif
 
-/* copied from libssh: libssh/include/libssh/priv.h*/
+/* copied from libssh: libssh/include/libssh/priv.h */
 #ifndef HAVE_HTONLL
 #ifdef WORDS_BIGENDIAN
 #define htonll(x) (x)

src/ssh.c

@@ -4,13 +4,15 @@
 #include <unistd.h>
 #include <stdlib.h>
 
-#include "libssh/callbacks.h"
-
 #include <ssh.h>
 #include <mscp.h>
 #include <strerrno.h>
 
+#include "libssh/callbacks.h"
+#include "libssh/options.h"
+
 static int ssh_verify_known_hosts(ssh_session session);
+static int ssh_authenticate_kbdint(ssh_session session);
 
 static int ssh_set_opts(ssh_session ssh, struct mscp_ssh_opts *opts)
 {
@@ -86,6 +88,27 @@ static int ssh_set_opts(ssh_session ssh, struct mscp_ssh_opts *opts)
 		return -1;
 	}
 
+	if (opts->proxyjump) {
+		char buf[256];
+		memset(buf, 0, sizeof(buf));
+		snprintf(buf, sizeof(buf), "proxyjump=%s", opts->proxyjump);
+		if (ssh_config_parse_string(ssh, buf) != SSH_OK) {
+			priv_set_errv("failed to set ssh option: %s", buf);
+			return -1;
+		}
+	}
+
+	if (opts->options) {
+		int n;
+		for (n = 0; opts->options[n]; n++) {
+			if (ssh_config_parse_string(ssh, opts->options[n]) != SSH_OK) {
+				priv_set_errv("failed to set ssh option: %s",
+					      opts->options[n]);
+				return -1;
+			}
+		}
+	}
+
 	return 0;
 }
 
@@ -100,17 +123,18 @@ static int ssh_authenticate(ssh_session ssh, struct mscp_ssh_opts *opts)
 		return 0;
 
 	auth_bit_mask = ssh_userauth_list(ssh, NULL);
-
 	if (auth_bit_mask & SSH_AUTH_METHOD_NONE &&
 	    ssh_userauth_none(ssh, NULL) == SSH_AUTH_SUCCESS)
 		return 0;
 
+	auth_bit_mask = ssh_userauth_list(ssh, NULL);
 	if (auth_bit_mask & SSH_AUTH_METHOD_PUBLICKEY) {
 		char *p = opts->passphrase ? opts->passphrase : NULL;
 		if (ssh_userauth_publickey_auto(ssh, NULL, p) == SSH_AUTH_SUCCESS)
 			return 0;
 	}
 
+	auth_bit_mask = ssh_userauth_list(ssh, NULL);
 	if (auth_bit_mask & SSH_AUTH_METHOD_PASSWORD) {
 		if (!opts->password) {
 			char buf[128] = {};
@@ -128,6 +152,12 @@ static int ssh_authenticate(ssh_session ssh, struct mscp_ssh_opts *opts)
 			return 0;
 	}
 
+	auth_bit_mask = ssh_userauth_list(ssh, NULL);
+	if (auth_bit_mask & SSH_AUTH_METHOD_INTERACTIVE) {
+		if (ssh_authenticate_kbdint(ssh) == SSH_AUTH_SUCCESS)
+			return 0;
+	}
+
 	return -1;
 }
 
@@ -191,7 +221,7 @@ static ssh_session ssh_init_session(const char *sshdst, struct mscp_ssh_opts *op
 		goto disconnect_out;
 	}
 
-	if (!opts->no_hostkey_check && ssh_verify_known_hosts(ssh) != 0) {
+	if (ssh_verify_known_hosts(ssh) != 0) {
 		priv_set_errv("ssh_veriy_known_hosts failed");
 		goto disconnect_out;
 	}
@@ -319,6 +349,54 @@ static int ssh_verify_known_hosts(ssh_session session)
 	return 0;
 }
 
+static int ssh_authenticate_kbdint(ssh_session ssh)
+{
+	/* Copied and bit modified from
+	 * https://api.libssh.org/stable/libssh_tutor_authentication.html */
+	int rc;
+
+	rc = ssh_userauth_kbdint(ssh, NULL, NULL);
+	while (rc == SSH_AUTH_INFO) {
+		const char *name, *instruction;
+		int nprompts, iprompt;
+
+		name = ssh_userauth_kbdint_getname(ssh);
+		instruction = ssh_userauth_kbdint_getinstruction(ssh);
+		nprompts = ssh_userauth_kbdint_getnprompts(ssh);
+
+		if (strlen(name) > 0)
+			printf("%s\n", name);
+		if (strlen(instruction) > 0)
+			printf("%s\n", instruction);
+		for (iprompt = 0; iprompt < nprompts; iprompt++) {
+			const char *prompt;
+			char echo;
+
+			prompt = ssh_userauth_kbdint_getprompt(ssh, iprompt, &echo);
+			if (echo) {
+				char buf[128], *ptr;
+
+				printf("%s", prompt);
+				if (fgets(buf, sizeof(buf), stdin) == NULL)
+					return SSH_AUTH_ERROR;
+				buf[sizeof(buf) - 1] = '\0';
+				if ((ptr = strchr(buf, '\n')) != NULL)
+					*ptr = '\0';
+				if (ssh_userauth_kbdint_setanswer(ssh, iprompt, buf) < 0)
+					return SSH_AUTH_ERROR;
+				memset(buf, 0, strlen(buf));
+			} else {
+				char *ptr;
+				ptr = getpass(prompt);
+				if (ssh_userauth_kbdint_setanswer(ssh, iprompt, ptr) < 0)
+					return SSH_AUTH_ERROR;
+			}
+		}
+		rc = ssh_userauth_kbdint(ssh, NULL, NULL);
+	}
+	return rc;
+}
+
 void ssh_sftp_close(sftp_session sftp)
 {
 	ssh_session ssh = sftp_ssh(sftp);

test/test_e2e.py

@@ -6,6 +6,7 @@ test_e2e.py: End-to-End test for mscp executable.
 import platform
 import pytest
 import getpass
+import datetime
 import time
 import os
 import shutil
@@ -67,7 +68,7 @@ param_single_copy = [
 @pytest.mark.parametrize("src, dst", param_single_copy)
 def test_single_copy(mscp, src_prefix, dst_prefix, src, dst):
     src.make()
-    run2ok([mscp, "-H", "-vvv", src_prefix + src.path, dst_prefix + dst.path])
+    run2ok([mscp, "-vvv", src_prefix + src.path, dst_prefix + dst.path])
     assert check_same_md5sum(src, dst)
     src.cleanup()
     dst.cleanup()
@@ -76,7 +77,7 @@ def test_single_copy(mscp, src_prefix, dst_prefix, src, dst):
 def test_failed_to_copy_nonexistent_file(mscp, src_prefix, dst_prefix):
     src = "nonexistent_src"
     dst = "nonexistent_dst"
-    run2ng([mscp, "-H", "-vvv", src_prefix + src, dst_prefix + dst])
+    run2ng([mscp, "-vvv", src_prefix + src, dst_prefix + dst])
 
 param_double_copy = [
     (File("src1", size = 1024 * 1024), File("src2", size = 1024 * 1024),
@@ -88,7 +89,7 @@ param_double_copy = [
 def test_double_copy(mscp, src_prefix, dst_prefix, s1, s2, d1, d2):
     s1.make()
     s2.make()
-    run2ok([mscp, "-H", "-vvv", src_prefix + s1.path, src_prefix + s2.path, dst_prefix + "dst"])
+    run2ok([mscp, "-vvv", src_prefix + s1.path, src_prefix + s2.path, dst_prefix + "dst"])
     assert check_same_md5sum(s1, d1)
     assert check_same_md5sum(s2, d2)
     s1.cleanup()
@@ -106,7 +107,7 @@ param_remote_v6_prefix = [
 def test_double_copy_with_ipv6_notation(mscp, src_prefix, dst_prefix, s1, s2, d1, d2):
     s1.make()
     s2.make()
-    run2ok([mscp, "-H", "-vvv",
+    run2ok([mscp, "-vvv",
             src_prefix + s1.path, src_prefix + s2.path, dst_prefix + "dst"])
     assert check_same_md5sum(s1, d1)
     assert check_same_md5sum(s2, d2)
@@ -126,7 +127,7 @@ def test_double_copy_with_user_and_ipv6_notation(mscp, src_prefix, dst_prefix,
                                                  s1, s2, d1, d2):
     s1.make()
     s2.make()
-    run2ok([mscp, "-H", "-vvv",
+    run2ok([mscp, "-vvv",
             src_prefix + s1.path, src_prefix + s2.path, dst_prefix + "dst"])
     assert check_same_md5sum(s1, d1)
     assert check_same_md5sum(s2, d2)
@@ -166,11 +167,11 @@ def test_dir_copy(mscp, src_prefix, dst_prefix, src_dir, dst_dir, src, dst, twic
     for f in src:
         f.make()
 
-    run2ok([mscp, "-H", "-vvv", src_prefix + src_dir, dst_prefix + dst_dir])
+    run2ok([mscp, "-vvv", src_prefix + src_dir, dst_prefix + dst_dir])
     for sf, df in zip(src, dst):
         assert check_same_md5sum(sf, df)
 
-    run2ok([mscp, "-H", "-vvv", src_prefix + src_dir, dst_prefix + dst_dir])
+    run2ok([mscp, "-vvv", src_prefix + src_dir, dst_prefix + dst_dir])
     for sf, df in zip(src, twice):
         assert check_same_md5sum(sf, df)
 
@@ -191,7 +192,7 @@ param_dir_copy_single = [
 def test_dir_copy_single(mscp, src_prefix, dst_prefix, src_dir, dst_dir, src, dst):
     src.make()
     os.mkdir(dst_dir)
-    run2ok([mscp, "-H", "-vvv", src_prefix + src_dir, dst_prefix + dst_dir])
+    run2ok([mscp, "-vvv", src_prefix + src_dir, dst_prefix + dst_dir])
     assert check_same_md5sum(src, dst)
     src.cleanup()
     dst.cleanup()
@@ -202,7 +203,7 @@ def test_override_single_file(mscp, src_prefix, dst_prefix):
     dst = File("dst", size = 128).make()
     assert not check_same_md5sum(src, dst)
 
-    run2ok([mscp, "-H", "-vvv", src_prefix + src.path, dst_prefix + dst.path])
+    run2ok([mscp, "-vvv", src_prefix + src.path, dst_prefix + dst.path])
     assert check_same_md5sum(src, dst)
 
     src.cleanup()
@@ -217,7 +218,7 @@ def test_copy_file_under_root_to_dir(mscp, src_prefix, dst_prefix):
     src = File("/mscp-test-src", size = 1024).make()
     dst = File("/tmp/mscp-test-src")
 
-    run2ok([mscp, "-H", "-vvv", src_prefix + src.path,
+    run2ok([mscp, "-vvv", src_prefix + src.path,
             dst_prefix + os.path.dirname(dst.path)])
     assert check_same_md5sum(src, dst)
     src.cleanup()
@@ -229,7 +230,7 @@ def test_min_chunk(mscp, src_prefix, dst_prefix):
     src = File("src", size = 16 * 1024).make()
     dst = File("dst")
 
-    run2ok([mscp, "-H", "-vvv", "-s", 32768, src_prefix + src.path, dst_prefix + dst.path])
+    run2ok([mscp, "-vvv", "-s", 32768, src_prefix + src.path, dst_prefix + dst.path])
     assert check_same_md5sum(src, dst)
 
     src.cleanup()
@@ -266,7 +267,7 @@ def test_glob_src_path(mscp, src_prefix, dst_prefix,
     for src in srcs:
         src.make(size = 1024 * 1024)
 
-    run2ok([mscp, "-H", "-vvv", src_prefix + src_glob_path, dst_prefix + dst_path])
+    run2ok([mscp, "-vvv", src_prefix + src_glob_path, dst_prefix + dst_path])
     for src, dst in zip(srcs, dsts):
         assert check_same_md5sum(src, dst)
         src.cleanup()
@@ -277,7 +278,7 @@ def test_thread_affinity(mscp, src_prefix, dst_prefix):
     src = File("src", size = 64 * 1024).make()
     dst = File("dst")
 
-    run2ok([mscp, "-H", "-vvv", "-n", 4, "-m", "0x01",
+    run2ok([mscp, "-vvv", "-n", 4, "-m", "0x01",
             src_prefix + src.path, dst_prefix + dst.path])
     assert check_same_md5sum(src, dst)
 
@@ -289,7 +290,7 @@ def test_cannot_override_file_with_dir(mscp, src_prefix, dst_prefix):
     src = File("src", size = 128).make()
     dst = File("dst").make()
 
-    run2ng([mscp, "-H", "-vvv", src_prefix + src.path, dst_prefix + "dst/src"])
+    run2ng([mscp, "-vvv", src_prefix + src.path, dst_prefix + "dst/src"])
 
     src.cleanup()
     dst.cleanup()
@@ -298,7 +299,7 @@ def test_cannot_override_file_with_dir(mscp, src_prefix, dst_prefix):
 def test_transfer_zero_bytes(mscp, src_prefix, dst_prefix):
     src = File("src", size = 0).make()
     dst = File("dst")
-    run2ok([mscp, "-H", "-vvv", src_prefix + src.path, dst_prefix + "dst"])
+    run2ok([mscp, "-vvv", src_prefix + src.path, dst_prefix + "dst"])
     assert os.path.exists("dst")
     src.cleanup()
     dst.cleanup()
@@ -307,7 +308,7 @@ def test_transfer_zero_bytes(mscp, src_prefix, dst_prefix):
 def test_override_dst_having_larger_size(mscp, src_prefix, dst_prefix):
     src = File("src", size = 1024 * 1024).make()
     dst = File("dst", size = 1024 * 1024 * 2).make()
-    run2ok([mscp, "-H", "-vvv", src_prefix + src.path, dst_prefix + "dst"])
+    run2ok([mscp, "-vvv", src_prefix + src.path, dst_prefix + "dst"])
     assert check_same_md5sum(src, dst)
     src.cleanup()
     dst.cleanup()
@@ -316,11 +317,26 @@ def test_override_dst_having_larger_size(mscp, src_prefix, dst_prefix):
 def test_dont_truncate_dst(mscp, src_prefix, dst_prefix):
     f = File("srcanddst", size = 1024 * 1024 * 128).make()
     md5_before = f.md5sum()
-    run2ok([mscp, "-H", "-vvv", src_prefix + f.path, dst_prefix + f.path])
+    run2ok([mscp, "-vvv", src_prefix + f.path, dst_prefix + f.path])
     md5_after = f.md5sum()
     assert md5_before == md5_after
     f.cleanup()
 
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_copy_readonly_file(mscp, src_prefix, dst_prefix):
+    """When a source file permission is r--r--r--, if chmod(r--r--r--)
+    runs first on the remote side, following truncate() and setutime()
+    fail due to permission deneid. So, run chmod() after truncate()
+    and setutime()
+
+    """
+    src = File("src", size = 1024 * 1024 * 128, perm = 0o444).make()
+    dst = File("dst")
+    run2ok([mscp, "-vvv", src_prefix + src.path, dst_prefix + dst.path])
+    assert check_same_md5sum(src, dst)
+    src.cleanup()
+    dst.cleanup()
+
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 def test_dont_make_conns_more_than_chunks(mscp, src_prefix, dst_prefix):
     # copy 100 files with -n 20 -I 1 options. if mscp creates 20 SSH
@@ -331,7 +347,7 @@ def test_dont_make_conns_more_than_chunks(mscp, src_prefix, dst_prefix):
         srcs.append(File("src/src-{:06d}".format(n), size=1024).make())
         dsts.append(File("dst/src-{:06d}".format(n)))
     start = time.time()
-    run2ok([mscp, "-H", "-v", "-n", "20", "-I", "1",
+    run2ok([mscp, "-v", "-n", "20", "-I", "1",
             src_prefix + "src", dst_prefix + "dst"])
     end = time.time()
     for s, d in zip(srcs, dsts):
@@ -341,25 +357,40 @@ def test_dont_make_conns_more_than_chunks(mscp, src_prefix, dst_prefix):
     assert((end - start) < 10)
 
 
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_bwlimit(mscp, src_prefix, dst_prefix):
+    """Copy 100MB file with 100Mbps bitrate, this requires 8 seconds."""
+    src = File("src", size = 100 * 1024 * 1024).make()
+    dst = File("dst")
+
+    start = datetime.datetime.now().timestamp()
+    run2ok([mscp, "-vvv", "-L", "100m", src_prefix + "src", dst_prefix + "dst"])
+    end = datetime.datetime.now().timestamp()
+    assert check_same_md5sum(src, dst)
+    src.cleanup()
+    dst.cleanup()
+    assert end - start > 7
+
+
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 @pytest.mark.parametrize("src, dst", param_single_copy)
 def test_set_port_ng(mscp, src_prefix, dst_prefix, src, dst):
     src.make()
-    run2ng([mscp, "-H", "-vvv", "-P", 21, src_prefix + src.path, dst_prefix + dst.path])
+    run2ng([mscp, "-vvv", "-P", 21, src_prefix + src.path, dst_prefix + dst.path])
     src.cleanup()
 
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 @pytest.mark.parametrize("src, dst", param_single_copy)
 def test_set_port_ok(mscp, src_prefix, dst_prefix, src, dst):
     src.make()
-    run2ok([mscp, "-H", "-vvv", "-P", 8022, src_prefix + src.path, dst_prefix + dst.path])
+    run2ok([mscp, "-vvv", "-P", 8022, src_prefix + src.path, dst_prefix + dst.path])
     src.cleanup()
 
 def test_v4only(mscp):
     src = File("src", size = 1024).make()
     dst = File("dst")
     dst_prefix = "localhost:{}/".format(os.getcwd())
-    run2ok([mscp, "-H", "-vvv", "-4", src.path, dst_prefix + dst.path])
+    run2ok([mscp, "-vvv", "-4", src.path, dst_prefix + dst.path])
     assert check_same_md5sum(src, dst)
     src.cleanup()
     dst.cleanup()
@@ -368,7 +399,7 @@ def test_v6only(mscp):
     src = File("src", size = 1024).make()
     dst = File("dst")
     dst_prefix = "ip6-localhost:{}/".format(os.getcwd())
-    run2ok([mscp, "-H", "-vvv", "-6", src.path, dst_prefix + dst.path])
+    run2ok([mscp, "-vvv", "-6", src.path, dst_prefix + dst.path])
     assert check_same_md5sum(src, dst)
     src.cleanup()
     dst.cleanup()
@@ -377,14 +408,14 @@ def test_v4_to_v6_should_fail(mscp):
     src = File("src", size = 1024).make()
     dst = File("dst")
     dst_prefix = "[::1]:{}/".format(os.getcwd())
-    run2ng([mscp, "-H", "-vvv", "-4", src.path, dst_prefix + dst.path])
+    run2ng([mscp, "-vvv", "-4", src.path, dst_prefix + dst.path])
     src.cleanup()
 
 def test_v6_to_v4_should_fail(mscp):
     src = File("src", size = 1024).make()
     dst = File("dst")
     dst_prefix = "127.0.0.1:{}/".format(os.getcwd())
-    run2ng([mscp, "-H", "-vvv", "-6", src.path, dst_prefix + dst.path])
+    run2ng([mscp, "-vvv", "-6", src.path, dst_prefix + dst.path])
     src.cleanup()
 
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
@@ -394,7 +425,7 @@ def test_set_conn_interval(mscp, src_prefix, dst_prefix):
     for x in range(500):
         srcs.append(File("src/file{}".format(x), size = 128).make())
         dsts.append(File("dst/file{}".format(x)))
-    run2ok([mscp, "-H", "-vvv", "-I", 1, src_prefix + "src", dst_prefix + "dst"])
+    run2ok([mscp, "-vvv", "-I", 1, src_prefix + "src", dst_prefix + "dst"])
 
     for src, dst in zip(srcs, dsts):
         assert check_same_md5sum(src, dst)
@@ -407,7 +438,7 @@ compressions = ["yes", "no", "none"]
 def test_compression(mscp, src_prefix, dst_prefix, compress):
     src = File("src", size = 1024 * 1024).make()
     dst = File("dst", size = 1024 * 1024 * 2).make()
-    run2ok([mscp, "-H", "-vvv", "-C", compress, src_prefix + src.path, dst_prefix + "dst"])
+    run2ok([mscp, "-vvv", "-C", compress, src_prefix + src.path, dst_prefix + "dst"])
     assert check_same_md5sum(src, dst)
     src.cleanup()
     dst.cleanup()
@@ -425,7 +456,7 @@ def test_ccalgo(mscp, src_prefix, dst_prefix):
         with open("/proc/sys/net/ipv4/tcp_allowed_congestion_control", "r") as f:
             algo = f.read().strip().split().pop()
         run = run2ok
-    run([mscp, "-H", "-vvv", "-g", algo, src_prefix + src.path, dst_prefix + "dst"])
+    run([mscp, "-vvv", "-g", algo, src_prefix + src.path, dst_prefix + "dst"])
 
 
 testhost = "mscptestlocalhost"
@@ -442,7 +473,7 @@ def test_config_ok(mscp, src_prefix, dst_prefix):
 
     src = File("src", size = 1024 * 1024).make()
     dst = File("dst", size = 1024 * 1024 * 2).make()
-    run2ok([mscp, "-H", "-vvv", "-F", config,
+    run2ok([mscp, "-vvv", "-F", config,
             src_prefix + src.path, dst_prefix + "dst"])
 
     os.remove(config)
@@ -458,20 +489,79 @@ def test_config_ng(mscp, src_prefix, dst_prefix):
 
     src = File("src", size = 1024 * 1024).make()
     dst = File("dst", size = 1024 * 1024 * 2).make()
-    run2ng([mscp, "-H", "-vvv", "-F", config,
+    run2ng([mscp, "-vvv", "-F", config,
             src_prefix + src.path, dst_prefix + "dst"])
 
     os.remove(config)
     src.cleanup()
     dst.cleanup()
 
+
+param_valid_option_ok = [
+    [ "-o", "Port=8022" ],
+    [ "-o", "Port=8022", "-o", "User=root" ],
+    [ "-o", "unknown-option-is-silently-ignored" ],
+]
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+@pytest.mark.parametrize("option", param_valid_option_ok)
+def test_inline_option_ok(mscp, src_prefix, dst_prefix, option):
+    """ change port number with -o option. it should be ok. """
+    src = File("src", size = 1024 * 1024).make()
+    dst = File("dst")
+    run2ok([mscp, "-vvv"] + option +
+           [src_prefix + src.path, dst_prefix + dst.path])
+    assert check_same_md5sum(src, dst)
+    src.cleanup()
+    dst.cleanup()
+
+
+param_valid_option_ng = [
+    [ "-o", "Port=8023" ],
+    [ "-o", "User=invaliduser" ],
+]
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+@pytest.mark.parametrize("option", param_valid_option_ng)
+def test_inline_option_ng(mscp, src_prefix, dst_prefix, option):
+    """ change port number with -o option. it should be ng. """
+    src = File("src", size = 1024 * 1024).make()
+    dst = File("dst")
+    run2ng([mscp, "-vvv"] + option +
+           [src_prefix + src.path, dst_prefix + dst.path])
+    src.cleanup()
+
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_porxyjump_ok(mscp, src_prefix, dst_prefix):
+    """ test -J proxyjump option"""
+    src = File("src", size = 10 * 1024 * 1024).make()
+    dst = File("dst")
+    # use small min-chunk-size to use multiple connections
+    run2ok([mscp, "-n", 4, "-s", 1024 * 1024, "-vvv",
+            "-J", "localhost:8022",
+            src_prefix + src.path, dst_prefix + dst.path])
+    assert check_same_md5sum(src, dst)
+    src.cleanup()
+    dst.cleanup()
+
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_porxyjump_ng(mscp, src_prefix, dst_prefix):
+    """ test -J proxyjump option, invalid jump node causes fail"""
+    src = File("src", size = 10 * 1024 * 1024).make()
+    dst = File("dst")
+    # use small min-chunk-size to use multiple connections
+    run2ng([mscp, "-n", 4, "-s", 1024 * 1024, "-vvv",
+            "-J", "invaliduser@localhost:8022",
+            src_prefix + src.path, dst_prefix + dst.path])
+    src.cleanup()
+
 # username test assumes that this test runs inside a container, see Dockerfiles
 def test_specify_passphrase_via_env(mscp):
     src = File(os.getcwd() + "/src", size = 1024).make()
     dst = File("/home/test/dst")
     env = os.environ
     env["MSCP_SSH_AUTH_PASSPHRASE"]  = "keypassphrase"
-    run2ok([mscp, "-H", "-vvv", "-l", "test", "-i", "/home/test/.ssh/id_rsa_test",
+    run2ok([mscp, "-vvv", "-l", "test", "-i", "/home/test/.ssh/id_rsa_test",
             src.path, "localhost:" + dst.path], env = env)
     assert check_same_md5sum(src, dst)
     src.cleanup()
@@ -482,7 +572,7 @@ def test_specify_invalid_passphrase_via_env(mscp):
     dst = File("/home/test/dst")
     env = os.environ
     env["MSCP_SSH_AUTH_PASSPHRASE"]  = "invalid-keypassphrase"
-    run2ng([mscp, "-H", "-vvv", "-l", "test", "-i", "/home/test/.ssh/id_rsa_test",
+    run2ng([mscp, "-vvv", "-l", "test", "-i", "/home/test/.ssh/id_rsa_test",
             src.path, "localhost:" + dst.path], env = env)
     src.cleanup()
 
@@ -491,7 +581,7 @@ def test_specify_password_via_env(mscp):
     dst = File("/home/test/dst")
     env = os.environ
     env["MSCP_SSH_AUTH_PASSWORD"]  = "userpassword"
-    run2ok([mscp, "-H", "-vvv", "-l", "test",
+    run2ok([mscp, "-vvv", "-l", "test",
             src.path, "localhost:" + dst.path], env = env)
     assert check_same_md5sum(src, dst)
     src.cleanup()
@@ -502,7 +592,7 @@ def test_specify_invalid_password_via_env(mscp):
     dst = File("/home/test/dst")
     env = os.environ
     env["MSCP_SSH_AUTH_PASSWORD"]  = "invalid-userpassword"
-    run2ng([mscp, "-H", "-vvv", "-l", "test",
+    run2ng([mscp, "-vvv", "-l", "test",
             src.path, "localhost:" + dst.path], env = env)
     src.cleanup()
 
@@ -513,7 +603,7 @@ def test_10k_files(mscp, src_prefix, dst_prefix):
     for n in range(10000):
         srcs.append(File("src/src-{:06d}".format(n), size=1024).make())
         dsts.append(File("dst/src-{:06d}".format(n)))
-    run2ok([mscp, "-H", "-v", src_prefix + "src", dst_prefix + "dst"])
+    run2ok([mscp, "-v", src_prefix + "src", dst_prefix + "dst"])
     for s, d in zip(srcs, dsts):
         assert check_same_md5sum(s, d)
     shutil.rmtree("src")
@@ -521,15 +611,15 @@ def test_10k_files(mscp, src_prefix, dst_prefix):
 
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 def test_checkpoint_dump_and_resume(mscp, src_prefix, dst_prefix):
-    src1 = File("src1", size = 512 * 1024 * 1024).make()
+    src1 = File("src1", size = 64 * 1024 * 1024).make()
-    src2 = File("src2", size = 512 * 1024 * 1024).make()
+    src2 = File("src2", size = 64 * 1024 * 1024).make()
     dst1 = File("dst/src1")
     dst2 = File("dst/src2")
-    run2ok([mscp, "-H", "-vvv", "-W", "checkpoint", "-D",
+    run2ok([mscp, "-vvv", "-W", "checkpoint", "-D",
             src_prefix + "src1", src_prefix + "src2", dst_prefix + "dst"])
     assert os.path.exists("checkpoint")
 
-    run2ok([mscp, "-H", "-vvv", "-R", "checkpoint"])
+    run2ok([mscp, "-vvv", "-R", "checkpoint"])
     assert check_same_md5sum(src1, dst1)
     assert check_same_md5sum(src2, dst2)
     src1.cleanup()
@@ -538,20 +628,20 @@ def test_checkpoint_dump_and_resume(mscp, src_prefix, dst_prefix):
     dst2.cleanup()
     os.remove("checkpoint")
 
-@pytest.mark.parametrize("timeout", [1,2,3])
+@pytest.mark.parametrize("timeout", [1, 2, 3, 4, 5, 6])
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 def test_checkpoint_interrupt_and_resume(mscp, timeout, src_prefix, dst_prefix):
-    src1 = File("src1", size = 1024 * 1024 * 1024).make()
+    """Copy two 100MB files with 200Mbps -> 4 sec + 4 sec """
-    src2 = File("src2", size = 1024 * 1024 * 1024).make()
+    src1 = File("src1", size = 100 * 1024 * 1024).make()
+    src2 = File("src2", size = 100 * 1024 * 1024).make()
     dst1 = File("dst/src1")
     dst2 = File("dst/src2")
-    run2ng([mscp, "-H", "-vv", "-W", "checkpoint",
+    run2ng([mscp, "-vv", "-W", "checkpoint", "-L", "200m",
-            "-n", 1, "-s", 8192, "-S", 16384,
             src_prefix + "src1", src_prefix + "src2", dst_prefix + "dst"],
            timeout = timeout)
     assert os.path.exists("checkpoint")
 
-    run2ok([mscp, "-H", "-vv", "-R", "checkpoint"])
+    run2ok([mscp, "-vv", "-R", "checkpoint"])
     assert check_same_md5sum(src1, dst1)
     assert check_same_md5sum(src2, dst2)
     src1.cleanup()
@@ -559,3 +649,4 @@ def test_checkpoint_interrupt_and_resume(mscp, timeout, src_prefix, dst_prefix):
     dst1.cleanup()
     dst2.cleanup()
     os.remove("checkpoint")
+