bump version to 0.2.1

and clean-up chunk_sz related parts.

VERSION

@@ -1 +1 @@
-0.2.0
+0.2.1

debian/changelog

@@ -1,4 +1,16 @@
-mscp (0.2.0) UNRELEASED; urgency=medium
+mscp (0.2.1) UNRELEASED; urgency=medium
+
+  * fix broken checkpoint files when copying multiple files (#16)
+  * fix broken password inputs for resume transfers (#17 and #18)
+  * add support [kKmMgG] units for -s, -S, and -b options (#20)
+  * change the default min chunk size to 16MB (#20)
+  * change the default max chunk size to filesize / (nr_conn * 4) (#20)
+  * -s and -S accept integers other than multiples of page sizes (#20)
+  * help now shows available ciphers and HMACs (#20)
+
+ -- Ryo Nakamura <upa@haeena.net>  Sat, 11 May 2024 14:49:52 +0900
+
+mscp (0.2.0) unstable; urgency=medium
 
   * add -J DESTINATION option for ProxyJump (#15)
   * add -o SSH_OPTION option

doc/mscp.1.in

@@ -130,7 +130,8 @@ pinned to any cores.
 
 .TP
 .B \-u \fIMAX_STARTUPS\fR
-Specifies the number of concurrent outgoing SSH connections.
+Specifies the number of concurrent unauthenticated SSH connection
+attempts.
 .B sshd
 limits the number of simultaneous SSH connection attempts by
 .I MaxStartups
@@ -192,12 +193,16 @@ and remove the checkpoint if it returns 0.
 Specifies the minimum chunk size.
 .B mscp
 divides a single file into chunks and copies the chunks in
-parallel. The default value is 67108864 (64MB).
+parallel. The default value is 16M bytes.
 
 .TP
 .B \-S \fIMAX_CHUNK_SIZE\fR
 Specifies the maximum chunk size. The default is file size divided by
-the number of connections.
+the number of connections and devided by 4. If the calculated value
+is smarller than the
+.B MIN_CHUNK_SIZE
+value,
+MIN_CHUNK_SIZE is used.
 
 .TP
 .B \-a \fINR_AHEAD\fR
@@ -289,6 +294,10 @@ connection to the jump host described by
 .TP
 .B \-c \fICIPHER\fR
 Selects the cipher to use for encrypting the data transfer. See
+.B mscp -h
+or
+.B Ciphers
+in
 .UR https://\:www\:.libssh\:.org/\:features/
 libssh features
 .UE .
@@ -296,6 +305,10 @@ libssh features
 .TP
 .B \-M \fIHMAC\fR
 Specifies MAC hash algorithms. See
+.B mscp -h
+or
+.B MAC hashes
+in
 .UR https://\:www\:.libssh\:.org/\:features/
 libssh features
 .UE .
@@ -311,6 +324,9 @@ libssh features
 .TP
 .B \-g \fICONGESTION\fR
 Specifies the TCP congestion control algorithm to use (Linux only).
+See
+.B sysctl net.ipv4.tcp_allowed_congestion_control
+for available values.
 
 .TP
 .B \-p

doc/mscp.rst

@@ -2,7 +2,7 @@
 MSCP
 ====
 
-:Date: v0.1.5-18-ge47d5b7
+:Date: v0.2.0-9-g675126a
 
 NAME
 ====
@@ -60,10 +60,10 @@ OPTIONS
    pinned to any cores.
 
 **-u MAX_STARTUPS**
-   Specifies the number of concurrent outgoing SSH connections. **sshd**
-   limits the number of simultaneous SSH connection attempts by
-   *MaxStartups* in *sshd_config.* The default *MaxStartups* is 10;
-   thus, we set the default MAX_STARTUPS 8.
+   Specifies the number of concurrent unauthenticated SSH connection
+   attempts. **sshd** limits the number of simultaneous SSH connection
+   attempts by *MaxStartups* in *sshd_config.* The default *MaxStartups*
+   is 10; thus, we set the default MAX_STARTUPS 8.
 
 **-I INTERVAL**
    Specifies the interval (in seconds) between SSH connection attempts.
@@ -97,12 +97,14 @@ OPTIONS
 
 **-s MIN_CHUNK_SIZE**
    Specifies the minimum chunk size. **mscp** divides a single file into
-   chunks and copies the chunks in parallel. The default value is
-   67108864 (64MB).
+   chunks and copies the chunks in parallel. The default value is 16M
+   bytes.
 
 **-S MAX_CHUNK_SIZE**
    Specifies the maximum chunk size. The default is file size divided by
-   the number of connections.
+   the number of connections and devided by 4. If the calculated value
+   is smarller than the **MIN_CHUNK_SIZE** value, MIN_CHUNK_SIZE is
+   used.
 
 **-a NR_AHEAD**
    Specifies the number of inflight SFTP commands. The default value is
@@ -166,11 +168,12 @@ OPTIONS
 
 **-c CIPHER**
    Selects the cipher to use for encrypting the data transfer. See
-   `libssh features <https://www.libssh.org/features/>`__.
+   **mscp -h** or **Ciphers** in `libssh
+   features <https://www.libssh.org/features/>`__.
 
 **-M HMAC**
-   Specifies MAC hash algorithms. See `libssh
-   features <https://www.libssh.org/features/>`__.
+   Specifies MAC hash algorithms. See **mscp -h** or **MAC hashes** in
+   `libssh features <https://www.libssh.org/features/>`__.
 
 **-C COMPRESS**
    Enables compression: yes, no, zlib, zlib@openssh.com. The default is
@@ -178,6 +181,8 @@ OPTIONS
 
 **-g CONGESTION**
    Specifies the TCP congestion control algorithm to use (Linux only).
+   See **sysctl net.ipv4.tcp_allowed_congestion_control** for available
+   values.
 
 **-p**
    Preserves modification times and access times (file mode bits are

include/mscp.h

@@ -294,4 +294,15 @@ enum {
 };
 
 
+/**
+ * @brief Return available ciphers.
+ */
+const char **mscp_ssh_ciphers(void);
+
+/**
+ * @brief Return available hmacs.
+ */
+ const char **mscp_ssh_hmacs(void);
+
+
 #endif /* _MSCP_H_ */

patch/libssh-0.10.6-2-g6f1b1e76.patch

@@ -37,7 +37,7 @@ index 1fce7b76..b64d1455 100644
  int ssh_buffer_validate_length(struct ssh_buffer_struct *buffer, size_t len);
  
 diff --git a/include/libssh/libssh.h b/include/libssh/libssh.h
-index 669a0a96..da5b4099 100644
+index 669a0a96..26b20f3f 100644
 --- a/include/libssh/libssh.h
 +++ b/include/libssh/libssh.h
 @@ -368,6 +368,7 @@ enum ssh_options_e {
@@ -64,11 +64,14 @@ index 669a0a96..da5b4099 100644
  LIBSSH_API void ssh_buffer_free(ssh_buffer buffer);
  #define SSH_BUFFER_FREE(x) \
      do { if ((x) != NULL) { ssh_buffer_free(x); x = NULL; } } while(0)
-@@ -843,6 +846,8 @@ LIBSSH_API void *ssh_buffer_get(ssh_buffer buffer);
+@@ -843,6 +846,11 @@ LIBSSH_API void *ssh_buffer_get(ssh_buffer buffer);
  LIBSSH_API uint32_t ssh_buffer_get_len(ssh_buffer buffer);
  LIBSSH_API int ssh_session_set_disconnect_message(ssh_session session, const char *message);
  
 +typedef ssize_t (*ssh_add_func) (void *ptr, size_t max_bytes, void *userdata);
++
++LIBSSH_API const char **ssh_ciphers(void);
++LIBSSH_API const char **ssh_hmacs(void);
 +
  #ifndef LIBSSH_LEGACY_0_4
  #include "libssh/legacy.h"
@@ -299,6 +302,60 @@ index 15cae644..02ef43b4 100644
          errno = 0;
          rc = connect(s, itr->ai_addr, itr->ai_addrlen);
          if (rc == -1 && (errno != 0) && (errno != EINPROGRESS)) {
+diff --git a/src/misc.c b/src/misc.c
+index 7081f12a..e3879fe4 100644
+--- a/src/misc.c
++++ b/src/misc.c
+@@ -71,6 +71,8 @@
+ #include "libssh/priv.h"
+ #include "libssh/misc.h"
+ #include "libssh/session.h"
++#include "libssh/wrapper.h"
++#include "libssh/crypto.h"
+ 
+ #ifdef HAVE_LIBGCRYPT
+ #define GCRYPT_STRING "/gnutls"
+@@ -2074,4 +2076,40 @@ int ssh_check_hostname_syntax(const char *hostname)
+     return SSH_OK;
+ }
+ 
++/**
++ * @brief Return supported cipher names
++ * @return	The list of cipher names.
++ */
++const char **ssh_ciphers(void)
++{
++     struct ssh_cipher_struct *tab=ssh_get_ciphertab();
++     static const char *ciphers[32];
++     int n;
++
++     memset(ciphers, 0, sizeof(*ciphers));
++
++     for (n = 0; tab[n].name != NULL; n++) {
++	  ciphers[n] = tab[n].name;
++     }
++     return ciphers;
++}
++
++/**
++ * @brief Return supported hmac names
++ * @return	The list of hmac names.
++ */
++const char **ssh_hmacs(void)
++{
++     struct ssh_hmac_struct *tab=ssh_get_hmactab();
++     static const char *hmacs[32];
++     int n;
++
++     memset(hmacs, 0, sizeof(*hmacs));
++
++     for (n = 0; tab[n].name != NULL; n++) {
++	  hmacs[n] = tab[n].name;
++     }
++     return hmacs;
++}
++
+ /** @} */
 diff --git a/src/options.c b/src/options.c
 index b3ecffe1..8de24ed6 100644
 --- a/src/options.c
@@ -392,10 +449,10 @@ index 8c509699..307388e5 100644
      session->opts.flags = SSH_OPT_FLAG_PASSWORD_AUTH |
                            SSH_OPT_FLAG_PUBKEY_AUTH |
 diff --git a/src/sftp.c b/src/sftp.c
-index e01012a8..3b86c3c6 100644
+index e01012a8..702623a0 100644
 --- a/src/sftp.c
 +++ b/src/sftp.c
-@@ -2228,6 +2228,135 @@ ssize_t sftp_write(sftp_file file, const void *buf, size_t count) {
+@@ -2228,6 +2228,132 @@ ssize_t sftp_write(sftp_file file, const void *buf, size_t count) {
    return -1; /* not reached */
  }
  
@@ -434,8 +491,7 @@ index e01012a8..3b86c3c6 100644
 +
 +  buffer = ssh_buffer_new_size(buf_sz, HEADROOM);
 +  if (buffer == NULL) {
-+    ssh_set_error(sftp->session, SSH_FATAL,
-+		  "ssh_buffer_new_size failed: Out of Memory");
++    ssh_set_error_oom(sftp->session);
 +    return -1;
 +  }
 +
@@ -449,16 +505,14 @@ index e01012a8..3b86c3c6 100644
 +                       count); /* len of datastring */
 +
 +  if (rc != SSH_OK){
-+    ssh_set_error(sftp->session, SSH_FATAL,
-+		  "ssh_buffer_pack failed: Out of Memory");
++    ssh_set_error_oom(sftp->session);
 +    ssh_buffer_free(buffer);
 +    return SSH_ERROR;
 +  }
 +
 +  actual = ssh_buffer_add_func(buffer, f, count, userdata);
 +  if (actual < 0){
-+    ssh_set_error(sftp->session, SSH_FATAL,
-+		  "ssh_buffer_add_func failed: %s", strerror(errno));
++    ssh_set_error_oom(sftp->session);
 +    ssh_buffer_free(buffer);
 +    return SSH_ERROR;
 +  }

rpm/mscp.spec.in

@@ -38,6 +38,9 @@ make -C build install DESTDIR=%{buildroot}
 
 
 %changelog
+* Sat May 11 2024 Ryo Nakamura <upa@haeena.net> - 0.2.1-1
+- RPM release for v0.2.1
+
 * Mon Apr 15 2024 Ryo Nakamura <upa@haeena.net> - 0.2.0-1
 - RPM release for v0.2.0
 

src/checkpoint.c

@@ -276,7 +276,7 @@ int checkpoint_save(const char *pathname, int dir, const char *user, const char
 	pool_for_each(path_pool, p, i) {
 		if (p->state == FILE_STATE_DONE)
 			continue;
-		if (checkpoint_write_path(fd, p, i) < 0)
+		if (checkpoint_write_path(fd, p, nr_paths) < 0)
 			return -1;
 		nr_paths++;
 	}
@@ -349,7 +349,8 @@ static int checkpoint_load_path(struct checkpoint_obj_hdr *hdr, pool *path_pool)
 		return -1;
 	}
 
-	pr_info("checkpoint:file: %s -> %s", p->path, p->dst_path);
+	pr_info("checkpoint:file: idx=%u %s -> %s", ntohl(path->idx),
+		p->path, p->dst_path);
 
 	return 0;
 }
@@ -375,7 +376,8 @@ static int checkpoint_load_chunk(struct checkpoint_obj_hdr *hdr, pool *path_pool
 		return -1;
 	}
 
-	pr_debug("checkpoint:chunk: %s 0x%lx-0x%lx", p->path, c->off, c->off + c->len);
+	pr_debug("checkpoint:chunk: idx=%u %s 0x%lx-0x%lx", ntohl(chunk->idx),
+		 p->path, c->off, c->off + c->len);
 
 	return 0;
 }

src/main.c

@@ -40,14 +40,14 @@ void usage(bool print_help)
 	printf("    -n NR_CONNECTIONS  number of connections "
 	       "(default: floor(log(cores)*2)+1)\n"
 	       "    -m COREMASK        hex value to specify cores where threads pinned\n"
-	       "    -u MAX_STARTUPS    number of concurrent SSH connection attempts "
+	       "    -u MAX_STARTUPS    number of concurrent unauthed SSH attempts "
 	       "(default: 8)\n"
 	       "    -I INTERVAL        interval between SSH connection attempts (default: 0)\n"
 	       "    -W CHECKPOINT      write states to the checkpoint if transfer fails\n"
 	       "    -R CHECKPOINT      resume transferring from the checkpoint\n"
 	       "\n"
-	       "    -s MIN_CHUNK_SIZE  min chunk size (default: 64MB)\n"
-	       "    -S MAX_CHUNK_SIZE  max chunk size (default: filesize/nr_conn)\n"
+	       "    -s MIN_CHUNK_SIZE  min chunk size (default: 16M bytes)\n"
+	       "    -S MAX_CHUNK_SIZE  max chunk size (default: filesize/nr_conn/4)\n"
 	       "    -a NR_AHEAD        number of inflight SFTP commands (default: 32)\n"
 	       "    -b BUF_SZ          buffer size for i/o and transfer\n"
 	       "    -L LIMIT_BITRATE   Limit the bitrate, n[KMG] (default: 0, no limit)\n"
@@ -75,6 +75,26 @@ void usage(bool print_help)
 	       "    -N                 enable Nagle's algorithm (default disabled)\n"
 	       "    -h                 print this help\n"
 	       "\n");
+
+	const char **ciphers = mscp_ssh_ciphers();
+	const char **hmacs = mscp_ssh_hmacs();
+	int n;
+
+	printf("Available ciphers: ");
+	for (n = 0; ciphers[n] != NULL; n++) {
+		printf("%s", ciphers[n]);
+		if (ciphers[n + 1])
+			printf(", ");
+	}
+	printf("\n\n");
+
+	printf("Available hmacs: ");
+	for (n = 0; hmacs[n] != NULL; n++) {
+		printf("%s", hmacs[n]);
+		if (hmacs[n + 1])
+			printf(", ");
+	}
+	printf("\n\n");
 }
 
 char *strip_brackets(char *s)
@@ -260,6 +280,37 @@ void print_cli(const char *fmt, ...)
 
 void print_stat(bool final);
 
+long atol_with_unit(char *value, bool i)
+{
+	/* value must be "\d+[kKmMgG]?" */
+
+	char *u = value + (strlen(optarg) - 1);
+	long k = i ? 1024 : 1000;
+	long factor = 1;
+	long v;
+
+	switch (*u) {
+	case 'k':
+	case 'K':
+		factor = k;
+		*u = '\0';
+		break;
+	case 'm':
+	case 'M':
+		factor = k * k;
+		*u = '\0';
+		break;
+	case 'g':
+	case 'G':
+		factor = k * k * k;
+		*u = '\0';
+		break;
+	}
+
+	v = atol(value);
+	return v * factor;
+}
+
 int main(int argc, char **argv)
 {
 	struct mscp_ssh_opts s;
@@ -271,8 +322,6 @@ int main(int argc, char **argv)
 	char *remote = NULL, *checkpoint_save = NULL, *checkpoint_load = NULL;
 	bool dryrun = false, resume = false;
 	int nr_options = 0;
-	size_t factor = 1;
-	char *unit;
 
 	memset(&s, 0, sizeof(s));
 	memset(&o, 0, sizeof(o));
@@ -305,36 +354,19 @@ int main(int argc, char **argv)
 			resume = true;
 			break;
 		case 's':
-			o.min_chunk_sz = atoi(optarg);
+			o.min_chunk_sz = atol_with_unit(optarg, true);
 			break;
 		case 'S':
-			o.max_chunk_sz = atoi(optarg);
+			o.max_chunk_sz = atol_with_unit(optarg, true);
 			break;
 		case 'a':
 			o.nr_ahead = atoi(optarg);
 			break;
 		case 'b':
-			o.buf_sz = atoi(optarg);
+			o.buf_sz = atol_with_unit(optarg, true);
 			break;
 		case 'L':
-			factor = 1;
-			unit = optarg + (strlen(optarg) - 1);
-			if (*unit == 'k' || *unit == 'K') {
-				factor = 1000;
-				*unit = '\0';
-			} else if (*unit == 'm' || *unit == 'M') {
-				factor = 1000000;
-				*unit = '\0';
-			} else if (*unit == 'g' || *unit == 'G') {
-				factor = 1000000000;
-				*unit = '\0';
-			}
-			o.bitrate = atol(optarg);
-			if (o.bitrate == 0) {
-				pr_err("invalid bitrate: %s", optarg);
-				return 1;
-			}
-			o.bitrate *= factor;
+			o.bitrate = atol_with_unit(optarg, false);
 			break;
 		case '4':
 			s.ai_family = AF_INET;
@@ -487,6 +519,18 @@ int main(int argc, char **argv)
 			pr_err("mscp_checkpoint_load: %s", priv_get_err());
 			return -1;
 		}
+
+		if (dryrun)
+			goto out;
+
+		/* create the first ssh connection to get password or
+		 * passphrase. The sftp session over it will be not
+		 * used for resume transfer in actuality. ToDo:
+		 * connectin managemnet should be improved. */
+		if (mscp_connect(m) < 0) {
+			pr_err("mscp_connect: %s", priv_get_err());
+			return -1;
+		}
 	}
 
 	if (dryrun) {

src/mscp.c

@@ -62,7 +62,7 @@ struct mscp {
 	struct mscp_thread scan; /* mscp_thread for mscp_scan_thread() */
 };
 
-#define DEFAULT_MIN_CHUNK_SZ (64 << 20) /* 64MB */
+#define DEFAULT_MIN_CHUNK_SZ (16 << 20) /* 16MB */
 #define DEFAULT_NR_AHEAD 32
 #define DEFAULT_BUF_SZ 16384
 /* XXX: we use 16384 byte buffer pointed by
@@ -158,23 +158,8 @@ static int validate_and_set_defaut_params(struct mscp_opts *o)
 
 	if (o->min_chunk_sz == 0)
 		o->min_chunk_sz = DEFAULT_MIN_CHUNK_SZ;
-	else {
-		if (o->min_chunk_sz < getpagesize() ||
-		    o->min_chunk_sz % getpagesize() != 0) {
-			priv_set_errv("min chunk size must be "
-				      "larget than and multiple of page size %d: %lu",
-				      getpagesize(), o->min_chunk_sz);
-			return -1;
-		}
-	}
 
 	if (o->max_chunk_sz) {
-		if (o->max_chunk_sz < getpagesize() ||
-		    o->max_chunk_sz % getpagesize() != 0) {
-			priv_set_errv("min chunk size must be larget than and "
-				      "multiple of page size %d: %lu",
-				      getpagesize(), o->max_chunk_sz);
-		}
 		if (o->min_chunk_sz > o->max_chunk_sz) {
 			priv_set_errv("smaller max chunk size than "
 				      "min chunk size: %lu < %lu",
@@ -345,18 +330,10 @@ int mscp_set_dst_path(struct mscp *m, const char *dst_path)
 	return 0;
 }
 
-static int get_page_mask(void)
+static size_t get_page_mask(void)
 {
-	long page_sz = sysconf(_SC_PAGESIZE);
-	size_t page_mask = 0;
-	int n;
-
-	for (n = 0; page_sz > 0; page_sz >>= 1, n++) {
-		page_mask <<= 1;
-		page_mask |= 1;
-	}
-
-	return page_mask >> 1;
+	size_t page_sz = sysconf(_SC_PAGESIZE);
+	return ~(page_sz - 1);
 }
 
 static void mscp_stop_copy_thread(struct mscp *m)

src/path.c

@@ -102,13 +102,10 @@ static int resolve_chunk(struct path *p, size_t size, struct path_resolve_args *
 	size_t chunk_sz, off, len;
 	size_t remaind;
 
-	if (size <= a->min_chunk_sz)
-		chunk_sz = size;
-	else if (a->max_chunk_sz)
+	if (a->max_chunk_sz)
 		chunk_sz = a->max_chunk_sz;
 	else {
-		chunk_sz = (size - (size % a->nr_conn)) / a->nr_conn;
-		chunk_sz &= ~a->chunk_align; /* align with page_sz */
+		chunk_sz = (size / (a->nr_conn * 4)) & a->chunk_align;
 		if (chunk_sz <= a->min_chunk_sz)
 			chunk_sz = a->min_chunk_sz;
 	}

src/ssh.c

@@ -407,3 +407,13 @@ void ssh_sftp_close(sftp_session sftp)
 	ssh_disconnect(ssh);
 	ssh_free(ssh);
 }
+
+const char **mscp_ssh_ciphers(void)
+{
+	return ssh_ciphers();
+}
+
+const char **mscp_ssh_hmacs(void)
+{
+	return ssh_hmacs();
+}

test/test_e2e.py

@@ -628,9 +628,9 @@ def test_checkpoint_dump_and_resume(mscp, src_prefix, dst_prefix):
     dst2.cleanup()
     os.remove("checkpoint")
 
-@pytest.mark.parametrize("timeout", [1, 2, 3, 4, 5, 6])
+@pytest.mark.parametrize("timeout", [ 1, 2, 3, 4, 5 ])
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
-def test_checkpoint_interrupt_and_resume(mscp, timeout, src_prefix, dst_prefix):
+def test_checkpoint_interrupt_large_file(mscp, timeout, src_prefix, dst_prefix):
     """Copy two 100MB files with 200Mbps -> 4 sec + 4 sec """
     src1 = File("src1", size = 100 * 1024 * 1024).make()
     src2 = File("src2", size = 100 * 1024 * 1024).make()
@@ -650,3 +650,31 @@ def test_checkpoint_interrupt_and_resume(mscp, timeout, src_prefix, dst_prefix):
     dst2.cleanup()
     os.remove("checkpoint")
 
+@pytest.mark.parametrize("timeout", [ 1, 2, 3, 4, 5 ])
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_checkpoint_interrupt_many_files(mscp, timeout, src_prefix, dst_prefix):
+    """Copy 100 1-MB files with 4 connections, and interrupt and
+    resume the transfer
+    """
+
+    files = []
+    for x in range(100):
+        files.append((
+            File("src/{:03d}".format(x), size = 1024 * 1024).make(),
+            File("dst/{:03d}".format(x))
+        ))
+
+    run2ng([mscp, "-vv", "-W", "checkpoint", "-L", "80m", "-n", 4,
+            src_prefix + "src",  dst_prefix + "dst"],
+           timeout = timeout)
+    assert os.path.exists("checkpoint")
+
+    run2ok([mscp, "-vv", "-R", "checkpoint"])
+
+    for src, dst in files:
+        assert check_same_md5sum(src, dst)
+        src.cleanup()
+        dst.cleanup()
+
+    os.remove("checkpoint")
+