add glob for source paths

https://github.com/upa/mscp/issues/3

src/fileops.c

@@ -97,18 +97,15 @@ MDIR *mscp_opendir_wrapped(const char *path)
 	return mscp_opendir(path, tls_sftp);
 }
 
-int mscp_closedir(MDIR *md)
+void mscp_closedir(MDIR *md)
 {
 	int ret;
-	if (md->remote) {
+	if (md->remote)
-		ret = sftp_closedir(md->remote);
+		sftp_closedir(md->remote);
-		if (ret < 0)
+	else
-			sftp_err_to_errno(md->remote->sftp);
+		closedir(md->local);
-	} else
-		ret = closedir(md->local);
 
 	free(md);
-	return ret;
 }
 
 
@@ -308,3 +305,29 @@ int mscp_chmod(const char *path, mode_t mode, sftp_session sftp)
 
 	return ret;
 }
+
+static int errfunc(const char *epath, int err)
+{
+	printf("errfunc for path %s\n", epath);
+	return 0;
+}
+
+int mscp_glob(const char *pattern, int flags, glob_t *pglob, sftp_session sftp)
+{
+	int ret;
+	if (sftp) {
+		pglob->gl_opendir = (void *(*)(const char *))mscp_opendir_wrapped;
+		pglob->gl_readdir = (struct dirent *(*)(void *))mscp_readdir;
+		pglob->gl_closedir = (void (*)(void *))mscp_closedir;
+		pglob->gl_lstat = mscp_lstat_wrapped;
+		pglob->gl_stat = mscp_stat_wrapped;
+		flags |= GLOB_ALTDIRFUNC;
+		set_tls_sftp_session(sftp);
+	}
+
+	ret =  glob(pattern, flags, errfunc, pglob);
+
+	if (sftp)
+		set_tls_sftp_session(NULL);
+	return ret;
+}

src/fileops.h

@@ -1,6 +1,7 @@
 
 #include <dirent.h>
 #include <sys/stat.h>
+#include <glob.h>
 
 #include <ssh.h>
 
@@ -10,17 +11,18 @@ void set_tls_sftp_session(sftp_session sftp);
  mscp_lstat_wrapped(). This _wrapped() functions exist for
  sftp_glob() */
 
+/* directory operations */
+
 struct mdir_struct {
 	DIR *local;
 	sftp_dir remote;
 };
 typedef struct mdir_struct MDIR;
 
-/* directory operations */
-MDIR *mscp_opendir(const char *path, sftp_session sftp);
 
+MDIR *mscp_opendir(const char *path, sftp_session sftp);
 MDIR *mscp_opendir_wrapped(const char *path);
-int mscp_closedir(MDIR *md);
+void mscp_closedir(MDIR *md);
 struct dirent *mscp_readdir(MDIR *md);
 
 int mscp_mkdir(const char *path, mode_t mode, sftp_session sftp);
@@ -45,3 +47,6 @@ mf *mscp_open(const char *path, int flags, mode_t mode, sftp_session sftp);
 void mscp_close(mf *f);
 int mscp_lseek(mf *f, size_t off);
 int mscp_chmod(const char *path, mode_t mode, sftp_session sftp);
+
+/* remote glob */
+int mscp_glob(const char *pattern, int flags, glob_t *pglob, sftp_session sftp);

src/mscp.c

@@ -378,6 +378,8 @@ void *mscp_scan_thread(void *arg)
 	struct path *p;
 	struct src *s;
 	struct stat ss, ds;
+	glob_t pglob;
+	int n;
 	
 	m->ret_scan = 0;
 
@@ -418,21 +420,33 @@ void *mscp_scan_thread(void *arg)
 
 	/* walk a src_path recusively, and resolve path->dst_path for each src */
 	list_for_each_entry(s, &m->src_list, list) {
-		if (mscp_stat(s->path, &ss, src_sftp) < 0) {
+		memset(&pglob, 0, sizeof(pglob));
-			mscp_set_error("stat: %s", strerrno());
+		if (mscp_glob(s->path, GLOB_NOCHECK, &pglob, src_sftp) < 0) {
+			mscp_set_error("mscp_glob: %s", strerrno());
 			goto err_out;
 		}
 
-		/* set path specific args */
+		for (n = 0; n < pglob.gl_pathc; n++) {
-		a.src_path = s->path;
+			if (mscp_stat(pglob.gl_pathv[n], &ss, src_sftp) < 0) {
-		a.dst_path = m->dst_path;
+				mscp_set_error("stat: %s %s", s->path, strerrno());
-		a.src_path_is_dir = S_ISDIR(ss.st_mode);
+				goto err_out;
+			}
 
-		INIT_LIST_HEAD(&tmp);
+			if (!a.dst_path_should_dir && pglob.gl_pathc > 1)
-		if (walk_src_path(src_sftp, s->path, &tmp, &a) < 0)
+				a.dst_path_should_dir = true; /* we have over 1 src */
-			goto err_out;
 
-		list_splice_tail(&tmp, m->path_list.prev);
+			/* set path specific args */
+			a.src_path = pglob.gl_pathv[n];
+			a.dst_path = m->dst_path;
+			a.src_path_is_dir = S_ISDIR(ss.st_mode);
+
+			INIT_LIST_HEAD(&tmp);
+			if (walk_src_path(src_sftp, pglob.gl_pathv[n], &tmp, &a) < 0)
+				goto err_out;
+
+			list_splice_tail(&tmp, m->path_list.prev);
+		}
+		globfree(&pglob);
 	}
 
 	mpr_info(m->msg_fp, "walk source path(s) done\n");

test/test_e2e.py

@@ -150,6 +150,33 @@ def test_min_chunk(mscp, src_prefix, dst_prefix):
     src.cleanup()
     dst.cleanup()
 
+
+param_glob_copy = [
+    (
+        "src*", "dstx",
+        [ File("src1"), File("src2"), File("src3") ],
+        [ File("dstx/src1"), File("dstx/src2"), File("dstx/src3") ],
+    ),
+    (
+        "src*", "dstx",
+        [ File("src1/s1"), File("src2/s2"), File("src3/s3") ],
+        [ File("dstx/s1"), File("dstx/s2"), File("dstx/s3") ],
+    )
+]
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+@pytest.mark.parametrize("src_glob_path, dst_path, srcs, dsts", param_glob_copy)
+def test_glob_src_path(mscp, src_prefix, dst_prefix,
+                       src_glob_path, dst_path, srcs, dsts):
+    for src in srcs:
+        src.make(size = 1024 * 1024)
+
+    run2ok([mscp, "-H", "-vvv", src_prefix + src_glob_path, dst_prefix + dst_path])
+    for src, dst in zip(srcs, dsts):
+        assert check_same_md5sum(src, dst)
+        src.cleanup()
+        dst.cleanup()
+
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 def test_thread_affinity(mscp, src_prefix, dst_prefix):
     src = File("src", size = 64 * 1024).make()

test/util.py

@@ -22,7 +22,10 @@ class File():
     def __str__(self):
         return self.path
 
-    def make(self):
+    def make(self, size = None):
+        if size:
+            self.size = size
+
         d = os.path.dirname(self.path)
         if d:
             os.makedirs(d, exist_ok = True)