start to impliment mscp as a library

this commit starts to refactor file.h|c to path.h|c and
add mscp.c|h. not completed yet.
This commit is contained in:
Ryo Nakamura
2023-02-25 22:17:29 +09:00
parent b4c021c954
commit 1be9b70808
9 changed files with 788 additions and 3 deletions

View File

@@ -208,6 +208,32 @@ static inline void list_splice(struct list_head *list, struct list_head *head)
__list_splice(list, head);
}
static inline void __list_splice_tail(struct list_head *list,
struct list_head *head)
{
struct list_head *first = list->next;
struct list_head *last = list->prev;
struct list_head *at = head->prev;
first->prev = at;
at->next = first;
last->next = head;
at->prev = last;
}
/**
* list_splice_tail - join two lists
* @list: the new list to add.
* @head: the place to add it in the first list.
*/
static inline void list_splice_tail(struct list_head *list, struct list_head *head)
{
if (!list_empty(list))
__list_splice_tail(list, head);
}
/**
* list_splice_init - join two lists and reinitialise the emptied list.
* @list: the new list to add.

190
src/mscp.c Normal file
View File

@@ -0,0 +1,190 @@
#include <stdbool.h>
#include <unistd.h>
#include <signal.h>
#include <sys/time.h>
#include <sys/ioctl.h>
#include <math.h>
#include <pthread.h>
#include <list.h>
#include <util.h>
#include <ssh.h>
#include <path.h>
#include <pprint.h>
#include <atomic.h>
#include <platform.h>
#include <mscp.h>
struct mscp {
const char *remote; /* remote host (and uername) */
struct mscp_opts *opts;
struct ssh_opts ssh_opts;
sftp_session first; /* first sftp session */
char dst_path[PATH_MAX];
struct list_head src_list;
struct list_head path_list;
struct list_head chunk_list;
lock chunk_lock;
struct mscp_thread *threads;
};
struct src {
struct list_head list;
char *path;
};
#define DEFAULT_MIN_CHUNK_SZ (64 << 20) /* 64MB */
#define DEFAULT_NR_AHEAD 32
#define DEFAULT_BUF_SZ 16384
struct mscp *mscp_init(const char *remote_host, struct mscp_opts *opts)
{
struct mscp *m;
m = malloc(sizeof(*m));
if (!m) {
pr_err("failed to allocate memory: %s\n", strerrno());
return NULL;
}
memset(m, 0, sizeof(*m));
INIT_LIST_HEAD(&m->src_list);
INIT_LIST_HEAD(&m->path_list);
INIT_LIST_HEAD(&m->chunk_list);
lock_init(&m->chunk_lock);
m->remote = strdup(remote_host);
if (!m->remote) {
pr_err("failed to allocate memory: %s\n", strerrno());
free(m);
return NULL;
}
m->opts = opts;
m->ssh_opts.login_name = opts->ssh_login_name;
m->ssh_opts.port = opts->ssh_port;
m->ssh_opts.identity = opts->ssh_identity;
m->ssh_opts.cipher = opts->ssh_cipher_spec;
m->ssh_opts.hmac = opts->ssh_hmac_spec;
m->ssh_opts.compress = opts->ssh_compress_level;
m->ssh_opts.debuglevel = opts->ssh_debug_level;
m->ssh_opts.no_hostkey_check = opts->ssh_no_hostkey_check;
m->ssh_opts.nodelay = opts->ssh_disable_tcp_nodely;
m->first = ssh_init_sftp_session(m->remote, &m->ssh_opts);
if (!m->first) {
free(m);
return NULL;
}
return m;
}
int mscp_add_src_path(struct mscp *m, const char *src_path)
{
struct src *s;
s = malloc(sizeof(*s));
if (!s) {
pr_err("failed to allocate memory: %s\n", strerrno());
return -1;
}
memset(s, 0, sizeof(*s));
s->path = strdup(src_path);
if (!s->path) {
pr_err("failed to allocate memory: %s\n", strerrno());
free(s);
return -1;
}
list_add_tail(&s->list, &m->src_list);
return 0;
}
static void mscp_free_src_list(struct mscp *m)
{
struct src *s, *n;
list_for_each_entry_safe(s, n, &m->src_list, list) {
free(s->path);
list_del(&s->list);
free(s);
}
}
int mscp_set_dst_path(struct mscp *m, const char *dst_path)
{
if (strlen(dst_path) + 1 >= PATH_MAX) {
pr_err("too long dst path: %s\n", dst_path);
return -1;
}
strncpy(m->dst_path, dst_path, PATH_MAX);
return 0;
}
int mscp_prepare(struct mscp *m)
{
sftp_session src_sftp = NULL, dst_sftp = NULL;
bool src_path_is_dir, dst_path_is_dir;
struct list_head tmp;
struct src *s;
mstat ss, ds;
switch (m->opts->direct) {
case MSCP_DIRECT_L2R:
src_sftp = NULL;
dst_sftp = m->first;
break;
case MSCP_DIRECT_R2L:
src_sftp = m->first;
dst_sftp = NULL;
break;
default:
pr_err("invalid mscp direction: %d\n", m->opts->direct);
return -1;
}
if (mscp_stat(m->dst_path, &ds, dst_sftp) == 0) {
if (mstat_is_dir(ds))
dst_path_is_dir = true;
mscp_stat_free(ds);
} else
dst_path_is_dir = false;
/* walk a src_path recusively, and resolve path->dst_path for each src */
list_for_each_entry(s, &m->src_list, list) {
if (mscp_stat(s->path, &ss, src_sftp) < 0) {
pr_err("stat: %s\n", mscp_stat_strerror(src_sftp));
return -1;
}
src_path_is_dir = mstat_is_dir(ss);
mscp_stat_free(ss);
INIT_LIST_HEAD(&tmp);
if (walk_src_path(src_sftp, s->path, &tmp) < 0)
return -1;
if (resolve_dst_path(src_sftp, s->path, m->dst_path, &tmp,
src_path_is_dir, dst_path_is_dir) < 0)
return -1;
list_splice_tail(&tmp, m->path_list.prev);
}
if (prepare_chunk(&m->path_list, &m->chunk_list, m->opts->nr_threads,
m->opts->max_chunk_sz, m->opts->min_chunk_sz) < 0)
return -1;
mscp_free_src_list(m);
return 0;
}
int mscp_start(struct mscp *m)
{
return 0;
}

43
src/mscp.h Normal file
View File

@@ -0,0 +1,43 @@
#ifndef _MSCP_H_
#define _MSCP_H_
#include <stdbool.h>
#define MSCP_DIRECT_L2R 1
#define MSCP_DIRECT_R2L 2
struct mscp_opts {
/* mscp options */
int direct; /* MSCP_DIRECT_ */
int nr_threads;
int nr_ahead;
size_t min_chunk_sz;
size_t max_chunk_sz;
size_t buf_sz;
int verbose_level;
bool quiet;
bool dryrun;
/* ssh options */
char ssh_login_name[64];
char ssh_port[32];
char ssh_identity[PATH_MAX];
char ssh_cipher_spec[64];
char ssh_hmac_spec[32];
int ssh_debug_level;
int ssh_compress_level;
bool ssh_no_hostkey_check;
bool ssh_disable_tcp_nodely;
};
struct mscp;
struct mscp *mscp_init(const char *remote_host, struct mscp_opts *opts);
int mscp_add_src_path(struct mscp *m, const char *src_path);
int mscp_set_dst_path(struct mscp *m, const char *dst_path);
int mscp_prepare(struct mscp *m);
int mscp_start(struct mscp *m);
#endif /* _MSCP_H_ */

250
src/path.c Normal file
View File

@@ -0,0 +1,250 @@
#include <string.h>
#include <unistd.h>
#include <dirent.h>
#include <sys/stat.h>
#include <libgen.h>
#include <ssh.h>
#include <util.h>
#include <list.h>
#include <atomic.h>
#include <path.h>
static int append_path(sftp_session sftp, const char *path, mstat s,
struct list_head *path_list)
{
struct path *p;
if (!(p = malloc(sizeof(*p)))) {
pr_err("failed to allocate memory: %s\n", strerrno());
return -1;
}
memset(p, 0, sizeof(*p));
INIT_LIST_HEAD(&p->list);
strncpy(p->path, path, PATH_MAX - 1);
p->size = mstat_size(s);
p->mode = mstat_mode(s);
p->state = FILE_STATE_INIT;
lock_init(&p->lock);
list_add_tail(&p->list, path_list);
return 0;
}
static bool check_path_should_skip(const char *path)
{
int len = strlen(path);
if ((len == 1 && strncmp(path, ".", 1) == 0) ||
(len == 2 && strncmp(path, "..", 2) == 0)) {
return true;
}
return false;
}
static int walk_path_recursive(sftp_session sftp, const char *path,
struct list_head *path_list)
{
char next_path[PATH_MAX];
mdirent *e;
mdir *d;
mstat s;
int ret;
if (mscp_stat(path, &s, sftp) < 0)
return -1;
if (mstat_is_regular(s)) {
/* this path is regular file. it is to be copied */
ret = append_path(sftp, path, s, path_list);
mscp_stat_free(s);
return ret;
}
if (!mstat_is_dir(s)) {
/* not regular file and not directory, skip it. */
mscp_stat_free(s);
return 0;
}
mscp_stat_free(s);
/* ok, this path is directory. walk it. */
if (!(d = mscp_opendir(path, sftp)))
return -1;
for (e = mscp_readdir(d); !mdirent_is_null(e); e = mscp_readdir(d)) {
if (check_path_should_skip(mdirent_name(e)))
continue;
if (strlen(path) + 1 + strlen(mdirent_name(e)) > PATH_MAX) {
pr_err("too long path: %s/%s\n", path, mdirent_name(e));
return -1;
}
snprintf(next_path, sizeof(next_path), "%s/%s", path, mdirent_name(e));
ret = walk_path_recursive(sftp, next_path, path_list);
if (ret < 0)
return ret;
}
mscp_closedir(d);
return 0;
}
int walk_src_path(sftp_session sftp, const char *src_path, struct list_head *path_list)
{
return walk_path_recursive(sftp, src_path, path_list);
}
static int src2dst_path(const char *src_path, const char *src_file_path,
const char *dst_path, char *dst_file_path, size_t len,
bool src_path_is_dir, bool dst_path_is_dir)
{
char copy[PATH_MAX];
char *prefix;
int offset;
strncpy(copy, src_path, PATH_MAX - 1);
prefix = dirname(copy);
if (!prefix) {
pr_err("dirname: %s\n", strerrno());
return -1;
}
if (strlen(prefix) == 1 && prefix[0] == '.')
offset = 0;
else
offset = strlen(prefix) + 1;
/* both are file */
if (!src_path_is_dir && !dst_path_is_dir)
strncpy(dst_file_path, dst_path, len);
/* src is file, and dst is dir */
if (!src_path_is_dir && dst_path_is_dir)
snprintf(dst_file_path, len, "%s/%s", dst_path, src_path + offset);
/* both are directory */
if (src_path_is_dir && dst_path_is_dir)
snprintf(dst_file_path, len, "%s/%s", dst_path, src_file_path + offset);
/* dst path does not exist. change dir name to dst_path */
if (src_path_is_dir && !dst_path_is_dir)
snprintf(dst_file_path, len, "%s/%s",
dst_path, src_file_path + strlen(src_path) + 1);
return 0;
}
int resolve_dst_path(sftp_session sftp, const char *src_path, const char *dst_path,
struct list_head *path_list, bool src_is_dir, bool dst_is_dir)
{
struct path *p;
list_for_each_entry(p, path_list, list) {
if (src2dst_path(src_path, p->path, dst_path, p->dst_path, PATH_MAX,
src_is_dir, dst_is_dir) < 0)
return -1;
}
return 0;
}
void path_dump(struct list_head *path_list)
{
struct path *p;
list_for_each_entry(p, path_list, list) {
printf("src: %s %lu-byte\n", p->path, p->size);
printf("dst: %s\n", p->dst_path);
}
}
/* chunk preparation */
static struct chunk *alloc_chunk(struct path *p)
{
struct chunk *c;
if (!(c = malloc(sizeof(*c)))) {
pr_err("%s\n", strerrno());
return NULL;
}
memset(c, 0, sizeof(*c));
c->p = p;
c->off = 0;
c->len = 0;
refcnt_inc(&p->refcnt);
return c;
}
static int get_page_mask(void)
{
long page_sz = sysconf(_SC_PAGESIZE);
size_t page_mask = 0;
int n;
for (n = 0; page_sz > 0; page_sz >>= 1, n++) {
page_mask <<= 1;
page_mask |= 1;
}
return page_mask >> 1;
}
int prepare_chunk(struct list_head *path_list, struct list_head *chunk_list,
int nr_conn, int min_chunk_sz, int max_chunk_sz)
{
struct chunk *c;
struct path *p;
size_t page_mask;
size_t chunk_sz;
size_t size;
page_mask = get_page_mask();
list_for_each_entry(p, path_list, list) {
if (p->size <= min_chunk_sz)
chunk_sz = p->size;
else if (max_chunk_sz)
chunk_sz = max_chunk_sz;
else {
chunk_sz = (p->size - (p->size % nr_conn)) / nr_conn;
chunk_sz &= ~page_mask; /* align with page_sz */
if (chunk_sz <= min_chunk_sz)
chunk_sz = min_chunk_sz;
}
/* for (size = f->size; size > 0;) does not create a
* file (chunk) when file size is 0. This do {} while
* (size > 0) creates just open/close a 0-byte file.
*/
size = p->size;
do {
c = alloc_chunk(p);
if (!c)
return -1;
c->off = p->size - size;
c->len = size < chunk_sz ? size : chunk_sz;
size -= c->len;
list_add_tail(&c->list, chunk_list);
} while (size > 0);
}
return 0;
}
void chunk_dump(struct list_head *chunk_list)
{
struct chunk *c;
list_for_each_entry(c, chunk_list, list) {
printf("chunk: %s 0x%lx-%lx bytes\n",
c->p->path, c->off, c->off + c->len);
}
}

192
src/path.h Normal file
View File

@@ -0,0 +1,192 @@
#ifndef _PATH_H_
#define _PATH_H_
#include <limits.h>
#include <fcntl.h>
#include <dirent.h>
#include <sys/stat.h>
#include <list.h>
#include <atomic.h>
#include <ssh.h>
struct path {
struct list_head list; /* mscp->path_list */
char path[PATH_MAX]; /* file path */
size_t size; /* size of file on this path */
mode_t mode; /* permission */
char dst_path[PATH_MAX]; /* copy dst path */
int state;
lock lock;
refcnt refcnt;
};
#define FILE_STATE_INIT 0
#define FILE_STATE_OPENED 1
#define FILE_STATE_DONE 2
struct chunk {
struct list_head list; /* mscp->chunk_list */
struct path *p;
size_t off; /* offset of this chunk on the file on path p */
size_t len; /* length of this chunk */
size_t done; /* copied bytes for this chunk by a thread */
};
/* recursivly walk through src_path and fill path_list for each file */
int walk_src_path(sftp_session sftp, const char *src_path, struct list_head *path_list);
/* fill path->dst_path for all files */
int resolve_dst_path(sftp_session sftp, const char *src_path, const char *dst_path,
struct list_head *path_list,
bool src_path_is_dir, bool dst_path_is_dir);
/* prepare chunk_list for files in the path_list */
int prepare_chunk(struct list_head *path_list, struct list_head *chunk_list,
int nr_conn, int min_chunk_sz, int max_chunk_sz);
/* just print contents. just for debugging */
void path_dump(struct list_head *path_list);
void chunk_dump(struct list_head *chunk_list);
/* wrap DIR/dirent and sftp_dir/sftp_attribute. not thread safe */
struct mscp_dir {
DIR *l;
sftp_dir r;
sftp_session sftp;
};
typedef struct mscp_dir mdir;
struct mscp_dirent {
struct dirent *l;
sftp_attributes r;
};
typedef struct mscp_dirent mdirent;
#define mdirent_name(e) ((e->l) ? e->l->d_name : e->r->name)
#define mdirent_is_dir(e) ((e->l) ? \
(e->l->d_type == DT_DIR) : \
(e->r->type == SSH_FILEXFER_TYPE_DIRECTORY))
#define mdirent_is_null(e) (e->l == NULL && e->r == NULL)
static mdir *mscp_opendir(const char *path, sftp_session sftp)
{
mdir *d;
if (!(d = malloc(sizeof(*d))))
return NULL;
memset(d, 0, sizeof(*d));
d->sftp = sftp;
if (sftp) {
d->r = sftp_opendir(sftp, path);
if (!d->r) {
pr_err("sftp_opendir: %s: %s\n", path, sftp_get_ssh_error(sftp));
free(d);
return NULL;
}
} else {
d->l = opendir(path);
if (!d->l) {
pr_err("opendir: %s: %s\n", path, strerrno());
free(d);
return NULL;
}
}
return d;
}
static int mscp_closedir(mdir *d)
{
int ret;
if (d->r)
ret = sftp_closedir(d->r);
else
ret = closedir(d->l);
free(d);
return ret;
}
static mdirent *mscp_readdir(mdir *d)
{
static mdirent e;
memset(&e, 0, sizeof(e));
if (d->r)
e.r = sftp_readdir(d->sftp, d->r);
else
e.l = readdir(d->l);
return &e;
}
/* warp stat/sftp_stat */
struct mscp_stat {
struct stat l;
sftp_attributes r;
};
typedef struct mscp_stat mstat;
static int mscp_stat(const char *path, mstat *s, sftp_session sftp)
{
memset(s, 0, sizeof(*s));
if (sftp) {
s->r = sftp_stat(sftp, path);
if (!s->r)
return -1;
} else {
if (stat(path, &s->l) < 0)
return -1;
}
return 0;
}
static const char *mscp_stat_strerror(sftp_session sftp)
{
if (sftp)
return sftp_get_ssh_error(sftp);
return strerrno();
}
static int mscp_stat_check_err_noent(sftp_session sftp)
{
if (sftp) {
if (sftp_get_error(sftp) == SSH_FX_NO_SUCH_PATH ||
sftp_get_error(sftp) == SSH_FX_NO_SUCH_FILE)
return 0;
} else {
if (errno == ENOENT)
return 0;
}
return -1;
}
static void mscp_stat_free(mstat s) {
if (s.r)
sftp_attributes_free(s.r);
}
#define mstat_size(s) ((s.r) ? s.r->size : s.l.st_size)
#define mstat_mode(s) ((s.r) ? \
s.r->permissions : \
s.l.st_mode & (S_IRWXU|S_IRWXG|S_IRWXO))
#define mstat_is_regular(s) ((s.r) ? \
(s.r->type == SSH_FILEXFER_TYPE_REGULAR) : \
S_ISREG(s.l.st_mode))
#define mstat_is_dir(s) ((s.r) ? \
(s.r->type == SSH_FILEXFER_TYPE_DIRECTORY) : \
S_ISDIR(s.l.st_mode))
#endif /* _PATH_H_ */

View File

@@ -148,7 +148,7 @@ static struct ssh_callbacks_struct cb = {
.userdata = NULL,
};
static ssh_session ssh_init_session(char *sshdst, struct ssh_opts *opts)
static ssh_session ssh_init_session(const char *sshdst, struct ssh_opts *opts)
{
ssh_session ssh = ssh_new();
@@ -187,7 +187,7 @@ free_out:
return NULL;
}
sftp_session ssh_init_sftp_session(char *sshdst, struct ssh_opts *opts)
sftp_session ssh_init_sftp_session(const char *sshdst, struct ssh_opts *opts)
{
sftp_session sftp;
ssh_session ssh = ssh_init_session(sshdst, opts);

View File

@@ -25,7 +25,7 @@ struct ssh_opts {
/* ssh_init_sftp_session() creates sftp_session. sshdst accpets
* user@hostname and hostname notations (by libssh).
*/
sftp_session ssh_init_sftp_session(char *sshdst, struct ssh_opts *opts);
sftp_session ssh_init_sftp_session(const char *sshdst, struct ssh_opts *opts);
void ssh_sftp_close(sftp_session sftp);
#define sftp_ssh(sftp) (sftp)->session

68
src/test.c Normal file
View File

@@ -0,0 +1,68 @@
#include <util.h>
#include <path.h>
int path_walk_test(int argc, char **argv)
{
struct list_head path_list, chunk_list, tmp;
mstat src, dst;
bool dst_is_dir = false, src_is_dir = false;
int ret, n;
if (mscp_stat(argv[argc - 1], &dst, NULL) == 0) {
if (mstat_is_dir(dst))
dst_is_dir = true;
}
INIT_LIST_HEAD(&path_list);
INIT_LIST_HEAD(&chunk_list);
for (n = 1; n < argc - 1; n++) {
if (mscp_stat(argv[n], &src, NULL) < 0) {
pr_err("%s not found: %s\n", argv[n], strerrno());
return -1;
}
src_is_dir = mstat_is_dir(src);
INIT_LIST_HEAD(&tmp);
ret = walk_src_path(NULL, argv[n], &tmp);
if (ret < 0)
return ret;
ret = resolve_dst_path(NULL, argv[n], argv[argc - 1], &tmp,
mstat_is_dir(src), dst_is_dir);
if (ret < 0)
return ret;
list_splice_tail(&tmp, &path_list);
}
path_dump(&path_list);
ret = prepare_chunk(&path_list, &chunk_list, 4, 1024 * 1024, 0);
if (ret < 0)
return ret;
//chunk_dump(&chunk_list);
return 0;
}
void usage()
{
printf("test [SRC_PATH] ... [DST_PATH]\n");
}
int main(int argc, char **argv)
{
if (argc < 3) {
usage();
return 1;
}
if (path_walk_test(argc, argv) < 0)
return 1;
return 0;
}