adjust mscp.rst for v0.2.4

This fixes #36, password auth does not occur.

.clang-format

@@ -0,0 +1,747 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# clang-format configuration file. Intended for clang-format >= 11.
+#
+# For more information, see:
+#
+#   Documentation/process/clang-format.rst
+#   https://clang.llvm.org/docs/ClangFormat.html
+#   https://clang.llvm.org/docs/ClangFormatStyleOptions.html
+#
+
+# clang-format configuration for Linux kernel, except that ColumnLimit is 90
+---
+AccessModifierOffset: -4
+AlignAfterOpenBracket: Align
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlines: Left
+AlignOperands: true
+AlignTrailingComments: false
+AllowAllParametersOfDeclarationOnNextLine: false
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: None
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: false
+AlwaysBreakTemplateDeclarations: false
+BinPackArguments: true
+BinPackParameters: true
+BraceWrapping:
+  AfterClass: false
+  AfterControlStatement: false
+  AfterEnum: false
+  AfterFunction: true
+  AfterNamespace: true
+  AfterObjCDeclaration: false
+  AfterStruct: false
+  AfterUnion: false
+  AfterExternBlock: false
+  BeforeCatch: false
+  BeforeElse: false
+  IndentBraces: false
+  SplitEmptyFunction: true
+  SplitEmptyRecord: true
+  SplitEmptyNamespace: true
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Custom
+BreakBeforeInheritanceComma: false
+BreakBeforeTernaryOperators: false
+BreakConstructorInitializersBeforeComma: false
+BreakConstructorInitializers: BeforeComma
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: false
+ColumnLimit: 90
+CommentPragmas: '^ IWYU pragma:'
+CompactNamespaces: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: false
+ConstructorInitializerIndentWidth: 8
+ContinuationIndentWidth: 8
+Cpp11BracedListStyle: false
+DerivePointerAlignment: false
+DisableFormat: false
+ExperimentalAutoDetectBinPacking: false
+FixNamespaceComments: false
+
+# Taken from:
+#   git grep -h '^#define [^[:space:]]*for_each[^[:space:]]*(' include/ tools/ \
+#   | sed "s,^#define \([^[:space:]]*for_each[^[:space:]]*\)(.*$,  - '\1'," \
+#   | LC_ALL=C sort -u
+ForEachMacros:
+  - '__ata_qc_for_each'
+  - '__bio_for_each_bvec'
+  - '__bio_for_each_segment'
+  - '__evlist__for_each_entry'
+  - '__evlist__for_each_entry_continue'
+  - '__evlist__for_each_entry_from'
+  - '__evlist__for_each_entry_reverse'
+  - '__evlist__for_each_entry_safe'
+  - '__for_each_mem_range'
+  - '__for_each_mem_range_rev'
+  - '__for_each_thread'
+  - '__hlist_for_each_rcu'
+  - '__map__for_each_symbol_by_name'
+  - '__pci_bus_for_each_res0'
+  - '__pci_bus_for_each_res1'
+  - '__pci_dev_for_each_res0'
+  - '__pci_dev_for_each_res1'
+  - '__perf_evlist__for_each_entry'
+  - '__perf_evlist__for_each_entry_reverse'
+  - '__perf_evlist__for_each_entry_safe'
+  - '__rq_for_each_bio'
+  - '__shost_for_each_device'
+  - '__sym_for_each'
+  - 'apei_estatus_for_each_section'
+  - 'ata_for_each_dev'
+  - 'ata_for_each_link'
+  - 'ata_qc_for_each'
+  - 'ata_qc_for_each_raw'
+  - 'ata_qc_for_each_with_internal'
+  - 'ax25_for_each'
+  - 'ax25_uid_for_each'
+  - 'bio_for_each_bvec'
+  - 'bio_for_each_bvec_all'
+  - 'bio_for_each_folio_all'
+  - 'bio_for_each_integrity_vec'
+  - 'bio_for_each_segment'
+  - 'bio_for_each_segment_all'
+  - 'bio_list_for_each'
+  - 'bip_for_each_vec'
+  - 'bond_for_each_slave'
+  - 'bond_for_each_slave_rcu'
+  - 'bpf_for_each'
+  - 'bpf_for_each_reg_in_vstate'
+  - 'bpf_for_each_reg_in_vstate_mask'
+  - 'bpf_for_each_spilled_reg'
+  - 'bpf_object__for_each_map'
+  - 'bpf_object__for_each_program'
+  - 'btree_for_each_safe128'
+  - 'btree_for_each_safe32'
+  - 'btree_for_each_safe64'
+  - 'btree_for_each_safel'
+  - 'card_for_each_dev'
+  - 'cgroup_taskset_for_each'
+  - 'cgroup_taskset_for_each_leader'
+  - 'cpu_aggr_map__for_each_idx'
+  - 'cpufreq_for_each_efficient_entry_idx'
+  - 'cpufreq_for_each_entry'
+  - 'cpufreq_for_each_entry_idx'
+  - 'cpufreq_for_each_valid_entry'
+  - 'cpufreq_for_each_valid_entry_idx'
+  - 'css_for_each_child'
+  - 'css_for_each_descendant_post'
+  - 'css_for_each_descendant_pre'
+  - 'damon_for_each_region'
+  - 'damon_for_each_region_from'
+  - 'damon_for_each_region_safe'
+  - 'damon_for_each_scheme'
+  - 'damon_for_each_scheme_safe'
+  - 'damon_for_each_target'
+  - 'damon_for_each_target_safe'
+  - 'damos_for_each_filter'
+  - 'damos_for_each_filter_safe'
+  - 'data__for_each_file'
+  - 'data__for_each_file_new'
+  - 'data__for_each_file_start'
+  - 'device_for_each_child_node'
+  - 'displayid_iter_for_each'
+  - 'dma_fence_array_for_each'
+  - 'dma_fence_chain_for_each'
+  - 'dma_fence_unwrap_for_each'
+  - 'dma_resv_for_each_fence'
+  - 'dma_resv_for_each_fence_unlocked'
+  - 'do_for_each_ftrace_op'
+  - 'drm_atomic_crtc_for_each_plane'
+  - 'drm_atomic_crtc_state_for_each_plane'
+  - 'drm_atomic_crtc_state_for_each_plane_state'
+  - 'drm_atomic_for_each_plane_damage'
+  - 'drm_client_for_each_connector_iter'
+  - 'drm_client_for_each_modeset'
+  - 'drm_connector_for_each_possible_encoder'
+  - 'drm_exec_for_each_locked_object'
+  - 'drm_exec_for_each_locked_object_reverse'
+  - 'drm_for_each_bridge_in_chain'
+  - 'drm_for_each_connector_iter'
+  - 'drm_for_each_crtc'
+  - 'drm_for_each_crtc_reverse'
+  - 'drm_for_each_encoder'
+  - 'drm_for_each_encoder_mask'
+  - 'drm_for_each_fb'
+  - 'drm_for_each_legacy_plane'
+  - 'drm_for_each_plane'
+  - 'drm_for_each_plane_mask'
+  - 'drm_for_each_privobj'
+  - 'drm_gem_for_each_gpuva'
+  - 'drm_gem_for_each_gpuva_safe'
+  - 'drm_gpuva_for_each_op'
+  - 'drm_gpuva_for_each_op_from_reverse'
+  - 'drm_gpuva_for_each_op_safe'
+  - 'drm_gpuvm_for_each_va'
+  - 'drm_gpuvm_for_each_va_range'
+  - 'drm_gpuvm_for_each_va_range_safe'
+  - 'drm_gpuvm_for_each_va_safe'
+  - 'drm_mm_for_each_hole'
+  - 'drm_mm_for_each_node'
+  - 'drm_mm_for_each_node_in_range'
+  - 'drm_mm_for_each_node_safe'
+  - 'dsa_switch_for_each_available_port'
+  - 'dsa_switch_for_each_cpu_port'
+  - 'dsa_switch_for_each_cpu_port_continue_reverse'
+  - 'dsa_switch_for_each_port'
+  - 'dsa_switch_for_each_port_continue_reverse'
+  - 'dsa_switch_for_each_port_safe'
+  - 'dsa_switch_for_each_user_port'
+  - 'dsa_tree_for_each_cpu_port'
+  - 'dsa_tree_for_each_user_port'
+  - 'dsa_tree_for_each_user_port_continue_reverse'
+  - 'dso__for_each_symbol'
+  - 'dsos__for_each_with_build_id'
+  - 'elf_hash_for_each_possible'
+  - 'elf_symtab__for_each_symbol'
+  - 'evlist__for_each_cpu'
+  - 'evlist__for_each_entry'
+  - 'evlist__for_each_entry_continue'
+  - 'evlist__for_each_entry_from'
+  - 'evlist__for_each_entry_reverse'
+  - 'evlist__for_each_entry_safe'
+  - 'flow_action_for_each'
+  - 'for_each_acpi_consumer_dev'
+  - 'for_each_acpi_dev_match'
+  - 'for_each_active_dev_scope'
+  - 'for_each_active_drhd_unit'
+  - 'for_each_active_iommu'
+  - 'for_each_active_route'
+  - 'for_each_aggr_pgid'
+  - 'for_each_and_bit'
+  - 'for_each_andnot_bit'
+  - 'for_each_available_child_of_node'
+  - 'for_each_bench'
+  - 'for_each_bio'
+  - 'for_each_board_func_rsrc'
+  - 'for_each_btf_ext_rec'
+  - 'for_each_btf_ext_sec'
+  - 'for_each_bvec'
+  - 'for_each_card_auxs'
+  - 'for_each_card_auxs_safe'
+  - 'for_each_card_components'
+  - 'for_each_card_dapms'
+  - 'for_each_card_pre_auxs'
+  - 'for_each_card_prelinks'
+  - 'for_each_card_rtds'
+  - 'for_each_card_rtds_safe'
+  - 'for_each_card_widgets'
+  - 'for_each_card_widgets_safe'
+  - 'for_each_cgroup_storage_type'
+  - 'for_each_child_of_node'
+  - 'for_each_clear_bit'
+  - 'for_each_clear_bit_from'
+  - 'for_each_clear_bitrange'
+  - 'for_each_clear_bitrange_from'
+  - 'for_each_cmd'
+  - 'for_each_cmsghdr'
+  - 'for_each_collection'
+  - 'for_each_comp_order'
+  - 'for_each_compatible_node'
+  - 'for_each_component_dais'
+  - 'for_each_component_dais_safe'
+  - 'for_each_conduit'
+  - 'for_each_console'
+  - 'for_each_console_srcu'
+  - 'for_each_cpu'
+  - 'for_each_cpu_and'
+  - 'for_each_cpu_andnot'
+  - 'for_each_cpu_or'
+  - 'for_each_cpu_wrap'
+  - 'for_each_dapm_widgets'
+  - 'for_each_dedup_cand'
+  - 'for_each_dev_addr'
+  - 'for_each_dev_scope'
+  - 'for_each_dma_cap_mask'
+  - 'for_each_dpcm_be'
+  - 'for_each_dpcm_be_rollback'
+  - 'for_each_dpcm_be_safe'
+  - 'for_each_dpcm_fe'
+  - 'for_each_drhd_unit'
+  - 'for_each_dss_dev'
+  - 'for_each_efi_memory_desc'
+  - 'for_each_efi_memory_desc_in_map'
+  - 'for_each_element'
+  - 'for_each_element_extid'
+  - 'for_each_element_id'
+  - 'for_each_endpoint_of_node'
+  - 'for_each_event'
+  - 'for_each_event_tps'
+  - 'for_each_evictable_lru'
+  - 'for_each_fib6_node_rt_rcu'
+  - 'for_each_fib6_walker_rt'
+  - 'for_each_free_mem_pfn_range_in_zone'
+  - 'for_each_free_mem_pfn_range_in_zone_from'
+  - 'for_each_free_mem_range'
+  - 'for_each_free_mem_range_reverse'
+  - 'for_each_func_rsrc'
+  - 'for_each_gpiochip_node'
+  - 'for_each_group_evsel'
+  - 'for_each_group_evsel_head'
+  - 'for_each_group_member'
+  - 'for_each_group_member_head'
+  - 'for_each_hstate'
+  - 'for_each_if'
+  - 'for_each_inject_fn'
+  - 'for_each_insn'
+  - 'for_each_insn_prefix'
+  - 'for_each_intid'
+  - 'for_each_iommu'
+  - 'for_each_ip_tunnel_rcu'
+  - 'for_each_irq_nr'
+  - 'for_each_lang'
+  - 'for_each_link_codecs'
+  - 'for_each_link_cpus'
+  - 'for_each_link_platforms'
+  - 'for_each_lru'
+  - 'for_each_matching_node'
+  - 'for_each_matching_node_and_match'
+  - 'for_each_media_entity_data_link'
+  - 'for_each_mem_pfn_range'
+  - 'for_each_mem_range'
+  - 'for_each_mem_range_rev'
+  - 'for_each_mem_region'
+  - 'for_each_member'
+  - 'for_each_memory'
+  - 'for_each_migratetype_order'
+  - 'for_each_missing_reg'
+  - 'for_each_mle_subelement'
+  - 'for_each_mod_mem_type'
+  - 'for_each_net'
+  - 'for_each_net_continue_reverse'
+  - 'for_each_net_rcu'
+  - 'for_each_netdev'
+  - 'for_each_netdev_continue'
+  - 'for_each_netdev_continue_rcu'
+  - 'for_each_netdev_continue_reverse'
+  - 'for_each_netdev_dump'
+  - 'for_each_netdev_feature'
+  - 'for_each_netdev_in_bond_rcu'
+  - 'for_each_netdev_rcu'
+  - 'for_each_netdev_reverse'
+  - 'for_each_netdev_safe'
+  - 'for_each_new_connector_in_state'
+  - 'for_each_new_crtc_in_state'
+  - 'for_each_new_mst_mgr_in_state'
+  - 'for_each_new_plane_in_state'
+  - 'for_each_new_plane_in_state_reverse'
+  - 'for_each_new_private_obj_in_state'
+  - 'for_each_new_reg'
+  - 'for_each_node'
+  - 'for_each_node_by_name'
+  - 'for_each_node_by_type'
+  - 'for_each_node_mask'
+  - 'for_each_node_state'
+  - 'for_each_node_with_cpus'
+  - 'for_each_node_with_property'
+  - 'for_each_nonreserved_multicast_dest_pgid'
+  - 'for_each_numa_hop_mask'
+  - 'for_each_of_allnodes'
+  - 'for_each_of_allnodes_from'
+  - 'for_each_of_cpu_node'
+  - 'for_each_of_pci_range'
+  - 'for_each_old_connector_in_state'
+  - 'for_each_old_crtc_in_state'
+  - 'for_each_old_mst_mgr_in_state'
+  - 'for_each_old_plane_in_state'
+  - 'for_each_old_private_obj_in_state'
+  - 'for_each_oldnew_connector_in_state'
+  - 'for_each_oldnew_crtc_in_state'
+  - 'for_each_oldnew_mst_mgr_in_state'
+  - 'for_each_oldnew_plane_in_state'
+  - 'for_each_oldnew_plane_in_state_reverse'
+  - 'for_each_oldnew_private_obj_in_state'
+  - 'for_each_online_cpu'
+  - 'for_each_online_node'
+  - 'for_each_online_pgdat'
+  - 'for_each_or_bit'
+  - 'for_each_path'
+  - 'for_each_pci_bridge'
+  - 'for_each_pci_dev'
+  - 'for_each_pcm_streams'
+  - 'for_each_physmem_range'
+  - 'for_each_populated_zone'
+  - 'for_each_possible_cpu'
+  - 'for_each_present_blessed_reg'
+  - 'for_each_present_cpu'
+  - 'for_each_prime_number'
+  - 'for_each_prime_number_from'
+  - 'for_each_probe_cache_entry'
+  - 'for_each_process'
+  - 'for_each_process_thread'
+  - 'for_each_prop_codec_conf'
+  - 'for_each_prop_dai_codec'
+  - 'for_each_prop_dai_cpu'
+  - 'for_each_prop_dlc_codecs'
+  - 'for_each_prop_dlc_cpus'
+  - 'for_each_prop_dlc_platforms'
+  - 'for_each_property_of_node'
+  - 'for_each_reg'
+  - 'for_each_reg_filtered'
+  - 'for_each_reloc'
+  - 'for_each_reloc_from'
+  - 'for_each_requested_gpio'
+  - 'for_each_requested_gpio_in_range'
+  - 'for_each_reserved_mem_range'
+  - 'for_each_reserved_mem_region'
+  - 'for_each_rtd_codec_dais'
+  - 'for_each_rtd_components'
+  - 'for_each_rtd_cpu_dais'
+  - 'for_each_rtd_dais'
+  - 'for_each_sband_iftype_data'
+  - 'for_each_script'
+  - 'for_each_sec'
+  - 'for_each_set_bit'
+  - 'for_each_set_bit_from'
+  - 'for_each_set_bit_wrap'
+  - 'for_each_set_bitrange'
+  - 'for_each_set_bitrange_from'
+  - 'for_each_set_clump8'
+  - 'for_each_sg'
+  - 'for_each_sg_dma_page'
+  - 'for_each_sg_page'
+  - 'for_each_sgtable_dma_page'
+  - 'for_each_sgtable_dma_sg'
+  - 'for_each_sgtable_page'
+  - 'for_each_sgtable_sg'
+  - 'for_each_sibling_event'
+  - 'for_each_sta_active_link'
+  - 'for_each_subelement'
+  - 'for_each_subelement_extid'
+  - 'for_each_subelement_id'
+  - 'for_each_sublist'
+  - 'for_each_subsystem'
+  - 'for_each_supported_activate_fn'
+  - 'for_each_supported_inject_fn'
+  - 'for_each_sym'
+  - 'for_each_test'
+  - 'for_each_thread'
+  - 'for_each_token'
+  - 'for_each_unicast_dest_pgid'
+  - 'for_each_valid_link'
+  - 'for_each_vif_active_link'
+  - 'for_each_vma'
+  - 'for_each_vma_range'
+  - 'for_each_vsi'
+  - 'for_each_wakeup_source'
+  - 'for_each_zone'
+  - 'for_each_zone_zonelist'
+  - 'for_each_zone_zonelist_nodemask'
+  - 'func_for_each_insn'
+  - 'fwnode_for_each_available_child_node'
+  - 'fwnode_for_each_child_node'
+  - 'fwnode_for_each_parent_node'
+  - 'fwnode_graph_for_each_endpoint'
+  - 'gadget_for_each_ep'
+  - 'genradix_for_each'
+  - 'genradix_for_each_from'
+  - 'genradix_for_each_reverse'
+  - 'hash_for_each'
+  - 'hash_for_each_possible'
+  - 'hash_for_each_possible_rcu'
+  - 'hash_for_each_possible_rcu_notrace'
+  - 'hash_for_each_possible_safe'
+  - 'hash_for_each_rcu'
+  - 'hash_for_each_safe'
+  - 'hashmap__for_each_entry'
+  - 'hashmap__for_each_entry_safe'
+  - 'hashmap__for_each_key_entry'
+  - 'hashmap__for_each_key_entry_safe'
+  - 'hctx_for_each_ctx'
+  - 'hists__for_each_format'
+  - 'hists__for_each_sort_list'
+  - 'hlist_bl_for_each_entry'
+  - 'hlist_bl_for_each_entry_rcu'
+  - 'hlist_bl_for_each_entry_safe'
+  - 'hlist_for_each'
+  - 'hlist_for_each_entry'
+  - 'hlist_for_each_entry_continue'
+  - 'hlist_for_each_entry_continue_rcu'
+  - 'hlist_for_each_entry_continue_rcu_bh'
+  - 'hlist_for_each_entry_from'
+  - 'hlist_for_each_entry_from_rcu'
+  - 'hlist_for_each_entry_rcu'
+  - 'hlist_for_each_entry_rcu_bh'
+  - 'hlist_for_each_entry_rcu_notrace'
+  - 'hlist_for_each_entry_safe'
+  - 'hlist_for_each_entry_srcu'
+  - 'hlist_for_each_safe'
+  - 'hlist_nulls_for_each_entry'
+  - 'hlist_nulls_for_each_entry_from'
+  - 'hlist_nulls_for_each_entry_rcu'
+  - 'hlist_nulls_for_each_entry_safe'
+  - 'i3c_bus_for_each_i2cdev'
+  - 'i3c_bus_for_each_i3cdev'
+  - 'idr_for_each_entry'
+  - 'idr_for_each_entry_continue'
+  - 'idr_for_each_entry_continue_ul'
+  - 'idr_for_each_entry_ul'
+  - 'in_dev_for_each_ifa_rcu'
+  - 'in_dev_for_each_ifa_rtnl'
+  - 'inet_bind_bucket_for_each'
+  - 'interval_tree_for_each_span'
+  - 'intlist__for_each_entry'
+  - 'intlist__for_each_entry_safe'
+  - 'kcore_copy__for_each_phdr'
+  - 'key_for_each'
+  - 'key_for_each_safe'
+  - 'klp_for_each_func'
+  - 'klp_for_each_func_safe'
+  - 'klp_for_each_func_static'
+  - 'klp_for_each_object'
+  - 'klp_for_each_object_safe'
+  - 'klp_for_each_object_static'
+  - 'kunit_suite_for_each_test_case'
+  - 'kvm_for_each_memslot'
+  - 'kvm_for_each_memslot_in_gfn_range'
+  - 'kvm_for_each_vcpu'
+  - 'libbpf_nla_for_each_attr'
+  - 'list_for_each'
+  - 'list_for_each_codec'
+  - 'list_for_each_codec_safe'
+  - 'list_for_each_continue'
+  - 'list_for_each_entry'
+  - 'list_for_each_entry_continue'
+  - 'list_for_each_entry_continue_rcu'
+  - 'list_for_each_entry_continue_reverse'
+  - 'list_for_each_entry_from'
+  - 'list_for_each_entry_from_rcu'
+  - 'list_for_each_entry_from_reverse'
+  - 'list_for_each_entry_lockless'
+  - 'list_for_each_entry_rcu'
+  - 'list_for_each_entry_reverse'
+  - 'list_for_each_entry_safe'
+  - 'list_for_each_entry_safe_continue'
+  - 'list_for_each_entry_safe_from'
+  - 'list_for_each_entry_safe_reverse'
+  - 'list_for_each_entry_srcu'
+  - 'list_for_each_from'
+  - 'list_for_each_prev'
+  - 'list_for_each_prev_safe'
+  - 'list_for_each_rcu'
+  - 'list_for_each_reverse'
+  - 'list_for_each_safe'
+  - 'llist_for_each'
+  - 'llist_for_each_entry'
+  - 'llist_for_each_entry_safe'
+  - 'llist_for_each_safe'
+  - 'lwq_for_each_safe'
+  - 'map__for_each_symbol'
+  - 'map__for_each_symbol_by_name'
+  - 'maps__for_each_entry'
+  - 'maps__for_each_entry_safe'
+  - 'mas_for_each'
+  - 'mci_for_each_dimm'
+  - 'media_device_for_each_entity'
+  - 'media_device_for_each_intf'
+  - 'media_device_for_each_link'
+  - 'media_device_for_each_pad'
+  - 'media_entity_for_each_pad'
+  - 'media_pipeline_for_each_entity'
+  - 'media_pipeline_for_each_pad'
+  - 'mlx5_lag_for_each_peer_mdev'
+  - 'msi_domain_for_each_desc'
+  - 'msi_for_each_desc'
+  - 'mt_for_each'
+  - 'nanddev_io_for_each_page'
+  - 'netdev_for_each_lower_dev'
+  - 'netdev_for_each_lower_private'
+  - 'netdev_for_each_lower_private_rcu'
+  - 'netdev_for_each_mc_addr'
+  - 'netdev_for_each_synced_mc_addr'
+  - 'netdev_for_each_synced_uc_addr'
+  - 'netdev_for_each_uc_addr'
+  - 'netdev_for_each_upper_dev_rcu'
+  - 'netdev_hw_addr_list_for_each'
+  - 'nft_rule_for_each_expr'
+  - 'nla_for_each_attr'
+  - 'nla_for_each_nested'
+  - 'nlmsg_for_each_attr'
+  - 'nlmsg_for_each_msg'
+  - 'nr_neigh_for_each'
+  - 'nr_neigh_for_each_safe'
+  - 'nr_node_for_each'
+  - 'nr_node_for_each_safe'
+  - 'of_for_each_phandle'
+  - 'of_property_for_each_string'
+  - 'of_property_for_each_u32'
+  - 'pci_bus_for_each_resource'
+  - 'pci_dev_for_each_resource'
+  - 'pcl_for_each_chunk'
+  - 'pcl_for_each_segment'
+  - 'pcm_for_each_format'
+  - 'perf_config_items__for_each_entry'
+  - 'perf_config_sections__for_each_entry'
+  - 'perf_config_set__for_each_entry'
+  - 'perf_cpu_map__for_each_cpu'
+  - 'perf_cpu_map__for_each_idx'
+  - 'perf_evlist__for_each_entry'
+  - 'perf_evlist__for_each_entry_reverse'
+  - 'perf_evlist__for_each_entry_safe'
+  - 'perf_evlist__for_each_evsel'
+  - 'perf_evlist__for_each_mmap'
+  - 'perf_hpp_list__for_each_format'
+  - 'perf_hpp_list__for_each_format_safe'
+  - 'perf_hpp_list__for_each_sort_list'
+  - 'perf_hpp_list__for_each_sort_list_safe'
+  - 'perf_tool_event__for_each_event'
+  - 'plist_for_each'
+  - 'plist_for_each_continue'
+  - 'plist_for_each_entry'
+  - 'plist_for_each_entry_continue'
+  - 'plist_for_each_entry_safe'
+  - 'plist_for_each_safe'
+  - 'pnp_for_each_card'
+  - 'pnp_for_each_dev'
+  - 'protocol_for_each_card'
+  - 'protocol_for_each_dev'
+  - 'queue_for_each_hw_ctx'
+  - 'radix_tree_for_each_slot'
+  - 'radix_tree_for_each_tagged'
+  - 'rb_for_each'
+  - 'rbtree_postorder_for_each_entry_safe'
+  - 'rdma_for_each_block'
+  - 'rdma_for_each_port'
+  - 'rdma_umem_for_each_dma_block'
+  - 'resort_rb__for_each_entry'
+  - 'resource_list_for_each_entry'
+  - 'resource_list_for_each_entry_safe'
+  - 'rhl_for_each_entry_rcu'
+  - 'rhl_for_each_rcu'
+  - 'rht_for_each'
+  - 'rht_for_each_entry'
+  - 'rht_for_each_entry_from'
+  - 'rht_for_each_entry_rcu'
+  - 'rht_for_each_entry_rcu_from'
+  - 'rht_for_each_entry_safe'
+  - 'rht_for_each_from'
+  - 'rht_for_each_rcu'
+  - 'rht_for_each_rcu_from'
+  - 'rq_for_each_bvec'
+  - 'rq_for_each_segment'
+  - 'rq_list_for_each'
+  - 'rq_list_for_each_safe'
+  - 'sample_read_group__for_each'
+  - 'scsi_for_each_prot_sg'
+  - 'scsi_for_each_sg'
+  - 'sctp_for_each_hentry'
+  - 'sctp_skb_for_each'
+  - 'sec_for_each_insn'
+  - 'sec_for_each_insn_continue'
+  - 'sec_for_each_insn_from'
+  - 'sec_for_each_sym'
+  - 'shdma_for_each_chan'
+  - 'shost_for_each_device'
+  - 'sk_for_each'
+  - 'sk_for_each_bound'
+  - 'sk_for_each_bound_bhash2'
+  - 'sk_for_each_entry_offset_rcu'
+  - 'sk_for_each_from'
+  - 'sk_for_each_rcu'
+  - 'sk_for_each_safe'
+  - 'sk_nulls_for_each'
+  - 'sk_nulls_for_each_from'
+  - 'sk_nulls_for_each_rcu'
+  - 'snd_array_for_each'
+  - 'snd_pcm_group_for_each_entry'
+  - 'snd_soc_dapm_widget_for_each_path'
+  - 'snd_soc_dapm_widget_for_each_path_safe'
+  - 'snd_soc_dapm_widget_for_each_sink_path'
+  - 'snd_soc_dapm_widget_for_each_source_path'
+  - 'strlist__for_each_entry'
+  - 'strlist__for_each_entry_safe'
+  - 'sym_for_each_insn'
+  - 'sym_for_each_insn_continue_reverse'
+  - 'symbols__for_each_entry'
+  - 'tb_property_for_each'
+  - 'tcf_act_for_each_action'
+  - 'tcf_exts_for_each_action'
+  - 'ttm_resource_manager_for_each_res'
+  - 'twsk_for_each_bound_bhash2'
+  - 'udp_portaddr_for_each_entry'
+  - 'udp_portaddr_for_each_entry_rcu'
+  - 'usb_hub_for_each_child'
+  - 'v4l2_device_for_each_subdev'
+  - 'v4l2_m2m_for_each_dst_buf'
+  - 'v4l2_m2m_for_each_dst_buf_safe'
+  - 'v4l2_m2m_for_each_src_buf'
+  - 'v4l2_m2m_for_each_src_buf_safe'
+  - 'virtio_device_for_each_vq'
+  - 'while_for_each_ftrace_op'
+  - 'xa_for_each'
+  - 'xa_for_each_marked'
+  - 'xa_for_each_range'
+  - 'xa_for_each_start'
+  - 'xas_for_each'
+  - 'xas_for_each_conflict'
+  - 'xas_for_each_marked'
+  - 'xbc_array_for_each_value'
+  - 'xbc_for_each_key_value'
+  - 'xbc_node_for_each_array_value'
+  - 'xbc_node_for_each_child'
+  - 'xbc_node_for_each_key_value'
+  - 'xbc_node_for_each_subkey'
+  - 'zorro_for_each_dev'
+  - 'pool_iter_for_each'
+  - 'pool_for_each'
+
+IncludeBlocks: Preserve
+IncludeCategories:
+  - Regex: '.*'
+    Priority: 1
+IncludeIsMainRegex: '(Test)?$'
+IndentCaseLabels: false
+IndentGotoLabels: false
+IndentPPDirectives: None
+IndentWidth: 8
+IndentWrappedFunctionNames: false
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: false
+MacroBlockBegin: ''
+MacroBlockEnd: ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBinPackProtocolList: Auto
+ObjCBlockIndentWidth: 8
+ObjCSpaceAfterProperty: true
+ObjCSpaceBeforeProtocolList: true
+
+# Taken from git's rules
+PenaltyBreakAssignment: 10
+PenaltyBreakBeforeFirstCallParameter: 30
+PenaltyBreakComment: 10
+PenaltyBreakFirstLessLess: 0
+PenaltyBreakString: 10
+PenaltyExcessCharacter: 100
+PenaltyReturnTypeOnItsOwnLine: 60
+
+PointerAlignment: Right
+ReflowComments: false
+SortIncludes: false
+SortUsingDeclarations: false
+SpaceAfterCStyleCast: false
+SpaceAfterTemplateKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeCtorInitializerColon: true
+SpaceBeforeInheritanceColon: true
+SpaceBeforeParens: ControlStatementsExceptForEachMacros
+SpaceBeforeRangeBasedForLoopColon: true
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 1
+SpacesInAngles: false
+SpacesInContainerLiterals: false
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard: Cpp03
+TabWidth: 8
+UseTab: Always
+...

.dockerignore

@@ -0,0 +1 @@
+build

.github/workflows/build-freebsd.yml

@@ -0,0 +1,31 @@
+name: build on FreeBSD
+
+on:
+  push:
+    branches: [ "main", "dev" ]
+  pull_request:
+    branches: [ "main", "dev" ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+
+    - name: apply the patch to libssh
+      run: |
+        git -C libssh fetch --all --tags --prune
+        patch -d libssh -p1 < patch/$(git -C libssh describe).patch
+
+    - name: Build in FreeBSD
+      uses: vmactions/freebsd-vm@v1
+      with:
+        prepare: |
+          pkg install -y git cmake
+        run: |
+          cmake -B build -DCMAKE_BUILD_TYPE=Release
+          cmake --build build
+          build/mscp -h

.github/workflows/build-macos.yml

@@ -2,9 +2,9 @@ name: build on macOS
 
 on:
   push:
-    branches: [ "main" ]
+    branches: [ "main", "dev" ]
   pull_request:
-    branches: [ "main" ]
+    branches: [ "main", "dev" ]
 
 env:
   # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
@@ -18,10 +18,15 @@ jobs:
     runs-on: macos-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
 
+    - name: apply the patch to libssh
+      run: |
+        git -C libssh fetch --all --tags --prune
+        patch -d libssh -p1 < patch/$(git -C libssh describe).patch
+
     - name: install build dependency
       run: ./scripts/install-build-deps.sh
 
@@ -29,13 +34,10 @@ jobs:
       id: brew-prefix
       run: echo "HOMEBREW_PREFIX=$(brew --prefix)" >> $GITHUB_OUTPUT
 
-    - name: patch to libssh
-      run: patch -d libssh -p1 < patch/libssh-0.10.4.patch
-
     - name: Configure CMake
       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
-      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DOPENSSL_ROOT_DIR=${{steps.brew-prefix.outputs.HOMEBREW_PREFIX}}/opt/openssl@1.1
+      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DOPENSSL_ROOT_DIR=${{steps.brew-prefix.outputs.HOMEBREW_PREFIX}}/opt/openssl@3
 
     - name: Build
       # Build your program with the given configuration

.github/workflows/build-ubuntu.yml

@@ -2,9 +2,9 @@ name: build on ubuntu
 
 on:
   push:
-    branches: [ "main" ]
+    branches: [ "main", "dev" ]
   pull_request:
-    branches: [ "main" ]
+    branches: [ "main", "dev" ]
 
 env:
   # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
@@ -18,15 +18,19 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
 
-    - name: install build dependency
-      run: sudo ./scripts/install-build-deps.sh
+    - name: apply the patch to libssh
+      run: |
+        git -C libssh fetch --all --tags --prune
+        patch -d libssh -p1 < patch/$(git -C libssh describe).patch
 
-    - name: patch to libssh
-      run: patch -d libssh -p1 < patch/libssh-0.10.4.patch
+    - name: install build dependency
+      run: |
+        sudo apt-get update
+        sudo ./scripts/install-build-deps.sh
 
     - name: Configure CMake
       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.

.github/workflows/codeql.yml

@@ -13,10 +13,10 @@ name: "CodeQL"
 
 on:
   push:
-    branches: [ "main" ]
+    branches: [ "main", "dev" ]
   pull_request:
     # The branches below must be a subset of the branches above
-    branches: [ "main" ]
+    branches: [ "main", "dev" ]
   schedule:
     - cron: '35 11 * * 5'
 
@@ -38,19 +38,23 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         submodules: true
 
-    - name: install build dependency
-      run: sudo ./scripts/install-build-deps.sh
+    - name: apply the patch to libssh
+      run: |
+        git -C libssh fetch --all --tags --prune
+        patch -d libssh -p1 < patch/$(git -C libssh describe).patch
 
-    - name: patch to libssh
-      run: patch -d libssh -p1 < patch/libssh-0.10.4.patch
+    - name: install build dependency
+      run: |
+        sudo apt-get update
+        sudo ./scripts/install-build-deps.sh
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -63,7 +67,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, Go, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -76,6 +80,6 @@ jobs:
     #   ./location_of_script_within_repo/buildscript.sh
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3
       with:
         category: "/language:${{matrix.language}}"

.github/workflows/release.yml

@@ -4,47 +4,37 @@ on:
   push:
     tags:
       - "v*.*.*"
+  workflow_dispatch:
 
 env:
     BUILD_TYPE: Release
 
 jobs:
-  build-and-release:
+  source-release:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: true
 
-      - name: patch to libssh
-        run: patch -d libssh -p1 < patch/libssh-0.10.4.patch
-
-      # TODO: just building docker does not require packages. fix CMakeLists
-      - name: install build dependency
-        run: sudo ./scripts/install-build-deps.sh
+      - name: apply the patch to libssh
+        run: |
+          git -C libssh fetch --all --tags --prune
+          patch -d libssh -p1 < patch/$(git -C libssh describe).patch
 
       - name: Set variables
         run: |
           VER=$(cat VERSION)
           echo "VERSION=$VER" >> $GITHUB_ENV
 
-      - name: Configure Cmake
-        run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
-
-      - name: Build Containers
-        run: make -C ${{github.workspace}}/build docker-build-all
-
-      - name: Test
-        run: make -C ${{github.workspace}}/build docker-test-all
-
-      - name: Retrieve packages from containers
-        run: make -C ${{github.workspace}}/build docker-pkg-all
+      - name: archive
+        run: |
+          cd ..
+          cp -r mscp mscp-${{env.VERSION}}
+          tar zcvf mscp-${{env.VERSION}}.tar.gz --exclude-vcs mscp-${{env.VERSION}}
 
       - name: Release
         uses: softprops/action-gh-release@v1
         with:
           files: |
-            ${{github.workspace}}/build/mscp_${{env.VERSION}}-ubuntu-20.04-x86_64.deb
-            ${{github.workspace}}/build/mscp_${{env.VERSION}}-ubuntu-22.04-x86_64.deb
-            ${{github.workspace}}/build/mscp_${{env.VERSION}}-centos-8-x86_64.rpm
-            ${{github.workspace}}/build/mscp_${{env.VERSION}}-rocky-8.6-x86_64.rpm
+            ${{github.workspace}}/../mscp-${{env.VERSION}}.tar.gz

.github/workflows/test.yml

@@ -2,9 +2,9 @@ name: test
 
 on:
   push:
-    branches: [ "main" ]
+    branches: [ "main", "dev" ]
   pull_request:
-    branches: [ "main" ]
+    branches: [ "main", "dev" ]
 
 env:
   BUILD_TYPE: Release
@@ -12,23 +12,39 @@ env:
 jobs:
   test:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        index: # see DIST_IDS and DIST_VERS lists in CMakeLists.txt
+        - ubuntu-20.04
+        - ubuntu-22.04
+        - ubuntu-24.04
+        - rocky-8.9
+        - rocky-9.3
+        - almalinux-9.3
+        - alpine-3.22
+        - arch-base
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
 
-    - name: patch to libssh
-      run: patch -d libssh -p1 < patch/libssh-0.10.4.patch
+    - name: apply the patch to libssh
+      run: |
+        git -C libssh fetch --all --tags --prune
+        patch -d libssh -p1 < patch/$(git -C libssh describe).patch
 
-    # TODO: just building docker does not require libssh. fix CMakeLists
+    # TODO: just building docker images does not require libssh. fix CMakeLists
     - name: install build dependency
-      run: sudo ./scripts/install-build-deps.sh
+      run: |
+        sudo apt-get update
+        sudo ./scripts/install-build-deps.sh
 
     - name: configure CMake
       run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
 
     - name: Build Containers
-      run: make -C ${{github.workspace}}/build docker-build-all
+      run: make -C ${{github.workspace}}/build docker-build-${{ matrix.index }}
 
     - name: Run Test
-      run: make -C ${{github.workspace}}/build docker-test-all
+      run: make -C ${{github.workspace}}/build docker-test-${{ matrix.index }}

.gitignore

@@ -1,3 +1,8 @@
 build
+html
 compile_commands.json
+CMakeUserPresets.json
 .*.swp
+.cache
+
+include/mscp_version.h

CMakeLists.txt

@@ -6,51 +6,150 @@ project(mscp
 	VERSION ${MSCP_VERSION}
 	LANGUAGES C)
 
+
+find_package(Git)
+if (Git_FOUND)
+	# based on https://github.com/nocnokneo/cmake-git-versioning-example
+	execute_process(
+		COMMAND	${GIT_EXECUTABLE} describe --tags --match "v*"
+		OUTPUT_VARIABLE	GIT_DESCRIBE_VERSION
+		RESULT_VARIABLE	GIT_DESCRIBE_ERROR_CODE
+		OUTPUT_STRIP_TRAILING_WHITESPACE)
+	if(NOT GIT_DESCRIBE_ERROR_CODE)
+		set(MSCP_BUILD_VERSION ${GIT_DESCRIBE_VERSION})
+	endif()
+endif()
+
+if (NOT MSCP_BUILD_VERSION)
+	message(STATUS "Failed to determine version via Git. Use VERSION file instead.")
+	set(MSCP_BUILD_VERSION v${MSCP_VERSION})
+endif()
+
+
+include(GNUInstallDirs)
+
 set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -DDEBUG")
 list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/modules)
 
-
-# add libssh static library
-add_subdirectory(libssh EXCLUDE_FROM_ALL)
-
 if(APPLE)
 	list(APPEND CMAKE_PREFIX_PATH /usr/local) # intel mac homebrew prefix
 	list(APPEND CMAKE_PREFIX_PATH /opt/homebrew) # arm mac homebrew prefix
 endif() # APPLE
 
 
-# mscp executable
-add_executable(mscp src/main.c src/platform.c src/ssh.c src/file.c src/pprint.c)
+option(BUILD_CONAN OFF) # Build mscp with conan
+if(BUILD_CONAN)
+	message(STATUS "Build mscp with conan")
+endif()
 
-set(MSCP_LINK_LIBS m pthread)
-set(MSCP_LINK_DIRS "")
-set(MSCP_COMPILE_OPTS "")
-set(MSCP_INCLUDE_DIRS ${mscp_SOURCE_DIR}/src)
+option(BUILD_STATIC OFF) # Build mscp with -static LD flag
+if (BUILD_STATIC)
+	message(STATUS "Build mscp with -static LD option")
+	if (NOT BUILD_CONAN)
+		message(WARNING
+			"BUILD_STATIC strongly recommended with BUILD_CONAN option")
+	endif()
+endif()
 
+option(USE_PODMAN OFF) # use podman instread of docker
+if(USE_PODMAN)
+	message(STATUS "Use podman instead of docker")
+	set(CE podman) # CE means Container Engine
+else()
+	set(CE docker)
+endif()
+
+
+# add libssh static library
+set(CMAKE_POLICY_DEFAULT_CMP0077 NEW)
+set(WITH_SERVER OFF)
+set(BUILD_SHARED_LIBS OFF)
+set(WITH_EXAMPLES OFF)
+set(BUILD_STATIC_LIB ON)
+if(BUILD_CONAN)
+	message(STATUS
+		"Disable libssh GSSAPI support because libkrb5 doesn't exist in conan")
+	set(WITH_GSSAPI OFF)
+endif()
+add_subdirectory(libssh EXCLUDE_FROM_ALL)
+
+
+
+# setup mscp compile options
 list(APPEND MSCP_COMPILE_OPTS -iquote ${CMAKE_CURRENT_BINARY_DIR}/libssh/include)
-list(APPEND MSCP_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/libssh/include)
+list(APPEND MSCP_BUILD_INCLUDE_DIRS
+	${mscp_SOURCE_DIR}/src
+	${CMAKE_CURRENT_BINARY_DIR}/libssh/include)
+
 list(APPEND MSCP_LINK_LIBS ssh-static)
+if(BUILD_CONAN)
+	find_package(ZLIB REQUIRED)
+	find_package(OpenSSL REQUIRED)
+	list(APPEND MSCP_LINK_LIBS ZLIB::ZLIB)
+	list(APPEND MSCP_LINK_LIBS OpenSSL::Crypto)
+endif()
 
-find_package(GSSAPI)
-list(APPEND MSCP_LINK_LIBS ${GSSAPI_LIBRARIES})
 
-find_package(OpenSSL)
-list(APPEND MSCP_LINK_LIBS ${OPENSSL_LIBRARIES})
+# Symbol check
+check_symbol_exists(htonll	arpa/inet.h	HAVE_HTONLL)
+check_symbol_exists(ntohll	arpa/inet.h	HAVE_NTOHLL)
+check_symbol_exists(strlcat	string.h	HAVE_STRLCAT)
+if (NOT HAVE_STRLCAT)
+	list(APPEND OPENBSD_COMPAT_SRC src/openbsd-compat/strlcat.c)
+endif()
 
-find_package(ZLIB)
-list(APPEND MSCP_LINK_LIBS ${ZLIB_LIBRARIES})
 
-target_compile_definitions(mscp PUBLIC ASYNC_WRITE=1)
+# generate config.h in build dir
+configure_file(
+	${mscp_SOURCE_DIR}/include/config.h.in
+	${CMAKE_CURRENT_BINARY_DIR}/include/config.h)
+list(APPEND MSCP_BUILD_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/include)
 
-target_include_directories(mscp PRIVATE ${MSCP_INCLUDE_DIRS})
-target_link_directories(mscp PRIVATE ${MSCP_LINK_DIRS})
-target_link_libraries(mscp PRIVATE ${MSCP_LINK_LIBS})
+
+# libmscp.a
+set(LIBMSCP_SRC
+	src/mscp.c src/ssh.c src/fileops.c src/path.c src/checkpoint.c
+	src/bwlimit.c src/platform.c src/print.c src/pool.c src/strerrno.c
+	${OPENBSD_COMPAT_SRC})
+add_library(mscp-static STATIC ${LIBMSCP_SRC})
+target_include_directories(mscp-static
+	PRIVATE ${MSCP_BUILD_INCLUDE_DIRS} ${mscp_SOURCE_DIR}/include)
+target_compile_options(mscp-static PRIVATE ${MSCP_COMPILE_OPTS})
+target_link_libraries(mscp-static PRIVATE ${MSCP_LINK_LIBS})
+set_target_properties(mscp-static
+	PROPERTIES
+	OUTPUT_NAME	mscp)
+
+# mscp executable
+list(APPEND MSCP_LINK_LIBS m pthread)
+
+add_executable(mscp src/main.c)
+target_include_directories(mscp
+	PRIVATE ${MSCP_BUILD_INCLUDE_DIRS} ${mscp_SOURCE_DIR}/include)
+target_link_libraries(mscp mscp-static ${MSCP_LINK_LIBS})
+if (BUILD_STATIC)
+	target_link_options(mscp PRIVATE -static)
+endif()
 target_compile_options(mscp PRIVATE ${MSCP_COMPILE_OPTS})
-target_compile_definitions(mscp PUBLIC _VERSION="${PROJECT_VERSION}")
+
 
 install(TARGETS mscp RUNTIME DESTINATION bin)
 
 
+# mscp manpage and document
+configure_file(
+	${mscp_SOURCE_DIR}/doc/mscp.1.in
+	${PROJECT_BINARY_DIR}/mscp.1)
+
+add_custom_target(update-rst
+	COMMENT "Update doc/mscp.rst from mscp.1.in"
+	WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
+	COMMAND
+	pandoc -s -f man mscp.1 -t rst -o ${PROJECT_SOURCE_DIR}/doc/mscp.rst)
+
+install(FILES ${PROJECT_BINARY_DIR}/mscp.1
+	DESTINATION ${CMAKE_INSTALL_MANDIR}/man1)
+
 
 # Test
 add_test(NAME	pytest
@@ -62,89 +161,138 @@ enable_testing()
 
 
 
-# CPACK Rules
-set(CPACK_SET_DESTDIR		true)
-set(CPACK_PROJECT_NAME		${PROJECT_NAME})
-set(CPACK_PROJECT_VERSION	${PROJECT_VERSION})
-set(CPACK_PACKAGE_CONTACT	"Ryo Nakamura <upa@haeena.net>")
-set(CPACK_PACKAGE_DESCRIPTION
-	"mscp, copy files over multiple ssh connections")
-
-execute_process(COMMAND uname -m
-	OUTPUT_VARIABLE ARCH OUTPUT_STRIP_TRAILING_WHITESPACE)
-
-if(UNIX AND NOT APPLE) # on linux
-	execute_process(COMMAND
-		bash "-c" "cat /etc/os-release|grep '^ID='|cut -d '=' -f 2|tr -d '\"'"
-		OUTPUT_VARIABLE DIST_NAME OUTPUT_STRIP_TRAILING_WHITESPACE)
-	execute_process(COMMAND
-		bash "-c" "cat /etc/os-release|grep '^VERSION_ID='|cut -d '=' -f 2|tr -d '\"'"
-		OUTPUT_VARIABLE DIST_VER OUTPUT_STRIP_TRAILING_WHITESPACE)
-	execute_process(COMMAND
-		bash "-c" "${mscp_SOURCE_DIR}/scripts/print-install-deps.sh ${DIST_NAME}-${DIST_VER}"
-		OUTPUT_VARIABLE DIST_DEP OUTPUT_STRIP_TRAILING_WHITESPACE)
-
-	set(PACKAGE_FILE_NAME
-		${PROJECT_NAME}_${PROJECT_VERSION}-${DIST_NAME}-${DIST_VER}-${ARCH})
-
-	set(CPACK_DEBIAN_FILE_NAME		${PACKAGE_FILE_NAME}.deb)
-	set(CPACK_DEBIAN_PACKAGE_DEPENDS	${DIST_DEP})
-	set(CPACK_DEBIAN_PACKAGE_HOMEPAGE	"https://github.com/upa/mscp")
-
-	set(CPACK_RPM_FILE_NAME			${PACKAGE_FILE_NAME}.rpm)
-	set(CPACK_RPM_PACKAGE_REQUIRES		${DIST_DEP})
-	set(CPACK_RPM_PACKAGE_HOMEPAGE		"https://github.com/upa/mscp")
-	set(CPACK_RPM_PACKAGE_DESCRIPTION	${CPACK_PACKAGE_DESCRIPTION})
-endif() # on linux
-
-include(CPack)
-
-
 
 # Custom targets to build and test mscp in docker containers.
 # foreach(IN ZIP_LISTS) (cmake >= 3.17) can shorten the following lists.
 # However, ubuntu 20.04 has cmake 3.16.3. So this is a roundabout trick.
-list(APPEND DIST_NAMES	ubuntu	ubuntu	centos	rocky)
-list(APPEND DIST_VERS	20.04	22.04	8	8.6)
-list(APPEND DIST_PKGS	deb	deb	rpm	rpm)
+#
+# When edit DIST_IDS and DIST_VERS, also edit .github/workflows/test.yaml
+list(APPEND DIST_IDS  ubuntu ubuntu ubuntu rocky rocky almalinux alpine arch)
+list(APPEND DIST_VERS  20.04  22.04  24.04   8.9   9.3       9.3   3.22 base)
 
-list(LENGTH DIST_NAMES _DIST_LISTLEN)
+list(LENGTH DIST_IDS _DIST_LISTLEN)
 math(EXPR DIST_LISTLEN "${_DIST_LISTLEN} - 1")
 
 foreach(x RANGE ${DIST_LISTLEN})
-	list(GET DIST_NAMES	${x} DIST_NAME)
+	list(GET DIST_IDS	${x} DIST_ID)
 	list(GET DIST_VERS	${x} DIST_VER)
-	list(GET DIST_PKGS	${x} DIST_PKG)
 
-	set(DOCKER_IMAGE mscp-${DIST_NAME}:${DIST_VER})
-	set(DOCKER_INDEX ${DIST_NAME}-${DIST_VER})
-	set(PKG_FILE_NAME
-		mscp_${PROJECT_VERSION}-${DIST_NAME}-${DIST_VER}-${ARCH}.${DIST_PKG})
+	set(DOCKER_IMAGE mscp-${DIST_ID}:${DIST_VER})
+	set(DOCKER_INDEX ${DIST_ID}-${DIST_VER})
+	execute_process(
+		COMMAND ${CMAKE_SOURCE_DIR}/scripts/install-build-deps.sh
+		--dont-install --platform Linux-${DIST_ID}
+		OUTPUT_VARIABLE REQUIREDPKGS
+		OUTPUT_STRIP_TRAILING_WHITESPACE)
 
 	add_custom_target(docker-build-${DOCKER_INDEX}
 		COMMENT "Build mscp in ${DOCKER_IMAGE} container"
 		WORKING_DIRECTORY ${mscp_SOURCE_DIR}
 		COMMAND
-		docker build -t ${DOCKER_IMAGE} -f docker/${DOCKER_INDEX}.Dockerfile .)
+		${CE} build --build-arg REQUIREDPKGS=${REQUIREDPKGS}
+		-t ${DOCKER_IMAGE} -f Dockerfile/${DOCKER_INDEX}.Dockerfile .)
+
+	add_custom_target(docker-build-${DOCKER_INDEX}-no-cache
+		COMMENT "Build mscp in ${DOCKER_IMAGE} container"
+		WORKING_DIRECTORY ${mscp_SOURCE_DIR}
+		COMMAND
+		${CE} build --build-arg REQUIREDPKGS=${REQUIREDPKGS} --no-cache
+		-t ${DOCKER_IMAGE} -f Dockerfile/${DOCKER_INDEX}.Dockerfile .)
 
 	add_custom_target(docker-test-${DOCKER_INDEX}
 		COMMENT "Test mscp in ${DOCKER_IMAGE} container"
 		WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
 		COMMAND
-		docker run --init --rm ${DOCKER_IMAGE} /mscp/scripts/test-in-container.sh)
+		${CE} run --init --rm --privileged
+		--sysctl net.ipv6.conf.all.disable_ipv6=0
+		--add-host=ip6-localhost:::1
+		${DOCKER_IMAGE} /mscp/scripts/test-in-container.sh)
 
-	add_custom_target(docker-pkg-${DOCKER_INDEX}
-		COMMENT "Retrieve mscp package from ${DOCKER_IMAGE} container"
+	add_custom_target(docker-run-${DOCKER_INDEX}
+		COMMENT "Start ${DOCKER_IMAGE} container"
 		WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
 		COMMAND
-		docker run --rm -v ${CMAKE_BINARY_DIR}:/out ${DOCKER_IMAGE}
-		cp /mscp/build/${PKG_FILE_NAME} /out/)
+		${CE} run --init --rm --privileged
+		--sysctl net.ipv6.conf.all.disable_ipv6=0
+		--add-host=ip6-localhost:::1
+		-it
+		${DOCKER_IMAGE} /mscp/scripts/test-in-container.sh bash)
 
-	list(APPEND DOCKER_BUILDS	docker-build-${DOCKER_INDEX})
-	list(APPEND DOCKER_TESTS	docker-test-${DOCKER_INDEX})
-	list(APPEND DOCKER_PKGS		docker-pkg-${DOCKER_INDEX})
+	list(APPEND DOCKER_BUILDS		docker-build-${DOCKER_INDEX})
+	list(APPEND DOCKER_BUILDS_NO_CACHE	docker-build-${DOCKER_INDEX}-no-cache)
+	list(APPEND DOCKER_TESTS		docker-test-${DOCKER_INDEX})
 endforeach()
 
-add_custom_target(docker-build-all	DEPENDS ${DOCKER_BUILDS})
-add_custom_target(docker-test-all	DEPENDS ${DOCKER_TESTS})
-add_custom_target(docker-pkg-all	DEPENDS ${DOCKER_PKGS})
+add_custom_target(docker-build-all		DEPENDS ${DOCKER_BUILDS})
+add_custom_target(docker-build-all-no-cache	DEPENDS ${DOCKER_BUILDS_NO_CACHE})
+add_custom_target(docker-test-all		DEPENDS ${DOCKER_TESTS})
+
+
+### debuild-related definitions
+
+set(DEBBUILDCONTAINER mscp-build-deb)
+execute_process(
+	COMMAND ${CMAKE_SOURCE_DIR}/scripts/install-build-deps.sh
+	--dont-install --platform Linux-ubuntu
+	OUTPUT_VARIABLE REQUIREDPKGS_DEB
+	OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+add_custom_target(build-deb
+	COMMENT "build mscp deb files inside a container"
+	WORKING_DIRECTORY ${mscp_SOURCE_DIR}
+	BYPRODUCTS ${CMAKE_BINARY_DIR}/debbuild
+	COMMAND
+	${CE} build --build-arg REQUIREDPKGS=${REQUIREDPKGS_DEB}
+	-t ${DEBBUILDCONTAINER} -f Dockerfile/build-deb.Dockerfile .
+	COMMAND
+	${CE} run --rm -v ${CMAKE_BINARY_DIR}:/out ${DEBBUILDCONTAINER}
+	cp -r /debbuild /out/)
+
+
+### rpmbuild-related definitions
+
+# generate files for rpmbuild
+configure_file(
+	${mscp_SOURCE_DIR}/rpm/mscp.spec.in
+	${mscp_SOURCE_DIR}/rpm/mscp.spec
+	@ONLY)
+#configure_file(
+#	${mscp_SOURCE_DIR}/Dockerfile/build-srpm.Dockerfile.in
+#	${mscp_SOURCE_DIR}/Dockerfile/build-srpm.Dockerfile
+#	@ONLY)
+
+# Custom target to build mscp as a src.rpm in docker.
+set(RPMBUILDCONTAINER mscp-build-srpm)
+execute_process(
+	COMMAND ${CMAKE_SOURCE_DIR}/scripts/install-build-deps.sh
+	--dont-install --platform Linux-rocky
+	OUTPUT_VARIABLE REQUIREDPKGS_RPM
+	OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+add_custom_target(build-srpm
+	COMMENT "Build mscp src.rpm inside a container"
+	WORKING_DIRECTORY ${mscp_SOURCE_DIR}
+	COMMAND
+	${CE} build --build-arg REQUIREDPKGS=${REQUIREDPKGS_RPM}
+	--build-arg MSCP_VERSION=${MSCP_VERSION}
+	-t ${RPMBUILDCONTAINER} -f Dockerfile/build-srpm.Dockerfile .
+	COMMAND
+	${CE} run --rm -v ${CMAKE_BINARY_DIR}:/out ${RPMBUILDCONTAINER}
+	bash -c "cp /root/rpmbuild/SRPMS/mscp-*.src.rpm /out/")
+
+### single-binary-build-related definitions
+
+# Custom target to get single binary mscp
+set(SINGLEBINARYFILE mscp.linux.${CMAKE_SYSTEM_PROCESSOR}.static)
+add_custom_target(build-single-binary
+	COMMENT "Build mscp as a single binary in alpine conatiner"
+	WORKING_DIRECTORY ${mscp_SOURCE_DIR}
+	BYPRODUCTS ${CMAKE_BINARY_DIR}/${SINGLEBINARYFILE}
+	DEPENDS docker-build-alpine-3.22
+	COMMAND
+	${CE} run --rm -v ${CMAKE_BINARY_DIR}:/out mscp-alpine:3.22
+	cp /mscp/build/mscp /out/${SINGLEBINARYFILE})
+
+
+add_custom_target(build-pkg-all
+	DEPENDS build-deb build-srpm build-single-binary)

Dockerfile/README.md

@@ -0,0 +1,7 @@
+
+Dockerfiles for building and testing mscp.
+
+cmake provides custom targets to build and test mscp in the containers
+See `make docker-*` targets. `make docker-build-all` builds all
+container images, and `make docker-test-all` runs the test in all
+container images.

Dockerfile/almalinux-9.3.Dockerfile

@@ -0,0 +1,40 @@
+FROM almalinux:9.3
+
+ARG REQUIREDPKGS
+
+# install pytest, sshd for test, and rpm-build
+RUN set -ex && yum -y install \
+	${REQUIREDPKGS} python3 python3-pip python3-devel \
+	openssh openssh-server openssh-clients rpm-build
+
+RUN python3 -m pip install pytest
+
+
+# preparation for sshd
+RUN mkdir /var/run/sshd        \
+	&& ssh-keygen -A	\
+        && ssh-keygen -f /root/.ssh/id_rsa -N ""                \
+        && cat /root/.ssh/id_rsa.pub > /root/.ssh/authorized_keys
+
+# create test user
+RUN useradd -m -d /home/test test       \
+        && echo "test:userpassword" | chpasswd \
+        && mkdir -p /home/test/.ssh     \
+        && ssh-keygen -f /home/test/.ssh/id_rsa_test -N "keypassphrase" \
+        && cat /home/test/.ssh/id_rsa_test.pub >> /home/test/.ssh/authorized_keys \
+        && chown -R test:test /home/test \
+        && chown -R test:test /home/test/.ssh
+
+RUN rm -rf /run/nologin
+
+ARG mscpdir="/mscp"
+
+COPY . ${mscpdir}
+
+# build
+RUN cd ${mscpdir}			\
+        && rm -rf build			\
+        && cmake -B build		\
+        && cd ${mscpdir}/build          \
+	&& make	-j 2			\
+	&& make install

Dockerfile/alpine-3.22.Dockerfile

@@ -0,0 +1,54 @@
+FROM alpine:3.22
+
+# do not use REQUIREDPKGS build argument because
+# this Dockerfile compiles mscp with conan,so we do not need
+# libssl-dev and zlib-dev
+
+# Build mscp with conan to create single binary mscp
+
+RUN apk add --no-cache \
+	gcc make cmake libc-dev \
+	linux-headers openssh bash perl \
+	python3 py3-pip python3-dev py3-pytest g++
+
+RUN pip3 install --break-system-packages conan
+
+# preparation for sshd
+RUN ssh-keygen -A \
+	&& mkdir /var/run/sshd        \
+	&& ssh-keygen -f /root/.ssh/id_rsa -N ""                \
+	&& cat /root/.ssh/id_rsa.pub > /root/.ssh/authorized_keys
+
+# disable PerSourcePenaltie, which would distrub test:
+# https://undeadly.org/cgi?action=article;sid=20240607042157
+RUN echo "PerSourcePenalties=no" > /etc/ssh/sshd_config.d/90-mscp-test.conf
+
+# create test user
+RUN addgroup -S test \
+	&& adduser -S test -G test \
+        && echo "test:userpassword" | chpasswd \
+        && mkdir -p /home/test/.ssh     \
+        && ssh-keygen -f /home/test/.ssh/id_rsa_test -N "keypassphrase" \
+        && cat /home/test/.ssh/id_rsa_test.pub >> /home/test/.ssh/authorized_keys \
+        && chown -R test:test /home/test \
+        && chown -R test:test /home/test/.ssh
+
+
+# Build mscp as a single binary
+RUN conan profile detect --force
+
+ARG mscpdir="/mscp"
+
+COPY . ${mscpdir}
+
+RUN cd ${mscpdir}							\
+	&& rm -rf build							\
+	&& conan install . --output-folder=build --build=missing	\
+	&& cd ${mscpdir}/build						\
+	&& cmake ..							\
+		-DCMAKE_BUILD_TYPE=Release				\
+		-DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake		\
+		-DBUILD_CONAN=ON -DBUILD_STATIC=ON			\
+	&& make	-j 2							\
+	&& make install
+

Dockerfile/arch-base.Dockerfile

@@ -0,0 +1,36 @@
+FROM archlinux:base
+
+ARG REQUIREDPKGS
+
+# install pyest and openssh for test
+RUN set -ex && pacman -Syy && pacman --noconfirm -S ${REQUIREDPKGS} openssh python-pytest
+
+RUN mkdir /var/run/sshd        \
+        && ssh-keygen -A        \
+        && ssh-keygen -f /root/.ssh/id_rsa -N ""                \
+        && cat /root/.ssh/id_rsa.pub > /root/.ssh/authorized_keys
+
+# disable PerSourcePenaltie, which would distrub test:
+# https://undeadly.org/cgi?action=article;sid=20240607042157
+RUN echo "PerSourcePenalties=no" > /etc/ssh/sshd_config.d/90-mscp-test.conf
+
+# create test user
+RUN useradd -m -d /home/test test       \
+        && echo "test:userpassword" | chpasswd \
+        && mkdir -p /home/test/.ssh     \
+        && ssh-keygen -f /home/test/.ssh/id_rsa_test -N "keypassphrase" \
+        && cat /home/test/.ssh/id_rsa_test.pub >> /home/test/.ssh/authorized_keys \
+        && chown -R test:test /home/test \
+        && chown -R test:test /home/test/.ssh
+
+ARG mscpdir="/mscp"
+
+COPY . ${mscpdir}
+
+# build
+RUN cd ${mscpdir}                       \
+        && rm -rf build                 \
+        && cmake -B build               \
+        && cd ${mscpdir}/build          \
+        && make -j 2                    \
+        && make install

Dockerfile/build-deb.Dockerfile

@@ -0,0 +1,20 @@
+FROM ubuntu:22.04
+
+ARG REQUIREDPKGS
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN set -ex && apt-get update && apt-get install -y --no-install-recommends \
+	${REQUIREDPKGS} ca-certificates \
+	build-essential devscripts debhelper gcc make cmake
+
+ARG mscpdir="/debbuild/mscp"
+
+COPY . ${mscpdir}
+
+# build
+RUN cd ${mscpdir} 	\
+	&& debuild -us -uc -S \
+	&& mv ${mscpdir} /
+
+# Then all debuild output files exsit at /debbuild
+

Dockerfile/build-srpm.Dockerfile

@@ -0,0 +1,22 @@
+FROM rockylinux:9
+
+ARG REQUIREDPKGS
+ARG MSCP_VERSION
+
+# install pytest, sshd for test, and rpm-build
+RUN set -ex && yum -y install ${REQUIREDPKGS} rpm-build rpmdevtools
+
+ARG mscpdir="/mscp-${MSCP_VERSION}"
+ARG mscptgz="mscp-${MSCP_VERSION}.tar.gz"
+
+COPY . ${mscpdir}
+
+# prepare rpmbuild
+RUN rpmdev-setuptree \
+	&& rm -rf ${mscpdir}/build	\
+	&& tar zcvf /${mscptgz} --exclude-vcs ${mscpdir}	\
+	&& cp /${mscptgz} ~/rpmbuild/SOURCES/	\
+	&& cp ${mscpdir}/rpm/mscp.spec ~/rpmbuild/SPECS/
+
+# build rpm and src.rpm
+RUN rpmbuild -ba ~/rpmbuild/SPECS/mscp.spec

Dockerfile/rocky-8.9.Dockerfile

@@ -0,0 +1,42 @@
+FROM rockylinux:8.9
+
+ARG REQUIREDPKGS
+
+# install pytest, sshd for test, and rpm-build
+RUN set -ex && yum -y install \
+	${REQUIREDPKGS}	\
+	python3 python3-pip python3-devel \
+	openssh openssh-server openssh-clients rpm-build
+
+RUN python3 -m pip install pytest
+
+
+# preparation for sshd
+RUN mkdir /var/run/sshd        \
+	&& ssh-keygen -A	\
+        && ssh-keygen -f /root/.ssh/id_rsa -N ""                \
+        && cat /root/.ssh/id_rsa.pub > /root/.ssh/authorized_keys
+
+# create test user
+RUN useradd -m -d /home/test test       \
+        && echo "test:userpassword" | chpasswd \
+        && mkdir -p /home/test/.ssh     \
+        && ssh-keygen -f /home/test/.ssh/id_rsa_test -N "keypassphrase" \
+        && cat /home/test/.ssh/id_rsa_test.pub >> /home/test/.ssh/authorized_keys \
+        && chown -R test:test /home/test \
+        && chown -R test:test /home/test/.ssh
+
+RUN rm -rf /run/nologin
+
+
+ARG mscpdir="/mscp"
+
+COPY . ${mscpdir}
+
+# build
+RUN cd ${mscpdir}			\
+        && rm -rf build			\
+        && cmake -B build		\
+        && cd ${mscpdir}/build          \
+	&& make	-j 2			\
+	&& make install

Dockerfile/rocky-9.3.Dockerfile

@@ -0,0 +1,42 @@
+FROM rockylinux:9.3
+
+ARG REQUIREDPKGS
+
+# install pytest, sshd for test, and rpm-build
+RUN set -ex && yum -y install \
+	${REQUIREDPKGS}	\
+	python3 python3-pip python3-devel \
+	openssh openssh-server openssh-clients rpm-build
+
+RUN python3 -m pip install pytest
+
+
+# preparation for sshd
+RUN mkdir /var/run/sshd        \
+	&& ssh-keygen -A	\
+        && ssh-keygen -f /root/.ssh/id_rsa -N ""                \
+        && cat /root/.ssh/id_rsa.pub > /root/.ssh/authorized_keys
+
+# create test user
+RUN useradd -m -d /home/test test       \
+        && echo "test:userpassword" | chpasswd \
+        && mkdir -p /home/test/.ssh     \
+        && ssh-keygen -f /home/test/.ssh/id_rsa_test -N "keypassphrase" \
+        && cat /home/test/.ssh/id_rsa_test.pub >> /home/test/.ssh/authorized_keys \
+        && chown -R test:test /home/test \
+        && chown -R test:test /home/test/.ssh
+
+RUN rm -rf /run/nologin
+
+ARG mscpdir="/mscp"
+
+COPY . ${mscpdir}
+
+# build
+RUN cd ${mscpdir}			\
+        && rm -rf build			\
+        && cmake -B build		\
+        && cd ${mscpdir}/build          \
+	&& make	-j 2			\
+	&& make install
+

Dockerfile/ubuntu-20.04.Dockerfile

@@ -0,0 +1,38 @@
+FROM ubuntu:20.04
+
+ARG REQUIREDPKGS
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN set -ex && apt-get update && apt-get install -y --no-install-recommends \
+	${REQUIREDPKGS} ca-certificates python3 python3-pip python3-dev openssh-server
+
+RUN python3 -m pip install pytest
+
+# preparation for sshd
+RUN mkdir /var/run/sshd        \
+	&& ssh-keygen -A	\
+        && ssh-keygen -f /root/.ssh/id_rsa -N ""                \
+        && cat /root/.ssh/id_rsa.pub > /root/.ssh/authorized_keys
+
+# create test user
+RUN useradd -m -d /home/test test       \
+        && echo "test:userpassword" | chpasswd \
+        && mkdir -p /home/test/.ssh     \
+        && ssh-keygen -f /home/test/.ssh/id_rsa_test -N "keypassphrase" \
+        && cat /home/test/.ssh/id_rsa_test.pub >> /home/test/.ssh/authorized_keys \
+        && chown -R test:test /home/test \
+        && chown -R test:test /home/test/.ssh
+
+
+ARG mscpdir="/mscp"
+
+COPY . ${mscpdir}
+
+# build
+RUN cd ${mscpdir}			\
+	&& rm -rf build			\
+	&& cmake -B build		\
+	&& cd ${mscpdir}/build		\
+	&& make	-j 2			\
+	&& make install
+

Dockerfile/ubuntu-22.04.Dockerfile

@@ -0,0 +1,37 @@
+FROM ubuntu:22.04
+
+ARG REQUIREDPKGS
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN set -ex && apt-get update && apt-get install -y --no-install-recommends \
+	${REQUIREDPKGS} ca-certificates python3 python3-pip python3-dev openssh-server
+
+RUN python3 -m pip install pytest
+
+# preparation for sshd
+RUN mkdir /var/run/sshd        \
+	&& ssh-keygen -A	\
+	&& ssh-keygen -f /root/.ssh/id_rsa -N ""                \
+	&& cat /root/.ssh/id_rsa.pub > /root/.ssh/authorized_keys
+
+# create test user
+RUN useradd -m -d /home/test test	\
+	&& echo "test:userpassword" | chpasswd \
+	&& mkdir -p /home/test/.ssh	\
+	&& ssh-keygen -f /home/test/.ssh/id_rsa_test -N "keypassphrase"	\
+	&& cat /home/test/.ssh/id_rsa_test.pub >> /home/test/.ssh/authorized_keys \
+	&& chown -R test:test /home/test \
+	&& chown -R test:test /home/test/.ssh
+
+
+ARG mscpdir="/mscp"
+
+COPY . ${mscpdir}
+
+# build
+RUN cd ${mscpdir}			\
+	&& rm -rf build			\
+	&& cmake -B build		\
+	&& cd ${mscpdir}/build		\
+	&& make	-j 2			\
+	&& make install

Dockerfile/ubuntu-24.04.Dockerfile

@@ -0,0 +1,37 @@
+FROM ubuntu:24.04
+
+ARG REQUIREDPKGS
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN set -ex && apt-get update && apt-get install -y --no-install-recommends \
+	${REQUIREDPKGS} ca-certificates openssh-server vim-tiny \
+	python3 python3-pip python3-dev python3-pytest
+
+
+# preparation for sshd
+RUN mkdir /var/run/sshd        \
+	&& ssh-keygen -A	\
+	&& ssh-keygen -f /root/.ssh/id_rsa -N ""                \
+	&& cat /root/.ssh/id_rsa.pub > /root/.ssh/authorized_keys
+
+# create test user
+RUN useradd -m -d /home/test test	\
+	&& echo "test:userpassword" | chpasswd \
+	&& mkdir -p /home/test/.ssh	\
+	&& ssh-keygen -f /home/test/.ssh/id_rsa_test -N "keypassphrase"	\
+	&& cat /home/test/.ssh/id_rsa_test.pub >> /home/test/.ssh/authorized_keys \
+	&& chown -R test:test /home/test \
+	&& chown -R test:test /home/test/.ssh
+
+
+ARG mscpdir="/mscp"
+
+COPY . ${mscpdir}
+
+# build
+RUN cd ${mscpdir}			\
+	&& rm -rf build			\
+	&& cmake -B build		\
+	&& cd ${mscpdir}/build		\
+	&& make	-j 2			\
+	&& make install

Doxyfile

@@ -0,0 +1,13 @@
+
+PROJECT_NAME	= libmscp
+
+GENERATE_HTML	= YES
+GENERATE_CHI	= NO
+GENERATE_LATEX	= NO
+GENERATE_RTF	= NO
+GENERATE_MAN	= NO
+
+SOURCE_BROWSER	= YES
+
+INPUT	= src include
+

README.md

@@ -1,69 +1,96 @@
-# mscp
+# mscp: multi-threaded scp
 
-[![build on ubuntu](https://github.com/upa/mscp/actions/workflows/build-ubuntu.yml/badge.svg)](https://github.com/upa/mscp/actions/workflows/build-ubuntu.yml) [![build on macOS](https://github.com/upa/mscp/actions/workflows/build-macos.yml/badge.svg)](https://github.com/upa/mscp/actions/workflows/build-macos.yml) [![test](https://github.com/upa/mscp/actions/workflows/test.yml/badge.svg)](https://github.com/upa/mscp/actions/workflows/test.yml)
+[![build on ubuntu](https://github.com/upa/mscp/actions/workflows/build-ubuntu.yml/badge.svg)](https://github.com/upa/mscp/actions/workflows/build-ubuntu.yml)
+[![build on macOS](https://github.com/upa/mscp/actions/workflows/build-macos.yml/badge.svg)](https://github.com/upa/mscp/actions/workflows/build-macos.yml)
+[![build on FreeBSD](https://github.com/upa/mscp/actions/workflows/build-freebsd.yml/badge.svg)](https://github.com/upa/mscp/actions/workflows/build-freebsd.yml)
+[![test](https://github.com/upa/mscp/actions/workflows/test.yml/badge.svg)](https://github.com/upa/mscp/actions/workflows/test.yml)
 
 
-`mscp`, a variant of `scp`, copies files over multiple ssh (SFTP)
-connections. Multiple threads and connections in mscp transfer (1)
-multiple files simultaneously and (2) a large file in parallel. It
-would shorten the waiting time for transferring a lot of/large files
-over networks.
 
-You can use `mscp` like `scp`, for example, `mscp
-user@example.com:srcfile /tmp/dstfile`. Remote hosts only need to run
-standard `sshd` supporting the SFTP subsystem, and you need to be able
-to ssh to the hosts (as usual). `mscp` does not require anything else.
+`mscp`, a variant of `scp`, copies files over multiple SSH (SFTP)
+connections by multiple threads. It enables transferring (1) multiple
+files simultaneously and (2) a large file in parallel, reducing the
+transfer time for a lot of/large files over networks.
+
+You can use `mscp` like `scp`, for example:
+
+```shell-session
+$ mscp srcfile user@example.com:dstfile
+```
+
+Remote hosts only need to run standard `sshd` supporting the SFTP
+subsystem (e.g. openssh-server), and you need to be able to ssh to the
+hosts as usual. `mscp` does not require anything else.
 
 
-Differences from `scp`:
+https://github.com/upa/mscp/assets/184632/19230f57-be7f-4ef0-98dd-cb4c460f570d
 
-- remote glob on remote shell expansion is not supported.
-- remote to remote copy is not supported.
-- `-r` option is not needed.
-- and any other differences I have not implemented and noticed...
+--------------------------------------------------------------------
 
+Major differences from `scp` on usage:
+
+- Remote-to-remote copy is not supported.
+- `-r` option is not needed to transfer directories.
+- Checkpointing for resuming failed transfer is supported.
+- and any other differences I have not implemented and noticed.
+
+Paper:
+- Ryo Nakamura and Yohei Kuga. 2023. Multi-threaded scp: Easy and Fast File Transfer over SSH. In Practice and Experience in Advanced Research Computing (PEARC '23). Association for Computing Machinery, New York, NY, USA, 320–323. https://doi.org/10.1145/3569951.3597582
 
 ## Install
 
-- homebrew
+- macOS
 
 ```console
+# Homebrew
 brew install upa/tap/mscp
+
+# MacPorts
+sudo port install mscp
 ```
 
-- Linux
+- Ubuntu
+```console
+sudo add-apt-repository ppa:upaa/mscp
+sudo apt-get install mscp
+```
 
-Download a package for your environment from [Releases
-page](https://github.com/upa/mscp/releases).
+- RHEL-based distributions
+```console
+sudo dnf copr enable upaaa/mscp
+sudo dnf install mscp
+```
 
 
 ## Build
 
-mscp depends on a patched [libssh](https://www.libssh.org/).  The
+mscp depends on a patched [libssh](https://www.libssh.org/). The
 patch introduces asynchronous SFTP Write, which is derived from
 https://github.com/limes-datentechnik-gmbh/libssh (see [Re: SFTP Write
 async](https://archive.libssh.org/libssh/2020-06/0000004.html)).
 
-Currently macOS and Linux (Ubuntu, CentOS, Rocky) are supported.
+We test building mscp on Linux (Ubuntu, Rocky, Alma, and Alpine),
+macOS, and FreeBSD.
+
 
 ```console
-# 1. clone this repository
+# clone this repository
 git clone https://github.com/upa/mscp.git
 cd mscp
 
-# 2. prepare patched libssh
+# prepare patched libssh
 git submodule update --init
-patch -d libssh -p1 < patch/libssh-0.10.4.patch
+patch -d libssh -p1 < patch/$(git -C libssh describe).patch
 
-# 3. install build dependency
+# install build dependency
 bash ./scripts/install-build-deps.sh
 
-# 4. configure mscp
-mkdir build && mv build
+# configure mscp
+mkdir build && cd build
 cmake ..
 
-## in macOS, you may need OPENSSL_ROOT_DIR for cmake:
-cmake .. -DOPENSSL_ROOT_DIR=$(brew --prefix)/opt/openssl@1.1
+# in macOS, you may need OPENSSL_ROOT_DIR for cmake:
+# cmake .. -DOPENSSL_ROOT_DIR=$(brew --prefix)/opt/openssl@3
 
 # build
 make
@@ -72,98 +99,11 @@ make
 make install
 ```
 
-## Run
-
-- Usage
-
-```console
-$ mscp
-mscp v0.0.2: copy files over multiple ssh connections
-
-Usage: mscp [vqDCHdh] [-n nr_conns] [-m coremask]
-            [-s min_chunk_sz] [-S max_chunk_sz] [-a nr_ahead]
-            [-l login_name] [-p port] [-i identity_file]
-            [-c cipher_spec] source ... target
-```
-
-- Example: copy an 8GB file on tmpfs over a 100Gbps link
-  - Two Intel Xeon Gold 6130 machines directly connected with Intel E810 100Gbps NICs.
-
-```console
-$ mscp /tmp/test.img 10.0.0.1:/tmp/
-[=====================================================] 100% 8GB/8GB 3.02GB/s 
-```
-
-- `-v` option increments verbose output level.
-
-```console
-$ mscp test 10.0.0.1:
-[=====================================================] 100% 13B/13B 2.41KB/s 
-
-$ mscp -v test 10.0.0.1:
-file test/test.txt (local) -> ./test/test.txt (remote) 9B
-file test/test2/2.txt (local) -> ./test/test2/2.txt (remote) 2B
-file test/1.txt (local) -> ./test/1.txt (remote) 2B
-copy start: test/test.txt
-copy start: test/1.txt
-copy start: test/test2/2.txt
-copy done: test/1.txt
-copy done: test/test2/2.txt
-copy done: test/test.txt
-[=====================================================] 100% 13B/13B 2.51KB/s 
-
-$ mscp -vv -n 4 test 10.0.0.1:
-connecting to 10.0.0.1 for checking destinations...
-file test/test.txt (local) -> ./test/test.txt (remote) 9B
-file test/test2/2.txt (local) -> ./test/test2/2.txt (remote) 2B
-file test/1.txt (local) -> ./test/1.txt (remote) 2B
-connecting to 10.0.0.1 for a copy thread...
-connecting to 10.0.0.1 for a copy thread...
-connecting to 10.0.0.1 for a copy thread...
-connecting to 10.0.0.1 for a copy thread...
-copy start: test/test.txt
-copy start: test/1.txt
-copy start: test/test2/2.txt
-copy done: test/test.txt
-copy done: test/test2/2.txt
-copy done: test/1.txt
-[=====================================================] 100% 13B/13B 3.27KB/s
-```
-
-- Full usage
-
-```console
-$ mscp -h
-mscp v0.0.2: copy files over multiple ssh connections
-
-Usage: mscp [vqDCHdh] [-n nr_conns] [-m coremask]
-            [-s min_chunk_sz] [-S max_chunk_sz] [-a nr_ahead]
-            [-l login_name] [-p port] [-i identity_file]
-            [-c cipher_spec] source ... target
-
-    -n NR_CONNECTIONS  number of connections (default: half of # of cpu cores)
-    -m COREMASK        hex value to specify cores where threads pinned
-    -s MIN_CHUNK_SIZE  min chunk size (default: 64MB)
-    -S MAX_CHUNK_SIZE  max chunk size (default: filesize / nr_conn)
-
-    -a NR_AHEAD        number of inflight SFTP commands (default: 16)
-
-    -v                 increment verbose output level
-    -q                 disable output
-    -D                 dry run
-
-    -l LOGIN_NAME      login name
-    -p PORT            port number
-    -i IDENTITY        identity file for public key authentication
-    -c CIPHER          cipher spec, see `ssh -Q cipher`
-    -C                 enable compression on libssh
-    -H                 disable hostkey check
-    -d                 increment ssh debug output level
-    -h                 print this help
-```
+Source tar balls (`mscp-X.X.X.tar.gz`, not `Source code`) in
+[Releases page](https://github.com/upa/mscp/releases) contain the patched version
+of libssh. So you can start from cmake with it.
 
 
+## Documentation
 
-
-Note: mscp is still under development, and the author is not
-responsible for any accidents due to mscp.
+[manpage](/doc/mscp.rst) is available.

VERSION

@@ -1 +1 @@
-0.0.2
+0.2.4

cmake/modules/FindGSSAPI.cmake

@@ -1,325 +0,0 @@
-# - Try to find GSSAPI
-# Once done this will define
-#
-#  KRB5_CONFIG - Path to krb5-config
-#  GSSAPI_ROOT_DIR - Set this variable to the root installation of GSSAPI
-#
-# Read-Only variables:
-#  GSSAPI_FLAVOR_MIT - set to TURE if MIT Kerberos has been found
-#  GSSAPI_FLAVOR_HEIMDAL - set to TRUE if Heimdal Keberos has been found
-#  GSSAPI_FOUND - system has GSSAPI
-#  GSSAPI_INCLUDE_DIR - the GSSAPI include directory
-#  GSSAPI_LIBRARIES - Link these to use GSSAPI
-#  GSSAPI_DEFINITIONS - Compiler switches required for using GSSAPI
-#
-#=============================================================================
-#  Copyright (c) 2013 Andreas Schneider <asn@cryptomilk.org>
-#
-#  Distributed under the OSI-approved BSD License (the "License");
-#  see accompanying file Copyright.txt for details.
-#
-#  This software is distributed WITHOUT ANY WARRANTY; without even the
-#  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-#  See the License for more information.
-#=============================================================================
-#
-
-find_path(GSSAPI_ROOT_DIR
-    NAMES
-        include/gssapi.h
-        include/gssapi/gssapi.h
-    HINTS
-        ${_GSSAPI_ROOT_HINTS}
-    PATHS
-        ${_GSSAPI_ROOT_PATHS}
-)
-mark_as_advanced(GSSAPI_ROOT_DIR)
-
-if (UNIX)
-    find_program(KRB5_CONFIG
-        NAMES
-            krb5-config
-        PATHS
-            ${GSSAPI_ROOT_DIR}/bin
-            /opt/local/bin)
-    mark_as_advanced(KRB5_CONFIG)
-
-    if (KRB5_CONFIG)
-        # Check if we have MIT KRB5
-        execute_process(
-            COMMAND
-                ${KRB5_CONFIG} --vendor
-            RESULT_VARIABLE
-                _GSSAPI_VENDOR_RESULT
-            OUTPUT_VARIABLE
-                _GSSAPI_VENDOR_STRING)
-
-        if ((_GSSAPI_VENDOR_STRING MATCHES ".*Massachusetts.*") OR (_GSSAPI_VENDOR_STRING
-                        MATCHES ".*MITKerberosShim.*"))
-            set(GSSAPI_FLAVOR_MIT TRUE)
-        else()
-            execute_process(
-                COMMAND
-                    ${KRB5_CONFIG} --libs gssapi
-                RESULT_VARIABLE
-                    _GSSAPI_LIBS_RESULT
-                OUTPUT_VARIABLE
-                    _GSSAPI_LIBS_STRING)
-
-            if (_GSSAPI_LIBS_STRING MATCHES ".*roken.*")
-                set(GSSAPI_FLAVOR_HEIMDAL TRUE)
-            endif()
-        endif()
-
-        # Get the include dir
-        execute_process(
-            COMMAND
-                ${KRB5_CONFIG} --cflags gssapi
-            RESULT_VARIABLE
-                _GSSAPI_INCLUDE_RESULT
-            OUTPUT_VARIABLE
-                _GSSAPI_INCLUDE_STRING)
-        string(REGEX REPLACE "(\r?\n)+$" "" _GSSAPI_INCLUDE_STRING "${_GSSAPI_INCLUDE_STRING}")
-        string(REGEX REPLACE " *-I" "" _GSSAPI_INCLUDEDIR "${_GSSAPI_INCLUDE_STRING}")
-    endif()
-
-    if (NOT GSSAPI_FLAVOR_MIT AND NOT GSSAPI_FLAVOR_HEIMDAL)
-        # Check for HEIMDAL
-        find_package(PkgConfig)
-        if (PKG_CONFIG_FOUND)
-            pkg_check_modules(_GSSAPI heimdal-gssapi)
-        endif (PKG_CONFIG_FOUND)
-
-        if (_GSSAPI_FOUND)
-            set(GSSAPI_FLAVOR_HEIMDAL TRUE)
-        else()
-            find_path(_GSSAPI_ROKEN
-                NAMES
-                    roken.h
-                PATHS
-                    ${GSSAPI_ROOT_DIR}/include
-                    ${_GSSAPI_INCLUDEDIR})
-            if (_GSSAPI_ROKEN)
-                set(GSSAPI_FLAVOR_HEIMDAL TRUE)
-            endif()
-        endif ()
-    endif()
-endif (UNIX)
-
-find_path(GSSAPI_INCLUDE_DIR
-    NAMES
-        gssapi.h
-        gssapi/gssapi.h
-    PATHS
-        ${GSSAPI_ROOT_DIR}/include
-        ${_GSSAPI_INCLUDEDIR}
-)
-
-if (GSSAPI_FLAVOR_MIT)
-    find_library(GSSAPI_LIBRARY
-        NAMES
-            gssapi_krb5
-        PATHS
-            ${GSSAPI_ROOT_DIR}/lib
-            ${_GSSAPI_LIBDIR}
-    )
-
-    find_library(KRB5_LIBRARY
-        NAMES
-            krb5
-        PATHS
-            ${GSSAPI_ROOT_DIR}/lib
-            ${_GSSAPI_LIBDIR}
-    )
-
-    find_library(K5CRYPTO_LIBRARY
-        NAMES
-            k5crypto
-        PATHS
-            ${GSSAPI_ROOT_DIR}/lib
-            ${_GSSAPI_LIBDIR}
-    )
-
-    find_library(COM_ERR_LIBRARY
-        NAMES
-            com_err
-        PATHS
-            ${GSSAPI_ROOT_DIR}/lib
-            ${_GSSAPI_LIBDIR}
-    )
-
-    if (GSSAPI_LIBRARY)
-        set(GSSAPI_LIBRARIES
-            ${GSSAPI_LIBRARIES}
-            ${GSSAPI_LIBRARY}
-        )
-    endif (GSSAPI_LIBRARY)
-
-    if (KRB5_LIBRARY)
-        set(GSSAPI_LIBRARIES
-            ${GSSAPI_LIBRARIES}
-            ${KRB5_LIBRARY}
-        )
-    endif (KRB5_LIBRARY)
-
-    if (K5CRYPTO_LIBRARY)
-        set(GSSAPI_LIBRARIES
-            ${GSSAPI_LIBRARIES}
-            ${K5CRYPTO_LIBRARY}
-        )
-    endif (K5CRYPTO_LIBRARY)
-
-    if (COM_ERR_LIBRARY)
-        set(GSSAPI_LIBRARIES
-            ${GSSAPI_LIBRARIES}
-            ${COM_ERR_LIBRARY}
-        )
-    endif (COM_ERR_LIBRARY)
-endif (GSSAPI_FLAVOR_MIT)
-
-if (GSSAPI_FLAVOR_HEIMDAL)
-    find_library(GSSAPI_LIBRARY
-        NAMES
-            gssapi
-        PATHS
-            ${GSSAPI_ROOT_DIR}/lib
-            ${_GSSAPI_LIBDIR}
-    )
-
-    find_library(KRB5_LIBRARY
-        NAMES
-            krb5
-        PATHS
-            ${GSSAPI_ROOT_DIR}/lib
-            ${_GSSAPI_LIBDIR}
-    )
-
-    find_library(HCRYPTO_LIBRARY
-        NAMES
-            hcrypto
-        PATHS
-            ${GSSAPI_ROOT_DIR}/lib
-            ${_GSSAPI_LIBDIR}
-    )
-
-    find_library(COM_ERR_LIBRARY
-        NAMES
-            com_err
-        PATHS
-            ${GSSAPI_ROOT_DIR}/lib
-            ${_GSSAPI_LIBDIR}
-    )
-
-    find_library(HEIMNTLM_LIBRARY
-        NAMES
-            heimntlm
-        PATHS
-            ${GSSAPI_ROOT_DIR}/lib
-            ${_GSSAPI_LIBDIR}
-    )
-
-    find_library(HX509_LIBRARY
-        NAMES
-            hx509
-        PATHS
-            ${GSSAPI_ROOT_DIR}/lib
-            ${_GSSAPI_LIBDIR}
-    )
-
-    find_library(ASN1_LIBRARY
-        NAMES
-            asn1
-        PATHS
-            ${GSSAPI_ROOT_DIR}/lib
-            ${_GSSAPI_LIBDIR}
-    )
-
-    find_library(WIND_LIBRARY
-        NAMES
-            wind
-        PATHS
-            ${GSSAPI_ROOT_DIR}/lib
-            ${_GSSAPI_LIBDIR}
-    )
-
-    find_library(ROKEN_LIBRARY
-        NAMES
-            roken
-        PATHS
-            ${GSSAPI_ROOT_DIR}/lib
-            ${_GSSAPI_LIBDIR}
-    )
-
-    if (GSSAPI_LIBRARY)
-        set(GSSAPI_LIBRARIES
-            ${GSSAPI_LIBRARIES}
-            ${GSSAPI_LIBRARY}
-        )
-    endif (GSSAPI_LIBRARY)
-
-    if (KRB5_LIBRARY)
-        set(GSSAPI_LIBRARIES
-            ${GSSAPI_LIBRARIES}
-            ${KRB5_LIBRARY}
-        )
-    endif (KRB5_LIBRARY)
-
-    if (HCRYPTO_LIBRARY)
-        set(GSSAPI_LIBRARIES
-            ${GSSAPI_LIBRARIES}
-            ${HCRYPTO_LIBRARY}
-        )
-    endif (HCRYPTO_LIBRARY)
-
-    if (COM_ERR_LIBRARY)
-        set(GSSAPI_LIBRARIES
-            ${GSSAPI_LIBRARIES}
-            ${COM_ERR_LIBRARY}
-        )
-    endif (COM_ERR_LIBRARY)
-
-    if (HEIMNTLM_LIBRARY)
-        set(GSSAPI_LIBRARIES
-            ${GSSAPI_LIBRARIES}
-            ${HEIMNTLM_LIBRARY}
-        )
-    endif (HEIMNTLM_LIBRARY)
-
-    if (HX509_LIBRARY)
-        set(GSSAPI_LIBRARIES
-            ${GSSAPI_LIBRARIES}
-            ${HX509_LIBRARY}
-        )
-    endif (HX509_LIBRARY)
-
-    if (ASN1_LIBRARY)
-        set(GSSAPI_LIBRARIES
-            ${GSSAPI_LIBRARIES}
-            ${ASN1_LIBRARY}
-        )
-    endif (ASN1_LIBRARY)
-
-    if (WIND_LIBRARY)
-        set(GSSAPI_LIBRARIES
-            ${GSSAPI_LIBRARIES}
-            ${WIND_LIBRARY}
-        )
-    endif (WIND_LIBRARY)
-
-    if (ROKEN_LIBRARY)
-        set(GSSAPI_LIBRARIES
-            ${GSSAPI_LIBRARIES}
-            ${WIND_LIBRARY}
-        )
-    endif (ROKEN_LIBRARY)
-endif (GSSAPI_FLAVOR_HEIMDAL)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(GSSAPI DEFAULT_MSG GSSAPI_LIBRARIES GSSAPI_INCLUDE_DIR)
-
-if (GSSAPI_INCLUDE_DIRS AND GSSAPI_LIBRARIES)
-    set(GSSAPI_FOUND TRUE)
-endif (GSSAPI_INCLUDE_DIRS AND GSSAPI_LIBRARIES)
-
-# show the GSSAPI_INCLUDE_DIRS and GSSAPI_LIBRARIES variables only in the advanced view
-mark_as_advanced(GSSAPI_INCLUDE_DIRS GSSAPI_LIBRARIES)

conanfile.txt

@@ -0,0 +1,7 @@
+[requires]
+zlib/1.2.11
+openssl/1.1.1t
+
+[generators]
+CMakeDeps
+CMakeToolchain

debian/.gitignore

@@ -0,0 +1,11 @@
+
+/.debhelper/
+/*/
+!/tests/
+!/source
+
+/*.log
+/*.substvars
+/*.debhelper
+/files
+/mscp

debian/changelog

@@ -0,0 +1,97 @@
+mscp (0.2.4) UNRELEASED; urgency=medium
+
+  * fix password auth does not work (issue #36)
+  * add Debian and Devuan to the install dep script (pr #38)
+  * fallback to default terminal size on no tty (pr #39)
+
+ -- Ryo Nakamura <upa@haeena.net>  Sat, 08 Nov 2025 19:23:07 +0900
+
+mscp (0.2.3) unstable; urgency=medium
+
+  * bump libssh version to 0.11.2 (#25)
+  * adopt new AIO read API of libssh
+  * fix path handling when remote dst path has suffix '/' (#24)
+  * fix remote path handling including '~' (partially)
+  * try pubkey auth first to avoid PerSourcePenalties
+  * remove the single-binary version of mscp from release
+
+ -- Ryo Nakamura <upa@haeena.net>  Tue, 12 Aug 2025 18:11:47 +0900
+
+mscp (0.2.2) unstable; urgency=medium
+
+  * bump cmake_minimum_version on libssh (#32)
+  * fix quiet mode (#30)
+  * use openssl@3 on macOS (#29)
+  * add archlinux support (#28)
+
+ -- Ryo Nakamura <upa@haeena.net>  Wed, 16 Apr 2025 17:01:17 +0900
+
+mscp (0.2.1) unstable; urgency=medium
+
+  * fix broken checkpoint files when copying multiple files (#16)
+  * fix broken password inputs for resume transfers (#17 and #18)
+  * add support [kKmMgG] units for -s, -S, and -b options (#20)
+  * change the default min chunk size to 16MB (#20)
+  * change the default max chunk size to filesize / (nr_conn * 4) (#20)
+  * -s and -S accept integers other than multiples of page sizes (#20)
+  * help now shows available ciphers and HMACs (#20)
+
+ -- Ryo Nakamura <upa@haeena.net>  Sat, 11 May 2024 14:49:52 +0900
+
+mscp (0.2.0) unstable; urgency=medium
+
+  * add -J DESTINATION option for ProxyJump (#15)
+  * add -o SSH_OPTION option
+  * add -L LIMIT_BITRATE option (#14)
+  * add keyboard interactive authentication support. Also, we have manually
+    tested that mscp works with cisco DUO MFA (#2)
+  * remove -H disable host key check option (ssh_config or -o option can do
+    this instead)
+  * fix copying files of permission r--r--r--
+  * update github actions: checkout v3 to v4, and codeql from v2 to v3
+  * add ubuntu 24.04 test
+
+ -- Ryo Nakamura <upa@haeena.net>  Mon, 15 Apr 2024 00:05:20 +0900
+
+mscp (0.1.5) unstable; urgency=medium
+
+  * add support for resuming failed transfer (#5 and #10)
+  * remove the list structure derived from the linux kernel and refactoring
+    for this change.
+  * add and fix test cases (changing port number and number of connections)
+
+ -- Ryo Nakamura <upa@haeena.net>  Thu, 14 Mar 2024 12:51:23 +0900
+
+mscp (0.1.4) unstable; urgency=medium
+
+  * add a test for builds on FreeBSD
+  * updat container images for tests
+  * dropp support for Python bindings
+  * clean up error message handling
+  * add MSCP_SSH_AUTH_PASSWORD/PASSPHRASE environment variables
+    to pass passwords to MSCP without interactive input (issue #9)
+  * add a -p option to preserve timestamps
+  * add -4 and -6 options to use IPv4 or IPv6, respectively
+  * introduc .clang-format, which is derived from the Linux kernel
+  * fix the manpage
+
+ -- Ryo Nakamura <upa@haeena.net>  Wed, 07 Feb 2024 15:56:58 +0900
+
+mscp (0.1.3) unstable; urgency=medium
+
+  * add -I option for inserting intervals between SSH attempts (issue #7)
+  * add -P option, equivalent to -p (just for compatibility)
+  * update libssh to 0.10.6-2-g6f1b1e76 for security fixes
+  * cleanup warning messages when scanning source files fails
+  * fix wrong destination paths for source paths under '/' (issue #8)
+  * reduce memory footprint for coping many (e.g., over 100k) files (issue #8)
+  * add SPDX-License-Identifer to the source files
+  * add manpage
+
+ -- Ryo Nakamura <upa@haeena.net>  Fri, 12 Jan 2024 22:20:24 +0900
+
+mscp (0.1.2) unstable; urgency=medium
+
+  * Initial release for debian packaging
+
+ -- Ryo Nakamura <upa@haeena.net>  Sun, 10 Dec 2023 21:51:49 +0900

debian/compat

@@ -0,0 +1 @@
+10

debian/control

@@ -0,0 +1,17 @@
+Source: mscp
+Section: net
+Priority: optional
+Maintainer: Ryo Nakamura <upa@haeena.net>
+Build-Depends: debhelper (>= 10~), cmake, zlib1g-dev, libssl-dev, libkrb5-dev
+Homepage: https://github.com/upa/mscp
+Standards-Version: 4.5.0.3
+
+
+Package: mscp
+Architecture: linux-any
+Depends: ${misc:Depends}, ${shlibs:Depends}
+Description: mscp, fast file transfer over multiple SSH connections
+ mscp transfers files over multiple SSH connections. Multiple threads
+ and connections in mscp transfer (1) multiple files simultaneously
+ and (2) a large file in parallel. It would shorten the waiting time
+ for transferring a lot of/large files over networks.

debian/copyright

@@ -0,0 +1,12 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Source: https://github.com/upa/mscp
+Upstream-Name: mscp
+Upstream-Contact: Ryo Nakamura <upa@haeena.net>
+
+Files: *
+Copyright: Ryo Nakamura
+License: GPL-3
+
+Files: src/list.h
+Copyright: kazutomo@mcs.anl.gov
+License: GPL

debian/rules

@@ -0,0 +1,9 @@
+#!/usr/bin/make -f
+
+%:
+	dh $@
+
+
+override_dh_auto_test:
+
+

debian/source/format

@@ -0,0 +1 @@
+3.0 (native)

doc/RELEASE.md

@@ -0,0 +1,74 @@
+
+
+## Build mscp as deb package
+
+`make build-deb` produces a mscp deb package and related files. This
+target builds mscp with `debuild` inside a docker container
+(Dockerfile is `docker/build-deb.Docerfile`).
+
+
+```console
+mkdir build && cd build && cmake ..
+make build-deb
+```
+
+After that:
+
+```console
+$ ls debbuild
+mscp_0.1.4.dsc		 mscp_0.1.4_source.buildinfo  mscp_0.1.4.tar.xz
+mscp_0.1.4_source.build  mscp_0.1.4_source.changes
+```
+
+### To publush mscp in launchpad PPA:
+
+1. write changes in `debian/changelog` at main branch (the date
+   command needed here is `date -R`)
+2. switch to `ppa-focal` or `ppa-jammy` branch
+3. rebase to the `main` branch and modify `debian/changes`:
+   * change `mscp (X.X.X) UNRELEASED;` to `mscp (X.X.X-1~RELEASENAME) RELEASENAME;`
+	 where `RELEASENAME` is `focal` or `jammy`.
+4. run `make build-deb` at the build directory and `cd debbuild`
+5. sign the files with `debsign -k [GPGKEYID] mscp_X.X.X~X_source.changes`
+5. upload the files with `dput ppa:upaa/mscp mscp_X.X.X~X_source.changes`
+
+
+## Build mscp as (source) rpm package
+
+`make build-srpm` produces a mscp src.rpm package. This target builts
+mscp with `rpmbuild` inside a docker container (Dockerfile is
+`docker/build-srpm.Dockerfile`, generated from
+`build-srpm.Dockerfile.in` by cmake).
+
+```console
+mkdir build && cd build && cmake ..
+make build-srpm
+```
+
+After that:
+
+```console
+$ ls *.rpm
+mscp-0.1.3-1.el9.src.rpm
+```
+
+### To publish mscp in COPR:
+
+1. update `rpm/mscp.spec.in`, the `changelog` section (the date
+   command needed here is `date "+%a %b %d %Y"`)
+2. run `make build-srpm`
+3. download `mscp-X.X.X-1.yyy.src.rpm`
+4. upload the src.rpm to Build page at COPR.
+
+
+
+## Update Document
+
+The docuemnt is `doc/mscp.rst` (at present). When `mscp.1.in` is
+modified, run `make update-rst` to make it up to date.
+
+```console
+mkdir build cd build && cmake ..
+make update-rst
+```
+

doc/mscp.1.in

@@ -0,0 +1,463 @@
+.TH MSCP 1 "@MSCP_BUILD_VERSION@" "mscp" "User Commands"
+
+.SH NAME
+mscp \- copy files over multiple SSH connections
+
+.SH SYNOPSIS
+
+.B mscp
+.RB [ \-46vqDpdNh ]
+[\c
+.BI \-n \ NR_CONNECTIONS\c
+]
+[\c
+.BI \-m \ COREMASK\c
+]
+[\c
+.BI \-u \ MAX_STARTUPS\c
+]
+[\c
+.BI \-I \ INTERVAL\c
+]
+[\c
+.BI \-W \ CHECKPOINT\c
+]
+[\c
+.BI \-R \ CHECKPOINT\c
+]
+[\c
+.BI \-s \ MIN_CHUNK_SIZE\c
+]
+[\c
+.BI \-S \ MAX_CHUNK_SIZE\c
+]
+[\c
+.BI \-a \ NR_AHEAD\c
+]
+[\c
+.BI \-b \ BUF_SIZE\c
+]
+[\c
+.BI \-L \ LIMIT_BITRATE\c
+]
+[\c
+.BI \-l \ LOGIN_NAME\c
+]
+[\c
+.BI \-P \ PORT\c
+]
+[\c
+.BI \-F \ SSH_CONFIG\c
+]
+[\c
+.BI \-o \ SSH_OPTION\c
+]
+[\c
+.BI \-i \ IDENTITY\c
+]
+[\c
+.BI \-J \ DESTINATION\c
+]
+[\c
+.BI \-c \ CIPHER\c
+]
+[\c
+.BI \-M \ HMAC\c
+]
+[\c
+.BI \-C \ COMPRESS\c
+]
+[\c
+.BI \-g \ CONGESTION\c
+]
+.I source ... target
+
+.SH DESCRIPTION
+
+.PP
+.B mscp
+copies files over multiple SSH (SFTP) connections by multiple
+threads. It enables transferring (1) multiple files simultaneously and
+(2) a large file in parallel, reducing the transfer time for a lot
+of/large files over networks.
+
+.PP
+The usage of
+.B mscp
+follows the
+.B scp
+command of
+.I OpenSSH,
+for example:
+
+.nf
+    $ mscp srcfile user@example.com:dstfile
+.fi
+
+Remote hosts only need to run standard
+.B sshd
+supporting the SFTP subsystem, and users need to be able to
+.B ssh
+to the hosts as usual.
+.B mscp
+does not require anything else.
+
+.PP
+.B mscp
+uses
+.UR https://\:www\:.libssh\:.org
+libssh
+.UE
+as its SSH implementation. Thus, supported SSH features, for example,
+authentication, encryption, and various options in ssh_config, follow
+what
+.I libssh
+supports.
+
+.SH OPTIONS
+.TP
+.B \-n \fINR_CONNECTIONS\fR
+Specifies the number of SSH connections. The default value is
+calculated from the number of CPU cores on the host with the following
+formula: floor(log(nr_cores)*2)+1.
+
+.TP
+.B \-m \fICOREMASK\fR
+Configures CPU cores to be used by the hexadecimal bitmask. For
+example, -m 0x25 pins threads onto CPU cores 0, 2, and 5. The default
+value is not specified: all CPU cores are used and no threads are
+pinned to any cores.
+
+.TP
+.B \-u \fIMAX_STARTUPS\fR
+Specifies the number of concurrent unauthenticated SSH connection
+attempts.
+.B sshd
+limits the number of simultaneous SSH connection attempts by
+.I MaxStartups
+in
+.I sshd_config.
+The default
+.I MaxStartups
+is 10; thus, we set the default MAX_STARTUPS 8.
+
+.TP
+.B \-I \fIINTERVAL\fR
+Specifies the interval (in seconds) between SSH connection
+attempts. Some firewall products treat SSH connection attempts from a
+single source IP address for a short period as a brute force attack.
+This option inserts intervals between the attempts to avoid being
+determined as an attack. The default value is 0.
+
+.TP
+.B \-W \fICHECKPOINT\fR
+Specifies a checkpoint file to save the state of a failed
+transfer. When transferring fails due to, for example, connection
+disruption or user interrupt,
+.B mscp
+writes the information about the remaining files and chunks to the
+specified checkpoint file.
+.B \-W
+option with
+.B \-D
+(dry-run mode) only writes a checkpoint file and exits.
+
+
+.TP
+.B \-R \fICHECKPOINT\fR
+Specifies a checkpoint file to resume a transfer. When a checkpoint
+file is passed,
+.B mscp
+reads the checkpoint to load a remote host, copy direction, and files
+and their chunks to be transferred. Namely,
+.B mscp
+can resume a past failed transfer from the checkpoint. Resuming with a
+checkpoint does not require
+.I source ... target
+arguments. Other SSH connection options, such as port number and
+config file, should be specified as with the failed run. In addition,
+checkpoint files have file paths as relative paths. Thus, you must run
+.B mscp
+in the same working directory as the failed run. You can see the
+contents of a checkpoint file with the
+.B mscp \-vv \-D \-R CHECKPOINT
+command (Dry-run mode).  Note that the checkpoint file is not
+automatically removed after the resumed transfer ends
+successfully. Users should check the return value of
+.B mscp
+and remove the checkpoint if it returns 0.
+
+
+.TP
+.B \-s \fIMIN_CHUNK_SIZE\fR
+Specifies the minimum chunk size.
+.B mscp
+divides a single file into chunks and copies the chunks in
+parallel. The default value is 16M bytes.
+
+.TP
+.B \-S \fIMAX_CHUNK_SIZE\fR
+Specifies the maximum chunk size. The default is file size divided by
+the number of connections and devided by 4. If the calculated value
+is smarller than the
+.B MIN_CHUNK_SIZE
+value,
+MIN_CHUNK_SIZE is used.
+
+.TP
+.B \-a \fINR_AHEAD\fR
+Specifies the number of inflight SFTP commands. The default value is
+32.
+
+.TP
+.B \-b \fIBUF_SIZE\fR
+Specifies the buffer size for I/O and transfer over SFTP. The default
+value is 16384. Note that the SSH specification restricts buffer size
+delivered over SSH. Changing this value is not recommended at present.
+
+.TP
+.B \-L \fILIMIT_BITRATE\fR
+Limits the bitrate, specified with k (K), m (M), and g (G), e.g., 100m
+indicates 100 Mbps.
+
+.TP
+.B \-4
+Uses IPv4 addresses only.
+
+.TP
+.B \-6
+Uses IPv6 addresses only.
+
+.TP
+.B \-v
+Increments the verbose output level.
+
+.TP
+.B \-q
+Quiet mode: turns off all outputs.
+
+.TP
+.B \-D
+Dry-run mode: it scans source files to be copied, calculates chunks,
+resolves destination file paths, and exits. Dry-run mode with
+.B -vv
+option can confirm files to be copied and their destination paths.
+
+.TP
+.B \-r
+No effect.
+.B mscp
+copies recursively if a source path is a directory. This option exists
+for just compatibility.
+
+.TP
+.B \-l \fILOGIN_NAME\fR
+Specifies the username to log in on the remote machine as with
+.I ssh(1).
+
+.TP
+.B \-P \fIPORT\fR
+Specifies the port number to connect to on the remote machine as with
+.I scp(1).
+
+.TP
+.B \-F \fISSH_CONFIG\fR
+Specifies an alternative per-user ssh configuration file. Note that
+acceptable options in the configuration file are what
+.I libssh
+supports.
+
+.TP
+.B \-o \fISSH_OPTION\fR
+Specifies ssh options in the format used in ssh_config. Note that
+acceptable options are what
+.I libssh
+supports.
+
+.TP
+.B \-i \fIIDENTITY\fR
+Specifies the identity file for public key authentication.
+
+.TP
+.B \-J \fIDESTINATION\fR
+A shortcut to define a
+.B ProxyJump
+configuration directive. Each SFTP session of
+.B mscp
+connects to the target host by first making an
+.B ssh
+connection to the jump host described by
+.I destination.
+
+
+
+.TP
+.B \-c \fICIPHER\fR
+Selects the cipher to use for encrypting the data transfer. See
+.B mscp -h
+or
+.B Ciphers
+in
+.UR https://\:www\:.libssh\:.org/\:features/
+libssh features
+.UE .
+
+.TP
+.B \-M \fIHMAC\fR
+Specifies MAC hash algorithms. See
+.B mscp -h
+or
+.B MAC hashes
+in
+.UR https://\:www\:.libssh\:.org/\:features/
+libssh features
+.UE .
+
+.TP
+.B \-C \fICOMPRESS\fR
+Enables compression: yes, no, zlib, zlib@openssh.com. The default is
+none. See
+.UR https://\:www\:.libssh\:.org/\:features/
+libssh features
+.UE .
+
+.TP
+.B \-g \fICONGESTION\fR
+Specifies the TCP congestion control algorithm to use (Linux only).
+See
+.B sysctl net.ipv4.tcp_allowed_congestion_control
+for available values.
+
+.TP
+.B \-p
+Preserves modification times and access times (file mode bits are
+preserved by default).
+
+.TP
+.B \-d
+Increments the ssh debug output level.
+
+.TP
+.B \-N
+Enables Nagle's algorithm. It is disabled by default.
+
+.TP
+.B \-h
+Prints help.
+
+.SH EXIT STATUS
+Exit status is 0 on success,  and >0 if an error occurs.
+
+.SH ENVIRONMENT
+
+.PP
+.B mscp
+recognizes the following environment variables.
+
+.TP
+.B MSCP_SSH_AUTH_PASSWORD
+This environment variable passes a password for password
+authentication to establish SSH connections.
+
+.TP
+.B MSCP_SSH_AUTH_PASSPHRASE
+This environment variable passes a passphrase for public-key
+authentication for establishing SSH connections.
+
+
+
+.SH NOTES
+
+.PP
+.B mscp
+uses glob(3) for globbing pathnames, including matching patterns for
+local and remote paths. However, globbing on the
+.I remote
+side does not work with musl libc (used in Alpine Linux and the
+single-binary version of mscp) because musl libc does not support
+GLOB_ALTDIRFUNC.
+
+.PP
+.B mscp
+does not support remote-to-remote copy, which
+.B scp
+supports.
+
+.SH EXAMPLES
+
+.PP
+Copy a local file to a remote host with different name:
+
+.nf
+    $ mscp ~/src-file 10.0.0.1:copied-file
+.fi
+
+.PP
+Copy a local file and a directory to /tmp at a remote host:
+
+.nf
+    $ mscp ~/src-file dir1 10.0.0.1:/tmp
+.fi
+
+.PP
+Save a checkpoint if transfer fails:
+
+.nf
+    $ mscp -W mscp.checkpoint many-large-files 10.0.0.1:dst/
+.fi
+
+.PP
+Check the remaining files and chunks, and resume the failed transfer:
+
+.nf
+    # Dump the content of a checkpoint and exit (dry-run mode)
+    $ mscp -vv -D -R mscp.checkpoint
+
+    # resume transferring from the checkpoint
+    $ mscp -R mscp.checkpoint
+.fi
+
+.PP
+In a long fat network, following options might improve performance:
+
+.nf
+    $ mscp -n 64 -m 0xffff -a 64 -c aes128-gcm@openssh.com src 10.0.0.1:
+.fi
+
+.B -n
+increases the number of SSH connections than default,
+.B -m
+pins threads to specific CPU cores,
+.B -a
+increases asynchronous inflight SFTP WRITE/READ commands, and
+.B -c aes128-gcm@openssh.com
+will be faster than the default chacha20-poly1305 cipher, particularly
+on hosts that support AES-NI.
+
+
+.SH "SEE ALSO"
+.BR scp (1),
+.BR ssh (1),
+.BR sshd (8).
+
+.SH "PAPER REFERENCE"
+
+
+Ryo Nakamura and Yohei Kuga. 2023. Multi-threaded scp: Easy and Fast
+File Transfer over SSH. In Practice and Experience in Advanced
+Research Computing (PEARC '23). Association for Computing Machinery,
+New York, NY, USA, 320–323.
+.UR https://\:doi\:.org/\:10.1145/\:3569951.3597582
+DOI
+.UE .
+
+
+.SH CONTACT INFORMATION
+.PP
+For patches, bug reports, or feature requests, please open an issue on
+.UR https://\:github\:.com/\:upa/\:mscp
+GitHub
+.UE .
+
+.SH AUTHORS
+Ryo Nakamura <upa@haeena.net>

doc/mscp.rst

@@ -0,0 +1,294 @@
+====
+MSCP
+====
+
+:Date: v0.2.4
+
+NAME
+====
+
+mscp - copy files over multiple SSH connections
+
+SYNOPSIS
+========
+
+**mscp** [**-46vqDpdNh**] [ **-n** *NR_CONNECTIONS* ] [ **-m**
+*COREMASK* ] [ **-u** *MAX_STARTUPS* ] [ **-I** *INTERVAL* ] [ **-W**
+*CHECKPOINT* ] [ **-R** *CHECKPOINT* ] [ **-s** *MIN_CHUNK_SIZE* ] [
+**-S** *MAX_CHUNK_SIZE* ] [ **-a** *NR_AHEAD* ] [ **-b** *BUF_SIZE* ] [
+**-L** *LIMIT_BITRATE* ] [ **-l** *LOGIN_NAME* ] [ **-P** *PORT* ] [
+**-F** *SSH_CONFIG* ] [ **-o** *SSH_OPTION* ] [ **-i** *IDENTITY* ] [
+**-J** *DESTINATION* ] [ **-c** *CIPHER* ] [ **-M** *HMAC* ] [ **-C**
+*COMPRESS* ] [ **-g** *CONGESTION* ] *source ... target*
+
+DESCRIPTION
+===========
+
+**mscp** copies files over multiple SSH (SFTP) connections by multiple
+threads. It enables transferring (1) multiple files simultaneously and
+(2) a large file in parallel, reducing the transfer time for a lot
+of/large files over networks.
+
+The usage of **mscp** follows the **scp** command of *OpenSSH,* for
+example:
+
+::
+
+       $ mscp srcfile user@example.com:dstfile
+
+Remote hosts only need to run standard **sshd** supporting the SFTP
+subsystem, and users need to be able to **ssh** to the hosts as usual.
+**mscp** does not require anything else.
+
+**mscp** uses `libssh <https://www.libssh.org>`__ as its SSH
+implementation. Thus, supported SSH features, for example,
+authentication, encryption, and various options in ssh_config, follow
+what *libssh* supports.
+
+OPTIONS
+=======
+
+**-n NR_CONNECTIONS**
+   Specifies the number of SSH connections. The default value is
+   calculated from the number of CPU cores on the host with the
+   following formula: floor(log(nr_cores)*2)+1.
+
+**-m COREMASK**
+   Configures CPU cores to be used by the hexadecimal bitmask. For
+   example, -m 0x25 pins threads onto CPU cores 0, 2, and 5. The default
+   value is not specified: all CPU cores are used and no threads are
+   pinned to any cores.
+
+**-u MAX_STARTUPS**
+   Specifies the number of concurrent unauthenticated SSH connection
+   attempts. **sshd** limits the number of simultaneous SSH connection
+   attempts by *MaxStartups* in *sshd_config.* The default *MaxStartups*
+   is 10; thus, we set the default MAX_STARTUPS 8.
+
+**-I INTERVAL**
+   Specifies the interval (in seconds) between SSH connection attempts.
+   Some firewall products treat SSH connection attempts from a single
+   source IP address for a short period as a brute force attack. This
+   option inserts intervals between the attempts to avoid being
+   determined as an attack. The default value is 0.
+
+**-W CHECKPOINT**
+   Specifies a checkpoint file to save the state of a failed transfer.
+   When transferring fails due to, for example, connection disruption or
+   user interrupt, **mscp** writes the information about the remaining
+   files and chunks to the specified checkpoint file. **-W** option with
+   **-D** (dry-run mode) only writes a checkpoint file and exits.
+
+**-R CHECKPOINT**
+   Specifies a checkpoint file to resume a transfer. When a checkpoint
+   file is passed, **mscp** reads the checkpoint to load a remote host,
+   copy direction, and files and their chunks to be transferred. Namely,
+   **mscp** can resume a past failed transfer from the checkpoint.
+   Resuming with a checkpoint does not require *source ... target*
+   arguments. Other SSH connection options, such as port number and
+   config file, should be specified as with the failed run. In addition,
+   checkpoint files have file paths as relative paths. Thus, you must
+   run **mscp** in the same working directory as the failed run. You can
+   see the contents of a checkpoint file with the **mscp -vv -D -R
+   CHECKPOINT** command (Dry-run mode). Note that the checkpoint file is
+   not automatically removed after the resumed transfer ends
+   successfully. Users should check the return value of **mscp** and
+   remove the checkpoint if it returns 0.
+
+**-s MIN_CHUNK_SIZE**
+   Specifies the minimum chunk size. **mscp** divides a single file into
+   chunks and copies the chunks in parallel. The default value is 16M
+   bytes.
+
+**-S MAX_CHUNK_SIZE**
+   Specifies the maximum chunk size. The default is file size divided by
+   the number of connections and devided by 4. If the calculated value
+   is smarller than the **MIN_CHUNK_SIZE** value, MIN_CHUNK_SIZE is
+   used.
+
+**-a NR_AHEAD**
+   Specifies the number of inflight SFTP commands. The default value is
+   32.
+
+**-b BUF_SIZE**
+   Specifies the buffer size for I/O and transfer over SFTP. The default
+   value is 16384. Note that the SSH specification restricts buffer size
+   delivered over SSH. Changing this value is not recommended at
+   present.
+
+**-L LIMIT_BITRATE**
+   Limits the bitrate, specified with k (K), m (M), and g (G), e.g.,
+   100m indicates 100 Mbps.
+
+**-4**
+   Uses IPv4 addresses only.
+
+**-6**
+   Uses IPv6 addresses only.
+
+**-v**
+   Increments the verbose output level.
+
+**-q**
+   Quiet mode: turns off all outputs.
+
+**-D**
+   Dry-run mode: it scans source files to be copied, calculates chunks,
+   resolves destination file paths, and exits. Dry-run mode with **-vv**
+   option can confirm files to be copied and their destination paths.
+
+**-r**
+   No effect. **mscp** copies recursively if a source path is a
+   directory. This option exists for just compatibility.
+
+**-l LOGIN_NAME**
+   Specifies the username to log in on the remote machine as with
+   *ssh(1).*
+
+**-P PORT**
+   Specifies the port number to connect to on the remote machine as with
+   *scp(1).*
+
+**-F SSH_CONFIG**
+   Specifies an alternative per-user ssh configuration file. Note that
+   acceptable options in the configuration file are what *libssh*
+   supports.
+
+**-o SSH_OPTION**
+   Specifies ssh options in the format used in ssh_config. Note that
+   acceptable options are what *libssh* supports.
+
+**-i IDENTITY**
+   Specifies the identity file for public key authentication.
+
+**-J DESTINATION**
+   A shortcut to define a **ProxyJump** configuration directive. Each
+   SFTP session of **mscp** connects to the target host by first making
+   an **ssh** connection to the jump host described by *destination.*
+
+**-c CIPHER**
+   Selects the cipher to use for encrypting the data transfer. See
+   **mscp -h** or **Ciphers** in `libssh
+   features <https://www.libssh.org/features/>`__.
+
+**-M HMAC**
+   Specifies MAC hash algorithms. See **mscp -h** or **MAC hashes** in
+   `libssh features <https://www.libssh.org/features/>`__.
+
+**-C COMPRESS**
+   Enables compression: yes, no, zlib, zlib@openssh.com. The default is
+   none. See `libssh features <https://www.libssh.org/features/>`__.
+
+**-g CONGESTION**
+   Specifies the TCP congestion control algorithm to use (Linux only).
+   See **sysctl net.ipv4.tcp_allowed_congestion_control** for available
+   values.
+
+**-p**
+   Preserves modification times and access times (file mode bits are
+   preserved by default).
+
+**-d**
+   Increments the ssh debug output level.
+
+**-N**
+   Enables Nagle's algorithm. It is disabled by default.
+
+**-h**
+   Prints help.
+
+EXIT STATUS
+===========
+
+Exit status is 0 on success, and >0 if an error occurs.
+
+ENVIRONMENT
+===========
+
+**mscp** recognizes the following environment variables.
+
+**MSCP_SSH_AUTH_PASSWORD**
+   This environment variable passes a password for password
+   authentication to establish SSH connections.
+
+**MSCP_SSH_AUTH_PASSPHRASE**
+   This environment variable passes a passphrase for public-key
+   authentication for establishing SSH connections.
+
+NOTES
+=====
+
+**mscp** uses glob(3) for globbing pathnames, including matching
+patterns for local and remote paths. However, globbing on the *remote*
+side does not work with musl libc (used in Alpine Linux and the
+single-binary version of mscp) because musl libc does not support
+GLOB_ALTDIRFUNC.
+
+**mscp** does not support remote-to-remote copy, which **scp** supports.
+
+EXAMPLES
+========
+
+Copy a local file to a remote host with different name:
+
+::
+
+       $ mscp ~/src-file 10.0.0.1:copied-file
+
+Copy a local file and a directory to /tmp at a remote host:
+
+::
+
+       $ mscp ~/src-file dir1 10.0.0.1:/tmp
+
+Save a checkpoint if transfer fails:
+
+::
+
+       $ mscp -W mscp.checkpoint many-large-files 10.0.0.1:dst/
+
+Check the remaining files and chunks, and resume the failed transfer:
+
+::
+
+       # Dump the content of a checkpoint and exit (dry-run mode)
+       $ mscp -vv -D -R mscp.checkpoint
+
+       # resume transferring from the checkpoint
+       $ mscp -R mscp.checkpoint
+
+In a long fat network, following options might improve performance:
+
+::
+
+       $ mscp -n 64 -m 0xffff -a 64 -c aes128-gcm@openssh.com src 10.0.0.1:
+
+**-n** increases the number of SSH connections than default, **-m** pins
+threads to specific CPU cores, **-a** increases asynchronous inflight
+SFTP WRITE/READ commands, and **-c aes128-gcm@openssh.com** will be
+faster than the default chacha20-poly1305 cipher, particularly on hosts
+that support AES-NI.
+
+SEE ALSO
+========
+
+**scp**\ (1), **ssh**\ (1), **sshd**\ (8).
+
+PAPER REFERENCE
+===============
+
+Ryo Nakamura and Yohei Kuga. 2023. Multi-threaded scp: Easy and Fast
+File Transfer over SSH. In Practice and Experience in Advanced Research
+Computing (PEARC '23). Association for Computing Machinery, New York,
+NY, USA, 320–323. `DOI <https://doi.org/10.1145/3569951.3597582>`__.
+
+CONTACT INFORMATION
+===================
+
+For patches, bug reports, or feature requests, please open an issue on
+`GitHub <https://github.com/upa/mscp>`__.
+
+AUTHORS
+=======
+
+Ryo Nakamura <upa@haeena.net>

docker/README.md

@@ -1,44 +0,0 @@
-
-Build docker containers.
-
-```console
-cd ..
-
-docker build -t mscp-ubuntu:20.04 -f docker/ubuntu-20.04.Dockerfile .
-
-docker build -t mscp-ubuntu:22.04 -f docker/ubuntu-22.04.Dockerfile .
-
-docker build -t mscp-centos:8 -f docker/centos-8.Dockerfile .
-
-docker build -t mscp-rocky:8.6 -f docker/rocky-8.6.Dockerfile .
-```
-
-Test `mscp` in the containers.
-
-```console
-docker run --init --rm mscp-ubuntu:20.04 /mscp/scripts/test-in-container.sh
-
-docker run --init --rm mscp-ubuntu:22.04 /mscp/scripts/test-in-container.sh
-
-docker run --init --rm mscp-centos:8 /mscp/scripts/test-in-container.sh
-
-docker run --init --rm mscp-rocky:8.6 /mscp/scripts/test-in-container.sh
-```
-
-Retrieve deb/rpm packages.
-
-```console
-docker run --rm -v (pwd):/out mscp-ubuntu:20.04 \
-       cp /mscp/build/mscp_0.0.0-ubuntu-20.04-x86_64.deb /out/
-
-docker run --rm -v (pwd):/out mscp-ubuntu:22.04 \
-       cp /mscp/build/mscp_0.0.0-ubuntu-22.04-x86_64.deb /out/
-
-docker run --rm -v (pwd):/out mscp-centos:8 \
-       cp /mscp/build/mscp_0.0.0-centos-8-x86_64.rpm /out/
-
-docker run --rm -v (pwd):/out mscp-rocky:8.6 \
-       cp /mscp/build/mscp_0.0.0-rocky-8.6-x86_64.rpm /out/
-```
-
-I don't know whether these are good way.

docker/centos-8.Dockerfile

@@ -1,36 +0,0 @@
-FROM centos:8
-
-ARG mscpdir="/mscp"
-
-COPY . ${mscpdir}
-
-# from https://stackoverflow.com/questions/70963985/error-failed-to-download-metadata-for-repo-appstream-cannot-prepare-internal
-RUN cd /etc/yum.repos.d/
-RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*
-RUN sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
-
-# install numpy and pytest, sshd for test, and rpm-build
-RUN set -ex && yum -y update && yum -y install \
-	python3 python3-pip openssh openssh-server openssh-clients rpm-build
-
-RUN python3 -m pip install numpy pytest
-
-
-# preparation for sshd
-RUN  mkdir /var/run/sshd        \
-	&& ssh-keygen -A	\
-        && ssh-keygen -f /root/.ssh/id_rsa -N ""                \
-        && mv /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
-
-# install build dependency
-RUN ${mscpdir}/scripts/install-build-deps.sh
-
-# build
-RUN cd ${mscpdir}			\
-        && rm -rf build			\
-        && cmake -B build 		\
-        && cd ${mscpdir}/build          \
-	&& make				\
-	&& cpack -G RPM CPackConfig.cmake \
-	&& rpm -iv *.rpm
-

docker/rocky-8.6.Dockerfile

@@ -1,31 +0,0 @@
-FROM rockylinux:8.6
-
-ARG mscpdir="/mscp"
-
-COPY . ${mscpdir}
-
-# install numpy and pytest, sshd for test, and rpm-build
-RUN set -ex && yum -y install \
-	python3 python3-pip openssh openssh-server openssh-clients rpm-build
-
-RUN python3 -m pip install numpy pytest
-
-
-# preparation for sshd
-RUN  mkdir /var/run/sshd        \
-	&& ssh-keygen -A	\
-        && ssh-keygen -f /root/.ssh/id_rsa -N ""                \
-        && mv /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
-
-# install build dependency
-RUN ${mscpdir}/scripts/install-build-deps.sh
-
-# build
-RUN cd ${mscpdir}			\
-        && rm -rf build			\
-        && cmake -B build		\
-        && cd ${mscpdir}/build          \
-	&& make				\
-	&& cpack -G RPM CPackConfig.cmake \
-	&& rpm -iv *.rpm
-

docker/ubuntu-20.04.Dockerfile

@@ -1,35 +0,0 @@
-FROM ubuntu:20.04
-
-ARG DEBIAN_FRONTEND=noninteractive
-ARG mscpdir="/mscp"
-
-COPY . ${mscpdir}
-
-RUN set -ex && apt-get update && apt-get install -y --no-install-recommends \
-	ca-certificates
-
-# install numpy and pytest, and sshd for test
-RUN apt-get install -y --no-install-recommends  \
-        python3 python3-pip openssh-server
-
-RUN python3 -m pip install numpy pytest
-
-
-# preparation for sshd
-RUN  mkdir /var/run/sshd        \
-        && ssh-keygen -f /root/.ssh/id_rsa -N ""                \
-        && mv /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
-
-
-# install build dependency
-RUN ${mscpdir}/scripts/install-build-deps.sh
-
-
-# build
-RUN cd ${mscpdir}			\
-	&& rm -rf build			\
-	&& cmake -B build		\
-	&& cd ${mscpdir}/build		\
-	&& make				\
-	&& cpack -G DEB CPackConfig.cmake \
-	&& dpkg -i *.deb

docker/ubuntu-22.04.Dockerfile

@@ -1,35 +0,0 @@
-FROM ubuntu:22.04
-
-ARG DEBIAN_FRONTEND=noninteractive
-ARG mscpdir="/mscp"
-
-COPY . ${mscpdir}
-
-RUN set -ex && apt-get update && apt-get install -y --no-install-recommends \
-	ca-certificates
-
-# install numpy and pytest, and sshd for test
-RUN apt-get install -y --no-install-recommends  \
-        python3 python3-pip openssh-server
-
-RUN python3 -m pip install numpy pytest
-
-
-# preparation for sshd
-RUN  mkdir /var/run/sshd        \
-        && ssh-keygen -f /root/.ssh/id_rsa -N ""                \
-        && mv /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
-
-
-# install build dependency
-RUN ${mscpdir}/scripts/install-build-deps.sh
-
-
-# build
-RUN cd ${mscpdir}			\
-	&& rm -rf build			\
-	&& cmake -B build		\
-	&& cd ${mscpdir}/build		\
-	&& make				\
-	&& cpack -G DEB CPackConfig.cmake \
-	&& dpkg -i *.deb

include/config.h.in

@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
+#ifndef _CONFIG_H_
+#define _CONFIG_H_
+
+#define MSCP_VERSION		"@MSCP_VERSION@"
+#define MSCP_BUILD_VERSION	"@MSCP_BUILD_VERSION@"
+
+
+/* Define to 1 if you have the strlcat function. */
+#cmakedefine HAVE_STRLCAT 1
+
+/* Define to 1 if you have the htonll function. */
+#cmakedefine HAVE_HTONLL 1
+
+/* Define to 1 if you have the ntohll function. */
+#cmakedefine HAVE_NTOHLL 1
+
+#endif /* _CONFIG_H_ */

include/mscp.h

@@ -0,0 +1,309 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
+#ifndef _MSCP_H_
+#define _MSCP_H_
+
+/**
+ * @file mscp.h
+ *
+ * @brief mscp library header file.
+ *
+ * @mainpage
+ *
+ * libmscp is a library for multi-threaded scp. Project page is
+ * https://github.com/upa/mscp.
+ *
+ * All public APIs of libmscp are defined in mscp.h. Basic usage of
+ * libmscp is follows:
+ *
+ * 1. create mscp instance with mscp_init()
+ * 2. set remote host and copy direction with mscp_set_remote()
+ * 3. connect to remote host with mscp_connect()
+ * 4. add path to source files with mscp_add_src_path()
+ * 5. set path to destination with mscp_set_dst_path()
+ * 6. start to scan source files with mscp_scan()
+ * 7. start copy with mscp_start()
+ * 8. wait for copy finished with mscp_join()
+ * 9. cleanup mscp instance with mscp_cleanup() and mscp_free()
+ */
+
+#include <stdbool.h>
+#include <limits.h>
+#include <stdlib.h>
+
+#define MSCP_DIRECTION_L2R	1	/** Indicates local to remote copy */
+#define MSCP_DIRECTION_R2L	2	/** Indicates remote to local copy */
+
+/**
+ * @struct	mscp_opts
+ * @brief	Structure configuring mscp.
+ */
+struct mscp_opts {
+	int	nr_threads;	/** number of copy threads */
+	int	nr_ahead;	/** number of SFTP commands on-the-fly */
+	size_t	min_chunk_sz;	/** minimum chunk size (default 64MB) */
+	size_t	max_chunk_sz;	/** maximum chunk size (default file size/nr_threads) */
+	size_t	buf_sz;		/** buffer size, default 16k. */
+	size_t	bitrate;	/** bits-per-seconds to limit bandwidth */
+	char	*coremask;	/** hex to specifiy usable cpu cores */
+	int	max_startups;	/** sshd MaxStartups concurrent connections */
+	int     interval;	/** interval between SSH connection attempts */
+	bool	preserve_ts;	/** preserve file timestamps */
+	int	severity; 	/** messaging severity. set MSCP_SERVERITY_* */
+};
+
+
+/**
+ * @struct	mscp_ssh_opts
+ * @brief	Structure configuring SSH connections
+ */
+struct mscp_ssh_opts {
+	/* ssh options */
+	char	*login_name;	/** ssh username */
+	char	*port;		/** ssh port */
+	int	ai_family;	/** address family */
+	char	*config;	/** path to ssh_config, default ~/.ssh/config*/
+	char	**options;	/** array of ssh_config options, terminated by NULL */
+	char	*identity;	/** path to private key */
+	char	*proxyjump;	/** ProxyJump configuration directive (shortcut) */
+	char	*cipher;	/** cipher spec */
+	char	*hmac;		/** hmacp spec */
+	char	*compress;	/** yes, no, zlib@openssh.com */
+	char	*ccalgo;	/** TCP cc algorithm */
+
+	char	*password;	/** password auth passowrd */
+	char	*passphrase;	/** passphrase for private key */
+
+	int	debug_level;		/** inclirement libssh debug output level */
+	bool	enable_nagle;		/** enable Nagle's algorithm if true */
+};
+
+/** @def
+ * Environment variable that passes password for ssh password auth
+ */
+#define ENV_SSH_AUTH_PASSWORD	"MSCP_SSH_AUTH_PASSWORD"
+
+/** @def
+ * Environment vraible that passes passphrase for private key
+ */
+#define ENV_SSH_AUTH_PASSPHRASE	"MSCP_SSH_AUTH_PASSPHRASE"
+
+
+/**
+ * @struct	mscp_stats
+ * @brief	Structure to get mscp statistics
+ */
+struct mscp_stats {
+	size_t total;	/** total bytes to be transferred */
+	size_t done;	/** total bytes transferred */
+};
+
+
+/** Structure representing mscp instance */
+struct mscp;
+
+/**
+ * @brief Creates a new mscp instance.
+ *
+ * @param o		options for configuring mscp.
+ * @param s		options for configuring ssh connections.
+ *
+ * @retrun 		A new mscp instance or NULL on error.
+ */
+struct mscp *mscp_init(struct mscp_opts *o, struct mscp_ssh_opts *s);
+
+/**
+ * @brief Set remote host and copy direction.
+ *
+ * @param remote_host	remote host for file transer.
+ * @param direction	copy direction, `MSCP_DIRECTION_L2R` or `MSCP_DIRECTION_R2L`
+ *
+ * @return              0 on success, < 0 if an error occured.
+ */
+int mscp_set_remote(struct mscp *m, const char *remote_host, int direction);
+
+/**
+ * @brief Connect the first SSH connection. mscp_connect connects to
+ * remote host and initialize a SFTP session over the
+ * connection. mscp_scan() and mscp_start() require mscp_connect()
+ * beforehand.
+ *
+ * @param m	mscp instance.
+ *
+ * @return 	0 on success, < 0 if an error occured.
+ */
+int mscp_connect(struct mscp *m);
+
+/* add a source file path to be copied */
+
+/**
+ * @brief Add a source file path to be copied. The path indicates
+ * either a file or directory.
+ *
+ * @param m		mscp instance.
+ * @param src_path	source file path to be copied.
+ *
+ * @return 		0 on success, < 0 if an error occured.
+ */
+int mscp_add_src_path(struct mscp *m, const char *src_path);
+
+/**
+ * @brief Set the destination file path. The path indicates either a
+ * file, directory, or nonexistent path.
+ *
+ * @param m		mscp instance.
+ * @param dst_path	destination path to which source files copied.
+ *
+ * @return 		0 on success, < 0 if an error occured.
+ */
+int mscp_set_dst_path(struct mscp *m, const char *dst_path);
+
+/* scan source files, resolve destination file paths for all source
+ * files, and calculate chunks for all files. */
+
+/**
+ * @brief Scan source paths and prepare. This function checks all
+ * source files (recursively), resolve paths on the destination side,
+ * and calculate file chunks. This function is non-blocking.
+ *
+ * @param m	mscp instance.
+ *
+ * @return 	0 on success, < 0 if an error occured.
+ */
+int mscp_scan(struct mscp *m);
+
+/**
+ * @brief Join scan thread invoked by mscp_scan() if it
+ * runs. mscp_join() involves mscp_can_join(). Thus, there is no need
+ * to call this function alone.
+ *
+ * @param m	mscp instance.
+ * @return	0 on success, < 0 if an error occured.
+ */
+int mscp_scan_join(struct mscp *m);
+
+/**
+ * @brief get information about remote host and copy direction from a
+ * checkpoint file specified by *pathname. This functions returns
+ * remote host name to *renote, and the copy direction into *dir.
+ * Thus, you can call mscp_init with those values.
+ *
+ * @param pathname	path to a checkpoint file.
+ * @param remote	char buffer to which remote hostname is stored.
+ * @param len		length of *remote.
+ * @param dir		int to which the copy direction is stored.
+ */
+int mscp_checkpoint_get_remote(const char *pathname, char *remote, size_t len, int *dir);
+
+/**
+ * @brief load information about untransferred files and chunks at the
+ * last transfer . mscp_checkpoint_load() loads files and associated
+ * chunks from the checkpoint file pointed by pathname. If you call
+ * mscp_checkpoint_load(), do not call mscp_scan().
+ *
+ * @param m		mscp instance.
+ * @param pathname	path to a checkpoint file.
+ * @return		0 on success, < 0 if an error occured.
+ */
+int mscp_checkpoint_load(struct mscp *m, const char *pathname);
+
+/**
+ * @brief save information about untransferred files and chunks to a
+ * checkpoint file.
+ *
+ * @param m		mscp instance.
+ * @param pathname	path to a checkpoint file.
+ * @return		0 on success, < 0 if an error occured.
+ */
+int mscp_checkpoint_save(struct mscp *m, const char *pathname);
+
+/**
+ * @brief Start to copy files. mscp_start() returns immediately. You
+ * can get statistics via mscp_get_stats() or messages via pipe set by
+ * mscp_opts.msg_fd or mscp_set_msg_fd(). mscp_stop() cancels mscp
+ * copy threads, and mscp_join() joins the threads.
+ *
+ * @param m	mscp instance.
+ *
+ * @return 	number of threads on success, < 0 if an error occured.
+ *
+ * @see		mscp_join()
+ */
+int mscp_start(struct mscp *m);
+
+
+/**
+ * @brief Stop coping files.
+ *
+ * @param m	mscp instance.
+ */
+void mscp_stop(struct mscp *m);
+
+
+/**
+ * @brief Join copy threads. This function is blocking until all copy
+ * have done.
+ *
+ * @param m	mscp instance.
+ *
+ * @return 	0 on success, < 0 if an error occured.
+ */
+int mscp_join(struct mscp *m);
+
+/**
+ * @brief Get statistics of copy.
+ *
+ * @param m		mscp instance.
+ * @param s[out]	statistics.
+ */
+void mscp_get_stats(struct mscp *m, struct mscp_stats *s);
+
+/**
+ * @brief Cleanup the mscp instance. Before calling mscp_cleanup(),
+ * must call mscp_join(). After mscp_cleanup() called, the mscp
+ * instance can restart from mscp_connect(). Note that do not call
+ * mscp_cleanup() before callign mscp_join(). It causes crash (ToDo:
+ * check status of copy threads and return error when they are
+ * running).
+ *
+ * @param m		mscp instance.
+ */
+void mscp_cleanup(struct mscp *m);
+
+/**
+ * @brief Release the mscp instance.  Note that do not call *
+ mscp_free() before calling mscp_join(). It causes crash (ToDo: check
+ * status of copy threads and return error when they are running).
+ *
+ * @param m		mscp instance.
+ */
+void mscp_free(struct mscp *m);
+
+
+/* messaging with mscp */
+
+/**
+ * @enum	mscp_serverity
+ * @brief 	Filter messages from libmscp with severity level.
+ */
+enum {
+	MSCP_SEVERITY_NONE	= -1,
+	MSCP_SEVERITY_ERR	= 0,
+	MSCP_SEVERITY_WARN	= 1,
+	MSCP_SEVERITY_NOTICE	= 2,
+        MSCP_SEVERITY_INFO	= 3,
+	MSCP_SEVERITY_DEBUG	= 4,
+};
+
+
+/**
+ * @brief Return available ciphers.
+ */
+const char **mscp_ssh_ciphers(void);
+
+/**
+ * @brief Return available hmacs.
+ */
+ const char **mscp_ssh_hmacs(void);
+
+
+#endif /* _MSCP_H_ */

patch/README.md

@@ -1,13 +1,6 @@
 
-Patch(es) in this directory introduces `sftp_async_write()` and
-`sftp_async_write_end()` to libssh. Those implementations are derived
-from https://github.com/limes-datentechnik-gmbh/libssh. See [Re: SFTP
-Write async](https://archive.libssh.org/libssh/2020-06/0000004.html).
+Patches in this directory introduce enhancements for libssh including
+`sftp_async_write()` and `sftp_async_write_end()`, derived from
+https://github.com/limes-datentechnik-gmbh/libssh. See [Re: SFTP Write
+async](https://archive.libssh.org/libssh/2020-06/0000004.html).
 
-```console
-git clone https://git.libssh.org/projects/libssh.git/ --depth=1 -b libssh-0.10.4
-cd libssh
-git apply ../pathc/libssh-0.10.4.patch
-
-# then build libssh
-```

patch/libssh-0.10.4.patch

@@ -1,41 +1,92 @@
-diff --git a/DefineOptions.cmake b/DefineOptions.cmake
-index 068db988..6db0fc0f 100644
---- a/DefineOptions.cmake
-+++ b/DefineOptions.cmake
-@@ -17,7 +17,7 @@ option(UNIT_TESTING "Build with unit tests" OFF)
- option(CLIENT_TESTING "Build with client tests; requires openssh" OFF)
- option(SERVER_TESTING "Build with server tests; requires openssh and dropbear" OFF)
- option(WITH_BENCHMARKS "Build benchmarks tools" OFF)
--option(WITH_EXAMPLES "Build examples" ON)
-+option(WITH_EXAMPLES "Build examples" OFF)
- option(WITH_NACL "Build with libnacl (curve25519)" ON)
- option(WITH_SYMBOL_VERSIONING "Build with symbol versioning" ON)
- option(WITH_ABI_BREAK "Allow ABI break" OFF)
-@@ -25,6 +25,7 @@ option(WITH_GEX "Enable DH Group exchange mechanisms" ON)
- option(WITH_INSECURE_NONE "Enable insecure none cipher and MAC algorithms (not suitable for production!)" OFF)
- option(FUZZ_TESTING "Build with fuzzer for the server and client (automatically enables none cipher!)" OFF)
- option(PICKY_DEVELOPER "Build with picky developer flags" OFF)
-+option(WITH_STATIC_LIB "Build static library" ON)
+diff --git a/ConfigureChecks.cmake b/ConfigureChecks.cmake
+index 7103f303..c64eb39d 100644
+--- a/ConfigureChecks.cmake
++++ b/ConfigureChecks.cmake
+@@ -258,6 +258,7 @@ if (UNIX)
+     check_library_exists(util forkpty "" HAVE_LIBUTIL)
+     check_function_exists(cfmakeraw HAVE_CFMAKERAW)
+     check_function_exists(__strtoull HAVE___STRTOULL)
++    check_symbol_exists(TCP_CONGESTION "netinet/tcp.h" HAVE_TCP_CONGESTION)
+ endif (UNIX)
  
- if (WITH_ZLIB)
-     set(WITH_LIBZ ON)
-@@ -60,3 +61,7 @@ endif (NOT GLOBAL_CLIENT_CONFIG)
- if (FUZZ_TESTING)
-   set(WITH_INSECURE_NONE ON)
- endif (FUZZ_TESTING)
+ set(LIBSSH_REQUIRED_LIBRARIES ${_REQUIRED_LIBRARIES} CACHE INTERNAL "libssh required system libraries")
+diff --git a/config.h.cmake b/config.h.cmake
+index 1357615b..1e915ead 100644
+--- a/config.h.cmake
++++ b/config.h.cmake
+@@ -237,6 +237,8 @@
+ 
+ #cmakedefine HAVE_GCC_BOUNDED_ATTRIBUTE 1
+ 
++#cmakedefine HAVE_TCP_CONGESTION 1
 +
-+if (WITH_STATIC_LIB)
-+  set(BUILD_STATIC_LIB ON)
-+endif()
+ /* Define to 1 if you want to enable GSSAPI */
+ #cmakedefine WITH_GSSAPI 1
+ 
+diff --git a/include/libssh/buffer.h b/include/libssh/buffer.h
+index a55a1b40..e34e075c 100644
+--- a/include/libssh/buffer.h
++++ b/include/libssh/buffer.h
+@@ -33,6 +33,8 @@ int ssh_buffer_add_u8(ssh_buffer buffer, uint8_t data);
+ int ssh_buffer_add_u16(ssh_buffer buffer, uint16_t data);
+ int ssh_buffer_add_u32(ssh_buffer buffer, uint32_t data);
+ int ssh_buffer_add_u64(ssh_buffer buffer, uint64_t data);
++ssize_t ssh_buffer_add_func(ssh_buffer buffer, ssh_add_func f, size_t max_bytes,
++			    void *userdata);
+ 
+ int ssh_buffer_validate_length(struct ssh_buffer_struct *buffer, size_t len);
+ 
+diff --git a/include/libssh/libssh.h b/include/libssh/libssh.h
+index 7857a77b..6b4d481c 100644
+--- a/include/libssh/libssh.h
++++ b/include/libssh/libssh.h
+@@ -402,6 +402,7 @@ enum ssh_options_e {
+   SSH_OPTIONS_GSSAPI_AUTH,
+   SSH_OPTIONS_GLOBAL_KNOWNHOSTS,
+   SSH_OPTIONS_NODELAY,
++  SSH_OPTIONS_CCALGO,
+   SSH_OPTIONS_PUBLICKEY_ACCEPTED_TYPES,
+   SSH_OPTIONS_PROCESS_CONFIG,
+   SSH_OPTIONS_REKEY_DATA,
+@@ -833,6 +834,7 @@ LIBSSH_API const char* ssh_get_hmac_in(ssh_session session);
+ LIBSSH_API const char* ssh_get_hmac_out(ssh_session session);
+ 
+ LIBSSH_API ssh_buffer ssh_buffer_new(void);
++LIBSSH_API ssh_buffer ssh_buffer_new_size(uint32_t size, uint32_t headroom);
+ LIBSSH_API void ssh_buffer_free(ssh_buffer buffer);
+ #define SSH_BUFFER_FREE(x) \
+     do { if ((x) != NULL) { ssh_buffer_free(x); x = NULL; } } while(0)
+@@ -843,6 +845,8 @@ LIBSSH_API void *ssh_buffer_get(ssh_buffer buffer);
+ LIBSSH_API uint32_t ssh_buffer_get_len(ssh_buffer buffer);
+ LIBSSH_API int ssh_session_set_disconnect_message(ssh_session session, const char *message);
+ 
++typedef ssize_t (*ssh_add_func) (void *ptr, size_t max_bytes, void *userdata);
++
+ #ifndef LIBSSH_LEGACY_0_4
+ #include "libssh/legacy.h"
+ #endif
+diff --git a/include/libssh/session.h b/include/libssh/session.h
+index d3e5787c..15183d1b 100644
+--- a/include/libssh/session.h
++++ b/include/libssh/session.h
+@@ -232,6 +232,7 @@ struct ssh_session_struct {
+         int gss_delegate_creds;
+         int flags;
+         int nodelay;
++        char *ccalgo;
+         bool config_processed;
+         uint8_t options_seen[SOC_MAX];
+         uint64_t rekey_data;
 diff --git a/include/libssh/sftp.h b/include/libssh/sftp.h
-index c855df8a..1fd1710a 100644
+index c855df8a..0fcdb9b8 100644
 --- a/include/libssh/sftp.h
 +++ b/include/libssh/sftp.h
-@@ -565,6 +565,9 @@ LIBSSH_API int sftp_async_read(sftp_file file, void *data, uint32_t len, uint32_
+@@ -565,6 +565,10 @@ LIBSSH_API int sftp_async_read(sftp_file file, void *data, uint32_t len, uint32_
   */
  LIBSSH_API ssize_t sftp_write(sftp_file file, const void *buf, size_t count);
  
-+LIBSSH_API int sftp_async_write(sftp_file file, const void *buf, size_t count, uint32_t* id);
++LIBSSH_API ssize_t sftp_async_write(sftp_file file, ssh_add_func f, size_t count,
++				    void *userdata, uint32_t* id);
 +LIBSSH_API int sftp_async_write_end(sftp_file file, uint32_t id, int blocking);
 +
  /**
@@ -57,26 +108,243 @@ index c090fef7..e2f86309 100644
  endif (BUILD_STATIC_LIB)
  
  message(STATUS "Threads_FOUND=${Threads_FOUND}")
+diff --git a/src/buffer.c b/src/buffer.c
+index e0068015..cc0caf35 100644
+--- a/src/buffer.c
++++ b/src/buffer.c
+@@ -141,6 +141,40 @@ struct ssh_buffer_struct *ssh_buffer_new(void)
+     return buf;
+ }
+ 
++/**
++ * @brief Create a new SSH buffer with a specified size and headroom.
++ *
++ * @param[in] len       length for newly initialized SSH buffer.
++ * @param[in] headroom  length for headroom
++ * @return A newly initialized SSH buffer, NULL on error.
++ */
++struct ssh_buffer_struct *ssh_buffer_new_size(uint32_t len, uint32_t headroom)
++{
++    struct ssh_buffer_struct *buf = NULL;
++    int rc;
++
++    if (len < headroom)
++	    return NULL;
++
++    buf = calloc(1, sizeof(struct ssh_buffer_struct));
++    if (buf == NULL) {
++        return NULL;
++    }
++
++    rc = ssh_buffer_allocate_size(buf, len);
++    if (rc != 0) {
++        SAFE_FREE(buf);
++        return NULL;
++    }
++
++    buf->pos += headroom;
++    buf->used += headroom;
++
++    buffer_verify(buf);
++
++    return buf;
++}
++
+ /**
+  * @brief Deallocate a SSH buffer.
+  *
+@@ -328,6 +362,49 @@ int ssh_buffer_add_data(struct ssh_buffer_struct *buffer, const void *data, uint
+     return 0;
+ }
+ 
++/**
++ * @brief Add data at the tail of a buffer by an external function
++ *
++ * @param[in]  buffer    The buffer to add data.
++ *
++ * @param[in]  f         function that adds data to the buffer.
++ *
++ * @param[in]  max_bytes The maximum length of the data to add.
++ *
++ * @return               actual bytes added on success, < 0 on error.
++ */
++ssize_t ssh_buffer_add_func(struct ssh_buffer_struct *buffer, ssh_add_func f,
++			    size_t max_bytes, void *userdata)
++{
++    ssize_t actual;
++
++    if (buffer == NULL) {
++        return -1;
++    }
++
++    buffer_verify(buffer);
++
++    if (buffer->used + max_bytes < max_bytes) {
++        return -1;
++    }
++
++    if (buffer->allocated < (buffer->used + max_bytes)) {
++        if (buffer->pos > 0) {
++            buffer_shift(buffer);
++        }
++        if (realloc_buffer(buffer, buffer->used + max_bytes) < 0) {
++            return -1;
++        }
++    }
++
++    if ((actual = f(buffer->data + buffer->used, max_bytes, userdata)) < 0)
++      return -1;
++
++    buffer->used += actual;
++    buffer_verify(buffer);
++    return actual;
++}
++
+ /**
+  * @brief Ensure the buffer has at least a certain preallocated size.
+  *
+diff --git a/src/connect.c b/src/connect.c
+index 57e37e63..c02397d5 100644
+--- a/src/connect.c
++++ b/src/connect.c
+@@ -156,6 +156,20 @@ static int set_tcp_nodelay(socket_t socket)
+                       sizeof(opt));
+ }
+ 
++static int set_tcp_ccalgo(socket_t socket, const char *ccalgo)
++{
++#ifdef HAVE_TCP_CONGESTION
++	return setsockopt(socket,
++			  IPPROTO_TCP,
++			  TCP_CONGESTION,
++			  (void *)ccalgo,
++			  strlen(ccalgo));
++#else
++	errno = ENOTSUP;
++	return -1;
++#endif
++}
++
+ /**
+  * @internal
+  *
+@@ -256,6 +270,18 @@ socket_t ssh_connect_host_nonblocking(ssh_session session, const char *host,
+             }
+         }
+ 
++        if (session->opts.ccalgo) {
++	     rc = set_tcp_ccalgo(s, session->opts.ccalgo);
++	     if (rc < 0) {
++		 ssh_set_error(session, SSH_FATAL,
++			       "Failed to set TCP_CONGESTION on socket: %s",
++			       ssh_strerror(errno, err_msg, SSH_ERRNO_MSG_MAX));
++		 ssh_connect_socket_close(s);
++		 s = -1;
++		 continue;
++	     }
++	}
++
+         errno = 0;
+         rc = connect(s, itr->ai_addr, itr->ai_addrlen);
+         if (rc == -1 && (errno != 0) && (errno != EINPROGRESS)) {
+diff --git a/src/options.c b/src/options.c
+index 49aaefa2..9f7360c3 100644
+--- a/src/options.c
++++ b/src/options.c
+@@ -210,6 +210,7 @@ int ssh_options_copy(ssh_session src, ssh_session *dest)
+     new->opts.gss_delegate_creds    = src->opts.gss_delegate_creds;
+     new->opts.flags                 = src->opts.flags;
+     new->opts.nodelay               = src->opts.nodelay;
++    new->opts.ccalgo                = src->opts.ccalgo;
+     new->opts.config_processed      = src->opts.config_processed;
+     new->common.log_verbosity       = src->common.log_verbosity;
+     new->common.callbacks           = src->common.callbacks;
+@@ -450,6 +451,10 @@ int ssh_options_set_algo(ssh_session session,
+  *                Set it to disable Nagle's Algorithm (TCP_NODELAY) on the
+  *                session socket. (int, 0=false)
+  *
++ *              - SSH_OPTIONS_CCALGO
++ *                Set it to specify TCP congestion control algorithm on the
++ *                session socket (Linux only). (int, 0=false)
++ *
+  *              - SSH_OPTIONS_PROCESS_CONFIG
+  *                Set it to false to disable automatic processing of per-user
+  *                and system-wide OpenSSH configuration files. LibSSH
+@@ -1013,6 +1018,20 @@ int ssh_options_set(ssh_session session, enum ssh_options_e type,
+                 session->opts.nodelay = (*x & 0xff) > 0 ? 1 : 0;
+             }
+             break;
++	case SSH_OPTIONS_CCALGO:
++            v = value;
++            if (v == NULL || v[0] == '\0') {
++                ssh_set_error_invalid(session);
++                return -1;
++            } else {
++                SAFE_FREE(session->opts.ccalgo);
++                session->opts.ccalgo = strdup(v);
++                if (session->opts.ccalgo == NULL) {
++                    ssh_set_error_oom(session);
++                    return -1;
++                }
++            }
++            break;
+         case SSH_OPTIONS_PROCESS_CONFIG:
+             if (value == NULL) {
+                 ssh_set_error_invalid(session);
+diff --git a/src/session.c b/src/session.c
+index 6025c133..6b197526 100644
+--- a/src/session.c
++++ b/src/session.c
+@@ -108,6 +108,7 @@ ssh_session ssh_new(void)
+     session->opts.fd = -1;
+     session->opts.compressionlevel = 7;
+     session->opts.nodelay = 0;
++    session->opts.ccalgo = NULL;
+ 
+     session->opts.flags = SSH_OPT_FLAG_PASSWORD_AUTH |
+                           SSH_OPT_FLAG_PUBKEY_AUTH |
 diff --git a/src/sftp.c b/src/sftp.c
-index e01012a8..7b5dc249 100644
+index e01012a8..702623a0 100644
 --- a/src/sftp.c
 +++ b/src/sftp.c
-@@ -2228,6 +2228,102 @@ ssize_t sftp_write(sftp_file file, const void *buf, size_t count) {
+@@ -2228,6 +2228,132 @@ ssize_t sftp_write(sftp_file file, const void *buf, size_t count) {
    return -1; /* not reached */
  }
  
 +/*
-+ * sftp_async_write and sftp_async_write_end are copied from
++ * sftp_async_write is based on and sftp_async_write_end is copied from
 + * https://github.com/limes-datentechnik-gmbh/libssh
++ *
++ * sftp_async_write has some optimizations:
++ * - use ssh_buffer_new_size() to reduce realoc_buffer.
++ * - use ssh_buffer_add_func() to avoid memcpy from read buffer to ssh buffer.
 + */
-+int sftp_async_write(sftp_file file, const void *buf, size_t count, uint32_t* id) {
++ssize_t sftp_async_write(sftp_file file, ssh_add_func f, size_t count, void *userdata,
++			 uint32_t* id) {
 +  sftp_session sftp = file->sftp;
 +  ssh_buffer buffer;
++  uint32_t buf_sz;
++  ssize_t actual;
 +  int len;
 +  int packetlen;
 +  int rc;
 +
-+  buffer = ssh_buffer_new();
++#define HEADROOM 16
++  /* sftp_packet_write() prepends a 5-bytes (uint32_t length and
++   * 1-byte type) header to the head of the payload by
++   * ssh_buffer_prepend_data(). Inserting headroom by
++   * ssh_buffer_new_size() eliminates memcpy for prepending the
++   * header.
++   */
++
++  buf_sz = (HEADROOM + /* for header */
++	    sizeof(uint32_t) + /* id */
++	    ssh_string_len(file->handle) + 4 + /* file->handle */
++	    sizeof(uint64_t) + /* file->offset */
++	    sizeof(uint32_t) + /* count */
++	    count); /* datastring */
++
++  buffer = ssh_buffer_new_size(buf_sz, HEADROOM);
 +  if (buffer == NULL) {
 +    ssh_set_error_oom(sftp->session);
 +    return -1;
@@ -85,17 +353,25 @@ index e01012a8..7b5dc249 100644
 +  *id = sftp_get_new_id(file->sftp);
 +
 +  rc = ssh_buffer_pack(buffer,
-+                       "dSqdP",
++                       "dSqd",
 +                       *id,
 +                       file->handle,
 +                       file->offset,
-+                       count, /* len of datastring */
-+                       (size_t)count, buf);
++                       count); /* len of datastring */
++
 +  if (rc != SSH_OK){
 +    ssh_set_error_oom(sftp->session);
 +    ssh_buffer_free(buffer);
 +    return SSH_ERROR;
 +  }
++
++  actual = ssh_buffer_add_func(buffer, f, count, userdata);
++  if (actual < 0){
++    ssh_set_error_oom(sftp->session);
++    ssh_buffer_free(buffer);
++    return SSH_ERROR;
++  }
++
 +  packetlen=ssh_buffer_get_len(buffer)+5;
 +  len = sftp_packet_write(file->sftp, SSH_FXP_WRITE, buffer);
 +  ssh_buffer_free(buffer);
@@ -109,9 +385,9 @@ index e01012a8..7b5dc249 100644
 +    return SSH_ERROR;
 +  }
 +
-+  file->offset += count;
++  file->offset += actual;
 +
-+  return SSH_OK;
++  return actual;
 +}
 +
 +int sftp_async_write_end(sftp_file file, uint32_t id, int blocking) {

patch/libssh-0.10.6-2-g6f1b1e76.patch

@@ -0,0 +1,597 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index a64b7708..c6344a5a 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -1,4 +1,4 @@
+-cmake_minimum_required(VERSION 3.3.0)
++cmake_minimum_required(VERSION 3.13.0)
+ cmake_policy(SET CMP0048 NEW)
+ 
+ # Specify search path for CMake modules to be loaded by include()
+diff --git a/ConfigureChecks.cmake b/ConfigureChecks.cmake
+index 9de10225..0f3d20ed 100644
+--- a/ConfigureChecks.cmake
++++ b/ConfigureChecks.cmake
+@@ -258,6 +258,7 @@ if (UNIX)
+     check_library_exists(util forkpty "" HAVE_LIBUTIL)
+     check_function_exists(cfmakeraw HAVE_CFMAKERAW)
+     check_function_exists(__strtoull HAVE___STRTOULL)
++    check_symbol_exists(TCP_CONGESTION "netinet/tcp.h" HAVE_TCP_CONGESTION)
+ endif (UNIX)
+ 
+ set(LIBSSH_REQUIRED_LIBRARIES ${_REQUIRED_LIBRARIES} CACHE INTERNAL "libssh required system libraries")
+diff --git a/config.h.cmake b/config.h.cmake
+index cc83734d..f74cd03b 100644
+--- a/config.h.cmake
++++ b/config.h.cmake
+@@ -237,6 +237,8 @@
+ 
+ #cmakedefine HAVE_GCC_BOUNDED_ATTRIBUTE 1
+ 
++#cmakedefine HAVE_TCP_CONGESTION 1
++
+ /* Define to 1 if you want to enable GSSAPI */
+ #cmakedefine WITH_GSSAPI 1
+ 
+diff --git a/include/libssh/buffer.h b/include/libssh/buffer.h
+index 1fce7b76..b64d1455 100644
+--- a/include/libssh/buffer.h
++++ b/include/libssh/buffer.h
+@@ -37,6 +37,8 @@ int ssh_buffer_add_u8(ssh_buffer buffer, uint8_t data);
+ int ssh_buffer_add_u16(ssh_buffer buffer, uint16_t data);
+ int ssh_buffer_add_u32(ssh_buffer buffer, uint32_t data);
+ int ssh_buffer_add_u64(ssh_buffer buffer, uint64_t data);
++ssize_t ssh_buffer_add_func(ssh_buffer buffer, ssh_add_func f, size_t max_bytes,
++			    void *userdata);
+ 
+ int ssh_buffer_validate_length(struct ssh_buffer_struct *buffer, size_t len);
+ 
+diff --git a/include/libssh/libssh.h b/include/libssh/libssh.h
+index 669a0a96..26b20f3f 100644
+--- a/include/libssh/libssh.h
++++ b/include/libssh/libssh.h
+@@ -368,6 +368,7 @@ enum ssh_options_e {
+   SSH_OPTIONS_HOST,
+   SSH_OPTIONS_PORT,
+   SSH_OPTIONS_PORT_STR,
++  SSH_OPTIONS_AI_FAMILY,
+   SSH_OPTIONS_FD,
+   SSH_OPTIONS_USER,
+   SSH_OPTIONS_SSH_DIR,
+@@ -402,6 +403,7 @@ enum ssh_options_e {
+   SSH_OPTIONS_GSSAPI_AUTH,
+   SSH_OPTIONS_GLOBAL_KNOWNHOSTS,
+   SSH_OPTIONS_NODELAY,
++  SSH_OPTIONS_CCALGO,
+   SSH_OPTIONS_PUBLICKEY_ACCEPTED_TYPES,
+   SSH_OPTIONS_PROCESS_CONFIG,
+   SSH_OPTIONS_REKEY_DATA,
+@@ -833,6 +835,7 @@ LIBSSH_API const char* ssh_get_hmac_in(ssh_session session);
+ LIBSSH_API const char* ssh_get_hmac_out(ssh_session session);
+ 
+ LIBSSH_API ssh_buffer ssh_buffer_new(void);
++LIBSSH_API ssh_buffer ssh_buffer_new_size(uint32_t size, uint32_t headroom);
+ LIBSSH_API void ssh_buffer_free(ssh_buffer buffer);
+ #define SSH_BUFFER_FREE(x) \
+     do { if ((x) != NULL) { ssh_buffer_free(x); x = NULL; } } while(0)
+@@ -843,6 +846,11 @@ LIBSSH_API void *ssh_buffer_get(ssh_buffer buffer);
+ LIBSSH_API uint32_t ssh_buffer_get_len(ssh_buffer buffer);
+ LIBSSH_API int ssh_session_set_disconnect_message(ssh_session session, const char *message);
+ 
++typedef ssize_t (*ssh_add_func) (void *ptr, size_t max_bytes, void *userdata);
++
++LIBSSH_API const char **ssh_ciphers(void);
++LIBSSH_API const char **ssh_hmacs(void);
++
+ #ifndef LIBSSH_LEGACY_0_4
+ #include "libssh/legacy.h"
+ #endif
+diff --git a/include/libssh/session.h b/include/libssh/session.h
+index 97936195..e4fc4fce 100644
+--- a/include/libssh/session.h
++++ b/include/libssh/session.h
+@@ -249,6 +249,7 @@ struct ssh_session_struct {
+         unsigned long timeout; /* seconds */
+         unsigned long timeout_usec;
+         uint16_t port;
++        int  ai_family;
+         socket_t fd;
+         int StrictHostKeyChecking;
+         char compressionlevel;
+@@ -258,6 +259,7 @@ struct ssh_session_struct {
+         int flags;
+         int exp_flags;
+         int nodelay;
++        char *ccalgo;
+         bool config_processed;
+         uint8_t options_seen[SOC_MAX];
+         uint64_t rekey_data;
+diff --git a/include/libssh/sftp.h b/include/libssh/sftp.h
+index c713466e..e27fe326 100644
+--- a/include/libssh/sftp.h
++++ b/include/libssh/sftp.h
+@@ -565,6 +565,10 @@ LIBSSH_API int sftp_async_read(sftp_file file, void *data, uint32_t len, uint32_
+  */
+ LIBSSH_API ssize_t sftp_write(sftp_file file, const void *buf, size_t count);
+ 
++LIBSSH_API ssize_t sftp_async_write(sftp_file file, ssh_add_func f, size_t count,
++				    void *userdata, uint32_t* id);
++LIBSSH_API int sftp_async_write_end(sftp_file file, uint32_t id, int blocking);
++
+ /**
+  * @brief Seek to a specific location in a file.
+  *
+diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
+index 807313b5..86487087 100644
+--- a/src/CMakeLists.txt
++++ b/src/CMakeLists.txt
+@@ -448,6 +448,11 @@ if (BUILD_STATIC_LIB)
+   if (WIN32)
+     target_compile_definitions(ssh-static PUBLIC "LIBSSH_STATIC")
+   endif (WIN32)
++
++  install(TARGETS ssh-static
++	  EXPORT libssh-config
++	  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
++	  COMPONENT libraries)
+ endif (BUILD_STATIC_LIB)
+ 
+ message(STATUS "Threads_FOUND=${Threads_FOUND}")
+diff --git a/src/buffer.c b/src/buffer.c
+index 8991e006..e0414801 100644
+--- a/src/buffer.c
++++ b/src/buffer.c
+@@ -142,6 +142,40 @@ struct ssh_buffer_struct *ssh_buffer_new(void)
+     return buf;
+ }
+ 
++/**
++ * @brief Create a new SSH buffer with a specified size and headroom.
++ *
++ * @param[in] len       length for newly initialized SSH buffer.
++ * @param[in] headroom  length for headroom
++ * @return A newly initialized SSH buffer, NULL on error.
++ */
++struct ssh_buffer_struct *ssh_buffer_new_size(uint32_t len, uint32_t headroom)
++{
++    struct ssh_buffer_struct *buf = NULL;
++    int rc;
++
++    if (len < headroom)
++	    return NULL;
++
++    buf = calloc(1, sizeof(struct ssh_buffer_struct));
++    if (buf == NULL) {
++        return NULL;
++    }
++
++    rc = ssh_buffer_allocate_size(buf, len);
++    if (rc != 0) {
++        SAFE_FREE(buf);
++        return NULL;
++    }
++
++    buf->pos += headroom;
++    buf->used += headroom;
++
++    buffer_verify(buf);
++
++    return buf;
++}
++
+ /**
+  * @brief Deallocate a SSH buffer.
+  *
+@@ -329,6 +363,49 @@ int ssh_buffer_add_data(struct ssh_buffer_struct *buffer, const void *data, uint
+     return 0;
+ }
+ 
++/**
++ * @brief Add data at the tail of a buffer by an external function
++ *
++ * @param[in]  buffer    The buffer to add data.
++ *
++ * @param[in]  f         function that adds data to the buffer.
++ *
++ * @param[in]  max_bytes The maximum length of the data to add.
++ *
++ * @return               actual bytes added on success, < 0 on error.
++ */
++ssize_t ssh_buffer_add_func(struct ssh_buffer_struct *buffer, ssh_add_func f,
++			    size_t max_bytes, void *userdata)
++{
++    ssize_t actual;
++
++    if (buffer == NULL) {
++        return -1;
++    }
++
++    buffer_verify(buffer);
++
++    if (buffer->used + max_bytes < max_bytes) {
++        return -1;
++    }
++
++    if (buffer->allocated < (buffer->used + max_bytes)) {
++        if (buffer->pos > 0) {
++            buffer_shift(buffer);
++        }
++        if (realloc_buffer(buffer, buffer->used + max_bytes) < 0) {
++            return -1;
++        }
++    }
++
++    if ((actual = f(buffer->data + buffer->used, max_bytes, userdata)) < 0)
++      return -1;
++
++    buffer->used += actual;
++    buffer_verify(buffer);
++    return actual;
++}
++
+ /**
+  * @brief Ensure the buffer has at least a certain preallocated size.
+  *
+diff --git a/src/connect.c b/src/connect.c
+index 15cae644..02ef43b4 100644
+--- a/src/connect.c
++++ b/src/connect.c
+@@ -114,7 +114,7 @@ static int ssh_connect_socket_close(socket_t s)
+ #endif
+ }
+ 
+-static int getai(const char *host, int port, struct addrinfo **ai)
++static int getai(const char *host, int port, int ai_family, struct addrinfo **ai)
+ {
+     const char *service = NULL;
+     struct addrinfo hints;
+@@ -123,7 +123,7 @@ static int getai(const char *host, int port, struct addrinfo **ai)
+     ZERO_STRUCT(hints);
+ 
+     hints.ai_protocol = IPPROTO_TCP;
+-    hints.ai_family = PF_UNSPEC;
++    hints.ai_family = ai_family > 0 ? ai_family : PF_UNSPEC;
+     hints.ai_socktype = SOCK_STREAM;
+ 
+     if (port == 0) {
+@@ -156,6 +156,20 @@ static int set_tcp_nodelay(socket_t socket)
+                       sizeof(opt));
+ }
+ 
++static int set_tcp_ccalgo(socket_t socket, const char *ccalgo)
++{
++#ifdef HAVE_TCP_CONGESTION
++	return setsockopt(socket,
++			  IPPROTO_TCP,
++			  TCP_CONGESTION,
++			  (void *)ccalgo,
++			  strlen(ccalgo));
++#else
++	errno = ENOTSUP;
++	return -1;
++#endif
++}
++
+ /**
+  * @internal
+  *
+@@ -173,7 +187,7 @@ socket_t ssh_connect_host_nonblocking(ssh_session session, const char *host,
+     struct addrinfo *ai = NULL;
+     struct addrinfo *itr = NULL;
+ 
+-    rc = getai(host, port, &ai);
++    rc = getai(host, port, session->opts.ai_family, &ai);
+     if (rc != 0) {
+         ssh_set_error(session, SSH_FATAL,
+                       "Failed to resolve hostname %s (%s)",
+@@ -199,7 +213,7 @@ socket_t ssh_connect_host_nonblocking(ssh_session session, const char *host,
+ 
+             SSH_LOG(SSH_LOG_PACKET, "Resolving %s", bind_addr);
+ 
+-            rc = getai(bind_addr, 0, &bind_ai);
++            rc = getai(bind_addr, 0, session->opts.ai_family, &bind_ai);
+             if (rc != 0) {
+                 ssh_set_error(session, SSH_FATAL,
+                               "Failed to resolve bind address %s (%s)",
+@@ -256,6 +270,18 @@ socket_t ssh_connect_host_nonblocking(ssh_session session, const char *host,
+             }
+         }
+ 
++        if (session->opts.ccalgo) {
++	     rc = set_tcp_ccalgo(s, session->opts.ccalgo);
++	     if (rc < 0) {
++		 ssh_set_error(session, SSH_FATAL,
++			       "Failed to set TCP_CONGESTION on socket: %s",
++			       ssh_strerror(errno, err_msg, SSH_ERRNO_MSG_MAX));
++		 ssh_connect_socket_close(s);
++		 s = -1;
++		 continue;
++	     }
++	}
++
+         errno = 0;
+         rc = connect(s, itr->ai_addr, itr->ai_addrlen);
+         if (rc == -1 && (errno != 0) && (errno != EINPROGRESS)) {
+diff --git a/src/misc.c b/src/misc.c
+index 7081f12a..e3879fe4 100644
+--- a/src/misc.c
++++ b/src/misc.c
+@@ -71,6 +71,8 @@
+ #include "libssh/priv.h"
+ #include "libssh/misc.h"
+ #include "libssh/session.h"
++#include "libssh/wrapper.h"
++#include "libssh/crypto.h"
+ 
+ #ifdef HAVE_LIBGCRYPT
+ #define GCRYPT_STRING "/gnutls"
+@@ -2074,4 +2076,40 @@ int ssh_check_hostname_syntax(const char *hostname)
+     return SSH_OK;
+ }
+ 
++/**
++ * @brief Return supported cipher names
++ * @return	The list of cipher names.
++ */
++const char **ssh_ciphers(void)
++{
++     struct ssh_cipher_struct *tab=ssh_get_ciphertab();
++     static const char *ciphers[32];
++     int n;
++
++     memset(ciphers, 0, sizeof(*ciphers));
++
++     for (n = 0; tab[n].name != NULL; n++) {
++	  ciphers[n] = tab[n].name;
++     }
++     return ciphers;
++}
++
++/**
++ * @brief Return supported hmac names
++ * @return	The list of hmac names.
++ */
++const char **ssh_hmacs(void)
++{
++     struct ssh_hmac_struct *tab=ssh_get_hmactab();
++     static const char *hmacs[32];
++     int n;
++
++     memset(hmacs, 0, sizeof(*hmacs));
++
++     for (n = 0; tab[n].name != NULL; n++) {
++	  hmacs[n] = tab[n].name;
++     }
++     return hmacs;
++}
++
+ /** @} */
+diff --git a/src/options.c b/src/options.c
+index b3ecffe1..8de24ed6 100644
+--- a/src/options.c
++++ b/src/options.c
+@@ -217,6 +217,7 @@ int ssh_options_copy(ssh_session src, ssh_session *dest)
+     new->opts.gss_delegate_creds    = src->opts.gss_delegate_creds;
+     new->opts.flags                 = src->opts.flags;
+     new->opts.nodelay               = src->opts.nodelay;
++    new->opts.ccalgo                = src->opts.ccalgo;
+     new->opts.config_processed      = src->opts.config_processed;
+     new->common.log_verbosity       = src->common.log_verbosity;
+     new->common.callbacks           = src->common.callbacks;
+@@ -268,6 +269,9 @@ int ssh_options_set_algo(ssh_session session,
+  *              - SSH_OPTIONS_PORT_STR:
+  *                The port to connect to (const char *).
+  *
++ *              - SSH_OPTIONS_AI_FAMILY:
++ *                The address family for connecting (int *).
++ *
+  *              - SSH_OPTIONS_FD:
+  *                The file descriptor to use (socket_t).\n
+  *                \n
+@@ -458,6 +462,10 @@ int ssh_options_set_algo(ssh_session session,
+  *                Set it to disable Nagle's Algorithm (TCP_NODELAY) on the
+  *                session socket. (int, 0=false)
+  *
++ *              - SSH_OPTIONS_CCALGO
++ *                Set it to specify TCP congestion control algorithm on the
++ *                session socket (Linux only). (int, 0=false)
++ *
+  *              - SSH_OPTIONS_PROCESS_CONFIG
+  *                Set it to false to disable automatic processing of per-user
+  *                and system-wide OpenSSH configuration files. LibSSH
+@@ -571,6 +579,21 @@ int ssh_options_set(ssh_session session, enum ssh_options_e type,
+                 session->opts.port = i & 0xffffU;
+             }
+             break;
++        case SSH_OPTIONS_AI_FAMILY:
++            if (value == NULL) {
++                session->opts.ai_family = 0;
++                ssh_set_error_invalid(session);
++                return -1;
++            } else {
++                int *x = (int *) value;
++                if (*x < 0) {
++		    session->opts.ai_family = 0;
++                    ssh_set_error_invalid(session);
++                    return -1;
++                }
++                session->opts.ai_family = *x;
++            }
++            break;
+         case SSH_OPTIONS_FD:
+             if (value == NULL) {
+                 session->opts.fd = SSH_INVALID_SOCKET;
+@@ -1017,6 +1040,20 @@ int ssh_options_set(ssh_session session, enum ssh_options_e type,
+                 session->opts.nodelay = (*x & 0xff) > 0 ? 1 : 0;
+             }
+             break;
++	case SSH_OPTIONS_CCALGO:
++            v = value;
++            if (v == NULL || v[0] == '\0') {
++                ssh_set_error_invalid(session);
++                return -1;
++            } else {
++                SAFE_FREE(session->opts.ccalgo);
++                session->opts.ccalgo = strdup(v);
++                if (session->opts.ccalgo == NULL) {
++                    ssh_set_error_oom(session);
++                    return -1;
++                }
++            }
++            break;
+         case SSH_OPTIONS_PROCESS_CONFIG:
+             if (value == NULL) {
+                 ssh_set_error_invalid(session);
+diff --git a/src/session.c b/src/session.c
+index 8c509699..307388e5 100644
+--- a/src/session.c
++++ b/src/session.c
+@@ -105,9 +105,11 @@ ssh_session ssh_new(void)
+     /* OPTIONS */
+     session->opts.StrictHostKeyChecking = 1;
+     session->opts.port = 22;
++    session->opts.ai_family = 0;
+     session->opts.fd = -1;
+     session->opts.compressionlevel = 7;
+     session->opts.nodelay = 0;
++    session->opts.ccalgo = NULL;
+ 
+     session->opts.flags = SSH_OPT_FLAG_PASSWORD_AUTH |
+                           SSH_OPT_FLAG_PUBKEY_AUTH |
+diff --git a/src/sftp.c b/src/sftp.c
+index e01012a8..702623a0 100644
+--- a/src/sftp.c
++++ b/src/sftp.c
+@@ -2228,6 +2228,132 @@ ssize_t sftp_write(sftp_file file, const void *buf, size_t count) {
+   return -1; /* not reached */
+ }
+ 
++/*
++ * sftp_async_write is based on and sftp_async_write_end is copied from
++ * https://github.com/limes-datentechnik-gmbh/libssh
++ *
++ * sftp_async_write has some optimizations:
++ * - use ssh_buffer_new_size() to reduce realoc_buffer.
++ * - use ssh_buffer_add_func() to avoid memcpy from read buffer to ssh buffer.
++ */
++ssize_t sftp_async_write(sftp_file file, ssh_add_func f, size_t count, void *userdata,
++			 uint32_t* id) {
++  sftp_session sftp = file->sftp;
++  ssh_buffer buffer;
++  uint32_t buf_sz;
++  ssize_t actual;
++  int len;
++  int packetlen;
++  int rc;
++
++#define HEADROOM 16
++  /* sftp_packet_write() prepends a 5-bytes (uint32_t length and
++   * 1-byte type) header to the head of the payload by
++   * ssh_buffer_prepend_data(). Inserting headroom by
++   * ssh_buffer_new_size() eliminates memcpy for prepending the
++   * header.
++   */
++
++  buf_sz = (HEADROOM + /* for header */
++	    sizeof(uint32_t) + /* id */
++	    ssh_string_len(file->handle) + 4 + /* file->handle */
++	    sizeof(uint64_t) + /* file->offset */
++	    sizeof(uint32_t) + /* count */
++	    count); /* datastring */
++
++  buffer = ssh_buffer_new_size(buf_sz, HEADROOM);
++  if (buffer == NULL) {
++    ssh_set_error_oom(sftp->session);
++    return -1;
++  }
++
++  *id = sftp_get_new_id(file->sftp);
++
++  rc = ssh_buffer_pack(buffer,
++                       "dSqd",
++                       *id,
++                       file->handle,
++                       file->offset,
++                       count); /* len of datastring */
++
++  if (rc != SSH_OK){
++    ssh_set_error_oom(sftp->session);
++    ssh_buffer_free(buffer);
++    return SSH_ERROR;
++  }
++
++  actual = ssh_buffer_add_func(buffer, f, count, userdata);
++  if (actual < 0){
++    ssh_set_error_oom(sftp->session);
++    ssh_buffer_free(buffer);
++    return SSH_ERROR;
++  }
++
++  packetlen=ssh_buffer_get_len(buffer)+5;
++  len = sftp_packet_write(file->sftp, SSH_FXP_WRITE, buffer);
++  ssh_buffer_free(buffer);
++  if (len < 0) {
++    return SSH_ERROR;
++  } else  if (len != packetlen) {
++    ssh_set_error(sftp->session, SSH_FATAL,
++      "Could only send %d of %d bytes to remote host!", len, packetlen);
++    SSH_LOG(SSH_LOG_PACKET,
++        "Could not write as much data as expected");
++    return SSH_ERROR;
++  }
++
++  file->offset += actual;
++
++  return actual;
++}
++
++int sftp_async_write_end(sftp_file file, uint32_t id, int blocking) {
++  sftp_session sftp = file->sftp;
++  sftp_message msg = NULL;
++  sftp_status_message status;
++
++  msg = sftp_dequeue(sftp, id);
++  while (msg == NULL) {
++    if (!blocking && ssh_channel_poll(sftp->channel, 0) == 0) {
++      /* we cannot block */
++      return SSH_AGAIN;
++    }
++    if (sftp_read_and_dispatch(sftp) < 0) {
++      /* something nasty has happened */
++      return SSH_ERROR;
++    }
++    msg = sftp_dequeue(sftp, id);
++  }
++
++  switch (msg->packet_type) {
++    case SSH_FXP_STATUS:
++      status = parse_status_msg(msg);
++      sftp_message_free(msg);
++      if (status == NULL) {
++        return SSH_ERROR;
++      }
++      sftp_set_error(sftp, status->status);
++      switch (status->status) {
++        case SSH_FX_OK:
++          status_msg_free(status);
++          return SSH_OK;
++        default:
++          break;
++      }
++      ssh_set_error(sftp->session, SSH_REQUEST_DENIED,
++          "SFTP server: %s", status->errormsg);
++      status_msg_free(status);
++      return SSH_ERROR;
++    default:
++      ssh_set_error(sftp->session, SSH_FATAL,
++          "Received message %d during write!", msg->packet_type);
++      sftp_message_free(msg);
++      return SSH_ERROR;
++  }
++
++  return SSH_ERROR; /* not reached */
++}
++
+ /* Seek to a specific location in a file. */
+ int sftp_seek(sftp_file file, uint32_t new_offset) {
+   if (file == NULL) {

patch/libssh-0.10.6.patch

@@ -0,0 +1,442 @@
+diff --git a/ConfigureChecks.cmake b/ConfigureChecks.cmake
+index 9de10225..0f3d20ed 100644
+--- a/ConfigureChecks.cmake
++++ b/ConfigureChecks.cmake
+@@ -258,6 +258,7 @@ if (UNIX)
+     check_library_exists(util forkpty "" HAVE_LIBUTIL)
+     check_function_exists(cfmakeraw HAVE_CFMAKERAW)
+     check_function_exists(__strtoull HAVE___STRTOULL)
++    check_symbol_exists(TCP_CONGESTION "netinet/tcp.h" HAVE_TCP_CONGESTION)
+ endif (UNIX)
+ 
+ set(LIBSSH_REQUIRED_LIBRARIES ${_REQUIRED_LIBRARIES} CACHE INTERNAL "libssh required system libraries")
+diff --git a/config.h.cmake b/config.h.cmake
+index cc83734d..f74cd03b 100644
+--- a/config.h.cmake
++++ b/config.h.cmake
+@@ -237,6 +237,8 @@
+ 
+ #cmakedefine HAVE_GCC_BOUNDED_ATTRIBUTE 1
+ 
++#cmakedefine HAVE_TCP_CONGESTION 1
++
+ /* Define to 1 if you want to enable GSSAPI */
+ #cmakedefine WITH_GSSAPI 1
+ 
+diff --git a/include/libssh/buffer.h b/include/libssh/buffer.h
+index 1fce7b76..b64d1455 100644
+--- a/include/libssh/buffer.h
++++ b/include/libssh/buffer.h
+@@ -37,6 +37,8 @@ int ssh_buffer_add_u8(ssh_buffer buffer, uint8_t data);
+ int ssh_buffer_add_u16(ssh_buffer buffer, uint16_t data);
+ int ssh_buffer_add_u32(ssh_buffer buffer, uint32_t data);
+ int ssh_buffer_add_u64(ssh_buffer buffer, uint64_t data);
++ssize_t ssh_buffer_add_func(ssh_buffer buffer, ssh_add_func f, size_t max_bytes,
++			    void *userdata);
+ 
+ int ssh_buffer_validate_length(struct ssh_buffer_struct *buffer, size_t len);
+ 
+diff --git a/include/libssh/libssh.h b/include/libssh/libssh.h
+index 669a0a96..b6a93ac7 100644
+--- a/include/libssh/libssh.h
++++ b/include/libssh/libssh.h
+@@ -402,6 +402,7 @@ enum ssh_options_e {
+   SSH_OPTIONS_GSSAPI_AUTH,
+   SSH_OPTIONS_GLOBAL_KNOWNHOSTS,
+   SSH_OPTIONS_NODELAY,
++  SSH_OPTIONS_CCALGO,
+   SSH_OPTIONS_PUBLICKEY_ACCEPTED_TYPES,
+   SSH_OPTIONS_PROCESS_CONFIG,
+   SSH_OPTIONS_REKEY_DATA,
+@@ -833,6 +834,7 @@ LIBSSH_API const char* ssh_get_hmac_in(ssh_session session);
+ LIBSSH_API const char* ssh_get_hmac_out(ssh_session session);
+ 
+ LIBSSH_API ssh_buffer ssh_buffer_new(void);
++LIBSSH_API ssh_buffer ssh_buffer_new_size(uint32_t size, uint32_t headroom);
+ LIBSSH_API void ssh_buffer_free(ssh_buffer buffer);
+ #define SSH_BUFFER_FREE(x) \
+     do { if ((x) != NULL) { ssh_buffer_free(x); x = NULL; } } while(0)
+@@ -843,6 +845,8 @@ LIBSSH_API void *ssh_buffer_get(ssh_buffer buffer);
+ LIBSSH_API uint32_t ssh_buffer_get_len(ssh_buffer buffer);
+ LIBSSH_API int ssh_session_set_disconnect_message(ssh_session session, const char *message);
+ 
++typedef ssize_t (*ssh_add_func) (void *ptr, size_t max_bytes, void *userdata);
++
+ #ifndef LIBSSH_LEGACY_0_4
+ #include "libssh/legacy.h"
+ #endif
+diff --git a/include/libssh/session.h b/include/libssh/session.h
+index 97936195..e4a7f80c 100644
+--- a/include/libssh/session.h
++++ b/include/libssh/session.h
+@@ -258,6 +258,7 @@ struct ssh_session_struct {
+         int flags;
+         int exp_flags;
+         int nodelay;
++        char *ccalgo;
+         bool config_processed;
+         uint8_t options_seen[SOC_MAX];
+         uint64_t rekey_data;
+diff --git a/include/libssh/sftp.h b/include/libssh/sftp.h
+index c713466e..e27fe326 100644
+--- a/include/libssh/sftp.h
++++ b/include/libssh/sftp.h
+@@ -565,6 +565,10 @@ LIBSSH_API int sftp_async_read(sftp_file file, void *data, uint32_t len, uint32_
+  */
+ LIBSSH_API ssize_t sftp_write(sftp_file file, const void *buf, size_t count);
+ 
++LIBSSH_API ssize_t sftp_async_write(sftp_file file, ssh_add_func f, size_t count,
++				    void *userdata, uint32_t* id);
++LIBSSH_API int sftp_async_write_end(sftp_file file, uint32_t id, int blocking);
++
+ /**
+  * @brief Seek to a specific location in a file.
+  *
+diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
+index 807313b5..86487087 100644
+--- a/src/CMakeLists.txt
++++ b/src/CMakeLists.txt
+@@ -448,6 +448,11 @@ if (BUILD_STATIC_LIB)
+   if (WIN32)
+     target_compile_definitions(ssh-static PUBLIC "LIBSSH_STATIC")
+   endif (WIN32)
++
++  install(TARGETS ssh-static
++	  EXPORT libssh-config
++	  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
++	  COMPONENT libraries)
+ endif (BUILD_STATIC_LIB)
+ 
+ message(STATUS "Threads_FOUND=${Threads_FOUND}")
+diff --git a/src/buffer.c b/src/buffer.c
+index 8991e006..e0414801 100644
+--- a/src/buffer.c
++++ b/src/buffer.c
+@@ -142,6 +142,40 @@ struct ssh_buffer_struct *ssh_buffer_new(void)
+     return buf;
+ }
+ 
++/**
++ * @brief Create a new SSH buffer with a specified size and headroom.
++ *
++ * @param[in] len       length for newly initialized SSH buffer.
++ * @param[in] headroom  length for headroom
++ * @return A newly initialized SSH buffer, NULL on error.
++ */
++struct ssh_buffer_struct *ssh_buffer_new_size(uint32_t len, uint32_t headroom)
++{
++    struct ssh_buffer_struct *buf = NULL;
++    int rc;
++
++    if (len < headroom)
++	    return NULL;
++
++    buf = calloc(1, sizeof(struct ssh_buffer_struct));
++    if (buf == NULL) {
++        return NULL;
++    }
++
++    rc = ssh_buffer_allocate_size(buf, len);
++    if (rc != 0) {
++        SAFE_FREE(buf);
++        return NULL;
++    }
++
++    buf->pos += headroom;
++    buf->used += headroom;
++
++    buffer_verify(buf);
++
++    return buf;
++}
++
+ /**
+  * @brief Deallocate a SSH buffer.
+  *
+@@ -329,6 +363,49 @@ int ssh_buffer_add_data(struct ssh_buffer_struct *buffer, const void *data, uint
+     return 0;
+ }
+ 
++/**
++ * @brief Add data at the tail of a buffer by an external function
++ *
++ * @param[in]  buffer    The buffer to add data.
++ *
++ * @param[in]  f         function that adds data to the buffer.
++ *
++ * @param[in]  max_bytes The maximum length of the data to add.
++ *
++ * @return               actual bytes added on success, < 0 on error.
++ */
++ssize_t ssh_buffer_add_func(struct ssh_buffer_struct *buffer, ssh_add_func f,
++			    size_t max_bytes, void *userdata)
++{
++    ssize_t actual;
++
++    if (buffer == NULL) {
++        return -1;
++    }
++
++    buffer_verify(buffer);
++
++    if (buffer->used + max_bytes < max_bytes) {
++        return -1;
++    }
++
++    if (buffer->allocated < (buffer->used + max_bytes)) {
++        if (buffer->pos > 0) {
++            buffer_shift(buffer);
++        }
++        if (realloc_buffer(buffer, buffer->used + max_bytes) < 0) {
++            return -1;
++        }
++    }
++
++    if ((actual = f(buffer->data + buffer->used, max_bytes, userdata)) < 0)
++      return -1;
++
++    buffer->used += actual;
++    buffer_verify(buffer);
++    return actual;
++}
++
+ /**
+  * @brief Ensure the buffer has at least a certain preallocated size.
+  *
+diff --git a/src/connect.c b/src/connect.c
+index 15cae644..e7520f40 100644
+--- a/src/connect.c
++++ b/src/connect.c
+@@ -156,6 +156,20 @@ static int set_tcp_nodelay(socket_t socket)
+                       sizeof(opt));
+ }
+ 
++static int set_tcp_ccalgo(socket_t socket, const char *ccalgo)
++{
++#ifdef HAVE_TCP_CONGESTION
++	return setsockopt(socket,
++			  IPPROTO_TCP,
++			  TCP_CONGESTION,
++			  (void *)ccalgo,
++			  strlen(ccalgo));
++#else
++	errno = ENOTSUP;
++	return -1;
++#endif
++}
++
+ /**
+  * @internal
+  *
+@@ -256,6 +270,18 @@ socket_t ssh_connect_host_nonblocking(ssh_session session, const char *host,
+             }
+         }
+ 
++        if (session->opts.ccalgo) {
++	     rc = set_tcp_ccalgo(s, session->opts.ccalgo);
++	     if (rc < 0) {
++		 ssh_set_error(session, SSH_FATAL,
++			       "Failed to set TCP_CONGESTION on socket: %s",
++			       ssh_strerror(errno, err_msg, SSH_ERRNO_MSG_MAX));
++		 ssh_connect_socket_close(s);
++		 s = -1;
++		 continue;
++	     }
++	}
++
+         errno = 0;
+         rc = connect(s, itr->ai_addr, itr->ai_addrlen);
+         if (rc == -1 && (errno != 0) && (errno != EINPROGRESS)) {
+diff --git a/src/options.c b/src/options.c
+index 38511455..a183605d 100644
+--- a/src/options.c
++++ b/src/options.c
+@@ -217,6 +217,7 @@ int ssh_options_copy(ssh_session src, ssh_session *dest)
+     new->opts.gss_delegate_creds    = src->opts.gss_delegate_creds;
+     new->opts.flags                 = src->opts.flags;
+     new->opts.nodelay               = src->opts.nodelay;
++    new->opts.ccalgo                = src->opts.ccalgo;
+     new->opts.config_processed      = src->opts.config_processed;
+     new->common.log_verbosity       = src->common.log_verbosity;
+     new->common.callbacks           = src->common.callbacks;
+@@ -458,6 +459,10 @@ int ssh_options_set_algo(ssh_session session,
+  *                Set it to disable Nagle's Algorithm (TCP_NODELAY) on the
+  *                session socket. (int, 0=false)
+  *
++ *              - SSH_OPTIONS_CCALGO
++ *                Set it to specify TCP congestion control algorithm on the
++ *                session socket (Linux only). (int, 0=false)
++ *
+  *              - SSH_OPTIONS_PROCESS_CONFIG
+  *                Set it to false to disable automatic processing of per-user
+  *                and system-wide OpenSSH configuration files. LibSSH
+@@ -1023,6 +1028,20 @@ int ssh_options_set(ssh_session session, enum ssh_options_e type,
+                 session->opts.nodelay = (*x & 0xff) > 0 ? 1 : 0;
+             }
+             break;
++	case SSH_OPTIONS_CCALGO:
++            v = value;
++            if (v == NULL || v[0] == '\0') {
++                ssh_set_error_invalid(session);
++                return -1;
++            } else {
++                SAFE_FREE(session->opts.ccalgo);
++                session->opts.ccalgo = strdup(v);
++                if (session->opts.ccalgo == NULL) {
++                    ssh_set_error_oom(session);
++                    return -1;
++                }
++            }
++            break;
+         case SSH_OPTIONS_PROCESS_CONFIG:
+             if (value == NULL) {
+                 ssh_set_error_invalid(session);
+diff --git a/src/session.c b/src/session.c
+index 8c509699..88602b6a 100644
+--- a/src/session.c
++++ b/src/session.c
+@@ -108,6 +108,7 @@ ssh_session ssh_new(void)
+     session->opts.fd = -1;
+     session->opts.compressionlevel = 7;
+     session->opts.nodelay = 0;
++    session->opts.ccalgo = NULL;
+ 
+     session->opts.flags = SSH_OPT_FLAG_PASSWORD_AUTH |
+                           SSH_OPT_FLAG_PUBKEY_AUTH |
+diff --git a/src/sftp.c b/src/sftp.c
+index e01012a8..702623a0 100644
+--- a/src/sftp.c
++++ b/src/sftp.c
+@@ -2228,6 +2228,132 @@ ssize_t sftp_write(sftp_file file, const void *buf, size_t count) {
+   return -1; /* not reached */
+ }
+ 
++/*
++ * sftp_async_write is based on and sftp_async_write_end is copied from
++ * https://github.com/limes-datentechnik-gmbh/libssh
++ *
++ * sftp_async_write has some optimizations:
++ * - use ssh_buffer_new_size() to reduce realoc_buffer.
++ * - use ssh_buffer_add_func() to avoid memcpy from read buffer to ssh buffer.
++ */
++ssize_t sftp_async_write(sftp_file file, ssh_add_func f, size_t count, void *userdata,
++			 uint32_t* id) {
++  sftp_session sftp = file->sftp;
++  ssh_buffer buffer;
++  uint32_t buf_sz;
++  ssize_t actual;
++  int len;
++  int packetlen;
++  int rc;
++
++#define HEADROOM 16
++  /* sftp_packet_write() prepends a 5-bytes (uint32_t length and
++   * 1-byte type) header to the head of the payload by
++   * ssh_buffer_prepend_data(). Inserting headroom by
++   * ssh_buffer_new_size() eliminates memcpy for prepending the
++   * header.
++   */
++
++  buf_sz = (HEADROOM + /* for header */
++	    sizeof(uint32_t) + /* id */
++	    ssh_string_len(file->handle) + 4 + /* file->handle */
++	    sizeof(uint64_t) + /* file->offset */
++	    sizeof(uint32_t) + /* count */
++	    count); /* datastring */
++
++  buffer = ssh_buffer_new_size(buf_sz, HEADROOM);
++  if (buffer == NULL) {
++    ssh_set_error_oom(sftp->session);
++    return -1;
++  }
++
++  *id = sftp_get_new_id(file->sftp);
++
++  rc = ssh_buffer_pack(buffer,
++                       "dSqd",
++                       *id,
++                       file->handle,
++                       file->offset,
++                       count); /* len of datastring */
++
++  if (rc != SSH_OK){
++    ssh_set_error_oom(sftp->session);
++    ssh_buffer_free(buffer);
++    return SSH_ERROR;
++  }
++
++  actual = ssh_buffer_add_func(buffer, f, count, userdata);
++  if (actual < 0){
++    ssh_set_error_oom(sftp->session);
++    ssh_buffer_free(buffer);
++    return SSH_ERROR;
++  }
++
++  packetlen=ssh_buffer_get_len(buffer)+5;
++  len = sftp_packet_write(file->sftp, SSH_FXP_WRITE, buffer);
++  ssh_buffer_free(buffer);
++  if (len < 0) {
++    return SSH_ERROR;
++  } else  if (len != packetlen) {
++    ssh_set_error(sftp->session, SSH_FATAL,
++      "Could only send %d of %d bytes to remote host!", len, packetlen);
++    SSH_LOG(SSH_LOG_PACKET,
++        "Could not write as much data as expected");
++    return SSH_ERROR;
++  }
++
++  file->offset += actual;
++
++  return actual;
++}
++
++int sftp_async_write_end(sftp_file file, uint32_t id, int blocking) {
++  sftp_session sftp = file->sftp;
++  sftp_message msg = NULL;
++  sftp_status_message status;
++
++  msg = sftp_dequeue(sftp, id);
++  while (msg == NULL) {
++    if (!blocking && ssh_channel_poll(sftp->channel, 0) == 0) {
++      /* we cannot block */
++      return SSH_AGAIN;
++    }
++    if (sftp_read_and_dispatch(sftp) < 0) {
++      /* something nasty has happened */
++      return SSH_ERROR;
++    }
++    msg = sftp_dequeue(sftp, id);
++  }
++
++  switch (msg->packet_type) {
++    case SSH_FXP_STATUS:
++      status = parse_status_msg(msg);
++      sftp_message_free(msg);
++      if (status == NULL) {
++        return SSH_ERROR;
++      }
++      sftp_set_error(sftp, status->status);
++      switch (status->status) {
++        case SSH_FX_OK:
++          status_msg_free(status);
++          return SSH_OK;
++        default:
++          break;
++      }
++      ssh_set_error(sftp->session, SSH_REQUEST_DENIED,
++          "SFTP server: %s", status->errormsg);
++      status_msg_free(status);
++      return SSH_ERROR;
++    default:
++      ssh_set_error(sftp->session, SSH_FATAL,
++          "Received message %d during write!", msg->packet_type);
++      sftp_message_free(msg);
++      return SSH_ERROR;
++  }
++
++  return SSH_ERROR; /* not reached */
++}
++
+ /* Seek to a specific location in a file. */
+ int sftp_seek(sftp_file file, uint32_t new_offset) {
+   if (file == NULL) {

patch/libssh-0.11.2.patch

@@ -0,0 +1,605 @@
+diff --git a/ConfigureChecks.cmake b/ConfigureChecks.cmake
+index 8765dc6e..766e7d16 100644
+--- a/ConfigureChecks.cmake
++++ b/ConfigureChecks.cmake
+@@ -209,6 +209,7 @@ if (UNIX)
+     check_library_exists(util forkpty "" HAVE_LIBUTIL)
+     check_function_exists(cfmakeraw HAVE_CFMAKERAW)
+     check_function_exists(__strtoull HAVE___STRTOULL)
++    check_symbol_exists(TCP_CONGESTION "netinet/tcp.h" HAVE_TCP_CONGESTION)
+ endif (UNIX)
+ 
+ set(LIBSSH_REQUIRED_LIBRARIES ${_REQUIRED_LIBRARIES} CACHE INTERNAL "libssh required system libraries")
+diff --git a/config.h.cmake b/config.h.cmake
+index 8dce5273..ef534762 100644
+--- a/config.h.cmake
++++ b/config.h.cmake
+@@ -219,6 +219,8 @@
+ 
+ #cmakedefine HAVE_GCC_BOUNDED_ATTRIBUTE 1
+ 
++#cmakedefine HAVE_TCP_CONGESTION 1
++
+ /* Define to 1 if you want to enable GSSAPI */
+ #cmakedefine WITH_GSSAPI 1
+ 
+diff --git a/include/libssh/buffer.h b/include/libssh/buffer.h
+index d22178e7..2d6aa0a7 100644
+--- a/include/libssh/buffer.h
++++ b/include/libssh/buffer.h
+@@ -37,6 +37,8 @@ int ssh_buffer_add_u8(ssh_buffer buffer, uint8_t data);
+ int ssh_buffer_add_u16(ssh_buffer buffer, uint16_t data);
+ int ssh_buffer_add_u32(ssh_buffer buffer, uint32_t data);
+ int ssh_buffer_add_u64(ssh_buffer buffer, uint64_t data);
++ssize_t ssh_buffer_add_func(ssh_buffer buffer, ssh_add_func f, size_t max_bytes,
++			    void *userdata);
+ 
+ int ssh_buffer_validate_length(struct ssh_buffer_struct *buffer, size_t len);
+ 
+diff --git a/include/libssh/libssh.h b/include/libssh/libssh.h
+index 3bddb019..1d5d7761 100644
+--- a/include/libssh/libssh.h
++++ b/include/libssh/libssh.h
+@@ -373,6 +373,7 @@ enum ssh_options_e {
+     SSH_OPTIONS_HOST,
+     SSH_OPTIONS_PORT,
+     SSH_OPTIONS_PORT_STR,
++    SSH_OPTIONS_AI_FAMILY,
+     SSH_OPTIONS_FD,
+     SSH_OPTIONS_USER,
+     SSH_OPTIONS_SSH_DIR,
+@@ -407,6 +408,7 @@ enum ssh_options_e {
+     SSH_OPTIONS_GSSAPI_AUTH,
+     SSH_OPTIONS_GLOBAL_KNOWNHOSTS,
+     SSH_OPTIONS_NODELAY,
++    SSH_OPTIONS_CCALGO,
+     SSH_OPTIONS_PUBLICKEY_ACCEPTED_TYPES,
+     SSH_OPTIONS_PROCESS_CONFIG,
+     SSH_OPTIONS_REKEY_DATA,
+@@ -876,6 +878,7 @@ LIBSSH_API const char* ssh_get_hmac_in(ssh_session session);
+ LIBSSH_API const char* ssh_get_hmac_out(ssh_session session);
+ 
+ LIBSSH_API ssh_buffer ssh_buffer_new(void);
++LIBSSH_API ssh_buffer ssh_buffer_new_size(uint32_t size, uint32_t headroom);
+ LIBSSH_API void ssh_buffer_free(ssh_buffer buffer);
+ #define SSH_BUFFER_FREE(x) \
+     do { if ((x) != NULL) { ssh_buffer_free(x); x = NULL; } } while(0)
+@@ -886,6 +889,12 @@ LIBSSH_API void *ssh_buffer_get(ssh_buffer buffer);
+ LIBSSH_API uint32_t ssh_buffer_get_len(ssh_buffer buffer);
+ LIBSSH_API int ssh_session_set_disconnect_message(ssh_session session, const char *message);
+ 
++typedef ssize_t (*ssh_add_func) (void *ptr, size_t max_bytes, void *userdata);
++
++LIBSSH_API const char **ssh_ciphers(void);
++LIBSSH_API const char **ssh_hmacs(void);
++LIBSSH_API void ssh_use_openssh_proxy_jumps(int);
++
+ #ifndef LIBSSH_LEGACY_0_4
+ #include "libssh/legacy.h"
+ #endif
+diff --git a/include/libssh/session.h b/include/libssh/session.h
+index aed94072..327cf4fe 100644
+--- a/include/libssh/session.h
++++ b/include/libssh/session.h
+@@ -255,6 +255,7 @@ struct ssh_session_struct {
+         unsigned long timeout; /* seconds */
+         unsigned long timeout_usec;
+         uint16_t port;
++        int  ai_family;
+         socket_t fd;
+         int StrictHostKeyChecking;
+         char compressionlevel;
+@@ -264,6 +265,7 @@ struct ssh_session_struct {
+         int flags;
+         int exp_flags;
+         int nodelay;
++        char *ccalgo;
+         bool config_processed;
+         uint8_t options_seen[SOC_MAX];
+         uint64_t rekey_data;
+diff --git a/include/libssh/sftp.h b/include/libssh/sftp.h
+index cf4458c3..1a864795 100644
+--- a/include/libssh/sftp.h
++++ b/include/libssh/sftp.h
+@@ -569,6 +569,10 @@ SSH_DEPRECATED LIBSSH_API int sftp_async_read(sftp_file file,
+                                               uint32_t len,
+                                               uint32_t id);
+ 
++LIBSSH_API ssize_t sftp_async_write(sftp_file file, ssh_add_func f, size_t count,
++				    void *userdata, uint32_t* id);
++LIBSSH_API int sftp_async_write_end(sftp_file file, uint32_t id, int blocking);
++
+ /**
+  * @brief Write to a file using an opened sftp file handle.
+  *
+diff --git a/src/buffer.c b/src/buffer.c
+index 449fa941..f49e8af6 100644
+--- a/src/buffer.c
++++ b/src/buffer.c
+@@ -142,6 +142,40 @@ struct ssh_buffer_struct *ssh_buffer_new(void)
+     return buf;
+ }
+ 
++/**
++ * @brief Create a new SSH buffer with a specified size and headroom.
++ *
++ * @param[in] len       length for newly initialized SSH buffer.
++ * @param[in] headroom  length for headroom
++ * @return A newly initialized SSH buffer, NULL on error.
++ */
++struct ssh_buffer_struct *ssh_buffer_new_size(uint32_t len, uint32_t headroom)
++{
++    struct ssh_buffer_struct *buf = NULL;
++    int rc;
++
++    if (len < headroom)
++	    return NULL;
++
++    buf = calloc(1, sizeof(struct ssh_buffer_struct));
++    if (buf == NULL) {
++        return NULL;
++    }
++
++    rc = ssh_buffer_allocate_size(buf, len);
++    if (rc != 0) {
++        SAFE_FREE(buf);
++        return NULL;
++    }
++
++    buf->pos += headroom;
++    buf->used += headroom;
++
++    buffer_verify(buf);
++
++    return buf;
++}
++
+ /**
+  * @brief Deallocate a SSH buffer.
+  *
+@@ -329,6 +363,49 @@ int ssh_buffer_add_data(struct ssh_buffer_struct *buffer, const void *data, uint
+     return 0;
+ }
+ 
++/**
++ * @brief Add data at the tail of a buffer by an external function
++ *
++ * @param[in]  buffer    The buffer to add data.
++ *
++ * @param[in]  f         function that adds data to the buffer.
++ *
++ * @param[in]  max_bytes The maximum length of the data to add.
++ *
++ * @return               actual bytes added on success, < 0 on error.
++ */
++ssize_t ssh_buffer_add_func(struct ssh_buffer_struct *buffer, ssh_add_func f,
++			    size_t max_bytes, void *userdata)
++{
++    ssize_t actual;
++
++    if (buffer == NULL) {
++        return -1;
++    }
++
++    buffer_verify(buffer);
++
++    if (buffer->used + max_bytes < max_bytes) {
++        return -1;
++    }
++
++    if (buffer->allocated < (buffer->used + max_bytes)) {
++        if (buffer->pos > 0) {
++            buffer_shift(buffer);
++        }
++        if (realloc_buffer(buffer, buffer->used + max_bytes) < 0) {
++            return -1;
++        }
++    }
++
++    if ((actual = f(buffer->data + buffer->used, max_bytes, userdata)) < 0)
++      return -1;
++
++    buffer->used += actual;
++    buffer_verify(buffer);
++    return actual;
++}
++
+ /**
+  * @brief Ensure the buffer has at least a certain preallocated size.
+  *
+diff --git a/src/connect.c b/src/connect.c
+index 2cb64037..51f4c87e 100644
+--- a/src/connect.c
++++ b/src/connect.c
+@@ -109,7 +109,7 @@ static int ssh_connect_socket_close(socket_t s)
+ #endif
+ }
+ 
+-static int getai(const char *host, int port, struct addrinfo **ai)
++static int getai(const char *host, int port, int ai_family, struct addrinfo **ai)
+ {
+     const char *service = NULL;
+     struct addrinfo hints;
+@@ -118,7 +118,7 @@ static int getai(const char *host, int port, struct addrinfo **ai)
+     ZERO_STRUCT(hints);
+ 
+     hints.ai_protocol = IPPROTO_TCP;
+-    hints.ai_family = PF_UNSPEC;
++    hints.ai_family = ai_family > 0 ? ai_family : PF_UNSPEC;
+     hints.ai_socktype = SOCK_STREAM;
+ 
+     if (port == 0) {
+@@ -151,6 +151,20 @@ static int set_tcp_nodelay(socket_t socket)
+                       sizeof(opt));
+ }
+ 
++static int set_tcp_ccalgo(socket_t socket, const char *ccalgo)
++{
++#ifdef HAVE_TCP_CONGESTION
++	return setsockopt(socket,
++			  IPPROTO_TCP,
++			  TCP_CONGESTION,
++			  (void *)ccalgo,
++			  strlen(ccalgo));
++#else
++	errno = ENOTSUP;
++	return -1;
++#endif
++}
++
+ /**
+  * @internal
+  *
+@@ -168,7 +182,7 @@ socket_t ssh_connect_host_nonblocking(ssh_session session, const char *host,
+     struct addrinfo *ai = NULL;
+     struct addrinfo *itr = NULL;
+ 
+-    rc = getai(host, port, &ai);
++    rc = getai(host, port, session->opts.ai_family, &ai);
+     if (rc != 0) {
+         ssh_set_error(session, SSH_FATAL,
+                       "Failed to resolve hostname %s (%s)",
+@@ -194,7 +208,7 @@ socket_t ssh_connect_host_nonblocking(ssh_session session, const char *host,
+ 
+             SSH_LOG(SSH_LOG_PACKET, "Resolving %s", bind_addr);
+ 
+-            rc = getai(bind_addr, 0, &bind_ai);
++            rc = getai(bind_addr, 0, session->opts.ai_family, &bind_ai);
+             if (rc != 0) {
+                 ssh_set_error(session, SSH_FATAL,
+                               "Failed to resolve bind address %s (%s)",
+@@ -251,6 +265,18 @@ socket_t ssh_connect_host_nonblocking(ssh_session session, const char *host,
+             }
+         }
+ 
++        if (session->opts.ccalgo) {
++	     rc = set_tcp_ccalgo(s, session->opts.ccalgo);
++	     if (rc < 0) {
++		 ssh_set_error(session, SSH_FATAL,
++			       "Failed to set TCP_CONGESTION on socket: %s",
++			       ssh_strerror(errno, err_msg, SSH_ERRNO_MSG_MAX));
++		 ssh_connect_socket_close(s);
++		 s = -1;
++		 continue;
++	     }
++	}
++
+         errno = 0;
+         rc = connect(s, itr->ai_addr, itr->ai_addrlen);
+         if (rc == -1) {
+diff --git a/src/misc.c b/src/misc.c
+index 774211fb..74e57959 100644
+--- a/src/misc.c
++++ b/src/misc.c
+@@ -71,6 +71,8 @@
+ #include "libssh/priv.h"
+ #include "libssh/misc.h"
+ #include "libssh/session.h"
++#include "libssh/wrapper.h"
++#include "libssh/crypto.h"
+ 
+ #ifdef HAVE_LIBGCRYPT
+ #define GCRYPT_STRING "/gcrypt"
+@@ -2054,6 +2056,42 @@ ssize_t ssh_readn(int fd, void *buf, size_t nbytes)
+     return total_bytes_read;
+ }
+ 
++/**
++ * @brief Return supported cipher names
++ * @return	The list of cipher names.
++ */
++const char **ssh_ciphers(void)
++{
++     struct ssh_cipher_struct *tab=ssh_get_ciphertab();
++     static const char *ciphers[32];
++     int n;
++
++     memset(ciphers, 0, sizeof(*ciphers));
++
++     for (n = 0; tab[n].name != NULL; n++) {
++	  ciphers[n] = tab[n].name;
++     }
++     return ciphers;
++}
++
++/**
++ * @brief Return supported hmac names
++ * @return	The list of hmac names.
++ */
++const char **ssh_hmacs(void)
++{
++     struct ssh_hmac_struct *tab=ssh_get_hmactab();
++     static const char *hmacs[32];
++     int n;
++
++     memset(hmacs, 0, sizeof(*hmacs));
++
++     for (n = 0; tab[n].name != NULL; n++) {
++	  hmacs[n] = tab[n].name;
++     }
++     return hmacs;
++}
++
+ /**
+  * @brief Write the requested number of bytes to a local file.
+  *
+@@ -2227,6 +2265,17 @@ ssh_proxyjumps_free(struct ssh_list *proxy_jump_list)
+     }
+ }
+ 
++static bool force_openssh_proxy_jumps;
++
++/**
++ * @breif set use openssh proxy jumps without the OPENSSH_PROXYJUMP env var
++ */
++void
++ssh_use_openssh_proxy_jumps(int v)
++{
++    force_openssh_proxy_jumps = (v > 0);
++}
++
+ /**
+  * @brief Check if libssh proxy jumps is enabled
+  *
+@@ -2241,7 +2290,12 @@ ssh_libssh_proxy_jumps(void)
+ {
+     const char *t = getenv("OPENSSH_PROXYJUMP");
+ 
++    if (force_openssh_proxy_jumps)
++	    return false;
++
+     return !(t != NULL && t[0] == '1');
+ }
+ 
++
++
+ /** @} */
+diff --git a/src/options.c b/src/options.c
+index 785296dd..a82d4d81 100644
+--- a/src/options.c
++++ b/src/options.c
+@@ -251,6 +251,7 @@ int ssh_options_copy(ssh_session src, ssh_session *dest)
+     new->opts.gss_delegate_creds    = src->opts.gss_delegate_creds;
+     new->opts.flags                 = src->opts.flags;
+     new->opts.nodelay               = src->opts.nodelay;
++    new->opts.ccalgo                = src->opts.ccalgo;
+     new->opts.config_processed      = src->opts.config_processed;
+     new->opts.control_master        = src->opts.control_master;
+     new->common.log_verbosity       = src->common.log_verbosity;
+@@ -326,6 +327,9 @@ int ssh_options_set_algo(ssh_session session,
+  *              - SSH_OPTIONS_PORT_STR:
+  *                The port to connect to (const char *).
+  *
++ *              - SSH_OPTIONS_AI_FAMILY:
++ *                The address family for connecting (int *).
++ *
+  *              - SSH_OPTIONS_FD:
+  *                The file descriptor to use (socket_t).\n
+  *                \n
+@@ -571,6 +575,10 @@ int ssh_options_set_algo(ssh_session session,
+  *                Set it to disable Nagle's Algorithm (TCP_NODELAY) on the
+  *                session socket. (int, 0=false)
+  *
++ *              - SSH_OPTIONS_CCALGO
++ *                Set it to specify TCP congestion control algorithm on the
++ *                session socket (Linux only). (int, 0=false)
++ *
+  *              - SSH_OPTIONS_PROCESS_CONFIG
+  *                Set it to false to disable automatic processing of per-user
+  *                and system-wide OpenSSH configuration files. LibSSH
+@@ -727,6 +735,21 @@ int ssh_options_set(ssh_session session, enum ssh_options_e type,
+                 session->opts.port = i & 0xffffU;
+             }
+             break;
++        case SSH_OPTIONS_AI_FAMILY:
++            if (value == NULL) {
++                session->opts.ai_family = 0;
++                ssh_set_error_invalid(session);
++                return -1;
++            } else {
++                int *x = (int *) value;
++                if (*x < 0) {
++		    session->opts.ai_family = 0;
++                    ssh_set_error_invalid(session);
++                    return -1;
++                }
++                session->opts.ai_family = *x;
++            }
++            break;
+         case SSH_OPTIONS_FD:
+             if (value == NULL) {
+                 session->opts.fd = SSH_INVALID_SOCKET;
+@@ -1241,6 +1264,20 @@ int ssh_options_set(ssh_session session, enum ssh_options_e type,
+                 session->opts.nodelay = (*x & 0xff) > 0 ? 1 : 0;
+             }
+             break;
++	case SSH_OPTIONS_CCALGO:
++            v = value;
++            if (v == NULL || v[0] == '\0') {
++                ssh_set_error_invalid(session);
++                return -1;
++            } else {
++                SAFE_FREE(session->opts.ccalgo);
++                session->opts.ccalgo = strdup(v);
++                if (session->opts.ccalgo == NULL) {
++                    ssh_set_error_oom(session);
++                    return -1;
++                }
++            }
++            break;
+         case SSH_OPTIONS_PROCESS_CONFIG:
+             if (value == NULL) {
+                 ssh_set_error_invalid(session);
+diff --git a/src/session.c b/src/session.c
+index 9fd5d946..ed9f908e 100644
+--- a/src/session.c
++++ b/src/session.c
+@@ -107,9 +107,11 @@ ssh_session ssh_new(void)
+     /* OPTIONS */
+     session->opts.StrictHostKeyChecking = 1;
+     session->opts.port = 22;
++    session->opts.ai_family = 0;
+     session->opts.fd = -1;
+     session->opts.compressionlevel = 7;
+     session->opts.nodelay = 0;
++    session->opts.ccalgo = NULL;
+     session->opts.identities_only = false;
+     session->opts.control_master = SSH_CONTROL_MASTER_NO;
+ 
+diff --git a/src/sftp.c b/src/sftp.c
+index 37b4133b..12b6d296 100644
+--- a/src/sftp.c
++++ b/src/sftp.c
+@@ -1488,6 +1488,132 @@ ssize_t sftp_write(sftp_file file, const void *buf, size_t count) {
+   return -1; /* not reached */
+ }
+ 
++/*
++ * sftp_async_write is based on and sftp_async_write_end is copied from
++ * https://github.com/limes-datentechnik-gmbh/libssh
++ *
++ * sftp_async_write has some optimizations:
++ * - use ssh_buffer_new_size() to reduce realoc_buffer.
++ * - use ssh_buffer_add_func() to avoid memcpy from read buffer to ssh buffer.
++ */
++ssize_t sftp_async_write(sftp_file file, ssh_add_func f, size_t count, void *userdata,
++			 uint32_t* id) {
++  sftp_session sftp = file->sftp;
++  ssh_buffer buffer;
++  uint32_t buf_sz;
++  ssize_t actual;
++  int len;
++  int packetlen;
++  int rc;
++
++#define HEADROOM 16
++  /* sftp_packet_write() prepends a 5-bytes (uint32_t length and
++   * 1-byte type) header to the head of the payload by
++   * ssh_buffer_prepend_data(). Inserting headroom by
++   * ssh_buffer_new_size() eliminates memcpy for prepending the
++   * header.
++   */
++
++  buf_sz = (HEADROOM + /* for header */
++	    sizeof(uint32_t) + /* id */
++	    ssh_string_len(file->handle) + 4 + /* file->handle */
++	    sizeof(uint64_t) + /* file->offset */
++	    sizeof(uint32_t) + /* count */
++	    count); /* datastring */
++
++  buffer = ssh_buffer_new_size(buf_sz, HEADROOM);
++  if (buffer == NULL) {
++    ssh_set_error_oom(sftp->session);
++    return -1;
++  }
++
++  *id = sftp_get_new_id(file->sftp);
++
++  rc = ssh_buffer_pack(buffer,
++                       "dSqd",
++                       *id,
++                       file->handle,
++                       file->offset,
++                       count); /* len of datastring */
++
++  if (rc != SSH_OK){
++    ssh_set_error_oom(sftp->session);
++    ssh_buffer_free(buffer);
++    return SSH_ERROR;
++  }
++
++  actual = ssh_buffer_add_func(buffer, f, count, userdata);
++  if (actual < 0){
++    ssh_set_error_oom(sftp->session);
++    ssh_buffer_free(buffer);
++    return SSH_ERROR;
++  }
++
++  packetlen=ssh_buffer_get_len(buffer)+5;
++  len = sftp_packet_write(file->sftp, SSH_FXP_WRITE, buffer);
++  ssh_buffer_free(buffer);
++  if (len < 0) {
++    return SSH_ERROR;
++  } else  if (len != packetlen) {
++    ssh_set_error(sftp->session, SSH_FATAL,
++      "Could only send %d of %d bytes to remote host!", len, packetlen);
++    SSH_LOG(SSH_LOG_PACKET,
++        "Could not write as much data as expected");
++    return SSH_ERROR;
++  }
++
++  file->offset += actual;
++
++  return actual;
++}
++
++int sftp_async_write_end(sftp_file file, uint32_t id, int blocking) {
++  sftp_session sftp = file->sftp;
++  sftp_message msg = NULL;
++  sftp_status_message status;
++
++  msg = sftp_dequeue(sftp, id);
++  while (msg == NULL) {
++    if (!blocking && ssh_channel_poll(sftp->channel, 0) == 0) {
++      /* we cannot block */
++      return SSH_AGAIN;
++    }
++    if (sftp_read_and_dispatch(sftp) < 0) {
++      /* something nasty has happened */
++      return SSH_ERROR;
++    }
++    msg = sftp_dequeue(sftp, id);
++  }
++
++  switch (msg->packet_type) {
++    case SSH_FXP_STATUS:
++      status = parse_status_msg(msg);
++      sftp_message_free(msg);
++      if (status == NULL) {
++        return SSH_ERROR;
++      }
++      sftp_set_error(sftp, status->status);
++      switch (status->status) {
++        case SSH_FX_OK:
++          status_msg_free(status);
++          return SSH_OK;
++        default:
++          break;
++      }
++      ssh_set_error(sftp->session, SSH_REQUEST_DENIED,
++          "SFTP server: %s", status->errormsg);
++      status_msg_free(status);
++      return SSH_ERROR;
++    default:
++      ssh_set_error(sftp->session, SSH_FATAL,
++          "Received message %d during write!", msg->packet_type);
++      sftp_message_free(msg);
++      return SSH_ERROR;
++  }
++
++  return SSH_ERROR; /* not reached */
++}
++
+ /* Seek to a specific location in a file. */
+ int sftp_seek(sftp_file file, uint32_t new_offset) {
+   if (file == NULL) {

patch/libssh-0.9.6.patch

@@ -0,0 +1,164 @@
+diff --git a/DefineOptions.cmake b/DefineOptions.cmake
+index b82a5018..f1f2ab9d 100644
+--- a/DefineOptions.cmake
++++ b/DefineOptions.cmake
+@@ -15,13 +15,14 @@ option(UNIT_TESTING "Build with unit tests" OFF)
+ option(CLIENT_TESTING "Build with client tests; requires openssh" OFF)
+ option(SERVER_TESTING "Build with server tests; requires openssh and dropbear" OFF)
+ option(WITH_BENCHMARKS "Build benchmarks tools" OFF)
+-option(WITH_EXAMPLES "Build examples" ON)
++option(WITH_EXAMPLES "Build examples" OFF)
+ option(WITH_NACL "Build with libnacl (curve25519)" ON)
+ option(WITH_SYMBOL_VERSIONING "Build with symbol versioning" ON)
+ option(WITH_ABI_BREAK "Allow ABI break" OFF)
+ option(WITH_GEX "Enable DH Group exchange mechanisms" ON)
+ option(FUZZ_TESTING "Build with fuzzer for the server" OFF)
+ option(PICKY_DEVELOPER "Build with picky developer flags" OFF)
++option(WITH_STATIC_LIB "Build static library" ON)
+ 
+ if (WITH_ZLIB)
+     set(WITH_LIBZ ON)
+@@ -53,3 +54,7 @@ endif (NOT GLOBAL_BIND_CONFIG)
+ if (NOT GLOBAL_CLIENT_CONFIG)
+   set(GLOBAL_CLIENT_CONFIG "/etc/ssh/ssh_config")
+ endif (NOT GLOBAL_CLIENT_CONFIG)
++
++if (WITH_STATIC_LIB)
++  set(BUILD_STATIC_LIB ON)
++endif()
+diff --git a/include/libssh/sftp.h b/include/libssh/sftp.h
+index 8c14b21d..95ac1d6b 100644
+--- a/include/libssh/sftp.h
++++ b/include/libssh/sftp.h
+@@ -565,6 +565,9 @@ LIBSSH_API int sftp_async_read(sftp_file file, void *data, uint32_t len, uint32_
+  */
+ LIBSSH_API ssize_t sftp_write(sftp_file file, const void *buf, size_t count);
+ 
++LIBSSH_API int sftp_async_write(sftp_file file, const void *buf, size_t count, uint32_t* id);
++LIBSSH_API int sftp_async_write_end(sftp_file file, uint32_t id, int blocking);
++
+ /**
+  * @brief Seek to a specific location in a file.
+  *
+diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
+index a576cf71..303a1c7f 100644
+--- a/src/CMakeLists.txt
++++ b/src/CMakeLists.txt
+@@ -412,6 +412,10 @@ if (BUILD_STATIC_LIB)
+   if (WIN32)
+     target_compile_definitions(ssh-static PUBLIC "LIBSSH_STATIC")
+   endif (WIN32)
++  install(TARGETS ssh-static
++	EXPORT libssh-config
++	LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
++	COMPONENT libraries)
+ endif (BUILD_STATIC_LIB)
+ 
+ message(STATUS "Threads_FOUND=${Threads_FOUND}")
+diff --git a/src/sftp.c b/src/sftp.c
+index a8346040..a4261ec9 100644
+--- a/src/sftp.c
++++ b/src/sftp.c
+@@ -2234,6 +2234,102 @@ ssize_t sftp_write(sftp_file file, const void *buf, size_t count) {
+   return -1; /* not reached */
+ }
+ 
++/*
++ * sftp_async_write and sftp_async_write_end are copied from
++ * https://github.com/limes-datentechnik-gmbh/libssh
++ */
++int sftp_async_write(sftp_file file, const void *buf, size_t count, uint32_t* id) {
++  sftp_session sftp = file->sftp;
++  ssh_buffer buffer;
++  int len;
++  int packetlen;
++  int rc;
++
++  buffer = ssh_buffer_new();
++  if (buffer == NULL) {
++    ssh_set_error_oom(sftp->session);
++    return -1;
++  }
++
++  *id = sftp_get_new_id(file->sftp);
++
++  rc = ssh_buffer_pack(buffer,
++                       "dSqdP",
++                       *id,
++                       file->handle,
++                       file->offset,
++                       count, /* len of datastring */
++                       (size_t)count, buf);
++  if (rc != SSH_OK){
++    ssh_set_error_oom(sftp->session);
++    ssh_buffer_free(buffer);
++    return SSH_ERROR;
++  }
++  packetlen=ssh_buffer_get_len(buffer)+5;
++  len = sftp_packet_write(file->sftp, SSH_FXP_WRITE, buffer);
++  ssh_buffer_free(buffer);
++  if (len < 0) {
++    return SSH_ERROR;
++  } else  if (len != packetlen) {
++    ssh_set_error(sftp->session, SSH_FATAL,
++      "Could only send %d of %d bytes to remote host!", len, packetlen);
++    SSH_LOG(SSH_LOG_PACKET,
++        "Could not write as much data as expected");
++    return SSH_ERROR;
++  }
++
++  file->offset += count;
++
++  return SSH_OK;
++}
++
++int sftp_async_write_end(sftp_file file, uint32_t id, int blocking) {
++  sftp_session sftp = file->sftp;
++  sftp_message msg = NULL;
++  sftp_status_message status;
++
++  msg = sftp_dequeue(sftp, id);
++  while (msg == NULL) {
++    if (!blocking && ssh_channel_poll(sftp->channel, 0) == 0) {
++      /* we cannot block */
++      return SSH_AGAIN;
++    }
++    if (sftp_read_and_dispatch(sftp) < 0) {
++      /* something nasty has happened */
++      return SSH_ERROR;
++    }
++    msg = sftp_dequeue(sftp, id);
++  }
++
++  switch (msg->packet_type) {
++    case SSH_FXP_STATUS:
++      status = parse_status_msg(msg);
++      sftp_message_free(msg);
++      if (status == NULL) {
++        return SSH_ERROR;
++      }
++      sftp_set_error(sftp, status->status);
++      switch (status->status) {
++        case SSH_FX_OK:
++          status_msg_free(status);
++          return SSH_OK;
++        default:
++          break;
++      }
++      ssh_set_error(sftp->session, SSH_REQUEST_DENIED,
++          "SFTP server: %s", status->errormsg);
++      status_msg_free(status);
++      return SSH_ERROR;
++    default:
++      ssh_set_error(sftp->session, SSH_FATAL,
++          "Received message %d during write!", msg->packet_type);
++      sftp_message_free(msg);
++      return SSH_ERROR;
++  }
++
++  return SSH_ERROR; /* not reached */
++}
++
+ /* Seek to a specific location in a file. */
+ int sftp_seek(sftp_file file, uint32_t new_offset) {
+   if (file == NULL) {

rpm/.gitignore

@@ -0,0 +1,3 @@
+
+# generated by cmake
+mscp.spec

rpm/mscp.spec.in

@@ -0,0 +1,63 @@
+Name:		mscp
+Version:	@MSCP_VERSION@
+Release:	1%{?dist}
+Summary:	mscp, fast file transfer over multiple SSH connections
+
+Group:		Applications/Internet
+License:	GPLv3
+URL:		https://github.com/upa/mscp
+Source0:	%{name}-%{version}.tar.gz
+
+BuildRequires:	gcc make cmake zlib-devel openssl-devel
+Requires:	glibc crypto-policies krb5-libs openssl-libs libcom_err
+
+%description
+mscp transfers files over multiple SSH connections. Multiple threads
+and connections in mscp transfer (1) multiple files simultaneously
+and (2) a large file in parallel. It would shorten the waiting time
+for transferring a lot of/large files over networks.
+
+
+%global debug_package %{nil}
+
+%prep
+%setup -q
+
+
+%build
+cmake -S . -B build -DINSTALL_EXECUTABLE_ONLY=ON
+make -C build %{?_smp_mflags}
+
+
+%install
+make -C build install DESTDIR=%{buildroot}
+
+%files
+/usr/local/bin/mscp
+/usr/local/share/man/man1/mscp.1
+
+
+%changelog
+* Sat Nov 08 2025 Ryo Nakamura <upa@haeena.net> - 0.2.4-1
+- RPM release for v0.2.4
+
+* Tue Aug 12 2025 Ryo Nakamura <upa@haeena.net> - 0.2.3-1
+- RPM release for v0.2.3
+
+* Wed Apr 16 2025 Ryo Nakamura <upa@haeena.net> - 0.2.2-1
+- RPM release for v0.2.2
+
+* Sat May 11 2024 Ryo Nakamura <upa@haeena.net> - 0.2.1-1
+- RPM release for v0.2.1
+
+* Mon Apr 15 2024 Ryo Nakamura <upa@haeena.net> - 0.2.0-1
+- RPM release for v0.2.0
+
+* Thu Mar 14 2024 Ryo Nakamura <upa@haeena.net> - 0.1.5-0
+- RPM release for v0.1.5
+
+* Wed Feb 07 2024 Ryo Nakamura <upa@haeena.net> - 0.1.4-0
+- RPM release for v0.1.4
+
+* Sat Feb 03 2024 Ryo Nakamura <upa@haeena.net> - 0.1.3-0
+- Initial release for rpm packaging

scripts/install-build-deps.sh

@@ -1,30 +1,77 @@
-#!/bin/bash -eu
+#!/usr/bin/env bash
 #
 # Install build dpenedencies.
 
+set -e
+#set -u
+
+function print_help() {
+	echo "$0 [options]"
+	echo "    --dont-install         Print required packages."
+	echo "    --platform [PLATFORM]  PLATFORM is Kernel-ID, e.g., Linux-ubuntu."
+	echo "                           Automatically detected if not specified."
+}
+
 platform=$(uname -s)
+doinstall=1
 
 if [ -e /etc/os-release ]; then
 	source /etc/os-release
 	platform=${platform}-${ID}
 fi
 
-set -x
+while getopts h-: opt; do
+        optarg="${!OPTIND}"
+        [[ "$opt" = - ]] && opt="-$OPTARG"
+	case "-${opt}" in
+		--dont-install)
+			doinstall=0
+			;;
+		--platform)
+			platform=$optarg
+			shift
+			;;
+		-h)
+			print_help
+			exit 0
+			;;
+		*)
+			print_help
+			exit 1
+			;;
+	esac
+done
 
 case $platform in
 	Darwin)
-		brew install openssl@1.1
+		cmd="brew install"
+		pkgs="openssl@3"
 		;;
-	Linux-ubuntu*)
-		apt-get install -y \
-			gcc make cmake zlib1g-dev libssl-dev libkrb5-dev
+	Linux-ubuntu* | Linux-debian* | Linux-devuan*)
+		cmd="apt-get install --no-install-recommends -y"
+		pkgs="gcc make cmake zlib1g-dev libssl-dev libkrb5-dev"
 		;;
-	Linux-centos* | Linux-rhel* | Linux-rocky*)
-		yum install -y \
-			gcc make cmake zlib-devel openssl-devel rpm-build
+	Linux-centos* | Linux-rhel* | Linux-rocky* | Linux-almalinux)
+		cmd="yum install -y"
+		pkgs="gcc make cmake zlib-devel openssl-devel rpm-build"
+		;;
+	Linux-arch*)
+		cmd="pacman --no-confirm -S"
+		pkgs="gcc make cmake"
+		;;
+	FreeBSD-freebsd)
+		cmd="pkg install"
+		pkgs="cmake"
 		;;
 	*)
 		echo "unsupported platform: $platform"
 		exit 1
 		;;
 esac
+
+if [ $doinstall -gt 0 ]; then
+	echo do "$cmd $pkgs"
+	$cmd $pkgs
+else
+	echo $pkgs
+fi

scripts/print-install-deps.sh

@@ -1,23 +0,0 @@
-#!/bin/bash -e
-#
-# Print install dpenedencies on Linux. CMake runs this script to obtain deps for CPACK.
-# mscp dependes on packages on which libssh depends.
-
-source /etc/os-release
-
-release=$1
-
-case $release in
-	ubuntu-20.04*)
-		echo "libc6 (>= 2.27), libgssapi-krb5-2 (>= 1.17), libssl1.1 (>= 1.1.1), zlib1g (>= 1:1.1.4)"
-		;;
-	ubuntu-22.04*)
-		echo "libc6 (>= 2.33), libgssapi-krb5-2 (>= 1.17), libssl3 (>= 3.0.0~~alpha1), zlib1g (>= 1:1.1.4)"
-		;;
-	centos* | rhel* | rocky*)
-		echo "glibc crypto-policies krb5-libs openssl-libs libcom_err"
-		;;
-	*)
-		echo "unsupported install dependency: $release"
-		exit 1
-esac

scripts/test-in-container.sh

@@ -8,10 +8,31 @@ cd $script_dir
 
 set -x
 
+# sshd Linsten on 22 and 8022
+echo "Port 22" >> /etc/ssh/sshd_config
+echo "Port 8022" >> /etc/ssh/sshd_config
+
+## Alpine default sshd disables TcpForwarding, which is required for proxyjump test
+sed -i -e 's/AllowTcpForwarding no/AllowTcpForwarding yes/' /etc/ssh/sshd_config
+
 # Run sshd
 if [ ! -e /var/run/sshd.pid ]; then
-	/usr/sbin/sshd
+	/usr/sbin/sshd -E /tmp/sshd.log
+	sleep 1
 fi
 
-# Run test
-python3 -m pytest ../test -v
+for port in 22 8022; do
+	ssh-keyscan -p $port localhost >> ${HOME}/.ssh/known_hosts
+	ssh-keyscan -p $port ip6-localhost >> ${HOME}/.ssh/known_hosts
+	ssh-keyscan -p $port 127.0.0.1 >> ${HOME}/.ssh/known_hosts
+	ssh-keyscan -p $port ::1 >> ${HOME}/.ssh/known_hosts
+done
+
+
+if [ $# -gt 0 ]; then
+	# command arguments are passed, exec them
+	"$@"
+else
+	# no arguments passed, run the test
+	python3 -m pytest -v ../test
+fi

src/atomic.h

@@ -1,9 +1,10 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
 #ifndef _ATOMIC_H_
 #define _ATOMIC_H_
 
 #include <stdlib.h>
+#include <assert.h>
 #include <pthread.h>
-#include <util.h>
 
 typedef int refcnt;
 
@@ -17,6 +18,7 @@ static inline refcnt refcnt_dec(refcnt *cnt)
 	return __sync_sub_and_fetch(cnt, 1);
 }
 
+/* mutex */
 
 typedef pthread_mutex_t lock;
 
@@ -28,31 +30,65 @@ static inline void lock_init(lock *l)
 static inline void lock_acquire(lock *l)
 {
 	int ret = pthread_mutex_lock(l);
-	if (ret < 0) {
-		switch (ret) {
-		case EINVAL:
-			pr_err("invalid mutex\n");
-			exit(1);
-		case EDEADLK:
-			pr_err("a deadlock would occur\n");
-			exit(1);
-		}
-	}
+	assert(ret == 0);
 }
 
 static inline void lock_release(lock *l)
 {
 	int ret = pthread_mutex_unlock(l);
-	if (ret < 0) {
-		switch (ret) {
-		case EINVAL:
-			pr_err("invalid mutex\n");
-			exit(1);
-		case EPERM:
-			pr_err("this thread does not hold this mutex\n");
-			exit(1);
-		}
-	}
+	assert(ret == 0);
 }
 
+static inline void lock_release_via_cleanup(void *l)
+{
+	lock_release(l);
+}
+
+#define LOCK_ACQUIRE(l)  \
+	lock_acquire(l); \
+	pthread_cleanup_push(lock_release_via_cleanup, l)
+
+#define LOCK_RELEASE() pthread_cleanup_pop(1)
+
+/* read/write lock */
+typedef pthread_rwlock_t rwlock;
+
+static inline void rwlock_init(rwlock *rw)
+{
+	pthread_rwlock_init(rw, NULL);
+}
+
+static inline void rwlock_read_acquire(rwlock *rw)
+{
+	int ret = pthread_rwlock_rdlock(rw);
+	assert(ret == 0);
+}
+
+static inline void rwlock_write_acquire(rwlock *rw)
+{
+	int ret = pthread_rwlock_wrlock(rw);
+	assert(ret == 0);
+}
+
+static inline void rwlock_release(rwlock *rw)
+{
+	int ret = pthread_rwlock_unlock(rw);
+	assert(ret == 0);
+}
+
+static inline void rwlock_release_via_cleanup(void *rw)
+{
+	rwlock_release(rw);
+}
+
+#define RWLOCK_READ_ACQUIRE(rw)  \
+	rwlock_read_acquire(rw); \
+	pthread_cleanup_push(rwlock_release_via_cleanup, rw)
+
+#define RWLOCK_WRITE_ACQUIRE(rw)  \
+	rwlock_write_acquire(rw); \
+	pthread_cleanup_push(rwlock_release_via_cleanup, rw)
+
+#define RWLOCK_RELEASE() pthread_cleanup_pop(1)
+
 #endif /* _ATOMIC_H_ */

src/bwlimit.c

@@ -0,0 +1,92 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
+#include <errno.h>
+
+#include <bwlimit.h>
+#include <platform.h>
+
+#define timespeczerorize(ts)    \
+	do {                    \
+		ts.tv_sec = 0;  \
+		ts.tv_nsec = 0; \
+	} while (0)
+
+int bwlimit_init(struct bwlimit *bw, uint64_t bps, uint64_t win)
+{
+	if (!(bw->sem = sem_create(1)))
+		return -1;
+
+	bw->bps = bps;
+	bw->win = win; /* msec window */
+	bw->amt = (double)bps / 8 / 1000 * win; /* bytes in a window (msec) */
+	bw->credit = bw->amt;
+	timespeczerorize(bw->wstart);
+	timespeczerorize(bw->wend);
+
+	return 0;
+}
+
+#define timespecisset(ts) ((ts).tv_sec || (ts).tv_nsec)
+
+#define timespecmsadd(a, msec, r)                                \
+	do {                                                     \
+		(r).tv_sec = (a).tv_sec;                         \
+		(r).tv_nsec = (a).tv_nsec + (msec * 1000000);    \
+		if ((r).tv_nsec > 1000000000) {                  \
+			(r).tv_sec += (r.tv_nsec) / 1000000000L; \
+			(r).tv_nsec = (r.tv_nsec) % 1000000000L; \
+		}                                                \
+	} while (0)
+
+#define timespecsub(a, b, r)                             \
+	do {                                             \
+		(r).tv_sec = (a).tv_sec - (b).tv_sec;    \
+		(r).tv_nsec = (a).tv_nsec - (b).tv_nsec; \
+		if ((r).tv_nsec < 0) {                   \
+			(r).tv_sec -= 1;                 \
+			(r).tv_nsec += 1000000000;       \
+		}                                        \
+	} while (0)
+
+#define timespeccmp(a, b, expr) \
+	((a.tv_sec * 1000000000 + a.tv_nsec) expr(b.tv_sec * 1000000000 + b.tv_nsec))
+
+int bwlimit_wait(struct bwlimit *bw, size_t nr_bytes)
+{
+	struct timespec now, end, rq, rm;
+
+	if (bw->bps == 0)
+		return 0; /* no bandwidth limit */
+
+	if (sem_wait(bw->sem) < 0)
+		return -1;
+
+	clock_gettime(CLOCK_MONOTONIC, &now);
+
+	if (!timespecisset(bw->wstart)) {
+		bw->wstart = now;
+		timespecmsadd(bw->wstart, bw->win, bw->wend);
+	}
+
+	bw->credit -= nr_bytes;
+
+	if (bw->credit < 0) {
+		/* no more credit on this window. sleep until the end
+		 * of this window + additional time for the remaining
+		 * bytes. */
+		uint64_t addition = (double)(bw->credit * -1) / (bw->bps / 8);
+		timespecmsadd(bw->wend, addition * 1000, end);
+		if (timespeccmp(end, now, >)) {
+			timespecsub(end, now, rq);
+			while (nanosleep(&rq, &rm) == -1) {
+				if (errno != EINTR)
+					break;
+				rq = rm;
+			}
+		}
+		bw->credit = bw->amt;
+		timespeczerorize(bw->wstart);
+	}
+
+	sem_post(bw->sem);
+	return 0;
+}

src/bwlimit.h

@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
+#ifndef _BWLIMIT_H_
+#define _BWLIMIT_H_
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <sys/types.h>
+#include <time.h>
+#include <semaphore.h>
+
+struct bwlimit {
+	sem_t	*sem;	/* semaphore */
+	size_t	bps;	/* limit bit-rate (bps) */
+	size_t	win;	/* window size (msec) */
+	size_t	amt;	/* amount of bytes can be sent in a window */
+
+	ssize_t		credit;	/* remaining bytes can be sent in a window */
+	struct timespec wstart, wend; /* window start time and end time */
+};
+
+int bwlimit_init(struct bwlimit *bw, uint64_t bps, uint64_t win);
+/* if bps is 0, it means that bwlimit is not active. If so,
+ * bwlimit_wait() returns immediately. */
+
+int bwlimit_wait(struct bwlimit *bw, size_t nr_bytes);
+
+
+#endif /* _BWLIMIT_H_ */

src/checkpoint.c

@@ -0,0 +1,505 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
+#include <fcntl.h>
+#include <sys/uio.h>
+#include <arpa/inet.h>
+
+#include <path.h>
+#include <print.h>
+#include <platform.h>
+#include <strerrno.h>
+#include <openbsd-compat/openbsd-compat.h>
+
+#include <checkpoint.h>
+
+#define MSCP_CHECKPOINT_MAGIC 0x7063736dUL /* mscp in ascii */
+#define MSCP_CHECKPOINT_VERSION 0x1
+
+/**
+ * mscp checkpoint file format. All values are network byte order.
+ *
+ * The file starts with the File header:
+ * 0                   1                   2                   3
+ *  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+ * +---------------------------------------------------------------+
+ * |                          Magic Code                           |
+ * +---------------+-----------------------------------------------+
+ * |     Version   |
+ * +---------------+
+ *
+ * Magic code: 0x7063736dUL
+ *
+ * Version: 1.
+ *
+ *
+ * Each object in a checkpoint always starts with an object header:
+ * +---------------+---------------+-------------------------------+
+ * |     Type      |      rsv      |             Length            |
+ * +---------------+---------------+-------------------------------+
+ *
+ * Type: 0x0A (meta), 0x0B (path), or 0x0C (chunk)
+ *
+ * Rsv: reserved
+ *
+ * Length: Length of this object including the object header.
+ *
+ *
+ * Meta object provides generaic information for the failed copy:
+ * +---------------+---------------+-------------------------------+
+ * |     Type      |      rsv      |             Length            |
+ * +---------------+---------------+-------------------------------+
+ * |  Direction    | Remote string ...
+ * +---------------+------------------
+ *
+ * Direction: 1 (Local-to-Remote copy) or 2 (Remote-to-Local copy)
+ *
+ * Remote string: Remote host, e.g., user@hostname and IP address,
+ * string including '\0'.
+ *
+ *
+ * Path object represnts a file with sourcen and destination paths:
+ * +---------------+---------------+-------------------------------+
+ * |     Type      |      rsv      |             Length            |
+ * +---------------+---------------+-------------------------------+
+ * |                             Index                             |
+ * +-------------------------------+-------------------------------+
+ * |         Source offset         |      Destination offset       |
+ * +-------------------------------+-------------------------------+
+ * //                                                             //
+ * //                     Source path string                      //
+ * //                                                             //
+ * +---------------------------------------------------------------+
+ * //                                                             //
+ * //                   Destination path string                   //
+ * //                                                             //
+ * +---------------------------------------------------------------+
+ *
+ * Index: 32-bit unsigned int indicating this path (used by chunks)
+ *
+ * Source offset: Offset of the Source path string from the head of
+ * this object. It is identical to the end of the Destination offset
+ * filed.
+ *
+ * Destination offset: Offset of the Destnation path string from the
+ * head of this object. It also indicates the end of the Source path
+ * string.
+ *
+ * Source path string: String of copy source path (including '\0').
+ *
+ * Destination path string: string of copy destination path (including
+ * '\0').
+ *
+ *
+ * Chunk object represents a chunk associated with a path object:
+ * +---------------+---------------+-------------------------------+
+ * |     Type      |      rsv      |             Length            |
+ * +---------------+---------------+-------------------------------+
+ * |                             Index                             |
+ * +---------------------------------------------------------------+
+ * |                             Chunk                             |
+ * |                             offset                            |
+ * +---------------------------------------------------------------+
+ * |                             Chunk                             |
+ * |                             length                            |
+ * +---------------------------------------------------------------+
+ *
+ * Index: 32 bit unsigned int indicating the index of a path object
+ * this chunk associated with.
+ *
+ * Chunk offset: 64 bit unsigned int indicating the offset of this
+ * chunk from the head of the associating a file.
+ *
+ * Chunk length: 64 bit unsigned int indicating the length (bytes) of
+ * this chunk.
+ */
+
+enum {
+	OBJ_TYPE_META = 0x0A,
+	OBJ_TYPE_PATH = 0x0B,
+	OBJ_TYPE_CHUNK = 0x0C,
+};
+
+struct checkpoint_file_hdr {
+	uint32_t magic;
+	uint8_t version;
+} __attribute__((packed));
+
+struct checkpoint_obj_hdr {
+	uint8_t type;
+	uint8_t rsv;
+	uint16_t len; /* length of an object including this hdr */
+} __attribute__((packed));
+
+struct checkpoint_obj_meta {
+	struct checkpoint_obj_hdr hdr;
+	uint8_t direction; /* L2R or R2L */
+
+	char remote[0];
+} __attribute__((packed));
+
+struct checkpoint_obj_path {
+	struct checkpoint_obj_hdr hdr;
+
+	uint32_t idx;
+	uint16_t src_off; /* offset to the src path string (including
+			   * \0) from the head of this object. */
+	uint16_t dst_off; /* offset to the dst path string (including
+			   * \0) from the head of this object */
+} __attribute__((packed));
+
+#define obj_path_src(o) ((char *)(o) + ntohs(o->src_off))
+#define obj_path_dst(o) ((char *)(o) + ntohs(o->dst_off))
+
+#define obj_path_src_len(o) (ntohs(o->dst_off) - ntohs(o->src_off))
+#define obj_path_dst_len(o) (ntohs(o->hdr.len) - ntohs(o->dst_off))
+
+#define obj_path_validate(o)				\
+	((ntohs(o->hdr.len) > ntohs(o->dst_off)) &&	\
+	 (ntohs(o->dst_off) > ntohs(o->src_off)) &&	\
+	 (obj_path_src_len(o) < PATH_MAX) &&		\
+	 (obj_path_dst_len(o) < PATH_MAX))
+
+struct checkpoint_obj_chunk {
+	struct checkpoint_obj_hdr hdr;
+
+	uint32_t idx; /* index indicating associating path */
+	uint64_t off;
+	uint64_t len;
+} __attribute__((packed));
+
+#define CHECKPOINT_OBJ_MAXLEN (sizeof(struct checkpoint_obj_path) + PATH_MAX * 2)
+
+static int checkpoint_write_path(int fd, struct path *p, unsigned int idx)
+{
+	char buf[CHECKPOINT_OBJ_MAXLEN];
+	struct checkpoint_obj_path *path = (struct checkpoint_obj_path *)buf;
+	size_t src_len, dst_len;
+	struct iovec iov[3];
+
+	p->data = idx; /* save idx to be pointed by chunks */
+
+	src_len = strlen(p->path) + 1;
+	dst_len = strlen(p->dst_path) + 1;
+
+	memset(buf, 0, sizeof(buf));
+	path->hdr.type = OBJ_TYPE_PATH;
+	path->hdr.len = htons(sizeof(*path) + src_len + dst_len);
+
+	path->idx = htonl(idx);
+	path->src_off = htons(sizeof(*path));
+	path->dst_off = htons(sizeof(*path) + src_len);
+
+	iov[0].iov_base = path;
+	iov[0].iov_len = sizeof(*path);
+	iov[1].iov_base = p->path;
+	iov[1].iov_len = src_len;
+	iov[2].iov_base = p->dst_path;
+	iov[2].iov_len = dst_len;
+
+	if (writev(fd, iov, 3) < 0) {
+		priv_set_errv("writev: %s", strerrno());
+		return -1;
+	}
+	return 0;
+}
+
+static int checkpoint_write_chunk(int fd, struct chunk *c)
+{
+	struct checkpoint_obj_chunk chunk;
+
+	memset(&chunk, 0, sizeof(chunk));
+	chunk.hdr.type = OBJ_TYPE_CHUNK;
+	chunk.hdr.len = htons(sizeof(chunk));
+
+	chunk.idx = htonl(c->p->data); /* index stored by checkpoint_write_path */
+	chunk.off = htonll(c->off);
+	chunk.len = htonll(c->len);
+
+	if (write(fd, &chunk, sizeof(chunk)) < 0) {
+		priv_set_errv("writev: %s", strerrno());
+		return -1;
+	}
+	return 0;
+}
+
+int checkpoint_save(const char *pathname, int dir, const char *user, const char *remote,
+		    pool *path_pool, pool *chunk_pool)
+{
+	struct checkpoint_file_hdr hdr;
+	struct checkpoint_obj_meta meta;
+	struct iovec iov[3];
+	struct chunk *c;
+	struct path *p;
+	char buf[1024];
+	unsigned int i, nr_paths, nr_chunks;
+	int fd, ret;
+
+	fd = open(pathname, O_WRONLY | O_CREAT | O_TRUNC,
+		  S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH);
+	if (fd < 0) {
+		priv_set_errv("open: %s: %s", pathname, strerrno());
+		return -1;
+	}
+
+	/* write file hdr */
+	hdr.magic = htonl(MSCP_CHECKPOINT_MAGIC);
+	hdr.version = MSCP_CHECKPOINT_VERSION;
+
+	/* write meta */
+	if (user)
+		ret = snprintf(buf, sizeof(buf), "%s@%s", user, remote);
+	else
+		ret = snprintf(buf, sizeof(buf), "%s", remote);
+	if (ret >= sizeof(buf)) {
+		priv_set_errv("too long username and/or remote");
+		return -1;
+	}
+
+	memset(&meta, 0, sizeof(meta));
+	meta.hdr.type = OBJ_TYPE_META;
+	meta.hdr.len = htons(sizeof(meta) + strlen(buf) + 1);
+	meta.direction = dir;
+
+	iov[0].iov_base = &hdr;
+	iov[0].iov_len = sizeof(hdr);
+	iov[1].iov_base = &meta;
+	iov[1].iov_len = sizeof(meta);
+	iov[2].iov_base = buf;
+	iov[2].iov_len = strlen(buf) + 1;
+
+	if (writev(fd, iov, 3) < 0) {
+		priv_set_errv("writev: %s", strerrno());
+		return -1;
+	}
+
+	/* write paths */
+	nr_paths = 0;
+	pool_for_each(path_pool, p, i) {
+		if (p->state == FILE_STATE_DONE)
+			continue;
+		if (checkpoint_write_path(fd, p, nr_paths) < 0)
+			return -1;
+		nr_paths++;
+	}
+
+	/* write chunks */
+	nr_chunks = 0;
+	pool_for_each(chunk_pool, c, i) {
+		if (c->state == CHUNK_STATE_DONE)
+			continue;
+		if (checkpoint_write_chunk(fd, c) < 0)
+			return -1;
+		nr_chunks++;
+	}
+
+	pr_notice("checkpoint: %u paths and %u chunks saved", nr_paths, nr_chunks);
+
+	return 0;
+}
+
+static int checkpoint_load_meta(struct checkpoint_obj_hdr *hdr, char *remote, size_t len,
+				int *dir)
+{
+	struct checkpoint_obj_meta *meta = (struct checkpoint_obj_meta *)hdr;
+
+	if (len < ntohs(hdr->len) - sizeof(*meta)) {
+		priv_set_errv("too short buffer");
+		return -1;
+	}
+	snprintf(remote, len, "%s", meta->remote);
+	*dir = meta->direction;
+
+	pr_notice("checkpoint: remote=%s direction=%s", meta->remote,
+		  meta->direction == MSCP_DIRECTION_L2R ? "local-to-remote" :
+		  meta->direction == MSCP_DIRECTION_R2L ? "remote-to-local" :
+							  "invalid");
+
+	return 0;
+}
+
+static int checkpoint_load_path(struct checkpoint_obj_hdr *hdr, pool *path_pool)
+{
+	struct checkpoint_obj_path *path = (struct checkpoint_obj_path *)hdr;
+	struct path *p;
+	char *s, *d;
+
+	if (!obj_path_validate(path)) {
+		priv_set_errv("invalid path object");
+		return -1;
+	}
+
+	if (!(s = strndup(obj_path_src(path), obj_path_src_len(path)))) {
+		priv_set_errv("strdup: %s", strerrno());
+		return -1;
+	}
+
+	if (!(d = strndup(obj_path_dst(path), obj_path_dst_len(path)))) {
+		priv_set_errv("strdup: %s", strerrno());
+		free(s);
+		return -1;
+	}
+
+	if (!(p = alloc_path(s, d))) {
+		free(s);
+		free(d);
+		return -1;
+	}
+
+	if (pool_push(path_pool, p) < 0) {
+		priv_set_errv("pool_push: %s", strerrno());
+		return -1;
+	}
+
+	pr_info("checkpoint:file: idx=%u %s -> %s", ntohl(path->idx),
+		p->path, p->dst_path);
+
+	return 0;
+}
+
+static int checkpoint_load_chunk(struct checkpoint_obj_hdr *hdr, pool *path_pool,
+				 pool *chunk_pool)
+{
+	struct checkpoint_obj_chunk *chunk = (struct checkpoint_obj_chunk *)hdr;
+	struct chunk *c;
+	struct path *p;
+
+	if (!(p = pool_get(path_pool, ntohl(chunk->idx)))) {
+		/* we assumes all paths are already loaded in the order */
+		priv_set_errv("path index %u not found", ntohl(chunk->idx));
+		return -1;
+	}
+
+	if (!(c = alloc_chunk(p, ntohll(chunk->off), ntohll(chunk->len))))
+		return -1;
+
+	if (pool_push(chunk_pool, c) < 0) {
+		priv_set_errv("pool_push: %s", strerrno());
+		return -1;
+	}
+
+	pr_debug("checkpoint:chunk: idx=%u %s 0x%lx-0x%lx", ntohl(chunk->idx),
+		 p->path, c->off, c->off + c->len);
+
+	return 0;
+}
+
+static int checkpoint_read_obj(int fd, void *buf, size_t count)
+{
+	struct checkpoint_obj_hdr *hdr = (struct checkpoint_obj_hdr *)buf;
+	ssize_t ret, objlen, objbuflen;
+
+	memset(buf, 0, count);
+
+	if (count < sizeof(*hdr)) {
+		priv_set_errv("too short buffer");
+		return -1;
+	}
+
+	ret = read(fd, hdr, sizeof(*hdr));
+	if (ret == 0)
+		return 0; /* no more objects */
+	if (ret < 0)
+		return -1;
+
+	objlen = ntohs(hdr->len) - sizeof(*hdr);
+	objbuflen = count - sizeof(*hdr);
+	if (objbuflen < objlen) {
+		priv_set_errv("too short buffer");
+		return -1;
+	}
+
+	ret = read(fd, buf + sizeof(*hdr), objlen);
+	if (ret < objlen) {
+		priv_set_errv("checkpoint truncated");
+		return -1;
+	}
+
+	return 1;
+}
+
+static int checkpoint_read_file_hdr(int fd)
+{
+	struct checkpoint_file_hdr hdr;
+	ssize_t ret;
+
+	ret = read(fd, &hdr, sizeof(hdr));
+	if (ret < 0) {
+		priv_set_errv("read: %s", strerrno());
+		return -1;
+	}
+	if (ret < sizeof(hdr)) {
+		priv_set_errv("checkpoint truncated");
+		return -1;
+	}
+
+	if (ntohl(hdr.magic) != MSCP_CHECKPOINT_MAGIC) {
+		priv_set_errv("checkpoint: invalid megic code");
+		return -1;
+	}
+
+	if (hdr.version != MSCP_CHECKPOINT_VERSION) {
+		priv_set_errv("checkpoint: unknown version %u", hdr.version);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int checkpoint_load(const char *pathname, char *remote, size_t len, int *dir,
+			   pool *path_pool, pool *chunk_pool)
+{
+	char buf[CHECKPOINT_OBJ_MAXLEN];
+	struct checkpoint_obj_hdr *hdr;
+	int fd, ret;
+
+	if ((fd = open(pathname, O_RDONLY)) < 0) {
+		priv_set_errv("open: %s: %s", pathname, strerrno());
+		return -1;
+	}
+
+	if (checkpoint_read_file_hdr(fd) < 0)
+		return -1;
+
+	hdr = (struct checkpoint_obj_hdr *)buf;
+	while ((ret = checkpoint_read_obj(fd, buf, sizeof(buf))) > 0) {
+		switch (hdr->type) {
+		case OBJ_TYPE_META:
+			if (!remote || !dir)
+				break;
+			if (checkpoint_load_meta(hdr, remote, len, dir) < 0)
+				return -1;
+			if (!path_pool || !chunk_pool)
+				goto out;
+			break;
+		case OBJ_TYPE_PATH:
+			if (!path_pool)
+				break;
+			if (checkpoint_load_path(hdr, path_pool) < 0)
+				return -1;
+			break;
+		case OBJ_TYPE_CHUNK:
+			if (!path_pool)
+				break;
+			if (checkpoint_load_chunk(hdr, path_pool, chunk_pool) < 0)
+				return -1;
+			break;
+		default:
+			priv_set_errv("unknown obj type %u", hdr->type);
+			return -1;
+		}
+	}
+
+out:
+	close(fd);
+
+	return 0;
+}
+
+int checkpoint_load_remote(const char *pathname, char *remote, size_t len, int *dir)
+{
+	return checkpoint_load(pathname, remote, len, dir, NULL, NULL);
+}
+
+int checkpoint_load_paths(const char *pathname, pool *path_pool, pool *chunk_pool)
+{
+	return checkpoint_load(pathname, NULL, 0, NULL, path_pool, chunk_pool);
+}

src/checkpoint.h

@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
+#ifndef _CHECKPOINT_H_
+#define _CHECKPOINT_H_
+
+#include <pool.h>
+
+/* checkpoint_save() stores states to a checkponint file (pathname) */
+int checkpoint_save(const char *pathname, int dir, const char *user, const char *remote,
+		    pool *path_pool, pool *chunk_pool);
+
+/* checkpoint_load_meta() reads a checkpoint file (pathname) and returns
+ * remote host string to *remote and transfer direction to *dir.
+ */
+int checkpoint_load_remote(const char *pathname, char *remote, size_t len, int *dir);
+
+/* checkpoint_load_paths() reads a checkpoint file (pathname) and
+ * fills path_pool and chunk_pool.
+ */
+int checkpoint_load_paths(const char *pathname, pool *path_pool, pool *chunk_pool);
+
+#endif /* _CHECKPOINT_H_ */

src/file.c

@@ -1,960 +0,0 @@
-#include <stdlib.h>
-#include <stdbool.h>
-#include <sys/stat.h>
-#include <dirent.h>
-#include <fcntl.h>
-#include <libgen.h>
-
-#include <ssh.h>
-#include <util.h>
-#include <file.h>
-#include <pprint.h>
-#include <platform.h>
-
-bool file_has_hostname(char *path)
-{
-	char *p;
-
-	p = strchr(path, ':');
-	if (p) {
-		if (p == path || ((p > path) && *(p - 1) == '\\'))
-			return false; /* first byte is colon or escaped colon, skip */
-		else
-			return true;
-	}
-
-	return false;
-}
-
-char *file_find_hostname(char *path)
-{
-	char *dup, *p;
-
-	dup = strdup(path);
-	if (!dup) {
-		pr_err("%s", strerrno());
-		return NULL;
-	}
-
-	p = strchr(dup, ':');
-	if (p) {
-		if (p == dup || ((p > dup) && *(p - 1) == '\\')) {
-			/* first byte is colon or escaped colon, skip */
-			free(dup);
-		} else {
-			/* handle this as remote hostname (with username) */
-			*p = '\0';
-			return dup;
-		}
-	}
-
-	return NULL;
-}
-
-static char *file_find_path(char *path)
-{
-	char *p;
-
-	p = strchr(path, ':');
-	if (p) {
-		if (p == path || ((p > path) && *(p - 1) == '\\')) {
-			/* first byte is colon or escaped colon, skip */
-			return path;
-		} else {
-			return p + 1;
-		}
-	}
-
-	return path;
-}
-
-/* return 1 when path is directory, 0 is not directory, and -1 on error */
-static int file_is_directory(char *path, sftp_session sftp, bool print_error)
-{
-	int ret = 0;
-
-	if (sftp) {
-		char *remote_path = file_find_path(path);
-		sftp_attributes attr;
-
-		char *p = *remote_path == '\0' ? "." : remote_path;
-		attr = sftp_stat(sftp, p);
-		if (!attr) {
-			if (print_error)
-				pr_err("sftp_stat %s: %s\n",
-				       path, sftp_get_ssh_error(sftp));
-			ret = -1;
-		} else if (attr->type == SSH_FILEXFER_TYPE_DIRECTORY)
-			ret = 1;
-		sftp_attributes_free(attr);
-	} else {
-		struct stat statbuf;
-		if (stat(path, &statbuf) < 0) {
-			if (print_error)
-				pr_err("stat %s: %s\n", path, strerrno());
-			ret = -1;
-		} else if (S_ISDIR(statbuf.st_mode))
-			ret = 1;
-	}
-
-	return ret;
-}
-
-/* return 1 when directory exists, 0 not exists, and -1 on error */
-int file_directory_exists(char *path, sftp_session sftp)
-{
-	int ret = 0;
-
-	if (sftp) {
-		sftp_attributes attr;
-		attr = sftp_stat(sftp, path);
-		if (!attr) {
-			if (sftp_get_error(sftp) == SSH_FX_NO_SUCH_PATH ||
-			    sftp_get_error(sftp) == SSH_FX_NO_SUCH_FILE)
-				ret = 0;
-			else {
-				pr_err("%s: %s\n", path, sftp_get_ssh_error(sftp));
-				ret = -1;
-			}
-		} else if (attr->type == SSH_FILEXFER_TYPE_DIRECTORY)
-			ret = 1;
-		sftp_attributes_free(attr);
-	} else {
-		struct stat statbuf;
-		if (stat(path, &statbuf) < 0) {
-			if (errno == ENOENT)
-				ret = 0;
-			else {
-				pr_err("%s: %s\n", path, strerrno());
-				ret = -1;
-			}
-		} else if ((statbuf.st_mode & S_IFMT) == S_IFDIR)
-			ret = 1;
-	}
-
-	return ret;
-}
-
-static struct file *file_alloc(char *src_path, size_t size, bool src_is_remote)
-{
-	struct file *f;
-
-	f = malloc(sizeof(*f));
-	if (!f) {
-		pr_err("%s\n", strerrno());
-		return NULL;
-	}
-	memset(f, 0, sizeof(*f));
-
-	strncpy(f->src_path, src_path, PATH_MAX - 1);
-	f->size = size;
-	f->src_is_remote = src_is_remote;
-	f->dst_is_remote = !src_is_remote;
-	lock_init(&f->lock);
-
-	return f;
-}
-
-static bool check_file_should_skip(char *path)
-{
-	int len = strlen(path);
-	if ((len == 1 && strncmp(path, ".", 1) == 0) ||
-	    (len == 2 && strncmp(path, "..", 2) == 0)) {
-		return true;
-	}
-	return false;
-}
-
-
-/* return -1 when error, 0 when should skip, and 1 when should be copied  */
-static int check_file_tobe_copied(char *path, sftp_session sftp, size_t *size)
-{
-	struct stat statbuf;
-	sftp_attributes attr;
-	int ret = 0;
-
-	if (!sftp) {
-		/* local */
-		if (stat(path, &statbuf) < 0) {
-			pr_err("stat %s: %s\n", path, strerrno());
-			return -1;
-		}
-		if (S_ISREG(statbuf.st_mode)) {
-			*size = statbuf.st_size;
-			return 1;
-		}
-		return 0;
-	}
-
-	/* remote */
-	attr = sftp_stat(sftp, path);
-	if (!attr) {
-		pr_err("sftp_stat %s: %s\n", path, sftp_get_ssh_error(sftp));
-		return -1;
-	}
-	if (attr->type == SSH_FILEXFER_TYPE_REGULAR ||
-	    attr->type == SSH_FILEXFER_TYPE_SYMLINK) {
-		*size = attr->size;
-		ret = 1;
-	}
-
-	sftp_attributes_free(attr);
-
-	return ret;
-}
-
-static int check_pathlen(const char *src, const char *dst)
-{
-	if ((strlen(src) + strlen(dst) + 1) > PATH_MAX) {
-		pr_err("too long path: %s/%s\n", src, dst);
-		return -1;
-	}
-	return 0;
-}
-
-static int file_fill_recursive(struct list_head *file_list,
-			       bool dst_is_remote, sftp_session sftp, char *src_path,
-			       char *rel_path, char *dst_path,
-			       bool dst_should_dir, bool replace_dir_name)
-{
-	char next_src_path[PATH_MAX], next_rel_path[PATH_MAX];
-	struct file *f;
-	size_t size;
-	int ret;
-
-	ret = file_is_directory(src_path, dst_is_remote ? NULL : sftp, true);
-	if (ret < 0)
-		return -1;
-
-	if (ret == 0) {
-		/* src_path is file */
-		ret = check_file_tobe_copied(src_path, dst_is_remote ? NULL : sftp,
-					     &size);
-		if (ret <= 0)
-			return ret; /* error or skip */
-
-		if ((f = file_alloc(src_path, size, !dst_is_remote)) == NULL) {
-			pr_err("%s\n", strerrno());
-			return -1;
-		}
-
-		if (dst_should_dir)
-			snprintf(f->dst_path, PATH_MAX, "%s/%s%s",
-				 dst_path, rel_path, basename(src_path));
-		else
-			snprintf(f->dst_path, PATH_MAX, "%s%s", rel_path, dst_path);
-
-		list_add_tail(&f->list, file_list);
-		pprint2("file %s %s -> %s %s %luB\n",
-			f->src_path, dst_is_remote ? "(local)" : "(remote)",
-			f->dst_path, dst_is_remote ? "(remote)" : "(local)",
-			f->size);
-
-		return 0;
-	}
-
-	/* src_path is directory */
-	if (dst_is_remote) {
-		/* src_path is local directory */
-		struct dirent *de;
-		DIR *dir;
-		if ((dir = opendir(src_path)) == NULL) {
-			pr_err("opendir '%s': %s\n", src_path, strerrno());
-			return -1;
-		}
-		while ((de = readdir(dir)) != NULL) {
-			if (check_file_should_skip(de->d_name))
-				continue;
-			if (check_pathlen(src_path, de->d_name) < 0 ||
-			    check_pathlen(rel_path, basename(src_path)) < 0)
-				return -1;
-
-			snprintf(next_src_path, sizeof(next_src_path),
-				 "%s/%s", src_path, de->d_name);
-			if (replace_dir_name)
-				memset(next_rel_path, 0, sizeof(next_rel_path));
-			else
-				snprintf(next_rel_path, sizeof(next_rel_path),
-					 "%s%s/", rel_path, basename(src_path));
-			ret = file_fill_recursive(file_list, dst_is_remote, sftp,
-						  next_src_path, next_rel_path,
-						  dst_path, dst_should_dir, false);
-			if (ret < 0)
-				return ret;
-		}
-	} else {
-		/* src_path is remote directory */
-		sftp_attributes attr;
-		sftp_dir dir;
-		if ((dir = sftp_opendir(sftp, src_path)) == NULL) {
-			pr_err("sftp_opendir: '%s': %s\n", src_path,
-			       sftp_get_ssh_error(sftp));
-			return -1;
-		}
-		while ((attr = sftp_readdir(sftp, dir)) != NULL) {
-			if (check_file_should_skip(attr->name))
-				continue;
-			if (check_pathlen(src_path, attr->name) < 0 ||
-			    check_pathlen(rel_path, basename(src_path)) < 0)
-				return -1;
-
-			snprintf(next_src_path, sizeof(next_src_path),
-				 "%s/%s", src_path, attr->name);
-			if (replace_dir_name)
-				memset(next_rel_path, 0, sizeof(next_rel_path));
-			else
-				snprintf(next_rel_path, sizeof(next_rel_path),
-					 "%s%s/", rel_path, basename(src_path));
-			ret = file_fill_recursive(file_list, dst_is_remote, sftp,
-						  next_src_path, next_rel_path,
-						  dst_path, dst_should_dir, false);
-			if (ret < 0)
-				return ret;
-		}
-	}
-
-	return 0;
-}
-
-int file_fill(sftp_session sftp, struct list_head *file_list, char **src_array, int cnt,
-	      char *dst)
-{
-	bool dst_is_remote, dst_is_dir, dst_dir_no_exist, dst_should_dir, dst_must_dir;
-	char *dst_path, *src_path;
-	int n, ret;
-
-	dst_path = file_find_path(dst);
-	dst_path = *dst_path == '\0' ? "." : dst_path;
-	dst_is_remote = file_find_hostname(dst) ? true : false;
-	dst_must_dir = cnt > 1 ? true : false;
-
-	if (file_is_directory(dst_path, dst_is_remote ? sftp : NULL, false) > 0)
-		dst_is_dir = true;
-	else
-		dst_is_dir = false;
-	dst_dir_no_exist = !dst_is_dir;
-
-	for (n = 0; n < cnt; n++) {
-		src_path = file_find_path(src_array[n]);
-
-		if (file_is_directory(src_path, dst_is_remote ? NULL : sftp, false) > 0)
-			dst_should_dir = true;
-		else
-			dst_should_dir = false;
-
-		ret = file_fill_recursive(file_list, dst_is_remote, sftp,
-					  src_path, "", dst_path,
-					  dst_should_dir | dst_must_dir | dst_is_dir,
-					  dst_dir_no_exist);
-		if (ret < 0)
-			return ret;
-	}
-
-	return 0;
-}
-
-/* based on
- * https://stackoverflow.com/questions/2336242/recursive-mkdir-system-call-on-unix */
-static int file_dst_prepare(struct file *f, sftp_session sftp)
-{
-	/* XXX: should reflect the permission of the original directory? */
-	mode_t mode =  S_IRWXU | S_IRWXG | S_IRWXO;
-	char path[PATH_MAX];
-	char *p;
-	int ret;
-
-	strncpy(path, f->dst_path, sizeof(path));
-
-	pr_debug("prepare for %s\n", path);
-
-	/* mkdir -p */
-	for (p = strchr(path + 1, '/'); p; p = strchr(p + 1, '/')) {
-		*p = '\0';
-
-		ret = file_directory_exists(path, sftp);
-		pr_debug("check %s ret=%d\n", path, ret);
-		if (ret < -1)
-			return -1;
-		if (ret == 1)
-			goto next;
-
-		pr_debug("mkdir %s\n", path);
-
-		if (sftp) {
-			ret = sftp_mkdir(sftp, path, mode);
-			if (ret < 0 &&
-			    sftp_get_error(sftp) != SSH_FX_FILE_ALREADY_EXISTS) {
-				pr_err("failed to create %s: %s\n",
-				       path, sftp_get_ssh_error(sftp));
-				return -1;
-			}
-		} else {
-			if (mkdir(path, mode) == -1 && errno != EEXIST) {
-				pr_err("failed to create %s: %s\n",
-				       path, strerrno());
-				return -1;
-			}
-		}
-	next:
-		*p = '/';
-	}
-
-	/* open file with O_TRUNC to set file size 0 */
-	mode = O_WRONLY|O_CREAT|O_TRUNC;
-	if (sftp) {
-		sftp_file sf;
-		if ((sf = sftp_open(sftp, f->dst_path, mode, S_IRUSR|S_IWUSR)) == NULL) {
-			pr_err("sftp_open: %s\n", sftp_get_ssh_error(sftp));
-			return -1;
-		}
-		if (sftp_close(sf) < 0) {
-			pr_err("sftp_close: %s\n", sftp_get_ssh_error(sftp));
-			return -1;
-		}
-	} else {
-		int fd;
-		if ((fd = open(f->dst_path, mode, S_IRUSR|S_IWUSR)) < 0) {
-			pr_err("open: %s\n", strerrno());
-			return -1;
-		}
-		if (close(fd) < 0) {
-			pr_err("close: %s\n", strerrno());
-			return -1;
-		}
-	}
-
-	return 0;
-}
-
-
-#ifdef DEBUG
-void file_dump(struct list_head *file_list)
-{
-	struct file *f;
-
-	list_for_each_entry(f, file_list, list) {
-		pr_debug("%s %s -> %s %s %lu-byte\n",
-			 f->src_path, strloc(f->src_is_remote),
-			 f->dst_path, strloc(f->dst_is_remote),
-			 f->size);
-	}
-}
-#endif
-
-
-static void *chunk_alloc(struct file *f)
-{
-	struct chunk *c;
-
-	c = malloc(sizeof(*c));
-	if (!c) {
-		pr_err("%s\n", strerrno());
-		return NULL;
-	}
-	memset(c, 0, sizeof(*c));
-
-	c->f = f;
-	c->off = 0;
-	c->len = 0;
-	refcnt_inc(&f->refcnt);
-	return c;
-}
-
-static int get_page_mask(void)
-{
-	int n;
-	long page_sz = sysconf(_SC_PAGESIZE);
-	size_t page_mask = 0;
-
-	for (n = 0; page_sz > 0; page_sz >>= 1, n++) {
-		page_mask <<= 1;
-		page_mask |= 1;
-	}
-
-	return page_mask >> 1;
-}
-
-int chunk_fill(struct list_head *file_list, struct list_head *chunk_list,
-	       int nr_conn, int min_chunk_sz, int max_chunk_sz)
-{
-	struct chunk *c;
-	struct file *f;
-	size_t page_mask;
-	size_t chunk_sz;
-	size_t size;
-
-	page_mask = get_page_mask();
-
-	list_for_each_entry(f, file_list, list) {
-		if (f->size <= min_chunk_sz)
-			chunk_sz = f->size;
-		else if (max_chunk_sz)
-			chunk_sz = max_chunk_sz;
-		else {
-			chunk_sz = (f->size - (f->size % nr_conn)) / nr_conn;
-			chunk_sz &= ~page_mask; /* align in page_sz */
-			if (chunk_sz <= min_chunk_sz)
-				chunk_sz = min_chunk_sz;
-		}
-
-		pr_debug("%s chunk_sz %lu-byte\n", f->src_path, chunk_sz);
-
-		/* for (size = f->size; size > 0;) does not create a
-		 * file (chunk) when file size is 0. This do {} while
-		 * (size > 0) creates just open/close a 0-byte file.
-		 */
-		size = f->size;
-		do {
-			c = chunk_alloc(f);
-			if (!c)
-				return -1;
-			c->off = f->size - size;
-			c->len = size < chunk_sz ? size : chunk_sz;
-			size -= c->len;
-			list_add_tail(&c->list, chunk_list);
-			pprint4("chunk %s 0x%010lx-0x%010lx %luB\n",
-				c->f->src_path, c->off, c->off + c->len, c->len);
-		} while (size > 0);
-	}
-
-	return 0;
-}
-
-#ifdef DEBUG
-void chunk_dump(struct list_head *chunk_list)
-{
-	struct chunk *c;
-
-	list_for_each_entry(c, chunk_list, list) {
-		pr_debug("%s %s 0x%010lx-0x%010lx %lu-byte\n",
-			 c->f->src_path, strloc(c->f->src_is_remote),
-			 c->off, c->off + c->len, c->len);
-	}
-}
-#endif
-
-
-struct chunk *chunk_acquire(struct list_head *chunk_list)
-{
-	/* under the lock for chunk_list */
-
-	struct list_head *first = chunk_list->next;
-	struct chunk *c = NULL;
-
-	if (list_empty(chunk_list))
-		return NULL; /* list is empty */
-
-	c = list_entry(first, struct chunk, list);
-	list_del(first);
-	return c;
-}
-
-int chunk_prepare(struct chunk *c, sftp_session sftp)
-{
-	struct file *f = c->f;
-	int ret = 0;
-
-	lock_acquire(&f->lock); /* XXX: is always acquiring lock per-chunk heavy? */
-	if (f->state == FILE_STATE_INIT) {
-		if (file_dst_prepare(f, f->dst_is_remote ? sftp : NULL) < 0) {
-			ret = -1;
-			goto out;
-		}
-		f->state = FILE_STATE_OPENED;
-		pprint2("copy start: %s\n", f->src_path);
-	}
-
-out:
-	lock_release(&f->lock);
-	return ret;
-}
-
-static mode_t chunk_get_mode(const char *path, sftp_session sftp)
-{
-	mode_t mode;
-
-	if (sftp) {
-		sftp_attributes attr = sftp_stat(sftp, path);
-		if (!attr) {
-			pr_err("sftp_stat %s: %s\n", path, sftp_get_ssh_error(sftp));
-			return -1;
-		}
-		mode = attr->permissions;
-		sftp_attributes_free(attr);
-	} else {
-		struct stat statbuf;
-		if (stat(path, &statbuf) < 0) {
-			pr_err("stat %s: %s\n", path, strerrno());
-			return -1;
-		}
-		mode = statbuf.st_mode & (S_IRWXU|S_IRWXG|S_IRWXO);
-	}
-	return mode;
-}
-
-static int chunk_set_mode(const char *path, mode_t mode, sftp_session sftp)
-{
-	if (sftp) {
-		if (sftp_chmod(sftp, path, mode) < 0) {
-			pr_err("sftp_chmod %s: %s\n", path, sftp_get_ssh_error(sftp));
-			return -1;
-		}
-	} else {
-		if (chmod(path, mode) < 0) {
-			pr_err("chmod %s: %s\n", path, strerrno());
-			return -1;
-		}
-	}
-
-	return 0;
-}
-
-static int chunk_open_local(const char *path, int flags, mode_t mode, size_t off)
-{
-	int fd;
-
-	fd = open(path, flags, mode);
-	if (fd < 0) {
-		pr_err("open failed for %s: %s\n", path, strerrno());
-		return -1;
-	}
-	if (lseek(fd, off, SEEK_SET) < 0) {
-		pr_err("seek error for %s: %s\n", path, strerrno());
-		close(fd);
-		return -1;
-	}
-
-	return fd;
-}
-
-static sftp_file chunk_open_remote(const char *path, int flags, mode_t mode, size_t off,
-				   sftp_session sftp)
-{
-	sftp_file sf;
-
-	sf = sftp_open(sftp, path, flags, mode);
-
-	if (!sf) {
-		pr_err("sftp_open %s: %s\n", path, sftp_get_ssh_error(sftp));
-		return NULL;
-	}
-
-	if (sftp_seek64(sf, off) < 0) {
-		pr_err("sftp_seek64 %s: %s\n", path, sftp_get_ssh_error(sftp));
-		return NULL;
-	}
-
-	return sf;
-}
-
-/*
- * TODO: handle case when read returns 0 (EOF).
- */
-static int _chunk_copy_local_to_remote(struct chunk *c, int fd, sftp_file sf,
-				       size_t sftp_buf_sz, size_t io_buf_sz,
-				       size_t *counter)
-{
-	ssize_t read_bytes, write_bytes, remaind;
-	char buf[io_buf_sz];
-
-	for (remaind = c->len; remaind > 0;) {
-
-		read_bytes = read(fd, buf, min(remaind, io_buf_sz));
-		if (read_bytes < 0) {
-			pr_err("read: %s\n", strerrno());
-			return -1;
-		}
-
-		write_bytes = sftp_write2(sf, buf, read_bytes, sftp_buf_sz);
-		if (write_bytes < 0) {
-			pr_err("sftp_write: %d\n", sftp_get_error(sf->sftp));
-			return -1;
-		}
-
-		if (write_bytes < read_bytes) {
-			pr_err("failed to write full bytes to %s\n", c->f->dst_path);
-			return -1;
-		}
-
-		*counter += write_bytes;
-		remaind -= write_bytes;
-	}
-
-	return 0;
-}
-
-#define XFER_BUF_SIZE	16384
-
-#ifdef ASYNC_WRITE
-static int _chunk_copy_local_to_remote_async(struct chunk *c, int fd,
-					     sftp_file sf, int nr_ahead, size_t *counter)
-{
-	ssize_t read_bytes, remaind, thrown;
-	char buf[XFER_BUF_SIZE];
-	int idx, ret;
-	struct {
-		uint32_t id;
-		ssize_t  len;
-	} reqs[nr_ahead];
-
-	if (c->len == 0)
-		return 0;
-
-	remaind = thrown = c->len;
-	for (idx = 0; idx < nr_ahead && thrown > 0; idx++) {
-		reqs[idx].len = min(thrown, sizeof(buf));
-		read_bytes = read(fd, buf, reqs[idx].len);
-		if (read_bytes < 0) {
-			pr_err("read: %s\n", strerrno());
-			return -1;
-		}
-		ret = sftp_async_write(sf, buf, reqs[idx].len, &reqs[idx].id);
-		if (ret < 0) {
-			pr_err("sftp_async_write: %d\n", sftp_get_error(sf->sftp));
-			return -1;
-		}
-		thrown -= reqs[idx].len;
-	}
-
-	for (idx = 0; remaind > 0; idx = (idx + 1) % nr_ahead) {
-		ret = sftp_async_write_end(sf, reqs[idx].id, 1);
-		if (ret != SSH_OK) {
-			pr_err("sftp_async_write_end: %d\n", sftp_get_error(sf->sftp));
-			return -1;
-		}
-
-		*counter += reqs[idx].len;
-		remaind -= reqs[idx].len;
-
-		if (remaind <= 0)
-			break;
-
-		if (thrown <= 0)
-			continue;
-
-		reqs[idx].len = min(thrown, sizeof(buf));
-		read_bytes = read(fd, buf, reqs[idx].len);
-		if (read_bytes < 0) {
-			pr_err("read: %s\n", strerrno());
-			return -1;
-		}
-
-		ret = sftp_async_write(sf, buf, reqs[idx].len, &reqs[idx].id);
-		if (ret < 0) {
-			pr_err("sftp_async_write: %d\n", sftp_get_error(sf->sftp));
-			return -1;
-		}
-		thrown -= reqs[idx].len;
-	}
-
-	if (remaind < 0) {
-		pr_err("invalid remaind bytes %ld. last async_write_end bytes %lu. "
-		       "last read bytes %ld\n",
-		       remaind, reqs[idx].len, read_bytes);
-		return -1;
-	}
-
-	return 0;
-}
-#endif
-
-static int _chunk_copy_remote_to_local(struct chunk *c, int fd, sftp_file sf,
-				       int nr_ahead, size_t *counter)
-{
-	ssize_t read_bytes, write_bytes, remaind, thrown;
-	char buf[XFER_BUF_SIZE];
-	int idx;
-	struct {
-		int id;
-		ssize_t len;
-	} reqs[nr_ahead];
-
-	/* TODO: sftp_buf_sz has no effect on remote to local copy. we
-	 * always use 16384 byte buffer pointed by
-	 * https://api.libssh.org/stable/libssh_tutor_sftp.html. The
-	 * larget read length from sftp_async_read is 65536 byte.
-	 * Read sizes larget than 65536 cause a situation where data
-	 * remainds but sftp_async_read returns 0.
-	 */
-
-	if (c->len == 0)
-		return 0;
-
-	remaind = thrown = c->len;
-
-	for (idx = 0; idx < nr_ahead && thrown > 0; idx++) {
-		reqs[idx].len = min(thrown, sizeof(buf));
-		reqs[idx].id = sftp_async_read_begin(sf, reqs[idx].len);
-		if (reqs[idx].id < 0) {
-			pr_err("sftp_async_read_begin: %d\n",
-			       sftp_get_error(sf->sftp));
-			return -1;
-		}
-		thrown -= reqs[idx].len;
-	}
-
-	for (idx = 0; remaind > 0; idx = (idx + 1) % nr_ahead) {
-		read_bytes = sftp_async_read(sf, buf, reqs[idx].len, reqs[idx].id);
-		if (read_bytes == SSH_ERROR) {
-			pr_err("sftp_async_read: %d\n", sftp_get_error(sf->sftp));
-			return -1;
-		}
-
-		if (thrown > 0) {
-			reqs[idx].len = min(thrown, sizeof(buf));
-			reqs[idx].id = sftp_async_read_begin(sf, reqs[idx].len);
-			thrown -= reqs[idx].len;
-		}
-
-		write_bytes = write(fd, buf, read_bytes);
-		if (write_bytes < 0) {
-			pr_err("write: %s\n", strerrno());
-			return -1;
-		}
-
-		if (write_bytes < read_bytes) {
-			pr_err("failed to write full bytes\n");
-			return -1;
-		}
-
-		*counter += write_bytes;
-		remaind -= read_bytes;
-	}
-
-	if (remaind < 0) {
-		pr_err("invalid remaind bytes %ld. last async_read bytes %ld. "
-		       "last write bytes %ld\n",
-		       remaind, read_bytes, write_bytes);
-		return -1;
-	}
-
-	return 0;
-}
-
-static int chunk_copy_local_to_remote(struct chunk *c, sftp_session sftp,
-				      size_t sftp_buf_sz, size_t io_buf_sz,
-				      int nr_ahead, size_t *counter)
-{
-	struct file *f = c->f;
-	sftp_file sf = NULL;
-	mode_t mode;
-	int ret = 0;
-	int fd = 0;
-	int flags;
-
-	flags = O_RDONLY;
-	mode = S_IRUSR;
-	if ((fd = chunk_open_local(f->src_path, flags, mode, c->off)) < 0) {
-		ret = -1;
-		goto out;
-	}
-
-	flags = O_WRONLY;
-	mode = S_IRUSR|S_IWUSR;
-	if (!(sf = chunk_open_remote(f->dst_path, flags, mode, c->off, sftp))) {
-		ret = -1;
-		goto out;
-	}
-
-#ifndef ASYNC_WRITE
-	ret = _chunk_copy_local_to_remote(c, fd, sf, sftp_buf_sz, io_buf_sz,
-						  counter);
-#else
-	ret = _chunk_copy_local_to_remote_async(c, fd, sf, nr_ahead, counter);
-#endif
-	if (ret < 0)
-		goto out;
-
-	if ((mode = chunk_get_mode(f->src_path, NULL)) < 0) {
-		ret = -1;
-		goto out;
-	}
-	if (chunk_set_mode(f->dst_path, mode, sftp) < 0) {
-		ret = -1;
-	}
-
-out:
-	if (fd > 0)
-		close(fd);
-	if (sf)
-		sftp_close(sf);
-	return ret;
-}
-
-static int chunk_copy_remote_to_local(struct chunk *c, sftp_session sftp,
-				      int nr_ahead, size_t *counter)
-{
-	struct file *f = c->f;
-	sftp_file sf = NULL;
-	mode_t mode;
-	int flags;
-	int fd = 0;
-	int ret = 0;
-
-	flags = O_WRONLY;
-	mode = S_IRUSR|S_IWUSR;
-	if ((fd = chunk_open_local(f->dst_path, flags, mode, c->off)) < 0) {
-		ret = -1;
-		goto out;
-	}
-
-	flags = O_RDONLY;
-	mode = S_IRUSR;
-	if (!(sf = chunk_open_remote(f->src_path, flags, mode, c->off, sftp))) {
-		ret = -1;
-		goto out;
-	}
-
-	ret = _chunk_copy_remote_to_local(c, fd, sf, nr_ahead, counter);
-	if (ret< 0)
-		goto out;
-
-out:
-	if (fd > 0)
-		close(fd);
-	if (sf)
-		sftp_close(sf);
-
-	return ret;
-}
-
-
-
-int chunk_copy(struct chunk *c, sftp_session sftp, size_t sftp_buf_sz, size_t io_buf_sz,
-	       int nr_ahead, size_t *counter)
-{
-	struct file *f = c->f;
-	int ret = 0;
-
-	pr_debug("copy %s %s -> %s %s off=0x%010lx\n",
-		 f->src_path, strloc(f->src_is_remote),
-		 f->dst_path, strloc(f->dst_is_remote), c->off);
-
-	pprint4("copy start: chunk %s 0x%010lx-0x%010lx %luB\n",
-		c->f->src_path, c->off, c->off + c->len, c->len);
-
-
-	if (f->dst_is_remote)
-		ret = chunk_copy_local_to_remote(c, sftp, sftp_buf_sz, io_buf_sz,
-						 nr_ahead, counter);
-	else
-		ret = chunk_copy_remote_to_local(c, sftp, nr_ahead, counter);
-
-	if (ret < 0)
-		return ret;
-
-	pr_debug("done %s %s -> %s %s off=0x%010lx\n",
-		 f->src_path, strloc(f->src_is_remote),
-		 f->dst_path, strloc(f->dst_is_remote), c->off);
-
-	pprint4("copy done: chunk %s 0x%010lx-0x%010lx %luB\n",
-		c->f->src_path, c->off, c->off + c->len, c->len);
-
-	if (refcnt_dec(&f->refcnt) == 0) {
-		f->state = FILE_STATE_DONE;
-		pprint2("copy done: %s\n", f->src_path);
-	}
-
-
-	return ret;
-}

src/file.h

@@ -1,84 +0,0 @@
-#ifndef _FILE_H_
-#define _FILE_H_
-
-#include <limits.h>
-#include <pthread.h>
-#include "libssh/libssh.h"
-#include "libssh/sftp.h"
-
-#include <list.h>
-#include <atomic.h>
-
-struct file {
-	struct list_head        list;   /* mscp->file_list */
-
-	char    src_path[PATH_MAX];	/* copy source path */
-	bool    src_is_remote;		/* source is remote */
-	size_t  size;			/* size of this file */
-
-	char    dst_path[PATH_MAX];	/* copy destination path */
-	bool    dst_is_remote;		/* destination is remote */
-
-	int     state;  /* destination file state */
-	lock    lock;   /* mutex to protect state */
-	refcnt  refcnt; /* chunks referencing this file */
-};
-#define FILE_STATE_INIT         0
-#define FILE_STATE_OPENED       1
-#define FILE_STATE_DONE         2
-
-#define strloc(is_remote) is_remote ? "(remote)" : "(local)"
-
-/* Allocating chunk increments refcnt of the associating file.
- * Multiple threads copying files follows:
- *
- * acquire a chunk (inside a global lock)
- *
- * if the file state of the chunk is INIT:
- *     acquire the file lock
- * *         if file state is INIT:
- *             create directory if necessary
- *             open file with O_TRUNC and close.
- *             set file state OPENED.
- *             // only the first thread in the lock open the destination file
- *     release the file lock
- * endif
- *
- * copy the chunk to the destination.
- * decrement the refcnt of the file.
- *
- * if refcnt == 0:
- *     all chunks are copied.
- *     set the file state DONE, print something useful output.
- * endif
- */
-
-struct chunk {
-	struct list_head        list;   /* mscp->chunk_list */
-	struct file *f;
-	size_t  off;    /* offset of this chunk on the file f */
-	size_t  len;    /* length of this chunk */
-	size_t  done;   /* copied bytes for this chunk by a thread */
-};
-
-char *file_find_hostname(char *path);
-bool file_has_hostname(char *path);
-
-int file_fill(sftp_session sftp, struct list_head *file_list, char **src_array, int cnt,
-	      char *dst);
-
-int chunk_fill(struct list_head *file_list, struct list_head *chunk_list,
-	       int nr_conn, int min_chunk_sz, int max_chunk_sz);
-
-struct chunk *chunk_acquire(struct list_head *chunk_list);
-int chunk_prepare(struct chunk *c, sftp_session sftp);
-int chunk_copy(struct chunk *c, sftp_session sftp, size_t sftp_buf_sz, size_t io_buf_sz,
-	       int nr_ahead, size_t *counter);
-
-#ifdef DEBUG
-void file_dump(struct list_head *file_list);
-void chunk_dump(struct list_head *chunk_list);
-#endif
-
-
-#endif /* _FILE_H_ */

src/fileops.c

@@ -0,0 +1,388 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <sys/times.h>
+#include <utime.h>
+
+#include <fileops.h>
+#include <ssh.h>
+#include <print.h>
+#include <platform.h>
+
+sftp_session __thread tls_sftp;
+/* tls_sftp is used *_wrapped() functions */
+
+void set_tls_sftp_session(sftp_session sftp)
+{
+	tls_sftp = sftp;
+}
+
+static void sftp_err_to_errno(sftp_session sftp)
+{
+	int sftperr = sftp_get_error(sftp);
+
+	switch (sftperr) {
+	case SSH_FX_OK:
+	case SSH_FX_EOF:
+		errno = 0;
+		break;
+	case SSH_FX_NO_SUCH_FILE:
+	case SSH_FX_NO_SUCH_PATH:
+		errno = ENOENT;
+		break;
+	case SSH_FX_PERMISSION_DENIED:
+		errno = EACCES;
+		break;
+	case SSH_FX_FAILURE:
+		errno = EINVAL;
+	case SSH_FX_BAD_MESSAGE:
+		errno = EBADMSG;
+	case SSH_FX_NO_CONNECTION:
+		errno = ENOTCONN;
+		break;
+	case SSH_FX_CONNECTION_LOST:
+		errno = ENETRESET;
+		break;
+	case SSH_FX_OP_UNSUPPORTED:
+		errno = EOPNOTSUPP;
+		break;
+	case SSH_FX_INVALID_HANDLE:
+		errno = EBADF;
+		break;
+	case SSH_FX_FILE_ALREADY_EXISTS:
+		errno = EEXIST;
+		break;
+	case SSH_FX_WRITE_PROTECT:
+		errno = EPERM;
+		break;
+	case SSH_FX_NO_MEDIA:
+		errno = ENODEV;
+		break;
+	default:
+		pr_warn("unkown SSH_FX response %d", sftperr);
+	}
+}
+
+MDIR *mscp_opendir(const char *path, sftp_session sftp)
+{
+	MDIR *md;
+
+	if (!(md = malloc(sizeof(*md))))
+		return NULL;
+	memset(md, 0, sizeof(*md));
+
+	if (sftp) {
+		md->remote = sftp_opendir(sftp, path);
+		sftp_err_to_errno(sftp);
+		if (!md->remote) {
+			goto free_out;
+		}
+	} else {
+		md->local = opendir(path);
+		if (!md->local) {
+			goto free_out;
+		}
+	}
+
+	return md;
+
+free_out:
+	free(md);
+	return NULL;
+}
+
+MDIR *mscp_opendir_wrapped(const char *path)
+{
+	return mscp_opendir(path, tls_sftp);
+}
+
+void mscp_closedir(MDIR *md)
+{
+	if (md->remote)
+		sftp_closedir(md->remote);
+	else
+		closedir(md->local);
+
+	free(md);
+}
+
+struct dirent __thread tls_dirent;
+/* tls_dirent contains dirent converted from sftp_attributes returned
+ * from sftp_readdir(). This trick is derived from openssh's
+ * fudge_readdir() */
+
+struct dirent *mscp_readdir(MDIR *md)
+{
+	sftp_attributes attr;
+	struct dirent *ret = NULL;
+	static int inum = 1;
+
+	if (md->remote) {
+		attr = sftp_readdir(md->remote->sftp, md->remote);
+		if (!attr) {
+			sftp_err_to_errno(md->remote->sftp);
+			return NULL;
+		}
+
+		memset(&tls_dirent, 0, sizeof(tls_dirent));
+		strncpy(tls_dirent.d_name, attr->name, sizeof(tls_dirent.d_name) - 1);
+		tls_dirent.d_ino = inum++;
+		if (!inum)
+			inum = 1;
+		ret = &tls_dirent;
+		sftp_attributes_free(attr);
+	} else
+		ret = readdir(md->local);
+
+	return ret;
+}
+
+int mscp_mkdir(const char *path, mode_t mode, sftp_session sftp)
+{
+	int ret;
+
+	if (sftp) {
+		ret = sftp_mkdir(sftp, path, mode);
+		sftp_err_to_errno(sftp);
+	} else
+		ret = mkdir(path, mode);
+
+	if (ret < 0 && errno == EEXIST) {
+		ret = 0;
+	}
+
+	return ret;
+}
+
+static void sftp_attr_to_stat(sftp_attributes attr, struct stat *st)
+{
+	memset(st, 0, sizeof(*st));
+	st->st_size = attr->size;
+	st->st_uid = attr->uid;
+	st->st_gid = attr->gid;
+	st->st_mode = attr->permissions;
+
+#if defined(__APPLE__)
+#define st_atim st_atimespec
+#define st_mtim st_mtimespec
+#define st_ctim st_ctimespec
+#endif
+	st->st_atim.tv_sec = attr->atime;
+	st->st_atim.tv_nsec = attr->atime_nseconds;
+	st->st_mtim.tv_sec = attr->mtime;
+	st->st_mtim.tv_nsec = attr->mtime_nseconds;
+	st->st_ctim.tv_sec = attr->createtime;
+	st->st_ctim.tv_nsec = attr->createtime_nseconds;
+
+	switch (attr->type) {
+	case SSH_FILEXFER_TYPE_REGULAR:
+		st->st_mode |= S_IFREG;
+		break;
+	case SSH_FILEXFER_TYPE_DIRECTORY:
+		st->st_mode |= S_IFDIR;
+		break;
+	case SSH_FILEXFER_TYPE_SYMLINK:
+		st->st_mode |= S_IFLNK;
+		break;
+	case SSH_FILEXFER_TYPE_SPECIAL:
+		st->st_mode |= S_IFCHR; /* or block? */
+		break;
+	case SSH_FILEXFER_TYPE_UNKNOWN:
+		st->st_mode |= S_IFIFO; /* really? */
+		break;
+	default:
+		pr_warn("unkown SSH_FILEXFER_TYPE %d", attr->type);
+	}
+}
+
+int mscp_stat(const char *path, struct stat *st, sftp_session sftp)
+{
+	sftp_attributes attr;
+	int ret = 0;
+
+	memset(st, 0, sizeof(*st));
+
+	if (sftp) {
+		attr = sftp_stat(sftp, path);
+		sftp_err_to_errno(sftp);
+		if (!attr)
+			return -1;
+
+		sftp_attr_to_stat(attr, st);
+		sftp_attributes_free(attr);
+		ret = 0;
+	} else
+		ret = stat(path, st);
+
+	return ret;
+}
+
+int mscp_stat_wrapped(const char *path, struct stat *st)
+{
+	return mscp_stat(path, st, tls_sftp);
+}
+
+int mscp_lstat(const char *path, struct stat *st, sftp_session sftp)
+{
+	sftp_attributes attr;
+	int ret = 0;
+
+	if (sftp) {
+		attr = sftp_lstat(sftp, path);
+		sftp_err_to_errno(sftp);
+		if (!attr)
+			return -1;
+
+		sftp_attr_to_stat(attr, st);
+		sftp_attributes_free(attr);
+		ret = 0;
+	} else
+		ret = lstat(path, st);
+
+	return ret;
+}
+
+int mscp_lstat_wrapped(const char *path, struct stat *st)
+{
+	return mscp_lstat(path, st, tls_sftp);
+}
+
+mf *mscp_open(const char *path, int flags, mode_t mode, sftp_session sftp)
+{
+	mf *f;
+
+	f = malloc(sizeof(*f));
+	if (!f)
+		return NULL;
+	memset(f, 0, sizeof(*f));
+
+	if (sftp) {
+		f->remote = sftp_open(sftp, path, flags, mode);
+		if (!f->remote) {
+			sftp_err_to_errno(sftp);
+			goto free_out;
+		}
+	} else {
+		f->local = open(path, flags, mode);
+		if (f->local < 0)
+			goto free_out;
+	}
+
+	return f;
+
+free_out:
+	free(f);
+	return NULL;
+}
+
+void mscp_close(mf *f)
+{
+	if (f->remote)
+		sftp_close(f->remote);
+	if (f->local > 0)
+		close(f->local);
+	free(f);
+}
+
+off_t mscp_lseek(mf *f, off_t off)
+{
+	off_t ret;
+
+	if (f->remote) {
+		ret = sftp_seek64(f->remote, off);
+		sftp_err_to_errno(f->remote->sftp);
+	} else
+		ret = lseek(f->local, off, SEEK_SET);
+
+	return ret;
+}
+
+int mscp_setstat(const char *path, struct stat *st, bool preserve_ts, sftp_session sftp)
+{
+	int ret;
+
+	if (sftp) {
+		struct sftp_attributes_struct attr;
+		memset(&attr, 0, sizeof(attr));
+		attr.permissions = st->st_mode;
+		attr.size = st->st_size;
+		attr.flags = (SSH_FILEXFER_ATTR_PERMISSIONS | SSH_FILEXFER_ATTR_SIZE);
+		if (preserve_ts) {
+			attr.atime = st->st_atim.tv_sec;
+			attr.atime_nseconds = st->st_atim.tv_nsec;
+			attr.mtime = st->st_mtim.tv_sec;
+			attr.mtime_nseconds = st->st_mtim.tv_nsec;
+			attr.flags |= (SSH_FILEXFER_ATTR_ACCESSTIME |
+				       SSH_FILEXFER_ATTR_MODIFYTIME |
+				       SSH_FILEXFER_ATTR_SUBSECOND_TIMES);
+		}
+		ret = sftp_setstat(sftp, path, &attr);
+		sftp_err_to_errno(sftp);
+	} else {
+		if ((ret = truncate(path, st->st_size)) < 0)
+			return ret;
+		if (preserve_ts) {
+			if ((ret = setutimes(path, st->st_atim, st->st_mtim)) < 0)
+				return ret;
+		}
+		if ((ret = chmod(path, st->st_mode)) < 0)
+			return ret;
+	}
+
+	return ret;
+}
+
+int mscp_glob(const char *pattern, int flags, glob_t *pglob, sftp_session sftp)
+{
+	int ret;
+	if (sftp) {
+#ifndef GLOB_ALTDIRFUNC
+#define GLOB_NOALTDIRMAGIC INT_MAX
+		/* musl does not implement GLOB_ALTDIRFUNC */
+		pglob->gl_pathc = 1;
+		pglob->gl_pathv = malloc(sizeof(char *));
+		pglob->gl_pathv[0] = strdup(pattern);
+		pglob->gl_offs = GLOB_NOALTDIRMAGIC;
+		return 0;
+#else
+		flags |= GLOB_ALTDIRFUNC;
+		set_tls_sftp_session(sftp);
+#if defined(__APPLE__) || defined(__FreeBSD__)
+		pglob->gl_opendir = (void *(*)(const char *))mscp_opendir_wrapped;
+		pglob->gl_readdir = (struct dirent * (*)(void *)) mscp_readdir;
+		pglob->gl_closedir = (void (*)(void *))mscp_closedir;
+		pglob->gl_lstat = mscp_lstat_wrapped;
+		pglob->gl_stat = mscp_stat_wrapped;
+#elif linux
+		pglob->gl_opendir = (void *(*)(const char *))mscp_opendir_wrapped;
+		pglob->gl_readdir = (void *(*)(void *))mscp_readdir;
+		pglob->gl_closedir = (void (*)(void *))mscp_closedir;
+		pglob->gl_lstat = (int (*)(const char *, void *))mscp_lstat_wrapped;
+		pglob->gl_stat = (int (*)(const char *, void *))mscp_stat_wrapped;
+#else
+#error unsupported platform
+#endif
+#endif
+	}
+
+	ret = glob(pattern, flags, NULL, pglob);
+
+	if (sftp)
+		set_tls_sftp_session(NULL);
+	return ret;
+}
+
+void mscp_globfree(glob_t *pglob)
+{
+#ifndef GLOB_ALTDIRFUNC
+	if (pglob->gl_offs == GLOB_NOALTDIRMAGIC) {
+		free(pglob->gl_pathv[0]);
+		free(pglob->gl_pathv);
+		return;
+	}
+#endif
+	globfree(pglob);
+}

src/fileops.h

@@ -0,0 +1,55 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
+#include <dirent.h>
+#include <sys/stat.h>
+#include <glob.h>
+
+#include <ssh.h>
+
+void set_tls_sftp_session(sftp_session sftp);
+/* sftp_session set by set_tls_sftp_session is sued in
+ mscp_open_wrapped(), mscp_stat_wrapped(), and
+ mscp_lstat_wrapped(). This _wrapped() functions exist for
+ sftp_glob() */
+
+/* directory operations */
+
+struct mdir_struct {
+	DIR *local;
+	sftp_dir remote;
+};
+typedef struct mdir_struct MDIR;
+
+MDIR *mscp_opendir(const char *path, sftp_session sftp);
+MDIR *mscp_opendir_wrapped(const char *path);
+void mscp_closedir(MDIR *md);
+struct dirent *mscp_readdir(MDIR *md);
+
+int mscp_mkdir(const char *path, mode_t mode, sftp_session sftp);
+
+/* stat operations */
+int mscp_stat(const char *path, struct stat *st, sftp_session sftp);
+int mscp_stat_wrapped(const char *path, struct stat *st);
+
+int mscp_lstat(const char *path, struct stat *st, sftp_session sftp);
+int mscp_lstat_wrapped(const char *path, struct stat *st);
+
+/* file operations */
+
+struct mf_struct {
+	sftp_file remote;
+	int local;
+};
+typedef struct mf_struct mf;
+
+mf *mscp_open(const char *path, int flags, mode_t mode, sftp_session sftp);
+void mscp_close(mf *f);
+off_t mscp_lseek(mf *f, off_t off);
+
+/* mscp_setstat() involves chmod and truncate. It executes both at
+ * once via a single SFTP command (sftp_setstat()).
+ */
+int mscp_setstat(const char *path, struct stat *st, bool preserve_ts, sftp_session sftp);
+
+/* remote glob */
+int mscp_glob(const char *pattern, int flags, glob_t *pglob, sftp_session sftp);
+void mscp_globfree(glob_t *pglob);

src/list.h

@@ -1,532 +0,0 @@
-/**
- * 
- * I grub it from linux kernel source code and fix it for user space
- * program. Of course, this is a GPL licensed header file.
- *
- * Here is a recipe to cook list.h for user space program
- *
- * 1. copy list.h from linux/include/list.h
- * 2. remove 
- *     - #ifdef __KERNE__ and its #endif
- *     - all #include line
- *     - prefetch() and rcu related functions
- * 3. add macro offsetof() and container_of
- *
- * - kazutomo@mcs.anl.gov
- */
-#ifndef _LINUX_LIST_H
-#define _LINUX_LIST_H
-
-/**
- * @name from other kernel headers
- */
-/*@{*/
-
-/**
- * Get offset of a member
- */
-#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
-
-/**
- * Casts a member of a structure out to the containing structure
- * @param ptr        the pointer to the member.
- * @param type       the type of the container struct this is embedded in.
- * @param member     the name of the member within the struct.
- *
- */
-#define container_of(ptr, type, member) ({                      \
-        const typeof( ((type *)0)->member ) *__mptr = (ptr);    \
-        (type *)( (char *)__mptr - offsetof(type,member) );})
-/*@}*/
-
-
-/*
- * These are non-NULL pointers that will result in page faults
- * under normal circumstances, used to verify that nobody uses
- * non-initialized list entries.
- */
-#define LIST_POISON1  ((void *) 0x00100100)
-#define LIST_POISON2  ((void *) 0x00200200)
-
-/**
- * Simple doubly linked list implementation.
- *
- * Some of the internal functions ("__xxx") are useful when
- * manipulating whole lists rather than single entries, as
- * sometimes we already know the next/prev entries and we can
- * generate better code by using them directly rather than
- * using the generic single-entry routines.
- */
-struct list_head {
-	struct list_head *next, *prev;
-};
-
-#define LIST_HEAD_INIT(name) { &(name), &(name) }
-
-#define LIST_HEAD(name) \
-	struct list_head name = LIST_HEAD_INIT(name)
-
-#define INIT_LIST_HEAD(ptr) do { \
-	(ptr)->next = (ptr); (ptr)->prev = (ptr); \
-} while (0)
-
-/*
- * Insert a new entry between two known consecutive entries.
- *
- * This is only for internal list manipulation where we know
- * the prev/next entries already!
- */
-static inline void __list_add(struct list_head *new,
-			      struct list_head *prev,
-			      struct list_head *next)
-{
-	next->prev = new;
-	new->next = next;
-	new->prev = prev;
-	prev->next = new;
-}
-
-/**
- * list_add - add a new entry
- * @new: new entry to be added
- * @head: list head to add it after
- *
- * Insert a new entry after the specified head.
- * This is good for implementing stacks.
- */
-static inline void list_add(struct list_head *new, struct list_head *head)
-{
-	__list_add(new, head, head->next);
-}
-
-/**
- * list_add_tail - add a new entry
- * @new: new entry to be added
- * @head: list head to add it before
- *
- * Insert a new entry before the specified head.
- * This is useful for implementing queues.
- */
-static inline void list_add_tail(struct list_head *new, struct list_head *head)
-{
-	__list_add(new, head->prev, head);
-}
-
-
-/*
- * Delete a list entry by making the prev/next entries
- * point to each other.
- *
- * This is only for internal list manipulation where we know
- * the prev/next entries already!
- */
-static inline void __list_del(struct list_head * prev, struct list_head * next)
-{
-	next->prev = prev;
-	prev->next = next;
-}
-
-/**
- * list_del - deletes entry from list.
- * @entry: the element to delete from the list.
- * Note: list_empty on entry does not return true after this, the entry is
- * in an undefined state.
- */
-static inline void list_del(struct list_head *entry)
-{
-	__list_del(entry->prev, entry->next);
-	entry->next = LIST_POISON1;
-	entry->prev = LIST_POISON2;
-}
-
-
-
-/**
- * list_del_init - deletes entry from list and reinitialize it.
- * @entry: the element to delete from the list.
- */
-static inline void list_del_init(struct list_head *entry)
-{
-	__list_del(entry->prev, entry->next);
-	INIT_LIST_HEAD(entry);
-}
-
-/**
- * list_move - delete from one list and add as another's head
- * @list: the entry to move
- * @head: the head that will precede our entry
- */
-static inline void list_move(struct list_head *list, struct list_head *head)
-{
-        __list_del(list->prev, list->next);
-        list_add(list, head);
-}
-
-/**
- * list_move_tail - delete from one list and add as another's tail
- * @list: the entry to move
- * @head: the head that will follow our entry
- */
-static inline void list_move_tail(struct list_head *list,
-				  struct list_head *head)
-{
-        __list_del(list->prev, list->next);
-        list_add_tail(list, head);
-}
-
-/**
- * list_empty - tests whether a list is empty
- * @head: the list to test.
- */
-static inline int list_empty(const struct list_head *head)
-{
-	return head->next == head;
-}
-
-static inline void __list_splice(struct list_head *list,
-				 struct list_head *head)
-{
-	struct list_head *first = list->next;
-	struct list_head *last = list->prev;
-	struct list_head *at = head->next;
-
-	first->prev = head;
-	head->next = first;
-
-	last->next = at;
-	at->prev = last;
-}
-
-/**
- * list_splice - join two lists
- * @list: the new list to add.
- * @head: the place to add it in the first list.
- */
-static inline void list_splice(struct list_head *list, struct list_head *head)
-{
-	if (!list_empty(list))
-		__list_splice(list, head);
-}
-
-/**
- * list_splice_init - join two lists and reinitialise the emptied list.
- * @list: the new list to add.
- * @head: the place to add it in the first list.
- *
- * The list at @list is reinitialised
- */
-static inline void list_splice_init(struct list_head *list,
-				    struct list_head *head)
-{
-	if (!list_empty(list)) {
-		__list_splice(list, head);
-		INIT_LIST_HEAD(list);
-	}
-}
-
-/**
- * list_entry - get the struct for this entry
- * @ptr:	the &struct list_head pointer.
- * @type:	the type of the struct this is embedded in.
- * @member:	the name of the list_struct within the struct.
- */
-#define list_entry(ptr, type, member) \
-	container_of(ptr, type, member)
-
-/**
- * list_for_each	-	iterate over a list
- * @pos:	the &struct list_head to use as a loop counter.
- * @head:	the head for your list.
- */
-
-#define list_for_each(pos, head) \
-  for (pos = (head)->next; pos != (head);	\
-       pos = pos->next)
-
-/**
- * __list_for_each	-	iterate over a list
- * @pos:	the &struct list_head to use as a loop counter.
- * @head:	the head for your list.
- *
- * This variant differs from list_for_each() in that it's the
- * simplest possible list iteration code, no prefetching is done.
- * Use this for code that knows the list to be very short (empty
- * or 1 entry) most of the time.
- */
-#define __list_for_each(pos, head) \
-	for (pos = (head)->next; pos != (head); pos = pos->next)
-
-/**
- * list_for_each_prev	-	iterate over a list backwards
- * @pos:	the &struct list_head to use as a loop counter.
- * @head:	the head for your list.
- */
-#define list_for_each_prev(pos, head) \
-	for (pos = (head)->prev; prefetch(pos->prev), pos != (head); \
-        	pos = pos->prev)
-
-/**
- * list_for_each_safe	-	iterate over a list safe against removal of list entry
- * @pos:	the &struct list_head to use as a loop counter.
- * @n:		another &struct list_head to use as temporary storage
- * @head:	the head for your list.
- */
-#define list_for_each_safe(pos, n, head) \
-	for (pos = (head)->next, n = pos->next; pos != (head); \
-		pos = n, n = pos->next)
-
-/**
- * list_for_each_entry	-	iterate over list of given type
- * @pos:	the type * to use as a loop counter.
- * @head:	the head for your list.
- * @member:	the name of the list_struct within the struct.
- */
-#define list_for_each_entry(pos, head, member)				\
-	for (pos = list_entry((head)->next, typeof(*pos), member);	\
-	     &pos->member != (head);					\
-	     pos = list_entry(pos->member.next, typeof(*pos), member))
-
-/**
- * list_for_each_entry_reverse - iterate backwards over list of given type.
- * @pos:	the type * to use as a loop counter.
- * @head:	the head for your list.
- * @member:	the name of the list_struct within the struct.
- */
-#define list_for_each_entry_reverse(pos, head, member)			\
-	for (pos = list_entry((head)->prev, typeof(*pos), member);	\
-	     &pos->member != (head); 	\
-	     pos = list_entry(pos->member.prev, typeof(*pos), member))
-
-/**
- * list_prepare_entry - prepare a pos entry for use as a start point in
- *			list_for_each_entry_continue
- * @pos:	the type * to use as a start point
- * @head:	the head of the list
- * @member:	the name of the list_struct within the struct.
- */
-#define list_prepare_entry(pos, head, member) \
-	((pos) ? : list_entry(head, typeof(*pos), member))
-
-/**
- * list_for_each_entry_continue -	iterate over list of given type
- *			continuing after existing point
- * @pos:	the type * to use as a loop counter.
- * @head:	the head for your list.
- * @member:	the name of the list_struct within the struct.
- */
-#define list_for_each_entry_continue(pos, head, member) 		\
-	for (pos = list_entry(pos->member.next, typeof(*pos), member);	\
-	     &pos->member != (head);	\
-	     pos = list_entry(pos->member.next, typeof(*pos), member))
-
-/**
- * list_for_each_entry_safe - iterate over list of given type safe against removal of list entry
- * @pos:	the type * to use as a loop counter.
- * @n:		another type * to use as temporary storage
- * @head:	the head for your list.
- * @member:	the name of the list_struct within the struct.
- */
-#define list_for_each_entry_safe(pos, n, head, member)			\
-	for (pos = list_entry((head)->next, typeof(*pos), member),	\
-		n = list_entry(pos->member.next, typeof(*pos), member);	\
-	     &pos->member != (head); 					\
-	     pos = n, n = list_entry(n->member.next, typeof(*n), member))
-
-/**
- * list_for_each_entry_safe_continue -	iterate over list of given type
- *			continuing after existing point safe against removal of list entry
- * @pos:	the type * to use as a loop counter.
- * @n:		another type * to use as temporary storage
- * @head:	the head for your list.
- * @member:	the name of the list_struct within the struct.
- */
-#define list_for_each_entry_safe_continue(pos, n, head, member) 		\
-	for (pos = list_entry(pos->member.next, typeof(*pos), member), 		\
-		n = list_entry(pos->member.next, typeof(*pos), member);		\
-	     &pos->member != (head);						\
-	     pos = n, n = list_entry(n->member.next, typeof(*n), member))
-
-/**
- * list_for_each_entry_safe_reverse - iterate backwards over list of given type safe against
- *				      removal of list entry
- * @pos:	the type * to use as a loop counter.
- * @n:		another type * to use as temporary storage
- * @head:	the head for your list.
- * @member:	the name of the list_struct within the struct.
- */
-#define list_for_each_entry_safe_reverse(pos, n, head, member)		\
-	for (pos = list_entry((head)->prev, typeof(*pos), member),	\
-		n = list_entry(pos->member.prev, typeof(*pos), member);	\
-	     &pos->member != (head); 					\
-	     pos = n, n = list_entry(n->member.prev, typeof(*n), member))
-
-
-
-
-/*
- * Double linked lists with a single pointer list head.
- * Mostly useful for hash tables where the two pointer list head is
- * too wasteful.
- * You lose the ability to access the tail in O(1).
- */
-
-struct hlist_head {
-	struct hlist_node *first;
-};
-
-struct hlist_node {
-	struct hlist_node *next, **pprev;
-};
-
-#define HLIST_HEAD_INIT { .first = NULL }
-#define HLIST_HEAD(name) struct hlist_head name = {  .first = NULL }
-#define INIT_HLIST_HEAD(ptr) ((ptr)->first = NULL)
-#define INIT_HLIST_NODE(ptr) ((ptr)->next = NULL, (ptr)->pprev = NULL)
-
-static inline int hlist_unhashed(const struct hlist_node *h)
-{
-	return !h->pprev;
-}
-
-static inline int hlist_empty(const struct hlist_head *h)
-{
-	return !h->first;
-}
-
-static inline void __hlist_del(struct hlist_node *n)
-{
-	struct hlist_node *next = n->next;
-	struct hlist_node **pprev = n->pprev;
-	*pprev = next;
-	if (next)
-		next->pprev = pprev;
-}
-
-static inline void hlist_del(struct hlist_node *n)
-{
-	__hlist_del(n);
-	n->next = LIST_POISON1;
-	n->pprev = LIST_POISON2;
-}
-
-
-static inline void hlist_del_init(struct hlist_node *n)
-{
-	if (n->pprev)  {
-		__hlist_del(n);
-		INIT_HLIST_NODE(n);
-	}
-}
-
-static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h)
-{
-	struct hlist_node *first = h->first;
-	n->next = first;
-	if (first)
-		first->pprev = &n->next;
-	h->first = n;
-	n->pprev = &h->first;
-}
-
-
-
-/* next must be != NULL */
-static inline void hlist_add_before(struct hlist_node *n,
-					struct hlist_node *next)
-{
-	n->pprev = next->pprev;
-	n->next = next;
-	next->pprev = &n->next;
-	*(n->pprev) = n;
-}
-
-static inline void hlist_add_after(struct hlist_node *n,
-					struct hlist_node *next)
-{
-	next->next = n->next;
-	n->next = next;
-	next->pprev = &n->next;
-
-	if(next->next)
-		next->next->pprev  = &next->next;
-}
-
-
-
-#define hlist_entry(ptr, type, member) container_of(ptr,type,member)
-
-#define hlist_for_each(pos, head) \
-	for (pos = (head)->first; pos && ({ prefetch(pos->next); 1; }); \
-	     pos = pos->next)
-
-#define hlist_for_each_safe(pos, n, head) \
-	for (pos = (head)->first; pos && ({ n = pos->next; 1; }); \
-	     pos = n)
-
-/**
- * hlist_for_each_entry	- iterate over list of given type
- * @tpos:	the type * to use as a loop counter.
- * @pos:	the &struct hlist_node to use as a loop counter.
- * @head:	the head for your list.
- * @member:	the name of the hlist_node within the struct.
- */
-#define hlist_for_each_entry(tpos, pos, head, member)			 \
-	for (pos = (head)->first;					 \
-	     pos && ({ prefetch(pos->next); 1;}) &&			 \
-		({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
-	     pos = pos->next)
-
-/**
- * hlist_for_each_entry_continue - iterate over a hlist continuing after existing point
- * @tpos:	the type * to use as a loop counter.
- * @pos:	the &struct hlist_node to use as a loop counter.
- * @member:	the name of the hlist_node within the struct.
- */
-#define hlist_for_each_entry_continue(tpos, pos, member)		 \
-	for (pos = (pos)->next;						 \
-	     pos && ({ prefetch(pos->next); 1;}) &&			 \
-		({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
-	     pos = pos->next)
-
-/**
- * hlist_for_each_entry_from - iterate over a hlist continuing from existing point
- * @tpos:	the type * to use as a loop counter.
- * @pos:	the &struct hlist_node to use as a loop counter.
- * @member:	the name of the hlist_node within the struct.
- */
-#define hlist_for_each_entry_from(tpos, pos, member)			 \
-	for (; pos && ({ prefetch(pos->next); 1;}) &&			 \
-		({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
-	     pos = pos->next)
-
-/**
- * hlist_for_each_entry_safe - iterate over list of given type safe against removal of list entry
- * @tpos:	the type * to use as a loop counter.
- * @pos:	the &struct hlist_node to use as a loop counter.
- * @n:		another &struct hlist_node to use as temporary storage
- * @head:	the head for your list.
- * @member:	the name of the hlist_node within the struct.
- */
-#define hlist_for_each_entry_safe(tpos, pos, n, head, member) 		 \
-	for (pos = (head)->first;					 \
-	     pos && ({ n = pos->next; 1; }) && 				 \
-		({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
-	     pos = n)
-
-
-/**
- * list_count - return length of list
- * @head        the head for your list.
- */
-static inline int list_count(struct list_head *head)
-{
-	int n = 0;
-	struct list_head *p;
-
-	list_for_each(p, head) n++;
-	return n;
-}
-
-
-#endif
-

src/minmax.h

@@ -0,0 +1,8 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
+#ifndef _MINMAX_H_
+#define _MINMAX_H_
+
+#define min(a, b) (((a) > (b)) ? (b) : (a))
+#define max(a, b) (((a) > (b)) ? (a) : (b))
+
+#endif /* _MINMAX_H_ */

src/mscp.c

@@ -0,0 +1,776 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
+#include <stdbool.h>
+#include <unistd.h>
+#include <math.h>
+#include <pthread.h>
+#include <semaphore.h>
+#include <sys/time.h>
+
+#include <pool.h>
+#include <minmax.h>
+#include <ssh.h>
+#include <path.h>
+#include <checkpoint.h>
+#include <fileops.h>
+#include <atomic.h>
+#include <platform.h>
+#include <print.h>
+#include <strerrno.h>
+#include <mscp.h>
+#include <bwlimit.h>
+
+#include <openbsd-compat/openbsd-compat.h>
+
+struct mscp_thread {
+	struct mscp *m;
+	sftp_session sftp;
+
+	/* attributes used by copy threads */
+	size_t copied_bytes;
+	int id;
+	int cpu;
+
+	/* thread-specific values */
+	pthread_t tid;
+	int ret;
+};
+
+struct mscp {
+	char *remote; /* remote host (and uername) */
+	int direction; /* copy direction */
+	char dst_path[PATH_MAX];
+
+	struct mscp_opts *opts;
+	struct mscp_ssh_opts *ssh_opts;
+
+	int *cores; /* usable cpu cores by COREMASK */
+	int nr_cores; /* length of array of cores */
+
+	sem_t *sem; /* semaphore for concurrent  connecting ssh sessions */
+
+	sftp_session first; /* first sftp session */
+
+	pool *src_pool, *path_pool, *chunk_pool, *thread_pool;
+
+	size_t total_bytes; /* total_bytes to be copied */
+	bool chunk_pool_ready;
+#define chunk_pool_is_ready(m) ((m)->chunk_pool_ready)
+#define chunk_pool_set_ready(m, b) ((m)->chunk_pool_ready = b)
+
+	struct bwlimit bw; /* bandwidth limit mechanism */
+
+	struct mscp_thread scan; /* mscp_thread for mscp_scan_thread() */
+};
+
+#define DEFAULT_MIN_CHUNK_SZ (16 << 20) /* 16MB */
+#define DEFAULT_NR_AHEAD 32
+#define DEFAULT_BUF_SZ 16384
+/* XXX: we use 16384 byte buffer pointed by
+ * https://api.libssh.org/stable/libssh_tutor_sftp.html. The larget
+ * read length from sftp_async_read is 65536 byte. Read sizes larger
+ * than 65536 cause a situation where data remainds but
+ * sftp_async_read returns 0.
+ */
+
+#define DEFAULT_MAX_STARTUPS 8
+
+#define non_null_string(s) (s[0] != '\0')
+
+static int expand_coremask(const char *coremask, int **cores, int *nr_cores)
+{
+	int n, *core_list, nr_usable, nr_all;
+	char c[2] = { 'x', '\0' };
+	const char *_coremask;
+	long v, needle;
+	int ncores = nr_cpus();
+
+	/*
+         * This function returns array of usable cores in `cores` and
+         * returns the number of usable cores (array length) through
+         * nr_cores.
+         */
+
+	if (strncmp(coremask, "0x", 2) == 0)
+		_coremask = coremask + 2;
+	else
+		_coremask = coremask;
+
+	core_list = realloc(NULL, sizeof(int) * 64);
+	if (!core_list) {
+		priv_set_errv("realloc: %s", strerrno());
+		return -1;
+	}
+
+	nr_usable = 0;
+	nr_all = 0;
+	for (n = strlen(_coremask) - 1; n >= 0; n--) {
+		c[0] = _coremask[n];
+		v = strtol(c, NULL, 16);
+		if (v == LONG_MIN || v == LONG_MAX) {
+			priv_set_errv("invalid coremask: %s", coremask);
+			return -1;
+		}
+
+		for (needle = 0x01; needle < 0x10; needle <<= 1) {
+			nr_all++;
+			if (nr_all > ncores)
+				break; /* too long coremask */
+			if (v & needle) {
+				nr_usable++;
+				core_list = realloc(core_list, sizeof(int) * nr_usable);
+				if (!core_list) {
+					priv_set_errv("realloc: %s", strerrno());
+					return -1;
+				}
+				core_list[nr_usable - 1] = nr_all - 1;
+			}
+		}
+	}
+
+	if (nr_usable < 1) {
+		priv_set_errv("invalid core mask: %s", coremask);
+		return -1;
+	}
+
+	*cores = core_list;
+	*nr_cores = nr_usable;
+	return 0;
+}
+
+static int default_nr_threads()
+{
+	return (int)(floor(log(nr_cpus()) * 2) + 1);
+}
+
+static int validate_and_set_defaut_params(struct mscp_opts *o)
+{
+	if (o->nr_threads < 0) {
+		priv_set_errv("invalid nr_threads: %d", o->nr_threads);
+		return -1;
+	} else if (o->nr_threads == 0)
+		o->nr_threads = default_nr_threads();
+
+	if (o->nr_ahead < 0) {
+		priv_set_errv("invalid nr_ahead: %d", o->nr_ahead);
+		return -1;
+	} else if (o->nr_ahead == 0)
+		o->nr_ahead = DEFAULT_NR_AHEAD;
+
+	if (o->min_chunk_sz == 0)
+		o->min_chunk_sz = DEFAULT_MIN_CHUNK_SZ;
+
+	if (o->max_chunk_sz) {
+		if (o->min_chunk_sz > o->max_chunk_sz) {
+			priv_set_errv("smaller max chunk size than "
+				      "min chunk size: %lu < %lu",
+				      o->max_chunk_sz, o->min_chunk_sz);
+			return -1;
+		}
+	}
+
+	if (o->buf_sz == 0)
+		o->buf_sz = DEFAULT_BUF_SZ;
+	else if (o->buf_sz == 0) {
+		priv_set_errv("invalid buf size: %lu", o->buf_sz);
+		return -1;
+	}
+
+	if (o->max_startups == 0)
+		o->max_startups = DEFAULT_MAX_STARTUPS;
+	else if (o->max_startups < 0) {
+		priv_set_errv("invalid max_startups: %d", o->max_startups);
+		return -1;
+	}
+
+	if (o->interval > 0) {
+		/* when the interval is set, establish SSH connections sequentially. */
+		o->max_startups = 1;
+	}
+
+	return 0;
+}
+
+int mscp_set_remote(struct mscp *m, const char *remote_host, int direction)
+{
+	if (!remote_host) {
+		priv_set_errv("empty remote host");
+		return -1;
+	}
+
+	if (!(direction == MSCP_DIRECTION_L2R || direction == MSCP_DIRECTION_R2L)) {
+		priv_set_errv("invalid copy direction: %d", direction);
+		return -1;
+	}
+
+	if (!(m->remote = strdup(remote_host))) {
+		priv_set_errv("strdup: %s", strerrno());
+		return -1;
+	}
+	m->direction = direction;
+
+	return 0;
+}
+
+struct mscp *mscp_init(struct mscp_opts *o, struct mscp_ssh_opts *s)
+{
+	struct mscp *m;
+	int n;
+
+	set_print_severity(o->severity);
+
+	if (validate_and_set_defaut_params(o) < 0) {
+		return NULL;
+	}
+
+	if (!(m = malloc(sizeof(*m)))) {
+		priv_set_errv("malloc: %s", strerrno());
+		return NULL;
+	}
+	memset(m, 0, sizeof(*m));
+	m->opts = o;
+	m->ssh_opts = s;
+	chunk_pool_set_ready(m, false);
+
+	if (!(m->src_pool = pool_new())) {
+		priv_set_errv("pool_new: %s", strerrno());
+		goto free_out;
+	}
+
+	if (!(m->path_pool = pool_new())) {
+		priv_set_errv("pool_new: %s", strerrno());
+		goto free_out;
+	}
+
+	if (!(m->chunk_pool = pool_new())) {
+		priv_set_errv("pool_new: %s", strerrno());
+		goto free_out;
+	}
+
+	if (!(m->thread_pool = pool_new())) {
+		priv_set_errv("pool_new: %s", strerrno());
+		goto free_out;
+	}
+
+	if ((m->sem = sem_create(o->max_startups)) == NULL) {
+		priv_set_errv("sem_create: %s", strerrno());
+		goto free_out;
+	}
+
+	if (o->coremask) {
+		if (expand_coremask(o->coremask, &m->cores, &m->nr_cores) < 0)
+			goto free_out;
+		char b[512], c[8];
+		memset(b, 0, sizeof(b));
+		for (n = 0; n < m->nr_cores; n++) {
+			memset(c, 0, sizeof(c));
+			snprintf(c, sizeof(c) - 1, " %d", m->cores[n]);
+			strlcat(b, c, sizeof(b));
+		}
+		pr_notice("usable cpu cores:%s", b);
+	}
+
+	if (bwlimit_init(&m->bw, o->bitrate, 100) < 0) { /* 100ms window (hardcoded) */
+		priv_set_errv("bwlimit_init: %s", strerrno());
+		goto free_out;
+	}
+
+	/* workaround: set libssh using openssh proxyjump
+	 * https://gitlab.com/libssh/libssh-mirror/-/issues/319 */
+	ssh_use_openssh_proxy_jumps(1);
+
+	/* call ssh_init() because libssh is statically linked */
+	ssh_init();
+
+	return m;
+
+free_out:
+	if (m->src_pool)
+		pool_free(m->src_pool);
+	if (m->path_pool)
+		pool_free(m->path_pool);
+	if (m->chunk_pool)
+		pool_free(m->chunk_pool);
+	if (m->thread_pool)
+		pool_free(m->thread_pool);
+	if (m->remote)
+		free(m->remote);
+	free(m);
+	return NULL;
+}
+
+int mscp_connect(struct mscp *m)
+{
+	m->first = ssh_init_sftp_session(m->remote, m->ssh_opts);
+	if (!m->first)
+		return -1;
+
+	return 0;
+}
+
+int mscp_add_src_path(struct mscp *m, const char *src_path)
+{
+	char *s = strdup(src_path);
+	if (!s) {
+		priv_set_errv("strdup: %s", strerrno());
+		return -1;
+	}
+	if (pool_push(m->src_pool, s) < 0) {
+		priv_set_errv("pool_push: %s", strerrno());
+		return -1;
+	}
+	return 0;
+}
+
+int mscp_set_dst_path(struct mscp *m, const char *dst_path)
+{
+	if (strlen(dst_path) + 1 >= PATH_MAX) {
+		priv_set_errv("too long dst path: %s", dst_path);
+		return -1;
+	}
+
+	if (!non_null_string(dst_path)) {
+		priv_set_errv("empty dst path");
+		return -1;
+	}
+
+	strncpy(m->dst_path, dst_path, PATH_MAX);
+
+	return 0;
+}
+
+static size_t get_page_mask(void)
+{
+	size_t page_sz = sysconf(_SC_PAGESIZE);
+	return ~(page_sz - 1);
+}
+
+static void mscp_stop_copy_thread(struct mscp *m)
+{
+	struct mscp_thread *t;
+	unsigned int idx;
+	pool_lock(m->thread_pool);
+	pool_for_each(m->thread_pool, t, idx) {
+		if (t->tid)
+			pthread_cancel(t->tid);
+	}
+	pool_unlock(m->thread_pool);
+}
+
+static void mscp_stop_scan_thread(struct mscp *m)
+{
+	if (m->scan.tid)
+		pthread_cancel(m->scan.tid);
+}
+
+void mscp_stop(struct mscp *m)
+{
+	mscp_stop_scan_thread(m);
+	mscp_stop_copy_thread(m);
+}
+
+void *mscp_scan_thread(void *arg)
+{
+	struct mscp_thread *t = arg;
+	struct mscp *m = t->m;
+	sftp_session src_sftp = NULL, dst_sftp = NULL;
+	struct path_resolve_args a;
+	struct path *p;
+	struct stat ss, ds;
+	char *src_path;
+	glob_t pglob;
+	int n;
+
+	switch (m->direction) {
+	case MSCP_DIRECTION_L2R:
+		src_sftp = NULL;
+		dst_sftp = t->sftp;
+		break;
+	case MSCP_DIRECTION_R2L:
+		src_sftp = t->sftp;
+		dst_sftp = NULL;
+		break;
+	default:
+		pr_err("invalid copy direction: %d", m->direction);
+		goto err_out;
+	}
+
+	/* initialize path_resolve_args */
+	memset(&a, 0, sizeof(a));
+	a.total_bytes = &m->total_bytes;
+
+	if (pool_size(m->src_pool) > 1)
+		a.dst_path_should_dir = true;
+
+	if (m->dst_path[strlen(m->dst_path) - 1] == '/')
+		a.dst_path_should_dir = true;
+
+	if (mscp_stat(m->dst_path, &ds, dst_sftp) == 0) {
+		if (S_ISDIR(ds.st_mode))
+			a.dst_path_is_dir = true;
+	}
+
+	a.path_pool = m->path_pool;
+	a.chunk_pool = m->chunk_pool;
+	a.nr_conn = m->opts->nr_threads;
+	a.min_chunk_sz = m->opts->min_chunk_sz;
+	a.max_chunk_sz = m->opts->max_chunk_sz;
+	a.chunk_align = get_page_mask();
+
+	pr_info("start to walk source path(s)");
+
+	/* walk each src_path recusively, and resolve path->dst_path for each src */
+	pool_iter_for_each(m->src_pool, src_path) {
+		memset(&pglob, 0, sizeof(pglob));
+		if (mscp_glob(src_path, GLOB_NOCHECK, &pglob, src_sftp) < 0) {
+			pr_err("mscp_glob: %s", strerrno());
+			goto err_out;
+		}
+
+		for (n = 0; n < pglob.gl_pathc; n++) {
+			if (mscp_stat(pglob.gl_pathv[n], &ss, src_sftp) < 0) {
+				pr_err("stat: %s %s", src_path, strerrno());
+				goto err_out;
+			}
+
+			if (!a.dst_path_should_dir && pglob.gl_pathc > 1)
+				a.dst_path_should_dir = true; /* we have over 1 srces */
+
+			/* set path specific args */
+			a.src_path = pglob.gl_pathv[n];
+			a.dst_path = m->dst_path;
+			a.src_path_is_dir = S_ISDIR(ss.st_mode);
+
+			if (walk_src_path(src_sftp, pglob.gl_pathv[n], &a) < 0)
+				goto err_out;
+		}
+		mscp_globfree(&pglob);
+	}
+
+	pr_info("walk source path(s) done");
+	t->ret = 0;
+	chunk_pool_set_ready(m, true);
+	return NULL;
+
+err_out:
+	t->ret = -1;
+	chunk_pool_set_ready(m, true);
+	return NULL;
+}
+
+int mscp_scan(struct mscp *m)
+{
+	struct mscp_thread *t = &m->scan;
+	int ret;
+
+	memset(t, 0, sizeof(*t));
+	t->m = m;
+	t->sftp = m->first;
+
+	if ((ret = pthread_create(&t->tid, NULL, mscp_scan_thread, t)) < 0) {
+		priv_set_err("pthread_create: %d", ret);
+		return -1;
+	}
+
+	/* We wait for there are over nr_threads chunks to determine
+	 * actual number of threads (and connections), or scan
+	 * finished. If the number of chunks are smaller than
+	 * nr_threads, we adjust nr_threads to the number of chunks.
+	 */
+	while (!chunk_pool_is_ready(m) && pool_size(m->chunk_pool) < m->opts->nr_threads)
+		usleep(100);
+
+	return 0;
+}
+
+int mscp_scan_join(struct mscp *m)
+{
+	struct mscp_thread *t = &m->scan;
+	if (t->tid) {
+		pthread_join(t->tid, NULL);
+		t->tid = 0;
+		return t->ret;
+	}
+	return 0;
+}
+
+int mscp_checkpoint_get_remote(const char *pathname, char *remote, size_t len, int *dir)
+{
+	return checkpoint_load_remote(pathname, remote, len, dir);
+}
+
+int mscp_checkpoint_load(struct mscp *m, const char *pathname)
+{
+	struct chunk *c;
+	unsigned int i;
+
+	if (checkpoint_load_paths(pathname, m->path_pool, m->chunk_pool) < 0)
+		return -1;
+
+	/* totaling up bytes to be transferred and set chunk_pool is
+	 * ready instead of the mscp_scan thread */
+	m->total_bytes = 0;
+	pool_for_each(m->chunk_pool, c, i) {
+		m->total_bytes += c->len;
+	}
+	chunk_pool_set_ready(m, true);
+
+	return 0;
+}
+
+int mscp_checkpoint_save(struct mscp *m, const char *pathname)
+{
+	return checkpoint_save(pathname, m->direction, m->ssh_opts->login_name, m->remote,
+			       m->path_pool, m->chunk_pool);
+}
+
+static void *mscp_copy_thread(void *arg);
+
+static struct mscp_thread *mscp_copy_thread_spawn(struct mscp *m, int id)
+{
+	struct mscp_thread *t;
+	int ret;
+
+	if (!(t = malloc(sizeof(*t)))) {
+		priv_set_errv("malloc: %s", strerrno());
+		return NULL;
+	}
+
+	memset(t, 0, sizeof(*t));
+	t->m = m;
+	t->id = id;
+	if (m->cores == NULL)
+		t->cpu = -1; /* not pinned to cpu */
+	else
+		t->cpu = m->cores[id % m->nr_cores];
+
+	if ((ret = pthread_create(&t->tid, NULL, mscp_copy_thread, t)) < 0) {
+		priv_set_errv("pthread_create: %d", ret);
+		free(t);
+		return NULL;
+	}
+
+	return t;
+}
+
+int mscp_start(struct mscp *m)
+{
+	struct mscp_thread *t;
+	int n, ret = 0;
+
+	if ((n = pool_size(m->chunk_pool)) < m->opts->nr_threads) {
+		pr_notice("we have %d chunk(s), set number of connections to %d", n, n);
+		m->opts->nr_threads = n;
+	}
+
+	pr_notice("threads: %d",m->opts->nr_threads);
+	pr_notice("bwlimit: %ld bps", m->bw.bps);
+
+	for (n = 0; n < m->opts->nr_threads; n++) {
+		t = mscp_copy_thread_spawn(m, n);
+		if (!t)
+			break;
+		if (pool_push_lock(m->thread_pool, t) < 0) {
+			priv_set_errv("pool_push_lock: %s", strerrno());
+			break;
+		}
+	}
+
+	return n;
+}
+
+int mscp_join(struct mscp *m)
+{
+	struct mscp_thread *t;
+	struct path *p;
+	unsigned int idx;
+	size_t total_copied_bytes = 0, nr_copied = 0, nr_tobe_copied = 0;
+	int n, ret = 0;
+
+	/* waiting for scan thread joins... */
+	ret = mscp_scan_join(m);
+
+	/* waiting for copy threads join... */
+	pool_for_each(m->thread_pool, t, idx) {
+		pthread_join(t->tid, NULL);
+	}
+
+	pool_for_each(m->thread_pool, t, idx) {
+		total_copied_bytes += t->copied_bytes;
+		if (t->ret != 0)
+			ret = t->ret;
+		if (t->sftp) {
+			ssh_sftp_close(t->sftp);
+			t->sftp = NULL;
+		}
+	}
+
+	/* count up number of transferred files */
+	pool_iter_for_each(m->path_pool, p) {
+		nr_tobe_copied++;
+		if (p->state == FILE_STATE_DONE) {
+			nr_copied++;
+		}
+	}
+
+	if (m->first) {
+		ssh_sftp_close(m->first);
+		m->first = NULL;
+	}
+
+	pr_notice("%lu/%lu bytes copied for %lu/%lu files", total_copied_bytes,
+		  m->total_bytes, nr_copied, nr_tobe_copied);
+
+	return ret;
+}
+
+/* copy thread-related functions */
+
+static void wait_for_interval(int interval)
+{
+	_Atomic static long next;
+	struct timeval t;
+	long now;
+
+	gettimeofday(&t, NULL);
+	now = t.tv_sec * 1000000 + t.tv_usec;
+
+	if (next - now > 0)
+		usleep(next - now);
+
+	next = now + interval * 1000000;
+}
+
+void *mscp_copy_thread(void *arg)
+{
+	sftp_session src_sftp, dst_sftp;
+	struct mscp_thread *t = arg;
+	struct mscp *m = t->m;
+	struct chunk *c;
+	bool next_chunk_exist;
+
+	/* when error occurs, each thread prints error messages
+	 * immediately with pr_* functions. */
+
+	if (t->cpu > -1) {
+		if (set_thread_affinity(pthread_self(), t->cpu) < 0) {
+			pr_err("set_thread_affinity: %s", priv_get_err());
+			goto err_out;
+		}
+		pr_notice("thread[%d]: pin to cpu core %d", t->id, t->cpu);
+	}
+
+	if (sem_wait(m->sem) < 0) {
+		pr_err("sem_wait: %s", strerrno());
+		goto err_out;
+	}
+
+	if ((next_chunk_exist = pool_iter_has_next_lock(m->chunk_pool))) {
+		if (m->opts->interval > 0)
+			wait_for_interval(m->opts->interval);
+		pr_notice("thread[%d]: connecting to %s", t->id, m->remote);
+		t->sftp = ssh_init_sftp_session(m->remote, m->ssh_opts);
+	}
+
+	if (sem_post(m->sem) < 0) {
+		pr_err("sem_post: %s", strerrno());
+		goto err_out;
+	}
+
+	if (!next_chunk_exist) {
+		pr_notice("thread[%d]: no more connections needed", t->id);
+		goto out;
+	}
+
+	if (!t->sftp) {
+		pr_err("thread[%d]: %s", t->id, priv_get_err());
+		goto err_out;
+	}
+
+	switch (m->direction) {
+	case MSCP_DIRECTION_L2R:
+		src_sftp = NULL;
+		dst_sftp = t->sftp;
+		break;
+	case MSCP_DIRECTION_R2L:
+		src_sftp = t->sftp;
+		dst_sftp = NULL;
+		break;
+	default:
+		assert(false);
+		goto err_out; /* not reached */
+	}
+
+	while (1) {
+		c = pool_iter_next_lock(m->chunk_pool);
+		if (c == NULL) {
+			if (!chunk_pool_is_ready(m)) {
+				/* a new chunk will be added. wait for it. */
+				usleep(100);
+				continue;
+			}
+			break; /* no more chunks */
+		}
+
+		if ((t->ret = copy_chunk(c, src_sftp, dst_sftp, m->opts->nr_ahead,
+					 m->opts->buf_sz, m->opts->preserve_ts, &m->bw,
+					 &t->copied_bytes)) < 0)
+			break;
+	}
+
+	if (t->ret < 0) {
+		pr_err("thread[%d]: copy failed: %s -> %s, 0x%010lx-0x%010lx, %s", t->id,
+		       c->p->path, c->p->dst_path, c->off, c->off + c->len,
+		       priv_get_err());
+	}
+
+	return NULL;
+
+err_out:
+	t->ret = -1;
+	return NULL;
+out:
+	t->ret = 0;
+	return NULL;
+}
+
+/* cleanup-related functions */
+
+void mscp_cleanup(struct mscp *m)
+{
+	if (m->first) {
+		ssh_sftp_close(m->first);
+		m->first = NULL;
+	}
+
+	pool_zeroize(m->src_pool, free);
+	pool_zeroize(m->path_pool, (pool_map_f)free_path);
+	pool_zeroize(m->chunk_pool, free);
+	pool_zeroize(m->thread_pool, free);
+}
+
+void mscp_free(struct mscp *m)
+{
+	pool_destroy(m->src_pool, free);
+	pool_destroy(m->path_pool, (pool_map_f)free_path);
+
+	if (m->remote)
+		free(m->remote);
+	if (m->cores)
+		free(m->cores);
+
+	sem_release(m->sem);
+	free(m);
+}
+
+void mscp_get_stats(struct mscp *m, struct mscp_stats *s)
+{
+	struct mscp_thread *t;
+	unsigned int idx;
+
+	s->total = m->total_bytes;
+	s->done = 0;
+
+	pool_for_each(m->thread_pool, t, idx) {
+		s->done += t->copied_bytes;
+	}
+}

src/openbsd-compat/openbsd-compat.h

@@ -0,0 +1,10 @@
+#ifndef _OPENBSD_COMPAT_H
+#define _OPENBSD_COMPAT_H
+
+#include "config.h"
+
+#ifndef HAVE_STRLCAT
+size_t strlcat(char *dst, const char *src, size_t siz);
+#endif
+
+#endif /* _OPENBSD_COMPAT_H_ */

src/openbsd-compat/strlcat.c

@@ -0,0 +1,62 @@
+/*	$OpenBSD: strlcat.c,v 1.13 2005/08/08 08:05:37 espie Exp $	*/
+
+/*
+ * Copyright (c) 1998 Todd C. Miller <Todd.Miller@courtesan.com>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+/* OPENBSD ORIGINAL: lib/libc/string/strlcat.c */
+
+#include "config.h"
+#ifndef HAVE_STRLCAT
+
+#include <sys/types.h>
+#include <string.h>
+
+/*
+ * Appends src to string dst of size siz (unlike strncat, siz is the
+ * full size of dst, not space left).  At most siz-1 characters
+ * will be copied.  Always NUL terminates (unless siz <= strlen(dst)).
+ * Returns strlen(src) + MIN(siz, strlen(initial dst)).
+ * If retval >= siz, truncation occurred.
+ */
+size_t
+strlcat(char *dst, const char *src, size_t siz)
+{
+	char *d = dst;
+	const char *s = src;
+	size_t n = siz;
+	size_t dlen;
+
+	/* Find the end of dst and adjust bytes left but don't go past end */
+	while (n-- != 0 && *d != '\0')
+		d++;
+	dlen = d - dst;
+	n = siz - dlen;
+
+	if (n == 0)
+		return(dlen + strlen(s));
+	while (*s != '\0') {
+		if (n != 1) {
+			*d++ = *s;
+			n--;
+		}
+		s++;
+	}
+	*d = '\0';
+
+	return(dlen + (s - src));	/* count does not include NUL */
+}
+
+#endif /* !HAVE_STRLCAT */

src/path.c

@@ -0,0 +1,564 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
+#include <string.h>
+#include <unistd.h>
+#include <dirent.h>
+#include <sys/stat.h>
+#include <libgen.h>
+#include <assert.h>
+
+#include <ssh.h>
+#include <minmax.h>
+#include <fileops.h>
+#include <atomic.h>
+#include <path.h>
+#include <strerrno.h>
+#include <print.h>
+
+/* paths of copy source resoltion */
+static char *resolve_dst_path(const char *src_file_path, struct path_resolve_args *a)
+{
+	char copy[PATH_MAX + 1], dst_file_path[PATH_MAX + 1];
+	char *prefix;
+	int offset;
+	int ret;
+
+	strncpy(copy, a->src_path, PATH_MAX);
+	prefix = dirname(copy);
+	if (!prefix) {
+		pr_err("dirname: %s", strerrno());
+		return NULL;
+	}
+
+	offset = strlen(prefix) + 1;
+	if (strlen(prefix) == 1) { /* corner cases */
+		switch (prefix[0]) {
+		case '.':
+			offset = 0;
+			break;
+		case '/':
+			offset = 1;
+			break;
+		}
+	}
+
+	if (!a->src_path_is_dir && !a->dst_path_is_dir) {
+		/* src path is file. dst path is (1) file, or (2) does not exist.
+                 * In the second case, we need to put src under the dst.
+                 */
+		if (a->dst_path_should_dir)
+			ret = snprintf(dst_file_path, PATH_MAX, "%s/%s", a->dst_path,
+				       a->src_path + offset);
+		else
+			ret = snprintf(dst_file_path, PATH_MAX, "%s", a->dst_path);
+	}
+
+	/* src is file, and dst is dir */
+	if (!a->src_path_is_dir && a->dst_path_is_dir)
+		ret = snprintf(dst_file_path, PATH_MAX, "%s/%s", a->dst_path,
+			       a->src_path + offset);
+
+	/* both are directory */
+	if (a->src_path_is_dir && a->dst_path_is_dir)
+		ret = snprintf(dst_file_path, PATH_MAX, "%s/%s", a->dst_path,
+			       src_file_path + offset);
+
+	/* dst path does not exist. change dir name to dst_path */
+	if (a->src_path_is_dir && !a->dst_path_is_dir)
+		ret = snprintf(dst_file_path, PATH_MAX, "%s/%s", a->dst_path,
+			       src_file_path + strlen(a->src_path) + 1);
+
+	if (ret >= PATH_MAX) {
+		pr_warn("Too long path: %s", dst_file_path);
+		return NULL;
+	}
+
+	pr_debug("file: %s -> %s", src_file_path, dst_file_path);
+
+	return strndup(dst_file_path, PATH_MAX);
+}
+
+/* chunk preparation */
+struct chunk *alloc_chunk(struct path *p, size_t off, size_t len)
+{
+	struct chunk *c;
+
+	if (!(c = malloc(sizeof(*c)))) {
+		pr_err("malloc %s", strerrno());
+		return NULL;
+	}
+	memset(c, 0, sizeof(*c));
+
+	c->p = p;
+	c->off = off;
+	c->len = len;
+	c->state = CHUNK_STATE_INIT;
+	refcnt_inc(&p->refcnt);
+	return c;
+}
+
+static int resolve_chunk(struct path *p, size_t size, struct path_resolve_args *a)
+{
+	struct chunk *c;
+	size_t chunk_sz, off, len;
+	size_t remaind;
+
+	if (a->max_chunk_sz)
+		chunk_sz = a->max_chunk_sz;
+	else {
+		chunk_sz = (size / (a->nr_conn * 4)) & a->chunk_align;
+		if (chunk_sz <= a->min_chunk_sz)
+			chunk_sz = a->min_chunk_sz;
+	}
+
+	/* for (size = size; size > 0;) does not create a file (chunk)
+         * when file size is 0. This do {} while (remaind > 0) creates
+         * just open/close a 0-byte file.
+         */
+	remaind = size;
+	do {
+		off = size - remaind;
+		len = remaind < chunk_sz ? remaind : chunk_sz;
+		c = alloc_chunk(p, off, len);
+		if (!c)
+			return -1;
+
+		remaind -= len;
+		if (pool_push_lock(a->chunk_pool, c) < 0) {
+			pr_err("pool_push_lock: %s", strerrno());
+			return -1;
+		}
+	} while (remaind > 0);
+
+	return 0;
+}
+
+void free_path(struct path *p)
+{
+	if (p->path)
+		free(p->path);
+	if (p->dst_path)
+		free(p->dst_path);
+	free(p);
+}
+
+struct path *alloc_path(char *path, char *dst_path)
+{
+	struct path *p;
+
+	if (!(p = malloc(sizeof(*p)))) {
+		pr_err("malloc: %s", strerrno());
+		return NULL;
+	}
+	memset(p, 0, sizeof(*p));
+
+	p->path = path;
+	p->dst_path = dst_path;
+	p->state = FILE_STATE_INIT;
+	lock_init(&p->lock);
+	p->data = 0;
+
+	return p;
+}
+
+static int append_path(sftp_session sftp, const char *path, struct stat st,
+		       struct path_resolve_args *a)
+{
+	struct path *p;
+	char *src, *dst;
+
+	if (!(src = strdup(path))) {
+		pr_err("strdup: %s", strerrno());
+		return -1;
+	}
+
+	if (!(dst = resolve_dst_path(src, a))) {
+		free(src);
+		return -1;
+	}
+
+	if (!(p = alloc_path(src, dst)))
+		return -1;
+
+	if (resolve_chunk(p, st.st_size, a) < 0)
+		return -1; /* XXX: do not free path becuase chunk(s)
+			    * was added to chunk pool already */
+
+	if (pool_push_lock(a->path_pool, p) < 0) {
+		pr_err("pool_push: %s", strerrno());
+		goto free_out;
+	}
+
+	*a->total_bytes += st.st_size;
+
+	return 0;
+
+free_out:
+	free_path(p);
+	return -1;
+}
+
+static bool check_path_should_skip(const char *path)
+{
+	int len = strlen(path);
+	if ((len == 1 && strncmp(path, ".", 1) == 0) ||
+	    (len == 2 && strncmp(path, "..", 2) == 0)) {
+		return true;
+	}
+	return false;
+}
+
+static int walk_path_recursive(sftp_session sftp, const char *path,
+			       struct path_resolve_args *a)
+{
+	char next_path[PATH_MAX + 1];
+	struct dirent *e;
+	struct stat st;
+	MDIR *d;
+	int ret;
+
+	if (mscp_stat(path, &st, sftp) < 0) {
+		pr_err("stat: %s: %s", path, strerrno());
+		return -1;
+	}
+
+	if (S_ISREG(st.st_mode)) {
+		/* this path is regular file. it is to be copied */
+		return append_path(sftp, path, st, a);
+	}
+
+	if (!S_ISDIR(st.st_mode))
+		return 0; /* not a regular file and not a directory, skip it. */
+
+	/* ok, this path is a directory. walk through it. */
+	if (!(d = mscp_opendir(path, sftp))) {
+		pr_err("opendir: %s: %s", path, strerrno());
+		return -1;
+	}
+
+	for (e = mscp_readdir(d); e; e = mscp_readdir(d)) {
+		if (check_path_should_skip(e->d_name))
+			continue;
+
+		ret = snprintf(next_path, PATH_MAX, "%s/%s", path, e->d_name);
+		if (ret >= PATH_MAX) {
+			pr_warn("Too long path: %s/%s", path, e->d_name);
+			continue;
+		}
+
+		walk_path_recursive(sftp, next_path, a);
+		/* do not stop even when walk_path_recursive returns
+		 * -1 due to an unreadable file. go to a next
+		 * file. Thus, do not pass error messages via
+		 * priv_set_err() under walk_path_recursive.  Print
+		 * the error with pr_err immediately.
+		 */
+	}
+
+	mscp_closedir(d);
+
+	return 0;
+}
+
+int walk_src_path(sftp_session src_sftp, const char *src_path,
+		  struct path_resolve_args *a)
+{
+	return walk_path_recursive(src_sftp, src_path, a);
+}
+
+/* based on
+ * https://stackoverflow.com/questions/2336242/recursive-mkdir-system-call-on-unix */
+static int touch_dst_path(struct path *p, sftp_session sftp)
+{
+	/* XXX: should reflect the permission of the original directory? */
+	mode_t mode = S_IRWXU | S_IRWXG | S_IRWXO;
+	struct stat st;
+	char path[PATH_MAX];
+	char *needle;
+	int ret;
+	mf *f;
+
+	strncpy(path, p->dst_path, sizeof(path));
+
+	/* mkdir -p.
+	 * XXX: this may be slow when dst is the remote side. need speed-up. */
+	for (needle = strchr(path + 1, '/'); needle; needle = strchr(needle + 1, '/')) {
+		*needle = '\0';
+
+		if (mscp_stat(path, &st, sftp) == 0) {
+			if (S_ISDIR(st.st_mode))
+				goto next; /* directory exists. go deeper */
+			else {
+				priv_set_errv("mscp_stat %s: not a directory", path);
+				return -1; /* path exists, but not directory. */
+			}
+		}
+
+		if (errno == ENOENT) {
+			/* no file on the path. create directory. */
+			if (mscp_mkdir(path, mode, sftp) < 0) {
+				priv_set_errv("mscp_mkdir %s: %s", path, strerrno());
+				return -1;
+			}
+		}
+next:
+		*needle = '/';
+	}
+
+	/* Do not set O_TRUNC here. Instead, do mscp_setstat() at the
+	 * end. see https://bugzilla.mindrot.org/show_bug.cgi?id=3431 */
+	f = mscp_open(p->dst_path, O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR, sftp);
+	if (!f) {
+		priv_set_errv("mscp_open %s: %s", p->dst_path, strerrno());
+		return -1;
+	}
+
+	mscp_close(f);
+
+	return 0;
+}
+
+static int prepare_dst_path(struct path *p, sftp_session dst_sftp)
+{
+	int ret = 0;
+
+	LOCK_ACQUIRE(&p->lock);
+	if (p->state == FILE_STATE_INIT) {
+		if (touch_dst_path(p, dst_sftp) < 0) {
+			ret = -1;
+			goto out;
+		}
+		p->state = FILE_STATE_OPENED;
+		pr_info("copy start: %s", p->path);
+	}
+
+out:
+	LOCK_RELEASE();
+	return ret;
+}
+
+/* functions for copy */
+
+static ssize_t read_to_buf(void *ptr, size_t len, void *userdata)
+{
+	int fd = *((int *)userdata);
+	return read(fd, ptr, len);
+}
+
+static int copy_chunk_l2r(struct chunk *c, int fd, sftp_file sf, int nr_ahead, int buf_sz,
+			  struct bwlimit *bw, size_t *counter)
+{
+	ssize_t read_bytes, remaind, thrown;
+	int idx, ret;
+	struct {
+		uint32_t id;
+		ssize_t len;
+	} reqs[nr_ahead];
+
+	if (c->len == 0)
+		return 0;
+
+	remaind = thrown = c->len;
+	for (idx = 0; idx < nr_ahead && thrown > 0; idx++) {
+		reqs[idx].len = min(thrown, buf_sz);
+		reqs[idx].len = sftp_async_write(sf, read_to_buf, reqs[idx].len, &fd,
+						 &reqs[idx].id);
+		if (reqs[idx].len < 0) {
+			priv_set_errv("sftp_async_write: %s",
+				      sftp_get_ssh_error(sf->sftp));
+			return -1;
+		}
+		thrown -= reqs[idx].len;
+		bwlimit_wait(bw, reqs[idx].len);
+	}
+
+	for (idx = 0; remaind > 0; idx = (idx + 1) % nr_ahead) {
+		ret = sftp_async_write_end(sf, reqs[idx].id, 1);
+		if (ret != SSH_OK) {
+			priv_set_errv("sftp_async_write_end: %s",
+				      sftp_get_ssh_error(sf->sftp));
+			return -1;
+		}
+
+		*counter += reqs[idx].len;
+		remaind -= reqs[idx].len;
+
+		if (remaind <= 0)
+			break;
+
+		if (thrown <= 0)
+			continue;
+
+		reqs[idx].len = min(thrown, buf_sz);
+		reqs[idx].len = sftp_async_write(sf, read_to_buf, reqs[idx].len, &fd,
+						 &reqs[idx].id);
+		if (reqs[idx].len < 0) {
+			priv_set_errv("sftp_async_write: %s",
+				      sftp_get_ssh_error(sf->sftp));
+			return -1;
+		}
+		thrown -= reqs[idx].len;
+		bwlimit_wait(bw, reqs[idx].len);
+	}
+
+	if (remaind < 0) {
+		priv_set_errv("invalid remaind bytes %ld. "
+			      "last async_write_end bytes %lu.",
+			      remaind, reqs[idx].len);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int copy_chunk_r2l(struct chunk *c, sftp_file sf, int fd,
+			   int nr_ahead, int buf_sz,
+			   struct bwlimit *bw, size_t *counter)
+{
+	ssize_t read_bytes, write_bytes, remain, thrown, len, requested;
+	sftp_aio reqs[nr_ahead];
+	char buf[buf_sz];
+	int i;
+
+	if (c->len == 0)
+		return 0;
+
+	remain = thrown = c->len;
+
+	for (i = 0; i < nr_ahead && thrown > 0; i++) {
+		len = min(thrown, sizeof(buf));
+		requested = sftp_aio_begin_read(sf, len, &reqs[i]);
+		if (requested == SSH_ERROR) {
+			priv_set_errv("sftp_aio_begin_read: %d",
+				      sftp_get_error(sf->sftp));
+			return -1;
+		}
+		thrown -= requested;
+		bwlimit_wait(bw, requested);
+	}
+
+	for (i = 0; remain > 0; i = (i + 1) % nr_ahead) {
+		read_bytes = sftp_aio_wait_read(&reqs[i], buf, sizeof(buf));
+		if (read_bytes == SSH_ERROR) {
+			priv_set_errv("sftp_aio_wait_read: %d",
+				      sftp_get_error(sf->sftp));
+			return -1;
+		}
+
+		if (thrown > 0) {
+			len = min(thrown, sizeof(buf));
+			requested = sftp_aio_begin_read(sf, len, &reqs[i]);
+			thrown -= requested;
+			bwlimit_wait(bw, requested);
+		}
+
+		write_bytes = write(fd, buf, read_bytes);
+		if (write_bytes < 0) {
+			priv_set_errv("write: %s", strerrno());
+			return -1;
+		}
+
+		if (write_bytes < read_bytes) {
+			priv_set_errv("failed to write full bytes");
+			return -1;
+		}
+
+		*counter += write_bytes;
+		remain -= write_bytes;
+	}
+
+	if (remain < 0) {
+		priv_set_errv("invalid remain bytes %ld. last async_read bytes %ld. "
+			      "last write bytes %ld",
+			      remain, read_bytes, write_bytes);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int _copy_chunk(struct chunk *c, mf *s, mf *d, int nr_ahead, int buf_sz,
+		       struct bwlimit *bw, size_t *counter)
+{
+	if (s->local && d->remote) /* local to remote copy */
+		return copy_chunk_l2r(c, s->local, d->remote, nr_ahead, buf_sz, bw,
+				      counter);
+	else if (s->remote && d->local) /* remote to local copy */
+		return copy_chunk_r2l(c, s->remote, d->local, nr_ahead, buf_sz, bw,
+				      counter);
+
+	assert(false);
+	return -1; /* not reached */
+}
+
+int copy_chunk(struct chunk *c, sftp_session src_sftp, sftp_session dst_sftp,
+	       int nr_ahead, int buf_sz, bool preserve_ts, struct bwlimit *bw,
+	       size_t *counter)
+{
+	mode_t mode;
+	int flags;
+	mf *s, *d;
+	int ret;
+
+	assert((src_sftp && !dst_sftp) || (!src_sftp && dst_sftp));
+
+	if (prepare_dst_path(c->p, dst_sftp) < 0)
+		return -1;
+
+	/* open src */
+	flags = O_RDONLY;
+	mode = S_IRUSR;
+	if (!(s = mscp_open(c->p->path, flags, mode, src_sftp))) {
+		priv_set_errv("mscp_open: %s: %s", c->p->path, strerrno());
+		return -1;
+	}
+	if (mscp_lseek(s, c->off) < 0) {
+		priv_set_errv("mscp_lseek: %s: %s", c->p->path, strerrno());
+		return -1;
+	}
+
+	/* open dst */
+	flags = O_WRONLY;
+	mode = S_IRUSR | S_IWUSR;
+	if (!(d = mscp_open(c->p->dst_path, flags, mode, dst_sftp))) {
+		mscp_close(s);
+		priv_set_errv("mscp_open: %s: %s", c->p->dst_path, strerrno());
+		return -1;
+	}
+	if (mscp_lseek(d, c->off) < 0) {
+		priv_set_errv("mscp_lseek: %s: %s", c->p->dst_path, strerrno());
+		return -1;
+	}
+
+	c->state = CHUNK_STATE_COPING;
+	pr_debug("copy chunk start: %s 0x%lx-0x%lx", c->p->path, c->off, c->off + c->len);
+
+	ret = _copy_chunk(c, s, d, nr_ahead, buf_sz, bw, counter);
+
+	pr_debug("copy chunk done: %s 0x%lx-0x%lx", c->p->path, c->off, c->off + c->len);
+
+	mscp_close(d);
+	mscp_close(s);
+	if (ret < 0)
+		return ret;
+
+	if (refcnt_dec(&c->p->refcnt) == 0) {
+		struct stat st;
+		c->p->state = FILE_STATE_DONE;
+
+		/* sync stat */
+		if (mscp_stat(c->p->path, &st, src_sftp) < 0) {
+			priv_set_errv("mscp_stat: %s: %s", c->p->path, strerrno());
+			return -1;
+		}
+		if (mscp_setstat(c->p->dst_path, &st, preserve_ts, dst_sftp) < 0) {
+			priv_set_errv("mscp_setstat: %s: %s", c->p->path, strerrno());
+			return -1;
+		}
+		pr_info("copy done: %s", c->p->path);
+	}
+
+	if (ret == 0)
+		c->state = CHUNK_STATE_DONE;
+
+	return ret;
+}

src/path.h

@@ -0,0 +1,73 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
+#ifndef _PATH_H_
+#define _PATH_H_
+
+#include <limits.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <sys/stat.h>
+#include <pool.h>
+#include <atomic.h>
+#include <ssh.h>
+#include <bwlimit.h>
+
+struct path {
+	char *path; /* file path */
+	char *dst_path; /* copy dst path */
+
+	refcnt refcnt; /* number of associated chunks */
+	lock lock;
+	int state;
+#define FILE_STATE_INIT 0
+#define FILE_STATE_OPENED 1
+#define FILE_STATE_DONE 2
+
+	uint64_t data; /* used by other components, i.e., checkpoint */
+};
+
+struct path *alloc_path(char *path, char *dst_path);
+
+struct chunk {
+	struct path *p;
+	size_t off; /* offset of this chunk on the file on path p */
+	size_t len; /* length of this chunk */
+	int state;
+#define CHUNK_STATE_INIT 0
+#define CHUNK_STATE_COPING 1
+#define CHUNK_STATE_DONE 2
+};
+
+struct chunk *alloc_chunk(struct path *p, size_t off, size_t len);
+
+struct path_resolve_args {
+	size_t *total_bytes;
+
+	/* args to resolve src path to dst path */
+	const char *src_path;
+	const char *dst_path;
+	bool src_path_is_dir;
+	bool dst_path_is_dir;
+	bool dst_path_should_dir;
+
+	/* args to resolve chunks for a path */
+	pool *path_pool;
+	pool *chunk_pool;
+	int nr_conn;
+	size_t min_chunk_sz;
+	size_t max_chunk_sz;
+	size_t chunk_align;
+};
+
+/* walk src_path recursivly and fill a->path_pool with found files */
+int walk_src_path(sftp_session src_sftp, const char *src_path,
+		  struct path_resolve_args *a);
+
+/* free struct path */
+void free_path(struct path *p);
+
+/* copy a chunk. either src_sftp or dst_sftp is not null, and another is null */
+int copy_chunk(struct chunk *c, sftp_session src_sftp, sftp_session dst_sftp,
+	       int nr_ahead, int buf_sz, bool preserve_ts, struct bwlimit *bw,
+	       size_t *counter);
+
+#endif /* _PATH_H_ */

src/platform.c

@@ -1,15 +1,31 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
 #ifdef __APPLE__
+#include <stdlib.h>
 #include <sys/types.h>
+#include <sys/time.h>
 #include <sys/sysctl.h>
 #elif linux
 #define _GNU_SOURCE
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
 #include <sched.h>
+#elif __FreeBSD__
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <pthread_np.h>
 #else
 #error unsupported platform
 #endif
 
-#include <util.h>
+#include <config.h>
 #include <platform.h>
+#include <strerrno.h>
+#include <print.h>
 
 #ifdef __APPLE__
 int nr_cpus()
@@ -18,7 +34,7 @@ int nr_cpus()
 	size_t size = sizeof(n);
 
 	if (sysctlbyname("machdep.cpu.core_count", &n, &size, NULL, 0) != 0) {
-		pr_err("failed to get number of cpu cores: %s\n", strerrno());
+		priv_set_errv("failed to get number of cpu cores: %s", strerrno());
 		return -1;
 	}
 
@@ -27,10 +43,56 @@ int nr_cpus()
 
 int set_thread_affinity(pthread_t tid, int core)
 {
-	pr_warn("setting thread afinity is not implemented on apple\n");
+	pr_warn("setting thread afinity is not implemented on apple");
 	return 0;
 }
 
+int setutimes(const char *path, struct timespec atime, struct timespec mtime)
+{
+	struct timeval tv[2] = {
+		{
+			.tv_sec = atime.tv_sec,
+			.tv_usec = atime.tv_nsec * 1000,
+		},
+		{
+			.tv_sec = mtime.tv_sec,
+			.tv_usec = mtime.tv_nsec * 1000,
+		},
+	};
+	return utimes(path, tv);
+}
+
+static void random_string(char *buf, size_t size)
+{
+	char chars[] = "abcdefhijklmnopqrstuvwxyz1234567890";
+	int n, x;
+
+	for (n = 0; n < size - 1; n++) {
+		x = arc4random() % (sizeof(chars) - 1);
+		buf[n] = chars[x];
+	}
+	buf[size - 1] = '\0';
+}
+
+sem_t *sem_create(int value)
+{
+	char sem_name[30] = "mscp-";
+	sem_t *sem;
+	int n;
+
+	n = strlen(sem_name);
+	random_string(sem_name + n, sizeof(sem_name) - n - 1);
+	if ((sem = sem_open(sem_name, O_CREAT, 600, value)) == SEM_FAILED)
+		return NULL;
+
+	return sem;
+}
+
+int sem_release(sem_t *sem)
+{
+	return sem_close(sem);
+}
+
 #endif
 
 #ifdef linux
@@ -41,6 +103,17 @@ int nr_cpus()
 		return CPU_COUNT(&cpu_set);
 	return -1;
 }
+#endif
+
+#ifdef __FreeBSD__
+int nr_cpus()
+{
+	long nr_cpus = sysconf(_SC_NPROCESSORS_ONLN);
+	return nr_cpus;
+}
+#endif
+
+#if defined(linux) || defined(__FreeBSD__)
 
 int set_thread_affinity(pthread_t tid, int core)
 {
@@ -51,9 +124,43 @@ int set_thread_affinity(pthread_t tid, int core)
 	CPU_SET(core, &target_cpu_set);
 	ret = pthread_setaffinity_np(tid, sizeof(target_cpu_set), &target_cpu_set);
 	if (ret < 0)
-		pr_err("failed to set thread/cpu affinity for core %d: %s",
-		       core, strerrno());
+		priv_set_errv("failed to set thread/cpu affinity for core %d: %s", core,
+			      strerrno());
 	return ret;
 }
-#endif
 
+int setutimes(const char *path, struct timespec atime, struct timespec mtime)
+{
+	struct timespec ts[2] = { atime, mtime };
+	int fd = open(path, O_WRONLY);
+	int ret;
+
+	if (fd < 0)
+		return -1;
+	ret = futimens(fd, ts);
+	close(fd);
+	return ret;
+}
+
+sem_t *sem_create(int value)
+{
+	sem_t *sem;
+
+	if ((sem = malloc(sizeof(*sem))) == NULL)
+		return NULL;
+
+	if (sem_init(sem, 0, value) < 0) {
+		free(sem);
+		return NULL;
+	}
+
+	return sem;
+}
+
+int sem_release(sem_t *sem)
+{
+	free(sem);
+	return 0;
+}
+
+#endif

src/platform.h

@@ -1,9 +1,47 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
 #ifndef _PLATFORM_H_
 #define _PLATFORM_H_
 
-#include <pthread.h>
+#include <config.h>
 
-int nr_cpus();
+#include <pthread.h>
+#include <semaphore.h>
+#include <stdint.h>
+
+int nr_cpus(void);
 int set_thread_affinity(pthread_t tid, int core);
+int setutimes(const char *path, struct timespec atime, struct timespec mtime);
+
+/*
+ * macOS does not support sem_init(). macOS (seems to) releases the
+ * named semaphore when associated mscp process finished. In linux,
+ * program (seems to) need to release named semaphore in /dev/shm by
+ * sem_unlink() explicitly. So, using sem_init() (unnamed semaphore)
+ * in linux and using sem_open() (named semaphore) in macOS without
+ * sem_unlink() are reasonable (?).
+ */
+sem_t *sem_create(int value);
+int sem_release(sem_t *sem);
+
+#ifdef HAVE_HTONLL
+#include <arpa/inet.h> /* Apple has htonll and ntohll in arpa/inet.h */
+#endif
+
+/* copied from libssh: libssh/include/libssh/priv.h */
+#ifndef HAVE_HTONLL
+#ifdef WORDS_BIGENDIAN
+#define htonll(x) (x)
+#else
+#define htonll(x) (((uint64_t)htonl((x)&0xFFFFFFFF) << 32) | htonl((x) >> 32))
+#endif
+#endif
+
+#ifndef HAVE_NTOHLL
+#ifdef WORDS_BIGENDIAN
+#define ntohll(x) (x)
+#else
+#define ntohll(x) (((uint64_t)ntohl((x)&0xFFFFFFFF) << 32) | ntohl((x) >> 32))
+#endif
+#endif
 
 #endif /* _PLATFORM_H_ */

src/pool.c

@@ -0,0 +1,123 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
+#include <string.h>
+#include <stdlib.h>
+#include <pool.h>
+
+#define DEFAULT_START_SIZE 16
+
+pool *pool_new(void)
+{
+	pool *p;
+	p = malloc(sizeof(*p));
+	if (!p)
+		return NULL;
+	memset(p, 0, sizeof(*p));
+
+	p->array = calloc(DEFAULT_START_SIZE, sizeof(void *));
+	if (!p->array) {
+		free(p);
+		return NULL;
+	}
+
+	p->len = DEFAULT_START_SIZE;
+	p->num = 0;
+	lock_init(&p->lock);
+	return p;
+}
+
+void pool_free(pool *p)
+{
+	if (p->array) {
+		free(p->array);
+		p->array = NULL;
+	}
+	free(p);
+}
+
+void pool_zeroize(pool *p, pool_map_f f)
+{
+	void *v;
+	pool_iter_for_each(p, v) {
+		f(v);
+	}
+	p->num = 0;
+}
+
+void pool_destroy(pool *p, pool_map_f f)
+{
+	pool_zeroize(p, f);
+	pool_free(p);
+}
+
+int pool_push(pool *p, void *v)
+{
+	if (p->num == p->len) {
+		/* expand array */
+		size_t newlen = p->len * 2;
+		void *new = realloc(p->array, newlen * sizeof(void *));
+		if (new == NULL)
+			return -1;
+		p->len = newlen;
+		p->array = new;
+	}
+	p->array[p->num] = v;
+	__sync_synchronize();
+	p->num++;
+	return 0;
+}
+
+int pool_push_lock(pool *p, void *v)
+{
+	int ret = -1;
+	pool_lock(p);
+	ret = pool_push(p, v);
+	pool_unlock(p);
+	return ret;
+}
+
+void *pool_pop(pool *p)
+{
+	return p->num == 0 ? NULL : p->array[--p->num];
+}
+
+void *pool_pop_lock(pool *p)
+{
+	void *v;
+	pool_lock(p);
+	v = pool_pop(p);
+	pool_unlock(p);
+	return v;
+}
+
+void *pool_get(pool *p, unsigned int idx)
+{
+	return p->num <= idx ? NULL : p->array[idx];
+}
+
+void *pool_iter_next(pool *p)
+{
+	if (p->num <= p->idx)
+		return NULL;
+
+	void *v = p->array[p->idx];
+	p->idx++;
+	return v;
+}
+
+void *pool_iter_next_lock(pool *p)
+{
+	void *v = NULL;
+	pool_lock(p);
+	v = pool_iter_next(p);
+	pool_unlock(p);
+	return v;
+}
+
+bool pool_iter_has_next_lock(pool *p)
+{
+	bool next_exist;
+	pool_lock(p);
+	next_exist = (p->idx < p->num);
+	pool_unlock(p);
+	return next_exist;
+}

src/pool.h

@@ -0,0 +1,94 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
+#ifndef _POOL_H_
+#define _POOL_H_
+
+#include <stdbool.h>
+#include <stddef.h>
+
+#include <atomic.h>
+
+/* A pool like a stack with an iterator walking from the bottom to the
+ * top. The memory foot print for a pool never shrinks. Thus this is
+ * not suitable for long-term uses. */
+
+struct pool_struct {
+	void **array;
+	size_t len; /* length of array */
+	size_t num; /* number of items in the array */
+	size_t idx; /* index used dy iter */
+	lock lock;
+};
+
+typedef struct pool_struct pool;
+
+/* allocate a new pool */
+pool *pool_new(void);
+
+/* func type applied to each item in a pool */
+typedef void (*pool_map_f)(void *v);
+
+/* apply f, which free an item, to all items and set num to 0 */
+void pool_zeroize(pool *p, pool_map_f f);
+
+/* free pool->array and pool */
+void pool_free(pool *p);
+
+/* free pool->array and pool after applying f to all items in p->array */
+void pool_destroy(pool *p, pool_map_f f);
+
+#define pool_lock(p) LOCK_ACQUIRE(&(p->lock))
+#define pool_unlock(p) LOCK_RELEASE()
+
+/*
+ * pool_push() pushes *v to pool *p. pool_push_lock() does this while
+ * locking *p.
+ */
+int pool_push(pool *p, void *v);
+int pool_push_lock(pool *p, void *v);
+
+/*
+ * pool_pop() pops the last *v pushed to *p. pool_pop_lock() does this
+ * while locking *p.
+ */
+void *pool_pop(pool *p);
+void *pool_pop_lock(pool *p);
+
+/* pool_get() returns value indexed by idx */
+void *pool_get(pool *p, unsigned int idx);
+
+#define pool_size(p) ((p)->num)
+#define pool_is_empty(p) (pool_size(p) == 0)
+
+/*
+ * pool->idx indicates next *v in an iteration. This has two
+ * use-cases.
+ *
+ * (1) A simple list: just a single thread has a pool, and the thread
+ * can call pool_iter_for_each() for the pool (not thread safe).
+ *
+ * (2) A thread-safe queue: one thread initializes the iterator for a
+ * pool by pool_iter_init(). Then, multiple threads get a next *v
+ * concurrently by pool_iter_next_lock(), which means dequeuing. At
+ * this time, other thread can add new *v by pool_push_lock(), which
+ * means enqueuing. During this, other threads must not intercept the
+ * pool by pool_iter_* functions.
+ */
+
+#define pool_iter_init(p) (p->idx = 0)
+void *pool_iter_next(pool *p);
+void *pool_iter_next_lock(pool *p);
+
+/* pool_iter_has_next_lock() returns true if pool_iter_next(_lock)
+ * function will retrun a next value, otherwise false, which means
+ * there is no more values in this iteration. */
+bool pool_iter_has_next_lock(pool *p);
+
+#define pool_iter_for_each(p, v) \
+	pool_iter_init(p);       \
+	for (v = pool_iter_next(p); v != NULL; v = pool_iter_next(p))
+
+#define pool_for_each(p, v, idx) \
+	idx = 0;                 \
+	for (v = pool_get(p, idx); v != NULL; v = pool_get(p, ++idx))
+
+#endif /* _POOL_H_ */

src/pprint.c

@@ -1,27 +0,0 @@
-#include <stdio.h>
-#include <stdarg.h>
-#include <pthread.h>
-
-static int pprint_level = 1;
-
-static pthread_mutex_t pprint_lock = PTHREAD_MUTEX_INITIALIZER;
-
-void pprint_set_level(int level)
-{
-	pprint_level = level;
-}
-
-void pprint(int level, const char *fmt, ...)
-{
-	va_list va;
-
-	if (level <= pprint_level) {
-		pthread_mutex_lock(&pprint_lock);
-		va_start(va, fmt);
-		vfprintf(stdout, fmt, va);
-		fflush(stdout);
-		va_end(va);
-		pthread_mutex_unlock(&pprint_lock);
-	}
-}
-

src/pprint.h

@@ -1,20 +0,0 @@
-#ifndef _PPRINT_H_
-#define _PPRINT_H_
-
-/* progress print functions */
-
-/* level 1: print progress bar only.
- * level 2: print copy start/done messages.
- * level 3: print ssh connection establishment/disconnection.
- * level 4: print chunk information.
- */
-void pprint_set_level(int level);
-void pprint(int level, const char *fmt, ...);
-
-#define pprint1(fmt, ...) pprint(1, "\r\033[K" fmt, ##__VA_ARGS__)
-#define pprint2(fmt, ...) pprint(2, "\r\033[K" fmt, ##__VA_ARGS__)
-#define pprint3(fmt, ...) pprint(3, "\r\033[K" fmt, ##__VA_ARGS__)
-#define pprint4(fmt, ...) pprint(4, "\r\033[K" fmt, ##__VA_ARGS__)
-
-
-#endif /* _PPRRINT_H_ */

src/print.c

@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
+
+#include <print.h>
+
+/* message print functions */
+static int __print_severity = MSCP_SEVERITY_WARN;
+
+void set_print_severity(int serverity)
+{
+	if (serverity < 0)
+		__print_severity = -1; /* no print */
+        else
+                __print_severity = serverity;
+}
+
+int get_print_severity()
+{
+	return __print_severity;
+}

src/print.h

@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
+#ifndef _PRINT_H_
+#define _PRINT_H_
+
+#include <libgen.h>
+#include <stdio.h>
+#include <errno.h>
+#include <string.h>
+
+#include <mscp.h>
+
+/* message print. printed messages are passed to application via msg_fd */
+void set_print_severity(int severity);
+int get_print_severity();
+
+#define __print(fp, severity, fmt, ...)                                  \
+	do {                                                             \
+		if (severity <= get_print_severity()) {                  \
+			fprintf(fp, "\r\033[K" fmt "\n", ##__VA_ARGS__); \
+			fflush(fp);                                      \
+		}                                                        \
+	} while (0)
+
+#define pr_err(fmt, ...) __print(stderr, MSCP_SEVERITY_ERR, fmt, ##__VA_ARGS__)
+#define pr_warn(fmt, ...) __print(stderr, MSCP_SEVERITY_WARN, fmt, ##__VA_ARGS__)
+#define pr_notice(fmt, ...) __print(stdout, MSCP_SEVERITY_NOTICE, fmt, ##__VA_ARGS__)
+#define pr_info(fmt, ...) __print(stdout, MSCP_SEVERITY_INFO, fmt, ##__VA_ARGS__)
+#define pr_debug(fmt, ...) __print(stdout, MSCP_SEVERITY_DEBUG, fmt, ##__VA_ARGS__)
+
+#endif /* _PRINT_H_ */

src/rename-logic.py

@@ -1,48 +0,0 @@
-#!/usr/bin/env python3
-
-from os.path import dirname, basename, isfile, isdir, exists
-from os import listdir
-import sys
-
-"""
-
-This file simply implements the src_path to dst_path conversion logic
-just for test. file_fill() and file_fill_recursive() in file.c
-implements this logic.
-
-"""
-
-
-def recursive(src, rel_path, dst, dst_should_dir, replace_dir_name):
-    
-    if isfile(src):
-        if dst_should_dir:
-            print("{} => {}/{}{}".format(src, dst, rel_path, basename(src)))
-        else:
-            print("{} => {}{}".format(src, rel_path, dst))
-        return
-        
-    # src is directory
-    for f in listdir(src):
-        next_src = "{}/{}".format(src, f)
-        if replace_dir_name and dst_should_dir:
-            next_rel_path = ""
-        else:
-            next_rel_path = "{}{}/".format(rel_path, basename(src))
-        recursive(next_src, next_rel_path, dst, dst_should_dir, False)
-
-
-def fill_dst(srclist, dst):
-    dst_must_dir = len(srclist) > 1
-    for src in srclist:
-        dst_should_dir = isdir(src) | isdir(dst)
-        replace_dir_name = not isdir(dst)
-        recursive(src, "", dst, dst_should_dir | dst_must_dir, replace_dir_name)
-
-
-def main():
-    if (len(sys.argv) < 2):
-        print("usage: {} source ... target".format(sys.argv[0]))
-    fill_dst(sys.argv[1:len(sys.argv) - 1], sys.argv[len(sys.argv) - 1])
-
-main()

src/ssh.c

@@ -1,112 +1,226 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
 #include <stdio.h>
 #include <string.h>
 #include <unistd.h>
+#include <stdlib.h>
 
 #include <ssh.h>
-#include <util.h>
+#include <mscp.h>
+#include <strerrno.h>
+
+#include "libssh/callbacks.h"
+#include "libssh/options.h"
 
 static int ssh_verify_known_hosts(ssh_session session);
+static int ssh_authenticate_kbdint(ssh_session session);
 
-
-static int ssh_set_opts(ssh_session ssh, struct ssh_opts *opts)
+static int ssh_set_opts(ssh_session ssh, struct mscp_ssh_opts *opts)
 {
-	ssh_set_log_level(opts->debuglevel);
+	ssh_set_log_level(opts->debug_level);
 
 	if (opts->login_name &&
 	    ssh_options_set(ssh, SSH_OPTIONS_USER, opts->login_name) < 0) {
-		pr_err("failed to set login name\n");
+		priv_set_errv("failed to set login name");
 		return -1;
 	}
 
-	if (opts->port &&
-	    ssh_options_set(ssh, SSH_OPTIONS_PORT_STR, opts->port) < 0) {
-		pr_err("failed to set port number\n");
+	if (opts->port && ssh_options_set(ssh, SSH_OPTIONS_PORT_STR, opts->port) < 0) {
+		priv_set_errv("failed to set port number");
+		return -1;
+	}
+
+	if (opts->ai_family &&
+	    ssh_options_set(ssh, SSH_OPTIONS_AI_FAMILY, &opts->ai_family) < 0) {
+		priv_set_errv("failed to set address family");
 		return -1;
 	}
 
 	if (opts->identity &&
 	    ssh_options_set(ssh, SSH_OPTIONS_IDENTITY, opts->identity) < 0) {
-		pr_err("failed to set identity\n");
+		priv_set_errv("failed to set identity");
 		return -1;
 	}
 
 	if (opts->cipher) {
 		if (ssh_options_set(ssh, SSH_OPTIONS_CIPHERS_C_S, opts->cipher) < 0) {
-			pr_err("failed to set cipher client to server\n");
+			priv_set_errv("failed to set cipher for client to server");
 			return -1;
 		}
 		if (ssh_options_set(ssh, SSH_OPTIONS_CIPHERS_S_C, opts->cipher) < 0) {
-			pr_err("failed to set cipher client to server\n");
+			priv_set_errv("failed to set cipher for server to client");
+			return -1;
+		}
+	}
+
+	if (opts->hmac) {
+		if (ssh_options_set(ssh, SSH_OPTIONS_HMAC_C_S, opts->hmac) < 0) {
+			priv_set_errv("failed to set hmac for client to server");
+			return -1;
+		}
+		if (ssh_options_set(ssh, SSH_OPTIONS_HMAC_S_C, opts->hmac) < 0) {
+			priv_set_errv("failed to set hmac for server to client");
 			return -1;
 		}
 	}
 
 	if (opts->compress &&
-	    ssh_options_set(ssh, SSH_OPTIONS_COMPRESSION, "yes") < 0) {
-		pr_err("failed to enable ssh compression\n");
+	    ssh_options_set(ssh, SSH_OPTIONS_COMPRESSION, opts->compress) < 0) {
+		priv_set_errv("failed to enable ssh compression");
+		return -1;
+	}
+
+	if (opts->ccalgo && ssh_options_set(ssh, SSH_OPTIONS_CCALGO, opts->ccalgo) < 0) {
+		priv_set_errv("failed to set cclago");
+		return -1;
+	}
+
+	/* if NOT specified to enable Nagle's algorithm, disable it (set TCP_NODELAY) */
+	if (!opts->enable_nagle) {
+		int v = 1;
+		if (ssh_options_set(ssh, SSH_OPTIONS_NODELAY, &v) < 0) {
+			priv_set_errv("failed to set TCP_NODELAY");
+			return -1;
+		}
+	}
+
+	if (opts->config && ssh_options_parse_config(ssh, opts->config) < 0) {
+		priv_set_errv("failed to parse ssh_config: %s", opts->config);
+		return -1;
+	}
+
+	if (opts->proxyjump) {
+		char buf[256];
+		memset(buf, 0, sizeof(buf));
+		snprintf(buf, sizeof(buf), "proxyjump=%s", opts->proxyjump);
+		if (ssh_config_parse_string(ssh, buf) != SSH_OK) {
+			priv_set_errv("failed to set ssh option: %s", buf);
+			return -1;
+		}
+	}
+
+	if (opts->options) {
+		int n;
+		for (n = 0; opts->options[n]; n++) {
+			if (ssh_config_parse_string(ssh, opts->options[n]) != SSH_OK) {
+				priv_set_errv("failed to set ssh option: %s",
+					      opts->options[n]);
+				return -1;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static int ssh_authenticate(ssh_session ssh, struct mscp_ssh_opts *opts)
+{
+	static int auth_bit_mask;
+	int ret;
+
+        if (auth_bit_mask == 0) {
+                /* the first authentication attempt. try none auth to
+                 * get available auth methods. */
+                if (ssh_userauth_none(ssh, NULL) == SSH_AUTH_SUCCESS)
+                        return 0;
+
+                /* save auth_bit_mask for further authentications */
+                auth_bit_mask = ssh_userauth_list(ssh, NULL);
+        }
+
+	if (auth_bit_mask & SSH_AUTH_METHOD_PUBLICKEY) {
+                char *p = opts->passphrase ? opts->passphrase : NULL;
+                if (ssh_userauth_publickey_auto(ssh, NULL, p) == SSH_AUTH_SUCCESS)
+                        return 0;
+        }
+
+	if (auth_bit_mask & SSH_AUTH_METHOD_PASSWORD) {
+		if (!opts->password) {
+			char buf[128] = {};
+			if (ssh_getpass("Password: ", buf, sizeof(buf), 0, 0) < 0) {
+				priv_set_errv("ssh_getpass failed");
+				return -1;
+			}
+			if (!(opts->password = strndup(buf, sizeof(buf)))) {
+				priv_set_errv("strndup: %s", strerrno());
+				return -1;
+			}
+		}
+
+		if (ssh_userauth_password(ssh, NULL, opts->password) == SSH_AUTH_SUCCESS)
+			return 0;
+	}
+
+	auth_bit_mask = ssh_userauth_list(ssh, NULL);
+	if (auth_bit_mask & SSH_AUTH_METHOD_INTERACTIVE) {
+		if (ssh_authenticate_kbdint(ssh) == SSH_AUTH_SUCCESS)
+			return 0;
+	}
+
+	return -1;
+}
+
+static int ssh_cache_passphrase(const char *prompt, char *buf, size_t len, int echo,
+				int verify, void *userdata)
+{
+	struct mscp_ssh_opts *opts = userdata;
+
+	/* This function is called on the first time for importing
+	 * priv key file with passphrase. It is not called on the
+	 * second time or after because cached passphrase is passed
+	 * to ssh_userauth_publickey_auto(). */
+
+	/* ToDo: use
+	 * ssh_userauth_publickey_auto_get_current_identity() to print
+	 * id for which we ask passphrase */
+
+	if (ssh_getpass("Passphrase: ", buf, len, echo, verify) < 0)
+		return -1;
+
+	/* cache the passphrase */
+	if (opts->passphrase)
+		free(opts->passphrase);
+
+	if (!(opts->passphrase = strndup(buf, len))) {
+		priv_set_errv("strndup: %s", strerrno());
 		return -1;
 	}
 
 	return 0;
 }
 
-static int ssh_authenticate(ssh_session ssh, struct ssh_opts *opts)
-{
-	int auth_bit_mask;
-	int ret;
+static struct ssh_callbacks_struct cb = {
+	.auth_function = ssh_cache_passphrase,
+	.userdata = NULL,
+};
 
-	/* none method */
-	ret = ssh_userauth_none(ssh, NULL);
-	if (ret == SSH_AUTH_SUCCESS)
-		return 0;
-
-	auth_bit_mask = ssh_userauth_list(ssh, NULL);
-
-	if (auth_bit_mask & SSH_AUTH_METHOD_NONE &&
-	    ssh_userauth_none(ssh, NULL) == SSH_AUTH_SUCCESS) {
-		return 0;
-	}
-
-	if (auth_bit_mask & SSH_AUTH_METHOD_PUBLICKEY &&
-	    ssh_userauth_publickey_auto(ssh, NULL, NULL) == SSH_AUTH_SUCCESS) {
-		return 0;
-	}
-
-	if (auth_bit_mask & SSH_AUTH_METHOD_PASSWORD) {
-		if (!opts->password) {
-			opts->password = getpass("Password: ");
-		}
-		if (ssh_userauth_password(ssh, NULL, opts->password) == SSH_AUTH_SUCCESS)
-			return 0;
-	}
-
-	pr_err("authentication failure: %s\n", ssh_get_error(ssh));
-	return -1;
-}
-
-static ssh_session ssh_make_ssh_session(char *sshdst, struct ssh_opts *opts)
+static ssh_session ssh_init_session(const char *sshdst, struct mscp_ssh_opts *opts)
 {
 	ssh_session ssh = ssh_new();
 
+	ssh_callbacks_init(&cb);
+	cb.userdata = opts;
+	ssh_set_callbacks(ssh, &cb);
+
+	if (ssh_options_set(ssh, SSH_OPTIONS_HOST, sshdst) != SSH_OK) {
+		priv_set_errv("failed to set destination host");
+		goto free_out;
+	}
+
 	if (ssh_set_opts(ssh, opts) != 0)
 		goto free_out;
 
-	if (ssh_options_set(ssh, SSH_OPTIONS_HOST, sshdst) != SSH_OK) {
-		pr_err("failed to set destination host\n");
-		goto free_out;
-	}
-
 	if (ssh_connect(ssh) != SSH_OK) {
-		pr_err("failed to connect ssh server: %s\n", ssh_get_error(ssh));
+		priv_set_errv("failed to connect ssh server: %s", ssh_get_error(ssh));
 		goto free_out;
 	}
 
 	if (ssh_authenticate(ssh, opts) != 0) {
-		pr_err("authentication failed: %s\n", ssh_get_error(ssh));
+		priv_set_errv("authentication failed: %s", ssh_get_error(ssh));
 		goto disconnect_out;
 	}
 
-	if (!opts->no_hostkey_check && ssh_verify_known_hosts(ssh) != 0) {
+	if (ssh_verify_known_hosts(ssh) != 0) {
+		priv_set_errv("ssh_veriy_known_hosts failed");
 		goto disconnect_out;
 	}
 
@@ -119,24 +233,23 @@ free_out:
 	return NULL;
 }
 
-sftp_session ssh_make_sftp_session(char *sshdst, struct ssh_opts *opts)
+sftp_session ssh_init_sftp_session(const char *sshdst, struct mscp_ssh_opts *opts)
 {
 	sftp_session sftp;
-	ssh_session ssh = ssh_make_ssh_session(sshdst, opts);
+	ssh_session ssh = ssh_init_session(sshdst, opts);
 
-	if (!ssh) {
+	if (!ssh)
 		return NULL;
-	}
 
 	sftp = sftp_new(ssh);
 	if (!sftp) {
-		pr_err("failed to allocate sftp session: %s\n", ssh_get_error(ssh));
+		priv_set_errv("failed to allocate sftp session: %s", ssh_get_error(ssh));
 		goto err_out;
 	}
 
 	if (sftp_init(sftp) != SSH_OK) {
-		pr_err("failed to initialize sftp session: err code %d\n",
-		       sftp_get_error(sftp));
+		priv_set_errv("failed to initialize sftp session: err code %d",
+			      sftp_get_error(sftp));
 		goto err_out;
 	}
 
@@ -147,7 +260,6 @@ err_out:
 	return NULL;
 }
 
-
 /* copied from https://api.libssh.org/stable/libssh_tutor_guided_tour.html*/
 static int ssh_verify_known_hosts(ssh_session session)
 {
@@ -166,10 +278,7 @@ static int ssh_verify_known_hosts(ssh_session session)
 		return -1;
 	}
 
-	rc = ssh_get_publickey_hash(srv_pubkey,
-				    SSH_PUBLICKEY_HASH_SHA1,
-				    &hash,
-				    &hlen);
+	rc = ssh_get_publickey_hash(srv_pubkey, SSH_PUBLICKEY_HASH_SHA1, &hash, &hlen);
 	ssh_key_free(srv_pubkey);
 	if (rc < 0) {
 		return -1;
@@ -190,8 +299,9 @@ static int ssh_verify_known_hosts(ssh_session session)
 		return -1;
 	case SSH_KNOWN_HOSTS_OTHER:
 		fprintf(stderr, "The host key for this server was not found but an other"
-			"type of key exists.\n");
-		fprintf(stderr, "An attacker might change the default server key to"
+				"type of key exists.\n");
+		fprintf(stderr,
+			"An attacker might change the default server key to"
 			"confuse your client into thinking the key does not exist\n");
 		ssh_clean_pubkey_hash(&hash);
 
@@ -199,13 +309,13 @@ static int ssh_verify_known_hosts(ssh_session session)
 	case SSH_KNOWN_HOSTS_NOT_FOUND:
 		fprintf(stderr, "Could not find known host file.\n");
 		fprintf(stderr, "If you accept the host key here, the file will be"
-			"automatically created.\n");
+				"automatically created.\n");
 
 		/* FALL THROUGH to SSH_SERVER_NOT_KNOWN behavior */
 
 	case SSH_KNOWN_HOSTS_UNKNOWN:
 		hexa = ssh_get_hexa(hash, hlen);
-		fprintf(stderr,"The server is unknown. Do you trust the host key?\n");
+		fprintf(stderr, "The server is unknown. Do you trust the host key?\n");
 		fprintf(stderr, "Public key hash: %s\n", hexa);
 		fprintf(stderr, "(yes/no): ");
 		ssh_string_free_char(hexa);
@@ -222,13 +332,13 @@ static int ssh_verify_known_hosts(ssh_session session)
 
 		rc = ssh_session_update_known_hosts(session);
 		if (rc < 0) {
-			fprintf(stderr, "Error %s\n", strerror(errno));
+			priv_set_errv("%s", ssh_get_error(session));
 			return -1;
 		}
 
 		break;
 	case SSH_KNOWN_HOSTS_ERROR:
-		fprintf(stderr, "Error %s", ssh_get_error(session));
+		fprintf(stderr, "known hosts error: %s", ssh_get_error(session));
 		ssh_clean_pubkey_hash(&hash);
 		return -1;
 	}
@@ -237,39 +347,71 @@ static int ssh_verify_known_hosts(ssh_session session)
 	return 0;
 }
 
+static int ssh_authenticate_kbdint(ssh_session ssh)
+{
+	/* Copied and bit modified from
+	 * https://api.libssh.org/stable/libssh_tutor_authentication.html */
+	int rc;
+
+	rc = ssh_userauth_kbdint(ssh, NULL, NULL);
+	while (rc == SSH_AUTH_INFO) {
+		const char *name, *instruction;
+		int nprompts, iprompt;
+
+		name = ssh_userauth_kbdint_getname(ssh);
+		instruction = ssh_userauth_kbdint_getinstruction(ssh);
+		nprompts = ssh_userauth_kbdint_getnprompts(ssh);
+
+		if (strlen(name) > 0)
+			printf("%s\n", name);
+		if (strlen(instruction) > 0)
+			printf("%s\n", instruction);
+		for (iprompt = 0; iprompt < nprompts; iprompt++) {
+			const char *prompt;
+			char echo;
+
+			prompt = ssh_userauth_kbdint_getprompt(ssh, iprompt, &echo);
+			if (echo) {
+				char buf[128], *ptr;
+
+				printf("%s", prompt);
+				if (fgets(buf, sizeof(buf), stdin) == NULL)
+					return SSH_AUTH_ERROR;
+				buf[sizeof(buf) - 1] = '\0';
+				if ((ptr = strchr(buf, '\n')) != NULL)
+					*ptr = '\0';
+				if (ssh_userauth_kbdint_setanswer(ssh, iprompt, buf) < 0)
+					return SSH_AUTH_ERROR;
+				memset(buf, 0, strlen(buf));
+			} else {
+				char *ptr;
+				ptr = getpass(prompt);
+				if (ssh_userauth_kbdint_setanswer(ssh, iprompt, ptr) < 0)
+					return SSH_AUTH_ERROR;
+			}
+		}
+		rc = ssh_userauth_kbdint(ssh, NULL, NULL);
+	}
+	return rc;
+}
+
 void ssh_sftp_close(sftp_session sftp)
 {
 	ssh_session ssh = sftp_ssh(sftp);
-	sftp_free(sftp);
+	/* XXX: sftp_free is stuck in ssh_poll_ctx_dopoll() when build type is Release.
+	 * skip sftp_free inappropriately...
+	 */
+	//sftp_free(sftp);
 	ssh_disconnect(ssh);
 	ssh_free(ssh);
 }
 
-
-ssize_t sftp_write2(sftp_file sf, const void *buf, size_t len, size_t sftp_buf_sz)
+const char **mscp_ssh_ciphers(void)
 {
-	ssize_t ret, nbytes;
-
-	for (nbytes = 0; nbytes < len;) {
-		ret = sftp_write(sf, buf + nbytes,
-				 min(len - nbytes, sftp_buf_sz));
-		if (ret < 0)
-			return ret;
-		nbytes += ret;
-	}
-	return nbytes;
+	return ssh_ciphers();
 }
 
-ssize_t sftp_read2(sftp_file sf, void *buf, size_t len, size_t sftp_buf_sz)
+const char **mscp_ssh_hmacs(void)
 {
-	ssize_t ret, nbytes;
-
-	for (nbytes = 0; nbytes < len;) {
-		ret = sftp_read(sf, buf + nbytes,
-				min(len - nbytes, sftp_buf_sz));
-		if (ret < 0)
-			return ret;
-		nbytes += ret;
-	}
-	return nbytes;
+	return ssh_hmacs();
 }

src/ssh.h

@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
 #ifndef _SSH_H_
 #define _SSH_H_
 
@@ -5,30 +6,15 @@
 #include "libssh/libssh.h"
 #include "libssh/sftp.h"
 
+#include <mscp.h>
 
-struct ssh_opts {
-	char    *login_name;		/* -l */
-	char    *port;			/* -p */
-	char    *identity;		/* -i */
-	char    *cipher;		/* -c */
-	int     compress;		/* -C */
-	int     debuglevel;		/* -v */
-	bool	no_hostkey_check;	/* -H */
-
-	char    *password;		/* filled at the first connecting phase */
-};
-
-/* ssh_make_sftp_session() creates sftp_session. sshdst accpets
+/* ssh_init_sftp_session() creates sftp_session. sshdst accpets
  * user@hostname and hostname notations (by libssh).
  */
-sftp_session ssh_make_sftp_session(char *sshdst, struct ssh_opts *opts);
+sftp_session ssh_init_sftp_session(const char *sshdst, struct mscp_ssh_opts *opts);
 void ssh_sftp_close(sftp_session sftp);
 
 #define sftp_ssh(sftp) (sftp)->session
 #define sftp_get_ssh_error(sftp) ssh_get_error(sftp_ssh(sftp))
 
-/* wrapping multiple sftp_read|write */
-ssize_t sftp_write2(sftp_file sf, const void *buf, size_t len, size_t sftp_buf_sz);
-ssize_t sftp_read2(sftp_file sf, void *buf, size_t len, size_t sftp_buf_sz);
-
 #endif /* _SSH_H_ */

src/strerrno.c

@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
+
+#include <errno.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdarg.h>
+
+#include <strerrno.h>
+
+#define STRERRNO_TLS_BUFSIZ 128
+__thread char tls_strerrno_buf[STRERRNO_TLS_BUFSIZ];
+
+const char *strerrno(void)
+{
+	snprintf(tls_strerrno_buf, sizeof(tls_strerrno_buf), "%s", "strerror_r error");
+	strerror_r(errno, tls_strerrno_buf, sizeof(tls_strerrno_buf));
+	return tls_strerrno_buf;
+}
+
+#define PRIV_ERR_BUFSIZ (1 << 12)
+__thread char priv_err_buf[PRIV_ERR_BUFSIZ], internal[PRIV_ERR_BUFSIZ];
+
+void priv_set_err(const char *fmt, ...)
+{
+	va_list va;
+	memset(internal, 0, sizeof(internal));
+	va_start(va, fmt);
+	vsnprintf(internal, sizeof(internal), fmt, va);
+	va_end(va);
+	snprintf(priv_err_buf, sizeof(priv_err_buf), "%s", internal);
+}
+
+const char *priv_get_err()
+{
+	return priv_err_buf;
+}

src/strerrno.h

@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: GPL-3.0-only */
+#ifndef _STRERRNO_
+#define _STRERRNO_
+
+#include <libgen.h> /* basename() */
+
+/**
+ * strerrno() returns error message string corresponding to errno.
+ * strerrno() is thread safe.
+ */
+const char *strerrno(void);
+
+/**
+ * priv_set_err() sets an error message into a thread-local private
+ * buffer. This error message can be accessed via priv_get_err().
+ *
+ * The top-level function in a thread should print errors using
+ * priv_get_err(), while lower-level functions should set error
+ * messages using priv_set_err().
+ */
+void priv_set_err(const char *fmt, ...);
+
+/**
+ * priv_set_errv(), a wrapper for priv_set_err(), just adds filename,
+ * line, and function name to the error message.
+ */
+#define priv_set_errv(fmt, ...)                                                      \
+	priv_set_err("[%s:%d:%s] " fmt "\0", basename(__FILE__), __LINE__, __func__, \
+		     ##__VA_ARGS__)
+
+/**
+ * priv_get_err() gets the error message sotred in the thread-local private buffer.
+ */
+const char *priv_get_err();
+
+#endif /* _STRERRNO_ */

src/util.h

@@ -1,39 +0,0 @@
-#ifndef _UTIL_H_
-#define _UTIL_H_
-
-#include <stdio.h>
-#include <string.h>
-#include <errno.h>
-
-#define likely(x)       __builtin_expect(!!(x), 1)
-#define unlikely(x)     __builtin_expect(!!(x), 0)
-
-
-#define pr(fmt, ...) fprintf(stderr, fmt, ##__VA_ARGS__)
-
-#define pr_info(fmt, ...) fprintf(stderr, "INFO:%s(): " fmt,    \
-				  __func__, ##__VA_ARGS__)
-
-#define pr_warn(fmt, ...) fprintf(stderr, "\x1b[1m\x1b[33m"	\
-				  "WARN:%s():\x1b[0m " fmt,	\
-				  __func__, ##__VA_ARGS__)
-
-#define pr_err(fmt, ...) fprintf(stderr, "\x1b[1m\x1b[31m"	\
-				 "ERR:%s():\x1b[0m " fmt,	\
-				 __func__, ##__VA_ARGS__)
-
-#ifdef DEBUG
-#define pr_debug(fmt, ...) fprintf(stderr, "\x1b[1m\x1b[33m"    \
-				   "DEBUG:%s():\x1b[0m " fmt,	\
-				   __func__, ##__VA_ARGS__);
-#else
-#define pr_debug(fmt, ...)
-#endif
-
-#define strerrno() strerror(errno)
-
-
-#define min(a, b) (((a) > (b)) ? (b) : (a))
-#define max(a, b) (((a) > (b)) ? (a) : (b))
-
-#endif /* _UTIL_H_ */

test/test_e2e.py

@@ -1,19 +1,75 @@
 
+"""
+test_e2e.py: End-to-End test for mscp executable.
+"""
+
+import platform
 import pytest
-import numpy
-import hashlib
+import getpass
+import datetime
+import time
 import os
+import re
+import shutil
 
-from subprocess import check_call, CalledProcessError, PIPE
+from subprocess import check_call, CalledProcessError
 from util import File, check_same_md5sum
 
 
-def run2ok(args):
-    check_call(list(map(str, args)))
+def run2ok(args, env = None, quiet = False):
+    cmd = list(map(str, args))
+    if not quiet:
+        print("cmd: {}".format(" ".join(cmd)))
+    check_call(cmd, env = env)
 
-def run2ng(args):
-    with pytest.raises(CalledProcessError) as e:
-        check_call(list(map(str, args)))
+def run2ng(args, env = None, timeout = None, quiet = False):
+    if timeout:
+        args = ["timeout", "-s", "INT", timeout] + args
+    cmd = list(map(str, args))
+    if not quiet:
+        print("cmd: {}".format(" ".join(cmd)))
+    with pytest.raises(CalledProcessError):
+        check_call(cmd, env = env)
+
+
+@pytest.fixture(autouse=True)
+def cleanup_files():
+    """
+    Cleanup files having the following `prefixes` or matching `paths`.
+    """
+
+    yield
+
+    prefixes = [
+        "src", "dst",
+        "non_existent_dstdir",
+    ]
+    paths = [
+        "/mscp-test-src", "/tmp/mscp-test-src",
+        "{}/src".format(os.environ["HOME"]),
+        "{}/dst".format(os.environ["HOME"]),
+        "/tmp/mscp_test_ssh_config",
+        "/home/test/dst",
+        "/home/test/src",
+        "checkpoint",
+    ]
+
+    def remove(path):
+        print(f"cleanup remove: {fname}")
+        if os.path.isdir(path):
+            shutil.rmtree(path)
+        else:
+            os.remove(path)
+
+    for fname in os.listdir(os.getcwd()):
+        for prefix in prefixes:
+            if fname.startswith(prefix):
+                remove(fname)
+                break
+
+    for path in paths:
+        if os.path.exists(path):
+            remove(path)
 
 
 """ usage test """
@@ -54,12 +110,16 @@ param_single_copy = [
 @pytest.mark.parametrize("src, dst", param_single_copy)
 def test_single_copy(mscp, src_prefix, dst_prefix, src, dst):
     src.make()
-    run2ok([mscp, "-H", src_prefix + src.path, dst_prefix + dst.path])
+    run2ok([mscp, "-vvv", src_prefix + src.path, dst_prefix + dst.path])
     assert check_same_md5sum(src, dst)
-    src.cleanup()
-    dst.cleanup()
 
 
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_failed_to_copy_nonexistent_file(mscp, src_prefix, dst_prefix):
+    src = "nonexistent_src"
+    dst = "nonexistent_dst"
+    run2ng([mscp, "-vvv", src_prefix + src, dst_prefix + dst])
+
 param_double_copy = [
     (File("src1", size = 1024 * 1024), File("src2", size = 1024 * 1024),
      File("dst/src1"), File("dst/src2")
@@ -70,13 +130,41 @@ param_double_copy = [
 def test_double_copy(mscp, src_prefix, dst_prefix, s1, s2, d1, d2):
     s1.make()
     s2.make()
-    run2ok([mscp, "-H", src_prefix + s1.path, src_prefix + s2.path, dst_prefix + "dst"])
+    run2ok([mscp, "-vvv", src_prefix + s1.path, src_prefix + s2.path, dst_prefix + "dst"])
     assert check_same_md5sum(s1, d1)
     assert check_same_md5sum(s2, d2)
-    s1.cleanup()
-    s2.cleanup()
-    d1.cleanup()
-    d2.cleanup()
+
+
+remote_v6_prefix = "[::1]:{}/".format(os.getcwd())
+param_remote_v6_prefix = [
+    ("", remote_v6_prefix), (remote_v6_prefix, "")
+]
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_v6_prefix)
+@pytest.mark.parametrize("s1, s2, d1, d2", param_double_copy)
+def test_double_copy_with_ipv6_notation(mscp, src_prefix, dst_prefix, s1, s2, d1, d2):
+    s1.make()
+    s2.make()
+    run2ok([mscp, "-vvv",
+            src_prefix + s1.path, src_prefix + s2.path, dst_prefix + "dst"])
+    assert check_same_md5sum(s1, d1)
+    assert check_same_md5sum(s2, d2)
+
+
+remote_user_v6_prefix = "{}@[::1]:{}/".format(getpass.getuser(), os.getcwd())
+param_remote_user_v6_prefix = [
+    ("", remote_user_v6_prefix), (remote_user_v6_prefix, "")
+]
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_user_v6_prefix)
+@pytest.mark.parametrize("s1, s2, d1, d2", param_double_copy)
+def test_double_copy_with_user_and_ipv6_notation(mscp, src_prefix, dst_prefix,
+                                                 s1, s2, d1, d2):
+    s1.make()
+    s2.make()
+    run2ok([mscp, "-vvv",
+            src_prefix + s1.path, src_prefix + s2.path, dst_prefix + "dst"])
+    assert check_same_md5sum(s1, d1)
+    assert check_same_md5sum(s2, d2)
+
 
 param_dir_copy = [
     ( "src_dir", "dst_dir",
@@ -107,18 +195,30 @@ def test_dir_copy(mscp, src_prefix, dst_prefix, src_dir, dst_dir, src, dst, twic
     for f in src:
         f.make()
 
-    run2ok([mscp, "-H", src_prefix + src_dir, dst_prefix + dst_dir])
+    run2ok([mscp, "-vvv", src_prefix + src_dir, dst_prefix + dst_dir])
     for sf, df in zip(src, dst):
         assert check_same_md5sum(sf, df)
 
-    run2ok([mscp, "-H", src_prefix + src_dir, dst_prefix + dst_dir])
+    run2ok([mscp, "-vvv", src_prefix + src_dir, dst_prefix + dst_dir])
     for sf, df in zip(src, twice):
         assert check_same_md5sum(sf, df)
 
-    for sf, df, tf in zip(src, dst, twice):
-        sf.cleanup()
-        df.cleanup()
-        tf.cleanup()
+
+
+param_dir_copy_single = [
+    ("src_dir", "dst_dir",
+     File("src_dir/t1", size = 1024 * 1024),
+     File("dst_dir/src_dir/t1"),
+     )
+]
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+@pytest.mark.parametrize("src_dir, dst_dir, src, dst", param_dir_copy_single)
+def test_dir_copy_single(mscp, src_prefix, dst_prefix, src_dir, dst_dir, src, dst):
+    src.make()
+    os.mkdir(dst_dir)
+    run2ok([mscp, "-vvv", src_prefix + src_dir, dst_prefix + dst_dir])
+    assert check_same_md5sum(src, dst)
+
 
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 def test_override_single_file(mscp, src_prefix, dst_prefix):
@@ -126,59 +226,517 @@ def test_override_single_file(mscp, src_prefix, dst_prefix):
     dst = File("dst", size = 128).make()
     assert not check_same_md5sum(src, dst)
 
-    run2ok([mscp, "-H", src_prefix + src.path, dst_prefix + dst.path])
+    run2ok([mscp, "-vvv", src_prefix + src.path, dst_prefix + dst.path])
+    assert check_same_md5sum(src, dst)
+
+
+absolute_remote_prefix = "localhost:"
+param_absolute_remote_prefix = [
+    ("", absolute_remote_prefix), (absolute_remote_prefix, "")
+]
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_absolute_remote_prefix)
+def test_copy_file_under_root_to_dir(mscp, src_prefix, dst_prefix):
+    src = File("/mscp-test-src", size = 1024).make()
+    dst = File("/tmp/mscp-test-src")
+
+    run2ok([mscp, "-vvv", src_prefix + src.path,
+            dst_prefix + os.path.dirname(dst.path)])
+    assert check_same_md5sum(src, dst)
+
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_dst_has_suffix_slash(mscp, src_prefix, dst_prefix):
+    """
+    if dst path has suffix '/' like "dir/" and does not exist,
+    mscp should create dir/ and put dir/src-file-name.
+    """
+    dstdir = "non_existent_dstdir/"
+
+    src = File("src", size = 1024 * 1024).make()
+    dst = File(f"{dstdir}/src")
+
+    run2ok([mscp, "-vvv", src_prefix + src.path,
+            dst_prefix + dstdir])
+
+    assert check_same_md5sum(src, dst)
+
+
+param_tilde_paths = [
+    ("src", "localhost:~/dst"),
+    ("localhost:~/src", "dst"),
+]
+@pytest.mark.parametrize("src_path, dst_path", param_tilde_paths)
+def test_remote_path_contains_tilde(mscp, src_path, dst_path):
+    """
+    if remote path contains '~' as prefix, it should be expanded as '.'.
+    Note that `~user` notation is not supported yet.
+    """
+    def extract_and_expand(path):
+        path = path if not ':' in path else path[path.index(':')+1:]
+        return path.replace('~', os.environ["HOME"])
+
+    src_f_path = extract_and_expand(src_path)
+    dst_f_path = extract_and_expand(dst_path)
+
+    src = File(src_f_path, size = 1024 * 1024).make()
+    dst = File(dst_f_path)
+
+    run2ok([mscp, "-vvv", src_path, dst_path])
+    assert check_same_md5sum(src, dst)
+
+
+def test_remote_path_contains_tilde2(mscp):
+    src = File("src", size = 1024 * 1024).make()
+    dst = File(f"{os.environ['HOME']}/src")
+
+    run2ok([mscp, "-vvv", src.path, f"localhost:~"])
     assert check_same_md5sum(src, dst)
 
-    src.cleanup()
-    dst.cleanup()
 
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 def test_min_chunk(mscp, src_prefix, dst_prefix):
     src = File("src", size = 16 * 1024).make()
     dst = File("dst")
 
-    run2ok([mscp, "-H", "-s", 8192, src_prefix + src.path, dst_prefix + dst.path])
+    run2ok([mscp, "-vvv", "-s", 32768, src_prefix + src.path, dst_prefix + dst.path])
     assert check_same_md5sum(src, dst)
 
-    src.cleanup()
-    dst.cleanup()
+
+def is_alpine():
+    if os.path.exists("/etc/os-release"):
+        with open("/etc/os-release", "r") as f:
+            for line in f:
+                if line.strip() == "ID=alpine":
+                    return True
+    return False
+
+param_glob_copy = [
+    (
+        "src*", "dstx",
+        [ File("src1"), File("src2"), File("src3") ],
+        [ File("dstx/src1"), File("dstx/src2"), File("dstx/src3") ],
+    ),
+    (
+        "src*", "dstx",
+        [ File("src1/s1"), File("src2/s2"), File("src3/s3") ],
+        [ File("dstx/s1"), File("dstx/s2"), File("dstx/s3") ],
+    )
+]
+
+@pytest.mark.skipif(is_alpine(),
+                    reason = "musl does not implement glob ALTDIRFUNC")
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+@pytest.mark.parametrize("src_glob_path, dst_path, srcs, dsts", param_glob_copy)
+def test_glob_src_path(mscp, src_prefix, dst_prefix,
+                       src_glob_path, dst_path, srcs, dsts):
+    for src in srcs:
+        src.make(size = 1024 * 1024)
+
+    run2ok([mscp, "-vvv", src_prefix + src_glob_path, dst_prefix + dst_path])
+    for src, dst in zip(srcs, dsts):
+        assert check_same_md5sum(src, dst)
+
 
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 def test_thread_affinity(mscp, src_prefix, dst_prefix):
     src = File("src", size = 64 * 1024).make()
     dst = File("dst")
 
-    run2ok([mscp, "-H", "-n", 4, "-m", "0x01", "-s", 8192, "-S", 65536,
+    run2ok([mscp, "-vvv", "-n", 4, "-m", "0x01",
             src_prefix + src.path, dst_prefix + dst.path])
     assert check_same_md5sum(src, dst)
 
-    src.cleanup()
-    dst.cleanup()
 
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 def test_cannot_override_file_with_dir(mscp, src_prefix, dst_prefix):
     src = File("src", size = 128).make()
     dst = File("dst").make()
 
-    run2ng([mscp, "-H", src_prefix + src.path, dst_prefix + "dst/src"])
+    run2ng([mscp, "-vvv", src_prefix + src.path, dst_prefix + "dst/src"])
 
-    src.cleanup()
-    dst.cleanup()
 
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 def test_transfer_zero_bytes(mscp, src_prefix, dst_prefix):
     src = File("src", size = 0).make()
     dst = File("dst")
-    run2ok([mscp, "-H", src_prefix + src.path, dst_prefix + "dst"])
+    run2ok([mscp, "-vvv", src_prefix + src.path, dst_prefix + "dst"])
     assert os.path.exists("dst")
-    src.cleanup()
-    dst.cleanup()
 
 @pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
 def test_override_dst_having_larger_size(mscp, src_prefix, dst_prefix):
     src = File("src", size = 1024 * 1024).make()
     dst = File("dst", size = 1024 * 1024 * 2).make()
-    run2ok([mscp, "-H", src_prefix + src.path, dst_prefix + "dst"])
+    run2ok([mscp, "-vvv", src_prefix + src.path, dst_prefix + "dst"])
     assert check_same_md5sum(src, dst)
-    src.cleanup()
-    dst.cleanup()
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_dont_truncate_dst(mscp, src_prefix, dst_prefix):
+    f = File("srcanddst", size = 1024 * 1024 * 128).make()
+    md5_before = f.md5sum()
+    run2ok([mscp, "-vvv", src_prefix + f.path, dst_prefix + f.path])
+    md5_after = f.md5sum()
+    assert md5_before == md5_after
+
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_copy_readonly_file(mscp, src_prefix, dst_prefix):
+    """When a source file permission is r--r--r--, if chmod(r--r--r--)
+    runs first on the remote side, following truncate() and setutime()
+    fail due to permission deneid. So, run chmod() after truncate()
+    and setutime()
+
+    """
+    src = File("src", size = 1024 * 1024 * 128, perm = 0o444).make()
+    dst = File("dst")
+    run2ok([mscp, "-vvv", src_prefix + src.path, dst_prefix + dst.path])
+    assert check_same_md5sum(src, dst)
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_dont_make_conns_more_than_chunks(mscp, src_prefix, dst_prefix):
+    # copy 100 files with -n 20 -I 1 options. if mscp creates 20 SSH
+    # connections although all files have been copied, it is error.
+    srcs = []
+    dsts = []
+    for n in range(100):
+        srcs.append(File("src/src-{:06d}".format(n), size=1024).make())
+        dsts.append(File("dst/src-{:06d}".format(n)))
+    start = time.time()
+    run2ok([mscp, "-v", "-n", "20", "-I", "1",
+            src_prefix + "src", dst_prefix + "dst"])
+    end = time.time()
+    for s, d in zip(srcs, dsts):
+        assert check_same_md5sum(s, d)
+
+    assert((end - start) < 10)
+
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_bwlimit(mscp, src_prefix, dst_prefix):
+    """Copy 100MB file with 100Mbps bitrate, this requires 8 seconds."""
+    src = File("src", size = 100 * 1024 * 1024).make()
+    dst = File("dst")
+
+    start = datetime.datetime.now().timestamp()
+    run2ok([mscp, "-vvv", "-L", "100m", src_prefix + "src", dst_prefix + "dst"])
+    end = datetime.datetime.now().timestamp()
+    assert check_same_md5sum(src, dst)
+    assert end - start > 7
+
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+@pytest.mark.parametrize("src, dst", param_single_copy)
+def test_set_port_ng(mscp, src_prefix, dst_prefix, src, dst):
+    src.make()
+    run2ng([mscp, "-vvv", "-P", 21, src_prefix + src.path, dst_prefix + dst.path])
+
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+@pytest.mark.parametrize("src, dst", param_single_copy)
+def test_set_port_ok(mscp, src_prefix, dst_prefix, src, dst):
+    src.make()
+    run2ok([mscp, "-vvv", "-P", 8022, src_prefix + src.path, dst_prefix + dst.path])
+
+def test_v4only(mscp):
+    src = File("src", size = 1024).make()
+    dst = File("dst")
+    dst_prefix = "localhost:{}/".format(os.getcwd())
+    run2ok([mscp, "-vvv", "-4", src.path, dst_prefix + dst.path])
+    assert check_same_md5sum(src, dst)
+
+def test_v6only(mscp):
+    src = File("src", size = 1024).make()
+    dst = File("dst")
+    dst_prefix = "ip6-localhost:{}/".format(os.getcwd())
+    run2ok([mscp, "-vvv", "-6", src.path, dst_prefix + dst.path])
+    assert check_same_md5sum(src, dst)
+
+def test_v4_to_v6_should_fail(mscp):
+    src = File("src", size = 1024).make()
+    dst = File("dst")
+    dst_prefix = "[::1]:{}/".format(os.getcwd())
+    run2ng([mscp, "-vvv", "-4", src.path, dst_prefix + dst.path])
+
+def test_v6_to_v4_should_fail(mscp):
+    src = File("src", size = 1024).make()
+    dst = File("dst")
+    dst_prefix = "127.0.0.1:{}/".format(os.getcwd())
+    run2ng([mscp, "-vvv", "-6", src.path, dst_prefix + dst.path])
+
+def test_quiet_mode(capsys, mscp):
+    src = File("src", size = 1024).make()
+    dst = File("dst")
+    dst_prefix = "127.0.0.1:{}/".format(os.getcwd())
+    run2ok([mscp, "-vvv", "-q", src.path, dst_prefix + dst.path], quiet=True)
+    assert check_same_md5sum(src, dst)
+
+    captured = capsys.readouterr()
+    assert not captured.out
+    assert not captured.err
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_set_conn_interval(mscp, src_prefix, dst_prefix):
+    srcs = []
+    dsts = []
+    for x in range(500):
+        srcs.append(File("src/file{}".format(x), size = 128).make())
+        dsts.append(File("dst/file{}".format(x)))
+    run2ok([mscp, "-vvv", "-I", 1, src_prefix + "src", dst_prefix + "dst"])
+
+    for src, dst in zip(srcs, dsts):
+        assert check_same_md5sum(src, dst)
+
+compressions = ["yes", "no", "none"]
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+@pytest.mark.parametrize("compress", compressions)
+def test_compression(mscp, src_prefix, dst_prefix, compress):
+    src = File("src", size = 1024 * 1024).make()
+    dst = File("dst", size = 1024 * 1024 * 2).make()
+    run2ok([mscp, "-vvv", "-C", compress, src_prefix + src.path, dst_prefix + "dst"])
+    assert check_same_md5sum(src, dst)
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_ccalgo(mscp, src_prefix, dst_prefix):
+    src = File("src", size = 1024 * 1024).make()
+    dst = File("dst").make()
+    if platform.system() == "Darwin":
+        # Darwin does not support TCP_CONGESTION
+        algo = "cubic"
+        run = run2ng
+    elif platform.system() == "Linux":
+        # Linux supports TCP_CONGESTION
+        with open("/proc/sys/net/ipv4/tcp_allowed_congestion_control", "r") as f:
+            algo = f.read().strip().split().pop()
+        run = run2ok
+    run([mscp, "-vvv", "-g", algo, src_prefix + src.path, dst_prefix + "dst"])
+
+
+testhost = "mscptestlocalhost"
+testhost_prefix = "{}:{}/".format(testhost, os.getcwd()) # use current dir
+param_testhost_prefix = [
+    ("", testhost_prefix), (testhost_prefix, "")
+]
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_testhost_prefix)
+def test_config_ok(mscp, src_prefix, dst_prefix):
+    config = "/tmp/mscp_test_ssh_config"
+    with open(config, "w") as f:
+        f.write("host {}\n".format(testhost))
+        f.write("    hostname localhost\n")
+
+    src = File("src", size = 1024 * 1024).make()
+    dst = File("dst", size = 1024 * 1024 * 2).make()
+    run2ok([mscp, "-vvv", "-F", config,
+            src_prefix + src.path, dst_prefix + "dst"])
+
+    os.remove(config)
+    assert check_same_md5sum(src, dst)
+
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_testhost_prefix)
+def test_config_ng(mscp, src_prefix, dst_prefix):
+    config = "/tmp/mscp_test_ssh_config"
+    with open(config, "w") as f:
+        f.write("\n") # use empty ssh_config
+
+    src = File("src", size = 1024 * 1024).make()
+    dst = File("dst", size = 1024 * 1024 * 2).make()
+    run2ng([mscp, "-vvv", "-F", config,
+            src_prefix + src.path, dst_prefix + "dst"])
+
+    os.remove(config)
+
+
+param_valid_option_ok = [
+    [ "-o", "Port=8022" ],
+    [ "-o", "Port=8022", "-o", "User=root" ],
+    [ "-o", "unknown-option-is-silently-ignored" ],
+]
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+@pytest.mark.parametrize("option", param_valid_option_ok)
+def test_inline_option_ok(mscp, src_prefix, dst_prefix, option):
+    """ change port number with -o option. it should be ok. """
+    src = File("src", size = 1024 * 1024).make()
+    dst = File("dst")
+    run2ok([mscp, "-vvv"] + option +
+           [src_prefix + src.path, dst_prefix + dst.path])
+    assert check_same_md5sum(src, dst)
+
+
+param_valid_option_ng = [
+    [ "-o", "Port=8023" ],
+    [ "-o", "User=invaliduser" ],
+]
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+@pytest.mark.parametrize("option", param_valid_option_ng)
+def test_inline_option_ng(mscp, src_prefix, dst_prefix, option):
+    """ change port number with -o option. it should be ng. """
+    src = File("src", size = 1024 * 1024).make()
+    dst = File("dst")
+    run2ng([mscp, "-vvv"] + option +
+           [src_prefix + src.path, dst_prefix + dst.path])
+
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_porxyjump_ok(mscp, src_prefix, dst_prefix):
+    """ test -J proxyjump option"""
+    src = File("src", size = 10 * 1024 * 1024).make()
+    dst = File("dst")
+    # use small min-chunk-size to use multiple connections
+    run2ok([mscp, "-n", 4, "-s", 1024 * 1024, "-vvv",
+            "-J", "localhost:8022",
+            src_prefix + src.path, dst_prefix + dst.path])
+    assert check_same_md5sum(src, dst)
+
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_porxyjump_ng(mscp, src_prefix, dst_prefix):
+    """ test -J proxyjump option, invalid jump node causes fail"""
+    src = File("src", size = 10 * 1024 * 1024).make()
+    dst = File("dst")
+    # use small min-chunk-size to use multiple connections
+    run2ng([mscp, "-n", 4, "-s", 1024 * 1024, "-vvv",
+            "-J", "invaliduser@localhost:8022",
+            src_prefix + src.path, dst_prefix + dst.path])
+
+# username test assumes that this test runs inside a container, see Dockerfiles
+def test_specify_passphrase_via_env(mscp):
+    src = File(os.getcwd() + "/src", size = 1024).make()
+    dst = File("/home/test/dst")
+    env = os.environ
+    env["MSCP_SSH_AUTH_PASSPHRASE"]  = "keypassphrase"
+    run2ok([mscp, "-vvv", "-l", "test", "-i", "/home/test/.ssh/id_rsa_test",
+            src.path, "localhost:" + dst.path], env = env)
+    assert check_same_md5sum(src, dst)
+
+def test_specify_invalid_passphrase_via_env(mscp):
+    src = File(os.getcwd() + "/src", size = 1024).make()
+    dst = File("/home/test/dst")
+    env = os.environ
+    env["MSCP_SSH_AUTH_PASSPHRASE"]  = "invalid-keypassphrase"
+    run2ng([mscp, "-vvv", "-l", "test", "-i", "/home/test/.ssh/id_rsa_test",
+            src.path, "localhost:" + dst.path], env = env)
+
+def test_specify_password_via_env(mscp):
+    src = File(os.getcwd() + "/src", size = 1024).make()
+    dst = File("/home/test/dst")
+    env = os.environ
+    env["MSCP_SSH_AUTH_PASSWORD"]  = "userpassword"
+    run2ok([mscp, "-vvv", "-l", "test",
+            src.path, "localhost:" + dst.path], env = env)
+    assert check_same_md5sum(src, dst)
+
+def test_specify_invalid_password_via_env(mscp):
+    src = File(os.getcwd() + "/src", size = 1024).make()
+    dst = File("/home/test/dst")
+    env = os.environ
+    env["MSCP_SSH_AUTH_PASSWORD"]  = "invalid-userpassword"
+    run2ng([mscp, "-vvv", "-l", "test",
+            src.path, "localhost:" + dst.path], env = env)
+
+
+@pytest.fixture
+def move_pubkey_temporally():
+    """
+    mv ~/.ssh/id_* to id_rsa.bak before test, and move it back after test.
+    """
+
+    sshdir = os.path.join(os.environ["HOME"], ".ssh")
+
+    # move pubkeys to /tmp
+    moved = []
+    for fname in os.listdir(sshdir):
+        if re.match(r"^id_[a-z0-9]+$", fname):
+            moved.append(fname)
+            shutil.move(f"{sshdir}/{fname}", f"/tmp/{fname}")
+
+    yield
+
+    # move back the keys
+    for fname in moved:
+        shutil.move(f"/tmp/{fname}", f"{sshdir}/{fname}")
+
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_passwordauth_without_pubkey(move_pubkey_temporally,
+                                     mscp, src_prefix, dst_prefix):
+    """
+    make sure password auth works (by removing public keys)
+    """
+    src = File(os.getcwd() + "/src", size = 1024).make()
+    dst = File("/home/test/dst")
+    env = os.environ
+    env["MSCP_SSH_AUTH_PASSWORD"]  = "userpassword"
+    run2ok([mscp, "-vvv", "-l", "test",
+            src.path, "localhost:" + dst.path], env = env)
+    assert check_same_md5sum(src, dst)
+
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_10k_files(mscp, src_prefix, dst_prefix):
+    srcs = []
+    dsts = []
+    for n in range(10000):
+        srcs.append(File("src/src-{:06d}".format(n), size=1024).make())
+        dsts.append(File("dst/src-{:06d}".format(n)))
+    run2ok([mscp, "-v", src_prefix + "src", dst_prefix + "dst"])
+    for s, d in zip(srcs, dsts):
+        assert check_same_md5sum(s, d)
+
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_checkpoint_dump_and_resume(mscp, src_prefix, dst_prefix):
+    src1 = File("src1", size = 64 * 1024 * 1024).make()
+    src2 = File("src2", size = 64 * 1024 * 1024).make()
+    dst1 = File("dst/src1")
+    dst2 = File("dst/src2")
+    run2ok([mscp, "-vvv", "-W", "checkpoint", "-D",
+            src_prefix + "src1", src_prefix + "src2", dst_prefix + "dst"])
+    assert os.path.exists("checkpoint")
+
+    run2ok([mscp, "-vvv", "-R", "checkpoint"])
+    assert check_same_md5sum(src1, dst1)
+    assert check_same_md5sum(src2, dst2)
+    os.remove("checkpoint")
+
+@pytest.mark.parametrize("timeout", [ 1, 2, 3, 4, 5 ])
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_checkpoint_interrupt_large_file(mscp, timeout, src_prefix, dst_prefix):
+    """Copy two 100MB files with 200Mbps -> 4 sec + 4 sec """
+    src1 = File("src1", size = 100 * 1024 * 1024).make()
+    src2 = File("src2", size = 100 * 1024 * 1024).make()
+    dst1 = File("dst/src1")
+    dst2 = File("dst/src2")
+    run2ng([mscp, "-vv", "-W", "checkpoint", "-L", "200m",
+            src_prefix + "src1", src_prefix + "src2", dst_prefix + "dst"],
+           timeout = timeout)
+    assert os.path.exists("checkpoint")
+
+    run2ok([mscp, "-vv", "-R", "checkpoint"])
+    assert check_same_md5sum(src1, dst1)
+    assert check_same_md5sum(src2, dst2)
+    os.remove("checkpoint")
+
+@pytest.mark.parametrize("timeout", [ 1, 2, 3, 4, 5 ])
+@pytest.mark.parametrize("src_prefix, dst_prefix", param_remote_prefix)
+def test_checkpoint_interrupt_many_files(mscp, timeout, src_prefix, dst_prefix):
+    """Copy 100 1-MB files with 4 connections, and interrupt and
+    resume the transfer
+    """
+
+    files = []
+    for x in range(100):
+        files.append((
+            File("src/{:03d}".format(x), size = 1024 * 1024).make(),
+            File("dst/{:03d}".format(x))
+        ))
+
+    run2ng([mscp, "-vv", "-W", "checkpoint", "-L", "80m", "-n", 4,
+            src_prefix + "src",  dst_prefix + "dst"],
+           timeout = timeout)
+    assert os.path.exists("checkpoint")
+
+    run2ok([mscp, "-vv", "-R", "checkpoint"])
+
+    for src, dst in files:
+        assert check_same_md5sum(src, dst)
+
+    os.remove("checkpoint")
+

test/util.py

@@ -1,6 +1,5 @@
 
 import hashlib
-import numpy
 import os
 
 
@@ -17,10 +16,16 @@ class File():
         self.content = content
         self.perm = perm
 
+    def __repr__(self):
+        return "<file:{} {}-bytes>".format(self.path, self.size)
+
     def __str__(self):
         return self.path
 
-    def make(self):
+    def make(self, size = None):
+        if size:
+            self.size = size
+
         d = os.path.dirname(self.path)
         if d:
             os.makedirs(d, exist_ok = True)
@@ -39,10 +44,12 @@ class File():
 
     def make_content_random(self):
         with open(self.path, "wb") as f:
-            f.write(numpy.random.bytes(self.size))
+            f.write(os.urandom(self.size))
 
-    def cleanup(self):
+    def cleanup(self, preserve_dir = False):
         os.remove(self.path)
+        if preserve_dir:
+            return
         tmp = os.path.dirname(self.path)
         while tmp and not tmp in [".", "/"]:
             if len(os.listdir(tmp)) == 0: