Ma/mscp
1
0
Fork 0
mirror of https://github.com/upa/mscp.git synced 2026-02-13 08:34:41 +08:00
Code Issues Packages Projects Releases 1 Wiki Activity

add .clang-format from Linux kernel and format sources

Browse Source 
The exception is that ColumnLimit is 90.
This commit is contained in:
Ryo Nakamura
2024-02-06 23:09:59 +09:00
parent 4f0669f8f8
commit 93f155cffe
14 changed files with 1295 additions and 602 deletions
Download Patch File Download Diff File Expand all files Collapse all files

745
.clang-format Normal file
View File

@@ -0,0 +1,745 @@
# SPDX-License-Identifier: GPL-2.0
#
# clang-format configuration file. Intended for clang-format >= 11.
#
# For more information, see:
#
# Documentation/process/clang-format.rst
# https://clang.llvm.org/docs/ClangFormat.html
# https://clang.llvm.org/docs/ClangFormatStyleOptions.html
#
# clang-format configuration for Linux kernel, except that ColumnLimit is 90
---
AccessModifierOffset: -4
AlignAfterOpenBracket: Align
AlignConsecutiveAssignments: false
AlignConsecutiveDeclarations: false
AlignEscapedNewlines: Left
AlignOperands: true
AlignTrailingComments: false
AllowAllParametersOfDeclarationOnNextLine: false
AllowShortBlocksOnASingleLine: false
AllowShortCaseLabelsOnASingleLine: false
AllowShortFunctionsOnASingleLine: None
AllowShortIfStatementsOnASingleLine: false
AllowShortLoopsOnASingleLine: false
AlwaysBreakAfterDefinitionReturnType: None
AlwaysBreakAfterReturnType: None
AlwaysBreakBeforeMultilineStrings: false
AlwaysBreakTemplateDeclarations: false
BinPackArguments: true
BinPackParameters: true
BraceWrapping:
AfterClass: false
AfterControlStatement: false
AfterEnum: false
AfterFunction: true
AfterNamespace: true
AfterObjCDeclaration: false
AfterStruct: false
AfterUnion: false
AfterExternBlock: false
BeforeCatch: false
BeforeElse: false
IndentBraces: false
SplitEmptyFunction: true
SplitEmptyRecord: true
SplitEmptyNamespace: true
BreakBeforeBinaryOperators: None
BreakBeforeBraces: Custom
BreakBeforeInheritanceComma: false
BreakBeforeTernaryOperators: false
BreakConstructorInitializersBeforeComma: false
BreakConstructorInitializers: BeforeComma
BreakAfterJavaFieldAnnotations: false
BreakStringLiterals: false
ColumnLimit: 90
CommentPragmas: '^ IWYU pragma:'
CompactNamespaces: false
ConstructorInitializerAllOnOneLineOrOnePerLine: false
ConstructorInitializerIndentWidth: 8
ContinuationIndentWidth: 8
Cpp11BracedListStyle: false
DerivePointerAlignment: false
DisableFormat: false
ExperimentalAutoDetectBinPacking: false
FixNamespaceComments: false
# Taken from:
# git grep -h '^#define [^[:space:]]*for_each[^[:space:]]*(' include/ tools/ \
# | sed "s,^#define \([^[:space:]]*for_each[^[:space:]]*\)(.*$, - '\1'," \
# | LC_ALL=C sort -u
ForEachMacros:
- '__ata_qc_for_each'
- '__bio_for_each_bvec'
- '__bio_for_each_segment'
- '__evlist__for_each_entry'
- '__evlist__for_each_entry_continue'
- '__evlist__for_each_entry_from'
- '__evlist__for_each_entry_reverse'
- '__evlist__for_each_entry_safe'
- '__for_each_mem_range'
- '__for_each_mem_range_rev'
- '__for_each_thread'
- '__hlist_for_each_rcu'
- '__map__for_each_symbol_by_name'
- '__pci_bus_for_each_res0'
- '__pci_bus_for_each_res1'
- '__pci_dev_for_each_res0'
- '__pci_dev_for_each_res1'
- '__perf_evlist__for_each_entry'
- '__perf_evlist__for_each_entry_reverse'
- '__perf_evlist__for_each_entry_safe'
- '__rq_for_each_bio'
- '__shost_for_each_device'
- '__sym_for_each'
- 'apei_estatus_for_each_section'
- 'ata_for_each_dev'
- 'ata_for_each_link'
- 'ata_qc_for_each'
- 'ata_qc_for_each_raw'
- 'ata_qc_for_each_with_internal'
- 'ax25_for_each'
- 'ax25_uid_for_each'
- 'bio_for_each_bvec'
- 'bio_for_each_bvec_all'
- 'bio_for_each_folio_all'
- 'bio_for_each_integrity_vec'
- 'bio_for_each_segment'
- 'bio_for_each_segment_all'
- 'bio_list_for_each'
- 'bip_for_each_vec'
- 'bond_for_each_slave'
- 'bond_for_each_slave_rcu'
- 'bpf_for_each'
- 'bpf_for_each_reg_in_vstate'
- 'bpf_for_each_reg_in_vstate_mask'
- 'bpf_for_each_spilled_reg'
- 'bpf_object__for_each_map'
- 'bpf_object__for_each_program'
- 'btree_for_each_safe128'
- 'btree_for_each_safe32'
- 'btree_for_each_safe64'
- 'btree_for_each_safel'
- 'card_for_each_dev'
- 'cgroup_taskset_for_each'
- 'cgroup_taskset_for_each_leader'
- 'cpu_aggr_map__for_each_idx'
- 'cpufreq_for_each_efficient_entry_idx'
- 'cpufreq_for_each_entry'
- 'cpufreq_for_each_entry_idx'
- 'cpufreq_for_each_valid_entry'
- 'cpufreq_for_each_valid_entry_idx'
- 'css_for_each_child'
- 'css_for_each_descendant_post'
- 'css_for_each_descendant_pre'
- 'damon_for_each_region'
- 'damon_for_each_region_from'
- 'damon_for_each_region_safe'
- 'damon_for_each_scheme'
- 'damon_for_each_scheme_safe'
- 'damon_for_each_target'
- 'damon_for_each_target_safe'
- 'damos_for_each_filter'
- 'damos_for_each_filter_safe'
- 'data__for_each_file'
- 'data__for_each_file_new'
- 'data__for_each_file_start'
- 'device_for_each_child_node'
- 'displayid_iter_for_each'
- 'dma_fence_array_for_each'
- 'dma_fence_chain_for_each'
- 'dma_fence_unwrap_for_each'
- 'dma_resv_for_each_fence'
- 'dma_resv_for_each_fence_unlocked'
- 'do_for_each_ftrace_op'
- 'drm_atomic_crtc_for_each_plane'
- 'drm_atomic_crtc_state_for_each_plane'
- 'drm_atomic_crtc_state_for_each_plane_state'
- 'drm_atomic_for_each_plane_damage'
- 'drm_client_for_each_connector_iter'
- 'drm_client_for_each_modeset'
- 'drm_connector_for_each_possible_encoder'
- 'drm_exec_for_each_locked_object'
- 'drm_exec_for_each_locked_object_reverse'
- 'drm_for_each_bridge_in_chain'
- 'drm_for_each_connector_iter'
- 'drm_for_each_crtc'
- 'drm_for_each_crtc_reverse'
- 'drm_for_each_encoder'
- 'drm_for_each_encoder_mask'
- 'drm_for_each_fb'
- 'drm_for_each_legacy_plane'
- 'drm_for_each_plane'
- 'drm_for_each_plane_mask'
- 'drm_for_each_privobj'
- 'drm_gem_for_each_gpuva'
- 'drm_gem_for_each_gpuva_safe'
- 'drm_gpuva_for_each_op'
- 'drm_gpuva_for_each_op_from_reverse'
- 'drm_gpuva_for_each_op_safe'
- 'drm_gpuvm_for_each_va'
- 'drm_gpuvm_for_each_va_range'
- 'drm_gpuvm_for_each_va_range_safe'
- 'drm_gpuvm_for_each_va_safe'
- 'drm_mm_for_each_hole'
- 'drm_mm_for_each_node'
- 'drm_mm_for_each_node_in_range'
- 'drm_mm_for_each_node_safe'
- 'dsa_switch_for_each_available_port'
- 'dsa_switch_for_each_cpu_port'
- 'dsa_switch_for_each_cpu_port_continue_reverse'
- 'dsa_switch_for_each_port'
- 'dsa_switch_for_each_port_continue_reverse'
- 'dsa_switch_for_each_port_safe'
- 'dsa_switch_for_each_user_port'
- 'dsa_tree_for_each_cpu_port'
- 'dsa_tree_for_each_user_port'
- 'dsa_tree_for_each_user_port_continue_reverse'
- 'dso__for_each_symbol'
- 'dsos__for_each_with_build_id'
- 'elf_hash_for_each_possible'
- 'elf_symtab__for_each_symbol'
- 'evlist__for_each_cpu'
- 'evlist__for_each_entry'
- 'evlist__for_each_entry_continue'
- 'evlist__for_each_entry_from'
- 'evlist__for_each_entry_reverse'
- 'evlist__for_each_entry_safe'
- 'flow_action_for_each'
- 'for_each_acpi_consumer_dev'
- 'for_each_acpi_dev_match'
- 'for_each_active_dev_scope'
- 'for_each_active_drhd_unit'
- 'for_each_active_iommu'
- 'for_each_active_route'
- 'for_each_aggr_pgid'
- 'for_each_and_bit'
- 'for_each_andnot_bit'
- 'for_each_available_child_of_node'
- 'for_each_bench'
- 'for_each_bio'
- 'for_each_board_func_rsrc'
- 'for_each_btf_ext_rec'
- 'for_each_btf_ext_sec'
- 'for_each_bvec'
- 'for_each_card_auxs'
- 'for_each_card_auxs_safe'
- 'for_each_card_components'
- 'for_each_card_dapms'
- 'for_each_card_pre_auxs'
- 'for_each_card_prelinks'
- 'for_each_card_rtds'
- 'for_each_card_rtds_safe'
- 'for_each_card_widgets'
- 'for_each_card_widgets_safe'
- 'for_each_cgroup_storage_type'
- 'for_each_child_of_node'
- 'for_each_clear_bit'
- 'for_each_clear_bit_from'
- 'for_each_clear_bitrange'
- 'for_each_clear_bitrange_from'
- 'for_each_cmd'
- 'for_each_cmsghdr'
- 'for_each_collection'
- 'for_each_comp_order'
- 'for_each_compatible_node'
- 'for_each_component_dais'
- 'for_each_component_dais_safe'
- 'for_each_conduit'
- 'for_each_console'
- 'for_each_console_srcu'
- 'for_each_cpu'
- 'for_each_cpu_and'
- 'for_each_cpu_andnot'
- 'for_each_cpu_or'
- 'for_each_cpu_wrap'
- 'for_each_dapm_widgets'
- 'for_each_dedup_cand'
- 'for_each_dev_addr'
- 'for_each_dev_scope'
- 'for_each_dma_cap_mask'
- 'for_each_dpcm_be'
- 'for_each_dpcm_be_rollback'
- 'for_each_dpcm_be_safe'
- 'for_each_dpcm_fe'
- 'for_each_drhd_unit'
- 'for_each_dss_dev'
- 'for_each_efi_memory_desc'
- 'for_each_efi_memory_desc_in_map'
- 'for_each_element'
- 'for_each_element_extid'
- 'for_each_element_id'
- 'for_each_endpoint_of_node'
- 'for_each_event'
- 'for_each_event_tps'
- 'for_each_evictable_lru'
- 'for_each_fib6_node_rt_rcu'
- 'for_each_fib6_walker_rt'
- 'for_each_free_mem_pfn_range_in_zone'
- 'for_each_free_mem_pfn_range_in_zone_from'
- 'for_each_free_mem_range'
- 'for_each_free_mem_range_reverse'
- 'for_each_func_rsrc'
- 'for_each_gpiochip_node'
- 'for_each_group_evsel'
- 'for_each_group_evsel_head'
- 'for_each_group_member'
- 'for_each_group_member_head'
- 'for_each_hstate'
- 'for_each_if'
- 'for_each_inject_fn'
- 'for_each_insn'
- 'for_each_insn_prefix'
- 'for_each_intid'
- 'for_each_iommu'
- 'for_each_ip_tunnel_rcu'
- 'for_each_irq_nr'
- 'for_each_lang'
- 'for_each_link_codecs'
- 'for_each_link_cpus'
- 'for_each_link_platforms'
- 'for_each_lru'
- 'for_each_matching_node'
- 'for_each_matching_node_and_match'
- 'for_each_media_entity_data_link'
- 'for_each_mem_pfn_range'
- 'for_each_mem_range'
- 'for_each_mem_range_rev'
- 'for_each_mem_region'
- 'for_each_member'
- 'for_each_memory'
- 'for_each_migratetype_order'
- 'for_each_missing_reg'
- 'for_each_mle_subelement'
- 'for_each_mod_mem_type'
- 'for_each_net'
- 'for_each_net_continue_reverse'
- 'for_each_net_rcu'
- 'for_each_netdev'
- 'for_each_netdev_continue'
- 'for_each_netdev_continue_rcu'
- 'for_each_netdev_continue_reverse'
- 'for_each_netdev_dump'
- 'for_each_netdev_feature'
- 'for_each_netdev_in_bond_rcu'
- 'for_each_netdev_rcu'
- 'for_each_netdev_reverse'
- 'for_each_netdev_safe'
- 'for_each_new_connector_in_state'
- 'for_each_new_crtc_in_state'
- 'for_each_new_mst_mgr_in_state'
- 'for_each_new_plane_in_state'
- 'for_each_new_plane_in_state_reverse'
- 'for_each_new_private_obj_in_state'
- 'for_each_new_reg'
- 'for_each_node'
- 'for_each_node_by_name'
- 'for_each_node_by_type'
- 'for_each_node_mask'
- 'for_each_node_state'
- 'for_each_node_with_cpus'
- 'for_each_node_with_property'
- 'for_each_nonreserved_multicast_dest_pgid'
- 'for_each_numa_hop_mask'
- 'for_each_of_allnodes'
- 'for_each_of_allnodes_from'
- 'for_each_of_cpu_node'
- 'for_each_of_pci_range'
- 'for_each_old_connector_in_state'
- 'for_each_old_crtc_in_state'
- 'for_each_old_mst_mgr_in_state'
- 'for_each_old_plane_in_state'
- 'for_each_old_private_obj_in_state'
- 'for_each_oldnew_connector_in_state'
- 'for_each_oldnew_crtc_in_state'
- 'for_each_oldnew_mst_mgr_in_state'
- 'for_each_oldnew_plane_in_state'
- 'for_each_oldnew_plane_in_state_reverse'
- 'for_each_oldnew_private_obj_in_state'
- 'for_each_online_cpu'
- 'for_each_online_node'
- 'for_each_online_pgdat'
- 'for_each_or_bit'
- 'for_each_path'
- 'for_each_pci_bridge'
- 'for_each_pci_dev'
- 'for_each_pcm_streams'
- 'for_each_physmem_range'
- 'for_each_populated_zone'
- 'for_each_possible_cpu'
- 'for_each_present_blessed_reg'
- 'for_each_present_cpu'
- 'for_each_prime_number'
- 'for_each_prime_number_from'
- 'for_each_probe_cache_entry'
- 'for_each_process'
- 'for_each_process_thread'
- 'for_each_prop_codec_conf'
- 'for_each_prop_dai_codec'
- 'for_each_prop_dai_cpu'
- 'for_each_prop_dlc_codecs'
- 'for_each_prop_dlc_cpus'
- 'for_each_prop_dlc_platforms'
- 'for_each_property_of_node'
- 'for_each_reg'
- 'for_each_reg_filtered'
- 'for_each_reloc'
- 'for_each_reloc_from'
- 'for_each_requested_gpio'
- 'for_each_requested_gpio_in_range'
- 'for_each_reserved_mem_range'
- 'for_each_reserved_mem_region'
- 'for_each_rtd_codec_dais'
- 'for_each_rtd_components'
- 'for_each_rtd_cpu_dais'
- 'for_each_rtd_dais'
- 'for_each_sband_iftype_data'
- 'for_each_script'
- 'for_each_sec'
- 'for_each_set_bit'
- 'for_each_set_bit_from'
- 'for_each_set_bit_wrap'
- 'for_each_set_bitrange'
- 'for_each_set_bitrange_from'
- 'for_each_set_clump8'
- 'for_each_sg'
- 'for_each_sg_dma_page'
- 'for_each_sg_page'
- 'for_each_sgtable_dma_page'
- 'for_each_sgtable_dma_sg'
- 'for_each_sgtable_page'
- 'for_each_sgtable_sg'
- 'for_each_sibling_event'
- 'for_each_sta_active_link'
- 'for_each_subelement'
- 'for_each_subelement_extid'
- 'for_each_subelement_id'
- 'for_each_sublist'
- 'for_each_subsystem'
- 'for_each_supported_activate_fn'
- 'for_each_supported_inject_fn'
- 'for_each_sym'
- 'for_each_test'
- 'for_each_thread'
- 'for_each_token'
- 'for_each_unicast_dest_pgid'
- 'for_each_valid_link'
- 'for_each_vif_active_link'
- 'for_each_vma'
- 'for_each_vma_range'
- 'for_each_vsi'
- 'for_each_wakeup_source'
- 'for_each_zone'
- 'for_each_zone_zonelist'
- 'for_each_zone_zonelist_nodemask'
- 'func_for_each_insn'
- 'fwnode_for_each_available_child_node'
- 'fwnode_for_each_child_node'
- 'fwnode_for_each_parent_node'
- 'fwnode_graph_for_each_endpoint'
- 'gadget_for_each_ep'
- 'genradix_for_each'
- 'genradix_for_each_from'
- 'genradix_for_each_reverse'
- 'hash_for_each'
- 'hash_for_each_possible'
- 'hash_for_each_possible_rcu'
- 'hash_for_each_possible_rcu_notrace'
- 'hash_for_each_possible_safe'
- 'hash_for_each_rcu'
- 'hash_for_each_safe'
- 'hashmap__for_each_entry'
- 'hashmap__for_each_entry_safe'
- 'hashmap__for_each_key_entry'
- 'hashmap__for_each_key_entry_safe'
- 'hctx_for_each_ctx'
- 'hists__for_each_format'
- 'hists__for_each_sort_list'
- 'hlist_bl_for_each_entry'
- 'hlist_bl_for_each_entry_rcu'
- 'hlist_bl_for_each_entry_safe'
- 'hlist_for_each'
- 'hlist_for_each_entry'
- 'hlist_for_each_entry_continue'
- 'hlist_for_each_entry_continue_rcu'
- 'hlist_for_each_entry_continue_rcu_bh'
- 'hlist_for_each_entry_from'
- 'hlist_for_each_entry_from_rcu'
- 'hlist_for_each_entry_rcu'
- 'hlist_for_each_entry_rcu_bh'
- 'hlist_for_each_entry_rcu_notrace'
- 'hlist_for_each_entry_safe'
- 'hlist_for_each_entry_srcu'
- 'hlist_for_each_safe'
- 'hlist_nulls_for_each_entry'
- 'hlist_nulls_for_each_entry_from'
- 'hlist_nulls_for_each_entry_rcu'
- 'hlist_nulls_for_each_entry_safe'
- 'i3c_bus_for_each_i2cdev'
- 'i3c_bus_for_each_i3cdev'
- 'idr_for_each_entry'
- 'idr_for_each_entry_continue'
- 'idr_for_each_entry_continue_ul'
- 'idr_for_each_entry_ul'
- 'in_dev_for_each_ifa_rcu'
- 'in_dev_for_each_ifa_rtnl'
- 'inet_bind_bucket_for_each'
- 'interval_tree_for_each_span'
- 'intlist__for_each_entry'
- 'intlist__for_each_entry_safe'
- 'kcore_copy__for_each_phdr'
- 'key_for_each'
- 'key_for_each_safe'
- 'klp_for_each_func'
- 'klp_for_each_func_safe'
- 'klp_for_each_func_static'
- 'klp_for_each_object'
- 'klp_for_each_object_safe'
- 'klp_for_each_object_static'
- 'kunit_suite_for_each_test_case'
- 'kvm_for_each_memslot'
- 'kvm_for_each_memslot_in_gfn_range'
- 'kvm_for_each_vcpu'
- 'libbpf_nla_for_each_attr'
- 'list_for_each'
- 'list_for_each_codec'
- 'list_for_each_codec_safe'
- 'list_for_each_continue'
- 'list_for_each_entry'
- 'list_for_each_entry_continue'
- 'list_for_each_entry_continue_rcu'
- 'list_for_each_entry_continue_reverse'
- 'list_for_each_entry_from'
- 'list_for_each_entry_from_rcu'
- 'list_for_each_entry_from_reverse'
- 'list_for_each_entry_lockless'
- 'list_for_each_entry_rcu'
- 'list_for_each_entry_reverse'
- 'list_for_each_entry_safe'
- 'list_for_each_entry_safe_continue'
- 'list_for_each_entry_safe_from'
- 'list_for_each_entry_safe_reverse'
- 'list_for_each_entry_srcu'
- 'list_for_each_from'
- 'list_for_each_prev'
- 'list_for_each_prev_safe'
- 'list_for_each_rcu'
- 'list_for_each_reverse'
- 'list_for_each_safe'
- 'llist_for_each'
- 'llist_for_each_entry'
- 'llist_for_each_entry_safe'
- 'llist_for_each_safe'
- 'lwq_for_each_safe'
- 'map__for_each_symbol'
- 'map__for_each_symbol_by_name'
- 'maps__for_each_entry'
- 'maps__for_each_entry_safe'
- 'mas_for_each'
- 'mci_for_each_dimm'
- 'media_device_for_each_entity'
- 'media_device_for_each_intf'
- 'media_device_for_each_link'
- 'media_device_for_each_pad'
- 'media_entity_for_each_pad'
- 'media_pipeline_for_each_entity'
- 'media_pipeline_for_each_pad'
- 'mlx5_lag_for_each_peer_mdev'
- 'msi_domain_for_each_desc'
- 'msi_for_each_desc'
- 'mt_for_each'
- 'nanddev_io_for_each_page'
- 'netdev_for_each_lower_dev'
- 'netdev_for_each_lower_private'
- 'netdev_for_each_lower_private_rcu'
- 'netdev_for_each_mc_addr'
- 'netdev_for_each_synced_mc_addr'
- 'netdev_for_each_synced_uc_addr'
- 'netdev_for_each_uc_addr'
- 'netdev_for_each_upper_dev_rcu'
- 'netdev_hw_addr_list_for_each'
- 'nft_rule_for_each_expr'
- 'nla_for_each_attr'
- 'nla_for_each_nested'
- 'nlmsg_for_each_attr'
- 'nlmsg_for_each_msg'
- 'nr_neigh_for_each'
- 'nr_neigh_for_each_safe'
- 'nr_node_for_each'
- 'nr_node_for_each_safe'
- 'of_for_each_phandle'
- 'of_property_for_each_string'
- 'of_property_for_each_u32'
- 'pci_bus_for_each_resource'
- 'pci_dev_for_each_resource'
- 'pcl_for_each_chunk'
- 'pcl_for_each_segment'
- 'pcm_for_each_format'
- 'perf_config_items__for_each_entry'
- 'perf_config_sections__for_each_entry'
- 'perf_config_set__for_each_entry'
- 'perf_cpu_map__for_each_cpu'
- 'perf_cpu_map__for_each_idx'
- 'perf_evlist__for_each_entry'
- 'perf_evlist__for_each_entry_reverse'
- 'perf_evlist__for_each_entry_safe'
- 'perf_evlist__for_each_evsel'
- 'perf_evlist__for_each_mmap'
- 'perf_hpp_list__for_each_format'
- 'perf_hpp_list__for_each_format_safe'
- 'perf_hpp_list__for_each_sort_list'
- 'perf_hpp_list__for_each_sort_list_safe'
- 'perf_tool_event__for_each_event'
- 'plist_for_each'
- 'plist_for_each_continue'
- 'plist_for_each_entry'
- 'plist_for_each_entry_continue'
- 'plist_for_each_entry_safe'
- 'plist_for_each_safe'
- 'pnp_for_each_card'
- 'pnp_for_each_dev'
- 'protocol_for_each_card'
- 'protocol_for_each_dev'
- 'queue_for_each_hw_ctx'
- 'radix_tree_for_each_slot'
- 'radix_tree_for_each_tagged'
- 'rb_for_each'
- 'rbtree_postorder_for_each_entry_safe'
- 'rdma_for_each_block'
- 'rdma_for_each_port'
- 'rdma_umem_for_each_dma_block'
- 'resort_rb__for_each_entry'
- 'resource_list_for_each_entry'
- 'resource_list_for_each_entry_safe'
- 'rhl_for_each_entry_rcu'
- 'rhl_for_each_rcu'
- 'rht_for_each'
- 'rht_for_each_entry'
- 'rht_for_each_entry_from'
- 'rht_for_each_entry_rcu'
- 'rht_for_each_entry_rcu_from'
- 'rht_for_each_entry_safe'
- 'rht_for_each_from'
- 'rht_for_each_rcu'
- 'rht_for_each_rcu_from'
- 'rq_for_each_bvec'
- 'rq_for_each_segment'
- 'rq_list_for_each'
- 'rq_list_for_each_safe'
- 'sample_read_group__for_each'
- 'scsi_for_each_prot_sg'
- 'scsi_for_each_sg'
- 'sctp_for_each_hentry'
- 'sctp_skb_for_each'
- 'sec_for_each_insn'
- 'sec_for_each_insn_continue'
- 'sec_for_each_insn_from'
- 'sec_for_each_sym'
- 'shdma_for_each_chan'
- 'shost_for_each_device'
- 'sk_for_each'
- 'sk_for_each_bound'
- 'sk_for_each_bound_bhash2'
- 'sk_for_each_entry_offset_rcu'
- 'sk_for_each_from'
- 'sk_for_each_rcu'
- 'sk_for_each_safe'
- 'sk_nulls_for_each'
- 'sk_nulls_for_each_from'
- 'sk_nulls_for_each_rcu'
- 'snd_array_for_each'
- 'snd_pcm_group_for_each_entry'
- 'snd_soc_dapm_widget_for_each_path'
- 'snd_soc_dapm_widget_for_each_path_safe'
- 'snd_soc_dapm_widget_for_each_sink_path'
- 'snd_soc_dapm_widget_for_each_source_path'
- 'strlist__for_each_entry'
- 'strlist__for_each_entry_safe'
- 'sym_for_each_insn'
- 'sym_for_each_insn_continue_reverse'
- 'symbols__for_each_entry'
- 'tb_property_for_each'
- 'tcf_act_for_each_action'
- 'tcf_exts_for_each_action'
- 'ttm_resource_manager_for_each_res'
- 'twsk_for_each_bound_bhash2'
- 'udp_portaddr_for_each_entry'
- 'udp_portaddr_for_each_entry_rcu'
- 'usb_hub_for_each_child'
- 'v4l2_device_for_each_subdev'
- 'v4l2_m2m_for_each_dst_buf'
- 'v4l2_m2m_for_each_dst_buf_safe'
- 'v4l2_m2m_for_each_src_buf'
- 'v4l2_m2m_for_each_src_buf_safe'
- 'virtio_device_for_each_vq'
- 'while_for_each_ftrace_op'
- 'xa_for_each'
- 'xa_for_each_marked'
- 'xa_for_each_range'
- 'xa_for_each_start'
- 'xas_for_each'
- 'xas_for_each_conflict'
- 'xas_for_each_marked'
- 'xbc_array_for_each_value'
- 'xbc_for_each_key_value'
- 'xbc_node_for_each_array_value'
- 'xbc_node_for_each_child'
- 'xbc_node_for_each_key_value'
- 'xbc_node_for_each_subkey'
- 'zorro_for_each_dev'
IncludeBlocks: Preserve
IncludeCategories:
- Regex: '.*'
Priority: 1
IncludeIsMainRegex: '(Test)?$'
IndentCaseLabels: false
IndentGotoLabels: false
IndentPPDirectives: None
IndentWidth: 8
IndentWrappedFunctionNames: false
JavaScriptQuotes: Leave
JavaScriptWrapImports: true
KeepEmptyLinesAtTheStartOfBlocks: false
MacroBlockBegin: ''
MacroBlockEnd: ''
MaxEmptyLinesToKeep: 1
NamespaceIndentation: None
ObjCBinPackProtocolList: Auto
ObjCBlockIndentWidth: 8
ObjCSpaceAfterProperty: true
ObjCSpaceBeforeProtocolList: true
# Taken from git's rules
PenaltyBreakAssignment: 10
PenaltyBreakBeforeFirstCallParameter: 30
PenaltyBreakComment: 10
PenaltyBreakFirstLessLess: 0
PenaltyBreakString: 10
PenaltyExcessCharacter: 100
PenaltyReturnTypeOnItsOwnLine: 60
PointerAlignment: Right
ReflowComments: false
SortIncludes: false
SortUsingDeclarations: false
SpaceAfterCStyleCast: false
SpaceAfterTemplateKeyword: true
SpaceBeforeAssignmentOperators: true
SpaceBeforeCtorInitializerColon: true
SpaceBeforeInheritanceColon: true
SpaceBeforeParens: ControlStatementsExceptForEachMacros
SpaceBeforeRangeBasedForLoopColon: true
SpaceInEmptyParentheses: false
SpacesBeforeTrailingComments: 1
SpacesInAngles: false
SpacesInContainerLiterals: false
SpacesInCStyleCastParentheses: false
SpacesInParentheses: false
SpacesInSquareBrackets: false
Standard: Cpp03
TabWidth: 8
UseTab: Always
...

24
src/atomic.h
View File

@@ -18,7 +18,6 @@ static inline refcnt refcnt_dec(refcnt *cnt)
return __sync_sub_and_fetch(cnt, 1);
}
/* mutex */
typedef pthread_mutex_t lock;
@@ -45,14 +44,11 @@ static inline void lock_release_via_cleanup(void *l)
lock_release(l);
}
#define LOCK_ACQUIRE(l) \
lock_acquire(l); \
#define LOCK_ACQUIRE(l) \
lock_acquire(l); \
pthread_cleanup_push(lock_release_via_cleanup, l)
#define LOCK_RELEASE() \
pthread_cleanup_pop(1)
#define LOCK_RELEASE() pthread_cleanup_pop(1)
/* read/write lock */
typedef pthread_rwlock_t rwlock;
@@ -85,18 +81,14 @@ static inline void rwlock_release_via_cleanup(void *rw)
rwlock_release(rw);
}
#define RWLOCK_READ_ACQUIRE(rw) \
rwlock_read_acquire(rw); \
#define RWLOCK_READ_ACQUIRE(rw) \
rwlock_read_acquire(rw); \
pthread_cleanup_push(rwlock_release_via_cleanup, rw)
#define RWLOCK_WRITE_ACQUIRE(rw) \
rwlock_write_acquire(rw); \
#define RWLOCK_WRITE_ACQUIRE(rw) \
rwlock_write_acquire(rw); \
pthread_cleanup_push(rwlock_release_via_cleanup, rw)
#define RWLOCK_RELEASE() \
pthread_cleanup_pop(1)
#define RWLOCK_RELEASE() pthread_cleanup_pop(1)
#endif /* _ATOMIC_H_ */

40
src/fileops.c
View File

@@ -12,7 +12,6 @@
#include <print.h>
#include <platform.h>
sftp_session __thread tls_sftp;
/* tls_sftp is used *_wrapped() functions */
@@ -25,7 +24,7 @@ static void sftp_err_to_errno(sftp_session sftp)
{
int sftperr = sftp_get_error(sftp);
switch (sftperr){
switch (sftperr) {
case SSH_FX_OK:
case SSH_FX_EOF:
errno = 0;
@@ -67,7 +66,6 @@ static void sftp_err_to_errno(sftp_session sftp)
}
}
MDIR *mscp_opendir(const char *path, sftp_session sftp)
{
MDIR *md;
@@ -112,7 +110,6 @@ void mscp_closedir(MDIR *md)
free(md);
}
struct dirent __thread tls_dirent;
/* tls_dirent contains dirent converted from sftp_attributes returned
* from sftp_readdir(). This trick is derived from openssh's
@@ -161,26 +158,25 @@ int mscp_mkdir(const char *path, mode_t mode, sftp_session sftp)
return ret;
}
static void sftp_attr_to_stat(sftp_attributes attr, struct stat *st)
{
memset(st, 0, sizeof(*st));
st->st_size = attr->size;
st->st_uid = attr->uid;
st->st_gid = attr->gid;
st->st_mode = attr->permissions;
memset(st, 0, sizeof(*st));
st->st_size = attr->size;
st->st_uid = attr->uid;
st->st_gid = attr->gid;
st->st_mode = attr->permissions;
#if defined(__APPLE__)
#define st_atim st_atimespec
#define st_mtim st_mtimespec
#define st_ctim st_ctimespec
#define st_atim st_atimespec
#define st_mtim st_mtimespec
#define st_ctim st_ctimespec
#endif
st->st_atim.tv_sec = attr->atime;
st->st_atim.tv_nsec = attr->atime_nseconds;
st->st_mtim.tv_sec = attr->mtime;
st->st_mtim.tv_nsec = attr->mtime_nseconds;
st->st_ctim.tv_sec = attr->createtime;
st->st_ctim.tv_nsec = attr->createtime_nseconds;
st->st_atim.tv_sec = attr->atime;
st->st_atim.tv_nsec = attr->atime_nseconds;
st->st_mtim.tv_sec = attr->mtime;
st->st_mtim.tv_nsec = attr->mtime_nseconds;
st->st_ctim.tv_sec = attr->createtime;
st->st_ctim.tv_nsec = attr->createtime_nseconds;
switch (attr->type) {
case SSH_FILEXFER_TYPE_REGULAR:
@@ -203,7 +199,6 @@ static void sftp_attr_to_stat(sftp_attributes attr, struct stat *st)
}
}
int mscp_stat(const char *path, struct stat *st, sftp_session sftp)
{
sftp_attributes attr;
@@ -256,7 +251,6 @@ int mscp_lstat_wrapped(const char *path, struct stat *st)
return mscp_lstat(path, st, tls_sftp);
}
mf *mscp_open(const char *path, int flags, mode_t mode, sftp_session sftp)
{
mf *f;
@@ -316,7 +310,7 @@ int mscp_setstat(const char *path, struct stat *st, bool preserve_ts, sftp_sessi
memset(&attr, 0, sizeof(attr));
attr.permissions = st->st_mode;
attr.size = st->st_size;
attr.flags = (SSH_FILEXFER_ATTR_PERMISSIONS|SSH_FILEXFER_ATTR_SIZE);
attr.flags = (SSH_FILEXFER_ATTR_PERMISSIONS | SSH_FILEXFER_ATTR_SIZE);
if (preserve_ts) {
attr.atime = st->st_atim.tv_sec;
attr.atime_nseconds = st->st_atim.tv_nsec;
@@ -357,7 +351,7 @@ int mscp_glob(const char *pattern, int flags, glob_t *pglob, sftp_session sftp)
set_tls_sftp_session(sftp);
#if defined(__APPLE__) || defined(__FreeBSD__)
pglob->gl_opendir = (void *(*)(const char *))mscp_opendir_wrapped;
pglob->gl_readdir = (struct dirent *(*)(void *))mscp_readdir;
pglob->gl_readdir = (struct dirent * (*)(void *)) mscp_readdir;
pglob->gl_closedir = (void (*)(void *))mscp_closedir;
pglob->gl_lstat = mscp_lstat_wrapped;
pglob->gl_stat = mscp_stat_wrapped;

2
src/fileops.h
View File

@@ -19,7 +19,6 @@ struct mdir_struct {
};
typedef struct mdir_struct MDIR;
MDIR *mscp_opendir(const char *path, sftp_session sftp);
MDIR *mscp_opendir_wrapped(const char *path);
void mscp_closedir(MDIR *md);
@@ -34,7 +33,6 @@ int mscp_stat_wrapped(const char *path, struct stat *st);
int mscp_lstat(const char *path, struct stat *st, sftp_session sftp);
int mscp_lstat_wrapped(const char *path, struct stat *st);
/* file operations */
struct mf_struct {

196
src/main.c
View File

@@ -16,10 +16,10 @@
#include <strerrno.h>
#include <print.h>
#include "config.h"
void usage(bool print_help) {
void usage(bool print_help)
{
printf("mscp " MSCP_BUILD_VERSION ": copy files over multiple ssh connections\n"
"\n"
"Usage: mscp [vqDpHdNh] [-n nr_conns] [-m coremask]\n"
@@ -184,8 +184,8 @@ struct target *validate_targets(char **arg, int len)
/* split remote:path into remote and path */
for (n = 0; n < len; n++) {
t[n].copy = split_user_host_path(arg[n], &t[n].user,
&t[n].host, &t[n].path);
t[n].copy =
split_user_host_path(arg[n], &t[n].user, &t[n].host, &t[n].path);
if (!t[n].copy)
goto free_target_out;
}
@@ -217,7 +217,8 @@ invalid_remotes:
free_split_out:
for (n = 0; n < len; n++)
if (t[n].copy) free(t[n].copy);
if (t[n].copy)
free(t[n].copy);
free_target_out:
free(t);
@@ -238,7 +239,7 @@ void *print_stat_thread(void *arg);
void print_cli(const char *fmt, ...)
{
va_list va;
va_list va;
va_start(va, fmt);
vfprintf(stdout, fmt, va);
fflush(stdout);
@@ -260,13 +261,13 @@ int main(int argc, char **argv)
memset(&o, 0, sizeof(o));
o.severity = MSCP_SEVERITY_WARN;
while ((ch = getopt(argc, argv,
"n:m:u:I:s:S:a:b:vqDrl:P:i:F:c:M:C:g:pHdNh")) != -1) {
while ((ch = getopt(argc, argv, "n:m:u:I:s:S:a:b:vqDrl:P:i:F:c:M:C:g:pHdNh")) !=
-1) {
switch (ch) {
case 'n':
o.nr_threads = atoi(optarg);
if (o.nr_threads < 1) {
pr_err( "invalid number of connections: %s", optarg);
pr_err("invalid number of connections: %s", optarg);
return 1;
}
break;
@@ -351,7 +352,6 @@ int main(int argc, char **argv)
s.password = getenv(ENV_SSH_AUTH_PASSWORD);
s.passphrase = getenv(ENV_SSH_AUTH_PASSPHRASE);
if (argc - optind < 2) {
/* mscp needs at lease 2 (src and target) argument */
usage(false);
@@ -389,7 +389,7 @@ int main(int argc, char **argv)
pr_err("mscp_add_src_path: %s", priv_get_err());
return -1;
}
}
}
if (mscp_set_dst_path(m, t[i - 1].path) < 0) {
pr_err("mscp_set_dst_path: %s", priv_get_err());
@@ -434,52 +434,49 @@ out:
return ret;
}
/* progress bar-related functions */
double calculate_timedelta(struct timeval *b, struct timeval *a)
{
double sec, usec;
double sec, usec;
if (a->tv_usec < b->tv_usec) {
a->tv_usec += 1000000;
a->tv_sec--;
}
if (a->tv_usec < b->tv_usec) {
a->tv_usec += 1000000;
a->tv_sec--;
}
sec = a->tv_sec - b->tv_sec;
usec = a->tv_usec - b->tv_usec;
sec += usec / 1000000;
sec = a->tv_sec - b->tv_sec;
usec = a->tv_usec - b->tv_usec;
sec += usec / 1000000;
return sec;
return sec;
}
double calculate_bps(size_t diff, struct timeval *b, struct timeval *a)
{
return (double)diff / calculate_timedelta(b, a);
return (double)diff / calculate_timedelta(b, a);
}
char *calculate_eta(size_t remain, size_t diff, struct timeval *b, struct timeval *a,
bool final)
{
static char buf[16];
static char buf[16];
#define bps_window_size 16
static double bps_window[bps_window_size];
static size_t sum, idx, count;
double elapsed = calculate_timedelta(b, a);
double elapsed = calculate_timedelta(b, a);
double bps = diff / elapsed;
double avg, eta;
/* early return when diff == 0 (stalled) or final output */
if (diff == 0) {
snprintf(buf, sizeof(buf), "--:-- ETA");
if (diff == 0) {
snprintf(buf, sizeof(buf), "--:-- ETA");
return buf;
}
if (final) {
snprintf(buf, sizeof(buf), "%02d:%02d ",
(int)(floor(elapsed / 60)), (int)round(elapsed) % 60);
snprintf(buf, sizeof(buf), "%02d:%02d ", (int)(floor(elapsed / 60)),
(int)round(elapsed) % 60);
return buf;
}
@@ -493,96 +490,97 @@ char *calculate_eta(size_t remain, size_t diff, struct timeval *b, struct timeva
/* calcuate ETA from avg of recent bps values */
avg = sum / min(count, bps_window_size);
eta = remain / avg;
snprintf(buf, sizeof(buf), "%02d:%02d ETA",
(int)floor(eta / 60), (int)round(eta) % 60);
snprintf(buf, sizeof(buf), "%02d:%02d ETA", (int)floor(eta / 60),
(int)round(eta) % 60);
return buf;
return buf;
}
void print_progress_bar(double percent, char *suffix)
{
int n, thresh, bar_width;
struct winsize ws;
char buf[128];
int n, thresh, bar_width;
struct winsize ws;
char buf[128];
/*
/*
* [=======> ] XX% SUFFIX
*/
buf[0] = '\0';
buf[0] = '\0';
if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws) < 0)
return; /* XXX */
bar_width = min(sizeof(buf), ws.ws_col) - strlen(suffix) - 7;
if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws) < 0)
return; /* XXX */
bar_width = min(sizeof(buf), ws.ws_col) - strlen(suffix) - 7;
memset(buf, 0, sizeof(buf));
if (bar_width > 8) {
thresh = floor(bar_width * (percent / 100)) - 1;
memset(buf, 0, sizeof(buf));
if (bar_width > 8) {
thresh = floor(bar_width * (percent / 100)) - 1;
for (n = 1; n < bar_width - 1; n++) {
if (n <= thresh)
buf[n] = '=';
else
buf[n] = ' ';
}
buf[thresh] = '>';
buf[0] = '[';
buf[bar_width - 1] = ']';
snprintf(buf + bar_width, sizeof(buf) - bar_width,
" %3d%% ", (int)floor(percent));
}
for (n = 1; n < bar_width - 1; n++) {
if (n <= thresh)
buf[n] = '=';
else
buf[n] = ' ';
}
buf[thresh] = '>';
buf[0] = '[';
buf[bar_width - 1] = ']';
snprintf(buf + bar_width, sizeof(buf) - bar_width, " %3d%% ",
(int)floor(percent));
}
print_cli("\r\033[K" "%s%s", buf, suffix);
print_cli("\r\033[K"
"%s%s",
buf, suffix);
}
void print_progress(struct timeval *b, struct timeval *a,
size_t total, size_t last, size_t done, bool final)
void print_progress(struct timeval *b, struct timeval *a, size_t total, size_t last,
size_t done, bool final)
{
char *bps_units[] = { "B/s ", "KB/s", "MB/s", "GB/s" };
char *byte_units[] = { "B ", "KB", "MB", "GB", "TB", "PB" };
char suffix[128];
int bps_u, byte_tu, byte_du;
double total_round, done_round;
int percent;
double bps;
char *bps_units[] = { "B/s ", "KB/s", "MB/s", "GB/s" };
char *byte_units[] = { "B ", "KB", "MB", "GB", "TB", "PB" };
char suffix[128];
int bps_u, byte_tu, byte_du;
double total_round, done_round;
int percent;
double bps;
#define array_size(a) (sizeof(a) / sizeof(a[0]))
if (total <= 0) {
print_cli("\r\033[K" "total 0 byte transferred");
return; /* copy 0-byte file(s) */
}
if (total <= 0) {
print_cli("\r\033[K"
"total 0 byte transferred");
return; /* copy 0-byte file(s) */
}
total_round = total;
for (byte_tu = 0; total_round > 1000 && byte_tu < array_size(byte_units) - 1;
byte_tu++)
total_round /= 1024;
total_round = total;
for (byte_tu = 0; total_round > 1000 && byte_tu < array_size(byte_units) - 1;
byte_tu++)
total_round /= 1024;
bps = calculate_bps(done - last, b, a);
for (bps_u = 0; bps > 1000 && bps_u < array_size(bps_units); bps_u++)
bps /= 1000;
bps = calculate_bps(done - last, b, a);
for (bps_u = 0; bps > 1000 && bps_u < array_size(bps_units); bps_u++)
bps /= 1000;
percent = floor(((double)(done) / (double)total) * 100);
percent = floor(((double)(done) / (double)total) * 100);
done_round = done;
for (byte_du = 0; done_round > 1024 && byte_du < array_size(byte_units) - 1;
byte_du++)
done_round /= 1024;
done_round = done;
for (byte_du = 0; done_round > 1024 && byte_du < array_size(byte_units) - 1;
byte_du++)
done_round /= 1024;
snprintf(suffix, sizeof(suffix), "%4.1lf%s/%.1lf%s %6.1f%s %s",
done_round, byte_units[byte_du], total_round, byte_units[byte_tu],
bps, bps_units[bps_u],
calculate_eta(total - done, done - last, b, a, final));
snprintf(suffix, sizeof(suffix), "%4.1lf%s/%.1lf%s %6.1f%s %s", done_round,
byte_units[byte_du], total_round, byte_units[byte_tu], bps,
bps_units[bps_u], calculate_eta(total - done, done - last, b, a, final));
print_progress_bar(percent, suffix);
print_progress_bar(percent, suffix);
}
struct xfer_stat {
struct timeval start, before, after;
size_t total;
size_t last;
size_t done;
struct timeval start, before, after;
size_t total;
size_t last;
size_t done;
};
struct xfer_stat x;
@@ -597,8 +595,8 @@ void print_stat(bool final)
mscp_get_stats(m, &s);
x.total = s.total;
x.done = s.done;
print_progress(!final ? &x.before : &x.start, &x.after,
x.total, !final ? x.last : 0, x.done, final);
print_progress(!final ? &x.before : &x.start, &x.after, x.total,
!final ? x.last : 0, x.done, final);
x.before = x.after;
x.last = x.done;
}
@@ -616,12 +614,12 @@ void *print_stat_thread(void *arg)
char buf[8192];
memset(&x, 0, sizeof(x));
gettimeofday(&x.start, NULL);
x.before = x.start;
gettimeofday(&x.start, NULL);
x.before = x.start;
pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL);
pthread_cleanup_push(print_stat_thread_cleanup, NULL);
pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL);
pthread_cleanup_push(print_stat_thread_cleanup, NULL);
while (true) {
print_stat(false);

287
src/mscp.c
View File

@@ -6,7 +6,7 @@
#include <semaphore.h>
#include <sys/time.h>
#include <list.h>
#include <list.h>
#include <minmax.h>
#include <ssh.h>
#include <path.h>
@@ -20,55 +20,54 @@
#include <openbsd-compat/openbsd-compat.h>
struct mscp {
char *remote; /* remote host (and uername) */
int direction; /* copy direction */
struct mscp_opts *opts;
struct mscp_ssh_opts *ssh_opts;
char *remote; /* remote host (and uername) */
int direction; /* copy direction */
struct mscp_opts *opts;
struct mscp_ssh_opts *ssh_opts;
int *cores; /* usable cpu cores by COREMASK */
int nr_cores; /* length of array of cores */
int *cores; /* usable cpu cores by COREMASK */
int nr_cores; /* length of array of cores */
sem_t *sem; /* semaphore for concurrent
sem_t *sem; /* semaphore for concurrent
* connecting ssh sessions */
sftp_session first; /* first sftp session */
sftp_session first; /* first sftp session */
char dst_path[PATH_MAX];
struct list_head src_list;
struct list_head path_list;
struct chunk_pool cp;
char dst_path[PATH_MAX];
struct list_head src_list;
struct list_head path_list;
struct chunk_pool cp;
pthread_t tid_scan; /* tid for scan thread */
int ret_scan; /* return code from scan thread */
pthread_t tid_scan; /* tid for scan thread */
int ret_scan; /* return code from scan thread */
size_t total_bytes; /* total bytes to be transferred */
size_t total_bytes; /* total bytes to be transferred */
struct list_head thread_list;
rwlock thread_rwlock;
struct list_head thread_list;
rwlock thread_rwlock;
};
struct mscp_thread {
struct list_head list; /* mscp->thread_list */
struct list_head list; /* mscp->thread_list */
struct mscp *m;
int id;
sftp_session sftp;
pthread_t tid;
int cpu;
size_t done;
bool finished;
int ret;
struct mscp *m;
int id;
sftp_session sftp;
pthread_t tid;
int cpu;
size_t done;
bool finished;
int ret;
};
struct src {
struct list_head list; /* mscp->src_list */
struct list_head list; /* mscp->src_list */
char *path;
};
#define DEFAULT_MIN_CHUNK_SZ (64 << 20) /* 64MB */
#define DEFAULT_NR_AHEAD 32
#define DEFAULT_BUF_SZ 16384
#define DEFAULT_MIN_CHUNK_SZ (64 << 20) /* 64MB */
#define DEFAULT_NR_AHEAD 32
#define DEFAULT_BUF_SZ 16384
/* XXX: we use 16384 byte buffer pointed by
* https://api.libssh.org/stable/libssh_tutor_sftp.html. The larget
* read length from sftp_async_read is 65536 byte. Read sizes larger
@@ -76,75 +75,74 @@ struct src {
* sftp_async_read returns 0.
*/
#define DEFAULT_MAX_STARTUPS 8
#define DEFAULT_MAX_STARTUPS 8
#define non_null_string(s) (s[0] != '\0')
static int expand_coremask(const char *coremask, int **cores, int *nr_cores)
{
int n, *core_list, core_list_len = 0, nr_usable, nr_all;
char c[2] = { 'x', '\0' };
const char *_coremask;
long v, needle;
int ncores = nr_cpus();
int n, *core_list, core_list_len = 0, nr_usable, nr_all;
char c[2] = { 'x', '\0' };
const char *_coremask;
long v, needle;
int ncores = nr_cpus();
/*
/*
* This function returns array of usable cores in `cores` and
* returns the number of usable cores (array length) through
* nr_cores.
*/
if (strncmp(coremask, "0x", 2) == 0)
_coremask = coremask + 2;
else
_coremask = coremask;
if (strncmp(coremask, "0x", 2) == 0)
_coremask = coremask + 2;
else
_coremask = coremask;
core_list = realloc(NULL, sizeof(int) * 64);
if (!core_list) {
priv_set_errv("failed to realloc: %s", strerrno());
return -1;
}
core_list = realloc(NULL, sizeof(int) * 64);
if (!core_list) {
priv_set_errv("failed to realloc: %s", strerrno());
return -1;
}
nr_usable = 0;
nr_all = 0;
for (n = strlen(_coremask) - 1; n >=0; n--) {
c[0] = _coremask[n];
v = strtol(c, NULL, 16);
if (v == LONG_MIN || v == LONG_MAX) {
priv_set_errv("invalid coremask: %s", coremask);
return -1;
}
nr_usable = 0;
nr_all = 0;
for (n = strlen(_coremask) - 1; n >= 0; n--) {
c[0] = _coremask[n];
v = strtol(c, NULL, 16);
if (v == LONG_MIN || v == LONG_MAX) {
priv_set_errv("invalid coremask: %s", coremask);
return -1;
}
for (needle = 0x01; needle < 0x10; needle <<= 1) {
nr_all++;
if (nr_all > ncores)
break; /* too long coremask */
if (v & needle) {
nr_usable++;
core_list = realloc(core_list, sizeof(int) * nr_usable);
if (!core_list) {
priv_set_errv("realloc: %s", strerrno());
return -1;
}
core_list[nr_usable - 1] = nr_all - 1;
}
}
}
for (needle = 0x01; needle < 0x10; needle <<= 1) {
nr_all++;
if (nr_all > ncores)
break; /* too long coremask */
if (v & needle) {
nr_usable++;
core_list = realloc(core_list, sizeof(int) * nr_usable);
if (!core_list) {
priv_set_errv("realloc: %s", strerrno());
return -1;
}
core_list[nr_usable - 1] = nr_all - 1;
}
}
}
if (nr_usable < 1) {
priv_set_errv("invalid core mask: %s", coremask);
return -1;
}
if (nr_usable < 1) {
priv_set_errv("invalid core mask: %s", coremask);
return -1;
}
*cores = core_list;
*nr_cores = nr_usable;
return 0;
*cores = core_list;
*nr_cores = nr_usable;
return 0;
}
static int default_nr_threads()
{
return (int)(floor(log(nr_cpus()) * 2) + 1);
return (int)(floor(log(nr_cpus()) * 2) + 1);
}
static int validate_and_set_defaut_params(struct mscp_opts *o)
@@ -167,8 +165,8 @@ static int validate_and_set_defaut_params(struct mscp_opts *o)
if (o->min_chunk_sz < getpagesize() ||
o->min_chunk_sz % getpagesize() != 0) {
priv_set_errv("min chunk size must be "
"larget than and multiple of page size %d: %lu",
getpagesize(), o->min_chunk_sz);
"larget than and multiple of page size %d: %lu",
getpagesize(), o->min_chunk_sz);
return -1;
}
}
@@ -177,13 +175,13 @@ static int validate_and_set_defaut_params(struct mscp_opts *o)
if (o->max_chunk_sz < getpagesize() ||
o->max_chunk_sz % getpagesize() != 0) {
priv_set_errv("min chunk size must be larget than and "
"multiple of page size %d: %lu",
getpagesize(), o->max_chunk_sz);
"multiple of page size %d: %lu",
getpagesize(), o->max_chunk_sz);
}
if (o->min_chunk_sz > o->max_chunk_sz) {
priv_set_errv("smaller max chunk size than "
"min chunk size: %lu < %lu",
o->max_chunk_sz, o->min_chunk_sz);
"min chunk size: %lu < %lu",
o->max_chunk_sz, o->min_chunk_sz);
return -1;
}
}
@@ -210,8 +208,8 @@ static int validate_and_set_defaut_params(struct mscp_opts *o)
return 0;
}
struct mscp *mscp_init(const char *remote_host, int direction,
struct mscp_opts *o, struct mscp_ssh_opts *s)
struct mscp *mscp_init(const char *remote_host, int direction, struct mscp_opts *o,
struct mscp_ssh_opts *s)
{
struct mscp *m;
int n;
@@ -221,8 +219,7 @@ struct mscp *mscp_init(const char *remote_host, int direction,
return NULL;
}
if (!(direction == MSCP_DIRECTION_L2R ||
direction == MSCP_DIRECTION_R2L)) {
if (!(direction == MSCP_DIRECTION_L2R || direction == MSCP_DIRECTION_R2L)) {
priv_set_errv("invalid copy direction: %d", direction);
return NULL;
}
@@ -329,16 +326,16 @@ int mscp_set_dst_path(struct mscp *m, const char *dst_path)
static int get_page_mask(void)
{
long page_sz = sysconf(_SC_PAGESIZE);
size_t page_mask = 0;
int n;
long page_sz = sysconf(_SC_PAGESIZE);
size_t page_mask = 0;
int n;
for (n = 0; page_sz > 0; page_sz >>= 1, n++) {
page_mask <<= 1;
page_mask |= 1;
}
for (n = 0; page_sz > 0; page_sz >>= 1, n++) {
page_mask <<= 1;
page_mask |= 1;
}
return page_mask >> 1;
return page_mask >> 1;
}
static void mscp_stop_copy_thread(struct mscp *m)
@@ -376,7 +373,7 @@ void *mscp_scan_thread(void *arg)
struct stat ss, ds;
glob_t pglob;
int n;
m->ret_scan = 0;
switch (m->direction) {
@@ -487,8 +484,6 @@ int mscp_scan_join(struct mscp *m)
return 0;
}
static void *mscp_copy_thread(void *arg);
static struct mscp_thread *mscp_copy_thread_spawn(struct mscp *m, int id)
@@ -497,7 +492,7 @@ static struct mscp_thread *mscp_copy_thread_spawn(struct mscp *m, int id)
int ret;
t = malloc(sizeof(*t));
if (!t){
if (!t) {
priv_set_errv("malloc: %s,", strerrno());
return NULL;
}
@@ -506,7 +501,7 @@ static struct mscp_thread *mscp_copy_thread_spawn(struct mscp *m, int id)
t->m = m;
t->id = id;
if (m->cores == NULL)
t->cpu = -1; /* not pinned to cpu */
t->cpu = -1; /* not pinned to cpu */
else
t->cpu = m->cores[id % m->nr_cores];
@@ -520,7 +515,6 @@ static struct mscp_thread *mscp_copy_thread_spawn(struct mscp *m, int id)
return t;
}
int mscp_start(struct mscp *m)
{
struct mscp_thread *t;
@@ -528,7 +522,8 @@ int mscp_start(struct mscp *m)
if ((n = chunk_pool_size(&m->cp)) < m->opts->nr_threads) {
pr_notice("we have only %d chunk(s). "
"set number of connections to %d", n, n);
"set number of connections to %d",
n, n);
m->opts->nr_threads = n;
}
@@ -541,7 +536,7 @@ int mscp_start(struct mscp *m)
RWLOCK_WRITE_ACQUIRE(&m->thread_rwlock);
list_add_tail(&t->list, &m->thread_list);
RWLOCK_RELEASE();
}
}
return n;
}
@@ -556,7 +551,7 @@ int mscp_join(struct mscp *m)
/* waiting for scan thread joins... */
ret = mscp_scan_join(m);
/* waiting for copy threads join... */
/* waiting for copy threads join... */
RWLOCK_READ_ACQUIRE(&m->thread_rwlock);
list_for_each_entry(t, &m->thread_list, list) {
pthread_join(t->tid, NULL);
@@ -570,8 +565,8 @@ int mscp_join(struct mscp *m)
}
RWLOCK_RELEASE();
if (m->first) {
ssh_sftp_close(m->first);
if (m->first) {
ssh_sftp_close(m->first);
m->first = NULL;
}
@@ -583,8 +578,8 @@ int mscp_join(struct mscp *m)
}
}
pr_notice("%lu/%lu bytes copied for %lu/%lu files",
done, m->total_bytes, nr_copied, nr_tobe_copied);
pr_notice("%lu/%lu bytes copied for %lu/%lu files", done, m->total_bytes,
nr_copied, nr_tobe_copied);
return ret;
}
@@ -608,8 +603,8 @@ static void wait_for_interval(int interval)
static void mscp_copy_thread_cleanup(void *arg)
{
struct mscp_thread *t = arg;
t->finished = true;
struct mscp_thread *t = arg;
t->finished = true;
}
void *mscp_copy_thread(void *arg)
@@ -620,12 +615,12 @@ void *mscp_copy_thread(void *arg)
struct chunk *c;
bool nomore;
pthread_cleanup_push(mscp_copy_thread_cleanup, t);
pthread_cleanup_push(mscp_copy_thread_cleanup, t);
if (t->cpu > -1) {
if (set_thread_affinity(pthread_self(), t->cpu) < 0)
if (t->cpu > -1) {
if (set_thread_affinity(pthread_self(), t->cpu) < 0)
goto err_out;
}
}
if (sem_wait(m->sem) < 0) {
pr_err("sem_wait: %s", strerrno());
@@ -654,42 +649,42 @@ void *mscp_copy_thread(void *arg)
goto err_out;
}
switch (m->direction) {
case MSCP_DIRECTION_L2R:
src_sftp = NULL;
dst_sftp = t->sftp;
break;
case MSCP_DIRECTION_R2L:
src_sftp = t->sftp;
dst_sftp = NULL;
break;
default:
return NULL; /* not reached */
}
switch (m->direction) {
case MSCP_DIRECTION_L2R:
src_sftp = NULL;
dst_sftp = t->sftp;
break;
case MSCP_DIRECTION_R2L:
src_sftp = t->sftp;
dst_sftp = NULL;
break;
default:
return NULL; /* not reached */
}
while (1) {
c = chunk_pool_pop(&m->cp);
while (1) {
c = chunk_pool_pop(&m->cp);
if (c == CHUNK_POP_WAIT) {
usleep(100); /* XXX: hard code */
continue;
}
if (!c)
break; /* no more chunks */
if (!c)
break; /* no more chunks */
if ((t->ret = copy_chunk(c, src_sftp, dst_sftp,
m->opts->nr_ahead, m->opts->buf_sz,
m->opts->preserve_ts, &t->done)) < 0)
if ((t->ret = copy_chunk(c, src_sftp, dst_sftp, m->opts->nr_ahead,
m->opts->buf_sz, m->opts->preserve_ts,
&t->done)) < 0)
break;
}
}
pthread_cleanup_pop(1);
pthread_cleanup_pop(1);
if (t->ret < 0)
pr_err("thread:%d copy failed: %s 0x%010lx-0x%010lx",
t->id, c->p->path, c->off, c->off + c->len);
if (t->ret < 0)
pr_err("thread:%d copy failed: %s 0x%010lx-0x%010lx", t->id, c->p->path,
c->off, c->off + c->len);
return NULL;
return NULL;
err_out:
t->finished = true;
@@ -701,7 +696,6 @@ out:
return NULL;
}
/* cleanup-related functions */
static void list_free_src(struct list_head *list)
@@ -728,8 +722,8 @@ static void list_free_thread(struct list_head *list)
void mscp_cleanup(struct mscp *m)
{
if (m->first) {
ssh_sftp_close(m->first);
if (m->first) {
ssh_sftp_close(m->first);
m->first = NULL;
}
@@ -778,4 +772,3 @@ void mscp_get_stats(struct mscp *m, struct mscp_stats *s)
s->finished = nr_threads > 0 ? (nr_finished == nr_threads) : false;
}

430
src/path.c
View File

@@ -16,8 +16,8 @@
#include <print.h>
/* chunk pool operations */
#define CHUNK_POOL_STATE_FILLING 0
#define CHUNK_POOL_STATE_FILLED 1
#define CHUNK_POOL_STATE_FILLING 0
#define CHUNK_POOL_STATE_FILLED 1
void chunk_pool_init(struct chunk_pool *cp)
{
@@ -83,9 +83,9 @@ struct chunk *chunk_pool_pop(struct chunk_pool *cp)
static void chunk_free(struct list_head *list)
{
struct chunk *c;
c = list_entry(list, typeof(*c), list);
free(c);
struct chunk *c;
c = list_entry(list, typeof(*c), list);
free(c);
}
void chunk_pool_release(struct chunk_pool *cp)
@@ -96,17 +96,17 @@ void chunk_pool_release(struct chunk_pool *cp)
/* paths of copy source resoltion */
static char *resolve_dst_path(const char *src_file_path, struct path_resolve_args *a)
{
char copy[PATH_MAX + 1], dst_file_path[PATH_MAX + 1];
char *prefix;
int offset;
char copy[PATH_MAX + 1], dst_file_path[PATH_MAX + 1];
char *prefix;
int offset;
int ret;
strncpy(copy, a->src_path, PATH_MAX);
prefix = dirname(copy);
if (!prefix) {
priv_set_errv("dirname: %s", strerrno());
return NULL;
}
strncpy(copy, a->src_path, PATH_MAX);
prefix = dirname(copy);
if (!prefix) {
priv_set_errv("dirname: %s", strerrno());
return NULL;
}
offset = strlen(prefix) + 1;
if (strlen(prefix) == 1) { /* corner cases */
@@ -120,93 +120,93 @@ static char *resolve_dst_path(const char *src_file_path, struct path_resolve_arg
}
}
if (!a->src_path_is_dir && !a->dst_path_is_dir) {
/* src path is file. dst path is (1) file, or (2) does not exist.
if (!a->src_path_is_dir && !a->dst_path_is_dir) {
/* src path is file. dst path is (1) file, or (2) does not exist.
* In the second case, we need to put src under the dst.
*/
if (a->dst_path_should_dir)
ret = snprintf(dst_file_path, PATH_MAX, "%s/%s",
a->dst_path, a->src_path + offset);
else
ret = snprintf(dst_file_path, PATH_MAX, "%s", a->dst_path);
}
if (a->dst_path_should_dir)
ret = snprintf(dst_file_path, PATH_MAX, "%s/%s", a->dst_path,
a->src_path + offset);
else
ret = snprintf(dst_file_path, PATH_MAX, "%s", a->dst_path);
}
/* src is file, and dst is dir */
if (!a->src_path_is_dir && a->dst_path_is_dir)
ret = snprintf(dst_file_path, PATH_MAX, "%s/%s",
a->dst_path, a->src_path + offset);
/* src is file, and dst is dir */
if (!a->src_path_is_dir && a->dst_path_is_dir)
ret = snprintf(dst_file_path, PATH_MAX, "%s/%s", a->dst_path,
a->src_path + offset);
/* both are directory */
if (a->src_path_is_dir && a->dst_path_is_dir)
ret = snprintf(dst_file_path, PATH_MAX, "%s/%s",
a->dst_path, src_file_path + offset);
/* both are directory */
if (a->src_path_is_dir && a->dst_path_is_dir)
ret = snprintf(dst_file_path, PATH_MAX, "%s/%s", a->dst_path,
src_file_path + offset);
/* dst path does not exist. change dir name to dst_path */
if (a->src_path_is_dir && !a->dst_path_is_dir)
ret = snprintf(dst_file_path, PATH_MAX, "%s/%s",
a->dst_path, src_file_path + strlen(a->src_path) + 1);
/* dst path does not exist. change dir name to dst_path */
if (a->src_path_is_dir && !a->dst_path_is_dir)
ret = snprintf(dst_file_path, PATH_MAX, "%s/%s", a->dst_path,
src_file_path + strlen(a->src_path) + 1);
if (ret >= PATH_MAX) {
pr_warn("Too long path: %s", dst_file_path);
return NULL;
}
pr_debug("file: %s -> %s", src_file_path, dst_file_path);
pr_debug("file: %s -> %s", src_file_path, dst_file_path);
return strndup(dst_file_path, PATH_MAX);
return strndup(dst_file_path, PATH_MAX);
}
/* chunk preparation */
static struct chunk *alloc_chunk(struct path *p)
{
struct chunk *c;
struct chunk *c;
if (!(c = malloc(sizeof(*c)))) {
priv_set_errv("malloc %s", strerrno());
return NULL;
}
memset(c, 0, sizeof(*c));
if (!(c = malloc(sizeof(*c)))) {
priv_set_errv("malloc %s", strerrno());
return NULL;
}
memset(c, 0, sizeof(*c));
c->p = p;
c->off = 0;
c->len = 0;
refcnt_inc(&p->refcnt);
return c;
c->p = p;
c->off = 0;
c->len = 0;
refcnt_inc(&p->refcnt);
return c;
}
static int resolve_chunk(struct path *p, struct path_resolve_args *a)
{
struct chunk *c;
size_t chunk_sz;
size_t size;
struct chunk *c;
size_t chunk_sz;
size_t size;
if (p->size <= a->min_chunk_sz)
chunk_sz = p->size;
else if (a->max_chunk_sz)
chunk_sz = a->max_chunk_sz;
else {
chunk_sz = (p->size - (p->size % a->nr_conn)) / a->nr_conn;
chunk_sz &= ~a->chunk_align; /* align with page_sz */
if (chunk_sz <= a->min_chunk_sz)
chunk_sz = a->min_chunk_sz;
}
if (p->size <= a->min_chunk_sz)
chunk_sz = p->size;
else if (a->max_chunk_sz)
chunk_sz = a->max_chunk_sz;
else {
chunk_sz = (p->size - (p->size % a->nr_conn)) / a->nr_conn;
chunk_sz &= ~a->chunk_align; /* align with page_sz */
if (chunk_sz <= a->min_chunk_sz)
chunk_sz = a->min_chunk_sz;
}
/* for (size = f->size; size > 0;) does not create a file
/* for (size = f->size; size > 0;) does not create a file
* (chunk) when file size is 0. This do {} while (size > 0)
* creates just open/close a 0-byte file.
*/
size = p->size;
do {
c = alloc_chunk(p);
if (!c)
return -1;
c->off = p->size - size;
c->len = size < chunk_sz ? size : chunk_sz;
size -= c->len;
chunk_pool_add(a->cp, c);
} while (size > 0);
size = p->size;
do {
c = alloc_chunk(p);
if (!c)
return -1;
c->off = p->size - size;
c->len = size < chunk_sz ? size : chunk_sz;
size -= c->len;
chunk_pool_add(a->cp, c);
} while (size > 0);
return 0;
return 0;
}
void free_path(struct path *p)
@@ -257,13 +257,13 @@ free_out:
}
static bool check_path_should_skip(const char *path)
{
int len = strlen(path);
if ((len == 1 && strncmp(path, ".", 1) == 0) ||
(len == 2 && strncmp(path, "..", 2) == 0)) {
return true;
}
return false;
{
int len = strlen(path);
if ((len == 1 && strncmp(path, ".", 1) == 0) ||
(len == 2 && strncmp(path, "..", 2) == 0)) {
return true;
}
return false;
}
static int walk_path_recursive(sftp_session sftp, const char *path,
@@ -276,7 +276,7 @@ static int walk_path_recursive(sftp_session sftp, const char *path,
int ret;
if (mscp_stat(path, &st, sftp) < 0) {
pr_err("stat: %s: %s", path, strerrno());
pr_err("stat: %s: %s", path, strerrno());
return -1;
}
@@ -293,11 +293,11 @@ static int walk_path_recursive(sftp_session sftp, const char *path,
pr_err("opendir: %s: %s", path, strerrno());
return -1;
}
for (e = mscp_readdir(d); e; e = mscp_readdir(d)) {
if (check_path_should_skip(e->d_name))
continue;
ret = snprintf(next_path, PATH_MAX, "%s/%s", path, e->d_name);
if (ret >= PATH_MAX) {
pr_warn("Too long path: %s/%s", path, e->d_name);
@@ -329,27 +329,25 @@ void path_dump(struct list_head *path_list)
printf("dst: %s\n", p->dst_path);
}
}
/* based on
* https://stackoverflow.com/questions/2336242/recursive-mkdir-system-call-on-unix */
static int touch_dst_path(struct path *p, sftp_session sftp)
{
/* XXX: should reflect the permission of the original directory? */
mode_t mode = S_IRWXU | S_IRWXG | S_IRWXO;
/* XXX: should reflect the permission of the original directory? */
mode_t mode = S_IRWXU | S_IRWXG | S_IRWXO;
struct stat st;
char path[PATH_MAX];
char *needle;
int ret;
char path[PATH_MAX];
char *needle;
int ret;
mf *f;
strncpy(path, p->dst_path, sizeof(path));
strncpy(path, p->dst_path, sizeof(path));
/* mkdir -p.
/* mkdir -p.
* XXX: this may be slow when dst is the remote side. need speed-up. */
for (needle = strchr(path + 1, '/'); needle; needle = strchr(needle + 1, '/')) {
*needle = '\0';
for (needle = strchr(path + 1, '/'); needle; needle = strchr(needle + 1, '/')) {
*needle = '\0';
if (mscp_stat(path, &st, sftp) == 0) {
if (S_ISDIR(st.st_mode))
@@ -367,13 +365,13 @@ static int touch_dst_path(struct path *p, sftp_session sftp)
return -1;
}
}
next:
*needle = '/';
}
next:
*needle = '/';
}
/* Do not set O_TRUNC here. Instead, do mscp_setstat() at the
* end. see https://bugzilla.mindrot.org/show_bug.cgi?id=3431 */
f = mscp_open(p->dst_path, O_WRONLY|O_CREAT, S_IRUSR|S_IWUSR, sftp);
f = mscp_open(p->dst_path, O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR, sftp);
if (!f) {
priv_set_errv("mscp_open %s: %s\n", p->dst_path, strerrno());
return -1;
@@ -381,7 +379,7 @@ static int touch_dst_path(struct path *p, sftp_session sftp)
mscp_close(f);
return 0;
return 0;
}
static int prepare_dst_path(struct path *p, sftp_session dst_sftp)
@@ -404,152 +402,149 @@ out:
return ret;
}
/* functions for copy */
static ssize_t read_to_buf(void *ptr, size_t len, void *userdata)
{
int fd = *((int *)userdata);
return read(fd, ptr, len);
int fd = *((int *)userdata);
return read(fd, ptr, len);
}
static int copy_chunk_l2r(struct chunk *c, int fd, sftp_file sf,
int nr_ahead, int buf_sz, size_t *counter)
static int copy_chunk_l2r(struct chunk *c, int fd, sftp_file sf, int nr_ahead, int buf_sz,
size_t *counter)
{
ssize_t read_bytes, remaind, thrown;
int idx, ret;
struct {
uint32_t id;
ssize_t len;
} reqs[nr_ahead];
ssize_t read_bytes, remaind, thrown;
int idx, ret;
struct {
uint32_t id;
ssize_t len;
} reqs[nr_ahead];
if (c->len == 0)
return 0;
if (c->len == 0)
return 0;
remaind = thrown = c->len;
for (idx = 0; idx < nr_ahead && thrown > 0; idx++) {
reqs[idx].len = min(thrown, buf_sz);
reqs[idx].len = sftp_async_write(sf, read_to_buf, reqs[idx].len, &fd,
&reqs[idx].id);
if (reqs[idx].len < 0) {
priv_set_errv("sftp_async_write: %s or %s",
sftp_get_ssh_error(sf->sftp), strerrno());
return -1;
}
thrown -= reqs[idx].len;
}
remaind = thrown = c->len;
for (idx = 0; idx < nr_ahead && thrown > 0; idx++) {
reqs[idx].len = min(thrown, buf_sz);
reqs[idx].len = sftp_async_write(sf, read_to_buf, reqs[idx].len, &fd,
&reqs[idx].id);
if (reqs[idx].len < 0) {
priv_set_errv("sftp_async_write: %s or %s",
sftp_get_ssh_error(sf->sftp), strerrno());
return -1;
}
thrown -= reqs[idx].len;
}
for (idx = 0; remaind > 0; idx = (idx + 1) % nr_ahead) {
ret = sftp_async_write_end(sf, reqs[idx].id, 1);
if (ret != SSH_OK) {
priv_set_errv("sftp_async_write_end: %s",
sftp_get_ssh_error(sf->sftp));
return -1;
}
for (idx = 0; remaind > 0; idx = (idx + 1) % nr_ahead) {
ret = sftp_async_write_end(sf, reqs[idx].id, 1);
if (ret != SSH_OK) {
priv_set_errv("sftp_async_write_end: %s",
sftp_get_ssh_error(sf->sftp));
return -1;
}
*counter += reqs[idx].len;
remaind -= reqs[idx].len;
*counter += reqs[idx].len;
remaind -= reqs[idx].len;
if (remaind <= 0)
break;
if (remaind <= 0)
break;
if (thrown <= 0)
continue;
if (thrown <= 0)
continue;
reqs[idx].len = min(thrown, buf_sz);
reqs[idx].len = sftp_async_write(sf, read_to_buf, reqs[idx].len, &fd,
&reqs[idx].id);
if (reqs[idx].len < 0) {
priv_set_errv("sftp_async_write: %s or %s",
sftp_get_ssh_error(sf->sftp), strerrno());
return -1;
}
thrown -= reqs[idx].len;
}
reqs[idx].len = min(thrown, buf_sz);
reqs[idx].len = sftp_async_write(sf, read_to_buf, reqs[idx].len, &fd,
&reqs[idx].id);
if (reqs[idx].len < 0) {
priv_set_errv("sftp_async_write: %s or %s",
sftp_get_ssh_error(sf->sftp), strerrno());
return -1;
}
thrown -= reqs[idx].len;
}
if (remaind < 0) {
priv_set_errv("invalid remaind bytes %ld. "
"last async_write_end bytes %lu.",
remaind, reqs[idx].len);
return -1;
}
return 0;
if (remaind < 0) {
priv_set_errv("invalid remaind bytes %ld. "
"last async_write_end bytes %lu.",
remaind, reqs[idx].len);
return -1;
}
return 0;
}
static int copy_chunk_r2l(struct chunk *c, sftp_file sf, int fd,
int nr_ahead, int buf_sz, size_t *counter)
static int copy_chunk_r2l(struct chunk *c, sftp_file sf, int fd, int nr_ahead, int buf_sz,
size_t *counter)
{
ssize_t read_bytes, write_bytes, remaind, thrown;
char buf[buf_sz];
int idx;
struct {
int id;
ssize_t len;
} reqs[nr_ahead];
ssize_t read_bytes, write_bytes, remaind, thrown;
char buf[buf_sz];
int idx;
struct {
int id;
ssize_t len;
} reqs[nr_ahead];
if (c->len == 0)
return 0;
if (c->len == 0)
return 0;
remaind = thrown = c->len;
remaind = thrown = c->len;
for (idx = 0; idx < nr_ahead && thrown > 0; idx++) {
reqs[idx].len = min(thrown, sizeof(buf));
reqs[idx].id = sftp_async_read_begin(sf, reqs[idx].len);
if (reqs[idx].id < 0) {
priv_set_errv("sftp_async_read_begin: %d",
sftp_get_error(sf->sftp));
return -1;
}
thrown -= reqs[idx].len;
}
for (idx = 0; idx < nr_ahead && thrown > 0; idx++) {
reqs[idx].len = min(thrown, sizeof(buf));
reqs[idx].id = sftp_async_read_begin(sf, reqs[idx].len);
if (reqs[idx].id < 0) {
priv_set_errv("sftp_async_read_begin: %d",
sftp_get_error(sf->sftp));
return -1;
}
thrown -= reqs[idx].len;
}
for (idx = 0; remaind > 0; idx = (idx + 1) % nr_ahead) {
read_bytes = sftp_async_read(sf, buf, reqs[idx].len, reqs[idx].id);
if (read_bytes == SSH_ERROR) {
priv_set_errv("sftp_async_read: %d",
sftp_get_error(sf->sftp));
return -1;
}
for (idx = 0; remaind > 0; idx = (idx + 1) % nr_ahead) {
read_bytes = sftp_async_read(sf, buf, reqs[idx].len, reqs[idx].id);
if (read_bytes == SSH_ERROR) {
priv_set_errv("sftp_async_read: %d", sftp_get_error(sf->sftp));
return -1;
}
if (thrown > 0) {
reqs[idx].len = min(thrown, sizeof(buf));
reqs[idx].id = sftp_async_read_begin(sf, reqs[idx].len);
thrown -= reqs[idx].len;
}
if (thrown > 0) {
reqs[idx].len = min(thrown, sizeof(buf));
reqs[idx].id = sftp_async_read_begin(sf, reqs[idx].len);
thrown -= reqs[idx].len;
}
write_bytes = write(fd, buf, read_bytes);
if (write_bytes < 0) {
priv_set_errv("write: %s", strerrno());
return -1;
}
write_bytes = write(fd, buf, read_bytes);
if (write_bytes < 0) {
priv_set_errv("write: %s", strerrno());
return -1;
}
if (write_bytes < read_bytes) {
priv_set_errv("failed to write full bytes");
return -1;
}
if (write_bytes < read_bytes) {
priv_set_errv("failed to write full bytes");
return -1;
}
*counter += write_bytes;
remaind -= read_bytes;
}
*counter += write_bytes;
remaind -= read_bytes;
}
if (remaind < 0) {
priv_set_errv("invalid remaind bytes %ld. last async_read bytes %ld. "
"last write bytes %ld",
remaind, read_bytes, write_bytes);
return -1;
}
if (remaind < 0) {
priv_set_errv("invalid remaind bytes %ld. last async_read bytes %ld. "
"last write bytes %ld",
remaind, read_bytes, write_bytes);
return -1;
}
return 0;
return 0;
}
static int _copy_chunk(struct chunk *c, mf *s, mf *d,
int nr_ahead, int buf_sz, size_t *counter)
static int _copy_chunk(struct chunk *c, mf *s, mf *d, int nr_ahead, int buf_sz,
size_t *counter)
{
if (s->local && d->remote) /* local to remote copy */
return copy_chunk_l2r(c, s->local, d->remote, nr_ahead, buf_sz, counter);
else if (s->remote && d->local) /* remote to local copy */
else if (s->remote && d->local) /* remote to local copy */
return copy_chunk_r2l(c, s->remote, d->local, nr_ahead, buf_sz, counter);
assert(false);
@@ -570,8 +565,8 @@ int copy_chunk(struct chunk *c, sftp_session src_sftp, sftp_session dst_sftp,
return -1;
/* open src */
flags = O_RDONLY;
mode = S_IRUSR;
flags = O_RDONLY;
mode = S_IRUSR;
s = mscp_open(c->p->path, flags, mode, src_sftp);
if (!s) {
priv_set_errv("mscp_open: %s: %s", c->p->path, strerrno());
@@ -583,8 +578,8 @@ int copy_chunk(struct chunk *c, sftp_session src_sftp, sftp_session dst_sftp,
}
/* open dst */
flags = O_WRONLY;
mode = S_IRUSR|S_IWUSR;
flags = O_WRONLY;
mode = S_IRUSR | S_IWUSR;
d = mscp_open(c->p->dst_path, flags, mode, dst_sftp);
if (!d) {
mscp_close(s);
@@ -596,14 +591,11 @@ int copy_chunk(struct chunk *c, sftp_session src_sftp, sftp_session dst_sftp,
return -1;
}
pr_debug("copy chunk start: %s 0x%lx-0x%lx",
c->p->path, c->off, c->off + c->len);
pr_debug("copy chunk start: %s 0x%lx-0x%lx", c->p->path, c->off, c->off + c->len);
ret = _copy_chunk(c, s, d, nr_ahead, buf_sz, counter);
pr_debug("copy chunk done: %s 0x%lx-0x%lx",
c->p->path, c->off, c->off + c->len);
pr_debug("copy chunk done: %s 0x%lx-0x%lx", c->p->path, c->off, c->off + c->len);
mscp_close(d);
mscp_close(s);

69
src/path.h
View File

@@ -12,39 +12,38 @@
#include <ssh.h>
struct path {
struct list_head list; /* mscp->path_list */
struct list_head list; /* mscp->path_list */
char *path; /* file path */
size_t size; /* size of file on this path */
mode_t mode; /* permission */
char *path; /* file path */
size_t size; /* size of file on this path */
mode_t mode; /* permission */
char *dst_path; /* copy dst path */
char *dst_path; /* copy dst path */
int state;
lock lock;
refcnt refcnt;
int state;
lock lock;
refcnt refcnt;
};
#define FILE_STATE_INIT 0
#define FILE_STATE_OPENED 1
#define FILE_STATE_DONE 2
#define FILE_STATE_INIT 0
#define FILE_STATE_OPENED 1
#define FILE_STATE_DONE 2
struct chunk {
struct list_head list; /* chunk_pool->list */
struct list_head list; /* chunk_pool->list */
struct path *p;
size_t off; /* offset of this chunk on the file on path p */
size_t len; /* length of this chunk */
size_t done; /* copied bytes for this chunk by a thread */
size_t off; /* offset of this chunk on the file on path p */
size_t len; /* length of this chunk */
size_t done; /* copied bytes for this chunk by a thread */
};
struct chunk_pool {
struct list_head list; /* list of struct chunk */
size_t count;
lock lock;
int state;
struct list_head list; /* list of struct chunk */
size_t count;
lock lock;
int state;
};
/* initialize chunk pool */
void chunk_pool_init(struct chunk_pool *cp);
@@ -53,7 +52,7 @@ void chunk_pool_init(struct chunk_pool *cp);
* added, or pointer to chunk.
*/
struct chunk *chunk_pool_pop(struct chunk_pool *cp);
#define CHUNK_POP_WAIT ((void *) -1)
#define CHUNK_POP_WAIT ((void *)-1)
/* set and check fillingchunks to this pool has finished */
void chunk_pool_set_filled(struct chunk_pool *cp);
@@ -68,24 +67,22 @@ bool chunk_pool_is_empty(struct chunk_pool *cp);
/* free chunks in the chunk_pool */
void chunk_pool_release(struct chunk_pool *cp);
struct path_resolve_args {
size_t *total_bytes;
size_t *total_bytes;
/* args to resolve src path to dst path */
const char *src_path;
const char *dst_path;
bool src_path_is_dir;
bool dst_path_is_dir;
bool dst_path_should_dir;
/* args to resolve src path to dst path */
const char *src_path;
const char *dst_path;
bool src_path_is_dir;
bool dst_path_is_dir;
bool dst_path_should_dir;
/* args to resolve chunks for a path */
struct chunk_pool *cp;
int nr_conn;
size_t min_chunk_sz;
size_t max_chunk_sz;
size_t chunk_align;
/* args to resolve chunks for a path */
struct chunk_pool *cp;
int nr_conn;
size_t min_chunk_sz;
size_t max_chunk_sz;
size_t chunk_align;
};
/* recursivly walk through src_path and fill path_list for each file */

30
src/platform.c
View File

@@ -26,7 +26,6 @@
#include <strerrno.h>
#include <print.h>
#ifdef __APPLE__
int nr_cpus()
{
@@ -49,7 +48,7 @@ int set_thread_affinity(pthread_t tid, int core)
int setutimes(const char *path, struct timespec atime, struct timespec mtime)
{
struct timeval tv[2] = {
struct timeval tv[2] = {
{
.tv_sec = atime.tv_sec,
.tv_usec = atime.tv_nsec * 1000,
@@ -64,14 +63,14 @@ int setutimes(const char *path, struct timespec atime, struct timespec mtime)
static void random_string(char *buf, size_t size)
{
char chars[] = "abcdefhijklmnopqrstuvwxyz1234567890";
int n, x;
char chars[] = "abcdefhijklmnopqrstuvwxyz1234567890";
int n, x;
for (n = 0; n < size - 1; n++) {
x = arc4random() % (sizeof(chars) - 1);
buf[n] = chars[x];
}
buf[size - 1] = '\0';
for (n = 0; n < size - 1; n++) {
x = arc4random() % (sizeof(chars) - 1);
buf[n] = chars[x];
}
buf[size - 1] = '\0';
}
sem_t *sem_create(int value)
@@ -80,10 +79,10 @@ sem_t *sem_create(int value)
sem_t *sem;
int n;
n = strlen(sem_name);
random_string(sem_name + n, sizeof(sem_name) - n - 1);
if ((sem = sem_open(sem_name, O_CREAT, 600, value)) == SEM_FAILED)
return NULL;
n = strlen(sem_name);
random_string(sem_name + n, sizeof(sem_name) - n - 1);
if ((sem = sem_open(sem_name, O_CREAT, 600, value)) == SEM_FAILED)
return NULL;
return sem;
}
@@ -124,8 +123,8 @@ int set_thread_affinity(pthread_t tid, int core)
CPU_SET(core, &target_cpu_set);
ret = pthread_setaffinity_np(tid, sizeof(target_cpu_set), &target_cpu_set);
if (ret < 0)
priv_set_errv("failed to set thread/cpu affinity for core %d: %s",
core, strerrno());
priv_set_errv("failed to set thread/cpu affinity for core %d: %s", core,
strerrno());
return ret;
}
@@ -164,4 +163,3 @@ int sem_release(sem_t *sem)
}
#endif

1
src/print.c
View File

@@ -16,4 +16,3 @@ int get_print_severity()
{
return __print_severity;
}

27
src/print.h
View File

@@ -13,23 +13,18 @@
void set_print_severity(int severity);
int get_print_severity();
#define __print(fp, severity, fmt, ...) \
do { \
if (severity <= get_print_severity()) { \
fprintf(fp, "\r\033[K" fmt "\n", ##__VA_ARGS__); \
fflush(fp); \
} \
#define __print(fp, severity, fmt, ...) \
do { \
if (severity <= get_print_severity()) { \
fprintf(fp, "\r\033[K" fmt "\n", ##__VA_ARGS__); \
fflush(fp); \
} \
} while (0)
#define pr_err(fmt, ...) \
__print(stderr, MSCP_SEVERITY_ERR, fmt, ##__VA_ARGS__)
#define pr_warn(fmt, ...) \
__print(stderr, MSCP_SEVERITY_WARN, fmt, ##__VA_ARGS__)
#define pr_notice(fmt, ...) \
__print(stdout, MSCP_SEVERITY_NOTICE, fmt, ##__VA_ARGS__)
#define pr_info(fmt, ...) \
__print(stdout, MSCP_SEVERITY_INFO, fmt, ##__VA_ARGS__)
#define pr_debug(fmt, ...) \
__print(stdout, MSCP_SEVERITY_DEBUG, fmt, ##__VA_ARGS__)
#define pr_err(fmt, ...) __print(stderr, MSCP_SEVERITY_ERR, fmt, ##__VA_ARGS__)
#define pr_warn(fmt, ...) __print(stderr, MSCP_SEVERITY_WARN, fmt, ##__VA_ARGS__)
#define pr_notice(fmt, ...) __print(stdout, MSCP_SEVERITY_NOTICE, fmt, ##__VA_ARGS__)
#define pr_info(fmt, ...) __print(stdout, MSCP_SEVERITY_INFO, fmt, ##__VA_ARGS__)
#define pr_debug(fmt, ...) __print(stdout, MSCP_SEVERITY_DEBUG, fmt, ##__VA_ARGS__)
#endif /* _PRINT_H_ */

28
src/ssh.c
View File

@@ -22,8 +22,7 @@ static int ssh_set_opts(ssh_session ssh, struct mscp_ssh_opts *opts)
return -1;
}
if (opts->port &&
ssh_options_set(ssh, SSH_OPTIONS_PORT_STR, opts->port) < 0) {
if (opts->port && ssh_options_set(ssh, SSH_OPTIONS_PORT_STR, opts->port) < 0) {
priv_set_errv("failed to set port number");
return -1;
}
@@ -62,8 +61,7 @@ static int ssh_set_opts(ssh_session ssh, struct mscp_ssh_opts *opts)
return -1;
}
if (opts->ccalgo &&
ssh_options_set(ssh, SSH_OPTIONS_CCALGO, opts->ccalgo) < 0) {
if (opts->ccalgo && ssh_options_set(ssh, SSH_OPTIONS_CCALGO, opts->ccalgo) < 0) {
priv_set_errv("failed to set cclago");
return -1;
}
@@ -77,8 +75,7 @@ static int ssh_set_opts(ssh_session ssh, struct mscp_ssh_opts *opts)
}
}
if (opts->config &&
ssh_options_parse_config(ssh, opts->config) < 0) {
if (opts->config && ssh_options_parse_config(ssh, opts->config) < 0) {
priv_set_errv("failed to parse ssh_config: %s", opts->config);
return -1;
}
@@ -157,7 +154,6 @@ static int ssh_cache_passphrase(const char *prompt, char *buf, size_t len, int e
return 0;
}
static struct ssh_callbacks_struct cb = {
.auth_function = ssh_cache_passphrase,
.userdata = NULL,
@@ -213,14 +209,13 @@ sftp_session ssh_init_sftp_session(const char *sshdst, struct mscp_ssh_opts *opt
sftp = sftp_new(ssh);
if (!sftp) {
priv_set_errv("failed to allocate sftp session: %s",
ssh_get_error(ssh));
priv_set_errv("failed to allocate sftp session: %s", ssh_get_error(ssh));
goto err_out;
}
if (sftp_init(sftp) != SSH_OK) {
priv_set_errv("failed to initialize sftp session: err code %d",
sftp_get_error(sftp));
sftp_get_error(sftp));
goto err_out;
}
@@ -231,7 +226,6 @@ err_out:
return NULL;
}
/* copied from https://api.libssh.org/stable/libssh_tutor_guided_tour.html*/
static int ssh_verify_known_hosts(ssh_session session)
{
@@ -250,10 +244,7 @@ static int ssh_verify_known_hosts(ssh_session session)
return -1;
}
rc = ssh_get_publickey_hash(srv_pubkey,
SSH_PUBLICKEY_HASH_SHA1,
&hash,
&hlen);
rc = ssh_get_publickey_hash(srv_pubkey, SSH_PUBLICKEY_HASH_SHA1, &hash, &hlen);
ssh_key_free(srv_pubkey);
if (rc < 0) {
return -1;
@@ -274,8 +265,9 @@ static int ssh_verify_known_hosts(ssh_session session)
return -1;
case SSH_KNOWN_HOSTS_OTHER:
fprintf(stderr, "The host key for this server was not found but an other"
"type of key exists.\n");
fprintf(stderr, "An attacker might change the default server key to"
"type of key exists.\n");
fprintf(stderr,
"An attacker might change the default server key to"
"confuse your client into thinking the key does not exist\n");
ssh_clean_pubkey_hash(&hash);
@@ -283,7 +275,7 @@ static int ssh_verify_known_hosts(ssh_session session)
case SSH_KNOWN_HOSTS_NOT_FOUND:
fprintf(stderr, "Could not find known host file.\n");
fprintf(stderr, "If you accept the host key here, the file will be"
"automatically created.\n");
"automatically created.\n");
/* FALL THROUGH to SSH_SERVER_NOT_KNOWN behavior */

12
src/strerrno.c
View File

@@ -7,7 +7,7 @@
#include <strerrno.h>
#define STRERRNO_TLS_BUFSIZ 128
#define STRERRNO_TLS_BUFSIZ 128
__thread char tls_strerrno_buf[STRERRNO_TLS_BUFSIZ];
const char *strerrno(void)
@@ -17,7 +17,7 @@ const char *strerrno(void)
return tls_strerrno_buf;
}
#define PRIV_ERR_BUFSIZ (1 << 12)
#define PRIV_ERR_BUFSIZ (1 << 12)
static char priv_err_buf[PRIV_ERR_BUFSIZ], internal[PRIV_ERR_BUFSIZ];
void priv_set_err(const char *fmt, ...)
@@ -27,10 +27,10 @@ void priv_set_err(const char *fmt, ...)
/* arguments may contains priv_err_buf. Thus, we build the
* string in a internal buffer, and then copy it to
* priv_err_buf. */
memset(internal, 0, sizeof(internal));
va_start(va, fmt);
vsnprintf(internal, sizeof(internal), fmt, va);
va_end(va);
memset(internal, 0, sizeof(internal));
va_start(va, fmt);
vsnprintf(internal, sizeof(internal), fmt, va);
va_end(va);
snprintf(priv_err_buf, sizeof(priv_err_buf), "%s", internal);
}

6
src/strerrno.h
View File

@@ -21,9 +21,9 @@ void priv_set_err(const char *fmt, ...);
* priv_set_errv(), a wrapper for priv_set_err(), just adds filename,
* line, and function name to the error message.
*/
#define priv_set_errv(fmt, ...) \
priv_set_err("[%s:%d:%s] " fmt "\0", \
basename(__FILE__), __LINE__, __func__, ##__VA_ARGS__)
#define priv_set_errv(fmt, ...) \
priv_set_err("[%s:%d:%s] " fmt "\0", basename(__FILE__), __LINE__, __func__, \
##__VA_ARGS__)
/**
* priv_get_err() gets the error message sotred in a private buffer.