add pre-copy hook for rclone hardlink over ssh

2 files changed, 157 insertions, 40 deletions

diff --git a/backomp b/backomp
index 1813ea2..c811a2a 100755
--- a/backomp
+++ b/backomp
@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+#
+# Copyright 2023 David Vazgenovich Shakaryan
 
 read -rd '' AWK_PREFILTER <<'EOF'
 /(\/|^)[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{6}$/ {
@@ -98,22 +100,36 @@ err() {
 parse_flags() {
 	local -n __arr="${1}"
 	local __str="${2}" __label="${3}"
+	local res
 
-	mapfile -td $'\0' __arr < <(
+	mapfile -td '' __arr < <(
 		printf '%s' "${__str}" | xargs -r printf '%s\0')
-	if ! wait "$!"; then
-		err "parsing ${__label} failed"
+	wait "$!"
+	res="$?"
+	if [[ "${res}" -ne 0 ]]; then
+		err "parsing ${__label} failed (${res})"
 		return 1
 	fi
 }
 
+qargs() {
+	local arg str=
+
+	for arg; do
+		[[ "${arg:-*}" =~ [^A-Za-z0-9%+./:=@_-] ]] && arg="${arg@Q}"
+		str+=" ${arg}"
+	done
+	printf '%s\n' "${str:1}"
+}
+
 run() {
-	local res
+	local str res
 
-	echo "* $*"
+	str="$(qargs "$@")"
+	echo "* ${str}"
 	"$@"
 	res="$?"
-	[[ "${res}" -ne 0 ]] && err "command failed: $*"
+	[[ "${res}" -ne 0 ]] && err "command failed (${res}): ${str}"
 	return "${res}"
 }
 
@@ -128,7 +144,8 @@ ssh_run() {
 	else
 		"${cmd[@]}"
 		res="$?"
-		[[ "${res}" -eq 255 ]] && err "command failed: ${cmd[*]}"
+		[[ "${res}" -eq 255 ]] &&
+			err "command failed (${res}): $(qargs "${cmd[@]}")"
 		return "${res}"
 	fi
 }
@@ -146,7 +163,7 @@ current() {
 			*) ((s += val)) ;;
 		esac
 
-		interval="${interval#*${BASH_REMATCH[0]}}"
+		interval="${interval#*"${BASH_REMATCH[0]}"}"
 	done
 	((s < 1)) && return 1
 
@@ -164,7 +181,7 @@ exists() {
 		res="$?"
 		[[ "${res}" -eq 0 ]] && return 0
 		[[ "${res}" -eq 3 ]] && return 1
-		err "command failed: ${cmd[*]}"
+		err "command failed (${res}): $(qargs "${cmd[@]}")"
 		return 2
 	fi
 
@@ -181,12 +198,15 @@ filter() {
 	local -n __src="${1}" __dest="${2}"
 	local __label="${3}" __script="${4}"
 	shift 4
+	local res
 
-	mapfile -td $'\0' __dest < <(
-		printf '%s\0' "${__src[@]}" | awk -v 'RS=\0' -v 'ORS=\0' \
-			"${@}" -- "${__script}")
-	if ! wait "$!"; then
-		err "${__label} failed"
+	mapfile -td '' __dest < <(
+		printf '%s\0' "${__src[@]}" | \
+			awk -v 'RS=\0' -v 'ORS=\0' "$@" -- "${__script}")
+	wait "$!"
+	res="$?"
+	if [[ "${res}" -ne 0 ]]; then
+		err "${__label} failed (${res})"
 		return 1
 	fi
 
@@ -200,15 +220,16 @@ filter() {
 
 populate() {
 	local -n __arr="${1}"
-	local  __dir="${2}"
-	local cmd
+	local __dir="${2}"
+	local cmd res
 
 	if [[ "${__dir}" == rclone://* ]]; then
 		cmd=(rclone lsf --dir-slash=false -- "${__dir#*//}")
 		mapfile -t __arr < <("${cmd[@]}" 2>/dev/null)
 		wait "$!"
-		if [[ "$?" -ne 0 && "$?" -ne 3 ]]; then
-			err "command failed: ${cmd[*]}"
+		res="$?"
+		if [[ "${res}" -ne 0 && "${res}" -ne 3 ]]; then
+			err "command failed (${res}): $(qargs "${cmd[@]}")"
 			return 1
 		fi
 
@@ -217,9 +238,9 @@ populate() {
 	fi
 
 	if [[ "${__dir}" =~ ^([^:/]*):(.*) ]]; then
-		mapfile -td $'\0' __arr < \
-			<(ssh_run "${BASH_REMATCH[1]}" \
-				"shopt -s nullglob; printf '%s\0' $(\
+		mapfile -td '' __arr < <(
+			ssh_run "${BASH_REMATCH[1]}" \
+				"shopt -s nullglob; printf '%s\0' $( \
 					printf '%q' "${BASH_REMATCH[2]}")/*")
 		wait "$!" || return 1
 
@@ -227,8 +248,8 @@ populate() {
 		return
 	fi
 
-	mapfile -td $'\0' __arr < \
-		<(shopt -s nullglob; printf '%s\0' "${__dir}/"*)
+	mapfile -td '' __arr < <(
+		shopt -s nullglob; printf '%s\0' "${__dir}/"*)
 }
 
 copy() {
@@ -245,15 +266,28 @@ copy() {
 		fi
 	done
 
+	if [[ -n "${opts['pre_copy']}" ]]; then
+		bash -c "$(declare -p src dest link_dest opts \
+			); temp_dest=$(printf '%q' "${dest}.tmp" \
+			); ${opts['pre_copy']}" < /dev/null
+		res="$?"
+		if [[ "${res}" -ne 0 ]]; then
+			err "pre-copy command failed (${res}):" \
+				"${opts['pre_copy']}"
+			return 1
+		fi
+	fi
+
 	if [[ "${dest}" == rclone://* ]]; then
 		r_dest="${dest}.tmp"
+		((${opts['rclone_use_copy_dest']:-1})) || link_dest=
 		if [[ -d "${src}" ]]; then
 			# emulates rsync src trailing slash behaviour
 			r_dest+="/${src##*/}"
 			link_dest+="${link_dest:+/${src##*/}}"
 		fi
 		parse_flags flags "${rclone_flags}" 'rclone_flags' || return 1
-		run rclone copy "${flags[@]}" \
+		run rclone "${opts['rclone_copy_command']}" "${flags[@]}" \
 			${link_dest:+"--copy-dest=${link_dest#*//}"} \
 			-- "${src}" "${r_dest#*//}" || return 1
 		run rclone move -- "${dest#*//}.tmp" "${dest#*//}"
@@ -267,8 +301,8 @@ copy() {
 
 	if [[ "${dest}" =~ ^([^:/]*):(.*) ]]; then
 		USE_RUN=1 ssh_run "${BASH_REMATCH[1]}" \
-			"mv -T -- $(\
-				printf '%q' "${BASH_REMATCH[2]}.tmp") $(\
+			"mv -T -- $( \
+				printf '%q' "${BASH_REMATCH[2]}.tmp") $( \
 				printf '%q' "${BASH_REMATCH[2]}")"
 		return
 	fi
@@ -338,8 +372,10 @@ declare -A def_opts=(
 	['retain']='*h' # e.g. 24h7d4w12M or 24^h10d*M
 	['interval']='0'
 	['week_start']='mon'
-	['rsync_flags']='-ac'
-	['rclone_flags']='--checksum --sftp-copy-is-hardlink'
+	['rsync_flags']='-a'
+	['rclone_flags']='--sftp-copy-is-hardlink'
+	['rclone_copy_command']='copy'
+	['rclone_use_copy_dest']=1
 )
 declare -A opts tainted_opts
 reset_opts() {
@@ -355,7 +391,8 @@ expand_opts() {
 	local str="${1}" default="${2}"
 	local res opt tainted=0
 
-	while [[ "${str}" =~ \\([\\$=!])|\$([A-Za-z0-9_]+)|\$\{([^}]*)\} ]]; do
+	while [[ "${str}" =~ \
+			\\([\\$=!%])|\$([A-Za-z0-9_]+)|\$\{([^}]*)\} ]]; do
 		res+="${str%%"${BASH_REMATCH[0]}"*}"
 		opt="${BASH_REMATCH[2]:-${BASH_REMATCH[3]}}"
 		if [[ -n "${BASH_REMATCH[1]}" ]]; then
@@ -400,23 +437,23 @@ while IFS= read -r line; do
 	[[ "${f1}" == \$?* ]] && opt="${f1:1}" || opt=
 	err_ctx="${opt:+opt }${opt:-task $((++bcount))} (${err_ctx})"
 
+	res=0
 	tainted_src=0
 	tainted_dest=0
 	if [[ "${f2}" == \!?* ]]; then
 		cmd="${f2:1}"
-		f2="$(default="${opt:+${def_opts["${opt}"]}}" eval "${cmd}" \
-			2> >(while read -r x || [[ -n "${x}" ]]; do
-				[[ "${x}" =~ ^"${0}: line "[^:]+": "(.*) ]] &&
-					x="bash: ${BASH_REMATCH[1]}"
-				echo "${0##*/}: ${err_ctx}: stderr: ${x}"
-			done >&2))"
-		ret="$?"
-		[[ "${ret}" -ne 0 ]] && err "command failed (${ret}): ${cmd}"
+		default="${opt:+${def_opts["${opt}"]}}"
+		f2="$(bash -c "$(declare -p default opts); ${cmd}" \
+			< /dev/null)"
+		res="$?"
+		[[ "${res}" -ne 0 ]] && err "command failed (${res}): ${cmd}"
+	elif [[ "${f2}" == %?* ]]; then
+		f2="${f2:1}"
 	else
 		f2="$(expand_opts "${f2}" "${opt:+${def_opts["${opt}"]}}")"
-		ret="$?"
+		res="$?"
 	fi
-	if [[ "${ret}" -ne 0 ]]; then
+	if [[ "${res}" -ne 0 ]]; then
 		if [[ -n "${opt}" ]]; then
 			tainted_opts["${opt}"]=1
 			((++errors))
@@ -425,7 +462,7 @@ while IFS= read -r line; do
 		tainted_src=1
 	elif [[ -n "${opt}" ]]; then
 		opts["${opt}"]="${f2}"
-		unset tainted_opts["${opt}"]
+		unset 'tainted_opts[${opt}]'
 		continue
 	fi
 
diff --git a/pre-copy-rclone-hardlink.sh b/pre-copy-rclone-hardlink.sh
new file mode 100755
index 0000000..01f7b6e
--- /dev/null
+++ b/pre-copy-rclone-hardlink.sh
@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+#
+# Copyright 2023 David Vazgenovich Shakaryan
+
+qargs() {
+	local arg str=
+
+	for arg; do
+		[[ "${arg:-*}" =~ [^A-Za-z0-9%+./:=@_-] ]] && arg="${arg@Q}"
+		str+=" ${arg}"
+	done
+	printf '%s\n' "${str:1}"
+}
+
+temp_dest="${1}"
+link_dest="${2}"
+[[ -n "${link_dest}" ]] || exit 0
+
+[[ "${temp_dest}" =~ ^(rclone://([^:]+):)(.+) ]] || exit 1
+prefix="${BASH_REMATCH[1]}"
+remote="${BASH_REMATCH[2]}"
+path="${BASH_REMATCH[3]}"
+[[ "${link_dest}" == "${prefix}"* ]] || exit 1
+link="${link_dest#"${prefix}"}"
+
+conf="$(rclone config dump)" || exit 1
+read -rd '' JQ <<'EOF'
+.[$k]
+| select(.type == "crypt" or .type == "sftp") // error("not crypt or sftp")
+| [.type, .remote]
+| @tsv
+EOF
+IFS=$'\t' read -r r_type r_remote < <(
+	jq -rc --arg k "${remote}" "${JQ}" <<<"${conf}") || exit 1
+
+if [[ "${r_type}" == 'crypt' ]]; then
+	crypt="${remote}"
+
+	[[ "${r_remote}" =~ ^([^:]+):(.*) ]] || exit 1
+	remote="${BASH_REMATCH[1]}"
+	base="${BASH_REMATCH[2]:+${BASH_REMATCH[2]}/}"
+
+	upath=0
+	ulink=0
+	while IFS=$'\t' read -r k v; do
+		[[ "${k% }" == "${path}" ]] && path="${base}${v# }" && upath=1
+		[[ "${k% }" == "${link}" ]] && link="${base}${v# }" && ulink=1
+	done < <(rclone cryptdecode --reverse "${crypt}:" "${path}" "${link}")
+	wait "$!" || exit 1
+	((ulink && upath)) || exit 1
+fi
+
+read -rd '' JQ <<'EOF'
+.[$k]
+| select(.type == "sftp") // error("not sftp")
+| select(.key_file_pass | not) // error("key uses passphrase")
+| [
+	(.key_file | select(.) // error("missing key")
+		| "-i", (sub("^~"; env.HOME) | @sh)),
+	(.known_hosts_file | select(.)
+		| "-o", ("UserKnownHostsFile=\(sub("^~"; env.HOME))" | @sh)),
+	(.port | select(.) | "-p", @sh),
+	("\(.user // error("missing user"))@\(.host // error("missing host"))"
+		| @sh)
+]
+| join(" ")
+EOF
+mapfile -td '' args < <(
+	jq -rc --arg k "${remote}" "${JQ}" <<<"${conf}" | \
+		xargs -r printf '%s\0'
+	[[ "${PIPESTATUS[0]}" -eq 0 && "${PIPESTATUS[1]}" -eq 0 ]])
+wait "$!" || exit 1
+
+ssh_cmd=(ssh -o BatchMode=yes "${args[@]}")
+cmd=("${ssh_cmd[@]}"
+	"cp -al $(printf '%q' "${link}") $(printf '%q' "${path}")")
+echo "* $(qargs "${cmd[@]}")"
+"${cmd[@]}" || exit 1
+"${ssh_cmd[@]}" "touch -c ${path@Q}"
+exit 0