path: root/backomp

#!/usr/bin/env bash
#
# Copyright 2023 David Vazgenovich Shakaryan

read -rd '' AWK_PREFILTER <<'EOF'
/(\/|^)[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{6}$/ {
	a[$0]
}

END {
	n = asorti(a)
	for (i = n; i >= 1; --i)
		print a[i]
}
EOF

read -rd '' AWK_FILTER <<'EOF'
function flr(n, s) {
	return int(n / s) * s
}

function mstart(t) {
	return mktime(strftime("%Y %m 01 00 00 00", t, 1), 1)
}

BEGIN {
	n = split("h d w M", arr)
	for (i = 1; i <= n; ++i) {
		b = arr[i]

		if (!match(opt_retain, "([*0-9]+)(\\^?)" b, md))
			continue

		ret[b] = md[1]
		newest_in[b] = (md[2] == "^")
		buckets[++n_buckets] = b
	}

	split("sun mon tue wed thu fri sat", arr)
	for (i in arr)
		arr2[arr[i]] = i - 1
	wshift = (11 - arr2[opt_week_start])*24*60*60
}

NR == 1 {
	++keep[$0]
}

{
	t = mktime(gensub(/(.*\/|^)(.+)-(..)-(..)-(..)(..)(..)$/,
		"\\2 \\3 \\4 \\5 \\6 \\7", 1), 1)
	if (t < 0) {
		++keep[$0]
		next
	}

	bt["h"] = flr(t, 60*60)
	bt["d"] = flr(t, 24*60*60)
	bt["w"] = flr(t + wshift, 7*24*60*60) - wshift
	bt["M"] = mstart(t)

	for (i = 1; i <= n_buckets; ++i) {
		b = buckets[i]

		if (b == "M" && "w" in ret) {
			if (newest_in["w"]) {
				if (mstart(bt["w"] + 6*24*60*60) > bt["M"])
					next
			} else if (bt["w"] < bt["M"]) {
				next
			}
		}

		if (newest_in[b] && bt[b] == last[b, b] && (i == 1 ||
				bt[buckets[i-1]] != last[b, buckets[i-1]]))
			next

		if ((b, bt[b]) in bkeep || ret[b] == "*" || bc[b]++ < ret[b])
			bkeep[b, bt[b]] = $0

		for (j in buckets)
			last[b, buckets[j]] = bt[buckets[j]]
	}
}

END {
	for (i in bkeep)
		++keep[bkeep[i]]

	n = asorti(keep)
	for (i = 1; i <= n; ++i)
		print keep[i]
}
EOF

err() {
	echo "${0##*/}: ${err_ctx}: error: $*" >&2
}

parse_flags() {
	local -n __arr="${1}"
	local __str="${2}" __label="${3}"
	local res

	mapfile -td '' __arr < <(
		printf '%s' "${__str}" | xargs -r printf '%s\0')
	wait "$!"
	res="$?"
	if [[ "${res}" -ne 0 ]]; then
		err "parsing ${__label} failed (${res})"
		return 1
	fi
}

qargs() {
	local arg str=

	for arg; do
		[[ "${arg:-*}" =~ [^A-Za-z0-9%+./:=@_-] ]] && arg="${arg@Q}"
		str+=" ${arg}"
	done
	printf '%s\n' "${str:1}"
}

run() {
	local str res

	str="$(qargs "$@")"
	echo "* ${str}"
	"$@"
	res="$?"
	[[ "${res}" -ne 0 ]] && err "command failed (${res}): ${str}"
	return "${res}"
}

ssh_run() {
	local host="${1}" cmd_str="${2}"
	local cmd res flags

	parse_flags flags "${ssh_flags}" 'ssh_flags' || return 255
	cmd=(ssh "${flags[@]}" -n "${host}" "${cmd_str}")
	if ((USE_RUN)); then
		run "${cmd[@]}"
	else
		"${cmd[@]}"
		res="$?"
		[[ "${res}" -eq 255 ]] &&
			err "command failed (${res}): $(qargs "${cmd[@]}")"
		return "${res}"
	fi
}

current() {
	local str="${1}" interval="${2}"
	local s=0 val d

	while [[ "${interval}" =~ ([0-9]+)(.?) ]]; do
		val="${BASH_REMATCH[1]}"
		case "${BASH_REMATCH[2]}" in
			d) ((s += val * 24*60*60)) ;;
			h) ((s += val * 60*60)) ;;
			m) ((s += val * 60)) ;;
			*) ((s += val)) ;;
		esac

		interval="${interval#*"${BASH_REMATCH[0]}"}"
	done
	((s < 1)) && return 1

	d="${str:0:10} ${str:11:2}:${str:13:2}:${str:15:2}"
	(($(date -u +%s) / s == $(date -ud "${d}" +%s) / s))
}

exists() {
	local path="${1}"
	local cmd res

	if [[ "${path}" == rclone://* ]]; then
		cmd=(rclone lsf -- "${path#*//}")
		"${cmd[@]}" &>/dev/null
		res="$?"
		[[ "${res}" -eq 0 ]] && return 0
		[[ "${res}" -eq 3 ]] && return 1
		err "command failed (${res}): $(qargs "${cmd[@]}")"
		return 2
	fi

	if [[ "${path}" =~ ^([^:/]*):(.*) ]]; then
		ssh_run "${BASH_REMATCH[1]}" \
			"test -e $(printf '%q' "${BASH_REMATCH[2]}")"
		return
	fi

	[[ -e "${path}" ]]
}

filter() {
	local -n __src="${1}" __dest="${2}"
	local __label="${3}" __script="${4}"
	shift 4
	local res

	mapfile -td '' __dest < <(
		printf '%s\0' "${__src[@]}" | \
			awk -v 'RS=\0' -v 'ORS=\0' "$@" -- "${__script}")
	wait "$!"
	res="$?"
	if [[ "${res}" -ne 0 ]]; then
		err "${__label} failed (${res})"
		return 1
	fi

	if ((${#__src[@]})); then
		exists "${__dest[0]}" && return 0
		[[ "$?" -eq 1 ]] &&
			err "${__label} returned empty or invalid list"
		return 1
	fi
}

populate() {
	local -n __arr="${1}"
	local __dir="${2}"
	local cmd res

	if [[ "${__dir}" == rclone://* ]]; then
		cmd=(rclone lsf --dir-slash=false -- "${__dir#*//}")
		mapfile -t __arr < <("${cmd[@]}" 2>/dev/null)
		wait "$!"
		res="$?"
		if [[ "${res}" -ne 0 && "${res}" -ne 3 ]]; then
			err "command failed (${res}): $(qargs "${cmd[@]}")"
			return 1
		fi

		__arr=("${__arr[@]/#/${__dir}/}")
		return
	fi

	if [[ "${__dir}" =~ ^([^:/]*):(.*) ]]; then
		mapfile -td '' __arr < <(
			ssh_run "${BASH_REMATCH[1]}" \
				"shopt -s nullglob; printf '%s\0' $( \
					printf '%q' "${BASH_REMATCH[2]}")/*")
		wait "$!" || return 1

		__arr=("${__arr[@]/#/${BASH_REMATCH[1]}:}")
		return
	fi

	mapfile -td '' __arr < <(
		shopt -s nullglob; printf '%s\0' "${__dir}/"*)
}

copy() {
	local src="${1}" dest="${2}" link_dest="${3}"
	local path res flags r_dest

	for path in "${dest}" "${dest}.tmp"; do
		exists "${path}"
		res="$?"
		if [[ "${res}" -ne 1 ]]; then
			[[ "${res}" -eq 0 ]] &&
				err "destination already exists: ${path}"
			return 1
		fi
	done

	if [[ -n "${opts['pre_copy']}" ]]; then
		bash -c "$(declare -p src dest link_dest opts \
			); temp_dest=$(printf '%q' "${dest}.tmp" \
			); ${opts['pre_copy']}" < /dev/null
		res="$?"
		if [[ "${res}" -ne 0 ]]; then
			err "pre-copy command failed (${res}):" \
				"${opts['pre_copy']}"
			return 1
		fi
	fi

	if [[ "${dest}" == rclone://* ]]; then
		r_dest="${dest}.tmp"
		((${opts['rclone_use_copy_dest']:-1})) || link_dest=
		if [[ -d "${src}" ]]; then
			# emulates rsync src trailing slash behaviour
			r_dest+="/${src##*/}"
			link_dest+="${link_dest:+/${src##*/}}"
		fi
		parse_flags flags "${rclone_flags}" 'rclone_flags' || return 1
		run rclone "${opts['rclone_copy_command']}" "${flags[@]}" \
			${link_dest:+"--copy-dest=${link_dest#*//}"} \
			-- "${src}" "${r_dest#*//}" || return 1
		run rclone move -- "${dest#*//}.tmp" "${dest#*//}"
		return
	fi

	parse_flags flags "${rsync_flags}" 'rsync_flags' || return 1
	run rsync "${flags[@]}" --mkpath \
		${link_dest:+"--link-dest=../${link_dest##*/}"} \
		-- "${src}" "${dest}.tmp/" || return 1

	if [[ "${dest}" =~ ^([^:/]*):(.*) ]]; then
		USE_RUN=1 ssh_run "${BASH_REMATCH[1]}" \
			"mv -T -- $( \
				printf '%q' "${BASH_REMATCH[2]}.tmp") $( \
				printf '%q' "${BASH_REMATCH[2]}")"
		return
	fi

	run mv -T -- "${dest}.tmp" "${dest}" || return 1
}

prune() {
	local -n __snaps="${1}" __keep="${2}"
	local __i __k

	for ((__i = ${#__snaps[@]} - 1, __k = 0; __i >= 0; --__i)); do
		if [[ "${__snaps[__i]}" == "${__keep[__k]}" ]]; then
			((++__k))
		else
			if [[ "${__snaps[__i]}" == rclone://* ]]; then
				run rclone purge -- "${__snaps[__i]#*//}"
			elif [[ "${__snaps[__i]}" =~ ^([^:/]*):(.*) ]]; then
				USE_RUN=1 ssh_run "${BASH_REMATCH[1]}" \
					"rm -rf -- $(printf '%q' \
						"${BASH_REMATCH[2]}")"
			else
				run rm -rf -- "${__snaps[__i]}"
			fi
		fi
	done
}

backup() {
	local src_path="${1}" dest_dir="${2}"
	local retain="${3}" interval="${4}" week_start="${5}"
	local rsync_flags="${6}" rclone_flags="${7}" ssh_flags="${8}"
	local dest_path snaps keep

	if [[ ! -e "${src_path}" ]]; then
		err "source does not exist: ${src_path}"
		return 1
	fi

	# double colon after host enables daemon mode in rsync
	[[ "${dest_dir}" =~ ^(rclone://)?[^:/]*:$ ]] && dest_dir+='.'

	populate snaps "${dest_dir}" || return 1
	filter snaps snaps 'pre-snapshot awk prefilter' "${AWK_PREFILTER}" \
		|| return 1
	[[ -n "${snaps}" ]] && current "${snaps[0]##*/}" "${interval}" \
		&& return

	dest_path="${dest_dir}/$(date -u '+%Y-%m-%d-%H%M%S')"
	copy "${src_path}" "${dest_path}" "${snaps[0]}" || return 1

	if [[ ! "${retain}" =~ ^( *([0-9]+|\*)\^?[hdwM] *)+$ ]]; then
		err "invalid \$retain setting: ${retain}"
		return 1
	fi

	snaps+=("${dest_path}")
	filter snaps snaps 'post-snapshot awk prefilter' "${AWK_PREFILTER}" \
		|| return 1
	filter snaps keep 'awk filter' "${AWK_FILTER}" \
		-v "opt_retain=${retain}" -v "opt_week_start=${week_start}" \
		|| return 1
	prune snaps keep
}

declare -A def_opts=(
	['retain']='*h' # e.g. 24h7d4w12M or 24^h10d*M
	['interval']='0'
	['week_start']='mon'
	['rsync_flags']='-a'
	['rclone_flags']='--sftp-copy-is-hardlink'
	['rclone_copy_command']='copy'
	['rclone_use_copy_dest']=1
)
declare -A opts tainted_opts
reset_opts() {
	opts=()
	tainted_opts=()
	for k in "${!def_opts[@]}"; do
		opts["${k}"]="${def_opts["${k}"]}"
	done
}
reset_opts

expand_opts() {
	local str="${1}" default="${2}"
	local res opt tainted=0

	while [[ "${str}" =~ \
			\\([\\$=!%])|\$([A-Za-z0-9_]+)|\$\{([^}]*)\} ]]; do
		res+="${str%%"${BASH_REMATCH[0]}"*}"
		opt="${BASH_REMATCH[2]:-${BASH_REMATCH[3]}}"
		if [[ -n "${BASH_REMATCH[1]}" ]]; then
			res+="${BASH_REMATCH[1]}"
		elif [[ "${opt}" == 'default' ]]; then
			res+="${default}"
		elif [[ -n "${opt}" ]]; then
			res+="${opts["${opt}"]}"
			if [[ ! -v opts["${opt}"] ]]; then
				err "reference to undefined opt: ${opt}"
				tainted=1
			elif [[ -v tainted_opts["${opt}"] ]]; then
				err "reference to tainted opt: ${opt}"
				tainted=1
			fi
		fi
		str="${str#*"${BASH_REMATCH[0]}"}"
	done
	printf '%s\n' "${res}${str}"
	return "${tainted}"
}

declare -A seen
while IFS= read -r line; do
	((++lnum))
	[[ -z "${line}" || "${line}" == \#* ]] && continue

	if [[ "${line}" =~ ^\[(.*)]$ ]]; then
		base_path="${BASH_REMATCH[1]}"
		reset_opts
		continue
	fi

	err_ctx="line ${lnum}"
	if [[ ! "${line}" =~ ^((\\[\\=]|[^=])+)=(.*) ]]; then
		err "expected '='"
		((++errors))
		continue
	fi
	f1="${BASH_REMATCH[1]}"
	f2="${BASH_REMATCH[3]}"
	[[ "${f1}" == \$?* ]] && opt="${f1:1}" || opt=
	err_ctx="${opt:+opt }${opt:-task $((++bcount))} (${err_ctx})"

	res=0
	tainted_src=0
	tainted_dest=0
	if [[ "${f2}" == \!?* ]]; then
		cmd="${f2:1}"
		default="${opt:+${def_opts["${opt}"]}}"
		f2="$(bash -c "$(declare -p default opts); ${cmd}" \
			< /dev/null)"
		res="$?"
		[[ "${res}" -ne 0 ]] && err "command failed (${res}): ${cmd}"
	elif [[ "${f2}" == %?* ]]; then
		f2="${f2:1}"
	else
		f2="$(expand_opts "${f2}" "${opt:+${def_opts["${opt}"]}}")"
		res="$?"
	fi
	if [[ "${res}" -ne 0 ]]; then
		if [[ -n "${opt}" ]]; then
			tainted_opts["${opt}"]=1
			((++errors))
			continue
		fi
		tainted_src=1
	elif [[ -n "${opt}" ]]; then
		opts["${opt}"]="${f2}"
		unset 'tainted_opts[${opt}]'
		continue
	fi

	src="${f2}"
	dest="${base_path}"
	[[ "${dest:-/}" != */ && ! "${dest}" =~ ^(rclone://)?[^:/]*:$ ]] &&
		dest+='/'
	f1_exp="$(expand_opts "${f1#./}")"
	tainted_dest="$?"
	[[ "${dest}" =~ ^[^:/]*:$ && "${f1_exp}" == :* ]] && dest+='./'
	dest+="${f1_exp}"
	printf '[%d] [%s] %s => %s\n' "${bcount}" "${opts['retain']}" \
		"${src}" "${dest}"

	skips=()
	((tainted_src)) && skips+=('tainted source')
	((tainted_dest)) && skips+=('tainted destination')
	for k in "${!def_opts[@]}"; do
		[[ -v tainted_opts["${k}"] ]] && skips+=("tainted opt: ${k}")
	done
	[[ -z "${src}" ]] && skips+=('source blank')
	if ((!tainted_dest)); then
		if [[ -z "${dest}" ]]; then
			skips+=('destination blank')
		else
			dest_rp="$(realpath -m -- "${dest}")"
			[[ -v seen["${dest_rp}"] ]] &&
				skips+=("destination already seen: ${dest}")
			seen["${dest_rp}"]=
		fi
	fi
	if ((${#skips[@]})); then
		for x in "${skips[@]}"; do
			err "${x}"
		done
		((++errors))
		continue
	fi

	backup "${src}" "${dest}" \
		"${opts['retain']}" \
		"${opts['interval']}" \
		"${opts['week_start']}" \
		"${opts['rsync_flags']}" \
		"${opts['rclone_flags']}" \
		"${opts['ssh_flags']}" \
		|| ((++errors))
done < "${1}"

((!errors))