414 lines
14 KiB
Bash
Executable File
414 lines
14 KiB
Bash
Executable File
#!/bin/bash
|
|
## Syncs all data in folder given to the configured destiny.
|
|
|
|
## Log levels for messages
|
|
LLMute=0
|
|
LLMdtry=1 # mandatory: messages that are always written to the log (except mute)
|
|
LLError=2
|
|
LLWarning=3
|
|
LLInfo=4
|
|
LLDebug=5
|
|
## Actual log level of the logger used
|
|
Log_Level=$LLDebug
|
|
|
|
## Write log of the script.
|
|
## $1 Loglevel to log the message with.
|
|
## $2 message to log.
|
|
function logger() {
|
|
local _logging_level=$1;
|
|
local _log_message=$2;
|
|
if [[ $Log_Level -lt $_logging_level ]]; then
|
|
return;
|
|
fi
|
|
local _now=$($DATE +%Y%m%d_%H:%M:%S);
|
|
if [[ $_logging_level -eq $LLInfo ]]; then
|
|
_type=" Info:";
|
|
elif [[ $_logging_level -eq $LLError ]]; then
|
|
_type=" ERROR!";
|
|
elif [[ $_logging_level -eq $LLWarning ]]; then
|
|
_type=" Warning:";
|
|
elif [[ $_logging_level -eq $LLDebug ]]; then
|
|
_type=" DEBUG:";
|
|
else
|
|
_type=":";
|
|
fi
|
|
printf "photosync ${_now}${_type} ${_log_message}\n" >> $Log_File;
|
|
if [[ $Log_Level -eq $LLDebug ]]; then
|
|
printf "photosync ${_now}${_type} ${_log_message}\n";
|
|
fi
|
|
}
|
|
|
|
## Display Help
|
|
function usage() {
|
|
echo "$0 syncs all data from a source folder to a target folder. Target folder may optionally be on a remote ssh server."
|
|
echo
|
|
echo "Syntax: scriptTemplate [-h|c|v|V]"
|
|
echo "options:"
|
|
echo "h Print this Help."
|
|
echo "c configuration file with all parameters for syncing your data."
|
|
# echo "s Verbose mode."
|
|
echo "V Print software version and exit."
|
|
echo
|
|
}
|
|
|
|
## Cleanup function to exit script gracefully
|
|
# param1 Exit value to end with
|
|
function cleanup() {
|
|
logger $LLMdtry "Done! Exiting with value ${1}.";
|
|
exit $1;
|
|
}
|
|
|
|
## Returns the binaries full path if existent.
|
|
## $1 referenced variable the found path to the binary is stored and returned in.
|
|
## $2 Name of the binary to look for.
|
|
## return 0 in case of success (binary found), 1 on fail. $1 then contains "$2 not found!".
|
|
function which_is() {
|
|
declare -n _ret_val=$1; # define _ret_val as a reference to the first parameter of the function.
|
|
local _executable=$2;
|
|
local _with_full_path=$(which ${_executable} 2>/dev/null);
|
|
if [ -x "$_with_full_path" ]; then
|
|
_ret_val="$_with_full_path";
|
|
return 0;
|
|
else
|
|
_ret_val="$_executable not found!";
|
|
return 1;
|
|
fi
|
|
}
|
|
|
|
## Checks given folder for files with certain extensions. Subfolders are checked recursively.
|
|
## $1 Folder to check.
|
|
## $2 String of space separated extensions to check for. e.g.: "*.jpg *.JPG *.mp4 *.MP4".
|
|
## returns Count of files found with the desired extension.
|
|
function check_folder() {
|
|
local _folder=$1;
|
|
local _extensions=$2;
|
|
local _file_count=0;
|
|
if [ -d "$_folder" ]; then # is a folder
|
|
local _item="";
|
|
local _fc=${#Files_N_Paths[@]};
|
|
pushd "$_folder" 1>/dev/null;
|
|
for _item in ${_extensions}; do # lookup all files with the desired extensions
|
|
if [[ -f "$_item" ]]; then # is a file
|
|
Files_N_Paths[$_fc]="$(pwd)/$_item";
|
|
((_fc+=1));
|
|
((_file_count+=1));
|
|
fi
|
|
done
|
|
# check subfolders if exist
|
|
local _sub_folders="$(/usr/bin/ls . 2>/dev/null)";
|
|
local _cnt=0;
|
|
for _sub_item in ${_sub_folders}; do
|
|
if [[ -d "$_sub_item" ]]; then
|
|
check_folder "$_sub_item" "$_extensions";
|
|
_cnt=$?
|
|
((_file_count+=_cnt));
|
|
fi
|
|
done
|
|
popd 1>/dev/null;
|
|
fi
|
|
return $_file_count
|
|
}
|
|
|
|
## Check filesizes of local and remote file
|
|
## $1 Path of local file
|
|
## $2 Path of remote file
|
|
## return 0 in case of matching sizes, 1 means size differs, 2 remote file does not exist.
|
|
check_local_remote_filesize() {
|
|
local _local_file=$1;
|
|
local _remote_file=$2;
|
|
local _local_file_size=$(stat -c%s "${_local_file}");
|
|
if [ ! -z $TOSSH ]; then
|
|
local _remote_file_size=$($S $TOSSH "stat -c%s ${_remote_file}" 2>/dev/null);
|
|
else
|
|
local _remote_file_size=$(stat -c%s "${_remote_file}" 2>/dev/null);
|
|
fi
|
|
if [[ $_remote_file_size != ?(-)+([0-9]) ]]; then # value should be integer to test true
|
|
return 2;
|
|
fi
|
|
if [[ $_local_file_size -ne $_remote_file_size ]]; then
|
|
return 1;
|
|
fi
|
|
return 0;
|
|
}
|
|
|
|
## Main function
|
|
function main() {
|
|
|
|
# create file list to move to copy to server
|
|
declare -A Files_N_Paths; # associative array
|
|
check_folder "$Cp_Folder" "$EXTENSIONS";
|
|
local _file_count=$?
|
|
logger $LLDebug "Count of files with extensions »$EXTENSIONS« is $_file_count.";
|
|
if [[ $_file_count -eq 0 ]]; then
|
|
logger $LLError "No files to copy!"
|
|
cleanup 1;
|
|
fi
|
|
|
|
# Make Lists for prefixes to replace
|
|
declare -A _prefixes;
|
|
local _pre_c=0;
|
|
for _prefx in $PREFIXES; do
|
|
_prefixes[$_pre_c]="$_prefx"; #echo "Arrayinhalt an der Stelle $INP_C: ${FILE[$INP_C]}";
|
|
((_pre_c+=1));
|
|
done
|
|
if [[ $_pre_c -eq 0 ]]; then
|
|
logger $LLError "Number of prefixes ($_pre_c) must not be 0! Check variables in <$Conf_File>!";
|
|
exit 1;
|
|
fi
|
|
|
|
# Make List with replacement prefixes
|
|
declare -A _replacements;
|
|
local _repl_c=0;
|
|
for _repl in $REPLACE; do
|
|
_replacements[$_repl_c]="$_repl";
|
|
((_repl_c+=+1));
|
|
done
|
|
if [[ $_repl_c -eq 0 ]]; then
|
|
logger $LLError "Number of replacements ($_repl_c) must not be 0! Check variables in <$Conf_File>!";
|
|
exit 1;
|
|
fi
|
|
if [[ $_repl_c -ne $_pre_c ]]; then
|
|
logger $LLError "Number of prefixes ($_pre_c) is not equal to number of replacements ($_repl_c)! Check variables in <$Conf_File>!";
|
|
exit 1;
|
|
fi
|
|
|
|
logger $LLMdtry "_______________________ ${0} __________________________";
|
|
logger $LLMdtry "$($DATE +'%h:%n %d.%m.%Y'), File count: $_file_count";
|
|
for((_pc=0;_pc<${#_prefixes[*]};_pc++)); do
|
|
logger $LLMdtry "${_prefixes[$_pc]} -> ${_replacements[$_pc]}";
|
|
done
|
|
|
|
local _succ_count=0; # successfully copied files
|
|
local _done_count=0; # files where already there
|
|
local _dup_diff=0; # there already is a duplicate file
|
|
local _miss_count=0; # after copy file sizes do not match
|
|
local _fail_count=0; # file is missing after copy.
|
|
|
|
declare -A _original_files;
|
|
declare -A _date_times;
|
|
# Start copy loop
|
|
for((_fc=0;_fc<${#Files_N_Paths[*]};_fc++)); do
|
|
|
|
# we only start copying, when there are no more copy procedures ongoing, than THRESHOLD!
|
|
while [ ${#_original_files[*]} -eq $THRESHOLD ]; do # as long as the length of original_files array is THRESHOLD
|
|
logger $LLDebug "$_fc Threshold reached.";
|
|
for _pid in "${!_original_files[@]}"; do # check all process ids in _original_files
|
|
if [ -n "${_pid}" -a -d "/proc/${_pid}" ]; then
|
|
# Copy process still running
|
|
logger $LLDebug "Copying of ${_original_files[$_pid]} (${_pid}) still running.";
|
|
else
|
|
# Copy process done, compare file sizes
|
|
logger $LLDebug "Check target »${_target_files[$_pid]}«";
|
|
check_local_remote_filesize "${_original_files[$_pid]}" "${_target_files[$_pid]}";
|
|
local _ret_clrfs=$?;
|
|
if [ $_ret_clrfs -eq 0 ]; then # sizes match
|
|
if [ ! -z "$TOSSH" ]; then # otherwise cp is done with -p
|
|
local _file_base_name=$(basename -- "${_target_files[$_pid]}");
|
|
$S $TOSSH "touch -d '${_date_times[$_pid]}' ${_target_files[$_pid]}"; # restore file creation date
|
|
fi
|
|
((_succ_count+=1))
|
|
logger $LLDebug "$SUC_C ${_original_files[$_pid]}";
|
|
elif [[ $_ret_clrfs -eq 1 ]]; then
|
|
logger $LLError "Sizes of local and remote file size do not match! »${_original_files[$_pid]}« <-> »${_target_files[$_pid]}«";
|
|
((_miss_count+=1))
|
|
else # 2: file does not exist
|
|
logger $LLError "Remote file »${_target_files[$_pid]}« does not exist!";
|
|
((_fail_count+=1))
|
|
fi
|
|
unset _original_files["${_pid}"];
|
|
unset _target_files["${_pid}"];
|
|
unset _date_times["${_pid}"];
|
|
fi
|
|
done # for _pid in "${!_original_files[@]}"; do
|
|
done
|
|
|
|
# set file creation date by exif date
|
|
local _datetime=$($EXIFTOOL -DateTimeOriginal -s -s -s -d '%F %H:%M:%S' "${Files_N_Paths[$_fc]}");
|
|
|
|
# construct server path
|
|
local _date_y=$(date +%Y --date="$_datetime");
|
|
local _date_m=$(date +%m --date="$_datetime");
|
|
local _date_d=$(date +%d --date="$_datetime");
|
|
# _target_path format : YYYY/YYYY_MM_DD_PREFIX
|
|
_target_path="${SYNC_TARGET}${_date_y}/${_date_y}_${_date_m}_${_date_d}_${_prefixes[0]}"; # Path on server for target file
|
|
_extra_path="${SYNC_EXTRA}${_date_y}/${_date_y}_${_date_m}_${_date_d}_${_prefixes[0]}"; # Path on server for target file
|
|
logger $LLDebug "Server target path »$_target_path«";
|
|
|
|
# create new file name
|
|
local _file_base_name=$(basename -- "${Files_N_Paths[$_fc]}");
|
|
for((_rc=0;_rc<${#_prefixes[*]};_rc++)); do
|
|
_pref="${_prefixes[$_rc]}";
|
|
_repl="${_replacements[$_rc]}";
|
|
_file_repl_name="${_file_base_name/$_repl/$_pref}";
|
|
done
|
|
logger $LLDebug "$_file_base_name -> $_file_repl_name";
|
|
# Test for existing files
|
|
check_local_remote_filesize "${Files_N_Paths[$_fc]}" "${_target_path}/${_file_repl_name}";
|
|
CLRFSres=$?
|
|
if [ $CLRFSres -eq 0 ]; then # file is already there
|
|
((_done_count+=1))
|
|
logger $LLInfo "File <${Files_N_Paths[$_fc]}> already exists as <${_target_path}/${_file_repl_name}>, skipping.";
|
|
elif [ $CLRFSres -eq 2 ]; then # file does not exist, just start copy
|
|
if [ ! -z $TOSSH ]; then #
|
|
logger $LLInfo "Copying file <${Files_N_Paths[$_fc]}> -> Server<${_target_path}/${_file_repl_name}>";
|
|
cat "${Files_N_Paths[$_fc]}" | $S $TOSSH "mkdir -p ${_target_path};cat > '${_target_path}/${_file_repl_name}'" &
|
|
_cp_pid=$!;
|
|
else # no TOSSH
|
|
if [ ! -e "${_target_path}" ]; then # file path does not exist
|
|
mkdir -p "${_target_path}";
|
|
fi
|
|
if [ -d "${_target_path}" ]; then
|
|
cp -p "${Files_N_Paths[$_fc]}" "${_target_path}/${_file_repl_name}" &
|
|
_cp_pid=$!;
|
|
else
|
|
logger $LLError "Unable to create ${_target_path}! Skipping file ${Files_N_Paths[$_fc]}!";
|
|
continue;
|
|
fi
|
|
fi
|
|
_original_files+=( ["${_cp_pid}"]="${Files_N_Paths[$_fc]}" );
|
|
_target_files+=( ["${_cp_pid}"]="${_target_path}/${_file_repl_name}" );
|
|
_date_times+=( ["${_cp_pid}"]="${_datetime}" );
|
|
else # 1:
|
|
logger $LLInfo "Copying file <${Files_N_Paths[$_fc]}> to Server<${_extra_path}/${_file_repl_name}>; creating link to ${_target_path}/${_file_repl_name}.";
|
|
if [ ! -z $TOSSH ]; then #
|
|
cat "${Files_N_Paths[$_fc]}" | $S $TOSSH "mkdir -p ${_extra_path};cat > ${_extra_path}/${_file_repl_name}" &
|
|
_cp_pid=$!;
|
|
$S $TOSSH "ln -s ${_target_path}/${_file_repl_name} ${_extra_path}/_${_file_repl_name}" &
|
|
else # no TOSSH
|
|
if [ ! -e "${_extra_path}" ]; then # path does not exist
|
|
mkdir -p "${_extra_path}";
|
|
fi
|
|
if [ -d "${_extra_path}" ]; then # check if target path exists as folder
|
|
cp -p "${Files_N_Paths[$_fc]}" "${_extra_path}/${_file_repl_name}" &
|
|
_cp_pid=$!;
|
|
ln -s "${_target_path}/${_file_repl_name}" "${_extra_path}/_${_file_repl_name}" &
|
|
else
|
|
logger $LLError "Unable to create ${_extra_path}! Skipping file ${Files_N_Paths[$_fc]}!";
|
|
((_miss_count+=1))
|
|
continue;
|
|
fi
|
|
fi
|
|
((_dup_diff+=1))
|
|
_original_files+=( ["${_cp_pid}"]="${Files_N_Paths[$_fc]}" );
|
|
_target_files+=( ["${_cp_pid}"]="${_target_path}/${_file_repl_name}" );
|
|
_date_times+=( ["${_cp_pid}"]="${_datetime}" );
|
|
fi
|
|
|
|
done # copy loop
|
|
logger $LLMdtry "Tried to copy $_file_count files.\n$_done_count where already there,\n$_succ_count where copied of which $_dup_diff where of different size and got copied to the extra location,\n$_miss_count have different size after copy and\n$_fail_count could not be copied.";
|
|
} ## end main
|
|
|
|
|
|
# Get the options from commandline
|
|
while getopts ":hc:f:" Option 2>/dev/null; do
|
|
case $Option in
|
|
h) # display Help
|
|
usage
|
|
cleanup 1
|
|
;;
|
|
c) # configuration file to use
|
|
Conf_File=$OPTARG;
|
|
;;
|
|
f) # folder to copy
|
|
Cp_Folder=$OPTARG;
|
|
;;
|
|
\?)
|
|
echo "Invalid option: -$OPTARG";
|
|
exit 1
|
|
;;
|
|
:)
|
|
echo "Option -$OPTARG requires an argument.";
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if [[ -z "$Conf_File" ]]; then
|
|
echo "The option -c is mandatory! Provide a configuration file!";
|
|
cleanup 1;
|
|
fi
|
|
if [[ -f "${Conf_File}" ]]; then
|
|
echo "Using configuration file $Conf_File.";
|
|
source $Conf_File;
|
|
else
|
|
echo "ERROR: $Conf_File is not a file!";
|
|
cleanup 1;
|
|
fi
|
|
|
|
if [[ -z "$Cp_Folder" ]]; then
|
|
echo "The option -f is mandatory! Provide a folder to copy!";
|
|
cleanup 1;
|
|
fi
|
|
if [[ -d "${Cp_Folder}" ]]; then
|
|
echo "Copying folder $Cp_Folder.";
|
|
else
|
|
echo "ERROR: $Cp_Folder is not a folder!";
|
|
cleanup 1;
|
|
fi
|
|
|
|
## Check binaries availability
|
|
which_is SUDO 'sudo';
|
|
if [ $? -ne 0 ]; then echo $SUDO; exit 1; fi
|
|
|
|
which_is DATE 'date';
|
|
if [ $? -ne 0 ]; then echo $DATE; exit 1; fi
|
|
|
|
which_is LN 'ln';
|
|
if [ $? -ne 0 ]; then echo $LN; exit 1; fi
|
|
|
|
which_is RM 'rm';
|
|
if [ $? -ne 0 ]; then echo $RM; exit 1; fi
|
|
|
|
which_is SSH 'ssh';
|
|
if [ $? -ne 0 ]; then echo $SSH; exit 1; fi
|
|
|
|
which_is TOUCH 'touch';
|
|
if [ $? -ne 0 ]; then echo $TOUCH; exit 1; fi
|
|
|
|
which_is CHOWN 'chown';
|
|
if [ $? -ne 0 ]; then echo $CHOWN; exit 1; fi
|
|
|
|
which_is EXIFTOOL 'exiftool';
|
|
if [ $? -ne 0 ]; then echo $EXIFTOOL; exit 1; fi
|
|
|
|
which_is NC 'nc';
|
|
if [ $? -ne 0 ]; then echo $NC; exit 1; fi
|
|
|
|
Run_Date=$($DATE +%Y_%m_%d)
|
|
Log_File="/tmp/${Run_Date}_photosync.log";
|
|
|
|
# Check if target is on remote server (TOSSH nonzero length)
|
|
if [ ! -z $TOSSH ]; then
|
|
logger $LLDebug "Will copy to »$TOSSH«!";
|
|
# Check validity of ssh data
|
|
if [ "$SSHUSER" ] && [ "$SSHPORT" ]; then
|
|
S="$SSH -p $SSHPORT -l $SSHUSER";
|
|
else
|
|
logger $LLError "Invalid ssh command, port or user: »$SSH« »$SSHPORT« «$SSHUSER«";
|
|
cleanup 1;
|
|
fi
|
|
# Check availability of target server
|
|
$NC -z -w1 $TOSSH $SSHPORT > /dev/null
|
|
if [ $? -ne 0 ]; then
|
|
logger $LLError "The server $TOSSH is not reachable.";
|
|
cleanup 1;
|
|
fi
|
|
fi
|
|
|
|
# Add slash to paths if missing
|
|
if [ "${SYNC_TARGET:${#SYNC_TARGET}-1:1}" != "/" ]; then
|
|
SYNC_TARGET=$SYNC_TARGET/
|
|
fi
|
|
|
|
if [ "${CONFIG_NC_DATA:${#CONFIG_NC_DATA}-1:1}" != "/" ]; then
|
|
CONFIG_NC_DATA=$CONFIG_NC_DATA/
|
|
fi
|
|
|
|
if [ "${CONFIG_NC_LOGPATH:${#CONFIG_NC_LOGPATH}-1:1}" != "/" ]; then
|
|
CONFIG_NC_LOGPATH=$CONFIG_NC_LOGPATH/
|
|
fi
|
|
# run the program after basics where checked.
|
|
|
|
main
|
|
|
|
cleanup 0;
|