390 lines
12 KiB
Bash
Executable File
390 lines
12 KiB
Bash
Executable File
#!/bin/bash
|
|
## Syncs all data in folder given to the configured destiny.
|
|
|
|
## Log levels for messages
|
|
LLMUTE=0
|
|
LLMDTRY=1
|
|
LLERROR=2
|
|
LLWARNING=3
|
|
LLINFO=4
|
|
LLDEBUG=5
|
|
## Actual log level of the logger used
|
|
LOGLEVEL=$LLDEBUG
|
|
|
|
## Write log of the script.
|
|
## $1 Loglevel to log the message with.
|
|
## $2 message to log.
|
|
function __logger() {
|
|
local LOGGINGLEVEL=$1;
|
|
local LOGMESSAGE=$2;
|
|
local _NOW=$($DATE +%Y%m%d_%H:%M:%S);
|
|
if [[ $LOGLEVEL -ge $LOGGINGLEVEL ]]; then
|
|
echo "photosync $_NOW: ${LOGMESSAGE}" >> $LOGFILE;
|
|
fi
|
|
if [[ $LOGLEVEL -eq $LLDEBUG ]]; then
|
|
echo "photosync $_NOW: ${LOGMESSAGE}";
|
|
fi
|
|
}
|
|
|
|
## Display Help
|
|
function __usage()
|
|
{
|
|
echo "$0 syncs all data from a source folder to a target folder. Target folder may optionally be on a remote ssh server."
|
|
echo
|
|
echo "Syntax: scriptTemplate [-h|c|v|V]"
|
|
echo "options:"
|
|
echo "h Print this Help."
|
|
echo "c configuration file with all parameters for syncing your data."
|
|
# echo "s Verbose mode."
|
|
echo "V Print software version and exit."
|
|
echo
|
|
}
|
|
|
|
## Cleanup function to exit script gracefully
|
|
# param1 Exit value to end with
|
|
function __cleanup() {
|
|
local EXITVAL=$1
|
|
|
|
|
|
exit $EXITVAL;
|
|
}
|
|
|
|
## Returns the binaries full path if existent.
|
|
## $1 referenced variable the found path to the binary is stored and returned in.
|
|
## $2 Name of the binary to look for.
|
|
## return 0 in case of success (binary found), 1 on fail. $1 then contains "$2 not found!".
|
|
function __which_is() {
|
|
declare -n RETVAL=$1;
|
|
EXECUTABLE=$2;
|
|
WITHFULLPATH=$(which ${EXECUTABLE} 2>/dev/null);
|
|
if [ -x "$WITHFULLPATH" ]; then
|
|
RETVAL="$WITHFULLPATH";
|
|
return 0;
|
|
else
|
|
RETVAL="$EXECUTABLE not found!";
|
|
return 1;
|
|
fi
|
|
}
|
|
|
|
## Checks given folder for files with certain extensions. Subfolders are checked recursively.
|
|
## $1 Folder to check.
|
|
## $2 space separated list of extensions to check for. e.g.: "*.jpg *.JPG *.mp4 *.MP4".
|
|
## returns Count of files found with the desired extension.
|
|
__check_folder() {
|
|
local _FOLDER=$1;
|
|
local _EXTENSIONS=$2;
|
|
local _FILE_C=0;
|
|
if [ -d "$_FOLDER" ]; then # is a folder
|
|
local _ITEM="";
|
|
local _FC=${#FILESNPATHS[@]};
|
|
pushd "$_FOLDER" 1>/dev/null;
|
|
for _ITEM in ${_EXTENSIONS}; do # lookup all files with the desired extensions
|
|
if [[ -f "$_ITEM" ]]; then # is a file
|
|
FILESNPATHS[$_FC]="$(pwd)/$_ITEM";
|
|
((_FC+=1));
|
|
((_FILE_C+=1));
|
|
fi
|
|
done
|
|
# check subfolders if exist
|
|
local _SUBFOLDERS="$(/usr/bin/ls . 2>/dev/null)";
|
|
local _CNT=0;
|
|
for _SUBITEM in ${_SUBFOLDERS}; do
|
|
if [[ -d "$_SUBITEM" ]]; then
|
|
__check_folder "$_SUBITEM" "$_EXTENSIONS";
|
|
_CNT=$?
|
|
((_FILE_C+=_CNT));
|
|
fi
|
|
done
|
|
popd 1>/dev/null;
|
|
fi
|
|
return $_FILE_C
|
|
}
|
|
|
|
## Check filesizes of local and remote file
|
|
## $1 Path of local file
|
|
## $2 Path of remote file
|
|
check_local_remote_filesize() {
|
|
local LocalFile=$1;
|
|
local RemoteFile=$2;
|
|
local LocFileSize=$(stat -c%s "${LocalFile}");
|
|
__logger $LLDEBUG "Local file size : $LocFileSize <${FILESNPATHS[$FILE_C]}>";
|
|
if [ -n $TOSSH ]; then
|
|
local RemFileSize=$($S $TOSSH "stat -c%s ${RemoteFile}" 2>/dev/null);
|
|
else
|
|
local RemFileSize=$(stat -c%s "${RemoteFile}");
|
|
fi
|
|
__logger $LLDEBUG "Remote file size : ${RemFileSize} <${TARGETPATH}/${FILEBASENAME}>";
|
|
if [[ $RemFileSize != ?(-)+([0-9]) ]]; then # value should be integer to test true
|
|
__logger $LLERROR "Remote file »${RemoteFile}« does not exist!";
|
|
return 2;
|
|
fi
|
|
if [ $LocFileSize -ne $RemFileSize ]; then
|
|
__logger $LLERROR "ERROR! Sizes of local and remote file size do not match! <${LocalFile}>!";
|
|
return 1;
|
|
fi
|
|
return 0;
|
|
}
|
|
|
|
## Main function
|
|
function __main() {
|
|
|
|
# create file list to move to copy to server
|
|
declare -A FILESNPATHS; # associative array
|
|
__check_folder "$CPFOLDER" "$EXTENSIONS";
|
|
FILE_COUNT=$?
|
|
__logger $LLDEBUG "Count of files with extensions »$EXTENSIONS« is $FILE_COUNT.";
|
|
if [[ $FILE_COUNT -eq 0 ]]; then
|
|
__logger $LLERROR "No files to copy!"
|
|
__cleanup 1;
|
|
fi
|
|
|
|
# Make Lists for prefixes to replace
|
|
declare -A PREFIXES;
|
|
PRE_C=0;
|
|
for PRE in $PREFIX; do
|
|
PREFIXES[$PRE_C]="$PRE"; #echo "Arrayinhalt an der Stelle $INP_C: ${FILE[$INP_C]}";
|
|
((PRE_C=PRE_C+1));
|
|
done
|
|
if [ $PRE_C -eq 0 ]; then
|
|
__logger $LLERROR "ERROR! Number of prefixes ($PRE_C) must not be 0! Check variables in <$SOURCECONFFILE>!";
|
|
exit 1;
|
|
fi
|
|
|
|
# Make List with replacement prefixes
|
|
declare -A REPLACEMENTS;
|
|
REP_C=0;
|
|
for REP in $REPLACE; do
|
|
REPLACEMENTS[$REP_C]="$REP";
|
|
((REP_C=REP_C+1));
|
|
done
|
|
if [ $REP_C -eq 0 ]; then
|
|
__logger $LLERROR "ERROR! Number of replacements ($REP_C) must not be 0! Check variables in <$SOURCECONFFILE>!";
|
|
exit 1;
|
|
fi
|
|
if [ $REP_C -ne $PRE_C ]; then
|
|
__logger $LLERROR "ERROR! Number of prefixes ($PRE_C) is not equal to number of replacements ($REP_C)! Check variables in <$SOURCECONFFILE>!";
|
|
exit 1;
|
|
fi
|
|
|
|
__logger $LLMDTRY "_______________________ cloud_photosync.sh __________________________";
|
|
__logger $LLMDTRY "$($DATE '+%h:%n %d.%m.%Y')";
|
|
__logger $LLMDTRY "File count: $FILE_COUNT";
|
|
for((C=0;C<${#PREFIXES[*]};C++)); do
|
|
__logger $LLMDTRY "${PREFIXES[C]} -> ${REPLACEMENTS[C]}";
|
|
done
|
|
|
|
# Start copy loop
|
|
for((FILE_C=0;FILE_C<${#FILESNPATHS[*]};FILE_C++)); do
|
|
# we only start copying, when there are no more copy procedures ongoing, than THRESHOLD!
|
|
while [ ${#ORIGINALFILES[*]} -eq $THRESHOLD ]; do
|
|
__logger $LLDEBUG "$FILE_C Threshold reached.";
|
|
for CPID in "${!ORIGINALFILES[@]}"; do
|
|
if [ -n "${CPID}" -a -d "/proc/${CPID}" ]; then
|
|
# Prozess läuft noch, check nicht möglich
|
|
__logger $LLDEBUG "${CPID} still running";
|
|
else
|
|
# hochladen beendet, kann Größe testen
|
|
__logger $LLDEBUG "# Check on target ${TARGETFILES[$CPID]}";
|
|
check_local_remote_filesize "${ORIGINALFILES[$CPID]}" "${TARGETFILES[$CPID]}";
|
|
RET=$?;
|
|
if [ $RET -eq 0 ]; then # sizes match
|
|
if [ ! -z "$TOSSH" ]; then # otherwise cp is done with -p
|
|
FILEBASENAME=$(basename -- "${TARGETFILES[$CPID]}");
|
|
$S $TOSSH "touch -d '${DATETIMES[$CPID]}' ${TARGETFILES[$CPID]}"; # restore file creation date
|
|
fi
|
|
((SUC_C=SUC_C+1))
|
|
__logger $LLDEBUG "$SUC_C ${ORIGINALFILES[$CPID]}";
|
|
else # sizes don't match or file does not exist
|
|
((MIS_C=MIS_C+1))
|
|
fi
|
|
unset ORIGINALFILES["${CPID}"];
|
|
unset TARGETFILES["${CPID}"];
|
|
unset DATETIMES["${CPID}"];
|
|
fi
|
|
done # for CPID in "${!ORIGINALFILES[@]}"; do
|
|
done
|
|
|
|
# set file creation date by exif date
|
|
DATETIME=$($EXIFTOOL -DateTimeOriginal -s -s -s -d '%F %H:%M:%S' "${FILESNPATHS[$FILE_C]}");
|
|
|
|
# construct server path
|
|
DATE_Y=$(date +%Y --date="$DATETIME");
|
|
DATE_M=$(date +%m --date="$DATETIME");
|
|
DATE_D=$(date +%d --date="$DATETIME");
|
|
# Targetpath format : YYYY/YYYY_MM_DD_PREFIX
|
|
TARGETPATH="${SYNC_TARGET}${DATE_Y}/${DATE_Y}_${DATE_M}_${DATE_D}_${PREFIXES[0]}"; # Path on server for target file
|
|
EXTRAPATH="${SYNC_EXTRA}${DATE_Y}/${DATE_Y}_${DATE_M}_${DATE_D}_${PREFIXES[0]}"; # Path on server for target file
|
|
__logger $LLDEBUG "Server target path »$TARGETPATH«";
|
|
|
|
# create new file name
|
|
FILEBASENAME=$(basename -- "${FILESNPATHS[$FILE_C]}");
|
|
for((REPL_C=0;REPL_C<${#PREFIXES[*]};REPL_C++)); do
|
|
PREF="${PREFIXES[$REPL_C]}";
|
|
REPL="${REPLACEMENTS[$REPL_C]}";
|
|
FILEBASENAME="${FILEBASENAME/$REPL/$PREF}";
|
|
done
|
|
# Test for existing files
|
|
check_local_remote_filesize "${FILESNPATHS[$FILE_C]}" "${TARGETPATH}/${FILEBASENAME}";
|
|
CLRFSres=$?
|
|
if [ $CLRFSres -eq 0 ]; then
|
|
__logger $LLINFO "File <${FILESNPATHS[$FILE_C]}> already exists as <${TARGETPATH}/${FILEBASENAME}>, skipping.";
|
|
elif [ $CLRFSres -eq 1 ]; then # file does not exist
|
|
if [ ! -z $TOSSH ]; then #
|
|
__logger $LLINFO "Copying file <${FILESNPATHS[$FILE_C]}> -> Server<${TARGETPATH}/${FILEBASENAME}>";
|
|
cat "${FILESNPATHS[$FILE_C]}" | $S $TOSSH "mkdir -p ${TARGETPATH};cat > '${TARGETPATH}/${FILEBASENAME}'" &
|
|
COPYPID=$!;
|
|
else # no TOSSH
|
|
if [ ! -e "${TARGETPATH}" ]; then # file path does not exist
|
|
mkdir -p "${TARGETPATH}";
|
|
fi
|
|
if [ -d "${TARGETPATH}" ]; then
|
|
cp -p "${FILESNPATHS[$FILE_C]}" "${TARGETPATH}/${FILEBASENAME}" &
|
|
COPYPID=$!;
|
|
else
|
|
__logger $LLERROR "Unable to create ${TARGETPATH}! Skipping file ${FILESNPATHS[$FILE_C]}!";
|
|
continue;
|
|
fi
|
|
fi
|
|
ORIGINALFILES+=( ["${COPYPID}"]="${FILESNPATHS[$FILE_C]}" );
|
|
TARGETFILES+=( ["${COPYPID}"]="${TARGETPATH}/${FILEBASENAME}" );
|
|
DATETIMES+=( ["${COPYPID}"]="${DATETIME}" );
|
|
else # 2: different size-> start copying
|
|
if [ ! -z $TOSSH ]; then #
|
|
__logger $LLINFO "Copying file <${FILESNPATHS[$FILE_C]}> -> Server<${EXTRAPATH}/${FILEBASENAME}>";
|
|
cat "${FILESNPATHS[$FILE_C]}" | $S $TOSSH "mkdir -p ${EXTRAPATH};cat > ${EXTRAPATH}/${FILEBASENAME} && ln -s ${TARGETPATH}/${FILEBASENAME} ${EXTRAPATH}/_${FILEBASENAME}" &
|
|
COPYPID=$!;
|
|
else # no TOSSH
|
|
if [ ! -e "${TARGETPATH}" ]; then # path does not exist
|
|
mkdir -p "${TARGETPATH}";
|
|
fi
|
|
if [ -d "${TARGETPATH}" ]; then # check if target path exists as folder
|
|
cp -p "${FILESNPATHS[$FILE_C]}" "${TARGETPATH}/${FILEBASENAME}" &
|
|
COPYPID=$!;
|
|
else
|
|
__logger $LLERROR "Unable to create ${TARGETPATH}! Skipping file ${FILESNPATHS[$FILE_C]}!";
|
|
continue;
|
|
fi
|
|
fi
|
|
ORIGINALFILES+=( ["${COPYPID}"]="${FILESNPATHS[$FILE_C]}" );
|
|
TARGETFILES+=( ["${COPYPID}"]="${TARGETPATH}/${FILEBASENAME}" );
|
|
DATETIMES+=( ["${COPYPID}"]="${DATETIME}" );
|
|
fi
|
|
|
|
done # Start copy loop
|
|
} ## end main
|
|
|
|
|
|
# Get the options from commandline
|
|
while getopts ":hc:f:" option 2>/dev/null; do
|
|
case $option in
|
|
h) # display Help
|
|
__usage
|
|
__cleanup 1
|
|
;;
|
|
c) # configuration file to use
|
|
CONFFILE=$OPTARG;
|
|
;;
|
|
f) # folder to copy
|
|
CPFOLDER=$OPTARG;
|
|
;;
|
|
\?)
|
|
echo "Invalid option: -$OPTARG";
|
|
exit 1
|
|
;;
|
|
:)
|
|
echo "Option -$OPTARG requires an argument.";
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if [[ -z "$CONFFILE" ]]; then
|
|
echo "The option -c is mandatory! Provide a configuration file!";
|
|
__cleanup 1;
|
|
fi
|
|
if [[ -f "${CONFFILE}" ]]; then
|
|
echo "Using configuration file $CONFFILE.";
|
|
source $CONFFILE;
|
|
else
|
|
echo "ERROR: $CONFFILE is not a file!";
|
|
__cleanup 1;
|
|
fi
|
|
|
|
if [[ -z "$CPFOLDER" ]]; then
|
|
echo "The option -f is mandatory! Provide a folder to copy!";
|
|
__cleanup 1;
|
|
fi
|
|
if [[ -d "${CPFOLDER}" ]]; then
|
|
echo "Copying folder $CPFOLDER.";
|
|
else
|
|
echo "ERROR: $CPFOLDER is not a folder!";
|
|
__cleanup 1;
|
|
fi
|
|
|
|
## Check binaries availability
|
|
__which_is SUDO 'sudo';
|
|
if [ $? -ne 0 ]; then echo $SUDO; exit 1; fi
|
|
|
|
__which_is DATE 'date';
|
|
if [ $? -ne 0 ]; then echo $DATE; exit 1; fi
|
|
|
|
__which_is LN 'ln';
|
|
if [ $? -ne 0 ]; then echo $LN; exit 1; fi
|
|
|
|
__which_is RM 'rm';
|
|
if [ $? -ne 0 ]; then echo $RM; exit 1; fi
|
|
|
|
__which_is SSH 'ssh';
|
|
if [ $? -ne 0 ]; then echo $SSH; exit 1; fi
|
|
|
|
__which_is TOUCH 'touch';
|
|
if [ $? -ne 0 ]; then echo $TOUCH; exit 1; fi
|
|
|
|
__which_is CHOWN 'chown';
|
|
if [ $? -ne 0 ]; then echo $CHOWN; exit 1; fi
|
|
|
|
__which_is EXIFTOOL 'exiftool';
|
|
if [ $? -ne 0 ]; then echo $EXIFTOOL; exit 1; fi
|
|
|
|
__which_is NC 'nc';
|
|
if [ $? -ne 0 ]; then echo $NC; exit 1; fi
|
|
|
|
RUNDATE=$($DATE +%Y_%m_%d)
|
|
LOGFILE="/tmp/${RUNDATE}_photosync.log";
|
|
|
|
# Check if target is on remote server (TOSSH nonzero length)
|
|
if [ ! -z $TOSSH ]; then
|
|
__logger $LLDEBUG "Will copy to »$TOSSH«!";
|
|
# Check validity of ssh data
|
|
if [ "$SSHUSER" ] && [ "$SSHPORT" ]; then
|
|
S="$SSH -p $SSHPORT -l $SSHUSER";
|
|
else
|
|
__logger $LLERROR "Invalid ssh command, port or user: »$SSH« »$SSHPORT« «$SSHUSER«";
|
|
__cleanup 1;
|
|
fi
|
|
# Check availability of target server
|
|
$NC -z -w1 $TOSSH $SSHPORT > /dev/null
|
|
if [ $? -ne 0 ]; then
|
|
__logger $LLERROR "The server $TOSSH is not reachable.";
|
|
__cleanup 1;
|
|
fi
|
|
fi
|
|
|
|
RUNDATE=$($DATE +%Y_%m_%d)
|
|
LOGFILE="/tmp/${RUNDATE}_cloud_photosync.log";
|
|
|
|
# Add slash to paths if missing
|
|
if [ "${SYNC_TARGET:${#SYNC_TARGET}-1:1}" != "/" ]; then
|
|
SYNC_TARGET=$SYNC_TARGET/
|
|
fi
|
|
|
|
if [ "${CONFIG_NC_DATA:${#CONFIG_NC_DATA}-1:1}" != "/" ]; then
|
|
CONFIG_NC_DATA=$CONFIG_NC_DATA/
|
|
fi
|
|
|
|
if [ "${CONFIG_NC_LOGPATH:${#CONFIG_NC_LOGPATH}-1:1}" != "/" ]; then
|
|
CONFIG_NC_LOGPATH=$CONFIG_NC_LOGPATH/
|
|
fi
|
|
# run the program after basics where checked.
|
|
|
|
__main
|