From eb087ba8efdbcafe300e4030a3722f52492eee08 Mon Sep 17 00:00:00 2001
From: Stephan <accounts@janeemussja.de>
Date: Mon, 20 Nov 2023 23:22:38 +0100
Subject: [PATCH] Repo description and first version of photosync script.

---
 README.md    |   7 +
 photosync.sh | 389 +++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 396 insertions(+)
 create mode 100644 README.md
 create mode 100755 photosync.sh

diff --git a/README.md b/README.md
new file mode 100644
index 0000000..1f7f6a4
--- /dev/null
+++ b/README.md
@@ -0,0 +1,7 @@
+Photosync copies all you photos (or other files) to a new destination folder.
+
+It separates them into folders of the structure yyyy/yyyy_mm_dd/ in the target folder. when setting an ssh server, the target folder may exist there.
+
+It can change the prefix from the generic "IMG_" to something you like. I use that to distinguish photos taken with different cameras, my daughters or my wifes…
+
+Optionally a "last" symlink identifies the latest file of the last run, so not everything in the source folder is copied or checked against the target again.
diff --git a/photosync.sh b/photosync.sh
new file mode 100755
index 0000000..26b0c5f
--- /dev/null
+++ b/photosync.sh
@@ -0,0 +1,389 @@
+#!/bin/bash
+## Syncs all data in folder given to the configured destiny.
+
+## Log levels for messages
+LLMUTE=0
+LLMDTRY=1
+LLERROR=2
+LLWARNING=3
+LLINFO=4
+LLDEBUG=5
+## Actual log level of the logger used
+LOGLEVEL=$LLDEBUG
+
+## Write log of the script.
+## $1 Loglevel to log the message with.
+## $2 message to log.
+function __logger() {
+	local LOGGINGLEVEL=$1;
+	local LOGMESSAGE=$2;
+	local _NOW=$($DATE +%Y%m%d_%H:%M:%S);
+	if [[ $LOGLEVEL -ge $LOGGINGLEVEL ]]; then
+		echo "photosync $_NOW: ${LOGMESSAGE}" >> $LOGFILE;
+	fi
+	if [[ $LOGLEVEL -eq $LLDEBUG ]]; then
+		echo "photosync $_NOW: ${LOGMESSAGE}";
+	fi
+}
+
+## Display Help
+function __usage()
+{
+   echo "$0 syncs all data from a source folder to a target folder. Target folder may optionally be on a remote ssh server."
+   echo
+   echo "Syntax: scriptTemplate [-h|c|v|V]"
+   echo "options:"
+   echo "h     Print this Help."
+   echo "c     configuration file with all parameters for syncing your data."
+#   echo "s     Verbose mode."
+   echo "V     Print software version and exit."
+   echo
+}
+
+## Cleanup function to exit script gracefully
+# param1 Exit value to end with
+function __cleanup() {
+	local EXITVAL=$1
+	
+
+	exit $EXITVAL;
+}
+
+## Returns the binaries full path if existent.
+## $1 referenced variable the found path to the binary is stored and returned in.
+## $2 Name of the binary to look for.
+## return 0 in case of success (binary found), 1 on fail. $1 then contains "$2 not found!".
+function __which_is() {
+	declare -n RETVAL=$1;
+	EXECUTABLE=$2;
+	WITHFULLPATH=$(which ${EXECUTABLE} 2>/dev/null);
+	if [ -x "$WITHFULLPATH" ]; then
+		RETVAL="$WITHFULLPATH";
+		return 0;
+	else
+		RETVAL="$EXECUTABLE not found!";
+		return 1;
+	fi
+}
+
+## Checks given folder for files with certain extensions. Subfolders are checked recursively. 
+## $1 Folder to check.
+## $2 space separated list of extensions to check for. e.g.: "*.jpg *.JPG *.mp4 *.MP4".
+## returns Count of files found with the desired extension.
+__check_folder() {
+	local _FOLDER=$1;
+	local _EXTENSIONS=$2;
+	local _FILE_C=0;
+	if [ -d "$_FOLDER" ]; then	# is a folder
+		local _ITEM="";
+		local _FC=${#FILESNPATHS[@]};
+		pushd "$_FOLDER" 1>/dev/null;
+		for _ITEM in ${_EXTENSIONS}; do # lookup all files with the desired extensions
+			if [[ -f "$_ITEM" ]]; then # is a file
+				FILESNPATHS[$_FC]="$(pwd)/$_ITEM";
+ 				((_FC+=1));
+ 				((_FILE_C+=1));
+			fi
+		done
+		# check subfolders if exist
+		local _SUBFOLDERS="$(/usr/bin/ls . 2>/dev/null)";
+		local _CNT=0;
+		for _SUBITEM in ${_SUBFOLDERS}; do
+			if [[ -d "$_SUBITEM" ]]; then
+				__check_folder "$_SUBITEM" "$_EXTENSIONS";
+				_CNT=$?
+				((_FILE_C+=_CNT));
+			fi
+		done
+		popd 1>/dev/null;
+	fi
+	return $_FILE_C
+}
+
+## Check filesizes of local and remote file
+## $1 Path of local file
+## $2 Path of remote file
+check_local_remote_filesize() {
+	local LocalFile=$1;
+	local RemoteFile=$2;
+	local LocFileSize=$(stat -c%s "${LocalFile}");
+	__logger $LLDEBUG "Local file size  : $LocFileSize <${FILESNPATHS[$FILE_C]}>";
+	if [ -n $TOSSH ]; then
+		local RemFileSize=$($S $TOSSH "stat -c%s ${RemoteFile}" 2>/dev/null);
+	else
+		local RemFileSize=$(stat -c%s "${RemoteFile}");
+	fi
+	__logger $LLDEBUG "Remote file size : ${RemFileSize} <${TARGETPATH}/${FILEBASENAME}>";
+	if [[ $RemFileSize != ?(-)+([0-9]) ]]; then  # value should be integer to test true
+		__logger $LLERROR "Remote file »${RemoteFile}« does not exist!";
+		return 2;
+	fi
+	if [ $LocFileSize -ne $RemFileSize ]; then
+		__logger $LLERROR "ERROR! Sizes of local and remote file size do not match! <${LocalFile}>!";
+		return 1;
+	fi
+	return 0;
+}
+
+## Main function
+function __main() {
+
+	# create file list to move to copy to server
+	declare -A FILESNPATHS;  # associative array
+	__check_folder "$CPFOLDER" "$EXTENSIONS";
+	FILE_COUNT=$?
+	__logger $LLDEBUG "Count of files with extensions »$EXTENSIONS« is $FILE_COUNT.";
+	if [[ $FILE_COUNT -eq 0 ]]; then
+		__logger $LLERROR "No files to copy!"
+		__cleanup 1;
+	fi
+	
+	# Make Lists for prefixes to replace
+	declare -A PREFIXES;
+	PRE_C=0;
+	for PRE in $PREFIX; do
+		PREFIXES[$PRE_C]="$PRE";  #echo "Arrayinhalt an der Stelle $INP_C: ${FILE[$INP_C]}";
+		((PRE_C=PRE_C+1));
+	done
+	if [ $PRE_C -eq 0 ]; then
+		__logger $LLERROR "ERROR! Number of prefixes ($PRE_C) must not be 0! Check variables in <$SOURCECONFFILE>!";
+		exit 1;
+	fi
+
+	# Make List with replacement prefixes
+	declare -A REPLACEMENTS;
+	REP_C=0;
+	for REP in $REPLACE; do
+		REPLACEMENTS[$REP_C]="$REP";
+		((REP_C=REP_C+1));
+	done
+	if [ $REP_C -eq 0 ]; then
+		__logger $LLERROR "ERROR! Number of replacements ($REP_C) must not be 0! Check variables in <$SOURCECONFFILE>!";
+		exit 1;
+	fi
+	if [ $REP_C -ne $PRE_C ]; then
+		__logger $LLERROR "ERROR! Number of prefixes ($PRE_C) is not equal to number of replacements ($REP_C)! Check variables in <$SOURCECONFFILE>!";
+		exit 1;
+	fi
+
+	__logger $LLMDTRY "_______________________ cloud_photosync.sh __________________________";
+	__logger $LLMDTRY "$($DATE '+%h:%n %d.%m.%Y')";
+	__logger $LLMDTRY "File count: $FILE_COUNT";
+	for((C=0;C<${#PREFIXES[*]};C++)); do
+		__logger $LLMDTRY "${PREFIXES[C]} -> ${REPLACEMENTS[C]}";
+	done
+
+	# Start copy loop
+	for((FILE_C=0;FILE_C<${#FILESNPATHS[*]};FILE_C++)); do	
+		# we only start copying, when there are no more copy procedures ongoing, than THRESHOLD!
+		while [ ${#ORIGINALFILES[*]} -eq $THRESHOLD ]; do
+			__logger $LLDEBUG "$FILE_C Threshold reached.";
+			for CPID in "${!ORIGINALFILES[@]}"; do
+				if [ -n "${CPID}" -a -d "/proc/${CPID}" ]; then
+				 	# Prozess läuft noch, check nicht möglich   
+					__logger $LLDEBUG "${CPID} still running";
+				else
+					# hochladen beendet, kann Größe testen
+					__logger $LLDEBUG "# Check on target ${TARGETFILES[$CPID]}";
+					check_local_remote_filesize "${ORIGINALFILES[$CPID]}" "${TARGETFILES[$CPID]}";
+					RET=$?;
+					if [ $RET -eq 0 ]; then # sizes match
+						if [ ! -z "$TOSSH" ]; then  # otherwise cp is done with -p
+							FILEBASENAME=$(basename -- "${TARGETFILES[$CPID]}");
+							$S $TOSSH "touch -d '${DATETIMES[$CPID]}' ${TARGETFILES[$CPID]}"; # restore file creation date
+						fi
+						((SUC_C=SUC_C+1))				
+						__logger $LLDEBUG "$SUC_C ${ORIGINALFILES[$CPID]}";
+					else	# sizes don't match or file does not exist
+						((MIS_C=MIS_C+1))				
+					fi
+					unset ORIGINALFILES["${CPID}"];
+					unset TARGETFILES["${CPID}"];
+					unset DATETIMES["${CPID}"];
+				fi
+			done  # for CPID in "${!ORIGINALFILES[@]}"; do
+		done
+
+		# set file creation date by exif date
+		DATETIME=$($EXIFTOOL -DateTimeOriginal -s -s -s -d '%F %H:%M:%S' "${FILESNPATHS[$FILE_C]}");
+
+		# construct server path	
+		DATE_Y=$(date +%Y --date="$DATETIME");
+		DATE_M=$(date +%m --date="$DATETIME");
+		DATE_D=$(date +%d --date="$DATETIME");
+		# Targetpath format : YYYY/YYYY_MM_DD_PREFIX
+		TARGETPATH="${SYNC_TARGET}${DATE_Y}/${DATE_Y}_${DATE_M}_${DATE_D}_${PREFIXES[0]}"; # Path on server for target file
+		EXTRAPATH="${SYNC_EXTRA}${DATE_Y}/${DATE_Y}_${DATE_M}_${DATE_D}_${PREFIXES[0]}"; # Path on server for target file
+		__logger $LLDEBUG "Server target path »$TARGETPATH«";
+		
+		# create new file name
+		FILEBASENAME=$(basename -- "${FILESNPATHS[$FILE_C]}");
+		for((REPL_C=0;REPL_C<${#PREFIXES[*]};REPL_C++)); do
+			PREF="${PREFIXES[$REPL_C]}";
+			REPL="${REPLACEMENTS[$REPL_C]}";
+			FILEBASENAME="${FILEBASENAME/$REPL/$PREF}";
+		done
+		# Test for existing files
+		check_local_remote_filesize "${FILESNPATHS[$FILE_C]}" "${TARGETPATH}/${FILEBASENAME}";
+		CLRFSres=$?
+		if [ $CLRFSres -eq 0 ]; then
+			__logger $LLINFO "File <${FILESNPATHS[$FILE_C]}> already exists as <${TARGETPATH}/${FILEBASENAME}>, skipping.";
+		elif [ $CLRFSres -eq 1 ]; then  # file does not exist
+			if [ ! -z $TOSSH ]; then  #
+				__logger $LLINFO "Copying file <${FILESNPATHS[$FILE_C]}> -> Server<${TARGETPATH}/${FILEBASENAME}>";
+				cat "${FILESNPATHS[$FILE_C]}" | $S $TOSSH "mkdir -p ${TARGETPATH};cat > '${TARGETPATH}/${FILEBASENAME}'" &
+				COPYPID=$!;
+			else  # no TOSSH
+				if [ ! -e "${TARGETPATH}" ]; then  # file path does not exist
+					mkdir -p "${TARGETPATH}";
+				fi
+				if [ -d "${TARGETPATH}" ]; then
+					cp -p "${FILESNPATHS[$FILE_C]}" "${TARGETPATH}/${FILEBASENAME}" &
+					COPYPID=$!;
+				else
+					__logger $LLERROR "Unable to create ${TARGETPATH}! Skipping file ${FILESNPATHS[$FILE_C]}!";
+					continue;
+				fi
+			fi
+			ORIGINALFILES+=( ["${COPYPID}"]="${FILESNPATHS[$FILE_C]}" );
+			TARGETFILES+=( ["${COPYPID}"]="${TARGETPATH}/${FILEBASENAME}" );
+			DATETIMES+=(  ["${COPYPID}"]="${DATETIME}" );
+		else # 2: different size-> start copying
+			if [ ! -z $TOSSH ]; then  #
+				__logger $LLINFO "Copying file <${FILESNPATHS[$FILE_C]}> -> Server<${EXTRAPATH}/${FILEBASENAME}>";			
+				cat "${FILESNPATHS[$FILE_C]}" | $S $TOSSH "mkdir -p ${EXTRAPATH};cat > ${EXTRAPATH}/${FILEBASENAME} && ln -s ${TARGETPATH}/${FILEBASENAME} ${EXTRAPATH}/_${FILEBASENAME}" &
+				COPYPID=$!;
+			else  # no TOSSH
+				if [ ! -e "${TARGETPATH}" ]; then  # path does not exist
+					mkdir -p "${TARGETPATH}";
+				fi
+				if [ -d "${TARGETPATH}" ]; then  # check if target path exists as folder
+					cp -p "${FILESNPATHS[$FILE_C]}" "${TARGETPATH}/${FILEBASENAME}" &
+					COPYPID=$!;
+				else
+					__logger $LLERROR "Unable to create ${TARGETPATH}! Skipping file ${FILESNPATHS[$FILE_C]}!";
+					continue;
+				fi
+			fi
+			ORIGINALFILES+=( ["${COPYPID}"]="${FILESNPATHS[$FILE_C]}" );
+			TARGETFILES+=( ["${COPYPID}"]="${TARGETPATH}/${FILEBASENAME}" );
+			DATETIMES+=( ["${COPYPID}"]="${DATETIME}" );
+		fi
+
+	done  # Start copy loop
+}  ## end main
+
+
+# Get the options from commandline
+while getopts ":hc:f:" option 2>/dev/null; do
+	case $option in
+		h) # display Help
+			__usage
+			__cleanup 1
+			;;
+		c) # configuration file to use
+			CONFFILE=$OPTARG;			
+			;;
+		f) # folder to copy
+			CPFOLDER=$OPTARG;
+			;;
+		\?)
+			echo "Invalid option: -$OPTARG";
+			exit 1
+			;;
+		:)
+      echo "Option -$OPTARG requires an argument.";
+      ;;
+	esac
+done
+
+if [[ -z "$CONFFILE" ]]; then
+	echo "The option -c is mandatory! Provide a configuration file!";
+	__cleanup 1;
+fi
+if [[ -f "${CONFFILE}" ]]; then
+	echo "Using configuration file $CONFFILE.";
+	source $CONFFILE;
+else
+	echo "ERROR: $CONFFILE is not a file!";
+	__cleanup 1;
+fi
+
+if [[ -z "$CPFOLDER" ]]; then
+	echo "The option -f is mandatory! Provide a folder to copy!";
+	__cleanup 1;
+fi
+if [[ -d "${CPFOLDER}" ]]; then
+	echo "Copying folder $CPFOLDER.";
+else
+	echo "ERROR: $CPFOLDER is not a folder!";
+	__cleanup 1;
+fi
+
+## Check binaries availability
+__which_is SUDO 'sudo';
+if [ $? -ne 0 ]; then echo $SUDO; exit 1; fi
+
+__which_is DATE 'date';
+if [ $? -ne 0 ]; then echo $DATE; exit 1; fi
+
+__which_is LN 'ln';
+if [ $? -ne 0 ]; then echo $LN; exit 1; fi
+
+__which_is RM 'rm';
+if [ $? -ne 0 ]; then echo $RM; exit 1; fi
+
+__which_is SSH 'ssh';
+if [ $? -ne 0 ]; then echo $SSH; exit 1; fi
+
+__which_is TOUCH 'touch';
+if [ $? -ne 0 ]; then echo $TOUCH; exit 1; fi
+
+__which_is CHOWN 'chown';
+if [ $? -ne 0 ]; then echo $CHOWN; exit 1; fi
+
+__which_is EXIFTOOL 'exiftool';
+if [ $? -ne 0 ]; then echo $EXIFTOOL; exit 1; fi
+
+__which_is NC 'nc';
+if [ $? -ne 0 ]; then echo $NC; exit 1; fi
+
+RUNDATE=$($DATE +%Y_%m_%d)
+LOGFILE="/tmp/${RUNDATE}_photosync.log";
+
+# Check if target is on remote server (TOSSH nonzero length)
+if [ ! -z $TOSSH ]; then
+	__logger $LLDEBUG "Will copy to »$TOSSH«!";
+  # Check validity of ssh data
+  if [ "$SSHUSER" ] && [ "$SSHPORT" ]; then
+    S="$SSH -p $SSHPORT -l $SSHUSER";
+  else
+    __logger $LLERROR "Invalid ssh command, port or user: »$SSH« »$SSHPORT« «$SSHUSER«";
+    __cleanup 1;
+  fi
+  # Check availability of target server
+  $NC -z -w1 $TOSSH $SSHPORT > /dev/null
+  if [ $? -ne 0 ]; then
+    __logger $LLERROR "The server $TOSSH is not reachable.";
+    __cleanup 1;
+  fi
+fi
+
+RUNDATE=$($DATE +%Y_%m_%d)
+LOGFILE="/tmp/${RUNDATE}_cloud_photosync.log";
+
+# Add slash to paths if missing
+if [ "${SYNC_TARGET:${#SYNC_TARGET}-1:1}" != "/" ]; then
+  SYNC_TARGET=$SYNC_TARGET/
+fi
+
+if [ "${CONFIG_NC_DATA:${#CONFIG_NC_DATA}-1:1}" != "/" ]; then
+  CONFIG_NC_DATA=$CONFIG_NC_DATA/
+fi
+
+if [ "${CONFIG_NC_LOGPATH:${#CONFIG_NC_LOGPATH}-1:1}" != "/" ]; then
+  CONFIG_NC_LOGPATH=$CONFIG_NC_LOGPATH/
+fi
+# run the program after basics where checked.
+
+__main