Merge branch 'development'

Basis ssh host config and ssh keygen script, does not hard create, but prints out commads
Fix sudo set check in init.sh file
Repository name to ssh config host name fix in the new clone script. Change was done to match ssh config create file

README.md

@@ -8,6 +8,8 @@ the clone base for one campaign and a simple crontab script to pull data from th
 - base_setup.sh: setup for the folder structure, users, etc
 - new_clone.sh: Basic clone script
 - git_sync.sh: The script to run in crontab, to sync the changes
+- switch_branch.sh: Switch from one branch to the other
+- create_ssh_config.sh: setup ssh key and ssh config entry
 
 ## Run commands
 
@@ -22,12 +24,22 @@ SUDO_USER="<SUDO USER NAME TO USE>"
 USE_SUDO=<1: use sudo, 0: no sudo used>
 ```
 
+The script will create the sudo user if needed automatically
+
 ### base_setup.sh
 
 as is, if the folder exists it will only copy the scripts, will not alter or change anything.
 
 Planned to get auto fixes for wrong ACL, etc or missing config settings
 
+### SSH Key generation
+
+A SSH Key has to be generated for each git respository that will be synced and the configuration has to be added to the ssh config file
+
+```sh
+create_ssh_config.sh [SSH Key name] [repo url full] ([jump proxy])
+```
+
 ### new_clone.sh
 
 Create a new clone
@@ -36,7 +48,7 @@ Create a new clone
 new_clone.sh [repository] [branch] ([host]) ([remote name])
 ```
 
-The [host] is the SSH Host name entry, this sill repalce any "[host]:" in the [repository]. If the [host] is not set the host set in the [repository] will be used. If nothing found the script will abort
+The [host] is the SSH Host name entry, this will replace any "[host]:" in the [repository]. If the [host] is not set the host set in the [repository] will be used. If nothing found the script will abort
 
 A [branch] name must be set all the time.
 
@@ -56,6 +68,17 @@ A [branch] name must be set all the time.
 
 An optional [remote name] can be set, if not set "origin" will be used.
 
+All sync progress will be written to the log folder inside the base folder, the log file has the name of [repository]
+
+Sample
+
+```log
+[2025-07-04 16:06:31] [<uniq id>] [START] git merge <repository folder> <remote name>/<branch>
+[2025-07-04 16:06:31] [<uniq id>] Updating <sha>..<sha>
+<git inforation>
+[2025-07-04 16:06:31] [<uniq id>] [END]
+```
+
 ## TODO
 
 Future versions will hold an incoming webhook handler and a polling scripts (systemd based)

src/bin/base_setup.sh

@@ -39,15 +39,33 @@ if [ "$(whoami)" != "root" ]; then
 	error=1;
 fi;
 
+if [ -z "$(command -v setfacl)" ]; then
+	echo "Missing setfacl command, aborting";
+	error=1;
+fi;
+if [ -z "$(command -v git)" ]; then
+	echo "Missing git command, aborting";
+	error=1;
+fi;
+
 if [ $error -eq 1 ]; then
 	exit;
 fi;
 
 # Define base folders
+# folder where all the repositories are located
 CLONE_BASE="clone-base/"
+# log folder for all log files
 LOG_FOLDER="log/"
-SCRIPT_FOLDER="scripts/"
+# sync/lone/etc scripts
+CLONE_SCRIPTS_FOLDER="scripts/"
+# any scripts that have to be run before deploy
+DEPLOY_SCRIPTS="deploy-scripts/";
+# any secrets that might be needed after clone
+SECRETS_FOLDER="secrets/"
+# overall config file
 CONFIG_FOLDER="config/"
+# admin/webhook web interface
 WWW_BASE="www/"
 WWW_WEBHOOK_INCOMING="${WWW_BASE}webhook-incoming";
 WWW_ADMIN="${WWW_BASE}admin";
@@ -66,14 +84,21 @@ if [ -d "${GIT_WEBHOOK_BASE_FOLDER}" ]; then
 	echo "[TODO] -> Not implemented: check folder, check ACL";
 	# copy scripts & default config
 	echo "~ Copy basic script and config files";
-	# git_sync.sh, init.sh, new_clone.sh, webhook.default.cfg
-	cp "${BASE_FOLDER}new_clone.sh" "${BASE_FOLDER}init.sh" "${BASE_FOLDER}git_sync.sh" "${GIT_WEBHOOK_BASE_FOLDER}${SCRIPT_FOLDER}";
-	cp "${CONFIG_BASE}/webhook.default.cfg" "${GIT_WEBHOOK_BASE_FOLDER}${CONFIG_FOLDER}";
+	# git_sync.sh, init.sh, new_clone.sh, switch_branch.sh, webhook.default.cfg
+	cp \
+		"${BASE_FOLDER}new_clone.sh" \
+		"${BASE_FOLDER}init.sh" \
+		"${BASE_FOLDER}git_sync.sh" \
+		"${BASE_FOLDER}switch_branch.sh" \
+		"${GIT_WEBHOOK_BASE_FOLDER}${CLONE_SCRIPTS_FOLDER}";
+	cp "${CONFIG_BASE}/webhook.default.cfg" \
+		"${GIT_WEBHOOK_BASE_FOLDER}${CONFIG_FOLDER}";
 	# and make sure they are all owned by the correct user
 	chown "${SUDO_USER}" \
 		"${BASE_FOLDER}new_clone.sh" \
 		"${BASE_FOLDER}init.sh" \
 		"${BASE_FOLDER}git_sync.sh" \
+		"${BASE_FOLDER}switch_branch.sh" \
 		"${CONFIG_BASE}/webhook.default.cfg";
 	# check config entries missing
 	exit;
@@ -118,20 +143,24 @@ EOF
 		fi;
 	fi;
 	# All other FOLDER
-	echo "+ Other folders for clone base: ${CLONE_BASE}, ${LOG_FOLDER}, ${SCRIPT_FOLDER}, ${CONFIG_FOLDER}, ${WWW_WEBHOOK_INCOMING}, ${WWW_ADMIN}"
+	echo "+ Other folders for clone base: ${CLONE_BASE}, ${LOG_FOLDER}, ${CLONE_SCRIPTS_FOLDER}, ${CONFIG_FOLDER}, ${WWW_WEBHOOK_INCOMING}, ${WWW_ADMIN}"
 	sudo -u "${SUDO_USER}" \
 		mkdir -p \
 		"${GIT_WEBHOOK_BASE_FOLDER}${CLONE_BASE}" \
 		"${GIT_WEBHOOK_BASE_FOLDER}${LOG_FOLDER}" \
-		"${GIT_WEBHOOK_BASE_FOLDER}${SCRIPT_FOLDER}" \
+		"${GIT_WEBHOOK_BASE_FOLDER}${CLONE_SCRIPTS_FOLDER}" \
+		"${GIT_WEBHOOK_BASE_FOLDER}${DEPLOY_SCRIPTS}" \
 		"${GIT_WEBHOOK_BASE_FOLDER}${CONFIG_FOLDER}" \
+		"${GIT_WEBHOOK_BASE_FOLDER}${SECRETS_FOLDER}" \
 		"${GIT_WEBHOOK_BASE_FOLDER}${WWW_WEBHOOK_INCOMING}" \
 		"${GIT_WEBHOOK_BASE_FOLDER}${WWW_ADMIN}";
 	# set basic folder rights, clone folder is excluded
 	sudo -u "${SUDO_USER}" chmod 700 \
 		"${GIT_WEBHOOK_BASE_FOLDER}${LOG_FOLDER}" \
-		"${GIT_WEBHOOK_BASE_FOLDER}${SCRIPT_FOLDER}" \
+		"${GIT_WEBHOOK_BASE_FOLDER}${CLONE_SCRIPTS_FOLDER}" \
+		"${GIT_WEBHOOK_BASE_FOLDER}${DEPLOY_SCRIPTS}" \
 		"${GIT_WEBHOOK_BASE_FOLDER}${CONFIG_FOLDER}" \
+		"${GIT_WEBHOOK_BASE_FOLDER}${SECRETS_FOLDER}" \
 		"${GIT_WEBHOOK_BASE_FOLDER}${WWW_WEBHOOK_INCOMING}" \
 		"${GIT_WEBHOOK_BASE_FOLDER}${WWW_ADMIN}";
 	# setfacl -m u:"${SUDO_USER}":rwx -R "${GIT_WEBHOOK_BASE_FOLDER}${CLONE_BASE}"
@@ -146,13 +175,22 @@ EOF
 	# Copy files
 	echo "+ Copy basic script and config files";
 	# git_sync.sh, init.sh, new_clone.sh, webhook.default.cfg
-	cp "${BASE_FOLDER}new_clone.sh" "${BASE_FOLDER}init.sh" "${BASE_FOLDER}git_sync.sh" "${GIT_WEBHOOK_BASE_FOLDER}${SCRIPT_FOLDER}";
-	cp "${CONFIG_BASE}/webhook.cfg" "${CONFIG_BASE}/webhook.default.cfg" "${GIT_WEBHOOK_BASE_FOLDER}${CONFIG_FOLDER}";
+	cp \
+		"${BASE_FOLDER}new_clone.sh" \
+		"${BASE_FOLDER}init.sh" \
+		"${BASE_FOLDER}git_sync.sh" \
+		"${BASE_FOLDER}switch_branch.sh" \
+		"${GIT_WEBHOOK_BASE_FOLDER}${CLONE_SCRIPTS_FOLDER}";
+	cp \
+		"${CONFIG_BASE}/webhook.cfg" \
+		"${CONFIG_BASE}/webhook.default.cfg" \
+		"${GIT_WEBHOOK_BASE_FOLDER}${CONFIG_FOLDER}";
 	# and make sure they are all owned by the correct user
 	chown "${SUDO_USER}" \
 		"${BASE_FOLDER}new_clone.sh" \
 		"${BASE_FOLDER}init.sh" \
 		"${BASE_FOLDER}git_sync.sh" \
+		"${BASE_FOLDER}switch_branch.sh" \
 		"${CONFIG_BASE}/webhook.cfg" \
 		"${CONFIG_BASE}/webhook.default.cfg";
 fi;

src/bin/create_ssh_config.sh

@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+
+# <s> [SSH Key name] [repo url] ([Jump Proxy])
+
+# ssh-keygen -t ed25519 -N "" -C "${repo_url}" -f "${ssh_key_name}"
+
+# CONFIG
+# Host <Repo name flattened>
+# 	Hostname <host name>
+# 	User git
+# 	IdentityFile ~/.ssh/<pem key name>
+# 	[ProxyJump <jump proxy>]
+
+REPOSITORY="$1";
+JUMP_PROXY="$2";
+# below are only to skip error
+BRANCH="-"
+
+BASE_FOLDER=$(dirname "$(readlink -f "$0")")"/";
+# shellcheck source=init.sh
+. "${BASE_FOLDER}init.sh";
+
+# base folder for ssh config
+SSH_CONFIG_BASE="${GIT_WEBHOOK_BASE_FOLDER}.ssh/";
+if [ ! -f "${SSH_CONFIG_BASE}config" ]; then
+	echo "[!] SSH config file does not exist: ${SSH_CONFIG_BASE}";
+	error=1;
+fi;
+
+if [[ "${REPOSITORY}" == *":"* ]]; then
+	REMOTE_HOST=$(echo "${REPOSITORY}" | cut -d ":" -f 1);
+else
+	echo "[!] Must set a full repository path with remote host for the repository";
+	error=1;
+fi;
+# if we have an ":" in the repository, split by it and replace it with the remote host
+if [[ "${REPOSITORY}" == *":"* ]]; then
+	REPOSITORY=$(echo "${REPOSITORY}" | cut -d ":" -f 2);
+fi;
+GIT_REPOSITORY_NAME=$(basename "${REPOSITORY}" .git);
+
+if [ -f "${SSH_CONFIG_BASE}${GIT_REPOSITORY_NAME}.pem" ]; then
+	echo "SSH Key: ${SSH_KEY_NAME} already exists";
+	error=1
+fi;
+
+if
+	[ -f "${SSH_CONFIG_BASE}config" ] &&
+	[ -n "${GIT_REPOSITORY_NAME}" ] &&
+	grep "Host ${GIT_REPOSITORY_NAME}" "${SSH_CONFIG_BASE}config";
+then
+	echo "[!] ssh config entry for Host '${GIT_REPOSITORY_NAME}' already exists";
+	error=1
+fi;
+if [ $error -eq 1 ]; then
+	exit;
+fi;
+
+# SUDO_COMMAND= as base
+# ssh-keygen -t ed25519 -N "" -C "${GIT_REPOSITORY_NAME}" -f "${SSH_CONFIG_BASE}${SSH_KEY_NAME}"
+# must add ".pem" if key name does not end in .pem
+SSH_COMMAND=("${SUDO_COMMAND[@]}" "ssh-keygen" "-t" "ed25519" "-N" "" "-C" "${GIT_REPOSITORY_NAME}" "-f" "${SSH_CONFIG_BASE}${GIT_REPOSITORY_NAME}.pem")
+
+# debug output for now
+echo "";
+echo "* SSH-KEYGEN:"
+echo "";
+echo "${SSH_COMMAND[*]}";
+echo "";
+echo "* ADD TO: ${SSH_CONFIG_BASE}config";
+echo "";
+echo "Host ${GIT_REPOSITORY_NAME}":
+echo "    Hostname ${REMOTE_HOST}";
+echo "    User git";
+echo "    PreferredAuthentications publickey";
+echo "    IdentityFile ~/.ssh/${GIT_REPOSITORY_NAME}.pem";
+if [ -n "${JUMP_PROXY}" ]; then
+	echo "    ProxyJump ${JUMP_PROXY}";
+fi;
+echo "";
+
+# __END__

src/bin/git_sync.sh

@@ -31,7 +31,7 @@ GIT_COMMAND=("${GIT_COMMAND_BASE[@]}" "-C" "${GIT_REPOSITORY_FOLDER}" "fetch" "-
 GIT_COMMAND=("${GIT_COMMAND_BASE[@]}" "-C" "${GIT_REPOSITORY_FOLDER}" "diff" "--stat" "HEAD" "${REMOTE_NAME}/${BRANCH}")
 changes=$("${GIT_COMMAND[@]}" 2>&1)
 if [ -n "${changes}" ]; then
-	echo "[$(date +"%Y-%m-%d %H:%M:%S")] [${unique_id}] [START] Changes" &>> "$LOG_FILE";
+	echo "[$(date +"%Y-%m-%d %H:%M:%S")] [${unique_id}] [START] git merge ${GIT_REPOSITORY_FOLDER} ${REMOTE_NAME}/${BRANCH}" &>> "$LOG_FILE";
 	GIT_COMMAND=("${GIT_COMMAND_BASE[@]}" "-C" "${GIT_REPOSITORY_FOLDER}" merge "${REMOTE_NAME}/${BRANCH}")
 	log_data=$("${GIT_COMMAND[@]}" 2>&1);
 	echo "[$(date +"%Y-%m-%d %H:%M:%S")] [${unique_id}] ${log_data}" &>> "$LOG_FILE";

src/bin/init.sh

@@ -23,7 +23,7 @@ if [ -z "$(command -v git)" ]; then
 fi;
 GIT_COMMAND_BASE=("git");
 SUDO_COMMAND=()
-if [ -n "${USE_SUDO}" ]; then
+if [ "${USE_SUDO}" == 1 ]; then
 	# if we are root -> ok, else we must be SUDO USER
 	if [ "$(whoami)" = "root" ]; then
 		SUDO_COMMAND=("sudo" "-u" "${SUDO_USER}");

src/bin/new_clone.sh

@@ -4,10 +4,13 @@
 # DATE: 2025/6/27
 # DESC: create a new basic clone
 
+# COMMAND: new_clone.sh <Repo.git> <branch> [<host>] [<Repo Target Name>] [<remote name>]
+
 REPOSITORY="$1";
 BRANCH="$2";
 REMOTE_HOST="$3";
-REMOTE_NAME="$4";
+REPOSITORY_FOLDER="$4"
+REMOTE_NAME="$5";
 if [ -z "${REMOTE_NAME}" ]; then
 	REMOTE_NAME="origin"
 fi;
@@ -24,7 +27,7 @@ if [ -z "${REMOTE_HOST}" ]; then
 	if [[ "${REPOSITORY}" == *":"* ]]; then
 		REMOTE_HOST=$(echo "${REPOSITORY}" | cut -d ":" -f 1);
 	else
-		echo "[!] Must set a remote host for the repository";
+		echo "[!] Must set a repository path with remote host for the repository";
 		error=1;
 	fi;
 fi;
@@ -32,14 +35,20 @@ fi;
 if [[ "${REPOSITORY}" == *":"* ]]; then
 	REPOSITORY=$(echo "${REPOSITORY}" | cut -d ":" -f 2);
 fi;
+# strip .git from the repository path, this is folder and ssh key Host name
+if [ -z "${REPOSITORY_FOLDER}" ]; then
+	GIT_REPOSITORY_NAME=$(basename "${REPOSITORY}" .git);
+else
+	GIT_REPOSITORY_NAME="${REPOSITORY_FOLDER}";
+fi;
 if [ $error -eq 1 ]; then
 	exit;
 fi;
 
 error=0
 echo "* Validate SSH PEM Key exist and SSH config";
-if ! grep "Host ${REMOTE_HOST}" "${GIT_WEBHOOK_BASE_FOLDER}"/.ssh/config; then
-	echo "ssh config entry for Host ${REMOTE_HOST} is missing";
+if ! grep "Host ${GIT_REPOSITORY_NAME}" "${GIT_WEBHOOK_BASE_FOLDER}"/.ssh/config; then
+	echo "[!] ssh config entry for Host ${GIT_REPOSITORY_NAME} is missing";
 	error=1;
 else
 	# make sure the identiy file is there
@@ -47,7 +56,7 @@ else
 	SSH_TEST=("${SUDO_COMMAND[@]}" "ssh" "${REMOTE_HOST}");
 	result=$("${SSH_TEST[@]}" 2>&1);
 	# this can be key or deploy key
-	validate_string="You've successfully authenticated with the "
+	validate_string="You've successfully authenticated"
 	if [[ "$result" != *"$validate_string"* ]]; then
 		echo "Could not connect to ${REMOTE_HOST}: ${result}";
 		error=1;
@@ -58,11 +67,23 @@ if [ $error -eq 1 ]; then
 	exit;
 fi;
 
-# from the repository get the last path without the .git so we have the target folder
-GIT_REPOSITORY_FOLDER=$(basename "${REPOSITORY}" .git);
+unique_id=$(uuidgen | tr -d '-' | head -c 8);
 
-echo "* New clone from ${REMOTE_HOST}:${REPOSITORY}::${BRANCH} into ${GIT_REPOSITORY_FOLDER}";
-GIT_COMMAND=("${GIT_COMMAND_BASE[@]}" "clone" "-b" "${BRANCH}" "--single-branch" "--depth" "1" "--origin" "${REMOTE_NAME}" "${REMOTE_HOST}:${REPOSITORY}" "${GIT_WEBHOOK_BASE_FOLDER}${CLONE_BASE}${GIT_REPOSITORY_FOLDER}")
-"${GIT_COMMAND[@]}";
+# log folder target
+LOG_FILE="${GIT_WEBHOOK_BASE_FOLDER}${LOG_FOLDER}${GIT_REPOSITORY_NAME}.log";
+# from the repository get the last path without the .git so we have the target folder
+echo "[$(date +"%Y-%m-%d %H:%M:%S")] [${unique_id}] [START] New clone from ${REMOTE_HOST}:${REPOSITORY}::${BRANCH} into ${GIT_REPOSITORY_NAME}" | tee -a "$LOG_FILE";
+# clone everything
+GIT_COMMAND=("${GIT_COMMAND_BASE[@]}" "clone" "-b" "${BRANCH}" "--single-branch" "--depth" "1" "--origin" "${REMOTE_NAME}" "${REMOTE_HOST}:${REPOSITORY}" "${GIT_WEBHOOK_BASE_FOLDER}${CLONE_BASE}${GIT_REPOSITORY_NAME}")
+"${GIT_COMMAND[@]}" 2>&1 | tee -a "$LOG_FILE";
+# set the repository folder
+GIT_REPOSITORY_FOLDER="${GIT_WEBHOOK_BASE_FOLDER}${CLONE_BASE}${GIT_REPOSITORY_NAME}";
+# show origin info
+GIT_COMMAND=("${GIT_COMMAND_BASE[@]}" "-C" "${GIT_REPOSITORY_FOLDER}" remote show "${REMOTE_NAME}" );
+"${GIT_COMMAND[@]}" 2>&1 | tee -a "$LOG_FILE";
+# get last log entry
+GIT_COMMAND=("${GIT_COMMAND_BASE[@]}" "-C" "${GIT_REPOSITORY_FOLDER}" log -n 1 --pretty=short --no-color);
+"${GIT_COMMAND[@]}" 2>&1 | tee -a "$LOG_FILE";
+echo "[$(date +"%Y-%m-%d %H:%M:%S")] [${unique_id}] [FINISH] clone completed" | tee -a "$LOG_FILE";
 
 # __END__

src/bin/switch_branch.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+# AUTHOR: Clemens Schwaighofer
+# DATE: 2025/7/15
+# DESC: Switch a branch, run this script if we have to switch to a different branch
+#       If not the new branch will merge into the branch that was originally selected
+
+REPOSITORY="$1";
+BRANCH="$2";
+REMOTE_NAME="$3";
+if [ -z "${REMOTE_NAME}" ]; then
+	REMOTE_NAME="origin"
+fi;
+BASE_FOLDER=$(dirname "$(readlink -f "$0")")"/";
+# shellcheck source=init.sh
+. "${BASE_FOLDER}init.sh";
+
+GIT_REPOSITORY_FOLDER="${GIT_WEBHOOK_BASE_FOLDER}${CLONE_BASE}${REPOSITORY}";
+if [ ! -d "${GIT_REPOSITORY_FOLDER}" ]; then
+	echo "[!] ${REPOSITORY} not found in clone folder";
+	echo "[!] Full path: ${GIT_REPOSITORY_FOLDER}";
+	exit;
+fi;
+LOG_FILE="${GIT_WEBHOOK_BASE_FOLDER}${LOG_FOLDER}${REPOSITORY}.log";
+unique_id=$(uuidgen | tr -d '-' | head -c 8);
+
+echo "[$(date +"%Y-%m-%d %H:%M:%S")] [${unique_id}] [START] On repository ${GIT_REPOSITORY_FOLDER} switch to branch ${REMOTE_NAME}/${BRANCH}" | tee -a "$LOG_FILE";
+# add new branch to remote
+GIT_COMMAND=("${GIT_COMMAND_BASE[@]}" "-C" "${GIT_REPOSITORY_FOLDER}" "remote" "set-branches" "--add" "${REMOTE_NAME}" "${BRANCH}")
+"${GIT_COMMAND[@]}" 2>&1 | tee -a "$LOG_FILE";
+# fetch new branch
+GIT_COMMAND=("${GIT_COMMAND_BASE[@]}" "-C" "${GIT_REPOSITORY_FOLDER}" "fetch" "--depth" "1" "${REMOTE_NAME}" "${BRANCH}")
+"${GIT_COMMAND[@]}" 2>&1 | tee -a "$LOG_FILE";
+# checkout new branch
+GIT_COMMAND=("${GIT_COMMAND_BASE[@]}" "-C" "${GIT_REPOSITORY_FOLDER}" "checkout" "${BRANCH}");
+"${GIT_COMMAND[@]}" 2>&1 | tee -a "$LOG_FILE";
+# get the latest changes from branch
+GIT_COMMAND=("${GIT_COMMAND_BASE[@]}" "-C" "${GIT_REPOSITORY_FOLDER}" "pull" "${REMOTE_NAME}" "${BRANCH}")
+"${GIT_COMMAND[@]}" 2>&1 | tee -a "$LOG_FILE";
+echo "[$(date +"%Y-%m-%d %H:%M:%S")] [${unique_id}] [END] branch switch done" | tee -a "$LOG_FILE";
+
+
+# __END__