ScriptsCollections/DumpDatabasePostgreSQL
4
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update with commands in array for calls, logging for single file restore

Browse Source 
All commands are now run through a single array variable
All logs are in logs folder for the restore which is a sub folder to where the SQL file is located
On restore abort if the basic database creation failed or skip to the next database in block restore
This commit is contained in:
Clemens Schwaighofer
2025-10-17 10:30:11 +09:00
parent 0b938f31c8
commit b979cdd51f
3 changed files with 271 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

255
pg_restore_db_file.sh
View File

@@ -181,14 +181,14 @@ if [ ! -d "$DUMP_FOLDER" ]; then
exit;
fi;
LOGS=$DUMP_FOLDER'/logs/';
LOG_PATH=$DUMP_FOLDER'/logs/';
# create logs folder if missing
if [ ! -d "$LOGS" ]; then
echo "Creating '$LOGS' folder";
mkdir -p "$LOGS";
if [ ! -d "$LOGS" ]; then
echo "Creation of '$LOGS' folder failed";
exit;
if [ ! -d "$LOG_PATH" ]; then
echo "+ Creating '$LOG_PATH' folder";
mkdir -p "$LOG_PATH";
if [ ! -d "$LOG_PATH" ]; then
echo "[!] Creation of '$LOG_PATH' folder failed";
exit 1;
fi;
fi;
@@ -263,7 +263,7 @@ PG_PSQL="psql";
# default port and host
EXCLUDE_LIST="pg_globals"; # space separated
LOGFILE="tee -a "$LOGS/PG_RESTORE_DB_FILE.$(date +"%Y%m%d_%H%M%S").log"";
LOG_FILE="tee -a "$LOG_PATH/PG_RESTORE_DB_FILE.$(date +"%Y%m%d_%H%M%S").log"";
# get the count for DBs to import
db_count=$(find "${DUMP_FOLDER}" -name "*.sql" -print | wc -l);
@@ -286,11 +286,11 @@ fi;
if [ ${DRY_RUN} ]; then
echo "**** [DRY RUN] ****";
fi;
echo "= Will import $db_count databases from $_DUMP_FOLDER" | $LOGFILE;
echo "= into the DB server $_HOST:$_PORT" | $LOGFILE;
echo "= running $MAX_JOBS jobs" | $LOGFILE;
echo "= import logs: $LOGS" | $LOGFILE;
echo "" | $LOGFILE;
echo "= Will import $db_count databases from $_DUMP_FOLDER" | $LOG_FILE;
echo "= into the DB server $_HOST:$_PORT" | $LOG_FILE;
echo "= running $MAX_JOBS jobs" | $LOG_FILE;
echo "= import logs: $LOG_PATH" | $LOG_FILE;
echo "" | $LOG_FILE;
pos=1;
# go through all the files an import them into the database
MASTERSTART=$(date +"%s");
@@ -300,7 +300,7 @@ if [ "$IMPORT_GLOBALS" -eq 1 ]; then
start_time=$(date +"%F %T");
START=$(date +"%s");
# get the pg_globals file
echo "=[Globals Restore]=START=[$start_time]==================================================>" | $LOGFILE;
echo "=[Globals Restore]=START=[$start_time]==================================================>" | $LOG_FILE;
# get newest and only the first one
file=$(find "$DUMP_FOLDER" -name "pg_global*" -type f -printf "%Ts\t%p\n" | sort -nr | head -1);
filename=$(basename "$file");
@@ -334,23 +334,24 @@ if [ "$IMPORT_GLOBALS" -eq 1 ]; then
PG_PATH_VERSION_LOCAL="${PG_PATH_VERSION}";
fi;
PG_PATH="${PG_BASE_PATH}${PG_PATH_VERSION_LOCAL}${PG_PATH_BIN}";
echo "+ Restore globals file: $filename to [$_host:$_port] @ $(date +"%F %T")" | $LOGFILE;
echo "+ Restore globals file: $filename to [$_host:$_port] @ $(date +"%F %T")" | $LOG_FILE;
_PG_PARAMS=("-U" "postgres");
_PG_PARAMS+=("${PG_PARAM_HOST[@]}");
_PG_PARAMS+=("${PG_PARAM_PORT[@]}");
_PG_PARAMS+=("-f" "$file" "-e" "-q" "-X" "template1");
PG_COMMAND=("${PG_PATH}${PG_PSQL}" "${_PG_PARAMS[@]}");
if [ ${DRY_RUN} -eq 0 ]; then
"${PG_PATH}${PG_PSQL}" "${_PG_PARAMS[@]}" | $LOGFILE;
"${PG_COMMAND[@]}" | $LOG_FILE;
else
echo "${PG_PATH}${PG_PSQL} ${_PG_PARAMS[*]}" | $LOGFILE;
echo "${PG_COMMAND[*]}" | $LOG_FILE;
fi;
DURATION=$(($(date +"%s")-START));
printf "=[Globals Restore]=END===[%s]========================================================>\n" "$(convert_time ${DURATION})" | $LOGFILE;
printf "=[Globals Restore]=END===[%s]========================================================>\n" "$(convert_time ${DURATION})" | $LOG_FILE;
fi;
for file in "$DUMP_FOLDER/"*.sql; do
start_time=$(date +"%F %T");
START=$(date +"%s");
echo "=[$pos/$db_count]=START=[$start_time]==================================================>" | $LOGFILE;
echo "=[$pos/$db_count]=START=[$start_time]==================================================>" | $LOG_FILE;
# the encoding
set_encoding='';
# get the filename
@@ -358,6 +359,22 @@ for file in "$DUMP_FOLDER/"*.sql; do
# get the databse, user
# default file name is <database>.<owner>.<encoding>.<type>-<version>_<host>_<port>_<date>_<time>_<sequence>
database=$(echo "$filename" | cut -d "." -f 1);
# check this is skip or not
exclude=0;
for exclude_db in $EXCLUDE_LIST; do
if [ "$exclude_db" = "$database" ]; then
exclude=1;
break;
fi;
done;
if [ $exclude -eq 1 ]; then
DURATION=0;
echo "# Skipped DB '$database'" | $LOG_FILE;
printf "=[$pos/$db_count]=END===[%s]========================================================>\n" "$(convert_time ${DURATION})" | $LOG_FILE;
pos=$((pos+1));
continue;
fi;
# restore DB
owner=$(echo "$filename" | cut -d "." -f 2);
__encoding=$(echo "$filename" | cut -d "." -f 3);
# the last _ part if for version 10
@@ -403,93 +420,129 @@ for file in "$DUMP_FOLDER/"*.sql; do
PG_PATH_VERSION_LOCAL="${PG_PATH_VERSION}";
fi;
PG_PATH="${PG_BASE_PATH}${PG_PATH_VERSION_LOCAL}${PG_PATH_BIN}";
# check this is skip or not
exclude=0;
for exclude_db in $EXCLUDE_LIST; do
if [ "$exclude_db" = "$database" ]; then
exclude=1;
fi;
done;
if [ $exclude -eq 0 ]; then
# create user if not exist yet
# check query for user
# for all calls
_PG_PARAMS_ALL=("-U" "postgres");
_PG_PARAMS_ALL+=("${PG_PARAM_HOST[@]}");
_PG_PARAMS_ALL+=("${PG_PARAM_PORT[@]}");
# for the call
# create user if not exist yet
# check query for user
# for all calls
_PG_PARAMS_ALL=("-U" "postgres");
_PG_PARAMS_ALL+=("${PG_PARAM_HOST[@]}");
_PG_PARAMS_ALL+=("${PG_PARAM_PORT[@]}");
# for the call
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("-A" "-F" "," "-t" "-q" "-X" "-c" "SELECT oid FROM pg_roles WHERE rolname = '$owner';" "template1");
user_oid=$("$PG_PSQL" "${_PG_PARAMS[@]}");
if [ -z "$user_oid" ]; then
echo "+ Create USER '$owner' for DB '$database' [$_host:$_port] @ $(date +"%F %T")" | $LOG_FILE;
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("-A" "-F" "," "-t" "-q" "-X" "-c" "SELECT oid FROM pg_roles WHERE rolname = '$owner';" "template1");
user_oid=$("$PG_PSQL" "${_PG_PARAMS[@]}");
if [ -z "$user_oid" ]; then
echo "+ Create USER '$owner' for DB '$database' [$_host:$_port] @ $(date +"%F %T")" | $LOGFILE;
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("-D" "-R" "-S" "$owner");
if [ ${DRY_RUN} -eq 0 ]; then
"${PG_PATH}${PG_CREATEUSER}" "${_PG_PARAMS[@]}";
else
echo "${PG_PATH}${PG_CREATEUSER} ${_PG_PARAMS[*]}";
_PG_PARAMS+=("-D" "-R" "-S" "$owner");
PG_COMMAND=("${PG_PATH}${PG_CREATEUSER}" "${_PG_PARAMS[@]}");
if [ ${DRY_RUN} -eq 0 ]; then
"${PG_COMMAND[@]}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!] Creation of user '$owner' failed, skipping database '$database'";
printf "=[$pos/$db_count]=END===[%s]========================================================>\n" "$(convert_time 0)" | $LOG_FILE;
pos=$((pos+1));
continue;
fi;
fi;
# before importing the data, drop this database
echo "- Drop DB '$database' [$_host:$_port] @ $(date +"%F %T")" | $LOGFILE;
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("$database");
if [ ${DRY_RUN} -eq 0 ]; then
"${PG_PATH}${PG_DROPDB}" "${_PG_PARAMS[@]}";
else
echo "${PG_PATH}${PG_DROPDB} ${_PG_PARAMS[*]}";
echo "${PG_COMMAND[*]}";
fi;
echo "+ Create DB '$database' with '$owner' [$_host:$_port] @ $(date +"%F %T")" | $LOGFILE;
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("-O" "$owner" "-E" "$set_encoding" "-T" "$TEMPLATEDB" "$database");
if [ ${DRY_RUN} -eq 0 ]; then
"${PG_PATH}${PG_CREATEDB}" "${_PG_PARAMS[@]}";
else
echo "${PG_PATH}${PG_CREATEDB} ${_PG_PARAMS[*]}";
fi;
if [ -f "${PG_PATH}${PG_CREATELANG}" ]; then
echo "+ Create plpgsql lang in DB '$database' [$_host:$_port] @ $(date +"%F %T")" | $LOGFILE;
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("plpgsql" "$database");
if [ ${DRY_RUN} -eq 0 ]; then
"${PG_PATH}${PG_CREATELANG}" "${_PG_PARAMS[@]}";
else
echo "${PG_PATH}${PG_CREATELANG} ${_PG_PARAMS[*]}";
fi;
fi;
echo "% Restore data from '$filename' to DB '$database' using $MAX_JOBS jobs [$_host:$_port] @ $(date +"%F %T")" | $LOGFILE;
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("-d" "$database" "-F" "c" "-v" "-c" "-j" "$MAX_JOBS" "$file");
if [ ${DRY_RUN} -eq 0 ]; then
"${PG_PATH}${PG_RESTORE}" "${_PG_PARAMS[@]}" 2>"$LOGS/errors.${database}.$(date +"%Y%m%d_%H%M%S").log";
else
echo "${PG_PATH}${PG_RESTORE} ${_PG_PARAMS[*]} 2>${LOGS}/errors.${database}.$(date +"%Y%m%d_%H%M%S").log";
fi;
# BUG FIX FOR POSTGRESQL 9.6.2 db_dump
# it does not dump the default public ACL so the owner of the DB cannot access the data,
# check if the ACL dump is missing and do a basic restore
if ! "${PG_PATH}${PG_RESTORE}" -l "$file" | grep -q -- "ACL - public postgres"; then
echo "? Fixing missing basic public schema ACLs from DB $database [$_host:$_port] @ $(date +"%F %T")";
# grant usage on schema public to public;
_PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("-Atq" "-c" "GRANT USAGE ON SCHEMA public TO public;" "${database}");
"${PG_PSQL}" "${_PG_PARAMS[@]}";
# grant create on schema public to public;
_PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("-Atq" "-c" "GRANT CREATE ON SCHEMA public TO public;" "${database}");
"${PG_PSQL}" "${_PG_PARAMS[@]}";
fi;
echo "$ Restore of data '$filename' for DB '$database' [$_host:$_port] finished" | $LOGFILE;
DURATION=$(($(date "+%s")-START));
echo "* Start at $start_time and end at $(date +"%F %T") and ran for $(convert_time ${DURATION}) seconds" | $LOGFILE;
else
DURATION=0;
echo "# Skipped DB '$database'" | $LOGFILE;
fi;
printf "=[$pos/$db_count]=END===[%s]========================================================>\n" "$(convert_time ${DURATION})" | $LOGFILE;
# before importing the data, drop this database
echo "- Drop DB '$database' [$_host:$_port] @ $(date +"%F %T")" | $LOG_FILE;
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("$database");
PG_COMMAND=("${PG_PATH}${PG_DROPDB}" "${_PG_PARAMS[@]}");
if [ ${DRY_RUN} -eq 0 ]; then
"${PG_COMMAND[@]}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!] Could not drop database, skipping database '$database'";
printf "=[$pos/$db_count]=END===[%s]========================================================>\n" "$(convert_time 0)" | $LOG_FILE;
pos=$((pos+1));
continue;
fi;
else
echo "${PG_COMMAND[*]}";
fi;
echo "+ Create DB '$database' with '$owner' [$_host:$_port] @ $(date +"%F %T")" | $LOG_FILE;
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("-O" "$owner" "-E" "$set_encoding" "-T" "$TEMPLATEDB" "$database");
PG_COMMAND=("${PG_PATH}${PG_CREATEDB}" "${_PG_PARAMS[@]}");
if [ ${DRY_RUN} -eq 0 ]; then
"${PG_COMMAND[@]}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!] Could not create database, skipping database '$database'";
printf "=[$pos/$db_count]=END===[%s]========================================================>\n" "$(convert_time 0)" | $LOG_FILE;
pos=$((pos+1));
continue;
fi;
else
echo "${PG_COMMAND[*]}";
fi;
if [ -f "${PG_PATH}${PG_CREATELANG}" ]; then
echo "+ Create plpgsql lang in DB '$database' [$_host:$_port] @ $(date +"%F %T")" | $LOG_FILE;
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("plpgsql" "$database");
PG_COMMAND=("${PG_PATH}${PG_CREATELANG}" "${_PG_PARAMS[@]}");
if [ ${DRY_RUN} -eq 0 ]; then
"${PG_COMMAND[@]}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!] Could not create plpgsql language, skipping database '$database'";
printf "=[$pos/$db_count]=END===[%s]========================================================>\n" "$(convert_time 0)" | $LOG_FILE;
pos=$((pos+1));
continue;
fi;
else
echo "${PG_COMMAND[*]}";
fi;
fi;
echo "% Restore data from '$filename' to DB '$database' using $MAX_JOBS jobs [$_host:$_port] @ $(date +"%F %T")" | $LOG_FILE;
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("-d" "$database" "-F" "c" "-v" "-c" "-j" "$MAX_JOBS" "$file");
PG_COMMAND=("${PG_PATH}${PG_RESTORE}" "${_PG_PARAMS[@]}");
LOG_ERROR_FILE="$LOG_PATH/errors.${database}.$(date +"%Y%m%d_%H%M%S").log";
if [ ${DRY_RUN} -eq 0 ]; then
"${PG_COMMAND[@]}" 2>"${LOG_ERROR_FILE}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!] Restore of database '$database' failed, see ${LOG_ERROR_FILE} for details";
fi;
else
echo "${PG_COMMAND[*]} 2>${LOG_ERROR_FILE}";
fi;
# BUG FIX FOR POSTGRESQL 9.6.2 db_dump
# it does not dump the default public ACL so the owner of the DB cannot access the data,
# check if the ACL dump is missing and do a basic restore
if ! "${PG_PATH}${PG_RESTORE}" -l "$file" | grep -q -- "ACL - public postgres"; then
echo "? Fixing missing basic public schema ACLs from DB $database [$_host:$_port] @ $(date +"%F %T")";
# grant usage on schema public to public;
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("-AtqX" "-c" "GRANT USAGE ON SCHEMA public TO public;" "${database}");
PG_COMMAND=("${PG_PSQL}" "${_PG_PARAMS[@]}");
if [ ${DRY_RUN} -eq 0 ]; then
"${PG_COMMAND[@]}";
else
echo "${PG_COMMAND[*]}";
fi;
# grant create on schema public to public;
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("-AtqX" "-c" "GRANT CREATE ON SCHEMA public TO public;" "${database}");
PG_COMMAND=("${PG_PSQL}" "${_PG_PARAMS[@]}");
if [ ${DRY_RUN} -eq 0 ]; then
"${PG_COMMAND[@]}";
else
echo "${PG_COMMAND[*]}";
fi;
fi;
echo "$ Restore of data '$filename' for DB '$database' [$_host:$_port] finished" | $LOG_FILE;
DURATION=$(($(date "+%s")-START));
echo "* Start at $start_time and end at $(date +"%F %T") and ran for $(convert_time ${DURATION}) seconds" | $LOG_FILE;
printf "=[$pos/$db_count]=END===[%s]========================================================>\n" "$(convert_time ${DURATION})" | $LOG_FILE;
pos=$((pos+1));
done;
DURATION=$(($(date "+%s")-MASTERSTART));
echo "" | $LOGFILE;
echo "= Start at $master_start_time and end at $(date +"%F %T") and ran for $(convert_time ${DURATION}) seconds. Imported $db_count databases." | $LOGFILE;
echo "" | $LOG_FILE;
echo "= Start at $master_start_time and end at $(date +"%F %T") and ran for $(convert_time ${DURATION}) seconds. Imported $db_count databases." | $LOG_FILE;