ScriptsCollections/DumpDatabasePostgreSQL
4
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update with commands in array for calls, logging for single file restore

Browse Source 
All commands are now run through a single array variable
All logs are in logs folder for the restore which is a sub folder to where the SQL file is located
On restore abort if the basic database creation failed or skip to the next database in block restore
This commit is contained in:
Clemens Schwaighofer
2025-10-17 10:30:11 +09:00
parent 0b938f31c8
commit b979cdd51f
3 changed files with 271 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

136
pg_drop_restore.sh
View File

@@ -39,6 +39,8 @@ DRY_RUN=0;
BC='/usr/bin/bc';
PORT_REGEX="^[0-9]{4,5}$";
OPTARG_REGEX="^-";
# log path
LOG_PATH='';
MAX_JOBS='';
PG_PARAMS=();
PG_PARAM_ROLE=();
@@ -359,6 +361,18 @@ else
PG_PATH="${PG_BASE_PATH}${ident}/bin/'";
fi;
# set log path, this is the base path of the file + logs
LOG_PATH=$(dirname "${file}")"/logs/";
# create logs folder if missing
if [ ! -d "$LOG_PATH" ]; then
echo "+ Creating '$LOG_PATH' folder";
mkdir -p "$LOG_PATH";
if [ ! -d "$LOG_PATH" ]; then
echo "[!] Creation of '$LOG_PATH' folder failed";
exit 1;
fi;
fi;
PG_DROPDB="${PG_PATH}dropdb";
PG_CREATEDB="${PG_PATH}createdb";
PG_CREATELANG="${PG_PATH}createlang";
@@ -390,11 +404,11 @@ fi;
if [ $DRY_RUN -eq 1 ]; then
echo "**** [DRY RUN] ****";
fi;
echo "Will drop database '$database' on host '$_host:$_port' and load file '$file' with user '$owner', set encoding '$encoding' and use database version '$ident'";
echo "[.] Will drop database '$database' on host '$_host:$_port' and load file '$file' with user '$owner', set encoding '$encoding' and use database version '$ident'";
if [ $SCHEMA_ONLY -eq 1 ]; then
echo "!!!!!!! WILL ONLY RESTORE SCHEMA, NO DATA !!!!!!!";
fi;
if [ $NO_ASK -eq 1 ]; then
if [ $NO_ASK -eq 1 ] || [ $DRY_RUN -eq 1 ]; then
go='yes';
else
echo "Continue? type 'yes'";
@@ -406,75 +420,135 @@ if [ "$go" != 'yes' ]; then
else
start_time=$(date +"%F %T");
START=$(date +'%s');
echo "Drop DB $database [$_host:$_port] @ $start_time";
echo "[1] - Drop DB $database [$_host:$_port] @ $start_time";
# DROP DATABASE
_PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("-U" "postgres" "${database}");
PG_COMMAND=("${PG_DROPDB}" "${_PG_PARAMS[@]}");
if [ $DRY_RUN -eq 0 ]; then
"${PG_DROPDB}" "${_PG_PARAMS[@]}";
"${PG_COMMAND[@]}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!:${RETURN_CODE}] Could not drop database $database, aborting";
exit 1;
fi;
else
echo "${PG_DROPDB} ${_PG_PARAMS[*]}";
echo "${PG_COMMAND[*]}";
fi;
# CREATE DATABASE
echo "Create DB $database with $owner and encoding $encoding on [$_host:$_port] @ $(date +"%F %T")";
echo "[2] + Create DB $database with $owner and encoding $encoding on [$_host:$_port] @ $(date +"%F %T")";
_PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("-U" "postgres" "-O" "${owner}" "-E" "${encoding}" "-T" "${TEMPLATEDB}" "${database}");
PG_COMMAND=("${PG_CREATEDB}" "${_PG_PARAMS[@]}");
if [ $DRY_RUN -eq 0 ]; then
"${PG_CREATEDB}" "${_PG_PARAMS[@]}";
"${PG_COMMAND[@]}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!:${RETURN_CODE}] Could not create database $database, aborting";
exit 1;
fi;
else
echo "${PG_CREATEDB} ${_PG_PARAMS[*]}";
echo "${PG_COMMAND[*]}";
fi;
# CREATE plpgsql LANG
if [ -f "$PG_CREATELANG" ]; then
_PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("-U" "postgres" "plpgsql" "${database}");
echo "Create plpgsql lang in DB $database on [$_host:$_port] @ $(date +"%F %T")";
PG_COMMAND=("${PG_CREATELANG}" "${_PG_PARAMS[@]}");
echo "[3] + Create plpgsql lang in DB $database on [$_host:$_port] @ $(date +"%F %T")";
if [ $DRY_RUN -eq 0 ]; then
"${PG_CREATELANG}" "${_PG_PARAMS[@]}";
"${PG_COMMAND[@]}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!:${RETURN_CODE}] Could not create plgpgsql language in $database, aborting";
exit 1;
fi;
else
echo "${PG_CREATELANG} ${_PG_PARAMS[*]}";
echo "${PG_COMMAND[*]}";
fi;
fi;
# RESTORE DATA
echo "Restore data from $file to DB $database on [$_host:$_port] with Jobs $MAX_JOBS @ $(date +"%F %T")";
echo "[4] % Restore data from $file to DB $database on [$_host:$_port] with Jobs $MAX_JOBS @ $(date +"%F %T")";
_PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("${PG_PARAM_ROLE[@]}")
_PG_PARAMS+=("-U" "postgres" "-d" "${database}" "-F" "c" "-v" "-c" "${schema}" "-j" "${MAX_JOBS}" "${file}");
if [ -n "$schema" ]; then
_PG_PARAMS+=("${schema}");
fi;
_PG_PARAMS+=("-U" "postgres" "-d" "${database}" "-F" "c" "-v" "-c" "-j" "${MAX_JOBS}" "${file}");
PG_COMMAND=("${PG_RESTORE}" "${_PG_PARAMS[@]}");
LOG_ERROR_FILE="${LOG_PATH}/restore_errors.${LOG_FILE_EXT}";
LOG_OUTPUT_FILE="${LOG_PATH}/restore_output.${LOG_FILE_EXT}";
if [ $DRY_RUN -eq 0 ]; then
"${PG_RESTORE}" "${_PG_PARAMS[@]}" 2>"restore_errors.${LOG_FILE_EXT}";
"${PG_COMMAND[@]}" 1>"${LOG_OUTPUT_FILE}" 2>"${LOG_ERROR_FILE}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!:${RETURN_CODE}] Could not restore the database $database successfully, check ${LOG_ERROR_FILE} for details";
fi;
else
echo "${PG_RESTORE} ${_PG_PARAMS[*]} 2>restore_errors.${LOG_FILE_EXT}";
echo "${PG_COMMAND[*]} 1>${LOG_OUTPUT_FILE} 2>${LOG_ERROR_FILE}";
fi;
# BUG FIX FOR POSTGRESQL 9.6.2 db_dump
# it does not dump the default public ACL so the owner of the DB cannot access the data, check if the ACL dump is missing and do a basic restore
if ! "${PG_RESTORE}" -l "$file" | grep -q -- "ACL - public postgres"; then
echo "Fixing missing basic public schema ACLs from DB $database [$_host:$_port] @ $(date +"%F %T")";
echo "[5] ? Fixing missing basic public schema ACLs from DB $database [$_host:$_port] @ $(date +"%F %T")";
# grant usage on schema public to public;
_PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("-U" "postgres" "-Atq" "-c" "GRANT USAGE ON SCHEMA public TO public;" "${database}");
"${PG_PSQL}" "${_PG_PARAMS[@]}";
_PG_PARAMS+=("-U" "postgres" "-AtqX" "-c" "GRANT USAGE ON SCHEMA public TO public;" "${database}");
PG_COMMAND=("${PG_PSQL}" "${_PG_PARAMS[@]}");
if [ $DRY_RUN -eq 0 ]; then
"${PG_COMMAND[@]}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!:${RETURN_CODE}] Could not fix usage access to basic public schema ACLs in $database";
fi;
else
echo "${PG_COMMAND[*]}";
fi;
# grant create on schema public to public;
_PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("-U" "postgres" "-Atq" "-c" "GRANT CREATE ON SCHEMA public TO public;" "${database}");
"${PG_PSQL}" "${_PG_PARAMS[@]}";
_PG_PARAMS+=("-U" "postgres" "-AtqX" "-c" "GRANT CREATE ON SCHEMA public TO public;" "${database}");
PG_COMMAND=("${PG_PSQL}" "${_PG_PARAMS[@]}");
if [ $DRY_RUN -eq 0 ]; then
"${PG_COMMAND[@]}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!:${RETURN_CODE}] Could not fix create access to basic public schema ACLs in $database";
fi;
else
echo "${PG_COMMAND[*]}";
fi;
fi;
# SEQUENCE RESET DATA COLLECTION
echo "Resetting all sequences from DB $database [$_host:$_port] @ $(date +"%F %T")";
echo "[6] ? Resetting all sequences from DB $database [$_host:$_port] @ $(date +"%F %T")";
reset_query="SELECT 'SELECT SETVAL(' ||quote_literal(S.relname)|| ', MAX(' ||quote_ident(C.attname)|| ') ) FROM ' ||quote_ident(T.relname)|| ';' FROM pg_class AS S, pg_depend AS D, pg_class AS T, pg_attribute AS C WHERE S.relkind = 'S' AND S.oid = D.objid AND D.refobjid = T.oid AND D.refobjid = C.attrelid AND D.refobjsubid = C.attnum ORDER BY S.relname;";
_PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("-U" "postgres" "-Atq" "-o" "${TEMP_FILE}" "-c" "${reset_query}" "${database}");
_PG_PARAMS+=("-U" "postgres" "-AtqX" "-o" "${TEMP_FILE}" "-c" "${reset_query}" "${database}");
PG_COMMAND=("${PG_PSQL}" "${_PG_PARAMS[@]}");
_PG_PARAMS_OUT=("${PG_PARAMS[@]}");
_PG_PARAMS_OUT+=("-U" "postgres" "-e" "-f" "${TEMP_FILE}" "${database}");
_PG_PARAMS_OUT+=("-U" "postgres" "-X" "-e" "-f" "${TEMP_FILE}" "${database}");
PG_COMMAND_OUT=("${PG_PSQL}" "${_PG_PARAMS_OUT[@]}");
LOG_OUTPUT_FILE="${LOG_PATH}/output_sequence.${LOG_FILE_EXT}";
LOG_ERROR_FILE="${LOG_PATH}/errors_sequence.${database}.${LOG_FILE_EXT}";
if [ $DRY_RUN -eq 0 ]; then
"${PG_PSQL}" "${_PG_PARAMS[@]}";
"${PG_PSQL}" "${_PG_PARAMS_OUT[@]}" 1>"output_sequence.${LOG_FILE_EXT}" 2>"errors_sequence.${database}.${LOG_FILE_EXT}";
rm "${TEMP_FILE}";
"${PG_COMMAND[@]}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!:${RETURN_CODE}] Could not create sequence reset query for database $database";
fi;
if [ -f "${TEMP_FILE}" ]; then
"${PG_COMMAND_OUT[@]}" 1>"${LOG_OUTPUT_FILE}" 2>"${LOG_ERROR_FILE}";
RETURN_CODE=$?;
if [ $RETURN_CODE -ne 0 ]; then
echo "[!:${RETURN_CODE}] Could not reset sequences for database $database";
fi;
rm "${TEMP_FILE}";
fi;
else
echo "${PG_PSQL}　${_PG_PARAMS[*]}";
echo "${PG_PSQL} ${_PG_PARAMS_OUT[*]} 1>output_sequence.${LOG_FILE_EXT} 2>errors_sequence.${database}.${LOG_FILE_EXT}";
echo "${PG_COMMAND[*]}";
echo "${PG_COMMAND_OUT[*]} 1>${LOG_OUTPUT_FILE} 2>${LOG_ERROR_FILE}";
fi;
echo "Restore of data $file for DB $database [$_host:$_port] finished";
echo "[.] $ Restore of data $file for DB $database [$_host:$_port] finished";
DURATION=$(($(date "+%s")-START));
echo "Start at $start_time and end at $(date +"%F %T") and ran for $(convert_time ${DURATION})";
echo "=== END RESTORE" >>"restore_errors.${LOG_FILE_EXT}";
echo "[.] * Start at $start_time and end at $(date +"%F %T") and ran for $(convert_time ${DURATION})";
echo "=== END RESTORE" >>"${LOG_PATH}/restore_errors.${LOG_FILE_EXT}";
fi;