ScriptsCollections/DumpDatabasePostgreSQL
4
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Some more clean up for code

Browse Source 
Fix ident get from file, they where subtly not correct
- wrong lookup for <10 postgresql version
- if version has a . inside the host/port lookups have to shift
- dry run notice for restore runs
This commit is contained in:
Clemens Schwaighofer
2025-04-15 09:40:06 +09:00
parent 01b434fca0
commit bb5c65818b
2 changed files with 28 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
pg_drop_restore.sh
View File

@@ -33,6 +33,7 @@ schema='';
NO_ASK=0; NO_ASK=0;
TEMPLATEDB='template0'; TEMPLATEDB='template0';
SCHEMA_ONLY=0; SCHEMA_ONLY=0;
ERROR=0;
REDHAT=0; REDHAT=0;
DRY_RUN=0; DRY_RUN=0;
BC='/usr/bin/bc'; BC='/usr/bin/bc';
@@ -274,10 +275,18 @@ if [ -r "$file" ] && { [ ! "$owner" ] || [ ! "$database" ] || [ ! "$encoding" ];
_owner=$(echo "${db_file}" | cut -d "." -f 2); _owner=$(echo "${db_file}" | cut -d "." -f 2);
__encoding=$(echo "${db_file}" | cut -d "." -f 3); __encoding=$(echo "${db_file}" | cut -d "." -f 3);
# set the others as optional # set the others as optional
_ident=$(echo "${db_file}" | cut -d "." -f 4 | cut -d "-" -f 2); # db version first part # the last _ is for version 10 or higher
_ident=$_ident'.'$(echo "${db_file}" | cut -d "." -f 5 | cut -d "_" -f 1); # db version, second part (after .) # db version, without prefix of DB type
__host=$(echo "${db_file}" | cut -d "." -f 4 | cut -d "_" -f 2); _ident=$(echo "${db_file}" | cut -d "." -f 4 | cut -d "-" -f 2 | cut -d "_" -f 1);
__port=$(echo "${db_file}" | cut -d "." -f 4 | cut -d "_" -f 3); cut_pos=4;
# if this is < 10 then we need the second part too
if [ "${_ident}" -lt 10 ]; then
# db version, second part (after .)
_ident=$_ident'.'$(echo "$db_file" | cut -d "." -f 5 | cut -d "_" -f 1);
cut_pos=5;
fi;
__host=$(echo "${db_file}" | cut -d "." -f ${cut_pos} | cut -d "_" -f 2);
__port=$(echo "${db_file}" | cut -d "." -f ${cut_pos} | cut -d "_" -f 3);
# if any of those are not set, override by the file name settings # if any of those are not set, override by the file name settings
if [ ! "$owner" ]; then if [ ! "$owner" ]; then
owner=$_owner; owner=$_owner;
@@ -371,14 +380,16 @@ fi;
# if I cannot connect with user postgres to template1, the restore won't work # if I cannot connect with user postgres to template1, the restore won't work
_PG_PARAMS=("${PG_PARAMS[@]}"); _PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("-U" "postgres" "template1" "-q" "-t" "-X" "-A" "-F" "," "-c" "SELECT version();"); _PG_PARAMS+=("-U" "postgres" "template1" "-q" "-t" "-X" "-A" "-F" "," "-c" "SELECT version();");
_output=$("${PG_PSQL}" "${PG_PARAMS[@]}" 2>&1); _output=$("${PG_PSQL}" "${_PG_PARAMS[@]}" 2>&1);
found=$(echo "$_output" | grep "PostgreSQL"); found=$(echo "$_output" | grep "PostgreSQL");
# if the output does not have the PG version string, we have an error and abort # if the output does not have the PG version string, we have an error and abort
if [ -z "$found" ]; then if [ -z "$found" ]; then
echo "Cannot connect to the database: $_output"; echo "Cannot connect to the database: $_output";
exit 1; exit 1;
fi; fi;
if [ ${DRY_RUN} ]; then
echo "**** [DRY RUN] ****";
fi;
echo "Will drop database '$database' on host '$_host:$_port' and load file '$file' with user '$owner', set encoding '$encoding' and use database version '$ident'"; echo "Will drop database '$database' on host '$_host:$_port' and load file '$file' with user '$owner', set encoding '$encoding' and use database version '$ident'";
if [ $SCHEMA_ONLY -eq 1 ]; then if [ $SCHEMA_ONLY -eq 1 ]; then
echo "!!!!!!! WILL ONLY RESTORE SCHEMA, NO DATA !!!!!!!"; echo "!!!!!!! WILL ONLY RESTORE SCHEMA, NO DATA !!!!!!!";
@@ -400,7 +411,6 @@ else
_PG_PARAMS=("${PG_PARAMS[@]}"); _PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("-U" "postgres" "${database}"); _PG_PARAMS+=("-U" "postgres" "${database}");
if [ $DRY_RUN -eq 0 ]; then if [ $DRY_RUN -eq 0 ]; then
# $PG_DROPDB -U postgres $host $port $database;
"${PG_DROPDB}" "${PG_PARAMS[@]}"; "${PG_DROPDB}" "${PG_PARAMS[@]}";
else else
echo "${PG_DROPDB} ${PG_PARAMS[*]}"; echo "${PG_DROPDB} ${PG_PARAMS[*]}";
@@ -410,7 +420,6 @@ else
_PG_PARAMS=("${PG_PARAMS[@]}"); _PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("-U" "postgres" "-O" "${owner}" "-E" "${encoding}" "-T" "${TEMPLATEDB}" "${database}"); _PG_PARAMS+=("-U" "postgres" "-O" "${owner}" "-E" "${encoding}" "-T" "${TEMPLATEDB}" "${database}");
if [ $DRY_RUN -eq 0 ]; then if [ $DRY_RUN -eq 0 ]; then
# $PG_CREATEDB -U postgres -O $owner -E $encoding -T $TEMPLATEDB $host $port $database;
"${PG_CREATEDB}" "${_PG_PARAMS[@]}"; "${PG_CREATEDB}" "${_PG_PARAMS[@]}";
else else
echo "${PG_CREATEDB} ${_PG_PARAMS[*]}"; echo "${PG_CREATEDB} ${_PG_PARAMS[*]}";
@@ -421,7 +430,6 @@ else
_PG_PARAMS+=("-U" "postgres" "plpgsql" "${database}"); _PG_PARAMS+=("-U" "postgres" "plpgsql" "${database}");
echo "Create plpgsql lang in DB $database on [$_host:$_port] @ $(date +"%F %T")"; echo "Create plpgsql lang in DB $database on [$_host:$_port] @ $(date +"%F %T")";
if [ $DRY_RUN -eq 0 ]; then if [ $DRY_RUN -eq 0 ]; then
# $PG_CREATELANG -U postgres plpgsql $host $port $database;
"${PG_CREATELANG}" "${_PG_PARAMS[@]}"; "${PG_CREATELANG}" "${_PG_PARAMS[@]}";
else else
echo "${PG_CREATELANG} ${_PG_PARAMS[*]}"; echo "${PG_CREATELANG} ${_PG_PARAMS[*]}";
@@ -433,18 +441,14 @@ else
_PG_PARAMS+=("${PG_PARAM_ROLE[@]}") _PG_PARAMS+=("${PG_PARAM_ROLE[@]}")
_PG_PARAMS+=("-U" "postgres" "-d" "${database}" "-F" "c" "-v" "-c" "${schema}" "-j" "${MAX_JOBS}" "${file}"); _PG_PARAMS+=("-U" "postgres" "-d" "${database}" "-F" "c" "-v" "-c" "${schema}" "-j" "${MAX_JOBS}" "${file}");
if [ $DRY_RUN -eq 0 ]; then if [ $DRY_RUN -eq 0 ]; then
# $PG_RESTORE -U postgres -d $database -F c -v -c $schema -j $MAX_JOBS $host $port $role $file 2>restore_errors.$LOG_FILE_EXT;
"${PG_RESTORE}" "${_PG_PARAMS[@]}" 2>"restore_errors.${LOG_FILE_EXT}"; "${PG_RESTORE}" "${_PG_PARAMS[@]}" 2>"restore_errors.${LOG_FILE_EXT}";
else else
echo "${PG_RESTORE} ${_PG_PARAMS[*]} 2>restore_errors.${LOG_FILE_EXT}"; echo "${PG_RESTORE} ${_PG_PARAMS[*]} 2>restore_errors.${LOG_FILE_EXT}";
fi; fi;
# BUG FIX FOR POSTGRESQL 9.6.2 db_dump # BUG FIX FOR POSTGRESQL 9.6.2 db_dump
# it does not dump the default public ACL so the owner of the DB cannot access the data, check if the ACL dump is missing and do a basic restore # it does not dump the default public ACL so the owner of the DB cannot access the data, check if the ACL dump is missing and do a basic restore
# if [ -z "$("$PG_RESTORE" -l "$file" | grep -- "ACL - public postgres")" ]; then if ! "${PG_RESTORE}" -l "$file" | grep -q -- "ACL - public postgres"; then
if ! "$("${PG_RESTORE}" -l "$file" | grep -q -- "ACL - public postgres")"; then
echo "Fixing missing basic public schema ACLs from DB $database [$_host:$_port] @ $(date +"%F %T")"; echo "Fixing missing basic public schema ACLs from DB $database [$_host:$_port] @ $(date +"%F %T")";
# echo "GRANT USAGE ON SCHEMA public TO public;" | $PG_PSQL -U postgres -Atq $host $port $database;
# echo "GRANT CREATE ON SCHEMA public TO public;" | $PG_PSQL -U postgres -Atq $host $port $database;
# grant usage on schema public to public; # grant usage on schema public to public;
_PG_PARAMS=("${PG_PARAMS[@]}"); _PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("-U" "postgres" "-Atq" "-c" "GRANT USAGE ON SCHEMA public TO public;" "${database}"); _PG_PARAMS+=("-U" "postgres" "-Atq" "-c" "GRANT USAGE ON SCHEMA public TO public;" "${database}");
@@ -462,13 +466,11 @@ else
_PG_PARAMS_OUT=("${PG_PARAMS[@]}"); _PG_PARAMS_OUT=("${PG_PARAMS[@]}");
_PG_PARAMS_OUT+=("-U" "postgres" "-e" "-f" "${TEMP_FILE}" "${database}"); _PG_PARAMS_OUT+=("-U" "postgres" "-e" "-f" "${TEMP_FILE}" "${database}");
if [ $DRY_RUN -eq 0 ]; then if [ $DRY_RUN -eq 0 ]; then
# echo "${reset_query}" | $PG_PSQL -U postgres -Atq $host $port -o $TEMP_FILE $database
"${PG_PSQL}" "${_PG_PARAMS[@]}"; "${PG_PSQL}" "${_PG_PARAMS[@]}";
# $PG_PSQL -U postgres $host $port -e -f $TEMP_FILE $database 1>output_sequence.$LOG_FILE_EXT 2>errors_sequence.$database.$LOG_FILE_EXT;
"${PG_PSQL}" "${_PG_PARAMS_OUT[@]}" 1>"output_sequence.${LOG_FILE_EXT}" 2>"errors_sequence.${database}.${LOG_FILE_EXT}"; "${PG_PSQL}" "${_PG_PARAMS_OUT[@]}" 1>"output_sequence.${LOG_FILE_EXT}" 2>"errors_sequence.${database}.${LOG_FILE_EXT}";
rm "${TEMP_FILE}"; rm "${TEMP_FILE}";
else else
echo "${reset_query}"; echo "${PG_PSQL}　${_PG_PARAMS[*]}";
echo "${PG_PSQL} ${_PG_PARAMS_OUT[*]} 1>output_sequence.${LOG_FILE_EXT} 2>errors_sequence.${database}.${LOG_FILE_EXT}"; echo "${PG_PSQL} ${_PG_PARAMS_OUT[*]} 1>output_sequence.${LOG_FILE_EXT} 2>errors_sequence.${database}.${LOG_FILE_EXT}";
fi; fi;
echo "Restore of data $file for DB $database [$_host:$_port] finished"; echo "Restore of data $file for DB $database [$_host:$_port] finished";

20
pg_restore_db_file.sh
View File

@@ -28,6 +28,7 @@ _host='local';
HOST=''; HOST='';
_encoding='UTF8'; _encoding='UTF8';
set_encoding=''; set_encoding='';
ERROR=0;
REDHAT=0; REDHAT=0;
IMPORT_GLOBALS=1; IMPORT_GLOBALS=1;
TEMPLATEDB='template0'; # truly empty for restore TEMPLATEDB='template0'; # truly empty for restore
@@ -282,6 +283,9 @@ if [ -z "${PORT}" ]; then
else else
_PORT=${PORT}; _PORT=${PORT};
fi; fi;
if [ ${DRY_RUN} ]; then
echo "**** [DRY RUN] ****";
fi;
echo "= Will import $db_count databases from $_DUMP_FOLDER" | $LOGFILE; echo "= Will import $db_count databases from $_DUMP_FOLDER" | $LOGFILE;
echo "= into the DB server $_HOST:$_PORT" | $LOGFILE; echo "= into the DB server $_HOST:$_PORT" | $LOGFILE;
echo "= running $MAX_JOBS jobs" | $LOGFILE; echo "= running $MAX_JOBS jobs" | $LOGFILE;
@@ -298,21 +302,22 @@ if [ "$IMPORT_GLOBALS" -eq 1 ]; then
# get the pg_globals file # get the pg_globals file
echo "=[Globals Restore]=START=[$start_time]==================================================>" | $LOGFILE; echo "=[Globals Restore]=START=[$start_time]==================================================>" | $LOGFILE;
# get newest and only the first one # get newest and only the first one
# file=$(ls -1t "$DUMP_FOLDER/pg_global"* | head -1);
file=$(find "$DUMP_FOLDER" -name "pg_global*" -type f -printf "%Ts\t%p\n" | sort -nr | head -1); file=$(find "$DUMP_FOLDER" -name "pg_global*" -type f -printf "%Ts\t%p\n" | sort -nr | head -1);
filename=$(basename "$file"); filename=$(basename "$file");
# the last _ is for version 10 or higher # the last _ is for version 10 or higher
# db version, without prefix of DB type # db version, without prefix of DB type
version=$(echo "$filename" | cut -d "." -f 4 | cut -d "-" -f 2 | cut -d "_" -f 1); version=$(echo "$filename" | cut -d "." -f 4 | cut -d "-" -f 2 | cut -d "_" -f 1);
cut_pos=4;
# if this is < 10 then we need the second part too # if this is < 10 then we need the second part too
if [ "${version}" -lt 10 ]; then if [ "${version}" -lt 10 ]; then
# db version, second part (after .) # db version, second part (after .)
version=$version'.'$(echo "$filename" | cut -d "." -f 5 | cut -d "_" -f 1); version=$version'.'$(echo "$filename" | cut -d "." -f 5 | cut -d "_" -f 1);
cut_pos=5;
fi; fi;
# hostname of original DB, can be used as target host too # hostname of original DB, can be used as target host too
__host=$(echo "$filename" | cut -d "." -f 5 | cut -d "_" -f 2); __host=$(echo "$filename" | cut -d "." -f ${cut_pos} | cut -d "_" -f 2);
# port of original DB, can be used as target port too # port of original DB, can be used as target port too
__port=$(echo "$filename" | cut -d "." -f 5 | cut -d "_" -f 3); __port=$(echo "$filename" | cut -d "." -f ${cut_pos} | cut -d "_" -f 3);
# override file port over given port if it differs and is valid # override file port over given port if it differs and is valid
if [ -z "$_port" ] && [ "$__port" != "$_port" ] && [[ "$__port" =~ $PORT_REGEX ]] ; then if [ -z "$_port" ] && [ "$__port" != "$_port" ] && [[ "$__port" =~ $PORT_REGEX ]] ; then
_port=$__port; _port=$__port;
@@ -415,14 +420,12 @@ for file in "$DUMP_FOLDER/"*.sql; do
# for the call # for the call
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}"); _PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("-A" "-F" "," "-t" "-q" "-X" "-c" "SELECT oid FROM pg_roles WHERE rolname = '$owner';" "template1"); _PG_PARAMS+=("-A" "-F" "," "-t" "-q" "-X" "-c" "SELECT oid FROM pg_roles WHERE rolname = '$owner';" "template1");
# user_oid=$(echo "SELECT oid FROM pg_roles WHERE rolname = '$owner';" | $PSQL -U postgres $host $port -A -F "," -t -q -X template1);
user_oid=$("$PG_PSQL" "${_PG_PARAMS[@]}"); user_oid=$("$PG_PSQL" "${_PG_PARAMS[@]}");
if [ -z "$user_oid" ]; then if [ -z "$user_oid" ]; then
echo "+ Create USER '$owner' for DB '$database' [$_host:$_port] @ $(date +"%F %T")" | $LOGFILE; echo "+ Create USER '$owner' for DB '$database' [$_host:$_port] @ $(date +"%F %T")" | $LOGFILE;
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}"); _PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("-D" "-R" "-S" "$owner"); _PG_PARAMS+=("-D" "-R" "-S" "$owner");
if [ ${DRY_RUN} -eq 0 ]; then if [ ${DRY_RUN} -eq 0 ]; then
# "${PG_PATH}${PG_CREATEUSER}" -U postgres -D -R -S $host $port $owner;
"${PG_PATH}${PG_CREATEUSER}" "${_PG_PARAMS[@]}"; "${PG_PATH}${PG_CREATEUSER}" "${_PG_PARAMS[@]}";
else else
echo "${PG_PATH}${PG_CREATEUSER} ${_PG_PARAMS[*]}"; echo "${PG_PATH}${PG_CREATEUSER} ${_PG_PARAMS[*]}";
@@ -441,7 +444,6 @@ for file in "$DUMP_FOLDER/"*.sql; do
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}"); _PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("-O" "$owner" "-E" "$set_encoding" "-T" "$TEMPLATEDB" "$database"); _PG_PARAMS+=("-O" "$owner" "-E" "$set_encoding" "-T" "$TEMPLATEDB" "$database");
if [ ${DRY_RUN} -eq 0 ]; then if [ ${DRY_RUN} -eq 0 ]; then
# "${PG_PATH}${PG_CREATEDB}" -U postgres -O $owner -E $set_encoding -T $TEMPLATEDB $host $port $database;
"${PG_PATH}${PG_CREATEDB}" "${_PG_PARAMS[@]}"; "${PG_PATH}${PG_CREATEDB}" "${_PG_PARAMS[@]}";
else else
echo "${PG_PATH}${PG_CREATEDB} ${_PG_PARAMS[*]}"; echo "${PG_PATH}${PG_CREATEDB} ${_PG_PARAMS[*]}";
@@ -451,7 +453,6 @@ for file in "$DUMP_FOLDER/"*.sql; do
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}"); _PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("plpgsql" "$database"); _PG_PARAMS+=("plpgsql" "$database");
if [ ${DRY_RUN} -eq 0 ]; then if [ ${DRY_RUN} -eq 0 ]; then
# "${PG_PATH}${PG_CREATELANG}" -U postgres plpgsql $host $port $database;
"${PG_PATH}${PG_CREATELANG}" "${_PG_PARAMS[@]}"; "${PG_PATH}${PG_CREATELANG}" "${_PG_PARAMS[@]}";
else else
echo "${PG_PATH}${PG_CREATELANG} ${_PG_PARAMS[*]}"; echo "${PG_PATH}${PG_CREATELANG} ${_PG_PARAMS[*]}";
@@ -461,7 +462,6 @@ for file in "$DUMP_FOLDER/"*.sql; do
_PG_PARAMS=("${_PG_PARAMS_ALL[@]}"); _PG_PARAMS=("${_PG_PARAMS_ALL[@]}");
_PG_PARAMS+=("-d" "$database" "-F" "c" "-v" "-c" "-j" "$MAX_JOBS" "$file"); _PG_PARAMS+=("-d" "$database" "-F" "c" "-v" "-c" "-j" "$MAX_JOBS" "$file");
if [ ${DRY_RUN} -eq 0 ]; then if [ ${DRY_RUN} -eq 0 ]; then
# "${PG_PATH}${PG_RESTORE}" -U postgres -d $database -F c -v -c -j $MAX_JOBS $host $port $file 2>"$LOGS/errors.${database}.$(date +"%Y%m%d_%H%M%S").log";
"${PG_PATH}${PG_RESTORE}" "${_PG_PARAMS[@]}" 2>"$LOGS/errors.${database}.$(date +"%Y%m%d_%H%M%S").log"; "${PG_PATH}${PG_RESTORE}" "${_PG_PARAMS[@]}" 2>"$LOGS/errors.${database}.$(date +"%Y%m%d_%H%M%S").log";
else else
echo "${PG_PATH}${PG_RESTORE} ${_PG_PARAMS[*]} 2>${LOGS}/errors.${database}.$(date +"%Y%m%d_%H%M%S").log"; echo "${PG_PATH}${PG_RESTORE} ${_PG_PARAMS[*]} 2>${LOGS}/errors.${database}.$(date +"%Y%m%d_%H%M%S").log";
@@ -469,10 +469,8 @@ for file in "$DUMP_FOLDER/"*.sql; do
# BUG FIX FOR POSTGRESQL 9.6.2 db_dump # BUG FIX FOR POSTGRESQL 9.6.2 db_dump
# it does not dump the default public ACL so the owner of the DB cannot access the data, # it does not dump the default public ACL so the owner of the DB cannot access the data,
# check if the ACL dump is missing and do a basic restore # check if the ACL dump is missing and do a basic restore
if ! "$("${PG_PATH}${PG_RESTORE}" -l "$file" | grep -q -- "ACL - public postgres")"; then if ! "${PG_PATH}${PG_RESTORE}" -l "$file" | grep -q -- "ACL - public postgres"; then
echo "? Fixing missing basic public schema ACLs from DB $database [$_host:$_port] @ $(date +"%F %T")"; echo "? Fixing missing basic public schema ACLs from DB $database [$_host:$_port] @ $(date +"%F %T")";
# echo "GRANT USAGE ON SCHEMA public TO public;" | "${PG_PATH}${PG_PSQL}" -U postgres -Atq $host $port $database;
# echo "GRANT CREATE ON SCHEMA public TO public;" | "${PG_PATH}${PG_PSQL}" -U postgres -Atq $host $port $database;
# grant usage on schema public to public; # grant usage on schema public to public;
_PG_PARAMS=("${PG_PARAMS[@]}"); _PG_PARAMS=("${PG_PARAMS[@]}");
_PG_PARAMS+=("-Atq" "-c" "GRANT USAGE ON SCHEMA public TO public;" "${database}"); _PG_PARAMS+=("-Atq" "-c" "GRANT USAGE ON SCHEMA public TO public;" "${database}");