From bb5c65818baac454c3ad4227869e7fce89351ca6 Mon Sep 17 00:00:00 2001 From: Clemens Schwaighofer Date: Tue, 15 Apr 2025 09:40:06 +0900 Subject: [PATCH] Some more clean up for code Fix ident get from file, they where subtly not correct - wrong lookup for <10 postgresql version - if version has a . inside the host/port lookups have to shift - dry run notice for restore runs --- pg_drop_restore.sh | 36 +++++++++++++++++++----------------- pg_restore_db_file.sh | 20 +++++++++----------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/pg_drop_restore.sh b/pg_drop_restore.sh index 5f01dcc..e02c87e 100755 --- a/pg_drop_restore.sh +++ b/pg_drop_restore.sh @@ -33,6 +33,7 @@ schema=''; NO_ASK=0; TEMPLATEDB='template0'; SCHEMA_ONLY=0; +ERROR=0; REDHAT=0; DRY_RUN=0; BC='/usr/bin/bc'; @@ -274,10 +275,18 @@ if [ -r "$file" ] && { [ ! "$owner" ] || [ ! "$database" ] || [ ! "$encoding" ]; _owner=$(echo "${db_file}" | cut -d "." -f 2); __encoding=$(echo "${db_file}" | cut -d "." -f 3); # set the others as optional - _ident=$(echo "${db_file}" | cut -d "." -f 4 | cut -d "-" -f 2); # db version first part - _ident=$_ident'.'$(echo "${db_file}" | cut -d "." -f 5 | cut -d "_" -f 1); # db version, second part (after .) - __host=$(echo "${db_file}" | cut -d "." -f 4 | cut -d "_" -f 2); - __port=$(echo "${db_file}" | cut -d "." -f 4 | cut -d "_" -f 3); + # the last _ is for version 10 or higher + # db version, without prefix of DB type + _ident=$(echo "${db_file}" | cut -d "." -f 4 | cut -d "-" -f 2 | cut -d "_" -f 1); + cut_pos=4; + # if this is < 10 then we need the second part too + if [ "${_ident}" -lt 10 ]; then + # db version, second part (after .) + _ident=$_ident'.'$(echo "$db_file" | cut -d "." -f 5 | cut -d "_" -f 1); + cut_pos=5; + fi; + __host=$(echo "${db_file}" | cut -d "." -f ${cut_pos} | cut -d "_" -f 2); + __port=$(echo "${db_file}" | cut -d "." -f ${cut_pos} | cut -d "_" -f 3); # if any of those are not set, override by the file name settings if [ ! "$owner" ]; then owner=$_owner; @@ -371,14 +380,16 @@ fi; # if I cannot connect with user postgres to template1, the restore won't work _PG_PARAMS=("${PG_PARAMS[@]}"); _PG_PARAMS+=("-U" "postgres" "template1" "-q" "-t" "-X" "-A" "-F" "," "-c" "SELECT version();"); -_output=$("${PG_PSQL}" "${PG_PARAMS[@]}" 2>&1); +_output=$("${PG_PSQL}" "${_PG_PARAMS[@]}" 2>&1); found=$(echo "$_output" | grep "PostgreSQL"); # if the output does not have the PG version string, we have an error and abort if [ -z "$found" ]; then echo "Cannot connect to the database: $_output"; exit 1; fi; - +if [ ${DRY_RUN} ]; then + echo "**** [DRY RUN] ****"; +fi; echo "Will drop database '$database' on host '$_host:$_port' and load file '$file' with user '$owner', set encoding '$encoding' and use database version '$ident'"; if [ $SCHEMA_ONLY -eq 1 ]; then echo "!!!!!!! WILL ONLY RESTORE SCHEMA, NO DATA !!!!!!!"; @@ -400,7 +411,6 @@ else _PG_PARAMS=("${PG_PARAMS[@]}"); _PG_PARAMS+=("-U" "postgres" "${database}"); if [ $DRY_RUN -eq 0 ]; then - # $PG_DROPDB -U postgres $host $port $database; "${PG_DROPDB}" "${PG_PARAMS[@]}"; else echo "${PG_DROPDB} ${PG_PARAMS[*]}"; @@ -410,7 +420,6 @@ else _PG_PARAMS=("${PG_PARAMS[@]}"); _PG_PARAMS+=("-U" "postgres" "-O" "${owner}" "-E" "${encoding}" "-T" "${TEMPLATEDB}" "${database}"); if [ $DRY_RUN -eq 0 ]; then - # $PG_CREATEDB -U postgres -O $owner -E $encoding -T $TEMPLATEDB $host $port $database; "${PG_CREATEDB}" "${_PG_PARAMS[@]}"; else echo "${PG_CREATEDB} ${_PG_PARAMS[*]}"; @@ -421,7 +430,6 @@ else _PG_PARAMS+=("-U" "postgres" "plpgsql" "${database}"); echo "Create plpgsql lang in DB $database on [$_host:$_port] @ $(date +"%F %T")"; if [ $DRY_RUN -eq 0 ]; then - # $PG_CREATELANG -U postgres plpgsql $host $port $database; "${PG_CREATELANG}" "${_PG_PARAMS[@]}"; else echo "${PG_CREATELANG} ${_PG_PARAMS[*]}"; @@ -433,18 +441,14 @@ else _PG_PARAMS+=("${PG_PARAM_ROLE[@]}") _PG_PARAMS+=("-U" "postgres" "-d" "${database}" "-F" "c" "-v" "-c" "${schema}" "-j" "${MAX_JOBS}" "${file}"); if [ $DRY_RUN -eq 0 ]; then - # $PG_RESTORE -U postgres -d $database -F c -v -c $schema -j $MAX_JOBS $host $port $role $file 2>restore_errors.$LOG_FILE_EXT; "${PG_RESTORE}" "${_PG_PARAMS[@]}" 2>"restore_errors.${LOG_FILE_EXT}"; else echo "${PG_RESTORE} ${_PG_PARAMS[*]} 2>restore_errors.${LOG_FILE_EXT}"; fi; # BUG FIX FOR POSTGRESQL 9.6.2 db_dump # it does not dump the default public ACL so the owner of the DB cannot access the data, check if the ACL dump is missing and do a basic restore - # if [ -z "$("$PG_RESTORE" -l "$file" | grep -- "ACL - public postgres")" ]; then - if ! "$("${PG_RESTORE}" -l "$file" | grep -q -- "ACL - public postgres")"; then + if ! "${PG_RESTORE}" -l "$file" | grep -q -- "ACL - public postgres"; then echo "Fixing missing basic public schema ACLs from DB $database [$_host:$_port] @ $(date +"%F %T")"; - # echo "GRANT USAGE ON SCHEMA public TO public;" | $PG_PSQL -U postgres -Atq $host $port $database; - # echo "GRANT CREATE ON SCHEMA public TO public;" | $PG_PSQL -U postgres -Atq $host $port $database; # grant usage on schema public to public; _PG_PARAMS=("${PG_PARAMS[@]}"); _PG_PARAMS+=("-U" "postgres" "-Atq" "-c" "GRANT USAGE ON SCHEMA public TO public;" "${database}"); @@ -462,13 +466,11 @@ else _PG_PARAMS_OUT=("${PG_PARAMS[@]}"); _PG_PARAMS_OUT+=("-U" "postgres" "-e" "-f" "${TEMP_FILE}" "${database}"); if [ $DRY_RUN -eq 0 ]; then - # echo "${reset_query}" | $PG_PSQL -U postgres -Atq $host $port -o $TEMP_FILE $database "${PG_PSQL}" "${_PG_PARAMS[@]}"; - # $PG_PSQL -U postgres $host $port -e -f $TEMP_FILE $database 1>output_sequence.$LOG_FILE_EXT 2>errors_sequence.$database.$LOG_FILE_EXT; "${PG_PSQL}" "${_PG_PARAMS_OUT[@]}" 1>"output_sequence.${LOG_FILE_EXT}" 2>"errors_sequence.${database}.${LOG_FILE_EXT}"; rm "${TEMP_FILE}"; else - echo "${reset_query}"; + echo "${PG_PSQL}　${_PG_PARAMS[*]}"; echo "${PG_PSQL} ${_PG_PARAMS_OUT[*]} 1>output_sequence.${LOG_FILE_EXT} 2>errors_sequence.${database}.${LOG_FILE_EXT}"; fi; echo "Restore of data $file for DB $database [$_host:$_port] finished"; diff --git a/pg_restore_db_file.sh b/pg_restore_db_file.sh index ab546b1..53955d9 100755 --- a/pg_restore_db_file.sh +++ b/pg_restore_db_file.sh @@ -28,6 +28,7 @@ _host='local'; HOST=''; _encoding='UTF8'; set_encoding=''; +ERROR=0; REDHAT=0; IMPORT_GLOBALS=1; TEMPLATEDB='template0'; # truly empty for restore @@ -282,6 +283,9 @@ if [ -z "${PORT}" ]; then else _PORT=${PORT}; fi; +if [ ${DRY_RUN} ]; then + echo "**** [DRY RUN] ****"; +fi; echo "= Will import $db_count databases from $_DUMP_FOLDER" | $LOGFILE; echo "= into the DB server $_HOST:$_PORT" | $LOGFILE; echo "= running $MAX_JOBS jobs" | $LOGFILE; @@ -298,21 +302,22 @@ if [ "$IMPORT_GLOBALS" -eq 1 ]; then # get the pg_globals file echo "=[Globals Restore]=START=[$start_time]==================================================>" | $LOGFILE; # get newest and only the first one - # file=$(ls -1t "$DUMP_FOLDER/pg_global"* | head -1); file=$(find "$DUMP_FOLDER" -name "pg_global*" -type f -printf "%Ts\t%p\n" | sort -nr | head -1); filename=$(basename "$file"); # the last _ is for version 10 or higher # db version, without prefix of DB type version=$(echo "$filename" | cut -d "." -f 4 | cut -d "-" -f 2 | cut -d "_" -f 1); + cut_pos=4; # if this is < 10 then we need the second part too if [ "${version}" -lt 10 ]; then # db version, second part (after .) version=$version'.'$(echo "$filename" | cut -d "." -f 5 | cut -d "_" -f 1); + cut_pos=5; fi; # hostname of original DB, can be used as target host too - __host=$(echo "$filename" | cut -d "." -f 5 | cut -d "_" -f 2); + __host=$(echo "$filename" | cut -d "." -f ${cut_pos} | cut -d "_" -f 2); # port of original DB, can be used as target port too - __port=$(echo "$filename" | cut -d "." -f 5 | cut -d "_" -f 3); + __port=$(echo "$filename" | cut -d "." -f ${cut_pos} | cut -d "_" -f 3); # override file port over given port if it differs and is valid if [ -z "$_port" ] && [ "$__port" != "$_port" ] && [[ "$__port" =~ $PORT_REGEX ]] ; then _port=$__port; @@ -415,14 +420,12 @@ for file in "$DUMP_FOLDER/"*.sql; do # for the call _PG_PARAMS=("${_PG_PARAMS_ALL[@]}"); _PG_PARAMS+=("-A" "-F" "," "-t" "-q" "-X" "-c" "SELECT oid FROM pg_roles WHERE rolname = '$owner';" "template1"); - # user_oid=$(echo "SELECT oid FROM pg_roles WHERE rolname = '$owner';" | $PSQL -U postgres $host $port -A -F "," -t -q -X template1); user_oid=$("$PG_PSQL" "${_PG_PARAMS[@]}"); if [ -z "$user_oid" ]; then echo "+ Create USER '$owner' for DB '$database' [$_host:$_port] @ $(date +"%F %T")" | $LOGFILE; _PG_PARAMS=("${_PG_PARAMS_ALL[@]}"); _PG_PARAMS+=("-D" "-R" "-S" "$owner"); if [ ${DRY_RUN} -eq 0 ]; then - # "${PG_PATH}${PG_CREATEUSER}" -U postgres -D -R -S $host $port $owner; "${PG_PATH}${PG_CREATEUSER}" "${_PG_PARAMS[@]}"; else echo "${PG_PATH}${PG_CREATEUSER} ${_PG_PARAMS[*]}"; @@ -441,7 +444,6 @@ for file in "$DUMP_FOLDER/"*.sql; do _PG_PARAMS=("${_PG_PARAMS_ALL[@]}"); _PG_PARAMS+=("-O" "$owner" "-E" "$set_encoding" "-T" "$TEMPLATEDB" "$database"); if [ ${DRY_RUN} -eq 0 ]; then - # "${PG_PATH}${PG_CREATEDB}" -U postgres -O $owner -E $set_encoding -T $TEMPLATEDB $host $port $database; "${PG_PATH}${PG_CREATEDB}" "${_PG_PARAMS[@]}"; else echo "${PG_PATH}${PG_CREATEDB} ${_PG_PARAMS[*]}"; @@ -451,7 +453,6 @@ for file in "$DUMP_FOLDER/"*.sql; do _PG_PARAMS=("${_PG_PARAMS_ALL[@]}"); _PG_PARAMS+=("plpgsql" "$database"); if [ ${DRY_RUN} -eq 0 ]; then - # "${PG_PATH}${PG_CREATELANG}" -U postgres plpgsql $host $port $database; "${PG_PATH}${PG_CREATELANG}" "${_PG_PARAMS[@]}"; else echo "${PG_PATH}${PG_CREATELANG} ${_PG_PARAMS[*]}"; @@ -461,7 +462,6 @@ for file in "$DUMP_FOLDER/"*.sql; do _PG_PARAMS=("${_PG_PARAMS_ALL[@]}"); _PG_PARAMS+=("-d" "$database" "-F" "c" "-v" "-c" "-j" "$MAX_JOBS" "$file"); if [ ${DRY_RUN} -eq 0 ]; then - # "${PG_PATH}${PG_RESTORE}" -U postgres -d $database -F c -v -c -j $MAX_JOBS $host $port $file 2>"$LOGS/errors.${database}.$(date +"%Y%m%d_%H%M%S").log"; "${PG_PATH}${PG_RESTORE}" "${_PG_PARAMS[@]}" 2>"$LOGS/errors.${database}.$(date +"%Y%m%d_%H%M%S").log"; else echo "${PG_PATH}${PG_RESTORE} ${_PG_PARAMS[*]} 2>${LOGS}/errors.${database}.$(date +"%Y%m%d_%H%M%S").log"; @@ -469,10 +469,8 @@ for file in "$DUMP_FOLDER/"*.sql; do # BUG FIX FOR POSTGRESQL 9.6.2 db_dump # it does not dump the default public ACL so the owner of the DB cannot access the data, # check if the ACL dump is missing and do a basic restore - if ! "$("${PG_PATH}${PG_RESTORE}" -l "$file" | grep -q -- "ACL - public postgres")"; then + if ! "${PG_PATH}${PG_RESTORE}" -l "$file" | grep -q -- "ACL - public postgres"; then echo "? Fixing missing basic public schema ACLs from DB $database [$_host:$_port] @ $(date +"%F %T")"; - # echo "GRANT USAGE ON SCHEMA public TO public;" | "${PG_PATH}${PG_PSQL}" -U postgres -Atq $host $port $database; - # echo "GRANT CREATE ON SCHEMA public TO public;" | "${PG_PATH}${PG_PSQL}" -U postgres -Atq $host $port $database; # grant usage on schema public to public; _PG_PARAMS=("${PG_PARAMS[@]}"); _PG_PARAMS+=("-Atq" "-c" "GRANT USAGE ON SCHEMA public TO public;" "${database}");