diff --git a/pg_db_dump_file.sh b/pg_db_dump_file.sh index 17873b0..b9cdd43 100755 --- a/pg_db_dump_file.sh +++ b/pg_db_dump_file.sh @@ -344,7 +344,8 @@ fi; # check if we can connect to template1 table, if not we abort here _PG_PARAMS_SELECT=("${PG_PARAMS_SELECT[@]}"); _PG_PARAMS_SELECT+=("SELECT datname FROM pg_catalog.pg_database WHERE datname = 'template1';"); -connect=$(${PG_PSQL} "${_PG_PARAMS_SELECT[@]}") || echo "[!] pgsql connect error"; +PG_COMMAND=("${PG_PSQL}" "${_PG_PARAMS_SELECT[@]}"); +connect=$("${PG_COMMAND[@]}") || echo "[!] pgsql connect error"; if [ "${connect}" != "template1" ]; then echo "Failed to connect to template1 with user '${DB_USER}' at host '${DB_HOST}' on port '${DB_PORT}'"; exit 0; @@ -491,7 +492,8 @@ function get_dump_databases fi; _PG_PARAMS_SELECT=("${PG_PARAMS_SELECT[@]}"); _PG_PARAMS_SELECT+=("SELECT pg_catalog.pg_get_userbyid(datdba) AS owner, datname, pg_catalog.pg_encoding_to_char(encoding) FROM pg_catalog.pg_database WHERE datname !~ 'template(0|1)';"); - for owner_db in $(${PG_PSQL} "${_PG_PARAMS_SELECT[@]}"); do + PG_COMMAND=("${PG_PSQL}" "${_PG_PARAMS_SELECT[@]}"); + for owner_db in $("${PG_COMMAND[@]}"); do db=$(echo "${owner_db}" | cut -d "," -f 2); # check if we exclude this db exclude=0; @@ -615,10 +617,11 @@ if [ ${GLOBALS} -eq 1 ]; then # build dump parms _PG_PARAMS_DUMP=("${PG_PARAMS[@]}"); _PG_PARAMS_DUMP+=("--globals-only") + PG_COMMAND=("${PG_DUMPALL}" "${_PG_PARAMS_DUMP[@]}"); if [ ${TEST} -eq 0 ]; then - ${PG_DUMPALL} "${_PG_PARAMS_DUMP[@]}" > "${filename}"; + "${PG_COMMAND[@]}" > "${filename}"; else - echo "${PG_DUMPALL} ${_PG_PARAMS_DUMP[*]} > ${filename}"; + echo "${PG_COMMAND[*]} > ${filename}"; fi; echo "done"; else @@ -641,7 +644,8 @@ fi; filesize_sum=0; _PG_PARAMS_SELECT=("${PG_PARAMS_SELECT[@]}"); _PG_PARAMS_SELECT+=("SELECT pg_catalog.pg_get_userbyid(datdba) AS owner, datname, pg_catalog.pg_encoding_to_char(encoding) AS encoding FROM pg_catalog.pg_database WHERE datname !~ 'template(0|1)' ORDER BY datname;"); -for owner_db in $(${PG_PSQL} "${_PG_PARAMS_SELECT[@]}"); do +PG_COMMAND=("${PG_PSQL}" "${_PG_PARAMS_SELECT[@]}"); +for owner_db in $("${PG_COMMAND[@]}"); do # get the user who owns the DB too owner=$(echo "${owner_db}" | cut -d "," -f 1); db=$(echo "${owner_db}" | cut -d "," -f 2); @@ -673,10 +677,11 @@ for owner_db in $(${PG_PSQL} "${_PG_PARAMS_SELECT[@]}"); do # build dump parms _PG_PARAMS_DUMP=("${PG_PARAMS[@]}"); _PG_PARAMS_DUMP+=("-c" "--format=c" "${db}") + PG_COMMAND=("${PG_DUMP}" "${_PG_PARAMS_DUMP[@]}"); if [ ${TEST} -eq 0 ]; then - ${PG_DUMP} "${_PG_PARAMS_DUMP[@]}" > "${filename}"; + "${PG_COMMAND[@]}" > "${filename}"; else - echo "${PG_DUMP} ${_PG_PARAMS_DUMP[*]} > ${filename}"; + echo "${PG_COMMAND[*]} > ${filename}"; fi; # get the file size for the dumped file and convert it to a human readable format filesize=0; diff --git a/pg_drop_restore.sh b/pg_drop_restore.sh index e22d9fb..f9ba968 100755 --- a/pg_drop_restore.sh +++ b/pg_drop_restore.sh @@ -39,6 +39,8 @@ DRY_RUN=0; BC='/usr/bin/bc'; PORT_REGEX="^[0-9]{4,5}$"; OPTARG_REGEX="^-"; +# log path +LOG_PATH=''; MAX_JOBS=''; PG_PARAMS=(); PG_PARAM_ROLE=(); @@ -359,6 +361,18 @@ else PG_PATH="${PG_BASE_PATH}${ident}/bin/'"; fi; +# set log path, this is the base path of the file + logs +LOG_PATH=$(dirname "${file}")"/logs/"; +# create logs folder if missing +if [ ! -d "$LOG_PATH" ]; then + echo "+ Creating '$LOG_PATH' folder"; + mkdir -p "$LOG_PATH"; + if [ ! -d "$LOG_PATH" ]; then + echo "[!] Creation of '$LOG_PATH' folder failed"; + exit 1; + fi; +fi; + PG_DROPDB="${PG_PATH}dropdb"; PG_CREATEDB="${PG_PATH}createdb"; PG_CREATELANG="${PG_PATH}createlang"; @@ -390,11 +404,11 @@ fi; if [ $DRY_RUN -eq 1 ]; then echo "**** [DRY RUN] ****"; fi; -echo "Will drop database '$database' on host '$_host:$_port' and load file '$file' with user '$owner', set encoding '$encoding' and use database version '$ident'"; +echo "[.] Will drop database '$database' on host '$_host:$_port' and load file '$file' with user '$owner', set encoding '$encoding' and use database version '$ident'"; if [ $SCHEMA_ONLY -eq 1 ]; then echo "!!!!!!! WILL ONLY RESTORE SCHEMA, NO DATA !!!!!!!"; fi; -if [ $NO_ASK -eq 1 ]; then +if [ $NO_ASK -eq 1 ] || [ $DRY_RUN -eq 1 ]; then go='yes'; else echo "Continue? type 'yes'"; @@ -406,75 +420,135 @@ if [ "$go" != 'yes' ]; then else start_time=$(date +"%F %T"); START=$(date +'%s'); - echo "Drop DB $database [$_host:$_port] @ $start_time"; + echo "[1] - Drop DB $database [$_host:$_port] @ $start_time"; # DROP DATABASE _PG_PARAMS=("${PG_PARAMS[@]}"); _PG_PARAMS+=("-U" "postgres" "${database}"); + PG_COMMAND=("${PG_DROPDB}" "${_PG_PARAMS[@]}"); if [ $DRY_RUN -eq 0 ]; then - "${PG_DROPDB}" "${_PG_PARAMS[@]}"; + "${PG_COMMAND[@]}"; + RETURN_CODE=$?; + if [ $RETURN_CODE -ne 0 ]; then + echo "[!:${RETURN_CODE}] Could not drop database $database, aborting"; + exit 1; + fi; else - echo "${PG_DROPDB} ${_PG_PARAMS[*]}"; + echo "${PG_COMMAND[*]}"; fi; # CREATE DATABASE - echo "Create DB $database with $owner and encoding $encoding on [$_host:$_port] @ $(date +"%F %T")"; + echo "[2] + Create DB $database with $owner and encoding $encoding on [$_host:$_port] @ $(date +"%F %T")"; _PG_PARAMS=("${PG_PARAMS[@]}"); _PG_PARAMS+=("-U" "postgres" "-O" "${owner}" "-E" "${encoding}" "-T" "${TEMPLATEDB}" "${database}"); + PG_COMMAND=("${PG_CREATEDB}" "${_PG_PARAMS[@]}"); if [ $DRY_RUN -eq 0 ]; then - "${PG_CREATEDB}" "${_PG_PARAMS[@]}"; + "${PG_COMMAND[@]}"; + RETURN_CODE=$?; + if [ $RETURN_CODE -ne 0 ]; then + echo "[!:${RETURN_CODE}] Could not create database $database, aborting"; + exit 1; + fi; else - echo "${PG_CREATEDB} ${_PG_PARAMS[*]}"; + echo "${PG_COMMAND[*]}"; fi; # CREATE plpgsql LANG if [ -f "$PG_CREATELANG" ]; then _PG_PARAMS=("${PG_PARAMS[@]}"); _PG_PARAMS+=("-U" "postgres" "plpgsql" "${database}"); - echo "Create plpgsql lang in DB $database on [$_host:$_port] @ $(date +"%F %T")"; + PG_COMMAND=("${PG_CREATELANG}" "${_PG_PARAMS[@]}"); + echo "[3] + Create plpgsql lang in DB $database on [$_host:$_port] @ $(date +"%F %T")"; if [ $DRY_RUN -eq 0 ]; then - "${PG_CREATELANG}" "${_PG_PARAMS[@]}"; + "${PG_COMMAND[@]}"; + RETURN_CODE=$?; + if [ $RETURN_CODE -ne 0 ]; then + echo "[!:${RETURN_CODE}] Could not create plgpgsql language in $database, aborting"; + exit 1; + fi; else - echo "${PG_CREATELANG} ${_PG_PARAMS[*]}"; + echo "${PG_COMMAND[*]}"; fi; fi; # RESTORE DATA - echo "Restore data from $file to DB $database on [$_host:$_port] with Jobs $MAX_JOBS @ $(date +"%F %T")"; + echo "[4] % Restore data from $file to DB $database on [$_host:$_port] with Jobs $MAX_JOBS @ $(date +"%F %T")"; _PG_PARAMS=("${PG_PARAMS[@]}"); _PG_PARAMS+=("${PG_PARAM_ROLE[@]}") - _PG_PARAMS+=("-U" "postgres" "-d" "${database}" "-F" "c" "-v" "-c" "${schema}" "-j" "${MAX_JOBS}" "${file}"); + if [ -n "$schema" ]; then + _PG_PARAMS+=("${schema}"); + fi; + _PG_PARAMS+=("-U" "postgres" "-d" "${database}" "-F" "c" "-v" "-c" "-j" "${MAX_JOBS}" "${file}"); + PG_COMMAND=("${PG_RESTORE}" "${_PG_PARAMS[@]}"); + LOG_ERROR_FILE="${LOG_PATH}/restore_errors.${LOG_FILE_EXT}"; + LOG_OUTPUT_FILE="${LOG_PATH}/restore_output.${LOG_FILE_EXT}"; if [ $DRY_RUN -eq 0 ]; then - "${PG_RESTORE}" "${_PG_PARAMS[@]}" 2>"restore_errors.${LOG_FILE_EXT}"; + "${PG_COMMAND[@]}" 1>"${LOG_OUTPUT_FILE}" 2>"${LOG_ERROR_FILE}"; + RETURN_CODE=$?; + if [ $RETURN_CODE -ne 0 ]; then + echo "[!:${RETURN_CODE}] Could not restore the database $database successfully, check ${LOG_ERROR_FILE} for details"; + fi; else - echo "${PG_RESTORE} ${_PG_PARAMS[*]} 2>restore_errors.${LOG_FILE_EXT}"; + echo "${PG_COMMAND[*]} 1>${LOG_OUTPUT_FILE} 2>${LOG_ERROR_FILE}"; fi; # BUG FIX FOR POSTGRESQL 9.6.2 db_dump # it does not dump the default public ACL so the owner of the DB cannot access the data, check if the ACL dump is missing and do a basic restore if ! "${PG_RESTORE}" -l "$file" | grep -q -- "ACL - public postgres"; then - echo "Fixing missing basic public schema ACLs from DB $database [$_host:$_port] @ $(date +"%F %T")"; + echo "[5] ? Fixing missing basic public schema ACLs from DB $database [$_host:$_port] @ $(date +"%F %T")"; # grant usage on schema public to public; _PG_PARAMS=("${PG_PARAMS[@]}"); - _PG_PARAMS+=("-U" "postgres" "-Atq" "-c" "GRANT USAGE ON SCHEMA public TO public;" "${database}"); - "${PG_PSQL}" "${_PG_PARAMS[@]}"; + _PG_PARAMS+=("-U" "postgres" "-AtqX" "-c" "GRANT USAGE ON SCHEMA public TO public;" "${database}"); + PG_COMMAND=("${PG_PSQL}" "${_PG_PARAMS[@]}"); + if [ $DRY_RUN -eq 0 ]; then + "${PG_COMMAND[@]}"; + RETURN_CODE=$?; + if [ $RETURN_CODE -ne 0 ]; then + echo "[!:${RETURN_CODE}] Could not fix usage access to basic public schema ACLs in $database"; + fi; + else + echo "${PG_COMMAND[*]}"; + fi; # grant create on schema public to public; _PG_PARAMS=("${PG_PARAMS[@]}"); - _PG_PARAMS+=("-U" "postgres" "-Atq" "-c" "GRANT CREATE ON SCHEMA public TO public;" "${database}"); - "${PG_PSQL}" "${_PG_PARAMS[@]}"; + _PG_PARAMS+=("-U" "postgres" "-AtqX" "-c" "GRANT CREATE ON SCHEMA public TO public;" "${database}"); + PG_COMMAND=("${PG_PSQL}" "${_PG_PARAMS[@]}"); + if [ $DRY_RUN -eq 0 ]; then + "${PG_COMMAND[@]}"; + RETURN_CODE=$?; + if [ $RETURN_CODE -ne 0 ]; then + echo "[!:${RETURN_CODE}] Could not fix create access to basic public schema ACLs in $database"; + fi; + else + echo "${PG_COMMAND[*]}"; + fi; fi; # SEQUENCE RESET DATA COLLECTION - echo "Resetting all sequences from DB $database [$_host:$_port] @ $(date +"%F %T")"; + echo "[6] ? Resetting all sequences from DB $database [$_host:$_port] @ $(date +"%F %T")"; reset_query="SELECT 'SELECT SETVAL(' ||quote_literal(S.relname)|| ', MAX(' ||quote_ident(C.attname)|| ') ) FROM ' ||quote_ident(T.relname)|| ';' FROM pg_class AS S, pg_depend AS D, pg_class AS T, pg_attribute AS C WHERE S.relkind = 'S' AND S.oid = D.objid AND D.refobjid = T.oid AND D.refobjid = C.attrelid AND D.refobjsubid = C.attnum ORDER BY S.relname;"; _PG_PARAMS=("${PG_PARAMS[@]}"); - _PG_PARAMS+=("-U" "postgres" "-Atq" "-o" "${TEMP_FILE}" "-c" "${reset_query}" "${database}"); + _PG_PARAMS+=("-U" "postgres" "-AtqX" "-o" "${TEMP_FILE}" "-c" "${reset_query}" "${database}"); + PG_COMMAND=("${PG_PSQL}" "${_PG_PARAMS[@]}"); _PG_PARAMS_OUT=("${PG_PARAMS[@]}"); - _PG_PARAMS_OUT+=("-U" "postgres" "-e" "-f" "${TEMP_FILE}" "${database}"); + _PG_PARAMS_OUT+=("-U" "postgres" "-X" "-e" "-f" "${TEMP_FILE}" "${database}"); + PG_COMMAND_OUT=("${PG_PSQL}" "${_PG_PARAMS_OUT[@]}"); + LOG_OUTPUT_FILE="${LOG_PATH}/output_sequence.${LOG_FILE_EXT}"; + LOG_ERROR_FILE="${LOG_PATH}/errors_sequence.${database}.${LOG_FILE_EXT}"; if [ $DRY_RUN -eq 0 ]; then - "${PG_PSQL}" "${_PG_PARAMS[@]}"; - "${PG_PSQL}" "${_PG_PARAMS_OUT[@]}" 1>"output_sequence.${LOG_FILE_EXT}" 2>"errors_sequence.${database}.${LOG_FILE_EXT}"; - rm "${TEMP_FILE}"; + "${PG_COMMAND[@]}"; + RETURN_CODE=$?; + if [ $RETURN_CODE -ne 0 ]; then + echo "[!:${RETURN_CODE}] Could not create sequence reset query for database $database"; + fi; + if [ -f "${TEMP_FILE}" ]; then + "${PG_COMMAND_OUT[@]}" 1>"${LOG_OUTPUT_FILE}" 2>"${LOG_ERROR_FILE}"; + RETURN_CODE=$?; + if [ $RETURN_CODE -ne 0 ]; then + echo "[!:${RETURN_CODE}] Could not reset sequences for database $database"; + fi; + rm "${TEMP_FILE}"; + fi; else - echo "${PG_PSQL}　${_PG_PARAMS[*]}"; - echo "${PG_PSQL} ${_PG_PARAMS_OUT[*]} 1>output_sequence.${LOG_FILE_EXT} 2>errors_sequence.${database}.${LOG_FILE_EXT}"; + echo "${PG_COMMAND[*]}"; + echo "${PG_COMMAND_OUT[*]} 1>${LOG_OUTPUT_FILE} 2>${LOG_ERROR_FILE}"; fi; - echo "Restore of data $file for DB $database [$_host:$_port] finished"; + echo "[.] $ Restore of data $file for DB $database [$_host:$_port] finished"; DURATION=$(($(date "+%s")-START)); - echo "Start at $start_time and end at $(date +"%F %T") and ran for $(convert_time ${DURATION})"; - echo "=== END RESTORE" >>"restore_errors.${LOG_FILE_EXT}"; + echo "[.] * Start at $start_time and end at $(date +"%F %T") and ran for $(convert_time ${DURATION})"; + echo "=== END RESTORE" >>"${LOG_PATH}/restore_errors.${LOG_FILE_EXT}"; fi; diff --git a/pg_restore_db_file.sh b/pg_restore_db_file.sh index 53955d9..ad810c7 100755 --- a/pg_restore_db_file.sh +++ b/pg_restore_db_file.sh @@ -181,14 +181,14 @@ if [ ! -d "$DUMP_FOLDER" ]; then exit; fi; -LOGS=$DUMP_FOLDER'/logs/'; +LOG_PATH=$DUMP_FOLDER'/logs/'; # create logs folder if missing -if [ ! -d "$LOGS" ]; then - echo "Creating '$LOGS' folder"; - mkdir -p "$LOGS"; - if [ ! -d "$LOGS" ]; then - echo "Creation of '$LOGS' folder failed"; - exit; +if [ ! -d "$LOG_PATH" ]; then + echo "+ Creating '$LOG_PATH' folder"; + mkdir -p "$LOG_PATH"; + if [ ! -d "$LOG_PATH" ]; then + echo "[!] Creation of '$LOG_PATH' folder failed"; + exit 1; fi; fi; @@ -263,7 +263,7 @@ PG_PSQL="psql"; # default port and host EXCLUDE_LIST="pg_globals"; # space separated -LOGFILE="tee -a "$LOGS/PG_RESTORE_DB_FILE.$(date +"%Y%m%d_%H%M%S").log""; +LOG_FILE="tee -a "$LOG_PATH/PG_RESTORE_DB_FILE.$(date +"%Y%m%d_%H%M%S").log""; # get the count for DBs to import db_count=$(find "${DUMP_FOLDER}" -name "*.sql" -print | wc -l); @@ -286,11 +286,11 @@ fi; if [ ${DRY_RUN} ]; then echo "**** [DRY RUN] ****"; fi; -echo "= Will import $db_count databases from $_DUMP_FOLDER" | $LOGFILE; -echo "= into the DB server $_HOST:$_PORT" | $LOGFILE; -echo "= running $MAX_JOBS jobs" | $LOGFILE; -echo "= import logs: $LOGS" | $LOGFILE; -echo "" | $LOGFILE; +echo "= Will import $db_count databases from $_DUMP_FOLDER" | $LOG_FILE; +echo "= into the DB server $_HOST:$_PORT" | $LOG_FILE; +echo "= running $MAX_JOBS jobs" | $LOG_FILE; +echo "= import logs: $LOG_PATH" | $LOG_FILE; +echo "" | $LOG_FILE; pos=1; # go through all the files an import them into the database MASTERSTART=$(date +"%s"); @@ -300,7 +300,7 @@ if [ "$IMPORT_GLOBALS" -eq 1 ]; then start_time=$(date +"%F %T"); START=$(date +"%s"); # get the pg_globals file - echo "=[Globals Restore]=START=[$start_time]==================================================>" | $LOGFILE; + echo "=[Globals Restore]=START=[$start_time]==================================================>" | $LOG_FILE; # get newest and only the first one file=$(find "$DUMP_FOLDER" -name "pg_global*" -type f -printf "%Ts\t%p\n" | sort -nr | head -1); filename=$(basename "$file"); @@ -334,23 +334,24 @@ if [ "$IMPORT_GLOBALS" -eq 1 ]; then PG_PATH_VERSION_LOCAL="${PG_PATH_VERSION}"; fi; PG_PATH="${PG_BASE_PATH}${PG_PATH_VERSION_LOCAL}${PG_PATH_BIN}"; - echo "+ Restore globals file: $filename to [$_host:$_port] @ $(date +"%F %T")" | $LOGFILE; + echo "+ Restore globals file: $filename to [$_host:$_port] @ $(date +"%F %T")" | $LOG_FILE; _PG_PARAMS=("-U" "postgres"); _PG_PARAMS+=("${PG_PARAM_HOST[@]}"); _PG_PARAMS+=("${PG_PARAM_PORT[@]}"); _PG_PARAMS+=("-f" "$file" "-e" "-q" "-X" "template1"); + PG_COMMAND=("${PG_PATH}${PG_PSQL}" "${_PG_PARAMS[@]}"); if [ ${DRY_RUN} -eq 0 ]; then - "${PG_PATH}${PG_PSQL}" "${_PG_PARAMS[@]}" | $LOGFILE; + "${PG_COMMAND[@]}" | $LOG_FILE; else - echo "${PG_PATH}${PG_PSQL} ${_PG_PARAMS[*]}" | $LOGFILE; + echo "${PG_COMMAND[*]}" | $LOG_FILE; fi; DURATION=$(($(date +"%s")-START)); - printf "=[Globals Restore]=END===[%s]========================================================>\n" "$(convert_time ${DURATION})" | $LOGFILE; + printf "=[Globals Restore]=END===[%s]========================================================>\n" "$(convert_time ${DURATION})" | $LOG_FILE; fi; for file in "$DUMP_FOLDER/"*.sql; do start_time=$(date +"%F %T"); START=$(date +"%s"); - echo "=[$pos/$db_count]=START=[$start_time]==================================================>" | $LOGFILE; + echo "=[$pos/$db_count]=START=[$start_time]==================================================>" | $LOG_FILE; # the encoding set_encoding=''; # get the filename @@ -358,6 +359,22 @@ for file in "$DUMP_FOLDER/"*.sql; do # get the databse, user # default file name is ...-____