ScriptsCollections/DumpDatabasePostgreSQL
4
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Clean up code for pg_db_dump_file

Browse Source 
Fixes for wrong empty settings
Fix log path not set correct if given as parameter

Fix all shell linting problems
This commit is contained in:
Clemens Schwaighofer
2025-04-14 15:58:41 +09:00
parent 6adee6abc4
commit 7d2f75366d
1 changed files with 134 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

231
pg_db_dump_file.sh
View File

@@ -56,7 +56,7 @@ OPTARG_REGEX="^-";
# log path # log path
LOG_PATH=''; LOG_PATH='';
# base path for PostgreSQL binary # base path for PostgreSQL binary
DBPATH_BASE=''; # DBPATH_BASE='';
# defaults # defaults
_BACKUPDIR='/mnt/backup/db_dumps_fc/'; _BACKUPDIR='/mnt/backup/db_dumps_fc/';
_DB_VERSION=$(pgv=$(pg_dump --version| grep "pg_dump" | cut -d " " -f 3); if [[ $(echo "${pgv}" | cut -d "." -f 1) -ge 10 ]]; then echo "${pgv}" | cut -d "." -f 1; else echo "${pgv}" | cut -d "." -f 1,2; fi ); _DB_VERSION=$(pgv=$(pg_dump --version| grep "pg_dump" | cut -d " " -f 3); if [[ $(echo "${pgv}" | cut -d "." -f 1) -ge 10 ]]; then echo "${pgv}" | cut -d "." -f 1; else echo "${pgv}" | cut -d "." -f 1,2; fi );
@@ -68,7 +68,7 @@ _EXCLUDE=''; # space separated list of database names
_INCLUDE=''; # space seperated list of database names _INCLUDE=''; # space seperated list of database names
REDHAT=0; REDHAT=0;
AMAZON=0; AMAZON=0;
CONN_DB_HOST=''; # CONN_DB_HOST='';
ERROR=0; ERROR=0;
# set options # set options
@@ -125,79 +125,97 @@ while getopts ":ctsgnk:b:i:d:e:u:h:p:l:L:ram" opt; do
fi; fi;
# set options # set options
case ${opt} in case ${opt} in
t|test) # t|test)
t)
TEST=1; TEST=1;
;; ;;
g|globals) # g|globals)
g)
GLOBALS=0; GLOBALS=0;
;; ;;
c|clean-up-before) # c|clean-up-before)
c)
PRE_RUN_CLEAN_UP=1; PRE_RUN_CLEAN_UP=1;
;; ;;
s|sslmode) # s|sslmode)
s)
SSLMODE=enable; SSLMODE=enable;
;; ;;
k|keep) # k|keep)
k)
KEEP=${OPTARG}; KEEP=${OPTARG};
;; ;;
n|number-keep) # n|number-keep)
n)
CLEAN_NUMBER=1; CLEAN_NUMBER=1;
;; ;;
b|backuppath) # b|backuppath)
b)
if [ -z "${BACKUPDIR}" ]; then if [ -z "${BACKUPDIR}" ]; then
BACKUPDIR="${OPTARG}"; BACKUPDIR="${OPTARG}";
fi; fi;
;; ;;
i|ident) # i|ident)
i)
if [ -z "${DB_VERSION}" ]; then if [ -z "${DB_VERSION}" ]; then
DB_VERSION=${OPTARG}; DB_VERSION=${OPTARG};
SET_IDENT=1; SET_IDENT=1;
fi; fi;
;; ;;
u|user) # u|user)
u)
if [ -z "${DB_USER}" ]; then if [ -z "${DB_USER}" ]; then
DB_USER=${OPTARG}; DB_USER=${OPTARG};
fi; fi;
;; ;;
h|hostname) # h|hostname)
h)
if [ -z "${DB_HOST}" ]; then if [ -z "${DB_HOST}" ]; then
DB_HOST=${OPTARG}; DB_HOST=${OPTARG};
fi; fi;
;; ;;
p|port) # p|port)
p)
if [ -z "${DB_PORT}" ]; then if [ -z "${DB_PORT}" ]; then
DB_PORT=${OPTARG}; DB_PORT=${OPTARG};
fi; fi;
;; ;;
l|login) # l|login)
l)
if [ -z "${DB_PASSWD}" ]; then if [ -z "${DB_PASSWD}" ]; then
DB_PASSWD=${OPTARG}; DB_PASSWD=${OPTARG};
fi; fi;
;; ;;
d|database) # d|database)
if [ ! -z "${INCLUDE}" ]; then d)
if [ -z "${INCLUDE}" ]; then
INCLUDE=${INCLUDE}" "; INCLUDE=${INCLUDE}" ";
fi; fi;
INCLUDE=${INCLUDE}${OPTARG}; INCLUDE=${INCLUDE}${OPTARG};
;; ;;
e|exclude) # e|exclude)
if [ ! -z "${EXCLUDE}" ]; then e)
if [ -z "${EXCLUDE}" ]; then
EXCLUDE=${EXCLUDE}" "; EXCLUDE=${EXCLUDE}" ";
fi; fi;
EXCLUDE=${EXCLUDE}${OPTARG}; EXCLUDE=${EXCLUDE}${OPTARG};
;; ;;
r|redhat) # r|redhat)
r)
REDHAT=1; REDHAT=1;
;; ;;
a|amazon) # a|amazon)
a)
AMAZON=1; AMAZON=1;
;; ;;
L|logpath) # L|logpath)
if [ ! -z "{$LOG_PATH}" ]; then L)
if [ -z "${LOG_PATH}" ]; then
LOG_PATH="${OPTARG}"; LOG_PATH="${OPTARG}";
fi; fi;
;; ;;
m|manual) # m|manual)
m)
usage; usage;
exit 0; exit 0;
;; ;;
@@ -222,7 +240,7 @@ if [ "${REDHAT}" -eq 1 ] && [ "${AMAZON}" -eq 1 ]; then
fi; fi;
# if we have numeric keep and keep number is set to 0 abort # if we have numeric keep and keep number is set to 0 abort
if [ ${CLEAN_NUMBER} -eq 1 ] && [ ${KEEP} -lt 1 ]; then if [ "${CLEAN_NUMBER}" -eq 1 ] && [ "${KEEP}" -lt 1 ]; then
echo "If keep in numbers is on, keep must be at least 1 or higher"; echo "If keep in numbers is on, keep must be at least 1 or higher";
exit 0; exit 0;
fi; fi;
@@ -233,7 +251,7 @@ for name in BACKUPDIR DB_VERSION DB_USER DB_PASSWD DB_HOST DB_PORT EXCLUDE INCLU
if [ -z "${!name}" ]; then if [ -z "${!name}" ]; then
# add the _ for the default name # add the _ for the default name
default="_"${name}; default="_"${name};
eval ${name}=\${!default}; declare $name=${!default}
fi; fi;
done; done;
@@ -273,7 +291,7 @@ exec &> >(tee -a "${LOG}");
# check DB port is valid number # check DB port is valid number
if ! [[ "${DB_PORT}" =~ ${PORT_REGEX} ]]; then if ! [[ "${DB_PORT}" =~ ${PORT_REGEX} ]]; then
echo "The port needs to be a valid number: ${_port}"; echo "The port needs to be a valid number: ${DB_PORT}";
exit 0; exit 0;
fi; fi;
@@ -284,13 +302,19 @@ else
BC_OK=0; BC_OK=0;
fi; fi;
# the default pg params
PG_PARAMS=("-U" "${DB_USER}" "-p" "${DB_PORT}")
# if DB_HOST is set, we need to add -h to the command line # if DB_HOST is set, we need to add -h to the command line
# if nothing is set, DB_HOST is set to local so we know this is a "port" connection for later automatic restore # if nothing is set, DB_HOST is set to local so we know this is a "port" connection for later automatic restore
if [ -z "${DB_HOST}" ]; then if [ -z "${DB_HOST}" ]; then
DB_HOST='local'; DB_HOST='local';
else else
CONN_DB_HOST='-h '${DB_HOST}; # CONN_DB_HOST='-h '${DB_HOST};
PG_PARAMS+=("-h" "${DB_HOST}");
fi; fi;
# copy to select
PG_PARAMS_SELECT=("${PG_PARAMS[@]}");
PG_PARAMS_SELECT+=("-d" "template1" "-t" "-A" "-F" "," "-X" "-q" "-c");
# set the binaries we need # set the binaries we need
PG_PATH=${PG_BASE_PATH}${DB_VERSION}'/bin/'; PG_PATH=${PG_BASE_PATH}${DB_VERSION}'/bin/';
@@ -301,7 +325,7 @@ DB_TYPE='pgsql';
db=''; db='';
# core abort if no core files found # core abort if no core files found
if [ ! -f ${PG_PSQL} ] || [ ! -f ${PG_DUMP} ] || [ ! -f ${PG_DUMPALL} ]; then if [ ! -f "${PG_PSQL}" ] || [ ! -f "${PG_DUMP}" ] || [ ! -f "${PG_DUMPALL}" ]; then
echo "One of the core binaries (psql, pg_dump, pg_dumpall) could not be found."; echo "One of the core binaries (psql, pg_dump, pg_dumpall) could not be found.";
echo "Search Path: ${PG_PATH}"; echo "Search Path: ${PG_PATH}";
echo "Perhaps manual ident set with -i is necessary"; echo "Perhaps manual ident set with -i is necessary";
@@ -309,26 +333,28 @@ if [ ! -f ${PG_PSQL} ] || [ ! -f ${PG_DUMP} ] || [ ! -f ${PG_DUMPALL} ]; then
exit 0; exit 0;
fi; fi;
if [ ! -d ${BACKUPDIR} ] ; then if [ ! -d "${BACKUPDIR}" ] ; then
if ! mkdir ${BACKUPDIR} ; then if ! mkdir "${BACKUPDIR}" ; then
echo "Cannot create backup directory: ${BACKUPDIR}" echo "Cannot create backup directory: ${BACKUPDIR}"
exit 0; exit 0;
fi fi
fi fi
# check if we can write into that folder # check if we can write into that folder
touch ${BACKUPDIR}/tmpfile || echo "[!] touch failed"; touch "${BACKUPDIR}/tmpfile" || echo "[!] touch failed";
if [ ! -f ${BACKUPDIR}/tmpfile ]; then if [ ! -f "${BACKUPDIR}/tmpfile" ]; then
echo "Cannot write to ${BACKUPDIR}"; echo "Cannot write to ${BACKUPDIR}";
exit 0; exit 0;
else else
rm -f ${BACKUPDIR}/tmpfile; rm -f "${BACKUPDIR}/tmpfile";
fi; fi;
# if backupdir is "." rewrite to pwd # if backupdir is "." rewrite to pwd
if [ "${BACKUPDIR}" == '.' ]; then if [ "${BACKUPDIR}" == '.' ]; then
BACKUPDIR=$(pwd); BACKUPDIR=$(pwd);
fi; fi;
# check if we can connect to template1 table, if not we abort here # check if we can connect to template1 table, if not we abort here
connect=$(${PG_PSQL} -U "${DB_USER}" ${CONN_DB_HOST} -p ${DB_PORT} -d template1 -t -A -F "," -X -q -c "SELECT datname FROM pg_catalog.pg_database WHERE datname = 'template1';") || echo "[!] pgsql connect error"; _PG_PARAMS_SELECT=("${PG_PARAMS_SELECT[@]}");
_PG_PARAMS_SELECT+=("SELECT datname FROM pg_catalog.pg_database WHERE datname = 'template1';");
connect=$(${PG_PSQL} "${_PG_PARAMS_SELECT[@]}") || echo "[!] pgsql connect error";
if [ "${connect}" != "template1" ]; then if [ "${connect}" != "template1" ]; then
echo "Failed to connect to template1 with user '${DB_USER}' at host '${DB_HOST}' on port '${DB_PORT}'"; echo "Failed to connect to template1 with user '${DB_USER}' at host '${DB_HOST}' on port '${DB_PORT}'";
exit 0; exit 0;
@@ -336,7 +362,7 @@ fi;
# if we have an ident override set, set a different DUMP VERSION here than the automatic one # if we have an ident override set, set a different DUMP VERSION here than the automatic one
if [ "${SET_IDENT}" -eq 1 ]; then if [ "${SET_IDENT}" -eq 1 ]; then
DUMP_DB_VERSION=$(pgv=$(${PG_PATH}/pg_dump --version| grep "pg_dump" | cut -d " " -f 3); if [[ $(echo "${pgv}" | cut -d "." -f 1) -ge 10 ]]; then echo "${pgv}" | cut -d "." -f 1; else echo "${pgv}" | cut -d "." -f 1,2; fi ); DUMP_DB_VERSION=$(pgv=$("${PG_PATH}/pg_dump" --version | grep "pg_dump" | cut -d " " -f 3); if [[ $(echo "${pgv}" | cut -d "." -f 1) -ge 10 ]]; then echo "${pgv}" | cut -d "." -f 1; else echo "${pgv}" | cut -d "." -f 1,2; fi );
else else
DUMP_DB_VERSION=${DB_VERSION}; DUMP_DB_VERSION=${DB_VERSION};
fi; fi;
@@ -355,15 +381,15 @@ function convert_time
{ {
timestamp=${1}; timestamp=${1};
# round to four digits for ms # round to four digits for ms
timestamp=$(printf "%1.4f" $timestamp); timestamp=$(printf "%1.4f" "$timestamp");
# get the ms part and remove any leading 0 # get the ms part and remove any leading 0
ms=$(echo ${timestamp} | cut -d "." -f 2 | sed -e 's/^0*//'); ms=$(echo "${timestamp}" | cut -d "." -f 2 | sed -e 's/^0*//');
timestamp=$(echo ${timestamp} | cut -d "." -f 1); timestamp=$(echo "${timestamp}" | cut -d "." -f 1);
timegroups=(86400 3600 60 1); # day, hour, min, sec timegroups=(86400 3600 60 1); # day, hour, min, sec
timenames=("d" "h" "m" "s"); # day, hour, min, sec timenames=("d" "h" "m" "s"); # day, hour, min, sec
output=( ); output=( );
time_string=''; time_string='';
for timeslice in ${timegroups[@]}; do for timeslice in "${timegroups[@]}"; do
# floor for the division, push to output # floor for the division, push to output
if [ ${BC_OK} -eq 1 ]; then if [ ${BC_OK} -eq 1 ]; then
output[${#output[*]}]=$(echo "${timestamp}/${timeslice}" | bc); output[${#output[*]}]=$(echo "${timestamp}/${timeslice}" | bc);
@@ -375,8 +401,8 @@ function convert_time
done; done;
for ((i=0; i<${#output[@]}; i++)); do for ((i=0; i<${#output[@]}; i++)); do
if [ ${output[$i]} -gt 0 ] || [ ! -z "$time_string" ]; then if [ "${output[$i]}" -gt 0 ] || [ -n "$time_string" ]; then
if [ ! -z "${time_string}" ]; then if [ -n "${time_string}" ]; then
time_string=${time_string}" "; time_string=${time_string}" ";
fi; fi;
time_string=${time_string}${output[$i]}${timenames[$i]}; time_string=${time_string}${output[$i]}${timenames[$i]};
@@ -384,9 +410,9 @@ function convert_time
done; done;
# milliseconds must be filled, but we also check that they are non "nan" string # milliseconds must be filled, but we also check that they are non "nan" string
# that can appear in the original value # that can appear in the original value
if [ ! -z ${ms} ] && [ "${ms}" != "nan" ]; then if [ -n "${ms}" ] && [ "${ms}" != "nan" ]; then
if [ ${ms} -gt 0 ]; then if [ "${ms}" -gt 0 ]; then
time_string=${time_string}" "${ms}"ms"; time_string="${time_string} ${ms}ms";
fi; fi;
fi; fi;
# just in case the time is 0 # just in case the time is 0
@@ -405,15 +431,15 @@ function convert_bytes
{ {
bytes=${1}; bytes=${1};
# use awk to calc it # use awk to calc it
echo -n $(echo ${bytes} | awk 'function human(x) { echo -n "$(echo "${bytes}" | awk 'function human(x) {
s=" B KB MB GB TB EB PB YB ZB" s=" B KB MB GB TB EB PB YB ZB"
while (x>=1024 && length(s)>1) while (x>=1024 && length(s)>1)
{x/=1024; s=substr(s,4)} {x/=1024; s=substr(s,4)}
s=substr(s,1,4) s=substr(s,1,4)
xf=(s==" B ")?"%d ":"%.2f" xf=(s==" B ")?"%d ":"%.2f"
return sprintf( xf"%s\n", x, s) return sprintf( xf"%s\n", x, s)
} }
{gsub(/^[0-9]+/, human($1)); print}'); {gsub(/^[0-9]+/, human($1)); print}')";
} }
# METHOD: get_dump_file_name # METHOD: get_dump_file_name
@@ -424,22 +450,20 @@ function convert_bytes
function get_dump_file_name function get_dump_file_name
{ {
# set base search for the files # set base search for the files
sequence=*; if [ "${db}" ]; then
if [ ${db} ]; then db_name="${db}.${owner}.${encoding}.";
db_name=${db}"."${owner}"."${encoding}".";
else else
db_name="pg_globals."${DB_USER}".NONE."; db_name="pg_globals.${DB_USER}.NONE.";
fi; fi;
file=${BACKUPDIR}"/"${db_name}${DB_TYPE}"-"${DUMP_DB_VERSION}"_"${DB_HOST}"_"${DB_PORT}"_"$(date +%Y%m%d)"_"$(date +%H%M)"_"${sequence}".c.sql";
seq=''; seq='';
# we need to find the next sequence number # we need to find the next sequence number
for i in $(ls -1 ${file} 2>/dev/null); do for i in "${BACKUPDIR}/${db_name}${DB_TYPE}-${DUMP_DB_VERSION}_${DB_HOST}_${DB_PORT}_$(date +%Y%m%d)_$(date +%H%M)"*.c.sql; do
# get the last sequence and cut any leading 0 so we can run +1 on it # get the last sequence and cut any leading 0 so we can run +1 on it
seq=$(echo $i | cut -d "." -f 3 | cut -d "_" -f 4 | sed -e "s/^0//g"); seq=$(echo "$i" | cut -d "." -f 4 | cut -d "_" -f 6 | sed -e "s/^0//g");
done; done;
if [ ! -z ${seq} ]; then if [ -n "${seq}" ]; then
# add +1 and if < 10 prefix with 0 # add +1 and if < 10 prefix with 0
let seq=${seq}+1; seq=$((seq+1));
if [ ${seq} -lt 10 ]; then if [ ${seq} -lt 10 ]; then
sequence="0"${seq}; sequence="0"${seq};
else else
@@ -449,7 +473,7 @@ function get_dump_file_name
sequence="01"; sequence="01";
fi; fi;
# now build correct file name # now build correct file name
filename=${BACKUPDIR}"/"${db_name}${DB_TYPE}"-"${DUMP_DB_VERSION}"_"${DB_HOST}"_"${DB_PORT}"_"$(date +%Y%m%d)"_"$(date +%H%M)"_"${sequence}".c.sql"; filename="${BACKUPDIR}/${db_name}${DB_TYPE}-${DUMP_DB_VERSION}_${DB_HOST}_${DB_PORT}_$(date +%Y%m%d)_$(date +%H%M)_${sequence}.c.sql";
echo "${filename}"; echo "${filename}";
} }
@@ -466,8 +490,10 @@ function get_dump_databases
if [ ${GLOBALS} -eq 1 ]; then if [ ${GLOBALS} -eq 1 ]; then
search_names+=("pg_globals.*"); search_names+=("pg_globals.*");
fi; fi;
for owner_db in $(${PG_PSQL} -U ${DB_USER} ${CONN_DB_HOST} -p ${DB_PORT} -d template1 -t -A -F "," -X -q -c "SELECT pg_catalog.pg_get_userbyid(datdba) AS owner, datname, pg_catalog.pg_encoding_to_char(encoding) FROM pg_catalog.pg_database WHERE datname "\!"~ 'template(0|1)';"); do _PG_PARAMS_SELECT=("${PG_PARAMS_SELECT[@]}");
db=$(echo ${owner_db} | cut -d "," -f 2); _PG_PARAMS_SELECT+=("SELECT pg_catalog.pg_get_userbyid(datdba) AS owner, datname, pg_catalog.pg_encoding_to_char(encoding) FROM pg_catalog.pg_database WHERE datname !~ 'template(0|1)';");
for owner_db in $(${PG_PSQL} "${_PG_PARAMS_SELECT[@]}"); do
db=$(echo "${owner_db}" | cut -d "," -f 2);
# check if we exclude this db # check if we exclude this db
exclude=0; exclude=0;
include=0; include=0;
@@ -477,7 +503,7 @@ function get_dump_databases
break; break;
fi; fi;
done; done;
if [ ! -z "${INCLUDE}" ]; then if [ -n "${INCLUDE}" ]; then
for incl_db in ${INCLUDE}; do for incl_db in ${INCLUDE}; do
if [ "${db}" = "${incl_db}" ]; then if [ "${db}" = "${incl_db}" ]; then
include=1; include=1;
@@ -500,7 +526,7 @@ function get_dump_databases
# DESC : checks for older files than given keep time/amount and removes them # DESC : checks for older files than given keep time/amount and removes them
function clean_up function clean_up
{ {
if [ -d ${BACKUPDIR} ]; then if [ -d "${BACKUPDIR}" ]; then
if [ ${CLEAN_NUMBER} -eq 0 ]; then if [ ${CLEAN_NUMBER} -eq 0 ]; then
echo "Cleanup older than ${KEEP} days backup in ${BACKUPDIR}"; echo "Cleanup older than ${KEEP} days backup in ${BACKUPDIR}";
else else
@@ -508,41 +534,43 @@ function clean_up
# if we run clean before backup, we need to clean up +1 # if we run clean before backup, we need to clean up +1
# so if we keep one, we must remove all data before running new # so if we keep one, we must remove all data before running new
if [ ${PRE_RUN_CLEAN_UP} -eq 1 ]; then if [ ${PRE_RUN_CLEAN_UP} -eq 1 ]; then
let KEEP=${KEEP}-1; # let KEEP=${KEEP}-1;
KEEP=$((KEEP-1));
fi; fi;
fi; fi;
# build the find string based on the search names patter # build the find string based on the search names patter
find_string=''; find_params=("${BACKUPDIR}");
for name in "${search_names[@]}"; do for name in "${search_names[@]}"; do
# for not number based, we build the find string here # for not number based, we build the find string here
# else we do the delete here already # else we do the delete here already
if [ ${CLEAN_NUMBER} -eq 0 ]; then if [ ${CLEAN_NUMBER} -eq 0 ]; then
if [ ! -z "${find_string}" ]; then if [ ${#find_params[@]} -gt 1 ]; then
find_string=${find_string}' -o '; find_params+=("-o");
fi; fi;
find_string=${find_string}"-mtime +${KEEP} -name "${name}${DB_TYPE}*.sql" -type f -delete -print"; find_params+=("-mtime" "+${KEEP}" "-name" "${name}${DB_TYPE}*.sql" "-type" "f" "-delete" "-print");
echo "- Remove old backups for '${name}'"; echo "- Remove old backups for '${name}'";
else else
# if we do number based delete of old data, but only if the number of # if we do number based delete of old data, but only if the number of
# files is bigger than the keep number or equal if we do PRE_RUN_CLEAN_UP # files is bigger than the keep number or equal if we do PRE_RUN_CLEAN_UP
# this can be error, but we allow it -> script should not abort here # this can be error, but we allow it -> script should not abort here
# note we have a wildcard in the name, so we can't put that into "" # note we have a wildcard in the name, so we can't put that into ""
count=$(ls "${BACKUPDIR}/"${name}${DB_TYPE}*.sql 2>/dev/null | wc -l) || true; count=$(find "${BACKUPDIR}" -name "${name}${DB_TYPE}*.sql" -type f -print | wc -l);
if [ ${PRE_RUN_CLEAN_UP} -eq 1 ]; then if [ ${PRE_RUN_CLEAN_UP} -eq 1 ]; then
let count=${count}+1; # let count=${count}+1;
count=$((count+1));
fi; fi;
if [ ${count} -gt ${KEEP} ]; then if [ "${count}" -gt "${KEEP}" ]; then
# calculate the amount to delete # calculate the amount to delete
# eg if we want to keep 1, and we have 3 files then we need to delete 2 # eg if we want to keep 1, and we have 3 files then we need to delete 2
# keep is always +1 (include the to backup count). # keep is always +1 (include the to backup count).
# count is +1 if we do a pre-run cleanup # count is +1 if we do a pre-run cleanup
# grouped by db name, db type # grouped by db name, db type
let TO_DELETE=${count}-${KEEP}; TO_DELETE=$((count-KEEP));
echo "- Remove old backups for '${name}', found ${count}, will delete ${TO_DELETE}"; echo "- Remove old backups for '${name}', found ${count}, will delete ${TO_DELETE}";
if [ ${TEST} -eq 0 ]; then if [ ${TEST} -eq 0 ]; then
ls -tr "${BACKUPDIR}/"${name}${DB_TYPE}*.sql 2>/dev/null | head -n ${TO_DELETE} | xargs rm; find "${BACKUPDIR}" -name "${name}${DB_TYPE}*.sql" -type f -printf "%Ts\t%p\n" | sort -nr | head -n ${TO_DELETE} | xargs rm;
else else
echo "ls -tr ${BACKUPDIR}/${name}${DB_TYPE}*.sql 2>/dev/null | head -n ${TO_DELETE} | xargs rm"; print "find \"${BACKUPDIR}\" -name \"${name}${DB_TYPE}*.sql\" -type f -printf \"%Ts\\t%p\\n\" | sort -nr | head -n ${TO_DELETE} | xargs rm";
fi; fi;
fi; fi;
fi; fi;
@@ -550,15 +578,15 @@ function clean_up
# if we do find (day based) delete of old data # if we do find (day based) delete of old data
if [ ${CLEAN_NUMBER} -eq 0 ]; then if [ ${CLEAN_NUMBER} -eq 0 ]; then
if [ ${TEST} -eq 0 ]; then if [ ${TEST} -eq 0 ]; then
find ${BACKUPDIR} ${find_string}; find "${find_params[@]}";
else else
echo "find ${BACKUPDIR} ${find_string}"; echo "find ${find_params[*]}";
fi; fi;
fi; fi;
fi fi
} }
if [ ! -z "${DB_PASSWD}" ]; then if [ -n "${DB_PASSWD}" ]; then
export PGPASSWORD=${DB_PASSWD}; export PGPASSWORD=${DB_PASSWD};
fi; fi;
START=$(date "+%s"); START=$(date "+%s");
@@ -580,11 +608,15 @@ if [ ${GLOBALS} -eq 1 ]; then
# reset any previous set db name from deletes so the correct global file name is set # reset any previous set db name from deletes so the correct global file name is set
db=''; db='';
filename=$(get_dump_file_name); filename=$(get_dump_file_name);
search_names+=("pg_globals.*"); # this is used for the find/delete part # this is used for the find/delete part
search_names+=("pg_globals.*");
# build dump parms
_PG_PARAMS_DUMP=("${PG_PARAMS[@]}");
_PG_PARAMS_DUMP+=("--globals-only")
if [ ${TEST} -eq 0 ]; then if [ ${TEST} -eq 0 ]; then
${PG_DUMPALL} -U ${DB_USER} ${CONN_DB_HOST} -p ${DB_PORT} --globals-only > "${filename}"; ${PG_DUMPALL} "${_PG_PARAMS_DUMP[@]}" > "${filename}";
else else
echo "${PG_DUMPALL} -U ${DB_USER} ${CONN_DB_HOST} -p ${DB_PORT} --globals-only > ${filename}"; echo "${PG_DUMPALL} ${_PG_PARAMS_DUMP[*]} > ${filename}";
fi; fi;
echo "done"; echo "done";
else else
@@ -595,21 +627,23 @@ echo -n "(+) Dump databases: ";
if [ -z "${INCLUDE}" ]; then if [ -z "${INCLUDE}" ]; then
echo "All"; echo "All";
else else
echo ${INCLUDE}; echo "${INCLUDE}";
fi; fi;
echo -n "(-) Exclude databases: "; echo -n "(-) Exclude databases: ";
if [ -z "${EXCLUDE}" ]; then if [ -z "${EXCLUDE}" ]; then
echo "None"; echo "None";
else else
echo ${EXCLUDE}; echo "${EXCLUDE}";
fi; fi;
filesize_sum=0; filesize_sum=0;
for owner_db in $(${PG_PSQL} -U ${DB_USER} ${CONN_DB_HOST} -p ${DB_PORT} -d template1 -t -A -F "," -X -q -c "SELECT pg_catalog.pg_get_userbyid(datdba) AS owner, datname, pg_catalog.pg_encoding_to_char(encoding) AS encoding FROM pg_catalog.pg_database WHERE datname "\!"~ 'template(0|1)' ORDER BY datname;"); do _PG_PARAMS_SELECT=("${PG_PARAMS_SELECT[@]}");
_PG_PARAMS_SELECT+=("SELECT pg_catalog.pg_get_userbyid(datdba) AS owner, datname, pg_catalog.pg_encoding_to_char(encoding) AS encoding FROM pg_catalog.pg_database WHERE datname !~ 'template(0|1)' ORDER BY datname;");
for owner_db in $(${PG_PSQL} "${_PG_PARAMS_SELECT[@]}"); do
# get the user who owns the DB too # get the user who owns the DB too
owner=$(echo ${owner_db} | cut -d "," -f 1); owner=$(echo "${owner_db}" | cut -d "," -f 1);
db=$(echo ${owner_db} | cut -d "," -f 2); db=$(echo "${owner_db}" | cut -d "," -f 2);
encoding=$(echo ${owner_db} | cut -d "," -f 3); encoding=$(echo "${owner_db}" | cut -d "," -f 3);
# check if we exclude this db # check if we exclude this db
exclude=0; exclude=0;
include=0; include=0;
@@ -619,7 +653,7 @@ for owner_db in $(${PG_PSQL} -U ${DB_USER} ${CONN_DB_HOST} -p ${DB_PORT} -d temp
break; break;
fi; fi;
done; done;
if [ ! -z "${INCLUDE}" ]; then if [ -n "${INCLUDE}" ]; then
for incl_db in ${INCLUDE}; do for incl_db in ${INCLUDE}; do
if [ "${db}" = "${incl_db}" ]; then if [ "${db}" = "${incl_db}" ]; then
include=1; include=1;
@@ -634,25 +668,28 @@ for owner_db in $(${PG_PSQL} -U ${DB_USER} ${CONN_DB_HOST} -p ${DB_PORT} -d temp
filename=$(get_dump_file_name); filename=$(get_dump_file_name);
search_names+=("${db}.*"); search_names+=("${db}.*");
SUBSTART=$(date "+%s"); SUBSTART=$(date "+%s");
# build dump parms
_PG_PARAMS_DUMP=("${PG_PARAMS[@]}");
_PG_PARAMS_DUMP+=("-c" "--format=c" "${db}")
if [ ${TEST} -eq 0 ]; then if [ ${TEST} -eq 0 ]; then
${PG_DUMP} -U ${DB_USER} ${CONN_DB_HOST} -p ${DB_PORT} -c --format=c ${db} > "${filename}"; ${PG_DUMP} "${_PG_PARAMS_DUMP[@]}" > "${filename}";
else else
echo "${PG_DUMP} -U ${DB_USER} ${CONN_DB_HOST} -p ${DB_PORT} -c --format=c ${db} > ${filename}"; echo "${PG_DUMP} ${_PG_PARAMS_DUMP[*]} > ${filename}";
fi; fi;
# get the file size for the dumped file and convert it to a human readable format # get the file size for the dumped file and convert it to a human readable format
filesize=0; filesize=0;
if [ -f "${filename}" ]; then if [ -f "${filename}" ]; then
filesize=$(wc -c "${filename}" | cut -f 1 -d ' '); filesize=$(wc -c "${filename}" | cut -f 1 -d ' ');
filesize_sum=$[$filesize+$filesize_sum]; filesize_sum=$((filesize+filesize_sum));
fi; fi;
DURATION=$[$(date "+%s")-${SUBSTART}]; DURATION=$(($(date "+%s")-SUBSTART));
printf "done (%s and %s)\n" "$(convert_time ${DURATION})" "$(convert_bytes ${filesize})"; printf "done (%s and %s)\n" "$(convert_time ${DURATION})" "$(convert_bytes "${filesize}")";
else else
printf -- "- Exclude database: %35s\n" "${db}"; printf -- "- Exclude database: %35s\n" "${db}";
fi; fi;
done done
printf "Backup ended at %s\n" "$(date '+%Y-%m-%d %H:%M:%S')"; printf "Backup ended at %s\n" "$(date '+%Y-%m-%d %H:%M:%S')";
if [ ! -z "${DB_PASSWD}" ]; then if [ -n "${DB_PASSWD}" ]; then
unset DB_PASSWD; unset DB_PASSWD;
fi; fi;
@@ -660,7 +697,7 @@ if [ ${PRE_RUN_CLEAN_UP} -eq 0 ]; then
clean_up; clean_up;
fi; fi;
DURATION=$[$(date "+%s")-${START}]; DURATION=$(($(date "+%s")-START));
printf "Cleanup ended at %s\n" "$(date '+%Y-%m-%d %H:%M:%S')"; printf "Cleanup ended at %s\n" "$(date '+%Y-%m-%d %H:%M:%S')";
printf "Finished backup in %s with %s\n" "$(convert_time ${DURATION})" "$(convert_bytes ${filesize_sum})"; printf "Finished backup in %s with %s\n" "$(convert_time ${DURATION})" "$(convert_bytes ${filesize_sum})";