diff --git a/.github/workflows/test-build-deploy.yml b/.github/workflows/build-deploy.yml similarity index 80% rename from .github/workflows/test-build-deploy.yml rename to .github/workflows/build-deploy.yml index 8d09fb8..a0da389 100644 --- a/.github/workflows/test-build-deploy.yml +++ b/.github/workflows/build-deploy.yml @@ -2,42 +2,15 @@ name: Build and deploy on: push: - branches: - - '**' tags: - '**' - paths-ignore: - - 'README.md' - pull_request: - branches: - - master jobs: - test: - name: Test - - runs-on: ubuntu-20.04 - - steps: - - uses: actions/checkout@v2 - with: - submodules: recursive - - - name: Launch Docker containers - run: docker compose -f docker-compose.test.yml up -d --build - - - name: Test - run: docker compose -f docker-compose.test.yml exec backup make test - build: name: Build runs-on: ubuntu-20.04 - needs: test - - if: startsWith(github.ref, 'refs/tags/') - steps: - uses: actions/checkout@v2 @@ -75,12 +48,10 @@ jobs: deploy: name: Deploy - needs: [test, build] + needs: [build] runs-on: ubuntu-20.04 - if: startsWith(github.ref, 'refs/tags/') - steps: - uses: FranzDiebold/github-env-vars-action@v2 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..c19cd3f --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,26 @@ +name: Test + +on: + push: + branches: + - '**' + pull_request: + branches: + - master + +jobs: + test: + name: Test + + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + + - name: Launch Docker containers + run: docker compose -f docker-compose.test.yml up -d --build + + - name: Test + run: docker compose -f docker-compose.test.yml exec backup make test diff --git a/Makefile b/Makefile index 6708567..90269eb 100644 --- a/Makefile +++ b/Makefile @@ -2,5 +2,9 @@ all: test .PHONY: test test: + @printf "▶ Check scripts syntax\n" @for file in $$(find ./src -type f); do shellcheck -e 1091,2012 --format=tty $$file; done; + @printf "\n▶ Run unit tests\n" @./test/bats/bin/bats test/unit + @printf "\n▶ Run functional tests\n" + @./test/bats/bin/bats test/functional diff --git a/README.md b/README.md index 6c3f061..2b43fcb 100644 --- a/README.md +++ b/README.md @@ -2,14 +2,36 @@ This Docker image will backup your MySQL/MariaDB databases following the Grandfather-Father-Son (GFS) retention scheme. +## Features + +* GFS backups retention scheme +* AES256 encryption/decryption +* Single or multiple databases backups +* Grouped or individual archives +* Parallelized compression + +## GFS retention scheme + This means that you'll always have: -* A backup for every day of the last week -* A backup for every week of the last month -* A backup for every month of the last year +* A backup for every day of the last week (6) +* A backup for every week of the last month (4) +* A backup for every month of the last year (12) +* A backup for every previous year (unlimited) + +For a 100MB backup, it will cost you around 2GB for the current year + 100MB for each previous year. + +In SuperSafe mode, you'll have: + +* A backup for every day of the last month (28~31) +* A backup for every week of the last year (48) +* A backup for every previous year (unlimited) + +For a 100MB backup, backups will cost you around 8GB for the current year + 100MB for each previous year. Backups are run by default every day at 00:00 UTC. + ## Usage ```bash @@ -32,6 +54,7 @@ As an example, you can use: `MYSQL_PASSWORD_FILE=/run/secret/mysql-root-password | Variable | Description | Default | | -------- | ----------- | ------- | +| `SUPERSAFE_MODE` | Run backups in SuperSafe mode. This means many more backups ([see details](#gfs-retention-scheme)). | `false` | | `MYSQL_HOST` | The host of your MySQL/MariaDB database. | `mysql` | | `MYSQL_PORT` | The port number of your MySQL/MariaDB database. | `3306` | | `MYSQL_USER` | The username of your MySQL/MariaDB database. | `root` | diff --git a/docker-compose.test.yml b/docker-compose.test.yml index d6ad6d8..ef531d1 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -17,3 +17,8 @@ services: restart: 'no' depends_on: - database + volumes: + - ./src:/usr/local/bin + - ./src:/usr/src/secure-mysql-backups/src + - ./test:/usr/src/secure-mysql-backups/test + - ./Makefile:/usr/src/secure-mysql-backups/Makefile diff --git a/src/backup b/src/backup index b31f59c..e773275 100755 --- a/src/backup +++ b/src/backup @@ -11,9 +11,13 @@ host="$(get_env "MYSQL_HOST" "mysql")" user="$(get_env "MYSQL_USER" "root")" port="$(get_env "MYSQL_PORT" "3306")" password="$(get_env "MYSQL_PASSWORD")" -databases="$(get_env "MYSQL_DATABASE" "$(get_databases_list "$host" "$port" "$user" "$password")")" individual_backups="$(get_env "INDIVIDUAL_BACKUPS" false)" backup_name="$(get_env "BACKUP_NAME" "main-backup")" +supersafe_mode="$(get_env "SUPERSAFE_MODE" false)" + +wait_for_database "$host" "$port" "$user" "$password" "60" + +databases="$(get_env "MYSQL_DATABASE" "$(get_databases_list "$host" "$port" "$user" "$password")")" echo "Backup started at $(date)." @@ -21,13 +25,23 @@ for database in $databases; do dump_database "$host" "$port" "$user" "$password" "$database" || continue if [ "$individual_backups" = true ]; then - archive_file="$(get_archive_name "$backup_name.$database")" + if [ "$supersafe_mode" = true ]; then + archive_file="$(get_archive_name_supersafe_mode "$backup_name.$database")" + else + archive_file="$(get_archive_name "$backup_name.$database")" + fi + create_archive "$archive_file" "$database.sql" fi done if [ "$individual_backups" = false ]; then - archive_file="$(get_archive_name "$backup_name")" + if [ "$supersafe_mode" = true ]; then + archive_file="$(get_archive_name_supersafe_mode "$backup_name")" + else + archive_file="$(get_archive_name "$backup_name")" + fi + create_archive "$archive_file" "*.sql" fi diff --git a/src/lib/file.sh b/src/lib/file.sh index 0a5515c..28d8e12 100755 --- a/src/lib/file.sh +++ b/src/lib/file.sh @@ -45,19 +45,19 @@ get_file_gid () { || ls -ld "$1" | awk '{print $4}' } -#/ Get an archive name corresponding to the current name +#/ Get an archive name corresponding to the current day #/ #/ @usage var="$(get_archive_name "my-backup-name")" #/ #/ @param $1 Backup name -#/ @return Prints the result +#/ @return Prints the generated archive name get_archive_name () { if [ "$#" -lt 1 ]; then echo "At least 1 argument required, $# provided" exit 1 fi - local day day_num month_num week_file backup_name archive_file + local day day_num month_num year_num week_file backup_name archive_file backup_name="$1" # Find which week of the month 1-4 it is. @@ -74,7 +74,10 @@ get_archive_name () { # Create archive filename. day=$(date +%A) - if [ "$(date +%-d)" -eq "$(date -d "$(date +%-m)/1 + 1 month - 1 day" "+%d")" ]; then + if [ "$(date +%m-%d)" = "12-31" ]; then + year_num=$(date +%Y) + archive_file="$backup_name.year$year_num.tgz" + elif [ "$(date +%-d)" -eq "$(date -d "$(date +%-m)/1 + 1 month - 1 day" "+%d")" ]; then month_num=$(date +%m) archive_file="$backup_name.month$month_num.tgz" elif [ "$day" != "Saturday" ]; then @@ -87,6 +90,38 @@ get_archive_name () { echo "$archive_file" } +#/ Get an archive name corresponding to the current day +#/ +#/ @usage var="$(get_archive_name "my-backup-name")" +#/ +#/ @param $1 Backup name +#/ @return Prints the generated archive name +get_archive_name_supersafe_mode () { + if [ "$#" -lt 1 ]; then + echo "At least 1 argument required, $# provided" + exit 1 + fi + + local day_num day_num_pad week_num year_num backup_name archive_file + backup_name="$1" + + day_num=$(date +%-d) + day_num_pad=$(date +%d) + week_num=$(date +%V) + year_num=$(date +%Y) + + # Create archive filename. + if [ "$(date +%m-%d)" = "12-31" ]; then + archive_file="$backup_name.year$year_num.tgz" + elif [ "$((day_num % 7))" -eq 0 ]; then + archive_file="$backup_name.week$week_num.tgz" + else + archive_file="$backup_name.day$day_num_pad.tgz" + fi + + echo "$archive_file" +} + #/ Create an archive given an archive name and files list #/ #/ @usage create_archive "my-backup-name.my_db.day1-Monday.tgz" "my_db.sql" @@ -120,8 +155,7 @@ create_archive () { fi bash -c "rm -f /tmp/backup/$2 /tmp/backup/$archive_file" - chown -f "$chown_files" "/backup/$archive_file" "/backup/$archive_file.aes" - exit 0 + chown -f "$chown_files" "/backup/$archive_file" "/backup/$archive_file.aes" || true } diff --git a/src/run-cron b/src/run-cron index 2e799ee..18bec2c 100755 --- a/src/run-cron +++ b/src/run-cron @@ -8,10 +8,12 @@ source "$SCRIPT_DIR/lib/env.sh" cron_minute=$(get_env "CRON_MINUTE" "0") cron_hour=$(get_env "CRON_HOUR" "0") cron_time=$(get_env "CRON_TIME" "$cron_minute $cron_hour * * *") - -tail -F /var/log/backup.log & - -echo "Launching cron service..." +test_env=$(get_env "TEST_ENV" false) echo "$cron_time backup >> /var/log/backup.log" > /etc/crontabs/root -exec crond -f -L /var/log/cron.log + +if [ "$test_env" = false ]; then + echo "Launching cron service..." + tail -F /var/log/backup.log & + exec crond -f -L /var/log/cron.log +fi diff --git a/test/functional/backup.bats b/test/functional/backup.bats new file mode 100644 index 0000000..0812a04 --- /dev/null +++ b/test/functional/backup.bats @@ -0,0 +1,129 @@ +setup() { + source "/usr/local/bin/lib/db.sh" + + wait_for_database "database" "3306" "root" "root" "60" + mkdir -p /tmp/backup /backup +} + +teardown() { + run bash -c "rm -f /tmp/backup/* /backup/*" + run mysql_command "database" "3306" "root" "root" "DROP DATABASE IF EXISTS alpha;" + run mysql_command "database" "3306" "root" "root" "DROP DATABASE IF EXISTS beta;" +} + +@test "creates a daily backup on a Thursday" { + export MYSQL_HOST="database" + export MYSQL_PASSWORD="root" + export FAKETIME='2022-08-11 06:43:24' + + run backup + [ "$status" -eq 0 ] + [ -f "/backup/main-backup.day4-Thursday.tgz" ] +} + +@test "creates a weekly backup on a Saturday" { + export MYSQL_HOST="database" + export MYSQL_PASSWORD="root" + export FAKETIME='2022-08-13 06:43:24' + + run backup + [ "$status" -eq 0 ] + [ -f "/backup/main-backup.week2.tgz" ] +} + +@test "creates a monthly backup at the end of the month" { + export MYSQL_HOST="database" + export MYSQL_PASSWORD="root" + export FAKETIME='2022-08-31 06:43:24' + + run backup + [ "$status" -eq 0 ] + [ -f "/backup/main-backup.month08.tgz" ] +} + +@test "creates a yearly backup at the end of the year" { + export MYSQL_HOST="database" + export MYSQL_PASSWORD="root" + export FAKETIME='2022-12-31 06:43:24' + + run backup + [ "$status" -eq 0 ] + [ -f "/backup/main-backup.year2022.tgz" ] +} + +@test "creates a daily backup on a Thursday - SuperSafe mode" { + export MYSQL_HOST="database" + export MYSQL_PASSWORD="root" + export FAKETIME='2022-08-11 06:43:24' + export SUPERSAFE_MODE=true + + run backup + [ "$status" -eq 0 ] + [ -f "/backup/main-backup.day11.tgz" ] +} + +@test "creates a weekly backup on the 14th - SuperSafe mode" { + export MYSQL_HOST="database" + export MYSQL_PASSWORD="root" + export FAKETIME='2022-08-14 06:43:24' + export SUPERSAFE_MODE=true + + run backup + [ "$status" -eq 0 ] + [ -f "/backup/main-backup.week32.tgz" ] +} + +@test "creates a yearly backup at the end of the year - SuperSafe mode" { + export MYSQL_HOST="database" + export MYSQL_PASSWORD="root" + export FAKETIME='2022-12-31 06:43:24' + export SUPERSAFE_MODE=true + + run backup + [ "$status" -eq 0 ] + [ -f "/backup/main-backup.year2022.tgz" ] +} + +@test "creates a encrypted backup" { + export MYSQL_HOST="database" + export MYSQL_PASSWORD="root" + export AES_PASSPHRASE="passphrase" + export FAKETIME='2022-08-11 06:43:24' + + run backup + [ "$status" -eq 0 ] + [ -f "/backup/main-backup.day4-Thursday.tgz.aes" ] +} + +@test "creates individual backups for each database" { + export MYSQL_HOST="database" + export MYSQL_PASSWORD="root" + export INDIVIDUAL_BACKUPS=true + export FAKETIME='2022-08-11 06:43:24' + + mysql_command "database" "3306" "root" "root" "CREATE DATABASE alpha;" + mysql_command "database" "3306" "root" "root" "CREATE DATABASE beta;" + + run backup + [ "$status" -eq 0 ] + [ -f "/backup/main-backup.alpha.day4-Thursday.tgz" ] + [ -f "/backup/main-backup.beta.day4-Thursday.tgz" ] + [ -f "/backup/main-backup.my_db.day4-Thursday.tgz" ] +} + +@test "creates individual encrypted backups for each database" { + export MYSQL_HOST="database" + export MYSQL_PASSWORD="root" + export AES_PASSPHRASE="passphrase" + export INDIVIDUAL_BACKUPS=true + export FAKETIME='2022-08-11 06:43:24' + + mysql_command "database" "3306" "root" "root" "CREATE DATABASE alpha;" + mysql_command "database" "3306" "root" "root" "CREATE DATABASE beta;" + + run backup + [ "$status" -eq 0 ] + [ -f "/backup/main-backup.alpha.day4-Thursday.tgz.aes" ] + [ -f "/backup/main-backup.beta.day4-Thursday.tgz.aes" ] + [ -f "/backup/main-backup.my_db.day4-Thursday.tgz.aes" ] +} diff --git a/test/functional/run_cron.bats b/test/functional/run_cron.bats new file mode 100644 index 0000000..b5b00c7 --- /dev/null +++ b/test/functional/run_cron.bats @@ -0,0 +1,34 @@ +@test "run-cron command creates default cron entry" { + export TEST_ENV=true + + run run-cron + [ "$status" -eq 0 ] + [ "$(cat /etc/crontabs/root)" = "0 0 * * * backup >> /var/log/backup.log" ] +} + +@test "run-cron command creates cron with overriden minutes" { + export TEST_ENV=true + export CRON_MINUTE=12 + + run run-cron + [ "$status" -eq 0 ] + [ "$(cat /etc/crontabs/root)" = "12 0 * * * backup >> /var/log/backup.log" ] +} + +@test "run-cron command creates cron with overriden hours" { + export TEST_ENV=true + export CRON_HOUR=5 + + run run-cron + [ "$status" -eq 0 ] + [ "$(cat /etc/crontabs/root)" = "0 5 * * * backup >> /var/log/backup.log" ] +} + +@test "run-cron command creates cron fully overriden" { + export TEST_ENV=true + export CRON_TIME="*/10 9 * * sun" + + run run-cron + [ "$status" -eq 0 ] + [ "$(cat /etc/crontabs/root)" = "*/10 9 * * sun backup >> /var/log/backup.log" ] +} diff --git a/test/unit/get_archive_name.bats b/test/unit/get_archive_name.bats index ea144a4..f306d4d 100644 --- a/test/unit/get_archive_name.bats +++ b/test/unit/get_archive_name.bats @@ -15,7 +15,6 @@ setup() { alias date="FAKETIME='2022-01-05 06:35:46' date" run get_archive_name "main-backup" - echo "$output" [ "$status" -eq 0 ] [ "$output" = "main-backup.day3-Wednesday.tgz" ] } @@ -25,7 +24,6 @@ setup() { alias date="FAKETIME='2022-02-19 12:54:11' date" run get_archive_name "main-backup" - echo "$output" [ "$status" -eq 0 ] [ "$output" = "main-backup.week3.tgz" ] } @@ -38,3 +36,12 @@ setup() { [ "$status" -eq 0 ] [ "$output" = "main-backup.month02.tgz" ] } + +@test "last day of the year creates a monthly backup" { + shopt -s expand_aliases + alias date="FAKETIME='2020-12-31 18:43:24' date" + + run get_archive_name "main-backup" + [ "$status" -eq 0 ] + [ "$output" = "main-backup.year2020.tgz" ] +} diff --git a/test/unit/get_archive_name_supersafe_mode.bats b/test/unit/get_archive_name_supersafe_mode.bats new file mode 100644 index 0000000..50e83b3 --- /dev/null +++ b/test/unit/get_archive_name_supersafe_mode.bats @@ -0,0 +1,38 @@ +setup() { + DIR="$( cd "$( dirname "$BATS_TEST_FILENAME" )" >/dev/null 2>&1 && pwd )" + PATH="$DIR/../../src:$DIR/../../src/lib:$PATH" + source file.sh +} + +@test "fails if no argument given" { + run get_archive_name_supersafe_mode + [ "$status" -eq 1 ] + [ "$output" = "At least 1 argument required, 0 provided" ] +} + +@test "5th of the month creates a daily backup" { + shopt -s expand_aliases + alias date="FAKETIME='2022-07-05 06:35:46' date" + + run get_archive_name_supersafe_mode "main-backup" + [ "$status" -eq 0 ] + [ "$output" = "main-backup.day05.tgz" ] +} + +@test "21st of the month creates a weekly backup all year" { + shopt -s expand_aliases + alias date="FAKETIME='2022-06-21 06:35:46' date" + + run get_archive_name_supersafe_mode "main-backup" + [ "$status" -eq 0 ] + [ "$output" = "main-backup.week25.tgz" ] +} + +@test "last day of the year creates a yearly backup" { + shopt -s expand_aliases + alias date="FAKETIME='2021-12-31 06:35:46' date" + + run get_archive_name_supersafe_mode "main-backup" + [ "$status" -eq 0 ] + [ "$output" = "main-backup.year2021.tgz" ] +}