Ian Wienand 51733e5623 borg-backup: implement saving a stream, use for database backups
Add facility to borg-backup role to run a command and save the output
of it to a separate archive file during the backup process.

This is mostly useful for database backups.  Compressed on-disk logs
are terrible for differential backups because revisions have
essentially no common data.  By saving the uncompressed stream
directly from mysqldump, we allow borg the chance to de-duplicate,
saving considerable space on the backup servers.

This is implemented for our ansible-managed servers currently doing
dumps.  We also add it to the testinfra.

This also separates the archive names for the filesystem and stream
backup with unique prefixes so they can be pruned separately.
Otherwise we end up keeping only one of the stream or filesystem
backups which isn't the intention.  However, due to issues with
--append-only mode we are not issuing prune commands at this time.

Note the updated dump commands are updated slightly, particularly with
"--skip-extended-insert" which was suggested by mordred and
significantly improves incremental diff-ability by being slightly more
verbose but keeping much more of the output stable across dumps.

Change-Id: I500062c1c52c74a567621df9aaa716de804ffae7
2021-02-03 11:43:12 +11:00

202 lines
6.0 KiB
YAML

- name: Ensure docker-compose directory exists
file:
state: directory
path: /etc/gitea-docker
mode: 0700
- name: Write docker-compose file
template:
src: docker-compose.yaml.j2
dest: /etc/gitea-docker/docker-compose.yaml
mode: 0600
- name: Ensure gitea volume directories exists
file:
state: directory
path: "/var/gitea/{{ item }}"
owner: 1000
group: 1000
loop:
- conf
- data
- logs
- certs
- db
- name: Write app.ini
template:
src: app.ini.j2
dest: /var/gitea/conf/app.ini
- name: Install distro packages
package:
name:
- docker-compose
- python3-requests
state: present
- name: Install reverse proxy
include_tasks: proxy.yaml
when: gitea_reverse_proxy
- name: Run docker-compose pull
shell:
cmd: docker-compose pull
chdir: /etc/gitea-docker/
register: docker_compose_pull
- name: Stop/Start gitea safely for Gerrit replication
when: "'downloaded newer image' in docker_compose_pull.stderr"
block:
- name: Run docker-compose stop
shell:
cmd: docker-compose stop --timeout 60
chdir: /etc/gitea-docker/
- name: Run docker-compose up mariadb gitea-web
shell:
cmd: docker-compose up -d --timeout 60 mariadb gitea-web
chdir: /etc/gitea-docker/
# We wait here for the main gitea service to start before starting
# the ssh service. This is friendly to gerrit replication.
- name: Wait until the web service is sufficiently up to start ssh
uri:
url: "https://localhost:3000/api/v1/users/root"
validate_certs: false
status_code: 200, 404
register: root_user_check
delay: 1
retries: 300
until: root_user_check and root_user_check.status in (200, 404)
- name: Run docker-compose up gitea-ssh
shell:
cmd: docker-compose up -d --timeout 60 gitea-ssh
chdir: /etc/gitea-docker/
- name: Run docker prune to cleanup unneeded images
shell:
cmd: docker image prune -f
# User management outside of service bringup to avoid confusion between
# the two stages.
- name: Check if root user exists
uri:
url: "https://localhost:3000/api/v1/users/root"
validate_certs: false
status_code: 200, 404
register: root_user_check
delay: 1
retries: 300
until: root_user_check and root_user_check.status in (200, 404)
- name: Create root user
when: root_user_check.status==404
block:
- name: Create root user
command: "docker exec -t gitea-docker_gitea-web_1 gitea admin create-user --name root --password {{ gitea_root_password }} --email {{ gitea_root_email }} --admin"
no_log: "{{ gitea_no_log }}"
- name: Check if gerrit user exists
uri:
url: "https://localhost:3000/api/v1/users/gerrit"
validate_certs: false
status_code: 200, 404
register: gerrit_user_check
- name: Create gerrit user
when: gerrit_user_check.status==404
no_log: true
uri:
url: "https://localhost:3000/api/v1/admin/users"
validate_certs: false
method: POST
user: root
password: "{{ gitea_root_password }}"
force_basic_auth: true
status_code: 201
body_format: json
body:
email: "gerrit@review.opendev.org"
full_name: Gerrit
login_name: gerrit
must_change_password: false
password: "{{ gitea_gerrit_password }}"
send_notify: false
source_id: 0
username: gerrit
- name: Check if gerrit ssh key exists
uri:
user: root
password: "{{ gitea_root_password }}"
force_basic_auth: true
url: "https://localhost:3000/api/v1/users/gerrit/keys"
validate_certs: false
status_code: 200
register: gerrit_key_check
no_log: true
- name: Delete old gerrit ssh key
when: gerrit_key_check.json | length > 0 and gerrit_key_check.json[0].key != gitea_gerrit_public_key
no_log: true
uri:
user: root
password: "{{ gitea_root_password }}"
force_basic_auth: true
url: "https://localhost:3000/api/v1/user/keys/{{ gerrit_key_check.json[0].id }}"
validate_certs: false
method: DELETE
status_code: 204
- name: Add gerrit ssh key
when: gerrit_key_check.json | length == 0
no_log: true
uri:
user: root
password: "{{ gitea_root_password }}"
force_basic_auth: true
url: "https://localhost:3000/api/v1/admin/users/gerrit/keys"
validate_certs: false
method: POST
status_code: 201
body_format: json
body:
key: "{{ gitea_gerrit_public_key }}"
read_only: false
title: "Gerrit replication key"
- name: Set up cron job to pack git refs
cron:
name: pack-git-refs
state: present
job: "docker exec -t gitea-docker_gitea-web_1 find /data/git/repositories/ -maxdepth 2 -name *.git -type d -execdir git --git-dir={} gc --quiet \\;"
minute: '{{ 59 | random(seed=inventory_hostname) }}'
hour: '{{ 23 | random(seed=inventory_hostname) }}'
weekday: '0'
- name: Create db backup dest
file:
state: directory
path: /var/backups/gitea-mariadb
mode: 0700
owner: root
group: root
- name: Set up cron job to backup the database
cron:
name: gitea-db-backup
state: present
user: root
job: >
/usr/local/bin/docker-compose -f /etc/gitea-docker/docker-compose.yaml exec -T mariadb
bash -c '/usr/bin/mysqldump --opt --databases gitea --single-transaction -uroot -p"$MYSQL_ROOT_PASSWORD"' |
gzip -9 > /var/backups/gitea-mariadb/gitea-mariadb.sql.gz
minute: 42
hour: 4
- name: Rotate db backups
include_role:
name: logrotate
vars:
logrotate_file_name: /var/backups/gitea-mariadb/gitea-mariadb.sql.gz
logrotate_compress: false
- name: Setup db backup streaming job
block:
- name: Create backup streaming config dir
file:
path: /etc/borg-streams
state: directory
- name: Create db streaming file
copy:
content: >-
/usr/local/bin/docker-compose -f /etc/gitea-docker/docker-compose.yaml exec -T mariadb
bash -c '/usr/bin/mysqldump --skip-extended-insert --databases gitea --single-transaction -uroot -p"$MYSQL_ROOT_PASSWORD"'T_PASSWORD"'
dest: /etc/borg-streams/mysql