Add support for other s3 providers (#90)

* Add support for other s3 providers

You can configure another s3 provider by supplying the node_backup_s3_provider, node_backup_s3_access_key_id, node_backup_s3_secret_access_key, node_backup_s3_endpoint and optionally node_backup_s3_region. To use this type of service, you have to set the backup type to s3-rclone.

* Node backups - only use no_check_bucket on R2

Restrict the no_check_bucket to Cloudflare as the impact on other sites is unknown.

Also bumps galaxy.yml

galaxy.yml

@@ -8,7 +8,7 @@ namespace: paritytech
 name: chain
 
 # The version of the collection. Must be compatible with semantic versioning
-version: 1.10.8
+version: 1.10.9
 
 # The path to the Markdown (.md) readme file. This path is relative to the root of the collection
 readme: README.md

roles/node_backup/defaults/main.yml

@@ -6,6 +6,14 @@ node_backup_r2_access_key_id: ""
 node_backup_r2_secret_access_key: ""
 node_backup_r2_api_url: ""
 
+# S3 Compatible configuration (defaults to filling with r2 configuration for backward compat)
+node_backup_s3_access_key_id: "{{ node_backup_r2_access_key_id }}"
+node_backup_s3_secret_access_key: "{{ node_backup_r2_secret_access_key }}"
+node_backup_s3_endpoint: "{{ node_backup_r2_api_url }}"
+node_backup_s3_region: ""
+# The rclone provider to use for the backup
+node_backup_s3_provider: Cloudflare
+
 node_backup_max_concurrent_requests: 50
 
 node_backup_schedule:
@@ -31,7 +39,7 @@ node_backup_targets: []
 #   # old way of backups. It takes more time to restore and backup
 #   # it's true by default
 #   tar: false
-#   # type of backup. can be 'gcp-native', 'gcp-rclone' or 'r2-rclone'
+#   # type of backup. can be 'gcp-native', 'gcp-rclone', 'r2-rclone' or 's3-rclone'
 #   type: 'gcp-rclone'
 #   # name of the bucket
 #   bucket_name: "backup"

roles/node_backup/tasks/main.yml

@@ -23,7 +23,7 @@
     - path: "{{ _node_backup_log_path }}"
       user: root
     - path: "{{ _node_backup_venv_path }}"
-      user: root
+      user: "{{ node_backup_user }}"
   tags: [node-backup]
 
 - name: node-backup | requirements

roles/node_backup/tasks/requirements.yml

@@ -36,7 +36,7 @@
         owner: root
         group: root
 
-    - name: node-backup | requirements | copy R2 config
+    - name: node-backup | requirements | copy rclone config
       ansible.builtin.template:
         src: rclone/rclone.conf.j2
         dest: /root/.config/rclone/rclone.conf

roles/node_backup/tasks/tests.yml

@@ -5,6 +5,12 @@
   when: node_backup_targets | json_query('[].type') | intersect(_node_backup_r2_types) | length > 0 and ( node_backup_r2_access_key_id == '' or node_backup_r2_secret_access_key
     == '' or node_backup_r2_api_url == '' )
 
+- name: node-backup | test | check s3 configuration
+  ansible.builtin.fail:
+    msg: If the S3 backups are used, 'node_backup_s3_access_key_id', 'node_backup_s3_secret_access_key', 'node_backup_s3_endpoint' amd 'node_backup_s3_provider' variables have to be specified
+  when: node_backup_targets | json_query('[].type') | intersect(_node_backup_rclone_types) | length > 0 and ( node_backup_s3_access_key_id == '' or node_backup_s3_secret_access_key
+    == '' or node_backup_s3_endpoint == '' or node_backup_s3_provider == '' )
+
 - name: node-backup | test | check variables
   ansible.builtin.fail:
     msg: "'service_name', 'rpc_port', 'type' and 'bucket_name' fields have to be specified for each item in 'node_backup_targets'"
@@ -20,5 +26,5 @@
 - name: node-backup | test | check backup types
   ansible.builtin.fail:
     msg: "{{ item.type }} is not a valid backup type"
-  when: item.type not in (_node_backup_gcp_types + _node_backup_r2_types)
+  when: item.type not in (_node_backup_gcp_types + _node_backup_rclone_types)
   loop: "{{ node_backup_targets }}"

roles/node_backup/templates/rclone/rclone.conf.j2

@@ -1,14 +1,20 @@
-{% if node_backup_targets | json_query('[].type') | intersect(_node_backup_r2_types) | length > 0 %}
-[R2backups]
+{% if node_backup_targets | json_query('[].type') | intersect(_node_backup_rclone_types) | length > 0 %}
+[S3backups]
 type = s3
-provider = Cloudflare
-access_key_id = {{ node_backup_r2_access_key_id }}
-secret_access_key = {{ node_backup_r2_secret_access_key }}
-endpoint = {{ node_backup_r2_api_url }}
+provider = {{ node_backup_s3_provider }}
+access_key_id = {{ node_backup_s3_access_key_id }}
+secret_access_key = {{ node_backup_s3_secret_access_key }}
+endpoint = {{ node_backup_s3_endpoint }}
+{% if node_backup_s3_region != "" %}
+region = {{ node_backup_s3_region }}
+{% endif %}
 acl = private
 upload_cutoff = 1024M
 upload_concurrency = {{ node_backup_max_concurrent_requests }}
 chunk_size = 256M
+{% if node_backup_s3_provider == "Cloudflare" %}
+no_check_bucket = true
+{% endif %}
 {% endif %}
 
 {% if node_backup_targets | json_query('[].type') | intersect(_node_backup_gcp_types) | length > 0 %}

roles/node_backup/templates/single-backup.sh.j2

@@ -128,7 +128,9 @@ echo -e "\n---\n$(date +%Y-%m-%d\ %H:%M:%S) Start the '{{ item.id }}' backup\n--
 {% if item.type == 'gcp-rclone' %}
 remote="GCPbackups"
 {% elif item.type == 'r2-rclone' %}
-remote="R2backups"
+remote="S3backups"
+{% elif item.type == 's3-rclone' %}
+remote="S3backups"
 {% else %}
 {{ "backup type must be defined."/0 }}
 {% endif %}

roles/node_backup/vars/main.yml

@@ -10,8 +10,9 @@ _node_backup_rclone_deb: https://downloads.rclone.org/v1.63.1/rclone-v1.63.1-lin
 
 _node_backup_r2_types: [r2-rclone]
 _node_backup_gcp_types: [gcp-native, gcp-rclone]
-_node_backup_rclone_types: [gcp-rclone, r2-rclone]
+_node_backup_rclone_types: [gcp-rclone, r2-rclone, s3-rclone]
 _node_backup_storages:
+  s3-rclone: s3
   r2-rclone: r2
   gcp-rclone: gcp
   gcp-native: gcp