diff --git a/src/datastore_mad/remotes/restic/restic.rb b/src/datastore_mad/remotes/restic/restic.rb index b515b4fa0df..4212defd6b0 100644 --- a/src/datastore_mad/remotes/restic/restic.rb +++ b/src/datastore_mad/remotes/restic/restic.rb @@ -130,6 +130,9 @@ def initialize(action, opts = {}) raise StandardError, 'Invalid value for RESTIC_PRUNE_MAX_UNUSED' \ unless @max_unused.nil? || @max_unused.match(/^\d+[kKmMgGtT%]?$|^unlimited$/) + # Max retries when waiting for an exclusive Restic lock (5 s between each retry). + @lock_retries = Integer(safe_get("#{prefix}TEMPLATE/RESTIC_LOCK_RETRIES", 720)) + create_repo_if_not_exists if @options[:create_repo] && @repo_id rescue StandardError => e raise StandardError, "Wrong restic datastore configuration: #{e.message}" @@ -176,12 +179,19 @@ def create_repo_if_not_exists script = <<~EOS set -e -o pipefail; shopt -qs failglob #{resticenv_sh(RESTIC_BIN_PATHS[:frontend])} - #{restic('stats')} || #{restic('init')} + #{restic('stats')} || { ec=$?; if [ "$ec" -eq 11 ]; then exit 11; fi; #{restic('init')}; } EOS - rc = LocalCommand.run '/bin/bash -s', nil, script + rc = nil + @lock_retries.times do + rc = LocalCommand.run '/bin/bash -s', nil, script + return if rc.code == 0 + break unless rc.code == 11 - raise StandardError, rc.stderr if rc.code != 0 + sleep 5 + end + + raise StandardError, rc.stderr end # Gets (from Restic) full metadata of a specific snapshot. @@ -268,7 +278,7 @@ def pull_snapshots(snaps, rhost = @sftp, wdir = nil) # @return [nil] def remove_snapshots(snaps, rhost = @sftp, opts = {}) options = { - :retries => 60, + :retries => @lock_retries, :delay => 5 # seconds }.merge!(opts) @@ -339,7 +349,7 @@ def run_action(name, script, rhost = @sftp) # @return [Object] RC struct def run_with_lock_retry(name, script, rhost, opts = {}) options = { - :retries => 60, + :retries => @lock_retries, :delay => 5 }.merge!(opts) diff --git a/src/fireedge/src/modules/components/Forms/Datastore/CreateForm/Steps/ConfigurationAttributes/Fields/restic.js b/src/fireedge/src/modules/components/Forms/Datastore/CreateForm/Steps/ConfigurationAttributes/Fields/restic.js index e15d2da821f..b1325498123 100644 --- a/src/fireedge/src/modules/components/Forms/Datastore/CreateForm/Steps/ConfigurationAttributes/Fields/restic.js +++ b/src/fireedge/src/modules/components/Forms/Datastore/CreateForm/Steps/ConfigurationAttributes/Fields/restic.js @@ -192,6 +192,19 @@ const RESTIC_SPARSIFY = { grid: { xs: 12, md: 6 }, } +/** @type {Field} - Max retries when waiting for an exclusive Restic lock */ +const RESTIC_LOCK_RETRIES = { + name: 'RESTIC_LOCK_RETRIES', + label: T.ResticLockRetries, + tooltip: T.ResticLockRetriesConcept, + dependOf: '$general.STORAGE_BACKEND', + type: INPUT_TYPES.TEXT, + htmlType: (type) => + typeIsOneOf(type, [isRestic]) ? 'number' : INPUT_TYPES.HIDDEN, + validation: number(), + grid: { xs: 12, md: 6 }, +} + export const RESTIC_FIELDS = [ RESTIC_PASSWORD, RESTIC_SFTP_SERVER, @@ -205,5 +218,6 @@ export const RESTIC_FIELDS = [ RESTIC_MAX_WIOPS, RESTIC_CPU_QUOTA, RESTIC_MAXPROC, + RESTIC_LOCK_RETRIES, RESTIC_SPARSIFY, ] diff --git a/src/fireedge/src/modules/constants/translates.js b/src/fireedge/src/modules/constants/translates.js index 3d97eef7195..08bf480fdc0 100644 --- a/src/fireedge/src/modules/constants/translates.js +++ b/src/fireedge/src/modules/constants/translates.js @@ -947,6 +947,9 @@ module.exports = { MaxNumberOSThreads: 'Max number of OS threads', MaxNumberOSThreadsConcept: 'Sets GOMAXPROCS for restic to limit the OS threads that execute user-level Go code simultaneously.', + ResticLockRetries: 'Restic lock retries', + ResticLockRetriesConcept: + 'Maximum number of retries (5 s apart) when waiting for an exclusive Restic repository lock to be released. Increase this for repositories where prune operations take a long time. Default: 720 (60 min).', Sparsify: 'Sparsify', SparsifyConcept: 'Runs virt-sparsify on flatten backups to reduce backup size. It requires libguestfs package.',