Fix: add exclusive lock retry to create_repo_if_not_exists and unify lock timeout

src/datastore_mad/remotes/restic/restic.rb

@@ -130,6 +130,9 @@ def initialize(action, opts = {})
         raise StandardError, 'Invalid value for RESTIC_PRUNE_MAX_UNUSED' \
             unless @max_unused.nil? || @max_unused.match(/^\d+[kKmMgGtT%]?$|^unlimited$/)
 
+        # Max retries when waiting for an exclusive Restic lock (5 s between each retry).
+        @lock_retries = Integer(safe_get("#{prefix}TEMPLATE/RESTIC_LOCK_RETRIES", 720))
+
         create_repo_if_not_exists if @options[:create_repo] && @repo_id
     rescue StandardError => e
         raise StandardError, "Wrong restic datastore configuration: #{e.message}"
@@ -176,12 +179,19 @@ def create_repo_if_not_exists
         script = <<~EOS
             set -e -o pipefail; shopt -qs failglob
             #{resticenv_sh(RESTIC_BIN_PATHS[:frontend])}
-            #{restic('stats')} || #{restic('init')}
+            #{restic('stats')} || { ec=$?; if [ "$ec" -eq 11 ]; then exit 11; fi; #{restic('init')}; }
         EOS
 
-        rc = LocalCommand.run '/bin/bash -s', nil, script
+        rc = nil
+        @lock_retries.times do
+            rc = LocalCommand.run '/bin/bash -s', nil, script
+            return if rc.code == 0
+            break unless rc.code == 11
 
-        raise StandardError, rc.stderr if rc.code != 0
+            sleep 5
+        end
+
+        raise StandardError, rc.stderr
     end
 
     # Gets (from Restic) full metadata of a specific snapshot.
@@ -268,7 +278,7 @@ def pull_snapshots(snaps, rhost = @sftp, wdir = nil)
     # @return [nil]
     def remove_snapshots(snaps, rhost = @sftp, opts = {})
         options = {
-            :retries => 60,
+            :retries => @lock_retries,
             :delay   => 5 # seconds
         }.merge!(opts)
 
@@ -339,7 +349,7 @@ def run_action(name, script, rhost = @sftp)
     # @return [Object] RC struct
     def run_with_lock_retry(name, script, rhost, opts = {})
         options = {
-            :retries => 60,
+            :retries => @lock_retries,
             :delay   => 5
         }.merge!(opts)
 

src/fireedge/src/modules/components/Forms/Datastore/CreateForm/Steps/ConfigurationAttributes/Fields/restic.js

@@ -192,6 +192,19 @@ const RESTIC_SPARSIFY = {
   grid: { xs: 12, md: 6 },
 }
 
+/** @type {Field} - Max retries when waiting for an exclusive Restic lock */
+const RESTIC_LOCK_RETRIES = {
+  name: 'RESTIC_LOCK_RETRIES',
+  label: T.ResticLockRetries,
+  tooltip: T.ResticLockRetriesConcept,
+  dependOf: '$general.STORAGE_BACKEND',
+  type: INPUT_TYPES.TEXT,
+  htmlType: (type) =>
+    typeIsOneOf(type, [isRestic]) ? 'number' : INPUT_TYPES.HIDDEN,
+  validation: number(),
+  grid: { xs: 12, md: 6 },
+}
+
 export const RESTIC_FIELDS = [
   RESTIC_PASSWORD,
   RESTIC_SFTP_SERVER,
@@ -205,5 +218,6 @@ export const RESTIC_FIELDS = [
   RESTIC_MAX_WIOPS,
   RESTIC_CPU_QUOTA,
   RESTIC_MAXPROC,
+  RESTIC_LOCK_RETRIES,
   RESTIC_SPARSIFY,
 ]

src/fireedge/src/modules/constants/translates.js

@@ -947,6 +947,9 @@ module.exports = {
   MaxNumberOSThreads: 'Max number of OS threads',
   MaxNumberOSThreadsConcept:
     'Sets GOMAXPROCS for restic to limit the OS threads that execute user-level Go code simultaneously.',
+  ResticLockRetries: 'Restic lock retries',
+  ResticLockRetriesConcept:
+    'Maximum number of retries (5 s apart) when waiting for an exclusive Restic repository lock to be released. Increase this for repositories where prune operations take a long time. Default: 720 (60 min).',
   Sparsify: 'Sparsify',
   SparsifyConcept:
     'Runs virt-sparsify on flatten backups to reduce backup size. It requires libguestfs package.',