| | | 1 | | // Copyright (c) Microsoft. All rights reserved. |
| | | 2 | | // Licensed under the MIT license. See LICENSE file in the project root for full license information. |
| | | 3 | | |
| | | 4 | | namespace Microsoft.Azure.ServiceBus |
| | | 5 | | { |
| | | 6 | | using System; |
| | | 7 | | using System.Collections.Generic; |
| | | 8 | | using System.Threading.Tasks; |
| | | 9 | | using System.Transactions; |
| | | 10 | | using Primitives; |
| | | 11 | | |
| | | 12 | | /// <summary> |
| | | 13 | | /// Represents an abstraction for retrying messaging operations. Users should not |
| | | 14 | | /// implement this class, and instead should use one of the provided implementations. |
| | | 15 | | /// <remarks>RetryPolicy will not be applied when an ambient transaction is found.</remarks> |
| | | 16 | | /// </summary> |
| | | 17 | | public abstract class RetryPolicy |
| | | 18 | | { |
| | 2 | 19 | | internal static readonly TimeSpan ServerBusyBaseSleepTime = TimeSpan.FromSeconds(10); |
| | | 20 | | |
| | | 21 | | const int DefaultRetryMaxCount = 5; |
| | 2 | 22 | | static readonly TimeSpan DefaultRetryMinBackoff = TimeSpan.Zero; |
| | 2 | 23 | | static readonly TimeSpan DefaultRetryMaxBackoff = TimeSpan.FromSeconds(30); |
| | | 24 | | |
| | 84 | 25 | | readonly object serverBusyLock = new object(); |
| | | 26 | | |
| | | 27 | | // This is a volatile copy of IsServerBusy. IsServerBusy is synchronized with a lock, whereas encounteredServerB |
| | | 28 | | volatile bool encounteredServerBusy; |
| | | 29 | | |
| | 84 | 30 | | protected RetryPolicy() |
| | | 31 | | { |
| | 84 | 32 | | } |
| | | 33 | | |
| | | 34 | | /// <summary> |
| | | 35 | | /// Returns the default retry policy, <see cref="RetryExponential"/>. |
| | | 36 | | /// </summary> |
| | 56 | 37 | | public static RetryPolicy Default => new RetryExponential(DefaultRetryMinBackoff, DefaultRetryMaxBackoff, Defaul |
| | | 38 | | |
| | | 39 | | /// <summary> |
| | | 40 | | /// Returns a <see cref="NoRetry"/> retry policy. |
| | | 41 | | /// </summary> |
| | 0 | 42 | | public static RetryPolicy NoRetry => new NoRetry(); |
| | | 43 | | |
| | | 44 | | /// <summary> |
| | | 45 | | /// Determines whether or not the server returned a busy error. |
| | | 46 | | /// </summary> |
| | 70 | 47 | | public bool IsServerBusy { get; protected set; } |
| | | 48 | | |
| | | 49 | | /// <summary> |
| | | 50 | | /// Gets the exception message when a server busy error is returned. |
| | | 51 | | /// </summary> |
| | 0 | 52 | | public string ServerBusyExceptionMessage { get; protected set; } |
| | | 53 | | |
| | | 54 | | /// <summary> |
| | | 55 | | /// Runs a <see cref="Func{T, TResult}"/>, using the current RetryPolicy. |
| | | 56 | | /// </summary> |
| | | 57 | | /// <param name="operation">A <see cref="Func{T, TResult}"/> to be executed.</param> |
| | | 58 | | /// <param name="operationTimeout">The timeout for the entire operation.</param> |
| | | 59 | | public async Task RunOperation(Func<Task> operation, TimeSpan operationTimeout) |
| | | 60 | | { |
| | 8 | 61 | | var currentRetryCount = 0; |
| | 8 | 62 | | List<Exception> exceptions = null; |
| | 8 | 63 | | var timeoutHelper = new TimeoutHelper(operationTimeout, true); |
| | | 64 | | |
| | 8 | 65 | | if (this.IsServerBusy && timeoutHelper.RemainingTime() < RetryPolicy.ServerBusyBaseSleepTime) |
| | | 66 | | { |
| | | 67 | | // We are in a server busy state before we start processing. |
| | | 68 | | // Since ServerBusyBaseSleepTime > remaining time for the operation, we don't wait for the entire Sleep |
| | 0 | 69 | | await Task.Delay(timeoutHelper.RemainingTime()).ConfigureAwait(false); |
| | 0 | 70 | | throw new ServerBusyException(this.ServerBusyExceptionMessage); |
| | | 71 | | } |
| | | 72 | | |
| | | 73 | | while (true) |
| | | 74 | | { |
| | 12 | 75 | | if (this.IsServerBusy) |
| | | 76 | | { |
| | 6 | 77 | | await Task.Delay(RetryPolicy.ServerBusyBaseSleepTime).ConfigureAwait(false); |
| | | 78 | | } |
| | | 79 | | |
| | | 80 | | try |
| | | 81 | | { |
| | 12 | 82 | | await operation().ConfigureAwait(false); |
| | | 83 | | |
| | | 84 | | // Its a successful operation. Preemptively reset ServerBusy status. |
| | 6 | 85 | | this.ResetServerBusy(); |
| | 6 | 86 | | break; |
| | | 87 | | } |
| | | 88 | | catch (Exception exception) |
| | | 89 | | { |
| | 6 | 90 | | currentRetryCount++; |
| | 6 | 91 | | if (exceptions == null) |
| | | 92 | | { |
| | 4 | 93 | | exceptions = new List<Exception>(); |
| | | 94 | | } |
| | 6 | 95 | | exceptions.Add(exception); |
| | | 96 | | |
| | 6 | 97 | | if (this.ShouldRetry( |
| | 6 | 98 | | timeoutHelper.RemainingTime(), currentRetryCount, exception, out var retryInterval) |
| | 6 | 99 | | && retryInterval < timeoutHelper.RemainingTime()) |
| | | 100 | | { |
| | | 101 | | // Log intermediate exceptions. |
| | 4 | 102 | | MessagingEventSource.Log.RunOperationExceptionEncountered(exception); |
| | 4 | 103 | | await Task.Delay(retryInterval).ConfigureAwait(false); |
| | 4 | 104 | | continue; |
| | | 105 | | } |
| | | 106 | | |
| | 2 | 107 | | throw; |
| | 0 | 108 | | } |
| | | 109 | | } |
| | 6 | 110 | | } |
| | | 111 | | |
| | | 112 | | /// <summary> |
| | | 113 | | /// Determines whether or not the exception can be retried. |
| | | 114 | | /// </summary> |
| | | 115 | | /// <returns>A bool indicating whether or not the operation can be retried.</returns> |
| | | 116 | | public virtual bool IsRetryableException(Exception exception) |
| | | 117 | | { |
| | 92 | 118 | | if (exception == null) |
| | | 119 | | { |
| | 0 | 120 | | throw Fx.Exception.ArgumentNull(nameof(exception)); |
| | | 121 | | } |
| | | 122 | | |
| | 92 | 123 | | var serviceBusException = exception as ServiceBusException; |
| | 92 | 124 | | return serviceBusException?.IsTransient == true; |
| | | 125 | | } |
| | | 126 | | |
| | | 127 | | internal bool ShouldRetry(TimeSpan remainingTime, int currentRetryCount, Exception lastException, out TimeSpan r |
| | | 128 | | { |
| | | 129 | | // There is no exception information or there's there's an ambient transaction - should not retry |
| | 94 | 130 | | if (lastException == null || Transaction.Current != null) |
| | | 131 | | { |
| | 2 | 132 | | retryInterval = TimeSpan.Zero; |
| | 2 | 133 | | return false; |
| | | 134 | | } |
| | | 135 | | |
| | 92 | 136 | | if (lastException is ServerBusyException) |
| | | 137 | | { |
| | 16 | 138 | | this.SetServerBusy(lastException.Message); |
| | | 139 | | } |
| | | 140 | | |
| | 92 | 141 | | if (this.IsRetryableException(lastException)) |
| | | 142 | | { |
| | 72 | 143 | | return this.OnShouldRetry(remainingTime, currentRetryCount, out retryInterval); |
| | | 144 | | } |
| | | 145 | | |
| | 20 | 146 | | retryInterval = TimeSpan.Zero; |
| | 20 | 147 | | return false; |
| | | 148 | | } |
| | | 149 | | |
| | | 150 | | internal void SetServerBusy(string exceptionMessage) |
| | | 151 | | { |
| | | 152 | | // multiple call to this method will not prolong the timer. |
| | 18 | 153 | | if (this.encounteredServerBusy) |
| | | 154 | | { |
| | 2 | 155 | | return; |
| | | 156 | | } |
| | | 157 | | |
| | 16 | 158 | | lock (this.serverBusyLock) |
| | | 159 | | { |
| | 16 | 160 | | if (!this.encounteredServerBusy) |
| | | 161 | | { |
| | 16 | 162 | | this.encounteredServerBusy = true; |
| | 16 | 163 | | this.ServerBusyExceptionMessage = string.IsNullOrWhiteSpace(exceptionMessage) ? |
| | 16 | 164 | | Resources.DefaultServerBusyException : exceptionMessage; |
| | 16 | 165 | | this.IsServerBusy = true; |
| | 16 | 166 | | TaskExtensionHelper.Schedule(ScheduleResetServerBusy); |
| | | 167 | | } |
| | 16 | 168 | | } |
| | 16 | 169 | | } |
| | | 170 | | |
| | | 171 | | internal void ResetServerBusy() |
| | | 172 | | { |
| | 22 | 173 | | if (!this.encounteredServerBusy) |
| | | 174 | | { |
| | 6 | 175 | | return; |
| | | 176 | | } |
| | | 177 | | |
| | 16 | 178 | | lock (this.serverBusyLock) |
| | | 179 | | { |
| | 16 | 180 | | if (this.encounteredServerBusy) |
| | | 181 | | { |
| | 16 | 182 | | this.encounteredServerBusy = false; |
| | 16 | 183 | | this.ServerBusyExceptionMessage = Resources.DefaultServerBusyException; |
| | 16 | 184 | | this.IsServerBusy = false; |
| | | 185 | | } |
| | 16 | 186 | | } |
| | 16 | 187 | | } |
| | | 188 | | |
| | | 189 | | protected abstract bool OnShouldRetry(TimeSpan remainingTime, int currentRetryCount, out TimeSpan retryInterval) |
| | | 190 | | |
| | | 191 | | private async Task ScheduleResetServerBusy() |
| | | 192 | | { |
| | 16 | 193 | | await Task.Delay(RetryPolicy.ServerBusyBaseSleepTime).ConfigureAwait(false); |
| | 16 | 194 | | ResetServerBusy(); |
| | 16 | 195 | | } |
| | | 196 | | } |
| | | 197 | | } |