| | 1 | | // Copyright (c) Microsoft Corporation. All rights reserved. |
| | 2 | | // Licensed under the MIT License. |
| | 3 | |
|
| | 4 | | using System; |
| | 5 | | using System.Collections.Generic; |
| | 6 | | using System.Diagnostics; |
| | 7 | | using System.Runtime.ExceptionServices; |
| | 8 | | using System.Threading; |
| | 9 | | using System.Threading.Tasks; |
| | 10 | | using Azure.Core.Diagnostics; |
| | 11 | |
|
| | 12 | | namespace Azure.Core.Pipeline |
| | 13 | | { |
| | 14 | | internal class RetryPolicy : HttpPipelinePolicy |
| | 15 | | { |
| | 16 | | private readonly RetryMode _mode; |
| | 17 | | private readonly TimeSpan _delay; |
| | 18 | | private readonly TimeSpan _maxDelay; |
| | 19 | | private readonly int _maxRetries; |
| | 20 | |
|
| 210 | 21 | | private readonly Random _random = new ThreadSafeRandom(); |
| | 22 | |
|
| 210 | 23 | | public RetryPolicy(RetryMode mode, TimeSpan delay, TimeSpan maxDelay, int maxRetries) |
| | 24 | | { |
| 210 | 25 | | _mode = mode; |
| 210 | 26 | | _delay = delay; |
| 210 | 27 | | _maxDelay = maxDelay; |
| 210 | 28 | | _maxRetries = maxRetries; |
| 210 | 29 | | } |
| | 30 | |
|
| | 31 | | private const string RetryAfterHeaderName = "Retry-After"; |
| | 32 | | private const string RetryAfterMsHeaderName = "retry-after-ms"; |
| | 33 | | private const string XRetryAfterMsHeaderName = "x-ms-retry-after-ms"; |
| | 34 | |
|
| | 35 | | public override void Process(HttpMessage message, ReadOnlyMemory<HttpPipelinePolicy> pipeline) |
| | 36 | | { |
| 888 | 37 | | ProcessAsync(message, pipeline, false).EnsureCompleted(); |
| 874 | 38 | | } |
| | 39 | |
|
| | 40 | | public override ValueTask ProcessAsync(HttpMessage message, ReadOnlyMemory<HttpPipelinePolicy> pipeline) |
| | 41 | | { |
| 902 | 42 | | return ProcessAsync(message, pipeline, true); |
| | 43 | | } |
| | 44 | |
|
| | 45 | | private async ValueTask ProcessAsync(HttpMessage message, ReadOnlyMemory<HttpPipelinePolicy> pipeline, bool asyn |
| | 46 | | { |
| 1790 | 47 | | int attempt = 0; |
| 1790 | 48 | | List<Exception>? exceptions = null; |
| 640 | 49 | | while (true) |
| | 50 | | { |
| 2430 | 51 | | Exception? lastException = null; |
| 2430 | 52 | | var before = Stopwatch.GetTimestamp(); |
| | 53 | | try |
| | 54 | | { |
| 2430 | 55 | | if (async) |
| | 56 | | { |
| 1226 | 57 | | await ProcessNextAsync(message, pipeline).ConfigureAwait(false); |
| | 58 | | } |
| | 59 | | else |
| | 60 | | { |
| 1204 | 61 | | ProcessNext(message, pipeline); |
| | 62 | | } |
| 2350 | 63 | | } |
| 80 | 64 | | catch (Exception ex) |
| | 65 | | { |
| 80 | 66 | | if (exceptions == null) |
| | 67 | | { |
| 48 | 68 | | exceptions = new List<Exception>(); |
| | 69 | | } |
| | 70 | |
|
| 80 | 71 | | exceptions.Add(ex); |
| | 72 | |
|
| 80 | 73 | | lastException = ex; |
| 80 | 74 | | } |
| | 75 | |
|
| 2430 | 76 | | var after = Stopwatch.GetTimestamp(); |
| 2430 | 77 | | double elapsed = (after - before) / (double)Stopwatch.Frequency; |
| | 78 | |
|
| | 79 | | TimeSpan delay; |
| | 80 | |
|
| 2430 | 81 | | attempt++; |
| | 82 | |
|
| 2430 | 83 | | var shouldRetry = attempt <= _maxRetries; |
| | 84 | |
|
| 2430 | 85 | | if (lastException != null) |
| | 86 | | { |
| 80 | 87 | | if (shouldRetry && message.ResponseClassifier.IsRetriable(message, lastException)) |
| | 88 | | { |
| 52 | 89 | | GetDelay(attempt, out delay); |
| | 90 | | } |
| | 91 | | else |
| | 92 | | { |
| | 93 | | // Rethrow a singular exception |
| 28 | 94 | | if (exceptions?.Count == 1) |
| | 95 | | { |
| 12 | 96 | | ExceptionDispatchInfo.Capture(lastException).Throw(); |
| | 97 | | } |
| | 98 | |
|
| 16 | 99 | | throw new AggregateException($"Retry failed after {attempt} tries.", exceptions); |
| | 100 | | } |
| | 101 | | } |
| 2350 | 102 | | else if (message.ResponseClassifier.IsErrorResponse(message)) |
| | 103 | | { |
| 700 | 104 | | if (shouldRetry && message.ResponseClassifier.IsRetriableResponse(message)) |
| | 105 | | { |
| 588 | 106 | | GetDelay(message, attempt, out delay); |
| | 107 | | } |
| | 108 | | else |
| | 109 | | { |
| | 110 | | return; |
| | 111 | | } |
| | 112 | | } |
| | 113 | | else |
| | 114 | | { |
| 1762 | 115 | | return; |
| | 116 | | } |
| | 117 | |
|
| 640 | 118 | | if (delay > TimeSpan.Zero) |
| | 119 | | { |
| 632 | 120 | | if (async) |
| | 121 | | { |
| 316 | 122 | | await WaitAsync(delay, message.CancellationToken).ConfigureAwait(false); |
| | 123 | | } |
| | 124 | | else |
| | 125 | | { |
| 316 | 126 | | Wait(delay, message.CancellationToken); |
| | 127 | | } |
| | 128 | | } |
| | 129 | |
|
| 640 | 130 | | if (message.HasResponse) |
| | 131 | | { |
| | 132 | | // Dispose the content stream to free up a connection if the request has any |
| 592 | 133 | | message.Response.ContentStream?.Dispose(); |
| | 134 | | } |
| | 135 | |
|
| 640 | 136 | | AzureCoreEventSource.Singleton.RequestRetrying(message.Request.ClientRequestId, attempt, elapsed); |
| 640 | 137 | | } |
| 1762 | 138 | | } |
| | 139 | |
|
| | 140 | | internal virtual async Task WaitAsync(TimeSpan time, CancellationToken cancellationToken) |
| | 141 | | { |
| 204 | 142 | | await Task.Delay(time, cancellationToken).ConfigureAwait(false); |
| 204 | 143 | | } |
| | 144 | |
|
| | 145 | | internal virtual void Wait(TimeSpan time, CancellationToken cancellationToken) |
| | 146 | | { |
| 204 | 147 | | cancellationToken.WaitHandle.WaitOne(time); |
| 204 | 148 | | } |
| | 149 | |
|
| | 150 | | protected virtual TimeSpan GetServerDelay(HttpMessage message) |
| | 151 | | { |
| 588 | 152 | | if (message.Response == null) |
| | 153 | | { |
| 0 | 154 | | return TimeSpan.Zero; |
| | 155 | | } |
| | 156 | |
|
| 588 | 157 | | if (message.Response.TryGetHeader(RetryAfterMsHeaderName, out var retryAfterValue) || |
| 588 | 158 | | message.Response.TryGetHeader(XRetryAfterMsHeaderName, out retryAfterValue)) |
| | 159 | | { |
| 32 | 160 | | if (int.TryParse(retryAfterValue, out var delaySeconds)) |
| | 161 | | { |
| 32 | 162 | | return TimeSpan.FromMilliseconds(delaySeconds); |
| | 163 | | } |
| | 164 | | } |
| | 165 | |
|
| 556 | 166 | | if (message.Response.TryGetHeader(RetryAfterHeaderName, out retryAfterValue)) |
| | 167 | | { |
| 44 | 168 | | if (int.TryParse(retryAfterValue, out var delaySeconds)) |
| | 169 | | { |
| 28 | 170 | | return TimeSpan.FromSeconds(delaySeconds); |
| | 171 | | } |
| 16 | 172 | | if (DateTimeOffset.TryParse(retryAfterValue, out DateTimeOffset delayTime)) |
| | 173 | | { |
| 8 | 174 | | return delayTime - DateTimeOffset.Now; |
| | 175 | | } |
| | 176 | | } |
| | 177 | |
|
| 520 | 178 | | return TimeSpan.Zero; |
| | 179 | | } |
| | 180 | |
|
| | 181 | | private void GetDelay(HttpMessage message, int attempted, out TimeSpan delay) |
| | 182 | | { |
| 588 | 183 | | delay = TimeSpan.Zero; |
| | 184 | |
|
| 588 | 185 | | switch (_mode) |
| | 186 | | { |
| | 187 | | case RetryMode.Fixed: |
| 76 | 188 | | delay = _delay; |
| 76 | 189 | | break; |
| | 190 | | case RetryMode.Exponential: |
| 512 | 191 | | delay = CalculateExponentialDelay(attempted); |
| | 192 | | break; |
| | 193 | | } |
| | 194 | |
|
| 588 | 195 | | TimeSpan serverDelay = GetServerDelay(message); |
| 588 | 196 | | if (serverDelay > delay) |
| | 197 | | { |
| 60 | 198 | | delay = serverDelay; |
| | 199 | | } |
| 588 | 200 | | } |
| | 201 | |
|
| | 202 | | private void GetDelay(int attempted, out TimeSpan delay) |
| | 203 | | { |
| 52 | 204 | | delay = CalculateExponentialDelay(attempted); |
| 52 | 205 | | } |
| | 206 | |
|
| | 207 | | private TimeSpan CalculateExponentialDelay(int attempted) |
| | 208 | | { |
| 564 | 209 | | return TimeSpan.FromMilliseconds( |
| 564 | 210 | | Math.Min( |
| 564 | 211 | | (1 << (attempted - 1)) * _random.Next((int)(_delay.TotalMilliseconds * 0.8), (int)(_delay.TotalMilli |
| 564 | 212 | | _maxDelay.TotalMilliseconds)); |
| | 213 | | } |
| | 214 | | } |
| | 215 | | } |