| | 1 | | // Copyright (c) Microsoft Corporation. All rights reserved. |
| | 2 | | // Licensed under the MIT License. |
| | 3 | |
|
| | 4 | | using System; |
| | 5 | | using System.Collections; |
| | 6 | | using System.Collections.Generic; |
| | 7 | | using System.Threading; |
| | 8 | | using System.Threading.Tasks; |
| | 9 | | using Azure.Core.Pipeline; |
| | 10 | | using Azure.Messaging.ServiceBus.Diagnostics; |
| | 11 | | using Azure.Messaging.ServiceBus.Plugins; |
| | 12 | |
|
| | 13 | | namespace Azure.Messaging.ServiceBus |
| | 14 | | { |
| | 15 | | /// <summary> |
| | 16 | | /// Represents a single receiver instance that multiple threads spawned by the ServiceBusProcessor |
| | 17 | | /// may be using to receive and process messages. The manager will delegate to the user provided |
| | 18 | | /// callbacks and handle automatic locking of messages. |
| | 19 | | /// </summary> |
| | 20 | | internal class ReceiverManager |
| | 21 | | { |
| 32 | 22 | | protected virtual ServiceBusReceiver Receiver { get; set; } |
| | 23 | | protected readonly ServiceBusConnection _connection; |
| | 24 | | protected readonly string _fullyQualifiedNamespace; |
| | 25 | | protected readonly string _entityPath; |
| | 26 | | protected readonly string _identifier; |
| | 27 | | protected readonly TimeSpan? _maxReceiveWaitTime; |
| | 28 | | private readonly ServiceBusReceiverOptions _receiverOptions; |
| | 29 | | protected readonly ServiceBusProcessorOptions _processorOptions; |
| | 30 | | private readonly Func<ProcessErrorEventArgs, Task> _errorHandler; |
| | 31 | | private readonly Func<ProcessMessageEventArgs, Task> _messageHandler; |
| | 32 | | protected readonly EntityScopeFactory _scopeFactory; |
| | 33 | | protected readonly IList<ServiceBusPlugin> _plugins; |
| | 34 | |
|
| 4 | 35 | | protected bool AutoRenewLock => _processorOptions.MaxAutoLockRenewalDuration > TimeSpan.Zero; |
| | 36 | |
|
| 4 | 37 | | public ReceiverManager( |
| 4 | 38 | | ServiceBusConnection connection, |
| 4 | 39 | | string fullyQualifiedNamespace, |
| 4 | 40 | | string entityPath, |
| 4 | 41 | | string identifier, |
| 4 | 42 | | ServiceBusProcessorOptions processorOptions, |
| 4 | 43 | | Func<ProcessMessageEventArgs, Task> messageHandler, |
| 4 | 44 | | Func<ProcessErrorEventArgs, Task> errorHandler, |
| 4 | 45 | | EntityScopeFactory scopeFactory, |
| 4 | 46 | | IList<ServiceBusPlugin> plugins) |
| | 47 | | { |
| 4 | 48 | | _connection = connection; |
| 4 | 49 | | _fullyQualifiedNamespace = fullyQualifiedNamespace; |
| 4 | 50 | | _entityPath = entityPath; |
| 4 | 51 | | _processorOptions = processorOptions; |
| 4 | 52 | | _receiverOptions = new ServiceBusReceiverOptions |
| 4 | 53 | | { |
| 4 | 54 | | ReceiveMode = _processorOptions.ReceiveMode, |
| 4 | 55 | | PrefetchCount = _processorOptions.PrefetchCount |
| 4 | 56 | | }; |
| 4 | 57 | | _maxReceiveWaitTime = _processorOptions.MaxReceiveWaitTime; |
| 4 | 58 | | _identifier = identifier; |
| 4 | 59 | | _plugins = plugins; |
| 4 | 60 | | Receiver = new ServiceBusReceiver( |
| 4 | 61 | | connection: _connection, |
| 4 | 62 | | entityPath: _entityPath, |
| 4 | 63 | | isSessionEntity: false, |
| 4 | 64 | | plugins: _plugins, |
| 4 | 65 | | options: _receiverOptions); |
| 4 | 66 | | _errorHandler = errorHandler; |
| 4 | 67 | | _messageHandler = messageHandler; |
| 4 | 68 | | _scopeFactory = scopeFactory; |
| 4 | 69 | | } |
| | 70 | |
|
| | 71 | | public virtual async Task CloseReceiverIfNeeded( |
| | 72 | | CancellationToken cancellationToken) |
| | 73 | | { |
| | 74 | | try |
| | 75 | | { |
| 4 | 76 | | await Receiver.DisposeAsync().ConfigureAwait(false); |
| 4 | 77 | | } |
| | 78 | | finally |
| | 79 | | { |
| 4 | 80 | | Receiver = null; |
| | 81 | | } |
| 4 | 82 | | } |
| | 83 | |
|
| | 84 | | public virtual async Task ReceiveAndProcessMessagesAsync(CancellationToken cancellationToken) |
| | 85 | | { |
| 4 | 86 | | ServiceBusErrorSource errorSource = ServiceBusErrorSource.Receive; |
| | 87 | | try |
| | 88 | | { |
| | 89 | | // loop within the context of this thread |
| 8 | 90 | | while (!cancellationToken.IsCancellationRequested) |
| | 91 | | { |
| 4 | 92 | | errorSource = ServiceBusErrorSource.Receive; |
| 4 | 93 | | ServiceBusReceivedMessage message = await Receiver.ReceiveMessageAsync( |
| 4 | 94 | | _maxReceiveWaitTime, |
| 4 | 95 | | cancellationToken).ConfigureAwait(false); |
| 4 | 96 | | if (message == null) |
| | 97 | | { |
| | 98 | | continue; |
| | 99 | | } |
| 4 | 100 | | await ProcessOneMessageWithinScopeAsync( |
| 4 | 101 | | message, |
| 4 | 102 | | DiagnosticProperty.ProcessMessageActivityName, |
| 4 | 103 | | cancellationToken).ConfigureAwait(false); |
| | 104 | | } |
| 4 | 105 | | } |
| | 106 | | catch (Exception ex) |
| | 107 | | // If the user manually throws a TCE, then we should log it. |
| 0 | 108 | | when (!(ex is TaskCanceledException) || |
| 0 | 109 | | !cancellationToken.IsCancellationRequested) |
| | 110 | | { |
| 0 | 111 | | if (ex is ServiceBusException sbException && sbException.ProcessorErrorSource.HasValue) |
| | 112 | | { |
| 0 | 113 | | errorSource = sbException.ProcessorErrorSource.Value; |
| | 114 | | } |
| 0 | 115 | | await RaiseExceptionReceived( |
| 0 | 116 | | new ProcessErrorEventArgs( |
| 0 | 117 | | ex, |
| 0 | 118 | | errorSource, |
| 0 | 119 | | _fullyQualifiedNamespace, |
| 0 | 120 | | _entityPath)) |
| 0 | 121 | | .ConfigureAwait(false); |
| | 122 | | } |
| 4 | 123 | | } |
| | 124 | |
|
| | 125 | | protected async Task ProcessOneMessageWithinScopeAsync(ServiceBusReceivedMessage message, string activityName, C |
| | 126 | | { |
| 4 | 127 | | using DiagnosticScope scope = _scopeFactory.CreateScope(activityName); |
| 4 | 128 | | scope.Start(); |
| 4 | 129 | | scope.SetMessageData(new ServiceBusReceivedMessage[] { message }); |
| | 130 | | try |
| | 131 | | { |
| 4 | 132 | | await ProcessOneMessage( |
| 4 | 133 | | message, |
| 4 | 134 | | cancellationToken) |
| 4 | 135 | | .ConfigureAwait(false); |
| 4 | 136 | | } |
| 0 | 137 | | catch (Exception ex) |
| | 138 | | { |
| 0 | 139 | | scope.Failed(ex); |
| 0 | 140 | | throw; |
| | 141 | | } |
| 4 | 142 | | } |
| | 143 | |
|
| | 144 | | /// <summary> |
| | 145 | | /// |
| | 146 | | /// </summary> |
| | 147 | | /// <param name="message"></param> |
| | 148 | | /// <param name="cancellationToken"></param> |
| | 149 | | /// <returns></returns> |
| | 150 | | private async Task ProcessOneMessage( |
| | 151 | | ServiceBusReceivedMessage message, |
| | 152 | | CancellationToken cancellationToken) |
| | 153 | | { |
| 4 | 154 | | ServiceBusErrorSource errorSource = ServiceBusErrorSource.Receive; |
| 4 | 155 | | CancellationTokenSource renewLockCancellationTokenSource = null; |
| 4 | 156 | | Task renewLock = null; |
| | 157 | |
|
| | 158 | | try |
| | 159 | | { |
| 4 | 160 | | if (!Receiver.IsSessionReceiver && |
| 4 | 161 | | Receiver.ReceiveMode == ReceiveMode.PeekLock && |
| 4 | 162 | | AutoRenewLock) |
| | 163 | | { |
| 0 | 164 | | renewLockCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken |
| 0 | 165 | | renewLock = RenewMessageLock( |
| 0 | 166 | | message, |
| 0 | 167 | | renewLockCancellationTokenSource); |
| | 168 | | } |
| | 169 | |
|
| 4 | 170 | | errorSource = ServiceBusErrorSource.UserCallback; |
| | 171 | |
|
| 4 | 172 | | await OnMessageHandler(message, cancellationToken).ConfigureAwait(false); |
| | 173 | |
|
| 4 | 174 | | if (Receiver.ReceiveMode == ReceiveMode.PeekLock && |
| 4 | 175 | | _processorOptions.AutoComplete && |
| 4 | 176 | | !message.IsSettled) |
| | 177 | | { |
| 0 | 178 | | errorSource = ServiceBusErrorSource.Complete; |
| | 179 | | // don't pass the processor cancellation token |
| | 180 | | // as we want in flight autocompletion to be able |
| | 181 | | // to finish |
| 0 | 182 | | await Receiver.CompleteMessageAsync( |
| 0 | 183 | | message.LockToken, |
| 0 | 184 | | CancellationToken.None) |
| 0 | 185 | | .ConfigureAwait(false); |
| | 186 | | } |
| | 187 | |
|
| 4 | 188 | | await CancelTask(renewLockCancellationTokenSource, renewLock).ConfigureAwait(false); |
| 4 | 189 | | } |
| | 190 | | catch (Exception ex) |
| | 191 | | // This prevents exceptions relating to processing a message from bubbling up all |
| | 192 | | // the way to the main thread when calling StopProcessingAsync, which we don't want |
| | 193 | | // as it isn't actionable. |
| 0 | 194 | | when (!(ex is TaskCanceledException) || !cancellationToken.IsCancellationRequested) |
| | 195 | | { |
| 0 | 196 | | ThrowIfSessionLockLost(ex, errorSource); |
| 0 | 197 | | await RaiseExceptionReceived( |
| 0 | 198 | | new ProcessErrorEventArgs( |
| 0 | 199 | | ex, |
| 0 | 200 | | errorSource, |
| 0 | 201 | | _fullyQualifiedNamespace, |
| 0 | 202 | | _entityPath)) |
| 0 | 203 | | .ConfigureAwait(false); |
| | 204 | |
|
| | 205 | | // if the user settled the message, or if the message or session lock was lost, |
| | 206 | | // do not attempt to abandon the message |
| 0 | 207 | | ServiceBusFailureReason? failureReason = (ex as ServiceBusException)?.Reason; |
| 0 | 208 | | if (!message.IsSettled && |
| 0 | 209 | | _receiverOptions.ReceiveMode == ReceiveMode.PeekLock && |
| 0 | 210 | | failureReason != ServiceBusFailureReason.SessionLockLost && |
| 0 | 211 | | failureReason != ServiceBusFailureReason.MessageLockLost) |
| | 212 | | { |
| | 213 | | try |
| | 214 | | { |
| | 215 | | // don't pass the processor cancellation token |
| | 216 | | // as we want in flight abandon to be able |
| | 217 | | // to finish even if user stopped processing |
| 0 | 218 | | await Receiver.AbandonMessageAsync( |
| 0 | 219 | | message.LockToken, |
| 0 | 220 | | cancellationToken: CancellationToken.None) |
| 0 | 221 | | .ConfigureAwait(false); |
| 0 | 222 | | } |
| 0 | 223 | | catch (Exception exception) |
| | 224 | | { |
| 0 | 225 | | ThrowIfSessionLockLost(exception, ServiceBusErrorSource.Abandon); |
| 0 | 226 | | await RaiseExceptionReceived( |
| 0 | 227 | | new ProcessErrorEventArgs( |
| 0 | 228 | | exception, |
| 0 | 229 | | ServiceBusErrorSource.Abandon, |
| 0 | 230 | | _fullyQualifiedNamespace, |
| 0 | 231 | | _entityPath)) |
| 0 | 232 | | .ConfigureAwait(false); |
| | 233 | | } |
| | 234 | | } |
| | 235 | | } |
| | 236 | | finally |
| | 237 | | { |
| 4 | 238 | | renewLockCancellationTokenSource?.Cancel(); |
| 4 | 239 | | renewLockCancellationTokenSource?.Dispose(); |
| | 240 | | } |
| 4 | 241 | | } |
| | 242 | |
|
| | 243 | | protected virtual async Task OnMessageHandler(ServiceBusReceivedMessage message, CancellationToken processorCanc |
| | 244 | | { |
| 4 | 245 | | var args = new ProcessMessageEventArgs( |
| 4 | 246 | | message, |
| 4 | 247 | | Receiver, |
| 4 | 248 | | processorCancellationToken); |
| 4 | 249 | | await _messageHandler(args).ConfigureAwait(false); |
| 4 | 250 | | } |
| | 251 | |
|
| | 252 | | /// <summary> |
| | 253 | | /// |
| | 254 | | /// </summary> |
| | 255 | | /// <param name="message"></param> |
| | 256 | | /// <param name="cancellationTokenSource"></param> |
| | 257 | | /// <returns></returns> |
| | 258 | | private async Task RenewMessageLock( |
| | 259 | | ServiceBusReceivedMessage message, |
| | 260 | | CancellationTokenSource cancellationTokenSource) |
| | 261 | | { |
| 0 | 262 | | cancellationTokenSource.CancelAfter(_processorOptions.MaxAutoLockRenewalDuration); |
| 0 | 263 | | CancellationToken cancellationToken = cancellationTokenSource.Token; |
| 0 | 264 | | while (!cancellationToken.IsCancellationRequested) |
| | 265 | | { |
| | 266 | | try |
| | 267 | | { |
| 0 | 268 | | ServiceBusEventSource.Log.ProcessorRenewMessageLockStart(_identifier, 1, message.LockToken); |
| 0 | 269 | | TimeSpan delay = CalculateRenewDelay(message.LockedUntil); |
| | 270 | |
|
| 0 | 271 | | await Task.Delay(delay, cancellationToken).ConfigureAwait(false); |
| 0 | 272 | | if (Receiver.IsDisposed) |
| | 273 | | { |
| 0 | 274 | | break; |
| | 275 | | } |
| | 276 | |
|
| 0 | 277 | | await Receiver.RenewMessageLockAsync(message, cancellationToken).ConfigureAwait(false); |
| 0 | 278 | | ServiceBusEventSource.Log.ProcessorRenewMessageLockComplete(_identifier); |
| 0 | 279 | | } |
| 0 | 280 | | catch (Exception ex) when (!(ex is TaskCanceledException)) |
| | 281 | | { |
| 0 | 282 | | ServiceBusEventSource.Log.ProcessorRenewMessageLockException(_identifier, ex.ToString()); |
| 0 | 283 | | await HandleRenewLockException(ex, cancellationToken).ConfigureAwait(false); |
| | 284 | |
|
| | 285 | | // if the error was not transient, break out of the loop |
| 0 | 286 | | if (!(ex as ServiceBusException)?.IsTransient == true) |
| | 287 | | { |
| | 288 | | break; |
| | 289 | | } |
| | 290 | | } |
| | 291 | | } |
| 0 | 292 | | } |
| | 293 | |
|
| | 294 | | /// <summary> |
| | 295 | | /// Cancels the specified cancellation source and awaits the specified task. |
| | 296 | | /// </summary> |
| | 297 | | /// <param name="cancellationSource">CancellationTokenSource to cancel</param> |
| | 298 | | /// <param name="task">Associated task to await</param> |
| | 299 | | protected static async Task CancelTask( |
| | 300 | | CancellationTokenSource cancellationSource, |
| | 301 | | Task task) |
| | 302 | | { |
| | 303 | | try |
| | 304 | | { |
| 4 | 305 | | if (cancellationSource != null) |
| | 306 | | { |
| 0 | 307 | | cancellationSource.Cancel(); |
| 0 | 308 | | await task.ConfigureAwait(false); |
| | 309 | | } |
| 4 | 310 | | } |
| 0 | 311 | | catch (Exception ex) when (ex is TaskCanceledException) |
| | 312 | | { |
| | 313 | | // Nothing to do here. These exceptions are expected. |
| 0 | 314 | | } |
| 4 | 315 | | } |
| | 316 | |
|
| | 317 | | private static void ThrowIfSessionLockLost( |
| | 318 | | Exception exception, |
| | 319 | | ServiceBusErrorSource errorSource) |
| | 320 | | { |
| | 321 | | // we need to propagate this in order to dispose the session receiver |
| | 322 | | // in the same place where we are creating them. |
| 0 | 323 | | var sbException = exception as ServiceBusException; |
| 0 | 324 | | if (sbException?.Reason == ServiceBusFailureReason.SessionLockLost) |
| | 325 | | { |
| 0 | 326 | | sbException.ProcessorErrorSource = errorSource; |
| 0 | 327 | | throw sbException; |
| | 328 | | } |
| 0 | 329 | | } |
| | 330 | |
|
| | 331 | | protected async Task HandleRenewLockException(Exception ex, CancellationToken cancellationToken) |
| | 332 | | { |
| | 333 | | // ObjectDisposedException should only happen here because the CancellationToken was disposed at which point |
| | 334 | | // this renew exception is not relevant anymore. Lets not bother user with this exception. |
| 0 | 335 | | if (!(ex is ObjectDisposedException) && !cancellationToken.IsCancellationRequested) |
| | 336 | | { |
| 0 | 337 | | await RaiseExceptionReceived( |
| 0 | 338 | | new ProcessErrorEventArgs( |
| 0 | 339 | | ex, |
| 0 | 340 | | ServiceBusErrorSource.RenewLock, |
| 0 | 341 | | _fullyQualifiedNamespace, |
| 0 | 342 | | _entityPath)).ConfigureAwait(false); |
| | 343 | | } |
| 0 | 344 | | } |
| | 345 | |
|
| | 346 | | /// <summary> |
| | 347 | | /// |
| | 348 | | /// </summary> |
| | 349 | | /// <param name="eventArgs"></param> |
| | 350 | | /// <returns></returns> |
| | 351 | | protected async Task RaiseExceptionReceived(ProcessErrorEventArgs eventArgs) |
| | 352 | | { |
| | 353 | | try |
| | 354 | | { |
| 0 | 355 | | await _errorHandler(eventArgs).ConfigureAwait(false); |
| 0 | 356 | | } |
| 0 | 357 | | catch (Exception exception) |
| | 358 | | { |
| | 359 | | // don't bubble up exceptions raised from customer exception handler |
| 0 | 360 | | ServiceBusEventSource.Log.ProcessorErrorHandlerThrewException(exception.ToString()); |
| 0 | 361 | | } |
| 0 | 362 | | } |
| | 363 | |
|
| | 364 | | /// <summary> |
| | 365 | | /// |
| | 366 | | /// </summary> |
| | 367 | | /// <param name="lockedUntil"></param> |
| | 368 | | /// <returns></returns> |
| | 369 | | protected static TimeSpan CalculateRenewDelay(DateTimeOffset lockedUntil) |
| | 370 | | { |
| 0 | 371 | | var remainingTime = lockedUntil - DateTimeOffset.UtcNow; |
| | 372 | |
|
| 0 | 373 | | if (remainingTime < TimeSpan.FromMilliseconds(400)) |
| | 374 | | { |
| 0 | 375 | | return TimeSpan.Zero; |
| | 376 | | } |
| | 377 | |
|
| 0 | 378 | | var buffer = TimeSpan.FromTicks(Math.Min(remainingTime.Ticks / 2, Constants.MaximumRenewBufferDuration.Ticks |
| 0 | 379 | | var renewAfter = remainingTime - buffer; |
| | 380 | |
|
| 0 | 381 | | return renewAfter; |
| | 382 | | } |
| | 383 | | } |
| | 384 | | } |