| | 1 | | // Copyright (c) Microsoft Corporation. All rights reserved. |
| | 2 | | // Licensed under the MIT License. |
| | 3 | |
|
| | 4 | | using System; |
| | 5 | | using System.Collections.Concurrent; |
| | 6 | | using System.Collections.Generic; |
| | 7 | | using System.ComponentModel; |
| | 8 | | using System.Diagnostics.CodeAnalysis; |
| | 9 | | using System.Globalization; |
| | 10 | | using System.Linq; |
| | 11 | | using System.Runtime.CompilerServices; |
| | 12 | | using System.Runtime.ExceptionServices; |
| | 13 | | using System.Threading; |
| | 14 | | using System.Threading.Channels; |
| | 15 | | using System.Threading.Tasks; |
| | 16 | | using Azure.Core; |
| | 17 | | using Azure.Messaging.EventHubs.Core; |
| | 18 | | using Azure.Messaging.EventHubs.Diagnostics; |
| | 19 | |
|
| | 20 | | namespace Azure.Messaging.EventHubs.Consumer |
| | 21 | | { |
| | 22 | | /// <summary> |
| | 23 | | /// A client responsible for reading <see cref="EventData" /> from a specific Event Hub |
| | 24 | | /// as a member of a specific consumer group. |
| | 25 | | /// |
| | 26 | | /// A consumer may be exclusive, which asserts ownership over associated partitions for the consumer |
| | 27 | | /// group to ensure that only one consumer from that group is reading the from the partition. |
| | 28 | | /// These exclusive consumers are sometimes referred to as "Epoch Consumers." |
| | 29 | | /// |
| | 30 | | /// A consumer may also be non-exclusive, allowing multiple consumers from the same consumer |
| | 31 | | /// group to be actively reading events from a given partition. These non-exclusive consumers are |
| | 32 | | /// sometimes referred to as "Non-Epoch Consumers." |
| | 33 | | /// </summary> |
| | 34 | | /// |
| | 35 | | public class EventHubConsumerClient : IAsyncDisposable |
| | 36 | | { |
| | 37 | | /// <summary>The name of the default consumer group in the Event Hubs service.</summary> |
| | 38 | | public const string DefaultConsumerGroupName = "$Default"; |
| | 39 | |
|
| | 40 | | /// <summary>The maximum wait time for receiving an event batch for the background publishing operation used for |
| 0 | 41 | | private readonly TimeSpan BackgroundPublishingWaitTime = TimeSpan.FromMilliseconds(250); |
| | 42 | |
|
| | 43 | | /// <summary>Indicates whether or not this instance has been closed.</summary> |
| | 44 | | private volatile bool _closed = false; |
| | 45 | |
|
| | 46 | | /// <summary> |
| | 47 | | /// The fully qualified Event Hubs namespace that the consumer is associated with. This is likely |
| | 48 | | /// to be similar to <c>{yournamespace}.servicebus.windows.net</c>. |
| | 49 | | /// </summary> |
| | 50 | | /// |
| 2 | 51 | | public string FullyQualifiedNamespace => Connection.FullyQualifiedNamespace; |
| | 52 | |
|
| | 53 | | /// <summary> |
| | 54 | | /// The name of the Event Hub that the consumer is connected to, specific to the |
| | 55 | | /// Event Hubs namespace that contains it. |
| | 56 | | /// </summary> |
| | 57 | | /// |
| 434 | 58 | | public string EventHubName => Connection.EventHubName; |
| | 59 | |
|
| | 60 | | /// <summary> |
| | 61 | | /// The name of the consumer group that this consumer is associated with. Events will be read |
| | 62 | | /// only in the context of this group. |
| | 63 | | /// </summary> |
| | 64 | | /// |
| 148 | 65 | | public string ConsumerGroup { get; } |
| | 66 | |
|
| | 67 | | /// <summary> |
| | 68 | | /// Indicates whether or not this <see cref="EventHubConsumerClient"/> has been closed. |
| | 69 | | /// </summary> |
| | 70 | | /// |
| | 71 | | /// <value> |
| | 72 | | /// <c>true</c> if the client is closed; otherwise, <c>false</c>. |
| | 73 | | /// </value> |
| | 74 | | /// |
| | 75 | | public bool IsClosed |
| | 76 | | { |
| 180 | 77 | | get => _closed; |
| 12 | 78 | | protected set => _closed = value; |
| | 79 | | } |
| | 80 | |
|
| | 81 | | /// <summary> |
| | 82 | | /// Indicates whether the client has ownership of the associated <see cref="EventHubConnection" /> |
| | 83 | | /// and should take responsibility for managing its lifespan. |
| | 84 | | /// </summary> |
| | 85 | | /// |
| 0 | 86 | | private bool OwnsConnection { get; } = true; |
| | 87 | |
|
| | 88 | | /// <summary> |
| | 89 | | /// The policy to use for determining retry behavior for when an operation fails. |
| | 90 | | /// </summary> |
| | 91 | | /// |
| 286 | 92 | | private EventHubsRetryPolicy RetryPolicy { get; } |
| | 93 | |
|
| | 94 | | /// <summary> |
| | 95 | | /// The active connection to the Azure Event Hubs service, enabling client communications for metadata |
| | 96 | | /// about the associated Event Hub and access to transport-aware consumers. |
| | 97 | | /// </summary> |
| | 98 | | /// |
| 652 | 99 | | private EventHubConnection Connection { get; } |
| | 100 | |
|
| | 101 | | /// <summary> |
| | 102 | | /// The set of active Event Hub transport-specific consumers created by this client for use with |
| | 103 | | /// delegated operations. |
| | 104 | | /// </summary> |
| | 105 | | /// |
| 0 | 106 | | private ConcurrentDictionary<string, TransportConsumer> ActiveConsumers { get; } = new ConcurrentDictionary<stri |
| | 107 | |
|
| | 108 | | /// <summary> |
| | 109 | | /// Initializes a new instance of the <see cref="EventHubConsumerClient"/> class. |
| | 110 | | /// </summary> |
| | 111 | | /// |
| | 112 | | /// <param name="consumerGroup">The name of the consumer group this consumer is associated with. Events are rea |
| | 113 | | /// <param name="connectionString">The connection string to use for connecting to the Event Hubs namespace; it i |
| | 114 | | /// |
| | 115 | | /// <remarks> |
| | 116 | | /// If the connection string is copied from the Event Hubs namespace, it will likely not contain the name of t |
| | 117 | | /// which is needed. In this case, the name can be added manually by adding ";EntityPath=[[ EVENT HUB NAME ]] |
| | 118 | | /// connection string. For example, ";EntityPath=telemetry-hub". |
| | 119 | | /// |
| | 120 | | /// If you have defined a shared access policy directly on the Event Hub itself, then copying the connection s |
| | 121 | | /// Event Hub will result in a connection string that contains the name. |
| | 122 | | /// </remarks> |
| | 123 | | /// |
| | 124 | | /// <seealso href="https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string"/> |
| | 125 | | /// |
| | 126 | | public EventHubConsumerClient(string consumerGroup, |
| 22 | 127 | | string connectionString) : this(consumerGroup, connectionString, null, null) |
| | 128 | | { |
| 6 | 129 | | } |
| | 130 | |
|
| | 131 | | /// <summary> |
| | 132 | | /// Initializes a new instance of the <see cref="EventHubConsumerClient"/> class. |
| | 133 | | /// </summary> |
| | 134 | | /// |
| | 135 | | /// <param name="consumerGroup">The name of the consumer group this consumer is associated with. Events are rea |
| | 136 | | /// <param name="connectionString">The connection string to use for connecting to the Event Hubs namespace; it i |
| | 137 | | /// <param name="clientOptions">The set of options to use for this consumer.</param> |
| | 138 | | /// |
| | 139 | | /// <remarks> |
| | 140 | | /// If the connection string is copied from the Event Hubs namespace, it will likely not contain the name of t |
| | 141 | | /// which is needed. In this case, the name can be added manually by adding ";EntityPath=[[ EVENT HUB NAME ]] |
| | 142 | | /// connection string. For example, ";EntityPath=telemetry-hub". |
| | 143 | | /// |
| | 144 | | /// If you have defined a shared access policy directly on the Event Hub itself, then copying the connection s |
| | 145 | | /// Event Hub will result in a connection string that contains the name. |
| | 146 | | /// </remarks> |
| | 147 | | /// |
| | 148 | | /// <seealso href="https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string"/> |
| | 149 | | /// |
| | 150 | | public EventHubConsumerClient(string consumerGroup, |
| | 151 | | string connectionString, |
| 6 | 152 | | EventHubConsumerClientOptions clientOptions) : this(consumerGroup, connectionStrin |
| | 153 | | { |
| 2 | 154 | | } |
| | 155 | |
|
| | 156 | | /// <summary> |
| | 157 | | /// Initializes a new instance of the <see cref="EventHubConsumerClient"/> class. |
| | 158 | | /// </summary> |
| | 159 | | /// |
| | 160 | | /// <param name="consumerGroup">The name of the consumer group this consumer is associated with. Events are rea |
| | 161 | | /// <param name="connectionString">The connection string to use for connecting to the Event Hubs namespace; it i |
| | 162 | | /// <param name="eventHubName">The name of the specific Event Hub to associate the consumer with.</param> |
| | 163 | | /// |
| | 164 | | /// <remarks> |
| | 165 | | /// If the connection string is copied from the Event Hub itself, it will contain the name of the desired Even |
| | 166 | | /// and can be used directly without passing the <paramref name="eventHubName" />. The name of the Event Hub |
| | 167 | | /// passed only once, either as part of the connection string or separately. |
| | 168 | | /// </remarks> |
| | 169 | | /// |
| | 170 | | /// <seealso href="https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string"/> |
| | 171 | | /// |
| | 172 | | public EventHubConsumerClient(string consumerGroup, |
| | 173 | | string connectionString, |
| 4 | 174 | | string eventHubName) : this(consumerGroup, connectionString, eventHubName, null) |
| | 175 | | { |
| 2 | 176 | | } |
| | 177 | |
|
| | 178 | | /// <summary> |
| | 179 | | /// Initializes a new instance of the <see cref="EventHubConsumerClient"/> class. |
| | 180 | | /// </summary> |
| | 181 | | /// |
| | 182 | | /// <param name="consumerGroup">The name of the consumer group this consumer is associated with. Events are rea |
| | 183 | | /// <param name="connectionString">The connection string to use for connecting to the Event Hubs namespace; it i |
| | 184 | | /// <param name="eventHubName">The name of the specific Event Hub to associate the consumer with.</param> |
| | 185 | | /// <param name="clientOptions">A set of options to apply when configuring the consumer.</param> |
| | 186 | | /// |
| | 187 | | /// <remarks> |
| | 188 | | /// If the connection string is copied from the Event Hub itself, it will contain the name of the desired Even |
| | 189 | | /// and can be used directly without passing the <paramref name="eventHubName" />. The name of the Event Hub |
| | 190 | | /// passed only once, either as part of the connection string or separately. |
| | 191 | | /// </remarks> |
| | 192 | | /// |
| | 193 | | /// <seealso href="https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string"/> |
| | 194 | | /// |
| 36 | 195 | | public EventHubConsumerClient(string consumerGroup, |
| 36 | 196 | | string connectionString, |
| 36 | 197 | | string eventHubName, |
| 36 | 198 | | EventHubConsumerClientOptions clientOptions) |
| | 199 | | { |
| 36 | 200 | | Argument.AssertNotNullOrEmpty(consumerGroup, nameof(consumerGroup)); |
| 32 | 201 | | Argument.AssertNotNullOrEmpty(connectionString, nameof(connectionString)); |
| | 202 | |
|
| 24 | 203 | | clientOptions = clientOptions?.Clone() ?? new EventHubConsumerClientOptions(); |
| | 204 | |
|
| 24 | 205 | | OwnsConnection = true; |
| 24 | 206 | | Connection = new EventHubConnection(connectionString, eventHubName, clientOptions.ConnectionOptions); |
| 10 | 207 | | ConsumerGroup = consumerGroup; |
| 10 | 208 | | RetryPolicy = clientOptions.RetryOptions.ToRetryPolicy(); |
| 10 | 209 | | } |
| | 210 | |
|
| | 211 | | /// <summary> |
| | 212 | | /// Initializes a new instance of the <see cref="EventHubConsumerClient"/> class. |
| | 213 | | /// </summary> |
| | 214 | | /// |
| | 215 | | /// <param name="consumerGroup">The name of the consumer group this consumer is associated with. Events are rea |
| | 216 | | /// <param name="fullyQualifiedNamespace">The fully qualified Event Hubs namespace to connect to. This is likel |
| | 217 | | /// <param name="eventHubName">The name of the specific Event Hub to associate the consumer with.</param> |
| | 218 | | /// <param name="credential">The Azure managed identity credential to use for authorization. Access controls ma |
| | 219 | | /// <param name="clientOptions">A set of options to apply when configuring the consumer.</param> |
| | 220 | | /// |
| 22 | 221 | | public EventHubConsumerClient(string consumerGroup, |
| 22 | 222 | | string fullyQualifiedNamespace, |
| 22 | 223 | | string eventHubName, |
| 22 | 224 | | TokenCredential credential, |
| 22 | 225 | | EventHubConsumerClientOptions clientOptions = default) |
| | 226 | | { |
| 22 | 227 | | Argument.AssertNotNullOrEmpty(consumerGroup, nameof(consumerGroup)); |
| 18 | 228 | | Argument.AssertWellFormedEventHubsNamespace(fullyQualifiedNamespace, nameof(fullyQualifiedNamespace)); |
| 12 | 229 | | Argument.AssertNotNullOrEmpty(eventHubName, nameof(eventHubName)); |
| 8 | 230 | | Argument.AssertNotNull(credential, nameof(credential)); |
| | 231 | |
|
| 6 | 232 | | clientOptions = clientOptions?.Clone() ?? new EventHubConsumerClientOptions(); |
| | 233 | |
|
| 6 | 234 | | OwnsConnection = true; |
| 6 | 235 | | Connection = new EventHubConnection(fullyQualifiedNamespace, eventHubName, credential, clientOptions.Connect |
| 6 | 236 | | ConsumerGroup = consumerGroup; |
| 6 | 237 | | RetryPolicy = clientOptions.RetryOptions.ToRetryPolicy(); |
| 6 | 238 | | } |
| | 239 | |
|
| | 240 | | /// <summary> |
| | 241 | | /// Initializes a new instance of the <see cref="EventHubConsumerClient"/> class. |
| | 242 | | /// </summary> |
| | 243 | | /// |
| | 244 | | /// <param name="consumerGroup">The name of the consumer group this consumer is associated with. Events are rea |
| | 245 | | /// <param name="connection">The <see cref="EventHubConnection" /> connection to use for communication with the |
| | 246 | | /// <param name="clientOptions">A set of options to apply when configuring the consumer.</param> |
| | 247 | | /// |
| 128 | 248 | | public EventHubConsumerClient(string consumerGroup, |
| 128 | 249 | | EventHubConnection connection, |
| 128 | 250 | | EventHubConsumerClientOptions clientOptions = default) |
| | 251 | | { |
| 128 | 252 | | Argument.AssertNotNullOrEmpty(consumerGroup, nameof(consumerGroup)); |
| 128 | 253 | | Argument.AssertNotNull(connection, nameof(connection)); |
| 126 | 254 | | clientOptions = clientOptions?.Clone() ?? new EventHubConsumerClientOptions(); |
| | 255 | |
|
| 126 | 256 | | OwnsConnection = false; |
| 126 | 257 | | Connection = connection; |
| 126 | 258 | | ConsumerGroup = consumerGroup; |
| 126 | 259 | | RetryPolicy = clientOptions.RetryOptions.ToRetryPolicy(); |
| 126 | 260 | | } |
| | 261 | |
|
| | 262 | | /// <summary> |
| | 263 | | /// Initializes a new instance of the <see cref="EventHubConsumerClient"/> class. |
| | 264 | | /// </summary> |
| | 265 | | /// |
| 0 | 266 | | protected EventHubConsumerClient() |
| | 267 | | { |
| 0 | 268 | | OwnsConnection = false; |
| 0 | 269 | | } |
| | 270 | |
|
| | 271 | | /// <summary> |
| | 272 | | /// Retrieves information about the Event Hub that the connection is associated with, including |
| | 273 | | /// the number of partitions present and their identifiers. |
| | 274 | | /// </summary> |
| | 275 | | /// |
| | 276 | | /// <param name="cancellationToken">An optional <see cref="CancellationToken"/> instance to signal the request t |
| | 277 | | /// |
| | 278 | | /// <returns>The set of information for the Event Hub that this client is associated with.</returns> |
| | 279 | | /// |
| | 280 | | public virtual async Task<EventHubProperties> GetEventHubPropertiesAsync(CancellationToken cancellationToken = d |
| | 281 | | { |
| 2 | 282 | | Argument.AssertNotClosed(IsClosed, nameof(EventHubConsumerClient)); |
| 2 | 283 | | return await Connection.GetPropertiesAsync(RetryPolicy, cancellationToken).ConfigureAwait(false); |
| 2 | 284 | | } |
| | 285 | |
|
| | 286 | | /// <summary> |
| | 287 | | /// Retrieves the set of identifiers for the partitions of an Event Hub. |
| | 288 | | /// </summary> |
| | 289 | | /// |
| | 290 | | /// <param name="cancellationToken">An optional <see cref="CancellationToken"/> instance to signal the request t |
| | 291 | | /// |
| | 292 | | /// <returns>The set of identifiers for the partitions within the Event Hub that this client is associated with. |
| | 293 | | /// |
| | 294 | | /// <remarks> |
| | 295 | | /// This method is synonymous with invoking <see cref="GetEventHubPropertiesAsync(CancellationToken)" /> and r |
| | 296 | | /// property that is returned. It is offered as a convenience for quick access to the set of partition identif |
| | 297 | | /// No new or extended information is presented. |
| | 298 | | /// </remarks> |
| | 299 | | /// |
| | 300 | | public virtual async Task<string[]> GetPartitionIdsAsync(CancellationToken cancellationToken = default) |
| | 301 | | { |
| | 302 | |
|
| 66 | 303 | | Argument.AssertNotClosed(IsClosed, nameof(EventHubConsumerClient)); |
| 66 | 304 | | return await Connection.GetPartitionIdsAsync(RetryPolicy, cancellationToken).ConfigureAwait(false); |
| 66 | 305 | | } |
| | 306 | |
|
| | 307 | | /// <summary> |
| | 308 | | /// Retrieves information about a specific partition for an Event Hub, including elements that describe the av |
| | 309 | | /// events in the partition event stream. |
| | 310 | | /// </summary> |
| | 311 | | /// |
| | 312 | | /// <param name="partitionId">The unique identifier of a partition associated with the Event Hub.</param> |
| | 313 | | /// <param name="cancellationToken">An optional <see cref="CancellationToken"/> instance to signal the request t |
| | 314 | | /// |
| | 315 | | /// <returns>The set of information for the requested partition under the Event Hub this client is associated wi |
| | 316 | | /// |
| | 317 | | public virtual async Task<PartitionProperties> GetPartitionPropertiesAsync(string partitionId, |
| | 318 | | CancellationToken cancellationToken = defau |
| | 319 | | { |
| 2 | 320 | | Argument.AssertNotClosed(IsClosed, nameof(EventHubConsumerClient)); |
| 2 | 321 | | return await Connection.GetPartitionPropertiesAsync(partitionId, RetryPolicy, cancellationToken).ConfigureAw |
| 2 | 322 | | } |
| | 323 | |
|
| | 324 | | /// <summary> |
| | 325 | | /// Reads events from the requested partition as an asynchronous enumerable, allowing events to be iterated as |
| | 326 | | /// become available on the partition, waiting as necessary should there be no events available. |
| | 327 | | /// |
| | 328 | | /// This enumerator may block for an indeterminate amount of time for an <c>await</c> if events are not availa |
| | 329 | | /// cancellation via the <paramref name="cancellationToken"/> to be requested in order to return control. It |
| | 330 | | /// which accepts a set of options for configuring read behavior for scenarios where a more deterministic maxi |
| | 331 | | /// </summary> |
| | 332 | | /// |
| | 333 | | /// <param name="partitionId">The identifier of the Event Hub partition from which events will be received.</par |
| | 334 | | /// <param name="startingPosition">The position within the partition where the consumer should begin reading eve |
| | 335 | | /// <param name="cancellationToken">An optional <see cref="CancellationToken"/> instance to signal the request t |
| | 336 | | /// |
| | 337 | | /// <returns>An <see cref="IAsyncEnumerable{T}"/> to be used for iterating over events in the partition.</return |
| | 338 | | /// |
| | 339 | | /// <remarks> |
| | 340 | | /// Each reader of events is presented with an independent iterator; if there are multiple readers, each recei |
| | 341 | | /// process, rather than competing for them. |
| | 342 | | /// </remarks> |
| | 343 | | /// |
| | 344 | | /// <seealso cref="ReadEventsFromPartitionAsync(string, EventPosition, ReadEventOptions, CancellationToken)"/> |
| | 345 | | /// |
| | 346 | | public virtual IAsyncEnumerable<PartitionEvent> ReadEventsFromPartitionAsync(string partitionId, |
| | 347 | | EventPosition startingPosition, |
| 28 | 348 | | CancellationToken cancellationToken |
| | 349 | |
|
| | 350 | | /// <summary> |
| | 351 | | /// Reads events from the requested partition as an asynchronous enumerable, allowing events to be iterated as |
| | 352 | | /// become available on the partition, waiting as necessary should there be no events available. |
| | 353 | | /// |
| | 354 | | /// This enumerator may block for an indeterminate amount of time for an <c>await</c> if events are not availa |
| | 355 | | /// cancellation via the <paramref name="cancellationToken"/> to be requested in order to return control. It |
| | 356 | | /// <see cref="ReadEventOptions.MaximumWaitTime" /> for scenarios where a more deterministic maximum waiting p |
| | 357 | | /// </summary> |
| | 358 | | /// |
| | 359 | | /// <param name="partitionId">The identifier of the Event Hub partition from which events will be received.</par |
| | 360 | | /// <param name="startingPosition">The position within the partition where the consumer should begin reading eve |
| | 361 | | /// <param name="readOptions">The set of options to use for configuring read behavior; if not specified the defa |
| | 362 | | /// <param name="cancellationToken">An optional <see cref="CancellationToken"/> instance to signal the request t |
| | 363 | | /// |
| | 364 | | /// <returns>An <see cref="IAsyncEnumerable{T}"/> to be used for iterating over events in the partition.</return |
| | 365 | | /// |
| | 366 | | /// <remarks> |
| | 367 | | /// Each reader of events is presented with an independent iterator; if there are multiple readers, each recei |
| | 368 | | /// process, rather than competing for them. |
| | 369 | | /// </remarks> |
| | 370 | | /// |
| | 371 | | /// <seealso cref="ReadEventsFromPartitionAsync(string, EventPosition, CancellationToken)"/> |
| | 372 | | /// |
| | 373 | | public virtual async IAsyncEnumerable<PartitionEvent> ReadEventsFromPartitionAsync(string partitionId, |
| | 374 | | EventPosition startingPositio |
| | 375 | | ReadEventOptions readOptions, |
| | 376 | | [EnumeratorCancellation] Canc |
| | 377 | | { |
| 42 | 378 | | Argument.AssertNotClosed(IsClosed, nameof(EventHubConsumerClient)); |
| 40 | 379 | | cancellationToken.ThrowIfCancellationRequested<TaskCanceledException>(); |
| | 380 | |
|
| 38 | 381 | | EventHubsEventSource.Log.ReadEventsFromPartitionStart(EventHubName, partitionId); |
| 38 | 382 | | readOptions = readOptions?.Clone() ?? new ReadEventOptions(); |
| | 383 | |
|
| 38 | 384 | | var transportConsumer = default(TransportConsumer); |
| 38 | 385 | | var partitionContext = default(PartitionContext); |
| 38 | 386 | | var emptyPartitionEvent = default(PartitionEvent); |
| 38 | 387 | | var closeException = default(Exception); |
| | 388 | |
|
| | 389 | | try |
| | 390 | | { |
| | 391 | | // Attempt to initialize the common objects. These will be used during the read and emit operations |
| | 392 | | // that follow. Because catch blocks cannot be used when yielding items, this step is wrapped individua |
| | 393 | | // to ensure that any exceptions are logged. |
| | 394 | |
|
| | 395 | | try |
| | 396 | | { |
| 38 | 397 | | transportConsumer = Connection.CreateTransportConsumer(ConsumerGroup, partitionId, startingPosition, |
| 38 | 398 | | partitionContext = new PartitionContext(partitionId, transportConsumer); |
| 38 | 399 | | emptyPartitionEvent = new PartitionEvent(partitionContext, null); |
| 38 | 400 | | } |
| 0 | 401 | | catch (Exception ex) |
| | 402 | | { |
| 0 | 403 | | EventHubsEventSource.Log.ReadEventsFromPartitionError(EventHubName, partitionId, ex.Message); |
| 0 | 404 | | throw; |
| | 405 | | } |
| | 406 | |
|
| | 407 | | // Process items. Because catch blocks cannot be used when yielding items, ensure that any exceptions d |
| | 408 | | // the receive operation are caught in an independent try/catch block. |
| | 409 | |
|
| 38 | 410 | | var emptyBatch = true; |
| 38 | 411 | | var eventBatch = default(IReadOnlyList<EventData>); |
| | 412 | |
|
| 66 | 413 | | while (!cancellationToken.IsCancellationRequested) |
| | 414 | | { |
| | 415 | | try |
| | 416 | | { |
| 64 | 417 | | emptyBatch = true; |
| 64 | 418 | | eventBatch = await transportConsumer.ReceiveAsync(readOptions.CacheEventCount, readOptions.Maxim |
| 52 | 419 | | } |
| 2 | 420 | | catch (TaskCanceledException) |
| | 421 | | { |
| 2 | 422 | | throw; |
| | 423 | | } |
| 2 | 424 | | catch (OperationCanceledException ex) |
| | 425 | | { |
| 2 | 426 | | throw new TaskCanceledException(ex.Message, ex); |
| | 427 | | } |
| 8 | 428 | | catch (Exception ex) |
| | 429 | | { |
| 8 | 430 | | EventHubsEventSource.Log.ReadEventsFromPartitionError(EventHubName, partitionId, ex.Message); |
| 8 | 431 | | throw; |
| | 432 | | } |
| | 433 | |
|
| | 434 | | // The batch will be available after the receive operation, regardless of whether there were events |
| | 435 | | // available or not; if there are any events in the set to yield, then mark the batch as non-empty. |
| | 436 | |
|
| 3414 | 437 | | foreach (var eventData in eventBatch) |
| | 438 | | { |
| 1664 | 439 | | cancellationToken.ThrowIfCancellationRequested<TaskCanceledException>(); |
| | 440 | |
|
| 1662 | 441 | | emptyBatch = false; |
| 1662 | 442 | | yield return new PartitionEvent(partitionContext, eventData); |
| | 443 | | } |
| | 444 | |
|
| | 445 | | // If there were no events received, only emit an empty event if there was a maximum wait time |
| | 446 | | // explicitly requested, otherwise, continue attempting to receive without emitting any value |
| | 447 | | // to the enumerable. |
| | 448 | |
|
| 34 | 449 | | if ((emptyBatch) && (readOptions.MaximumWaitTime.HasValue)) |
| | 450 | | { |
| 8 | 451 | | yield return emptyPartitionEvent; |
| | 452 | | } |
| | 453 | | } |
| | 454 | | } |
| | 455 | | finally |
| | 456 | | { |
| 38 | 457 | | if (transportConsumer != null) |
| | 458 | | { |
| | 459 | | try |
| | 460 | | { |
| 38 | 461 | | await transportConsumer.CloseAsync(CancellationToken.None).ConfigureAwait(false); |
| 38 | 462 | | } |
| 0 | 463 | | catch (Exception ex) |
| | 464 | | { |
| 0 | 465 | | EventHubsEventSource.Log.ReadEventsFromPartitionError(EventHubName, partitionId, ex.Message); |
| 0 | 466 | | closeException = ex; |
| 0 | 467 | | } |
| | 468 | | } |
| | 469 | |
|
| 38 | 470 | | EventHubsEventSource.Log.ReadEventsFromPartitionComplete(EventHubName, partitionId); |
| | 471 | | } |
| | 472 | |
|
| | 473 | | // If an exception was captured when closing the transport consumer, surface it. |
| | 474 | |
|
| 2 | 475 | | if (closeException != default) |
| | 476 | | { |
| 0 | 477 | | ExceptionDispatchInfo.Capture(closeException).Throw(); |
| | 478 | | } |
| | 479 | |
|
| | 480 | | // If cancellation was requested, then surface the expected exception. |
| | 481 | |
|
| 2 | 482 | | cancellationToken.ThrowIfCancellationRequested<TaskCanceledException>(); |
| 26 | 483 | | } |
| | 484 | |
|
| | 485 | | /// <summary> |
| | 486 | | /// Reads events from all partitions of the event hub as an asynchronous enumerable, allowing events to be ite |
| | 487 | | /// become available on the partition, waiting as necessary should there be no events available. |
| | 488 | | /// |
| | 489 | | /// This enumerator may block for an indeterminate amount of time for an <c>await</c> if events are not availa |
| | 490 | | /// cancellation via the <paramref name="cancellationToken"/> to be requested in order to return control. It |
| | 491 | | /// <see cref="ReadEventOptions.MaximumWaitTime" /> for scenarios where a more deterministic maximum waiting p |
| | 492 | | /// </summary> |
| | 493 | | /// |
| | 494 | | /// <param name="cancellationToken">An optional <see cref="CancellationToken"/> instance to signal the request t |
| | 495 | | /// |
| | 496 | | /// <returns>An <see cref="IAsyncEnumerable{T}"/> to be used for iterating over events in the partition.</return |
| | 497 | | /// |
| | 498 | | /// <remarks> |
| | 499 | | /// This method is not recommended for production use; the <c>EventProcessorClient</c> should be used for read |
| | 500 | | /// production scenario, as it offers a much more robust experience with higher throughput. |
| | 501 | | /// |
| | 502 | | /// It is important to note that this method does not guarantee fairness amongst the partitions during iterati |
| | 503 | | /// events to be read by the enumerator. Depending on service communication, there may be a clustering of eve |
| | 504 | | /// bias for a given partition or subset of partitions. |
| | 505 | | /// |
| | 506 | | /// Each reader of events is presented with an independent iterator; if there are multiple readers, each recei |
| | 507 | | /// process, rather than competing for them. |
| | 508 | | /// </remarks> |
| | 509 | | /// |
| | 510 | | /// <seealso href="https://www.nuget.org/packages/Azure.Messaging.EventHubs.Processor" /> |
| | 511 | | /// <seealso cref="ReadEventsAsync(ReadEventOptions, CancellationToken)"/> |
| | 512 | | /// |
| 52 | 513 | | public virtual IAsyncEnumerable<PartitionEvent> ReadEventsAsync(CancellationToken cancellationToken = default) = |
| | 514 | |
|
| | 515 | | /// <summary> |
| | 516 | | /// Reads events from all partitions of the event hub as an asynchronous enumerable, allowing events to be ite |
| | 517 | | /// become available on the partition, waiting as necessary should there be no events available. |
| | 518 | | /// |
| | 519 | | /// This enumerator may block for an indeterminate amount of time for an <c>await</c> if events are not availa |
| | 520 | | /// cancellation via the <paramref name="cancellationToken"/> to be requested in order to return control. It |
| | 521 | | /// <see cref="ReadEventOptions.MaximumWaitTime" /> for scenarios where a more deterministic maximum waiting p |
| | 522 | | /// </summary> |
| | 523 | | /// |
| | 524 | | /// <param name="readOptions">The set of options to use for configuring read behavior; if not specified the defa |
| | 525 | | /// <param name="cancellationToken">An optional <see cref="CancellationToken"/> instance to signal the request t |
| | 526 | | /// |
| | 527 | | /// <returns>An <see cref="IAsyncEnumerable{T}"/> to be used for iterating over events in the partition.</return |
| | 528 | | /// |
| | 529 | | /// <remarks> |
| | 530 | | /// This method is not recommended for production use; the <c>EventProcessorClient</c> should be used for read |
| | 531 | | /// production scenario, as it offers a much more robust experience with higher throughput. |
| | 532 | | /// |
| | 533 | | /// It is important to note that this method does not guarantee fairness amongst the partitions during iterati |
| | 534 | | /// events to be read by the enumerator. Depending on service communication, there may be a clustering of eve |
| | 535 | | /// bias for a given partition or subset of partitions. |
| | 536 | | /// |
| | 537 | | /// Each reader of events is presented with an independent iterator; if there are multiple readers, each recei |
| | 538 | | /// process, rather than competing for them. |
| | 539 | | /// </remarks> |
| | 540 | | /// |
| | 541 | | /// <seealso href="https://www.nuget.org/packages/Azure.Messaging.EventHubs.Processor" /> |
| | 542 | | /// <seealso cref="ReadEventsAsync(CancellationToken)"/> |
| | 543 | | /// |
| | 544 | | public virtual IAsyncEnumerable<PartitionEvent> ReadEventsAsync(ReadEventOptions readOptions, |
| 60 | 545 | | CancellationToken cancellationToken = default) = |
| | 546 | |
|
| | 547 | | /// <summary> |
| | 548 | | /// Reads events from all partitions of the event hub as an asynchronous enumerable, allowing events to be ite |
| | 549 | | /// become available on the partition, waiting as necessary should there be no events available. |
| | 550 | | /// |
| | 551 | | /// This enumerator may block for an indeterminate amount of time for an <c>await</c> if events are not availa |
| | 552 | | /// cancellation via the <paramref name="cancellationToken"/> to be requested in order to return control. It |
| | 553 | | /// <see cref="ReadEventOptions.MaximumWaitTime" /> for scenarios where a more deterministic maximum waiting p |
| | 554 | | /// </summary> |
| | 555 | | /// |
| | 556 | | /// <param name="startReadingAtEarliestEvent"><c>true</c> to begin reading at the first events available in each |
| | 557 | | /// <param name="readOptions">The set of options to use for configuring read behavior; if not specified the defa |
| | 558 | | /// <param name="cancellationToken">An optional <see cref="CancellationToken"/> instance to signal the request t |
| | 559 | | /// |
| | 560 | | /// <returns>An <see cref="IAsyncEnumerable{T}"/> to be used for iterating over events in the partition.</return |
| | 561 | | /// |
| | 562 | | /// <remarks> |
| | 563 | | /// This method is not recommended for production use; the <c>EventProcessorClient</c> should be used for read |
| | 564 | | /// production scenario, as it offers a much more robust experience with higher throughput. |
| | 565 | | /// |
| | 566 | | /// It is important to note that this method does not guarantee fairness amongst the partitions during iterati |
| | 567 | | /// events to be read by the enumerator. Depending on service communication, there may be a clustering of eve |
| | 568 | | /// bias for a given partition or subset of partitions. |
| | 569 | | /// |
| | 570 | | /// Each reader of events is presented with an independent iterator; if there are multiple readers, each recei |
| | 571 | | /// process, rather than competing for them. |
| | 572 | | /// </remarks> |
| | 573 | | /// |
| | 574 | | /// <seealso href="https://www.nuget.org/packages/Azure.Messaging.EventHubs.Processor" /> |
| | 575 | | /// <seealso cref="ReadEventsAsync(CancellationToken)"/> |
| | 576 | | /// <seealso cref="ReadEventsAsync(ReadEventOptions, CancellationToken)"/> |
| | 577 | | /// |
| | 578 | | public virtual async IAsyncEnumerable<PartitionEvent> ReadEventsAsync(bool startReadingAtEarliestEvent, |
| | 579 | | ReadEventOptions readOptions = default, |
| | 580 | | [EnumeratorCancellation] CancellationToken |
| | 581 | | { |
| 56 | 582 | | Argument.AssertNotClosed(IsClosed, nameof(EventHubConsumerClient)); |
| 54 | 583 | | cancellationToken.ThrowIfCancellationRequested<TaskCanceledException>(); |
| | 584 | |
|
| 52 | 585 | | EventHubsEventSource.Log.ReadAllEventsStart(EventHubName); |
| | 586 | |
|
| 52 | 587 | | var cancelPublishingAsync = default(Func<Task>); |
| 52 | 588 | | var eventChannel = default(Channel<PartitionEvent>); |
| 52 | 589 | | var options = readOptions?.Clone() ?? new ReadEventOptions(); |
| 52 | 590 | | var startingPosition = startReadingAtEarliestEvent ? EventPosition.Earliest : EventPosition.Latest; |
| | 591 | |
|
| 52 | 592 | | using var cancellationSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); |
| | 593 | |
|
| | 594 | | try |
| | 595 | | { |
| | 596 | | // Determine the partitions for the Event Hub and create the shared channel. |
| | 597 | |
|
| 52 | 598 | | var partitions = await GetPartitionIdsAsync(cancellationToken).ConfigureAwait(false); |
| | 599 | |
|
| 52 | 600 | | var channelSize = options.CacheEventCount * partitions.Length * 2L; |
| 52 | 601 | | eventChannel = CreateEventChannel((int)Math.Min(channelSize, int.MaxValue)); |
| | 602 | |
|
| | 603 | | // Start publishing for all partitions. |
| | 604 | |
|
| 52 | 605 | | var publishingTasks = new Task<Func<Task>>[partitions.Length]; |
| | 606 | |
|
| 312 | 607 | | for (var index = 0; index < partitions.Length; ++index) |
| | 608 | | { |
| 104 | 609 | | publishingTasks[index] = PublishPartitionEventsToChannelAsync(partitions[index], startingPosition, o |
| | 610 | | } |
| | 611 | |
|
| | 612 | | // Capture the callbacks to cancel publishing for all events. |
| | 613 | |
|
| 52 | 614 | | var cancelPublishingCallbacks = await Task.WhenAll(publishingTasks).ConfigureAwait(false); |
| 208 | 615 | | cancelPublishingAsync = () => Task.WhenAll(cancelPublishingCallbacks.Select(cancelCallback => cancelCall |
| 52 | 616 | | } |
| 0 | 617 | | catch (Exception ex) |
| | 618 | | { |
| 0 | 619 | | EventHubsEventSource.Log.ReadAllEventsError(EventHubName, ex.Message); |
| 0 | 620 | | cancellationSource?.Cancel(); |
| | 621 | |
|
| 0 | 622 | | if (cancelPublishingAsync != null) |
| | 623 | | { |
| 0 | 624 | | await cancelPublishingAsync().ConfigureAwait(false); |
| | 625 | | } |
| | 626 | |
|
| 0 | 627 | | EventHubsEventSource.Log.ReadAllEventsComplete(EventHubName); |
| 0 | 628 | | throw; |
| | 629 | | } |
| | 630 | |
|
| | 631 | | // Iterate the events from the channel. |
| | 632 | |
|
| | 633 | | try |
| | 634 | | { |
| 10962 | 635 | | await foreach (var partitionEvent in eventChannel.Reader.EnumerateChannel(options.MaximumWaitTime, cance |
| | 636 | | { |
| 5438 | 637 | | yield return partitionEvent; |
| | 638 | | } |
| | 639 | | } |
| | 640 | | finally |
| | 641 | | { |
| 52 | 642 | | cancellationSource?.Cancel(); |
| 52 | 643 | | await cancelPublishingAsync().ConfigureAwait(false); |
| 28 | 644 | | EventHubsEventSource.Log.ReadAllEventsComplete(EventHubName); |
| | 645 | | } |
| | 646 | |
|
| | 647 | | // If cancellation was requested, then surface the expected exception. |
| | 648 | |
|
| 0 | 649 | | cancellationToken.ThrowIfCancellationRequested<TaskCanceledException>(); |
| 22 | 650 | | } |
| | 651 | |
|
| | 652 | | /// <summary> |
| | 653 | | /// Closes the consumer. |
| | 654 | | /// </summary> |
| | 655 | | /// |
| | 656 | | /// <param name="cancellationToken">An optional <see cref="CancellationToken"/> instance to signal the request t |
| | 657 | | /// |
| | 658 | | /// <returns>A task to be resolved on when the operation has completed.</returns> |
| | 659 | | /// |
| | 660 | | public virtual async Task CloseAsync(CancellationToken cancellationToken = default) |
| | 661 | | { |
| 12 | 662 | | cancellationToken.ThrowIfCancellationRequested<TaskCanceledException>(); |
| | 663 | |
|
| 12 | 664 | | if (IsClosed) |
| | 665 | | { |
| 0 | 666 | | return; |
| | 667 | | } |
| | 668 | |
|
| 12 | 669 | | IsClosed = true; |
| | 670 | |
|
| 12 | 671 | | var clientHash = GetHashCode().ToString(CultureInfo.InvariantCulture); |
| 12 | 672 | | EventHubsEventSource.Log.ClientCloseStart(nameof(EventHubConsumerClient), EventHubName, clientHash); |
| | 673 | |
|
| | 674 | | // Attempt to close the active transport consumers. In the event that an exception is encountered, |
| | 675 | | // it should not impact the attempt to close the connection, assuming ownership. |
| | 676 | |
|
| 12 | 677 | | var transportConsumerException = default(Exception); |
| | 678 | |
|
| | 679 | | try |
| | 680 | | { |
| 12 | 681 | | var pendingCloses = new List<Task>(); |
| | 682 | |
|
| 32 | 683 | | foreach (var consumer in ActiveConsumers.Values) |
| | 684 | | { |
| 4 | 685 | | pendingCloses.Add(consumer.CloseAsync(CancellationToken.None)); |
| | 686 | | } |
| | 687 | |
|
| 12 | 688 | | ActiveConsumers.Clear(); |
| 12 | 689 | | await Task.WhenAll(pendingCloses).ConfigureAwait(false); |
| 10 | 690 | | } |
| 2 | 691 | | catch (Exception ex) |
| | 692 | | { |
| 2 | 693 | | EventHubsEventSource.Log.ClientCloseError(nameof(EventHubConsumerClient), EventHubName, clientHash, ex.M |
| 2 | 694 | | transportConsumerException = ex; |
| 2 | 695 | | } |
| | 696 | |
|
| | 697 | | // An exception when closing the connection supersedes one observed when closing the |
| | 698 | | // individual transport clients. |
| | 699 | |
|
| | 700 | | try |
| | 701 | | { |
| 12 | 702 | | if (OwnsConnection) |
| | 703 | | { |
| 2 | 704 | | await Connection.CloseAsync().ConfigureAwait(false); |
| | 705 | | } |
| 12 | 706 | | } |
| 0 | 707 | | catch (Exception ex) |
| | 708 | | { |
| 0 | 709 | | EventHubsEventSource.Log.ClientCloseError(nameof(EventHubConsumerClient), EventHubName, clientHash, ex.M |
| 0 | 710 | | transportConsumerException = null; |
| 0 | 711 | | throw; |
| | 712 | | } |
| | 713 | | finally |
| | 714 | | { |
| 12 | 715 | | EventHubsEventSource.Log.ClientCloseComplete(nameof(EventHubConsumerClient), EventHubName, clientHash); |
| | 716 | | } |
| | 717 | |
|
| | 718 | | // If there was an active exception pending from closing the individual |
| | 719 | | // transport consumers, surface it now. |
| | 720 | |
|
| 12 | 721 | | if (transportConsumerException != default) |
| | 722 | | { |
| 2 | 723 | | ExceptionDispatchInfo.Capture(transportConsumerException).Throw(); |
| | 724 | | } |
| 10 | 725 | | } |
| | 726 | |
|
| | 727 | | /// <summary> |
| | 728 | | /// Performs the task needed to clean up resources used by the <see cref="EventHubConsumerClient" />, |
| | 729 | | /// including ensuring that the client itself has been closed. |
| | 730 | | /// </summary> |
| | 731 | | /// |
| | 732 | | /// <returns>A task to be resolved on when the operation has completed.</returns> |
| | 733 | | /// |
| | 734 | | [SuppressMessage("Usage", "AZC0002:Ensure all service methods take an optional CancellationToken parameter.", Ju |
| 0 | 735 | | public virtual async ValueTask DisposeAsync() => await CloseAsync().ConfigureAwait(false); |
| | 736 | |
|
| | 737 | | /// <summary> |
| | 738 | | /// Determines whether the specified <see cref="System.Object" /> is equal to this instance. |
| | 739 | | /// </summary> |
| | 740 | | /// |
| | 741 | | /// <param name="obj">The <see cref="System.Object" /> to compare with this instance.</param> |
| | 742 | | /// |
| | 743 | | /// <returns><c>true</c> if the specified <see cref="System.Object" /> is equal to this instance; otherwise, <c> |
| | 744 | | /// |
| | 745 | | [EditorBrowsable(EditorBrowsableState.Never)] |
| 0 | 746 | | public override bool Equals(object obj) => base.Equals(obj); |
| | 747 | |
|
| | 748 | | /// <summary> |
| | 749 | | /// Returns a hash code for this instance. |
| | 750 | | /// </summary> |
| | 751 | | /// |
| | 752 | | /// <returns>A hash code for this instance, suitable for use in hashing algorithms and data structures like a ha |
| | 753 | | /// |
| | 754 | | [EditorBrowsable(EditorBrowsableState.Never)] |
| 12 | 755 | | public override int GetHashCode() => base.GetHashCode(); |
| | 756 | |
|
| | 757 | | /// <summary> |
| | 758 | | /// Converts the instance to string representation. |
| | 759 | | /// </summary> |
| | 760 | | /// |
| | 761 | | /// <returns>A <see cref="System.String" /> that represents this instance.</returns> |
| | 762 | | /// |
| | 763 | | [EditorBrowsable(EditorBrowsableState.Never)] |
| 0 | 764 | | public override string ToString() => base.ToString(); |
| | 765 | |
|
| | 766 | | /// <summary> |
| | 767 | | /// Publishes events for the requested <paramref name="partitionId"/> to the provided |
| | 768 | | /// <paramref name="channel" /> in the background, using a dedicated transport consumer |
| | 769 | | /// instance. |
| | 770 | | /// </summary> |
| | 771 | | /// |
| | 772 | | /// <param name="partitionId">The identifier of the partition from which events should be read.</param> |
| | 773 | | /// <param name="startingPosition">The position within the partition's event stream that reading should begin fr |
| | 774 | | /// <param name="trackLastEnqueuedEventProperties">Indicates whether information on the last enqueued event on t |
| | 775 | | /// <param name="receiveBatchSize">The batch size to use when receiving events.</param> |
| | 776 | | /// <param name="ownerLevel">The relative priority to associate with the link; for a non-exclusive link, this va |
| | 777 | | /// <param name="prefetchCount">The count of events requested eagerly and queued without regard to whether a rea |
| | 778 | | /// <param name="channel">The channel to which events should be published.</param> |
| | 779 | | /// <param name="publishingCancellationSource">A cancellation source which can be used for signaling publication |
| | 780 | | /// |
| | 781 | | /// <returns>A function to invoke when publishing should stop; once complete, background publishing is no longer |
| | 782 | | /// |
| | 783 | | /// <remarks> |
| | 784 | | /// This method assumes co-ownership of the <paramref name="channel" />, marking its writer as completed |
| | 785 | | /// when publishing is complete or when an exception is encountered. |
| | 786 | | /// </remarks> |
| | 787 | | /// |
| | 788 | | private async Task<Func<Task>> PublishPartitionEventsToChannelAsync(string partitionId, |
| | 789 | | EventPosition startingPosition, |
| | 790 | | bool trackLastEnqueuedEventProperties, |
| | 791 | | int receiveBatchSize, |
| | 792 | | long? ownerLevel, |
| | 793 | | uint prefetchCount, |
| | 794 | | Channel<PartitionEvent> channel, |
| | 795 | | CancellationTokenSource publishingCancellati |
| | 796 | | { |
| 104 | 797 | | publishingCancellationSource.Token.ThrowIfCancellationRequested<TaskCanceledException>(); |
| 104 | 798 | | EventHubsEventSource.Log.PublishPartitionEventsToChannelStart(EventHubName, partitionId); |
| | 799 | |
|
| 104 | 800 | | var transportConsumer = default(TransportConsumer); |
| 104 | 801 | | var publishingTask = default(Task); |
| 104 | 802 | | var observedException = default(Exception); |
| 104 | 803 | | var publisherId = Guid.NewGuid().ToString(); |
| | 804 | |
|
| | 805 | | // Termination must take place in multiple places due to the a catch block being |
| | 806 | | // disallowed with the use of "yield return". Create a local function that encapsulates |
| | 807 | | // the cleanup tasks needed. |
| | 808 | |
|
| | 809 | | async Task performCleanup() |
| | 810 | | { |
| 104 | 811 | | publishingCancellationSource?.Cancel(); |
| | 812 | |
|
| 104 | 813 | | if (publishingTask != null) |
| | 814 | | { |
| | 815 | | try |
| | 816 | | { |
| 104 | 817 | | await publishingTask.ConfigureAwait(false); |
| 84 | 818 | | channel.Writer.TryComplete(); |
| 84 | 819 | | } |
| 20 | 820 | | catch (Exception ex) when ((ex is TaskCanceledException) || (ex is ChannelClosedException)) |
| | 821 | | { |
| | 822 | | // Due to the non-determinism when requesting cancellation of the background |
| | 823 | | // publishing, it may surface as the expected cancellation or may result in |
| | 824 | | // an attempt to write to the shared channel after another publisher has |
| | 825 | | // marked it as final. |
| | 826 | | // |
| | 827 | | // These are expected scenarios; no action is needed. |
| 20 | 828 | | } |
| | 829 | | } |
| | 830 | |
|
| 104 | 831 | | if (transportConsumer != null) |
| | 832 | | { |
| 104 | 833 | | ActiveConsumers.TryRemove(publisherId, out var _); |
| 104 | 834 | | await transportConsumer.CloseAsync(CancellationToken.None).ConfigureAwait(false); |
| | 835 | | } |
| | 836 | |
|
| | 837 | | try |
| | 838 | | { |
| 104 | 839 | | if (observedException != default) |
| | 840 | | { |
| 34 | 841 | | EventHubsEventSource.Log.PublishPartitionEventsToChannelError(EventHubName, partitionId, observe |
| 34 | 842 | | ExceptionDispatchInfo.Capture(observedException).Throw(); |
| | 843 | | } |
| 70 | 844 | | } |
| | 845 | | finally |
| | 846 | | { |
| 104 | 847 | | EventHubsEventSource.Log.PublishPartitionEventsToChannelComplete(EventHubName, partitionId); |
| | 848 | | } |
| 70 | 849 | | } |
| | 850 | |
|
| | 851 | | // Setup the publishing context and begin publishing to the channel in the background. |
| | 852 | |
|
| | 853 | | try |
| | 854 | | { |
| 104 | 855 | | transportConsumer = Connection.CreateTransportConsumer(ConsumerGroup, partitionId, startingPosition, Ret |
| | 856 | |
|
| 104 | 857 | | if (!ActiveConsumers.TryAdd(publisherId, transportConsumer)) |
| | 858 | | { |
| 0 | 859 | | await transportConsumer.CloseAsync(CancellationToken.None).ConfigureAwait(false); |
| 0 | 860 | | transportConsumer = null; |
| 0 | 861 | | throw new EventHubsException(false, EventHubName, string.Format(CultureInfo.CurrentCulture, Resource |
| | 862 | | } |
| | 863 | |
|
| | 864 | | void exceptionCallback(Exception ex) |
| | 865 | | { |
| | 866 | | // Ignore the known exception cases that present during cancellation across |
| | 867 | | // background publishing for multiple partitions. |
| | 868 | |
|
| 48 | 869 | | if ((ex is ChannelClosedException) || (ex is TaskCanceledException)) |
| | 870 | | { |
| 14 | 871 | | return; |
| | 872 | | } |
| | 873 | |
|
| 34 | 874 | | observedException = ex; |
| 34 | 875 | | } |
| | 876 | |
|
| 104 | 877 | | publishingTask = StartBackgroundChannelPublishingAsync |
| 104 | 878 | | ( |
| 104 | 879 | | transportConsumer, |
| 104 | 880 | | channel, |
| 104 | 881 | | new PartitionContext(partitionId, transportConsumer), |
| 104 | 882 | | receiveBatchSize, |
| 104 | 883 | | exceptionCallback, |
| 104 | 884 | | publishingCancellationSource.Token |
| 104 | 885 | | ); |
| 104 | 886 | | } |
| 0 | 887 | | catch (Exception ex) |
| | 888 | | { |
| 0 | 889 | | EventHubsEventSource.Log.PublishPartitionEventsToChannelError(EventHubName, partitionId, ex.Message); |
| 0 | 890 | | await performCleanup().ConfigureAwait(false); |
| | 891 | |
|
| 0 | 892 | | throw; |
| | 893 | | } |
| | 894 | |
|
| 104 | 895 | | return performCleanup; |
| 104 | 896 | | } |
| | 897 | |
|
| | 898 | | /// <summary> |
| | 899 | | /// Begins the background process responsible for receiving from the specified <see cref="TransportConsumer" / |
| | 900 | | /// and publishing to the requested <see cref="Channel{PartitionEvent}" />. |
| | 901 | | /// </summary> |
| | 902 | | /// |
| | 903 | | /// <param name="transportConsumer">The consumer to use for receiving events.</param> |
| | 904 | | /// <param name="channel">The channel to which received events should be published.</param> |
| | 905 | | /// <param name="partitionContext">The context that represents the partition from which events being received.</ |
| | 906 | | /// <param name="receiveBatchSize">The batch size to use when receiving events.</param> |
| | 907 | | /// <param name="notifyException">An action to be invoked when an exception is encountered during publishing.</p |
| | 908 | | /// <param name="cancellationToken">The <see cref="CancellationToken"/> to signal the request to cancel the back |
| | 909 | | /// |
| | 910 | | /// <returns>A task to be resolved on when the operation has completed.</returns> |
| | 911 | | /// |
| | 912 | | private Task StartBackgroundChannelPublishingAsync(TransportConsumer transportConsumer, |
| | 913 | | Channel<PartitionEvent> channel, |
| | 914 | | PartitionContext partitionContext, |
| | 915 | | int receiveBatchSize, |
| | 916 | | Action<Exception> notifyException, |
| | 917 | | CancellationToken cancellationToken) => |
| 108 | 918 | | Task.Run(async () => |
| 108 | 919 | | { |
| 198 | 920 | | var failedAttemptCount = 0; |
| 198 | 921 | | var writtenItems = 0; |
| 198 | 922 | | var receivedItems = default(IReadOnlyList<EventData>); |
| 198 | 923 | | var retryDelay = default(TimeSpan?); |
| 198 | 924 | | var activeException = default(Exception); |
| 108 | 925 | |
|
| 51802 | 926 | | while (!cancellationToken.IsCancellationRequested) |
| 108 | 927 | | { |
| 108 | 928 | | try |
| 108 | 929 | | { |
| 108 | 930 | | // Receive items in batches and then write them to the subscribed channels. The channels will n |
| 108 | 931 | | // block if they reach their maximum queue size, so there is no need to throttle publishing. |
| 108 | 932 | |
|
| 51768 | 933 | | if (receivedItems == default) |
| 108 | 934 | | { |
| 51762 | 935 | | receivedItems = await transportConsumer.ReceiveAsync(receiveBatchSize, BackgroundPublishingW |
| 108 | 936 | | } |
| 108 | 937 | |
|
| 118792 | 938 | | foreach (EventData item in receivedItems) |
| 108 | 939 | | { |
| 7858 | 940 | | await channel.Writer.WriteAsync(new PartitionEvent(partitionContext, item), cancellationToke |
| 7842 | 941 | | ++writtenItems; |
| 108 | 942 | | } |
| 108 | 943 | |
|
| 51692 | 944 | | receivedItems = default; |
| 51692 | 945 | | writtenItems = 0; |
| 51692 | 946 | | failedAttemptCount = 0; |
| 51692 | 947 | | } |
| 116 | 948 | | catch (TaskCanceledException ex) |
| 108 | 949 | | { |
| 108 | 950 | | // In the case that a task was canceled, if cancellation was requested, then publishing should |
| 108 | 951 | | // is already terminating. Otherwise, something unexpected canceled the operation and it should |
| 108 | 952 | | // be treated as an exception to ensure that the channels are marked final and consumers are mad |
| 108 | 953 | | // aware. |
| 108 | 954 | |
|
| 116 | 955 | | activeException = (cancellationToken.IsCancellationRequested) ? null : ex; |
| 116 | 956 | | break; |
| 108 | 957 | | } |
| 114 | 958 | | catch (OperationCanceledException ex) |
| 108 | 959 | | { |
| 114 | 960 | | activeException = new TaskCanceledException(ex.Message, ex); |
| 114 | 961 | | break; |
| 108 | 962 | | } |
| 132 | 963 | | catch (EventHubsException ex) when |
| 132 | 964 | | (ex.Reason == EventHubsException.FailureReason.ConsumerDisconnected |
| 132 | 965 | | || ex.Reason == EventHubsException.FailureReason.ClientClosed) |
| 108 | 966 | | { |
| 108 | 967 | | // If the consumer was disconnected or closed, it is known to be unrecoverable; do not offer the |
| 108 | 968 | |
|
| 0 | 969 | | activeException = ex; |
| 0 | 970 | | break; |
| 108 | 971 | | } |
| 170 | 972 | | catch (Exception ex) when (ex.IsFatalException()) |
| 108 | 973 | | { |
| 0 | 974 | | channel.Writer.TryComplete(ex); |
| 0 | 975 | | throw; |
| 108 | 976 | | } |
| 170 | 977 | | catch (Exception ex) |
| 108 | 978 | | { |
| 108 | 979 | | // Determine if there should be a retry for the next attempt; if so enforce the delay but do not |
| 108 | 980 | | // Otherwise, mark the exception as active and break out of the loop. |
| 108 | 981 | |
|
| 170 | 982 | | ++failedAttemptCount; |
| 170 | 983 | | retryDelay = RetryPolicy.CalculateRetryDelay(ex, failedAttemptCount); |
| 108 | 984 | |
|
| 170 | 985 | | if (retryDelay.HasValue) |
| 108 | 986 | | { |
| 108 | 987 | | // If items were being emitted at the time of the exception, skip to the |
| 108 | 988 | | // last active item that was published to the channel so that publishing |
| 108 | 989 | | // resumes at the next event in sequence and duplicates are not written. |
| 108 | 990 | |
|
| 130 | 991 | | if ((receivedItems != default) && (writtenItems > 0)) |
| 108 | 992 | | { |
| 114 | 993 | | receivedItems = receivedItems.Skip(writtenItems).ToList(); |
| 108 | 994 | | } |
| 108 | 995 | |
|
| 130 | 996 | | await Task.Delay(retryDelay.Value, cancellationToken).ConfigureAwait(false); |
| 128 | 997 | | activeException = null; |
| 108 | 998 | | } |
| 108 | 999 | | else |
| 108 | 1000 | | { |
| 148 | 1001 | | activeException = ex; |
| 148 | 1002 | | break; |
| 108 | 1003 | | } |
| 108 | 1004 | | } |
| 108 | 1005 | | } |
| 108 | 1006 | |
|
| 108 | 1007 | | // Publishing has terminated; if there was an active exception, then take the necessary steps to mark pu |
| 108 | 1008 | | // than completed normally. |
| 108 | 1009 | |
|
| 196 | 1010 | | if (activeException != null) |
| 108 | 1011 | | { |
| 158 | 1012 | | channel.Writer.TryComplete(activeException); |
| 158 | 1013 | | notifyException(activeException); |
| 108 | 1014 | | } |
| 108 | 1015 | |
|
| 196 | 1016 | | }, cancellationToken); |
| | 1017 | |
|
| | 1018 | | /// <summary> |
| | 1019 | | /// Creates a channel for publishing events to local subscribers. |
| | 1020 | | /// </summary> |
| | 1021 | | /// |
| | 1022 | | /// <param name="capacity">The maximum amount of events that can be queued in the channel.</param> |
| | 1023 | | /// |
| | 1024 | | /// <returns>A bounded channel, configured for 1:many read/write usage.</returns> |
| | 1025 | | /// |
| | 1026 | | private Channel<PartitionEvent> CreateEventChannel(int capacity) => |
| 52 | 1027 | | Channel.CreateBounded<PartitionEvent>(new BoundedChannelOptions(capacity) |
| 52 | 1028 | | { |
| 52 | 1029 | | FullMode = BoundedChannelFullMode.Wait, |
| 52 | 1030 | | SingleWriter = false, |
| 52 | 1031 | | SingleReader = true |
| 52 | 1032 | | }); |
| | 1033 | | } |
| | 1034 | | } |