| | 1 | | // Copyright (c) Microsoft Corporation. All rights reserved. |
| | 2 | | // Licensed under the MIT License. |
| | 3 | |
|
| | 4 | | using System; |
| | 5 | | using System.Collections.Generic; |
| | 6 | | using System.Text.Json; |
| | 7 | | using System.Threading; |
| | 8 | | using System.Threading.Tasks; |
| | 9 | | using Azure.Storage.Blobs.Models; |
| | 10 | |
|
| | 11 | | namespace Azure.Storage.Blobs.ChangeFeed |
| | 12 | | { |
| | 13 | | internal class ChangeFeedFactory |
| | 14 | | { |
| | 15 | | private readonly SegmentFactory _segmentFactory; |
| | 16 | | private readonly BlobContainerClient _containerClient; |
| | 17 | |
|
| 76 | 18 | | public ChangeFeedFactory( |
| 76 | 19 | | BlobServiceClient blobServiceClient) |
| | 20 | | { |
| 76 | 21 | | _containerClient = blobServiceClient.GetBlobContainerClient(Constants.ChangeFeed.ChangeFeedContainerName); |
| 76 | 22 | | _segmentFactory = new SegmentFactory( |
| 76 | 23 | | _containerClient, |
| 76 | 24 | | new ShardFactory( |
| 76 | 25 | | _containerClient, |
| 76 | 26 | | new ChunkFactory( |
| 76 | 27 | | _containerClient, |
| 76 | 28 | | new LazyLoadingBlobStreamFactory(), |
| 76 | 29 | | new AvroReaderFactory()))); |
| 76 | 30 | | } |
| | 31 | |
|
| 20 | 32 | | public ChangeFeedFactory( |
| 20 | 33 | | BlobContainerClient containerClient, |
| 20 | 34 | | SegmentFactory segmentFactory) |
| | 35 | | { |
| 20 | 36 | | _containerClient = containerClient; |
| 20 | 37 | | _segmentFactory = segmentFactory; |
| 20 | 38 | | } |
| | 39 | |
|
| | 40 | | public async Task<ChangeFeed> BuildChangeFeed( |
| | 41 | | DateTimeOffset? startTime, |
| | 42 | | DateTimeOffset? endTime, |
| | 43 | | string continuation, |
| | 44 | | bool async, |
| | 45 | | CancellationToken cancellationToken) |
| | 46 | | { |
| | 47 | | DateTimeOffset lastConsumable; |
| 92 | 48 | | Queue<string> years = new Queue<string>(); |
| 92 | 49 | | Queue<string> segments = new Queue<string>(); |
| 92 | 50 | | ChangeFeedCursor cursor = null; |
| | 51 | |
|
| | 52 | | // Create cursor |
| 92 | 53 | | if (continuation != null) |
| | 54 | | { |
| 36 | 55 | | cursor = JsonSerializer.Deserialize<ChangeFeedCursor>(continuation); |
| 36 | 56 | | ValidateCursor(_containerClient, cursor); |
| 36 | 57 | | startTime = BlobChangeFeedExtensions.ToDateTimeOffset(cursor.CurrentSegmentCursor.SegmentPath).Value; |
| 36 | 58 | | endTime = cursor.EndTime; |
| | 59 | | } |
| | 60 | | // Round start and end time if we are not using the cursor. |
| | 61 | | else |
| | 62 | | { |
| 56 | 63 | | startTime = startTime.RoundDownToNearestHour(); |
| 56 | 64 | | endTime = endTime.RoundUpToNearestHour(); |
| | 65 | | } |
| | 66 | |
|
| | 67 | | // Check if Change Feed has been abled for this account. |
| | 68 | | bool changeFeedContainerExists; |
| | 69 | |
|
| 92 | 70 | | if (async) |
| | 71 | | { |
| 84 | 72 | | changeFeedContainerExists = await _containerClient.ExistsAsync(cancellationToken: cancellationToken).Con |
| | 73 | | } |
| | 74 | | else |
| | 75 | | { |
| 8 | 76 | | changeFeedContainerExists = _containerClient.Exists(cancellationToken: cancellationToken); |
| | 77 | | } |
| | 78 | |
|
| 92 | 79 | | if (!changeFeedContainerExists) |
| | 80 | | { |
| 0 | 81 | | throw new ArgumentException("Change Feed hasn't been enabled on this account, or is currently being enab |
| | 82 | | } |
| | 83 | |
|
| | 84 | | // Get last consumable |
| 92 | 85 | | BlobClient blobClient = _containerClient.GetBlobClient(Constants.ChangeFeed.MetaSegmentsPath); |
| | 86 | | BlobDownloadInfo blobDownloadInfo; |
| 92 | 87 | | if (async) |
| | 88 | | { |
| 84 | 89 | | blobDownloadInfo = await blobClient.DownloadAsync(cancellationToken: cancellationToken).ConfigureAwait(f |
| | 90 | | } |
| | 91 | | else |
| | 92 | | { |
| 8 | 93 | | blobDownloadInfo = blobClient.Download(cancellationToken: cancellationToken); |
| | 94 | | } |
| | 95 | |
|
| | 96 | | JsonDocument jsonMetaSegment; |
| 92 | 97 | | if (async) |
| | 98 | | { |
| 84 | 99 | | jsonMetaSegment = await JsonDocument.ParseAsync( |
| 84 | 100 | | blobDownloadInfo.Content, |
| 84 | 101 | | cancellationToken: cancellationToken |
| 84 | 102 | | ).ConfigureAwait(false); |
| | 103 | | } |
| | 104 | | else |
| | 105 | | { |
| 8 | 106 | | jsonMetaSegment = JsonDocument.Parse(blobDownloadInfo.Content); |
| | 107 | | } |
| | 108 | |
|
| 92 | 109 | | lastConsumable = jsonMetaSegment.RootElement.GetProperty("lastConsumable").GetDateTimeOffset(); |
| | 110 | |
|
| | 111 | | // Get year paths |
| 92 | 112 | | years = await GetYearPathsInternal( |
| 92 | 113 | | async, |
| 92 | 114 | | cancellationToken).ConfigureAwait(false); |
| | 115 | |
|
| | 116 | | // Dequeue any years that occur before start time |
| 92 | 117 | | if (startTime.HasValue) |
| | 118 | | { |
| 108 | 119 | | while (years.Count > 0 |
| 108 | 120 | | && BlobChangeFeedExtensions.ToDateTimeOffset(years.Peek()) < startTime.RoundDownToNearestYear()) |
| | 121 | | { |
| 20 | 122 | | years.Dequeue(); |
| | 123 | | } |
| | 124 | | } |
| | 125 | |
|
| | 126 | | // There are no years. |
| 92 | 127 | | if (years.Count == 0) |
| | 128 | | { |
| 4 | 129 | | return ChangeFeed.Empty(); |
| | 130 | | } |
| | 131 | |
|
| 180 | 132 | | while (segments.Count == 0 && years.Count > 0) |
| | 133 | | { |
| | 134 | | // Get Segments for year |
| 92 | 135 | | segments = await BlobChangeFeedExtensions.GetSegmentsInYearInternal( |
| 92 | 136 | | containerClient: _containerClient, |
| 92 | 137 | | yearPath: years.Dequeue(), |
| 92 | 138 | | startTime: startTime, |
| 92 | 139 | | endTime: BlobChangeFeedExtensions.MinDateTime(lastConsumable, endTime), |
| 92 | 140 | | async: async, |
| 92 | 141 | | cancellationToken: cancellationToken) |
| 92 | 142 | | .ConfigureAwait(false); |
| | 143 | | } |
| | 144 | |
|
| | 145 | | // We were on the last year, and there were no more segments. |
| 88 | 146 | | if (segments.Count == 0) |
| | 147 | | { |
| 0 | 148 | | return ChangeFeed.Empty(); |
| | 149 | | } |
| | 150 | |
|
| 88 | 151 | | Segment currentSegment = await _segmentFactory.BuildSegment( |
| 88 | 152 | | async, |
| 88 | 153 | | segments.Dequeue(), |
| 88 | 154 | | cursor?.CurrentSegmentCursor) |
| 88 | 155 | | .ConfigureAwait(false); |
| | 156 | |
|
| 88 | 157 | | return new ChangeFeed( |
| 88 | 158 | | _containerClient, |
| 88 | 159 | | _segmentFactory, |
| 88 | 160 | | years, |
| 88 | 161 | | segments, |
| 88 | 162 | | currentSegment, |
| 88 | 163 | | lastConsumable, |
| 88 | 164 | | startTime, |
| 88 | 165 | | endTime); |
| 92 | 166 | | } |
| | 167 | |
|
| | 168 | | private static void ValidateCursor( |
| | 169 | | BlobContainerClient containerClient, |
| | 170 | | ChangeFeedCursor cursor) |
| | 171 | | { |
| 36 | 172 | | if (BlobChangeFeedExtensions.ComputeMD5(containerClient.Uri.AbsoluteUri) != cursor.UrlHash) |
| | 173 | | { |
| 0 | 174 | | throw new ArgumentException("Cursor URL does not match container URL."); |
| | 175 | | } |
| 36 | 176 | | if (cursor.CursorVersion != 1) |
| | 177 | | { |
| 0 | 178 | | throw new ArgumentException("Unsupported cursor version."); |
| | 179 | | } |
| 36 | 180 | | } |
| | 181 | |
|
| | 182 | | internal async Task<Queue<string>> GetYearPathsInternal( |
| | 183 | | bool async, |
| | 184 | | CancellationToken cancellationToken) |
| | 185 | | { |
| 96 | 186 | | List<string> list = new List<string>(); |
| | 187 | |
|
| 96 | 188 | | if (async) |
| | 189 | | { |
| 560 | 190 | | await foreach (BlobHierarchyItem blobHierarchyItem in _containerClient.GetBlobsByHierarchyAsync( |
| 86 | 191 | | prefix: Constants.ChangeFeed.SegmentPrefix, |
| 86 | 192 | | delimiter: "/", |
| 86 | 193 | | cancellationToken: cancellationToken).ConfigureAwait(false)) |
| | 194 | | { |
| 194 | 195 | | if (blobHierarchyItem.Prefix.Contains(Constants.ChangeFeed.InitalizationSegment)) |
| | 196 | | continue; |
| | 197 | |
|
| 108 | 198 | | list.Add(blobHierarchyItem.Prefix); |
| | 199 | | } |
| | 200 | | } |
| | 201 | | else |
| | 202 | | { |
| 104 | 203 | | foreach (BlobHierarchyItem blobHierarchyItem in _containerClient.GetBlobsByHierarchy( |
| 10 | 204 | | prefix: Constants.ChangeFeed.SegmentPrefix, |
| 10 | 205 | | delimiter: "/", |
| 10 | 206 | | cancellationToken: cancellationToken)) |
| | 207 | | { |
| 42 | 208 | | if (blobHierarchyItem.Prefix.Contains(Constants.ChangeFeed.InitalizationSegment)) |
| | 209 | | continue; |
| | 210 | |
|
| 32 | 211 | | list.Add(blobHierarchyItem.Prefix); |
| | 212 | | } |
| | 213 | | } |
| 96 | 214 | | return new Queue<string>(list); |
| 96 | 215 | | } |
| | 216 | | } |
| | 217 | | } |