| | | 1 | | // Copyright (c) Microsoft and contributors. All rights reserved. |
| | | 2 | | // |
| | | 3 | | // Licensed under the Apache License, Version 2.0 (the "License"); |
| | | 4 | | // you may not use this file except in compliance with the License. |
| | | 5 | | // You may obtain a copy of the License at |
| | | 6 | | // http://www.apache.org/licenses/LICENSE-2.0 |
| | | 7 | | // |
| | | 8 | | // Unless required by applicable law or agreed to in writing, software |
| | | 9 | | // distributed under the License is distributed on an "AS IS" BASIS, |
| | | 10 | | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| | | 11 | | // |
| | | 12 | | // See the License for the specific language governing permissions and |
| | | 13 | | // limitations under the License. |
| | | 14 | | |
| | | 15 | | using Microsoft.Azure.Batch.Conventions.Files.Utilities; |
| | | 16 | | using Microsoft.WindowsAzure.Storage; |
| | | 17 | | using Microsoft.WindowsAzure.Storage.Blob; |
| | | 18 | | using System; |
| | | 19 | | using System.Collections.Generic; |
| | | 20 | | using System.Linq; |
| | | 21 | | using System.Text; |
| | | 22 | | using System.Threading; |
| | | 23 | | using System.Threading.Tasks; |
| | | 24 | | using static Microsoft.WindowsAzure.Storage.Blob.SharedAccessBlobPermissions; |
| | | 25 | | |
| | | 26 | | namespace Microsoft.Azure.Batch.Conventions.Files |
| | | 27 | | { |
| | | 28 | | /// <summary> |
| | | 29 | | /// Provides methods for working with the outputs of a <see cref="CloudJob"/>. |
| | | 30 | | /// </summary> |
| | | 31 | | public static class CloudJobExtensions |
| | | 32 | | { |
| | 0 | 33 | | private static readonly TimeSpan DefaultSasExpiry = TimeSpan.FromDays(7); |
| | | 34 | | |
| | | 35 | | /// <summary> |
| | | 36 | | /// Gets the <see cref="JobOutputStorage"/> for a specified <see cref="CloudJob"/>. |
| | | 37 | | /// </summary> |
| | | 38 | | /// <param name="job">The job for which to get output storage.</param> |
| | | 39 | | /// <param name="storageAccount">The storage account linked to the Azure Batch account.</param> |
| | | 40 | | /// <returns>A JobOutputStorage for the specified job.</returns> |
| | | 41 | | public static JobOutputStorage OutputStorage(this CloudJob job, CloudStorageAccount storageAccount) |
| | | 42 | | { |
| | 2 | 43 | | if (job == null) |
| | | 44 | | { |
| | 1 | 45 | | throw new ArgumentNullException(nameof(job)); |
| | | 46 | | } |
| | 1 | 47 | | if (storageAccount == null) |
| | | 48 | | { |
| | 1 | 49 | | throw new ArgumentNullException(nameof(storageAccount)); |
| | | 50 | | } |
| | | 51 | | |
| | 0 | 52 | | return new JobOutputStorage(storageAccount, job.Id); |
| | | 53 | | } |
| | | 54 | | |
| | | 55 | | /// <summary> |
| | | 56 | | /// Creates an Azure blob storage container for the outputs of a <see cref="CloudJob"/>. |
| | | 57 | | /// </summary> |
| | | 58 | | /// <param name="job">The job for which to create the container.</param> |
| | | 59 | | /// <param name="storageAccount">The storage account linked to the Azure Batch account.</param> |
| | | 60 | | /// <param name="cancellationToken">A <see cref="CancellationToken"/> for controlling the lifetime of the asynch |
| | | 61 | | /// <returns>A <see cref="Task"/> that represents the asynchronous operation.</returns> |
| | | 62 | | public static async Task PrepareOutputStorageAsync(this CloudJob job, CloudStorageAccount storageAccount, Cancel |
| | | 63 | | { |
| | 1 | 64 | | if (job == null) |
| | | 65 | | { |
| | 1 | 66 | | throw new ArgumentNullException(nameof(job)); |
| | | 67 | | } |
| | 0 | 68 | | if (storageAccount == null) |
| | | 69 | | { |
| | 0 | 70 | | throw new ArgumentNullException(nameof(storageAccount)); |
| | | 71 | | } |
| | | 72 | | |
| | 0 | 73 | | var jobOutputContainerName = ContainerNameUtils.GetSafeContainerName(job.Id); |
| | 0 | 74 | | var jobOutputContainer = storageAccount.CreateCloudBlobClient().GetContainerReference(jobOutputContainerName |
| | | 75 | | |
| | 0 | 76 | | await jobOutputContainer.CreateIfNotExistsAsync(BlobContainerPublicAccessType.Off, null, null, cancellationT |
| | 0 | 77 | | } |
| | | 78 | | |
| | | 79 | | /// <summary> |
| | | 80 | | /// Gets the URL, including a Shared Access Signature (SAS) that permits writing, for the job's |
| | | 81 | | /// output storage container in Azure blob storage. This URL is suitable for passing to tasks |
| | | 82 | | /// so they can use the <see cref="JobOutputStorage.JobOutputStorage(Uri)"/> or |
| | | 83 | | /// <see cref="TaskOutputStorage.TaskOutputStorage(Uri, string)"/> constructors that take a <see cref="Uri"/>. |
| | | 84 | | /// </summary> |
| | | 85 | | /// <param name="job">The job for which to create the container.</param> |
| | | 86 | | /// <param name="storageAccount">The storage account linked to the Azure Batch account.</param> |
| | | 87 | | /// <returns>The URL, including SAS, of the job output container.</returns> |
| | | 88 | | /// <remarks>The SAS expires after 7 days. This default is chosen to match the maximum time that |
| | | 89 | | /// tasks can remain active.</remarks> |
| | | 90 | | public static string GetOutputStorageContainerUrl(this CloudJob job, CloudStorageAccount storageAccount) |
| | | 91 | | { |
| | 0 | 92 | | return GetOutputStorageContainerUrl(job, storageAccount, DefaultSasExpiry); |
| | | 93 | | } |
| | | 94 | | |
| | | 95 | | /// <summary> |
| | | 96 | | /// Gets the URL, including a Shared Access Signature (SAS) that permits writing, for the job's |
| | | 97 | | /// output storage container in Azure blob storage. This URL is suitable for passing to tasks |
| | | 98 | | /// so they can use the <see cref="JobOutputStorage.JobOutputStorage(Uri)"/> or |
| | | 99 | | /// <see cref="TaskOutputStorage.TaskOutputStorage(Uri, string)"/> constructors that take a <see cref="Uri"/>. |
| | | 100 | | /// </summary> |
| | | 101 | | /// <param name="job">The job for which to create the container.</param> |
| | | 102 | | /// <param name="storageAccount">The storage account linked to the Azure Batch account.</param> |
| | | 103 | | /// <param name="expiryTime">The duration for which the SAS is valid. This should be long enough |
| | | 104 | | /// to allow all tasks of the job to be created and run to completion, including leeway for errors |
| | | 105 | | /// and retries.</param> |
| | | 106 | | /// <returns>The URL, including SAS, of the job output container.</returns> |
| | | 107 | | public static string GetOutputStorageContainerUrl(this CloudJob job, CloudStorageAccount storageAccount, TimeSpa |
| | | 108 | | { |
| | 4 | 109 | | if (job == null) |
| | | 110 | | { |
| | 1 | 111 | | throw new ArgumentNullException(nameof(job)); |
| | | 112 | | } |
| | 3 | 113 | | if (storageAccount == null) |
| | | 114 | | { |
| | 1 | 115 | | throw new ArgumentNullException(nameof(storageAccount)); |
| | | 116 | | } |
| | 2 | 117 | | if (expiryTime <= TimeSpan.Zero) |
| | | 118 | | { |
| | 2 | 119 | | throw new ArgumentException("Shared access signature expiry time must be greater than zero", nameof(expi |
| | | 120 | | } |
| | | 121 | | |
| | 0 | 122 | | var jobOutputContainerName = ContainerNameUtils.GetSafeContainerName(job.Id); |
| | 0 | 123 | | var container = storageAccount.CreateCloudBlobClient().GetContainerReference(jobOutputContainerName); |
| | 0 | 124 | | var accessPolicy = CreateWriteAccessPolicy(expiryTime); |
| | 0 | 125 | | var containerSas = container.GetSharedAccessSignature(accessPolicy); |
| | 0 | 126 | | var containerUrl = container.Uri.AbsoluteUri + containerSas; |
| | | 127 | | |
| | 0 | 128 | | return containerUrl; |
| | | 129 | | } |
| | | 130 | | |
| | | 131 | | /// <summary> |
| | | 132 | | /// Gets the name of the Azure blob storage container for the outputs of a <see cref="CloudJob"/>. |
| | | 133 | | /// </summary> |
| | | 134 | | /// <param name="job">The job for which to get the container name.</param> |
| | | 135 | | /// <returns>The name of the container in which to save the outputs of this job.</returns> |
| | | 136 | | public static string OutputStorageContainerName(this CloudJob job) |
| | | 137 | | { |
| | 101 | 138 | | if (job == null) |
| | | 139 | | { |
| | 1 | 140 | | throw new ArgumentNullException(nameof(job)); |
| | | 141 | | } |
| | | 142 | | |
| | 100 | 143 | | var jobOutputContainerName = ContainerNameUtils.GetSafeContainerName(job.Id); |
| | | 144 | | |
| | 100 | 145 | | return jobOutputContainerName; |
| | | 146 | | } |
| | | 147 | | |
| | | 148 | | /// <summary> |
| | | 149 | | /// Gets the Blob name prefix/folder where files of the given kind are stored |
| | | 150 | | /// </summary> |
| | | 151 | | /// <param name="job">The job to calculate the output storage destination for.</param> |
| | | 152 | | /// <param name="kind">The output kind.</param> |
| | | 153 | | /// <returns>The Blob name prefix/folder where files of the given kind are stored.</returns> |
| | | 154 | | public static string GetOutputStoragePath(this CloudJob job, JobOutputKind kind) |
| | 3 | 155 | | => StoragePath.JobStoragePath.BlobNamePrefixImpl(kind); |
| | | 156 | | |
| | | 157 | | private static SharedAccessBlobPolicy CreateWriteAccessPolicy(TimeSpan expiryTime) |
| | | 158 | | { |
| | 0 | 159 | | return new SharedAccessBlobPolicy |
| | 0 | 160 | | { |
| | 0 | 161 | | Permissions = Add | Create | List | Read | Write, |
| | 0 | 162 | | SharedAccessExpiryTime = DateTime.UtcNow.Add(expiryTime), |
| | 0 | 163 | | }; |
| | | 164 | | } |
| | | 165 | | } |
| | | 166 | | } |