public async Task <GrainStateRecord> Read(string partitionKey, string rowKey)
{
if (logger.IsVerbose3)
{
logger.Verbose3((int)AzureProviderErrorCode.AzureTableProvider_Storage_Reading, "Reading: PartitionKey={0} RowKey={1} from Table={2}", partitionKey, rowKey, TableName);
}
try
{
Tuple <GrainStateEntity, string> data = await tableManager.ReadSingleTableEntryAsync(partitionKey, rowKey);
if (data == null || data.Item1 == null)
{
if (logger.IsVerbose2)
{
logger.Verbose2((int)AzureProviderErrorCode.AzureTableProvider_DataNotFound, "DataNotFound reading: PartitionKey={0} RowKey={1} from Table={2}", partitionKey, rowKey, TableName);
}
return(null);
}
GrainStateEntity stateEntity = data.Item1;
var record = new GrainStateRecord {
Entity = stateEntity, ETag = data.Item2
};
if (logger.IsVerbose3)
{
logger.Verbose3((int)AzureProviderErrorCode.AzureTableProvider_Storage_DataRead, "Read: PartitionKey={0} RowKey={1} from Table={2} with ETag={3}", stateEntity.PartitionKey, stateEntity.RowKey, TableName, record.ETag);
}
return(record);
}
catch (Exception exc)
{
if (AzureStorageUtils.TableStorageDataNotFound(exc))
{
if (logger.IsVerbose2)
{
logger.Verbose2((int)AzureProviderErrorCode.AzureTableProvider_DataNotFound, "DataNotFound reading (exception): PartitionKey={0} RowKey={1} from Table={2} Exception={3}", partitionKey, rowKey, TableName, TraceLogger.PrintException(exc));
}
return(null); // No data
}
throw;
}
}
public string WriteLog(PayPalLogEntity.NotificationKind kind, string txnId, string logData)
{
if (String.IsNullOrEmpty(txnId))
{
throw new ArgumentException();
}
var rowKey = KeyUtils.GetCurrentTimeKey();
var entity = new PayPalLogEntity
{
PartitionKey = txnId,
RowKey = rowKey,
Kind = kind.ToString(),
LogData = logData,
};
AzureStorageUtils.InsertEntity(AzureStorageUtils.TableNames.PaymentLog, entity);
return(rowKey);
}
public async Task <IHttpActionResult> InboundWebhook(InboundWebhookFormModel form)
{
// The form is empty on a HEAD request
if (form != null)
{
var mandrill_events = form.mandrill_events;
var partitionKey = KeyUtils.GetCurrentTimeKey();
var entity = new ExternalSessionLogEntity
{
PartitionKey = partitionKey,
RowKey = "InboundWebhook",
Data = mandrill_events,
};
await AzureStorageUtils.InsertEntityAsync(AzureStorageUtils.TableNames.ExternalSessions, entity);
await SendSmsMessage("InboundWebhook " + partitionKey, partitionKey);
}
return(StatusCode(HttpStatusCode.NoContent));
}
public async Task <IHttpActionResult> GetAllReviewPieces(int exerciseId)
{
var filterPartition = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, ReviewPiece.GetPartitionKey(exerciseId));
var query = new TableQuery <ReviewPiece>().Where(filterPartition);
var allPieces = await AzureStorageUtils.ExecuteQueryAsync(AzureStorageUtils.TableNames.ReviewPieces, query);
// Enforce access rights. The exercise author cannot see review items in an unfinished review. An access entity is written when a review is finished. See ReviewsApiController.PostFinishReview
var userAccessCode = ReviewPiece.PieceTypes.Viewer + KeyUtils.IntToKey(this.GetUserId());
// Find the ReviewIds which are allowed to access.
var reviewIds = allPieces
.Where(i => ReviewPiece.GetUserAccessCode(i.RowKey) == userAccessCode)
.Select(i => ReviewPiece.GetReviewId(i.RowKey))
.ToList();
RemoveAccessEntries(allPieces);
// Filter the record set.
var accessablePieces = allPieces.Where(i => reviewIds.Contains(ReviewPiece.GetReviewId(i.RowKey)));
var piecesArr = accessablePieces.Select(i => i.Json).ToArray();
return(Ok(piecesArr));
}
public async Task <ActionResult> Claim(string id)
{
var userIdKey = KeyUtils.IntToKey(0);
var longTimeKey = id + this.GetExtId();
var blobName = ExerciseUtils.FormatBlobName(userIdKey, longTimeKey, "metadata", "json");
var blob = AzureStorageUtils.GetBlob(AzureStorageUtils.ContainerNames.Artifacts, blobName);
var metadataJson = await blob.DownloadTextAsync();
var metadata = JObject.Parse(metadataJson);
var serviceType = (string)metadata["serviceType"];
var cardId = (Guid?)metadata["cardId"];
var title = (string)metadata["title"];
var comment = (string)metadata["comment"];
var details = metadata["recordingDetails"];
var recordingDetails = details.ToObject <RecordingDetails>();
var exerciseId = await ExerciseUtils.CreateExercise(recordingDetails.BlobName, this.GetUserId(),
serviceType, ArtifactType.Mp3, recordingDetails.TotalDuration, title, cardId, comment, details.ToString(Formatting.None));
//~~ Redirect to the View exercise page.
return(RedirectToAction("View", new { Id = exerciseId }));
}
private async Task ExecuteQueryAsync(string processName, TableQuery <T> rangeQuery, Func <T, bool> filter,
Func <IEnumerable <T>, Task> yieldData)
{
try
{
TableContinuationToken tableContinuationToken = null;
var table = GetTable();
do
{
var queryResponse = await table.ExecuteQuerySegmentedAsync(rangeQuery, tableContinuationToken);
tableContinuationToken = queryResponse.ContinuationToken;
await yieldData(AzureStorageUtils.ApplyFilter(queryResponse.Results, filter));
} while (tableContinuationToken != null);
}
catch (Exception ex)
{
_log?.WriteFatalError("Table storage: " + _tableName, processName, rangeQuery.FilterString ?? "[null]",
ex).Wait();
throw;
}
}
public Task InsertAsync(IEnumerable <T> items)
{
items = items.ToArray();
try
{
if (items.Any())
{
var insertBatchOperation = new TableBatchOperation();
foreach (var item in items)
{
insertBatchOperation.Insert(item);
}
return(GetTable().ExecuteBatchAsync(insertBatchOperation));
}
}
catch (Exception ex)
{
_log?.WriteFatalError("Table storage: " + _tableName, "InsertAsync batch",
AzureStorageUtils.PrintItems(items), ex);
}
return(Task.CompletedTask);
}
public async Task AQ_Standalone_1()
{
queueName = "Test-1-".ToLower() + Guid.NewGuid();
AzureQueueDataManager manager = await GetTableManager(queueName);
Assert.AreEqual(0, await manager.GetApproximateMessageCount());
CloudQueueMessage inMessage = new CloudQueueMessage("Hello, World");
await manager.AddQueueMessage(inMessage);
//Nullable<int> count = manager.ApproximateMessageCount;
Assert.AreEqual(1, await manager.GetApproximateMessageCount());
CloudQueueMessage outMessage1 = await manager.PeekQueueMessage();
logger.Info("PeekQueueMessage 1: {0}", AzureStorageUtils.PrintCloudQueueMessage(outMessage1));
Assert.AreEqual(inMessage.AsString, outMessage1.AsString);
CloudQueueMessage outMessage2 = await manager.PeekQueueMessage();
logger.Info("PeekQueueMessage 2: {0}", AzureStorageUtils.PrintCloudQueueMessage(outMessage2));
Assert.AreEqual(inMessage.AsString, outMessage2.AsString);
CloudQueueMessage outMessage3 = await manager.GetQueueMessage();
logger.Info("GetQueueMessage 3: {0}", AzureStorageUtils.PrintCloudQueueMessage(outMessage3));
Assert.AreEqual(inMessage.AsString, outMessage3.AsString);
Assert.AreEqual(1, await manager.GetApproximateMessageCount());
CloudQueueMessage outMessage4 = await manager.GetQueueMessage();
Assert.IsNull(outMessage4);
Assert.AreEqual(1, await manager.GetApproximateMessageCount());
await manager.DeleteQueueMessage(outMessage3);
Assert.AreEqual(0, await manager.GetApproximateMessageCount());
}
public async Task <IHttpActionResult> DeletePiece(string partitionKey, string rowKey)
{
// Check access rights.
var reviewId = ReviewPiece.GetReviewId(rowKey);
var table = AzureStorageUtils.GetCloudTable(AzureStorageUtils.TableNames.ReviewPieces);
var userIsEditor = await UserIsEditor(partitionKey, reviewId, table);
if (userIsEditor)
{
var entity = new ReviewPiece()
{
PartitionKey = partitionKey,
RowKey = rowKey,
ETag = "*",
};
var deleteOperation = TableOperation.Delete(entity);
// If the piece is not yet saved, we will get "The remote server returned an error: (404) Not Found."
// The approved solution from MS is to try to retrieve the entity first. Catching exception is a hack which is appropriate for a single entity, it is not compatible with a Batch operation.
try
{
await table.ExecuteAsync(deleteOperation);
}
catch (StorageException ex)
{
if (ex.RequestInformation.HttpStatusCode != (int)HttpStatusCode.NotFound)
{
throw;
}
}
// Notify the exercise author in real-time.
var pieceType = ReviewPiece.GetType(rowKey);
var pieceId = ReviewPiece.GetPieceId(rowKey);
this.GetAuthorConnections(partitionKey).PieceDeleted(reviewId, pieceType, pieceId);
}
return(StatusCode(userIsEditor ? HttpStatusCode.NoContent : HttpStatusCode.BadRequest));
}
public async Task PublishAssetAsync(string file, string blobPath, PushOptions options, SemaphoreSlim clientThrottle = null)
{
using (await SemaphoreLock.LockAsync(clientThrottle))
{
blobPath = blobPath.Replace("\\", "/");
var blobClient = CreateBlobClient(blobPath);
if (!options.AllowOverwrite && await blobClient.ExistsAsync())
{
if (options.PassIfExistingItemIdentical)
{
if (!await blobClient.IsFileIdenticalToBlobAsync(file))
{
_log.LogError($"Asset '{file}' already exists with different contents at '{blobPath}'");
}
return;
}
_log.LogError($"Asset '{file}' already exists at '{blobPath}'");
return;
}
_log.LogMessage($"Uploading '{file}' to '{blobPath}'");
try
{
BlobUploadOptions blobUploadOptions = new()
{
HttpHeaders = AzureStorageUtils.GetBlobHeadersByExtension(file)
};
await blobClient.UploadAsync(file, blobUploadOptions);
}
catch (Exception e)
{
_log.LogError($"Unexpected exception publishing file {file} to {blobPath}: {e.Message}");
}
}
}
public void AzureTableErrorCode_ExtractRestErrorCode_ServerBusy()
{
string xml =
"<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\"?>" +
"<error xmlns=\"http://schemas.microsoft.com/ado/2007/08/dataservices/metadata\">" +
"<code>ServerBusy</code>" +
"<message xml:lang=\"en-US\">The server is busy." +
"RequestId:14ab4de6-fe80-4a45-a364-b0c2a27a4a86" +
"Time:2013-09-24T23:29:13.4913945Z</message>" +
"</error>";
var exc = new DataServiceClientException(xml);
var strCode = AzureStorageUtils.ExtractRestErrorCode(exc);
Assert.AreEqual(StorageErrorCodeStrings.ServerBusy, strCode);
var wrapped = new AggregateException(exc);
strCode = AzureStorageUtils.ExtractRestErrorCode(wrapped);
Assert.AreEqual(StorageErrorCodeStrings.ServerBusy, strCode);
//Assert.IsTrue(Async_AzureTableDataManager<SiloMetricsData>.IsRetriableHttpError((HttpStatusCode)500, "ServerBusy"));
}
// Extract text from the HTML page
private string FindText(string html, string startSnippet, string endSnippet)
{
var i1 = html.IndexOf(startSnippet);
var i2 = html.IndexOf(endSnippet, i1 + 1);
if ((i1 == -1) || (i2 == -1))
{
// Write the poison HTML page to the log.
var partitionKey = KeyUtils.GetCurrentTimeKey();
var entity = new ExternalSessionLogEntity
{
PartitionKey = partitionKey,
RowKey = "Error_FindText",
Data = html,
};
AzureStorageUtils.InsertEntity(AzureStorageUtils.TableNames.ExternalSessions, entity);
throw new FormatException("Parsing the external service data. Snippet not found. " + startSnippet.Length.ToString() + " " + endSnippet.Length.ToString());
}
var shift = startSnippet.Length;
return(html.Substring(i1 + shift, i2 - i1 - shift));
}
public async Task DeleteAsync(IEnumerable <T> items)
{
items = items.ToArray();
try
{
if (items.Any())
{
var deleteBatchOperation = new TableBatchOperation();
foreach (var item in items)
{
deleteBatchOperation.Delete(item);
}
var table = await GetTable();
await table.ExecuteBatchAsync(deleteBatchOperation);
}
}
catch (Exception ex)
{
_log?.WriteFatalErrorAsync("Table storage: " + _tableName, "DeleteAsync batch",
AzureStorageUtils.PrintItems(items), ex);
}
}
private async Task <DapperHelper.PageItems <ResourceDto> > GetHistoryResources(int userId, string secondaryFilter, int offset = 0, int limit = 0)
{
var result = new DapperHelper.PageItems <ResourceDto>();
var partitionKey = KeyUtils.IntToKey(userId);
var filterPartition = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionKey);
var combinedFilter = !String.IsNullOrEmpty(secondaryFilter) ? TableQuery.CombineFilters(filterPartition, TableOperators.And, secondaryFilter) : filterPartition;
var query = new TableQuery <LibraryHistoryEntity>().Where(combinedFilter);
var entities = await AzureStorageUtils.ExecuteQueryAsync(AzureStorageUtils.TableNames.LibraryHistory, query);
result.TotalCount = entities.Count();
if (entities.Any())
{
var historyItems = (limit != 0)
? entities.Skip(offset).Take(limit)
: entities;
result.Items = await HydrateHistoryItems(historyItems, userId);
}
return(result);
}
// GET: /history
public async Task <ActionResult> History()
{
// Send all the days there are records for. We will enable/disable days in the calendar on the page accordingly. RowKeys in the table are "inverted" local time.
var days = new List <string>();
if (this.IsAuthenticated())
{
var userId = this.GetUserId();
var partitionKey = KeyUtils.IntToKey(userId);
var filterPartition = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionKey);
var query = new TableQuery <TableEntity>().Where(filterPartition);
var table = AzureStorageUtils.GetCloudTable(AzureStorageUtils.TableNames.LibraryHistory);
TableQuerySegment <TableEntity> currentSegment = null;
while (currentSegment == null || currentSegment.ContinuationToken != null)
{
currentSegment = await table.ExecuteQuerySegmentedAsync <TableEntity>(
query,
currentSegment != null?currentSegment.ContinuationToken : null
);
// Format 2014-01-21 as "140121"
var localDays = currentSegment.Results
.GroupBy(i => i.RowKey.Substring(0, 6))
.Select(i => KeyUtils.InvertedKeyToLocalTime(i.Key, 3, "", "d2").Substring(2))
;
days.AddRange(localDays);
}
}
//var daysParam = days.Distinct();
//ViewBag.DaysParamJson = JsonUtils.SerializeAsJson(daysParam);
ViewBag.DaysParam = days.Distinct();
return(View());
}
internal async Task <string> UpsertRow(ReminderTableEntry reminderEntry)
{
try
{
return(await UpsertTableEntryAsync(reminderEntry));
}
catch (Exception exc)
{
HttpStatusCode httpStatusCode;
string restStatus;
if (AzureStorageUtils.EvaluateException(exc, out httpStatusCode, out restStatus))
{
if (Logger.IsEnabled(LogLevel.Trace))
{
Logger.Trace("UpsertRow failed with httpStatusCode={0}, restStatus={1}", httpStatusCode, restStatus);
}
if (AzureStorageUtils.IsContentionError(httpStatusCode))
{
return(null); // false;
}
}
throw;
}
}
private IEnumerable <T> ExecuteQuery(string processName, TableQuery <T> rangeQuery, Func <T, bool> filter)
{
TableContinuationToken tableContinuationToken = null;
do
{
TableQuerySegment <T> queryResponse;
try
{
queryResponse = GetTable().ExecuteQuerySegmented(rangeQuery, tableContinuationToken);
tableContinuationToken = queryResponse.ContinuationToken;
}
catch (Exception ex)
{
_log?.WriteFatalError("Table storage: " + _tableName, processName, rangeQuery.FilterString ?? "[null]", ex);
throw;
}
foreach (var itm in AzureStorageUtils.ApplyFilter(queryResponse.Results, filter))
{
yield return(itm);
}
} while (tableContinuationToken != null);
}
public async Task <IHttpActionResult> PutReviewPieces([FromBody] IEnumerable <ReviewPiece> pieces)
{
// All pieces must belong to the same exercise and review.
var partitionKey = pieces
.Select(i => i.PartitionKey)
.GroupBy(i => i)
.Select(i => i.Key)
.Single();
var reviewId = pieces
.Select(i => ReviewPiece.GetReviewId(i.RowKey))
.GroupBy(i => i)
.Select(i => i.Key)
.Single();
// Ensure that the user is the actual reviewer. Check the presense of the access entry for the user. All pieces must belong to the same exercise and review.
var table = AzureStorageUtils.GetCloudTable(AzureStorageUtils.TableNames.ReviewPieces);
var userIsEditor = await UserIsEditor(partitionKey, reviewId, table);
if (userIsEditor)
{
var batchOperation = new TableBatchOperation();
foreach (var piece in pieces)
{
batchOperation.InsertOrReplace(piece);
}
await table.ExecuteBatchAsync(batchOperation);
}
// Notify the exercise author in real-time.
var piecesArr = pieces.Select(i => i.Json).ToArray();
this.GetAuthorConnections(partitionKey).PiecesChanged(piecesArr);
return(StatusCode(userIsEditor ? HttpStatusCode.NoContent : HttpStatusCode.BadRequest));
}
public void AzureTableErrorCode_BadTableName()
{
string tableName = "abc-123";
AzureStorageUtils.ValidateTableName(tableName);
}
public static string MakeRowKey(string partition)
{
string key = $"partition_{partition}";
return(AzureStorageUtils.SanitizeTableProperty(key));
}
public static string MakePartitionKey(string streamProviderName, string checkpointNamespace)
{
string key = $"EventHubCheckpoints_{streamProviderName}_{checkpointNamespace}";
return(AzureStorageUtils.SanitizeTableProperty(key));
}
public Task <IEnumerable <T> > GetDataAsync(string partition, Func <T, bool> filter = null)
{
return(Task.Run(() => AzureStorageUtils.ApplyFilter(this[partition], filter)));
}
public async Task InsertAsync(T item, params int[] notLogCodes)
{
await _lockSlim.WaitAsync();
try
{
if (HasElement(item))
{
var message =
string.Format("Can not insert. Item is already in the table. Data:{0}", AzureStorageUtils.PrintItem(item));
var exception = StorageException.TranslateException(new Exception(message),
new RequestResult
{
HttpStatusCode = AzureStorageUtils.Conflict
}
);
throw exception;
}
PrivateInsert(item);
}
finally
{
_lockSlim.Release();
}
}
/// <summary> Decodes Storage exceptions.</summary>
public bool DecodeException(Exception e, out HttpStatusCode httpStatusCode, out string restStatus, bool getRESTErrors = false)
{
return(AzureStorageUtils.EvaluateException(e, out httpStatusCode, out restStatus, getRESTErrors));
}
private string GetKeyString(GrainReference grainReference)
{
var key = String.Format("{0}_{1}", this.clusterOptions.ServiceId, grainReference.ToKeyString());
return(AzureStorageUtils.SanitizeTableProperty(key));
}
} // Part of PartitionKey
public static string ConstructRowKey(GrainReference grainRef, string reminderName)
{
var key = String.Format("{0}-{1}", grainRef.ToKeyString(), reminderName); //grainRef.ToString(), reminderName);
return(AzureStorageUtils.SanitizeTableProperty(key));
}
public static string MakeRowKey(string partition)
{
string key = String.Format("partition_{0}", partition);
return(AzureStorageUtils.SanitizeTableProperty(key));
}
public async Task AzureTableDataManager_UpdateTableEntryAsync()
{
var data = GenerateNewData();
try
{
await manager.UpdateTableEntryAsync(data, AzureStorageUtils.ANY_ETAG);
Assert.True(false, "Should have thrown StorageException.");
}
catch (StorageException exc)
{
Assert.Equal((int)HttpStatusCode.NotFound, exc.RequestInformation.HttpStatusCode); // "Update before insert."
HttpStatusCode httpStatusCode;
string restStatus;
AzureStorageUtils.EvaluateException(exc, out httpStatusCode, out restStatus, true);
Assert.Equal(HttpStatusCode.NotFound, httpStatusCode);
Assert.Equal(StorageErrorCodeStrings.ResourceNotFound, restStatus);
}
await manager.UpsertTableEntryAsync(data);
var tuple = await manager.ReadSingleTableEntryAsync(data.PartitionKey, data.RowKey);
Assert.Equal(data.StringData, tuple.Item1.StringData);
var data2 = data.Clone();
data2.StringData = "NewData";
string eTag1 = await manager.UpdateTableEntryAsync(data2, AzureStorageUtils.ANY_ETAG);
tuple = await manager.ReadSingleTableEntryAsync(data2.PartitionKey, data2.RowKey);
Assert.Equal(data2.StringData, tuple.Item1.StringData);
var data3 = data.Clone();
data3.StringData = "EvenNewerData";
string ignoredETag = await manager.UpdateTableEntryAsync(data3, eTag1);
tuple = await manager.ReadSingleTableEntryAsync(data3.PartitionKey, data3.RowKey);
Assert.Equal(data3.StringData, tuple.Item1.StringData);
try
{
string eTag3 = await manager.UpdateTableEntryAsync(data3.Clone(), eTag1);
Assert.True(false, "Should have thrown StorageException.");
}
catch (StorageException exc)
{
Assert.Equal((int)HttpStatusCode.PreconditionFailed, exc.RequestInformation.HttpStatusCode); // "Wrong eTag"
HttpStatusCode httpStatusCode;
string restStatus;
AzureStorageUtils.EvaluateException(exc, out httpStatusCode, out restStatus, true);
Assert.Equal(HttpStatusCode.PreconditionFailed, httpStatusCode);
Assert.True(restStatus == TableErrorCodeStrings.UpdateConditionNotSatisfied ||
restStatus == StorageErrorCodeStrings.ConditionNotMet, restStatus);
}
}
/// <summary>
/// Determine whether a local package is the same as a package on an AzDO feed.
/// </summary>
/// <param name="localPackageFullPath"></param>
/// <param name="packageContentUrl"></param>
/// <param name="client"></param>
/// <returns></returns>
/// <remarks>
/// Open a stream to the local file and an http request to the package. There are a couple possibilities:
/// - The returned headers includes a content MD5 header, in which case we can
/// hash the local file and just compare those.
/// - No content MD5 hash, and the streams must be compared in blocks. This is a bit trickier to do efficiently,
/// since we do not necessarily want to read all bytes if we can help it. Thus, we should compare in blocks. However,
/// the streams make no gaurantee that they will return a full block each time when read operations are performed, so we
/// must be sure to only compare the minimum number of bytes returned.
/// </remarks>
public static async Task <PackageFeedStatus> CompareLocalPackageToFeedPackage(string localPackageFullPath, string packageContentUrl, HttpClient client, TaskLoggingHelper log)
{
log.LogMessage($"Getting package content from {packageContentUrl} and comparing to {localPackageFullPath}");
PackageFeedStatus result = PackageFeedStatus.Unknown;
ExponentialRetry RetryHandler = new ExponentialRetry
{
MaxAttempts = MaxRetries
};
bool success = await RetryHandler.RunAsync(async attempt =>
{
try
{
using (Stream localFileStream = File.OpenRead(localPackageFullPath))
using (HttpResponseMessage response = await client.GetAsync(packageContentUrl))
{
response.EnsureSuccessStatusCode();
// Check the headers for content length and md5
bool md5HeaderAvailable = response.Headers.TryGetValues("Content-MD5", out var md5);
bool lengthHeaderAvailable = response.Headers.TryGetValues("Content-Length", out var contentLength);
if (lengthHeaderAvailable && long.Parse(contentLength.Single()) != localFileStream.Length)
{
log.LogMessage(MessageImportance.Low, $"Package '{localPackageFullPath}' has different length than remote package '{packageContentUrl}'.");
result = PackageFeedStatus.ExistsAndDifferent;
return(true);
}
if (md5HeaderAvailable)
{
var localMD5 = AzureStorageUtils.CalculateMD5(localPackageFullPath);
if (!localMD5.Equals(md5.Single(), StringComparison.OrdinalIgnoreCase))
{
log.LogMessage(MessageImportance.Low, $"Package '{localPackageFullPath}' has different MD5 hash than remote package '{packageContentUrl}'.");
}
result = PackageFeedStatus.ExistsAndDifferent;
return(true);
}
const int BufferSize = 64 * 1024;
// Otherwise, compare the streams
var remoteStream = await response.Content.ReadAsStreamAsync();
var streamsMatch = await GeneralUtils.CompareStreamsAsync(localFileStream, remoteStream, BufferSize);
result = streamsMatch ? PackageFeedStatus.ExistsAndIdenticalToLocal : PackageFeedStatus.ExistsAndDifferent;
return(true);
}
}
// String based comparison because the status code isn't exposed in HttpRequestException
// see here: https://github.com/dotnet/runtime/issues/23648
catch (HttpRequestException e)
{
if (e.Message.Contains("404 (Not Found)"))
{
result = PackageFeedStatus.DoesNotExist;
return(true);
}
// Retry this. Could be an http client timeout, 500, etc.
return(false);
}
});
return(result);
}
private async Task <bool> ExecuteAsync()
{
try
{
string accessId = IsInternal ? "int" : "pub";
string baseFeedName = $"darc-{accessId}-{RepositoryName}-{CommitSha}";
string versionedFeedName = baseFeedName;
bool needsUniqueName = false;
int subVersion = 0;
var containerName = string.Empty;
Log.LogMessage(MessageImportance.High, $"Creating a new Azure Storage internal feed ...");
Match m = Regex.Match(AzureStorageFeedsBaseUrl, baseUrlRegex);
if (m.Success)
{
containerName = m.Groups["containername"].Value;
}
else
{
Log.LogError($"Could not parse {nameof(AzureStorageFeedsBaseUrl)} to extract the container name: '{AzureStorageFeedsBaseUrl}'");
return(false);
}
AzureStorageUtils azUtils = new AzureStorageUtils(AzureStorageAccountName, AzureStorageAccountKey, containerName);
// Create container if it doesn't already exist
if (!await azUtils.CheckIfContainerExistsAsync())
{
BlobContainerPermissions permissions = new BlobContainerPermissions
{
PublicAccess = IsInternal ? BlobContainerPublicAccessType.Off : BlobContainerPublicAccessType.Container
};
await azUtils.CreateContainerAsync(permissions);
}
// Create folder inside the container. Note that AzureStorage requires a folder
// to have at least one file.
do
{
if (await azUtils.CheckIfBlobExistsAsync($"{versionedFeedName}/index.json"))
{
versionedFeedName = $"{baseFeedName}-{++subVersion}";
needsUniqueName = true;
}
else
{
baseFeedName = versionedFeedName;
needsUniqueName = false;
}
} while (needsUniqueName);
// Initialize the feed using sleet
SleetSource sleetSource = new SleetSource()
{
Name = baseFeedName,
Type = "azure",
BaseUri = $"{AzureStorageFeedsBaseUrl}{baseFeedName}",
AccountName = AzureStorageAccountName,
Container = containerName,
FeedSubPath = $"{baseFeedName}",
ConnectionString = $"DefaultEndpointsProtocol=https;AccountName={AzureStorageAccountName};AccountKey={AzureStorageAccountKey};EndpointSuffix=core.windows.net"
};
BlobFeedAction bfAction = new BlobFeedAction(sleetSource, AzureStorageAccountKey, Log);
await bfAction.InitAsync();
TargetFeedURL = $"{AzureStorageFeedsBaseUrl}{baseFeedName}";
Log.LogMessage(MessageImportance.High, $"Feed '{TargetFeedURL}' created successfully!");
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
return(!Log.HasLoggedErrors);
}
