public async Task <int> ExecuteAsync()
{
Console.WriteLine($"Importing {DatasetUrl}");
using (var httpClient = new HttpClient())
{
var result = await httpClient.GetAsync(DatasetUrl);
result.EnsureSuccessStatusCode();
var dataset = JsonConvert.DeserializeObject <DatasetStorageItem>(await result.Content.ReadAsStringAsync());
Console.WriteLine($"Found Doc: {dataset.Id} - {dataset.Name}");
var storageCredentials = new StorageCredentials(StorageOptions.Account, StorageOptions.Key);
var storageAccount = new CloudStorageAccount(storageCredentials, true);
var blobClient = storageAccount.CreateCloudBlobClient();
var containerRef = blobClient.GetContainerReference(StorageName);
using (var client =
new DocumentClient(
new Uri($"https://{CosmosOptions.Endpoint}.documents.azure.com/"),
CosmosOptions.Key,
new ConnectionPolicy
{
ConnectionMode = ConnectionMode.Direct,
ConnectionProtocol = Protocol.Tcp
},
ConsistencyLevel.Session))
{
await client.OpenAsync();
var license = await client.ReadDocumentAsync <LicenseStorageItem>(
UriFactory.CreateDocumentUri(CosmosOptions.Database, CosmosOptions.DatasetsCollection, dataset.LicenseId.ToString()),
new RequestOptions
{
PartitionKey = new PartitionKey(WellKnownIds.LicenseDatasetId.ToString())
});
// Attachment details
var blob = new
{
StorageType = "blob",
StorageOptions.Account,
Container = StorageName,
MediaLink = containerRef.Uri.ToString(),
};
// Convert Dataset to Nomination
var nomination = DatasetConvert.DatasetToNomination(dataset, ContactOptions, license);
Console.WriteLine(JsonConvert.SerializeObject(dataset, Formatting.Indented));
Console.WriteLine(JsonConvert.SerializeObject(nomination, Formatting.Indented));
Console.WriteLine(JsonConvert.SerializeObject(blob, Formatting.Indented));
// Add the nomination document
var uri = UriFactory.CreateDocumentCollectionUri(
CosmosOptions.Database,
CosmosOptions.UserDataCollection);
var options = new RequestOptions
{
PartitionKey = new PartitionKey(nomination.DatasetId.ToString())
};
var newDoc = await client.UpsertDocumentAsync(uri, nomination, options);
// Add the attachment to the nomination (with blob storage details)
var datasetRecordLink = new Attachment
{
Id = "Content",
ContentType = "x-azure-blockstorage",
MediaLink = blob.MediaLink,
};
datasetRecordLink.SetPropertyValue("storageType", "blob");
datasetRecordLink.SetPropertyValue("container", blob.Container);
datasetRecordLink.SetPropertyValue("account", blob.Account);
await client.UpsertAttachmentAsync(newDoc.Resource.SelfLink, datasetRecordLink, options);
Console.WriteLine($"Added nomination document.");
}
}
return(0);
}
public async Task <int> ExecuteAsync()
{
Console.WriteLine($"Re-nominating dataset {DatasetId}");
using (var client =
new DocumentClient(
new Uri($"https://{CosmosOptions.Endpoint}.documents.azure.com/"),
CosmosOptions.Key,
new ConnectionPolicy
{
ConnectionMode = ConnectionMode.Direct,
ConnectionProtocol = Protocol.Tcp
},
ConsistencyLevel.Session))
{
await client.OpenAsync();
// Read Dataset Document
var uri = UriFactory.CreateDocumentUri(
CosmosOptions.Database,
CosmosOptions.DatasetsCollection,
DatasetId);
var options = new RequestOptions
{
PartitionKey = new PartitionKey(DatasetId)
};
var doc = await client.ReadDocumentAsync <DatasetStorageItem>(uri, options);
var dataset = doc.Document;
Console.WriteLine($"Found Doc: {dataset.Id} - {dataset.Name}");
var license = await client.ReadDocumentAsync <LicenseStorageItem>(
UriFactory.CreateDocumentUri(CosmosOptions.Database, CosmosOptions.DatasetsCollection, dataset.LicenseId.ToString()),
new RequestOptions
{
PartitionKey = new PartitionKey(WellKnownIds.LicenseDatasetId.ToString())
});
// Read Dataset Document attachment (with blob storage details)
var documentLink = UriFactory.CreateAttachmentUri(
CosmosOptions.Database,
CosmosOptions.DatasetsCollection,
DatasetId,
"Content");
var response = await client.ReadAttachmentAsync(documentLink, options);
var resource = response?.Resource;
if (resource == null)
{
throw new InvalidOperationException("Could not find storage information for dataset.");
}
var storageType = resource.GetPropertyValue <string>("storageType") ?? string.Empty;
if (storageType != "blob")
{
throw new InvalidOperationException($"Unknown storage type, \"{storageType}\", for dataset.");
}
var blob = new
{
StorageType = storageType,
Account = resource.GetPropertyValue <string>("account"),
Container = resource.GetPropertyValue <string>("container"),
MediaLink = resource.MediaLink,
};
// Convert Dataset to Nomination with either PendingApproval or Importing status.
var nomination = DatasetConvert.DatasetToNomination(dataset, ContactOptions, license);
nomination.NominationStatus = QueueJob
? NominationStatus.Importing
: NominationStatus.PendingApproval;
Console.WriteLine(JsonConvert.SerializeObject(dataset, Formatting.Indented));
Console.WriteLine(JsonConvert.SerializeObject(nomination, Formatting.Indented));
Console.WriteLine(JsonConvert.SerializeObject(blob, Formatting.Indented));
// Add the nomination document
uri = UriFactory.CreateDocumentCollectionUri(
CosmosOptions.Database,
CosmosOptions.UserDataCollection);
options = new RequestOptions
{
PartitionKey = new PartitionKey(nomination.DatasetId.ToString())
};
var newDoc = await client.UpsertDocumentAsync(uri, nomination, options);
// Add the attachment to the nomination (with blob storage details)
var datasetRecordLink = new Attachment
{
Id = "Content",
ContentType = "x-azure-blockstorage",
MediaLink = blob.MediaLink,
};
datasetRecordLink.SetPropertyValue("storageType", "blob");
datasetRecordLink.SetPropertyValue("container", blob.Container);
datasetRecordLink.SetPropertyValue("account", blob.Account);
await client.UpsertAttachmentAsync(newDoc.Resource.SelfLink, datasetRecordLink, options);
Console.WriteLine($"Added nomination document.");
if (QueueJob)
{
// Add import job to queue
var batchCreds = new BatchSharedKeyCredentials(BatchOptions.Url, BatchOptions.Account, BatchOptions.Key);
using (var batchClient = BatchClient.Open(batchCreds))
{
var taskName = GetUniqueTaskName(BatchConstants.ImportDatasetAppName);
var commandLine = GetCommandLine(
BatchConstants.ImportDatasetAppName,
BatchConstants.ImportDatasetExeName,
nomination.Id.ToString());
var task = new CloudTask(taskName, commandLine)
{
DisplayName = $"Importing: {nomination.Name} ({nomination.Id})"
};
await batchClient.JobOperations.AddTaskAsync(BatchConstants.DatasetJobId, task);
Console.WriteLine($"Queued Job to import dataset.");
}
}
}
return(0);
}
