public async Task <int> ExecuteAsync()
        {
            Console.WriteLine($"Importing {DatasetUrl}");

            using (var httpClient = new HttpClient())
            {
                var result = await httpClient.GetAsync(DatasetUrl);

                result.EnsureSuccessStatusCode();
                var dataset = JsonConvert.DeserializeObject <DatasetStorageItem>(await result.Content.ReadAsStringAsync());

                Console.WriteLine($"Found Doc: {dataset.Id} - {dataset.Name}");

                var storageCredentials = new StorageCredentials(StorageOptions.Account, StorageOptions.Key);
                var storageAccount     = new CloudStorageAccount(storageCredentials, true);
                var blobClient         = storageAccount.CreateCloudBlobClient();

                var containerRef = blobClient.GetContainerReference(StorageName);

                using (var client =
                           new DocumentClient(
                               new Uri($"https://{CosmosOptions.Endpoint}.documents.azure.com/"),
                               CosmosOptions.Key,
                               new ConnectionPolicy
                {
                    ConnectionMode = ConnectionMode.Direct,
                    ConnectionProtocol = Protocol.Tcp
                },
                               ConsistencyLevel.Session))
                {
                    await client.OpenAsync();

                    var license = await client.ReadDocumentAsync <LicenseStorageItem>(
                        UriFactory.CreateDocumentUri(CosmosOptions.Database, CosmosOptions.DatasetsCollection, dataset.LicenseId.ToString()),
                        new RequestOptions
                    {
                        PartitionKey = new PartitionKey(WellKnownIds.LicenseDatasetId.ToString())
                    });

                    // Attachment details
                    var blob = new
                    {
                        StorageType = "blob",
                        StorageOptions.Account,
                        Container = StorageName,
                        MediaLink = containerRef.Uri.ToString(),
                    };

                    // Convert Dataset to Nomination
                    var nomination = DatasetConvert.DatasetToNomination(dataset, ContactOptions, license);
                    Console.WriteLine(JsonConvert.SerializeObject(dataset, Formatting.Indented));
                    Console.WriteLine(JsonConvert.SerializeObject(nomination, Formatting.Indented));
                    Console.WriteLine(JsonConvert.SerializeObject(blob, Formatting.Indented));

                    // Add the nomination document
                    var uri = UriFactory.CreateDocumentCollectionUri(
                        CosmosOptions.Database,
                        CosmosOptions.UserDataCollection);
                    var options = new RequestOptions
                    {
                        PartitionKey = new PartitionKey(nomination.DatasetId.ToString())
                    };
                    var newDoc = await client.UpsertDocumentAsync(uri, nomination, options);

                    // Add the attachment to the nomination (with blob storage details)
                    var datasetRecordLink = new Attachment
                    {
                        Id          = "Content",
                        ContentType = "x-azure-blockstorage",
                        MediaLink   = blob.MediaLink,
                    };
                    datasetRecordLink.SetPropertyValue("storageType", "blob");
                    datasetRecordLink.SetPropertyValue("container", blob.Container);
                    datasetRecordLink.SetPropertyValue("account", blob.Account);
                    await client.UpsertAttachmentAsync(newDoc.Resource.SelfLink, datasetRecordLink, options);

                    Console.WriteLine($"Added nomination document.");
                }
            }

            return(0);
        }
Beispiel #2
0
        public async Task <int> ExecuteAsync()
        {
            Console.WriteLine($"Re-nominating dataset {DatasetId}");

            using (var client =
                       new DocumentClient(
                           new Uri($"https://{CosmosOptions.Endpoint}.documents.azure.com/"),
                           CosmosOptions.Key,
                           new ConnectionPolicy
            {
                ConnectionMode = ConnectionMode.Direct,
                ConnectionProtocol = Protocol.Tcp
            },
                           ConsistencyLevel.Session))
            {
                await client.OpenAsync();

                // Read Dataset Document

                var uri = UriFactory.CreateDocumentUri(
                    CosmosOptions.Database,
                    CosmosOptions.DatasetsCollection,
                    DatasetId);
                var options = new RequestOptions
                {
                    PartitionKey = new PartitionKey(DatasetId)
                };

                var doc = await client.ReadDocumentAsync <DatasetStorageItem>(uri, options);

                var dataset = doc.Document;
                Console.WriteLine($"Found Doc: {dataset.Id} - {dataset.Name}");

                var license = await client.ReadDocumentAsync <LicenseStorageItem>(
                    UriFactory.CreateDocumentUri(CosmosOptions.Database, CosmosOptions.DatasetsCollection, dataset.LicenseId.ToString()),
                    new RequestOptions
                {
                    PartitionKey = new PartitionKey(WellKnownIds.LicenseDatasetId.ToString())
                });

                // Read Dataset Document attachment (with blob storage details)

                var documentLink = UriFactory.CreateAttachmentUri(
                    CosmosOptions.Database,
                    CosmosOptions.DatasetsCollection,
                    DatasetId,
                    "Content");
                var response = await client.ReadAttachmentAsync(documentLink, options);

                var resource = response?.Resource;
                if (resource == null)
                {
                    throw new InvalidOperationException("Could not find storage information for dataset.");
                }

                var storageType = resource.GetPropertyValue <string>("storageType") ?? string.Empty;
                if (storageType != "blob")
                {
                    throw new InvalidOperationException($"Unknown storage type, \"{storageType}\", for dataset.");
                }
                var blob = new
                {
                    StorageType = storageType,
                    Account     = resource.GetPropertyValue <string>("account"),
                    Container   = resource.GetPropertyValue <string>("container"),
                    MediaLink   = resource.MediaLink,
                };

                // Convert Dataset to Nomination with either PendingApproval or Importing status.
                var nomination = DatasetConvert.DatasetToNomination(dataset, ContactOptions, license);
                nomination.NominationStatus = QueueJob
                    ? NominationStatus.Importing
                    : NominationStatus.PendingApproval;
                Console.WriteLine(JsonConvert.SerializeObject(dataset, Formatting.Indented));
                Console.WriteLine(JsonConvert.SerializeObject(nomination, Formatting.Indented));
                Console.WriteLine(JsonConvert.SerializeObject(blob, Formatting.Indented));

                // Add the nomination document

                uri = UriFactory.CreateDocumentCollectionUri(
                    CosmosOptions.Database,
                    CosmosOptions.UserDataCollection);
                options = new RequestOptions
                {
                    PartitionKey = new PartitionKey(nomination.DatasetId.ToString())
                };

                var newDoc = await client.UpsertDocumentAsync(uri, nomination, options);

                // Add the attachment to the nomination (with blob storage details)

                var datasetRecordLink = new Attachment
                {
                    Id          = "Content",
                    ContentType = "x-azure-blockstorage",
                    MediaLink   = blob.MediaLink,
                };

                datasetRecordLink.SetPropertyValue("storageType", "blob");
                datasetRecordLink.SetPropertyValue("container", blob.Container);
                datasetRecordLink.SetPropertyValue("account", blob.Account);

                await client.UpsertAttachmentAsync(newDoc.Resource.SelfLink, datasetRecordLink, options);

                Console.WriteLine($"Added nomination document.");

                if (QueueJob)
                {
                    // Add import job to queue
                    var batchCreds = new BatchSharedKeyCredentials(BatchOptions.Url, BatchOptions.Account, BatchOptions.Key);
                    using (var batchClient = BatchClient.Open(batchCreds))
                    {
                        var taskName    = GetUniqueTaskName(BatchConstants.ImportDatasetAppName);
                        var commandLine = GetCommandLine(
                            BatchConstants.ImportDatasetAppName,
                            BatchConstants.ImportDatasetExeName,
                            nomination.Id.ToString());
                        var task = new CloudTask(taskName, commandLine)
                        {
                            DisplayName = $"Importing: {nomination.Name} ({nomination.Id})"
                        };

                        await batchClient.JobOperations.AddTaskAsync(BatchConstants.DatasetJobId, task);

                        Console.WriteLine($"Queued Job to import dataset.");
                    }
                }
            }

            return(0);
        }