public async Task ShouldPublishUpdateDatasetContent() { await TestUtils.ExecAndCleanup(async cleanup => { var dataset = CreateDatasetDoc(); cleanup.Push(await SetupDataset(dataset)); var editService = Services.GetService <DatasetEditStorageService>(); var user = CreateTestUser(); var updated = await editService.InitiateDatasetContentEdit(dataset.Id, user, default); var blobClient = await Services.GetBlobClient(); var container = blobClient.GetContainerReference(updated.ContentEditContainer); foreach (var testFile in TestFiles) { var blob = container.GetBlockBlobReference($"{testFile}-updated.txt"); var content = $"{testFile}, updated {DateTime.UtcNow.ToString()}"; await blob.UploadTextAsync(content); } var result = await editService.PublishUpdatedDataset(dataset.Id, user, default); Assert.True(result.IsPublished); Assert.Equal(DatasetEditStatus.ContentsModified, result.UsingStatus); Assert.True(result.ShouldQueueBatchOperation); Assert.Equal(result.DatasetId, dataset.Id); Assert.Equal(result.DatasetName, dataset.Name); var cosmosClient = await Services.GetCosmosClient(); var cosmosConfig = Services.GetService <IOptions <CosmosConfiguration> >().Value; var docUri = UriFactory.CreateDocumentUri(cosmosConfig.Database, cosmosConfig.UserDataCollection, dataset.Id.ToString()); var reqOpts = new RequestOptions { PartitionKey = new PartitionKey(WellKnownIds.DatasetNominationDatasetId.ToString()) }; cleanup.Push(async() => { await cosmosClient.DeleteDocumentAsync(docUri, reqOpts); await container.DeleteAsync(); }); updated = await editService.GetDatasetEditById(dataset.Id, user, default); Assert.Equal(dataset.Id, updated.Id); Assert.Equal(DatasetEditStatus.Importing, updated.EditStatus); Assert.NotNull(updated.ContentEditAccount); Assert.NotNull(updated.ContentEditContainer); Assert.NotNull(updated.OriginalStorageAccount); Assert.NotNull(updated.OriginalStorageContainer); var nomination = (await cosmosClient.ReadDocumentAsync <DatasetNominationStorageItem>(docUri, reqOpts)).Document; Assert.Equal(dataset.Id, nomination.Id); Assert.Equal(dataset.Name, nomination.Name); Assert.Equal(NominationStatus.Importing, nomination.NominationStatus); var nominationLink = await cosmosClient.ReadAttachmentAsync( UriFactory.CreateAttachmentUri(cosmosConfig.Database, cosmosConfig.UserDataCollection, dataset.Id.ToString(), "Content"), reqOpts); Assert.Equal(container.Uri.ToString(), nominationLink.Resource.MediaLink); }); }
public static async Task <Attachment> GetAttachment(string formId, string attachId) { Document d = await client.ReadDocumentAsync(UriFactory.CreateDocumentUri(DatabaseId, CollectionId, formId)); Attachment a = await client.ReadAttachmentAsync(UriFactory.CreateAttachmentUri(DatabaseId, CollectionId, formId, attachId)); return(a); }
private static async Task MigrateSingleCollection( DocumentClient client, Database database, string cosmosCollectionName, string ravenCollectionName, JsonTextWriter jsonTextWriter, StreamWriter streamWriter) { var collection = UriFactory.CreateDocumentCollectionUri(database.Id, cosmosCollectionName); var options = new FeedOptions { MaxItemCount = 128 }; var query = client.CreateDocumentQuery <ExpandoObject>(collection, options).AsDocumentQuery(); var attachmentNumber = 0; while (query.HasMoreResults) { var result = await query.ExecuteNextAsync <ExpandoObject>(); foreach (var document in result) { var dictionary = (IDictionary <string, object>)document; var documentId = dictionary[CosmosDocumentId].ToString(); const string selfLinkKey = "_self"; var selfLink = dictionary[selfLinkKey].ToString(); var attachments = new List <Dictionary <string, object> >(); foreach (var attachment in client.CreateAttachmentQuery(selfLink)) { var attachmentUri = UriFactory.CreateAttachmentUri(database.Id, cosmosCollectionName, documentId, attachment.Id); var attachmentResponse = await client.ReadAttachmentAsync(attachmentUri); var resourceMediaLink = attachmentResponse.Resource.MediaLink; var mediaLinkResponse = await client.ReadMediaAsync(resourceMediaLink); attachmentNumber++; var attachmentInfo = await MigrationHelpers.WriteAttachment( mediaLinkResponse.Media, mediaLinkResponse.Media.Length, attachmentResponse.Resource.Id, ravenCollectionName, mediaLinkResponse.ContentType, attachmentNumber, jsonTextWriter, streamWriter); attachments.Add(attachmentInfo); } MigrationHelpers.WriteDocument( document, documentId, ravenCollectionName, PropertiesToRemove, jsonTextWriter, attachments); } } }
public async Task <ActionResult> EditAsync(PictureItem item, [Bind("oldCategory")] string oldCategory, IFormFile file) { if (ModelState.IsValid) { await this.galleryRepository.InitAsync("Pictures"); Document document = null; if (item.Category == oldCategory) { document = await this.galleryRepository.UpdateItemAsync(item.Id, item); if (file != null) { var attachLink = UriFactory.CreateAttachmentUri("Gallery", "Pictures", document.Id, "wallpaper"); Attachment attachment = await this.galleryRepository.ReadAttachmentAsync(attachLink.ToString(), item.Category); var input = new byte[file.OpenReadStream().Length]; file.OpenReadStream().Read(input, 0, input.Length); attachment.SetPropertyValue("file", input); ResourceResponse <Attachment> createdAttachment = await this.galleryRepository.ReplaceAttachmentAsync(attachment, new RequestOptions() { PartitionKey = new PartitionKey(item.Category) }); } } else { await this.galleryRepository.DeleteItemAsync(item.Id, oldCategory); document = await this.galleryRepository.CreateItemAsync(item); if (file != null) { var attachment = new Attachment { ContentType = file.ContentType, Id = "wallpaper", MediaLink = string.Empty }; var input = new byte[file.OpenReadStream().Length]; file.OpenReadStream().Read(input, 0, input.Length); attachment.SetPropertyValue("file", input); ResourceResponse <Attachment> createdAttachment = await this.galleryRepository.CreateAttachmentAsync(document.AttachmentsLink, attachment, new RequestOptions() { PartitionKey = new PartitionKey(item.Category) }); } } return(RedirectToAction("Index")); } return(View(item)); }
public async Task <MediaResponse> GetAttachmentMediaAsync(string documentId, string attachmentId) { var attachmentResponse = await _documentClient.ReadAttachmentAsync(UriFactory.CreateAttachmentUri(_applicationConfig.Database, _applicationConfig.Collection, documentId, attachmentId)); MediaResponse response = null; if (attachmentResponse != null) { response = await _documentClient.ReadMediaAsync(attachmentResponse.Resource.MediaLink); } return(response); }
public Attachment ReadResponseAttachment(IResponseContext responseContext, string attachmentId) { Attachment attachment = null; try { var rootFormName = responseContext.RootFormName; var rootResponseId = responseContext.RootResponseId; var attachmentUri = UriFactory.CreateAttachmentUri(DatabaseName, rootFormName, rootResponseId, attachmentId); attachment = Client.ReadAttachmentAsync(attachmentUri).Result; return(attachment); } catch (Exception ex) { attachment = null; return(attachment); } }
protected async Task <TAttachment> GetDocumentAttachmentAsync <TDocument, TAttachment>( string documentRecordId, string partitionKey, AttachmentTypeMapping <TDocument, TAttachment> attachmentMapping) { var client = GetClient(); var attachmentUri = UriFactory.CreateAttachmentUri( _dbAccess.DbConfig.DatabaseId, _dbAccess.DbConfig.CollectionName, documentRecordId, CreateAttachmentId(documentRecordId, attachmentMapping.AttachmentName)); var requestOptions = new RequestOptions { PartitionKey = new PartitionKey(partitionKey) }; var attachmentResponse = await MakeClientCall( async() => await client.ReadAttachmentAsync(attachmentUri, requestOptions), "Failed to read document attachment"); if (attachmentResponse == null) { // Attachment not found. return(default(TAttachment)); } var mediaResponse = await MakeClientCall( async() => await client.ReadMediaAsync(attachmentResponse.Resource.MediaLink), "Failed to read document attachment media"); if (mediaResponse == null) { // Media link not found. This would indicate a bug in attachment storage process. return(default(TAttachment)); } return(DeserialiseAttachment(attachmentMapping, mediaResponse)); }
public static async Task <dynamic> GetAttachment([ActivityTrigger] Binder binder) { try { var document = binder.BindingData.GetValueOrDefault("documentId").ToString(); var attachment = binder.BindingData.GetValueOrDefault("attachmentId").ToString(); var result = await Client.ReadAttachmentAsync(UriFactory.CreateAttachmentUri(Database, Collection, document, attachment)); return(result.Resource.MediaLink); } catch (DocumentClientException e) { var resp = new HttpResponseMessage { StatusCode = HttpStatusCode.BadRequest, Content = new StringContent(e.Message) }; return(resp); } }
public async Task <IViewComponentResult> InvokeAsync(PictureItem item) { await this.galleryRepository.InitAsync("Pictures"); string image = string.Empty; Document document = await this.galleryRepository.GetDocumentAsync(item.Id, item.Category); var attachLink = UriFactory.CreateAttachmentUri("Gallery", "Pictures", document.Id, "wallpaper"); Attachment attachment = await this.galleryRepository.ReadAttachmentAsync(attachLink.ToString(), item.Category); var file = attachment.GetPropertyValue <byte[]>("file"); if (file != null) { string bytes = Convert.ToBase64String(file); image = string.Format("data:{0};base64,{1}", attachment.ContentType, bytes); } return(View(new ImageVM() { Id = "img-" + item.Id, Src = image })); }
private async void deleteAttachmentButton_Click(object sender, EventArgs e) { try { var attachment = attachmentsGrid.SelectedRows?[0]?.DataBoundItem as Attachment; if (attachment == null) { throw new Exception("No attachment selected"); } var attachmentsLink = UriFactory.CreateAttachmentUri(DatabaseName, CollectionName, docIdTextBox.Text, attachment.Id); var requestOptions = new RequestOptions() { PartitionKey = new Microsoft.Azure.Documents.PartitionKey(tenantId.ToString()) }; var response = await client.DeleteAttachmentAsync(attachmentsLink, requestOptions); MessageBox.Show(response.StatusCode.ToString()); } catch (Exception ex) { MessageBox.Show(ex.Message); } }
public async Task <int> ExecuteAsync() { Console.WriteLine($"Opening {CosmosOptions.Endpoint} CosmosDB"); var storageCredentials = new StorageCredentials(StorageOptions.Account, StorageOptions.Key); var storageAccount = new CloudStorageAccount(storageCredentials, true); var blobClient = storageAccount.CreateCloudBlobClient(); var indexClient = new SearchIndexClient( SearchOptions.Name, IndexOptions.DatasetIndex, new SearchCredentials(SearchOptions.Key)); using (var client = new DocumentClient( new Uri($"https://{CosmosOptions.Endpoint}.documents.azure.com/"), CosmosOptions.RawKey, new ConnectionPolicy { ConnectionMode = ConnectionMode.Direct, ConnectionProtocol = Protocol.Tcp }, ConsistencyLevel.Session)) { await client.OpenAsync(); var queryText = "SELECT c.datasetId, c.name FROM c WHERE c.dataType = 'dataset'"; var queryable = client .CreateDocumentQuery <DatasetEntry>( UriFactory.CreateDocumentCollectionUri(CosmosOptions.Database, CosmosOptions.DatasetsCollection), queryText, new FeedOptions { EnableCrossPartitionQuery = true, MaxItemCount = -1 }) .AsDocumentQuery(); var entries = new List <DatasetEntry>(); while (queryable.HasMoreResults) { var response = await queryable.ExecuteNextAsync <DatasetEntry>(); entries.AddRange(response); } int errorCount = 0; foreach (var entry in entries) { bool hasErrors = false; Console.WriteLine($"{entry.DatasetId} - {entry.Name}"); var indexDoc = await indexClient.Documents.GetAsync <DatasetStorageItem>(entry.DatasetId); if (indexDoc == null || indexDoc.Id.ToString() != entry.DatasetId || indexDoc.Name != entry.Name) { Console.Error.WriteLine($"Search index document not found."); hasErrors = true; } var options = new RequestOptions { PartitionKey = new PartitionKey(entry.DatasetId) }; var documentLink = UriFactory.CreateAttachmentUri( CosmosOptions.Database, CosmosOptions.DatasetsCollection, entry.DatasetId, "Content"); var response = await client.ReadAttachmentAsync(documentLink, options); var resource = response?.Resource; if (resource == null) { Console.Error.WriteLine($"Attachment not found."); ++errorCount; continue; } var storageType = resource.GetPropertyValue <string>("storageType") ?? string.Empty; if (storageType != "blob") { Console.Error.WriteLine($"Unknown storage type, \"{storageType}\", for dataset."); ++errorCount; continue; } var account = resource.GetPropertyValue <string>("account"); var container = resource.GetPropertyValue <string>("container"); Console.WriteLine($" Account: {account}"); Console.WriteLine($" Container: {container}"); Console.WriteLine($" MediaLink: {resource.MediaLink}"); var containerNames = new HashSet <string>(); if (string.IsNullOrWhiteSpace(account) || account != StorageOptions.Account) { Console.Error.WriteLine($"Invalid account."); hasErrors = true; } if (string.IsNullOrWhiteSpace(container) || containerNames.Contains(container)) { Console.Error.WriteLine($"Invalid container."); hasErrors = true; } containerNames.Add(container); var containerRef = blobClient.GetContainerReference(container); bool exists = await containerRef.ExistsAsync(); if (!exists) { Console.Error.WriteLine($"Associated container does not exist."); hasErrors = true; } var containerUrl = containerRef.Uri.ToString(); if (string.IsNullOrWhiteSpace(resource.MediaLink) || resource.MediaLink != containerUrl) { Console.Error.WriteLine($"Invalid MediaLink."); Console.Error.WriteLine($"Expected {containerUrl}"); hasErrors = true; } if (hasErrors) { if (FixDatasets && exists) { var datasetRecordLink = new Attachment { Id = "Content", ContentType = "x-azure-blockstorage", MediaLink = containerUrl, }; datasetRecordLink.SetPropertyValue("storageType", "blob"); datasetRecordLink.SetPropertyValue("container", container); datasetRecordLink.SetPropertyValue("account", StorageOptions.Account); var uri = UriFactory.CreateDocumentUri( CosmosOptions.Database, CosmosOptions.DatasetsCollection, entry.DatasetId); await client.UpsertAttachmentAsync(uri, datasetRecordLink, options); } else { ++errorCount; } } } if (errorCount == 0) { Console.WriteLine("All datasets verified."); } else { Console.Error.WriteLine($"{errorCount} ERRORS FOUND."); } } return(0); }
/// <summary> /// Given a database, collection, document, and attachment id, this method creates an attachment link. /// </summary> /// <param name="entityId">document id</param> /// <param name="idAttachment">attachment id</param> /// <returns></returns> private Uri GetAttachmentUri(string entityId, string idAttachment) { return(UriFactory.CreateAttachmentUri(_databaseId, _collectionName, entityId, idAttachment)); }
/// <summary> /// Creates a user data document attachment URI reference. /// </summary> /// <param name="documentId">The document identifier.</param> /// <param name="attachmentName">Name of the attachment.</param> /// <returns>Uri reference for the document</returns> protected Uri CreateUserDataDocumentAttachmentUri(Guid documentId, string attachmentName) { var uri = UriFactory.CreateAttachmentUri(this.Configuration.Database, this.Configuration.UserDataCollection, documentId.ToString(), attachmentName); return(uri); }
/// <summary> /// Creates a attachment URI for the specified document identifier /// </summary> /// <param name="id">The document</param> /// <param name="attachmentName">The attachment name</param> /// <returns>The URI</returns> protected Uri CreateAttachmentUri(Guid id, string attachmentName) { return(UriFactory.CreateAttachmentUri(this.Database, this.Collection, id.ToString(), attachmentName)); }
public async Task <int> ExecuteAsync() { Console.WriteLine($"Re-nominating dataset {DatasetId}"); using (var client = new DocumentClient( new Uri($"https://{CosmosOptions.Endpoint}.documents.azure.com/"), CosmosOptions.Key, new ConnectionPolicy { ConnectionMode = ConnectionMode.Direct, ConnectionProtocol = Protocol.Tcp }, ConsistencyLevel.Session)) { await client.OpenAsync(); // Read Dataset Document var uri = UriFactory.CreateDocumentUri( CosmosOptions.Database, CosmosOptions.DatasetsCollection, DatasetId); var options = new RequestOptions { PartitionKey = new PartitionKey(DatasetId) }; var doc = await client.ReadDocumentAsync <DatasetStorageItem>(uri, options); var dataset = doc.Document; Console.WriteLine($"Found Doc: {dataset.Id} - {dataset.Name}"); var license = await client.ReadDocumentAsync <LicenseStorageItem>( UriFactory.CreateDocumentUri(CosmosOptions.Database, CosmosOptions.DatasetsCollection, dataset.LicenseId.ToString()), new RequestOptions { PartitionKey = new PartitionKey(WellKnownIds.LicenseDatasetId.ToString()) }); // Read Dataset Document attachment (with blob storage details) var documentLink = UriFactory.CreateAttachmentUri( CosmosOptions.Database, CosmosOptions.DatasetsCollection, DatasetId, "Content"); var response = await client.ReadAttachmentAsync(documentLink, options); var resource = response?.Resource; if (resource == null) { throw new InvalidOperationException("Could not find storage information for dataset."); } var storageType = resource.GetPropertyValue <string>("storageType") ?? string.Empty; if (storageType != "blob") { throw new InvalidOperationException($"Unknown storage type, \"{storageType}\", for dataset."); } var blob = new { StorageType = storageType, Account = resource.GetPropertyValue <string>("account"), Container = resource.GetPropertyValue <string>("container"), MediaLink = resource.MediaLink, }; // Convert Dataset to Nomination with either PendingApproval or Importing status. var nomination = DatasetConvert.DatasetToNomination(dataset, ContactOptions, license); nomination.NominationStatus = QueueJob ? NominationStatus.Importing : NominationStatus.PendingApproval; Console.WriteLine(JsonConvert.SerializeObject(dataset, Formatting.Indented)); Console.WriteLine(JsonConvert.SerializeObject(nomination, Formatting.Indented)); Console.WriteLine(JsonConvert.SerializeObject(blob, Formatting.Indented)); // Add the nomination document uri = UriFactory.CreateDocumentCollectionUri( CosmosOptions.Database, CosmosOptions.UserDataCollection); options = new RequestOptions { PartitionKey = new PartitionKey(nomination.DatasetId.ToString()) }; var newDoc = await client.UpsertDocumentAsync(uri, nomination, options); // Add the attachment to the nomination (with blob storage details) var datasetRecordLink = new Attachment { Id = "Content", ContentType = "x-azure-blockstorage", MediaLink = blob.MediaLink, }; datasetRecordLink.SetPropertyValue("storageType", "blob"); datasetRecordLink.SetPropertyValue("container", blob.Container); datasetRecordLink.SetPropertyValue("account", blob.Account); await client.UpsertAttachmentAsync(newDoc.Resource.SelfLink, datasetRecordLink, options); Console.WriteLine($"Added nomination document."); if (QueueJob) { // Add import job to queue var batchCreds = new BatchSharedKeyCredentials(BatchOptions.Url, BatchOptions.Account, BatchOptions.Key); using (var batchClient = BatchClient.Open(batchCreds)) { var taskName = GetUniqueTaskName(BatchConstants.ImportDatasetAppName); var commandLine = GetCommandLine( BatchConstants.ImportDatasetAppName, BatchConstants.ImportDatasetExeName, nomination.Id.ToString()); var task = new CloudTask(taskName, commandLine) { DisplayName = $"Importing: {nomination.Name} ({nomination.Id})" }; await batchClient.JobOperations.AddTaskAsync(BatchConstants.DatasetJobId, task); Console.WriteLine($"Queued Job to import dataset."); } } } return(0); }
public async Task <int> ExecuteAsync() { using (var client = new DocumentClient( new Uri($"https://{CosmosOptions.Endpoint}.documents.azure.com/"), CosmosOptions.RawKey, new ConnectionPolicy { ConnectionMode = ConnectionMode.Direct, ConnectionProtocol = Protocol.Tcp }, ConsistencyLevel.Session)) { await client.OpenAsync(); Console.WriteLine("========DATASET========"); var datasetUri = UriFactory.CreateDocumentCollectionUri( CosmosOptions.Database, CosmosOptions.DatasetsCollection); var dataset = (await client.CreateDocumentQuery( datasetUri, new FeedOptions { PartitionKey = new PartitionKey(DatasetId) }) .Where(d => d.Id == DatasetId) .AsDocumentQuery() .GetQueryResultsAsync()) .SingleOrDefault(); if (dataset == null) { Console.WriteLine($"Dataset document {DatasetId} not found."); } else { Console.WriteLine(dataset.ToString()); Console.WriteLine("========ATTACHMENT========"); try { var documentLink = UriFactory.CreateAttachmentUri( CosmosOptions.Database, CosmosOptions.DatasetsCollection, DatasetId, "Content"); var response = await client.ReadAttachmentAsync(documentLink, new RequestOptions { PartitionKey = new PartitionKey(DatasetId) }); Console.WriteLine(response?.Resource?.ToString()); } catch (DocumentClientException ex) when(ex.StatusCode == HttpStatusCode.NotFound) { Console.WriteLine("None."); } } Console.WriteLine("========DATASET-EDIT========"); datasetUri = UriFactory.CreateDocumentCollectionUri( CosmosOptions.Database, CosmosOptions.UserDataCollection); dataset = (await client.CreateDocumentQuery( datasetUri, new FeedOptions { PartitionKey = new PartitionKey(WellKnownIds.DatasetEditDatasetId.ToString()) }) .Where(d => d.Id == DatasetId) .AsDocumentQuery() .GetQueryResultsAsync()) .SingleOrDefault(); if (dataset == null) { Console.WriteLine($"No Dataset Edit found for {DatasetId}."); } else { Console.WriteLine(dataset.ToString()); Console.WriteLine("========ATTACHMENT========"); try { var documentLink = UriFactory.CreateAttachmentUri( CosmosOptions.Database, CosmosOptions.UserDataCollection, DatasetId, "Content"); var response = await client.ReadAttachmentAsync(documentLink, new RequestOptions { PartitionKey = new PartitionKey(WellKnownIds.DatasetEditDatasetId.ToString()) }); Console.WriteLine(response?.Resource?.ToString()); } catch (DocumentClientException ex) when(ex.StatusCode == HttpStatusCode.NotFound) { Console.WriteLine("None."); } } Console.WriteLine("========NOMINATION========"); datasetUri = UriFactory.CreateDocumentCollectionUri( CosmosOptions.Database, CosmosOptions.UserDataCollection); dataset = (await client.CreateDocumentQuery( datasetUri, new FeedOptions { PartitionKey = new PartitionKey(WellKnownIds.DatasetNominationDatasetId.ToString()) }) .Where(d => d.Id == DatasetId) .AsDocumentQuery() .GetQueryResultsAsync()) .SingleOrDefault(); if (dataset == null) { Console.WriteLine($"No Nomination found for {DatasetId}."); } else { Console.WriteLine(dataset.ToString()); Console.WriteLine("========ATTACHMENT========"); try { var documentLink = UriFactory.CreateAttachmentUri( CosmosOptions.Database, CosmosOptions.UserDataCollection, DatasetId, "Content"); var response = await client.ReadAttachmentAsync(documentLink, new RequestOptions { PartitionKey = new PartitionKey(WellKnownIds.DatasetNominationDatasetId.ToString()) }); Console.WriteLine(response?.Resource?.ToString()); } catch (DocumentClientException ex) when(ex.StatusCode == HttpStatusCode.NotFound) { Console.WriteLine("None."); } } } Console.WriteLine("================"); return(0); }