internal async Task SaveOdiDocuments(dynamic doc, StorageAdapter adapter, string newName) { if (doc == null) { throw new ArgumentNullException($"Failed to persist document because {nameof(doc)} is null."); } // Ask the adapter to make it happen. try { var oldDocumentPath = doc.DocumentPath; var newDocumentPath = oldDocumentPath.Substring(0, oldDocumentPath.Length - FetchOdiExtension().Length) + newName; var content = JsonConvert.SerializeObject(doc, Formatting.Indented, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore, ContractResolver = new CamelCasePropertyNamesContractResolver() }); await adapter.WriteAsync(newDocumentPath, content); } catch (Exception e) { Logger.Error(nameof(PersistenceLayer), (ResolveContext)this.Ctx, $"Failed to write to the file '{doc.DocumentPath}' for reason {e.Message}.", nameof(SaveOdiDocuments)); } // Save linked documents. if (doc.LinkedDocuments != null) { foreach (var linkedDoc in doc.LinkedDocuments) { await SaveOdiDocuments(linkedDoc, adapter, newName); } } }
public async Task WhenNoEnrollmentProvided_ThenDeviceCertiticateAuthenticationIsOff( [Credential(Role = PredefinedRole.ComputeViewer)] ResourceTask <ICredential> credential) { var adapter = new StorageAdapter(await credential); Assert.IsFalse(adapter.IsDeviceCertiticateAuthenticationEnabled); }
/// <summary> /// Mounts a namespace to the specified adapter. /// </summary> public void Mount(string nameSpace, StorageAdapter adapter) { if (string.IsNullOrEmpty(nameSpace)) { Logger.Error(nameof(StorageManager), this.Ctx, "The namespace cannot be null or empty.", nameof(Mount)); return; } if (adapter != null) { if (adapter is StorageAdapterBase adapterBase) { adapterBase.Ctx = this.Ctx; } this.NamespaceAdapters[nameSpace] = adapter; CdmFolderDefinition fd = new CdmFolderDefinition(this.Ctx, ""); fd.Corpus = this.Corpus as CdmCorpusDefinition; fd.Namespace = nameSpace; fd.FolderPath = "/"; this.NamespaceFolders[nameSpace] = fd; this.systemDefinedNamespaces.Remove(nameSpace); } else { Logger.Error(nameof(StorageManager), this.Ctx, "The adapter cannot be null.", nameof(Mount)); } }
/// <summary> /// Takes a corpus path, figures out the right adapter to use and then returns an adapter domain path. /// </summary> public string CorpusPathToAdapterPath(string corpusPath) { if (string.IsNullOrEmpty(corpusPath)) { Logger.Error(nameof(CdmCorpusDefinition), (ResolveContext)this.Ctx, $"The corpus path is null or empty", nameof(CorpusPathToAdapterPath)); return(null); } string result = ""; // break the corpus path into namespace and ... path Tuple <string, string> pathTuple = SplitNamespacePath(corpusPath); string nameSpace = pathTuple.Item1; if (string.IsNullOrWhiteSpace(nameSpace)) { nameSpace = this.DefaultNamespace; } // get the adapter registered for this namespace StorageAdapter namespaceAdapter = this.FetchAdapter(nameSpace); if (namespaceAdapter == null) { Logger.Error(nameof(CdmCorpusDefinition), (ResolveContext)this.Ctx, $"The namespace '{nameSpace}' has not been registered", nameof(CorpusPathToAdapterPath)); } else { // ask the storage adapter to 'adapt' this path result = namespaceAdapter.CreateAdapterPath(pathTuple.Item2); } return(result); }
/// <summary> /// Mounts a namespace to the specified adapter. /// </summary> public void Mount(string nameSpace, StorageAdapter adapter) { using (Logger.EnterScope(nameof(StorageManager), Ctx, nameof(Mount))) { if (string.IsNullOrEmpty(nameSpace)) { Logger.Error(this.Ctx, Tag, nameof(Mount), null, CdmLogCode.ErrStorageNullNamespace); return; } if (adapter != null) { if (adapter is StorageAdapterBase adapterBase) { adapterBase.Ctx = this.Ctx; } this.NamespaceAdapters[nameSpace] = adapter; CdmFolderDefinition fd = new CdmFolderDefinition(this.Ctx, ""); fd.Corpus = this.Corpus as CdmCorpusDefinition; fd.Namespace = nameSpace; fd.FolderPath = "/"; this.NamespaceFolders[nameSpace] = fd; this.systemDefinedNamespaces.Remove(nameSpace); } else { Logger.Error(this.Ctx, Tag, nameof(Mount), null, CdmLogCode.ErrStorageNullAdapter); } } }
/// <summary> /// Allow replacing a storage adapter with another one for testing, leaving folders intact. /// </summary> internal void SetAdapter(string nameSpace, StorageAdapter adapter) { if (adapter != null) { this.NamespaceAdapters[nameSpace] = adapter; } }
public void CreateStorageAdapterExceptionIsWrapped() { var adapter = new StorageAdapter("path/to/database/file.db") { _databaseDirectory = Mock.Of <Directory>() }; var sourceException = new System.IO.PathTooLongException(); // Mock the directory to throw when created. Mock.Get(adapter._databaseDirectory).Setup(directory => directory.Create()).Throws(sourceException); const string databaseDirectory = "databaseDirectory"; var databasePath = System.IO.Path.Combine(databaseDirectory, "database.db"); Exception actualException = null; try { adapter.InitializeStorageAsync().Wait(); } catch (AggregateException ex) { actualException = ex.InnerException; } Assert.IsInstanceOfType(actualException, typeof(StorageException)); Assert.IsInstanceOfType(actualException?.InnerException, typeof(System.IO.PathTooLongException)); }
public async Task WhenUserNotInRole_ThenListBucketsAsyncThrowsResourceAccessDeniedException( [Credential(Role = PredefinedRole.StorageObjectViewer)] ResourceTask <ICredential> credential) { var adapter = new StorageAdapter(await credential); AssertEx.ThrowsAggregateException <ResourceAccessDeniedException>( () => adapter.ListBucketsAsync( TestProject.ProjectId, CancellationToken.None).Wait()); }
public async Task WhenUserNotInRole_ThenDownloadObjectToMemoryAsyncThrowsResourceAccessDeniedException( [Credential(Role = PredefinedRole.ComputeViewer)] ResourceTask <ICredential> credential) { var adapter = new StorageAdapter(await credential); AssertEx.ThrowsAggregateException <ResourceAccessDeniedException>( () => adapter.DownloadObjectToMemoryAsync( SampleLocator, CancellationToken.None).Wait()); }
public void CreateStorageAdapterDoesNotCreateDirectoryWhenNull() { var adapter = new StorageAdapter("databaseAtRoot.db"); // Verify that a directory object was not created. Assert.IsNull(adapter._databaseDirectory); // Should not crash even if directory is null. adapter.InitializeStorageAsync().Wait(); }
public async Task ShouldThrowExceptionWhenTryingToLoadNoteThatDoesNotExist() { //GIVEN var id = Any.Guid(); using var storage = new StorageAdapter(); //WHEN - THEN (await storage.Awaiting(s => s.UserTodosDao.Load(id, Any.CancellationToken())) .Should().ThrowExactlyAsync <NoteNotFoundException>()).WithMessage($"*{id}"); }
private CdmCorpusDefinition CreateTestCorpus(StorageAdapter adapter) { var cdmCorpus = new CdmCorpusDefinition(); cdmCorpus.SetEventCallback(new EventCallback { Invoke = CommonDataModelLoader.ConsoleStatusReport }, CdmStatusLevel.Warning); cdmCorpus.Storage.Mount("local", adapter); cdmCorpus.Storage.DefaultNamespace = "local"; return(cdmCorpus); }
public void InitializeStorageCreatesStorageDirectory() { var adapter = new StorageAdapter("path/to/database/file.db"); // Verify that a directory object was created. Assert.IsNotNull(adapter._databaseDirectory); // Replace the directory with a mock and initialize. adapter._databaseDirectory = Mock.Of <Directory>(); adapter.InitializeStorageAsync().Wait(); Mock.Get(adapter._databaseDirectory).Verify(directory => directory.Create()); }
public async Task WhenObjectExists_ThenDownloadObjectToMemoryAsyncReturnsObject( [Credential(Role = PredefinedRole.StorageObjectViewer)] ResourceTask <ICredential> credential) { var adapter = new StorageAdapter(await credential); var stream = await adapter.DownloadObjectToMemoryAsync( SampleLocator, CancellationToken.None); Assert.AreEqual( SampleData, new StreamReader(stream).ReadToEnd()); }
public void TestInitialize() { Microsoft.AppCenter.Utils.Constants.AppCenterDatabasePath = DatabasePath; try { System.IO.File.Delete(DatabasePath); } catch { // Db file might not exist or might fail to be deleted. } _adapter = new StorageAdapter(); }
public void InitializeTest() { _mockNetworkAdapter = Mock.Of <IHttpNetworkAdapter>(); _storageAdapter = new StorageAdapter(); _storagePath = $"{Guid.NewGuid()}.db"; var storage = new Storage.Storage(_storageAdapter, _storagePath); var ingestion = new IngestionHttp(_mockNetworkAdapter); var channelGroup = new ChannelGroup(ingestion, storage, "app secret"); Crashes.Instance = new Crashes(); Crashes.SetEnabledAsync(true).Wait(); Crashes.Instance.OnChannelGroupReady(channelGroup, "app secret"); }
public async Task WhenBucketExists_ThenListBucketsAsyncReturnsObject( [Credential(Role = PredefinedRole.StorageAdmin)] ResourceTask <ICredential> credential) { var adapter = new StorageAdapter(await credential); var buckets = await adapter.ListBucketsAsync( TestProject.ProjectId, CancellationToken.None); Assert.IsNotNull(buckets); CollectionAssert.Contains( buckets.Select(o => o.Name).ToList(), GcsTestData.Bucket); }
public async Task WhenObjectExists_ThenListObjectsAsyncReturnsObject( [Credential(Role = PredefinedRole.StorageObjectViewer)] ResourceTask <ICredential> credential) { var adapter = new StorageAdapter(await credential); var objects = await adapter.ListObjectsAsync( GcsTestData.Bucket, null, CancellationToken.None); var objectNames = objects.Select(o => o.Name).ToList(); CollectionAssert.Contains(objectNames, SampleLocator.ObjectName); }
public void TestCleanup() { try { // Try to clean up resources but don't fail the test if that throws an error. _storageAdapter.Dispose(); _storageAdapter = null; File.Delete(_storagePath); } catch { // No-op. } }
/// <summary> /// Mounts a namespace to the specified adapter. /// </summary> public void Mount(string nameSpace, StorageAdapter adapter) { if (adapter != null) { this.NamespaceAdapters[nameSpace] = adapter; CdmFolderDefinition fd = new CdmFolderDefinition(this.Ctx, ""); fd.Corpus = this.Corpus as CdmCorpusDefinition; fd.Namespace = nameSpace; this.NamespaceFolders[nameSpace] = fd; } else { Logger.Error(nameof(StorageManager), this.Ctx, "The adapter cannot be null.", "Mount"); } }
/// <summary> /// Allow replacing a storage adapter with another one for testing, leaving folders intact. /// </summary> internal void SetAdapter(string nameSpace, StorageAdapter adapter) { if (string.IsNullOrEmpty(nameSpace)) { Logger.Error(this.Ctx, Tag, nameof(SetAdapter), null, CdmLogCode.ErrStorageNullNamespace); return; } if (adapter != null) { this.NamespaceAdapters[nameSpace] = adapter; } else { Logger.Error(this.Ctx, Tag, nameof(SetAdapter), null, CdmLogCode.ErrStorageNullAdapter); } }
public void FailToGetALog() { // Prepare data. StorageAdapter adapter = new StorageAdapter(); adapter.Initialize(_databasePath); var tables = new[] { ColumnIdName, ColumnChannelName, ColumnLogName }; var types = new[] { "INTEGER PRIMARY KEY AUTOINCREMENT", "TEXT NOT NULL", "TEXT NOT NULL" }; adapter.CreateTable(TableName, tables, types); adapter.Insert(TableName, tables, new List<object[]> { new object[] { 100, StorageTestChannelName, "good luck deserializing me!" } }); var storage = new Microsoft.AppCenter.Storage.Storage(adapter, _databasePath); var logs = new List<Log>(); var batchId = storage.GetLogsAsync(StorageTestChannelName, 4, logs).RunNotAsync(); var count = storage.CountLogsAsync(StorageTestChannelName).RunNotAsync(); Assert.IsNull(batchId); Assert.AreEqual(0, logs.Count); Assert.AreEqual(0, count); }
/// <summary> /// Allow replacing a storage adapter with another one for testing, leaving folders intact. /// </summary> internal void SetAdapter(string nameSpace, StorageAdapter adapter) { if (string.IsNullOrEmpty(nameSpace)) { Logger.Error(nameof(StorageManager), this.Ctx, "The namespace cannot be null or empty.", nameof(SetAdapter)); return; } if (adapter != null) { this.NamespaceAdapters[nameSpace] = adapter; } else { Logger.Error(nameof(StorageManager), this.Ctx, "The adapter cannot be null.", nameof(SetAdapter)); } }
public ServiceLogicRoot( TokenValidationParameters tokenValidationParameters, ILoggerFactory loggerFactory) { var storageAdapter = new StorageAdapter(); var appLogicRoot = new AppLogicRoot( storageAdapter.UserTodosDao, new NewGuidBasedIdSequence()); var endpointsAdapter = new EndpointsAdapter( appLogicRoot.TodoCommandFactory, appLogicRoot.TodoCommandFactory, tokenValidationParameters, LoggingAdapter.CreateServiceSupport(loggerFactory)); _endpointsAdapter = endpointsAdapter; _storageAdapter = new StorageAdapter(); }
public async Task ShouldAllowReadingSavedNotes() { //GIVEN using var storage = new StorageAdapter(); var id = Any.Guid(); var savedNote = Any.Instance <CreateTodoRequestData>(); await storage.UserTodosDao.Save(id, savedNote, Any.CancellationToken()); //WHEN var(guid, title, content, immutableHashSet) = await storage.UserTodosDao.Load(id, Any.CancellationToken()); //THEN content.Should().Be(savedNote.Content); title.Should().Be(savedNote.Title); guid.Should().Be(id); immutableHashSet.Should().Equal(ImmutableHashSet <Guid> .Empty); }
public void TestCleanup() { try { _adapter.Dispose(); _adapter = null; } catch (Exception e) { Assert.Fail("Failed to dispose storage adapter: {0}", e.Message); } try { System.IO.File.Delete(DatabasePath); } catch { // Db file might not exist or might fail to be deleted. } }
/// <summary> /// Takes a corpus path, figures out the right adapter to use and then returns an adapter domain path. /// </summary> public string CorpusPathToAdapterPath(string corpusPath) { using (Logger.EnterScope(nameof(StorageManager), Ctx, nameof(CorpusPathToAdapterPath))) { if (string.IsNullOrEmpty(corpusPath)) { Logger.Error(this.Ctx, Tag, nameof(CorpusPathToAdapterPath), null, CdmLogCode.ErrStorageNullCorpusPath); return(null); } string result = ""; // break the corpus path into namespace and ... path Tuple <string, string> pathTuple = StorageUtils.SplitNamespacePath(corpusPath); if (pathTuple == null) { Logger.Error(this.Ctx, Tag, nameof(CorpusPathToAdapterPath), null, CdmLogCode.ErrStorageNullCorpusPath); return(null); } string nameSpace = pathTuple.Item1; if (string.IsNullOrWhiteSpace(nameSpace)) { nameSpace = this.DefaultNamespace; } // get the adapter registered for this namespace StorageAdapter namespaceAdapter = this.FetchAdapter(nameSpace); if (namespaceAdapter == null) { Logger.Error(this.Ctx, Tag, nameof(CorpusPathToAdapterPath), null, CdmLogCode.ErrStorageNamespaceNotRegistered, nameSpace); } else { // ask the storage adapter to 'adapt' this path result = namespaceAdapter.CreateAdapterPath(pathTuple.Item2); } return(result); } }
/// <summary> /// Saves adapters config into a file. /// </summary> /// <param name="name">The name of a file.</param> /// <param name="adapter">The adapter used to save the config to a file.</param> public async Task SaveAdaptersConfigAsync(string name, StorageAdapter adapter) { await adapter.WriteAsync(name, FetchConfig()); }
/// <summary> /// Loads a document from the folder path. /// </summary> /// <param name="folder">The folder that contains the document we want to load.</param> /// <param name="docName">The document name.</param> /// <param name="docContainer">The loaded document, if it was previously loaded.</param> /// <returns>The loaded document.</returns> internal async Task <CdmDocumentDefinition> LoadDocumentFromPathAsync(CdmFolderDefinition folder, string docName, CdmDocumentDefinition docContainer) { // This makes sure date values are consistently parsed exactly as they appear. // Default behavior auto formats date values. JsonConvert.DefaultSettings = () => new JsonSerializerSettings { DateParseHandling = DateParseHandling.None }; dynamic docContent = null; string jsonData = null; DateTimeOffset?fsModifiedTime = null; string docPath = folder.FolderPath + docName; StorageAdapter adapter = this.Corpus.Storage.FetchAdapter(folder.Namespace); try { if (adapter.CanRead()) { jsonData = await adapter.ReadAsync(docPath); fsModifiedTime = await adapter.ComputeLastModifiedTimeAsync(docPath); Logger.Info(nameof(PersistenceLayer), this.Ctx, $"read file: {docPath}", nameof(LoadDocumentFromPathAsync)); } } catch (Exception e) { Logger.Error(nameof(PersistenceLayer), (ResolveContext)this.Ctx, $"Could not read '{docPath}' from the '{folder.Namespace}' namespace. Reason '{e.Message}'", nameof(LoadDocumentFromPathAsync)); return(null); } if (string.IsNullOrWhiteSpace(docName)) { Logger.Error(nameof(PersistenceLayer), (ResolveContext)this.Ctx, $"Document name cannot be null or empty.", nameof(LoadDocumentFromPathAsync)); return(null); } // If loading an odi.json/model.json file, check that it is named correctly. if (docName.EndWithOrdinalIgnoreCase(FetchOdiExtension()) && !docName.EqualsWithOrdinalIgnoreCase(FetchOdiExtension())) { Logger.Error(nameof(PersistenceLayer), (ResolveContext)this.Ctx, $"Failed to load '{docName}', as it's not an acceptable file name. It must be {FetchOdiExtension()}.", nameof(LoadDocumentFromPathAsync)); return(null); } if (docName.EndWithOrdinalIgnoreCase(FetchModelJsonExtension()) && !docName.EqualsWithOrdinalIgnoreCase(FetchModelJsonExtension())) { Logger.Error(nameof(PersistenceLayer), (ResolveContext)this.Ctx, $"Failed to load '{docName}', as it's not an acceptable file name. It must be {FetchModelJsonExtension()}.", nameof(LoadDocumentFromPathAsync)); return(null); } // Fetch the correct persistence class to use. Type persistenceClass = FetchRegisteredPersistenceFormat(docName); if (persistenceClass != null) { try { MethodInfo method = persistenceClass.GetMethod(nameof(FromData)); object[] parameters = new object[] { this.Ctx, docName, jsonData, folder }; // Check if FromData() is asynchronous for this persistence class. if (!isRegisteredPersistenceAsync.ContainsKey(persistenceClass)) { // Cache whether this persistence class has async methods. isRegisteredPersistenceAsync.Add(persistenceClass, (bool)persistenceClass.GetField("IsPersistenceAsync").GetValue(null)); } if (isRegisteredPersistenceAsync[persistenceClass]) { var task = (Task)method.Invoke(null, parameters); await task; docContent = task.GetType().GetProperty("Result").GetValue(task); } else { docContent = method.Invoke(null, parameters); } } catch (Exception e) { Logger.Error(nameof(PersistenceLayer), (ResolveContext)this.Ctx, $"Could not convert '{docName}'. Reason '{e.Message}'", nameof(LoadDocumentFromPathAsync)); return(null); } } else { // Could not find a registered persistence class to handle this document type. Logger.Error(nameof(PersistenceLayer), (ResolveContext)this.Ctx, $"Could not find a persistence class to handle the file '{docName}'", nameof(LoadDocumentFromPathAsync)); return(null); } // Add document to the folder, this sets all the folder/path things, caches name to content association and may trigger indexing on content if (docContent != null) { if (docContainer != null) { // there are situations where a previously loaded document must be re-loaded. // the end of that chain of work is here where the old version of the document has been removed from // the corpus and we have created a new document and loaded it from storage and after this call we will probably // add it to the corpus and index it, etc. // it would be really rude to just kill that old object and replace it with this replicant, especially because // the caller has no idea this happened. so... sigh ... instead of returning the new object return the one that // was just killed off but make it contain everything the new document loaded. docContent = docContent.Copy(new ResolveOptions(docContainer), docContainer) as CdmDocumentDefinition; } folder.Documents.Add(docContent, docName); docContent._fileSystemModifiedTime = fsModifiedTime; docContent.IsDirty = false; } return(docContent); }
public BookListService(StorageAdapter storageAdapter) { this.storageAdapter = storageAdapter; }
public static async Task <CdmDocumentDefinition> LoadDocumentFromPathAsync(CdmFolderDefinition folder, string docName) { // This makes sure date values are consistently parsed exactly as they appear. // Default behavior auto formats date values. JsonConvert.DefaultSettings = () => new JsonSerializerSettings { DateParseHandling = DateParseHandling.None }; CdmDocumentDefinition docContent = null; string jsonData = null; DateTimeOffset? fsModifiedTime = null; CdmCorpusContext ctx = folder.Ctx; string docPath = folder.FolderPath + docName; StorageAdapter adapter = ctx.Corpus.Storage.FetchAdapter(folder.Namespace); try { if (adapter.CanRead()) { jsonData = await adapter.ReadAsync(docPath); fsModifiedTime = await adapter.ComputeLastModifiedTimeAsync(docPath); Logger.Info(nameof(CdmFolderDefinition), ctx, $"read file: {docPath}", "LoadDocumentFromPathAsync"); } } catch (Exception e) { Logger.Error(nameof(CdmFolderDefinition), (ResolveContext)ctx, $"Could not read '{docPath}' from the '{ctx.Corpus.Namespace}' namespace. Reason '{e.Message}'", "LoadDocumentFromPathAsync"); return(null); } try { // Check file extensions, which performs a case-insensitive ordinal string comparison if (docPath.EndsWith(CdmCorpusDefinition.FetchManifestExtension(), StringComparison.OrdinalIgnoreCase) || docPath.EndsWith(CdmCorpusDefinition.FetchFolioExtension(), StringComparison.OrdinalIgnoreCase)) { docContent = ManifestPersistence.FromData(ctx, docName, folder.Namespace, folder.FolderPath, JsonConvert.DeserializeObject <ManifestContent>(jsonData)) as CdmDocumentDefinition; } else if (docPath.EndsWith(CdmCorpusDefinition.FetchModelJsonExtension(), StringComparison.OrdinalIgnoreCase)) { docContent = await ModelJson.ManifestPersistence.FromData(ctx, JsonConvert.DeserializeObject <Model>(jsonData), folder); } else { docContent = DocumentPersistence.FromData(ctx, docName, folder.Namespace, folder.FolderPath, JsonConvert.DeserializeObject <DocumentContent>(jsonData)); } } catch (Exception e) { Logger.Error(nameof(CdmFolderDefinition), (ResolveContext)ctx, $"Could not convert '{docPath}'. Reason '{e.Message}'", "LoadDocumentFromPathAsync"); return(null); } // Add document to the folder, this sets all the folder/path things, caches name to content association and may trigger indexing on content if (docContent != null) { folder.Documents.Add(docContent, docName); docContent._fileSystemModifiedTime = fsModifiedTime; docContent.IsDirty = false; } return(docContent); }