public async Task ReadSimpleDocuments_FilterIntegerProperty_MatchingDataRead()
{
var documents = SampleData.GetSimpleDocuments(5);
foreach (var document in documents)
{
var bsonDocument = new BsonDocument(document);
await Collection.InsertOneAsync(bsonDocument);
document["_id"] = bsonDocument["_id"].ToString();
}
var configuration =
Mocks
.Of <IMongoDbSourceAdapterConfiguration>(c =>
c.ConnectionString == Settings.MongoConnectionString &&
c.Collection == CollectionName &&
c.Query == "{ IntegerProperty: { $gt: 2 } }")
.First();
List <IDataItem> readResults;
using (var adapter = await new MongoDbSourceAdapterFactory()
.CreateAsync(configuration, DataTransferContextMock.Instance, CancellationToken.None))
{
readResults = await ReadDataAsync(adapter);
}
DataItemCollectionAssert.AreEquivalent(
documents.Where(d => (int)d["IntegerProperty"] > 2),
readResults,
TestResources.InvalidDocumentsRead);
}
public async Task ReadSampleData_AllDataRead()
{
var configuration =
Mocks
.Of <IDocumentDbSourceAdapterConfiguration>(c =>
c.ConnectionString == ConnectionString &&
c.Collection == CollectionName &&
c.InternalFields == false)
.First();
var readResults = new List <IDataItem>();
using (var adapter = await new DocumentDbSourceAdapterFactory()
.CreateAsync(configuration, DataTransferContextMock.Instance))
{
IDataItem dataItem;
var readOutput = new ReadOutputByRef();
while ((dataItem = await adapter.ReadNextAsync(readOutput, CancellationToken.None)) != null)
{
readResults.Add(dataItem);
Assert.IsNotNull(readOutput.DataItemId, CommonTestResources.MissingDataItemId);
readOutput.Wipe();
}
}
DataItemCollectionAssert.AreEquivalent(sampleData, readResults, TestResources.InvalidDocumentsRead);
}
public async Task ReadData_ApplyBelow40Query_MatchingDocumentsRead()
{
var configuration =
Mocks
.Of <IRavenDbSourceAdapterConfiguration>(c =>
c.ConnectionString == connectionString &&
c.Query == "Age: [* TO 40]" &&
c.Index == "AllDocs/ByAge" &&
c.ExcludeId == true)
.First();
var readResults = new List <IDataItem>();
using (var adapter = await new RavenDbSourceAdapterFactory()
.CreateAsync(configuration, DataTransferContextMock.Instance))
{
IDataItem dataItem;
var readOutput = new ReadOutputByRef();
while ((dataItem = await adapter.ReadNextAsync(readOutput, CancellationToken.None)) != null)
{
readResults.Add(dataItem);
Assert.IsNotNull(readOutput.DataItemId, CommonTestResources.MissingDataItemId);
readOutput.Wipe();
}
}
DataItemCollectionAssert.AreEquivalent(
sampleData.Where(d => (int)d["Age"] <= 40), readResults, TestResources.InvalidDocumentsRead);
}
public async Task ReadCompressedFile_AllFieldsRead()
{
DataItemCollectionAssert.AreEquivalent(
ExpectedData,
await ReadData(@"InputData\mongoexport.gz", true),
TestResources.InvalidDocumentsRead);
}
public async Task ReadSampleFileWithEmptyLines_AllFieldsRead()
{
DataItemCollectionAssert.AreEquivalent(
ExpectedData,
await ReadData(@"InputData\mongoexport_emptylines.json", false),
TestResources.InvalidDocumentsRead);
}
public async Task ReadSimpleDocument_AllFieldsRead()
{
var documents = SampleData.GetSimpleDocuments(5);
foreach (var document in documents)
{
var hashDocument = new MongoDocument(document);
Collection.Insert(hashDocument);
document["_id"] = hashDocument.Id.ToString();
}
var readResults = new List <IDataItem>();
using (var adapter = await new MongoDbSourceAdapterFactory().CreateAsync(Configuration, DataTransferContextMock.Instance))
{
var readOutput = new ReadOutputByRef();
IDataItem dataItem;
while ((dataItem = await adapter.ReadNextAsync(readOutput, CancellationToken.None)) != null)
{
readResults.Add(dataItem);
Assert.IsNotNull(readOutput.DataItemId, CommonTestResources.MissingDataItemId);
readOutput.Wipe();
}
}
DataItemCollectionAssert.AreEquivalent(documents, readResults, TestResources.InvalidDocumentsRead);
}
public async Task ReadEntitiesWithFilter_First100RecordsRead()
{
const string IntegerPropertyName = "IntegerProperty";
var configuration = Mocks
.Of <IAzureTableSourceAdapterConfiguration>(c =>
c.ConnectionString == Settings.AzureStorageConnectionString &&
c.Table == tableName &&
c.Filter == IntegerPropertyName + " lt 100" &&
c.InternalFields == AzureTableInternalFields.None)
.First();
var readResults = new List <IDataItem>();
using (var adapter = await new AzureTableSourceAdapterFactory()
.CreateAsync(configuration, DataTransferContextMock.Instance))
{
IDataItem dataItem;
var readOutput = new ReadOutputByRef();
while ((dataItem = await adapter.ReadNextAsync(readOutput, CancellationToken.None)) != null)
{
readResults.Add(dataItem);
Assert.IsNotNull(readOutput.DataItemId, CommonTestResources.MissingDataItemId);
readOutput.Wipe();
}
}
DataItemCollectionAssert.AreEquivalent(sampleData.Where(e => (int)e[IntegerPropertyName] < 100), readResults, TestResources.InvalidDocumentsRead);
}
protected static void VerifyData(IEnumerable <IDataItem> expected, IEnumerable <IReadOnlyDictionary <string, object> > actual)
{
var persistedData = actual
.Select(i => new DictionaryDataItem(i
// Exclude all internal properties
.Where(p => !p.Key.StartsWith("_"))
.ToDictionary(p => p.Key, p => p.Value)))
.ToList();
DataItemCollectionAssert.AreEquivalent(expected, persistedData, TestResources.InvalidDocumentsPersisted);
}
public async Task Read_QuerySampleDataForIntegerOver50_FilteredDataRead()
{
var configuration =
Mocks
.Of <IDynamoDbSourceAdapterConfiguration>(c =>
c.ConnectionString == Settings.DynamoDbConnectionString &&
c.Request == File.ReadAllText(@"DynamoDb\FilteredQueryTemplate.json").Replace("%TABLENAME%", TableName))
.First();
DataItemCollectionAssert.AreEquivalent(Data.Where(d => (int)d["IntegerProperty"] > 50),
await ReadData(configuration), TestResources.InvalidDocumentsRead);
}
public async Task Read_QuerySampleData_AllDataRead()
{
var configuration =
Mocks
.Of <IDynamoDbSourceAdapterConfiguration>(c =>
c.ConnectionString == Settings.DynamoDbConnectionString &&
c.Request == File.ReadAllText(@"DynamoDb\SimpleQueryTemplate.json").Replace("%TABLENAME%", TableName))
.First();
DataItemCollectionAssert.AreEquivalent(Data,
await ReadData(configuration), TestResources.InvalidDocumentsRead);
}
public async Task ReadAllEntities_AllDataRead()
{
var configuration = Mocks
.Of <IAzureTableSourceAdapterConfiguration>(c =>
c.ConnectionString == Settings.AzureStorageConnectionString &&
c.Table == tableName &&
c.InternalFields == AzureTableInternalFields.None)
.First();
DataItemCollectionAssert.AreEquivalent(sampleData,
await ReadData(configuration), TestResources.InvalidDocumentsRead);
}
public async Task ReadSampleData_AllDataRead()
{
var configuration =
Mocks
.Of <IRavenDbSourceAdapterConfiguration>(c =>
c.ConnectionString == connectionString &&
c.ExcludeId == true)
.First();
DataItemCollectionAssert.AreEquivalent(sampleData,
await ReadData(configuration), TestResources.InvalidDocumentsRead);
}
public async Task ReadData_ApplyBelow40Query_MatchingDocumentsRead()
{
var configuration =
Mocks
.Of <IRavenDbSourceAdapterConfiguration>(c =>
c.ConnectionString == connectionString &&
c.Query == "Age: [* TO 40]" &&
c.Index == "AllDocs/ByAge" &&
c.ExcludeId == true)
.First();
DataItemCollectionAssert.AreEquivalent(sampleData.Where(d => (int)d["Age"] <= 40),
await ReadData(configuration), TestResources.InvalidDocumentsRead);
}
public async Task ReadEntitiesWithFilter_First100RecordsRead()
{
const string IntegerPropertyName = "IntegerProperty";
var configuration = Mocks
.Of <IAzureTableSourceAdapterConfiguration>(c =>
c.ConnectionString == Settings.AzureStorageConnectionString &&
c.Table == tableName &&
c.Filter == IntegerPropertyName + " lt 100" &&
c.InternalFields == AzureTableInternalFields.None)
.First();
DataItemCollectionAssert.AreEquivalent(sampleData.Where(e => (int)e[IntegerPropertyName] < 100),
await ReadData(configuration), TestResources.InvalidDocumentsRead);
}
public async Task ReadSampleData_IncludeRowId_AllDataRead()
{
var configuration =
Mocks
.Of <IHBaseSourceAdapterConfiguration>(c =>
c.ConnectionString == Settings.HBaseConnectionString &&
c.Table == tableName)
.First();
DataItemCollectionAssert.AreEquivalent(
sampleData.Select(i => i
.ToDictionary(
p => p.Key == HBaseHelper.RowIdPropertyName ? p.Key : "data:" + p.Key,
p => (object)p.Value.ToString())),
await ReadData(configuration),
TestResources.InvalidDocumentsRead);
}
public async Task ReadSampleData_AllDataRead()
{
var configuration =
Mocks
.Of <IDocumentDbSourceAdapterConfiguration>(c =>
c.ConnectionString == ConnectionString &&
c.Collection == CollectionName &&
c.InternalFields == false)
.First();
List <IDataItem> readResults;
using (var adapter = await new DocumentDbSourceAdapterFactory()
.CreateAsync(configuration, DataTransferContextMock.Instance, CancellationToken.None))
{
readResults = await ReadDataAsync(adapter);
}
DataItemCollectionAssert.AreEquivalent(sampleData, readResults, TestResources.InvalidDocumentsRead);
}
public async Task ReadEntitiesWithProjection_OnlyStringPropertyRead()
{
const string StringPropertyName = "StringProperty";
var configuration = Mocks
.Of <IAzureTableSourceAdapterConfiguration>(c =>
c.ConnectionString == Settings.AzureStorageConnectionString &&
c.Table == tableName &&
c.Projection == new[] { StringPropertyName } &&
c.InternalFields == AzureTableInternalFields.None)
.First();
DataItemCollectionAssert.AreEquivalent(
sampleData
.Select(i => new Dictionary <string, object> {
{ StringPropertyName, i[StringPropertyName] }
})
.ToArray(),
await ReadData(configuration),
TestResources.InvalidDocumentsRead);
}
public async Task ReadSimpleDocument_AllFieldsRead()
{
var documents = SampleData.GetSimpleDocuments(5);
foreach (var document in documents)
{
var hashDocument = new MongoDocument(document);
Collection.Insert(hashDocument);
document["_id"] = hashDocument.Id.ToString();
}
List <IDataItem> readResults;
using (var adapter = await new MongoDbSourceAdapterFactory()
.CreateAsync(Configuration, DataTransferContextMock.Instance, CancellationToken.None))
{
readResults = await ReadDataAsync(adapter);
}
DataItemCollectionAssert.AreEquivalent(documents, readResults, TestResources.InvalidDocumentsRead);
}
public async Task ReadSampleData_AllDataRead()
{
const string TestCollectionName = "TestCollection";
const string TestQuery = "SELECT * FROM Collection";
const int NumberOfItems = 100;
var sampleData = SampleData.GetSimpleDocuments(NumberOfItems);
var client = Mocks
.Of <IDocumentDbReadClient>()
.Where(m =>
m.QueryDocumentsAsync(TestCollectionName, TestQuery, CancellationToken.None) ==
Task.FromResult <IAsyncEnumerator <IReadOnlyDictionary <string, object> > >(
new AsyncEnumeratorMock <Dictionary <string, object> >(sampleData.OfType <Dictionary <string, object> >().GetEnumerator())))
.First();
var configuration = Mocks
.Of <IDocumentDbSourceAdapterInstanceConfiguration>()
.Where(m =>
m.Collection == TestCollectionName &&
m.Query == TestQuery)
.First();
var readResults = new List <IDataItem>();
using (var adapter = new DocumentDbSourceAdapter(client, PassThroughTransformation.Instance, configuration))
{
await adapter.InitializeAsync(CancellationToken.None);
IDataItem dataItem;
while ((dataItem = await adapter.ReadNextAsync(ReadOutputByRef.None, CancellationToken.None)) != null)
{
readResults.Add(dataItem);
}
}
DataItemCollectionAssert.AreEquivalent(sampleData, readResults, TestResources.InvalidDataItemsTransferred);
}
public async Task ReadEntitiesWithProjection_OnlyStringPropertyRead()
{
const string StringPropertyName = "StringProperty";
var configuration = Mocks
.Of <IAzureTableSourceAdapterConfiguration>(c =>
c.ConnectionString == Settings.AzureStorageConnectionString &&
c.Table == tableName &&
c.Projection == new[] { StringPropertyName } &&
c.InternalFields == AzureTableInternalFields.None)
.First();
var readResults = new List <IDataItem>();
using (var adapter = await new AzureTableSourceAdapterFactory()
.CreateAsync(configuration, DataTransferContextMock.Instance))
{
IDataItem dataItem;
var readOutput = new ReadOutputByRef();
while ((dataItem = await adapter.ReadNextAsync(readOutput, CancellationToken.None)) != null)
{
readResults.Add(dataItem);
Assert.IsNotNull(readOutput.DataItemId, CommonTestResources.MissingDataItemId);
readOutput.Wipe();
}
}
DataItemCollectionAssert.AreEquivalent(
sampleData
.Select(i => new Dictionary <string, object> {
{ StringPropertyName, i[StringPropertyName] }
})
.ToArray(),
readResults,
TestResources.InvalidDocumentsRead);
}
