protected async Task<List<IDataItem>> ReadDataAsync(IDataSourceAdapter sourceAdapter) { var results = new List<IDataItem>(); IDataItem dataItem; var readOutput = new ReadOutputByRef(); while (true) { try { dataItem = await sourceAdapter.ReadNextAsync(readOutput, CancellationToken.None); } catch (NonFatalReadException) { continue; } if (dataItem == null) break; results.Add(dataItem); Assert.IsNotNull(readOutput.DataItemId, CommonTestResources.MissingDataItemId); readOutput.Wipe(); } return results; }
public async Task ExecuteAsync(IDataSourceAdapter source, IDataSinkAdapter sink, ITransferStatistics statistics, CancellationToken cancellation) { Guard.NotNull("source", source); Guard.NotNull("sink", sink); Guard.NotNull("statistics", statistics); var writeTasks = Enumerable .Range(0, sink.MaxDegreeOfParallelism) .Select(i => (Task)Task.FromResult<object>(null)) .ToArray(); var fatalExceptions = new List<Exception>(); var readOutput = new ReadOutputByRef(); IDataItem dataItem; while (!cancellation.IsCancellationRequested) { readOutput.Wipe(); try { dataItem = await source.ReadNextAsync(readOutput, cancellation); } catch (NonFatalReadException nonFatalException) { statistics.AddError(readOutput.DataItemId, nonFatalException); continue; } catch (Exception exception) { fatalExceptions.Add(exception); break; } if (dataItem == null || cancellation.IsCancellationRequested) break; var completed = await Task.WhenAny(writeTasks); writeTasks[Array.IndexOf(writeTasks, completed)] = TransferDataItem(sink, readOutput.DataItemId, dataItem, statistics, cancellation); } // Report completion to the sink try { await sink.CompleteAsync(cancellation); } catch (Exception exception) { fatalExceptions.Add(exception); } // Wait for all on-going writes to complete for (var index = 0; index < writeTasks.Length; ++index) await writeTasks[index]; // Throw fatal exceptions, if any if (fatalExceptions.Any()) throw new AggregateException(fatalExceptions); }
private async Task TestSqlDataSourceAsync(ISqlDataSourceAdapterConfiguration configuration) { var columnKeys = new[] { "IntColumn", "BitColumn", "NVarCharMaxColumn", "FloatColumn", "DateTimeColumn", }; var columnMappings = new Dictionary<string, string> { { columnKeys[0], "int primary key" }, { columnKeys[1], "bit" }, { columnKeys[2], "nvarchar(max)" }, { columnKeys[3], "float" }, { columnKeys[4], "datetime" }, }; var rows = new[] { new Dictionary<string, object> { { columnKeys[0], 1 }, { columnKeys[1], false }, { columnKeys[2], "String1" }, { columnKeys[3], 2.3 }, { columnKeys[4], GetSampleDateTime() } }, new Dictionary<string, object> { { columnKeys[0], 2 }, { columnKeys[1], true }, { columnKeys[2], "String2" }, { columnKeys[3], 4.5 }, { columnKeys[4], GetSampleDateTime() } }, }; using (var connection = new SqlConnection(ConnectionString)) { connection.Open(); try { CreateTable(connection, tableName, columnMappings); AddRows(connection, tableName, rows); using (var adapter = await new SqlDataSourceAdapterFactory() .CreateAsync(configuration, DataTransferContextMock.Instance, CancellationToken.None)) { var readOutput = new ReadOutputByRef(); for (var rowIndex = 0; rowIndex < rows.Length; ++rowIndex) { var dataItem = await adapter.ReadNextAsync(readOutput, CancellationToken.None); Assert.IsNotNull(dataItem, TestResources.MoreDataItemsExpected); Assert.IsNotNull(readOutput.DataItemId, CommonTestResources.MissingDataItemId); readOutput.Wipe(); VerifyDataItem(rows[rowIndex], dataItem); } } } finally { DropTable(connection, tableName); } } }
private static async Task ReadAndVerifyFields(IAzureTableSourceAdapterConfiguration configuration, string[] expectedInternalProperties) { using (var adapter = await new AzureTableSourceAdapterFactory() .CreateAsync(configuration, DataTransferContextMock.Instance, CancellationToken.None)) { IDataItem dataItem; var readOutput = new ReadOutputByRef(); while ((dataItem = await adapter.ReadNextAsync(readOutput, CancellationToken.None)) != null) { var fieldNames = dataItem.GetFieldNames().ToArray(); foreach (var expectedInternalProperty in expectedInternalProperties) { CollectionAssert.Contains(fieldNames, expectedInternalProperty, TestResources.MissingDataItemFieldFormat, expectedInternalProperty); Assert.IsNotNull(dataItem.GetValue(expectedInternalProperty), TestResources.EmptyDataItemFieldValueFormat, expectedInternalProperty); } Assert.IsNotNull(readOutput.DataItemId, CommonTestResources.MissingDataItemId); readOutput.Wipe(); } } }
private async Task TestSqlDataSourceAsync(ISqlDataSourceAdapterConfiguration configuration) { var columnKeys = new[] { "IntColumn", "GeographyColumn" }; var columnMappings = new Dictionary<string, string> { { columnKeys[0], "int primary key" }, { columnKeys[1], "geography" } }; var rows = new[] { new Dictionary<string, object> { { columnKeys[0], 1 }, { columnKeys[1], "geography::STGeomFromText('POINT(-122.16 43.656)', 4326)" } }, new Dictionary<string, object> { { columnKeys[0], 2 }, { columnKeys[1], "geography::STGeomFromText('LINESTRING(-122.360 47.656, -122.343 47.656 )', 4326)" } }, new Dictionary<string, object> { { columnKeys[0], 3 }, { columnKeys[1], "geography::STGeomFromText('POLYGON((-122.358 47.653 , -122.348 47.649, -122.348 47.658, -122.358 47.658, -122.358 47.653))', 4326)" } }, }; var expectedRows = new[] { new Dictionary<string, object> { { columnKeys[0], 1 }, { columnKeys[1], SampleData.Geospatial.AsPoint(new GeographyPosition(43.656, -122.16)) } }, new Dictionary<string, object> { { columnKeys[0], 2 }, { columnKeys[1], SampleData.Geospatial.AsLineString(new[] { new GeographyPosition(47.656, -122.360), new GeographyPosition(47.656, -122.343) }) } }, new Dictionary<string, object> { { columnKeys[0], 3 }, { columnKeys[1], SampleData.Geospatial.AsPolygon(new[] { new GeographyPosition(47.653, -122.358), new GeographyPosition(47.649, -122.348), new GeographyPosition(47.658, -122.348), new GeographyPosition(47.658, -122.358), new GeographyPosition(47.653, -122.358) }) } } }; using (var connection = new SqlConnection(ConnectionString)) { connection.Open(); try { CreateTable(connection, tableName, columnMappings); AddRows(connection, tableName, rows, true); using (var adapter = await new SqlDataSourceAdapterFactory() .CreateAsync(configuration, DataTransferContextMock.Instance, CancellationToken.None)) { var readOutput = new ReadOutputByRef(); for (var rowIndex = 0; rowIndex < rows.Length; ++rowIndex) { var dataItem = await adapter.ReadNextAsync(readOutput, CancellationToken.None); Assert.IsNotNull(dataItem, TestResources.MoreDataItemsExpected); Assert.IsNotNull(readOutput.DataItemId, CommonTestResources.MissingDataItemId); readOutput.Wipe(); VerifyDataItem(expectedRows[rowIndex], dataItem); } } } finally { DropTable(connection, tableName); } } }