public void WriteIntoHttpClient() { //Arrange Mock<HttpMessageHandler> handlerMock = CreateHandlerMoq(); HttpClient httpClient = CreateHttpClient(handlerMock); MemorySource<MySimpleRow> source = new MemorySource<MySimpleRow>(); source.DataAsList.Add(new MySimpleRow() { Col1 = 1, Col2 = "Test1" }); //Act JsonDestination<MySimpleRow> dest = new JsonDestination<MySimpleRow>("http://test.test", ResourceType.Http); dest.HttpClient = httpClient; source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert handlerMock.Protected().Verify( "SendAsync", Times.Exactly(1), ItExpr.Is<HttpRequestMessage>(req => req.Method == HttpMethod.Get && req.RequestUri.Equals(new Uri("http://test.test")) ), ItExpr.IsAny<CancellationToken>() ); }
public void NoErrorHandling() { //Arrange MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>(); source.DataAsList = new List <MySimpleRow>() { new MySimpleRow() { Col1 = "X" }, new MySimpleRow() { Col1 = "1" }, new MySimpleRow() { Col1 = null } }; JsonDestination <MySimpleRow> dest = new JsonDestination <MySimpleRow>("ErrorFile.json", ResourceType.File); //Act //Assert Assert.ThrowsAny <Exception>(() => { source.LinkTo(dest); source.Execute(); dest.Wait(); }); }
public void IgnoreWithStringArray() { //Arrange MemorySource <string[]> source = new MemorySource <string[]>(); source.DataAsList = new List <string[]>() { null, new string[] { "1", "Test1" }, null, new string[] { "2", "Test2" }, new string[] { "3", "Test3" }, null }; //Act JsonDestination <string[]> dest = new JsonDestination <string[]>("./IgnoreNullValuesStringArray.json", ResourceType.File); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert Assert.Equal(File.ReadAllText("./IgnoreNullValuesStringArray.json"), File.ReadAllText("res/JsonDestination/TwoColumnsStringArray.json")); }
public void IgnoreWithObject() { //Arrange MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>(); source.DataAsList = new List <MySimpleRow>() { null, new MySimpleRow() { Col1 = 1, Col2 = "Test1" }, null, new MySimpleRow() { Col1 = 2, Col2 = "Test2" }, new MySimpleRow() { Col1 = 3, Col2 = "Test3" }, null }; //Act JsonDestination <MySimpleRow> dest = new JsonDestination <MySimpleRow>("./IgnoreNullValues.json", ResourceType.File); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert Assert.Equal(File.ReadAllText("./IgnoreNullValues.json"), File.ReadAllText("res/JsonDestination/TwoColumns.json")); }
private static void StoreLastSyncKey(SyncData syncData) { var memsource = new MemorySource <SyncData>(); memsource.DisableLogging = true; memsource.DataAsList.Add(syncData); var syncdest = new JsonDestination <SyncData>("LastSyncId.json"); syncdest.DisableLogging = true; memsource.LinkTo(syncdest); Network.Execute(memsource); }
public void RedirectBatch() { //Arrange MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>(); source.DataAsList = new List <MySimpleRow>() { new MySimpleRow() { Col1 = "X" }, new MySimpleRow() { Col1 = "1" }, new MySimpleRow() { Col1 = "2" }, new MySimpleRow() { Col1 = null }, new MySimpleRow() { Col1 = "3" }, }; JsonDestination <MySimpleRow> dest = new JsonDestination <MySimpleRow>("ErrorFile.json", ResourceType.File); MemoryDestination <ETLBoxError> errorDest = new MemoryDestination <ETLBoxError>(); //Act source.LinkTo(dest); dest.LinkErrorTo(errorDest); source.Execute(); dest.Wait(); errorDest.Wait(); //Assert Assert.Equal(File.ReadAllText("./ErrorFile.json"), File.ReadAllText("res/JsonDestination/TwoColumnsErrorLinking.json")); Assert.Collection <ETLBoxError>(errorDest.Data, d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)), d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)) ); }
public void SimpleFlowWithObject() { //Arrange TwoColumnsTableFixture s2C = new TwoColumnsTableFixture("JsonDestSimple"); s2C.InsertTestDataSet3(); DbSource <MySimpleRow> source = new DbSource <MySimpleRow>(SqlConnection, "JsonDestSimple"); //Act JsonDestination <MySimpleRow> dest = new JsonDestination <MySimpleRow>("./SimpleWithObject.json", ResourceType.File); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert Assert.Equal(File.ReadAllText("res/JsonDestination/TwoColumnsSet3.json") , File.ReadAllText("./SimpleWithObject.json")); }
public void SimpleNonGeneric() { //Arrange TwoColumnsTableFixture s2C = new TwoColumnsTableFixture("JsonDestSimpleNonGeneric"); s2C.InsertTestDataSet3(); DBSource source = new DBSource(SqlConnection, "JsonDestSimpleNonGeneric"); //Act JsonDestination dest = new JsonDestination("./SimpleNonGeneric.json"); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert Assert.Equal(File.ReadAllText("./SimpleNonGeneric.json"), File.ReadAllText("res/JsonDestination/TwoColumnsSet3StringArray.json")); }
static void Main(string[] args) { StartWebServer(); JsonSource <Order> source = new JsonSource <Order>("https://www.etlbox.net/demo/api/orders", ResourceType.Http); source.HttpClient = CreateDefaultHttpClient(); ColumnRename <Order> rename = new ColumnRename <Order>(); rename.RenameColumns = new[] { new RenameColumn() { CurrentName = "Id", NewName = "OrderId" }, new RenameColumn() { CurrentName = "CustomerId", NewName = "CId" }, new RenameColumn() { CurrentName = "Description", RemoveColumn = true } }; JsonDestination destination = new JsonDestination(); destination.ResourceType = ResourceType.Http; destination.HttpClient = CreateDefaultHttpClient(); destination.HttpRequestMessage.Method = HttpMethod.Post; destination.HasNextUri = (streamMetaData, row) => true; destination.GetNextUri = (streamMetaData, row) => { streamMetaData.HttpRequestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", "Some token"); return($"http://localhost:61456/post/{streamMetaData.ProgressCount}"); }; source.LinkTo(rename); rename.LinkTo(destination); Network.Execute(source); WriteServerLog(); }
public void SimpleFlowWithObject() { //Arrange TwoColumnsTableFixture s2C = new TwoColumnsTableFixture("JsonDestDynamic"); s2C.InsertTestDataSet3(); DBSource <ExpandoObject> source = new DBSource <ExpandoObject>(SqlConnection, "JsonDestDynamic"); //Act JsonDestination <ExpandoObject> dest = new JsonDestination <ExpandoObject>("./SimpleWithDynamicObject.json"); source.LinkTo(dest); source.Execute(); dest.Wait(); //Assert //Null values can't be ignored: //https://github.com/JamesNK/Newtonsoft.Json/issues/1466 Assert.Equal(File.ReadAllText("res/JsonDestination/TwoColumnsSet3DynamicObject.json"), File.ReadAllText("./SimpleWithDynamicObject.json")); }
static void Main(string[] args) { var scraper = new Scraper(); var terms = scraper.GetTerms().ToList(); Console.Clear(); Console.WriteLine("Select a term to Scrape:"); int t = ChooseOptions(terms.Select(x => x.Name).ToArray()); var term = terms[t - 1]; Console.Clear(); Console.WriteLine("Will scrape " + term.Name); Console.WriteLine("Select a data destination:"); int d = ChooseOptions( "Scrape to Blank SQL Database", "Scrape to JSON file", "Scrape to XML file", "Scrape to CSV"); ISectionDestination dest; string path = null; if (d > 1) { Console.Write("Enter Output File Path: "); path = Console.ReadLine(); } switch (d) { case 1: dest = new DatabaseContext(); break; case 2: dest = new JsonDestination(path); break; case 3: dest = new XmlDestination(path); break; case 4: dest = new CsvDestination(path); break; default: throw new Exception("Invalid Option"); } Console.Clear(); var watch = Stopwatch.StartNew(); double prog = 0; var schools = scraper.GetSchools(term).ToList(); double progPerSchool = 1.0 / schools.Count; foreach (var school in schools) { var subjects = scraper.GetSubjects(school).ToList(); double progPerSubject = progPerSchool / subjects.Count; foreach (var subject in subjects) { var crns = scraper.GetCrns(subject).ToList(); double progPerCrn = progPerSubject / crns.Count; foreach (var crn in crns) { var cd = scraper.GetDetailedClass(crn); var cs = new ClassSection(cd) { School = school.Name, Term = term.Name, Subject = subject.Name }; dest.Add(cs); prog += progPerCrn; Console.Write("{0:P2} ({1:F2}s)\r", prog, watch.ElapsedMilliseconds / 1000.0); } dest.CommitChanges(); } } Console.WriteLine("\nClosing..."); dest.Close(); watch.Stop(); Console.WriteLine("Finished ({0}ms)", watch.ElapsedMilliseconds); }
public static async Task Run([TimerTrigger("0 */1 * * * *" //, RunOnStartup=true) //only for testing purposes )] TimerInfo myTimer, ILogger log) { Logging.LogInstance = log; string sqlConnectionString = Environment.GetEnvironmentVariable("SqlServerConnectionString", EnvironmentVariableTarget.Process); string storageConnString = Environment.GetEnvironmentVariable("AzureWebJobsStorage", EnvironmentVariableTarget.Process); SqlConnectionManager conn = new SqlConnectionManager(sqlConnectionString); if (!DemoHelper.WasInitialized) { containerName = DemoHelper.PrepareForDemo(storageConnString, conn); } SyncData syncDataLastRun = ReadLastSyncKey(); var parameter = new[] { new QueryParameter() { Name = "syncId", Value = syncDataLastRun.SyncId } }; var dbSource = new DbSource <Order>() { ConnectionManager = conn, Sql = $"SELECT Id, Number, Details, Date FROM Orders WHERE Id > @syncId ORDER BY Date", SqlParameter = parameter }; var jsonDest = new JsonDestination <Order>(); jsonDest.ResourceType = ResourceType.AzureBlob; jsonDest.AzureBlobStorage.ConnectionString = storageConnString; jsonDest.AzureBlobStorage.ContainerName = containerName; var currentDate = new DateTime(1900, 1, 1); jsonDest.HasNextUri = (_, order) => { if (order.Date.Date > currentDate.Date) { currentDate = order.Date; return(true); } return(false); }; jsonDest.GetNextUri = (_, order) => "OrderData_" + order.Date.ToString("yyyy-MM-dd") + ".json"; var multicast = new Multicast <Order>(); var aggregation = new Aggregation <Order, SyncData>(); aggregation.AggregateColumns = new[] { new AggregateColumn() { InputValuePropName = "Id", AggregatedValuePropName = "SyncId", AggregationMethod = AggregationMethod.Max } }; var syncMemoryDest = new MemoryDestination <SyncData>(); /* * |---> jsonDest ("OrderData_2020-01-01.json", "OrderData_2020-01-02.json", ..) * | * dbSource --> multicast * | * |---> aggregation --> syncMemoryDest (1st run: SyncId = 5, 2nd run: SyncId = 7) */ dbSource.LinkTo(multicast); multicast.LinkTo(jsonDest); multicast.LinkTo(aggregation); aggregation.LinkTo(syncMemoryDest); Network.Execute(dbSource); if (syncMemoryDest.Data.Count > 0) { SyncData syncDataThisRun = syncMemoryDest.Data.First(); StoreLastSyncKey(syncDataThisRun); } }