public void WriteIntoHttpClient()
{
//Arrange
Mock<HttpMessageHandler> handlerMock = CreateHandlerMoq();
HttpClient httpClient = CreateHttpClient(handlerMock);
MemorySource<MySimpleRow> source = new MemorySource<MySimpleRow>();
source.DataAsList.Add(new MySimpleRow() { Col1 = 1, Col2 = "Test1" });
//Act
JsonDestination<MySimpleRow> dest = new JsonDestination<MySimpleRow>("http://test.test", ResourceType.Http);
dest.HttpClient = httpClient;
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
handlerMock.Protected().Verify(
"SendAsync",
Times.Exactly(1),
ItExpr.Is<HttpRequestMessage>(req =>
req.Method == HttpMethod.Get
&& req.RequestUri.Equals(new Uri("http://test.test"))
),
ItExpr.IsAny<CancellationToken>()
);
}
public void NoErrorHandling()
{
//Arrange
MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>();
source.DataAsList = new List <MySimpleRow>()
{
new MySimpleRow()
{
Col1 = "X"
},
new MySimpleRow()
{
Col1 = "1"
},
new MySimpleRow()
{
Col1 = null
}
};
JsonDestination <MySimpleRow> dest = new JsonDestination <MySimpleRow>("ErrorFile.json", ResourceType.File);
//Act
//Assert
Assert.ThrowsAny <Exception>(() =>
{
source.LinkTo(dest);
source.Execute();
dest.Wait();
});
}
public void IgnoreWithStringArray()
{
//Arrange
MemorySource <string[]> source = new MemorySource <string[]>();
source.DataAsList = new List <string[]>()
{
null,
new string[] { "1", "Test1" },
null,
new string[] { "2", "Test2" },
new string[] { "3", "Test3" },
null
};
//Act
JsonDestination <string[]> dest = new JsonDestination <string[]>("./IgnoreNullValuesStringArray.json", ResourceType.File);
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(File.ReadAllText("./IgnoreNullValuesStringArray.json"),
File.ReadAllText("res/JsonDestination/TwoColumnsStringArray.json"));
}
public void IgnoreWithObject()
{
//Arrange
MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>();
source.DataAsList = new List <MySimpleRow>()
{
null,
new MySimpleRow()
{
Col1 = 1, Col2 = "Test1"
},
null,
new MySimpleRow()
{
Col1 = 2, Col2 = "Test2"
},
new MySimpleRow()
{
Col1 = 3, Col2 = "Test3"
},
null
};
//Act
JsonDestination <MySimpleRow> dest = new JsonDestination <MySimpleRow>("./IgnoreNullValues.json", ResourceType.File);
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(File.ReadAllText("./IgnoreNullValues.json"),
File.ReadAllText("res/JsonDestination/TwoColumns.json"));
}
private static void StoreLastSyncKey(SyncData syncData)
{
var memsource = new MemorySource <SyncData>();
memsource.DisableLogging = true;
memsource.DataAsList.Add(syncData);
var syncdest = new JsonDestination <SyncData>("LastSyncId.json");
syncdest.DisableLogging = true;
memsource.LinkTo(syncdest);
Network.Execute(memsource);
}
public void RedirectBatch()
{
//Arrange
MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>();
source.DataAsList = new List <MySimpleRow>()
{
new MySimpleRow()
{
Col1 = "X"
},
new MySimpleRow()
{
Col1 = "1"
},
new MySimpleRow()
{
Col1 = "2"
},
new MySimpleRow()
{
Col1 = null
},
new MySimpleRow()
{
Col1 = "3"
},
};
JsonDestination <MySimpleRow> dest = new JsonDestination <MySimpleRow>("ErrorFile.json", ResourceType.File);
MemoryDestination <ETLBoxError> errorDest = new MemoryDestination <ETLBoxError>();
//Act
source.LinkTo(dest);
dest.LinkErrorTo(errorDest);
source.Execute();
dest.Wait();
errorDest.Wait();
//Assert
Assert.Equal(File.ReadAllText("./ErrorFile.json"),
File.ReadAllText("res/JsonDestination/TwoColumnsErrorLinking.json"));
Assert.Collection <ETLBoxError>(errorDest.Data,
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText))
);
}
public void SimpleFlowWithObject()
{
//Arrange
TwoColumnsTableFixture s2C = new TwoColumnsTableFixture("JsonDestSimple");
s2C.InsertTestDataSet3();
DbSource <MySimpleRow> source = new DbSource <MySimpleRow>(SqlConnection, "JsonDestSimple");
//Act
JsonDestination <MySimpleRow> dest = new JsonDestination <MySimpleRow>("./SimpleWithObject.json", ResourceType.File);
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(File.ReadAllText("res/JsonDestination/TwoColumnsSet3.json")
, File.ReadAllText("./SimpleWithObject.json"));
}
public void SimpleNonGeneric()
{
//Arrange
TwoColumnsTableFixture s2C = new TwoColumnsTableFixture("JsonDestSimpleNonGeneric");
s2C.InsertTestDataSet3();
DBSource source = new DBSource(SqlConnection, "JsonDestSimpleNonGeneric");
//Act
JsonDestination dest = new JsonDestination("./SimpleNonGeneric.json");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(File.ReadAllText("./SimpleNonGeneric.json"),
File.ReadAllText("res/JsonDestination/TwoColumnsSet3StringArray.json"));
}
static void Main(string[] args)
{
StartWebServer();
JsonSource <Order> source = new JsonSource <Order>("https://www.etlbox.net/demo/api/orders", ResourceType.Http);
source.HttpClient = CreateDefaultHttpClient();
ColumnRename <Order> rename = new ColumnRename <Order>();
rename.RenameColumns = new[]
{
new RenameColumn()
{
CurrentName = "Id", NewName = "OrderId"
},
new RenameColumn()
{
CurrentName = "CustomerId", NewName = "CId"
},
new RenameColumn()
{
CurrentName = "Description", RemoveColumn = true
}
};
JsonDestination destination = new JsonDestination();
destination.ResourceType = ResourceType.Http;
destination.HttpClient = CreateDefaultHttpClient();
destination.HttpRequestMessage.Method = HttpMethod.Post;
destination.HasNextUri = (streamMetaData, row) => true;
destination.GetNextUri = (streamMetaData, row) =>
{
streamMetaData.HttpRequestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", "Some token");
return($"http://localhost:61456/post/{streamMetaData.ProgressCount}");
};
source.LinkTo(rename);
rename.LinkTo(destination);
Network.Execute(source);
WriteServerLog();
}
public void SimpleFlowWithObject()
{
//Arrange
TwoColumnsTableFixture s2C = new TwoColumnsTableFixture("JsonDestDynamic");
s2C.InsertTestDataSet3();
DBSource <ExpandoObject> source = new DBSource <ExpandoObject>(SqlConnection, "JsonDestDynamic");
//Act
JsonDestination <ExpandoObject> dest = new JsonDestination <ExpandoObject>("./SimpleWithDynamicObject.json");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
//Null values can't be ignored:
//https://github.com/JamesNK/Newtonsoft.Json/issues/1466
Assert.Equal(File.ReadAllText("res/JsonDestination/TwoColumnsSet3DynamicObject.json"),
File.ReadAllText("./SimpleWithDynamicObject.json"));
}
static void Main(string[] args)
{
var scraper = new Scraper();
var terms = scraper.GetTerms().ToList();
Console.Clear();
Console.WriteLine("Select a term to Scrape:");
int t = ChooseOptions(terms.Select(x => x.Name).ToArray());
var term = terms[t - 1];
Console.Clear();
Console.WriteLine("Will scrape " + term.Name);
Console.WriteLine("Select a data destination:");
int d = ChooseOptions(
"Scrape to Blank SQL Database",
"Scrape to JSON file",
"Scrape to XML file",
"Scrape to CSV");
ISectionDestination dest;
string path = null;
if (d > 1)
{
Console.Write("Enter Output File Path: ");
path = Console.ReadLine();
}
switch (d)
{
case 1:
dest = new DatabaseContext();
break;
case 2:
dest = new JsonDestination(path);
break;
case 3:
dest = new XmlDestination(path);
break;
case 4:
dest = new CsvDestination(path);
break;
default:
throw new Exception("Invalid Option");
}
Console.Clear();
var watch = Stopwatch.StartNew();
double prog = 0;
var schools = scraper.GetSchools(term).ToList();
double progPerSchool = 1.0 / schools.Count;
foreach (var school in schools)
{
var subjects = scraper.GetSubjects(school).ToList();
double progPerSubject = progPerSchool / subjects.Count;
foreach (var subject in subjects)
{
var crns = scraper.GetCrns(subject).ToList();
double progPerCrn = progPerSubject / crns.Count;
foreach (var crn in crns)
{
var cd = scraper.GetDetailedClass(crn);
var cs = new ClassSection(cd)
{
School = school.Name,
Term = term.Name,
Subject = subject.Name
};
dest.Add(cs);
prog += progPerCrn;
Console.Write("{0:P2} ({1:F2}s)\r", prog, watch.ElapsedMilliseconds / 1000.0);
}
dest.CommitChanges();
}
}
Console.WriteLine("\nClosing...");
dest.Close();
watch.Stop();
Console.WriteLine("Finished ({0}ms)", watch.ElapsedMilliseconds);
}
public static async Task Run([TimerTrigger("0 */1 * * * *"
//, RunOnStartup=true) //only for testing purposes
)] TimerInfo myTimer, ILogger log)
{
Logging.LogInstance = log;
string sqlConnectionString = Environment.GetEnvironmentVariable("SqlServerConnectionString", EnvironmentVariableTarget.Process);
string storageConnString = Environment.GetEnvironmentVariable("AzureWebJobsStorage", EnvironmentVariableTarget.Process);
SqlConnectionManager conn = new SqlConnectionManager(sqlConnectionString);
if (!DemoHelper.WasInitialized)
{
containerName = DemoHelper.PrepareForDemo(storageConnString, conn);
}
SyncData syncDataLastRun = ReadLastSyncKey();
var parameter = new[] { new QueryParameter()
{
Name = "syncId", Value = syncDataLastRun.SyncId
} };
var dbSource = new DbSource <Order>()
{
ConnectionManager = conn,
Sql = $"SELECT Id, Number, Details, Date FROM Orders WHERE Id > @syncId ORDER BY Date",
SqlParameter = parameter
};
var jsonDest = new JsonDestination <Order>();
jsonDest.ResourceType = ResourceType.AzureBlob;
jsonDest.AzureBlobStorage.ConnectionString = storageConnString;
jsonDest.AzureBlobStorage.ContainerName = containerName;
var currentDate = new DateTime(1900, 1, 1);
jsonDest.HasNextUri = (_, order) => {
if (order.Date.Date > currentDate.Date)
{
currentDate = order.Date;
return(true);
}
return(false);
};
jsonDest.GetNextUri = (_, order) => "OrderData_" + order.Date.ToString("yyyy-MM-dd") + ".json";
var multicast = new Multicast <Order>();
var aggregation = new Aggregation <Order, SyncData>();
aggregation.AggregateColumns = new[] {
new AggregateColumn()
{
InputValuePropName = "Id",
AggregatedValuePropName = "SyncId",
AggregationMethod = AggregationMethod.Max
}
};
var syncMemoryDest = new MemoryDestination <SyncData>();
/*
* |---> jsonDest ("OrderData_2020-01-01.json", "OrderData_2020-01-02.json", ..)
* |
* dbSource --> multicast
* |
* |---> aggregation --> syncMemoryDest (1st run: SyncId = 5, 2nd run: SyncId = 7)
*/
dbSource.LinkTo(multicast);
multicast.LinkTo(jsonDest);
multicast.LinkTo(aggregation);
aggregation.LinkTo(syncMemoryDest);
Network.Execute(dbSource);
if (syncMemoryDest.Data.Count > 0)
{
SyncData syncDataThisRun = syncMemoryDest.Data.First();
StoreLastSyncKey(syncDataThisRun);
}
}
