public void NoErrorHandling()
{
//Arrange
MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>();
source.DataAsList = new List <MySimpleRow>()
{
new MySimpleRow()
{
Col1 = "X"
},
new MySimpleRow()
{
Col1 = "1"
},
new MySimpleRow()
{
Col1 = null
}
};
CsvDestination <MySimpleRow> dest = new CsvDestination <MySimpleRow>("ErrorFileNoError.csv");
//Act
//Assert
Assert.ThrowsAny <Exception>(() =>
{
source.LinkTo(dest);
source.Execute();
dest.Wait();
});
}
public void TestCsvDestination_Tabs()
{
var outDir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory);
var opts = new IsIdentifiableRelationalDatabaseOptions
{
// This is slash t, not an tab
DestinationCsvSeparator = "\\t",
DestinationNoWhitespace = true,
DestinationCsvFolder = outDir.FullName
};
var dest = new CsvDestination(opts, "test", false);
var report = new TestFailureReport(dest);
report.WriteToDestinations();
report.CloseReport();
string fileCreatedContents = File.ReadAllText(Path.Combine(outDir.FullName, "test.csv"));
fileCreatedContents = fileCreatedContents.Replace("\r\n", Environment.NewLine);
TestHelpers.AreEqualIgnoringLineEndings(@"col1 col2
cell1 with some new lines and tabs cell2
", fileCreatedContents);
}
public static async Task <HttpResponseMessage> Run(
[HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] HttpRequest req,
ILogger log)
{
log.LogInformation("Received a json file to transform.");
string filename = Guid.NewGuid().ToString() + ".csv";
log.LogInformation("Temp file used to hold data: " + filename);
var jsonSource = new JsonSource()
{
CreateStreamReader = _ => new StreamReader(req.Body)
};
var csvFileDest = new CsvDestination(filename);
jsonSource.LinkTo(csvFileDest);
await Network.ExecuteAsync(jsonSource);
log.LogInformation("Successfully stored data in file - now returning the content as response.");
return(new HttpResponseMessage()
{
StatusCode = HttpStatusCode.OK,
Content = new StreamContent(new FileStream(filename, FileMode.Open))
});
}
public void IgnoreWithObject()
{
//Arrange
MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>();
source.DataAsList = new List <MySimpleRow>()
{
null,
new MySimpleRow()
{
Col1 = 1, Col2 = "Test1"
},
null,
new MySimpleRow()
{
Col1 = 2, Col2 = "Test2"
},
new MySimpleRow()
{
Col1 = 3, Col2 = "Test3"
},
null
};
//Act
CsvDestination <MySimpleRow> dest = new CsvDestination <MySimpleRow>("./IgnoreNullValues.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(File.ReadAllText("./IgnoreNullValues.csv"),
File.ReadAllText("res/CsvDestination/TwoColumns.csv"));
}
public void SerializingDateTime()
{
int rowCount = 0;
//Arrange
CustomSource <MySeriRow> source = new CustomSource <MySeriRow>(
() =>
new MySeriRow()
{
Col1 = 1,
Col2 = new DateTime(2010, 02, 05)
},
() => rowCount++ == 1);
//Act
CsvDestination <MySeriRow> dest = new CsvDestination <MySeriRow>("./DateTimeSerialization.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(File.ReadAllText("./DateTimeSerialization.csv"),
File.ReadAllText("res/CsvDestination/DateTimeSerialization.csv"));
}
public void IgnoreWithStringArray()
{
//Arrange
MemorySource <string[]> source = new MemorySource <string[]>();
source.DataAsList = new List <string[]>()
{
null,
new string[] { "1", "Test1" },
null,
new string[] { "2", "Test2" },
new string[] { "3", "Test3" },
null
};
//Act
CsvDestination <string[]> dest = new CsvDestination <string[]>("./IgnoreNullValuesStringArray.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(File.ReadAllText("./IgnoreNullValuesStringArray.csv"),
File.ReadAllText("res/CsvDestination/TwoColumnsNoHeader.csv"));
}
static void Main(string[] args)
{
var source = new CsvSource <InputData>("Accounts_Quartal1.csv");
source.Configuration.Delimiter = ";";
var trans = new RowMultiplication <InputData, PivotedOutput>();
trans.MultiplicationFunc = row =>
{
List <PivotedOutput> result = new List <PivotedOutput>();
result.Add(new PivotedOutput()
{
Account = row.Account,
Month = nameof(InputData.January),
MonthlyValue = row.January
});
result.Add(new PivotedOutput()
{
Account = row.Account,
Month = nameof(InputData.February),
MonthlyValue = row.February
});
result.Add(new PivotedOutput()
{
Account = row.Account,
Month = nameof(InputData.March),
MonthlyValue = row.March
});
return(result);
};
var dest = new CsvDestination <PivotedOutput>("AccountNumbers_Pivoted.csv");
dest.Configuration.HasHeaderRecord = false;
source.LinkTo(trans);
trans.LinkTo(dest);
Network.Execute(source);
/* AccountNumbers_Pivoted.csv output:
* 4711,January,10
* 4711,February,11
* 4711,March,12
* 4712,January,20
* 4712,February,21
* 4712,March,22
* 4713,January,30
* 4713,February,31
* 4713,March,32
*/
}
public void RedirectSingleRecordWithObject()
{
//Arrange
MemorySource <MySimpleRow> source = new MemorySource <MySimpleRow>();
source.DataAsList = new List <MySimpleRow>()
{
new MySimpleRow()
{
Col1 = "X"
},
new MySimpleRow()
{
Col1 = "1"
},
new MySimpleRow()
{
Col1 = "2"
},
new MySimpleRow()
{
Col1 = null
},
new MySimpleRow()
{
Col1 = "3"
},
};
CsvDestination <MySimpleRow> dest = new CsvDestination <MySimpleRow>("ErrorFile.csv");
MemoryDestination <ETLBoxError> errorDest = new MemoryDestination <ETLBoxError>();
//Act
source.LinkTo(dest);
dest.LinkErrorTo(errorDest);
source.Execute();
dest.Wait();
errorDest.Wait();
//Assert
Assert.Equal(File.ReadAllText("./ErrorFile.csv"),
File.ReadAllText("res/CsvDestination/TwoColumnsErrorLinking.csv"));
Assert.Collection <ETLBoxError>(errorDest.Data,
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText)),
d => Assert.True(!string.IsNullOrEmpty(d.RecordAsJson) && !string.IsNullOrEmpty(d.ErrorText))
);
}
public static void Main(string[] args)
{
PrepareSqlLiteDestination();
var currentYear = StartYear;
var source = new CustomBatchSource <Accident>();
source.ReadBatchFunc = _ => {
var accidents = ParseAccidentsFromUrl($"https://aviation-safety.net/database/dblist.php?Year={currentYear}");
currentYear++;
return(accidents);
};
source.ReadingCompleted = _ => currentYear > EndYear;
var filter = new FilterTransformation <Accident>();
filter.FilterPredicate = accident => accident.Year <= 1;
var multicast = new Multicast <Accident>();
var memDest = new MemoryDestination <Accident>();
var sqlLiteDest = new DbDestination <Accident>(SQLiteConnection, "Accidents");
var aggregation = new Aggregation <Accident, AccidentsPerYear>();
var csvDest = new CsvDestination <AccidentsPerYear>("aggregated.csv");
source.LinkTo(filter);
filter.LinkTo(multicast);
multicast.LinkTo(memDest);
multicast.LinkTo(sqlLiteDest);
multicast.LinkTo(aggregation, row => row.Year > 1);
aggregation.LinkTo(csvDest);
Network.Execute(source);
Console.WriteLine($"Imported {memDest.Data.Count} rows from aviation-safety.net");
for (int year = StartYear; year <= EndYear; year++)
{
Console.WriteLine($"There were {memDest.Data.Where(a => a.Year == year).Count()} accidents in {year}");
}
}
public void SimpleFlow()
{
//Arrange
TwoColumnsTableFixture s2C = new TwoColumnsTableFixture("CSVDestDynamicObject");
s2C.InsertTestDataSet3();
DbSource <ExpandoObject> source = new DbSource <ExpandoObject>(SqlConnection, "CSVDestDynamicObject");
//Act
CsvDestination <ExpandoObject> dest = new CsvDestination <ExpandoObject>("./SimpleWithDynamicObject.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(File.ReadAllText("./SimpleWithDynamicObject.csv"),
File.ReadAllText("res/CsvDestination/TwoColumnsSet3DynamicObject.csv"));
}
public void SimpleFlowWithObject()
{
//Arrange
TwoColumnsTableFixture s2C = new TwoColumnsTableFixture("CSVDestSimple");
s2C.InsertTestDataSet3();
DbSource <MySimpleRow> source = new DbSource <MySimpleRow>("CSVDestSimple", SqlConnection);
//Act
CsvDestination <MySimpleRow> dest = new CsvDestination <MySimpleRow>("./SimpleWithObject.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(File.ReadAllText("./SimpleWithObject.csv"),
File.ReadAllText("res/CsvDestination/TwoColumnsSet3.csv"));
}
public void SimpleNonGeneric()
{
//Arrange
TwoColumnsTableFixture s2C = new TwoColumnsTableFixture("CSVDestSimpleNonGeneric");
s2C.InsertTestDataSet3();
DbSource <string[]> source = new DbSource <string[]>(SqlConnection, "CSVDestSimpleNonGeneric");
//Act
CsvDestination <string[]> dest = new CsvDestination <string[]>("./SimpleNonGeneric.csv");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
//Assert
Assert.Equal(File.ReadAllText("./SimpleNonGeneric.csv"),
File.ReadAllText("res/CsvDestination/TwoColumnsSet3NoHeader.csv"));
}
public void Run()
{
PostgresConnectionManager conMan = new PostgresConnectionManager(PostgresConnectionString);
//Import CSV
CsvSource sourceCSV = new CsvSource("NameList.csv");
DbDestination importDest = new DbDestination(conMan, "NameTable");
sourceCSV.LinkTo(importDest);
sourceCSV.Execute();
importDest.Wait();
//Export again
DbSource <NameListElement> sourceTable = new DbSource <NameListElement>(conMan, "NameTable");
CsvDestination <NameListElement> destCSV = new CsvDestination <NameListElement>("Export.csv");
destCSV.Configuration.Delimiter = ";";
sourceTable.LinkTo(destCSV);
sourceTable.Execute();
destCSV.Wait();
}
public void DisableHeader()
{
//Arrange
TwoColumnsTableFixture s2c = new TwoColumnsTableFixture("CsvSourceNoHeader");
s2c.InsertTestData();
DbSource <MySimpleRow> source = new DbSource <MySimpleRow>(SqlConnection, "CsvSourceNoHeader");
//Act
CsvDestination <MySimpleRow> dest = new CsvDestination <MySimpleRow>("./ConfigurationNoHeader.csv");
dest.Configuration.HasHeaderRecord = false;
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Equal(File.ReadAllText("./ConfigurationNoHeader.csv"),
File.ReadAllText("res/CsvDestination/TwoColumnsNoHeader.csv"));
}
/// <summary>
/// Creates report destinations. Can be overriden to add headers or to initialize the destination in some way.
/// </summary>
/// <param name="opts"></param>
public virtual void AddDestinations(IsIdentifiableAbstractOptions opts)
{
IReportDestination destination;
// Default is to write out CSV results
if (!string.IsNullOrWhiteSpace(opts.DestinationCsvFolder))
{
destination = new CsvDestination(opts, _reportName);
}
else if (!string.IsNullOrWhiteSpace(opts.DestinationConnectionString))
{
destination = new DatabaseDestination(opts, _reportName);
}
else
{
opts.DestinationCsvFolder = Environment.CurrentDirectory;
destination = new CsvDestination(opts, _reportName);
}
Destinations.Add(destination);
}
public void should_write_the_data_to_a_file_using_the_specified_field_separator()
{
var result = new StringBuilder();
Func <string, TextWriter> writerCreator = s => new StringWriter(result);
var destination = new CsvDestination(";", writerCreator);
var data = new[] {
new TestData {
Bar = 1, Foo = "Hello"
},
new TestData {
Bar = 2, Foo = "World"
},
};
destination.Load(data);
Check.That(result.ToString())
.Contains("Hello;1")
.And.Contains("World;2");
}
public void UsingGetInputRecordKeyFunc()
{
var source = new CsvSource <InputRow>("InputData.csv");
source.Configuration.MissingFieldFound = null;
var lookupSource = new MemorySource <LookupRow>();
lookupSource.DataAsList = new List <LookupRow>()
{
new LookupRow()
{
LookupId = "idstringa", LookupValue = "A"
},
new LookupRow()
{
LookupId = "idstringb", LookupValue = "B"
},
new LookupRow()
{
LookupId = "idstringc", LookupValue = "C"
}
};
var lookup = new LookupTransformation <InputRow, LookupRow>();
lookup.Source = lookupSource;
lookup.GetInputRecordKeyFunc = row => row.Id.ToLower();
lookup.GetSourceRecordKeyFunc = row => row.LookupId;
var dest = new CsvDestination <InputRow>("output1.csv");
source.LinkTo(lookup).LinkTo(dest);
Network.Execute(source);
PrintFile("InputData.csv");
PrintFile("output1.csv");
}
public void WriteIntoMultipleDestinations()
{
//Arrange
var source = new MemorySource <string[]>();
source.DataAsList.Add(new string[] { "Test" });
var trans = new RowTransformation <string[]>();
trans.TransformationFunc = r => throw new Exception();
var dest = new MemoryDestination <string[]>();
CreateErrorTableTask.Create(SqlConnection, "error_log");
var mc = new Multicast <ETLBoxError>();
var errorMem = new MemoryDestination <ETLBoxError>();
var errorDb = new DbDestination <ETLBoxError>(SqlConnection, "error_log");
var errorCsv = new CsvDestination <ETLBoxError>("error_csv.csv");
source.LinkTo(trans);
trans.LinkTo(dest);
//Act
trans.LinkErrorTo(mc);
mc.LinkTo(errorMem);
mc.LinkTo(errorDb);
mc.LinkTo(errorCsv);
source.Execute();
dest.Wait();
errorMem.Wait();
errorDb.Wait();
errorCsv.Wait();
//Assert
Assert.True(errorMem.Data.Count > 0);
Assert.True(RowCountTask.Count(SqlConnection, "error_log") > 0);
Assert.True(File.ReadAllText("error_csv.csv").Length > 0);
}
public void OverwritingComparisonInObject()
{
var source = new CsvSource <MyInputRow>("InputData.csv");
source.Configuration.MissingFieldFound = null;
var lookupSource = new MemorySource <MyLookupRow>();
lookupSource.DataAsList = new List <MyLookupRow>()
{
new MyLookupRow()
{
LookupId = new ComparableObject("idstringa"), LookupValue = "A"
},
new MyLookupRow()
{
LookupId = new ComparableObject("idstringb"), LookupValue = "B"
},
new MyLookupRow()
{
LookupId = new ComparableObject("idstringc"), LookupValue = "C"
}
};
var lookup = new LookupTransformation <MyInputRow, MyLookupRow>();
lookup.Source = lookupSource;
var dest = new CsvDestination <MyInputRow>("output2.csv");
source.LinkTo(lookup).LinkTo(dest);
Network.Execute(source);
PrintFile("InputData.csv");
PrintFile("output1.csv");
}
public void WriteAsyncAndCheckLock(string filename, int noRecords)
{
//Arrange
if (File.Exists(filename))
{
File.Delete(filename);
}
MemorySource <string[]> source = new MemorySource <string[]>();
for (int i = 0; i < noRecords; i++)
{
source.DataAsList.Add(new string[] { HashHelper.RandomString(100) });
}
CsvDestination <string[]> dest = new CsvDestination <string[]>(filename);
bool onCompletionRun = false;
//Act
source.LinkTo(dest);
Task sT = source.ExecuteAsync();
Task dt = dest.Completion;
while (!File.Exists(filename))
{
Task.Delay(10).Wait();
}
//Assert
dest.OnCompletion = () =>
{
Assert.False(IsFileLocked(filename), "StreamWriter should be disposed and file unlocked");
onCompletionRun = true;
};
Assert.True(IsFileLocked(filename), "Right after start the file should still be locked.");
dt.Wait();
Assert.True(onCompletionRun, "OnCompletion action and assertion did run");
}
public void ErrorIntoCsv()
{
var source = new MemorySource <MyXmlRow>();
source.DataAsList.Add(new MyXmlRow()
{
Xml = _validXml
});
source.DataAsList.Add(new MyXmlRow()
{
Xml = _invalidXml
});
source.DataAsList.Add(new MyXmlRow()
{
Xml = _validXml
});
XmlSchemaValidation <MyXmlRow> valid = new XmlSchemaValidation <MyXmlRow>(xsdMarkup, i => i.Xml);
CsvDestination <ETLBoxError> errDest = new CsvDestination <ETLBoxError>("res/XmlValidation/Error.csv");
CsvDestination <MyXmlRow> dest = new CsvDestination <MyXmlRow>("res/XmlValidation/NoError.csv");
source.LinkTo(valid);
valid.LinkTo(dest);
valid.LinkErrorTo(errDest);
source.Execute();
dest.Wait();
errDest.Wait();
Assert.Equal(File.ReadAllText("res/XmlValidation/ValidXml.csv"),
File.ReadAllText("res/XmlValidation/NoError.csv"), ignoreCase: true, ignoreLineEndingDifferences: true, ignoreWhiteSpaceDifferences: true);
Assert.StartsWith(@"ErrorText,ReportTime,ExceptionType,RecordAsJson
The element 'Root' has invalid child element 'Child3'. List of possible elements expected: 'Child2'.",
File.ReadAllText("res/XmlValidation/Error.csv"));
}
/// <summary>
/// Connects to the database and runs all failures through the rules base performing redactions as required
/// </summary>
/// <returns></returns>
public int Run()
{
//In RulesOnly mode this will be null
var server = _target?.Discover();
List <Exception> errors = new List <Exception>();
var storeReport = new FailureStoreReport(_outputFile.Name, 100);
Stopwatch sw = new Stopwatch();
sw.Start();
using (var storeReportDestination = new CsvDestination(new IsIdentifiableDicomFileOptions(), _outputFile))
{
IsIdentifiableRule updateRule;
storeReport.AddDestination(storeReportDestination);
while (_reportReader.Next())
{
bool noUpdate;
try
{
noUpdate = _updater.OnLoad(server, _reportReader.Current, out updateRule);
}
catch (Exception e)
{
errors.Add(e);
continue;
}
//is it novel for updater
if (noUpdate)
{
//is it novel for ignorer
if (_ignorer.OnLoad(_reportReader.Current, out IsIdentifiableRule ignoreRule))
{
//we can't process it unattended
storeReport.Add(_reportReader.Current);
Unresolved++;
}
else
{
if (!_ignoreRulesUsed.ContainsKey(ignoreRule))
{
_ignoreRulesUsed.Add(ignoreRule, 1);
}
else
{
_ignoreRulesUsed[ignoreRule]++;
}
Ignores++;
}
}
else
{
if (!_updateRulesUsed.ContainsKey(updateRule))
{
_updateRulesUsed.Add(updateRule, 1);
}
else
{
_updateRulesUsed[updateRule]++;
}
Updates++;
}
Total++;
if (Total % 10000 == 0 || sw.ElapsedMilliseconds > 5000)
{
Log($"Done {Total:N0} u={Updates:N0} i={Ignores:N0} o={Unresolved:N0} err={errors.Count:N0}", true);
sw.Restart();
}
}
storeReport.CloseReport();
}
Log($"Ignore Rules Used:" + Environment.NewLine + string.Join(Environment.NewLine,
_ignoreRulesUsed.OrderBy(k => k.Value).Select(k => $"{k.Key.IfPattern} - {k.Value:N0}")), false);
Log($"Update Rules Used:" + Environment.NewLine + string.Join(Environment.NewLine,
_updateRulesUsed.OrderBy(k => k.Value).Select(k => $"{k.Key.IfPattern} - {k.Value:N0}")), false);
Log("Errors:" + Environment.NewLine + string.Join(Environment.NewLine, errors.Select(e => e.ToString())), false);
Log($"Finished {Total:N0} updates={Updates:N0} ignored={Ignores:N0} out={Unresolved:N0} err={errors.Count:N0}", true);
return(0);
}
static void Main(string[] args)
{
var scraper = new Scraper();
var terms = scraper.GetTerms().ToList();
Console.Clear();
Console.WriteLine("Select a term to Scrape:");
int t = ChooseOptions(terms.Select(x => x.Name).ToArray());
var term = terms[t - 1];
Console.Clear();
Console.WriteLine("Will scrape " + term.Name);
Console.WriteLine("Select a data destination:");
int d = ChooseOptions(
"Scrape to Blank SQL Database",
"Scrape to JSON file",
"Scrape to XML file",
"Scrape to CSV");
ISectionDestination dest;
string path = null;
if (d > 1)
{
Console.Write("Enter Output File Path: ");
path = Console.ReadLine();
}
switch (d)
{
case 1:
dest = new DatabaseContext();
break;
case 2:
dest = new JsonDestination(path);
break;
case 3:
dest = new XmlDestination(path);
break;
case 4:
dest = new CsvDestination(path);
break;
default:
throw new Exception("Invalid Option");
}
Console.Clear();
var watch = Stopwatch.StartNew();
double prog = 0;
var schools = scraper.GetSchools(term).ToList();
double progPerSchool = 1.0 / schools.Count;
foreach (var school in schools)
{
var subjects = scraper.GetSubjects(school).ToList();
double progPerSubject = progPerSchool / subjects.Count;
foreach (var subject in subjects)
{
var crns = scraper.GetCrns(subject).ToList();
double progPerCrn = progPerSubject / crns.Count;
foreach (var crn in crns)
{
var cd = scraper.GetDetailedClass(crn);
var cs = new ClassSection(cd)
{
School = school.Name,
Term = term.Name,
Subject = subject.Name
};
dest.Add(cs);
prog += progPerCrn;
Console.Write("{0:P2} ({1:F2}s)\r", prog, watch.ElapsedMilliseconds / 1000.0);
}
dest.CommitChanges();
}
}
Console.WriteLine("\nClosing...");
dest.Close();
watch.Stop();
Console.WriteLine("Finished ({0}ms)", watch.ElapsedMilliseconds);
}
