public void SingleNullColumnResultsNullKey()
{
var values = new SlimRow();
var result = values.GenerateKey("name");
Assert.IsNull(result);
}
public void MultipleNullColumnsResultsNonNullKey()
{
var values = new SlimRow();
var result = values.GenerateKey("id", "name");
Assert.IsNotNull(result);
}
public ContinuousAggregate(object tag)
{
ResultRow = new SlimRow()
{
Tag = tag,
};
}
public void MergeTestWithRow()
{
var context = TestExecuter.GetContext();
context.SetRowType <DictionaryRow>();
var row = new TrackedRow(context.CreateRow(null));
row["a"] = 1;
row["b"] = "dog";
row["c"] = 7.1d;
var newRow = new SlimRow()
{
["b"] = "cat",
["c"] = 7.1d,
["d"] = 8m,
};
row.MergeWith(newRow.Values);
Assert.AreEqual(4, row.ColumnCount);
Assert.AreEqual(1, row["a"]);
Assert.AreEqual("cat", row["b"]);
Assert.AreEqual(7.1d, row["c"]);
Assert.AreEqual(8m, row["d"]);
}
public void HasErrorFalse()
{
var values = new SlimRow()
{
["id"] = 12,
["name"] = "A",
};
Assert.IsFalse(values.HasError());
}
public void TimeSpanKeyIsInvariantWithDaysAndMilliseconds()
{
var values = new SlimRow()
{
["time"] = new TimeSpan(1, 1, 0),
};
var result = values.GenerateKey("time");
Assert.AreEqual("0:01:01:00.0000000", result);
}
public void KeyCaseIgnored()
{
var values = new SlimRow()
{
["date"] = new DateTime(2020, 02, 20, 12, 12, 0, 666),
};
var result = values["DATE"];
Assert.AreEqual(new DateTime(2020, 02, 20, 12, 12, 0, 666), result);
}
public void DateTimeOffsetKeyIsInvariantWithMilliseconds()
{
var values = new SlimRow()
{
["dto"] = new DateTimeOffset(2020, 02, 20, 12, 12, 0, 666, new TimeSpan(2, 0, 0)),
};
var result = values.GenerateKey("dto");
Assert.AreEqual("2020.02.20 12:12:00.6660000 +02:00", result);
}
public void DateTimeKeyIsInvariantWithMilliseconds()
{
var values = new SlimRow()
{
["date"] = new DateTime(2020, 02, 20, 12, 12, 0, 666),
};
var result = values.GenerateKey("date");
Assert.AreEqual("2020.02.20 12:12:00.6660000", result);
}
public void HasErrorTrue()
{
var values = new SlimRow()
{
["id"] = 12,
["name"] = "A",
["err"] = new EtlRowError(9),
};
Assert.IsTrue(values.HasError());
}
public void IntKeyIsInvariant()
{
var values = new SlimRow()
{
["id"] = 1234567,
["date"] = new DateTime(2020, 02, 20, 12, 12, 0, 666),
};
var result = values.GenerateKey("id", "date");
Assert.IsTrue(result.Contains("1234567", StringComparison.Ordinal));
Assert.IsTrue(result.Contains("2020.02.20 12:12:00.6660000", StringComparison.Ordinal));
}
public void EtlRowErrorUnEquality()
{
var a = new SlimRow()
{
["id"] = 12, ["name"] = new EtlRowError("x"),
};
var b = new SlimRow()
{
["id"] = 12, ["name"] = new EtlRowError("y"),
};
var result = new ColumnBasedRowEqualityComparer().Equals(a, b);
Assert.IsFalse(result);
}
public void ComplexTestCombinedWithSerializer()
{
var context = TestExecuter.GetContext();
var builder = new ProcessBuilder()
{
InputProcess = TestData.Person(context),
Mutators = new MutatorList()
{
new InPlaceConvertMutator(context)
{
Columns = new[] { "birthDate" },
TypeConverter = new DateConverterAuto(new CultureInfo("hu-HU")),
ActionIfInvalid = InvalidValueAction.Throw,
},
new ExplodeMutator(context)
{
RowCreator = row =>
{
var newRow = new SlimRow
{
["personModel"] = new PersonModel()
{
Id = row.GetAs <int>("id"),
Name = row.GetAs <string>("name"),
Age = row.GetAs <int?>("age"),
BirthDate = row.GetAs <DateTime?>("birthDate"),
}
};
return(new[] { newRow });
},
},
new DataContractXmlSerializerMutator <PersonModel>(context)
{
SourceColumn = "personModel",
TargetColumn = "personModelXml",
},
new RemoveColumnMutator(context)
{
Columns = new[] { "personModel" },
},
new InPlaceConvertMutator(context)
{
Columns = new[] { "personModelXml" },
TypeConverter = new DataContractXmlDeSerializerConverter <PersonModel>(),
},
new RenameColumnMutator(context)
{
Columns = new()
{
public void EqualityOnAllColumns()
{
var a = new SlimRow()
{
["id"] = 12, ["name"] = "x",
};
var b = new SlimRow()
{
["id"] = 12, ["name"] = "x",
};
var result = new ColumnBasedRowEqualityComparer().Equals(a, b);
Assert.IsTrue(result);
}
public void ReferenceUnEquality()
{
var a = new SlimRow()
{
["person"] = new TestData.PersonModel(),
};
var b = new SlimRow()
{
["person"] = new TestData.PersonModel(),
};
var result = new ColumnBasedRowEqualityComparer().Equals(a, b);
Assert.IsFalse(result);
}
public void StringUnequalityOnAnyColumns()
{
var a = new SlimRow()
{
["id"] = 12, ["name"] = "x",
};
var b = new SlimRow()
{
["id"] = 12, ["name"] = "y",
};
var result = new ColumnBasedRowEqualityComparer().Equals(a, b);
Assert.IsFalse(result);
}
public void ColorUnEquality()
{
var a = new SlimRow()
{
["color"] = Color.Red,
};
var b = new SlimRow()
{
["color"] = Color.Black,
};
var result = new ColumnBasedRowEqualityComparer().Equals(a, b);
Assert.IsFalse(result);
}
public void DoubleEquality()
{
var a = new SlimRow()
{
["id"] = -6.5d,
};
var b = new SlimRow()
{
["id"] = -13d / 2d,
};
var result = new ColumnBasedRowEqualityComparer().Equals(a, b);
Assert.IsTrue(result);
}
public void DoubleUnEquality()
{
var a = new SlimRow()
{
["id"] = -1d,
};
var b = new SlimRow()
{
["id"] = -1.01d,
};
var result = new ColumnBasedRowEqualityComparer().Equals(a, b);
Assert.IsFalse(result);
}
public void IntegerUnEquality()
{
var a = new SlimRow()
{
["id"] = 12,
};
var b = new SlimRow()
{
["id"] = 13,
};
var result = new ColumnBasedRowEqualityComparer().Equals(a, b);
Assert.IsFalse(result);
}
public void EqualityOnSelectedColumns()
{
var a = new SlimRow()
{
["id"] = 12, ["name"] = "x",
};
var b = new SlimRow()
{
["id"] = 12, ["name"] = "y",
};
var result = new ColumnBasedRowEqualityComparer()
{
Columns = new[] { "id" }
}.Equals(a, b);
Assert.IsTrue(result);
}
private void processSlimRow(IList <IList <string> > resultTable, IList <string> row)
{
IRowWrapper <string> currentRow = new SlimRow(row);
try
{
processRow(currentRow);
}
catch (Exception e)
{
LOG.Error(e, "Exception raised when processing row {0}", row[0]);
_restFixture.Formatter.exception(currentRow.getCell(0), e);
}
finally
{
IList <string> rowAsList = mapSlimRow(row, currentRow);
resultTable.Add(rowAsList);
}
}
public void NullValuesAreNotStored()
{
var values = new SlimRow()
{
["id"] = 12,
["name"] = "A",
["age"] = null,
};
Assert.AreEqual(2, values.ColumnCount);
Assert.IsTrue(values.Values.All(kvp => kvp.Value != null));
values["age"] = 7;
Assert.AreEqual(3, values.ColumnCount);
Assert.IsTrue(values.Values.All(kvp => kvp.Value != null));
values["name"] = null;
Assert.AreEqual(2, values.ColumnCount);
Assert.IsTrue(values.Values.All(kvp => kvp.Value != null));
}
public void ComplexTestCombinedWithSerializer()
{
var topic = TestExecuter.GetTopic();
var builder = new ProcessBuilder()
{
InputProcess = TestData.Person(topic),
Mutators = new MutatorList()
{
new InPlaceConvertMutator(topic, null)
{
Columns = new[] { "birthDate" },
TypeConverter = new DateConverterAuto(new CultureInfo("hu-HU")),
ActionIfInvalid = InvalidValueAction.Throw,
},
new ExplodeMutator(topic, null)
{
RowCreator = (proc, row) =>
{
var newRow = new SlimRow
{
["personModel"] = new PersonModel()
{
Id = row.GetAs <int>("id"),
Name = row.GetAs <string>("name"),
Age = row.GetAs <int?>("age"),
BirthDate = row.GetAs <DateTime?>("birthDate"),
}
};
return(new[] { newRow });
},
},
new DataContractXmlSerializerMutator <PersonModel>(topic, "serialize to XML byte[]")
{
ColumnConfiguration = new ColumnCopyConfiguration("personModel", "personModelXml"),
},
new RemoveColumnMutator(topic, null)
{
Columns = new[] { "personModel" },
},
new InPlaceConvertMutator(topic, "deserialize from XML byte[]")
{
Columns = new[] { "personModelXml" },
TypeConverter = new DataContractXmlDeSerializerConverter <PersonModel>(),
},
new RenameColumnMutator(topic, null)
{
ColumnConfiguration = new List <ColumnRenameConfiguration>()
{
new ColumnRenameConfiguration("personModelXml", "personModel"),
},
},
new ExplodeMutator(topic, null)
{
RowCreator = (proc, row) =>
{
var personModel = row.GetAs <PersonModel>("personModel");
var newRow = new SlimRow()
{
["id"] = personModel.Id,
["name"] = personModel.Name,
["age"] = personModel.Age,
["birthDate"] = personModel.BirthDate,
};
return(new[] { newRow });
},
},
},
};
var result = TestExecuter.Execute(builder);
Assert.AreEqual(7, result.MutatedRows.Count);
Assert.That.ExactMatch(result.MutatedRows, new List <CaseInsensitiveStringKeyDictionary <object> >()
{
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 0, ["name"] = "A", ["age"] = 17, ["birthDate"] = new DateTime(2010, 12, 9, 0, 0, 0, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 1, ["name"] = "B", ["age"] = 8, ["birthDate"] = new DateTime(2011, 2, 1, 0, 0, 0, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 2, ["name"] = "C", ["age"] = 27, ["birthDate"] = new DateTime(2014, 1, 21, 0, 0, 0, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 3, ["name"] = "D", ["age"] = 39, ["birthDate"] = new DateTime(2018, 7, 11, 0, 0, 0, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 4, ["name"] = "E", ["age"] = -3
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 5, ["name"] = "A", ["age"] = 11, ["birthDate"] = new DateTime(2013, 5, 15, 0, 0, 0, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 6, ["name"] = "fake", ["birthDate"] = new DateTime(2018, 1, 9, 0, 0, 0, 0)
}
});
var exceptions = topic.Context.GetExceptions();
Assert.AreEqual(0, exceptions.Count);
}
protected override IEnumerable <IRow> EvaluateImpl(Stopwatch netTimeStopwatch)
{
var groupRows = new List <IReadOnlySlimRow>();
string lastKey = null;
netTimeStopwatch.Stop();
var enumerator = Input.TakeRowsAndTransferOwnership(this).GetEnumerator();
netTimeStopwatch.Start();
var success = true;
var rowCount = 0;
var ignoredRowCount = 0;
var groupCount = 0;
var aggregateCount = 0;
while (!Context.IsTerminating)
{
netTimeStopwatch.Stop();
var finished = !enumerator.MoveNext();
if (finished)
{
break;
}
var row = enumerator.Current;
netTimeStopwatch.Start();
if (row.Tag is HeartBeatTag)
{
netTimeStopwatch.Stop();
yield return(row);
netTimeStopwatch.Start();
continue;
}
var apply = false;
if (RowFilter != null)
{
try
{
apply = RowFilter.Invoke(row);
}
catch (Exception ex)
{
AddException(ex, row);
break;
}
if (!apply)
{
ignoredRowCount++;
netTimeStopwatch.Stop();
yield return(row);
netTimeStopwatch.Start();
continue;
}
}
if (RowTagFilter != null)
{
try
{
apply = RowTagFilter.Invoke(row.Tag);
}
catch (Exception ex)
{
AddException(ex, row);
break;
}
if (!apply)
{
ignoredRowCount++;
netTimeStopwatch.Stop();
yield return(row);
netTimeStopwatch.Start();
continue;
}
}
rowCount++;
var key = KeyGenerator.Invoke(row);
if (key != lastKey)
{
lastKey = key;
if (groupRows.Count > 0)
{
var aggregates = new List <SlimRow>();
groupCount++;
try
{
Operation.TransformGroup(groupRows, () =>
{
var aggregate = new SlimRow
{
Tag = groupRows[0].Tag
};
if (FixColumns != null)
{
foreach (var column in FixColumns)
{
aggregate[column.Key] = groupRows[0][column.Value ?? column.Key];
}
}
aggregates.Add(aggregate);
return(aggregate);
});
}
catch (Exception ex)
{
var exception = new MemoryAggregationException(this, Operation, groupRows, ex);
AddException(exception);
success = false;
break;
}
foreach (var groupRow in groupRows)
{
Context.SetRowOwner(groupRow as IRow, null);
}
groupRows.Clear();
foreach (var aggregate in aggregates)
{
aggregateCount++;
var aggregateRow = Context.CreateRow(this, aggregate);
netTimeStopwatch.Stop();
yield return(aggregateRow);
netTimeStopwatch.Start();
}
}
}
groupRows.Add(row);
}
netTimeStopwatch.Start();
if (success && groupRows.Count > 0)
{
var aggregates = new List <SlimRow>();
groupCount++;
try
{
Operation.TransformGroup(groupRows, () =>
{
var aggregate = new SlimRow();
if (FixColumns != null)
{
foreach (var col in FixColumns)
{
aggregate[col.Key] = groupRows[0][col.Value ?? col.Key];
}
}
aggregates.Add(aggregate);
return(aggregate);
});
}
catch (Exception ex)
{
var exception = new MemoryAggregationException(this, Operation, groupRows, ex);
AddException(exception);
success = false;
}
foreach (var groupRow in groupRows)
{
Context.SetRowOwner(groupRow as IRow, null);
}
groupRows.Clear();
if (success)
{
foreach (var aggregate in aggregates)
{
aggregateCount++;
var aggregateRow = Context.CreateRow(this, aggregate);
netTimeStopwatch.Stop();
yield return(aggregateRow);
netTimeStopwatch.Start();
}
}
}
Context.Log(LogSeverity.Debug, this, "evaluated {RowCount} input rows, created {GroupCount} groups and created {AggregateCount} aggregates in {Elapsed}/{ElapsedWallClock}, ignored: {IgnoredRowCount}",
rowCount, groupCount, aggregateCount, InvocationInfo.LastInvocationStarted.Elapsed, netTimeStopwatch.Elapsed, ignoredRowCount);
}
public void CombinedTest()
{
var context = TestExecuter.GetContext();
var builder = SequenceBuilder.Fluent
.ReadFrom(TestData.Person(context))
.ConvertValue(new InPlaceConvertMutator(context)
{
Columns = new[] { "birthDate" },
TypeConverter = new DateConverterAuto(new CultureInfo("hu-HU"))
{
EpochDate = null,
},
ActionIfInvalid = InvalidValueAction.Throw,
})
.Explode(new ExplodeMutator(context)
{
RowCreator = row =>
{
var newRow = new SlimRow
{
["personModel"] = new TestData.PersonModel()
{
Id = row.GetAs <int>("id"),
Name = row.GetAs <string>("name"),
Age = row.GetAs <int?>("age"),
BirthDate = row.GetAs <DateTime?>("birthDate"),
}
};
return(new[] { newRow });
},
})
.SerializeToXml(new DataContractXmlSerializerMutator <TestData.PersonModel>(context)
{
SourceColumn = "personModel",
TargetColumn = "personModelXml",
})
.RemoveColumn(new RemoveColumnMutator(context)
{
Columns = new[] { "personModel" },
})
.DeSerializeFromXml(new DataContractXmlDeSerializerMutator <TestData.PersonModel>(context)
{
SourceColumn = "personModelXml",
TargetColumn = "personModel",
})
.Explode(new ExplodeMutator(context)
{
RowCreator = row =>
{
var personModel = row.GetAs <TestData.PersonModel>("personModel");
var newRow = new SlimRow()
{
["id"] = personModel.Id,
["name"] = personModel.Name,
["age"] = personModel.Age,
["birthDate"] = personModel.BirthDate,
};
return(new[] { newRow });
},
});
var result = TestExecuter.Execute(builder);
Assert.AreEqual(7, result.MutatedRows.Count);
Assert.That.ExactMatch(result.MutatedRows, new List <CaseInsensitiveStringKeyDictionary <object> >()
{
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 0, ["name"] = "A", ["age"] = 17, ["birthDate"] = new DateTime(2010, 12, 9, 0, 0, 0, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 1, ["name"] = "B", ["age"] = 8, ["birthDate"] = new DateTime(2011, 2, 1, 0, 0, 0, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 2, ["name"] = "C", ["age"] = 27, ["birthDate"] = new DateTime(2014, 1, 21, 0, 0, 0, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 3, ["name"] = "D", ["age"] = 39, ["birthDate"] = new DateTime(2018, 7, 11, 0, 0, 0, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 4, ["name"] = "E", ["age"] = -3
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 5, ["name"] = "A", ["age"] = 11, ["birthDate"] = new DateTime(2013, 5, 15, 0, 0, 0, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 6, ["name"] = "fake", ["birthDate"] = new DateTime(2018, 1, 9, 0, 0, 0, 0)
}
});
var exceptions = context.GetExceptions();
Assert.AreEqual(0, exceptions.Count);
}