public void DummyForDevelopment2()
{
var context = TestExecuter.GetContext();
var builder = SequenceBuilder.Fluent
.ReadFrom(TestData.Person(context))
.CustomCode(new CustomMutator(context));
var process = builder.Build();
process.Execute(null);
var msg = context.GetExceptions()[0].FormatExceptionWithDetails(true);
Debug.WriteLine(msg);
Debugger.Break();
}
public void DummyForDevelopment3()
{
var context = TestExecuter.GetContext();
var builder = SequenceBuilder.Fluent
.ReadFrom(TestData.Person(context))
.Join(new JoinMutator(context)
{
LookupBuilder = new RowLookupBuilder()
{
Process = TestData.PersonEyeColor(context),
KeyGenerator = row => row.GenerateKey("personId"),
},
RowKeyGenerator = row => row.GenerateKey("id"),
NoMatchAction = new NoMatchAction(MatchMode.Throw),
Columns = new(),
});
public void HasErrorFalse()
{
var context = TestExecuter.GetContext();
context.SetRowType <DictionaryRow>();
var initialValues = new Dictionary <string, object>()
{
["id"] = 12,
["name"] = "A",
};
var row = context.CreateRow(null, initialValues);
Assert.IsFalse(row.HasError());
}
public void DateTimeKeyIsInvariantWithMilliseconds()
{
var context = TestExecuter.GetContext();
context.SetRowType <DictionaryRow>();
var initialValues = new Dictionary <string, object>()
{
["date"] = new DateTime(2020, 02, 20, 12, 12, 0, 666),
};
var row = context.CreateRow(null, initialValues);
var result = row.GenerateKey("date");
Assert.AreEqual("2020.02.20 12:12:00.6660000", result);
}
public void TimeSpanKeyIsInvariantWithDaysAndMilliseconds()
{
var context = TestExecuter.GetContext();
context.SetRowType <DictionaryRow>();
var initialValues = new Dictionary <string, object>()
{
["time"] = new TimeSpan(1, 1, 0),
};
var row = context.CreateRow(null, initialValues);
var result = row.GenerateKey("time");
Assert.AreEqual("0:01:01:00.0000000", result);
}
public void KeyCaseIgnored()
{
var context = TestExecuter.GetContext();
context.SetRowType <DictionaryRow>();
var initialValues = new Dictionary <string, object>()
{
["date"] = new DateTime(2020, 02, 20, 12, 12, 0, 666),
};
var row = context.CreateRow(null, initialValues);
var result = row["DATE"];
Assert.AreEqual(new DateTime(2020, 02, 20, 12, 12, 0, 666), result);
}
public void NoNewLevelColumn()
{
var context = TestExecuter.GetContext();
var builder = SequenceBuilder.Fluent
.ReadFrom(TestData.RoleHierarchy(context))
.ResolveHierarchy(new ResolveHierarchyMutator(context)
{
IdentityColumn = "id",
NewColumnWithParentId = "parentId",
LevelColumns = new[] { "level1", "level2", "level3" },
RemoveLevelColumns = true,
});
var result = TestExecuter.Execute(builder);
Assert.AreEqual(6, result.MutatedRows.Count);
Assert.That.ExactMatch(result.MutatedRows, new List <CaseInsensitiveStringKeyDictionary <object> >()
{
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 0, ["code"] = "A"
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 1, ["code"] = "B", ["parentId"] = 0
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 2, ["code"] = "C", ["parentId"] = 1
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 3, ["code"] = "D", ["parentId"] = 1
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 4, ["code"] = "E", ["parentId"] = 0
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 5, ["code"] = "F", ["parentId"] = 0
}
});
var exceptions = context.GetExceptions();
Assert.AreEqual(0, exceptions.Count);
}
public void FullCopy()
{
var context = TestExecuter.GetContext();
var builder = SequenceBuilder.Fluent
.ImportEnumerable(new EnumerableImporter(context)
{
InputGenerator = caller => TestData.Person(context).TakeRowsAndReleaseOwnership(caller),
});
var result = TestExecuter.Execute(builder);
Assert.AreEqual(7, result.MutatedRows.Count);
Assert.That.ExactMatch(result.MutatedRows, new List <CaseInsensitiveStringKeyDictionary <object> >()
{
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 0, ["name"] = "A", ["age"] = 17, ["height"] = 160, ["eyeColor"] = "brown", ["countryId"] = 1, ["birthDate"] = new DateTime(2010, 12, 9, 0, 0, 0, 0), ["lastChangedTime"] = new DateTime(2015, 12, 19, 12, 0, 1, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 1, ["name"] = "B", ["age"] = 8, ["height"] = 190, ["countryId"] = 1, ["birthDate"] = new DateTime(2011, 2, 1, 0, 0, 0, 0), ["lastChangedTime"] = new DateTime(2015, 12, 19, 13, 2, 0, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 2, ["name"] = "C", ["age"] = 27, ["height"] = 170, ["eyeColor"] = "green", ["countryId"] = 2, ["birthDate"] = new DateTime(2014, 1, 21, 0, 0, 0, 0), ["lastChangedTime"] = new DateTime(2015, 11, 21, 17, 11, 58, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 3, ["name"] = "D", ["age"] = 39, ["height"] = 160, ["eyeColor"] = "fake", ["birthDate"] = "2018.07.11", ["lastChangedTime"] = new DateTime(2017, 8, 1, 4, 9, 1, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 4, ["name"] = "E", ["age"] = -3, ["height"] = 160, ["countryId"] = 1, ["lastChangedTime"] = new DateTime(2019, 1, 1, 23, 59, 59, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 5, ["name"] = "A", ["age"] = 11, ["height"] = 140, ["birthDate"] = new DateTime(2013, 5, 15, 0, 0, 0, 0), ["lastChangedTime"] = new DateTime(2018, 1, 1, 0, 0, 0, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 6, ["name"] = "fake", ["height"] = 140, ["countryId"] = 5, ["birthDate"] = new DateTime(2018, 1, 9, 0, 0, 0, 0)
}
});
var exceptions = context.GetExceptions();
Assert.AreEqual(0, exceptions.Count);
}
public void KeyCaseIsIgnoredDuringUnitTests()
{
var topic = TestExecuter.GetTopic();
var builder = new ProcessBuilder()
{
InputProcess = TestData.Person(topic),
Mutators = new MutatorList(),
};
var result = TestExecuter.Execute(builder);
Assert.AreEqual(7, result.MutatedRows.Count);
Assert.That.ExactMatch(result.MutatedRows, new List <CaseInsensitiveStringKeyDictionary <object> >()
{
new CaseInsensitiveStringKeyDictionary <object>()
{
["ID"] = 0, ["name"] = "A", ["age"] = 17, ["height"] = 160, ["eyeColor"] = "brown", ["countryId"] = 1, ["birthDate"] = new DateTime(2010, 12, 9, 0, 0, 0, 0), ["lastChangedTime"] = new DateTime(2015, 12, 19, 12, 0, 1, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["ID"] = 1, ["NAME"] = "B", ["Age"] = 8, ["height"] = 190, ["countryId"] = 1, ["birthDate"] = new DateTime(2011, 2, 1, 0, 0, 0, 0), ["lastChangedTime"] = new DateTime(2015, 12, 19, 13, 2, 0, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["ID"] = 2, ["name"] = "C", ["age"] = 27, ["height"] = 170, ["eyeColor"] = "green", ["countryId"] = 2, ["birthDate"] = new DateTime(2014, 1, 21, 0, 0, 0, 0), ["lastChangedTime"] = new DateTime(2015, 11, 21, 17, 11, 58, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["ID"] = 3, ["Name"] = "D", ["agE"] = 39, ["height"] = 160, ["eyeColor"] = "fake", ["birthDate"] = "2018.07.11", ["lastChangedTime"] = new DateTime(2017, 8, 1, 4, 9, 1, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 4, ["name"] = "E", ["age"] = -3, ["height"] = 160, ["countryId"] = 1, ["lastChangedTime"] = new DateTime(2019, 1, 1, 23, 59, 59, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 5, ["NaME"] = "A", ["aGe"] = 11, ["height"] = 140, ["birthDate"] = new DateTime(2013, 5, 15, 0, 0, 0, 0), ["lastChangedTime"] = new DateTime(2018, 1, 1, 0, 0, 0, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["ID"] = 6, ["name"] = "fake", ["height"] = 140, ["countryId"] = 5, ["birthDate"] = new DateTime(2018, 1, 9, 0, 0, 0, 0)
}
});
var exceptions = topic.Context.GetExceptions();
Assert.AreEqual(0, exceptions.Count);
}
public void RemoveWhenMatchAndEquals()
{
var topic = TestExecuter.GetTopic();
var builder = ProcessBuilder.Fluent
.ReadFrom(TestData.Person(topic))
.CompareWithRow(new CompareWithRowMutator(topic, null)
{
LookupBuilder = new RowLookupBuilder()
{
Process = TestData.PersonChanged(topic),
KeyGenerator = row => row.GenerateKey("id"),
},
RowKeyGenerator = row => row.GenerateKey("id"),
EqualityComparer = new ColumnBasedRowEqualityComparer(),
MatchAndEqualsAction = new MatchAction(MatchMode.Remove),
});
var result = TestExecuter.Execute(builder);
Assert.AreEqual(5, result.MutatedRows.Count);
Assert.That.ExactMatch(result.MutatedRows, new List <CaseInsensitiveStringKeyDictionary <object> >()
{
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 0, ["name"] = "A", ["age"] = 17, ["height"] = 160, ["eyeColor"] = "brown", ["countryId"] = 1, ["birthDate"] = new DateTime(2010, 12, 9, 0, 0, 0, 0), ["lastChangedTime"] = new DateTime(2015, 12, 19, 12, 0, 1, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 3, ["name"] = "D", ["age"] = 39, ["height"] = 160, ["eyeColor"] = "fake", ["birthDate"] = "2018.07.11", ["lastChangedTime"] = new DateTime(2017, 8, 1, 4, 9, 1, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 4, ["name"] = "E", ["age"] = -3, ["height"] = 160, ["countryId"] = 1, ["lastChangedTime"] = new DateTime(2019, 1, 1, 23, 59, 59, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 5, ["name"] = "A", ["age"] = 11, ["height"] = 140, ["birthDate"] = new DateTime(2013, 5, 15, 0, 0, 0, 0), ["lastChangedTime"] = new DateTime(2018, 1, 1, 0, 0, 0, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 6, ["name"] = "fake", ["height"] = 140, ["countryId"] = 5, ["birthDate"] = new DateTime(2018, 1, 9, 0, 0, 0, 0)
}
});
var exceptions = topic.Context.GetExceptions();
Assert.AreEqual(0, exceptions.Count);
}
public void MissingFileThrowsFileReadException()
{
var context = TestExecuter.GetContext();
var reader = GetReader(context, @".\TestData\MissingFile.xlsx");
var builder = SequenceBuilder.Fluent
.ReadFrom(reader)
.ThrowExceptionOnRowError();
var result = TestExecuter.Execute(builder);
Assert.AreEqual(0, result.MutatedRows.Count);
var exceptions = context.GetExceptions();
Assert.AreEqual(1, exceptions.Count);
Assert.IsTrue(exceptions[0] is LocalFileReadException);
}
public void StageNotApplied()
{
var topic = TestExecuter.GetTopic();
var builder = ProcessBuilder.Fluent
.ReadFrom(TestData.Person(topic))
.CustomCode(new CustomMutator(topic, null)
{
Then = (proc, row) =>
{
row.SetStagedValue("test", "test");
if (row.GetAs <int>("id") < 4)
{
row.ApplyStaging();
}
return(true);
}
});
var result = TestExecuter.Execute(builder);
Assert.AreEqual(4, result.MutatedRows.Count);
Assert.That.ExactMatch(result.MutatedRows, new List <CaseInsensitiveStringKeyDictionary <object> >()
{
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 0, ["name"] = "A", ["age"] = 17, ["height"] = 160, ["eyeColor"] = "brown", ["countryId"] = 1, ["birthDate"] = new DateTime(2010, 12, 9, 0, 0, 0, 0), ["lastChangedTime"] = new DateTime(2015, 12, 19, 12, 0, 1, 0), ["test"] = "test"
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 1, ["name"] = "B", ["age"] = 8, ["height"] = 190, ["countryId"] = 1, ["birthDate"] = new DateTime(2011, 2, 1, 0, 0, 0, 0), ["lastChangedTime"] = new DateTime(2015, 12, 19, 13, 2, 0, 0), ["test"] = "test"
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 2, ["name"] = "C", ["age"] = 27, ["height"] = 170, ["eyeColor"] = "green", ["countryId"] = 2, ["birthDate"] = new DateTime(2014, 1, 21, 0, 0, 0, 0), ["lastChangedTime"] = new DateTime(2015, 11, 21, 17, 11, 58, 0), ["test"] = "test"
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 3, ["name"] = "D", ["age"] = 39, ["height"] = 160, ["eyeColor"] = "fake", ["birthDate"] = "2018.07.11", ["lastChangedTime"] = new DateTime(2017, 8, 1, 4, 9, 1, 0), ["test"] = "test"
}
});
var exceptions = topic.Context.GetExceptions();
Assert.AreEqual(1, exceptions.Count);
Assert.IsTrue(exceptions[0] is ProcessExecutionException);
}
public void RemoveAll()
{
var context = TestExecuter.GetContext();
var builder = SequenceBuilder.Fluent
.ReadFrom(TestData.Person(context))
.RemoveRow(new RemoveRowMutator(context)
{
RowFilter = row => true,
});
var result = TestExecuter.Execute(builder);
Assert.AreEqual(0, result.MutatedRows.Count);
var exceptions = context.GetExceptions();
Assert.AreEqual(0, exceptions.Count);
}
public void HasErrorTrue()
{
var context = TestExecuter.GetContext();
context.SetRowType <DictionaryRow>();
var initialValues = new Dictionary <string, object>()
{
["id"] = 12,
["name"] = "A",
["err"] = new EtlRowError(9),
};
var row = new TrackedRow(context.CreateRow(null, initialValues));
Assert.IsTrue(row.HasError());
}
public void ToDebugStringStartsWithUid()
{
var context = TestExecuter.GetContext();
context.SetRowType <DictionaryRow>();
var initialValues = new Dictionary <string, object>()
{
["id"] = 12,
["name"] = "x",
};
var row = new TrackedRow(context.CreateRow(null, initialValues));
var result = row.ToDebugString();
Assert.IsTrue(result.StartsWith("uid", StringComparison.InvariantCultureIgnoreCase));
}
public void RemoveAll()
{
var context = TestExecuter.GetContext();
var builder = SequenceBuilder.Fluent
.ReadFrom(TestData.Person(context))
.RemoveColumn(new RemoveColumnMutator(context)
{
Columns = TestData.PersonColumns,
});
var result = TestExecuter.Execute(builder);
Assert.AreEqual(0, result.MutatedRows.Sum(x => x.ColumnCount));
var exceptions = context.GetExceptions();
Assert.AreEqual(0, exceptions.Count);
}
public void SingleKey()
{
var topic = TestExecuter.GetTopic();
var builder = new ProcessBuilder()
{
InputProcess = TestData.Person(topic),
Mutators = new MutatorList()
{
new ThrowExceptionOnDuplicateKeyMutator(topic, null)
{
RowKeyGenerator = row => row.GenerateKey("name"),
},
},
};
var result = TestExecuter.Execute(builder);
Assert.AreEqual(5, result.MutatedRows.Count);
Assert.That.ExactMatch(result.MutatedRows, new List <CaseInsensitiveStringKeyDictionary <object> >()
{
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 0, ["name"] = "A", ["age"] = 17, ["height"] = 160, ["eyeColor"] = "brown", ["countryId"] = 1, ["birthDate"] = new DateTime(2010, 12, 9, 0, 0, 0, 0), ["lastChangedTime"] = new DateTime(2015, 12, 19, 12, 0, 1, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 1, ["name"] = "B", ["age"] = 8, ["height"] = 190, ["countryId"] = 1, ["birthDate"] = new DateTime(2011, 2, 1, 0, 0, 0, 0), ["lastChangedTime"] = new DateTime(2015, 12, 19, 13, 2, 0, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 2, ["name"] = "C", ["age"] = 27, ["height"] = 170, ["eyeColor"] = "green", ["countryId"] = 2, ["birthDate"] = new DateTime(2014, 1, 21, 0, 0, 0, 0), ["lastChangedTime"] = new DateTime(2015, 11, 21, 17, 11, 58, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 3, ["name"] = "D", ["age"] = 39, ["height"] = 160, ["eyeColor"] = "fake", ["birthDate"] = "2018.07.11", ["lastChangedTime"] = new DateTime(2017, 8, 1, 4, 9, 1, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 4, ["name"] = "E", ["age"] = -3, ["height"] = 160, ["countryId"] = 1, ["lastChangedTime"] = new DateTime(2019, 1, 1, 23, 59, 59, 0)
}
});
var exceptions = topic.Context.GetExceptions();
Assert.AreEqual(1, exceptions.Count);
Assert.IsTrue(exceptions[0] is ProcessExecutionException);
}
public void NullIfNoKey()
{
var context = TestExecuter.GetContext();
var builder = SequenceBuilder.Fluent
.ReadFrom(TestData.Person(context))
.AddKeyHash("eyeColor-hash", "eyeColor");
var result = TestExecuter.Execute(builder);
Assert.AreEqual(7, result.MutatedRows.Count);
Assert.That.ExactMatch(result.MutatedRows, new List <CaseInsensitiveStringKeyDictionary <object> >()
{
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 0, ["name"] = "A", ["age"] = 17, ["height"] = 160, ["eyeColor"] = "brown", ["countryId"] = 1, ["birthDate"] = new DateTime(2010, 12, 9, 0, 0, 0, 0), ["lastChangedTime"] = new DateTime(2015, 12, 19, 12, 0, 1, 0), ["eyeColor-hash"] = "5eb67f9f8409b9c3f739735633cbdf92121393d0e13bd0f464b1b2a6a15ad2dc"
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 1, ["name"] = "B", ["age"] = 8, ["height"] = 190, ["countryId"] = 1, ["birthDate"] = new DateTime(2011, 2, 1, 0, 0, 0, 0), ["lastChangedTime"] = new DateTime(2015, 12, 19, 13, 2, 0, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 2, ["name"] = "C", ["age"] = 27, ["height"] = 170, ["eyeColor"] = "green", ["countryId"] = 2, ["birthDate"] = new DateTime(2014, 1, 21, 0, 0, 0, 0), ["lastChangedTime"] = new DateTime(2015, 11, 21, 17, 11, 58, 0), ["eyeColor-hash"] = "ba4788b226aa8dc2e6dc74248bb9f618cfa8c959e0c26c147be48f6839a0b088"
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 3, ["name"] = "D", ["age"] = 39, ["height"] = 160, ["eyeColor"] = "fake", ["birthDate"] = "2018.07.11", ["lastChangedTime"] = new DateTime(2017, 8, 1, 4, 9, 1, 0), ["eyeColor-hash"] = "b5d54c39e66671c9731b9f471e585d8262cd4f54963f0c93082d8dcf334d4c78"
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 4, ["name"] = "E", ["age"] = -3, ["height"] = 160, ["countryId"] = 1, ["lastChangedTime"] = new DateTime(2019, 1, 1, 23, 59, 59, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 5, ["name"] = "A", ["age"] = 11, ["height"] = 140, ["birthDate"] = new DateTime(2013, 5, 15, 0, 0, 0, 0), ["lastChangedTime"] = new DateTime(2018, 1, 1, 0, 0, 0, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 6, ["name"] = "fake", ["height"] = 140, ["countryId"] = 5, ["birthDate"] = new DateTime(2018, 1, 9, 0, 0, 0, 0)
}
});
var exceptions = context.GetExceptions();
Assert.AreEqual(0, exceptions.Count);
}
public void OneColumn()
{
var context = TestExecuter.GetContext();
var builder = SequenceBuilder.Fluent
.ReadFrom(TestData.Person(context))
.AddKeyHash("height-hash", "height");
var result = TestExecuter.Execute(builder);
Assert.AreEqual(7, result.MutatedRows.Count);
Assert.That.ExactMatch(result.MutatedRows, new List <CaseInsensitiveStringKeyDictionary <object> >()
{
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 0, ["name"] = "A", ["age"] = 17, ["height"] = 160, ["eyeColor"] = "brown", ["countryId"] = 1, ["birthDate"] = new DateTime(2010, 12, 9, 0, 0, 0, 0), ["lastChangedTime"] = new DateTime(2015, 12, 19, 12, 0, 1, 0), ["height-hash"] = "a512db2741cd20693e4b16f19891e72b9ff12cead72761fc5e92d2aaf34740c1"
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 1, ["name"] = "B", ["age"] = 8, ["height"] = 190, ["countryId"] = 1, ["birthDate"] = new DateTime(2011, 2, 1, 0, 0, 0, 0), ["lastChangedTime"] = new DateTime(2015, 12, 19, 13, 2, 0, 0), ["height-hash"] = "2397346b45823e070f6fc72ac94c0a999d234c472479f0e26b30cdf5942db854"
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 2, ["name"] = "C", ["age"] = 27, ["height"] = 170, ["eyeColor"] = "green", ["countryId"] = 2, ["birthDate"] = new DateTime(2014, 1, 21, 0, 0, 0, 0), ["lastChangedTime"] = new DateTime(2015, 11, 21, 17, 11, 58, 0), ["height-hash"] = "734d0759cdb4e0d0a35e4fd73749aee287e4fdcc8648b71a8d6ed591b7d4cb3f"
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 3, ["name"] = "D", ["age"] = 39, ["height"] = 160, ["eyeColor"] = "fake", ["birthDate"] = "2018.07.11", ["lastChangedTime"] = new DateTime(2017, 8, 1, 4, 9, 1, 0), ["height-hash"] = "a512db2741cd20693e4b16f19891e72b9ff12cead72761fc5e92d2aaf34740c1"
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 4, ["name"] = "E", ["age"] = -3, ["height"] = 160, ["countryId"] = 1, ["lastChangedTime"] = new DateTime(2019, 1, 1, 23, 59, 59, 0), ["height-hash"] = "a512db2741cd20693e4b16f19891e72b9ff12cead72761fc5e92d2aaf34740c1"
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 5, ["name"] = "A", ["age"] = 11, ["height"] = 140, ["birthDate"] = new DateTime(2013, 5, 15, 0, 0, 0, 0), ["lastChangedTime"] = new DateTime(2018, 1, 1, 0, 0, 0, 0), ["height-hash"] = "dbae772db29058a88f9bd830e957c695347c41b6162a7eb9a9ea13def34be56b"
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 6, ["name"] = "fake", ["height"] = 140, ["countryId"] = 5, ["birthDate"] = new DateTime(2018, 1, 9, 0, 0, 0, 0), ["height-hash"] = "dbae772db29058a88f9bd830e957c695347c41b6162a7eb9a9ea13def34be56b"
}
});
var exceptions = context.GetExceptions();
Assert.AreEqual(0, exceptions.Count);
}
public void RemoveSome()
{
var context = TestExecuter.GetContext();
var builder = new ProcessBuilder()
{
InputProcess = TestData.Person(context),
Mutators = new MutatorList()
{
new RemoveRowMutator(context)
{
RowFilter = row => row.GetAs <string>("name") == "A",
},
},
};
var result = TestExecuter.Execute(builder);
Assert.AreEqual(5, result.MutatedRows.Count);
Assert.That.ExactMatch(result.MutatedRows, new List <CaseInsensitiveStringKeyDictionary <object> >()
{
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 1, ["name"] = "B", ["age"] = 8, ["height"] = 190, ["countryId"] = 1, ["birthDate"] = new DateTime(2011, 2, 1, 0, 0, 0, 0), ["lastChangedTime"] = new DateTime(2015, 12, 19, 13, 2, 0, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 2, ["name"] = "C", ["age"] = 27, ["height"] = 170, ["eyeColor"] = "green", ["countryId"] = 2, ["birthDate"] = new DateTime(2014, 1, 21, 0, 0, 0, 0), ["lastChangedTime"] = new DateTime(2015, 11, 21, 17, 11, 58, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 3, ["name"] = "D", ["age"] = 39, ["height"] = 160, ["eyeColor"] = "fake", ["birthDate"] = "2018.07.11", ["lastChangedTime"] = new DateTime(2017, 8, 1, 4, 9, 1, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 4, ["name"] = "E", ["age"] = -3, ["height"] = 160, ["countryId"] = 1, ["lastChangedTime"] = new DateTime(2019, 1, 1, 23, 59, 59, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 6, ["name"] = "fake", ["height"] = 140, ["countryId"] = 5, ["birthDate"] = new DateTime(2018, 1, 9, 0, 0, 0, 0)
}
});
var exceptions = context.GetExceptions();
Assert.AreEqual(0, exceptions.Count);
}
public void IntKeyIsInvariant()
{
var context = TestExecuter.GetContext();
context.SetRowType <DictionaryRow>();
var initialValues = new Dictionary <string, object>()
{
["id"] = 1234567,
["date"] = new DateTime(2020, 02, 20, 12, 12, 0, 666),
};
var row = new TrackedRow(context.CreateRow(null, initialValues));
var result = row.GenerateKey("id", "date");
Assert.IsTrue(result.Contains("1234567", StringComparison.Ordinal));
Assert.IsTrue(result.Contains("2020.02.20 12:12:00.6660000", StringComparison.Ordinal));
}
public void StagedValueIsNotVisibleUntilApplied()
{
var context = TestExecuter.GetContext();
context.SetRowType <DictionaryRow>();
var initialValues = new Dictionary <string, object>()
{
["name"] = "A",
};
var row = context.CreateRow(null, initialValues);
row.SetStagedValue("name", "B");
Assert.AreEqual("A", row.GetAs <string>("name"));
Assert.IsTrue(row.HasStaging);
}
public void SingleKey()
{
var context = TestExecuter.GetContext();
var builder = SequenceBuilder.Fluent
.ReadFrom(TestData.Person(context))
.RemoveDuplicateRows(new RemoveDuplicateRowsMutator(context)
{
KeyGenerator = row => row.GenerateKey("name"),
});
var result = TestExecuter.Execute(builder);
Assert.AreEqual(6, result.MutatedRows.Count);
Assert.That.ExactMatch(result.MutatedRows, new List <CaseInsensitiveStringKeyDictionary <object> >()
{
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 0, ["name"] = "A", ["age"] = 17, ["height"] = 160, ["eyeColor"] = "brown", ["countryId"] = 1, ["birthDate"] = new DateTime(2010, 12, 9, 0, 0, 0, 0), ["lastChangedTime"] = new DateTime(2015, 12, 19, 12, 0, 1, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 1, ["name"] = "B", ["age"] = 8, ["height"] = 190, ["countryId"] = 1, ["birthDate"] = new DateTime(2011, 2, 1, 0, 0, 0, 0), ["lastChangedTime"] = new DateTime(2015, 12, 19, 13, 2, 0, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 2, ["name"] = "C", ["age"] = 27, ["height"] = 170, ["eyeColor"] = "green", ["countryId"] = 2, ["birthDate"] = new DateTime(2014, 1, 21, 0, 0, 0, 0), ["lastChangedTime"] = new DateTime(2015, 11, 21, 17, 11, 58, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 3, ["name"] = "D", ["age"] = 39, ["height"] = 160, ["eyeColor"] = "fake", ["birthDate"] = "2018.07.11", ["lastChangedTime"] = new DateTime(2017, 8, 1, 4, 9, 1, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 4, ["name"] = "E", ["age"] = -3, ["height"] = 160, ["countryId"] = 1, ["lastChangedTime"] = new DateTime(2019, 1, 1, 23, 59, 59, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 6, ["name"] = "fake", ["height"] = 140, ["countryId"] = 5, ["birthDate"] = new DateTime(2018, 1, 9, 0, 0, 0, 0)
}
});
var exceptions = context.GetExceptions();
Assert.AreEqual(0, exceptions.Count);
}
public void TagFilterDelegate()
{
var context = TestExecuter.GetContext();
var builder = ProcessBuilder.Fluent
.ReadFrom(TestData.Person(context))
.SetTag(new SetTagMutator(context)
{
RowFilter = row => row.GetAs <int>("id") > 2,
Tag = RowKind.test,
})
.CustomCode(new CustomMutator(context)
{
RowTagFilter = tag => tag is RowKind rk && rk == RowKind.test,
Action = row =>
{
row["test"] = "test";
return(true);
}
});
public void SplitterTestWithMerger()
{
var context = TestExecuter.GetContext();
var splitter = new Splitter <DefaultRowQueue>(context)
{
InputProcess = new EnumerableImporter(context)
{
InputGenerator = caller => TestData.Person(context).Evaluate(caller).TakeRowsAndReleaseOwnership(),
},
};
var merger = new ParallelMerger(context)
{
ProcessList = new List <IProducer>(),
};
for (var i = 0; i < 3; i++)
{
merger.ProcessList.Add(new CustomMutator(context)
{
InputProcess = splitter,
Action = row =>
{
Thread.Sleep(new Random().Next(10));
row["ThreadIndex"] = i;
return(true);
},
});
}
var result = merger.Evaluate().TakeRowsAndReleaseOwnership().ToList();
Assert.AreEqual(7, result.Count);
foreach (var p in TestData.Person(context).Evaluate().TakeRowsAndReleaseOwnership())
{
Assert.IsTrue(result.Any(m => m.GetAs <int>("id") == p.GetAs <int>("id")));
}
var exceptions = context.GetExceptions();
Assert.AreEqual(0, exceptions.Count);
}
public void NoMatchCustom()
{
var context = TestExecuter.GetContext();
var builder = ProcessBuilder.Fluent
.ReadFrom(TestData.Person(context))
.Join(new JoinMutator(context)
{
LookupBuilder = new RowLookupBuilder()
{
Process = TestData.PersonEyeColor(context),
KeyGenerator = row => row.GenerateKey("personId"),
},
RowKeyGenerator = row => row.GenerateKey("id"),
NoMatchAction = new NoMatchAction(MatchMode.Custom)
{
CustomAction = row => row["eyeColor"] = "not found",
},
Columns = new()
{
public void QuatedDelimiter_ThrowOnMissingClose()
{
var topic = TestExecuter.GetTopic();
var builder = new ProcessBuilder()
{
InputProcess = GetReader(topic, @"TestData\SampleQuatedDelimiterMissingClose.csv", throwOnMissingDoubleQuoteClose: true),
Mutators = new MutatorList
{
new ThrowExceptionOnRowErrorMutator(topic)
}
};
var result = TestExecuter.Execute(builder);
Assert.AreEqual(0, result.MutatedRows.Count);
var exceptions = topic.Context.GetExceptions();
Assert.AreEqual(1, exceptions.Count);
}
public void FluentProcessBuilderTest()
{
var context = TestExecuter.GetContext();
var n = 0;
var builder = ProcessBuilder.Fluent
.ImportEnumerable(new EnumerableImporter(context)
{
InputGenerator = caller => TestData.Person(context).Evaluate(caller).TakeRowsAndReleaseOwnership(),
})
.ProcessOnMultipleThreads(3, (i, mb) => mb
.CustomCode(new CustomMutator(context)
{
Action = row =>
{
Thread.Sleep(new Random().Next(10));
row["ThreadIndex"] = i;
return(true);
},
})
)
.CustomCode(new CustomMutator(context)
{
Action = row =>
{
row["AbsoluteFinalIndex"] = n++;
return(true);
},
});
var result = TestExecuter.Execute(builder);
Assert.AreEqual(7, result.MutatedRows.Count);
foreach (var p in TestData.Person(context).Evaluate().TakeRowsAndReleaseOwnership())
{
Assert.IsTrue(result.MutatedRows.Any(m => m.GetAs <int>("id") == p.GetAs <int>("id")));
}
var exceptions = context.GetExceptions();
Assert.AreEqual(0, exceptions.Count);
}
public void NullableMulti()
{
var topic = TestExecuter.GetTopic();
var lookup = new CountableOnlyRowLookup();
var builder = new RowLookupBuilder()
{
Process = TestData.Person(topic),
KeyGenerator = row => row.GenerateKey("name"),
};
builder.Append(lookup, null);
Assert.AreEqual(6, lookup.Keys.Count());
Assert.AreEqual(2, lookup.CountByKey("A"));
Assert.AreEqual(1, lookup.CountByKey("B"));
Assert.AreEqual(1, lookup.CountByKey("C"));
Assert.AreEqual(1, lookup.CountByKey("D"));
Assert.AreEqual(1, lookup.CountByKey("E"));
Assert.AreEqual(1, lookup.CountByKey("fake"));
}
public void TagFilterDelegate()
{
var topic = TestExecuter.GetTopic();
var builder = ProcessBuilder.Fluent
.ReadFrom(TestData.Person(topic))
.SetTag(new SetTagMutator(topic, null)
{
If = row => row.GetAs <int>("id") > 2,
Tag = RowKind.test,
})
.CustomCode(new CustomMutator(topic, null)
{
TagFilter = tag => tag is RowKind rk && rk == RowKind.test,
Then = (proc, row) =>
{
row.SetValue("test", "test");
return(true);
}
});
