/// <exception cref="System.Exception"/> public virtual void TestPercentFilter() { Log.Info("Testing Percent Filter with frequency: 1000"); // set the filter class SequenceFileInputFilter.SetFilterClass(job, typeof(SequenceFileInputFilter.PercentFilter )); SequenceFileInputFilter.PercentFilter.SetFrequency(job.GetConfiguration(), 1000); // clean input dir fs.Delete(inDir, true); // for a variety of lengths for (int length = 0; length < MaxLength; length += random.Next(MaxLength / 10) + 1) { Log.Info("******Number of records: " + length); CreateSequenceFile(length); int count = CountRecords(1); Log.Info("Accepted " + count + " records"); int expectedCount = length / 1000; if (expectedCount * 1000 != length) { expectedCount++; } NUnit.Framework.Assert.AreEqual(count, expectedCount); } // clean up fs.Delete(inDir, true); }
/// <exception cref="System.Exception"/> public virtual void TestMD5Filter() { // set the filter class Log.Info("Testing MD5 Filter with frequency: 1000"); SequenceFileInputFilter.SetFilterClass(job, typeof(SequenceFileInputFilter.MD5Filter )); SequenceFileInputFilter.MD5Filter.SetFrequency(job.GetConfiguration(), 1000); // clean input dir fs.Delete(inDir, true); // for a variety of lengths for (int length = 0; length < MaxLength; length += random.Next(MaxLength / 10) + 1) { Log.Info("******Number of records: " + length); CreateSequenceFile(length); Log.Info("Accepted " + CountRecords(0) + " records"); } // clean up fs.Delete(inDir, true); }
/// <exception cref="System.Exception"/> public virtual void TestRegexFilter() { // set the filter class Log.Info("Testing Regex Filter with patter: \\A10*"); SequenceFileInputFilter.SetFilterClass(job, typeof(SequenceFileInputFilter.RegexFilter )); SequenceFileInputFilter.RegexFilter.SetPattern(job.GetConfiguration(), "\\A10*"); // clean input dir fs.Delete(inDir, true); // for a variety of lengths for (int length = 1; length < MaxLength; length += random.Next(MaxLength / 10) + 1) { Log.Info("******Number of records: " + length); CreateSequenceFile(length); int count = CountRecords(0); NUnit.Framework.Assert.AreEqual(count, length == 0 ? 0 : (int)Math.Log10(length) + 1); } // clean up fs.Delete(inDir, true); }