public void ShortSet_CapacityHandling()
{
// Verify values above capacity are not reported, even if there are bits for them
ShortSet s1 = new ShortSet(10);
s1.Not();
Assert.AreEqual("0, 1, 2, 3, 4, 5, 6, 7, 8, 9", String.Join(", ", s1.Values));
// Verify the last value is not truncated if the this set in an operation is larger,
// and that values above the other capacity aren't involved in operations.
ShortSet s2 = new ShortSet(120);
s2.Not();
ShortSet s3 = new ShortSet(64);
s3.Not();
s2.And(s3);
Assert.IsTrue(s2.Contains(63));
s2.Or(s3);
Assert.IsTrue(s2.Contains(63));
s3.Not();
s2.AndNot(s3);
Assert.IsTrue(s2.Contains(63));
}
private static void FromAndPerformance()
{
ShortSet s0 = new ShortSet(ushort.MaxValue);
ShortSet s1 = new ShortSet(ushort.MaxValue);
s1.Not();
ShortSet s2 = Arriba.Test.ShortSetTests.BuildRandom(ushort.MaxValue, 10000, new Random());
s2.Count();
int iterations = 3000000;
Stopwatch w = Stopwatch.StartNew();
for (int i = 0; i < iterations; ++i)
{
s0.FromAnd(s1, s2);
}
w.Stop();
double milliseconds = w.ElapsedMilliseconds;
double operationsPerMillisecond = iterations / milliseconds;
Trace.Write(String.Format("{0:n0}\r\n", s0.Count()));
Trace.Write(String.Format("{0:n0} operations in {1:n0} milliseconds; {2:n0} per millisecond.\r\n", iterations, milliseconds, operationsPerMillisecond));
}
private static void SetCountPerformance()
{
ShortSet s0 = new ShortSet(ushort.MaxValue);
ShortSet s1 = new ShortSet(ushort.MaxValue);
s1.Not();
ShortSet s2 = Arriba.Test.ShortSetTests.BuildRandom(ushort.MaxValue, 10000, new Random());
ushort value = 0;
ushort value2 = 0;
ushort value3 = 0;
int iterations = 1000000;
Stopwatch w = Stopwatch.StartNew();
for (int i = 0; i < iterations; ++i)
{
value = s0.Count();
value2 = s1.Count();
value3 = s2.Count();
//value = (ushort)ShortSet.CallOverheadTest();
//value2 = (ushort)ShortSet.CallOverheadTest();
//value3 = (ushort)ShortSet.CallOverheadTest();
}
w.Stop();
double milliseconds = w.ElapsedMilliseconds;
double operationsPerMillisecond = iterations / milliseconds;
Trace.Write(String.Format("{0:n0}, {1:n0}, {2:n0}\r\n", value, value2, value3));
Trace.Write(String.Format("{0:n0} operations in {1:n0} milliseconds; {2:n0} per millisecond.\r\n", iterations, milliseconds, operationsPerMillisecond));
}
public void ShortSet_Performance_Count()
{
// Goal: Count is <10k instructions. 2GHz is 2B/sec, so 2M/ms, so 10k each means 200 iterations per ms (Release build)
// Count is used for COUNT(*) aggregate and to compute IntelliSense rank for words in the word index.
ShortSet s0 = new ShortSet(ushort.MaxValue);
ShortSet s1 = new ShortSet(ushort.MaxValue);
s1.Not();
ShortSet s2 = BuildRandom(ushort.MaxValue, 10000, new Random());
int iterations = 10000;
Stopwatch w = Stopwatch.StartNew();
for (int i = 0; i < iterations; ++i)
{
Assert.AreEqual(0, s0.Count());
Assert.AreEqual(ushort.MaxValue, s1.Count());
Assert.AreEqual(10000, s2.Count());
}
double milliseconds = w.ElapsedMilliseconds;
double operationsPerMillisecond = (3 * iterations) / milliseconds;
Trace.Write(String.Format("{0:n0} operations in {1:n0} milliseconds; {2:n0} per millisecond.", iterations, milliseconds, operationsPerMillisecond));
Assert.IsTrue(operationsPerMillisecond > 50, "Not within 200% of goal.");
}
public BooleanColumn(bool defaultValue)
{
this.DefaultValue = defaultValue;
ushort recommendedSize = (ushort)ArrayExtensions.RecommendedSize(ArrayExtensions.MinimumSize, ArrayExtensions.MinimumSize, ushort.MaxValue);
_trueItems = new ShortSet(recommendedSize);
if (defaultValue == true)
{
_trueItems.Not();
}
}
public void ShortSet_CapacityZero()
{
ShortSet s1 = new ShortSet(0);
Assert.AreEqual(0, s1.Count());
s1.Not();
Assert.AreEqual(0, s1.Count());
ShortSet s2 = new ShortSet(10);
s2.Not();
s1.And(s2);
s1.Or(s2);
Assert.AreEqual(0, s1.Count());
}
public void ShortSet_Performance_Set()
{
// Goal: Set operations are <10k instructions, so at 2M instructions per millisecond, 200 per millisecond (Release build)
// Set operations are used to combine where clauses and sets for specific words when word searching.
Random r = new Random();
ShortSet s1 = BuildRandom(ushort.MaxValue, 1000, r);
ShortSet s2 = BuildRandom(ushort.MaxValue, 10000, r);
ShortSet s3 = BuildRandom(ushort.MaxValue, 50000, r);
ushort[] s4 = { 1, 126, 950, 1024, 1025, 1670, 19240 };
ShortSet scratch = new ShortSet(ushort.MaxValue);
// 9 Operations x 10k iterations = 90k operations.
// Goal is 100ms.
int iterations = 2500;
Stopwatch w = Stopwatch.StartNew();
for (int i = 0; i < iterations; ++i)
{
// Singleton Operations / Reset
scratch.Not();
scratch.Clear();
scratch.Or(s1);
// Enumerable Operations
scratch.And(s4);
scratch.Or(s4);
scratch.AndNot(s4);
// ShortSet Operations
scratch.Or(s2);
scratch.And(s3);
scratch.AndNot(s2);
}
int operations = (9 * iterations);
double milliseconds = w.ElapsedMilliseconds;
double operationsPerMillisecond = operations / milliseconds;
Trace.Write(String.Format("{0:n0} operations in {1:n0} milliseconds; {2:n0} per millisecond.", operations, milliseconds, operationsPerMillisecond));
Assert.IsTrue(operationsPerMillisecond > 100, "Not within 200% of goal.");
}
public void TryEvaluate(Partition partition, ShortSet result, ExecutionDetails details)
{
if (details == null)
{
throw new ArgumentNullException("details");
}
if (result == null)
{
throw new ArgumentNullException("result");
}
if (partition == null)
{
throw new ArgumentNullException("partition");
}
// Include all items - clear any and then add everything.
// ShortSet will scope the set to the ID range valid within the data set.
result.Clear();
result.Not();
}
public void TryEvaluate(Partition partition, ShortSet result, ExecutionDetails details)
{
if (details == null)
{
throw new ArgumentNullException("details");
}
if (result == null)
{
throw new ArgumentNullException("result");
}
if (partition == null)
{
throw new ArgumentNullException("partition");
}
ushort itemCount = partition.Count;
ShortSet partResults = new ShortSet(itemCount);
_part[0].TryEvaluate(partition, partResults, details);
partResults.Not();
result.Or(partResults);
}
public void TryWhere(Operator op, ByteBlock value, ShortSet result, ExecutionDetails details)
{
// Convert the value to an IP Range. Error if not.
IpRange valueRange;
if (!IpRange.TryParse(value.ToString(), out valueRange))
{
details.AddError(ExecutionDetails.UnableToConvertType, value, this.Name, "IP Range");
return;
}
if (op == Operator.Matches)
{
// Matches finds rows which overlap the passed range.
// Get rows which *don't overlap* because the start address is after the range being searched for
this.StartAddressColumn.TryWhere(Operator.GreaterThan, valueRange.EndInclusive, result, details);
// Add rows which *don't overlap* because the end address is before the range being searched for
this.EndAddressColumn.TryWhere(Operator.LessThan, valueRange.StartInclusive, result, details);
// Negate to find the set which *do* overlap
result.Not();
}
else if (op == Operator.Equals || op == Operator.MatchesExact || op == Operator.NotEquals)
{
// Equals and MatchExact find rows which exactly equal the range being searched for
// Find rows with the wrong start
this.StartAddressColumn.TryWhere(Operator.NotEquals, valueRange.StartInclusive, result, details);
// Add rows the wrong end
this.EndAddressColumn.TryWhere(Operator.NotEquals, valueRange.EndInclusive, result, details);
// Negate to find the set which are equal (both start and end match)
if (op != Operator.NotEquals)
{
result.Not();
}
}
else if (op == Operator.LessThan)
{
// Find rows which end before the start of the search range
this.EndAddressColumn.TryWhere(Operator.LessThan, valueRange.StartInclusive, result, details);
}
else if (op == Operator.GreaterThan)
{
// Find rows start after the end of the search range
this.StartAddressColumn.TryWhere(Operator.GreaterThan, valueRange.EndInclusive, result, details);
}
else if (op == Operator.LessThanOrEqual)
{
// Find rows which end before the end of the search range
this.EndAddressColumn.TryWhere(Operator.LessThanOrEqual, valueRange.EndInclusive, result, details);
}
else if (op == Operator.GreaterThanOrEqual)
{
// Find rows which start after the start of the search range
this.StartAddressColumn.TryWhere(Operator.GreaterThanOrEqual, valueRange.StartInclusive, result, details);
}
else
{
details.AddError(ExecutionDetails.ColumnDoesNotSupportOperator, op, this.Name);
}
}
public override void VerifyConsistency(VerificationLevel level, ExecutionDetails details)
{
base.VerifyConsistency(level, details);
// Verify SortedIDCount agrees with ItemCount
if (this.SortedIDCount != this.Count)
{
if (details != null)
{
details.AddError(ExecutionDetails.ColumnDoesNotHaveEnoughValues, this.Name, this.SortedIDCount, this.Count);
}
}
// Verify that all IDs are in SortedIDs, all values are ordered, and no unexpected values are found
ushort lastID = 0;
IComparable lastValue = null;
ShortSet idsInList = new ShortSet(this.Count);
for (int i = 0; i < this.Count; ++i)
{
ushort id = this.SortedIDs[i];
if (id >= this.Count)
{
if (details != null)
{
details.AddError(ExecutionDetails.SortedIdOutOfRange, this.Name, id, this.Count);
}
}
else if (idsInList.Contains(id))
{
if (details != null)
{
details.AddError(ExecutionDetails.SortedIdAppearsMoreThanOnce, this.Name, id);
}
}
else
{
idsInList.Add(id);
IComparable value = (IComparable)this[id];
if (lastValue != null)
{
int compareResult = lastValue.CompareTo(value);
if (compareResult > 0)
{
if (details != null)
{
details.AddError(ExecutionDetails.SortedValuesNotInOrder, this.Name, lastID, lastValue, id, value);
}
}
}
lastValue = value;
lastID = id;
}
}
idsInList.Not();
if (idsInList.Count() > 0)
{
if (details != null)
{
details.AddError(ExecutionDetails.SortedColumnMissingIDs, this.Name, String.Join(", ", idsInList.Values));
}
}
}
public void ShortSet_Basic()
{
// Constructor
ShortSet s1 = new ShortSet(100);
// Empty
Assert.AreEqual("", String.Join(", ", s1.Values));
// Set value and enumerate
s1.Add(0);
Assert.AreEqual("0", String.Join(", ", s1.Values));
// Set additional values
s1.Add(15);
s1.Add(64);
Assert.AreEqual("0, 15, 64", String.Join(", ", s1.Values));
// Clear values
s1.Remove(64);
Assert.AreEqual("0, 15", String.Join(", ", s1.Values));
// Or
ShortSet s2 = new ShortSet(120);
s2.Or(new ushort[] { 0, 1, 2 });
s1.Or(s2);
Assert.AreEqual("0, 1, 2, 15", String.Join(", ", s1.Values));
Assert.AreEqual("0, 1, 2", String.Join(", ", s2.Values));
Verify.Exception <ArgumentNullException>(() => s1.Or((ShortSet)null));
Verify.Exception <ArgumentNullException>(() => s1.Or((IEnumerable <ushort>)null));
// OrNot [only 15, 16 not set, so only they should be added]
ShortSet s3 = new ShortSet(100);
s3.Not();
s3.Remove(15);
s3.Remove(16);
s1.OrNot(s3);
Assert.AreEqual("0, 1, 2, 15, 16", String.Join(", ", s1.Values));
Verify.Exception <ArgumentNullException>(() => s1.OrNot((ShortSet)null));
// And
s1.And(s2);
s1.And(new ushort[] { 1, 2 });
Assert.AreEqual("1, 2", String.Join(", ", s1.Values));
s1.And(new ushort[] { 1 });
Assert.AreEqual("1", String.Join(", ", s1.Values));
Verify.Exception <ArgumentNullException>(() => s1.And((ShortSet)null));
Verify.Exception <ArgumentNullException>(() => s1.And((IEnumerable <ushort>)null));
// AndNot
s1.Add(96);
s1.Add(64);
s1.AndNot(s2);
s1.AndNot(new ushort[] { 96 });
Assert.AreEqual("64", String.Join(", ", s1.Values));
Verify.Exception <ArgumentNullException>(() => s1.AndNot((ShortSet)null));
Verify.Exception <ArgumentNullException>(() => s1.AndNot((IEnumerable <ushort>)null));
// Clear
s1.Clear();
Assert.AreEqual("", String.Join(", ", s1.Values));
// From
s1.From(s2);
Assert.AreEqual("0, 1, 2", String.Join(", ", s1.Values));
Verify.Exception <ArgumentNullException>(() => s1.From((ShortSet)null));
// FromAnd
ShortSet s4 = new ShortSet(100);
s4.Or(new ushort[] { 1, 2, 3 });
s1.Clear();
s1.Not();
s1.FromAnd(s2, s4);
Assert.AreEqual("1, 2", String.Join(", ", s1.Values));
Verify.Exception <ArgumentNullException>(() => s1.FromAnd((ShortSet)null, s2));
Verify.Exception <ArgumentNullException>(() => s1.FromAnd(s2, (ShortSet)null));
// ToString
Assert.AreEqual("[1, 2]", s1.ToString());
}
