public void TestSweetSpotTf()
{
SweetSpotSimilarity ss = new SweetSpotSimilarity();
TFIDFSimilarity d = new DefaultSimilarity();
TFIDFSimilarity s = ss;
// tf equal
ss.SetBaselineTfFactors(0.0f, 0.0f);
for (int i = 1; i < 1000; i++)
{
assertEquals("tf: i=" + i,
d.Tf(i), s.Tf(i), 0.0f);
}
// tf higher
ss.SetBaselineTfFactors(1.0f, 0.0f);
for (int i = 1; i < 1000; i++)
{
assertTrue("tf: i=" + i + " : d=" + d.Tf(i) +
" < s=" + s.Tf(i),
d.Tf(i) < s.Tf(i));
}
// tf flat
ss.SetBaselineTfFactors(1.0f, 6.0f);
for (int i = 1; i <= 6; i++)
{
assertEquals("tf flat1: i=" + i, 1.0f, s.Tf(i), 0.0f);
}
ss.SetBaselineTfFactors(2.0f, 6.0f);
for (int i = 1; i <= 6; i++)
{
assertEquals("tf flat2: i=" + i, 2.0f, s.Tf(i), 0.0f);
}
for (int i = 6; i <= 1000; i++)
{
assertTrue("tf: i=" + i + " : s=" + s.Tf(i) +
" < d=" + d.Tf(i),
s.Tf(i) < d.Tf(i));
}
// stupidity
assertEquals("tf zero", 0.0f, s.Tf(0), 0.0f);
}
public void TestSweetSpotComputeNorm()
{
SweetSpotSimilarity ss = new SweetSpotSimilarity();
ss.SetLengthNormFactors(1, 1, 0.5f, true);
Similarity d = new DefaultSimilarity();
Similarity s = ss;
// base case, should degrade
FieldInvertState invertState = new FieldInvertState("bogus");
invertState.Boost = 1.0f;
for (int i = 1; i < 1000; i++)
{
invertState.Length = i;
assertEquals("base case: i=" + i,
ComputeAndGetNorm(d, invertState),
ComputeAndGetNorm(s, invertState),
0.0f);
}
// make a sweet spot
ss.SetLengthNormFactors(3, 10, 0.5f, true);
for (int i = 3; i <= 10; i++)
{
invertState.Length = i;
assertEquals("3,10: spot i=" + i,
1.0f,
ComputeAndDecodeNorm(ss, ss, invertState),
0.0f);
}
for (int i = 10; i < 1000; i++)
{
invertState.Length = (i - 9);
byte normD = ComputeAndGetNorm(d, invertState);
invertState.Length = i;
byte normS = ComputeAndGetNorm(s, invertState);
assertEquals("3,10: 10<x : i=" + i,
normD,
normS,
0.0f);
}
// separate sweet spot for certain fields
SweetSpotSimilarity ssBar = new SweetSpotSimilarity();
ssBar.SetLengthNormFactors(8, 13, 0.5f, false);
SweetSpotSimilarity ssYak = new SweetSpotSimilarity();
ssYak.SetLengthNormFactors(6, 9, 0.5f, false);
SweetSpotSimilarity ssA = new SweetSpotSimilarity();
ssA.SetLengthNormFactors(5, 8, 0.5f, false);
SweetSpotSimilarity ssB = new SweetSpotSimilarity();
ssB.SetLengthNormFactors(5, 8, 0.1f, false);
Similarity sp = new PerFieldSimilarityWrapperHelper(ssBar, ssYak, ssA, ssB, ss);
invertState = new FieldInvertState("foo");
invertState.Boost = 1.0f;
for (int i = 3; i <= 10; i++)
{
invertState.Length = i;
assertEquals("f: 3,10: spot i=" + i,
1.0f,
ComputeAndDecodeNorm(ss, sp, invertState),
0.0f);
}
for (int i = 10; i < 1000; i++)
{
invertState.Length = (i - 9);
byte normD = ComputeAndGetNorm(d, invertState);
invertState.Length = (i);
byte normS = ComputeAndGetNorm(sp, invertState);
assertEquals("f: 3,10: 10<x : i=" + i,
normD,
normS,
0.0f);
}
invertState = new FieldInvertState("bar");
invertState.Boost = (1.0f);
for (int i = 8; i <= 13; i++)
{
invertState.Length = (i);
assertEquals("f: 8,13: spot i=" + i,
1.0f,
ComputeAndDecodeNorm(ss, sp, invertState),
0.0f);
}
invertState = new FieldInvertState("yak");
invertState.Boost = (1.0f);
for (int i = 6; i <= 9; i++)
{
invertState.Length = (i);
assertEquals("f: 6,9: spot i=" + i,
1.0f,
ComputeAndDecodeNorm(ss, sp, invertState),
0.0f);
}
invertState = new FieldInvertState("bar");
invertState.Boost = (1.0f);
for (int i = 13; i < 1000; i++)
{
invertState.Length = (i - 12);
byte normD = ComputeAndGetNorm(d, invertState);
invertState.Length = (i);
byte normS = ComputeAndGetNorm(sp, invertState);
assertEquals("f: 8,13: 13<x : i=" + i,
normD,
normS,
0.0f);
}
invertState = new FieldInvertState("yak");
invertState.Boost = (1.0f);
for (int i = 9; i < 1000; i++)
{
invertState.Length = (i - 8);
byte normD = ComputeAndGetNorm(d, invertState);
invertState.Length = (i);
byte normS = ComputeAndGetNorm(sp, invertState);
assertEquals("f: 6,9: 9<x : i=" + i,
normD,
normS,
0.0f);
}
// steepness
for (int i = 9; i < 1000; i++)
{
invertState = new FieldInvertState("a");
invertState.Boost = (1.0f);
invertState.Length = (i);
byte normSS = ComputeAndGetNorm(sp, invertState);
invertState = new FieldInvertState("b");
invertState.Boost = (1.0f);
invertState.Length = (i);
byte normS = ComputeAndGetNorm(sp, invertState);
assertTrue("s: i=" + i + " : a=" + normSS +
" < b=" + normS,
normSS < normS);
}
}
public static float ComputeAndDecodeNorm(SweetSpotSimilarity decode, Similarity encode, FieldInvertState state)
{
return(decode.DecodeNormValue(ComputeAndGetNorm(encode, state)));
}
public void TestSweetSpotComputeNorm()
{
SweetSpotSimilarity ss = new SweetSpotSimilarity();
ss.SetLengthNormFactors(1, 1, 0.5f, true);
Similarity d = new DefaultSimilarity();
Similarity s = ss;
// base case, should degrade
FieldInvertState invertState = new FieldInvertState("bogus");
invertState.Boost = 1.0f;
for (int i = 1; i < 1000; i++)
{
invertState.Length = i;
assertEquals("base case: i=" + i,
ComputeAndGetNorm(d, invertState),
ComputeAndGetNorm(s, invertState),
0.0f);
}
// make a sweet spot
ss.SetLengthNormFactors(3, 10, 0.5f, true);
for (int i = 3; i <= 10; i++)
{
invertState.Length = i;
assertEquals("3,10: spot i=" + i,
1.0f,
ComputeAndDecodeNorm(ss, ss, invertState),
0.0f);
}
for (int i = 10; i < 1000; i++)
{
invertState.Length = (i - 9);
byte normD = ComputeAndGetNorm(d, invertState);
invertState.Length = i;
byte normS = ComputeAndGetNorm(s, invertState);
assertEquals("3,10: 10<x : i=" + i,
normD,
normS,
0.0f);
}
// separate sweet spot for certain fields
SweetSpotSimilarity ssBar = new SweetSpotSimilarity();
ssBar.SetLengthNormFactors(8, 13, 0.5f, false);
SweetSpotSimilarity ssYak = new SweetSpotSimilarity();
ssYak.SetLengthNormFactors(6, 9, 0.5f, false);
SweetSpotSimilarity ssA = new SweetSpotSimilarity();
ssA.SetLengthNormFactors(5, 8, 0.5f, false);
SweetSpotSimilarity ssB = new SweetSpotSimilarity();
ssB.SetLengthNormFactors(5, 8, 0.1f, false);
Similarity sp = new PerFieldSimilarityWrapperHelper(ssBar, ssYak, ssA, ssB, ss);
invertState = new FieldInvertState("foo");
invertState.Boost = 1.0f;
for (int i = 3; i <= 10; i++)
{
invertState.Length = i;
assertEquals("f: 3,10: spot i=" + i,
1.0f,
ComputeAndDecodeNorm(ss, sp, invertState),
0.0f);
}
for (int i = 10; i < 1000; i++)
{
invertState.Length = (i - 9);
byte normD = ComputeAndGetNorm(d, invertState);
invertState.Length = (i);
byte normS = ComputeAndGetNorm(sp, invertState);
assertEquals("f: 3,10: 10<x : i=" + i,
normD,
normS,
0.0f);
}
invertState = new FieldInvertState("bar");
invertState.Boost = (1.0f);
for (int i = 8; i <= 13; i++)
{
invertState.Length = (i);
assertEquals("f: 8,13: spot i=" + i,
1.0f,
ComputeAndDecodeNorm(ss, sp, invertState),
0.0f);
}
invertState = new FieldInvertState("yak");
invertState.Boost = (1.0f);
for (int i = 6; i <= 9; i++)
{
invertState.Length = (i);
assertEquals("f: 6,9: spot i=" + i,
1.0f,
ComputeAndDecodeNorm(ss, sp, invertState),
0.0f);
}
invertState = new FieldInvertState("bar");
invertState.Boost = (1.0f);
for (int i = 13; i < 1000; i++)
{
invertState.Length = (i - 12);
byte normD = ComputeAndGetNorm(d, invertState);
invertState.Length = (i);
byte normS = ComputeAndGetNorm(sp, invertState);
assertEquals("f: 8,13: 13<x : i=" + i,
normD,
normS,
0.0f);
}
invertState = new FieldInvertState("yak");
invertState.Boost = (1.0f);
for (int i = 9; i < 1000; i++)
{
invertState.Length = (i - 8);
byte normD = ComputeAndGetNorm(d, invertState);
invertState.Length = (i);
byte normS = ComputeAndGetNorm(sp, invertState);
assertEquals("f: 6,9: 9<x : i=" + i,
normD,
normS,
0.0f);
}
// steepness
for (int i = 9; i < 1000; i++)
{
invertState = new FieldInvertState("a");
invertState.Boost = (1.0f);
invertState.Length = (i);
byte normSS = ComputeAndGetNorm(sp, invertState);
invertState = new FieldInvertState("b");
invertState.Boost = (1.0f);
invertState.Length = (i);
byte normS = ComputeAndGetNorm(sp, invertState);
assertTrue("s: i=" + i + " : a=" + normSS +
" < b=" + normS,
normSS < normS);
}
}
public void TestSweetSpotTf()
{
SweetSpotSimilarity ss = new SweetSpotSimilarity();
TFIDFSimilarity d = new DefaultSimilarity();
TFIDFSimilarity s = ss;
// tf equal
ss.SetBaselineTfFactors(0.0f, 0.0f);
for (int i = 1; i < 1000; i++)
{
assertEquals("tf: i=" + i,
d.Tf(i), s.Tf(i), 0.0f);
}
// tf higher
ss.SetBaselineTfFactors(1.0f, 0.0f);
for (int i = 1; i < 1000; i++)
{
assertTrue("tf: i=" + i + " : d=" + d.Tf(i) +
" < s=" + s.Tf(i),
d.Tf(i) < s.Tf(i));
}
// tf flat
ss.SetBaselineTfFactors(1.0f, 6.0f);
for (int i = 1; i <= 6; i++)
{
assertEquals("tf flat1: i=" + i, 1.0f, s.Tf(i), 0.0f);
}
ss.SetBaselineTfFactors(2.0f, 6.0f);
for (int i = 1; i <= 6; i++)
{
assertEquals("tf flat2: i=" + i, 2.0f, s.Tf(i), 0.0f);
}
for (int i = 6; i <= 1000; i++)
{
assertTrue("tf: i=" + i + " : s=" + s.Tf(i) +
" < d=" + d.Tf(i),
s.Tf(i) < d.Tf(i));
}
// stupidity
assertEquals("tf zero", 0.0f, s.Tf(0), 0.0f);
}
public static float ComputeAndDecodeNorm(SweetSpotSimilarity decode, Similarity encode, FieldInvertState state)
{
return decode.DecodeNormValue(ComputeAndGetNorm(encode, state));
}
