private static void CalculateLshForListingSet(List <Listing> listings, string job_id, Dictionary <long, long> duplicates)
{
var numSimilarityBuckets = (int)Math.Ceiling(listings.Count / 100M);
// First make 2 dimensional array (docs by min-hashes)
var matrix = new int[listings.Count, minHashCount];
for (int listing = 0; listing < listings.Count; listing++)
{
for (int hash = 0; hash < listings[listing].minhash_description.Count; hash++)
{
matrix[listing, hash] = (int)listings[listing].minhash_description[hash];
}
}
// Now set LSH
var lsh = new LSH(matrix, numSimilarityBuckets);
lsh.Calc();
// Set closes duplicate on each listing
var duplicatesFound = new Dictionary <long, long>();
var singleItemProgress = ProgressManager.CalculateLoopIncrement(listings.Count(), 0.2M);
for (int listing = 0; listing < listings.Count; listing++)
{
ProgressManager.IncrementJobPercentBy(job_id, singleItemProgress);
var nearest = lsh.GetNearest(listing);
if (!nearest.Any())
{
continue;
}
var thisListing = listings[listing];
var nearestListing = listings[nearest[0]];
var priceRatio = nearestListing.buy_now_price / thisListing.buy_now_price;
if (priceRatio < 0.8M || priceRatio > 1.2M)
{
continue;
}
if (duplicatesFound.ContainsKey(nearestListing.id))
{
continue;
}
listings[listing].likely_duplicate_id_by_description = nearestListing.id;
listings[listing].similarity_description = Jaccard.Calc(ArrayHelpers.GetRow <int>(matrix, listing).ToList(), nearest);
duplicates[nearestListing.id] = thisListing.id;
duplicates[thisListing.id] = nearestListing.id;
}
}
public Point3D Evaluate(double u, double v)
{
var subpoints = new List <Point3D>();
for (var i = 0; i < 4; ++i)
{
var subpoint = BernsteinPolynomial.Evaluate3DPolynomial(
ArrayHelpers.GetRow(ControlPoints, i), u
);
subpoints.Add(subpoint);
}
return(BernsteinPolynomial.Evaluate3DPolynomial(subpoints, v));
}
public Vector3D Derivative(
double u, double v,
DerivativeParameter parameter
)
{
double firstParameter, secondParameter;
Func <int, Point3D[]> generator;
switch (parameter)
{
case DerivativeParameter.U:
generator = (i) => ArrayHelpers.GetColumn(ControlPoints, i);
firstParameter = v;
secondParameter = u;
break;
case DerivativeParameter.V:
generator = (i) => ArrayHelpers.GetRow(ControlPoints, i);
firstParameter = u;
secondParameter = v;
break;
default:
throw new ArgumentOutOfRangeException(
nameof(parameter),
parameter,
null
);
}
var subpoints = new List <Point3D>();
for (var i = 0; i < 4; ++i)
{
subpoints.Add(BernsteinPolynomial.Evaluate3DPolynomial(
generator(i),
firstParameter
));
}
var derivative = BernsteinPolynomial.CalculateDerivative(subpoints);
return((Vector3D)BernsteinPolynomial.Evaluate3DPolynomial(
derivative,
secondParameter
));
}
