/// <summary>
/// Checks whether the column index is in range.
/// </summary>
public void CheckColumnInRange(int col)
{
Contracts.CheckParam(0 <= col && col < _cumulativeColCounts[_cumulativeColCounts.Length - 1], nameof(col), "Column index out of range");
}
public ColumnType GetColumnType(int col)
{
Contracts.CheckParam(0 <= col && col < ColumnCount, nameof(col));
return(_input.GetColumnType(Sources[col]));
}
public ColumnType GetMetadataTypeOrNull(string kind, int col)
{
Contracts.CheckNonEmpty(kind, nameof(kind));
Contracts.CheckParam(0 <= col && col < ColumnCount, nameof(col));
return(_input.GetMetadataTypeOrNull(kind, Sources[col]));
}
public ColumnType GetColumnType(int col)
{
Contracts.CheckParam(col == 0, nameof(col));
return(_colType);
}
public string GetColumnName(int col)
{
Contracts.CheckParam(0 <= col && col < ColumnCount, nameof(col));
return(Infos[col].Name);
}
public SchemaShape(IEnumerable <Column> columns)
{
Contracts.CheckValue(columns, nameof(columns));
_columns = columns.ToArray();
Contracts.CheckParam(columns.All(c => c.IsValid), nameof(columns), "Some items are not initialized properly.");
}
public string GetColumnName(int col)
{
Contracts.CheckParam(col == 0, nameof(col));
return(RoleMappedSchema.ColumnRole.Feature.Value);
}
public TransformInfo(int rank, int dim)
{
Dimension = dim;
Rank = rank;
Contracts.CheckParam(0 < Rank && Rank <= Dimension, nameof(Rank), "Rank must be positive, and at most the dimension of untransformed data");
}
/// <summary>
/// Copy from this buffer to the given destination, making sure to explicitly include the
/// first count indices in indicesInclude. Note that indicesInclude should be sorted
/// with each index less than this.Length. Note that this can make the destination be
/// dense even if "this" is sparse.
/// </summary>
public void CopyTo(ref VBuffer <T> dst, int[] indicesInclude, int count)
{
Contracts.CheckParam(count >= 0, nameof(count));
Contracts.CheckParam(Utils.Size(indicesInclude) >= count, nameof(indicesInclude));
Contracts.CheckParam(Utils.Size(indicesInclude) <= Length, nameof(indicesInclude));
// REVIEW: Ideally we should Check that indicesInclude is sorted and in range. Would that
// check be too expensive?
#if DEBUG
int prev = -1;
for (int i = 0; i < count; i++)
{
Contracts.Assert(prev < indicesInclude[i]);
prev = indicesInclude[i];
}
Contracts.Assert(prev < Length);
#endif
if (IsDense || count == 0)
{
CopyTo(ref dst);
return;
}
if (count >= Length / 2 || Count >= Length / 2)
{
CopyToDense(ref dst);
return;
}
var indices = dst.Indices;
var values = dst.Values;
if (Count == 0)
{
// No values in "this".
if (Utils.Size(indices) < count)
{
indices = new int[count];
}
Array.Copy(indicesInclude, indices, count);
if (Utils.Size(values) < count)
{
values = new T[count];
}
else
{
Array.Clear(values, 0, count);
}
dst = new VBuffer <T>(Length, count, values, indices);
return;
}
int size = 0;
int max = count + Count;
Contracts.Assert(max < Length);
int ii1;
int ii2;
if (max >= Length / 2 || Utils.Size(values) < max || Utils.Size(indices) < max)
{
// Compute the needed size.
ii1 = 0;
ii2 = 0;
for (; ;)
{
Contracts.Assert(ii1 < Count);
Contracts.Assert(ii2 < count);
size++;
int diff = Indices[ii1] - indicesInclude[ii2];
if (diff == 0)
{
ii1++;
ii2++;
if (ii1 >= Count)
{
size += count - ii2;
break;
}
if (ii2 >= count)
{
size += Count - ii1;
break;
}
}
else if (diff < 0)
{
if (++ii1 >= Count)
{
size += count - ii2;
break;
}
}
else
{
if (++ii2 >= count)
{
size += Count - ii1;
break;
}
}
}
Contracts.Assert(size >= count && size >= Count);
if (size == Count)
{
CopyTo(ref dst);
return;
}
if (size >= Length / 2)
{
CopyToDense(ref dst);
return;
}
if (Utils.Size(values) < size)
{
values = new T[size];
}
if (Utils.Size(indices) < size)
{
indices = new int[size];
}
max = size;
}
int ii = 0;
ii1 = 0;
ii2 = 0;
for (; ;)
{
Contracts.Assert(ii < max);
Contracts.Assert(ii1 < Count);
Contracts.Assert(ii2 < count);
int i1 = Indices[ii1];
int i2 = indicesInclude[ii2];
if (i1 <= i2)
{
indices[ii] = i1;
values[ii] = Values[ii1];
ii++;
if (i1 == i2)
{
ii2++;
}
if (++ii1 >= Count)
{
if (ii2 >= count)
{
break;
}
Array.Clear(values, ii, count - ii2);
Array.Copy(indicesInclude, ii2, indices, ii, count - ii2);
ii += count - ii2;
break;
}
if (ii2 >= count)
{
Array.Copy(Values, ii1, values, ii, Count - ii1);
Array.Copy(Indices, ii1, indices, ii, Count - ii1);
ii += Count - ii1;
break;
}
}
else
{
indices[ii] = i2;
values[ii] = default(T);
ii++;
if (++ii2 >= count)
{
Array.Copy(Values, ii1, values, ii, Count - ii1);
Array.Copy(Indices, ii1, indices, ii, Count - ii1);
ii += Count - ii1;
break;
}
}
}
Contracts.Assert(size == ii || size == 0);
dst = new VBuffer <T>(Length, ii, values, indices);
}
internal static VectorType GetCategoricalType(int rangeCount)
{
Contracts.CheckParam(rangeCount > 0, nameof(rangeCount), "must be known size");
return(new VectorType(NumberType.I4, rangeCount, 2));
}
public static IDataTransform CreateTransformCore(
OutputKind argsOutputKind,
OneToOneColumn[] columns,
List <OutputKind?> columnOutputKinds,
IDataTransform input,
IHost h,
IHostEnvironment env,
CategoricalHashTransform.Arguments catHashArgs = null)
{
Contracts.CheckValue(columns, nameof(columns));
Contracts.CheckValue(columnOutputKinds, nameof(columnOutputKinds));
Contracts.CheckParam(columns.Length == columnOutputKinds.Count, nameof(columns));
using (var ch = h.Start("Create Transform Core"))
{
// Create the KeyToVectorTransform, if needed.
List <KeyToVectorTransform.Column> cols = new List <KeyToVectorTransform.Column>();
bool binaryEncoding = argsOutputKind == OutputKind.Bin;
for (int i = 0; i < columns.Length; i++)
{
var column = columns[i];
if (!column.TrySanitize())
{
throw h.ExceptUserArg(nameof(Column.Name));
}
bool? bag;
OutputKind kind = columnOutputKinds[i].HasValue ? columnOutputKinds[i].Value : argsOutputKind;
switch (kind)
{
default:
throw env.ExceptUserArg(nameof(Column.OutputKind));
case OutputKind.Key:
continue;
case OutputKind.Bin:
binaryEncoding = true;
bag = false;
break;
case OutputKind.Ind:
bag = false;
break;
case OutputKind.Bag:
bag = true;
break;
}
var col = new KeyToVectorTransform.Column();
col.Name = column.Name;
col.Source = column.Name;
col.Bag = bag;
cols.Add(col);
}
if (cols.Count == 0)
{
return(input);
}
IDataTransform transform;
if (binaryEncoding)
{
if ((catHashArgs?.InvertHash ?? 0) != 0)
{
ch.Warning("Invert hashing is being used with binary encoding.");
}
var keyToBinaryArgs = new KeyToBinaryVectorTransform.Arguments();
keyToBinaryArgs.Column = cols.ToArray();
transform = new KeyToBinaryVectorTransform(h, keyToBinaryArgs, input);
}
else
{
var keyToVecArgs = new KeyToVectorTransform.Arguments
{
Bag = argsOutputKind == OutputKind.Bag,
Column = cols.ToArray()
};
transform = new KeyToVectorTransform(h, keyToVecArgs, input);
}
ch.Done();
return(transform);
}
}
internal static VectorType GetNamesType(int size)
{
Contracts.CheckParam(size > 0, nameof(size), "must be known size");
return(new VectorType(TextType.Instance, size));
}
/// <summary>
/// Computes the real and the complex roots of a real monic polynomial represented as:
/// coefficients[0] + coefficients[1] * X + coefficients[2] * X^2 + ... + coefficients[n-1] * X^(n-1) + X^n
/// by computing the eigenvalues of the Companion matrix. (https://en.wikipedia.org/wiki/Companion_matrix)
/// </summary>
/// <param name="coefficients">The monic polynomial coefficients in the ascending order</param>
/// <param name="roots">The computed (complex) roots</param>
/// <param name="roundOffDigits">The number decimal digits to keep after round-off</param>
/// <param name="doublePrecision">The machine precision</param>
/// <returns>A boolean flag indicating whether the algorithm was successful.</returns>
public static bool FindPolynomialRoots(Double[] coefficients, ref Complex[] roots,
int roundOffDigits = 6, Double doublePrecision = 2.22 * 1e-100)
{
Contracts.CheckParam(doublePrecision > 0, nameof(doublePrecision), "The double precision must be positive.");
Contracts.CheckParam(Utils.Size(coefficients) >= 1, nameof(coefficients), "There must be at least one input coefficient.");
int i;
int n = coefficients.Length;
bool result = true;
_tol = doublePrecision;
if (Utils.Size(roots) < n)
{
roots = new Complex[n];
}
// Extracting the zero roots
for (i = 0; i < n; ++i)
{
if (IsZero(coefficients[i]))
{
roots[n - i - 1] = Complex.Zero;
}
else
{
break;
}
}
if (i == n) // All zero roots
{
return(true);
}
if (i == n - 1) // Handling the linear case
{
roots[0] = new Complex(-coefficients[i], 0);
}
else if (i == n - 2) // Handling the quadratic case
{
FindQuadraticRoots(coefficients[i + 1], coefficients[i], out roots[0], out roots[1]);
}
else // Handling higher-order cases by computing the eigenvalues of the Companion matrix
{
var coeff = coefficients;
if (i > 0)
{
coeff = new Double[n - i];
Array.Copy(coefficients, i, coeff, 0, n - i);
}
// REVIEW: the eigen decomposition of the companion matrix should be done using the FactorizedCompanionMatrix class
// instead of MKL.
//FactorizedCompanionMatrix companionMatrix = new FactorizedCompanionMatrix(coeff);
//result = companionMatrix.ComputeEigenValues(ref roots);
Double[] companionMatrix = null;
var realPart = new Double[n - i];
var imaginaryPart = new Double[n - i];
var dummy = new Double[1];
CreateFullCompanionMatrix(coeff, ref companionMatrix);
var info = EigenUtils.Dhseqr(EigenUtils.Layout.ColMajor, EigenUtils.Job.EigenValues, EigenUtils.Compz.None,
n - i, 1, n - i, companionMatrix, n - i, realPart, imaginaryPart, dummy, n - i);
if (info != 0)
{
return(false);
}
for (var j = 0; j < n - i; ++j)
{
roots[j] = new Complex(realPart[j], imaginaryPart[j]);
}
}
return(result);
}
/// <summary>
/// Computes the coefficients of a real monic polynomial given its real and complex roots.
/// The final monic polynomial is represented as:
/// coefficients[0] + coefficients[1] * X + coefficients[2] * X^2 + ... + coefficients[n-1] * X^(n-1) + X^n
///
/// Note: the constant 1 coefficient of the highest degree term is implicit and not included in the output of the method.
/// </summary>
/// <param name="roots">The input (complex) roots</param>
/// <param name="coefficients">The output real coefficients</param>
/// <returns>A boolean flag indicating whether the algorithm was successful.</returns>
public static bool FindPolynomialCoefficients(Complex[] roots, ref Double[] coefficients)
{
Contracts.CheckParam(Utils.Size(roots) > 0, nameof(roots), "There must be at least 1 input root.");
int i;
int n = roots.Length;
var hash = new Dictionary <Complex, FactorMultiplicity>();
int destinationOffset = 0;
var factors = new List <PolynomialFactor>();
for (i = 0; i < n; ++i)
{
if (Double.IsNaN(roots[i].Real) || Double.IsNaN(roots[i].Imaginary))
{
return(false);
}
if (roots[i].Equals(Complex.Zero)) // Zero roots
{
destinationOffset++;
}
else if (roots[i].Imaginary == 0) // Real roots
{
var f = new PolynomialFactor(new[] { (decimal) - roots[i].Real });
factors.Add(f);
}
else // Complex roots
{
var conj = Complex.Conjugate(roots[i]);
FactorMultiplicity temp;
if (hash.TryGetValue(conj, out temp))
{
temp.Multiplicity--;
var f = new PolynomialFactor(new[]
{
(decimal)(roots[i].Real * roots[i].Real + roots[i].Imaginary * roots[i].Imaginary),
(decimal)(-2 * roots[i].Real)
});
factors.Add(f);
if (temp.Multiplicity <= 0)
{
hash.Remove(conj);
}
}
else
{
if (hash.TryGetValue(roots[i], out temp))
{
temp.Multiplicity++;
}
else
{
hash.Add(roots[i], new FactorMultiplicity());
}
}
}
}
if (hash.Count > 0)
{
return(false);
}
var comparer = new ByMaximumCoefficient();
factors.Sort(comparer);
if (destinationOffset < n - 1)
{
if (Utils.Size(PolynomialFactor.Destination) < n)
{
PolynomialFactor.Destination = new decimal[n];
}
while (factors.Count > 1)
{
var k1 = Math.Abs(factors.ElementAt(0).Key);
var k2 = Math.Abs(factors.ElementAt(factors.Count - 1).Key);
PolynomialFactor f1;
if (k1 < k2)
{
f1 = factors.ElementAt(0);
factors.RemoveAt(0);
}
else
{
f1 = factors.ElementAt(factors.Count - 1);
factors.RemoveAt(factors.Count - 1);
}
var ind = factors.BinarySearch(new PolynomialFactor(-f1.Key), comparer);
if (ind < 0)
{
ind = ~ind;
}
ind = Math.Min(factors.Count - 1, ind);
var f2 = factors.ElementAt(ind);
factors.RemoveAt(ind);
f1.Multiply(f2);
ind = factors.BinarySearch(f1, comparer);
if (ind >= 0)
{
factors.Insert(ind, f1);
}
else
{
factors.Insert(~ind, f1);
}
}
}
if (Utils.Size(coefficients) < n)
{
coefficients = new Double[n];
}
for (i = 0; i < destinationOffset; ++i)
{
coefficients[i] = 0;
}
if (destinationOffset < n)
{
var coeff = factors.ElementAt(0).Coefficients;
for (i = destinationOffset; i < n; ++i)
{
coefficients[i] = Decimal.ToDouble(coeff[i - destinationOffset]);
}
}
return(true);
}
public NormStr GetNormStrById(int id)
{
Contracts.CheckParam(0 <= id && id < _cns, nameof(id));
return(GetNs(id));
}
public static JToken Union(params JToken[] types)
{
Contracts.CheckParam(Utils.Size(types) >= 2, nameof(types), "Union must have at least two types");
return(new JArray(types));
}
internal DictCountTableBuilder(float garbageThreshold)
{
Contracts.CheckParam(garbageThreshold >= 0, nameof(garbageThreshold), "Garbage threshold must be non-negative");
_garbageThreshold = garbageThreshold;
}
protected MetadataDispatcherBase(int colCount)
{
Contracts.CheckParam(colCount >= 0, nameof(colCount));
_infos = new ColInfo[colCount];
}
private static Stream OpenStream(IMultiStreamSource files)
{
Contracts.CheckValue(files, nameof(files));
Contracts.CheckParam(files.Count == 1, nameof(files), "Parquet loader must be created with one file");
return(files.Open(0));
}
/// <summary>
/// Returns the ColInfo registered for the given column index, if there is one. This may be called
/// before or after Seal is called.
/// </summary>
protected ColInfo GetColInfoOrNull(int index)
{
Contracts.CheckParam(0 <= index && index < _infos.Length, nameof(index));
return(_infos[index]);
}
public override string GetNameOrNull(int index)
{
Contracts.CheckParam(0 <= index && index < _count, nameof(index));
return(index < _names.Length ? _names[index] : null);
}
private FuncInstanceMethodInfo1(MethodInfo methodInfo)
: base(methodInfo)
{
Contracts.CheckParam(!GenericMethodDefinition.IsStatic, nameof(methodInfo), "Should be an instance method");
Contracts.CheckParam(GenericMethodDefinition.DeclaringType == typeof(TTarget), nameof(methodInfo), _targetTypeCheckMessage);
}
public IEnumerable <KeyValuePair <string, ColumnType> > GetMetadataTypes(int col)
{
Contracts.CheckParam(col == 0, nameof(col));
yield return(new KeyValuePair <string, ColumnType>(MetadataUtils.Kinds.SlotNames, _slotNamesType));
}
public SchemaShape(IEnumerable <Column> columns)
{
Contracts.CheckValue(columns, nameof(columns));
Columns = columns.ToArray();
Contracts.CheckParam(columns.All(c => c != null), nameof(columns), "No items should be null.");
}
public ColumnType GetColumnType(int col)
{
Contracts.CheckParam(0 <= col && col < ColumnCount, nameof(col));
return(Infos[col].TypeSrc);
}
private Float FindMinimum(DiffFunc1D func, Float initValue, Float initDeriv)
{
Contracts.CheckParam(initDeriv < 0, nameof(initDeriv), "Cannot search in direction of ascent!");
StepValueDeriv lo = new StepValueDeriv(func, 0, initValue, initDeriv);
StepValueDeriv hi = new StepValueDeriv(func, _step);
// bracket minimum
while (hi.Deriv < 0)
{
Swap(ref lo, ref hi);
if (lo.Step >= MaxStep)
{
return(MaxStep);
}
hi.Step = lo.Step * 2;
}
Float window = 1;
StepValueDeriv mid = new StepValueDeriv(func);
for (int numSteps = 1; ; ++numSteps)
{
Float interp = CubicInterp(lo, hi);
if (window <= MinWindow || numSteps == MaxNumSteps)
{
return(interp);
}
// insure minimal progress to narrow interval
Float minProgressStep = _minProgress * (hi.Step - lo.Step);
Float maxMid = hi.Step - minProgressStep;
if (interp > maxMid)
{
interp = maxMid;
}
Float minMid = lo.Step + minProgressStep;
if (interp < minMid)
{
interp = minMid;
}
mid.Step = interp;
if (mid.Deriv == 0 || mid.Step == lo.Step || mid.Step == hi.Step)
{
return(mid.Step);
}
if (mid.Deriv < 0)
{
Swap(ref lo, ref mid);
}
else
{
Swap(ref hi, ref mid);
}
if (lo.Step >= MaxStep)
{
return(MaxStep);
}
window = (hi.Step - lo.Step) / hi.Step;
}
}
public IEnumerable <KeyValuePair <string, ColumnType> > GetMetadataTypes(int col)
{
Contracts.CheckParam(0 <= col && col < ColumnCount, nameof(col));
return(_input.GetMetadataTypes(Sources[col]));
}
private Scalar <T> Load <T>(DataKind kind, int ordinal)
{
Contracts.CheckParam(ordinal >= 0, nameof(ordinal), "Should be non-negative");
return(new MyScalar <T>(_rec, kind, ordinal));
}
public void GetMetadata <TValue>(string kind, int col, ref TValue value)
{
Contracts.CheckNonEmpty(kind, nameof(kind));
Contracts.CheckParam(0 <= col && col < ColumnCount, nameof(col));
_input.GetMetadata(kind, Sources[col], ref value);
}
/// <summary>
/// Creates a minimum waiter.
/// </summary>
/// <param name="waiters">The initial number of waiters</param>
public MinWaiter(int waiters)
{
Contracts.CheckParam(waiters > 0, nameof(waiters), "Must have at least one waiter");
_maxWaiters = waiters;
_waiters = new Heap <WaitStats>((s1, s2) => s1.Line > s2.Line);
}
