private static string GetTerms(IDataView data, string colName)
{
Contracts.AssertValue(data);
Contracts.AssertNonWhiteSpace(colName);
var schema = data.Schema;
var col = schema.GetColumnOrNull(colName);
if (!col.HasValue)
return null;
var type = col.Value.Metadata.Schema.GetColumnOrNull(MetadataUtils.Kinds.KeyValues)?.Type as VectorType;
if (type == null || !type.IsKnownSize || !(type.ItemType is TextType))
return null;
var metadata = default(VBuffer<ReadOnlyMemory<char>>);
col.Value.GetKeyValues(ref metadata);
if (!metadata.IsDense)
return null;
var sb = new StringBuilder();
var pre = "";
var metadataValues = metadata.GetValues();
for (int i = 0; i < metadataValues.Length; i++)
{
sb.Append(pre);
sb.AppendMemory(metadataValues[i]);
pre = ",";
}
return sb.ToString();
}
private static string GetTerms(IDataView data, string colName)
{
Contracts.AssertValue(data);
Contracts.AssertNonWhiteSpace(colName);
int col;
var schema = data.Schema;
if (!schema.TryGetColumnIndex(colName, out col))
{
return(null);
}
var type = schema.GetMetadataTypeOrNull(MetadataUtils.Kinds.KeyValues, col);
if (type == null || !type.IsKnownSizeVector || !type.ItemType.IsText)
{
return(null);
}
var metadata = default(VBuffer <DvText>);
schema.GetMetadata(MetadataUtils.Kinds.KeyValues, col, ref metadata);
if (!metadata.IsDense)
{
return(null);
}
var sb = new StringBuilder();
var pre = "";
for (int i = 0; i < metadata.Length; i++)
{
sb.Append(pre);
metadata.Values[i].AddToStringBuilder(sb);
pre = ",";
}
return(sb.ToString());
}
protected AggregatorDictionaryBase(RoleMappedSchema schema, string stratCol, Func <string, TAgg> createAgg)
{
Contracts.AssertValue(schema);
Contracts.AssertNonWhiteSpace(stratCol);
Contracts.AssertValue(createAgg);
Schema = schema;
CreateAgg = createAgg;
ColName = stratCol;
}
public static AggregatorDictionaryBase Create(RoleMappedSchema schema, string stratCol, ColumnType stratType,
Func <string, TAgg> createAgg)
{
Contracts.AssertNonWhiteSpace(stratCol);
Contracts.AssertValue(createAgg);
if (stratType.KeyCount == 0 && !stratType.IsText)
{
throw Contracts.ExceptUserArg(nameof(MamlEvaluatorBase.ArgumentsBase.StratColumn),
"Stratification column '{0}' has type '{1}', but must be a known count key or text", stratCol, stratType);
}
return(Utils.MarshalInvoke(CreateDictionary <int>, stratType.RawType, schema, stratCol, stratType, createAgg));
}
public static string LoadFromFileOrDownloadFromWeb(string path, string fileName, Uri url, IChannel ch)
{
Contracts.AssertNonWhiteSpace(fileName, "Filename can't be empty");
var contents = "";
var filePath = Path.Combine(path, fileName);
if (!File.Exists(filePath))
{
try
{
using var webClient = new WebClient();
contents = webClient.DownloadString(url);
}
catch (WebException e)
{
throw new WebException($"File {fileName} not found and cannot be downloaded from {url}.\n" +
$"Error message: {e.Message}", e);
}
try
{
File.WriteAllText(filePath, contents);
ch.Info($"File {fileName} successfully downloaded from {url} and saved to {path}.");
}
catch (Exception e)
{
ch.Warning($"{DateTime.Now} - WARNING: File {fileName} successfully downloaded from {url}, " +
$"but error occurs when saving file {fileName} into {path}.\n" +
$"Error message: {e.Message}");
}
}
else
{
try
{
contents = File.ReadAllText(filePath);
}
catch (Exception e)
{
throw new IOException($"Problems met when reading {filePath}.\n" +
$"Error message: {e.Message}", e);
}
}
return(contents);
}
/// <summary>
/// Attempts to find a file that is expected to be distributed with a TLC component. Searches
/// in the following order:
/// 1. In the customSearchDir directory, if it is provided.
/// 2. In the custom search directory specified by the
/// <seealso cref="CustomSearchDirEnvVariable"/> environment variable.
/// 3. In the root folder of the provided assembly.
/// 4. In the folder of this assembly.
/// In each case it searches the file in the directory provided and combined with folderPrefix.
///
/// If any of these locations contain the file, a full local path will be returned, otherwise this
/// method will return null.
/// </summary>
/// <param name="fileName">File name to find</param>
/// <param name="folderPrefix">folder prefix, relative to the current or customSearchDir</param>
/// <param name="customSearchDir">
/// Custom directory to search for resources.
/// If null, the path specified in the environment variable <seealso cref="CustomSearchDirEnvVariable"/>
/// will be used.
/// </param>
/// <param name="assemblyForBasePath">
/// Assembly type to search the path of.
/// </param>
/// <returns>
/// Path to the existing file. Null if not found.
/// </returns>
public static string FindExistentFileOrNull(string fileName, string folderPrefix = null, string customSearchDir = null, System.Type assemblyForBasePath = null)
{
Contracts.AssertNonWhiteSpace(fileName);
string candidate;
// 1. Search in customSearchDir.
if (!string.IsNullOrWhiteSpace(customSearchDir) &&
TryFindFile(fileName, folderPrefix, customSearchDir, out candidate))
{
return(candidate);
}
// 2. Search in the path specified by the environment variable.
var envDir = Environment.GetEnvironmentVariable(CustomSearchDirEnvVariable);
if (!string.IsNullOrWhiteSpace(envDir) &&
TryFindFile(fileName, folderPrefix, envDir, out candidate))
{
return(candidate);
}
// 3. Search in the path specified by the assemblyForBasePath.
if (assemblyForBasePath != null)
{
// For CoreTLC we have MLCore located in main folder, and dlls with learners and transforms located
// in AutoLoad folder or (ClrWin|ClrLinux) folder so we need to check folder for provided type.
var assemblyDir = Path.GetDirectoryName(assemblyForBasePath.Assembly.Location);
if (assemblyDir != null && customSearchDir != null)
{
assemblyDir = Path.Combine(assemblyDir, customSearchDir);
}
if (TryFindFile(fileName, folderPrefix, assemblyDir, out candidate))
{
return(candidate);
}
}
// 4. Fallback to the root path of the current assembly
TryFindFile(fileName, folderPrefix, DllDir, out candidate);
return(candidate);
}
private static string GetTerms(IDataView data, string colName)
{
Contracts.AssertValue(data);
Contracts.AssertNonWhiteSpace(colName);
int col;
var schema = data.Schema;
if (!schema.TryGetColumnIndex(colName, out col))
{
return(null);
}
var type = schema[col].Metadata.Schema.GetColumnOrNull(MetadataUtils.Kinds.KeyValues)?.Type;
if (type == null || !type.IsKnownSizeVector || !type.ItemType.IsText)
{
return(null);
}
var metadata = default(VBuffer <ReadOnlyMemory <char> >);
schema[col].Metadata.GetValue(MetadataUtils.Kinds.KeyValues, ref metadata);
if (!metadata.IsDense)
{
return(null);
}
var sb = new StringBuilder();
var pre = "";
var metadataValues = metadata.GetValues();
for (int i = 0; i < metadataValues.Length; i++)
{
sb.Append(pre);
sb.AppendMemory(metadataValues[i]);
pre = ",";
}
return(sb.ToString());
}
public DictionaryKeyParameterBinding(string name, string key)
: base(name)
{
Contracts.AssertNonWhiteSpace(key);
Key = key;
}
protected ParameterBinding(string name)
{
Contracts.AssertNonWhiteSpace(name);
ParameterName = name;
}
protected VariableBinding(string varName)
{
Contracts.AssertNonWhiteSpace(varName);
VariableName = varName;
}
