public void Initialize(IDataView dataView)
{
if (dataView == null)
throw new ArgumentNullException();
if (_dataCollection != dataView)
{
if (_dataCollection != null)
{
this.ProcessingCompleted -= _dataCollection.ProcessingCompleted;
}
_dataCollection = dataView;
this.ProcessingCompleted += _dataCollection.ProcessingCompleted;
// Initialize all moving average calculators
foreach (int id in _movingAverageCalculators.Keys)
{
IMovingAverageCalculator ma = _movingAverageCalculators[id].Value;
ma.Reset();
}
// Initialize the trend calculator
InitializeTrendCalculator();
// Initialize the gap calculator
InitializeGapCalculator();
// Initialize the trend calculator
InitializeTrendCalculator();
}
}
protected override void Render(HtmlTextWriter output, IDataView dataView, string filter, Item root, Item folder)
{
Assert.ArgumentNotNull(output, "output");
Assert.ArgumentNotNull(dataView, "dataView");
Assert.ArgumentNotNull(filter, "filter");
Assert.ArgumentNotNull(root, "root");
Assert.ArgumentNotNull(folder, "folder");
output.Write("<div id=\"");
output.Write(this.ID);
output.Write("\" onclick=\"javascript:return Sitecore.Treeview.onTreeClick(this,event");
if (!string.IsNullOrEmpty(this.Click))
{
output.Write(",'");
output.Write(StringUtil.EscapeQuote(this.Click));
output.Write("'");
}
output.Write(")\"");
if (!string.IsNullOrEmpty(this.DblClick))
{
output.Write(" ondblclick=\"");
output.Write(AjaxScriptManager.GetEventReference(this.DblClick));
output.Write("\"");
}
if (!string.IsNullOrEmpty(this.ContextMenu))
{
output.Write(" oncontextmenu=\"");
output.Write(AjaxScriptManager.GetEventReference(this.ContextMenu));
output.Write("\"");
}
if (this.AllowDragging)
{
output.Write(" onmousedown=\"javascript:return Sitecore.Treeview.onTreeDrag(this,event)\" onmousemove=\"javascript:return Sitecore.Treeview.onTreeDrag(this,event)\" ondragstart=\"javascript:return Sitecore.Treeview.onTreeDrag(this,event)\" ondragover=\"javascript:return Sitecore.Treeview.onTreeDrop(this,event)\" ondrop=\"javascript:return Sitecore.Treeview.onTreeDrop(this,event)\"");
}
output.Write(">");
output.Write("<input id=\"");
output.Write(this.ID);
output.Write("_Selected\" type=\"hidden\" value=\"" + folder.ID.ToShortID() + "\"/>");
output.Write("<input id=\"");
output.Write(this.ID);
output.Write("_Database\" type=\"hidden\" value=\"" + folder.Database.Name + "\"/>");
output.Write("<input id=\"");
output.Write(this.ID);
output.Write("_Parameters\" type=\"hidden\" value=\"" + this.GetParameters() + "\"/>");
if (this.ShowRoot)
{
this.RenderNode(output, dataView, filter, root, root, folder);
}
else
{
foreach (Item item in dataView.GetChildren(root, string.Empty, true, 0, 0, this.GetFilter()))
{
this.RenderNode(output, dataView, filter, root, item, folder);
}
}
output.Write("</div>");
}
public static IDataView LoadPipeWithPredictor(IHostEnvironment env, Stream modelStream, IDataView view)
{
// Load transforms.
var pipe = env.LoadTransforms(modelStream, view);
// Load predictor (if present) and apply default scorer.
// REVIEW: distinguish the case of predictor / no predictor?
var predictor = env.LoadPredictorOrNull(modelStream);
if (predictor != null)
{
var roles = ModelFileUtils.LoadRoleMappingsOrNull(env, modelStream);
pipe = roles != null
? env.CreateDefaultScorer(new RoleMappedData(pipe, roles, opt : true), predictor)
: env.CreateDefaultScorer(new RoleMappedData(pipe, label : null, "Features"), predictor);
}
return(pipe);
}
private void RenderNodeBegin(HtmlTextWriter output, IDataView dataView, string filter, Item item, bool active, bool isExpanded)
{
Assert.ArgumentNotNull(output, "output");
Assert.ArgumentNotNull(dataView, "dataView");
Assert.ArgumentNotNull(filter, "filter");
Assert.ArgumentNotNull(item, "item");
string shortID = item.ID.ToShortID().ToString();
string nodeID = this.GetNodeID(shortID);
output.Write("<div id=\"");
output.Write(nodeID);
output.Write("\" class=\"scContentTreeNode\">");
RenderTreeNodeGlyph(output, dataView, filter, item, shortID, isExpanded);
string str3 = (active || this.SelectedIDs.Contains(shortID)) ? "scContentTreeNodeActive" : "scContentTreeNodeNormal";
string style = this.GetStyle(item);
output.Write("<a href=\"#\" class=\"" + str3 + "\"");
if (!string.IsNullOrEmpty(item.Help.Text))
{
output.Write(" title=\"");
output.Write(StringUtil.EscapeQuote(item.Help.Text));
output.Write("\"");
}
output.Write(style);
output.Write(">");
RenderTreeNodeIcon(output, item);
output.Write("<span>{0}</span>", this.GetTreeNodeName(item));
output.Write("</a>");
}
public ArrangeManager(Form owner, IDataView dv)
{
this.OwnerForm = owner;
this.Dataview = dv;
}
protected virtual void Render(HtmlTextWriter output, IDataView dataView, string filter, Item root, Item folder)
{
Assert.ArgumentNotNull(output, "output");
Assert.ArgumentNotNull(dataView, "dataView");
Assert.ArgumentNotNull(filter, "filter");
Assert.ArgumentNotNull(root, "root");
Assert.ArgumentNotNull(folder, "folder");
output.Write(string.Format("<div id=\"{0}\"", this.ID));
//double click custom event
output.Write(string.Format(" onclick=\"javascript:return Sitecore.Treeview.onTreeClick(this,event);\" ondblclick=\"FieldSuite.Fields.Treelist.AddItem('{0}',event);", this.ParentId));
output.Write("\"");
if (!string.IsNullOrEmpty(this.ContextMenu))
{
output.Write(" oncontextmenu=\"");
output.Write(AjaxScriptManager.GetEventReference(this.ContextMenu));
output.Write("\"");
}
if (this.AllowDragging)
{
output.Write(
" onmousedown=\"javascript:return Sitecore.Treeview.onTreeDrag(this,event)\" onmousemove=\"javascript:return Sitecore.Treeview.onTreeDrag(this,event)\" ondragstart=\"javascript:return Sitecore.Treeview.onTreeDrag(this,event)\" ondragover=\"javascript:return Sitecore.Treeview.onTreeDrop(this,event)\" ondrop=\"javascript:return Sitecore.Treeview.onTreeDrop(this,event)\"");
}
output.Write(">");
output.Write("<input id=\"");
output.Write(this.ID);
output.Write("_Selected\" type=\"hidden\" value=\"" + folder.ID.ToShortID() + "\"/>");
output.Write("<input id=\"");
output.Write(this.ID);
output.Write("_Database\" type=\"hidden\" value=\"" + folder.Database.Name + "\"/>");
output.Write("<input id=\"");
output.Write(this.ID);
output.Write("_Parameters\" type=\"hidden\" value=\"" + this.GetParameters() + "\"/>");
output.Write("<input id=\"");
output.Write(this.ID);
output.Write("_ExcludedTemplatesForSelection\" type=\"hidden\" value=\"" + this.GetExcludedTemplates() + "\"/>");
output.Write("<input id=\"");
output.Write(this.ID);
output.Write("_IncludedTemplatesForSelection\" type=\"hidden\" value=\"" + this.GetIncludedTemplates() + "\"/>");
if (this.ShowRoot)
{
this.RenderNode(output, dataView, filter, root, root, folder);
}
else
{
ItemCollection children = dataView.GetChildren(root, string.Empty, true, 0, 0, this.GetFilter());
foreach (Item item in children)
{
this.RenderNode(output, dataView, filter, root, item, folder);
}
}
output.Write("</div>");
}
public Pager(IDataView dataView, Grid grid, jQueryObject pagerElement)
{
}
public ClusteringScorer(IHostEnvironment env, Arguments args, IDataView data, ISchemaBoundMapper mapper, RoleMappedSchema trainSchema)
: base(args, env, data, mapper, trainSchema, RegistrationName, MetadataUtils.Const.ScoreColumnKind.Clustering,
MetadataUtils.Const.ScoreValueKind.Score, OutputTypeMatches, GetPredColType)
{
}
public static NormalizeTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input)
{
Contracts.CheckValue(env, nameof(env));
var h = env.Register("Normalize");
h.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel(GetVersionInfo());
h.CheckValue(input, nameof(input));
return(h.Apply("Loading Model",
ch =>
{
// *** Binary format ***
// int: sizeof(Float)
// <remainder handled in ctors>
int cbFloat = ctx.Reader.ReadInt32();
ch.CheckDecode(cbFloat == sizeof(Float));
return new NormalizeTransform(h, ctx, input);
}));
}
private NormalizeTransform(IHost host, ArgumentsBase args, IDataView input,
Func <int, int, ColumnType, IRowCursor, IColumnFunctionBuilder> fnCreate,
params int[] extraTrainColumnIds)
: base(host, host.CheckRef(args, nameof(args)).GetColumns(), input, args.TestType)
{
Host.AssertNonEmpty(Infos);
Host.Assert(Utils.Size(Infos) == Utils.Size(args.GetColumns()));
bool[] activeInput = new bool[Source.Schema.ColumnCount];
if (Utils.Size(extraTrainColumnIds) > 0)
{
foreach (var colId in extraTrainColumnIds)
{
Host.Assert(0 <= colId && colId < activeInput.Length);
activeInput[colId] = true;
}
}
foreach (var info in Infos)
{
activeInput[info.Source] = true;
}
var functionBuilders = new IColumnFunctionBuilder[Infos.Length];
var needMoreData = new bool[Infos.Length];
// Go through the input data and pass it to the column function builders.
using (var pch = Host.StartProgressChannel("Normalize"))
{
long numRows = 0;
pch.SetHeader(new ProgressHeader("examples"), e => e.SetProgress(0, numRows));
using (var cursor = Source.GetRowCursor(col => activeInput[col]))
{
for (int i = 0; i < Infos.Length; i++)
{
needMoreData[i] = true;
var info = Infos[i];
functionBuilders[i] = fnCreate(i, info.Source, info.TypeSrc, cursor);
}
while (cursor.MoveNext())
{
// If the row has bad values, the good values are still being used for training.
// The comparisons in the code below are arranged so that NaNs in the input are not recorded.
// REVIEW: Should infinities and/or NaNs be filtered before the normalization? Should we not record infinities for min/max?
// Currently, infinities are recorded and will result in zero scale which in turn will result in NaN output for infinity input.
bool any = false;
for (int i = 0; i < Infos.Length; i++)
{
if (!needMoreData[i])
{
continue;
}
var info = Infos[i];
Host.Assert(!info.TypeSrc.IsVector || info.TypeSrc.IsVector && info.TypeSrc.IsKnownSizeVector);
Host.Assert(functionBuilders[i] != null);
any |= needMoreData[i] = functionBuilders[i].ProcessValue();
}
numRows++;
if (!any)
{
break;
}
}
}
pch.Checkpoint(numRows);
_functions = new IColumnFunction[Infos.Length];
for (int i = 0; i < Infos.Length; i++)
{
_functions[i] = functionBuilders[i].CreateColumnFunction();
}
}
SetMetadata();
}
public DateTimeTransformer Fit(IDataView input)
{
return(new DateTimeTransformer(_host, _options.Source, _options.Prefix, _options.Country, input.Schema));
}
private static void ValidateUserType(SchemaDefinition schemaDefinition, Type userType, IDataView data)
{
//Get memberInfos
MemberInfo[] memberInfos = null;
if (schemaDefinition == null)
{
memberInfos = SchemaDefinition.GetMemberInfos(userType, SchemaDefinition.Direction.Write);
if (memberInfos == null)
{
return;
}
foreach (var memberInfo in memberInfos)
{
ValidateMemberInfo(memberInfo, data);
}
}
else
{
for (int i = 0; i < schemaDefinition.Count; ++i)
{
var col = schemaDefinition[i];
if (col.MemberName == null)
{
throw Contracts.ExceptParam(nameof(schemaDefinition), "Null field name detected in schema definition");
}
MemberInfo memberInfo = null;
// Infer the column name.
var colName = string.IsNullOrEmpty(col.ColumnName) ? col.MemberName : col.ColumnName;
if (col.Generator == null)
{
memberInfo = userType.GetField(col.MemberName);
if (memberInfo == null)
{
memberInfo = userType.GetProperty(col.MemberName);
}
if ((memberInfo is FieldInfo && (memberInfo as FieldInfo).FieldType == typeof(IChannel)) ||
(memberInfo is PropertyInfo && (memberInfo as PropertyInfo).PropertyType == typeof(IChannel)))
{
continue;
}
}
else
{
memberInfo = col.ReturnType;
}
ValidateMemberInfo(memberInfo, data);
}
}
}
public BinaryPredictionTransformer <CalibratedModelParametersBase <LightGbmBinaryModelParameters, PlattCalibrator> > Train(IDataView trainData, IDataView validationData = null) => TrainTransformer(trainData, validationData);
private void CompareSelectedR4ScalarColumns(string leftColumnName, string rightColumnName, IDataView left, IDataView right, int precision = 6)
{
var leftColumnIndex = left.Schema[leftColumnName].Index;
var rightColumnIndex = right.Schema[rightColumnName].Index;
using (var expectedCursor = left.GetRowCursor(left.Schema[leftColumnIndex]))
using (var actualCursor = right.GetRowCursor(right.Schema[rightColumnIndex]))
{
float expected = default;
VBuffer <float> actual = default;
var expectedGetter = expectedCursor.GetGetter <float>(leftColumnIndex);
var actualGetter = actualCursor.GetGetter <VBuffer <float> >(rightColumnIndex);
while (expectedCursor.MoveNext() && actualCursor.MoveNext())
{
expectedGetter(ref expected);
actualGetter(ref actual);
// Scalar such as R4 (float) is converted to [1, 1]-tensor in ONNX format for consitency of making batch prediction.
Assert.Equal(1, actual.Length);
Assert.Equal(expected, actual.GetItemOrDefault(0), precision);
}
}
}
private void CompareSelectedR4VectorColumns(string leftColumnName, string rightColumnName, IDataView left, IDataView right, int precision = 6)
{
var leftColumnIndex = left.Schema[leftColumnName].Index;
var rightColumnIndex = right.Schema[rightColumnName].Index;
using (var expectedCursor = left.GetRowCursor(left.Schema[leftColumnIndex]))
using (var actualCursor = right.GetRowCursor(right.Schema[rightColumnIndex]))
{
VBuffer <float> expected = default;
VBuffer <float> actual = default;
var expectedGetter = expectedCursor.GetGetter <VBuffer <float> >(leftColumnIndex);
var actualGetter = actualCursor.GetGetter <VBuffer <float> >(rightColumnIndex);
while (expectedCursor.MoveNext() && actualCursor.MoveNext())
{
expectedGetter(ref expected);
actualGetter(ref actual);
Assert.Equal(expected.Length, actual.Length);
for (int i = 0; i < expected.Length; ++i)
{
Assert.Equal(expected.GetItemOrDefault(i), actual.GetItemOrDefault(i), precision);
}
}
}
}
public SsaChangePointDetector Fit(IDataView input)
{
_host.CheckValue(input, nameof(input));
return(new SsaChangePointDetector(_host, _args, input));
}
// Factory method for SignatureDataScorer.
private static IDataScorerTransform Create(IHostEnvironment env, Arguments args, IDataView data, ISchemaBoundMapper mapper, RoleMappedSchema trainSchema)
{
Contracts.CheckValue(env, nameof(env));
env.CheckValue(data, nameof(data));
env.CheckValue(mapper, nameof(mapper));
if (args.Top < 0)
{
throw env.Except($"Number of top contribution must be non negative");
}
if (args.Bottom < 0)
{
throw env.Except($"Number of bottom contribution must be non negative");
}
var contributionMapper = mapper as RowMapper;
env.CheckParam(mapper != null, nameof(mapper), "Unexpected mapper");
var scorer = ScoreUtils.GetScorerComponent(env, contributionMapper);
var scoredPipe = scorer.CreateComponent(env, data, contributionMapper, trainSchema);
return(scoredPipe);
}
/// <summary> /// Take the data in, make transformations, output the data. /// Note that <see cref="IDataView"/>'s are lazy, so no actual transformations happen here, just schema validation. /// </summary> public IDataView Transform(IDataView input) => InternalTransform.Transform(input);
private ClusteringScorer(IHostEnvironment env, ClusteringScorer transform, IDataView newSource)
: base(env, transform, newSource, RegistrationName)
{
}
/// <summary> /// Creates an IDataTransform from an IDataView. /// </summary> internal IDataTransform MakeDataTransform(IDataView input) => InternalTransform.MakeDataTransform(input);
public PageManager(int pageSize, int cacheDeep, int cacheWidth, IDataView dataSupply)
{
if (pageSize <= 0)
throw new Exception("pageSize");
if (cacheDeep < 0)
throw new Exception("cacheDeep");
_pageSize = pageSize;
_cacheDeep = cacheDeep;
_cache = new Page[Math.Max(cacheDeep * 2 + 2, cacheWidth)];
_dataSupply = dataSupply;
_asyncPool = new AsyncLoadPool(_id);
//ResetPages();
}
public static ProduceIdTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input)
{
Contracts.CheckValue(env, nameof(env));
var h = env.Register(LoaderSignature);
h.CheckValue(ctx, nameof(ctx));
h.CheckValue(input, nameof(input));
ctx.CheckAtModel(GetVersionInfo());
return(h.Apply("Loading Model", ch => new ProduceIdTransform(h, ctx, input)));
}
//List<ExtremePoint> _extremePoints;
public ProcessingService()
{
_dataCollection = null;
_movingAverageCalculators = new Dictionary<int, KeyValuePair<PriceCalculator, IMovingAverageCalculator>>();
}
private static async Task StartMachineLearning(string TrainDataPath, string TestDataPath)
{
if (File.Exists(TrainDataPath) && File.Exists(TestDataPath))
{
var BestModel = await TrainAndGetBestModel(TrainDataPath);
Console.WriteLine("=============== 模型训练完成 ===============");
Console.WriteLine("=============== 正在读取测试数据文件 ===============");
IDataView TestDataView = MLCProvider.Current.Data.LoadFromTextFile <BookRating>(TestDataPath, ',', true);
EvaluateModel(TestDataView, BestModel);
Console.WriteLine("=============== 请输入UserID和ISBN以预测Rating ===============");
FLAG1:
Console.WriteLine("UserID:");
string UserID = Console.ReadLine();
if (string.IsNullOrWhiteSpace(UserID))
{
Console.WriteLine("此项必填，不可为空");
Console.WriteLine();
goto FLAG1;
}
Console.WriteLine();
FLAG2:
Console.WriteLine("Age:");
string Age = Console.ReadLine();
if (string.IsNullOrWhiteSpace(Age))
{
Console.WriteLine("此项必填，不可为空");
Console.WriteLine();
goto FLAG2;
}
Console.WriteLine();
FLAG3:
Console.WriteLine("ISBN:");
string ISBN = Console.ReadLine();
if (string.IsNullOrWhiteSpace(Age))
{
Console.WriteLine("此项必填，不可为空");
Console.WriteLine();
goto FLAG3;
}
Console.WriteLine();
using (PredictionEngine <BookRating, BookRatingPrediction> Engine = MLCProvider.Current.Model.CreatePredictionEngine <BookRating, BookRatingPrediction>(BestModel))
{
BookRating PredictInput = new BookRating
{
UserId = UserID,
ISBN = ISBN,
Age = Age
};
BookRatingPrediction PredictResult = Engine.Predict(PredictInput);
Console.WriteLine("预测结果如下：" + (PredictResult.PredictedLabel ? "推荐" : "不推荐") + "，预测该书籍评分:" + Sigmoid(PredictResult.Score).ToString("0.##"));
}
}
else
{
Console.WriteLine("=============== 数据文件路径无效 ===============");
}
}
public RankingPredictionTransformer <LightGbmRankingModelParameters> Train(IDataView trainData, IDataView validationData = null) => TrainTransformer(trainData, validationData);
internal SsaChangePointDetector(IHostEnvironment env, Arguments args, IDataView input)
: this(env, args)
{
Model.Train(new RoleMappedData(input, null, InputColumnName));
}
private void RenderNode(HtmlTextWriter output, IDataView dataView, string filter, Item root, Item parent, Item folder)
{
Assert.ArgumentNotNull(output, "output");
Assert.ArgumentNotNull(dataView, "dataView");
Assert.ArgumentNotNull(filter, "filter");
Assert.ArgumentNotNull(root, "root");
Assert.ArgumentNotNull(parent, "parent");
Assert.ArgumentNotNull(folder, "folder");
bool isExpanded = (parent.ID == root.ID) || (parent.Axes.IsAncestorOf(folder) && (parent.ID != folder.ID));
this.RenderNodeBegin(output, dataView, filter, parent, parent.ID == folder.ID, isExpanded);
if (isExpanded)
{
ItemCollection itemC = dataView.GetChildren(parent, string.Empty, true, 0, 0, this.GetFilter());
if (itemC != null)
{
foreach (Item item in itemC)
{
this.RenderNode(output, dataView, filter, root, item, folder);
}
}
}
RenderNodeEnd(output);
}
// Factory method for SignatureLoadDataTransform.
private static IDataTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input)
{
Contracts.CheckValue(env, nameof(env));
env.CheckValue(ctx, nameof(ctx));
env.CheckValue(input, nameof(input));
return(new SsaChangePointDetector(env, ctx).MakeDataTransform(input));
}
private static void RenderTreeNodeGlyph(HtmlTextWriter output, IDataView dataView, string filter, Item item, string id, bool isExpanded)
{
Assert.ArgumentNotNull(output, "output");
Assert.ArgumentNotNull(dataView, "dataView");
Assert.ArgumentNotNull(filter, "filter");
Assert.ArgumentNotNull(item, "item");
Assert.ArgumentNotNullOrEmpty(id, "id");
ImageBuilder builder2 = new ImageBuilder {
Class = "scContentTreeNodeGlyph"
};
ImageBuilder builder = builder2;
if (dataView.HasChildren(item, filter))
{
if (isExpanded)
{
builder.Src = "images/collapse15x15.gif";
}
else
{
builder.Src = "images/expand15x15.gif";
}
}
else
{
builder.Src = "images/noexpand15x15.gif";
}
output.Write(builder.ToString());
}
public static RowShufflingTransformer Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input)
{
Contracts.CheckValue(env, nameof(env));
var h = env.Register(RegistrationName);
h.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel(GetVersionInfo());
h.CheckValue(input, nameof(input));
return(h.Apply("Loading Model", ch => new RowShufflingTransformer(h, ctx, input)));
}
