public void Initialize(IDataView dataView) { if (dataView == null) throw new ArgumentNullException(); if (_dataCollection != dataView) { if (_dataCollection != null) { this.ProcessingCompleted -= _dataCollection.ProcessingCompleted; } _dataCollection = dataView; this.ProcessingCompleted += _dataCollection.ProcessingCompleted; // Initialize all moving average calculators foreach (int id in _movingAverageCalculators.Keys) { IMovingAverageCalculator ma = _movingAverageCalculators[id].Value; ma.Reset(); } // Initialize the trend calculator InitializeTrendCalculator(); // Initialize the gap calculator InitializeGapCalculator(); // Initialize the trend calculator InitializeTrendCalculator(); } }
protected override void Render(HtmlTextWriter output, IDataView dataView, string filter, Item root, Item folder) { Assert.ArgumentNotNull(output, "output"); Assert.ArgumentNotNull(dataView, "dataView"); Assert.ArgumentNotNull(filter, "filter"); Assert.ArgumentNotNull(root, "root"); Assert.ArgumentNotNull(folder, "folder"); output.Write("<div id=\""); output.Write(this.ID); output.Write("\" onclick=\"javascript:return Sitecore.Treeview.onTreeClick(this,event"); if (!string.IsNullOrEmpty(this.Click)) { output.Write(",'"); output.Write(StringUtil.EscapeQuote(this.Click)); output.Write("'"); } output.Write(")\""); if (!string.IsNullOrEmpty(this.DblClick)) { output.Write(" ondblclick=\""); output.Write(AjaxScriptManager.GetEventReference(this.DblClick)); output.Write("\""); } if (!string.IsNullOrEmpty(this.ContextMenu)) { output.Write(" oncontextmenu=\""); output.Write(AjaxScriptManager.GetEventReference(this.ContextMenu)); output.Write("\""); } if (this.AllowDragging) { output.Write(" onmousedown=\"javascript:return Sitecore.Treeview.onTreeDrag(this,event)\" onmousemove=\"javascript:return Sitecore.Treeview.onTreeDrag(this,event)\" ondragstart=\"javascript:return Sitecore.Treeview.onTreeDrag(this,event)\" ondragover=\"javascript:return Sitecore.Treeview.onTreeDrop(this,event)\" ondrop=\"javascript:return Sitecore.Treeview.onTreeDrop(this,event)\""); } output.Write(">"); output.Write("<input id=\""); output.Write(this.ID); output.Write("_Selected\" type=\"hidden\" value=\"" + folder.ID.ToShortID() + "\"/>"); output.Write("<input id=\""); output.Write(this.ID); output.Write("_Database\" type=\"hidden\" value=\"" + folder.Database.Name + "\"/>"); output.Write("<input id=\""); output.Write(this.ID); output.Write("_Parameters\" type=\"hidden\" value=\"" + this.GetParameters() + "\"/>"); if (this.ShowRoot) { this.RenderNode(output, dataView, filter, root, root, folder); } else { foreach (Item item in dataView.GetChildren(root, string.Empty, true, 0, 0, this.GetFilter())) { this.RenderNode(output, dataView, filter, root, item, folder); } } output.Write("</div>"); }
public static IDataView LoadPipeWithPredictor(IHostEnvironment env, Stream modelStream, IDataView view) { // Load transforms. var pipe = env.LoadTransforms(modelStream, view); // Load predictor (if present) and apply default scorer. // REVIEW: distinguish the case of predictor / no predictor? var predictor = env.LoadPredictorOrNull(modelStream); if (predictor != null) { var roles = ModelFileUtils.LoadRoleMappingsOrNull(env, modelStream); pipe = roles != null ? env.CreateDefaultScorer(new RoleMappedData(pipe, roles, opt : true), predictor) : env.CreateDefaultScorer(new RoleMappedData(pipe, label : null, "Features"), predictor); } return(pipe); }
private void RenderNodeBegin(HtmlTextWriter output, IDataView dataView, string filter, Item item, bool active, bool isExpanded) { Assert.ArgumentNotNull(output, "output"); Assert.ArgumentNotNull(dataView, "dataView"); Assert.ArgumentNotNull(filter, "filter"); Assert.ArgumentNotNull(item, "item"); string shortID = item.ID.ToShortID().ToString(); string nodeID = this.GetNodeID(shortID); output.Write("<div id=\""); output.Write(nodeID); output.Write("\" class=\"scContentTreeNode\">"); RenderTreeNodeGlyph(output, dataView, filter, item, shortID, isExpanded); string str3 = (active || this.SelectedIDs.Contains(shortID)) ? "scContentTreeNodeActive" : "scContentTreeNodeNormal"; string style = this.GetStyle(item); output.Write("<a href=\"#\" class=\"" + str3 + "\""); if (!string.IsNullOrEmpty(item.Help.Text)) { output.Write(" title=\""); output.Write(StringUtil.EscapeQuote(item.Help.Text)); output.Write("\""); } output.Write(style); output.Write(">"); RenderTreeNodeIcon(output, item); output.Write("<span>{0}</span>", this.GetTreeNodeName(item)); output.Write("</a>"); }
public ArrangeManager(Form owner, IDataView dv) { this.OwnerForm = owner; this.Dataview = dv; }
protected virtual void Render(HtmlTextWriter output, IDataView dataView, string filter, Item root, Item folder) { Assert.ArgumentNotNull(output, "output"); Assert.ArgumentNotNull(dataView, "dataView"); Assert.ArgumentNotNull(filter, "filter"); Assert.ArgumentNotNull(root, "root"); Assert.ArgumentNotNull(folder, "folder"); output.Write(string.Format("<div id=\"{0}\"", this.ID)); //double click custom event output.Write(string.Format(" onclick=\"javascript:return Sitecore.Treeview.onTreeClick(this,event);\" ondblclick=\"FieldSuite.Fields.Treelist.AddItem('{0}',event);", this.ParentId)); output.Write("\""); if (!string.IsNullOrEmpty(this.ContextMenu)) { output.Write(" oncontextmenu=\""); output.Write(AjaxScriptManager.GetEventReference(this.ContextMenu)); output.Write("\""); } if (this.AllowDragging) { output.Write( " onmousedown=\"javascript:return Sitecore.Treeview.onTreeDrag(this,event)\" onmousemove=\"javascript:return Sitecore.Treeview.onTreeDrag(this,event)\" ondragstart=\"javascript:return Sitecore.Treeview.onTreeDrag(this,event)\" ondragover=\"javascript:return Sitecore.Treeview.onTreeDrop(this,event)\" ondrop=\"javascript:return Sitecore.Treeview.onTreeDrop(this,event)\""); } output.Write(">"); output.Write("<input id=\""); output.Write(this.ID); output.Write("_Selected\" type=\"hidden\" value=\"" + folder.ID.ToShortID() + "\"/>"); output.Write("<input id=\""); output.Write(this.ID); output.Write("_Database\" type=\"hidden\" value=\"" + folder.Database.Name + "\"/>"); output.Write("<input id=\""); output.Write(this.ID); output.Write("_Parameters\" type=\"hidden\" value=\"" + this.GetParameters() + "\"/>"); output.Write("<input id=\""); output.Write(this.ID); output.Write("_ExcludedTemplatesForSelection\" type=\"hidden\" value=\"" + this.GetExcludedTemplates() + "\"/>"); output.Write("<input id=\""); output.Write(this.ID); output.Write("_IncludedTemplatesForSelection\" type=\"hidden\" value=\"" + this.GetIncludedTemplates() + "\"/>"); if (this.ShowRoot) { this.RenderNode(output, dataView, filter, root, root, folder); } else { ItemCollection children = dataView.GetChildren(root, string.Empty, true, 0, 0, this.GetFilter()); foreach (Item item in children) { this.RenderNode(output, dataView, filter, root, item, folder); } } output.Write("</div>"); }
public Pager(IDataView dataView, Grid grid, jQueryObject pagerElement) { }
public ClusteringScorer(IHostEnvironment env, Arguments args, IDataView data, ISchemaBoundMapper mapper, RoleMappedSchema trainSchema) : base(args, env, data, mapper, trainSchema, RegistrationName, MetadataUtils.Const.ScoreColumnKind.Clustering, MetadataUtils.Const.ScoreValueKind.Score, OutputTypeMatches, GetPredColType) { }
public static NormalizeTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input) { Contracts.CheckValue(env, nameof(env)); var h = env.Register("Normalize"); h.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(GetVersionInfo()); h.CheckValue(input, nameof(input)); return(h.Apply("Loading Model", ch => { // *** Binary format *** // int: sizeof(Float) // <remainder handled in ctors> int cbFloat = ctx.Reader.ReadInt32(); ch.CheckDecode(cbFloat == sizeof(Float)); return new NormalizeTransform(h, ctx, input); })); }
private NormalizeTransform(IHost host, ArgumentsBase args, IDataView input, Func <int, int, ColumnType, IRowCursor, IColumnFunctionBuilder> fnCreate, params int[] extraTrainColumnIds) : base(host, host.CheckRef(args, nameof(args)).GetColumns(), input, args.TestType) { Host.AssertNonEmpty(Infos); Host.Assert(Utils.Size(Infos) == Utils.Size(args.GetColumns())); bool[] activeInput = new bool[Source.Schema.ColumnCount]; if (Utils.Size(extraTrainColumnIds) > 0) { foreach (var colId in extraTrainColumnIds) { Host.Assert(0 <= colId && colId < activeInput.Length); activeInput[colId] = true; } } foreach (var info in Infos) { activeInput[info.Source] = true; } var functionBuilders = new IColumnFunctionBuilder[Infos.Length]; var needMoreData = new bool[Infos.Length]; // Go through the input data and pass it to the column function builders. using (var pch = Host.StartProgressChannel("Normalize")) { long numRows = 0; pch.SetHeader(new ProgressHeader("examples"), e => e.SetProgress(0, numRows)); using (var cursor = Source.GetRowCursor(col => activeInput[col])) { for (int i = 0; i < Infos.Length; i++) { needMoreData[i] = true; var info = Infos[i]; functionBuilders[i] = fnCreate(i, info.Source, info.TypeSrc, cursor); } while (cursor.MoveNext()) { // If the row has bad values, the good values are still being used for training. // The comparisons in the code below are arranged so that NaNs in the input are not recorded. // REVIEW: Should infinities and/or NaNs be filtered before the normalization? Should we not record infinities for min/max? // Currently, infinities are recorded and will result in zero scale which in turn will result in NaN output for infinity input. bool any = false; for (int i = 0; i < Infos.Length; i++) { if (!needMoreData[i]) { continue; } var info = Infos[i]; Host.Assert(!info.TypeSrc.IsVector || info.TypeSrc.IsVector && info.TypeSrc.IsKnownSizeVector); Host.Assert(functionBuilders[i] != null); any |= needMoreData[i] = functionBuilders[i].ProcessValue(); } numRows++; if (!any) { break; } } } pch.Checkpoint(numRows); _functions = new IColumnFunction[Infos.Length]; for (int i = 0; i < Infos.Length; i++) { _functions[i] = functionBuilders[i].CreateColumnFunction(); } } SetMetadata(); }
public DateTimeTransformer Fit(IDataView input) { return(new DateTimeTransformer(_host, _options.Source, _options.Prefix, _options.Country, input.Schema)); }
private static void ValidateUserType(SchemaDefinition schemaDefinition, Type userType, IDataView data) { //Get memberInfos MemberInfo[] memberInfos = null; if (schemaDefinition == null) { memberInfos = SchemaDefinition.GetMemberInfos(userType, SchemaDefinition.Direction.Write); if (memberInfos == null) { return; } foreach (var memberInfo in memberInfos) { ValidateMemberInfo(memberInfo, data); } } else { for (int i = 0; i < schemaDefinition.Count; ++i) { var col = schemaDefinition[i]; if (col.MemberName == null) { throw Contracts.ExceptParam(nameof(schemaDefinition), "Null field name detected in schema definition"); } MemberInfo memberInfo = null; // Infer the column name. var colName = string.IsNullOrEmpty(col.ColumnName) ? col.MemberName : col.ColumnName; if (col.Generator == null) { memberInfo = userType.GetField(col.MemberName); if (memberInfo == null) { memberInfo = userType.GetProperty(col.MemberName); } if ((memberInfo is FieldInfo && (memberInfo as FieldInfo).FieldType == typeof(IChannel)) || (memberInfo is PropertyInfo && (memberInfo as PropertyInfo).PropertyType == typeof(IChannel))) { continue; } } else { memberInfo = col.ReturnType; } ValidateMemberInfo(memberInfo, data); } } }
public BinaryPredictionTransformer <CalibratedModelParametersBase <LightGbmBinaryModelParameters, PlattCalibrator> > Train(IDataView trainData, IDataView validationData = null) => TrainTransformer(trainData, validationData);
private void CompareSelectedR4ScalarColumns(string leftColumnName, string rightColumnName, IDataView left, IDataView right, int precision = 6) { var leftColumnIndex = left.Schema[leftColumnName].Index; var rightColumnIndex = right.Schema[rightColumnName].Index; using (var expectedCursor = left.GetRowCursor(left.Schema[leftColumnIndex])) using (var actualCursor = right.GetRowCursor(right.Schema[rightColumnIndex])) { float expected = default; VBuffer <float> actual = default; var expectedGetter = expectedCursor.GetGetter <float>(leftColumnIndex); var actualGetter = actualCursor.GetGetter <VBuffer <float> >(rightColumnIndex); while (expectedCursor.MoveNext() && actualCursor.MoveNext()) { expectedGetter(ref expected); actualGetter(ref actual); // Scalar such as R4 (float) is converted to [1, 1]-tensor in ONNX format for consitency of making batch prediction. Assert.Equal(1, actual.Length); Assert.Equal(expected, actual.GetItemOrDefault(0), precision); } } }
private void CompareSelectedR4VectorColumns(string leftColumnName, string rightColumnName, IDataView left, IDataView right, int precision = 6) { var leftColumnIndex = left.Schema[leftColumnName].Index; var rightColumnIndex = right.Schema[rightColumnName].Index; using (var expectedCursor = left.GetRowCursor(left.Schema[leftColumnIndex])) using (var actualCursor = right.GetRowCursor(right.Schema[rightColumnIndex])) { VBuffer <float> expected = default; VBuffer <float> actual = default; var expectedGetter = expectedCursor.GetGetter <VBuffer <float> >(leftColumnIndex); var actualGetter = actualCursor.GetGetter <VBuffer <float> >(rightColumnIndex); while (expectedCursor.MoveNext() && actualCursor.MoveNext()) { expectedGetter(ref expected); actualGetter(ref actual); Assert.Equal(expected.Length, actual.Length); for (int i = 0; i < expected.Length; ++i) { Assert.Equal(expected.GetItemOrDefault(i), actual.GetItemOrDefault(i), precision); } } } }
public SsaChangePointDetector Fit(IDataView input) { _host.CheckValue(input, nameof(input)); return(new SsaChangePointDetector(_host, _args, input)); }
// Factory method for SignatureDataScorer. private static IDataScorerTransform Create(IHostEnvironment env, Arguments args, IDataView data, ISchemaBoundMapper mapper, RoleMappedSchema trainSchema) { Contracts.CheckValue(env, nameof(env)); env.CheckValue(data, nameof(data)); env.CheckValue(mapper, nameof(mapper)); if (args.Top < 0) { throw env.Except($"Number of top contribution must be non negative"); } if (args.Bottom < 0) { throw env.Except($"Number of bottom contribution must be non negative"); } var contributionMapper = mapper as RowMapper; env.CheckParam(mapper != null, nameof(mapper), "Unexpected mapper"); var scorer = ScoreUtils.GetScorerComponent(env, contributionMapper); var scoredPipe = scorer.CreateComponent(env, data, contributionMapper, trainSchema); return(scoredPipe); }
/// <summary> /// Take the data in, make transformations, output the data. /// Note that <see cref="IDataView"/>'s are lazy, so no actual transformations happen here, just schema validation. /// </summary> public IDataView Transform(IDataView input) => InternalTransform.Transform(input);
private ClusteringScorer(IHostEnvironment env, ClusteringScorer transform, IDataView newSource) : base(env, transform, newSource, RegistrationName) { }
/// <summary> /// Creates an IDataTransform from an IDataView. /// </summary> internal IDataTransform MakeDataTransform(IDataView input) => InternalTransform.MakeDataTransform(input);
public PageManager(int pageSize, int cacheDeep, int cacheWidth, IDataView dataSupply) { if (pageSize <= 0) throw new Exception("pageSize"); if (cacheDeep < 0) throw new Exception("cacheDeep"); _pageSize = pageSize; _cacheDeep = cacheDeep; _cache = new Page[Math.Max(cacheDeep * 2 + 2, cacheWidth)]; _dataSupply = dataSupply; _asyncPool = new AsyncLoadPool(_id); //ResetPages(); }
public static ProduceIdTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input) { Contracts.CheckValue(env, nameof(env)); var h = env.Register(LoaderSignature); h.CheckValue(ctx, nameof(ctx)); h.CheckValue(input, nameof(input)); ctx.CheckAtModel(GetVersionInfo()); return(h.Apply("Loading Model", ch => new ProduceIdTransform(h, ctx, input))); }
//List<ExtremePoint> _extremePoints; public ProcessingService() { _dataCollection = null; _movingAverageCalculators = new Dictionary<int, KeyValuePair<PriceCalculator, IMovingAverageCalculator>>(); }
private static async Task StartMachineLearning(string TrainDataPath, string TestDataPath) { if (File.Exists(TrainDataPath) && File.Exists(TestDataPath)) { var BestModel = await TrainAndGetBestModel(TrainDataPath); Console.WriteLine("=============== 模型训练完成 ==============="); Console.WriteLine("=============== 正在读取测试数据文件 ==============="); IDataView TestDataView = MLCProvider.Current.Data.LoadFromTextFile <BookRating>(TestDataPath, ',', true); EvaluateModel(TestDataView, BestModel); Console.WriteLine("=============== 请输入UserID和ISBN以预测Rating ==============="); FLAG1: Console.WriteLine("UserID:"); string UserID = Console.ReadLine(); if (string.IsNullOrWhiteSpace(UserID)) { Console.WriteLine("此项必填,不可为空"); Console.WriteLine(); goto FLAG1; } Console.WriteLine(); FLAG2: Console.WriteLine("Age:"); string Age = Console.ReadLine(); if (string.IsNullOrWhiteSpace(Age)) { Console.WriteLine("此项必填,不可为空"); Console.WriteLine(); goto FLAG2; } Console.WriteLine(); FLAG3: Console.WriteLine("ISBN:"); string ISBN = Console.ReadLine(); if (string.IsNullOrWhiteSpace(Age)) { Console.WriteLine("此项必填,不可为空"); Console.WriteLine(); goto FLAG3; } Console.WriteLine(); using (PredictionEngine <BookRating, BookRatingPrediction> Engine = MLCProvider.Current.Model.CreatePredictionEngine <BookRating, BookRatingPrediction>(BestModel)) { BookRating PredictInput = new BookRating { UserId = UserID, ISBN = ISBN, Age = Age }; BookRatingPrediction PredictResult = Engine.Predict(PredictInput); Console.WriteLine("预测结果如下:" + (PredictResult.PredictedLabel ? "推荐" : "不推荐") + ",预测该书籍评分:" + Sigmoid(PredictResult.Score).ToString("0.##")); } } else { Console.WriteLine("=============== 数据文件路径无效 ==============="); } }
public RankingPredictionTransformer <LightGbmRankingModelParameters> Train(IDataView trainData, IDataView validationData = null) => TrainTransformer(trainData, validationData);
internal SsaChangePointDetector(IHostEnvironment env, Arguments args, IDataView input) : this(env, args) { Model.Train(new RoleMappedData(input, null, InputColumnName)); }
private void RenderNode(HtmlTextWriter output, IDataView dataView, string filter, Item root, Item parent, Item folder) { Assert.ArgumentNotNull(output, "output"); Assert.ArgumentNotNull(dataView, "dataView"); Assert.ArgumentNotNull(filter, "filter"); Assert.ArgumentNotNull(root, "root"); Assert.ArgumentNotNull(parent, "parent"); Assert.ArgumentNotNull(folder, "folder"); bool isExpanded = (parent.ID == root.ID) || (parent.Axes.IsAncestorOf(folder) && (parent.ID != folder.ID)); this.RenderNodeBegin(output, dataView, filter, parent, parent.ID == folder.ID, isExpanded); if (isExpanded) { ItemCollection itemC = dataView.GetChildren(parent, string.Empty, true, 0, 0, this.GetFilter()); if (itemC != null) { foreach (Item item in itemC) { this.RenderNode(output, dataView, filter, root, item, folder); } } } RenderNodeEnd(output); }
// Factory method for SignatureLoadDataTransform. private static IDataTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input) { Contracts.CheckValue(env, nameof(env)); env.CheckValue(ctx, nameof(ctx)); env.CheckValue(input, nameof(input)); return(new SsaChangePointDetector(env, ctx).MakeDataTransform(input)); }
private static void RenderTreeNodeGlyph(HtmlTextWriter output, IDataView dataView, string filter, Item item, string id, bool isExpanded) { Assert.ArgumentNotNull(output, "output"); Assert.ArgumentNotNull(dataView, "dataView"); Assert.ArgumentNotNull(filter, "filter"); Assert.ArgumentNotNull(item, "item"); Assert.ArgumentNotNullOrEmpty(id, "id"); ImageBuilder builder2 = new ImageBuilder { Class = "scContentTreeNodeGlyph" }; ImageBuilder builder = builder2; if (dataView.HasChildren(item, filter)) { if (isExpanded) { builder.Src = "images/collapse15x15.gif"; } else { builder.Src = "images/expand15x15.gif"; } } else { builder.Src = "images/noexpand15x15.gif"; } output.Write(builder.ToString()); }
public static RowShufflingTransformer Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input) { Contracts.CheckValue(env, nameof(env)); var h = env.Register(RegistrationName); h.CheckValue(ctx, nameof(ctx)); ctx.CheckAtModel(GetVersionInfo()); h.CheckValue(input, nameof(input)); return(h.Apply("Loading Model", ch => new RowShufflingTransformer(h, ctx, input))); }