public bool DataSampleExtractComplex(string node, DataSampleModel complexObj)
        {
            var result = true;

            try
            {
                //to do
            }
            catch (Exception ex)
            {
                _logger.LogError("ExtractSimple失败:" + ex.ToString());
                result = false;
            }
            return(result);
        }
Exemple #2
0
 public JsonResult DataSampleExtractSimple(string node, DataSampleModel simObj)
 {
     try
     {
         _logRepository.Add("画布-样本管理-简单抽样");
         var result = true;
         var msg    = "提取成功";
         result = _dataFlowRepository.DataSampleExtractSimple(node, simObj);
         if (!result)
         {
             msg = "提取失败";
         }
         return(Json(new { success = result, msg }));
     }
     catch (Exception ex)
     {
         return(Json(new { success = false, msg = ex.ToString() }));
     }
 }
Exemple #3
0
        public BigModule(IClientFactory clientFactory, ITableNameCache tableNameCache)
        {
            Get["/big/tables"] = _ =>
            {
                var model = new TablesModel();

                using (var adminClient = clientFactory.GetAdminClient())
                {
                    var tables = adminClient.ListTablesAsync().Result;

                    model.Tables.AddRange(tables.Select(x => new TablesEntryModel
                    {
                        Name = x.Name
                    }));
                }

                return(View["Tables.sshtml", model]);
            };



            Get["/big/data/list"] = _ =>
            {
                var model = new DataListModel
                {
                    Table      = Request.Query["table"],
                    StartKey   = Request.Query["skey"],
                    CurrentKey = Request.Query["ckey"],
                    EndKey     = Request.Query["ekey"]
                };

                model.SetTableNames(tableNameCache.Names);
                model.SetMaxRows(Request.Query["maxrows"]);

                if (string.IsNullOrEmpty(model.CurrentKey))
                {
                    model.CurrentKey = model.StartKey;
                }

                if (!string.IsNullOrEmpty(model.Table) && !string.IsNullOrEmpty(model.StartKey) && !string.IsNullOrEmpty(model.EndKey))
                {
                    using (var dataClient = clientFactory.GetDataClient())
                    {
                        var table = new BigTable(model.Table);

                        var rows = dataClient.GetRowsAsync(table, model.CurrentKey, model.EndKey, model.MaxRows + 1).Result;

                        foreach (var row in rows)
                        {
                            if (model.KeyCount < model.MaxRows)
                            {
                                model.AddRow(row);
                            }
                            else
                            {
                                model.NextKey = row.KeyString;
                            }
                        }
                    }
                }

                return(View["DataList.sshtml", model]);
            };



            Get["/big/data/sample"] = _ =>
            {
                const string timeSpanFormat = @"hh\:mm\:ss\.ff";

                var model = new DataSampleModel
                {
                    Table = Request.Query["table"]
                };

                model.SetTableNames(tableNameCache.Names);
                model.SetMaxRows(Request.Query["maxrows"]);

                if (!string.IsNullOrEmpty(model.Table))
                {
                    using (var dataClient = clientFactory.GetDataClient())
                    {
                        var table = new BigTable(model.Table);

                        Stopwatch watch = Stopwatch.StartNew();
                        var       keys  = dataClient.SampleRowKeysAsync(table).Result.ToList();
                        watch.Stop();
                        model.SampleElapsed = watch.Elapsed.ToString(timeSpanFormat);
                        model.SampleCount   = keys.Count;

                        watch = Stopwatch.StartNew();

                        foreach (var batch in keys.Take(model.MaxRows).Batch(50))
                        {
                            var rowTasks = batch
                                           .Select(x => new
                            {
                                Task = dataClient.GetRowsAsync(table, x.Key, null, 1),
                                x.KeyString
                            })
                                           .ToArray();

                            Task.WhenAll(rowTasks.Select(x => x.Task)).Wait();      // silly syntax to avoid resharper warning

                            foreach (var rowset in rowTasks)
                            {
                                if (rowset.Task.Result != null)
                                {
                                    var row = rowset.Task.Result.FirstOrDefault();
                                    if (row != null)
                                    {
                                        model.AddRow(row);
                                    }
                                }
                            }
                        }

                        watch.Stop();
                        model.GetRowsElapsed = watch.Elapsed.ToString(timeSpanFormat);
                    }
                }

                return(View["DataSample.sshtml", model]);
            };
        }
        public bool DataSampleExtractSimple(string node, DataSampleModel simObj)
        {
            var result = true;

            try
            {
                var       cacheData = GetDataFlowCache(node);
                DataTable dtRel     = new DataTable();
                var       dt        = cacheData.DataTable;
                int       dataCount = dt.Rows.Count;
                var       rowList   = new List <DataRow>();
                switch (simObj.Method)
                {
                case "extract":
                    var count = Convert.ToInt32(dataCount / simObj.SimVal);
                    dtRel = dt.AsEnumerable().Take(count).CopyToDataTable();
                    break;

                case "pecent":
                    bool valBoo = System.Text.RegularExpressions.Regex.IsMatch(simObj.SimVal.ToString(), @"^[1-9]\d*$");
                    if (!valBoo)
                    {
                        simObj.SimVal = simObj.SimVal * 100;
                    }
                    if (simObj.SimMax > 0)
                    {
                        double dc = 0;
                        if (valBoo)
                        {
                            dc = dataCount * ((double)(simObj.SimVal) / 100);
                        }
                        else
                        {
                            dc = dataCount * ((double)(simObj.SimVal));
                        }
                        if (simObj.SimMax > dc)
                        {
                            dtRel = dt.AsEnumerable().Take(simObj.SimMax).CopyToDataTable();
                        }
                        else
                        {
                            dtRel = dt.AsEnumerable().Take(Convert.ToInt32(Math.Round(simObj.SimMax / (double)(simObj.SimVal / 100), 0))).CopyToDataTable();
                            dtRel = dtRel.AsEnumerable().Take(Convert.ToInt32(Math.Round(dataCount * (simObj.SimVal / 100), 0))).CopyToDataTable();
                        }
                    }
                    else
                    {
                        dtRel = dt.AsEnumerable().Take(Convert.ToInt32(Math.Round(dataCount * (simObj.SimVal / 100), 0))).CopyToDataTable();
                    }
                    break;

                default:
                    dtRel = dt.AsEnumerable().Take(Convert.ToInt32(Math.Round(simObj.SimVal, 0))).CopyToDataTable();
                    break;
                }

                cacheData.DataTable = dtRel;
                cacheData.DataCount = dtRel.Rows.Count;
                var handler = new LegacyCodeHandler(_extractEngineDataHandlerlogger);
                cacheData.DataColumns = handler.GenerateColumnDataForCache(cacheData.DataTable, cacheData.DataColumns);
                SetDataFlowCache(cacheData, node);
            }
            catch (Exception ex)
            {
                _logger.LogError("ExtractSimple失败:" + ex.ToString());
                result = false;
            }
            return(result);
        }