public async Task <Dictionary <string, PlotlyData> > Read(SwitchReader reader, int sampleRate = 1, bool disableNormalization = false,
                                                                  DateTime?minimumDate = null, double?minimumFitness = null)
        {
            minimumDate = minimumDate ?? DateTime.MinValue;

            var    data = new Dictionary <string, PlotlyData>();
            var    rand = new XorRandom();
            string line;

            while ((line = await reader.ReadLineAsync()) != null)
            {
                if (rand.Next(1, sampleRate) != 1)
                {
                    continue;
                }

                try
                {
                    var split = line.Split(',').Select(s => s.Trim()).ToArray();
                    var time  = DateTime.Parse(split[0].Substring(0, 24));

                    if (time < minimumDate)
                    {
                        continue;
                    }

                    foreach (var item in split.Skip(1))
                    {
                        if (item.Contains(": ") && !item.StartsWith("Start") && !item.StartsWith("End"))
                        {
                            var pair = item.Split(new[] { ": " }, StringSplitOptions.RemoveEmptyEntries);
                            if (!double.TryParse(pair[1], out var parsed))
                            {
                                continue;
                            }
                            if (!data.ContainsKey(pair[0]))
                            {
                                data.Add(pair[0], new PlotlyData {
                                    Name = pair[0]
                                });
                            }
                            data[pair[0]].X.Add(time.ToString("yyyy-MM-dd hh:mm:ss.ffff"));
                            data[pair[0]].Y.Add(double.Parse(pair[1]));
                            data[pair[0]].Text.Add(pair[1]);
                        }
                    }
                    //System.Diagnostics.Debug.WriteLine("Processing...");
                }
                catch (Exception)
                {
                    Console.WriteLine("Failed to parse:" + line);
                }
            }

            if (minimumFitness.HasValue)
            {
                var fitness = data.Last().Value;

                var removing = new List <int>();
                for (int i = 0; i < fitness.Y.Count(); i++)
                {
                    if (fitness.Y[i] < minimumFitness)
                    {
                        removing.Add(i);
                    }
                }
                foreach (var index in removing.OrderByDescending(o => o))
                {
                    foreach (var item in data.Where(d => d.Value.Y.Any()))
                    {
                        item.Value.Y.RemoveAt(index);
                        item.Value.Text.RemoveAt(index);
                    }
                }
            }

            if (!disableNormalization)
            {
                var fitness  = data.Last().Value;
                var nonEmpty = data.Take(data.Count() - 1).Where(d => d.Value.Y.Any());

                //on second pass reuse min/max
                if (_max == null || _min == null)
                {
                    _max = fitness.Y.Max();
                    _min = fitness.Y.Min();
                }
                var normalizer = new SharpLearning.FeatureTransformations.Normalization.LinearNormalizer();

                foreach (var list in nonEmpty)
                {
                    var oldMax = list.Value.Y.Max();
                    var oldMin = list.Value.Y.Min();
                    for (int i = 0; i < list.Value.Y.Count(); i++)
                    {
                        list.Value.Y[i] = normalizer.Normalize(_min.Value, _max.Value, oldMin, oldMax, list.Value.Y[i]);
                    }
                    //System.Diagnostics.Debug.WriteLine("Added to set:" + list.Key);
                }

                data = nonEmpty.Concat(new[] { data.Last() }).ToDictionary(k => k.Key, v => v.Value);
            }

            return(data);
        }
Beispiel #2
0
        public async Task <Dictionary <string, PlotlyData> > Read(SwitchReader reader, int sampleRate = 1, bool disableNormalization = false,
                                                                  DateTime?minimumDate = null, double?minimumFitness = null, IActivityLogger activityLogger = null)
        {
            minimumDate     = minimumDate ?? DateTime.MinValue;
            _activityLogger = activityLogger;

            var    data = new Dictionary <string, PlotlyData>();
            var    rand = new XorRandom();
            string line;
            int    counter = 0;

            //todo: chart bind progress
            //_activityLogger?.Add($"Processing: ", counter);
            //_activityLogger?.ShowMessageAlways();

            while ((line = await reader.ReadLineAsync()) != null)
            {
                if (sampleRate > 1 && rand.Next(1, sampleRate) != 1)
                {
                    continue;
                }

                try
                {
                    var split = line.Split(',').Select(s => s.Trim()).ToArray();
                    var time  = DateTime.Parse(split[0].Substring(0, 24));

                    if (time < minimumDate)
                    {
                        continue;
                    }

                    for (int i = 1; i < split.Length; i++)
                    {
                        var item = split[i];

                        if (item.Contains(": ") && !item.StartsWith("Start:") && !item.StartsWith("End:"))
                        {
                            var pair = item.Split(new[] { ": " }, StringSplitOptions.RemoveEmptyEntries);
                            if (!double.TryParse(pair[1], out var parsed))
                            {
                                continue;
                            }
                            if (!data.ContainsKey(pair[0]))
                            {
                                data.Add(pair[0], new PlotlyData {
                                    Name = pair[0]
                                });
                            }
                            data[pair[0]].X.Add(time.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
                            data[pair[0]].Y.Add(double.Parse(pair[1]));
                            data[pair[0]].Text.Add(pair[1]);
                        }
                    }
                    //counter++;
                    //if (counter % 1000 == 0)
                    //{
                    //_activityLogger?.Add($"Processing: ", counter);
                    //_activityLogger?.ShowMessageAlways();
                    //}
                }
                catch (Exception)
                {
                    //todo: feedback bad data but do not fail
                    Console.WriteLine("Failed to parse:" + line);
                }
            }

            if (minimumFitness.HasValue)
            {
                var fitness = data.Last().Value;

                var removing = new List <int>();
                for (int i = 0; i < fitness.Y.Count(); i++)
                {
                    if (fitness.Y[i] < minimumFitness)
                    {
                        removing.Add(i);
                    }
                }
                foreach (var index in removing.OrderByDescending(o => o))
                {
                    foreach (var item in data.Where(d => d.Value.Y.Any()))
                    {
                        item.Value.Y.RemoveAt(index);
                        item.Value.X.RemoveAt(index);
                        item.Value.Text.RemoveAt(index);
                    }
                }
            }

            if (!disableNormalization)
            {
                var fitness  = data.Last().Value;
                var nonEmpty = data.Take(data.Count() - 1).Where(d => d.Value.Y.Any());

                var max = fitness.Y.Max();
                var min = fitness.Y.Min();

                var normalizer = new SharpLearning.FeatureTransformations.Normalization.LinearNormalizer();

                foreach (var list in nonEmpty)
                {
                    var oldMax = list.Value.Y.Max();
                    var oldMin = list.Value.Y.Min();
                    for (int i = 0; i < list.Value.Y.Count(); i++)
                    {
                        list.Value.Y[i] = normalizer.Normalize(min, max, oldMin, oldMax, list.Value.Y[i]);
                    }
                }

                data = nonEmpty.Concat(new[] { data.Last() }).ToDictionary(k => k.Key, v => v.Value);
            }

            return(data);
        }
Beispiel #3
0
        public async Task <Dictionary <string, PlotlyData> > Read(SwitchReader reader, int sampleRate = 1, bool disableNormalization = false,
                                                                  DateTime?minimumDate = null, double?minimumFitness = null, IActivityLogger activityLogger = null)
        {
            minimumDate     = minimumDate ?? DateTime.MinValue;
            _activityLogger = activityLogger;

            _data = new Dictionary <string, PlotlyData>();
            string line;

            var counter = 0;

            _chunk = new List <string>(ChunkSize);

            while ((line = await reader.ReadLineAsync()) != null)
            {
                var rand = new XorRandom();
                if (sampleRate > 1 && rand.Next(1, sampleRate) != 1)
                {
                    continue;
                }

                if (DateTime.Parse(line.Substring(0, 24)) < minimumDate)
                {
                    continue;
                }

                _chunk.Add(line);
                counter++;

                if (counter % ChunkSize == 0)
                {
                    await SplitChunk();

                    _chunk.Clear();
                }
            }
            await SplitChunk();

            if (minimumFitness.HasValue)
            {
                var fitness = _data.Last().Value;

                var removing = new List <int>();
                for (int i = 0; i < fitness.Y.Count(); i++)
                {
                    if (fitness.Y[i] < minimumFitness)
                    {
                        removing.Add(i);
                    }
                }
                foreach (var index in removing.OrderByDescending(o => o))
                {
                    foreach (var item in _data.Where(d => d.Value.Y.Any()))
                    {
                        item.Value.Y.RemoveAt(index);
                        item.Value.X.RemoveAt(index);
                        item.Value.Text.RemoveAt(index);
                    }
                }
            }

            if (!disableNormalization)
            {
                var fitness  = _data.Last().Value;
                var nonEmpty = _data.Take(_data.Count() - 1).Where(d => d.Value.Y.Any());

                var max = fitness.Y.Max();
                var min = fitness.Y.Min();

                var normalizer = new SharpLearning.FeatureTransformations.Normalization.LinearNormalizer();

                foreach (var list in nonEmpty)
                {
                    var oldMax = list.Value.Y.Max();
                    var oldMin = list.Value.Y.Min();
                    for (int i = 0; i < list.Value.Y.Count(); i++)
                    {
                        list.Value.Y[i] = normalizer.Normalize(min, max, oldMin, oldMax, list.Value.Y[i]);
                    }
                }

                _data = nonEmpty.Concat(new[] { _data.Last() }).ToDictionary(k => k.Key, v => v.Value);
            }

            return(_data);
        }