Example #1
0
        public override int ReadChunk(out InputTextCunk chunk, int max_char_count)
        {
            List<String> list = new List<string>(max_char_count);
            int count = 0;
            for (int i = 0; i < max_char_count; i++)
            {
                var s = ReadRecord();
                if (s == null)
                    break;
                count += s.Length;
                list.Add(s);
                if (count > max_char_count)
                    break;
            }

            chunk = new InputTextCunk(list, count);
            return count;
        }
Example #2
0
        private IEnumerable <Dictionary <InterKey, List <InterValue> > > doMap(InputTextCunk chunk, int thread_num = 0)
        {
            Stopwatch watch         = new Stopwatch();
            var       input_records = chunk.Records;
            var       char_count    = chunk.CharCount;

            watch.Restart();
            var             dics   = new ConcurrentBag <Dictionary <InterKey, List <InterValue> > >();
            ParallelOptions option = new ParallelOptions();

            if (thread_num != 0)
            {
                option.MaxDegreeOfParallelism = thread_num;
            }

            Parallel.ForEach(Partitioner.Create(0, input_records.Count), option, (range) =>
            {
                var dic     = new Dictionary <InterKey, List <InterValue> >();
                var context = new MapContext <InterKey, InterValue>(dic);
                for (int i = range.Item1; i < range.Item2; i++)
                {
                    mapFunc.Invoke(input_records[i], context);
                }
                dics.Add(dic);
                Interlocked.Add(ref mapperInfo.MapEmits, context.EmitCount);
            });
            watch.Stop();

            mapperInfo.ProcessedRecords += input_records.Count;
            mapperInfo.ProcessedChars   += char_count;

            if (watch.Elapsed > maxWorkPeriod)
            {
                maxChunkSize = Math.Min(maxChunkSize / 2, maxCharsToMap);
            }
            if (watch.Elapsed < minWorkPeriod)
            {
                maxChunkSize = Math.Min(maxChunkSize * 2, maxCharsToMap);
            }

            logger.Debug("Mapped a chunk with {0} chars in {1}", StringFormatter.DigitGrouped(char_count), watch.Elapsed);
            return(dics);
        }
Example #3
0
        public override int ReadChunk(out InputTextCunk chunk, int max_char_count)
        {
            List <String> list  = new List <string>(max_char_count);
            int           count = 0;

            for (int i = 0; i < max_char_count; i++)
            {
                var s = ReadRecord();
                if (s == null)
                {
                    break;
                }
                count += s.Length;
                list.Add(s);
                if (count > max_char_count)
                {
                    break;
                }
            }

            chunk = new InputTextCunk(list, count);
            return(count);
        }