private static IEnumerable <ArraySegment <byte> > CalculateSegmentStrip(IEnumerable <ArraySegment <byte> > Input, InputOptions Options)
        {
            bool[] lookup;
            if (Options.HasFlag(InputOptions.StripWhitespaceAndPunctuation))
            {
                lookup = WhitespacePunctuationCharLookup;
            }
            else if (Options.HasFlag(InputOptions.StripWhitespace))
            {
                lookup = WhitespaceCharLookup;
            }
            else if (Options.HasFlag(InputOptions.StripPunctuation))
            {
                lookup = PunctuationCharLookup;
            }
            else
            {
                foreach (var segment in Input)
                {
                    yield return(segment);
                }
                yield break;
            }

            if (lookup == null || lookup.Length < 256)
            {
                throw new InvalidOperationException("Lookup should never be null");
            }

            foreach (var segment in Input)
            {
                // trim start
                var offset = segment.Offset;
                var end    = offset + segment.Count;
                var buffer = segment.Array;
                for (; offset < end; offset++)
                {
                    if (!lookup[buffer[offset]])
                    {
                        goto trimEnd;
                    }
                }
                continue;
trimEnd:
                end--;
                for (; end > offset; end--)
                {
                    if (!lookup[buffer[end]])
                    {
                        yield return(new ArraySegment <byte>(buffer, offset, end - offset));

                        continue;
                    }
                }
            }
        }
        private static ShocoModel GenerateModelFromSegments(IEnumerable <ArraySegment <byte> > Input, InputOptions Options, int MaximumLeadingBits, int MaxSuccessorBits, int EncodingTypes, bool OptimizeEncoding)
        {
            var input = Input;

            if (Options.HasFlag(InputOptions.SplitNewLine) || Options.HasFlag(InputOptions.SplitWhitespaceAndNewLine))
            {
                input = CalculateSegmentSplit(input, Options);
            }
            if (Options.HasFlag(InputOptions.StripPunctuation) || Options.HasFlag(InputOptions.StripWhitespace))
            {
                input = CalculateSegmentStrip(input, Options);
            }

            return(GenerateModelFromSegments(input, MaximumLeadingBits, MaxSuccessorBits, EncodingTypes, OptimizeEncoding));
        }
        private static IEnumerable <ArraySegment <byte> > CalculateSegments(IEnumerable <string> Input, Encoding InputEncoding, InputOptions Options)
        {
            var input = Input;

            if (Options.HasFlag(InputOptions.SplitWhitespaceAndNewLine))
            {
                input = input.SelectMany(i => i.Split(WhitespaceChars, StringSplitOptions.RemoveEmptyEntries));
            }
            else if (Options.HasFlag(InputOptions.SplitNewLine))
            {
                input = input.SelectMany(i => i.Split(NewLineChars, StringSplitOptions.RemoveEmptyEntries));
            }
            if (Options.HasFlag(InputOptions.StripWhitespaceAndPunctuation))
            {
                input = input.Select(i => i.Trim(WhitespacePunctuationChars));
            }
            else if (Options.HasFlag(InputOptions.StripWhitespace))
            {
                input = input.Select(i => i.Trim(WhitespaceChars));
            }
            else if (Options.HasFlag(InputOptions.StripPunctuation))
            {
                input = input.Select(i => i.Trim(PunctuationChars));
            }

            var buffer = new byte[2048];

            foreach (var segment in input)
            {
                if (buffer.Length < InputEncoding.GetMaxByteCount(segment.Length))
                {
                    buffer = new byte[InputEncoding.GetMaxByteCount(segment.Length)];
                }

                var length = InputEncoding.GetBytes(segment, 0, segment.Length, buffer, 0);

                if (length <= 1)
                {
                    continue;
                }

                yield return(new ArraySegment <byte>(buffer, 0, length));
            }
        }
        private static IEnumerable <ArraySegment <byte> > CalculateSegmentSplit(IEnumerable <ArraySegment <byte> > Input, InputOptions Options)
        {
            if (Options.HasFlag(InputOptions.SplitWhitespaceAndNewLine))
            {
                foreach (var segment in Input)
                {
                    var offset = segment.Offset;
                    var buffer = segment.Array;
                    var index  = offset;
                    for (; index < segment.Count; index++)
                    {
                        if (WhitespaceCharLookup[buffer[index]])
                        {
                            if (offset < index)
                            {
                                yield return(new ArraySegment <byte>(buffer, offset, offset - index));

                                offset = index + 1;
                            }
                        }
                    }
                    if (offset < index)
                    {
                        yield return(new ArraySegment <byte>(buffer, offset, offset - index));
                    }
                }
            }
            else if (Options.HasFlag(InputOptions.SplitNewLine))
            {
                foreach (var segment in Input)
                {
                    var offset = segment.Offset;
                    var buffer = segment.Array;
                    var index  = offset;
                    for (; index < segment.Count; index++)
                    {
                        if (buffer[index] == 10 || // '\n'
                            buffer[index] == 13)   // '\r'
                        {
                            if (offset < index)
                            {
                                yield return(new ArraySegment <byte>(buffer, offset, offset - index));

                                offset = index + 1;
                            }
                        }
                    }
                    if (offset < index)
                    {
                        yield return(new ArraySegment <byte>(buffer, offset, offset - index));
                    }
                }
            }
            else
            {
                foreach (var segment in Input)
                {
                    yield return(segment);
                }
            }
        }
        private static IEnumerable <ArraySegment <byte> > CalculateSegments(IEnumerable <Stream> Input, InputOptions Options)
        {
            var segment     = new byte[128];
            var buffer      = new byte[2048];
            var splitLookup = FalseCharLookup;
            var stripLookup = FalseCharLookup;

            if (Options.HasFlag(InputOptions.SplitWhitespaceAndNewLine))
            {
                splitLookup = WhitespaceCharLookup;
            }
            else if (Options.HasFlag(InputOptions.SplitNewLine))
            {
                splitLookup = NewLineCharLookup;
            }

            if (Options.HasFlag(InputOptions.StripWhitespaceAndPunctuation))
            {
                stripLookup = WhitespacePunctuationCharLookup;
            }
            else if (Options.HasFlag(InputOptions.StripWhitespace))
            {
                stripLookup = WhitespaceCharLookup;
            }
            else if (Options.HasFlag(InputOptions.StripPunctuation))
            {
                stripLookup = PunctuationCharLookup;
            }

            foreach (var stream in Input)
            {
                int segmentIndex = 0;
                var bufferLength = stream.Read(buffer, 0, buffer.Length);
                while (bufferLength > 0)
                {
                    for (int index = 0; index < bufferLength; index++)
                    {
                        var c = buffer[index];
                        if (splitLookup[c])
                        {
                            // trim end
                            for (segmentIndex--; segmentIndex > 0; segmentIndex--)
                            {
                                if (!stripLookup[segment[segmentIndex]])
                                {
                                    break;
                                }
                            }

                            if (segmentIndex++ > 0)
                            {
                                yield return(new ArraySegment <byte>(segment, 0, segmentIndex));
                            }

                            segmentIndex = 0;
                            continue;
                        }
                        if (segmentIndex > 0)
                        {
                            if (segment.Length <= segmentIndex)
                            {
                                Array.Resize(ref segment, segment.Length * 2);
                            }

                            segment[segmentIndex++] = c;
                            continue;
                        }
                        if (stripLookup[c])
                        {
                            continue; // trim start
                        }
                        else
                        {
                            segment[segmentIndex++] = c;
                        }
                    }
                    bufferLength = stream.Read(buffer, 0, buffer.Length);
                }

                if (segmentIndex++ > 0)
                {
                    yield return(new ArraySegment <byte>(segment, 0, segmentIndex));
                }
            }
        }