Ejemplo n.º 1
0
 private ThreadAheadReadable(CharReadable actual, int bufferSize) : base(actual)
 {
     this._actual            = actual;
     this._theOtherBuffer    = new SectionedCharBuffer(bufferSize);
     this._sourceDescription = actual.SourceDescription();
     start();
 }
Ejemplo n.º 2
0
        /// <summary>
        /// Extracts the first line, i.e characters until the first newline or end of stream.
        /// Reads one character at a time to be sure not to read too far ahead. The stream is left
        /// in a state of either exhausted or at the beginning of the next line of data.
        /// </summary>
        /// <param name="source"> <seealso cref="CharReadable"/> to read from. </param>
        /// <returns> char[] containing characters until the first newline character or end of stream. </returns>
        /// <exception cref="IOException"> on I/O reading error. </exception>
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: public static char[] extractFirstLineFrom(CharReadable source) throws java.io.IOException
        public static char[] ExtractFirstLineFrom(CharReadable source)
        {
            char[] result = new char[100];
            int    cursor = 0;
            int    read;
            bool   foundEol = false;

            do
            {
                // Grow on demand
                if (cursor >= result.Length)
                {
                    result = Arrays.copyOf(result, cursor * 2);
                }

                // Read one character
                read = source.Read(result, cursor, 1);
                if (read > 0)
                {
                    foundEol = BufferedCharSeeker.IsEolChar(result[cursor]);
                    if (!foundEol)
                    {
                        cursor++;
                    }
                }
            } while (read > 0 && !foundEol);
            return(Arrays.copyOf(result, cursor));
        }
Ejemplo n.º 3
0
        private void WarnAboutDuplicateSourceFiles(ISet <string> seenSourceFiles, CharReadable source)
        {
            string sourceDescription = source.SourceDescription();

            if (!seenSourceFiles.Add(sourceDescription))
            {
                _monitor.duplicateSourceFile(sourceDescription);
            }
        }
Ejemplo n.º 4
0
        /// <summary>
        /// Instantiates a <seealso cref="BufferedCharSeeker"/> with optional <seealso cref="ThreadAheadReadable read-ahead"/> capability.
        /// </summary>
        /// <param name="reader"> the <seealso cref="CharReadable"/> which is the source of data, f.ex. a <seealso cref="System.IO.StreamReader_FileReader"/>. </param>
        /// <param name="config"> <seealso cref="Configuration"/> for the resulting <seealso cref="CharSeeker"/>. </param>
        /// <param name="readAhead"> whether or not to start a <seealso cref="ThreadAheadReadable read-ahead thread"/>
        /// which strives towards always keeping one buffer worth of data read and available from I/O when it's
        /// time for the <seealso cref="BufferedCharSeeker"/> to read more data. </param>
        /// <returns> a <seealso cref="CharSeeker"/> with optional <seealso cref="ThreadAheadReadable read-ahead"/> capability. </returns>
        public static CharSeeker CharSeeker(CharReadable reader, Configuration config, bool readAhead)
        {
            if (readAhead)
            {               // Thread that always has one buffer read ahead
                reader = threadAhead(reader, config.BufferSize());
            }

            // Give the reader to the char seeker
            return(new BufferedCharSeeker(new AutoReadingSource(reader, config.BufferSize()), config));
        }
Ejemplo n.º 5
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: private long[] sample(Iterable<DataFactory> dataFactories, Header.Factory headerFactory, System.Func<org.neo4j.values.storable.Value[], int> valueSizeCalculator, System.Func<org.neo4j.unsafe.impl.batchimport.input.InputEntity, int> additionalCalculator) throws java.io.IOException
        private long[] Sample(IEnumerable <DataFactory> dataFactories, Header.Factory headerFactory, System.Func <Value[], int> valueSizeCalculator, System.Func <InputEntity, int> additionalCalculator)
        {
            long[] estimates = new long[4];               // [entity count, property count, property size, labels (for nodes only)]
            using (CsvInputChunkProxy chunk = new CsvInputChunkProxy())
            {
                // One group of input files
                int groupId = 0;
                foreach (DataFactory dataFactory in dataFactories)                           // one input group
                {
                    groupId++;
                    Header header = null;
                    Data   data   = dataFactory.Create(_config);
                    RawIterator <CharReadable, IOException> sources = data.Stream();
                    while (sources.HasNext())
                    {
                        using (CharReadable source = sources.Next())
                        {
                            if (header == null)
                            {
                                // Extract the header from the first file in this group
                                header = extractHeader(source, headerFactory, _idType, _config, _groups);
                            }
                            using (CsvInputIterator iterator = new CsvInputIterator(source, data.Decorator(), header, _config, _idType, EMPTY, extractors(_config), groupId), InputEntity entity = new InputEntity())
                            {
                                int entities     = 0;
                                int properties   = 0;
                                int propertySize = 0;
                                int additional   = 0;
                                while (iterator.Position() < _estimateSampleSize && iterator.Next(chunk))
                                {
                                    for ( ; chunk.Next(entity); entities++)
                                    {
                                        properties   += entity.PropertyCount();
                                        propertySize += calculatePropertySize(entity, valueSizeCalculator);
                                        additional   += additionalCalculator(entity);
                                    }
                                }
                                if (entities > 0)
                                {
                                    long entityCountInSource = ( long )((( double )source.Length() / iterator.Position()) * entities);
                                    estimates[0] += entityCountInSource;
                                    estimates[1] += ( long )((( double )properties / entities) * entityCountInSource);
                                    estimates[2] += ( long )((( double )propertySize / entities) * entityCountInSource);
                                    estimates[3] += ( long )((( double )additional / entities) * entityCountInSource);
                                }
                            }
                        }
                    }
                }
            }
            return(estimates);
        }
Ejemplo n.º 6
0
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void shouldExtractOnlyLine() throws Exception
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
        public virtual void ShouldExtractOnlyLine()
        {
            // given
            string       firstLine = "characters of only line";
            CharReadable reader    = Readables.Wrap(firstLine);

            // when
            char[] firstLineCharacters = Readables.ExtractFirstLineFrom(reader);
            int    readAfterwards      = reader.Read(new char[1], 0, 1);

            // then
            assertArrayEquals(firstLine.ToCharArray(), firstLineCharacters);
            assertEquals(-1, readAfterwards);
        }
Ejemplo n.º 7
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: private void warnAboutDuplicateSourceFiles(java.util.Set<String> seenSourceFiles, Iterable<DataFactory> dataFactories) throws java.io.IOException
        private void WarnAboutDuplicateSourceFiles(ISet <string> seenSourceFiles, IEnumerable <DataFactory> dataFactories)
        {
            foreach (DataFactory dataFactory in dataFactories)
            {
                RawIterator <CharReadable, IOException> stream = dataFactory.Create(_config).stream();
                while (stream.HasNext())
                {
                    using (CharReadable source = stream.Next())
                    {
                        WarnAboutDuplicateSourceFiles(seenSourceFiles, source);
                    }
                }
            }
        }
Ejemplo n.º 8
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: private boolean goToNextSource() throws java.io.IOException
        private bool GoToNextSource()
        {
            if (_actual.hasNext())
            {
                if (_current != null)
                {
                    _previousPosition += _current.position();
                }
                CloseCurrent();
                _current = _actual.next();
                return(true);
            }
            return(false);
        }
Ejemplo n.º 9
0
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test void shouldFailIfNoNewlineInChunk() throws Exception
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
        internal virtual void ShouldFailIfNoNewlineInChunk()
        {
            // GIVEN
            CharReadable reader = Readables.Wrap("1234567\n89012345678901234");

            // (next chunks):                                   ^
            // (actual chunks):                             ^
            using (ClosestNewLineChunker source = new ClosestNewLineChunker(reader, 12))
            {
                // WHEN
                Source_Chunk chunk = source.NewChunk();
                assertTrue(source.NextChunk(chunk));
                assertArrayEquals("1234567\n".ToCharArray(), CharactersOf(chunk));
                assertThrows(typeof(System.InvalidOperationException), () => assertFalse(source.NextChunk(chunk)));
            }
        }
Ejemplo n.º 10
0
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void shouldExtractFirstLine() throws Exception
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
        public virtual void ShouldExtractFirstLine()
        {
            // given
            string       firstLine  = "characters of first line";
            string       secondLine = "here on the second line";
            CharReadable reader     = Readables.Wrap(firstLine + "\n" + secondLine);

            // when
            char[] firstLineCharacters  = Readables.ExtractFirstLineFrom(reader);
            char[] secondLineCharacters = new char[secondLine.Length];
            reader.Read(secondLineCharacters, 0, secondLineCharacters.Length);

            // then
            assertArrayEquals(firstLine.ToCharArray(), firstLineCharacters);
            assertArrayEquals(secondLine.ToCharArray(), secondLineCharacters);
        }
Ejemplo n.º 11
0
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test void shouldHandleReadAheadEmptyData() throws Exception
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
        internal virtual void ShouldHandleReadAheadEmptyData()
        {
            // GIVEN
            TrackingReader actual        = new TrackingReader(0);
            int            bufferSize    = 10;
            CharReadable   aheadReadable = ThreadAheadReadable.ThreadAhead(actual, bufferSize);

            // WHEN
            actual.AwaitCompletedReadAttempts(1);

            // THEN
            SectionedCharBuffer buffer = new SectionedCharBuffer(bufferSize);

            buffer = aheadReadable.Read(buffer, buffer.Front());
            assertEquals(buffer.Pivot(), buffer.Back());
            assertEquals(buffer.Pivot(), buffer.Front());
        }
Ejemplo n.º 12
0
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test void shouldReadAhead() throws Exception
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
        internal virtual void ShouldReadAhead()
        {
            // GIVEN
            TrackingReader      actual      = new TrackingReader(23);
            int                 bufferSize  = 5;
            CharReadable        aheadReader = ThreadAheadReadable.ThreadAhead(actual, bufferSize);
            SectionedCharBuffer buffer      = new SectionedCharBuffer(bufferSize);

            // WHEN starting it up it should read and fill the buffer to the brim
            assertEquals(bufferSize, actual.AwaitCompletedReadAttempts(1));

            // WHEN we read one buffer
            int read = 0;

            buffer = aheadReader.Read(buffer, buffer.Front());
            AssertBuffer(Chars(read, bufferSize), buffer, 0, bufferSize);
            read += buffer.Available();

            // and simulate reading all characters, i.e. back section will be empty in the new buffer
            buffer = aheadReader.Read(buffer, buffer.Front());
            AssertBuffer(Chars(read, bufferSize), buffer, 0, bufferSize);
            read += buffer.Available();

            // then simulate reading some characters, i.e. back section will contain some characters
            int keep = 2;

            buffer = aheadReader.Read(buffer, buffer.Front() - keep);
            AssertBuffer(Chars(read - keep, bufferSize + keep), buffer, keep, bufferSize);
            read += buffer.Available();

            keep   = 3;
            buffer = aheadReader.Read(buffer, buffer.Front() - keep);
            AssertBuffer(Chars(read - keep, bufferSize + keep), buffer, keep, bufferSize);
            read += buffer.Available();

            keep   = 1;
            buffer = aheadReader.Read(buffer, buffer.Front() - keep);
            assertEquals(3, buffer.Available());
            AssertBuffer(Chars(read - keep, buffer.Available() + keep), buffer, keep, 3);
            read += buffer.Available();
            assertEquals(23, read);
        }
Ejemplo n.º 13
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: private void shouldComplyWithSpecifiedCharset(java.nio.charset.Charset charset) throws Exception
        private void ShouldComplyWithSpecifiedCharset(Charset charset)
        {
            // GIVEN
            string data = "abcåäö[]{}";
            File   file = WriteToFile(data, charset);

            // WHEN
            CharReadable        reader = Readables.Files(charset, file);
            SectionedCharBuffer buffer = new SectionedCharBuffer(100);

            buffer = reader.Read(buffer, buffer.Front());

            // THEN
            char[] expected = data.ToCharArray();
            char[] array    = buffer.Array();
            assertEquals(expected.Length, buffer.Available());
            for (int i = 0; i < expected.Length; i++)
            {
                assertEquals(expected[i], array[buffer.Pivot() + i]);
            }
        }
Ejemplo n.º 14
0
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test void shouldBackUpChunkToClosestNewline() throws Exception
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
        internal virtual void ShouldBackUpChunkToClosestNewline()
        {
            // GIVEN
            CharReadable reader = Readables.Wrap("1234567\n8901234\n5678901234");

            // (next chunks):                                   ^            ^
            // (actual chunks):                             ^        ^
            using (ClosestNewLineChunker source = new ClosestNewLineChunker(reader, 12))
            {
                // WHEN
                Source_Chunk chunk = source.NewChunk();
                assertTrue(source.NextChunk(chunk));
                assertArrayEquals("1234567\n".ToCharArray(), CharactersOf(chunk));
                assertTrue(source.NextChunk(chunk));
                assertArrayEquals("8901234\n".ToCharArray(), CharactersOf(chunk));
                assertTrue(source.NextChunk(chunk));
                assertArrayEquals("5678901234".ToCharArray(), CharactersOf(chunk));

                // THEN
                assertFalse(source.NextChunk(chunk));
            }
        }
Ejemplo n.º 15
0
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void shouldTrackPosition() throws Exception
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
        public virtual void ShouldTrackPosition()
        {
            // GIVEN
            string data = "1234567890";
            //                 ^   ^
            CharReadable        reader = Readables.Wrap(data);
            SectionedCharBuffer buffer = new SectionedCharBuffer(4);

            // WHEN
            int expected = 0;

            do
            {
                buffer    = reader.Read(buffer, buffer.Front());
                expected += buffer.Available();

                // THEN
                assertEquals(expected, reader.Position());
            } while (buffer.HasAvailable());

            // and THEN
            assertEquals(data.ToCharArray().length, expected);
        }
Ejemplo n.º 16
0
 public static CharReadable ThreadAhead(CharReadable actual, int bufferSize)
 {
     return(new ThreadAheadReadable(actual, bufferSize));
 }
Ejemplo n.º 17
0
 public ClosestNewLineChunker(CharReadable reader, int chunkSize) : base(reader, chunkSize)
 {
 }
Ejemplo n.º 18
0
 internal TrackingReader(int length)
 {
     this.Bytes  = length * 2;
     this.Actual = Readables.Wrap(new CharArrayReader(Chars(0, length)), length * 2);
 }
Ejemplo n.º 19
0
 public CharReadableChunker(CharReadable reader, int chunkSize)
 {
     this.Reader      = reader;
     this.ChunkSize   = chunkSize;
     this._backBuffer = new char[chunkSize >> 4];
 }
Ejemplo n.º 20
0
 public AutoReadingSource(CharReadable reader, SectionedCharBuffer charBuffer)
 {
     this._reader     = reader;
     this._charBuffer = charBuffer;
 }
Ejemplo n.º 21
0
 public AutoReadingSource(CharReadable reader, int bufferSize) : this(reader, new SectionedCharBuffer(bufferSize))
 {
 }
Ejemplo n.º 22
0
        /// <summary>
        /// Instantiates a <seealso cref="BufferedCharSeeker"/> with optional <seealso cref="ThreadAheadReadable read-ahead"/> capability.
        /// </summary>
        /// <param name="reader"> the <seealso cref="CharReadable"/> which is the source of data, f.ex. a <seealso cref="System.IO.StreamReader_FileReader"/>. </param>
        /// <param name="bufferSize"> buffer size of the seeker and, if enabled, the read-ahead thread. </param>
        /// <param name="readAhead"> whether or not to start a <seealso cref="ThreadAheadReadable read-ahead thread"/>
        /// which strives towards always keeping one buffer worth of data read and available from I/O when it's
        /// time for the <seealso cref="BufferedCharSeeker"/> to read more data. </param>
        /// <param name="quotationCharacter"> character to interpret quotation character. </param>
        /// <returns> a <seealso cref="CharSeeker"/> with optional <seealso cref="ThreadAheadReadable read-ahead"/> capability. </returns>
//JAVA TO C# CONVERTER WARNING: 'final' parameters are ignored unless the option to convert to C# 7.2 'in' parameters is selected:
//ORIGINAL LINE: public static CharSeeker charSeeker(CharReadable reader, final int bufferSize, boolean readAhead, final char quotationCharacter)
        public static CharSeeker CharSeeker(CharReadable reader, int bufferSize, bool readAhead, char quotationCharacter)
        {
            return(charSeeker(reader, new Configuration_OverriddenAnonymousInnerClass(DEFAULT, bufferSize, quotationCharacter)
                              , readAhead));
        }
Ejemplo n.º 23
0
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: private void assertReadText(CharReadable readable, String text) throws java.io.IOException
        private void AssertReadText(CharReadable readable, string text)
        {
            char[] readText = ReadMethod.read(readable, text.ToCharArray().length);
            assertArrayEquals(readText, text.ToCharArray());
        }