public async Task <IHttpRequest> Provide(IStreamReader reader) { // parse the http request var request = await reader.ReadLine().ConfigureAwait(false); if (request == null) { return(null); } var firstSpace = request.IndexOf(' '); var lastSpace = request.LastIndexOf(' '); var tokens = new [] { request.Substring(0, firstSpace), request.Substring(firstSpace + 1, lastSpace - firstSpace - 1), request.Substring(lastSpace + 1) }; if (tokens.Length != 3) { return(null); } var httpProtocol = tokens[2]; var url = tokens[1]; var queryString = GetQueryStringData(ref url); var uri = new Uri(url, UriKind.Relative); var headersRaw = new List <KeyValuePair <string, string> >(); // get the headers string line; while (!string.IsNullOrEmpty((line = await reader.ReadLine().ConfigureAwait(false)))) { string currentLine = line; var headerKvp = SplitHeader(currentLine); headersRaw.Add(headerKvp); } IHttpHeaders headers = new HttpHeaders(headersRaw.ToDictionary(k => k.Key, k => k.Value, StringComparer.InvariantCultureIgnoreCase)); IHttpPost post = await GetPostData(reader, headers).ConfigureAwait(false); string verb; if (!headers.TryGetByName("_method", out verb)) { verb = tokens[0]; } var httpMethod = HttpMethodProvider.Default.Provide(verb); return(new HttpRequest(headers, httpMethod, httpProtocol, uri, uri.OriginalString.Split(Separators, StringSplitOptions.RemoveEmptyEntries), queryString, post)); }
public async Task<IHttpRequest> Provide(IStreamReader reader) { // parse the http request var request = await reader.ReadLine().ConfigureAwait(false); if (request == null) return null; var firstSpace = request.IndexOf(' '); var lastSpace = request.LastIndexOf(' '); var tokens = new [] { request.Substring(0, firstSpace), request.Substring(firstSpace + 1, lastSpace - firstSpace - 1), request.Substring(lastSpace + 1) }; if (tokens.Length != 3) { return null; } var httpProtocol = tokens[2]; var url = tokens[1]; var queryString = GetQueryStringData(ref url); var uri = new Uri(url, UriKind.Relative); var headersRaw = new List<KeyValuePair<string, string>>(); // get the headers string line; while (!string.IsNullOrEmpty((line = await reader.ReadLine().ConfigureAwait(false)))) { string currentLine = line; var headerKvp = SplitHeader(currentLine); headersRaw.Add(headerKvp); } IHttpHeaders headers = new HttpHeaders(headersRaw.ToDictionary(k => k.Key, k => k.Value, StringComparer.InvariantCultureIgnoreCase)); IHttpPost post = await GetPostData(reader, headers).ConfigureAwait(false); string verb; if (!headers.TryGetByName("_method", out verb)) { verb = tokens[0]; } var httpMethod = HttpMethodProvider.Default.Provide(verb); return new HttpRequest(headers, httpMethod, httpProtocol, uri, uri.OriginalString.Split(Separators, StringSplitOptions.RemoveEmptyEntries), queryString, post); }
private void ReadDimensions(MazeParams mazeParams) { try { var dimensions = streamReader.ReadLine().Split(','); mazeParams.Width = Int32.Parse(dimensions[0]); mazeParams.Length = Int32.Parse(dimensions[1]); ReadDummySectionEndDelimeter(); } catch (SystemException) { Console.WriteLine("Exception while reading maze dimensions"); } }
public string ReadTrimmedStatement() { if ((_currentStatement = _streamReader.ReadLine()) == null) { return(_currentStatement); } _currentStatement = _statementTrimmer.Trim(_currentStatement); if (_multiLineStatementVerifier.IsMultiLineStatement(_currentStatement)) { ReadTillStatementEnd(); } return(_currentStatement); }
public List <Driver> RegisterDrivers(IStreamReader reader) { const string driverCommand = "driver"; string line; string[] input; List <Driver> drivers = new List <Driver>(); while (!reader.EndOfStream) { line = reader.ReadLine(); if (!string.IsNullOrEmpty(line) && line.ToLower().Contains(driverCommand)) { input = Regex.Split(line, @"\s+"); string driverName = input[1]; Driver driver = new Driver() { Name = driverName }; drivers.Add(driver); } } return(drivers); }
public void RegisterTrips_skips_TripEntry_if_DriverDoesNotExist() { // arrange List <Driver> drivers = new List <Driver> { new Driver() { Name = "Dan" }, new Driver() { Name = "Lauren" }, new Driver() { Name = "Kumi" } }; IStreamReader reader = Substitute.For <IStreamReader>(); reader.CreateReader(Arg.Any <string>()); reader.EndOfStream.Returns(false, true); reader.ReadLine().Returns("Trip Bob 07:15 07:45 17.3"); TripLogic tripLogic = new TripLogic(); // act List <Trip> trips = tripLogic.RegisterTrips(reader, drivers); // assert Assert.IsFalse(trips.Any()); }
public Input Provide(string arg) { arg = arg?.Trim() ?? string.Empty; if (!_fileProcessor.Exists(arg)) { throw new FileNotFoundException(arg); } var input = new Input(); _streamReader.TargetFile(arg); if (_streamReader.EndOfStream()) { throw new InputNotValidException("File should not be empty"); } string surfaceParameter = _streamReader.ReadLine(); input.SurfaceParameter = surfaceParameter.Trim(); while (!_streamReader.EndOfStream()) { string vehicleParameter = _streamReader.ReadLine()?.Trim(); if (string.IsNullOrEmpty(vehicleParameter)) { break; } if (_streamReader.EndOfStream()) { throw new InputNotValidException("Each vehicle should contains at least one move command"); } string moveCommandsParameter = _streamReader.ReadLine()?.Trim(); if (string.IsNullOrEmpty(moveCommandsParameter)) { throw new InputNotValidException("Each vehicle should contains at least one move command"); } var vehicleAndMoveCommandsPair = new Tuple <string, string>(vehicleParameter, moveCommandsParameter); input.VehicleAndCommandsParameterList.Add(vehicleAndMoveCommandsPair); } return(input); }
public string ReadLineWithEofLine() { var line = _streamReader.ReadLine(); if (line == null && !_isEndReached) { line = MarkerLines.EndOfFile; _isEndReached = true; } if (line != null) { ++CurrentLineNumber; } return(line); }
/// <summary> /// Gets the rows from the CSV stream reader as indexed dictionaries. /// </summary> /// <param name="reader">The CSV reader.</param> /// <returns></returns> public IEnumerable <CsvLine> GetRows(IStreamReader reader) { if (!this.FirstRowContainsHeaders && this.headers.Count() == 0) { throw new InvalidOperationException( "FirstRowContainsHeaders is set False while headers are empty. Headers are required for indexing."); } var isFirstTime = true; while (!reader.EndOfStream) { if (this.FirstRowContainsHeaders && isFirstTime) { this.headers = new HeaderCollection(reader.ReadLine().Elements); isFirstTime = false; } else { var line = reader.ReadLine(); if (!line.IsEmpty) { if (definition.ElementProcessing == ElementProcessing.Strict) { this.ValidateTooManyColumns(line); this.ValidateTooLessColumns(line); } else if (definition.ElementProcessing == ElementProcessing.OnlyTooFew) { this.ValidateTooLessColumns(line); } else if (definition.ElementProcessing == ElementProcessing.OnlyTooMany) { this.ValidateTooManyColumns(line); } yield return(this.CreateDictionaryLine(line)); } } } }
private void ReadLines(char delimiter, List <Person> persons) { string line; while ((line = _streamReaderWrapper.ReadLine()) != null) { string[] parsedRecord = line.Split(delimiter); CheckArraySize(parsedRecord); Person person = GetPerson(parsedRecord); persons.Add(person); } }
public static void WriteRecordToStream(StreamWriter writer, IStreamReader reader, Record record) { Int64 position = reader.Position; reader.Position = record.Start; while (reader.Position < record.End) { writer.WriteLine(reader.ReadLine()); } reader.Position = position; }
/// <summary> /// Reads a set of words from a stream, and adds them to the trie with the specified root. /// </summary> /// <param name="trieRoot">The root of the trie.</param> /// <param name="reader">The stream reader to used to read the set of words (allows specifying a mocked reader for unit testing).</param> /// <param name="trieBuilder">The trie builder to use to add the words to the trie (allows specifying a mocked builder for unit testing).</param> /// <param name="wordFilterFunction">A Func to filter whether or not the specified word should be added to the trie. Accepts the word as a parameter, and returns a bookean indicating whether that word should be added to the trie.</param> public void LoadWordsFromFile(Dictionary <Char, TrieNode <Char> > trieRoot, IStreamReader reader, ICharacterTrieBuilder trieBuilder, Func <String, Boolean> wordFilterFunction) { using (reader) { while (reader.EndOfStream == false) { String currentWord = reader.ReadLine(); if (wordFilterFunction.Invoke(currentWord) == true) { trieBuilder.AddWord(trieRoot, currentWord, true); } } } }
public void RegisterDriver_returns_NonEmptyDrivers() { // arrange IStreamReader reader = Substitute.For <IStreamReader>(); reader.CreateReader(Arg.Any <string>()); reader.EndOfStream.Returns(false, true); reader.ReadLine().Returns("Driver Test"); TripLogic tripLogic = new TripLogic(); // act List <Driver> drivers = tripLogic.RegisterDrivers(reader); // assert Assert.IsTrue(drivers.Any()); }
public List <Trip> RegisterTrips(IStreamReader reader, List <Driver> drivers) { const string tripCommand = "trip"; string line; string[] input; List <Trip> trips = new List <Trip>(); reader.SetPosition(1); while (!reader.EndOfStream) { line = reader.ReadLine(); if (!string.IsNullOrEmpty(line) && line.ToLower().Contains(tripCommand)) { input = Regex.Split(line, @"\s+"); string driverName = input[1]; TimeSpan startTime = TimeSpan.Parse(input[2], CultureInfo.CurrentCulture); TimeSpan endTime = TimeSpan.Parse(input[3], CultureInfo.CurrentCulture); decimal milesDriven = decimal.Parse(input[4]); Driver currentDriver = drivers.FirstOrDefault(x => x.Name != null && x.Name.Equals(driverName, StringComparison.CurrentCultureIgnoreCase)); if (currentDriver == null || (startTime > endTime)) { continue; } Trip trip = new Trip() { Driver = currentDriver, StartTime = startTime, EndTime = endTime, MilesDriven = milesDriven }; trips.Add(trip); } } return(trips); }
public void Start(CancellationToken token) { IStreamReader streamReader = GetStreamReader(); while (token.IsCancellationRequested == false) { try { string value = streamReader.ReadLine(); _logger.Trace($"Received: {value}"); ReceivedEventArgs args = new ReceivedEventArgs(value); OnReaded(args); } catch { streamReader = GetStreamReader(); } } }
public Record ReadRecord(IStreamReader reader) { Record record = null; var lineIDStart = (Int32)this.descriptor.LineIDStart; var lineIDLength = (Int32)this.descriptor.LineIDLength; var termStart = (Int32)this.descriptor.Term.Start; var termLength = (Int32)this.descriptor.Term.Length; while (!reader.EndOfStream) { Int64 position = reader.Position; var line = reader.ReadLine(); var lineID = line.Substring(lineIDStart, lineIDLength); if (lineID == descriptor.HeaderID) { if (record == null) { record = new Record { Start = position }; } else { record.End = position; reader.Position = position; return record; } } if (lineID == descriptor.Term.LineID) { record.Term = line.Substring(termStart, termLength); } } if (record != null) { record.End = reader.Position; } return record; }
public Record ReadRecord(IStreamReader streamReader) { Record record = null; while (!streamReader.EndOfStream) { Int64 position = streamReader.Position; var line = streamReader.ReadLine(); var lineIDTerm = line.ExtractField(this.descriptor.Delimiter, this.descriptor.Qualifier, this.descriptor.LineIDIndex); if (lineIDTerm == this.descriptor.HeaderID) { if (record == null) { record = new Record { Start = position }; } else { record.End = position; streamReader.Position = position; return record; } } if (lineIDTerm != this.descriptor.Term.LineID) { continue; } record.Term = line.ExtractField(this.descriptor.Delimiter, this.descriptor.Qualifier, this.descriptor.Term.Index); } if (record != null) { record.End = streamReader.Position; } return record; }
public void RegisterTrips_returns_CorrectTripDataForDriver() { // arrange List <Driver> drivers = new List <Driver> { new Driver() { Name = "Dan" } }; IStreamReader reader = Substitute.For <IStreamReader>(); reader.CreateReader(Arg.Any <string>()); reader.EndOfStream.Returns(false, true); reader.ReadLine().Returns("Trip Dan 07:15 07:45 17.3"); var expectedTripData = new Trip() { Driver = drivers[0], StartTime = TimeSpan.Parse("07:15"), EndTime = TimeSpan.Parse("07:45"), MilesDriven = (decimal)17.3 }; TripLogic tripLogic = new TripLogic(); // act List <Trip> trips = tripLogic.RegisterTrips(reader, drivers); // assert Assert.AreEqual(expectedTripData.Driver.Name, trips[0].Driver.Name); Assert.AreEqual(expectedTripData.StartTime, trips[0].StartTime); Assert.AreEqual(expectedTripData.EndTime, trips[0].EndTime); Assert.AreEqual(expectedTripData.MilesDriven, trips[0].MilesDriven); }
/// <summary> /// Reads a set of words from a stream, and adds the words and statistics pertaining to the words to a set of data structures. /// </summary> /// <param name="reader">The stream reader to used to read the set of words (allows specifying a mocked reader for unit testing).</param> /// <param name="trieBuilder">The trie builder to use to add the words to the trie (allows specifying a mocked builder for unit testing).</param> /// <param name="wordFilterFunction">A Func to filter whether or not the specified word should be added to the trie. Accepts the word as a parameter, and returns a bookean indicating whether that word should be added to the trie.</param> /// <param name="allWordsTrieRoot">The root of a character trie to populate with the words.</param> /// <param name="allWords">A HashSet to populate with the words.</param> /// <param name="fromCharacterFrequencies">A FrequencyTable to populate with the number of times each character is the 'from' character in a substitution.</param> /// <param name="characterSubstitutionFrequencies">A FrequencyTable to populate with the number of times each pair of characters in a substitution occur.</param> public void PopulateAdjacentWordDataStructures(IStreamReader reader, ICharacterTrieBuilder trieBuilder, Func <String, Boolean> wordFilterFunction, Dictionary <Char, TrieNode <Char> > allWordsTrieRoot, HashSet <String> allWords, FrequencyTable <Char> fromCharacterFrequencies, FrequencyTable <CharacterSubstitution> characterSubstitutionFrequencies) { // Read all words and add them to the HashSet and trie using (reader) { while (reader.EndOfStream == false) { String currentWord = reader.ReadLine(); if (wordFilterFunction.Invoke(currentWord) == true) { if (allWords.Contains(currentWord) == false) { allWords.Add(currentWord); trieBuilder.AddWord(allWordsTrieRoot, currentWord, true); } } } } // Populate the frequency tables CharacterTrieUtilities trieUtilities = new CharacterTrieUtilities(); WordUtilities wordUtilities = new WordUtilities(); foreach (String currentWord in allWords) { foreach (String adjacentWord in trieUtilities.FindAdjacentWords(allWordsTrieRoot, currentWord)) { // Find the character which was substitued between the word and the adjacent word Tuple <Char, Char> differingCharacters = wordUtilities.FindDifferingCharacters(currentWord, adjacentWord); Char fromCharacter = differingCharacters.Item1, toCharacter = differingCharacters.Item2; // Increment the data structures fromCharacterFrequencies.Increment(fromCharacter); characterSubstitutionFrequencies.Increment(new CharacterSubstitution(fromCharacter, toCharacter)); } } }
/******************************************************** * CLASS METHODS *********************************************************/ /// <summary> /// Try and create a new instance of the object and return it using the MoveNext enumeration pattern ("Current" public variable). /// </summary> /// <remarks>This is a highly called method and should be kept lean as possible.</remarks> /// <returns>Boolean true on successful move next. Set Current public property.</returns> public bool MoveNext() { // yield the aux data first if (AuxiliaryData.Count != 0) { Previous = Current; Current = AuxiliaryData.Dequeue(); return(true); } BaseData instance = null; var instanceMarketOpen = false; //Log.Debug("SubscriptionDataReader.MoveNext(): Starting MoveNext..."); try { //Calls this when no file, first "moveNext()" in refresh source. if (_endOfStream || _reader == null || _reader.EndOfStream) { if (_reader == null) { //Handle the 1% of time:: getReader failed e.g. missing day so skip day: Current = null; } else { //This is a MoveNext() after reading the last line of file: _lastBarOfStream = Current; } _endOfStream = true; return(false); } //Log.Debug("SubscriptionDataReader.MoveNext(): Launching While-InstanceNotNull && not EOS: " + reader.EndOfStream); //Keep looking until output's an instance: while (instance == null && !_reader.EndOfStream) { //Get the next string line from file, create instance of BaseData: var line = _reader.ReadLine(); try { instance = _dataFactory.Reader(_config, line, _date, _feedEndpoint); } catch (Exception err) { //Log.Debug("SubscriptionDataReader.MoveNext(): Error invoking instance: " + err.Message); Engine.ResultHandler.RuntimeError("Error invoking " + _config.Symbol + " data reader. Line: " + line + " Error: " + err.Message, err.StackTrace); _endOfStream = true; continue; } if (instance != null) { instanceMarketOpen = _security.Exchange.DateTimeIsOpen(instance.Time); //Apply custom user data filters: try { if (!_security.DataFilter.Filter(_security, instance)) { instance = null; continue; } } catch (Exception err) { Log.Error("SubscriptionDataReader.MoveNext(): Error applying filter: " + err.Message); Engine.ResultHandler.RuntimeError("Runtime error applying data filter. Assuming filter pass: "******"SubscriptionDataReader.MoveNext(): Instance null, continuing..."); continue; } //Check if we're in date range of the data request if (instance.Time < _periodStart) { _lastBarOutsideMarketHours = instance; instance = null; continue; } if (instance.Time > _periodFinish) { instance = null; continue; } //Save bar for extended market hours (fill forward). if (!instanceMarketOpen) { _lastBarOutsideMarketHours = instance; } //However, if we only want market hours data, don't return yet: Discard and continue looping. if (!_config.ExtendedMarketHours && !instanceMarketOpen) { instance = null; } } } //Handle edge conditions: First Bar Read: // -> Use previous bar from yesterday if available if (Current == null) { //Handle first loop where not set yet: if (_lastBarOfStream == null) { //For first bar, fill forward from premarket data where possible _lastBarOfStream = _lastBarOutsideMarketHours ?? instance; } //If current not set yet, set Previous to yesterday/last bar read. Previous = _lastBarOfStream; } else { Previous = Current; } Current = instance; //End of Stream: rewind reader to last if (_reader.EndOfStream && instance == null) { //Log.Debug("SubscriptionDataReader.MoveNext(): Reader EOS."); _endOfStream = true; if (_isFillForward && Previous != null) { //If instance == null, current is null, so clone previous to record the final sample: Current = Previous.Clone(true); //When market closes fastforward current bar to the last bar fill forwarded to close time. Current.Time = _security.Exchange.TimeOfDayClosed(Previous.Time); // Save the previous bar as last bar before next stream (for fill forwrd). _lastBarOfStream = Previous; } return(false); } return(true); } catch (Exception err) { Log.Error("SubscriptionDataReader.MoveNext(): " + err.Message); return(false); } }
/// <summary> /// Gets the next line/batch of content from the stream /// </summary> public string ReadLine() { return(_streamReader.ReadLine()); }
/// <summary> /// Parses the data. /// </summary> /// <param name="streamReader">The stream reader.</param> private void ParseData(IStreamReader streamReader) { // ProcessDate:2014-01-05%TotalItems:5%TotalPrice:160.00%PackageNumber:1%Order:Food.Dogfood%Price:10.50 string dataLine; do { dataLine = streamReader.ReadLine(); _logger.LogInfo("Parsing: " + dataLine); if (dataLine != null) { string[] allDataFromALine = dataLine.Split('%'); // Parse data DateTime processDateFromDataline = DateTime.Parse(GetDataFromInfoArray("ProcessDate", allDataFromALine)); TotalNumberOfOrdersFromInput = int.Parse(GetDataFromInfoArray("TotalItems", allDataFromALine)); TotalPriceOfOrdersFromInput = double.Parse(GetDataFromInfoArray("TotalPrice", allDataFromALine)); int packageNumberFromDataline = int.Parse(GetDataFromInfoArray("PackageNumber", allDataFromALine)); string orderNameFromDataline = GetDataFromInfoArray("Order", allDataFromALine); double priceNameFromDataline = double.Parse(GetDataFromInfoArray("Price", allDataFromALine)); // Find existing Package Package package = _packages.SingleOrDefault(p => p.PackageNumber == packageNumberFromDataline); if (package == null) { package = new Package(packageNumberFromDataline, DateTime.Now); Packages.Add(package); } // Add the order to the package Order newOrder = new Order(orderNameFromDataline, priceNameFromDataline); package.Orders.Add(newOrder); // Set ProcessDate ProcessDate = processDateFromDataline; } } while (dataLine != null); // Checksum data foreach (var package in Packages) { TotalNumberOfOrders += package.Orders.Count; TotalPriceOfOrders += package.Orders.Sum(o => o.Price); TotalPriceOfOrdersWithDiscount += package.TotalOrderPriceWithDiscount; } if (TotalNumberOfOrders != TotalNumberOfOrdersFromInput) { Console.Out.WriteLine("Total Orders expected: {0}, actual: {1}", TotalNumberOfOrdersFromInput, TotalNumberOfOrders); } else { Console.Out.WriteLine("Checksum total # of orders are oke."); } if (TotalPriceOfOrders != TotalPriceOfOrdersFromInput) { Console.Out.WriteLine("Total price of shipment expected: {0:C}, actual: {1:C}", TotalPriceOfOrdersFromInput, TotalPriceOfOrders); } else { Console.Out.WriteLine("Checksum total price is oke."); } Console.Out.WriteLine("Total price (with Discount): {0:C}", TotalPriceOfOrdersWithDiscount); Console.Out.WriteLine("Total price (without Discount): {0:C}", TotalPriceOfOrders); }
/// <summary> /// Reads the specified <paramref name="source"/> /// </summary> /// <param name="source">The source to be read</param> /// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns> public override IEnumerable <BaseData> Read(SubscriptionDataSource source) { SubscriptionDataSourceReader.CheckRemoteFileCache(); IStreamReader reader = null; try { reader = CreateStreamReader(source); if (reader == null) { yield break; } var raw = ""; while (!reader.EndOfStream) { BaseDataCollection instances = null; try { raw = reader.ReadLine(); var result = _factory.Reader(_config, raw, _date, IsLiveMode); instances = result as BaseDataCollection; if (instances == null && !reader.ShouldBeRateLimited) { OnInvalidSource(source, new Exception("Reader must generate a BaseDataCollection with the FileFormat.Collection")); continue; } } catch (Exception err) { OnReaderError(raw, err); if (!reader.ShouldBeRateLimited) { continue; } } if (IsLiveMode // this shouldn't happen, rest reader is the only one to be rate limited // and in live mode, but just in case... || instances == null && reader.ShouldBeRateLimited) { // in live trading these data points will be unrolled at the // 'LiveCustomDataSubscriptionEnumeratorFactory' level yield return(instances); } else { foreach (var instance in instances.Data) { if (instance != null && instance.EndTime != default(DateTime)) { yield return(instance); } } } } } finally { reader.DisposeSafely(); } }