public ICFTCParserResult Parse(ICFTCParserParams parserParams)
{
ICFTCParserResult result = new CFTCParserResult();
// preparing list of files to be downloaded and parsed
List <string> files = new List <string>();
files.Add(parserParams.RecentDataPath);
if (!parserParams.OnlyLast)
{
files.AddRange(parserParams.HistorticalDataPaths);
}
foreach (var path in files)
{
try
{
string url = string.Format("https://www.cftc.gov{0}", path);
// downloading
byte[] data = ServiceCall(url);
if (data != null)
{
Stream dataStream = Decompress(data);
data = null;
System.GC.Collect();
if (dataStream != null)
{
StreamReader sr = new StreamReader(dataStream);
string txt = sr.ReadToEnd();
ParseCSV(txt, true, parserParams, result);
dataStream.Close();
}
}
}
catch (Exception ex)
{
// suppressing errors in case if one of the files was not parsed
}
}
return(result);
}
public IQuotesSourceGetQuotesResult GetQuotes(IQuotesSourceGetQuotesParams getQuotesParams)
{
// For COT reports we don't extract selected items only because it will be too expensive
// Rather we're loading the whole report with all items there
// For this purposes we only identifying the types of items we need to extract based on indicator prefixes
IQuotesSourceGetQuotesResult result = new CFTCSourceGetQuotesResult();
CFTCParser cftcParser = new CFTCParser();
// checking which types we need and preparing parameters
List <ICFTCParserParams> parserParams = new List <ICFTCParserParams>();
foreach (var pt in s_paramTypes)
{
ICFTCParserParams cotTypeParams = null;
// at least one ticker has a prefix of given type - adding corresponding params object to parse proper report
// if no tickers are specified - importing all
if ((getQuotesParams.Tickers == null || getQuotesParams.Tickers.Count() == 0) || getQuotesParams.Tickers.Count(x => x.Contains(pt.TickerPrefix)) > 0)
{
cotTypeParams = pt.Clone();
cotTypeParams.OnlyLast = getQuotesParams.PeriodStart.Year == DateTime.Now.Year ? true : false;
parserParams.Add(cotTypeParams);
}
}
// for the list of parameters - calling parser to load proper report
foreach (var parserParam in parserParams)
{
ICFTCParserResult parserResult = cftcParser.Parse(parserParam);
foreach (CFTCInstrumentQuotes i in parserResult.Instruments.Values)
{
IQuotesData qd = new BaseQuotesData();
qd.Country = getQuotesParams.Country;
qd.Ticker = i.Ticker;
qd.Name = i.Description;
qd.Unit = TickerUnit(i.Ticker);
qd.Type = TickerType(i.Ticker);
qd.TimeFrame = ETimeFrame.Weekly;
foreach (var q in i.Quotes)
{
ITimeSeriesRecord tsr = new CustomTimeseriesRecord(i.Timeseries, q.ReportDate, q.Values);
qd.AddRecord(tsr);
}
qd.AgencyCode = s_agencyCode;
result.QuotesData.Add(qd);
}
}
result.Success = result.QuotesData.Count > 0;
if (result.Success && result.QuotesData.Count < getQuotesParams.Tickers.Count)
{
result.AddError(Interfaces.EErrorCodes.QuotesNotFound, Interfaces.EErrorType.Warning, "Not all quotes were found");
}
else if (!result.Success)
{
result.AddError(Interfaces.EErrorCodes.QuotesNotFound, Interfaces.EErrorType.Error, "Requested tickers are not supported or quotes for them not found");
}
return(result);
}
private void ParseCSV(string text, bool header, ICFTCParserParams parserParams, ICFTCParserResult result)
{
CFTCInstrumentQuotes curInstrument = null;
int colName = 0;
int colReportDate = 2;
int colCFTCMarketCode = 3;
StringReader sr = new StringReader(text);
string line = null;
bool headerSkipped = false;
do
{
line = sr.ReadLine();
if (line != null && header && !headerSkipped)
{
headerSkipped = true;
}
else if (line != null)
{
string[] vals = parserParams.SplitLine(line.Trim());
if (vals.Count() >= parserParams.ColumnsToTimeseriesMapping.Keys.Max() + 1)
{
// reading common values
CultureInfo enUS = new CultureInfo("en-US");
DateTime repDate;
// WARNING! There can be different time formats in the files
if (!DateTime.TryParse(vals[colReportDate], out repDate) &&
!DateTime.TryParseExact(vals[colReportDate], "M/dd/yyyy hh:mm:ss tt", enUS, DateTimeStyles.None, out repDate))
{
// fixing line with bad comma in description
int badComma = line.IndexOf(',');
StringBuilder sb = new StringBuilder(line);
sb[badComma] = ':';
vals = parserParams.SplitLine(sb.ToString().Trim());
DateTime.TryParse(vals[colReportDate], out repDate);
}
string name = vals[colName].Trim(new char[] { '"' });
string marketCode = vals[colCFTCMarketCode];
// checking - if its new instrument
if (curInstrument == null || !curInstrument.Ticker.Equals(parserParams.TickerPrefix + marketCode))
{
curInstrument = new CFTCInstrumentQuotes()
{
Ticker = parserParams.TickerPrefix + marketCode,
Description = name
};
curInstrument.Timeseries.AddRange(parserParams.ColumnsToTimeseriesMapping.Values);
CFTCInstrumentQuotes instrument = null;
if (!result.Instruments.TryGetValue(curInstrument.Ticker, out instrument))
{
result.Instruments.Add(curInstrument.Ticker, curInstrument);
}
else
{
curInstrument = instrument;
}
}
// creating new record
CFTCRecord rec = new CFTCRecord(parserParams.ColumnsToTimeseriesMapping.Count);
rec.ReportDate = repDate;
int i = 0;
foreach (int col in parserParams.ColumnsToTimeseriesMapping.Keys)
{
rec[i] = Decimal.Parse(vals[col]);
++i;
}
curInstrument.Quotes.Add(rec);
}
}
}while (line != null);
}
