// parse a single file
private void parse(CharSource charSource, ListMultimap <string, CurveSensitivities> parsed, IList <FailureItem> failures)
{
try
{
using (CsvIterator csv = CsvIterator.of(charSource, true))
{
if (!csv.containsHeader(TENOR_HEADER) && !csv.containsHeader(DATE_HEADER))
{
failures.Add(FailureItem.of(FailureReason.PARSING, "CSV file could not be parsed as sensitivities, invalid format"));
}
else if (csv.containsHeader(REFERENCE_HEADER) && csv.containsHeader(TYPE_HEADER) && csv.containsHeader(VALUE_HEADER))
{
parseStandardFormat(csv, parsed, failures);
}
else if (csv.containsHeader(REFERENCE_HEADER))
{
parseListFormat(csv, parsed, failures);
}
else
{
parseGridFormat(csv, parsed, failures);
}
}
}
catch (Exception ex)
{
failures.Add(FailureItem.of(FailureReason.PARSING, ex, "CSV file could not be parsed: {}", ex.Message));
}
}
//-------------------------------------------------------------------------
// parses the file in grid format
private void parseGridFormat(CsvIterator csv, ListMultimap <string, CurveSensitivities> parsed, IList <FailureItem> failures)
{
// find the applicable reference columns
IDictionary <string, CurveName> references = new LinkedHashMap <string, CurveName>();
foreach (string header in csv.headers())
{
string headerLowerCase = header.ToLower(Locale.ENGLISH);
if (!REF_HEADERS.contains(headerLowerCase) && !resolver.isInfoColumn(headerLowerCase))
{
references[header] = CurveName.of(header);
}
}
// loop around all rows, peeking to match batches with the same identifier
// no exception catch at this level to avoid infinite loops
while (csv.hasNext())
{
CsvRow peekedRow = csv.peek();
PortfolioItemInfo info = parseInfo(peekedRow);
//JAVA TO C# CONVERTER TODO TASK: Method reference arbitrary object instance method syntax is not converted by Java to C# Converter:
string id = info.Id.map(StandardId::toString).orElse("");
// process in batches, where the ID is the same
CurveSensitivitiesBuilder builder = CurveSensitivities.builder(info);
IList <CsvRow> batchRows = csv.nextBatch(r => matchId(r, id));
foreach (CsvRow batchRow in batchRows)
{
try
{
ParameterMetadata metadata = parseMetadata(batchRow, true);
CurveSensitivitiesType type = batchRow.findValue(TYPE_HEADER).map(str => CurveSensitivitiesType.of(str)).orElse(CurveSensitivitiesType.ZERO_RATE_DELTA);
foreach (KeyValuePair <string, CurveName> entry in references.SetOfKeyValuePairs())
{
CurveName reference = entry.Value;
CurveName resolvedCurveName = resolver.checkCurveName(reference);
string valueStr = batchRow.getField(entry.Key);
Currency currency = parseCurrency(batchRow, reference);
if (valueStr.Length > 0)
{
double value = LoaderUtils.parseDouble(valueStr);
builder.add(type, resolvedCurveName, currency, metadata, value);
}
}
}
catch (System.ArgumentException ex)
{
failures.Add(FailureItem.of(PARSING, "CSV file could not be parsed at line {}: {}", batchRow.lineNumber(), ex.Message));
}
}
CurveSensitivities sens = builder.build();
if (!sens.TypedSensitivities.Empty)
{
parsed.put(sens.Id.map(object.toString).orElse(""), sens);
}
}
}
public virtual void test_load_invalidNoType()
{
PositionCsvLoader test = PositionCsvLoader.standard();
ValueWithFailures <IList <Position> > trades = test.parse(ImmutableList.of(CharSource.wrap("Id")));
assertEquals(trades.Failures.size(), 1);
FailureItem failure = trades.Failures.get(0);
assertEquals(failure.Reason, FailureReason.PARSING);
assertEquals(failure.Message.Contains("CSV file does not contain 'Strata Position Type' header"), true);
}
public void test_parse_list_unableToGetCurrency()
{
CharSource source = CharSource.wrap("Reference,Sensitivity Tenor,Zero Rate Delta\n" + "X,1D,1.0");
ValueWithFailures <ListMultimap <string, CurveSensitivities> > test = LOADER.parse(ImmutableList.of(source));
assertEquals(test.Failures.size(), 1);
FailureItem failure0 = test.Failures.get(0);
assertEquals(failure0.Reason, FailureReason.PARSING);
assertEquals(failure0.Message, "CSV file could not be parsed at line 2: Unable to parse currency from reference, " + "consider adding a 'Currency' column");
}
public virtual void test_load_invalidUnknownType()
{
PositionCsvLoader test = PositionCsvLoader.standard();
ValueWithFailures <IList <Position> > trades = test.parse(ImmutableList.of(CharSource.wrap("Strata Position Type\nFoo")));
assertEquals(trades.Failures.size(), 1);
FailureItem failure = trades.Failures.get(0);
assertEquals(failure.Reason, FailureReason.PARSING);
assertEquals(failure.Message, "CSV file position type 'Foo' is not known at line 2");
}
public void test_parse_grid_missingColumns()
{
CharSource source = CharSource.wrap("GBP\n" + "1");
assertEquals(LOADER.isKnownFormat(source), false);
ValueWithFailures <ListMultimap <string, CurveSensitivities> > test = LOADER.parse(ImmutableList.of(source));
assertEquals(test.Failures.size(), 1);
assertEquals(test.Value.size(), 0);
FailureItem failure0 = test.Failures.get(0);
assertEquals(failure0.Reason, FailureReason.PARSING);
assertEquals(failure0.Message, "CSV file could not be parsed as sensitivities, invalid format");
}
public void test_parse_grid_badTenorWithValidDateColumn()
{
CharSource source = CharSource.wrap("Sensitivity Type,Sensitivity Tenor,Sensitivity Date,GBP\n" + "ZeroRateGamma,XXX,2018-06-30,1\n");
assertEquals(LOADER.isKnownFormat(source), true);
ValueWithFailures <ListMultimap <string, CurveSensitivities> > test = LOADER.parse(ImmutableList.of(source));
assertEquals(test.Failures.size(), 1);
assertEquals(test.Value.size(), 0);
FailureItem failure0 = test.Failures.get(0);
assertEquals(failure0.Reason, FailureReason.PARSING);
assertEquals(failure0.Message, "CSV file could not be parsed at line 2: Invalid tenor 'XXX', must be expressed as nD, nW, nM or nY");
}
//-------------------------------------------------------------------------
public void test_parse_list_allRowsBadNoEmptySensitvityCreated()
{
CharSource source = CharSource.wrap("Reference,Sensitivity Tenor,ZeroRateDelta\n" + "GBP,XX,1\n");
assertEquals(LOADER_DATE.isKnownFormat(source), true);
ValueWithFailures <ListMultimap <string, CurveSensitivities> > test = LOADER_DATE.parse(ImmutableList.of(source));
assertEquals(test.Failures.size(), 1);
assertEquals(test.Value.size(), 0);
FailureItem failure0 = test.Failures.get(0);
assertEquals(failure0.Reason, FailureReason.PARSING);
assertEquals(failure0.Message, "CSV file could not be parsed at line 2: Invalid tenor 'XX', must be expressed as nD, nW, nM or nY");
}
public void test_parse_standard_dateInTenorColumn()
{
CharSource source = CharSource.wrap("Reference,Sensitivity Type,Sensitivity Tenor,Value\n" + "GBP,ZeroRateGamma,2018-06-30,1\n");
assertEquals(LOADER_DATE.isKnownFormat(source), true);
ValueWithFailures <ListMultimap <string, CurveSensitivities> > test = LOADER_DATE.parse(ImmutableList.of(source));
assertEquals(test.Failures.size(), 1);
assertEquals(test.Value.size(), 0);
FailureItem failure0 = test.Failures.get(0);
assertEquals(failure0.Reason, FailureReason.PARSING);
assertEquals(failure0.Message, "CSV file could not be parsed at line 2: Invalid tenor '2018-06-30', must be expressed as nD, nW, nM or nY");
}
public void test_parse_grid_dateButTenorRequired()
{
CharSource source = CharSource.wrap("Sensitivity Type,Sensitivity Tenor,Sensitivity Date,GBP\n" + "ZeroRateGamma,,2018-06-30,1\n");
assertEquals(LOADER.isKnownFormat(source), true);
ValueWithFailures <ListMultimap <string, CurveSensitivities> > test = LOADER.parse(ImmutableList.of(source));
assertEquals(test.Failures.size(), 1);
assertEquals(test.Value.size(), 0);
FailureItem failure0 = test.Failures.get(0);
assertEquals(failure0.Reason, FailureReason.PARSING);
assertEquals(failure0.Message, "CSV file could not be parsed at line 2: Missing value for 'Sensitivity Tenor' column");
}
public virtual void test_load_invalidNoQuantity()
{
EtdContractSpecId specId = EtdContractSpecId.of("OG-ETD", "F-ECAG-FGBL");
EtdContractSpec contract = EtdContractSpec.builder().id(specId).type(EtdType.FUTURE).exchangeId(ExchangeIds.ECAG).contractCode(FGBL).description("Dummy").priceInfo(SecurityPriceInfo.of(Currency.GBP, 100)).build();
ReferenceData refData = ImmutableReferenceData.of(specId, contract);
PositionCsvLoader test = PositionCsvLoader.of(refData);
ValueWithFailures <IList <Position> > trades = test.parse(ImmutableList.of(CharSource.wrap("Strata Position Type,Exchange,Contract Code,Expiry\nFUT,ECAG,FGBL,2017-06")));
assertEquals(trades.Failures.size(), 1);
FailureItem failure = trades.Failures.get(0);
assertEquals(failure.Reason, FailureReason.PARSING);
assertEquals(failure.Message, "CSV file position could not be parsed at line 2: " + "Security must contain a quantity column, either 'Quantity' or 'Long Quantity' and 'Short Quantity'");
}
public void test_parse_grid_neitherTenorNorDate()
{
CharSource source = CharSource.wrap("Sensitivity Type,Sensitivity Tenor,Sensitivity Date,GBP\n" + "ZeroRateGamma,,,1\n");
assertEquals(LOADER_DATE.isKnownFormat(source), true);
ValueWithFailures <ListMultimap <string, CurveSensitivities> > test = LOADER_DATE.parse(ImmutableList.of(source));
assertEquals(test.Failures.size(), 1);
assertEquals(test.Value.size(), 0);
FailureItem failure0 = test.Failures.get(0);
assertEquals(failure0.Reason, FailureReason.PARSING);
assertEquals(failure0.Message, "CSV file could not be parsed at line 2: Unable to parse tenor or date, " + "check 'Sensitivity Tenor' and 'Sensitivity Date' columns");
}
/// <summary>
/// Parses one or more CSV format trade files with a quiet type filter.
/// <para>
/// A type is specified to filter the trades.
/// Trades that do not match the type are silently dropped.
/// </para>
/// <para>
/// CSV files sometimes contain a Unicode Byte Order Mark.
/// Callers are responsible for handling this, such as by using <seealso cref="UnicodeBom"/>.
///
/// </para>
/// </summary>
/// @param <T> the trade type </param>
/// <param name="charSources"> the CSV character sources </param>
/// <param name="tradeType"> the trade type to return </param>
/// <returns> the loaded trades, all errors are captured in the result </returns>
public ValueWithFailures <IList <T> > parse <T>(ICollection <CharSource> charSources, Type <T> tradeType) where T : com.opengamma.strata.product.Trade
{
try
{
ValueWithFailures <IList <T> > result = ValueWithFailures.of(ImmutableList.of());
foreach (CharSource charSource in charSources)
{
ValueWithFailures <IList <T> > singleResult = parseFile(charSource, tradeType);
result = result.combinedWith(singleResult, Guavate.concatToList);
}
return(result);
}
catch (Exception ex)
{
return(ValueWithFailures.of(ImmutableList.of(), FailureItem.of(FailureReason.ERROR, ex)));
}
}
// loads a single CSV file, filtering by trade type
private ValueWithFailures <IList <T> > parseFile <T>(CharSource charSource, Type <T> tradeType) where T : com.opengamma.strata.product.Trade
{
try
{
using (CsvIterator csv = CsvIterator.of(charSource, true))
{
if (!csv.headers().contains(TYPE_FIELD))
{
return(ValueWithFailures.of(ImmutableList.of(), FailureItem.of(FailureReason.PARSING, "CSV file does not contain '{header}' header: {}", TYPE_FIELD, charSource)));
}
return(parseFile(csv, tradeType));
}
}
catch (Exception ex)
{
return(ValueWithFailures.of(ImmutableList.of(), FailureItem.of(FailureReason.PARSING, ex, "CSV file could not be parsed: {exceptionMessage}: {}", ex.Message, charSource)));
}
}
//-------------------------------------------------------------------------
// parses the file in standard format
private void parseStandardFormat(CsvIterator csv, ListMultimap <string, CurveSensitivities> parsed, IList <FailureItem> failures)
{
// loop around all rows, peeking to match batches with the same identifier
// no exception catch at this level to avoid infinite loops
while (csv.hasNext())
{
CsvRow peekedRow = csv.peek();
PortfolioItemInfo info = parseInfo(peekedRow);
//JAVA TO C# CONVERTER TODO TASK: Method reference arbitrary object instance method syntax is not converted by Java to C# Converter:
string id = info.Id.map(StandardId::toString).orElse("");
// process in batches, where the ID is the same
CurveSensitivitiesBuilder builder = CurveSensitivities.builder(info);
IList <CsvRow> batchRows = csv.nextBatch(r => matchId(r, id));
foreach (CsvRow batchRow in batchRows)
{
try
{
CurveName reference = CurveName.of(batchRow.getValue(REFERENCE_HEADER));
CurveName resolvedCurveName = resolver.checkCurveName(reference);
CurveSensitivitiesType type = CurveSensitivitiesType.of(batchRow.getValue(TYPE_HEADER));
ParameterMetadata metadata = parseMetadata(batchRow, false);
Currency currency = parseCurrency(batchRow, reference);
string valueStr = batchRow.getField(VALUE_HEADER);
if (valueStr.Length > 0)
{
double value = LoaderUtils.parseDouble(valueStr);
builder.add(type, resolvedCurveName, currency, metadata, value);
}
}
catch (System.ArgumentException ex)
{
failures.Add(FailureItem.of(PARSING, "CSV file could not be parsed at line {}: {}", batchRow.lineNumber(), ex.Message));
}
}
CurveSensitivities sens = builder.build();
if (!sens.TypedSensitivities.Empty)
{
parsed.put(sens.Id.map(object.toString).orElse(""), sens);
}
}
}
/// <summary>
/// Parses one or more CSV format trade files with an error-creating type filter.
/// <para>
/// A list of types is specified to filter the trades.
/// Trades that do not match the type will be included in the failure list.
/// </para>
/// <para>
/// CSV files sometimes contain a Unicode Byte Order Mark.
/// Callers are responsible for handling this, such as by using <seealso cref="UnicodeBom"/>.
///
/// </para>
/// </summary>
/// <param name="charSources"> the CSV character sources </param>
/// <param name="tradeTypes"> the trade types to return </param>
/// <returns> the loaded trades, all errors are captured in the result </returns>
public ValueWithFailures <IList <Trade> > parse(ICollection <CharSource> charSources, IList <Type> tradeTypes)
{
ValueWithFailures <IList <Trade> > parsed = parse(charSources, typeof(Trade));
IList <Trade> valid = new List <Trade>();
IList <FailureItem> failures = new List <FailureItem>(parsed.Failures);
foreach (Trade trade in parsed.Value)
{
if (tradeTypes.Contains(trade.GetType()))
{
valid.Add(trade);
}
else
{
//JAVA TO C# CONVERTER WARNING: The .NET Type.FullName property will not always yield results identical to the Java Class.getName method:
//JAVA TO C# CONVERTER TODO TASK: Most Java stream collectors are not converted by Java to C# Converter:
failures.Add(FailureItem.of(FailureReason.PARSING, "Trade type not allowed {tradeType}, only these types are supported: {}", trade.GetType().FullName, tradeTypes.Select(t => t.SimpleName).collect(joining(", "))));
}
}
return(ValueWithFailures.of(valid, failures));
}
// loads a single CSV file
private ValueWithFailures <IList <T> > parseFile <T>(CsvIterator csv, Type <T> posType) where T : com.opengamma.strata.product.Position
{
IList <T> positions = new List <T>();
IList <FailureItem> failures = new List <FailureItem>();
int line = 2;
foreach (CsvRow row in (IEnumerable <CsvRow>)() => csv)
{
try
{
PositionInfo info = parsePositionInfo(row);
Optional <string> typeRawOpt = row.findValue(TYPE_FIELD);
if (typeRawOpt.Present)
{
// type specified
string type = typeRawOpt.get().ToUpper(Locale.ENGLISH);
switch (type.ToUpper(Locale.ENGLISH))
{
case "SEC":
case "SECURITY":
if (posType == typeof(SecurityPosition) || posType == typeof(ResolvableSecurityPosition))
{
positions.Add(posType.cast(SecurityCsvLoader.parseSecurityPosition(row, info, resolver)));
}
else if (posType == typeof(GenericSecurityPosition) || posType == typeof(Position))
{
Position parsed = SecurityCsvLoader.parseNonEtdPosition(row, info, resolver);
if (posType.IsInstanceOfType(parsed))
{
positions.Add(posType.cast(parsed));
}
}
break;
case "FUT":
case "FUTURE":
if (posType == typeof(EtdPosition) || posType == typeof(EtdFuturePosition) || posType == typeof(ResolvableSecurityPosition) || posType == typeof(Position))
{
positions.Add(posType.cast((Position)resolver.parseEtdFuturePosition(row, info)));
}
else if (posType == typeof(SecurityPosition))
{
positions.Add(posType.cast(resolver.parseEtdFutureSecurityPosition(row, info)));
}
break;
case "OPT":
case "OPTION":
if (posType == typeof(EtdPosition) || posType == typeof(EtdOptionPosition) || posType == typeof(ResolvableSecurityPosition) || posType == typeof(Position))
{
positions.Add(posType.cast(resolver.parseEtdOptionPosition(row, info)));
}
else if (posType == typeof(SecurityPosition))
{
positions.Add(posType.cast(resolver.parseEtdOptionSecurityPosition(row, info)));
}
break;
default:
failures.Add(FailureItem.of(FailureReason.PARSING, "CSV file position type '{positionType}' is not known at line {lineNumber}", typeRawOpt.get(), line));
break;
}
}
else
{
// infer type
if (posType == typeof(SecurityPosition))
{
positions.Add(posType.cast(SecurityCsvLoader.parsePositionLightweight(row, info, resolver)));
}
else
{
Position position = SecurityCsvLoader.parsePosition(row, info, resolver);
if (posType.IsInstanceOfType(position))
{
positions.Add(posType.cast(position));
}
}
}
}
catch (Exception ex)
{
failures.Add(FailureItem.of(FailureReason.PARSING, ex, "CSV file position could not be parsed at line {lineNumber}: {exceptionMessage}", line, ex.Message));
}
line++;
}
return(ValueWithFailures.of(positions, failures));
}
// loads a single CSV file
private ValueWithFailures <IList <T> > parseFile <T>(CsvIterator csv, Type <T> tradeType) where T : com.opengamma.strata.product.Trade
{
IList <T> trades = new List <T>();
IList <FailureItem> failures = new List <FailureItem>();
while (csv.hasNext())
{
CsvRow row = csv.next();
try
{
string typeRaw = row.getField(TYPE_FIELD);
TradeInfo info = parseTradeInfo(row);
switch (typeRaw.ToUpper(Locale.ENGLISH))
{
case "FRA":
if (tradeType == typeof(FraTrade) || tradeType == typeof(Trade))
{
trades.Add(tradeType.cast(FraTradeCsvLoader.parse(row, info, resolver)));
}
break;
case "SECURITY":
if (tradeType == typeof(SecurityTrade) || tradeType == typeof(GenericSecurityTrade) || tradeType == typeof(ResolvableSecurityTrade) || tradeType == typeof(Trade))
{
SecurityQuantityTrade parsed = SecurityCsvLoader.parseTrade(row, info, resolver);
if (tradeType.IsInstanceOfType(parsed))
{
trades.Add(tradeType.cast(parsed));
}
}
break;
case "SWAP":
if (tradeType == typeof(SwapTrade) || tradeType == typeof(Trade))
{
IList <CsvRow> variableRows = new List <CsvRow>();
while (csv.hasNext() && csv.peek().getField(TYPE_FIELD).ToUpper(Locale.ENGLISH).Equals("VARIABLE"))
{
variableRows.Add(csv.next());
}
trades.Add(tradeType.cast(SwapTradeCsvLoader.parse(row, variableRows, info, resolver)));
}
break;
case "TERMDEPOSIT":
case "TERM DEPOSIT":
if (tradeType == typeof(TermDepositTrade) || tradeType == typeof(Trade))
{
trades.Add(tradeType.cast(TermDepositTradeCsvLoader.parse(row, info, resolver)));
}
break;
case "VARIABLE":
failures.Add(FailureItem.of(FailureReason.PARSING, "CSV file contained a 'Variable' type at line {lineNumber} that was not preceeded by a 'Swap'", row.lineNumber()));
break;
case "FX":
case "FXSINGLE":
case "FX SINGLE":
if (tradeType == typeof(FxSingleTrade) || tradeType == typeof(FxTrade) || tradeType == typeof(Trade))
{
trades.Add(tradeType.cast(FxSingleTradeCsvLoader.parse(row, info, resolver)));
}
break;
case "FXSWAP":
case "FX SWAP":
if (tradeType == typeof(FxSwapTrade) || tradeType == typeof(FxTrade) || tradeType == typeof(Trade))
{
trades.Add(tradeType.cast(FxSwapTradeCsvLoader.parse(row, info, resolver)));
}
break;
default:
failures.Add(FailureItem.of(FailureReason.PARSING, "CSV file trade type '{tradeType}' is not known at line {lineNumber}", typeRaw, row.lineNumber()));
break;
}
}
catch (Exception ex)
{
failures.Add(FailureItem.of(FailureReason.PARSING, ex, "CSV file trade could not be parsed at line {lineNumber}: {exceptionMessage}", row.lineNumber(), ex.Message));
}
}
return(ValueWithFailures.of(trades, failures));
}
