[Timeout(36000000)] // These can take a long time in code coverage mode
public void TestRunQuantification()
{
var cache = new QrFactorizationCache();
var csvReader = new DsvFileReader(GetTextReader("quant.csv"), ',');
var dataRowsByProtein = ToDataRows(ReadCsvFile(csvReader));
var expectedResultsByProtein = ReadCsvFile(new DsvFileReader(GetTextReader("runquantdata.csv"), ',')).ToLookup(row => row["Protein"]);
foreach (var entry in dataRowsByProtein)
{
var expectedResultsByRun = expectedResultsByProtein[entry.Key].ToLookup(row => row["RUN"]);
FoldChangeDataSet dataSet = FoldChangeCalculator.MakeDataSet(entry.Value);
var designMatrix = DesignMatrix.GetRunQuantificationDesignMatrix(dataSet);
var runNames = FoldChangeCalculator.GetUniqueList(entry.Value.Select(row => row.Run));
var results = designMatrix.PerformLinearFit(cache);
for (int i = 0; i < dataSet.RunCount; i++)
{
string message = string.Format("Protein:{0} Run:{1}", entry.Key, runNames[i]);
var expectedRow = expectedResultsByRun[runNames[i]].FirstOrDefault();
Assert.IsNotNull(expectedRow);
Assert.AreEqual(double.Parse(expectedRow["LogIntensities"], CultureInfo.InvariantCulture), results[i].EstimatedValue, .000001, message);
Assert.AreEqual(int.Parse(expectedRow["NumFeature"], CultureInfo.InvariantCulture), dataSet.FeatureCount, message);
Assert.AreEqual(int.Parse(expectedRow["NumPeaks"], CultureInfo.InvariantCulture), dataSet.GetFeatureCountForRun(i), message);
}
}
}
public IList <PeakData> ImportFile(TextReader peakViewReader)
{
var peakDatas = new List <PeakData>();
var msFileNames = new List <string>();
var fileReader = new DsvFileReader(peakViewReader, TextUtil.SEPARATOR_TSV);
Assert.AreEqual(fileReader.GetFieldIndex(PROTEIN), 0);
Assert.AreEqual(fileReader.GetFieldIndex(PEPTIDE), 1);
Assert.AreEqual(fileReader.GetFieldIndex(LABEL), 2);
Assert.AreEqual(fileReader.GetFieldIndex(PRECURSOR), 3);
Assert.AreEqual(fileReader.GetFieldIndex(CHARGE), 4);
Assert.AreEqual(fileReader.GetFieldIndex(RT), 5);
Assert.AreEqual(fileReader.GetFieldIndex(DECOY), 6);
for (int i = 7; i < fileReader.NumberOfFields; ++i)
{
msFileNames.Add(fileReader.FieldNames[i]);
}
while (fileReader.ReadLine() != null)
{
string modifiedSequence = fileReader.GetFieldByName(PEPTIDE);
string decoyString = fileReader.GetFieldByName(DECOY);
bool decoy;
Assert.IsTrue(bool.TryParse(decoyString, out decoy));
foreach (var msFileName in msFileNames)
{
string dataFieldString = fileReader.GetFieldByName(msFileName);
double dataField;
Assert.IsTrue(double.TryParse(dataFieldString, out dataField));
var peakData = new PeakData(dataField, modifiedSequence, msFileName, decoy);
peakDatas.Add(peakData);
}
}
return(peakDatas);
}
private void TestGroupComparison(TextReader textReader, bool includeInteraction, IDictionary <string, LinearFitResult> expectedResults)
{
var csvReader = new DsvFileReader(textReader, ',');
var dataRowsByProtein = ToDataRows(ReadCsvFile(csvReader));
Assert.AreNotEqual(0, dataRowsByProtein.Count);
var cache = new QrFactorizationCache();
foreach (var entry in dataRowsByProtein)
{
FoldChangeDataSet dataSet = FoldChangeCalculator.MakeDataSet(entry.Value);
var designMatrix = DesignMatrix.GetDesignMatrix(dataSet, includeInteraction);
var foldChange = designMatrix.PerformLinearFit(cache).First();
LinearFitResult expectedResult = null;
if (null != expectedResults)
{
Assert.IsTrue(expectedResults.TryGetValue(entry.Key, out expectedResult));
}
if (null != expectedResult)
{
Assert.AreEqual(expectedResult.EstimatedValue, foldChange.EstimatedValue, 1E-6);
Assert.AreEqual(expectedResult.DegreesOfFreedom, foldChange.DegreesOfFreedom);
Assert.AreEqual(expectedResult.StandardError, foldChange.StandardError, 1E-6);
Assert.AreEqual(expectedResult.TValue, foldChange.TValue, 1E-6);
Assert.AreEqual(expectedResult.PValue, foldChange.PValue, 1E-6);
}
}
}
public SrmDocument ReadAnnotationsFromFile(CancellationToken cancellationToken, string filename)
{
using (var streamReader = new StreamReader(filename))
{
var dsvReader = new DsvFileReader(streamReader, TextUtil.SEPARATOR_CSV);
return(ReadAllAnnotations(cancellationToken, dsvReader));
}
}
protected static IEnumerable <string> ReadCsvRow(string row)
{
if (string.IsNullOrEmpty(row))
{
return(new string[0]);
}
var csvFileReader = new DsvFileReader(new StringReader(row), CSV_SEPARATOR_CHAR, false);
return(csvFileReader.ReadLine());
}
public SrmDocument ReadAnnotationsFromFile(CancellationToken cancellationToken, string filename)
{
DataSchema.BeginBatchModifyDocument();
using (var streamReader = new StreamReader(filename))
{
var dsvReader = new DsvFileReader(streamReader, TextUtil.SEPARATOR_CSV);
ReadAllAnnotations(cancellationToken, dsvReader);
}
DataSchema.CommitBatchModifyDocument(string.Empty, null);
return(DataSchema.Document);
}
public List <string> TranscribeAndModifyFile(StreamWriter writer, TextReader reader, List <string> fields, bool first, int currentFileCount)
{
var fileReader = new DsvFileReader(reader, SEPARATOR);
if (first)
{
fields = fileReader.FieldNames;
for (int i = 0; i < fields.Count; ++i)
{
if (i > 0)
{
writer.Write(SEPARATOR);
}
writer.WriteDsvField(fileReader.FieldNames[i], SEPARATOR);
}
writer.WriteLine();
}
Assert.AreEqual(fileReader.NumberOfFields, fields.Count);
for (int i = 0; i < fields.Count; ++i)
{
Assert.AreEqual(fileReader.FieldNames[i], fields[i]);
}
while (fileReader.ReadLine() != null)
{
for (int i = 0; i < fields.Count; ++i)
{
string modifiedField = fileReader.GetFieldByIndex(i);
switch (fileReader.FieldNames[i])
{
case FILE_NAME:
modifiedField = Path.GetFileNameWithoutExtension(Path.GetFileNameWithoutExtension(modifiedField)) + MS_FILE_TYPE;
break;
case TRANSITION_GROUP:
modifiedField = modifiedField + currentFileCount;
break;
case RUN_ID:
modifiedField = currentFileCount.ToString(CultureInfo.CurrentCulture);
break;
}
if (i > 0)
{
writer.Write(SEPARATOR);
}
writer.WriteDsvField(modifiedField, SEPARATOR);
}
writer.WriteLine();
}
return(fields);
}
private IList <Dictionary <string, string> > ReadCsvFile(DsvFileReader fileReader)
{
var result = new List <Dictionary <string, string> >();
while (null != fileReader.ReadLine())
{
Dictionary <string, string> row = new Dictionary <string, string>();
for (int i = 0; i < fileReader.NumberOfFields; i++)
{
var value = fileReader.GetFieldByIndex(i);
if (null != value)
{
row.Add(fileReader.FieldNames[i], value);
}
}
result.Add(row);
}
return(result);
}
public static IDictionary <string, LinearFitResult> ReadExpectedResults(Type type, string resourceName)
{
var result = new Dictionary <string, LinearFitResult>();
using (var reader = GetTextReaderForManifestResource(type, resourceName))
{
var csvReader = new DsvFileReader(reader, ',');
while (null != csvReader.ReadLine())
{
string protein = csvReader.GetFieldByName("Protein");
var linearFitResult = new LinearFitResult(Convert.ToDouble(csvReader.GetFieldByName("log2FC"), CultureInfo.InvariantCulture))
.SetStandardError(Convert.ToDouble(csvReader.GetFieldByName("SE"), CultureInfo.InvariantCulture))
.SetTValue(Convert.ToDouble(csvReader.GetFieldByName("Tvalue"), CultureInfo.InvariantCulture))
.SetDegreesOfFreedom(Convert.ToInt32(csvReader.GetFieldByName("DF"), CultureInfo.InvariantCulture))
.SetPValue(Convert.ToDouble(csvReader.GetFieldByName("pvalue"), CultureInfo.InvariantCulture));
result.Add(protein, linearFitResult);
}
}
return(result);
}
public SrmDocument ReadAllAnnotations(CancellationToken cancellationToken, DsvFileReader fileReader)
{
var document = Document;
var columns = new Columns(fileReader.FieldNames, Document.Settings.DataSettings.AnnotationDefs);
string[] row;
while ((row = fileReader.ReadLine()) != null)
{
cancellationToken.ThrowIfCancellationRequested();
ElementLocator elementLocator = columns.GetElementLocator(row);
var elementRef = ElementRefs.FromObjectReference(elementLocator);
var annotations = GetAnnotations(document, elementRef);
var newAnnotations = columns.ReadAnnotations(CultureInfo, elementRef, annotations, row);
if (!Equals(newAnnotations, annotations))
{
document = ChangeAnnotations(document, elementRef, newAnnotations);
}
}
return(document);
}
public void DsvHeadersTest()
{
// test reading DSV files with and without headers
string[] lines = { "dog,1.1,cat,2.2", "pony,3.3,fish,4.4" }; // Not L10N
const char csvSeperator = ',';
var withheaders = new DsvFileReader(new StringReader("0,1,2,3\n" + String.Join("\n", lines)), csvSeperator); // Not L10N
var withoutheaders = new DsvFileReader(new StringReader(String.Join("\n", lines)), csvSeperator, hasHeaders: false); // Not L10N
Assert.AreEqual(withheaders.NumberOfFields, withoutheaders.NumberOfFields);
for (int h = 0; h < withoutheaders.NumberOfFields; h++)
{
var f = String.Format("{0}", h); // verify that a headerless CSV file will pretend to have a header of form "0,1,2,..." // Not L10N
Assert.AreEqual(withheaders.GetFieldIndex(f), withoutheaders.GetFieldIndex(f));
}
int linenumber = 0;
while (true)
{
var fields = withoutheaders.ReadLine();
var fieldsTest = withheaders.ReadLine();
Assert.IsTrue((fields == null) == (fieldsTest == null));
if (fields == null)
{
break;
}
for (int f = 0; f < withoutheaders.NumberOfFields; f++)
{
Assert.AreEqual(fields[f], fieldsTest[f]); // quick string compare
Assert.AreEqual(lines[linenumber].Split(csvSeperator)[f], fields[f]);
}
linenumber++;
}
Assert.AreEqual(2, linenumber);
// let's also check the precision handling and field exception in AssertEx.FieldsEqual
var A = String.Join("\n", lines).Replace("cat", "hamster"); // change col 2, then ignore it // Not L10N
var B = String.Join("\n", lines).Replace("1.1", "1.09"); // add error at limits of precision, then allow for it // Not L10N
AssertEx.FieldsEqual(A, B, 4, 2, true);
}
[Timeout(36000000)] // These can take a long time in code coverage mode
public void TestGroupComparisonWithRunQuantification()
{
var csvReader = new DsvFileReader(GetTextReader("quant.csv"), ',');
var dataRowsByProtein = ToDataRows(ReadCsvFile(csvReader));
var expectedResultsByProtein = ReadCsvFile(new DsvFileReader(GetTextReader("result_newtesting_v2.csv"), ','))
.ToDictionary(row => row["Protein"]);
var cache = new QrFactorizationCache();
foreach (var entry in dataRowsByProtein)
{
FoldChangeDataSet dataSet = FoldChangeCalculator.MakeDataSet(entry.Value);
var quantifiedRuns = DesignMatrix.GetRunQuantificationDesignMatrix(dataSet).PerformLinearFit(cache);
var subjects = new List <int>();
for (int run = 0; run < quantifiedRuns.Count; run++)
{
int iRow = dataSet.Runs.IndexOf(run);
subjects.Add(dataSet.Subjects[iRow]);
}
var abundances = quantifiedRuns.Select(result => result.EstimatedValue).ToArray();
var quantifiedDataSet = new FoldChangeDataSet(
abundances,
Enumerable.Repeat(0, quantifiedRuns.Count).ToArray(),
Enumerable.Range(0, quantifiedRuns.Count).ToArray(),
subjects,
dataSet.SubjectControls);
var foldChangeResult = DesignMatrix.GetDesignMatrix(quantifiedDataSet, false).PerformLinearFit(cache).First();
var expectedResult = expectedResultsByProtein[entry.Key];
string message = entry.Key;
Assert.AreEqual(double.Parse(expectedResult["logFC"], CultureInfo.InvariantCulture), foldChangeResult.EstimatedValue, 1E-6, message);
Assert.AreEqual(double.Parse(expectedResult["SE"], CultureInfo.InvariantCulture), foldChangeResult.StandardError, 1E-6, message);
Assert.AreEqual(int.Parse(expectedResult["DF"], CultureInfo.InvariantCulture), foldChangeResult.DegreesOfFreedom, message);
if (Math.Abs(foldChangeResult.EstimatedValue) > 1E-8)
{
Assert.AreEqual(double.Parse(expectedResult["pvalue"], CultureInfo.InvariantCulture), foldChangeResult.PValue, 1E-6, message);
Assert.AreEqual(double.Parse(expectedResult["Tvalue"], CultureInfo.InvariantCulture), foldChangeResult.TValue, 1E-6, message);
}
}
}
Dictionary <DataProcessedRowKey, double?> ReadDataProcessedRows(TextReader reader)
{
DsvFileReader csvReader = new DsvFileReader(reader, TextUtil.SEPARATOR_CSV);
var rows = new Dictionary <DataProcessedRowKey, double?>();
while (null != csvReader.ReadLine())
{
DataProcessedRowKey dataProcessedRow = new DataProcessedRowKey
{
Protein = csvReader.GetFieldByName("PROTEIN"),
Peptide = csvReader.GetFieldByName("PEPTIDE"),
Transition = csvReader.GetFieldByName("TRANSITION"),
Run = int.Parse(csvReader.GetFieldByName("RUN"), CultureInfo.InvariantCulture),
};
String strAbundance = csvReader.GetFieldByName("ABUNDANCE");
double?abundance = "NA" == strAbundance
? default(double?)
: double.Parse(strAbundance, CultureInfo.InvariantCulture);
rows.Add(dataProcessedRow, abundance);
}
return(rows);
}
protected override void DoTest()
{
Assert.AreEqual(_language, CultureInfo.CurrentUICulture);
RunUI(() => SkylineWindow.Paste(TextUtil.LineSeparate("ELVIS", "LIVES")));
var exportLiveReportDlg = ShowDialog <ExportLiveReportDlg>(SkylineWindow.ShowExportReportDialog);
RunUI(() => exportLiveReportDlg.ReportName = Resources.Resources_ReportSpecList_GetDefaults_Peptide_Quantification);
Directory.CreateDirectory(TestContext.TestDir);
string outputFile = Path.Combine(TestContext.TestDir, "file.csv");
OkDialog(exportLiveReportDlg, () => exportLiveReportDlg.OkDialog(outputFile, TextUtil.CsvSeparator));
using (var textReader = new StreamReader(outputFile))
{
var csvReader = new DsvFileReader(textReader, TextUtil.CsvSeparator);
CollectionAssert.Contains(csvReader.FieldNames, ColumnCaptions.CalibrationCurve);
Assert.IsNotNull(csvReader.ReadLine());
string actualCalibrationCurve = csvReader.GetFieldByName(ColumnCaptions.CalibrationCurve);
string expectedCalibrationCurve = string.Format(
QuantificationStrings.CalibrationCurve_ToString_Slope___0_,
1.0.ToString(Formats.CalibrationCurve));
Assert.AreEqual(expectedCalibrationCurve, actualCalibrationCurve);
}
}
public void ReadAllAnnotations(CancellationToken cancellationToken, DsvFileReader fileReader)
{
var fieldNames = fileReader.FieldNames;
int locatorColumnIndex = fieldNames.IndexOf(COLUMN_LOCATOR);
if (locatorColumnIndex < 0)
{
throw new InvalidDataException(string.Format(Resources.Columns_Columns_Missing_column___0__,
COLUMN_LOCATOR));
}
string[] row;
while ((row = fileReader.ReadLine()) != null)
{
cancellationToken.ThrowIfCancellationRequested();
ElementLocator elementLocator = ElementLocator.Parse(row[locatorColumnIndex]);
var elementRef = ElementRefs.FromObjectReference(elementLocator);
ElementHandler handler;
if (!_elementHandlers.TryGetValue(elementRef.ElementType, out handler))
{
throw ElementNotSupportedException(elementRef);
}
SkylineObject element = handler.FindElement(elementRef);
if (element == null)
{
throw ElementNotFoundException(elementRef);
}
for (int icol = 0; icol < fieldNames.Count; icol++)
{
if (icol == locatorColumnIndex)
{
continue;
}
string fieldName = fieldNames[icol];
TextColumnWrapper propertyInfo = null;
AnnotationDef annotationDef = null;
if (fieldName.StartsWith(PROPERTY_PREFIX))
{
propertyInfo = handler.FindProperty(fieldName.Substring(PROPERTY_PREFIX.Length));
}
else if (fieldName.StartsWith(ANNOTATION_PREFIX))
{
annotationDef = handler.FindAnnotation(fieldName.Substring(ANNOTATION_PREFIX.Length));
}
if (propertyInfo == null && annotationDef == null)
{
propertyInfo = handler.FindProperty(fieldName);
if (propertyInfo == null)
{
annotationDef = handler.FindAnnotation(fieldName);
}
}
string fieldValue = row[icol];
if (propertyInfo == null && annotationDef == null)
{
if (string.IsNullOrEmpty(fieldValue))
{
continue;
}
throw AnnotationDoesNotApplyException(fieldName, elementRef);
}
if (propertyInfo != null)
{
object value = propertyInfo.ParseTextValue(CultureInfo, fieldValue);
propertyInfo.SetValue(element, value);
}
if (annotationDef != null)
{
SetAnnotationValue(element, annotationDef, fieldValue);
}
}
}
}
private IList <Dictionary <string, string> > ReadCsvFile(DsvFileReader fileReader)
{
return(MsStatsTestUtil.ReadCsvFile(fileReader));
}