//public void Step_09_BuildCommonProbeFile(string root)
//{
// var celFiles = GetCelFiles(chipDir);
// var rmafiles = Directory.GetFiles(root, "Step_08_*_justRMA.tsv");
// var commonProbes = new HashSet<string>();
// foreach (var rmafile in rmafiles)
// {
// Console.WriteLine(rmafile);
// var map = new MapItemReader(0, 1).ReadFromFile(rmafile);
// //var curprobes = (from k in map.Keys select k.ToLower()).ToList();
// var curprobes = map.Keys.ToList();
// if (commonProbes.Count == 0)
// {
// commonProbes = new HashSet<string>(curprobes);
// }
// else
// {
// commonProbes.IntersectWith(curprobes);
// }
// }
// Console.WriteLine("Common probes = {0}", commonProbes.Count);
// var probes = (from p in commonProbes
// orderby p
// select p).ToList();
// var values = (from rmafile in rmafiles
// select ReadFile(rmafile, commonProbes)).ToList();
// using (var sw = new StreamWriter(root + "\\Step_09_expression_commonprobes.tsv"))
// {
// sw.WriteLine("Probe\t{0}", (from v in values
// from s in v.Samples
// select s).Merge("\t"));
// for (int i = 0; i < probes.Count; i++)
// {
// var p = probes[i];
// sw.Write(p);
// foreach (var v in values)
// {
// var vv = v.Values[i];
// sw.Write("\t{0}", v.Values[i].Second);
// }
// sw.WriteLine();
// }
// }
//Assert.IsTrue(RHelper.ExtractCelData(celFiles, false), string.Format("Extrace tsv from cel failed"));
//var tsvFiles = Directory.GetFiles(chipDir, "*.tsv");
//var reader = new ExpressionDataRawReader(2, 1);
//var datas = (from tsvfile in tsvFiles
// select reader.ReadFromFile(tsvfile)).ToList();
//var commonGenes = datas.GetCommonGenes();
//Console.WriteLine("Common genes = {0}", commonGenes.Count);
////get sample files
//string[] files;
//if (smallDataset)
//{
// files = (from dir in Directory.GetDirectories(root)
// from file in Directory.GetFiles(dir, "*.tsv").Take(3)
// select file).ToArray();
//}
//else
//{
// files = (from dir in Directory.GetDirectories(root)
// from file in Directory.GetFiles(dir, "*.tsv")
// select file).ToArray();
//}
////get batch information
//var list = (from tsvFile in files
// let dir = Path.GetDirectoryName(tsvFile)
// let file = FileUtils.ChangeExtension(tsvFile, "")
// select new BarInfo()
// {
// FileName = tsvFile,
// Dataset = Path.GetFileName(dir).Replace("-", ""),
// BarCode = "S_" + GetBarCode(file),
// ChipType = CelFile.GetChipType(file).Replace("-", "")
// }).ToList();
//var dsmap = list.GroupBy(m => m.Dataset).ToList();
//foreach (var ds in dsmap)
//{
// var dss = ds.GroupBy(m => m.ChipType).ToList();
// if (dss.Count > 1)
// {
// foreach (var info in ds)
// {
// info.BatchName = info.Dataset + "_" + info.ChipType;
// }
// }
// else
// {
// foreach (var info in ds)
// {
// info.BatchName = info.Dataset;
// }
// }
//}
//var map = list.ToDictionary(m => m.FileName);
////output all informations
//var batchDefinitionFile = targetFile + ".batchdefinition";
//Console.WriteLine("Total {0} files", files.Length);
//var icount = 0;
//var genes = commonGenes.ToList();
//genes.Sort();
//using (StreamWriter sw = new StreamWriter(targetFile))
//{
// using (StreamWriter rw = new StreamWriter(batchDefinitionFile))
// {
// sw.Write("GENE");
// genes.ForEach(m => sw.Write("\tG_" + m));
// sw.WriteLine();
// foreach (var file in files)
// {
// icount++;
// BarInfo bi = map[file];
// rw.WriteLine(bi.BatchName + "\t" + bi.BarCode + "\t" + bi.FileName);
// Console.WriteLine("reading {0}/{1} : {2}", icount, files.Length, file);
// try
// {
// var data = reader.ReadFromFile(file);
// data.Values.RemoveAll(m => !commonGenes.Contains(m.Name));
// data.Values.Sort((m1, m2) => m1.Name.CompareTo(m2.Name));
// Assert.AreEqual(commonGenes.Count, data.Values.Count, "Gene count should equal to common gene count : " + Path.GetFileName(file));
// sw.Write(bi.BarCode);
// data.Values.ForEach(m =>
// {
// if (double.IsNaN(m.Value))
// {
// sw.Write("\tNA");
// }
// else
// {
// sw.Write("\t{0:0.00}", Math.Pow(2, m.Value));
// }
// });
// sw.WriteLine();
// }
// catch (Exception ex)
// {
// Console.Error.WriteLine(file + " : " + ex.Message);
// throw;
// }
// }
// }
//}
//Console.WriteLine("Finished!");
//}
private ProbeExpressionFile ReadFile(string rmafile, HashSet <string> commonProbes)
{
var result = new ProbeExpressionFile();
using (var sr = new StreamReader(rmafile))
{
var line = sr.ReadLine();
result.Samples = (from sample in line.Split('\t').Skip(1) select Path.GetFileNameWithoutExtension(sample)).ToArray();
result.Values = new List <Pair <string, string> >();
while ((line = sr.ReadLine()) != null)
{
var name = line.StringBefore("\t");
if (commonProbes.Contains(name))
{
result.Values.Add(new Pair <string, string>(name, line.StringAfter("\t")));
}
}
}
result.Values.Sort((m1, m2) => m1.First.CompareTo(m2.First));
return(result);
}
//public void Step_09_BuildCommonProbeFile(string root)
//{
// var celFiles = GetCelFiles(chipDir);
// var rmafiles = Directory.GetFiles(root, "Step_08_*_justRMA.tsv");
// var commonProbes = new HashSet<string>();
// foreach (var rmafile in rmafiles)
// {
// Console.WriteLine(rmafile);
// var map = new MapItemReader(0, 1).ReadFromFile(rmafile);
// //var curprobes = (from k in map.Keys select k.ToLower()).ToList();
// var curprobes = map.Keys.ToList();
// if (commonProbes.Count == 0)
// {
// commonProbes = new HashSet<string>(curprobes);
// }
// else
// {
// commonProbes.IntersectWith(curprobes);
// }
// }
// Console.WriteLine("Common probes = {0}", commonProbes.Count);
// var probes = (from p in commonProbes
// orderby p
// select p).ToList();
// var values = (from rmafile in rmafiles
// select ReadFile(rmafile, commonProbes)).ToList();
// using (var sw = new StreamWriter(root + "\\Step_09_expression_commonprobes.tsv"))
// {
// sw.WriteLine("Probe\t{0}", (from v in values
// from s in v.Samples
// select s).Merge("\t"));
// for (int i = 0; i < probes.Count; i++)
// {
// var p = probes[i];
// sw.Write(p);
// foreach (var v in values)
// {
// var vv = v.Values[i];
// sw.Write("\t{0}", v.Values[i].Second);
// }
// sw.WriteLine();
// }
// }
//Assert.IsTrue(RHelper.ExtractCelData(celFiles, false), string.Format("Extrace tsv from cel failed"));
//var tsvFiles = Directory.GetFiles(chipDir, "*.tsv");
//var reader = new ExpressionDataRawReader(2, 1);
//var datas = (from tsvfile in tsvFiles
// select reader.ReadFromFile(tsvfile)).ToList();
//var commonGenes = datas.GetCommonGenes();
//Console.WriteLine("Common genes = {0}", commonGenes.Count);
////get sample files
//string[] files;
//if (smallDataset)
//{
// files = (from dir in Directory.GetDirectories(root)
// from file in Directory.GetFiles(dir, "*.tsv").Take(3)
// select file).ToArray();
//}
//else
//{
// files = (from dir in Directory.GetDirectories(root)
// from file in Directory.GetFiles(dir, "*.tsv")
// select file).ToArray();
//}
////get batch information
//var list = (from tsvFile in files
// let dir = Path.GetDirectoryName(tsvFile)
// let file = FileUtils.ChangeExtension(tsvFile, "")
// select new BarInfo()
// {
// FileName = tsvFile,
// Dataset = Path.GetFileName(dir).Replace("-", ""),
// BarCode = "S_" + GetBarCode(file),
// ChipType = CelFile.GetChipType(file).Replace("-", "")
// }).ToList();
//var dsmap = list.GroupBy(m => m.Dataset).ToList();
//foreach (var ds in dsmap)
//{
// var dss = ds.GroupBy(m => m.ChipType).ToList();
// if (dss.Count > 1)
// {
// foreach (var info in ds)
// {
// info.BatchName = info.Dataset + "_" + info.ChipType;
// }
// }
// else
// {
// foreach (var info in ds)
// {
// info.BatchName = info.Dataset;
// }
// }
//}
//var map = list.ToDictionary(m => m.FileName);
////output all informations
//var batchDefinitionFile = targetFile + ".batchdefinition";
//Console.WriteLine("Total {0} files", files.Length);
//var icount = 0;
//var genes = commonGenes.ToList();
//genes.Sort();
//using (StreamWriter sw = new StreamWriter(targetFile))
//{
// using (StreamWriter rw = new StreamWriter(batchDefinitionFile))
// {
// sw.Write("GENE");
// genes.ForEach(m => sw.Write("\tG_" + m));
// sw.WriteLine();
// foreach (var file in files)
// {
// icount++;
// BarInfo bi = map[file];
// rw.WriteLine(bi.BatchName + "\t" + bi.BarCode + "\t" + bi.FileName);
// Console.WriteLine("reading {0}/{1} : {2}", icount, files.Length, file);
// try
// {
// var data = reader.ReadFromFile(file);
// data.Values.RemoveAll(m => !commonGenes.Contains(m.Name));
// data.Values.Sort((m1, m2) => m1.Name.CompareTo(m2.Name));
// Assert.AreEqual(commonGenes.Count, data.Values.Count, "Gene count should equal to common gene count : " + Path.GetFileName(file));
// sw.Write(bi.BarCode);
// data.Values.ForEach(m =>
// {
// if (double.IsNaN(m.Value))
// {
// sw.Write("\tNA");
// }
// else
// {
// sw.Write("\t{0:0.00}", Math.Pow(2, m.Value));
// }
// });
// sw.WriteLine();
// }
// catch (Exception ex)
// {
// Console.Error.WriteLine(file + " : " + ex.Message);
// throw;
// }
// }
// }
//}
//Console.WriteLine("Finished!");
//}
private ProbeExpressionFile ReadFile(string rmafile, HashSet<string> commonProbes)
{
var result = new ProbeExpressionFile();
using (var sr = new StreamReader(rmafile))
{
var line = sr.ReadLine();
result.Samples = (from sample in line.Split('\t').Skip(1) select Path.GetFileNameWithoutExtension(sample)).ToArray();
result.Values = new List<Pair<string, string>>();
while ((line = sr.ReadLine()) != null)
{
var name = line.StringBefore("\t");
if (commonProbes.Contains(name))
{
result.Values.Add(new Pair<string, string>(name, line.StringAfter("\t")));
}
}
}
result.Values.Sort((m1, m2) => m1.First.CompareTo(m2.First));
return result;
}