public static void GenerateFactTableDataInParallel()
{
List <string> repos = new List <string>
{
"devdiv",
"roslyn",
"VSO",
"VSOWithSrc",
"VsTest"
};
int projectSk = 1000;
int branchSK = 10000;
int buildPipeLineSK = 10000;
var taskList = new List <Task>();
for (; projectSk < 1005; projectSk++)
{
int buildId = 100000;
foreach (var repo in repos)
{
var factTableData = new FactTableData(repo, projectSk, branchSK, buildPipeLineSK, buildId);
var task = new Task(() => GenerateFacttable(factTableData));
taskList.Add(task);
branchSK += 10;
buildPipeLineSK += 10;
buildId += 100000;
}
}
foreach (var repoTask in taskList)
{
repoTask.Start();
}
Task.WaitAll(taskList.ToArray());
}
public static void GenerateFacttable(FactTableData factTableData)
{
string repoName = factTableData.repoName;
int projectSk = factTableData.projectSk;
int startingBranchSK = factTableData.startingBranchSK;
int startingBuildPipeLineSK = factTableData.startingBuildPipeLineSK;
int buildId = factTableData.startingBuildId;
Random randomNumberGenerator = new Random();
//var taskList = new List<Task>();
Dictionary <string, FileProps> durableSK = new Dictionary <string, FileProps>();
long gFileSK = 0;
// read from file
var files = File.ReadAllLines($"{repoName}.txt");
var watch = Stopwatch.StartNew();
foreach (var file in files)
{
durableSK.Add(file, new FileProps
{
FileSK = gFileSK++,
FileDurableSK = gFileSK - 1,
TotalLines = randomNumberGenerator.Next(240, 300),
ExecutableLines = randomNumberGenerator.Next(200, 220),
CoveredLines = randomNumberGenerator.Next(150, 180)
});
}
Console.WriteLine(watch.ElapsedMilliseconds);
Directory.CreateDirectory(string.Format(@"C:\AnalyticsPlayground\Data\{0}", repoName));
string filepath = string.Format(@"C:\AnalyticsPlayground\Data\{1}\FileCoverageFact-default-{0}.csv", DateTime.Now.ToString("yyyyMMddHHmmssfff"), repoName);
//StringBuilder builder = new StringBuilder();
using (var fileStream = new StreamWriter(filepath))
{
fileStream.WriteLine("DateSK,FileSK,DurableFileSK,BuildPipeLineSK,BranchSK,ProjectSK,BuildID,CodeChurn,TotalLines,ExecutableLines,CoveredLines,FullPath");
//builder.AppendLine("DateSK,FileSK,DurableFileSK,BuildPipeLineSK,BranchSK,ProjectSK,BuildID,CodeChurn,TotalLines,ExecutableLines,CoveredLines,FullPath");
fileStream.Flush();
for (int branch = 0; branch < 10; ++branch)
{
var sqlTimeStamp = DateTime.Now.ToString("yyyyMMdd");
var watch2 = Stopwatch.StartNew();
foreach (var file in files)
{
var line = string.Format("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11}",
sqlTimeStamp,
durableSK[file].FileSK,
durableSK[file].FileDurableSK,
startingBuildPipeLineSK + ((branch % 2 == 0) ? 0 : branch),
startingBranchSK + ((branch % 2 == 0) ? 0 : branch),
projectSk,
buildId,
0,
durableSK[file].TotalLines,
durableSK[file].ExecutableLines,
durableSK[file].CoveredLines,
file);
fileStream.WriteLine(line);
//builder.AppendLine(line);
}
fileStream.Flush();
Console.WriteLine(watch2.ElapsedMilliseconds);
}
buildId++;
}
//File.WriteAllText(filepath, builder.ToString());
// var task = new Task(() => SqlBulkInserter.InsertIntoDatabase(filepath));
// taskList.Add(task);
for (int day = 0; day < 30; ++day)
{
//builder = new StringBuilder();
var sqlTimeStamp = Convert.ToInt32(DateTime.Now.ToString("yyyyMMdd")) + day + 1;
filepath = string.Format(@"C:\AnalyticsPlayground\Data\{2}\FileCoverageFact-{0}-{1}.csv", day, DateTime.Now.ToString("yyyyMMddHHmmssfff"), repoName);
using (var fileStream = new StreamWriter(filepath))
{
fileStream.WriteLine("DateSK,FileSK,DurableFileSK,BuildPipeLineSK,BranchSK,ProjectSK,BuildID,CodeChurn,TotalLines,ExecutableLines,CoveredLines,FullPath");
fileStream.Flush();
//builder.AppendLine("DateSK,FileSK,DurableFileSK,BuildPipeLineSK,BranchSK,ProjectSK,BuildID,CodeChurn,TotalLines,ExecutableLines,CoveredLines,FullPath");
for (int build = 0; build <= 10; ++build)
{
for (int branch = 0; branch < 10; ++branch)
{
var randomSkip = new Random().Next(900, 90000);
var randomTake = 100;
var tempFiles = files.ToList();
var resultSet = tempFiles.Skip(randomSkip).Take(randomTake).ToList();
foreach (var result in resultSet)
{
tempFiles.Remove(result);
var fileProp = durableSK[result];
fileProp.FileSK = gFileSK++;
var newFile = result + day + build + branch;
if (!durableSK.ContainsKey(newFile))
{
durableSK.Add(newFile, fileProp);
}
tempFiles.Add(newFile);
}
var watch3 = Stopwatch.StartNew();
var codeChurn = new Random().Next(1, 10);
foreach (var file in tempFiles)
{
var line = string.Format("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11}",
sqlTimeStamp,
durableSK[file].FileSK,
durableSK[file].FileDurableSK,
startingBuildPipeLineSK + ((branch % 2 == 0) ? 0 : branch),
startingBranchSK + ((branch % 2 == 0) ? 0 : branch),
projectSk,
buildId,
codeChurn,
//files.Contains(file) ? 0 : codeChurn,
randomNumberGenerator.Next(240, 300),
randomNumberGenerator.Next(200, 220),
randomNumberGenerator.Next(150, 180),
file);
fileStream.WriteLine(line);
//builder.AppendLine(line);
}
fileStream.Flush();
Console.WriteLine(watch3.ElapsedMilliseconds);
Console.WriteLine("Done buildID {0}, day {1}, build {2}, branch {3}", buildId, day, build, branch);
files.Union(tempFiles);
}
buildId++;
}
}
//File.WriteAllText(filepath, builder.ToString());
//var insertTask = new Task(() => SqlBulkInserter.InsertIntoDatabase(filepath));
//taskList.Add(insertTask);
}
// foreach (var sqlBuildInsertTask in taskList)
// {
// sqlBuildInsertTask.Start();
// }
// Task.WaitAll(taskList.ToArray());
}
