private void ProcessSnapshot(short snap)
{
DebugOut.PrintLine("PROCESSING SNAPSHOT " + snap);
// using this only if needed
HsmlReader hsmlReader = new HsmlReader();
// and make the directory, just to be safe
try
{
Directory.CreateDirectory(GetSnapDir(outPath,snap));
}
catch (IOException e)
{
System.Console.WriteLine(e.Message);
}
int curFile = 0; // index of current read/write file
int numFiles = 1; // total number of files
if (snapshotFile >= 0)
{
curFile = snapshotFile;
numFiles = snapshotFile+1;
}
if (useHsml)
hsmlReader = new HsmlReader(GetHsmlPath(inPath, snap));
while (curFile < numFiles)
{
DebugOut.PrintLine("..file " + curFile + "/" + numFiles);
string filename = "";
try
{
filename = GetSnapFile(inPath, snap, curFile);
}
catch (Exception e)
{
MessageBox.Show(e.Message);
return;
}
// now load the file
SnapFile curSnap = new SnapFile(filename, samplingRate);
// and open the stream for writing
using (SqlBinaryWriter binwriter = new SqlBinaryWriter(new FileStream(GetSnapDefault(outPath, snap, snapshotFilePrefix, curFile), FileMode.Create)))
{
Structs[] parts = new Structs[curSnap.numSample];
// now write each particle into the array
for (int i = 0; i < curSnap.numSample; i++)
{
parts[i].x = curSnap.pos[i, 0];
parts[i].y = curSnap.pos[i, 1];
parts[i].z = curSnap.pos[i, 2];
parts[i].vx = curSnap.vel[i, 0];
parts[i].vy = curSnap.vel[i, 1];
parts[i].vz = curSnap.vel[i, 2];
parts[i].snapnum = snap;
parts[i].id = curSnap.id[i];
// add in highest-order bit
parts[i].id |= ((UInt64)curSnap.nLargeSims[1]) << 32;
// make ph-key
parts[i].phkey = GetPHKey(parts[i].x, parts[i].y, parts[i].z);
// read hsml, if desired
if (useHsml) // TODO fix this part for random sampling
{
parts[i].hsml = hsmlReader.GetHsml();
parts[i].veldisp = hsmlReader.GetVelDisp();
parts[i].density = hsmlReader.GetDensity();
// and advance read pointer
hsmlReader.Next();
}
}
// now sort before writing files
Array.Sort<Structs>(parts, new ParticleComparator());
// and then write output
/*
if (useHsml)
for (int i = 0; i < curSnap.numSample; i++)
parts[i].WriteBinary(binwriter);
else
for (int i = 0; i < curSnap.numSample; i++)
parts[i].WriteBinaryNoHsml(binwriter);
*/
for (int i = 0; i < curSnap.numSample; i++)
binwriter.Write(parts[i]);
// and add a bulk insert
DebugOut.AddCommand(GetSnapDefault(outPath, snap, snapshotFilePrefix,curFile), snapshotTable);
DebugOut.PrintLine("..wrote " + curSnap.numSample + "/" + curSnap.numTotals[1] + " points");
}
// and set numFiles
numFiles = (int)curSnap.numSubfiles;
curFile++;
// avoid outofmemory errors
GC.Collect();
GC.WaitForPendingFinalizers();
}
}
private void ProcessSnapshotWithArrays(short snap)
{
//DebugOut.PrintLine("PROCESSING SNAPSHOT WITH ARRAYS" + snap);
// using this only if needed
bool writeParticlesBool = true;
bool writeIndexBool = true;
List<SnapFileSummary> results = new List<SnapFileSummary>();
//and make the directory, just to be safe
try
{
Directory.CreateDirectory(outPath);
}
catch (IOException e)
{
globals.summary.addError(e.Message);
return;
}
int curFile = firstSnapshotFile; // index of current read/write file
int numFile = 0;
if (lastSnapshotFile < firstSnapshotFile)
{
lastSnapshotFile = firstSnapshotFile;
numFile = -1;
}
while (curFile <= lastSnapshotFile)
{
//DebugOut.PrintLine("..file " + curFile + "/" + lastSnapshotFile);
Console.WriteLine("..file " + curFile + "/" + lastSnapshotFile);
SnapFileSummary currentFileSummary = new SnapFileSummary();
results.Add(currentFileSummary);
currentFileSummary.start = DateTime.Now;
currentFileSummary.fileNumber = curFile;
string filename = "";
try
{
filename = GetSnapFile(inPath, snap, snapshotFilePrefix, curFile);
currentFileSummary.inFileName = filename;
}
catch (Exception e)
{
currentFileSummary.badStatus = true;
currentFileSummary.statusMessage = e.Message;
globals.summary.setFileSummaries(results);
return;
}
// now load the file
#region mainloop
try
{
SnapFile curSnap = new SnapFile(filename);
// and open the stream for writing
using (SqlBinaryWriter particleBinWriter = new SqlBinaryWriter(new FileStream(GetSnapDefault(outPath, snap, snapshotFilePrefix, curFile), FileMode.Create)),
indexBinWriter = new SqlBinaryWriter(new FileStream(GetSnapDefault(outPath, snap, "rev_index_", curFile), FileMode.Create)))
{
Structs[] parts = new Structs[curSnap.numSample];
//record destination path in file summary
currentFileSummary.outFileName = GetSnapDefault(outPath, snap, snapshotFilePrefix, curFile);
currentFileSummary.indexFileName = GetSnapDefault(outPath, snap, "rev_index_", curFile);
// now write each particle into the array
for (int i = 0; i < curSnap.numSample; i++)
{
parts[i].x = curSnap.pos[i, 0];
parts[i].y = curSnap.pos[i, 1];
parts[i].z = curSnap.pos[i, 2];
parts[i].vx = curSnap.vel[i, 0];
parts[i].vy = curSnap.vel[i, 1];
parts[i].vz = curSnap.vel[i, 2];
parts[i].snapnum = snap;
parts[i].id = curSnap.id[i];
// add in highest-order bit
parts[i].id |= ((UInt64)curSnap.nLargeSims[1]) << 32;
// make ph-key
parts[i].phkey = GetPHKey(parts[i].x, parts[i].y, parts[i].z);
}
// now sort before writing files
// TODO
// TBD this may not be necessary if the particles
// are sorted in the files already.
Array.Sort<Structs>(parts, new ParticleComparator());
/*
* Sue: This is where you will want to put in a partitioning key
* Every time there is a call to particleBinWriter.WriteCell(cell), a single row is written.
* The best way to do it would be to just add an argument to the WriteCell method
* which is in SqlBinaryWriter.cs on line 113
*/
Cell cell = new Cell(snap);
int currentPHkey = -1;
LoaderParamSingleton pars = LoaderParamSingleton.getInstance();
for (int i = 0; i < curSnap.numSample; i++)
{
if (parts[i].phkey != currentPHkey)
{
if (cell.Count > 0)
{
// TODO(INDEX) create Cell index here
if (writeParticlesBool)
{
particleBinWriter.WriteCell(cell);
}
//*********************************************************
if (writeIndexBool)
{
List<ReverseIndexEntry> index = cell.createIndex();
indexBinWriter.WriteReverseIndex(index);
}
//*********************************************************
/*
if (pars.firstSnapLoaded)
{
foreach (ReverseIndexEntry entry in index)
{
entry.WriteBinaryFirstSnap(indexBinWriter);
}
}
else
{
foreach (ReverseIndexEntry entry in index)
{
entry.WriteBinaryForMerge(indexBinWriter);
}
}*/
}
currentPHkey = parts[i].phkey;
cell.Init(currentPHkey);
}
cell.AddToCell(parts[i]);
}
if (cell.Count > 0)
{
if (writeParticlesBool)
{
particleBinWriter.WriteCell(cell);
}
//*********************************************************
if (writeIndexBool)
{
List<ReverseIndexEntry> index = cell.createIndex();
indexBinWriter.WriteReverseIndex(index);
}
//*********************************************************
/*if (pars.firstSnapLoaded)
{
foreach (ReverseIndexEntry entry in index)
{
entry.WriteBinaryFirstSnap(indexBinWriter);
}
}
else
{
foreach (ReverseIndexEntry entry in index)
{
entry.WriteBinaryForMerge(indexBinWriter);
}
}
*/
}
// and add a bulk insert
//DebugOut.AddCommand(GetSnapDefault(outPath, snap, snapshotFilePrefix, curFile), snapshotTable);
//Console.WriteLine("..wrote " + curSnap.numSample + "/" + curSnap.numTotals[1] + " points");
//DebugOut.PrintLine("..wrote " + curSnap.numSample + "/" + curSnap.numTotals[1] + " points");
}
// and set numFiles
if (numFile == -1)
{
numFile = (int)curSnap.numSubfiles - firstSnapshotFile;
lastSnapshotFile = (int)curSnap.numSubfiles - 1;
}
currentFileSummary.end = DateTime.Now;
currentFileSummary.duration = results[curFile].end - results[curFile].start;
currentFileSummary.numParticles = curSnap.numSample;
if (curSnap.numSample < (int)(0.01 * LoaderParamSingleton.getInstance().particlesPerSnap))
{
currentFileSummary.warning = true;
currentFileSummary.warningMessage = "Less than 1% of particles in snapshot in this file";
}
if (curSnap.numSample > LoaderParamSingleton.getInstance().particlesPerSnap)
{
currentFileSummary.warning = true;
currentFileSummary.warningMessage = "More particles in file than are supposed to be in snapshot";
}
if(currentFileSummary.outFileName != null)
{
globals.summary.AddSnapBCPCommand(currentFileSummary.outFileName);
globals.summary.AddIndexBCPCommand(currentFileSummary.indexFileName);
}
curFile++;
// avoid outofmemory errors
GC.Collect();
GC.WaitForPendingFinalizers();
}
catch (Exception e)
{
currentFileSummary.badStatus = true;
currentFileSummary.statusMessage = e.Message;
globals.summary.setFileSummaries(results);
return;
}
#endregion mainloop
}
globals.summary.setFileSummaries(results);
}
private void ProcessSnapshot(short snap)
{
DebugOut.PrintLine("PROCESSING SNAPSHOT WITHOUT ARRAYS " + snap);
// using this only if needed
List<SnapFileSummary> results = new List<SnapFileSummary>();
// and make the directory, just to be safe
try
{
//Directory.CreateDirectory(GetSnapDir(outPath, snap));
Directory.CreateDirectory(outPath);
}
//SUMMARY TODO
catch (IOException e)
{
System.Console.WriteLine(e.Message);
}
int curFile = firstSnapshotFile; // index of current read/write file
int numFile = 0;
if (lastSnapshotFile < firstSnapshotFile)
{
lastSnapshotFile = firstSnapshotFile;
numFile = -1;
}
while (curFile <= lastSnapshotFile)
{
//DebugOut.PrintLine("..file " + curFile + "/" + lastSnapshotFile);
Console.WriteLine("..file " + curFile + "/" + lastSnapshotFile);
SnapFileSummary currentFileSummary = new SnapFileSummary();
results.Add(currentFileSummary);
currentFileSummary.start = DateTime.Now;
string filename = "";
try
{
filename = GetSnapFile(inPath, snap, snapshotFilePrefix, curFile);
currentFileSummary.inFileName = filename;
}
catch (Exception e)
{
Console.WriteLine(e.Message);
currentFileSummary.badStatus = true;
currentFileSummary.statusMessage = e.Message;
return;
}
// now load the file
SnapFile curSnap = new SnapFile(filename);
// and open the stream for writing
using (BinaryWriter binwriter = new BinaryWriter(new FileStream(GetSnapDefault(outPath, snap, snapshotFilePrefix, curFile), FileMode.Create)))
{
currentFileSummary.outFileName = GetSnapDefault(outPath, snap, snapshotFilePrefix, curFile);
Structs[] parts = new Structs[curSnap.numSample];
// now write each particle into the array
for (int i = 0; i < curSnap.numSample; i++)
{
parts[i].x = curSnap.pos[i, 0];
parts[i].y = curSnap.pos[i, 1];
parts[i].z = curSnap.pos[i, 2];
parts[i].vx = curSnap.vel[i, 0];
parts[i].vy = curSnap.vel[i, 1];
parts[i].vz = curSnap.vel[i, 2];
parts[i].snapnum = snap;
parts[i].id = curSnap.id[i];
// add in highest-order bit
parts[i].id |= ((UInt64)curSnap.nLargeSims[1]) << 32;
// make ph-key
parts[i].phkey = GetPHKey(parts[i].x, parts[i].y, parts[i].z);
// read hsml, if desired
}
// now sort before writing files
Array.Sort<Structs>(parts, new ParticleComparator());
// and then write output
for (int i = 0; i < curSnap.numSample; i++)
parts[i].WriteBinaryNoHsml(binwriter);
// and add a bulk insert
//TODO sql commands file
DebugOut.AddCommand(GetSnapDefault(outPath, snap, snapshotFilePrefix, curFile), snapshotTable);
DebugOut.PrintLine("..wrote " + curSnap.numSample + "/" + curSnap.numTotals[1] + " points");
}
// and set numFiles
if (numFile == -1)
{
numFile = (int)curSnap.numSubfiles - firstSnapshotFile + 1;
lastSnapshotFile = numFile - 1;
}
currentFileSummary.end = DateTime.Now;
currentFileSummary.duration = results[curFile].start - results[curFile].end;
currentFileSummary.numParticles = curSnap.numSample;
curFile++;
// avoid outofmemory errors
GC.Collect();
GC.WaitForPendingFinalizers();
}
globals.summary.setFileSummaries(results);
}
private void ProcessSnapshotWithArrays(short snap)
{
//DebugOut.PrintLine("PROCESSING SNAPSHOT WITH ARRAYS" + snap);
// using this only if needed
List<SnapFileSummary> results = new List<SnapFileSummary>();
//and make the directory, just to be safe
try
{
Directory.CreateDirectory(outPath);
}
catch (IOException e)
{
globals.summary.addError(e.Message);
return;
}
int curFile = firstSnapshotFile; // index of current read/write file
int numFile = 0;
if (lastSnapshotFile < firstSnapshotFile)
{
lastSnapshotFile = firstSnapshotFile;
numFile = -1;
}
while (curFile <= lastSnapshotFile)
{
//DebugOut.PrintLine("..file " + curFile + "/" + lastSnapshotFile);
Console.WriteLine("..file " + curFile + "/" + lastSnapshotFile);
SnapFileSummary currentFileSummary = new SnapFileSummary();
results.Add(currentFileSummary);
currentFileSummary.start = DateTime.Now;
currentFileSummary.fileNumber = curFile;
string filename = "";
try
{
filename = GetSnapFile(inPath, snap, snapshotFilePrefix, curFile);
currentFileSummary.inFileName = filename;
}
catch (Exception e)
{
currentFileSummary.badStatus = true;
currentFileSummary.statusMessage = e.Message;
globals.summary.setFileSummaries(results);
return;
}
// now load the file
try
{
SnapFile curSnap = new SnapFile(filename);
// and open the stream for writing
using (SqlBinaryWriter binwriter = new SqlBinaryWriter(new FileStream(GetSnapDefault(outPath, snap, snapshotFilePrefix, curFile), FileMode.Create)))
{
Structs[] parts = new Structs[curSnap.numSample];
//record destination path in file summary
currentFileSummary.outFileName = GetSnapDefault(outPath, snap, snapshotFilePrefix, curFile);
// now write each particle into the array
for (int i = 0; i < curSnap.numSample; i++)
{
parts[i].x = curSnap.pos[i, 0];
parts[i].y = curSnap.pos[i, 1];
parts[i].z = curSnap.pos[i, 2];
parts[i].vx = curSnap.vel[i, 0];
parts[i].vy = curSnap.vel[i, 1];
parts[i].vz = curSnap.vel[i, 2];
parts[i].snapnum = snap;
parts[i].id = curSnap.id[i];
// add in highest-order bit
parts[i].id |= ((UInt64)curSnap.nLargeSims[1]) << 32;
// make ph-key
parts[i].phkey = GetPHKey(parts[i].x, parts[i].y, parts[i].z);
}
// now sort before writing files
// TODO
//TBD this may not be necessary if the particles
// are sorted in the files already.
Array.Sort<Structs>(parts, new ParticleComparator());
Cell cell = new Cell(snap);
int currentPHkey = -1;
for (int i = 0; i < curSnap.numSample; i++)
{
if (parts[i].phkey != currentPHkey)
{
if (cell.Count > 0)
binwriter.WriteCell(cell);
currentPHkey = parts[i].phkey;
cell.Init(currentPHkey);
}
cell.AddToCell(parts[i]);
}
if (cell.Count > 0)
binwriter.WriteCell(cell);
// and add a bulk insert
//DebugOut.AddCommand(GetSnapDefault(outPath, snap, snapshotFilePrefix, curFile), snapshotTable);
Console.WriteLine("..wrote " + curSnap.numSample + "/" + curSnap.numTotals[1] + " points");
DebugOut.PrintLine("..wrote " + curSnap.numSample + "/" + curSnap.numTotals[1] + " points");
}
// and set numFiles
if (numFile == -1)
{
numFile = (int)curSnap.numSubfiles - firstSnapshotFile;
lastSnapshotFile = (int)curSnap.numSubfiles - 1;
}
currentFileSummary.end = DateTime.Now;
currentFileSummary.duration = results[curFile].end - results[curFile].start;
currentFileSummary.numParticles = curSnap.numSample;
if (curSnap.numSample < (int)(0.01 * LoaderConstants.particlesPerSnap))
{
currentFileSummary.warning = true;
currentFileSummary.warningMessage = "Less than 1% of particles in snapshot in this file";
}
if (curSnap.numSample > LoaderConstants.particlesPerSnap)
{
currentFileSummary.warning = true;
currentFileSummary.warningMessage = "More particles in file than are supposed to be in snapshot";
}
if(currentFileSummary.outFileName != null)
{
globals.summary.AddSnapBCPCommand(currentFileSummary.outFileName);
}
curFile++;
// avoid outofmemory errors
GC.Collect();
GC.WaitForPendingFinalizers();
}
catch (Exception e)
{
currentFileSummary.badStatus = true;
currentFileSummary.statusMessage = e.Message;
globals.summary.setFileSummaries(results);
return;
}
}
globals.summary.setFileSummaries(results);
}
private void ProcessSnapshotWithArrays(short snap)
{
DebugOut.PrintLine("PROCESSING SNAPSHOT WITH ARRAYS" + snap);
// using this only if needed
HsmlReader hsmlReader = new HsmlReader();
// and make the directory, just to be safe
try
{
Directory.CreateDirectory(GetSnapDir(outPath, snap));
}
catch (IOException e)
{
System.Console.WriteLine(e.Message);
}
int curFile = firstSnapshotFile; // index of current read/write file
int numFile = 0;
if (useHsml)
hsmlReader = new HsmlReader(GetHsmlPath(inPath, snap));
if (lastSnapshotFile < firstSnapshotFile)
{
lastSnapshotFile = firstSnapshotFile;
numFile = -1;
}
while (curFile <= lastSnapshotFile)
{
DebugOut.PrintLine("..file " + curFile + "/" + lastSnapshotFile);
string filename = "";
try
{
filename = GetSnapFile(inPath, snap, snapshotFilePrefix, curFile);
}
catch (Exception e)
{
MessageBox.Show(e.Message);
return;
}
// now load the file
SnapFile curSnap = new SnapFile(filename, samplingRate);
// and open the stream for writing
using (SqlBinaryWriter binwriter = new SqlBinaryWriter(new FileStream(GetSnapDefault(outPath, snap, snapshotFilePrefix, curFile), FileMode.Create)))
{
Structs[] parts = new Structs[curSnap.numSample];
// now write each particle into the array
for (int i = 0; i < curSnap.numSample; i++)
{
parts[i].x = curSnap.pos[i, 0];
parts[i].y = curSnap.pos[i, 1];
parts[i].z = curSnap.pos[i, 2];
parts[i].vx = curSnap.vel[i, 0];
parts[i].vy = curSnap.vel[i, 1];
parts[i].vz = curSnap.vel[i, 2];
parts[i].snapnum = snap;
parts[i].id = curSnap.id[i];
// add in highest-order bit
parts[i].id |= ((UInt64)curSnap.nLargeSims[1]) << 32;
// make ph-key
parts[i].phkey = GetPHKey(parts[i].x, parts[i].y, parts[i].z);
// read hsml, if desired
if (useHsml) // TODO fix this part for random sampling
{
parts[i].hsml = hsmlReader.GetHsml();
parts[i].veldisp = hsmlReader.GetVelDisp();
parts[i].density = hsmlReader.GetDensity();
// and advance read pointer
hsmlReader.Next();
}
}
// now sort before writing files
// TBD this may not be necessary if the particles
// are sorted in the files already.
Array.Sort<Structs>(parts, new ParticleComparator());
Cell cell = new Cell(snap);
int currentPHkey = -1;
for (int i = 0; i < curSnap.numSample; i++)
{
if (parts[i].phkey != currentPHkey)
{
if(cell.Count > 0)
binwriter.Write(cell);
currentPHkey = parts[i].phkey;
cell.Init(currentPHkey);
}
cell.Add(parts[i]);
}
if (cell.Count > 0)
binwriter.Write(cell);
// and add a bulk insert
DebugOut.AddCommand(GetSnapDefault(outPath, snap, snapshotFilePrefix, curFile), snapshotTable);
DebugOut.PrintLine("..wrote " + curSnap.numSample + "/" + curSnap.numTotals[1] + " points");
}
// and set numFiles
if (numFile == -1)
{
numFile = (int)curSnap.numSubfiles - firstSnapshotFile + 1;
lastSnapshotFile = numFile - 1;
}
curFile++;
// avoid outofmemory errors
GC.Collect();
GC.WaitForPendingFinalizers();
}
}
public void startColliding()
{
String filename = prefix + j;
SnapFile curSnap = new SnapFile(filename);
Console.Out.WriteLine("Processing file " + j);
Console.Out.Flush();
Structs[] parts = new Structs[curSnap.numSample];
// now write each particle into the array
for (int i = 0; i < curSnap.numSample; i++)
{
parts[i].x = curSnap.pos[i, 0];
parts[i].y = curSnap.pos[i, 1];
parts[i].z = curSnap.pos[i, 2];
parts[i].vx = curSnap.vel[i, 0];
parts[i].vy = curSnap.vel[i, 1];
parts[i].vz = curSnap.vel[i, 2];
parts[i].snapnum = snap;
parts[i].id = curSnap.id[i];
// add in highest-order bit
parts[i].id |= ((UInt64)curSnap.nLargeSims[1]) << 32;
// make ph-key
parts[i].phkey = GetPHKey(parts[i].x, parts[i].y, parts[i].z);
}
// now sort before writing files
// TODO
//TBD this may not be necessary if the particles
// are sorted in the files already.
Array.Sort<Structs>(parts, new ParticleComparator());
Cell cell = new Cell(snap);
int currentPHkey = -1;
//for (int i = 0; i < 10000; i++)
for (int i = 0; i < curSnap.numSample; i++)
{
if (parts[i].phkey != currentPHkey)
{
if (cell.Count > 0)
processCell(cell, datapoints);
currentPHkey = parts[i].phkey;
cell.Init(currentPHkey);
}
cell.AddToCell(parts[i]);
if (i % 100000 == 0)
{
Console.Out.WriteLine("Particle " + i + " from file " + j);
Console.Out.Flush();
}
}
if (cell.Count > 0)
processCell(cell, datapoints);
}