public void ProcessIndraFFTData(int isnap)
{
DebugOut.PrintLine("PROCESSING FOF Groups from Snapshot " + isnap);
int L = 128; //TODO eventually, magic number, assume that there will always
// be 128 modes?
int lhalf = L / 2;
float[, ,] fft_re = new float[lhalf + +1, L + 1, L + 1];
float[, ,] fft_im = new float[lhalf + +1, L + 1, L + 1];
//float[, ,] fft_re = new float[L + 1, L + 1, lhalf + +1];
//float[, ,] fft_im = new float[L + 1, L + 1, lhalf + +1];
int nsize = 0;
double time2 = 0.0;
string fftfile = GetIndraFFTFILE(inPath, isnap, filePrefix, fileExtension);
DebugOut.PrintLine(fftfile);
using (BinaryReader reader = new BinaryReader(new FileStream(fftfile, FileMode.Open, FileAccess.Read)))
{
time2 = reader.ReadDouble();
//nsize = reader.ReadSingle();
nsize = reader.ReadInt32();
for (int y = 0; y <= L; y++)
{
for (int x = 0; x <= L; x++)
{
for (int z = 0; z <= lhalf; z++)
{
fft_re[z, x, y] = reader.ReadSingle();
}
}
}
for (int y = 0; y <= L; y++)
{
for (int x = 0; x <= L; x++)
{
for (int z = 0; z <= lhalf; z++)
{
fft_im[z, x, y] = reader.ReadSingle();
}
}
}
}
string outfilepath = outPath + "\\" + "trans_" + filePrefix + isnap;
using (SqlBinaryWriter binwriter = new SqlBinaryWriter(
new FileStream(outfilepath, FileMode.Create)))
{
binwriter.WriteFFTModes((short)isnap, time2, nsize, fft_re, fft_im);
}
GC.Collect();
GC.WaitForPendingFinalizers();
DebugOut.PrintLine("Completed writing snapshot: " + isnap);
}
public void ProcessIndraFofs(int isnap)
{
DebugOut.PrintLine("PROCESSING FOF Groups from Snapshot " + isnap);
int skip = 0;
//long skip_sub = 0;
int filenumber = 0;
string groupTabFile = "";
string groupIDFile = "";
int[] GroupLen = new int[0];
int[] GroupOffset = new int[0];
long[] IDs = new long[0];
bool done = false;
int Ntask = 0;
int nGroups = 0;
int totNGroups = 0;
int NIds = 0;
FOFSummary summary = new FOFSummary();
summary.start = DateTime.Now;
while (!done)
{
groupTabFile = GetSimDBFOFFile(this.inPath, isnap, this.groupTabFilePrefix, filenumber);
using (BinaryReader reader = new BinaryReader(new FileStream(groupTabFile, FileMode.Open, FileAccess.Read)))
{
nGroups = reader.ReadInt32();
NIds = reader.ReadInt32();
totNGroups = reader.ReadInt32();
Ntask = reader.ReadInt32();
if (filenumber == 0)
{
GroupLen = new int[totNGroups];
GroupOffset = new int[totNGroups];
}
if (nGroups > 0)
{
for (int i = skip; i < skip + nGroups; i++)
{
GroupLen[i] = reader.ReadInt32();
}
for (int i = skip; i < skip + nGroups; i++)
{
GroupOffset[i] = reader.ReadInt32();
}
skip += nGroups;
}
}
DebugOut.PrintLine("Ngroups: " + nGroups + " totNGroups: " + totNGroups + " NIds: " + NIds + " Ntask: " + Ntask);
filenumber++;
//if (filenumber > lastSnapshotFile)
// {
// done = true;
//}
if (filenumber == Ntask)
{
done = true;
}
}
nGroups = 0;
totNGroups = 0;
NIds = 0;
Ntask = 0;
done = false;
filenumber = 0;
skip = 0;
while (!done)
{
groupIDFile = GetSimDBFOFFile(this.inPath, isnap, this.groupIDFilePrefix, filenumber);
using (BinaryReader reader = new BinaryReader(new FileStream(groupIDFile, FileMode.Open, FileAccess.Read)))
{
nGroups = reader.ReadInt32();
NIds = reader.ReadInt32();
totNGroups = reader.ReadInt32();
Ntask = reader.ReadInt32();
int totNIds = 0;
for (int i = 0; i < GroupLen.Length; i++)
{
totNIds += GroupLen[i];
}
if (filenumber == 0)
{
IDs = new long[totNIds];
}
if (NIds > 0)
{
for (int i = skip; i < skip + NIds; i++)
{
IDs[i] = reader.ReadInt64();
IDs[i] &= ((((long)1) << 34) - 1);
}
skip += NIds;
}
}
//DebugOut.PrintLine("Ngroups: " + nGroups + " totNGroups: " + totNGroups + " NIds: " + NIds + " Ntask: " + Ntask);
filenumber++;
//if (filenumber > lastSnapshotFile)
// {
// done = true;
// }
if (filenumber == Ntask)
{
done = true;
}
}
string outfilepath = this.outPath + "//groups_snap" + isnap;
using (SqlBinaryWriter binwriter = new SqlBinaryWriter(
new FileStream(outfilepath, FileMode.Create)))
{
for (int i = 0; i < totNGroups; i++)
{
if (GroupLen[i] > 0)
{
long[] curGroupIds = new long[GroupLen[i]];
Array.Copy(IDs, GroupOffset[i], curGroupIds, 0, curGroupIds.Length);
// Now add the ID to a bloom filter
BloomFilter.Filter <long> filter = new Filter <long>(LoaderConstants.expectedSize);
for (int j = 0; j < curGroupIds.Length; j++)
{
filter.Add(curGroupIds[j]);
}
binwriter.WriteFoFGroup(curGroupIds, (short)isnap, i, filter);
}
}
}
summary.end = DateTime.Now;
summary.duration = summary.end - summary.start;
summary.numGroups = totNGroups;
globals.summary.fofSummary = summary;
GC.Collect();
GC.WaitForPendingFinalizers();
DebugOut.PrintLine("Completed writing snapshot: " + isnap);
}
private void ProcessSnapshotWithArrays(short snap)
{
//DebugOut.PrintLine("PROCESSING SNAPSHOT WITH ARRAYS" + snap);
// using this only if needed
List <SnapFileSummary> results = new List <SnapFileSummary>();
//and make the directory, just to be safe
try
{
Directory.CreateDirectory(outPath);
}
catch (IOException e)
{
globals.summary.addError(e.Message);
return;
}
int curFile = firstSnapshotFile; // index of current read/write file
int numFile = 0;
if (lastSnapshotFile < firstSnapshotFile)
{
lastSnapshotFile = firstSnapshotFile;
numFile = -1;
}
while (curFile <= lastSnapshotFile)
{
//DebugOut.PrintLine("..file " + curFile + "/" + lastSnapshotFile);
Console.WriteLine("..file " + curFile + "/" + lastSnapshotFile);
SnapFileSummary currentFileSummary = new SnapFileSummary();
results.Add(currentFileSummary);
currentFileSummary.start = DateTime.Now;
currentFileSummary.fileNumber = curFile;
string filename = "";
try
{
filename = GetSnapFile(inPath, snap, snapshotFilePrefix, curFile);
currentFileSummary.inFileName = filename;
}
catch (Exception e)
{
currentFileSummary.badStatus = true;
currentFileSummary.statusMessage = e.Message;
globals.summary.setFileSummaries(results);
return;
}
// now load the file
try
{
SnapFile curSnap = new SnapFile(filename);
// and open the stream for writing
using (SqlBinaryWriter binwriter = new SqlBinaryWriter(new FileStream(GetSnapDefault(outPath, snap, snapshotFilePrefix, curFile), FileMode.Create)))
{
Structs[] parts = new Structs[curSnap.numSample];
//record destination path in file summary
currentFileSummary.outFileName = GetSnapDefault(outPath, snap, snapshotFilePrefix, curFile);
// now write each particle into the array
for (int i = 0; i < curSnap.numSample; i++)
{
parts[i].x = curSnap.pos[i, 0];
parts[i].y = curSnap.pos[i, 1];
parts[i].z = curSnap.pos[i, 2];
parts[i].vx = curSnap.vel[i, 0];
parts[i].vy = curSnap.vel[i, 1];
parts[i].vz = curSnap.vel[i, 2];
parts[i].snapnum = snap;
parts[i].id = curSnap.id[i];
// add in highest-order bit
parts[i].id |= ((UInt64)curSnap.nLargeSims[1]) << 32;
// make ph-key
parts[i].phkey = GetPHKey(parts[i].x, parts[i].y, parts[i].z);
}
// now sort before writing files
// TODO
//TBD this may not be necessary if the particles
// are sorted in the files already.
Array.Sort <Structs>(parts, new ParticleComparator());
Cell cell = new Cell(snap);
int currentPHkey = -1;
for (int i = 0; i < curSnap.numSample; i++)
{
if (parts[i].phkey != currentPHkey)
{
if (cell.Count > 0)
{
binwriter.WriteCell(cell);
}
currentPHkey = parts[i].phkey;
cell.Init(currentPHkey);
}
cell.AddToCell(parts[i]);
}
if (cell.Count > 0)
{
binwriter.WriteCell(cell);
}
// and add a bulk insert
//DebugOut.AddCommand(GetSnapDefault(outPath, snap, snapshotFilePrefix, curFile), snapshotTable);
Console.WriteLine("..wrote " + curSnap.numSample + "/" + curSnap.numTotals[1] + " points");
DebugOut.PrintLine("..wrote " + curSnap.numSample + "/" + curSnap.numTotals[1] + " points");
}
// and set numFiles
if (numFile == -1)
{
numFile = (int)curSnap.numSubfiles - firstSnapshotFile;
lastSnapshotFile = (int)curSnap.numSubfiles - 1;
}
currentFileSummary.end = DateTime.Now;
currentFileSummary.duration = results[curFile].end - results[curFile].start;
currentFileSummary.numParticles = curSnap.numSample;
if (curSnap.numSample < (int)(0.01 * LoaderConstants.particlesPerSnap))
{
currentFileSummary.warning = true;
currentFileSummary.warningMessage = "Less than 1% of particles in snapshot in this file";
}
if (curSnap.numSample > LoaderConstants.particlesPerSnap)
{
currentFileSummary.warning = true;
currentFileSummary.warningMessage = "More particles in file than are supposed to be in snapshot";
}
if (currentFileSummary.outFileName != null)
{
globals.summary.AddSnapBCPCommand(currentFileSummary.outFileName);
}
curFile++;
// avoid outofmemory errors
GC.Collect();
GC.WaitForPendingFinalizers();
}
catch (Exception e)
{
currentFileSummary.badStatus = true;
currentFileSummary.statusMessage = e.Message;
globals.summary.setFileSummaries(results);
return;
}
}
globals.summary.setFileSummaries(results);
}
