private void ProcessSnapshotWithArrays(short snap) { DebugOut.PrintLine("PROCESSING SNAPSHOT WITH ARRAYS" + snap); // using this only if needed HsmlReader hsmlReader = new HsmlReader(); // and make the directory, just to be safe try { Directory.CreateDirectory(GetSnapDir(outPath, snap)); } catch (IOException e) { System.Console.WriteLine(e.Message); } int curFile = firstSnapshotFile; // index of current read/write file int numFile = 0; if (useHsml) hsmlReader = new HsmlReader(GetHsmlPath(inPath, snap)); if (lastSnapshotFile < firstSnapshotFile) { lastSnapshotFile = firstSnapshotFile; numFile = -1; } while (curFile <= lastSnapshotFile) { DebugOut.PrintLine("..file " + curFile + "/" + lastSnapshotFile); string filename = ""; try { filename = GetSnapFile(inPath, snap, snapshotFilePrefix, curFile); } catch (Exception e) { MessageBox.Show(e.Message); return; } // now load the file SnapFile curSnap = new SnapFile(filename, samplingRate); // and open the stream for writing using (SqlBinaryWriter binwriter = new SqlBinaryWriter(new FileStream(GetSnapDefault(outPath, snap, snapshotFilePrefix, curFile), FileMode.Create))) { Structs[] parts = new Structs[curSnap.numSample]; // now write each particle into the array for (int i = 0; i < curSnap.numSample; i++) { parts[i].x = curSnap.pos[i, 0]; parts[i].y = curSnap.pos[i, 1]; parts[i].z = curSnap.pos[i, 2]; parts[i].vx = curSnap.vel[i, 0]; parts[i].vy = curSnap.vel[i, 1]; parts[i].vz = curSnap.vel[i, 2]; parts[i].snapnum = snap; parts[i].id = curSnap.id[i]; // add in highest-order bit parts[i].id |= ((UInt64)curSnap.nLargeSims[1]) << 32; // make ph-key parts[i].phkey = GetPHKey(parts[i].x, parts[i].y, parts[i].z); // read hsml, if desired if (useHsml) // TODO fix this part for random sampling { parts[i].hsml = hsmlReader.GetHsml(); parts[i].veldisp = hsmlReader.GetVelDisp(); parts[i].density = hsmlReader.GetDensity(); // and advance read pointer hsmlReader.Next(); } } // now sort before writing files // TBD this may not be necessary if the particles // are sorted in the files already. Array.Sort<Structs>(parts, new ParticleComparator()); Cell cell = new Cell(snap); int currentPHkey = -1; for (int i = 0; i < curSnap.numSample; i++) { if (parts[i].phkey != currentPHkey) { if(cell.Count > 0) binwriter.Write(cell); currentPHkey = parts[i].phkey; cell.Init(currentPHkey); } cell.Add(parts[i]); } if (cell.Count > 0) binwriter.Write(cell); // and add a bulk insert DebugOut.AddCommand(GetSnapDefault(outPath, snap, snapshotFilePrefix, curFile), snapshotTable); DebugOut.PrintLine("..wrote " + curSnap.numSample + "/" + curSnap.numTotals[1] + " points"); } // and set numFiles if (numFile == -1) { numFile = (int)curSnap.numSubfiles - firstSnapshotFile + 1; lastSnapshotFile = numFile - 1; } curFile++; // avoid outofmemory errors GC.Collect(); GC.WaitForPendingFinalizers(); } }
private void ProcessSnapshot(short snap) { DebugOut.PrintLine("PROCESSING SNAPSHOT " + snap); // using this only if needed HsmlReader hsmlReader = new HsmlReader(); // and make the directory, just to be safe try { Directory.CreateDirectory(GetSnapDir(outPath,snap)); } catch (IOException e) { System.Console.WriteLine(e.Message); } int curFile = 0; // index of current read/write file int numFiles = 1; // total number of files if (snapshotFile >= 0) { curFile = snapshotFile; numFiles = snapshotFile+1; } if (useHsml) hsmlReader = new HsmlReader(GetHsmlPath(inPath, snap)); while (curFile < numFiles) { DebugOut.PrintLine("..file " + curFile + "/" + numFiles); string filename = ""; try { filename = GetSnapFile(inPath, snap, curFile); } catch (Exception e) { MessageBox.Show(e.Message); return; } // now load the file SnapFile curSnap = new SnapFile(filename, samplingRate); // and open the stream for writing using (SqlBinaryWriter binwriter = new SqlBinaryWriter(new FileStream(GetSnapDefault(outPath, snap, snapshotFilePrefix, curFile), FileMode.Create))) { Structs[] parts = new Structs[curSnap.numSample]; // now write each particle into the array for (int i = 0; i < curSnap.numSample; i++) { parts[i].x = curSnap.pos[i, 0]; parts[i].y = curSnap.pos[i, 1]; parts[i].z = curSnap.pos[i, 2]; parts[i].vx = curSnap.vel[i, 0]; parts[i].vy = curSnap.vel[i, 1]; parts[i].vz = curSnap.vel[i, 2]; parts[i].snapnum = snap; parts[i].id = curSnap.id[i]; // add in highest-order bit parts[i].id |= ((UInt64)curSnap.nLargeSims[1]) << 32; // make ph-key parts[i].phkey = GetPHKey(parts[i].x, parts[i].y, parts[i].z); // read hsml, if desired if (useHsml) // TODO fix this part for random sampling { parts[i].hsml = hsmlReader.GetHsml(); parts[i].veldisp = hsmlReader.GetVelDisp(); parts[i].density = hsmlReader.GetDensity(); // and advance read pointer hsmlReader.Next(); } } // now sort before writing files Array.Sort<Structs>(parts, new ParticleComparator()); // and then write output /* if (useHsml) for (int i = 0; i < curSnap.numSample; i++) parts[i].WriteBinary(binwriter); else for (int i = 0; i < curSnap.numSample; i++) parts[i].WriteBinaryNoHsml(binwriter); */ for (int i = 0; i < curSnap.numSample; i++) binwriter.Write(parts[i]); // and add a bulk insert DebugOut.AddCommand(GetSnapDefault(outPath, snap, snapshotFilePrefix,curFile), snapshotTable); DebugOut.PrintLine("..wrote " + curSnap.numSample + "/" + curSnap.numTotals[1] + " points"); } // and set numFiles numFiles = (int)curSnap.numSubfiles; curFile++; // avoid outofmemory errors GC.Collect(); GC.WaitForPendingFinalizers(); } }
public void ProcessSimDBFofs(int isnap) { DebugOut.PrintLine("PROCESSING FOF Groups from Snapshot " + isnap); // and make the directory, just to be safe try { Directory.CreateDirectory(GetSnapDir(this.outPath, isnap)); } catch (IOException e) { System.Console.WriteLine(e.Message); } int skip = 0; //long skip_sub = 0; int filenumber = 0; string groupTabFile = ""; string groupIDFile = ""; int[] GroupLen = new int[0]; int[] GroupOffset = new int[0]; int[] IDs = new int[0]; bool done = false; int Ntask = 0; int nGroups = 0; int totNGroups = 0; int NIds = 0; while(!done) { groupTabFile = GetSimDBFOFFile(this.inPath, isnap, this.groupTabFilePrefix, filenumber); using (BinaryReader reader = new BinaryReader(new FileStream(groupTabFile, FileMode.Open, FileAccess.Read))) { nGroups = reader.ReadInt32(); NIds = reader.ReadInt32(); totNGroups = reader.ReadInt32(); Ntask = reader.ReadInt32(); if(filenumber == 0) { GroupLen = new int[totNGroups]; GroupOffset = new int[totNGroups]; } if (nGroups > 0) { for (int i = skip; i < skip + nGroups; i++) { GroupLen[i] = reader.ReadInt32(); } for (int i = skip; i < skip + nGroups; i++) { GroupOffset[i] = reader.ReadInt32(); } skip += nGroups; } } //DebugOut.PrintLine("Ngroups: " + nGroups + " totNGroups: " + totNGroups + " NIds: " + NIds + " Ntask: " + Ntask); filenumber++; /*if (filenumber > lastSnapshotFile) { done = true; }*/ if (filenumber == Ntask) { done = true; } } nGroups = 0; totNGroups = 0; NIds = 0; Ntask = 0; done = false; filenumber = 0; skip = 0; while (!done) { groupIDFile = GetSimDBFOFFile(this.inPath, isnap, this.groupIDFilePrefix, filenumber); using (BinaryReader reader = new BinaryReader(new FileStream(groupIDFile, FileMode.Open, FileAccess.Read))) { nGroups = reader.ReadInt32(); NIds = reader.ReadInt32(); totNGroups = reader.ReadInt32(); Ntask = reader.ReadInt32(); //Will I get integer overflow here??? int totNIds = 0; for (int i = 0; i < GroupLen.Length; i++) { totNIds += GroupLen[i]; } if (filenumber == 0) { IDs = new int[totNIds]; } if (NIds > 0) { for (int i = skip; i < skip + NIds; i++) { IDs[i] = reader.ReadInt32(); } skip += NIds; } } //DebugOut.PrintLine("Ngroups: " + nGroups + " totNGroups: " + totNGroups + " NIds: " + NIds + " Ntask: " + Ntask); filenumber++; /*if (filenumber > lastSnapshotFile) { done = true; }*/ if (filenumber == Ntask) { done = true; } } string outfilepath = this.outPath + "//groups_snap" + isnap; using (SqlBinaryWriter binwriter = new SqlBinaryWriter( new FileStream(outfilepath, FileMode.Create))) { for (int i = 0; i < totNGroups; i++) { if (GroupLen[i] > 0) { int[] curGroupIds = new int[GroupLen[i]]; Array.Copy(IDs, GroupOffset[i], curGroupIds, 0, curGroupIds.Length); binwriter.Write(curGroupIds, (short)isnap, i); } } } GC.Collect(); GC.WaitForPendingFinalizers(); DebugOut.PrintLine("Completed writing snapshot: " + isnap); }