public void command(int narg, string[] args)
{
string[] arg = new string[narg];
Array.Copy(args, 1, arg, 0, narg);
if (narg < 1)
{
sparta.error.all("Illegal run command");
}
if (sparta.grid.exist == 0)
{
sparta.error.all("Run command before grid is defined");
}
if (sparta.grid.exist_ghost == 0)
{
sparta.error.all("Run command before grid ghost cells are defined");
}
bigint nsteps_input = Int64.Parse(arg[0]);
// parse optional args
int uptoflag = 0;
int startflag = 0;
int stopflag = 0;
bigint start = 0, stop = 0;
int preflag = 1;
int postflag = 1;
int nevery = 0;
int ncommands = 0;
int first = 0, last = 0;
int iarg = 1;
while (iarg < narg)
{
switch (arg[iarg])
{
case "upto":
if (iarg + 1 > narg)
{
sparta.error.all("Illegal run command");
}
uptoflag = 1;
iarg += 1;
break;
case "start":
if (iarg + 2 > narg)
{
sparta.error.all("Illegal run command");
}
startflag = 1;
start = Int64.Parse(arg[iarg + 1]);
iarg += 2;
break;
case "stop":
if (iarg + 2 > narg)
{
sparta.error.all("Illegal run command");
}
stopflag = 1;
stop = Int64.Parse(arg[iarg + 1]);
iarg += 2;
break;
case "pre":
if (iarg + 2 > narg)
{
sparta.error.all("Illegal run command");
}
if (string.Equals(arg[iarg + 1], "no"))
{
preflag = 0;
}
else if (string.Equals(arg[iarg + 1], "yes"))
{
preflag = 1;
}
else
{
sparta.error.all("Illegal run command");
}
iarg += 2;
break;
case "post":
if (iarg + 2 > narg)
{
sparta.error.all("Illegal run command");
}
if (string.Equals(arg[iarg + 1], "no"))
{
postflag = 0;
}
else if (string.Equals(arg[iarg + 1], "yes"))
{
postflag = 1;
}
else
{
sparta.error.all("Illegal run command");
}
iarg += 2;
// all remaining args are commands
// first,last = arg index of first/last commands
// set ncommands = 0 if single command and it is NULL
break;
case "every":
if (iarg + 3 > narg)
{
sparta.error.all("Illegal run command");
}
nevery = int.Parse(arg[iarg + 1]);
if (nevery <= 0)
{
sparta.error.all("Illegal run command");
}
first = iarg + 2;
last = narg - 1;
ncommands = last - first + 1;
if (ncommands == 1 && string.Equals(arg[first], "NULL"))
{
ncommands = 0;
}
iarg = narg;
break;
default:
sparta.error.all("Illegal run command");
break;
}
}
int nsteps;
if (uptoflag == 0)
{
if (nsteps_input < 0 || nsteps_input > MAXSMALLINT)
{
sparta.error.all("Invalid run command N value");
}
nsteps = Convert.ToInt32(nsteps_input);
}
else
{
bigint delta = nsteps_input - sparta.update.ntimestep;
if (delta < 0 || delta > MAXSMALLINT)
{
sparta.error.all("Invalid run command upto value");
}
nsteps = Convert.ToInt32(delta);
}
// error check
if (startflag != 0)
{
if (start < 0 || start > MAXBIGINT)
{
sparta.error.all("Invalid run command start/stop value");
}
if (start > sparta.update.ntimestep)
{
sparta.error.all("Run command start value is after start of run");
}
}
if (stopflag != 0)
{
if (stop < 0 || stop > MAXBIGINT)
{
sparta.error.all("Invalid run command start/stop value");
}
if (stop < sparta.update.ntimestep + nsteps)
{
sparta.error.all("Run command stop value is before end of run");
}
}
// if nevery, make copies of arg strings that are commands
// required because re-parsing commands via input.one() will wipe out args
string[] commands = null;
if (nevery != 0 && ncommands > 0)
{
commands = new string[ncommands];
ncommands = 0;
for (int i = first; i <= last; i++)
{
int n = arg[i].Length + 1;
commands[ncommands] = string.Copy(arg[i]);
ncommands++;
}
}
// perform a single run
// use start/stop to set begin/end step
// if pre or 1st run, do System init/setup,
// else just init timer and setup output
// if post, do full Finish, else just print time
sparta.update.runflag = 1;
if (nevery == 0)
{
sparta.update.nsteps = nsteps;
sparta.update.firststep = sparta.update.ntimestep;
sparta.update.laststep = sparta.update.ntimestep + nsteps;
if (sparta.update.laststep < 0 || sparta.update.laststep > MAXBIGINT)
{
sparta.error.all("Too many timesteps");
}
if (startflag != 0)
{
sparta.update.beginstep = start;
}
else
{
sparta.update.beginstep = sparta.update.firststep;
}
if (stopflag != 0)
{
sparta.update.endstep = stop;
}
else
{
sparta.update.endstep = sparta.update.laststep;
}
if (preflag != 0 || sparta.update.first_update == 0)
{
sparta.init();
sparta.update.setup();
}
else
{
sparta.output.setup(0);
}
sparta.timer.init();
sparta.timer.barrier_start((int)Timer.Enum1.TIME_LOOP);
sparta.update.run(nsteps);
sparta.timer.barrier_stop((int)Timer.Enum1.TIME_LOOP);
Finish finish = new Finish(sparta);
finish.end(postflag, 0.0);
// perform multiple runs optionally interleaved with invocation command(s)
// use start/stop to set begin/end step
// if pre or 1st iteration of multiple runs, do System init/setup,
// else just init timer and setup output
// if post or last iteration, do full Finish, else just print time
}
}
public void command(int narg, string[] args, int outflag = 1)
{
string[] arg = new string[narg];
Array.Copy(args, 1, arg, 0, narg);
if (sparta.grid.exist == 0)
{
sparta.error.all("Cannot balance grid before grid is defined");
}
if (narg < 1)
{
sparta.error.all("Illegal balance_grid command");
}
int order = 0, bstyle = 0;
int px = 0, py = 0, pz = 0;
int rcbwt = 0, rcbflip = 0;
if (string.Equals(arg[0], "none"))
{
if (narg != 1)
{
sparta.error.all("Illegal balance_grid command");
}
bstyle = (int)Enum1.NONE;
}
else if (string.Equals(arg[0], "stride"))
{
if (narg != 2)
{
sparta.error.all("Illegal balance_grid command");
}
bstyle = (int)Enum1.STRIDE;
if (string.Equals(arg[1], "xyz"))
{
order = (int)Enum2.XYZ;
}
else if (string.Equals(arg[1], "xzy"))
{
order = (int)Enum2.XZY;
}
else if (string.Equals(arg[1], "yxz"))
{
order = (int)Enum2.YXZ;
}
else if (string.Equals(arg[1], "yzx"))
{
order = (int)Enum2.YZX;
}
else if (string.Equals(arg[1], "zxy"))
{
order = (int)Enum2.ZXY;
}
else if (string.Equals(arg[1], "zyx"))
{
order = (int)Enum2.ZYX;
}
else
{
sparta.error.all("Illegal balance_grid command");
}
}
else if (string.Equals(arg[0], "clump"))
{
if (narg != 2)
{
sparta.error.all("Illegal balance_grid command");
}
bstyle = (int)Enum1.CLUMP;
if (string.Equals(arg[1], "xyz"))
{
order = (int)Enum2.XYZ;
}
else if (string.Equals(arg[1], "xzy"))
{
order = (int)Enum2.XZY;
}
else if (string.Equals(arg[1], "yxz"))
{
order = (int)Enum2.YXZ;
}
else if (string.Equals(arg[1], "yzx"))
{
order = (int)Enum2.YZX;
}
else if (string.Equals(arg[1], "zxy"))
{
order = (int)Enum2.ZXY;
}
else if (string.Equals(arg[1], "zyx"))
{
order = (int)Enum2.ZYX;
}
else
{
sparta.error.all("Illegal balance_grid command");
}
}
else if (string.Equals(arg[0], "block"))
{
if (narg != 4)
{
sparta.error.all("Illegal balance_grid command");
}
bstyle = (int)Enum1.BLOCK;
if (string.Equals(arg[1], "*"))
{
px = 0;
}
else
{
px = int.Parse(arg[1]);
}
if (string.Equals(arg[2], "*"))
{
py = 0;
}
else
{
py = int.Parse(arg[2]);
}
if (string.Equals(arg[3], "*"))
{
pz = 0;
}
else
{
pz = int.Parse(arg[3]);
}
}
else if (string.Equals(arg[0], "random"))
{
if (narg != 1)
{
sparta.error.all("Illegal balance_grid command");
}
bstyle = (int)Enum1.RANDOM;
}
else if (string.Equals(arg[0], "proc"))
{
if (narg != 1)
{
sparta.error.all("Illegal balance_grid command");
}
bstyle = (int)Enum1.PROC;
}
else if (string.Equals(arg[0], "rcb"))
{
if (narg != 2 && narg != 3)
{
sparta.error.all("Illegal balance_grid command");
}
bstyle = (int)Enum1.BISECTION;
if (string.Equals(arg[1], "cell"))
{
rcbwt = (int)Enum3.CELL;
}
else if (string.Equals(arg[1], "part"))
{
rcbwt = (int)Enum3.PARTICLE;
}
else
{
sparta.error.all("Illegal balance_grid command");
}
// undocumented optional 3rd arg
// rcbflip = 3rd arg = 1 forces rcb.compute() to flip sign
// of all grid cell "dots" to force totally different
// assignment of grid cells to procs and induce
// complete rebalance data migration
rcbflip = 0;
if (narg == 3)
{
rcbflip = int.Parse(arg[2]);
}
}
else
{
sparta.error.all("Illegal balance_grid command");
}
// error check on methods only allowed for a uniform grid
if (bstyle == (int)Enum1.STRIDE || bstyle == (int)Enum1.CLUMP || bstyle == (int)Enum1.BLOCK)
{
if (sparta.grid.uniform == 0)
{
sparta.error.all("Invalid balance_grid style for non-uniform grid");
}
}
// re-assign each of my local child cells to a proc
// only assign unsplit and split cells
// do not assign sub-cells since they migrate with their split cell
// set nmigrate = # of cells that will migrate to a new proc
// reset proc field in cells for migrating cells
// style NONE performs no re-assignment
sparta.mpi.MPI_Barrier(sparta.world);
double time1 = sparta.mpi.MPI_Wtime();
Grid.ChildCell[] cells = sparta.grid.cells;
Grid.ChildInfo[] cinfo = sparta.grid.cinfo;
int nglocal = sparta.grid.nlocal;
int nprocs = sparta.comm.nprocs;
int newproc;
int nmigrate = 0;
if (bstyle == (int)Enum1.STRIDE)
{
cellint idm1, ix, iy, iz, nth = 0;
cellint nx = sparta.grid.unx;
cellint ny = sparta.grid.uny;
cellint nz = sparta.grid.unz;
for (int icell = 0; icell < nglocal; icell++)
{
if (cells[icell].nsplit <= 0)
{
continue;
}
idm1 = cells[icell].id - 1;
ix = idm1 % nx;
iy = (idm1 / nx) % ny;
iz = idm1 / (nx * ny);
if (order == (int)Enum2.XYZ)
{
nth = iz * nx * ny + iy * nx + ix;
}
else if (order == (int)Enum2.XZY)
{
nth = iy * nx * nz + iz * nx + ix;
}
else if (order == (int)Enum2.YXZ)
{
nth = iz * ny * nx + ix * ny + iy;
}
else if (order == (int)Enum2.YZX)
{
nth = ix * ny * nz + iz * ny + iy;
}
else if (order == (int)Enum2.ZXY)
{
nth = iy * nz * nx + ix * nz + iz;
}
else if (order == (int)Enum2.ZYX)
{
nth = ix * nz * ny + iy * nz + iz;
}
newproc = nth % nprocs;
if (newproc != cells[icell].proc)
{
nmigrate++;
}
cells[icell].proc = newproc;
}
}
else if (bstyle == (int)Enum1.BLOCK)
{
cellint idm1, ix, iy, iz;
int ipx, ipy, ipz;
int nx = sparta.grid.unx;
int ny = sparta.grid.uny;
int nz = sparta.grid.unz;
procs2grid(nx, ny, nz, ref px, ref py, ref pz);
if (px * py * pz != nprocs)
{
sparta.error.all("Bad grid of processors for balance_grid block");
}
for (int icell = 0; icell < nglocal; icell++)
{
if (cells[icell].nsplit <= 0)
{
continue;
}
idm1 = cells[icell].id - 1;
ix = idm1 % nx;
iy = (idm1 / nx) % ny;
iz = idm1 / (nx * ny);
ipx = ix * px / nx;
ipy = iy * py / ny;
ipz = iz * pz / nz;
newproc = ipz * px * py + ipy * px + ipx;
if (newproc != cells[icell].proc)
{
nmigrate++;
}
cells[icell].proc = newproc;
}
}
else if (bstyle == (int)Enum1.RANDOM)
{
RanPark random = new RanPark(sparta.update.ranmaster.uniform());
double seed = sparta.update.ranmaster.uniform();
random.reset(seed, sparta.comm.me, 100);
for (int icell = 0; icell < nglocal; icell++)
{
if (cells[icell].nsplit <= 0)
{
continue;
}
newproc = Convert.ToInt32(nprocs * random.uniform());
if (newproc != cells[icell].proc)
{
nmigrate++;
}
cells[icell].proc = newproc;
}
//delete random;
}
else if (bstyle == (int)Enum1.BISECTION)
{
RCB rcb = new RCB(sparta);
double[,] x = new double[nglocal, 3];
double[] lo, hi;
int nbalance = 0;
for (int icell = 0; icell < nglocal; icell++)
{
if (cells[icell].nsplit <= 0)
{
continue;
}
lo = cells[icell].lo;
hi = cells[icell].hi;
x[nbalance, 0] = 0.5 * (lo[0] + hi[0]);
x[nbalance, 1] = 0.5 * (lo[1] + hi[1]);
x[nbalance, 2] = 0.5 * (lo[2] + hi[2]);
nbalance++;
}
double[] wt;
if (rcbwt == (int)Enum3.PARTICLE)
{
sparta.particle.sort();
int zero = 0;
int n;
wt = new double[nglocal];
nbalance = 0;
for (int icell = 0; icell < nglocal; icell++)
{
if (cells[icell].nsplit <= 0)
{
continue;
}
n = cinfo[icell].count;
if (n != 0)
{
wt[nbalance++] = n;
}
else
{
wt[nbalance++] = ZEROPARTICLE;
zero++;
}
}
}
else
{
wt = new double[nglocal];
}
rcb.compute(nbalance, x, wt, rcbflip);
rcb.invert();
nbalance = 0;
int[] sendproc = rcb.sendproc;
for (int icell = 0; icell < nglocal; icell++)
{
if (cells[icell].nsplit <= 0)
{
continue;
}
cells[icell].proc = sendproc[nbalance++];
}
nmigrate = nbalance - rcb.nkeep;
}
// set clumped of not, depending on style
// NONE style does not change clumping
if (nprocs == 1 || bstyle == (int)Enum1.CLUMP || bstyle == (int)Enum1.BLOCK || bstyle == (int)Enum1.BISECTION)
{
sparta.grid.clumped = 1;
}
else if (bstyle != (int)Enum1.NONE)
{
sparta.grid.clumped = 0;
}
sparta.mpi.MPI_Barrier(sparta.world);
double time2 = sparta.mpi.MPI_Wtime();
// sort particles
// NOTE: not needed again if rcbwt = PARTICLE for bstyle = BISECTION ??
sparta.particle.sort();
sparta.mpi.MPI_Barrier(sparta.world);
double time3 = sparta.mpi.MPI_Wtime();
// invoke init() so all grid cell info, including collide & fixes,
// is ready to migrate
// for init, do not require surfs be assigned collision models
// this allows balance call early in script, e.g. from ReadRestart
// migrate grid cells and their particles to new owners
// invoke grid methods to complete grid setup
int ghost_previous = sparta.grid.exist_ghost;
sparta.domain.boundary_collision_check = 0;
sparta.surf.surf_collision_check = 0;
sparta.init();
sparta.domain.boundary_collision_check = 1;
sparta.surf.surf_collision_check = 1;
sparta.grid.unset_neighbors();
sparta.grid.remove_ghosts();
sparta.comm.migrate_cells(nmigrate);
sparta.mpi.MPI_Barrier(sparta.world);
double time4 = sparta.mpi.MPI_Wtime();
sparta.grid.setup_owned();
sparta.grid.acquire_ghosts();
if (ghost_previous != 0)
{
sparta.grid.reset_neighbors();
}
else
{
sparta.grid.find_neighbors();
}
sparta.comm.reset_neighbors();
// reallocate per grid arrays in per grid dumps
for (int i = 0; i < sparta.output.ndump; i++)
{
sparta.output.dump[i].reset_grid();
}
sparta.mpi.MPI_Barrier(sparta.world);
double time5 = sparta.mpi.MPI_Wtime();
// stats on balance operation
// only print if outflag = 1
// some callers suppress output, e.g. ReadRestart
bigint count = nmigrate;
bigint nmigrate_all = 0;
sparta.mpi.MPI_Allreduce(ref count, ref nmigrate_all, 1, MPI.MPI_LONG_LONG, MPI.MPI_SUM, sparta.world);
double time_total = time5 - time1;
if (sparta.comm.me == 0 && outflag != 0)
{
string str1 = string.Format("Balance grid migrated {0} cells\n", nmigrate_all);
string str2 = string.Format(" CPU time = {0:G} secs\n", time_total);
string str3 = string.Format(" reassign/sort/migrate/ghost percent = {0:G6} {1:G6} {2:G6} {3:G6}\n",
100.0 * (time2 - time1) / time_total, 100.0 * (time3 - time2) / time_total,
100.0 * (time4 - time3) / time_total, 100.0 * (time5 - time4) / time_total);
Console.WriteLine(str1 + str2 + str3);
if (sparta.screen != null)
{
new StreamWriter(sparta.screen).WriteLine(str1 + str2 + str3);
//fprintf(screen, "Balance grid migrated " BIGINT_FORMAT " cells\n",
// nmigrate_all);
//fprintf(screen, " CPU time = %g secs\n", time_total);
//fprintf(screen, " reassign/sort/migrate/ghost percent = %g %g %g %g\n",
// 100.0 * (time2 - time1) / time_total, 100.0 * (time3 - time2) / time_total,
// 100.0 * (time4 - time3) / time_total, 100.0 * (time5 - time4) / time_total);
}
if (sparta.logfile != null)
{
new StreamWriter(sparta.logfile).WriteLine(str1 + str2 + str3);
}
}
}
