private void BlockTransposeXtoY(CustomFftPlan data, UnsafeArray src, UnsafeArray dst)
{
var ny = data.Ny;
var locNx = data.LocalNx;
var nm = data.Nc;
var nk = data.NumberOfLocalFftsAlongSecondDimension;
dst.ReShape(nm, locNx, ny);
//Linear writting, nonlinear reading
Iterate(nm, im =>
{
//for (int im = 0; im < nm; im++)
for (int ix = 0; ix < locNx; ix++)
{
for (int iy = 0; iy < ny; iy++)
{
var l0 = iy + im * ny; // place in XZ
var m0 = l0 - l0 % nk; //Ik = m0
var l = l0 + ix * nk + m0 * (locNx - 1); //Ix + (Iy - 1)Nx + (Ic - 1)NxNyLoc
dst[im, ix, iy] = src[l];
}
}
});
}
protected virtual void DistributedTranspose(CustomFftPlan data, UnsafeArray src, UnsafeArray dst)
{
FftwMpi.ExecuteR2r(data.TransposePlan.Handler, new IntPtr(src.Ptr), new IntPtr(dst.Ptr));
//int size = data.NumberOfLocalFftsAlongSecondDimension * data.LocalNx;
//_mpi.AllToAll(data.Output.Ptr, size, data.Input.Ptr, size);
//All2All
}
private void BlockTransposeYtoX(CustomFftPlan data, UnsafeArray src, UnsafeArray dst)
{
var locNx = data.LocalNx;
var ny = data.Ny;
var np = data.NumberOfMpiProcesses;
var nk = data.NumberOfLocalFftsAlongSecondDimension;
var bufferSize = nk * np * locNx - 1;
dst.ReShape(np, locNx, nk);
//Linear writting, nonlinear reading
Iterate(np, ip =>
{
//for (int ip = 0; ip < np; ip++)
for (int ix = 0; ix < locNx; ix++)
{
for (int ik = 0; ik < nk; ik++)
{
var l0 = ik + (ip) * nk; // place in XZ
var m0 = l0 - l0 % ny;
var l = l0 + ix * ny + m0 * (locNx - 1); //Ix + (Iy - 1)Nx + (Ic - 1)NxNyLoc
l = (l + bufferSize - Abs(bufferSize - l)) / 2; // if l > M we are in "addition" zone
dst[ip, ix, ik] = src[l];
}
}
});
}
protected virtual void InitialTranspose(CustomFftPlan data, UnsafeArray src, UnsafeArray dst)
{
var nxy = data.LocalNx * data.Ny;
var nz = data.Nc;
LocalTranspose(src, dst, nz, nxy);
}
private void FinalTranspose(CustomFftPlan data, UnsafeArray src, UnsafeArray dst)
{
var nxy = data.LocalNx * data.Ny;
var nz = data.Nc;
LocalTranspose(src, dst, nxy, nz);
}
private void CreateInternalPlans(CustomFftPlan data)
{
CreateTransposePlan(data, _flagsTranspose);
Create1DFftPlansY(data, _flags1d);
Create1DFftPlansX(data, _flags1d);
if (data.Dimension == 3)
{
Create1DFftPlansZ(data, _flags1d);
}
}
private void CreateTransposePlan(CustomFftPlan data, uint flags)
{
var n = new IntPtr(2 * data.NumberOfLocalFftsAlongSecondDimension * data.LocalNx);
var np = new IntPtr(data.NumberOfMpiProcesses);
var one = new IntPtr(1);
var input = new IntPtr(data.Input.Ptr);
var output = new IntPtr(data.Output.Ptr);
var handler = FftwMpi.PlanManyTranspose(np, np, n, one, one, input, output, _mpi.Communicator, flags);
data.TransposePlan = new FftwPlan(np.ToInt32(), np.ToInt32(), n.ToInt32(), handler);
}
public CustomFftPlan CreatePlan(Complex *input, Complex *output, Mpi mpi, int fullNx, int fullNy, int nc, int dimension)
{
var localNx = mpi.CalcLocalNxLength(fullNx);
var numberOfLocalFftsAlongSecondDimension = GetNumberOfLocalFftsAlongSecondDimension(mpi, fullNy, nc);
var plan = new CustomFftPlan(input, output, dimension)
{
Ny = fullNy,
Nc = nc,
NumberOfMpiProcesses = mpi.Size,
LocalNx = localNx,
NumberOfLocalFftsAlongSecondDimension = numberOfLocalFftsAlongSecondDimension
};
CreateInternalPlans(plan);
return(plan);
}
private void Create1DFftPlansX(CustomFftPlan data, uint flags)
{
var pNx = new int[] { data.LocalNx *data.NumberOfMpiProcesses };
var howmany = data.NumberOfLocalFftsAlongSecondDimension;
var input = new IntPtr(data.Input.Ptr);
var output = new IntPtr(data.Output.Ptr);
var handlerPlanXForward
= Fftw.PlanManyDft(1, pNx, howmany, input, pNx, howmany, 1, output, pNx, howmany, 1, (int)FftWrap.Direction.Forward, flags);
var handlerPlanXBackward
= Fftw.PlanManyDft(1, pNx, howmany, input, pNx, howmany, 1, output, pNx, howmany, 1, (int)FftWrap.Direction.Backward, flags);
data.ForwardX = new FftwPlan(0, 0, 0, handlerPlanXForward);
data.BackwardX = new FftwPlan(0, 0, 0, handlerPlanXBackward);
}
private void Create1DFftPlansY(CustomFftPlan data, uint flags)
{
var pNy = new int[] { data.Ny, };
var dy = data.Ny;
var howmany = data.LocalNx * data.Nc;
var input = new IntPtr(data.Output.Ptr);
var output = new IntPtr(data.Input.Ptr);
var handlerPlanYForward
= Fftw.PlanManyDft(1, pNy, howmany, input, pNy, 1, dy, output, pNy, 1, dy, (int)FftWrap.Direction.Forward,
flags);
var handlerPlanYBackward
= Fftw.PlanManyDft(1, pNy, howmany, input, pNy, 1, dy, output, pNy, 1, dy, (int)FftWrap.Direction.Backward,
flags);
data.ForwardY = new FftwPlan(0, 0, 0, handlerPlanYForward);
data.BackwardY = new FftwPlan(0, 0, 0, handlerPlanYBackward);
}
private void Create1DFftPlansZ(CustomFftPlan data, uint flags)
{
var pNx = new int[] { data.Nc };
var howmany = data.LocalNx * data.Ny;
var input = new IntPtr(data.Input.Ptr);
var output = new IntPtr(data.Output.Ptr);
int stride = 1;
int dist = data.Nc;
var handlerPlanZForward
= Fftw.PlanManyDft(1, pNx, howmany, input, pNx, stride, dist, output, pNx, stride, dist, (int)FftWrap.Direction.Forward, flags);
var handlerPlanZBackward
= Fftw.PlanManyDft(1, pNx, howmany, input, pNx, stride, dist, output, pNx, stride, dist, (int)FftWrap.Direction.Backward, flags);
data.ForwardZ = new FftwPlan(0, 0, 0, handlerPlanZForward);
data.BackwardZ = new FftwPlan(0, 0, 0, handlerPlanZBackward);
}
private void Compute(CustomFftPlan data, Direction direction)
{
var buff1 = direction == Direction.Forward ? data.Input : data.Output;
var buff2 = direction == Direction.Forward ? data.Output : data.Input;
using (_profiler?.StartAuto(ProfilerEvent.CustomFft))
{
if (data.Dimension == 3)
{
WithProfiler(ProfilerEvent.CustomFftFourierZ, () => FourierZ(data, direction, buff2, buff1));
}
WithProfiler(ProfilerEvent.CustomFftInitialTranspose, () => InitialTranspose(data, buff1, buff2));
WithProfiler(ProfilerEvent.CustomFftFourierY, () => FourierY(data, direction, buff2, buff1));
WithProfiler(ProfilerEvent.CustomFftBlockTransposeYtoX, () => BlockTransposeYtoX(data, buff1, buff2));
WithProfiler(ProfilerEvent.CustomFftDistributedTranspose, () => DistributedTranspose(data, buff2, buff1));
WithProfiler(ProfilerEvent.CustomFftFourierX, () => FourierX(data, direction, buff1, buff2));
WithProfiler(ProfilerEvent.CustomFftDistributedTranspose, () => DistributedTranspose(data, buff2, buff1));
WithProfiler(ProfilerEvent.CustomFftBlockTransposeXtoY, () => BlockTransposeXtoY(data, buff1, buff2));
WithProfiler(ProfilerEvent.CustomFftFinalTranspose, () => FinalTranspose(data, buff2, buff1));
}
}
private void FourierZ(CustomFftPlan data, Direction direction, UnsafeArray src, UnsafeArray dst)
{
var plan = direction == Direction.Forward ? data.ForwardZ : data.BackwardZ;
Fftw.ExecuteDft(plan.Handler, new IntPtr(src.Ptr), new IntPtr(dst.Ptr));
}
public void ComputeBackward(CustomFftPlan data) => Compute(data, Direction.Backward);
public void ComputeForward(CustomFftPlan data) => Compute(data, Direction.Forward);
public CustomBuffer(Complex *buffer1Ptr, Complex *buffer2Ptr, int bufferLength, CustomFftPlan customFftPlan) : base(buffer1Ptr, buffer2Ptr, bufferLength)
{
CustomFftPlan = customFftPlan;
}