/// <summary>
/// Extract the job information from a folder with logs on the local machine.
/// </summary>
/// <param name="jobRootFolder">Folder with logs for the specified job.</param>
/// <returns>The job information, or null if not found.</returns>
private ClusterJobInformation GetJobInfo(string jobRootFolder)
{
Uri uri = DfsFile.UriFromPath(this.config.JobsFolderUri, jobRootFolder);
long time;
long size;
this.config.DfsClient.GetFileStatus(uri, out time, out size);
DateTime date = DfsFile.TimeFromLong(time);
ClusterJobInformation.ClusterJobStatus status = ClusterJobInformation.ClusterJobStatus.Unknown;
string jobName = Path.GetFileName(jobRootFolder);
string errorMsg = "";
try
{
var jobinfo = this.yarnClient.QueryJob(jobName, uri);
var jobstatus = jobinfo.GetStatus();
errorMsg = jobinfo.ErrorMsg;
switch (jobstatus)
{
case JobStatus.NotSubmitted:
case JobStatus.Waiting:
status = ClusterJobInformation.ClusterJobStatus.Unknown;
break;
case JobStatus.Running:
status = ClusterJobInformation.ClusterJobStatus.Running;
break;
case JobStatus.Success:
status = ClusterJobInformation.ClusterJobStatus.Succeeded;
break;
case JobStatus.Cancelled:
status = ClusterJobInformation.ClusterJobStatus.Cancelled;
break;
case JobStatus.Failure:
status = ClusterJobInformation.ClusterJobStatus.Failed;
break;
default:
throw new ArgumentOutOfRangeException();
}
}
catch (Exception)
{
}
TimeSpan running = TimeSpan.Zero;
var info = new ClusterJobInformation(config.Name, "", jobName, jobName, Environment.UserName, date, running, status);
return(info);
}
/// <summary>
/// Create a cluster containing just the local machine.
/// </summary>
/// <param name="conf">Configuration for the local machine.</param>
public HdfsClusterStatus(ClusterConfiguration conf)
: base(conf)
{
if (!(conf is HdfsClusterConfiguration))
{
throw new ArgumentException("Expected an HdfsClusterConfiguration, got a " + conf.GetType());
}
this.config = conf as HdfsClusterConfiguration;
// make a fake call to initialize the cluster on the foreground thread
// HDFS does not work if initialized on the background thread.
Uri uri = DfsFile.UriFromPath(this.config.JobsFolderUri, "");
this.config.DfsClient.IsFileExists(uri); // ignore result
this.yarnClient = new NativeYarnClient(this.config.StatusNode, this.config.StatusNodePort, new HdfsClient(this.config.UserName));
}
/// <summary>
/// Force the recomputation of the cluster job list.
/// </summary>
/// <param name="virtualCluster">Virtual cluster to use (defined only for some cluster types).</param>
/// <param name="manager">Communication manager.</param>
// ReSharper disable once UnusedParameter.Global
protected override void RecomputeClusterJobList(string virtualCluster, CommManager manager)
{
this.clusterJobs = new Dictionary <string, ClusterJobInformation>();
var uri = DfsFile.UriFromPath(this.config.JobsFolderUri, "");
var jobs = this.config.DfsClient.EnumerateSubdirectories(uri).ToList();
int done = 0;
foreach (var job in jobs)
{
manager.Token.ThrowIfCancellationRequested();
ClusterJobInformation info = this.GetJobInfo(DfsFile.PathFromUri(this.config.JobsFolderUri, job));
if (info != null)
{
// ReSharper disable once AssignNullToNotNullAttribute
this.clusterJobs.Add(info.ClusterJobID, info);
}
manager.Progress(100 * done++ / jobs.Count);
}
manager.Progress(100);
}
/// <summary>
/// Create a new file, being the difference of two files.
/// <para>
/// The two input files must be equal in structure, e.g. coming
/// from the same simulation but giving different results.
/// Header and static data must be identical, only difference
/// must be in values of the dynamic data.
/// </para>
/// </summary>
public static void CreateDiffFile(string file1, string file2, string filediff = null)
{
IDfsFile dfs1 = DfsFileFactory.DfsGenericOpen(file1);
IDfsFile dfs2 = DfsFileFactory.DfsGenericOpen(file2);
// Validate that it has the same number of items.
if (dfs1.ItemInfo.Count != dfs2.ItemInfo.Count)
{
throw new Exception("Number of dynamic items does not match");
}
int numItems = dfs1.ItemInfo.Count;
// In case number of time steps does not match, take the smallest.
int numTimes = dfs1.FileInfo.TimeAxis.NumberOfTimeSteps;
if (numTimes > dfs2.FileInfo.TimeAxis.NumberOfTimeSteps)
{
numTimes = dfs2.FileInfo.TimeAxis.NumberOfTimeSteps;
Console.Out.WriteLine("Number of time steps does not match, using the smallest number");
}
// For recording max difference for every item
double[] maxDiff = new double[dfs1.ItemInfo.Count];
// Index in time (index) of maximum and first difference. -1 if no difference
int[] maxDiffTime = new int[dfs1.ItemInfo.Count];
int[] firstDiffTime = new int[dfs1.ItemInfo.Count];
for (int i = 0; i < dfs1.ItemInfo.Count; i++)
{
maxDiffTime[i] = -1;
firstDiffTime[i] = -1;
}
// Copy over info from the first file, assuming the second file contains the same data.
IDfsFileInfo fileInfo = dfs1.FileInfo;
DfsBuilder builder = null;
if (!string.IsNullOrEmpty(filediff))
{
builder = DfsBuilder.Create(fileInfo.FileTitle, fileInfo.ApplicationTitle, fileInfo.ApplicationVersion);
// Set up the header
builder.SetDataType(fileInfo.DataType);
builder.SetGeographicalProjection(fileInfo.Projection);
builder.SetTemporalAxis(fileInfo.TimeAxis);
builder.SetItemStatisticsType(fileInfo.StatsType);
builder.DeleteValueByte = fileInfo.DeleteValueByte;
builder.DeleteValueDouble = fileInfo.DeleteValueDouble;
builder.DeleteValueFloat = fileInfo.DeleteValueFloat;
builder.DeleteValueInt = fileInfo.DeleteValueInt;
builder.DeleteValueUnsignedInt = fileInfo.DeleteValueUnsignedInt;
// Transfer compression keys.
if (fileInfo.IsFileCompressed)
{
int[] xkey;
int[] ykey;
int[] zkey;
fileInfo.GetEncodeKey(out xkey, out ykey, out zkey);
builder.SetEncodingKey(xkey, ykey, zkey);
}
// Copy custom blocks
foreach (IDfsCustomBlock customBlock in fileInfo.CustomBlocks)
{
builder.AddCustomBlock(customBlock);
}
}
// Copy dynamic item definitions
bool[] floatItems = new bool[dfs1.ItemInfo.Count];
for (int i = 0; i < dfs1.ItemInfo.Count; i++)
{
var itemInfo = dfs1.ItemInfo[i];
// Validate item sizes
var itemInfo2 = dfs2.ItemInfo[i];
if (itemInfo.ElementCount != itemInfo2.ElementCount)
{
throw new Exception("Dynamic items must have same size, item number " + (i + 1) +
" has different sizes in the two files");
}
// Validate the data type, only supporting floats and doubles.
if (itemInfo.DataType == DfsSimpleType.Float)
{
floatItems[i] = true;
}
else if (itemInfo.DataType != DfsSimpleType.Double)
{
throw new Exception("Dynamic item must be double or float, item number " + (i + 1) + " is of type " +
(itemInfo.DataType));
}
builder?.AddDynamicItem(itemInfo);
}
// Create file
builder?.CreateFile(filediff);
if (builder != null)
{
// Copy over static items from file 1, assuming the static items of file 2 are identical
IDfsStaticItem si1;
while (null != (si1 = dfs1.ReadStaticItemNext()))
{
builder.AddStaticItem(si1);
}
}
// Get the file
DfsFile diff = builder?.GetFile();
// Write dynamic data to the file, being the difference between the two
for (int i = 0; i < numTimes; i++)
{
for (int j = 0; j < numItems; j++)
{
if (floatItems[j])
{
IDfsItemData <float> data1 = dfs1.ReadItemTimeStepNext() as IDfsItemData <float>;
IDfsItemData <float> data2 = dfs2.ReadItemTimeStepNext() as IDfsItemData <float>;
for (int k = 0; k < data1.Data.Length; k++)
{
float valuediff = data1.Data[k] - data2.Data[k];
data1.Data[k] = valuediff;
float absValueDiff = Math.Abs(valuediff);
if (absValueDiff > maxDiff[j])
{
maxDiff[j] = absValueDiff;
maxDiffTime[j] = i;
if (firstDiffTime[j] == -1)
{
firstDiffTime[j] = i;
}
}
}
diff?.WriteItemTimeStepNext(data1.Time, data1.Data);
}
else
{
IDfsItemData <double> data1 = dfs1.ReadItemTimeStepNext() as IDfsItemData <double>;
IDfsItemData <double> data2 = dfs2.ReadItemTimeStepNext() as IDfsItemData <double>;
for (int k = 0; k < data1.Data.Length; k++)
{
double valuediff = data1.Data[k] - data2.Data[k];
data1.Data[k] = valuediff;
double absValueDiff = Math.Abs(valuediff);
if (absValueDiff > maxDiff[j])
{
maxDiff[j] = absValueDiff;
maxDiffTime[j] = i;
if (firstDiffTime[j] == -1)
{
firstDiffTime[j] = i;
}
}
}
diff?.WriteItemTimeStepNext(data1.Time, data1.Data);
}
}
}
System.Console.WriteLine("Difference statistics:");
for (int i = 0; i < maxDiffTime.Length; i++)
{
if (maxDiffTime[i] < 0)
{
Console.WriteLine("{0,-30}: no difference", dfs1.ItemInfo[i].Name);
}
else
{
Console.WriteLine("{0,-30}: Max difference at timestep {1,3}: {2}. First difference at timestep {3}", dfs1.ItemInfo[i].Name, maxDiffTime[i], maxDiff[i], firstDiffTime[i]);
}
}
dfs1.Close();
dfs2.Close();
diff?.Close();
}
/// <summary>
/// Example of how to copy a Dfs file.
/// <para>
/// This example is intended to show how to generically copy a file. In
/// case a copy with modified data is required, this could be used as a base
/// for the copy.
/// </para>
/// </summary>
/// <param name="sourceFilename">Path and name of the source dfs file</param>
/// <param name="filename">Path and name of the new file to create</param>
public static void CopyDfsFile(string sourceFilename, string filename)
{
IDfsFile source = DfsFileFactory.DfsGenericOpen(sourceFilename);
IDfsFileInfo fileInfo = source.FileInfo;
DfsBuilder builder = DfsBuilder.Create(fileInfo.FileTitle, fileInfo.ApplicationTitle, fileInfo.ApplicationVersion);
// Set up the header
builder.SetDataType(fileInfo.DataType);
builder.SetGeographicalProjection(fileInfo.Projection);
builder.SetTemporalAxis(fileInfo.TimeAxis);
builder.SetItemStatisticsType(fileInfo.StatsType);
builder.DeleteValueByte = fileInfo.DeleteValueByte;
builder.DeleteValueDouble = fileInfo.DeleteValueDouble;
builder.DeleteValueFloat = fileInfo.DeleteValueFloat;
builder.DeleteValueInt = fileInfo.DeleteValueInt;
builder.DeleteValueUnsignedInt = fileInfo.DeleteValueUnsignedInt;
// Transfer compression keys - if any.
if (fileInfo.IsFileCompressed)
{
int[] xkey;
int[] ykey;
int[] zkey;
fileInfo.GetEncodeKey(out xkey, out ykey, out zkey);
builder.SetEncodingKey(xkey, ykey, zkey);
}
// Copy custom blocks - if any
foreach (IDfsCustomBlock customBlock in fileInfo.CustomBlocks)
{
builder.AddCustomBlock(customBlock);
}
// Copy dynamic items
foreach (var itemInfo in source.ItemInfo)
{
builder.AddDynamicItem(itemInfo);
}
// Create file
builder.CreateFile(filename);
// Copy static items
IDfsStaticItem sourceStaticItem;
while (null != (sourceStaticItem = source.ReadStaticItemNext()))
{
builder.AddStaticItem(sourceStaticItem);
}
// Get the file
DfsFile file = builder.GetFile();
// Copy dynamic item data
IDfsItemData sourceData;
while (null != (sourceData = source.ReadItemTimeStepNext()))
{
file.WriteItemTimeStepNext(sourceData.Time, sourceData.Data);
}
source.Close();
file.Close();
}
/// <summary>
/// Example of how to merge two or more dfs files. The merger is on dynamic item basis,
/// i.e. add all dynamic items of a number of dfs files to a new dfs file.
/// <para>
/// It is assumed that all files has the same time stepping layout. It will merge
/// as many time steps as the file with the least number of timesteps.
/// </para>
/// <para>
/// If merging one of the specific types of dfs files, dfs0 or dfs1 or dfs2 or dfs3,
/// the structure of the files must be identical, i.e. the sizes of the axis must equal.
/// Otherwise, the outcome will not be a valid dfs0/1/2/3 file.
/// </para>
/// </summary>
/// <param name="targetFilename">Path and name of the new file to create</param>
/// <param name="sourcesFilenames">Path and name of the source dfs files</param>
public static void MergeDfsFileItems(string targetFilename, IList <string> sourcesFilenames)
{
// List of sources to be merged - in case of more than one, just extend this.
List <IDfsFile> sources = new List <IDfsFile>();
for (int i = 0; i < sourcesFilenames.Count; i++)
{
sources.Add(DfsFileFactory.DfsGenericOpen(sourcesFilenames[i]));
}
// Use the first file as skeleton for header and static items.
IDfsFile source = sources[0];
IDfsFileInfo fileInfo = source.FileInfo;
DfsBuilder builder = DfsBuilder.Create(fileInfo.FileTitle, fileInfo.ApplicationTitle, fileInfo.ApplicationVersion);
// Set up the header
builder.SetDataType(fileInfo.DataType);
builder.SetGeographicalProjection(fileInfo.Projection);
builder.SetTemporalAxis(fileInfo.TimeAxis);
builder.SetItemStatisticsType(fileInfo.StatsType);
builder.DeleteValueByte = fileInfo.DeleteValueByte;
builder.DeleteValueDouble = fileInfo.DeleteValueDouble;
builder.DeleteValueFloat = fileInfo.DeleteValueFloat;
builder.DeleteValueInt = fileInfo.DeleteValueInt;
builder.DeleteValueUnsignedInt = fileInfo.DeleteValueUnsignedInt;
// Transfer compression keys - if any.
if (fileInfo.IsFileCompressed)
{
int[] xkey;
int[] ykey;
int[] zkey;
fileInfo.GetEncodeKey(out xkey, out ykey, out zkey);
builder.SetEncodingKey(xkey, ykey, zkey);
}
// Copy custom blocks - if any
foreach (IDfsCustomBlock customBlock in fileInfo.CustomBlocks)
{
builder.AddCustomBlock(customBlock);
}
int minNumTimesteps = int.MaxValue;
// Copy dynamic items for all source files
for (int j = 0; j < sources.Count; j++)
{
if (sources[j].FileInfo.TimeAxis.NumberOfTimeSteps < minNumTimesteps)
{
minNumTimesteps = sources[j].FileInfo.TimeAxis.NumberOfTimeSteps;
}
foreach (var itemInfo in sources[j].ItemInfo)
{
builder.AddDynamicItem(itemInfo);
}
}
// Create file
builder.CreateFile(targetFilename);
// Copy static items - add only from main file
IDfsStaticItem sourceStaticItem;
while (null != (sourceStaticItem = source.ReadStaticItemNext()))
{
builder.AddStaticItem(sourceStaticItem);
}
// Get the file
DfsFile file = builder.GetFile();
// Copy dynamic item data
IDfsItemData sourceData;
for (int i = 0; i < minNumTimesteps; i++)
{
for (int j = 0; j < sources.Count; j++)
{
IDfsFile sourcej = sources[j];
// Copy all items for this source
for (int k = 0; k < sourcej.ItemInfo.Count; k++)
{
sourceData = sourcej.ReadItemTimeStepNext();
file.WriteItemTimeStepNext(sourceData.Time, sourceData.Data);
}
}
}
foreach (IDfsFile sourcej in sources)
{
sourcej.Close();
}
file.Close();
}
/// <summary>
/// Example on how to extract dfs0 data from a 2D dfsu file for certain elements. All items
/// from dfsu file are extracted.
/// </summary>
/// <param name="dfsuFileNamePath">Name, including path, of 2D dfsu file</param>
/// <param name="elmtsIndices">Indices of elements to extract data from</param>
/// <param name="useStream">Use stream when writing dfs0 files - then more than 400 files can be created simultaneously</param>
public static void ExtractDfs0FromDfsu(string dfsuFileNamePath, IList <int> elmtsIndices, bool useStream)
{
// If not using stream approach, at most 400 elements at a time can be processed.
// There is a limit on how many files you can have open at the same time using
// the standard approach. It will fail in a nasty way, if the maximum number of
// file handles are exceeded. This is not an issue when using .NET streams.
if (!useStream && elmtsIndices.Count > 400)
{
throw new ArgumentException("At most 400 elements at a time");
}
// Open source dfsu file
IDfsuFile source;
Stream stream = null;
if (useStream)
{
stream = new FileStream(dfsuFileNamePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
source = DfsuFile.Open(stream);
}
else
{
source = DfsuFile.Open(dfsuFileNamePath);
}
// Figure out "basic" dfs0 file name
string dfsuFilename = Path.GetFileNameWithoutExtension(dfsuFileNamePath);
string path = Path.GetDirectoryName(dfsuFileNamePath);
string dfs0BaseFilename = Path.Combine(path, "test_" + dfsuFilename + "-");
// Factory for creating dfs objects
DfsFactory factory = new DfsFactory();
// Create a dfs0 file for each element in elmtsIndices
DfsFile[] dfs0Files = new DfsFile[elmtsIndices.Count];
Stream[] dfs0Streams = new Stream [elmtsIndices.Count];
double timeSpan = source.TimeStepInSeconds * source.NumberOfTimeSteps;
for (int k = 0; k < elmtsIndices.Count; k++)
{
// Index of element to create dfs0 for
int elmtsIndex = elmtsIndices[k];
// Calculate element center coordinates, to be stored in dfs0 items.
// Stored as float in dfs0, hence possible loss of precision...
float x = 0, y = 0, z = 0;
int[] nodeNumbers = source.ElementTable[elmtsIndex];
for (int i = 0; i < nodeNumbers.Length; i++)
{
int nodeIndex = nodeNumbers[i] - 1; // from number to index
x += (float)source.X[nodeIndex];
y += (float)source.Y[nodeIndex];
z += source.Z[nodeIndex];
}
x /= nodeNumbers.Length;
y /= nodeNumbers.Length;
z /= nodeNumbers.Length;
// Start building dfs0 file header
DfsBuilder builder = DfsBuilder.Create("fileTitle", "appTitle", 1);
builder.SetDataType(1); // standard dfs0 value
builder.SetGeographicalProjection(source.Projection);
builder.SetTemporalAxis(factory.CreateTemporalEqCalendarAxis(eumUnit.eumUsec, source.StartDateTime, 0, source.TimeStepInSeconds));
// Add all dynamic items from dfsu file to dfs0 file
for (int j = 0; j < source.ItemInfo.Count; j++)
{
IDfsSimpleDynamicItemInfo sourceItem = source.ItemInfo[j];
DfsDynamicItemBuilder itemBuilder = builder.CreateDynamicItemBuilder();
itemBuilder.Set(sourceItem.Name, sourceItem.Quantity, sourceItem.DataType);
itemBuilder.SetAxis(factory.CreateAxisEqD0());
itemBuilder.SetValueType(sourceItem.ValueType);
itemBuilder.SetReferenceCoordinates(x, y, z); // optional
builder.AddDynamicItem(itemBuilder.GetDynamicItemInfo());
}
// Create and get file, store them in dfs0s array
string dfs0Filename = dfs0BaseFilename + (elmtsIndex).ToString("000000") + ".dfs0";
if (useStream)
{
// Create file using C# streams - necessary to provie number of time steps and timespan of data
builder.SetNumberOfTimeSteps(source.NumberOfTimeSteps);
builder.SetTimeInfo(0, timeSpan);
Stream dfs0FileStream = new FileStream(dfs0Filename, FileMode.Create, FileAccess.Write, FileShare.ReadWrite);
builder.CreateStream(dfs0FileStream);
dfs0Streams[k] = dfs0FileStream;
}
else
{
// Create file in the ordinary way. Will include statistics (of delete values etc).
builder.CreateFile(dfs0Filename);
}
dfs0Files[k] = builder.GetFile();
}
// For performance, use predefined itemdata objects when reading data from dfsu
IDfsItemData <float>[] dfsuItemDatas = new IDfsItemData <float> [source.ItemInfo.Count];
for (int j = 0; j < source.ItemInfo.Count; j++)
{
dfsuItemDatas[j] = (IDfsItemData <float>)source.ItemInfo[j].CreateEmptyItemData();
}
// Read data from dfsu and store in dfs0
float[] dfs0Data = new float[1];
for (int i = 0; i < source.NumberOfTimeSteps; i++)
{
for (int j = 0; j < source.ItemInfo.Count; j++)
{
// Read data from dfsu
IDfsItemData <float> dfsuItemData = dfsuItemDatas[j];
bool ok = source.ReadItemTimeStep(dfsuItemData, i);
float[] floats = dfsuItemData.Data;
// write data to dfs0's
for (int k = 0; k < elmtsIndices.Count; k++)
{
int elmtsIndex = elmtsIndices[k];
dfs0Data[0] = floats[elmtsIndex];
dfs0Files[k].WriteItemTimeStepNext(0, dfs0Data);
}
}
}
// Close dfsu files
source.Close();
if (stream != null)
{
stream.Close();
}
// Close all dfs0 files
for (int k = 0; k < elmtsIndices.Count; k++)
{
dfs0Files[k].Close();
if (dfs0Streams[k] != null)
{
dfs0Streams[k].Close();
}
}
}
