private static void PrepareCommonData(string sessionId)
{
if (common_data_size > 0)
{
#if HPCPACK
byte[] common_data = new byte[common_data_size];
Random r = new Random();
r.NextBytes(common_data);
commonData_dataClientId = sessionId.ToString();
Log("Begin to write common data.");
data.CreateDataClientStart = DateTime.Now;
using (DataClient dataClient = DataClient.Create(headnode, commonData_dataClientId, null, !onDemand))
{
data.CreateDataClientEnd = DateTime.Now;
dataClient.SetDataLifeCycle(new DataLifeCycle(sessionId));
data.WriteDataClientStart = DateTime.Now;
dataClient.WriteRawBytesAll(common_data, common_data_compress);
data.WriteDataClientEnd = DateTime.Now;
dataClient.Close();
}
Log("Common data written done.");
#else
throw new NotSupportedException("No common data support yet.");
#endif
}
else
{
Log("Common data size is 0, skip creating common data.");
}
}
static void Main(string[] args)
{
//create prime number table of prime numbers between 1 and 200000
Console.WriteLine("Generating prime number table...");
List <int> PrimeNumberTable = GeneratePrimeNumberTable(200000);
const string headnode = "head.contoso.com";
string dataId = "PRIME_NUMBER_TABLE";
Console.WriteLine("Creating data client {0}", dataId);
try
{
//create DataClient to send data
using (DataClient dataClient = DataClient.Create(headnode, dataId))
{
//WriteAll can be called only once per data client
dataClient.WriteAll <List <int> >(PrimeNumberTable);
}
}
catch (DataException ex)
{
//If data client already exists, delete it and try again
if (ex.ErrorCode == 103809028)
{
Console.WriteLine("{0} already exists, delete it and send again.", dataId);
DataClient.Delete(headnode, dataId);
using (DataClient dataClient = DataClient.Create(headnode, dataId))
{
dataClient.WriteAll <List <int> >(PrimeNumberTable);
}
}
}
Console.WriteLine("{0} has been sent to cluster", dataId);
Console.WriteLine("Press any key to exit");
Console.ReadKey();
}
/// <summary>
/// Upload Azure files
/// </summary>
/// <param name="startInfo">The session start info</param>
private IList <string> UploadAzureFiles(SessionStartInfo startInfo)
{
List <string> newClientIdList = null;
if (startInfo.DependFiles != null && startInfo.DependFiles.Count > 0)
{
bool allUploaded = false;
SessionBase.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "Start to upload data files to Azure blob");
newClientIdList = new List <string>();
List <string> depList = new List <string>();
try
{
foreach (string srcfile in startInfo.DependFiles.Keys)
{
if (!File.Exists(srcfile))
{
throw new IOException(string.Format("The file {0} doesn't exist", srcfile));
}
string md5ByteStr = string.Empty;
string md5base64Str = string.Empty;
using (var md5 = MD5.Create())
{
using (var stream = File.OpenRead(srcfile))
{
var hashBytes = md5.ComputeHash(stream);
md5base64Str = Convert.ToBase64String(hashBytes);
md5ByteStr = BitConverter.ToString(hashBytes).Replace("-", string.Empty);
SessionBase.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "Source file {0} md5: {1}", srcfile, md5ByteStr);
}
}
string clientId = string.Format("{0}_{1}", startInfo.Username.Replace(@"\", "-").ToLower(), md5ByteStr);
bool removeClient = false;
DataClient client = null;
try
{
client = DataClient.Open(startInfo, clientId);
string contentMd5 = client.GetContentMd5();
if (contentMd5 != md5base64Str)
{
SessionBase.TraceSource.TraceEvent(TraceEventType.Information, 0, "The content of data client {0} is not correct: expected md5 {1}, real md5 {2}", clientId, md5base64Str, contentMd5);
removeClient = true;
}
}
catch (DataException e)
{
if (e.ErrorCode == DataErrorCode.DataTransferToAzureFailed || e.ErrorCode == DataErrorCode.DataInconsistent)
{
// DataTransferToAzureFailed: Blob primary data container and the data transfer to blob failed.
// DataInconsistent: The data is inconsistent
SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "The data client {0} already exists but data corrupt: {1}", clientId, e);
removeClient = true;
}
else if (e.ErrorCode == DataErrorCode.ConnectDataServiceFailure)
{
SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "Open data client {0} failed due to communication failure: {1}", clientId, e);
if (e.InnerException is MessageSecurityException)
{
// If caused by MessageSecurityException, we shall throw InnerException, so that prompt for username/password
throw e.InnerException;
}
else
{
throw;
}
}
else if (e.ErrorCode != DataErrorCode.DataClientNotFound)
{
SessionBase.TraceSource.TraceEvent(TraceEventType.Error, 0, "Open data client {0} failed: {1}", clientId, e);
throw;
}
}
try
{
if (removeClient)
{
DataClient.Delete(startInfo, clientId);
client = null;
}
if (client == null)
{
SessionBase.TraceSource.TraceEvent(TraceEventType.Information, 0, "Create data client {0} for file {1}", clientId, srcfile);
client = DataClient.Create(startInfo, clientId, true);
newClientIdList.Add(clientId);
client.WriteRawBytesAll(File.ReadAllBytes(srcfile));
}
depList.Add(string.Format("{0}={1}", clientId, startInfo.DependFiles[srcfile]));
}
catch (Exception e)
{
SessionBase.TraceSource.TraceEvent(TraceEventType.Error, 0, "Failed to create data client {0}: {1}", clientId, e);
throw;
}
}
allUploaded = true;
}
finally
{
if (!allUploaded && newClientIdList.Count > 0)
{
SessionBase.TraceSource.TraceEvent(TraceEventType.Information, 0, "Failed to upload some files, remove the files already uploaded");
CleanUpDataClients(startInfo, newClientIdList);
newClientIdList = null;
}
}
startInfo.Data.DependFiles = string.Join(";", depList.ToArray());
}
return(newClientIdList);
}
static void Main(string[] args)
{
string hostname = "[headnode]";
// DataClient Id used to identify the data, this should be unique across the cluster
string raw_data_id = "raw_data_id";
string dictionary_data_id = "dictionary_data_id";
Console.WriteLine("Start. " + DateTime.Now.ToLongTimeString());
// Create a DataClient to store a Dictionary data
using (DataClient client = DataClient.Create(hostname, dictionary_data_id))
{
Console.WriteLine("Data {0} Created. {1} ", dictionary_data_id, DateTime.Now.ToLongTimeString());
// Here we have a DataClient whose life cycle is not managed by SOA
Dictionary <string, string> objects = new Dictionary <string, string>();
objects.Add("key1", "value1");
objects.Add("key2", "value2");
// WriteAll() can only be called once on a data client.
client.WriteAll <Dictionary <string, string> >(objects);
}
SessionStartInfo sessionStartInfo = new SessionStartInfo(hostname, "CommonDataService");
// Pass DataClient Id in SOA session's environment variable so that it could be read from service code
sessionStartInfo.Environments.Add("DICTIONARY_DATA_ID", dictionary_data_id);
using (Session session = Session.CreateSession(sessionStartInfo))
{
Console.WriteLine("Session {0} Created. {1} ", session.Id, DateTime.Now.ToLongTimeString());
// Create a DataClient to store the raw data read from the file
using (DataClient client = DataClient.Create(hostname, raw_data_id))
{
Console.WriteLine("Data {0} Created {1}. ", client.Id, DateTime.Now.ToLongTimeString());
// Add a data life cycle management so that it'll be deleted when session is done
// Otherwise, the data will have to be cleaned up by client
client.SetDataLifeCycle(new DataLifeCycle(session.Id));
// WriteRawBytesAll() doesn't serialize the object and will write the byte stream directly.
// Use this when you want to transport a file or have non-.Net code that cannot handle .net serialization.
client.WriteRawBytesAll(File.ReadAllBytes("DataFile.txt"));
}
// Send/Receive SOA requests
using (BrokerClient <ICommonDataService> client = new BrokerClient <ICommonDataService>(session))
{
Console.WriteLine("Send Requests. " + DateTime.Now.ToLongTimeString());
client.SendRequest <GetDataRequest>(new GetDataRequest(raw_data_id));
client.EndRequests();
Console.WriteLine("Get Response. " + DateTime.Now.ToLongTimeString());
foreach (BrokerResponse <GetDataResponse> resp in client.GetResponses <GetDataResponse>())
{
string result = resp.Result.GetDataResult;
Console.WriteLine(result);
}
}
Console.WriteLine("Start closing session. " + DateTime.Now.ToLongTimeString());
}
Console.WriteLine("Session Closed. " + DateTime.Now.ToLongTimeString());
Console.WriteLine("Start cleaning up the Data {0}. {1} ", dictionary_data_id, DateTime.Now.ToLongTimeString());
// We should delete the DataClient "dictionary_data_id" here since it's not managed by SOA
DataClient.Delete(hostname, dictionary_data_id);
Console.WriteLine("Press any key to exit.");
Console.ReadKey();
}