public static void UploadToDfs(Uri dfsDirectory, string localPath)
{
DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsDirectory.AbsoluteUri);
try
{
if (dfsDirectory.Scheme == "hdfs")
{
using (var hdfs = new Microsoft.Research.Peloponnese.Hdfs.HdfsInstance(dfsDirectory))
{
string dfsPath = dfsDirectory.AbsolutePath.TrimEnd('/') + "/" + Path.GetFileName(localPath);
DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsPath);
hdfs.UploadAll(localPath, dfsPath);
}
}
else if (dfsDirectory.Scheme == "azureblob")
{
string account, key, container, blob;
Microsoft.Research.Peloponnese.Azure.Utils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob);
var azure = new Microsoft.Research.Peloponnese.Azure.AzureDfsClient(account, key, container);
string dfsPath = blob.TrimEnd('/') + "/" + Path.GetFileName(localPath);
Uri dfsUri = Microsoft.Research.Peloponnese.Azure.Utils.ToAzureUri(account, container, dfsPath, null, key);
DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsUri.AbsoluteUri);
azure.PutDfsFile(dfsUri, localPath);
}
else if (dfsDirectory.Scheme == "file")
{
string dstPath = Path.Combine(dfsDirectory.AbsolutePath, Path.GetFileName(localPath));
File.Copy(localPath, dstPath);
}
}
catch (Exception e)
{
DryadLogger.LogWarning("Failed to upload query plan: " + e.ToString());
}
}
public static void UploadToDfs(Uri dfsDirectory, string dfsName, string payload)
{
byte[] payloadBytes = System.Text.Encoding.UTF8.GetBytes(payload);
DryadLogger.LogInformation("Uploading payload to " + dfsName + " at " + dfsDirectory.AbsoluteUri);
try
{
if (dfsDirectory.Scheme == "hdfs")
{
using (var hdfs = new Microsoft.Research.Peloponnese.Hdfs.HdfsInstance(dfsDirectory))
{
string dfsPath = dfsDirectory.AbsolutePath + dfsName;
hdfs.WriteAll(dfsPath, payloadBytes);
}
}
else if (dfsDirectory.Scheme == "azureblob")
{
string account, key, container, blob;
Microsoft.Research.Peloponnese.Azure.Utils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob);
var azure = new Microsoft.Research.Peloponnese.Azure.AzureDfsClient(account, key, container);
string dfsPath = blob + dfsName;
Uri dfsUri = Microsoft.Research.Peloponnese.Azure.Utils.ToAzureUri(account, container, dfsPath, null, key);
azure.PutDfsFile(dfsUri, payloadBytes);
}
else if (dfsDirectory.Scheme == "file")
{
string dstPath = Path.Combine(dfsDirectory.AbsolutePath, dfsName);
File.WriteAllBytes(dstPath, payloadBytes);
}
}
catch (Exception e)
{
DryadLogger.LogWarning("Failed to upload final output: " + e.ToString());
}
}
public static void UploadToDfs(Uri dfsDirectory, string localPath)
{
DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsDirectory.AbsoluteUri);
try
{
if (dfsDirectory.Scheme == "hdfs")
{
using (var hdfs = new Microsoft.Research.Peloponnese.Hdfs.HdfsInstance(dfsDirectory))
{
string dfsPath = dfsDirectory.AbsolutePath.TrimEnd('/') + "/" + Path.GetFileName(localPath);
DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsPath);
hdfs.UploadAll(localPath, dfsPath);
}
}
else if (dfsDirectory.Scheme == "azureblob")
{
string account, key, container, blob;
Microsoft.Research.Peloponnese.Azure.Utils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob);
var azure = new Microsoft.Research.Peloponnese.Azure.AzureDfsClient(account, key, container);
string dfsPath = blob.TrimEnd('/') + "/" + Path.GetFileName(localPath);
Uri dfsUri = Microsoft.Research.Peloponnese.Azure.Utils.ToAzureUri(account, container, dfsPath, null, key);
DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsUri.AbsoluteUri);
azure.PutDfsFile(dfsUri, localPath);
}
else if (dfsDirectory.Scheme == "file")
{
string dstPath = Path.Combine(dfsDirectory.AbsolutePath, Path.GetFileName(localPath));
File.Copy(localPath, dstPath);
}
}
catch (Exception e)
{
DryadLogger.LogWarning("Failed to upload query plan: " + e.ToString());
}
}
public static void UploadToDfs(Uri dfsDirectory, string dfsName, string payload)
{
byte[] payloadBytes = System.Text.Encoding.UTF8.GetBytes(payload);
DryadLogger.LogInformation("Uploading payload to " + dfsName + " at " + dfsDirectory.AbsoluteUri);
try
{
if (dfsDirectory.Scheme == "hdfs")
{
using (var hdfs = new Microsoft.Research.Peloponnese.Hdfs.HdfsInstance(dfsDirectory))
{
string dfsPath = dfsDirectory.AbsolutePath + dfsName;
hdfs.WriteAll(dfsPath, payloadBytes);
}
}
else if (dfsDirectory.Scheme == "azureblob")
{
string account, key, container, blob;
Microsoft.Research.Peloponnese.Azure.Utils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob);
var azure = new Microsoft.Research.Peloponnese.Azure.AzureDfsClient(account, key, container);
string dfsPath = blob + dfsName;
Uri dfsUri = Microsoft.Research.Peloponnese.Azure.Utils.ToAzureUri(account, container, dfsPath, null, key);
azure.PutDfsFile(dfsUri, payloadBytes);
}
else if (dfsDirectory.Scheme == "file")
{
string dstPath = Path.Combine(dfsDirectory.AbsolutePath, dfsName);
File.WriteAllBytes(dstPath, payloadBytes);
}
}
catch (Exception e)
{
DryadLogger.LogWarning("Failed to upload final output: " + e.ToString());
}
}