public static void LogToHadoop()
{
try
{
//set variables
string srcFileName = "C:\\Users\\deepakk\\Desktop\\Log.txt";
string destFolderName = "/log_error";
string destFileName = "err2.txt";
//connect to hadoop cluster
Uri myUri = new Uri("http://localhost:50070/");
string userName = "******";
WebHDFSClient myClient = new WebHDFSClient(myUri, userName);
//drop destination directory (if exists)
myClient.DeleteDirectory(destFolderName, true).Wait();
//create destination directory
myClient.CreateDirectory(destFolderName).Wait();
//load file to destination directory
myClient.CreateFile(srcFileName, destFolderName + "/" + destFileName).Wait();
//Console.ReadLine();
}
catch (Exception ex)
{
Console.WriteLine("Custom HADOOP " + ex.Message);
}
}
/// <summary>
/// Put data into hdfs,if sucess return true,else return false
/// </summary>
/// <param name="srcPath">local file path</param>
/// <param name="trgtPath">hdfs file path</param>
/// <returns></returns>
public bool PutDataToHdfs(string srcPath, string trgtPath)
{
string sucessPath = string.Empty;
sucessPath = _con.CreateFile(srcPath, trgtPath).Result;
return(!string.IsNullOrEmpty(sucessPath));
}
static void Main(string[] args)
{
var blobAdapter = new BlobStorageAdapter("joydivision",
"RMfUv/odmgsSdOcLqYfCLUKDIk9QI556Ly+3kxH8M6YgKcXwKMaoHLd6di201F/v1HVxHOjSMBJKPj1DXNSBQw==",
"joydivision",
true);
var webHdfsClient = new WebHDFSClient("hadoop", blobAdapter);
// Upload the ozzie config to appPath in order to submit job
string source = AppDomain.CurrentDomain.BaseDirectory + @"\Workflow.xml";
string target = "/handsonlabs/app/Workflow.xml".Replace('\\', '/');
Console.WriteLine("Uploading to " + target);
var createdFile = webHdfsClient.CreateFile(source, target);
createdFile.Wait();
var oozieClient = new OozieHttpClient(AzureCluster, AzureUserName, AzurePassword);
string outputPath = AzureStorageVaultRoot + "/handsonlabs/output/oozieJob/" + Guid.NewGuid().ToString("N");
var oozieJobProperties = new OozieJobProperties(
AzureUserName,
AzureStorageVaultRoot,
"headnodehost:9010",
AzureStorageVaultRoot + "/handsonlabs/app/Workflow.xml",
"",
outputPath);
var submitJob = oozieClient.SubmitJob(oozieJobProperties.ToDictionary());
string id = HttpClientTools.GetTaskResults(submitJob).Value<string>("id");
oozieClient.StartJob(id);
submitJob.Wait();
// Check job status
Task<HttpResponseMessage> t1 = oozieClient.GetJobInfo(id);
var res = HttpClientTools.GetTaskResults(t1);
}
public static void SaveFile(string srcFileName, string destFolderName, string destFileName, string uri)
{
try
{
//connect to hadoop cluster
Uri myUri = new Uri(uri);
string userName = "******";
WebHDFSClient myClient = new WebHDFSClient(myUri, userName);
//drop destination directory (if exists)
myClient.DeleteDirectory(destFolderName, true);
//create destination directory
myClient.CreateDirectory(destFolderName);
string newpathfile = destFolderName + "/" + destFileName;
//load file to destination directory
var s = myClient.CreateFile(srcFileName, newpathfile);
Console.WriteLine(s.Result);
//keep command window open until user presses enter
Console.ReadLine();
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
}
public static void LogToHadoop()
{
try
{
//set variables
string srcFileName = "C:\\Users\\deepakk\\Desktop\\Log.txt";
string destFolderName = "/log_error";
string destFileName = "err2.txt";
//connect to hadoop cluster
Uri myUri = new Uri("http://*****:*****@"C:\Users\Administrator\Desktop\integer.txt", "/user/hadoop/integer.txt")
//.ContinueWith(t => Console.WriteLine("new file located at " + t.Result))
//.ContinueWith(t => myClient.OpenFile("/user/hadoop/integer.txt")
//.ContinueWith(resp => resp.Result.Content.ReadAsStringAsync()
//.ContinueWith(bigString => Console.WriteLine("new file is " + bigString.Result.Length + " bytes long"))
//.ContinueWith(t2 => myClient.DeleteDirectory("/user/Administrator/'demosimplenewin'")
// .ContinueWith(b => Console.WriteLine("Successfully deleted file."))
// )
// )
// )
// );
//-----------------------------------------------------
//keep command window open until user presses enter
Console.ReadLine();
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
public void Map(IEnumerable <Message> sensorMessages)
{
var groupedMessages = sensorMessages.GroupBy(msg => msg.DeliveryId);
foreach (var group in groupedMessages)
{
var delivery = group.MapToDelivery();
var client = new WebHDFSClient(new Uri(@"http://127.0.0.1:50070/"), "Camper");
var remotePath = string.Format("{0}/{1}_{2}", DirectoryPath, delivery.Vehicle.VehicleId, group.Key);
var memStream = new MemoryStream(SerializationHelper.Serialize(delivery));
client.CreateFile(memStream, remotePath).ContinueWith(tsk => memStream.Dispose());
}
}
public void Map(IEnumerable<Message> sensorMessages)
{
var groupedMessages = sensorMessages.GroupBy(msg => msg.DeliveryId);
foreach (var group in groupedMessages)
{
var delivery = group.MapToDelivery();
var client = new WebHDFSClient(new Uri(@"http://127.0.0.1:50070/"), "Camper");
var remotePath = string.Format("{0}/{1}_{2}", DirectoryPath, delivery.Vehicle.VehicleId, group.Key);
var memStream = new MemoryStream(SerializationHelper.Serialize(delivery));
client.CreateFile(memStream, remotePath).ContinueWith(tsk => memStream.Dispose());
}
}
public HDFSCloudComputingDriveInfo(string hostname, string userName) : base("HDFS")
{
try {
string host = hostname;
hdfsClient = new WebHDFSClient(new Uri("http://" + host + ":50070"), userName);
contentSummary = hdfsClient.GetContentSummary("/");
byte[] myByteArray = System.Text.Encoding.UTF8.GetBytes("test");
MemoryStream ms = new MemoryStream(myByteArray);
hdfsClient.CreateFile(ms, "/test.hdfs", true);
}
catch (SafeModeException e) {
this.error = e.Message;
}
catch (Exception e) {
this.error = e.Message;
}
}
public override void ProcessInput(int inputID, PipelineBuffer buffer)
{
//Debugger.Launch();
int columnIndex = ComponentMetaData.CustomPropertyCollection[Constants.SOURCE_COLUMN_INDEX].Value;
string remotePath = ComponentMetaData.CustomPropertyCollection[Constants.HDFS_PATH_PROPERTY].Value;
while (buffer.NextRow())
{
string strFullFileName = buffer.GetString(columnIndex);
string fileName = Path.GetFileName(strFullFileName);
string remoteFileName = remotePath + "/" + fileName;
client.CreateFile(strFullFileName, remoteFileName).Wait();
}
}
public void Write(IEnumerable <Message> messages)
{
var sensorData =
messages.OrderBy(msg => msg.VehicleId)
.ThenBy(msg => msg.Timestamp)
.Select(msg => msg.Serialize())
.ToArray();
const int chunkSize = 50000;
var maxIndex = (int)(Math.Ceiling(sensorData.Length / (double)chunkSize));
var client = new WebHDFSClient(new Uri(@"http://127.0.0.1:50070/"), "Camper");
for (int index = 0; index < maxIndex; index++)
{
var chunk = sensorData.Skip(index * chunkSize).Take(chunkSize);
var content = string.Join("\n", chunk);
var memStream = new MemoryStream(Encoding.UTF8.GetBytes(content));
string remoteFile = string.Format("{0}/SensorData_{1}", DirectoryPath, index + 1);
var task = client.CreateFile(memStream, remoteFile);
task.Wait();
}
}
static void Main(string[] args)
{
var blobAdapter = new BlobStorageAdapter("joydivision",
"RMfUv/odmgsSdOcLqYfCLUKDIk9QI556Ly+3kxH8M6YgKcXwKMaoHLd6di201F/v1HVxHOjSMBJKPj1DXNSBQw==",
"joydivision",
true);
var webHdfsClient = new WebHDFSClient("hadoop", blobAdapter);
// Upload the ozzie config to appPath in order to submit job
string source = AppDomain.CurrentDomain.BaseDirectory + @"\Workflow.xml";
string target = "/handsonlabs/app/Workflow.xml".Replace('\\', '/');
Console.WriteLine("Uploading to " + target);
var createdFile = webHdfsClient.CreateFile(source, target);
createdFile.Wait();
var oozieClient = new OozieHttpClient(AzureCluster, AzureUserName, AzurePassword);
string outputPath = AzureStorageVaultRoot + "/handsonlabs/output/oozieJob/" + Guid.NewGuid().ToString("N");
var oozieJobProperties = new OozieJobProperties(
AzureUserName,
AzureStorageVaultRoot,
"headnodehost:9010",
AzureStorageVaultRoot + "/handsonlabs/app/Workflow.xml",
"",
outputPath);
var submitJob = oozieClient.SubmitJob(oozieJobProperties.ToDictionary());
string id = HttpClientTools.GetTaskResults(submitJob).Value <string>("id");
oozieClient.StartJob(id);
submitJob.Wait();
// Check job status
Task <HttpResponseMessage> t1 = oozieClient.GetJobInfo(id);
var res = HttpClientTools.GetTaskResults(t1);
}
public void Write(IEnumerable<Message> messages)
{
var sensorData =
messages.OrderBy(msg => msg.VehicleId)
.ThenBy(msg => msg.Timestamp)
.Select(msg => msg.Serialize())
.ToArray();
const int chunkSize = 50000;
var maxIndex = (int)(Math.Ceiling(sensorData.Length / (double)chunkSize));
var client = new WebHDFSClient(new Uri(@"http://127.0.0.1:50070/"), "Camper");
for (int index = 0; index < maxIndex; index++)
{
var chunk = sensorData.Skip(index * chunkSize).Take(chunkSize);
var content = string.Join("\n", chunk);
var memStream = new MemoryStream(Encoding.UTF8.GetBytes(content));
string remoteFile = string.Format("{0}/SensorData_{1}", DirectoryPath, index + 1);
var task = client.CreateFile(memStream, remoteFile);
task.Wait();
}
}