public static void LogToHadoop() { try { //set variables string srcFileName = "C:\\Users\\deepakk\\Desktop\\Log.txt"; string destFolderName = "/log_error"; string destFileName = "err2.txt"; //connect to hadoop cluster Uri myUri = new Uri("http://localhost:50070/"); string userName = "******"; WebHDFSClient myClient = new WebHDFSClient(myUri, userName); //drop destination directory (if exists) myClient.DeleteDirectory(destFolderName, true).Wait(); //create destination directory myClient.CreateDirectory(destFolderName).Wait(); //load file to destination directory myClient.CreateFile(srcFileName, destFolderName + "/" + destFileName).Wait(); //Console.ReadLine(); } catch (Exception ex) { Console.WriteLine("Custom HADOOP " + ex.Message); } }
/// <summary> /// Put data into hdfs,if sucess return true,else return false /// </summary> /// <param name="srcPath">local file path</param> /// <param name="trgtPath">hdfs file path</param> /// <returns></returns> public bool PutDataToHdfs(string srcPath, string trgtPath) { string sucessPath = string.Empty; sucessPath = _con.CreateFile(srcPath, trgtPath).Result; return(!string.IsNullOrEmpty(sucessPath)); }
static void Main(string[] args) { var blobAdapter = new BlobStorageAdapter("joydivision", "RMfUv/odmgsSdOcLqYfCLUKDIk9QI556Ly+3kxH8M6YgKcXwKMaoHLd6di201F/v1HVxHOjSMBJKPj1DXNSBQw==", "joydivision", true); var webHdfsClient = new WebHDFSClient("hadoop", blobAdapter); // Upload the ozzie config to appPath in order to submit job string source = AppDomain.CurrentDomain.BaseDirectory + @"\Workflow.xml"; string target = "/handsonlabs/app/Workflow.xml".Replace('\\', '/'); Console.WriteLine("Uploading to " + target); var createdFile = webHdfsClient.CreateFile(source, target); createdFile.Wait(); var oozieClient = new OozieHttpClient(AzureCluster, AzureUserName, AzurePassword); string outputPath = AzureStorageVaultRoot + "/handsonlabs/output/oozieJob/" + Guid.NewGuid().ToString("N"); var oozieJobProperties = new OozieJobProperties( AzureUserName, AzureStorageVaultRoot, "headnodehost:9010", AzureStorageVaultRoot + "/handsonlabs/app/Workflow.xml", "", outputPath); var submitJob = oozieClient.SubmitJob(oozieJobProperties.ToDictionary()); string id = HttpClientTools.GetTaskResults(submitJob).Value<string>("id"); oozieClient.StartJob(id); submitJob.Wait(); // Check job status Task<HttpResponseMessage> t1 = oozieClient.GetJobInfo(id); var res = HttpClientTools.GetTaskResults(t1); }
public static void SaveFile(string srcFileName, string destFolderName, string destFileName, string uri) { try { //connect to hadoop cluster Uri myUri = new Uri(uri); string userName = "******"; WebHDFSClient myClient = new WebHDFSClient(myUri, userName); //drop destination directory (if exists) myClient.DeleteDirectory(destFolderName, true); //create destination directory myClient.CreateDirectory(destFolderName); string newpathfile = destFolderName + "/" + destFileName; //load file to destination directory var s = myClient.CreateFile(srcFileName, newpathfile); Console.WriteLine(s.Result); //keep command window open until user presses enter Console.ReadLine(); } catch (Exception ex) { Console.WriteLine(ex.ToString()); } }
public static void LogToHadoop() { try { //set variables string srcFileName = "C:\\Users\\deepakk\\Desktop\\Log.txt"; string destFolderName = "/log_error"; string destFileName = "err2.txt"; //connect to hadoop cluster Uri myUri = new Uri("http://*****:*****@"C:\Users\Administrator\Desktop\integer.txt", "/user/hadoop/integer.txt") //.ContinueWith(t => Console.WriteLine("new file located at " + t.Result)) //.ContinueWith(t => myClient.OpenFile("/user/hadoop/integer.txt") //.ContinueWith(resp => resp.Result.Content.ReadAsStringAsync() //.ContinueWith(bigString => Console.WriteLine("new file is " + bigString.Result.Length + " bytes long")) //.ContinueWith(t2 => myClient.DeleteDirectory("/user/Administrator/'demosimplenewin'") // .ContinueWith(b => Console.WriteLine("Successfully deleted file.")) // ) // ) // ) // ); //----------------------------------------------------- //keep command window open until user presses enter Console.ReadLine(); } catch (Exception ex) { Console.WriteLine(ex.Message); } }
public void Map(IEnumerable <Message> sensorMessages) { var groupedMessages = sensorMessages.GroupBy(msg => msg.DeliveryId); foreach (var group in groupedMessages) { var delivery = group.MapToDelivery(); var client = new WebHDFSClient(new Uri(@"http://127.0.0.1:50070/"), "Camper"); var remotePath = string.Format("{0}/{1}_{2}", DirectoryPath, delivery.Vehicle.VehicleId, group.Key); var memStream = new MemoryStream(SerializationHelper.Serialize(delivery)); client.CreateFile(memStream, remotePath).ContinueWith(tsk => memStream.Dispose()); } }
public void Map(IEnumerable<Message> sensorMessages) { var groupedMessages = sensorMessages.GroupBy(msg => msg.DeliveryId); foreach (var group in groupedMessages) { var delivery = group.MapToDelivery(); var client = new WebHDFSClient(new Uri(@"http://127.0.0.1:50070/"), "Camper"); var remotePath = string.Format("{0}/{1}_{2}", DirectoryPath, delivery.Vehicle.VehicleId, group.Key); var memStream = new MemoryStream(SerializationHelper.Serialize(delivery)); client.CreateFile(memStream, remotePath).ContinueWith(tsk => memStream.Dispose()); } }
public HDFSCloudComputingDriveInfo(string hostname, string userName) : base("HDFS") { try { string host = hostname; hdfsClient = new WebHDFSClient(new Uri("http://" + host + ":50070"), userName); contentSummary = hdfsClient.GetContentSummary("/"); byte[] myByteArray = System.Text.Encoding.UTF8.GetBytes("test"); MemoryStream ms = new MemoryStream(myByteArray); hdfsClient.CreateFile(ms, "/test.hdfs", true); } catch (SafeModeException e) { this.error = e.Message; } catch (Exception e) { this.error = e.Message; } }
public override void ProcessInput(int inputID, PipelineBuffer buffer) { //Debugger.Launch(); int columnIndex = ComponentMetaData.CustomPropertyCollection[Constants.SOURCE_COLUMN_INDEX].Value; string remotePath = ComponentMetaData.CustomPropertyCollection[Constants.HDFS_PATH_PROPERTY].Value; while (buffer.NextRow()) { string strFullFileName = buffer.GetString(columnIndex); string fileName = Path.GetFileName(strFullFileName); string remoteFileName = remotePath + "/" + fileName; client.CreateFile(strFullFileName, remoteFileName).Wait(); } }
public void Write(IEnumerable <Message> messages) { var sensorData = messages.OrderBy(msg => msg.VehicleId) .ThenBy(msg => msg.Timestamp) .Select(msg => msg.Serialize()) .ToArray(); const int chunkSize = 50000; var maxIndex = (int)(Math.Ceiling(sensorData.Length / (double)chunkSize)); var client = new WebHDFSClient(new Uri(@"http://127.0.0.1:50070/"), "Camper"); for (int index = 0; index < maxIndex; index++) { var chunk = sensorData.Skip(index * chunkSize).Take(chunkSize); var content = string.Join("\n", chunk); var memStream = new MemoryStream(Encoding.UTF8.GetBytes(content)); string remoteFile = string.Format("{0}/SensorData_{1}", DirectoryPath, index + 1); var task = client.CreateFile(memStream, remoteFile); task.Wait(); } }
static void Main(string[] args) { var blobAdapter = new BlobStorageAdapter("joydivision", "RMfUv/odmgsSdOcLqYfCLUKDIk9QI556Ly+3kxH8M6YgKcXwKMaoHLd6di201F/v1HVxHOjSMBJKPj1DXNSBQw==", "joydivision", true); var webHdfsClient = new WebHDFSClient("hadoop", blobAdapter); // Upload the ozzie config to appPath in order to submit job string source = AppDomain.CurrentDomain.BaseDirectory + @"\Workflow.xml"; string target = "/handsonlabs/app/Workflow.xml".Replace('\\', '/'); Console.WriteLine("Uploading to " + target); var createdFile = webHdfsClient.CreateFile(source, target); createdFile.Wait(); var oozieClient = new OozieHttpClient(AzureCluster, AzureUserName, AzurePassword); string outputPath = AzureStorageVaultRoot + "/handsonlabs/output/oozieJob/" + Guid.NewGuid().ToString("N"); var oozieJobProperties = new OozieJobProperties( AzureUserName, AzureStorageVaultRoot, "headnodehost:9010", AzureStorageVaultRoot + "/handsonlabs/app/Workflow.xml", "", outputPath); var submitJob = oozieClient.SubmitJob(oozieJobProperties.ToDictionary()); string id = HttpClientTools.GetTaskResults(submitJob).Value <string>("id"); oozieClient.StartJob(id); submitJob.Wait(); // Check job status Task <HttpResponseMessage> t1 = oozieClient.GetJobInfo(id); var res = HttpClientTools.GetTaskResults(t1); }
public void Write(IEnumerable<Message> messages) { var sensorData = messages.OrderBy(msg => msg.VehicleId) .ThenBy(msg => msg.Timestamp) .Select(msg => msg.Serialize()) .ToArray(); const int chunkSize = 50000; var maxIndex = (int)(Math.Ceiling(sensorData.Length / (double)chunkSize)); var client = new WebHDFSClient(new Uri(@"http://127.0.0.1:50070/"), "Camper"); for (int index = 0; index < maxIndex; index++) { var chunk = sensorData.Skip(index * chunkSize).Take(chunkSize); var content = string.Join("\n", chunk); var memStream = new MemoryStream(Encoding.UTF8.GetBytes(content)); string remoteFile = string.Format("{0}/SensorData_{1}", DirectoryPath, index + 1); var task = client.CreateFile(memStream, remoteFile); task.Wait(); } }