/// <summary>
/// Send webrequest for given URL and protocol
/// </summary>
/// <param name="url">URL</param>
/// <param name="protocol">Protocol</param>
/// <param name="clusterType"></param>
/// <returns>Returns success status of web request <true>return true if success</true></returns>
private bool SendWebRequest(string url, string protocol, ClusterType clusterType)
{
try
{
WebRequest request = WebRequest.Create(url);
if (clusterType.Equals(ClusterType.Secure))
{
request.Credentials = new NetworkCredential(userName, password);
}
else if (clusterType.Equals(ClusterType.Azure))
{
request.Headers.Add("Username", userName);
request.Headers.Add("Password", password);
}
request.Method = protocol;
using (var response = request.GetResponse())
{
response.Close();
}
return(true);
}
catch (Exception e)
{
Console.Error.WriteLine("/**SendWebRequest(string url, string protocol)**/" + e.Message + "\n" + e.StackTrace);
return(false);
}
}
/// <summary>
/// Renames Selected file/folder(s)
/// </summary>
/// <param name="host"></param>
/// <param name="clusterType"></param>
/// <param name="sourceName">Current file name</param>
/// <param name="replication"></param>
/// <returns>If Rename succeeds returns true else returns false</returns>
public bool SetReplication(string host, ClusterType clusterType, string sourceName, int replication)
{
string url = (clusterType.Equals(ClusterType.Secure))
? ("https://" + host + ":50470/webhdfs/v1" + sourceName + "/?op=SETREPLICATION&replication=" +
replication).Replace("#", "%23")
: (clusterType.Equals(ClusterType.Azure))
? ("https://" + host + ":8000/webhdfs/v1/" + sourceName + "/?user.name=SYSTEM" +
"&op=SETREPLICATION&replication=" + replication).Replace("#", "%23")
: ("http://" + host + ":50070/webhdfs/v1" + sourceName +
"/?user.name=SYSTEM&op=SETREPLICATION&replication=" + replication).Replace("#", "%23");
return(SendWebRequest(url, "PUT", clusterType));
}
/// <summary>
/// Deletes a selected file
/// </summary>
/// <param name="clusterType"></param>
/// <param name="filePath">HDFS path of file needs to be deleted</param>
/// <param name="host"></param>
/// <returns>If Delete succeeds returns true else returns false</returns>
public bool DeleteFile(string host, ClusterType clusterType, string filePath)
{
string url = (clusterType.Equals(ClusterType.Secure))
? ("https://" + host + ":50470/webhdfs/v1/" + filePath.TrimStart('/') + "/?op=DELETE&recursive=true")
.Replace("#", "%23")
: (clusterType.Equals(ClusterType.Azure))
? ("https://" + host + ":8000/webhdfs/v1/" + filePath.TrimStart('/') + "/?user.name=" + userName +
"&op=DELETE&recursive=true").Replace("#", "%23").Replace("+", "%2b")
: ("http://" + host + ":50070/webhdfs/v1/" + filePath.TrimStart('/') +
"/?user.name=SYSTEM&op=DELETE&recursive=true").Replace("#", "%23").Replace("+", "%2b");
return(SendWebRequest(url, "DELETE", clusterType));
}
/// <summary>
/// Creates a new folder
/// </summary>
/// <param name="clusterType"></param>
/// <param name="folderPath">HDFS Path of folder to be created</param>
/// <param name="host"></param>
/// <returns>If folder is created returns true else returns false</returns>
public bool CreateDirectory(string host, ClusterType clusterType, string folderPath)
{
string url = (clusterType.Equals(ClusterType.Secure))
? ("https://" + host + ":50470/webhdfs/v1/" + folderPath.TrimStart('/') + "/?op=MKDIRS").Replace("#",
"%23")
: (clusterType.Equals(ClusterType.Azure))
? ("https://" + host + ":8000/webhdfs/v1/" + folderPath.TrimStart('/') + "/?user.name=" + userName +
"&op=MKDIRS").Replace("#", "%23").Replace("+", "%2b")
: ("http://" + host + ":50070/webhdfs/v1/" + folderPath.TrimStart('/') +
"/?user.name=SYSTEM&op=MKDIRS")
.Replace("#", "%23");
return(SendWebRequest(url, "PUT", clusterType));
}
/// <summary>
/// Upload file operation
/// </summary>
/// <param name="host"></param>
/// <param name="clusterType"></param>
/// <param name="fileToUpload"></param>
/// <param name="hdfspathToUpload"></param>
private void UploadFile(string host, ClusterType clusterType, string fileToUpload, string hdfspathToUpload)
{
try
{
string url = string.Empty;
hdfspathToUpload = hdfspathToUpload.TrimEnd('/').TrimStart('/');
//Creating the url
url = (clusterType.Equals(ClusterType.Secure))
? ("https://" + host + ":50470/webhdfs/v1/" + hdfspathToUpload + "/?op=CREATE&data=true").Replace(
"#", "%23")
: (clusterType.Equals(ClusterType.Azure))
? ("https://" + host + ":8000/webhdfs/v1/" + hdfspathToUpload + "/?user.name=" + userName +
"&op=CREATE&data=true").Replace("#", "%23").Replace("+", "%2b")
: ("http://" + host + ":50070/webhdfs/v1/" + hdfspathToUpload +
"/?user.name=SYSTEM&op=CREATE&data=true").Replace("#", "%23");
var webRequest = WebRequest.Create(url);
if (clusterType.Equals(ClusterType.Secure))
{
webRequest.Credentials = new NetworkCredential(userName, password);
}
else if (clusterType.Equals(ClusterType.Azure))
{
webRequest.Headers.Add("Username", userName);
webRequest.Headers.Add("Password", password);
}
webRequest.Method = "PUT";
const int chunkSize = 1024 * 1024; // read the file by chunks of 10 MB
using (var file = new FileStream(fileToUpload, FileMode.Open, FileAccess.Read))
{
var buffer = new byte[chunkSize];
long fileLength = file.Length;
if (fileLength <= chunkSize)
{
buffer = new byte[file.Length];
}
webRequest.ContentType = "application/octet-stream";
webRequest.ContentLength = buffer.Length;
file.Read(buffer, 0, buffer.Length);
Stream objStream = webRequest.GetRequestStream();
objStream.Write(buffer, 0, buffer.Length);
objStream.Close();
objStream = webRequest.GetResponse().GetResponseStream();
url = (clusterType.Equals(ClusterType.Secure))
? ("https://" + host + ":50470/webhdfs/v1/" + hdfspathToUpload +
"/?op=SETREPLICATION&replication=1").Replace("#", "%23")
: (clusterType.Equals(ClusterType.Azure))
? ("https://" + host + ":8000/webhdfs/v1/" + hdfspathToUpload + "/?user.name=SYSTEM" +
"&op=SETREPLICATION&replication=1").Replace("#", "%23")
: ("http://" + host + ":50070/webhdfs/v1/" + hdfspathToUpload +
"/?user.name=SYSTEM" + "&op=SETREPLICATION&replication=1").Replace("#", "%23");
SendWebRequest(url, "PUT", clusterType);
buffer = (file.Length - file.Position) > chunkSize ? new byte[chunkSize] : new byte[(file.Length - file.Position)];
while (file.Read(buffer, 0, buffer.Length) > 0)
{
//Append File
url = (clusterType.Equals(ClusterType.Secure))
? ("https://" + host + ":50470/webhdfs/v1/" +
hdfspathToUpload +
"/?Op=APPEND&buffersize=" + webRequest.ContentLength)
: (clusterType.Equals(ClusterType.Azure))
? ("https://" + host + ":8000/webhdfs/v1/" + hdfspathToUpload + "/?user.name=" +
userName + "&op=APPEND&data=true")
: ("http://" + host + ":50070/webhdfs/v1/" +
hdfspathToUpload +
"/?user.name=SYSTEM&Op=APPEND&buffersize=" + webRequest.ContentLength);
WebRequest webRequest1 = WebRequest.Create(url);
webRequest1.Credentials = new NetworkCredential(userName, password);
webRequest1.Method = "POST";
webRequest1.ContentType = "application/octet-stream";
webRequest1.ContentLength = buffer.Length;
Stream objStreamAppend = webRequest1.GetRequestStream();
objStreamAppend.Write(buffer, 0, buffer.Length);
objStreamAppend.Close();
webRequest1.GetResponse().GetResponseStream();
buffer = (file.Length - file.Position) > chunkSize ? new byte[chunkSize] : new byte[(file.Length - file.Position)];
}
}
if (host == "localhost" || host == "127.0.0.1")
{
url = ("http://" + host + ":50070/webhdfs/v1/" + hdfspathToUpload +
"/?user.name=SYSTEM" + "&op=SETREPLICATION&replication=1").Replace("#", "%23");
}
else
{
url = (clusterType.Equals(ClusterType.Secure))
? ("https://" + host + ":50470/webhdfs/v1/" + hdfspathToUpload +
"/?op=SETREPLICATION&replication=3").Replace("#", "%23")
: (clusterType.Equals(ClusterType.Azure))
? ("https://" + host + ":8000/webhdfs/v1/" + hdfspathToUpload + "/?user.name=SYSTEM" +
"&op=SETREPLICATION&replication=3").Replace("#", "%23")
: ("http://" + host + ":50070/webhdfs/v1/" + hdfspathToUpload +
"/?user.name=SYSTEM" + "&op=SETREPLICATION&replication=3").Replace("#", "%23");
}
SendWebRequest(url, "PUT", clusterType);
}
catch (Exception e)
{
Console.Error.WriteLine(
"/**UploadHdfsFile(string nameNodeIP, string fileToUpload, string hdfspathToUpload)**/" +
e.Message + "\n" + e.StackTrace);
}
}