public static List <HdfsFileInfo> ParseJsonArray(string json)
{
try
{
if (string.IsNullOrWhiteSpace(json))
{
return(null);
}
var jObj = JsonConvert.DeserializeObject(json);
if (null == jObj)
{
throw new Exception("HdfsFileInfo Json解析错误");
}
var statusArray = (jObj as JObject)["FileStatuses"]["FileStatus"] as JArray;
List <HdfsFileInfo> files = new List <HdfsFileInfo>();
foreach (var item in statusArray)
{
files.Add(FromJson(item as JObject));
}
return(files);
}
catch (Exception ex)
{
DebugHelper.Error(ex);
return(null);
}
}
public async Task <string> Upload(HttpPostedFileBase captcha)
{
if (ModelState.IsValid)
{
try
{
if (captcha != null)
{
using (MemoryStream ms = new MemoryStream())
{
captcha.InputStream.CopyTo(ms);
var captchaModel = new Captcha()
{
Image = ms.GetBuffer(),
Passed = false,
CreatedTime = DateTime.Now
};
db.Captchas.Add(captchaModel);
db.SaveChanges();
var captchaResult = await GetCaptChaResult(captchaModel.Id);
return(SerializerHelper.SerializeJsonObject(captchaResult));
}
}
}
catch (Exception ex)
{
DebugHelper.Error(ex);
}
}
return("");
}
public void Receive(string from, string to, string content, string date)
{
if (ModelState.IsValid)
{
try
{
if (!string.IsNullOrEmpty(content))
{
var OTPModel = new OTP()
{
Content = content,
Passed = false,
CreatedTime = DateTime.Parse(date),
From = from,
To = to
};
ParseOTP(OTPModel);
db.OTPs.Add(OTPModel);
db.SaveChanges();
}
}
catch (Exception ex)
{
DebugHelper.Error(ex);
}
}
}
public static HttpResult ReadFilePart(string file, long offset, long size)
{
try
{
if (string.IsNullOrWhiteSpace(file))
{
throw new ArgumentNullException("file");
}
if (null == GetStatus(file))
{
return new HttpResult()
{
Code = -1,
ErrorLog = "File Not Exist."
}
}
;
string args = string.Format("&offset={0}&length={1}", offset, size);
string url = BuildUrl(file, WebHdfsAPI.OPEN, "user.name=hadoop&noredirect=false" + args);
string redirUrl = HttpClientHelperV45.GetRedirectedURL(url, HttpMethod.GET);
if (!string.IsNullOrWhiteSpace(redirUrl))
{
url = TranslateHostname(redirUrl);
}
return(HttpClientHelperV45.Request(url, new TimeSpan(0, 0, 6)));
}
catch (Exception ex)
{
DebugHelper.OutLog("获取HDFS文件失败：" + ex.Message);
DebugHelper.Error(ex, ex.Message);
return(null);
}
}
/// <summary>
/// 上传文件（自动生成上级目录）
/// </summary>
/// <param name="srcFile">本地文件路径</param>
/// <param name="hdfsPath">HDFS目标路径</param>
/// <param name="errlog">日志信息</param>
/// <returns></returns>
public static bool UploadFile(string srcFile, string hdfsPath, out string errlog)
{
try
{
// 确保目录存在
MkDir(hdfsPath);
// 自动拼接文件名
string fileName = new FileInfo(srcFile).Name;
if (!hdfsPath.Contains(fileName))
{
hdfsPath += "/" + fileName;
}
// Step 1. redirect to datanode
// CAUSE: avoid sending file data before the server redirection. and Jetty 6 server didn't implement "Expect: 100-continue" protocol.
string url = BuildUrl(hdfsPath, WebHdfsAPI.UPLOAD,
"user.name=hadoop&overwrite=true&noredirect=true");
/*string result = HttpClientHelperV45.PutResponse(url);
* if (!string.IsNullOrWhiteSpace(result) && result.Contains("Location"))
* {
* url = result.Substring(result.IndexOf("http:", StringComparison.OrdinalIgnoreCase));
* url = url.Remove(url.IndexOf("\""));
* }
**/
//Hadoop 2.7.1 服务端总是会重定向（307 TEMPORARY_REDIRECT），noredirect=true参数根本不管用
string redirUrl = HttpClientHelperV45.GetRedirectedURL(url, HttpMethod.PUT);
if (!string.IsNullOrWhiteSpace(redirUrl))
{
url = TranslateHostname(redirUrl);
DebugHelper.OutLog("Redirect to " + url);
}
// Step 2. upload file to datanode
using (FileStream fs = new FileStream(srcFile, FileMode.Open, FileAccess.Read, FileShare.Read))
{
string response = HttpClientHelperV45.HttpUpload(url, fs, HttpMethod.PUT);
fs.Close();
}
errlog = "succeed";
return(true);
}
catch (System.Exception ex)
{
string err = ex.Message;
if (ex is WebException)
{
err = HttpClientHelperV45.ParseHdfsWebException(ex as WebException);
}
DebugHelper.OutLog("上传文件到HDFS失败：" + err);
DebugHelper.Error(ex, err);
errlog = err;
return(false);
}
}
/// <summary>
/// 列举当前目录内容
/// </summary>
/// <param name="path">HDFS路径</param>
/// <returns></returns>
public static List <HdfsFileInfo> LsDir(string path)
{
try
{
string url = BuildUrl(path, WebHdfsAPI.LIST);
string result = HttpClientHelperV45.Get(url);
return(HdfsFileInfo.ParseJsonArray(result));
}
catch (Exception ex)
{
DebugHelper.Error(ex);
return(null);
}
}
/// <summary>
/// 删除路径
/// </summary>
/// <param name="path">文件或路径名</param>
/// <param name="recursively">是否删除所有下级子目录和文件</param>
/// <returns></returns>
public static bool RmDir(string path, bool recursively = true)
{
try
{
string url = BuildUrl(path, WebHdfsAPI.RM, "recursive=true");
HttpClientHelperV45.Delete(url);
return(true);
}
catch (Exception ex)
{
DebugHelper.Error(ex);
return(false);
}
}
/// <summary>
/// 创建目录（自动创建上级目录）
/// </summary>
/// <param name="path">HDFS路径</param>
/// <returns></returns>
public static bool MkDir(string path)
{
try
{
string url = BuildUrl(path, WebHdfsAPI.MKDIR, "user.name=hadoop");
HttpClientHelperV45.Put(url);
return(true);
}
catch (Exception ex)
{
DebugHelper.Error(ex);
return(false);
}
}
/// <summary>
/// 获取文件或路径的详细信息
/// </summary>
/// <param name="file">文件或路径名</param>
/// <returns></returns>
public static HdfsFileInfo GetStatus(string file)
{
try
{
string url = BuildUrl(file, WebHdfsAPI.FILESTATUS);
string result = HttpClientHelperV45.Get(url);
return(HdfsFileInfo.FromJson(result));
}
catch (Exception ex)
{
DebugHelper.Error(ex);
return(null);
}
}
protected override async void OnCreate(Bundle savedInstanceState)
{
base.OnCreate(savedInstanceState);
SetContentView(Resource.Drawable.splash);
this.imgSplash = FindViewById <ImageView>(Resource.Id.imgSplash);
try
{
RandomAdvice();
GetData();
}
catch (Exception ex)
{
DebugHelper.Error(ex);
}
}
void LoadDataFromCache()
{
try
{
DebugHelper.Method(MethodBase.GetCurrentMethod());
this.GlobalReport = SharedPreferencesHandler.GetCovidReport(this);
this.CountriesReport = SharedPreferencesHandler.GetCountriesReport(this);
}
catch (Exception ex)
{
DebugHelper.Error(ex);
Toast.MakeText(this, Resources.GetString(Resource.String.impossible_to_start), ToastLength.Short);
Finish();
}
}
public static bool CombineFiles(string destHdfsFile, params string[] srcHdfsFiles)
{
try
{
string srcs = string.Join(",", srcHdfsFiles);
string url = BuildUrl(destHdfsFile, WebHdfsAPI.CONCAT, "user.name=hadoop&sources=" + srcs);
string statcode = "";
HttpClientHelperV45.Post(url, "", out statcode);
return("OK".Equals(statcode.ToUpper()));
}
catch (Exception ex)
{
DebugHelper.Error(ex);
return(false);
}
}
public async Task <string> Fetch(string phoneNumber, string Reference)
{
try
{
if (!string.IsNullOrWhiteSpace(phoneNumber))
{
var OTPResult = await GetOTPResult(phoneNumber, Reference);
return(SerializerHelper.SerializeJsonObject(OTPResult));
}
}
catch (Exception ex)
{
DebugHelper.Error(ex);
}
return("");
}
public ActionResult Input(int Id, string Ref, string Result)
{
try
{
var OTPModel = db.OTPs.Where(x => x.Id == Id).FirstOrDefault();
OTPModel.Ref = Ref;
OTPModel.Result = Result;
db.Entry(OTPModel).State = EntityState.Modified;
db.SaveChanges();
}
catch (Exception ex)
{
DebugHelper.Error(ex);
}
var OTPModels = db.OTPs.Where(x => x.Passed == false).ToList();
return(View(OTPModels));
}
public bool Login(string username, string password)
{
if (string.IsNullOrEmpty(username) || string.IsNullOrEmpty(password))
{
throw new InvalidParameterException("Username/Password cannot be empty.");
}
var dt = GetDbTable(username, password);
var dbErrorCode = dt.Rows[0].Field <int>("ErrorCode");
if (dbErrorCode.Equals(0))
{
return(true);
}
DebugHelper.Error($"{username} login failed. DbErrorCode:{dbErrorCode}");
return(false);
}
/// <summary>
/// 新建文件
/// </summary>
/// <param name="hdfsFile">文件名（绝对路径）</param>
/// <returns></returns>
public static bool NewFile(string hdfsFile)
{
try
{
string dir = hdfsFile.Substring(0, hdfsFile.LastIndexOf('/'));
if (string.IsNullOrWhiteSpace(dir))
{
dir = "/";
}
string fileName = hdfsFile.Substring(hdfsFile.LastIndexOf('/') + 1);
return(UploadStream(new MemoryStream(), 0, 0, dir, fileName));
}
catch (System.Exception ex)
{
DebugHelper.Error(ex);
return(false);
}
}
public static string Put(string url, string content = "")
{
try
{
if (url.StartsWith("https"))
{
System.Net.ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls;
}
var httpClient = new HttpClient();
httpClient.DefaultRequestHeaders.Accept.Add(
new MediaTypeWithQualityHeaderValue("application/json"));
HttpContent httpContent = new StringContent(content);
httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json");
httpContent.Headers.ContentType.CharSet = "utf-8";
HttpResponseMessage response = httpClient.PutAsync(url, httpContent).Result;
string statusCode = response.StatusCode.ToString();
if (response.IsSuccessStatusCode)
{
string result = response.Content.ReadAsStringAsync().Result;
return(result);
}
return(null);
}
catch (WebException webex)
{
var resp = (HttpWebResponse)webex.Response;
string err = webex.Message;
if (null != resp)
{
err = string.Format("{0} {1}", (int)resp.StatusCode, resp.StatusCode);
}
DebugHelper.OutLog(err);
return(null);
}
catch (Exception ex)
{
DebugHelper.Error(ex);
return(null);
}
}
/// <summary>
/// 下载HDFS文件
/// </summary>
/// <param name="file">文件名</param>
/// <returns></returns>
public static byte[] GetFile(string file)
{
try
{
if (string.IsNullOrWhiteSpace(file))
{
throw new ArgumentNullException("file");
}
if (null == GetStatus(file))
{
return(Encoding.UTF8.GetBytes("File Not Exist."));
}
string url = BuildUrl(file, WebHdfsAPI.OPEN, "user.name=atlas&noredirect=false");
string redirUrl = HttpClientHelperV45.GetRedirectedURL(url, HttpMethod.GET);
if (!string.IsNullOrWhiteSpace(redirUrl))
{
url = TranslateHostname(redirUrl);
}
HttpResult res = HttpClientHelperV45.Request(url, new TimeSpan(1, 0, 0));
if (200 == res.Code)
{
return(res.Content);
}
else
{
throw new Exception(res.ErrorLog);
}
}
catch (Exception ex)
{
string err = ex.Message;
if (ex is WebException)
{
err = HttpClientHelperV45.ParseHdfsWebException(ex as WebException);
}
DebugHelper.OutLog("获取HDFS文件失败：" + err);
DebugHelper.Error(ex, err);
return(null);
}
}
public static bool UploadStream(Stream ins, long offset, long length, string hdfsDir, string fileName)
{
try
{
// 确保目录存在
MkDir(hdfsDir);
// 拼接文件名
string hdfsFile = hdfsDir + "/" + fileName;
if (hdfsDir.EndsWith("/"))
{
hdfsFile = hdfsDir + fileName;
}
// Step 1. redirect to datanode
string url = BuildUrl(hdfsFile, WebHdfsAPI.UPLOAD,
"user.name=hadoop&overwrite=true&noredirect=true");
string redirUrl = HttpClientHelperV45.GetRedirectedURL(url, HttpMethod.PUT);
if (!string.IsNullOrWhiteSpace(redirUrl))
{
url = TranslateHostname(redirUrl);
DebugHelper.OutLog("Redirect to " + url);
}
// Step 2. upload file to datanode
string response = HttpClientHelperV45.HttpUpload(url, ins, offset, length, HttpMethod.PUT);
return(null != response);
}
catch (System.Exception ex)
{
string err = ex.Message;
if (ex is WebException)
{
err = HttpClientHelperV45.ParseHdfsWebException(ex as WebException);
}
DebugHelper.OutLog("上传文件到HDFS失败：" + err);
DebugHelper.Error(ex, err);
return(false);
}
}
public static HdfsFileInfo FromJson(string status)
{
if (string.IsNullOrWhiteSpace(status))
{
return(null);
}
var jObj = JsonConvert.DeserializeObject(status);
if (null == jObj)
{
throw new Exception("HdfsFileInfo Json解析错误");
}
try
{
var jStatus = (jObj as JObject)["FileStatus"] as JObject;
return(FromJson(jStatus));
}
catch (Exception ex)
{
DebugHelper.Error(ex);
return(null);
}
}
/// <summary>
/// 获取服务重定向后的URL地址，便于手动处理重定向
/// </summary>
/// <param name="url">请求的服务地址</param>
/// <param name="method">请求方法</param>
/// <returns></returns>
public static string GetRedirectedURL(string url, HttpMethod method)
{
try
{
string redirectUrl = null;
HttpWebReqWrapper.Call(url, method,
// 请求参数设置
(HttpWebRequest req) =>
{
//req.Headers[HttpRequestHeader.Authorization] = "token";
req.AllowAutoRedirect = false;
return(req);
},
// 响应结果提取
(HttpWebResponse rep) =>
{
int code = (int)rep.StatusCode;
if (307 == code || 301 == code)
{
redirectUrl = rep.Headers[HttpResponseHeader.Location];
}
return(rep);
});
return(redirectUrl);
}
catch (Exception ex)
{
string err = ex.Message;
if (ex is WebException)
{
err = ParseHdfsWebException(ex as WebException);
}
DebugHelper.Error(ex);
return(null);
}
}
public static Dictionary <string, string> GetDataNodes()
{
Dictionary <string, string> dict = new Dictionary <string, string>();
try
{
string url = "HTTP://" + WebHdfsHost + WebHdfsAPI.HDFS_LIVENODES;
string jsonstr = HttpClientHelperV45.Get(url);
if (string.IsNullOrWhiteSpace(jsonstr))
{
return(null);
}
var jObj = JsonConvert.DeserializeObject(jsonstr);
if (null == jObj)
{
throw new Exception("HDFS LiveNodes Json解析错误");
}
var liveNodes = (jObj as JObject)["beans"][0]["LiveNodes"] as JValue;
var jobjNodes = JsonConvert.DeserializeObject(liveNodes.ToString()) as JObject;
foreach (var jprop in jobjNodes.Properties())
{
string hname = jprop.Name.Split(':')[0];
string addr = (jprop.Value as JObject)["infoAddr"].ToString();
string ip = addr.Split(':')[0];
dict.Add(hname, ip);
}
return(dict);
}
catch (Exception ex)
{
DebugHelper.Error(ex);
return(null);
}
}
public static string HttpUpload(string url, Stream ins, long offset, long length,
HttpMethod method = HttpMethod.PUT)
{
try
{
if (url.StartsWith("https"))
{
System.Net.ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls;
}
string result = null;
HttpWebReqWrapper.Call(url, method,
// 请求参数设置
(HttpWebRequest req) =>
{
//req.Headers[HttpRequestHeader.ContentType] = "application/octet-stream"; // 特定标头不能这么设置
req.ContentType = "application/octet-stream";
req.Timeout = IO_TIMEOUT;
req.ReadWriteTimeout = IO_TIMEOUT;
req.SendChunked = true;
using (Stream outstream = req.GetRequestStream())
{
ins.Seek(offset, SeekOrigin.Begin);
long endPos = offset + length;
int c = 1024 * 8;
byte[] buffer = new byte[c];
while (ins.Position < endPos)
{
int l = ins.Read(buffer, 0, c);
if (ins.Position > endPos)
{
l = l - (int)(ins.Position - endPos);
}
if (l <= 0)
{
break;
}
outstream.Write(buffer, 0, l);
outstream.Flush();
}
outstream.Close();
}
return(req);
},
// 响应结果提取
(HttpWebResponse rep) =>
{
using (Stream outstream = rep.GetResponseStream())
{
StreamReader reader = new StreamReader(outstream, Encoding.UTF8);
result = reader.ReadToEnd();
reader.Close();
}
return(rep);
});
return(result);
}
catch (WebException webex)
{
var resp = (HttpWebResponse)webex.Response;
string err = webex.Message;
if (null != resp)
{
err = string.Format("{0} {1}", (int)resp.StatusCode, resp.StatusCode);
}
DebugHelper.OutLog(err);
return(null);
}
catch (Exception ex)
{
DebugHelper.Error(ex); //DebugHelper.OutLog(ex.Message);
return(null);
}
}
