/// <summary>
/// https://hadoop.apache.org/docs/r2.8.0/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#Open_and_Read_a_File
/// Note: You are responsible for closing the stream you send in.
/// </summary>
public async Task <bool> ReadStream(
Stream stream,
string path,
long?offset = null,
long?length = null,
int?buffersize = null)
{
var query = HttpUtility.ParseQueryString(string.Empty);
WebHDFSHttpQueryParameter.SetOp(query, WebHDFSHttpQueryParameter.Op.OPEN);
WebHDFSHttpQueryParameter.SetOffset(query, offset);
WebHDFSHttpQueryParameter.SetLength(query, length);
WebHDFSHttpQueryParameter.SetBuffersize(query, buffersize);
string requestPath = _baseAPI + path + '?' + query;
var response = await _httpClient.GetAsync(requestPath);
if (response.StatusCode.Equals(HttpStatusCode.TemporaryRedirect))
{
var response2 = await _httpClient.GetAsync(response.Headers.Location);
response2.EnsureSuccessStatusCode();
if (response2.IsSuccessStatusCode)
{
await response2.Content.CopyToAsync(stream);
}
return(response2.IsSuccessStatusCode);
}
throw new InvalidOperationException("Should get a 307. Instead we got: " +
response.StatusCode + " " + response.ReasonPhrase);
}
/// <summary>
/// https://hadoop.apache.org/docs/r2.8.0/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#Create_and_Write_to_a_File
/// Note: You are responsible for closing the stream you send in.
/// </summary>
public async Task <bool> WriteStream(
Stream stream,
string path,
bool overwrite = false,
long?blocksize = null,
short?replication = null,
string permission = null,
int?buffersize = null)
{
var query = HttpUtility.ParseQueryString(string.Empty);
WebHDFSHttpQueryParameter.SetOp(query, WebHDFSHttpQueryParameter.Op.CREATE);
WebHDFSHttpQueryParameter.SetOverwrite(query, overwrite);
WebHDFSHttpQueryParameter.setBlocksize(query, blocksize);
WebHDFSHttpQueryParameter.SetReplication(query, replication);
WebHDFSHttpQueryParameter.SetPermission(query, permission);
WebHDFSHttpQueryParameter.SetBuffersize(query, buffersize);
string requestPath = _baseAPI + path + '?' + query;
var response = await _httpClient.PutAsync(requestPath, new ByteArrayContent(new byte[] {}));
if (response.StatusCode.Equals(HttpStatusCode.TemporaryRedirect))
{
var response2 = await _httpClient.PutAsync(response.Headers.Location, new StreamContent(stream));
response2.EnsureSuccessStatusCode();
return(response2.IsSuccessStatusCode);
}
throw new InvalidOperationException("Should get a 307. Instead we got: " +
response.StatusCode + " " + response.ReasonPhrase);
}
/// <summary>
/// https://hadoop.apache.org/docs/r2.8.0/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#Append_to_a_File
/// Note: You are responsible for closing the stream you send in.
/// </summary>
public async Task <bool> AppendStream(
Stream stream,
string path,
int?buffersize = null)
{
var query = HttpUtility.ParseQueryString(string.Empty);
WebHDFSHttpQueryParameter.SetOp(query, WebHDFSHttpQueryParameter.Op.APPEND);
WebHDFSHttpQueryParameter.SetBuffersize(query, buffersize);
string requestPath = _baseAPI + path + '?' + query;
var response = await _httpClient.PostAsync(requestPath, new ByteArrayContent(new byte[] { }));
if (response.StatusCode.Equals(HttpStatusCode.TemporaryRedirect))
{
var response2 = await _httpClient.PostAsync(response.Headers.Location, new StreamContent(stream));
response2.EnsureSuccessStatusCode();
return(response2.IsSuccessStatusCode);
}
throw new InvalidOperationException("Should get a 307. Instead we got: " +
response.StatusCode + " " + response.ReasonPhrase);
}