コード例 #1
0
        /// <summary>
        /// https://hadoop.apache.org/docs/r2.8.0/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#Open_and_Read_a_File
        /// Note: You are responsible for closing the stream you send in.
        /// </summary>
        public async Task <bool> ReadStream(
            Stream stream,
            string path,
            long?offset    = null,
            long?length    = null,
            int?buffersize = null)
        {
            var query = HttpUtility.ParseQueryString(string.Empty);

            WebHDFSHttpQueryParameter.SetOp(query, WebHDFSHttpQueryParameter.Op.OPEN);
            WebHDFSHttpQueryParameter.SetOffset(query, offset);
            WebHDFSHttpQueryParameter.SetLength(query, length);
            WebHDFSHttpQueryParameter.SetBuffersize(query, buffersize);

            string requestPath = _baseAPI + path + '?' + query;

            var response = await _httpClient.GetAsync(requestPath);

            if (response.StatusCode.Equals(HttpStatusCode.TemporaryRedirect))
            {
                var response2 = await _httpClient.GetAsync(response.Headers.Location);

                response2.EnsureSuccessStatusCode();
                if (response2.IsSuccessStatusCode)
                {
                    await response2.Content.CopyToAsync(stream);
                }
                return(response2.IsSuccessStatusCode);
            }
            throw new InvalidOperationException("Should get a 307. Instead we got: " +
                                                response.StatusCode + " " + response.ReasonPhrase);
        }
コード例 #2
0
        /// <summary>
        /// https://hadoop.apache.org/docs/r2.8.0/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#Create_and_Write_to_a_File
        /// Note: You are responsible for closing the stream you send in.
        /// </summary>
        public async Task <bool> WriteStream(
            Stream stream,
            string path,
            bool overwrite    = false,
            long?blocksize    = null,
            short?replication = null,
            string permission = null,
            int?buffersize    = null)
        {
            var query = HttpUtility.ParseQueryString(string.Empty);

            WebHDFSHttpQueryParameter.SetOp(query, WebHDFSHttpQueryParameter.Op.CREATE);
            WebHDFSHttpQueryParameter.SetOverwrite(query, overwrite);
            WebHDFSHttpQueryParameter.setBlocksize(query, blocksize);
            WebHDFSHttpQueryParameter.SetReplication(query, replication);
            WebHDFSHttpQueryParameter.SetPermission(query, permission);
            WebHDFSHttpQueryParameter.SetBuffersize(query, buffersize);

            string requestPath = _baseAPI + path + '?' + query;

            var response = await _httpClient.PutAsync(requestPath, new ByteArrayContent(new byte[] {}));

            if (response.StatusCode.Equals(HttpStatusCode.TemporaryRedirect))
            {
                var response2 = await _httpClient.PutAsync(response.Headers.Location, new StreamContent(stream));

                response2.EnsureSuccessStatusCode();
                return(response2.IsSuccessStatusCode);
            }
            throw new InvalidOperationException("Should get a 307. Instead we got: " +
                                                response.StatusCode + " " + response.ReasonPhrase);
        }
コード例 #3
0
        /// <summary>
        /// https://hadoop.apache.org/docs/r2.8.0/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#Append_to_a_File
        /// Note: You are responsible for closing the stream you send in.
        /// </summary>
        public async Task <bool> AppendStream(
            Stream stream,
            string path,
            int?buffersize = null)
        {
            var query = HttpUtility.ParseQueryString(string.Empty);

            WebHDFSHttpQueryParameter.SetOp(query, WebHDFSHttpQueryParameter.Op.APPEND);
            WebHDFSHttpQueryParameter.SetBuffersize(query, buffersize);

            string requestPath = _baseAPI + path + '?' + query;

            var response = await _httpClient.PostAsync(requestPath, new ByteArrayContent(new byte[] { }));

            if (response.StatusCode.Equals(HttpStatusCode.TemporaryRedirect))
            {
                var response2 = await _httpClient.PostAsync(response.Headers.Location, new StreamContent(stream));

                response2.EnsureSuccessStatusCode();
                return(response2.IsSuccessStatusCode);
            }
            throw new InvalidOperationException("Should get a 307. Instead we got: " +
                                                response.StatusCode + " " + response.ReasonPhrase);
        }