Esempio n. 1
0
        /// <summary>
        /// https://hadoop.apache.org/docs/r2.8.0/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#Open_and_Read_a_File
        /// Note: You are responsible for closing the stream you send in.
        /// </summary>
        public async Task <bool> ReadStream(
            Stream stream,
            string path,
            long?offset    = null,
            long?length    = null,
            int?buffersize = null)
        {
            var query = HttpUtility.ParseQueryString(string.Empty);

            WebHDFSHttpQueryParameter.SetOp(query, WebHDFSHttpQueryParameter.Op.OPEN);
            WebHDFSHttpQueryParameter.SetOffset(query, offset);
            WebHDFSHttpQueryParameter.SetLength(query, length);
            WebHDFSHttpQueryParameter.SetBuffersize(query, buffersize);

            string requestPath = _baseAPI + path + '?' + query;

            var response = await _httpClient.GetAsync(requestPath);

            if (response.StatusCode.Equals(HttpStatusCode.TemporaryRedirect))
            {
                var response2 = await _httpClient.GetAsync(response.Headers.Location);

                response2.EnsureSuccessStatusCode();
                if (response2.IsSuccessStatusCode)
                {
                    await response2.Content.CopyToAsync(stream);
                }
                return(response2.IsSuccessStatusCode);
            }
            throw new InvalidOperationException("Should get a 307. Instead we got: " +
                                                response.StatusCode + " " + response.ReasonPhrase);
        }
Esempio n. 2
0
        /// <summary>
        /// https://hadoop.apache.org/docs/r2.8.0/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#Create_and_Write_to_a_File
        /// Note: You are responsible for closing the stream you send in.
        /// </summary>
        public async Task <bool> WriteStream(
            Stream stream,
            string path,
            bool overwrite    = false,
            long?blocksize    = null,
            short?replication = null,
            string permission = null,
            int?buffersize    = null)
        {
            var query = HttpUtility.ParseQueryString(string.Empty);

            WebHDFSHttpQueryParameter.SetOp(query, WebHDFSHttpQueryParameter.Op.CREATE);
            WebHDFSHttpQueryParameter.SetOverwrite(query, overwrite);
            WebHDFSHttpQueryParameter.setBlocksize(query, blocksize);
            WebHDFSHttpQueryParameter.SetReplication(query, replication);
            WebHDFSHttpQueryParameter.SetPermission(query, permission);
            WebHDFSHttpQueryParameter.SetBuffersize(query, buffersize);

            string requestPath = _baseAPI + path + '?' + query;

            var response = await _httpClient.PutAsync(requestPath, new ByteArrayContent(new byte[] {}));

            if (response.StatusCode.Equals(HttpStatusCode.TemporaryRedirect))
            {
                var response2 = await _httpClient.PutAsync(response.Headers.Location, new StreamContent(stream));

                response2.EnsureSuccessStatusCode();
                return(response2.IsSuccessStatusCode);
            }
            throw new InvalidOperationException("Should get a 307. Instead we got: " +
                                                response.StatusCode + " " + response.ReasonPhrase);
        }
Esempio n. 3
0
        /// <summary>
        /// https://hadoop.apache.org/docs/r2.7.3/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#Check_access
        /// </summary>
        public async Task <bool> CheckAccess(
            string path,
            string fsaction)
        {
            var query = HttpUtility.ParseQueryString(string.Empty);

            WebHDFSHttpQueryParameter.SetOp(query, WebHDFSHttpQueryParameter.Op.CHECKACCESS);
            WebHDFSHttpQueryParameter.SetFSAction(query, fsaction);

            string requestPath = _baseAPI + path + '?' + query;
            var    response    = await _httpClient.GetAsync(requestPath);

            response.EnsureSuccessStatusCode();
            return(response.IsSuccessStatusCode);
        }
Esempio n. 4
0
        /// <summary>
        /// https://hadoop.apache.org/docs/r2.8.0/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#Status_of_a_FileDirectory
        /// </summary>
        public async Task <string> GetHomeDirectory()
        {
            var query = HttpUtility.ParseQueryString(string.Empty);

            WebHDFSHttpQueryParameter.SetOp(query, WebHDFSHttpQueryParameter.Op.GETHOMEDIRECTORY);

            string requestPath = _baseAPI + '?' + query;

            var response = await _httpClient.GetAsync(requestPath);

            response.EnsureSuccessStatusCode();
            var serializer = new DataContractJsonSerializer(typeof(PathClass));

            return(((PathClass)serializer.ReadObject(await response.Content.ReadAsStreamAsync())).Path);
        }
Esempio n. 5
0
        /// <summary>
        /// https://hadoop.apache.org/docs/r2.8.0/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#Set_Permission
        /// </summary>
        public async Task <bool> SetPermission(
            string path,
            string permission = null)
        {
            var query = HttpUtility.ParseQueryString(string.Empty);

            WebHDFSHttpQueryParameter.SetOp(query, WebHDFSHttpQueryParameter.Op.SETPERMISSION);
            WebHDFSHttpQueryParameter.SetPermission(query, permission);

            string requestPath = _baseAPI + path + '?' + query;

            var response = await _httpClient.PutAsync(requestPath, new ByteArrayContent(new byte[] { }));

            response.EnsureSuccessStatusCode();
            return(response.IsSuccessStatusCode);
        }
Esempio n. 6
0
        /// <summary>
        /// https://hadoop.apache.org/docs/r2.8.0/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#Concat_Files
        /// </summary>
        public async Task <bool> Concat(
            string path,
            string sources)
        {
            var query = HttpUtility.ParseQueryString(string.Empty);

            WebHDFSHttpQueryParameter.SetOp(query, WebHDFSHttpQueryParameter.Op.CONCAT);
            WebHDFSHttpQueryParameter.SetSources(query, sources);

            string requestPath = _baseAPI + path + '?' + query;

            var response = await _httpClient.PostAsync(requestPath, new ByteArrayContent(new byte[] { }));

            response.EnsureSuccessStatusCode();
            return(response.IsSuccessStatusCode);
        }
Esempio n. 7
0
        /// <summary>
        /// https://hadoop.apache.org/docs/r2.8.0/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#Get_Content_Summary_of_a_Directory
        /// </summary>
        public async Task <ContentSummary> GetContentSummary(
            string path)
        {
            var query = HttpUtility.ParseQueryString(string.Empty);

            WebHDFSHttpQueryParameter.SetOp(query, WebHDFSHttpQueryParameter.Op.GETCONTENTSUMMARY);

            string requestPath = _baseAPI + path + '?' + query;

            var response = await _httpClient.GetAsync(requestPath);

            response.EnsureSuccessStatusCode();
            var serializer = new DataContractJsonSerializer(typeof(ContentSummaryClass));

            return(((ContentSummaryClass)serializer.ReadObject(await response.Content.ReadAsStreamAsync())).ContentSummary);
        }
Esempio n. 8
0
        /// <summary>
        /// https://hadoop.apache.org/docs/r2.8.0/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#List_a_Directory
        /// </summary>
        public async Task <FileStatuses> ListStatus(
            string path)
        {
            var query = HttpUtility.ParseQueryString(string.Empty);

            WebHDFSHttpQueryParameter.SetOp(query, WebHDFSHttpQueryParameter.Op.LISTSTATUS);

            string requestPath = _baseAPI + path + '?' + query;

            var response = await _httpClient.GetAsync(requestPath);

            response.EnsureSuccessStatusCode();
            var serializer = new DataContractJsonSerializer(typeof(FileStatusesClass));

            return(((FileStatusesClass)serializer.ReadObject(await response.Content.ReadAsStreamAsync())).FileStatuses);
        }
Esempio n. 9
0
        /// <summary>
        /// https://hadoop.apache.org/docs/r2.8.0/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#Set_Replication_Factor
        /// </summary>
        public async Task <Boolean> SetReplication(
            string path,
            short?replication = null)
        {
            var query = HttpUtility.ParseQueryString(string.Empty);

            WebHDFSHttpQueryParameter.SetOp(query, WebHDFSHttpQueryParameter.Op.SETREPLICATION);
            WebHDFSHttpQueryParameter.SetReplication(query, replication);

            string requestPath = _baseAPI + path + '?' + query;

            var response = await _httpClient.PutAsync(requestPath, new ByteArrayContent(new byte[] { }));

            response.EnsureSuccessStatusCode();
            var serializer = new DataContractJsonSerializer(typeof(Boolean));

            return((Boolean)serializer.ReadObject(await response.Content.ReadAsStreamAsync()));
        }
Esempio n. 10
0
        /// <summary>
        /// https://hadoop.apache.org/docs/r2.8.0/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#Set_Owner
        /// </summary>
        public async Task <bool> SetOwner(
            string path,
            string owner = null,
            string group = null)
        {
            var query = HttpUtility.ParseQueryString(string.Empty);

            WebHDFSHttpQueryParameter.SetOp(query, WebHDFSHttpQueryParameter.Op.SETOWNER);
            WebHDFSHttpQueryParameter.SetOwner(query, owner);
            WebHDFSHttpQueryParameter.SetGroup(query, group);

            string requestPath = _baseAPI + path + '?' + query;

            var response = await _httpClient.PutAsync(requestPath, new ByteArrayContent(new byte[] { }));

            response.EnsureSuccessStatusCode();
            return(response.IsSuccessStatusCode);
        }
Esempio n. 11
0
        /// <summary>
        /// https://hadoop.apache.org/docs/r2.8.0/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#Create_a_Symbolic_Link
        /// </summary>
        public async Task <bool> CreateSymlink(
            string path,
            string destination,
            bool?createParent)
        {
            var query = HttpUtility.ParseQueryString(string.Empty);

            WebHDFSHttpQueryParameter.SetOp(query, WebHDFSHttpQueryParameter.Op.CREATESYMLINK);
            WebHDFSHttpQueryParameter.SetDestination(query, destination);
            WebHDFSHttpQueryParameter.SetCreateParent(query, createParent);

            string requestPath = _baseAPI + path + '?' + query;

            var response = await _httpClient.PutAsync(requestPath, new ByteArrayContent(new byte[] { }));

            response.EnsureSuccessStatusCode();
            return(response.IsSuccessStatusCode);
        }
Esempio n. 12
0
        /// <summary>
        /// https://hadoop.apache.org/docs/r2.8.0/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#Truncate_a_File
        /// </summary>
        public async Task <Boolean> Truncate(
            string path,
            long newlength)
        {
            var query = HttpUtility.ParseQueryString(string.Empty);

            WebHDFSHttpQueryParameter.SetOp(query, WebHDFSHttpQueryParameter.Op.TRUNCATE);
            WebHDFSHttpQueryParameter.SetNewLength(query, newlength);

            string requestPath = _baseAPI + path + '?' + query;

            var response = await _httpClient.PostAsync(requestPath, new ByteArrayContent(new byte[] { }));

            response.EnsureSuccessStatusCode();
            var serializer = new DataContractJsonSerializer(typeof(Boolean));

            return((Boolean)serializer.ReadObject(await response.Content.ReadAsStreamAsync()));
        }
Esempio n. 13
0
        /// <summary>
        /// https://hadoop.apache.org/docs/r2.8.0/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#Delete_a_FileDirectory
        /// </summary>
        public async Task <Boolean> Delete(
            string path,
            bool?recursive = null)
        {
            var query = HttpUtility.ParseQueryString(string.Empty);

            WebHDFSHttpQueryParameter.SetOp(query, WebHDFSHttpQueryParameter.Op.DELETE);
            WebHDFSHttpQueryParameter.SetRecursive(query, recursive);

            string requestPath = _baseAPI + path + '?' + query;

            var response = await _httpClient.DeleteAsync(requestPath);

            response.EnsureSuccessStatusCode();
            var serializer = new DataContractJsonSerializer(typeof(Boolean));

            return((Boolean)serializer.ReadObject(await response.Content.ReadAsStreamAsync()));
        }
Esempio n. 14
0
        /// <summary>
        /// https://hadoop.apache.org/docs/r2.8.0/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#Get_File_Checksum
        /// </summary>
        public async Task <FileChecksum> GetFileChecksum(
            string path)
        {
            var query = HttpUtility.ParseQueryString(string.Empty);

            WebHDFSHttpQueryParameter.SetOp(query, WebHDFSHttpQueryParameter.Op.GETFILECHECKSUM);

            string requestPath = _baseAPI + path + '?' + query;

            var response = await _httpClient.GetAsync(requestPath);

            if (response.StatusCode.Equals(HttpStatusCode.TemporaryRedirect))
            {
                var response2 = await _httpClient.GetAsync(response.Headers.Location);

                response2.EnsureSuccessStatusCode();
                var serializer = new DataContractJsonSerializer(typeof(FileChecksumClass));
                return(((FileChecksumClass)serializer.ReadObject(await response2.Content.ReadAsStreamAsync())).FileChecksum);
            }
            throw new InvalidOperationException("Should get a 307. Instead we got: " +
                                                response.StatusCode + " " + response.ReasonPhrase);
        }
Esempio n. 15
0
        /// <summary>
        /// https://hadoop.apache.org/docs/r2.8.0/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#Append_to_a_File
        /// Note: You are responsible for closing the stream you send in.
        /// </summary>
        public async Task <bool> AppendStream(
            Stream stream,
            string path,
            int?buffersize = null)
        {
            var query = HttpUtility.ParseQueryString(string.Empty);

            WebHDFSHttpQueryParameter.SetOp(query, WebHDFSHttpQueryParameter.Op.APPEND);
            WebHDFSHttpQueryParameter.SetBuffersize(query, buffersize);

            string requestPath = _baseAPI + path + '?' + query;

            var response = await _httpClient.PostAsync(requestPath, new ByteArrayContent(new byte[] { }));

            if (response.StatusCode.Equals(HttpStatusCode.TemporaryRedirect))
            {
                var response2 = await _httpClient.PostAsync(response.Headers.Location, new StreamContent(stream));

                response2.EnsureSuccessStatusCode();
                return(response2.IsSuccessStatusCode);
            }
            throw new InvalidOperationException("Should get a 307. Instead we got: " +
                                                response.StatusCode + " " + response.ReasonPhrase);
        }