예제 #1
0
        public Stream OpenFileStreamForAppend(string file)
        {
            _fileSystem = GetHdfsFileSystemConnection();
            if (!_fileSystem.IsValid())
            {
                return(null);
            }

            _fileStream = _fileSystem.OpenFileStream(file, HdfsFileAccess.Append);
            return(_fileStream.IsValid() ? _fileStream : null);
        }
예제 #2
0
        public Stream OpenFileStreamForAppend(string file)
        {
            _fileSystem = GetHdfsFileSystemConnection();
            if (!_fileSystem.IsValid()) return null;

            _fileStream = _fileSystem.OpenFileStream(file, HdfsFileAccess.Append);
            return _fileStream.IsValid() ? _fileStream : null;
        }
예제 #3
0
        /// <summary>
        /// Tests the FileStream Interface
        /// </summary>
        private static void FileTestStream(HdfsFileSystem hdfsSystem)
        {
            string localfilename   = "MobileSampleData.txt";
            string localfilepath   = Path.GetFullPath(@"..\..\..\Sample") + @"\" + localfilename;
            string localfilewrite  = Path.GetFullPath(@"..\..\..\Sample") + @"\HdfsStream." + localfilename;
            string localfileline   = Path.GetFullPath(@"..\..\..\Sample") + @"\HdfsStreamLine." + localfilename;
            string localfilestream = Path.GetFullPath(@"..\..\..\Sample") + @"\HdfsStreamCopy." + localfilename;

            string filename = BaseHdfsPath + "/testdata.txt";

            int chunksize = 1024;
            int chunk;

            // First write out some data
            Console.WriteLine("Working Directory is: " + hdfsSystem.GetWorkingDirectory());
            Console.WriteLine("Creating New File: " + filename);
            string data = "I am some unstructured data.\nThat will be written.\n";

            byte[] dataBytes = Encoding.UTF8.GetBytes(data);
            int    dataLen   = dataBytes.Length;

            using (HdfsFileStream file = hdfsSystem.OpenFileStream(filename, HdfsFileAccess.Write, chunksize))
            {
                file.Write(dataBytes, 0, data.Length);
                file.WriteByte((byte)47);
                file.Flush();
            }

            // Ensure you can read the data
            using (HdfsFileStream file = hdfsSystem.OpenFileStream(filename, HdfsFileAccess.Read))
            {
                Console.WriteLine("Data Written:");

                byte[] newDataBytes = new byte[dataLen];
                file.Read(newDataBytes, 0, newDataBytes.Length);
                Console.Write(Encoding.UTF8.GetString(newDataBytes));

                Console.WriteLine();
                Console.WriteLine("Setting file times and owner to be 'isotope'");
                file.Chown("isotope", null);
                file.SetTimes(DateTime.Now.AddDays(-2), DateTime.Now.AddDays(-1));

                HdfsFileInfoEntry fileinfo = file.GetInformation();
                if (fileinfo != null)
                {
                    Console.WriteLine(string.Format(@"'{0}', Modified/Accessed:""{1:G}, {2:G}"", Owner:""{3}""", fileinfo.Name, fileinfo.LastModified, fileinfo.LastAccessed, fileinfo.Owner));
                }
            }

            // Read the full contents of a file using bytes
            Console.WriteLine();
            Console.WriteLine("Reading all Byte Data:");
            byte[] readBytes = new byte[chunksize];

            using (HdfsFileStream file = hdfsSystem.OpenFileStream(filename, HdfsFileAccess.Read))
            {
                while ((chunk = file.Read(readBytes, 0, chunksize)) > 0)
                {
                    Console.Write(Encoding.UTF8.GetString(readBytes, 0, chunk));
                }
            }

            if (File.Exists(localfilepath))
            {
                // Copy a file from the local file system into Hadoop
                Console.WriteLine();
                Console.WriteLine("Copying Local File: " + localfilename);
                byte[] localbytes = new byte[chunksize];

                string localhdfsfilename = BaseHdfsPath + @"\" + localfilename;
                using (HdfsFileStream file = hdfsSystem.OpenFileStream(localhdfsfilename, HdfsFileAccess.Write, chunksize))
                {
                    using (FileStream stream = new FileStream(localfilepath, FileMode.Open, FileAccess.Read))
                    {
                        while ((chunk = stream.Read(localbytes, 0, chunksize)) > 0)
                        {
                            file.Write(localbytes, 0, chunk);
                        }
                    }

                    file.Flush();
                }

                using (HdfsFileStream file = hdfsSystem.OpenFileStream(localhdfsfilename, HdfsFileAccess.Write, chunksize))
                {
                    using (FileStream stream = new FileStream(localfilepath, FileMode.Open, FileAccess.Read))
                    {
                        stream.CopyTo(file);
                    }
                }

                // Copy the HDFS file to the local system
                Console.WriteLine();
                Console.WriteLine("Copying file to local:");

                using (HdfsFileStream file = hdfsSystem.OpenFileStream(localhdfsfilename, HdfsFileAccess.Read, chunksize))
                {
                    using (FileStream stream = new FileStream(localfilewrite, FileMode.Create, FileAccess.Write))
                    {
                        while ((chunk = file.Read(readBytes, 0, chunksize)) > 0)
                        {
                            stream.Write(readBytes, 0, chunk);
                        }
                    }
                }

                Console.WriteLine(String.Format("File '{0}' Created.", localfilewrite));

                // Copy the HDFS file to the local system using ReadLine
                Console.WriteLine();
                Console.WriteLine("Copying file to local using ReadLine:");

                using (StreamReader reader = new StreamReader(hdfsSystem.OpenFileStream(localhdfsfilename, HdfsFileAccess.Read, chunksize)))
                {
                    using (StreamWriter writer = new StreamWriter(localfileline, false, Encoding.UTF8))
                    {
                        string line;
                        while ((line = reader.ReadLine()) != null)
                        {
                            writer.WriteLine(line);
                        }
                    }
                }

                Console.WriteLine(String.Format("File '{0}' Created.", localfileline));

                // Now process using Stream Copy To interfaces
                Console.WriteLine();
                Console.WriteLine("Copying file to local using stream:");
                using (HdfsFileStream hdfsStream = hdfsSystem.OpenFileStream(localhdfsfilename, HdfsFileAccess.Read))
                {
                    using (FileStream fileStream = new FileStream(localfilestream, FileMode.Create, FileAccess.Write))
                    {
                        hdfsStream.CopyTo(fileStream);
                    }
                }

                Console.WriteLine(String.Format("File '{0}' Created.", localfilestream));
                Console.WriteLine();
            }
        }