/// <summary>Writes to a stream the given number of bytes specified</summary> /// <param name="byteAm">the file size in number of bytes to write</param> /// <param name="out">the outputstream to write to</param> /// <returns>the number of bytes written + time taken</returns> /// <exception cref="System.IO.IOException"/> internal virtual DataWriter.GenerateOutput WriteSegment(long byteAm, OutputStream @out) { long headerLen = GetHeaderLength(); if (byteAm < headerLen) { // not enough bytes to write even the header return(new DataWriter.GenerateOutput(0, 0)); } // adjust for header length byteAm -= headerLen; if (byteAm < 0) { byteAm = 0; } DataWriter.WriteInfo header = WriteHeader(@out, byteAm); DataHasher hasher = new DataHasher(header.GetHashValue()); DataWriter.GenerateOutput pRes = WritePieces(byteAm, 0, hasher, @out); long bytesWritten = pRes.GetBytesWritten() + header.GetBytesWritten(); long timeTaken = header.GetTimeTaken() + pRes.GetTimeTaken(); return(new DataWriter.GenerateOutput(bytesWritten, timeTaken)); }
public virtual void TestDataWriting() { long byteAm = 100; FilePath fn = GetTestFile(); DataWriter writer = new DataWriter(rnd); FileOutputStream fs = new FileOutputStream(fn); DataWriter.GenerateOutput ostat = writer.WriteSegment(byteAm, fs); Log.Info(ostat); fs.Close(); NUnit.Framework.Assert.IsTrue(ostat.GetBytesWritten() == byteAm); DataVerifier vf = new DataVerifier(); FileInputStream fin = new FileInputStream(fn); DataVerifier.VerifyOutput vfout = vf.VerifyFile(byteAm, new DataInputStream(fin)); Log.Info(vfout); fin.Close(); NUnit.Framework.Assert.AreEqual(vfout.GetBytesRead(), byteAm); NUnit.Framework.Assert.IsTrue(vfout.GetChunksDifferent() == 0); }
internal override IList <OperationOutput> Run(FileSystem fs) { // Operation IList <OperationOutput> @out = base.Run(fs); OutputStream os = null; try { Path fn = GetAppendFile(); // determine file status for file length requirement // to know if should fill in partial bytes Range <long> appendSizeRange = GetConfig().GetAppendSize(); if (GetConfig().ShouldAppendUseBlockSize()) { appendSizeRange = GetConfig().GetBlockSize(); } long appendSize = Range.BetweenPositive(GetRandom(), appendSizeRange); long timeTaken = 0; long bytesAppended = 0; DataWriter writer = new DataWriter(GetRandom()); Log.Info("Attempting to append to file at " + fn + " of size " + Helper.ToByteInfo (appendSize)); { // open long startTime = Timer.Now(); os = fs.Append(fn); timeTaken += Timer.Elapsed(startTime); // append given length DataWriter.GenerateOutput stats = writer.WriteSegment(appendSize, os); timeTaken += stats.GetTimeTaken(); bytesAppended += stats.GetBytesWritten(); // capture close time startTime = Timer.Now(); os.Close(); os = null; timeTaken += Timer.Elapsed(startTime); } @out.AddItem(new OperationOutput(OperationOutput.OutputType.Long, GetType(), ReportWriter .BytesWritten, bytesAppended)); @out.AddItem(new OperationOutput(OperationOutput.OutputType.Long, GetType(), ReportWriter .OkTimeTaken, timeTaken)); @out.AddItem(new OperationOutput(OperationOutput.OutputType.Long, GetType(), ReportWriter .Successes, 1L)); Log.Info("Appended " + Helper.ToByteInfo(bytesAppended) + " to file " + fn + " in " + timeTaken + " milliseconds"); } catch (FileNotFoundException e) { @out.AddItem(new OperationOutput(OperationOutput.OutputType.Long, GetType(), ReportWriter .NotFound, 1L)); Log.Warn("Error with appending", e); } catch (IOException e) { @out.AddItem(new OperationOutput(OperationOutput.OutputType.Long, GetType(), ReportWriter .Failures, 1L)); Log.Warn("Error with appending", e); } finally { if (os != null) { try { os.Close(); } catch (IOException e) { Log.Warn("Error with closing append stream", e); } } } return(@out); }
internal override IList <OperationOutput> Run(FileSystem fs) { // Operation IList <OperationOutput> @out = base.Run(fs); FSDataOutputStream os = null; try { Path fn = GetCreateFile(); Range <long> writeSizeRange = GetConfig().GetWriteSize(); long writeSize = 0; long blockSize = DetermineBlockSize(); short replicationAmount = DetermineReplication(); if (GetConfig().ShouldWriteUseBlockSize()) { writeSizeRange = GetConfig().GetBlockSize(); } writeSize = Range.BetweenPositive(GetRandom(), writeSizeRange); long bytesWritten = 0; long timeTaken = 0; int bufSize = GetBufferSize(); bool overWrite = false; DataWriter writer = new DataWriter(GetRandom()); Log.Info("Attempting to create file at " + fn + " of size " + Helper.ToByteInfo(writeSize ) + " using blocksize " + Helper.ToByteInfo(blockSize) + " and replication amount " + replicationAmount); { // open & create long startTime = Timer.Now(); os = fs.Create(fn, overWrite, bufSize, replicationAmount, blockSize); timeTaken += Timer.Elapsed(startTime); // write the given length DataWriter.GenerateOutput stats = writer.WriteSegment(writeSize, os); bytesWritten += stats.GetBytesWritten(); timeTaken += stats.GetTimeTaken(); // capture close time startTime = Timer.Now(); os.Close(); os = null; timeTaken += Timer.Elapsed(startTime); } Log.Info("Created file at " + fn + " of size " + Helper.ToByteInfo(bytesWritten) + " bytes using blocksize " + Helper.ToByteInfo(blockSize) + " and replication amount " + replicationAmount + " in " + timeTaken + " milliseconds"); // collect all the stats @out.AddItem(new OperationOutput(OperationOutput.OutputType.Long, GetType(), ReportWriter .OkTimeTaken, timeTaken)); @out.AddItem(new OperationOutput(OperationOutput.OutputType.Long, GetType(), ReportWriter .BytesWritten, bytesWritten)); @out.AddItem(new OperationOutput(OperationOutput.OutputType.Long, GetType(), ReportWriter .Successes, 1L)); } catch (IOException e) { @out.AddItem(new OperationOutput(OperationOutput.OutputType.Long, GetType(), ReportWriter .Failures, 1L)); Log.Warn("Error with creating", e); } finally { if (os != null) { try { os.Close(); } catch (IOException e) { Log.Warn("Error closing create stream", e); } } } return(@out); }