public void TestCache2() { // Writer DurableFileStream dfs = new DurableFileStream("TestCache2.dat", true, 5000); byte[] writeBuffer = new byte[4096]; Helper.SetArrayRandomly(writeBuffer); dfs.Write(writeBuffer, 0, writeBuffer.Length); byte[] writeBuffer2 = new byte[4096]; Helper.SetArrayRandomly(writeBuffer2); dfs.Write(writeBuffer2, 0, writeBuffer2.Length); byte[] writeBuffer3 = new byte[4096]; Helper.SetArrayRandomly(writeBuffer3); dfs.Write(writeBuffer3, 0, writeBuffer2.Length); dfs.Commit(); byte[] writeBuffer4 = new byte[4096]; Helper.SetArrayRandomly(writeBuffer4); dfs.Write(writeBuffer4, 0, writeBuffer2.Length); dfs.Seek(-2 * 4096, System.IO.SeekOrigin.Current); byte[] readBuffer = new byte[4096]; dfs.Read(readBuffer, 0, readBuffer.Length); dfs.Seek(-2 * 4096, System.IO.SeekOrigin.Current); byte[] readBuffer2 = new byte[4096]; dfs.Read(readBuffer2, 0, readBuffer2.Length); dfs.Close(); }
public void TestMisc1() { // // write // int bufferSize = DurableFileStream.BLOCK_SIZE; byte[] writeBuffer = new byte[bufferSize]; Helper.SetArrayRandomly(writeBuffer); DurableFileStream dfs = new DurableFileStream("TestMisc1.dat", true); dfs.Write(writeBuffer, 0, bufferSize); dfs.Abort(); Helper.SetArrayRandomly(writeBuffer); dfs.Write(writeBuffer, 0, bufferSize); dfs.Commit(); dfs.Close(); // // read // dfs = new DurableFileStream("TestMisc1.dat", false); byte[] readBuffer = new byte[bufferSize]; dfs.Read(readBuffer, 0, bufferSize); Assert.IsTrue(Helper.IsArrayEmpty(readBuffer)); dfs.Read(readBuffer, 0, bufferSize); Assert.IsTrue(Helper.EqualArray(readBuffer, writeBuffer)); dfs.Close(); }
public void TestWriteReadRandom_Seek2() { DurableFileStream dfs = new DurableFileStream("TestWriteReadRandom_Seek2.dat", true); // seek long seekedPos = dfs.Seek(2000, SeekOrigin.Begin); Assert.AreEqual<long>(seekedPos, 2000); Assert.AreEqual<long>(dfs.Position, 2000); Assert.AreEqual<long>(dfs.Length, 0); seekedPos = dfs.Seek(-1000, SeekOrigin.Current); Assert.AreEqual<long>(seekedPos, 1000); Assert.AreEqual<long>(dfs.Position, 1000); Assert.AreEqual<long>(dfs.Length, 0); // write int bufferSize = DurableFileStream.BLOCK_SIZE; byte[] writeBuffer = new byte[bufferSize]; Helper.SetArrayRandomly(writeBuffer); dfs.Write(writeBuffer, 0, bufferSize); long lastPosition = seekedPos + bufferSize; Assert.AreEqual<long>(dfs.Position, lastPosition); Assert.AreEqual<long>(dfs.Length, lastPosition); seekedPos = dfs.Seek(-1000, SeekOrigin.End); Assert.AreEqual<long>(seekedPos, lastPosition - 1000); Assert.AreEqual<long>(dfs.Position, lastPosition - 1000); Assert.AreEqual<long>(dfs.Length, lastPosition); dfs.Commit(); dfs.Close(); }
public void TestRenewCheckpoint2() { // RENEW_CHECKPOINT_THRESHOLD = 10 * 1024; int bufferSize = 20000; byte[] writeBuffer1 = new byte[bufferSize]; Helper.SetArrayRandomly(writeBuffer1); byte[] writeBuffer2 = new byte[1000]; Helper.SetArrayRandomly(writeBuffer2); // writer DurableFileStream dfs = new DurableFileStream("TestRenewCheckpoint2.dat", true); dfs.Write(writeBuffer1, 0, bufferSize); dfs.Commit(); dfs.Write(writeBuffer2, 0, writeBuffer2.Length); dfs.Commit(); dfs.Close(); // reader dfs = new DurableFileStream("TestRenewCheckpoint2.dat", false); byte[] readBuffer1 = new byte[bufferSize]; dfs.Read(readBuffer1, 0, bufferSize); Assert.IsTrue(Helper.EqualArray(readBuffer1, writeBuffer1)); byte[] readBuffer2 = new byte[1000]; dfs.Read(readBuffer2, 0, readBuffer2.Length); Assert.IsTrue(Helper.EqualArray(readBuffer2, writeBuffer2)); dfs.Close(); }
public void TestWriteReadRandom_Seek1() { DurableFileStream dfs = new DurableFileStream("TestWriteReadRandom_Seek1.dat", true); long seekedPos = dfs.Seek(2000, SeekOrigin.Begin); Assert.AreEqual<long>(seekedPos, 2000); Assert.AreEqual<long>(dfs.Position, 2000); Assert.AreEqual<long>(dfs.Length, 0); seekedPos = dfs.Seek(-1000, SeekOrigin.Current); Assert.AreEqual<long>(seekedPos, 1000); Assert.AreEqual<long>(dfs.Position, 1000); Assert.AreEqual<long>(dfs.Length, 0); dfs.Close(); }
public void TestRecreateLogFile1() { byte[] writeBuffer = new byte[4096]; // Writer DurableFileStream dfs = new DurableFileStream("TestRecreateLogFile1.dat", true); for (int i = 0; i < /*13000*/100; i++) { Helper.SetArrayRandomly(writeBuffer); dfs.Write(writeBuffer, 0, writeBuffer.Length); } dfs.Commit(); dfs.Close(); }
private void Check_NO_UNDO(string filename, DurableFileStream.FailedCommitCode failedCommitCode) { int bufferSize = 10005; byte[] writeBuffer = new byte[bufferSize]; Helper.SetArrayRandomly(writeBuffer); // Writer try { DurableFileStream dfs = new DurableFileStream(filename, true); dfs.Write(writeBuffer, 0, bufferSize); dfs.__FailedCommitSimulation__(failedCommitCode); Assert.Fail("should not pass here!"); dfs.Close(); } catch (Exception) { DurableFileStream dfs = new DurableFileStream(filename, false); Assert.AreEqual<long>(dfs.Position, 0); Assert.AreEqual<long>(dfs.Length, 0); dfs.Write(writeBuffer, 0, bufferSize); dfs.Commit(); Assert.AreEqual<long>(dfs.Position, bufferSize); Assert.AreEqual<long>(dfs.Length, bufferSize); long x = 9876543210; dfs.Write(BitConverter.GetBytes(x), 0, 8); dfs.Commit(); Assert.AreEqual<long>(dfs.Position, bufferSize + 8); Assert.AreEqual<long>(dfs.Length, bufferSize + 8); dfs.Close(); dfs = new DurableFileStream(filename, false); byte[] readBuffer = new byte[bufferSize]; dfs.Read(readBuffer, 0, bufferSize); Assert.IsTrue(Helper.EqualArray(writeBuffer, readBuffer)); dfs.Read(readBuffer, 0, 8); long y = BitConverter.ToInt64(readBuffer, 0); Assert.AreEqual<long>(x, y); dfs.Close(); } }
public void TestLogWriteOptimization2() { Random rand = new Random(); byte[] writeBuffer1 = new byte[2500]; Helper.SetArrayRandomly(writeBuffer1); byte[] writeBuffer2 = new byte[1000]; Helper.SetArrayRandomly(writeBuffer2); // // Writer // DurableFileStream dfs = new DurableFileStream("TestLogWriteOptimization2.dat", true); dfs.Write(writeBuffer1, 0, writeBuffer1.Length); Assert.AreEqual<long>(dfs.Position, writeBuffer1.Length); Assert.AreEqual<long>(dfs.Length, writeBuffer1.Length); dfs.Seek(500, System.IO.SeekOrigin.Begin); Assert.AreEqual<long>(dfs.Position, 500); dfs.Write(writeBuffer2, 0, writeBuffer2.Length); Assert.AreEqual<long>(dfs.Position, 500 + writeBuffer2.Length); Assert.AreEqual<long>(dfs.Length, writeBuffer1.Length); dfs.Commit(); dfs.Close(); // // Reader // dfs = new DurableFileStream("TestLogWriteOptimization2.dat", false); byte[] readBuffer = new byte[writeBuffer1.Length]; dfs.Read(readBuffer, 0, writeBuffer1.Length); Assert.AreEqual<long>(dfs.Position, writeBuffer1.Length); Assert.AreEqual<long>(dfs.Length, writeBuffer1.Length); byte[] mergedWriteBuffer = new byte[writeBuffer1.Length]; Array.Copy(writeBuffer1, 0, mergedWriteBuffer, 0, writeBuffer1.Length); Array.Copy(writeBuffer2, 0, mergedWriteBuffer, 500, writeBuffer2.Length); dfs.Close(); // Validate Assert.IsTrue(Helper.EqualArray(readBuffer, mergedWriteBuffer)); }
public void TestMisc2() { // // write // int bufferSize = DurableFileStream.BLOCK_SIZE; byte[] writeBuffer1 = new byte[bufferSize]; Helper.SetArrayRandomly(writeBuffer1); DurableFileStream dfs = new DurableFileStream("TestMisc2.dat", true); dfs.Write(writeBuffer1, 0, bufferSize); dfs.Commit(); byte[] writeBuffer2 = new byte[bufferSize]; Helper.SetArrayRandomly(writeBuffer2); dfs.Write(writeBuffer2, 0, bufferSize); Assert.AreEqual<long>(dfs.Length, 2 * DurableFileStream.BLOCK_SIZE); Assert.AreEqual<long>(dfs.Position, 2 * DurableFileStream.BLOCK_SIZE); dfs.Abort(); Assert.AreEqual<long>(dfs.Length, 1 * DurableFileStream.BLOCK_SIZE); Assert.AreEqual<long>(dfs.Position, 2 * DurableFileStream.BLOCK_SIZE); dfs.Close(); // // read // dfs = new DurableFileStream("TestMisc2.dat", false); Assert.AreEqual<long>(dfs.Length, DurableFileStream.BLOCK_SIZE); Assert.AreEqual<long>(dfs.Position, 0); byte[] readBuffer = new byte[bufferSize]; dfs.Read(readBuffer, 0, bufferSize); Assert.IsTrue(Helper.EqualArray(readBuffer, writeBuffer1)); Assert.AreEqual<long>(dfs.Length, DurableFileStream.BLOCK_SIZE); Assert.AreEqual<long>(dfs.Position, DurableFileStream.BLOCK_SIZE); dfs.Close(); }
public void TestLogWriteOptimization1() { Random rand = new Random(); byte[] writeBuffer1 = new byte[1000]; Helper.SetArrayRandomly(writeBuffer1); byte[] writeBuffer2 = new byte[1500]; Helper.SetArrayRandomly(writeBuffer2); // Writer DurableFileStream dfs = new DurableFileStream("TestLogWriteOptimization1.dat", true); dfs.Write(writeBuffer1, 0, writeBuffer1.Length); Assert.AreEqual<long>(dfs.Position, writeBuffer1.Length); Assert.AreEqual<long>(dfs.Length, writeBuffer1.Length); dfs.Write(writeBuffer2, 0, writeBuffer2.Length); Assert.AreEqual<long>(dfs.Position, writeBuffer1.Length + writeBuffer2.Length); Assert.AreEqual<long>(dfs.Length, writeBuffer1.Length + writeBuffer2.Length); dfs.Commit(); dfs.Close(); // Reader dfs = new DurableFileStream("TestLogWriteOptimization1.dat", false); byte[] readBuffer1 = new byte[writeBuffer1.Length]; dfs.Read(readBuffer1, 0, writeBuffer1.Length); Assert.AreEqual<long>(dfs.Position, writeBuffer1.Length); Assert.AreEqual<long>(dfs.Length, writeBuffer1.Length + writeBuffer2.Length); byte[] readBuffer2 = new byte[writeBuffer2.Length]; dfs.Read(readBuffer2, 0, writeBuffer2.Length); Assert.AreEqual<long>(dfs.Position, writeBuffer1.Length + writeBuffer2.Length); Assert.AreEqual<long>(dfs.Length, writeBuffer1.Length + writeBuffer2.Length); dfs.Close(); // Validate Assert.IsTrue(Helper.EqualArray(readBuffer1, writeBuffer1)); Assert.IsTrue(Helper.EqualArray(readBuffer2, writeBuffer2)); }
private void ExecuteWriteRead_NoCommit_Close(string filename, byte[] writeBuffer) { int bufferSize = writeBuffer.Length; // Writer DurableFileStream dfs = new DurableFileStream(filename, true); dfs.Write(writeBuffer, 0, bufferSize); Assert.AreEqual<long>(dfs.Position, bufferSize); Assert.AreEqual<long>(dfs.Length, bufferSize); dfs.Close(); // Reader dfs = new DurableFileStream(filename, false); byte[] readBuffer = new byte[bufferSize]; dfs.Read(readBuffer, 0, bufferSize); Assert.AreEqual<long>(dfs.Position, bufferSize); Assert.AreEqual<long>(dfs.Length, bufferSize); dfs.Close(); // Validate Assert.IsTrue(Helper.EqualArray(writeBuffer, readBuffer)); }
public void __Test_Commit__(DurableFileStream.FailedCommitCode failedCommitCode) { if (_writeLogRecordDict.Count == 0) return; List<WriteLogRecord> writeLogRecordList = GetSortedWriteLogRecordList(); NextTransactionID(); long prevLSN = _LSN; prevLSN = LogBegin(); FlushBuffer(); if (failedCommitCode == DurableFileStream.FailedCommitCode.AFTER_LOG_BEGIN) throw new Exception("Failed Commit: After Log Begin"); foreach (WriteLogRecord record in writeLogRecordList) { prevLSN = LogWrite(prevLSN, record); FlushBuffer(); if (failedCommitCode == DurableFileStream.FailedCommitCode.AFTER_LOG_WRITE) throw new Exception("Failed Commit: After Log Write"); } prevLSN = LogCommit(prevLSN); FlushBuffer(); if (failedCommitCode == DurableFileStream.FailedCommitCode.AFTER_LOG_COMMIT) throw new Exception("Failed Commit: After Log Commit"); //------------------------------------------------------ // Finalize Transaction by writing the File //------------------------------------------------------ WriteToDataFile(writeLogRecordList); _writeLogRecordDict.Clear(); if (failedCommitCode == DurableFileStream.FailedCommitCode.AFTER_DISK_WRITE) throw new Exception("Failed Commit: After Disk Write"); prevLSN = LogEnd(prevLSN); FlushBuffer(); if (failedCommitCode == DurableFileStream.FailedCommitCode.AFTER_LOG_END) throw new Exception("Failed Commit: After Log End"); //------------------------------------------------------ if (_fsLog.Length >= RECREATE_LOG_THRESHOLD) { CreateOrOpenCommitFile(true); // delete log file then create a new one } if (_fsLog.Length - _checkpointAddress >= RENEW_CHECKPOINT_THRESHOLD) { __Test_WriteNewCheckpointAddress__(failedCommitCode); // change check point address } }
private void SampleCode2() { // Reader DurableFile.DurableFileStream dfs = new DurableFile.DurableFileStream("example1.dat", false); byte[] buffer = new byte[8]; dfs.Read(buffer, 0, 8); long data = BitConverter.ToInt64(buffer, 0); Console.WriteLine(data); dfs.Read(buffer, 0, 8); data = BitConverter.ToInt64(buffer, 0); Console.WriteLine(data); dfs.Read(buffer, 0, 8); data = BitConverter.ToInt64(buffer, 0); Console.WriteLine(data); dfs.Close(); }
public void TestRenewCheckpoint_AfterLogBegin1() { int bufferSize = 20000; byte[] writeBuffer = new byte[bufferSize]; Helper.SetArrayRandomly(writeBuffer); try { DurableFileStream dfs = new DurableFileStream("TestRenewCheckpoint_AfterLogBegin1.dat", true); dfs.Write(writeBuffer, 0, bufferSize); dfs.__FailedCommitSimulation__(DurableFileStream.FailedCommitCode.CHECKPOINT_AFTER_LOG_BEGIN); Assert.Fail("should not pass here!"); dfs.Close(); } catch (Exception) { // reader DurableFileStream dfs = new DurableFileStream("TestRenewCheckpoint_AfterLogBegin1.dat", false); byte[] readBuffer = new byte[bufferSize]; dfs.Read(readBuffer, 0, bufferSize); Assert.IsTrue(Helper.EqualArray(readBuffer, writeBuffer)); dfs.Close(); } }
private void Check_REDO(string filename, DurableFileStream.FailedCommitCode failedCommitCode) { int bufferSize = 10005; byte[] writeBuffer = new byte[bufferSize]; Helper.SetArrayRandomly(writeBuffer); try { // Writer DurableFileStream dfs = new DurableFileStream(filename, true); dfs.Write(writeBuffer, 0, bufferSize); dfs.__FailedCommitSimulation__(failedCommitCode); Assert.Fail("should not pass here!"); dfs.Close(); } catch (Exception) { // reader DurableFileStream dfs = new DurableFileStream(filename, false); Assert.AreEqual<long>(dfs.Position, 0); Assert.AreEqual<long>(dfs.Length, bufferSize); byte[] readBuffer = new byte[bufferSize]; dfs.Read(readBuffer, 0, bufferSize); Assert.IsTrue(Helper.EqualArray(writeBuffer, readBuffer)); Assert.AreEqual<long>(dfs.Position, bufferSize); Assert.AreEqual<long>(dfs.Length, bufferSize); dfs.Close(); } }
private void __Test_WriteNewCheckpointAddress__(DurableFileStream.FailedCommitCode failedCommitCode) { NextTransactionID(); // clear first block on disk before writing log _fsLog.Seek(0, SeekOrigin.Begin); // go to first block ClearBuffer(); _bufferIdx = BLOCK_SIZE; FlushBuffer(); // write log to the first block after clearing _fsLog.Seek(0, SeekOrigin.Begin); // go to first block WriteLogRecord ckpWriteRecord = new WriteLogRecord(); ckpWriteRecord.filename = _logPath; ckpWriteRecord.blockNo = 1; ckpWriteRecord.count = 8; ckpWriteRecord.AFIM = BitConverter.GetBytes(_fsLog.Length); _LSN = 0; long prev_lsn = LogBegin(); FlushBuffer(); if (failedCommitCode == DurableFileStream.FailedCommitCode.CHECKPOINT_AFTER_LOG_BEGIN) throw new Exception("Failed Commit: Checkpoint After Log Begin"); prev_lsn = LogWrite(prev_lsn, ckpWriteRecord); FlushBuffer(); long lastLogPos = _fsLog.Position; if (failedCommitCode == DurableFileStream.FailedCommitCode.CHECKPOINT_AFTER_LOG_WRITE) throw new Exception("Failed Commit: Checkpoint After Log Write"); _fsLog.Seek(BLOCK_SIZE, SeekOrigin.Begin); _fsLog.Write(ckpWriteRecord.AFIM, 0, 8); _fsLog.Flush(true); if (failedCommitCode == DurableFileStream.FailedCommitCode.CHECKPOINT_AFTER_DISK_WRITE) throw new Exception("Failed Commit: Checkpoint After Disk Write"); _fsLog.Seek(lastLogPos, SeekOrigin.Begin); // go to first block prev_lsn = LogEnd(prev_lsn); FlushBuffer(); if (failedCommitCode == DurableFileStream.FailedCommitCode.CHECKPOINT_AFTER_LOG_END) throw new Exception("Failed Commit: Checkpoint After Log End"); _checkpointAddress = _fsLog.Length; // reset LSN _fsLog.Seek(0, SeekOrigin.End); _LSN = _fsLog.Position; }
public CommitLog(DurableFileStream dataFileStream, bool create) { _dataFileStream = dataFileStream; _logPath = dataFileStream.Path + ".log"; _writeLogRecordDict = new Dictionary<long, WriteLogRecord>(); CreateOrOpenCommitFile(create); if (_fsLog.Length > 0) { // existing file _recoveryBuffer = new byte[8496]; ReadHeaderBlock(); ReadCheckpointAddress(); Recover(); _fsLog.Seek(Math.Max(START_LOG_POSITION, _fsLog.Length), SeekOrigin.Begin); _LSN = _fsLog.Position; _recoveryBuffer = null; } }
public void TestRenewCheckpoint_AfterDiskWrite() { int bufferSize = 20000; byte[] writeBuffer1 = new byte[bufferSize]; Helper.SetArrayRandomly(writeBuffer1); byte[] writeBuffer2 = new byte[bufferSize]; Helper.SetArrayRandomly(writeBuffer2); long checkpointAdr = 0; try { // writer DurableFileStream dfs = new DurableFileStream("TestRenewCheckpoint_4.dat", true); dfs.Write(writeBuffer1, 0, bufferSize); dfs.Commit(); dfs.Write(writeBuffer2, 0, writeBuffer2.Length); dfs.__FailedCommitSimulation__(DurableFileStream.FailedCommitCode.CHECKPOINT_AFTER_DISK_WRITE); Assert.Fail("should not pass here!"); dfs.Close(); } catch (Exception) { // reader DurableFileStream dfs = new DurableFileStream("TestRenewCheckpoint_4.dat", false); byte[] readBuffer1 = new byte[bufferSize]; dfs.Read(readBuffer1, 0, bufferSize); Assert.IsTrue(Helper.EqualArray(readBuffer1, writeBuffer1)); byte[] readBuffer2 = new byte[writeBuffer2.Length]; dfs.Read(readBuffer2, 0, readBuffer2.Length); dfs.Close(); } }
private void SampleCode1() { // Writer DurableFile.DurableFileStream dfs = new DurableFile.DurableFileStream("example1.dat", true); long data = 78739; // write to cache buffer only dfs.Write(BitConverter.GetBytes(data), 0, 8); data = 10254; // write to cache buffer only dfs.Write(BitConverter.GetBytes(data), 0, 8); data = 85471; // write to cache buffer only dfs.Write(BitConverter.GetBytes(data), 0, 8); // persist writes permanently to disk dfs.Commit(); dfs.Close(); }
public void TestOffset1() { int bufferSize = 4096; byte[] writeBuffer = new byte[bufferSize]; Helper.SetArrayRandomly(writeBuffer); // // Writer // DurableFileStream dfs = new DurableFileStream("TestOffset1.dat", true); int offset = 100; int count = 500; dfs.Write(writeBuffer, offset, count); dfs.Seek(0, System.IO.SeekOrigin.Begin); byte[] readBuffer = new byte[count * 2]; int readOffest = 300; dfs.Read(readBuffer, readOffest, count); bool equal = true; for (int i = offset; i < count; i++) { if (writeBuffer[i] != readBuffer[i - offset + readOffest]) { equal = false; break; } } Assert.IsTrue(equal); dfs.Commit(); Assert.AreEqual<long>(dfs.Position, count); Assert.AreEqual<long>(dfs.Length, count); dfs.Seek(0, System.IO.SeekOrigin.Begin); readBuffer = new byte[count * 2]; dfs.Read(readBuffer, readOffest, count); equal = true; for (int i = offset; i < count; i++) { if (writeBuffer[i] != readBuffer[i - offset + readOffest]) { equal = false; break; } } Assert.IsTrue(equal); dfs.Close(); // // Reader // dfs = new DurableFileStream("TestOffset1.dat", false); Assert.AreEqual<long>(dfs.Position, 0); Assert.AreEqual<long>(dfs.Length, count); int count2 = count - 10; readBuffer = new byte[count2 * 3]; readOffest = 750; dfs.Read(readBuffer, readOffest, count2); equal = true; for (int i = offset; i < count2; i++) { if (writeBuffer[i] != readBuffer[i - offset + readOffest]) { equal = false; break; } } Assert.IsTrue(equal); Assert.AreEqual<long>(dfs.Position, count2); Assert.AreEqual<long>(dfs.Length, count); dfs.Close(); }