public static FileStream CreateFileStream(string filename, StreamStateBehaviour streamStateBehaviour)
{
FileStream result;
switch (streamStateBehaviour)
{
case StreamStateBehaviour.OpenNewStreamForReading:
//
// we have to create a new one, and use recursion to get the actual return value
//
result = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
break;
case StreamStateBehaviour.OpenNewStreamForReadingAndWriting:
//
// we have to create a new one, and use recursion to get the actual return value
//
result = new FileStream(filename, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite);
break;
default:
throw new NotSupportedException(string.Format("unknown streamStateBehaviour {0}", streamStateBehaviour));
}
return(result);
}
public static FileStream CreateFileStream(string filename, StreamStateBehaviour streamStateBehaviour)
{
FileStream result;
switch (streamStateBehaviour)
{
case StreamStateBehaviour.OpenNewStreamForReading:
//
// we have to create a new one, and use recursion to get the actual return value
//
result = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
break;
case StreamStateBehaviour.OpenNewStreamForReadingAndWriting:
//
// we have to create a new one, and use recursion to get the actual return value
//
result = new FileStream(filename, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite);
break;
default:
throw new NotSupportedException(string.Format("unknown streamStateBehaviour {0}", streamStateBehaviour));
}
return result;
}
public void StoreStream(Guid dataIdentifier, Stream streamToStore, int numberOfBytes, ICustomMetaData customMetaData, AddFileBehaviour addFileBehaviour, StreamStateBehaviour indexStreamStateBehaviour, StreamStateBehaviour dataStreamStateBehaviour)
{
using (var binaryReader = new BinaryReader(streamToStore))
{
byte[] fileData = binaryReader.ReadBytes(numberOfBytes);
StoreBytes(dataIdentifier, fileData, customMetaData, addFileBehaviour, indexStreamStateBehaviour, dataStreamStateBehaviour);
}
}
public void StoreFile(Guid dataIdentifier, string filename, ICustomMetaData customMetaData, AddFileBehaviour addFileBehaviour, StreamStateBehaviour indexStreamStateBehaviour, StreamStateBehaviour dataStreamStateBehaviour)
{
StoreBytes(dataIdentifier, File.ReadAllBytes(filename), customMetaData, addFileBehaviour, indexStreamStateBehaviour, dataStreamStateBehaviour);
}
public void StoreHttpRequest(Guid dataIdentifier, string url, ICustomMetaData customMetaData, AddFileBehaviour addFileBehaviour, string userAgent, StreamStateBehaviour indexStreamStateBehaviour, StreamStateBehaviour dataStreamStateBehaviour)
{
var request = (HttpWebRequest) WebRequest.Create(url);
request.UserAgent = userAgent;
// execute the web request
var response = (HttpWebResponse) request.GetResponse();
using (Stream webStream = response.GetResponseStream())
{
StoreBytes(dataIdentifier, webStream.ReadAllBytes(), customMetaData, addFileBehaviour, indexStreamStateBehaviour, dataStreamStateBehaviour);
}
}
public bool Exists(Guid dataIdentifier, StreamStateBehaviour indexStreamStateBehaviour)
{
bool result;
long offset = GetOffset(dataIdentifier, indexStreamStateBehaviour);
if (offset == 0)
{
//
// does not exist
//
result = false;
}
else
{
//
// does exist
//
result = true;
}
return result;
}
/// <summary>
/// Returns 0 if the dataIdentifier does not exist, or the offset of the dataIdentifier
/// in the datafile if it does exist.
/// </summary>
private Int64 GetOffset(Guid dataIdentifier, StreamStateBehaviour indexStreamStateBehaviour)
{
Int64 result;
switch (indexStreamStateBehaviour)
{
case StreamStateBehaviour.OpenNewStreamForReading:
//
// we have to create a new one, and use recursion to get the actual return value
//
using (IndexStream = FileStreamFactory.CreateFileStream(IndexFilename, indexStreamStateBehaviour))
{
result = GetOffset(dataIdentifier, StreamStateBehaviour.UseExistingStream);
}
break;
case StreamStateBehaviour.UseExistingStream:
result = GetOffsetInCurrentStream(dataIdentifier, 0, OffsetFirstIndexBlock);
break;
default:
throw new NotSupportedException(string.Format("unknown indexStreamStateBehaviour {0}", indexStreamStateBehaviour));
}
return result;
}
public void StoreString(Guid dataIdentifier, Encoding encoding, string stringToStore, ICustomMetaData customMetaData, AddFileBehaviour addFileBehaviour, StreamStateBehaviour indexStreamStateBehaviour, StreamStateBehaviour dataStreamStateBehaviour)
{
Byte[] bytes = encoding.GetBytes(stringToStore);
StoreBytes(dataIdentifier, bytes, customMetaData, addFileBehaviour, indexStreamStateBehaviour, dataStreamStateBehaviour);
}
private void GetDataIdentifiersRecursiveUsingIndex(IEnumerable<byte> bytesSoFar, int depth, Int64 offset, ICollection<Guid> result, StreamStateBehaviour indexStreamStateBehaviour, ExposeProgressDelegate anotherDataIdentifierFoundDelegate)
{
AllocationTableStruct indexStructs = GetIndexStruct(offset, indexStreamStateBehaviour);
for (int currentByteValue = 0; currentByteValue <= 255; currentByteValue++)
{
if (indexStructs.Pointers[currentByteValue] == 0)
{
// not used, recursion stops
}
else
{
//
// clone bytes so far, extend with current byte
//
var bytesForGuid = new List<byte>();
bytesForGuid.AddRange(bytesSoFar);
bytesForGuid.Add((byte) currentByteValue);
if (depth == 15)
{
var newGuid = new Guid(bytesForGuid.ToArray());
result.Add(newGuid);
//
// notify outside world we found another hit, if the outside world wants to hear about it
//
if (anotherDataIdentifierFoundDelegate != null)
{
anotherDataIdentifierFoundDelegate.Invoke(newGuid);
}
//
// recursion stops; this is the last byte
//
}
else
{
//
// continue recursion
//
long newOffset = indexStructs.Pointers[currentByteValue];
GetDataIdentifiersRecursiveUsingIndex(bytesForGuid, depth + 1, newOffset, result, indexStreamStateBehaviour, anotherDataIdentifierFoundDelegate);
}
}
}
}
public List<Guid> GetAllDataIdentifiersBasedUponFileStorageIndexFile(StreamStateBehaviour indexStreamStateBehaviour, ExposeProgressDelegate exposeProgressDelegate)
{
List<Guid> result;
switch (indexStreamStateBehaviour)
{
case StreamStateBehaviour.UseExistingStream:
result = new List<Guid>();
long offset = OffsetFirstIndexBlock;
var bytes = new List<byte>();
int depth = 0;
GetDataIdentifiersRecursiveUsingIndex(bytes, depth, offset, result, indexStreamStateBehaviour, exposeProgressDelegate);
break;
case StreamStateBehaviour.OpenNewStreamForReading:
//
// open the index stream for reading and make a recursive call
//
using (IndexStream = new FileStream(IndexFilename, FileMode.Open, FileAccess.Read))
{
result = GetAllDataIdentifiersBasedUponFileStorageIndexFile(StreamStateBehaviour.UseExistingStream, exposeProgressDelegate);
}
break;
default:
throw new NotSupportedException(string.Format("indexStreamStateBehaviour {0} is not supported", indexStreamStateBehaviour));
}
if (exposeProgressDelegate != null)
{
exposeProgressDelegate.Invoke(true);
}
return result;
}
public List<Guid> GetAllDataIdentifiersBasedUponFileStorageIndexFile(StreamStateBehaviour indexStreamStateBehaviour)
{
return GetAllDataIdentifiersBasedUponFileStorageIndexFile(indexStreamStateBehaviour, null);
}
public byte[] GetFileByteData(Guid dataIdentifier, StreamStateBehaviour indexStreamStateBehaviour, StreamStateBehaviour dataStreamStateBehaviour)
{
byte[] result;
switch (indexStreamStateBehaviour)
{
case StreamStateBehaviour.OpenNewStreamForReading:
//
// open the stream and make a recursive call
//
using (IndexStream = FileStreamFactory.CreateFileStream(IndexFilename, indexStreamStateBehaviour))
{
result = GetFileByteData(dataIdentifier, StreamStateBehaviour.UseExistingStream, dataStreamStateBehaviour);
}
break;
case StreamStateBehaviour.UseExistingStream:
//
// check state of data file stream
//
switch (dataStreamStateBehaviour)
{
case StreamStateBehaviour.OpenNewStreamForReading:
//
// open the stream and make a recursive call
//
using (DataStream = FileStreamFactory.CreateFileStream(DataFilename, dataStreamStateBehaviour))
{
result = GetFileByteData(dataIdentifier, indexStreamStateBehaviour, StreamStateBehaviour.UseExistingStream);
}
break;
case StreamStateBehaviour.UseExistingStream:
//
//
//
result = GetFileByteDataForExistingStreams(dataIdentifier);
//
break;
default:
throw new NotSupportedException(string.Format("Unexpected dataStreamStateBehaviour {0}", dataStreamStateBehaviour));
}
//
break;
default:
throw new NotSupportedException(string.Format("Unexpected indexStreamStateBehaviour {0}", indexStreamStateBehaviour));
}
return result;
}
public IDynamiteXml GetObjectData(Guid dataIdentifier, StreamStateBehaviour indexStreamStateBehaviour, StreamStateBehaviour dataStreamStateBehaviour)
{
string serializedObjectString = GetStringData(dataIdentifier, indexStreamStateBehaviour, dataStreamStateBehaviour);
return DynamiteXmlLogic.Deserialize(serializedObjectString);
}
public string GetStringData(Guid dataIdentifier, Encoding encoding, StreamStateBehaviour indexStreamStateBehaviour, StreamStateBehaviour dataStreamStateBehaviour)
{
byte[] bytes = GetFileByteData(dataIdentifier, indexStreamStateBehaviour, dataStreamStateBehaviour);
return encoding.GetString(bytes);
}
public string GetStringData(Guid dataIdentifier, StreamStateBehaviour indexStreamStateBehaviour, StreamStateBehaviour dataStreamStateBehaviour)
{
var encoding = new ASCIIEncoding();
return GetStringData(dataIdentifier, encoding, indexStreamStateBehaviour, dataStreamStateBehaviour);
}
public void StoreObject(Guid dataIdentifier, IDynamiteXml objectToStore, ICustomMetaData customMetaData, AddFileBehaviour addFileBehaviour, StreamStateBehaviour indexStreamStateBehaviour, StreamStateBehaviour dataStreamStateBehaviour)
{
string serializedObject = DynamiteXmlLogic.Serialize(objectToStore);
StoreString(dataIdentifier, serializedObject, customMetaData, addFileBehaviour, indexStreamStateBehaviour, dataStreamStateBehaviour);
}
public void StoreBytes(Guid dataIdentifier, byte[] fileData, ICustomMetaData customMetaData, AddFileBehaviour addFileBehaviour, StreamStateBehaviour indexStreamStateBehaviour, StreamStateBehaviour dataStreamStateBehaviour)
{
switch (indexStreamStateBehaviour)
{
case StreamStateBehaviour.OpenNewStreamForReadingAndWriting:
//
// open the stream and make a recursive call
//
using (IndexStream = FileStreamFactory.CreateFileStream(IndexFilename, indexStreamStateBehaviour))
{
StoreBytes(dataIdentifier, fileData, customMetaData, addFileBehaviour, StreamStateBehaviour.UseExistingStream, dataStreamStateBehaviour);
}
break;
case StreamStateBehaviour.UseExistingStream:
//
// check state of data file stream
//
switch (dataStreamStateBehaviour)
{
case StreamStateBehaviour.OpenNewStreamForReadingAndWriting:
//
// open the stream and make a recursive call
//
using (DataStream = FileStreamFactory.CreateFileStream(DataFilename, dataStreamStateBehaviour))
{
StoreBytes(dataIdentifier, fileData, customMetaData, addFileBehaviour, indexStreamStateBehaviour, StreamStateBehaviour.UseExistingStream);
}
break;
case StreamStateBehaviour.UseExistingStream:
//
//
//
StoreBytesForExistingStreams(dataIdentifier, fileData, customMetaData, addFileBehaviour);
//
break;
default:
throw new NotSupportedException(string.Format("Unexpected dataStreamStateBehaviour {0}", dataStreamStateBehaviour));
}
//
break;
default:
throw new NotSupportedException(string.Format("Unexpected indexStreamStateBehaviour {0}", indexStreamStateBehaviour));
}
}
/// <summary>
/// Marks the file in the index as deleted.
/// Note only the pointer to the file is removed, the data itself remains stored in the data file (data is not purged).
/// </summary>
public void DeleteDataIdentifier(Guid dataIdentifier, DeleteFileBehaviour deleteFileBehaviour, StreamStateBehaviour indexStreamStateBehaviour)
{
switch (indexStreamStateBehaviour)
{
case StreamStateBehaviour.OpenNewStreamForReadingAndWriting:
//
// open the stream and make a recursive call
//
using (IndexStream = FileStreamFactory.CreateFileStream(IndexFilename, indexStreamStateBehaviour))
{
DeleteDataIdentifier(dataIdentifier, deleteFileBehaviour, StreamStateBehaviour.UseExistingStream);
}
break;
case StreamStateBehaviour.UseExistingStream:
switch (deleteFileBehaviour)
{
case DeleteFileBehaviour.IgnoreWhenNotExists:
break;
case DeleteFileBehaviour.ThrowExceptionWhenNotExists:
if (!Exists(dataIdentifier, StreamStateBehaviour.UseExistingStream))
{
throw new FileNotFoundException(string.Format("File does not exist ({0})", dataIdentifier));
}
break;
default:
throw new NotSupportedException(string.Format("Unsupported delete file behaviour ({0})", deleteFileBehaviour));
}
DeleteDataIdentifierInExistingStream(dataIdentifier, 0, OffsetFirstIndexBlock, indexStreamStateBehaviour);
break;
default:
throw new NotSupportedException(string.Format("Unexpected indexStreamStateBehaviour {0}", indexStreamStateBehaviour));
}
}
public void BuildIndex(Guid dataIdentifier, Int64 dataOffset, BuildIndexBehaviour buildIndexBehaviour, StreamStateBehaviour indexStreamStateBehaviour)
{
int indexInGuid = 0;
long currentOffsetInIndexFile = OffsetFirstIndexBlock;
BuildIndex(dataIdentifier, indexInGuid, currentOffsetInIndexFile, dataOffset, buildIndexBehaviour, indexStreamStateBehaviour);
}
private void DeleteDataIdentifierInExistingStream(Guid dataIdentifier, int currentIndexInGuid, Int64 currentOffset, StreamStateBehaviour indexStreamStateBehaviour)
{
byte[] bytes = dataIdentifier.ToByteArray();
int indexerInGuid = bytes[currentIndexInGuid];
AllocationTableStruct currentIndexStruct = GetIndexStruct(currentOffset, indexStreamStateBehaviour);
if (currentIndexInGuid < 15)
{
long nextOffset = currentIndexStruct.Pointers[indexerInGuid];
if (nextOffset == 0)
{
throw new Exception("Unexpected; file was already deleted? (should have been checked before)");
}
//
// recursively call BuildIndex for the next index in the guid
//
DeleteDataIdentifierInExistingStream(dataIdentifier, currentIndexInGuid + 1, nextOffset, indexStreamStateBehaviour);
}
else
{
switch (currentIndexInGuid)
{
case 15:
if (currentIndexStruct.Pointers[indexerInGuid] == 0)
{
throw new Exception(
"Unexpected; file was already deleted? (should have been checked before)");
}
currentIndexStruct.Pointers[indexerInGuid] = 0;
StoreStruct(currentOffset, currentIndexStruct);
break;
default:
throw new IndexOutOfRangeException(currentIndexInGuid.ToString());
}
}
}
public AllocationTableStruct GetIndexStruct(Int64 currentOffset, StreamStateBehaviour indexStreamStateBehaviour)
{
if (indexStreamStateBehaviour != StreamStateBehaviour.UseExistingStream)
{
throw new NotSupportedException("Only useexistingstream is currently supported ");
}
var result = new AllocationTableStruct
{
Pointers = new Int64[256]
};
IndexStream.Seek(currentOffset, SeekOrigin.Begin);
//
// the stream is intentionaly left open
//
var binaryReader = new BinaryReader(IndexStream);
for (int elementIndex = 0; elementIndex < 256; elementIndex++)
{
result.Pointers[elementIndex] = binaryReader.ReadInt64();
}
return result;
}
public void RestoreIndexFile(AddFileBehaviour addFileBehaviour, StreamStateBehaviour indexStreamStateBehaviour, StreamStateBehaviour dataStreamStateBehaviour, ExposeProgressDelegate exposeProgressDelegate)
{
//
// the following method also checks whether the index file exists or not,
// if exists, it throws an exception
//
CreateNewFileStorage_IndexFile(IndexFilename);
switch (indexStreamStateBehaviour)
{
case StreamStateBehaviour.OpenNewStreamForReadingAndWriting:
// ok
break;
default:
throw new NotSupportedException(string.Format("Unexpected indexStreamStateBehaviour {0}", indexStreamStateBehaviour));
}
switch (dataStreamStateBehaviour)
{
case StreamStateBehaviour.OpenNewStreamForReading:
// ok
break;
default:
throw new NotSupportedException(string.Format("Unexpected dataStreamStateBehaviour {0}", dataStreamStateBehaviour));
}
using (IndexStream = FileStreamFactory.CreateFileStream(IndexFilename, StreamStateBehaviour.OpenNewStreamForReadingAndWriting))
{
using (DataStream = FileStreamFactory.CreateFileStream(DataFilename, StreamStateBehaviour.OpenNewStreamForReading))
{
long dataSize = DataStream.Length;
long offset = OffsetFirstDataBlock;
while (true)
{
InterpretedDataFileDataStruct retrieveFileStruct = GetInterpretedDataStructUsingExistingStream(offset);
//
// the offset in the index file should point to the starting location
// of the data struct, this is not the offset where the actual data of
// the file is stored, but the start of the struct before the actual data.
//
Guid dataIdentifier = retrieveFileStruct.DataIdentification;
//
// if the data file contains duplicate dataIdentifiers, we will override previous ones (which
// is acceptable, as new files are appeneded to the data file, so more up to date files are stored
// after older ones).
//
BuildIndex(dataIdentifier, offset, BuildIndexBehaviour.OverrideWhenAlreadyExists, StreamStateBehaviour.UseExistingStream);
//
// jump to the next data block
//
offset = retrieveFileStruct.BinaryDataOffset + retrieveFileStruct.BinaryDataSizeInBytes;
//
// inform the outside world about the progress we've made,
// in case the outside world is interested
//
if (exposeProgressDelegate != null)
{
exposeProgressDelegate.Invoke(dataIdentifier);
}
if (offset >= dataSize)
{
//
// reach the end of the file
//
break;
}
}
}
}
if (exposeProgressDelegate != null)
{
exposeProgressDelegate.Invoke(true);
}
}
public void ExportToOtherFileStorage(Guid dataIdentifier, FileStorageHandler targetFileStorageHandler, AddFileBehaviour addFileBehaviour, StreamStateBehaviour sourceIndexStreamStateBehaviour, StreamStateBehaviour sourceDataStreamStateBehaviour, StreamStateBehaviour targetIndexStreamStateBehaviour, StreamStateBehaviour targetDataStreamStateBehaviour)
{
if (targetFileStorageHandler.FileStorageName.Equals(FileStorageName))
{
throw new NotSupportedException(string.Format("Cannot export file from a filestorage to another filestorage with the same name ({0})", FileStorageName));
}
byte[] fileData = GetFileByteData(dataIdentifier, sourceIndexStreamStateBehaviour, sourceDataStreamStateBehaviour);
MetaDataContainer metaDataContainer = GetMetaDataContainer(dataIdentifier, sourceIndexStreamStateBehaviour, sourceDataStreamStateBehaviour);
targetFileStorageHandler.StoreBytes(dataIdentifier, fileData, metaDataContainer.CustomMetaData, addFileBehaviour, targetIndexStreamStateBehaviour, targetDataStreamStateBehaviour);
}
private void BuildIndex(Guid dataIdentifier, int currentIndexInGuid, Int64 currentOffset, Int64 dataOffset, BuildIndexBehaviour buildIndexBehaviour, StreamStateBehaviour indexStreamStateBehaviour)
{
byte[] bytes = dataIdentifier.ToByteArray();
int indexerInGuid = bytes[currentIndexInGuid];
Int64 endOfFileOffset = IndexStream.Length;
AllocationTableStruct currentIndexStruct = GetIndexStruct(currentOffset, indexStreamStateBehaviour);
if (currentIndexInGuid < 15)
{
long nextOffset = currentIndexStruct.Pointers[indexerInGuid];
if (nextOffset == 0)
{
//
// does not point to anywhere; this means we have to;
// * append a new block at the end of the file, with 0-pointers for each guid indexer over there
// * update the reference for the specific index in the guid to the end of the filestream
//
// update the struct
currentIndexStruct.Pointers[indexerInGuid] = endOfFileOffset;
// write the struct
StoreStruct(currentOffset, currentIndexStruct);
// create a new struct with offsets 0 for each index in guid
var newIndexStruct = new AllocationTableStruct
{
Pointers = new Int64[256]
};
StoreStruct(endOfFileOffset, newIndexStruct);
nextOffset = endOfFileOffset;
//
// recursively call BuildIndex for the next index in the guid
//
BuildIndex(dataIdentifier, currentIndexInGuid + 1, nextOffset, dataOffset, buildIndexBehaviour, indexStreamStateBehaviour);
}
else
{
//
// recursively call BuildIndex for the next index in the guid
//
BuildIndex(dataIdentifier, currentIndexInGuid + 1, nextOffset, dataOffset, buildIndexBehaviour, indexStreamStateBehaviour);
}
}
else if (currentIndexInGuid == 15)
{
//
// we reached the last index struct
//
switch (buildIndexBehaviour)
{
case BuildIndexBehaviour.OverrideWhenAlreadyExists:
//
// no matter if it already existed or not, we will override it
//
break;
case BuildIndexBehaviour.ThrowExceptionWhenIndexAlreadyExists:
if (currentIndexStruct.Pointers[indexerInGuid] == 0)
{
//
// its still free, continue
//
break;
}
//
// if we reach this point, it means the pointer was already allocated
//
throw new NotSupportedException(string.Format("Index {0} is already allocated", dataIdentifier));
default:
throw new NotSupportedException(string.Format("{0} is an unsupported BuildIndexBehaviour", buildIndexBehaviour));
}
currentIndexStruct.Pointers[indexerInGuid] = dataOffset;
// store the index block
StoreStruct(currentOffset, currentIndexStruct);
}
else
{
throw new IndexOutOfRangeException(currentIndexInGuid.ToString());
}
}
