/// <summary>
/// Makes a clone of a source filestorage, and re-allocates the data identifiers in the destination filestorage (the first
/// data identifier will get 0001, the next one 0002, etc. Next to outputting a new target filestorage with new indexes,
/// the method also produces a sql file that should be used to update your DAL logic, since the references need to be
/// updated too ofcourse.
/// </summary>
public static void Upgrade(string sourceFileStorageName, string destinationFileStorageName, FileStorageHandler.ExposeProgressDelegate exposeProgressDelegateA, FileStorageHandler.ExposeProgressDelegate exposeProgressDelegateB)
{
Create(destinationFileStorageName, CreateFileStorageBehaviour.ThrowExceptionWhenExists);
FileStorageHandler sourceFileStorageHandler = FileStorageHandler.Open(sourceFileStorageName, VersionBehaviour.BypassVersionCheck);
FileStorageHandler destinationFileStorageHandler = FileStorageHandler.Open(destinationFileStorageName);
List <Guid> dataIdentifiers = sourceFileStorageHandler.GetAllDataIdentifiersBasedUponFileStorageIndexFile(StreamStateBehaviour.OpenNewStreamForReading, exposeProgressDelegateA);
for (int currentDataIdentifierIndex = 0; currentDataIdentifierIndex < dataIdentifiers.Count; currentDataIdentifierIndex++)
{
Guid currentSourceDataIdentifier = dataIdentifiers[currentDataIdentifierIndex];
byte[] theBytes = sourceFileStorageHandler.GetFileByteData(currentSourceDataIdentifier, StreamStateBehaviour.OpenNewStreamForReading, StreamStateBehaviour.OpenNewStreamForReading);
ICustomMetaData customMetaData = sourceFileStorageHandler.SupportsFeature(FileStorageFeatureEnum.StoreMetaData)
? sourceFileStorageHandler.GetMetaDataContainer(currentSourceDataIdentifier, StreamStateBehaviour.OpenNewStreamForReading, StreamStateBehaviour.OpenNewStreamForReading).CustomMetaData
: new EmptyCustomMetaData();
Guid currentDestinationDataIdentifier = currentSourceDataIdentifier;
//
// store item in destination
//
destinationFileStorageHandler.StoreBytes(currentDestinationDataIdentifier, theBytes, customMetaData, AddFileBehaviour.ThrowExceptionWhenAlreadyExists, StreamStateBehaviour.OpenNewStreamForReadingAndWriting, StreamStateBehaviour.OpenNewStreamForReadingAndWriting);
if (exposeProgressDelegateB != null)
{
exposeProgressDelegateB.Invoke(currentDestinationDataIdentifier);
}
}
}
/// <summary>
/// Makes a clone of a source filestorage, and re-allocates the data identifiers in the destination filestorage (the first
/// data identifier will get 0001, the next one 0002, etc. Next to outputting a new target filestorage with new indexes,
/// the method also produces a sql file that should be used to update your DAL logic, since the references need to be
/// updated too ofcourse.
/// </summary>
public static void DefragDataIdentifiers(string sourceFileStorageName, string destinationFileStorageName, string sqlTable, string sqlColumn, FileStorageHandler.ExposeProgressDelegate exposeProgressDelegateA, FileStorageHandler.ExposeProgressDelegate exposeProgressDelegateB)
{
Create(destinationFileStorageName, CreateFileStorageBehaviour.ThrowExceptionWhenExists);
string sqlOutputFileName = destinationFileStorageName + ".FileStorage.index.fc.sql";
if (File.Exists(sqlOutputFileName))
{
throw new Exception(string.Format("File {0} already exists", sqlOutputFileName));
}
FileStorageHandler sourceFileStorageHandler = FileStorageHandler.Open(sourceFileStorageName);
FileStorageHandler destinationFileStorageHandler = FileStorageHandler.Open(destinationFileStorageName);
using (StreamWriter sw = File.CreateText(sqlOutputFileName))
{
sw.WriteLine("-- SQL Patch script to adjust the dataidentifiers");
List <Guid> dataIdentifiers = sourceFileStorageHandler.GetAllDataIdentifiersBasedUponFileStorageIndexFile(StreamStateBehaviour.OpenNewStreamForReading, exposeProgressDelegateA);
for (int currentDataIdentifierIndex = 0; currentDataIdentifierIndex < dataIdentifiers.Count; currentDataIdentifierIndex++)
{
Guid currentSourceDataIdentifier = dataIdentifiers[currentDataIdentifierIndex];
byte[] theBytes = sourceFileStorageHandler.GetFileByteData(currentSourceDataIdentifier, StreamStateBehaviour.OpenNewStreamForReading, StreamStateBehaviour.OpenNewStreamForReading);
ICustomMetaData customMetaData = sourceFileStorageHandler.SupportsFeature(FileStorageFeatureEnum.StoreMetaData)
? sourceFileStorageHandler.GetMetaDataContainer(currentSourceDataIdentifier, StreamStateBehaviour.OpenNewStreamForReading, StreamStateBehaviour.OpenNewStreamForReading).CustomMetaData
: new EmptyCustomMetaData();
//
// determine the new identifier
//
int b1AsInt = (currentDataIdentifierIndex >> 24) & 255;
var b1 = (byte)b1AsInt;
int b2AsInt = (currentDataIdentifierIndex >> 16) & 255;
var b2 = (byte)b2AsInt;
int b3AsInt = (currentDataIdentifierIndex >> 8) & 255;
var b3 = (byte)b3AsInt;
int b4AsInt = (currentDataIdentifierIndex >> 0) & 255;
var b4 = (byte)b4AsInt;
var currentDestinationDataIdentifier = new Guid(0, 0, 0, 0, 0, 0, 0, b1, b2, b3, b4);
string line = String.Format("update {0} set {1}='{2}' where {1}='{3}'", sqlTable, sqlColumn, currentSourceDataIdentifier, currentDestinationDataIdentifier);
sw.WriteLine(line);
//
// store item in destination
//
destinationFileStorageHandler.StoreBytes(currentDestinationDataIdentifier, theBytes, customMetaData, AddFileBehaviour.ThrowExceptionWhenAlreadyExists, StreamStateBehaviour.OpenNewStreamForReadingAndWriting, StreamStateBehaviour.OpenNewStreamForReadingAndWriting);
if (exposeProgressDelegateB != null)
{
exposeProgressDelegateB.Invoke(currentDestinationDataIdentifier);
}
}
sw.WriteLine("-- EOF");
}
}
public void StoreBytesForExistingStreams(Guid dataIdentifier, byte[] fileData, ICustomMetaData customMetaData, AddFileBehaviour addFileBehaviour)
{
#region Preconditions and initialization ...
if (fileData.LongLength > fileData.Length)
{
throw new Exception("NFileStorage is (currently) limited to a max length of 32bit (about 4.294.967.296 bytes)");
}
//
// first check the behaviour conditions
//
switch (addFileBehaviour)
{
case AddFileBehaviour.SkipWhenAlreadyExists:
if (Exists(dataIdentifier, StreamStateBehaviour.UseExistingStream))
{
return;
}
break;
case AddFileBehaviour.ThrowExceptionWhenAlreadyExists:
if (Exists(dataIdentifier, StreamStateBehaviour.UseExistingStream))
{
throw new Exception(string.Format("File {0} already exists", dataIdentifier));
}
break;
case AddFileBehaviour.OverrideWhenAlreadyExists:
if (Exists(dataIdentifier, StreamStateBehaviour.UseExistingStream))
{
DeleteDataIdentifier(dataIdentifier, DeleteFileBehaviour.ThrowExceptionWhenNotExists,
StreamStateBehaviour.UseExistingStream);
}
break;
default:
throw new NotSupportedException(string.Format("Unsupported behaviour; {0}", addFileBehaviour));
}
#endregion
bool supportsMetaData = SupportsFeature(FileStorageFeatureEnum.StoreMetaData);
long dataOffset = DataStream.Length;
int dataLength = fileData.Length;
Int64 metaDataOffsetInFileFromBegin;
if (supportsMetaData)
{
if (customMetaData == null)
{
customMetaData = new EmptyCustomMetaData();
}
metaDataOffsetInFileFromBegin = dataOffset + DataFileDataStructSize + dataLength;
}
else
{
metaDataOffsetInFileFromBegin = 0;
}
// create a new struct with offsets 0 for each index in guid
var newFileStruct = new DataFileDataStruct
{
DataIdentification = dataIdentifier,
BinaryDataSizeInBytes = dataLength,
DataStateEnumID = 0,
MetaDataOffsetInFileFromBegin = metaDataOffsetInFileFromBegin,
ReservedB = 0
};
#region Store the information in the data file
#region Store struct in data file
StoreStruct(dataOffset, newFileStruct);
#endregion
#region Store actual data in data file
//
// the stream is intentionally left open
//
var bufferedDataFileStream = new BufferedStream(DataStream);
// write the actual data
bufferedDataFileStream.Write(fileData, 0, dataLength);
// flush, to ensure we don't have bytes in the queue when we close the file
bufferedDataFileStream.Flush();
#endregion
#region Store meta data in data file
if (supportsMetaData)
{
//
// use the DynamiteXML feature to store metadata
//
var metaDataContainer = new MetaDataContainer(customMetaData, DateTime.UtcNow, dataLength);
string metaDataString = DynamiteXmlLogic.Serialize(metaDataContainer);
byte[] metaDataBytes = Encoding.UTF8.GetBytes(metaDataString);
//
// write the length of the bytes of the metadata
//
var binaryWriter = new BinaryWriter(DataStream);
binaryWriter.Write(metaDataBytes.Length);
bufferedDataFileStream.Write(metaDataBytes, 0, metaDataBytes.Length);
// flush, to ensure we don't have bytes in the queue when we close the file
bufferedDataFileStream.Flush();
}
#endregion
#endregion
#region Store the information in the index file
//
// derive the build index behaviour from the add file behaviour
//
BuildIndexBehaviour buildIndexBehaviour;
switch (addFileBehaviour)
{
case AddFileBehaviour.ThrowExceptionWhenAlreadyExists:
buildIndexBehaviour = BuildIndexBehaviour.ThrowExceptionWhenIndexAlreadyExists;
break;
case AddFileBehaviour.SkipWhenAlreadyExists:
//
// this should have been intercepted before, should be impossible
//
buildIndexBehaviour = BuildIndexBehaviour.ThrowExceptionWhenIndexAlreadyExists;
break;
case AddFileBehaviour.OverrideWhenAlreadyExists:
buildIndexBehaviour = BuildIndexBehaviour.OverrideWhenAlreadyExists;
break;
default:
throw new NotSupportedException(string.Format("Unexpected addFileBehaviour {0}", addFileBehaviour));
}
// we update the index file
BuildIndex(dataIdentifier, dataOffset, buildIndexBehaviour, StreamStateBehaviour.UseExistingStream);
#endregion
}
public void StoreString(Guid dataIdentifier, Encoding encoding, string stringToStore, ICustomMetaData customMetaData, AddFileBehaviour addFileBehaviour, StreamStateBehaviour indexStreamStateBehaviour, StreamStateBehaviour dataStreamStateBehaviour)
{
Byte[] bytes = encoding.GetBytes(stringToStore);
StoreBytes(dataIdentifier, bytes, customMetaData, addFileBehaviour, indexStreamStateBehaviour, dataStreamStateBehaviour);
}
public static void StoreHttpRequest(string fileStorageName, Guid dataIdentifier, string url, ICustomMetaData customMetaData, AddFileBehaviour addFileBehaviour, string userAgent)
{
var fileStorageHandler = FileStorageHandler.Open(fileStorageName);
fileStorageHandler.StoreHttpRequest(dataIdentifier, url, customMetaData, addFileBehaviour, userAgent, StreamStateBehaviour.OpenNewStreamForReadingAndWriting, StreamStateBehaviour.OpenNewStreamForReadingAndWriting);
}
public static void StoreBytes(string fileStorageName, Guid dataIdentifier, byte[] fileData, ICustomMetaData customMetaData, AddFileBehaviour addFileBehaviour)
{
FileStorageHandler fileStorageHandler = FileStorageHandler.Open(fileStorageName);
fileStorageHandler.StoreBytes(dataIdentifier, fileData, customMetaData, addFileBehaviour, StreamStateBehaviour.OpenNewStreamForReadingAndWriting, StreamStateBehaviour.OpenNewStreamForReadingAndWriting);
}
public void StoreFile(Guid dataIdentifier, string filename, ICustomMetaData customMetaData, AddFileBehaviour addFileBehaviour, StreamStateBehaviour indexStreamStateBehaviour, StreamStateBehaviour dataStreamStateBehaviour)
{
StoreBytes(dataIdentifier, File.ReadAllBytes(filename), customMetaData, addFileBehaviour, indexStreamStateBehaviour, dataStreamStateBehaviour);
}
public static void StoreStream(string fileStorageName, Guid dataIdentifier, Stream streamToStore, int numberOfBytes, ICustomMetaData customMetaData, AddFileBehaviour addFileBehaviour)
{
var fileStorageHandler = FileStorageHandler.Open(fileStorageName);
fileStorageHandler.StoreStream(dataIdentifier, streamToStore, numberOfBytes, customMetaData, addFileBehaviour, StreamStateBehaviour.OpenNewStreamForReadingAndWriting, StreamStateBehaviour.OpenNewStreamForReadingAndWriting);
}
public static void StoreObject(string fileStorageName, Guid dataIdentifier, IDynamiteXml objectToStore, ICustomMetaData customMetaData, AddFileBehaviour addFileBehaviour)
{
FileStorageHandler fileStorageHandler = FileStorageHandler.Open(fileStorageName);
fileStorageHandler.StoreObject(dataIdentifier, objectToStore, customMetaData, addFileBehaviour, StreamStateBehaviour.OpenNewStreamForReadingAndWriting, StreamStateBehaviour.OpenNewStreamForReadingAndWriting);
}
public void StoreHttpRequest(Guid dataIdentifier, string url, ICustomMetaData customMetaData, AddFileBehaviour addFileBehaviour, string userAgent, StreamStateBehaviour indexStreamStateBehaviour, StreamStateBehaviour dataStreamStateBehaviour)
{
var request = (HttpWebRequest) WebRequest.Create(url);
request.UserAgent = userAgent;
// execute the web request
var response = (HttpWebResponse) request.GetResponse();
using (Stream webStream = response.GetResponseStream())
{
StoreBytes(dataIdentifier, webStream.ReadAllBytes(), customMetaData, addFileBehaviour, indexStreamStateBehaviour, dataStreamStateBehaviour);
}
}
public static void StoreHttpRequest(string fileStorageName, Guid dataIdentifier, string url, ICustomMetaData customMetaData, AddFileBehaviour addFileBehaviour, string userAgent)
{
FileStorageHandler fileStorageHandler = FileStorageHandler.Open(fileStorageName);
fileStorageHandler.StoreHttpRequest(dataIdentifier, url, customMetaData, addFileBehaviour, userAgent, StreamStateBehaviour.OpenNewStreamForReadingAndWriting, StreamStateBehaviour.OpenNewStreamForReadingAndWriting);
}
public static void StoreString(string fileStorageName, Guid dataIdentifier, Encoding encoding, string stringToStore, ICustomMetaData customMetaData, AddFileBehaviour addFileBehaviour)
{
FileStorageHandler fileStorageHandler = FileStorageHandler.Open(fileStorageName);
fileStorageHandler.StoreString(dataIdentifier, encoding, stringToStore, customMetaData, addFileBehaviour, StreamStateBehaviour.OpenNewStreamForReadingAndWriting, StreamStateBehaviour.OpenNewStreamForReadingAndWriting);
}
public static void StoreStream(string fileStorageName, Guid dataIdentifier, Stream streamToStore, int numberOfBytes, ICustomMetaData customMetaData, AddFileBehaviour addFileBehaviour)
{
FileStorageHandler fileStorageHandler = FileStorageHandler.Open(fileStorageName);
fileStorageHandler.StoreStream(dataIdentifier, streamToStore, numberOfBytes, customMetaData, addFileBehaviour, StreamStateBehaviour.OpenNewStreamForReadingAndWriting, StreamStateBehaviour.OpenNewStreamForReadingAndWriting);
}
public MetaDataContainer(ICustomMetaData customMetaData, DateTime creationDateUtc, Int64 binaryDataSizeInBytes)
{
CustomMetaDataString = DynamiteXmlLogic.Serialize(customMetaData);
CreationDateUtc = creationDateUtc;
BinarySizeInBytes = binaryDataSizeInBytes;
}
public void StoreStream(Guid dataIdentifier, Stream streamToStore, int numberOfBytes, ICustomMetaData customMetaData, AddFileBehaviour addFileBehaviour, StreamStateBehaviour indexStreamStateBehaviour, StreamStateBehaviour dataStreamStateBehaviour)
{
using (var binaryReader = new BinaryReader(streamToStore))
{
byte[] fileData = binaryReader.ReadBytes(numberOfBytes);
StoreBytes(dataIdentifier, fileData, customMetaData, addFileBehaviour, indexStreamStateBehaviour, dataStreamStateBehaviour);
}
}
public static void StoreBytes(string fileStorageName, Guid dataIdentifier, byte[] fileData, ICustomMetaData customMetaData, AddFileBehaviour addFileBehaviour)
{
var fileStorageHandler = FileStorageHandler.Open(fileStorageName);
fileStorageHandler.StoreBytes(dataIdentifier, fileData, customMetaData, addFileBehaviour, StreamStateBehaviour.OpenNewStreamForReadingAndWriting, StreamStateBehaviour.OpenNewStreamForReadingAndWriting);
}
public void StoreObject(Guid dataIdentifier, IDynamiteXml objectToStore, ICustomMetaData customMetaData, AddFileBehaviour addFileBehaviour, StreamStateBehaviour indexStreamStateBehaviour, StreamStateBehaviour dataStreamStateBehaviour)
{
string serializedObject = DynamiteXmlLogic.Serialize(objectToStore);
StoreString(dataIdentifier, serializedObject, customMetaData, addFileBehaviour, indexStreamStateBehaviour, dataStreamStateBehaviour);
}
public static void StoreString(string fileStorageName, Guid dataIdentifier, Encoding encoding, string stringToStore, ICustomMetaData customMetaData, AddFileBehaviour addFileBehaviour)
{
var fileStorageHandler = FileStorageHandler.Open(fileStorageName);
fileStorageHandler.StoreString(dataIdentifier, encoding, stringToStore, customMetaData, addFileBehaviour, StreamStateBehaviour.OpenNewStreamForReadingAndWriting, StreamStateBehaviour.OpenNewStreamForReadingAndWriting);
}
public void StoreBytes(Guid dataIdentifier, byte[] fileData, ICustomMetaData customMetaData, AddFileBehaviour addFileBehaviour, StreamStateBehaviour indexStreamStateBehaviour, StreamStateBehaviour dataStreamStateBehaviour)
{
switch (indexStreamStateBehaviour)
{
case StreamStateBehaviour.OpenNewStreamForReadingAndWriting:
//
// open the stream and make a recursive call
//
using (IndexStream = FileStreamFactory.CreateFileStream(IndexFilename, indexStreamStateBehaviour))
{
StoreBytes(dataIdentifier, fileData, customMetaData, addFileBehaviour, StreamStateBehaviour.UseExistingStream, dataStreamStateBehaviour);
}
break;
case StreamStateBehaviour.UseExistingStream:
//
// check state of data file stream
//
switch (dataStreamStateBehaviour)
{
case StreamStateBehaviour.OpenNewStreamForReadingAndWriting:
//
// open the stream and make a recursive call
//
using (DataStream = FileStreamFactory.CreateFileStream(DataFilename, dataStreamStateBehaviour))
{
StoreBytes(dataIdentifier, fileData, customMetaData, addFileBehaviour, indexStreamStateBehaviour, StreamStateBehaviour.UseExistingStream);
}
break;
case StreamStateBehaviour.UseExistingStream:
//
//
//
StoreBytesForExistingStreams(dataIdentifier, fileData, customMetaData, addFileBehaviour);
//
break;
default:
throw new NotSupportedException(string.Format("Unexpected dataStreamStateBehaviour {0}", dataStreamStateBehaviour));
}
//
break;
default:
throw new NotSupportedException(string.Format("Unexpected indexStreamStateBehaviour {0}", indexStreamStateBehaviour));
}
}
public static void StoreObject(string fileStorageName, Guid dataIdentifier, IDynamiteXml objectToStore, ICustomMetaData customMetaData, AddFileBehaviour addFileBehaviour)
{
var fileStorageHandler = FileStorageHandler.Open(fileStorageName);
fileStorageHandler.StoreObject(dataIdentifier, objectToStore, customMetaData, addFileBehaviour, StreamStateBehaviour.OpenNewStreamForReadingAndWriting, StreamStateBehaviour.OpenNewStreamForReadingAndWriting);
}
public MetaDataContainer(ICustomMetaData customMetaData, DateTime creationDateUtc, Int64 binaryDataSizeInBytes)
{
CustomMetaDataString = DynamiteXmlLogic.Serialize(customMetaData);
CreationDateUtc = creationDateUtc;
BinarySizeInBytes = binaryDataSizeInBytes;
}
