public override void Load(Stream stream)
{
TestForEditable();
byte version = stream.ReadNextByte();
switch (version)
{
case 1:
m_directoryMethod = (ArchiveDirectoryMethod)stream.ReadInt32();
m_prefix = stream.ReadString();
m_pendingExtension = stream.ReadString();
m_finalExtension = stream.ReadString();
m_desiredRemainingSpace = stream.ReadInt64();
m_encodingMethod = new EncodingDefinition(stream);
int cnt = stream.ReadInt32();
m_writePath.Clear();
while (cnt > 0)
{
cnt--;
m_writePath.Add(stream.ReadString());
}
cnt = stream.ReadInt32();
m_flags.Clear();
while (cnt > 0)
{
cnt--;
m_flags.Add(stream.ReadGuid());
}
break;
default:
throw new VersionNotFoundException("Unknown Version Code: " + version);
}
}
/// <summary>
/// Creates a <see cref="ArchiveInitializer{TKey,TValue}"/> that will reside on the disk.
/// </summary>
/// <param name="paths">the paths to place the files.</param>
/// <param name="desiredRemainingSpace">The desired free space to leave on the disk before moving to another disk.</param>
/// <param name="directoryMethod">the method for storing files in a directory.</param>
/// <param name="encodingMethod">the encoding method to use for the archive file.</param>
/// <param name="prefix">the prefix to affix to the files created.</param>
/// <param name="pendingExtension">the extension file name</param>
/// <param name="finalExtension">the final extension to specify</param>
/// <param name="flags">flags to include in the archive that is created.</param>
/// <returns></returns>
public void ConfigureOnDisk(IEnumerable <string> paths, long desiredRemainingSpace, ArchiveDirectoryMethod directoryMethod, EncodingDefinition encodingMethod, string prefix, string pendingExtension, string finalExtension, params Guid[] flags)
{
TestForEditable();
Initialize();
DirectoryMethod = directoryMethod;
PendingExtension = pendingExtension;
FinalExtension = finalExtension;
Flags.AddRange(flags);
Prefix = prefix;
WritePath.AddRange(paths);
DesiredRemainingSpace = desiredRemainingSpace;
EncodingMethod = encodingMethod;
}
private void Initialize()
{
m_directoryMethod = ArchiveDirectoryMethod.TopDirectoryOnly;
m_isMemoryArchive = false;
m_prefix = string.Empty;
m_fileExtension = ".d2i";
m_desiredRemainingSpace = 5 * 1024 * 1024 * 1024L; //5GB
m_encodingMethod = EncodingDefinition.FixedSizeCombinedEncoding;
m_writePath = new ImmutableList <string>((x) =>
{
PathHelpers.ValidatePathName(x);
return(x);
});
m_flags = new ImmutableList <Guid>();
}
/// <summary>
/// Gets a database config.
/// </summary>
public AdvancedServerDatabaseConfig(string databaseName, string mainPath, bool supportsWriting)
{
m_supportsWriting = supportsWriting;
m_databaseName = databaseName;
m_mainPath = mainPath;
m_intermediateFileExtension = ".d2i";
m_finalFileExtension = ".d2";
m_importPaths = new List <string>();
m_finalWritePaths = new List <string>();
m_archiveEncodingMethod = EncodingDefinition.FixedSizeCombinedEncoding;
m_streamingEncodingMethods = new List <EncodingDefinition>();
m_targetFileSize = 2 * 1024 * 1024 * 1024L;
m_stagingCount = 3;
m_directoryMethod = ArchiveDirectoryMethod.TopDirectoryOnly;
m_diskFlushInterval = 10000;
m_cacheFlushInterval = 100;
}
private void FastMigration(object state)
{
try
{
long operationStartTime = DateTime.UtcNow.Ticks;
Dictionary <string, string> parameters = state as Dictionary <string, string>;
if ((object)parameters == null)
{
throw new ArgumentNullException("state", "Could not interpret thread state as parameters dictionary");
}
ClearUpdateMessages();
ShowUpdateMessage("Scanning source files...");
if (!Directory.Exists(parameters["sourceFilesLocation"]))
{
throw new DirectoryNotFoundException(string.Format("Source directory \"{0}\" not found.", parameters["sourceFilesLocation"]));
}
IEnumerable <string> sourceFiles = Directory.EnumerateFiles(parameters["sourceFilesLocation"], "*.d", SearchOption.TopDirectoryOnly);
if (Directory.Exists(parameters["sourceFilesOffloadLocation"]))
{
sourceFiles = sourceFiles.Concat(Directory.EnumerateFiles(parameters["sourceFilesOffloadLocation"], "*.d", SearchOption.TopDirectoryOnly));
}
int maxThreads, methodIndex;
if (!int.TryParse(parameters["maxThreads"], out maxThreads))
{
maxThreads = m_defaultMaxThreads;
}
if (!int.TryParse(parameters["directoryNamingMethod"], out methodIndex) || !Enum.IsDefined(typeof(ArchiveDirectoryMethod), methodIndex))
{
methodIndex = (int)ArchiveDirectoryMethod.YearThenMonth;
}
ArchiveDirectoryMethod method = (ArchiveDirectoryMethod)methodIndex;
HistorianFileEncoding encoder = new HistorianFileEncoding();
string[] sourceFileNames = sourceFiles.ToArray();
string destinationPath = parameters["destinationFilesLocation"];
string instanceName = parameters["instanceName"];
bool ignoreDuplicates = parameters["ignoreDuplicates"].ParseBoolean();
long totalProcessedPoints = 0;
int processedFiles = 0;
SetProgressMaximum(sourceFileNames.Length);
using (StreamWriter duplicateDataOutput = File.CreateText(FilePath.GetAbsolutePath("DuplicateData.txt")))
{
Parallel.ForEach(sourceFileNames, new ParallelOptions
{
MaxDegreeOfParallelism = maxThreads
},
(sourceFileName, loopState) =>
{
ShowUpdateMessage("Migrating \"{0}\"...", FilePath.GetFileName(sourceFileName));
long fileConversionStartTime = DateTime.UtcNow.Ticks;
long migratedPoints;
if (ignoreDuplicates)
{
// Migrate using SortedTreeFileSimpleWriter.CreateNonSequential() with raw unsorted historian file read
migratedPoints = ConvertArchiveFile.ConvertVersion1FileIgnoreDuplicates(
sourceFileName,
GetDestinationFileName(sourceFileName, instanceName, destinationPath, method),
encoder.EncodingMethod);
}
else
{
// Migrate using SortedTreeFileSimpleWriter.Create() with API sorted historian file read with duplicate handling
using (GSFHistorianStream stream = new GSFHistorianStream(this, sourceFileName, instanceName, duplicateDataOutput))
{
string completeFileName = GetDestinationFileName(stream.ArchiveFile, sourceFileName, instanceName, destinationPath, method);
string pendingFileName = Path.Combine(FilePath.GetDirectoryName(completeFileName), FilePath.GetFileNameWithoutExtension(completeFileName) + ".~d2i");
SortedTreeFileSimpleWriter <HistorianKey, HistorianValue> .Create(pendingFileName, completeFileName, 4096, null, encoder.EncodingMethod, stream);
migratedPoints = stream.PointCount;
}
}
Ticks totalTime = DateTime.UtcNow.Ticks - fileConversionStartTime;
ShowUpdateMessage(
"{0}Migrated {1:#,##0} points for last file in {2} at {3:#,##0} points per second.{0}",
Environment.NewLine,
migratedPoints,
totalTime.ToElapsedTimeString(3),
migratedPoints / totalTime.ToSeconds());
Interlocked.Increment(ref processedFiles);
Interlocked.Add(ref totalProcessedPoints, migratedPoints);
UpdateProgressBar(processedFiles);
if (m_formClosing)
{
loopState.Break();
}
});
}
if (m_formClosing)
{
ShowUpdateMessage("Migration canceled.");
UpdateProgressBar(0);
}
else
{
Ticks totalTime = DateTime.UtcNow.Ticks - operationStartTime;
ShowUpdateMessage("*** Migration Complete ***");
ShowUpdateMessage("Total migration time {0} at {1:#,##0} points per second.", totalTime.ToElapsedTimeString(3), totalProcessedPoints / totalTime.ToSeconds());
UpdateProgressBar(sourceFileNames.Length);
}
}
catch (Exception ex)
{
ShowUpdateMessage("Failure during migration: {0}", ex.Message);
}
finally
{
m_operationStarted = false;
}
}
private string GetDestinationFileName(ArchiveFile file, string sourceFileName, string instanceName, string destinationPath, ArchiveDirectoryMethod method)
{
string destinationFileName = FilePath.GetFileNameWithoutExtension(sourceFileName) + ".d2";
string archiveFileName = FilePath.GetFileName(sourceFileName);
string archiveInstanceName = archiveFileName.Substring(0, archiveFileName.LastIndexOf("_archive", StringComparison.OrdinalIgnoreCase));
DateTime startTime, endTime;
// Use source archive instance name for destination database instance name if not specified
if (string.IsNullOrEmpty(instanceName))
{
instanceName = archiveInstanceName;
}
startTime = file.Fat.FileStartTime.ToDateTime();
endTime = file.Fat.FileEndTime.ToDateTime();
destinationFileName = instanceName.ToLower() + "-" + startTime.ToString("yyyy-MM-dd HH.mm.ss.fff") + "_to_" + endTime.ToString("yyyy-MM-dd HH.mm.ss.fff") + "-" + DateTime.UtcNow.Ticks + ".d2";
switch (method)
{
case ArchiveDirectoryMethod.Year:
destinationPath = Path.Combine(destinationPath, string.Format("{0}\\", startTime.Year));
break;
case ArchiveDirectoryMethod.YearMonth:
destinationPath = Path.Combine(destinationPath, string.Format("{0}{1:00}\\", startTime.Year, startTime.Month));
break;
case ArchiveDirectoryMethod.YearThenMonth:
destinationPath = Path.Combine(destinationPath, string.Format("{0}\\{1:00}\\", startTime.Year, startTime.Month));
break;
}
if (!Directory.Exists(destinationPath))
{
Directory.CreateDirectory(destinationPath);
}
return(Path.Combine(destinationPath, destinationFileName));
}
private string GetDestinationFileName(string sourceFileName, string instanceName, string destinationPath, ArchiveDirectoryMethod method)
{
using (ArchiveFile file = OpenArchiveFile(sourceFileName, ref instanceName))
{
return(GetDestinationFileName(file, sourceFileName, instanceName, destinationPath, method));
}
}
