private void GetStreams()
{
List <DatabaseStream> streams = DatabaseHandler.Streams;
List <DatabaseStream> streamsValid = new List <DatabaseStream>();
List <DatabaseScheme> schemes = DatabaseHandler.Schemes;
foreach (DatabaseStream stream in streams)
{
foreach (DatabaseScheme scheme in schemes)
{
bool template = mode == Mode.TRAIN || mode == Mode.COMPLETE;
if (getTrainer(stream, scheme, template).Count > 0)
{
streamsValid.Add(stream);
break;
}
}
}
StreamsBox.ItemsSource = streamsValid;
if (StreamsBox.Items.Count > 0)
{
DatabaseStream stream = ((List <DatabaseStream>)StreamsBox.ItemsSource).Find(s => s.Name == Properties.Settings.Default.CMLDefaultStream);
if (stream != null)
{
StreamsBox.SelectedItem = stream;
}
if (StreamsBox.SelectedItem == null)
{
StreamsBox.SelectedIndex = 0;
}
StreamsBox.ScrollIntoView(StreamsBox.SelectedItem);
}
}
private List <Chain> getChains(DatabaseStream stream)
{
List <Chain> chains = new List <Chain>();
string chainDir = Properties.Settings.Default.CMLDirectory +
"\\" + Defaults.CML.ChainFolderName +
"\\" + stream.Type;
if (Directory.Exists(chainDir))
{
string[] chainFiles = Directory.GetFiles(chainDir, "*." + Defaults.CML.ChainFileExtension, SearchOption.AllDirectories);
foreach (string chainFile in chainFiles)
{
Chain chain = new Chain()
{
Path = chainFile
};
if (parseChainFile(ref chain))
{
chains.Add(chain);
}
}
}
return(chains);
}
private async Task CopyDatabaseTo(string pathto)
{
DatabaseStream.Position = 0;
using (var tostream = File.OpenWrite(pathto))
{
await DatabaseStream.CopyToAsync(tostream);
tostream.SetLength(DatabaseStream.Length);
}
}
public async Task SaveData <T>(T data) where T : IDatabaseEntry
{
await StreamSemaphore.WaitAsync();
DatabaseStream.Position = 0;
var tempPath = Path.Combine(Path.GetTempPath(), "MatchRecorder");
var tempFilePath = Path.Combine(tempPath, SharedSettings.DatabaseFile);
if (!Directory.Exists(tempPath))
{
Directory.CreateDirectory(tempPath);
}
await CopyDatabaseTo(tempFilePath);
DatabaseStream.Position = 0;
var typeName = typeof(T).Name;
var jsonPath = $"{typeof( T ).Name}['{data.DatabaseIndex}']";
//jesus christ what a pain in the ass, seems like every bloody thing here closes the stream
//so this looks needlessly UGLY
using (var fileCopyStream = File.OpenRead(tempFilePath))
using (var streamReader = new StreamReader(fileCopyStream))
using (var reader = new JsonTextReader(streamReader))
using (var streamWriter = new StreamWriter(DatabaseStream, Encoding.UTF8, 1024, true))
using (var writer = new JsonTextWriter(streamWriter)
{
Formatting = Formatting.Indented,
CloseOutput = false
})
{
bool wroteData = false;
await writer.WriteStartObjectAsync();
//go through all the top properties, MatchData etc
if (!wroteData)
{
}
await writer.WriteEndObjectAsync();
DatabaseStream.SetLength(DatabaseStream.Position);
await writer.FlushAsync();
}
StreamSemaphore.Release();
}
private void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing)
{
DatabaseStream?.Dispose();
DatabaseStream = null;
}
disposedValue = true;
}
}
private void EditStream_Click(object sender, RoutedEventArgs e)
{
if (StreamsBox.SelectedItem != null)
{
string name = (string)StreamsBox.SelectedItem;
DatabaseStream stream = new DatabaseStream()
{
Name = name
};
if (DatabaseHandler.GetStream(ref stream))
{
Dictionary <string, UserInputWindow.Input> input = new Dictionary <string, UserInputWindow.Input>();
input["name"] = new UserInputWindow.Input()
{
Label = "Name", DefaultValue = stream.Name
};
input["fileExt"] = new UserInputWindow.Input()
{
Label = "File extension", DefaultValue = stream.FileExt
};
input["type"] = new UserInputWindow.Input()
{
Label = "Type", DefaultValue = stream.Type
};
input["sr"] = new UserInputWindow.Input()
{
Label = "Sample rate", DefaultValue = stream.SampleRate.ToString()
};
UserInputWindow dialog = new UserInputWindow("Edit stream type", input);
dialog.ShowDialog();
if (dialog.DialogResult == true)
{
stream.Name = dialog.Result("name");
stream.FileExt = dialog.Result("fileExt");
stream.Type = dialog.Result("type");
double sr = 25.0;
double.TryParse(dialog.Result("sr"), out sr);
stream.SampleRate = sr;
if (DatabaseHandler.UpdateStream(name, stream))
{
GetStreamTypes(name);
}
}
}
}
}
public void GetChains()
{
ChainsBox.Items.Clear();
if (StreamsBox.SelectedItem != null)
{
DatabaseStream stream = (DatabaseStream)StreamsBox.SelectedItem;
List <Chain> chains = getChains(stream);
foreach (Chain chain in chains)
{
ChainsBox.Items.Add(chain);
}
}
if (ChainsBox.Items.Count > 0)
{
Chain chain = null;
foreach (Chain c in ChainsBox.Items)
{
if (c.Name == Properties.Settings.Default.CMLDefaultChain)
{
chain = c;
break;
}
}
if (chain != null)
{
ChainsBox.SelectedItem = chain;
}
if (ChainsBox.SelectedItem == null)
{
ChainsBox.SelectedIndex = 0;
}
}
if (ChainsBox.SelectedItem != null)
{
Chain chain = (Chain)ChainsBox.SelectedItem;
ChainPathLabel.Content = chain.Path;
LeftContextTextBox.Text = chain.LeftContext;
FrameStepTextBox.Text = chain.FrameStep;
RightContextTextBox.Text = chain.RightContext;
}
}
private void AddStream_Click(object sender, RoutedEventArgs e)
{
Dictionary <string, UserInputWindow.Input> input = new Dictionary <string, UserInputWindow.Input>();
input["name"] = new UserInputWindow.Input()
{
Label = "Name", DefaultValue = ""
};
input["fileExt"] = new UserInputWindow.Input()
{
Label = "File extension", DefaultValue = ""
};
input["type"] = new UserInputWindow.Input()
{
Label = "Type", DefaultValue = ""
};
input["sr"] = new UserInputWindow.Input()
{
Label = "Sample rate", DefaultValue = ""
};
UserInputWindow dialog = new UserInputWindow("Add new stream type", input);
dialog.ShowDialog();
if (dialog.DialogResult == true)
{
string name = dialog.Result("name");
string fileExt = dialog.Result("fileExt");
string type = dialog.Result("type");
double sr = 25.0;
double.TryParse(dialog.Result("sr"), out sr);
DatabaseStream streamType = new DatabaseStream()
{
Name = name, FileExt = fileExt, Type = type, SampleRate = sr
};
if (DatabaseHandler.AddStream(streamType))
{
GetStreamTypes(dialog.Result("name"));
}
}
}
public void CanSerialize()
{
var table = new DataTable("User");
table.Columns.Add("Id", typeof(int));
table.Columns.Add("Name", typeof(string));
table.Columns.Add("Salary", typeof(decimal));
var row = table.NewRow();
row["Id"] = 1;
row["Name"] = "John Doe";
row["Salary"] = 15.5m;
table.Rows.Add(row);
var dbStream = new DatabaseStream(table.CreateDataReader());
String EmpData;
dbStream.dtReader.Read();
EmpData = dbStream.dtReader[0].ToString() + "," + dbStream.dtReader[1].ToString() + "," + dbStream.dtReader[2].ToString();
byte[] bytes = Encoding.ASCII.GetBytes(EmpData.ToString());
int numBytesToRead = (int)EmpData.Length;
int numBytesRead = 0;
if (dbStream.Read(bytes, numBytesRead, numBytesRead) == 1)
{
byte[] XMLSerial = dbStream.xmlBytes;
}
Console.Write("Success");
}
private List <Trainer> getTrainer(DatabaseStream stream, DatabaseScheme scheme, bool isTemplate)
{
List <Trainer> trainers = new List <Trainer>();
if (scheme.Type == AnnoScheme.TYPE.CONTINUOUS)
{
if (stream.SampleRate != scheme.SampleRate)
{
return(trainers);
}
}
string streamName = "";
string[] streamParts = stream.Name.Split('.');
if (streamParts.Length <= 1)
{
streamName = stream.Name;
}
else
{
streamName = streamParts[1];
for (int i = 2; i < streamParts.Length; i++)
{
streamName += "." + streamParts[i];
}
}
string trainerDir = null;
if (isTemplate)
{
trainerDir = Properties.Settings.Default.CMLDirectory + "\\" +
Defaults.CML.ModelsFolderName + "\\" +
Defaults.CML.ModelsTemplatesFolderName + "\\" +
scheme.Type.ToString().ToLower() + "\\" +
stream.Type;
}
else
{
trainerDir = Properties.Settings.Default.CMLDirectory + "\\" +
Defaults.CML.ModelsFolderName + "\\" +
Defaults.CML.ModelsTrainerFolderName + "\\" +
scheme.Type.ToString().ToLower() + "\\" +
scheme.Name + "\\" +
stream.Type + "{" +
streamName + "}\\";
}
if (Directory.Exists(trainerDir))
{
string[] searchDirs = Directory.GetDirectories(trainerDir);
foreach (string searchDir in searchDirs)
{
string[] trainerFiles = Directory.GetFiles(searchDir, "*." + Defaults.CML.TrainerFileExtension);
foreach (string trainerFile in trainerFiles)
{
Trainer trainer = new Trainer()
{
Path = trainerFile
};
if (parseTrainerFile(ref trainer, isTemplate))
{
trainers.Add(trainer);
}
}
}
}
return(trainers);
}
private void GetTrainers()
{
if (RolesBox.SelectedItem == null || AnnotatorsBox.SelectedItem == null || SchemesBox.SelectedItem == null)
{
return;
}
TrainerPathComboBox.ItemsSource = null;
if (StreamsBox.SelectedItem != null)
{
DatabaseStream stream = (DatabaseStream)StreamsBox.SelectedItem;
DatabaseScheme scheme = (DatabaseScheme)SchemesBox.SelectedItem;
if (scheme != null)
{
bool isDiscrete = scheme.Type == AnnoScheme.TYPE.DISCRETE;
FillGapCheckBox.IsEnabled = isDiscrete;
FillGapTextBox.IsEnabled = isDiscrete;
RemoveLabelCheckBox.IsEnabled = isDiscrete;
RemoveLabelTextBox.IsEnabled = isDiscrete;
bool template = mode == Mode.TRAIN || mode == Mode.COMPLETE;
List <Trainer> trainers = getTrainer(stream, scheme, template);
if (trainers.Count > 0)
{
TrainerPathComboBox.ItemsSource = trainers;
}
}
}
if (TrainerPathComboBox.Items.Count > 0)
{
Trainer trainer = ((List <Trainer>)TrainerPathComboBox.ItemsSource).Find(t => t.Name == Properties.Settings.Default.CMLDefaultTrainer);
if (trainer != null)
{
TrainerPathComboBox.SelectedItem = trainer;
}
else
{
TrainerPathComboBox.SelectedIndex = 0;
}
}
if (TrainerPathComboBox.SelectedItem != null)
{
Trainer trainer = (Trainer)TrainerPathComboBox.SelectedItem;
LeftContextTextBox.Text = trainer.LeftContext;
RightContextTextBox.Text = trainer.RightContext;
BalanceComboBox.SelectedIndex = 0;
if (trainer.Balance.ToLower() == "under")
{
BalanceComboBox.SelectedIndex = 1;
}
else if (trainer.Balance.ToLower() == "over")
{
BalanceComboBox.SelectedIndex = 2;
}
string database = "";
if (DatabasesBox.SelectedItem != null)
{
database = DatabasesBox.SelectedItem.ToString();
}
TrainerNameTextBox.Text = mode == Mode.COMPLETE ? Path.GetFileName(tempTrainerPath) : database;
TrainerPathLabel.Content = trainer.Path;
}
}
private void GetStreams(DatabaseStream selected = null)
{
List <DatabaseStream> streams = DatabaseHandler.Streams;
List <DatabaseStream> streamsValid = new List <DatabaseStream>();
foreach (DatabaseStream stream in streams)
{
if (mode == Mode.MERGE)
{
if (stream.Type == Defaults.CML.StreamTypeNameFeature)
{
streamsValid.Add(stream);
}
}
else
{
if (getChains(stream).Count > 0)
{
streamsValid.Add(stream);
}
}
}
List <DatabaseStream> selectedStreams = null;
if (mode == Mode.MERGE)
{
selectedStreams = new List <DatabaseStream>();
foreach (DatabaseStream s in StreamsBox.SelectedItems)
{
selectedStreams.Add(s);
}
}
if (StreamsBox.HasItems)
{
StreamsBox.ItemsSource = null;
}
StreamsBox.ItemsSource = streamsValid;
if (mode == Mode.MERGE && selectedStreams != null)
{
foreach (DatabaseStream s in selectedStreams)
{
StreamsBox.SelectedItems.Add(s);
}
}
if (mode == Mode.EXTRACT && StreamsBox.Items.Count > 0)
{
DatabaseStream stream = ((List <DatabaseStream>)StreamsBox.ItemsSource).Find(s => s.Name == Properties.Settings.Default.CMLDefaultStream);
if (stream != null)
{
StreamsBox.SelectedItem = stream;
}
if (StreamsBox.SelectedItem == null)
{
StreamsBox.SelectedIndex = 0;
}
if (selected != null)
{
StreamsBox.SelectedItem = selected;
}
StreamsBox.ScrollIntoView(StreamsBox.SelectedItem);
}
}
private void Extract()
{
if (ChainsBox.SelectedItem == null)
{
MessageTools.Warning("select a chain first");
return;
}
string featureName = FeatureNameTextBox.Text;
if (featureName == "" || featureName.IndexOfAny(Path.GetInvalidFileNameChars()) >= 0)
{
MessageTools.Warning("not a valid feature name");
return;
}
Chain chain = (Chain)ChainsBox.SelectedItem;
bool force = ForceCheckBox.IsChecked.Value;
if (!File.Exists(chain.Path))
{
MessageTools.Warning("file does not exist '" + chain.Path + "'");
return;
}
string database = DatabaseHandler.DatabaseName;
var sessions = SessionsBox.SelectedItems;
var roles = RolesBox.SelectedItems;
DatabaseStream stream = (DatabaseStream)StreamsBox.SelectedItem;
string leftContext = LeftContextTextBox.Text;
string frameStep = FrameStepTextBox.Text;
string rightContext = RightContextTextBox.Text;
int nParallel = 1;
int.TryParse(NParallelTextBox.Text, out nParallel);
logTextBox.Text = "";
// prepare lists
int nFiles = 0;
using (StreamWriter fileListIn = new StreamWriter(tempInListPath))
{
using (StreamWriter fileListOut = new StreamWriter(tempOutListPath))
{
foreach (DatabaseSession session in sessions)
{
foreach (DatabaseRole role in roles)
{
string fromPath = Properties.Settings.Default.DatabaseDirectory + "\\"
+ database + "\\"
+ session.Name + "\\"
+ role.Name + "." + stream.Name + "." + stream.FileExt;
string toPath = Path.GetDirectoryName(fromPath) + "\\"
+ role.Name + "." + featureName + ".stream";
if (force || !File.Exists(toPath))
{
nFiles++;
fileListIn.WriteLine(fromPath);
fileListOut.WriteLine(toPath);
}
else
{
logTextBox.Text += "skip " + fromPath + "\n";
}
}
}
}
}
// start feature extraction
Chain selectedChain = (Chain)ChainsBox.SelectedItem;
DatabaseStream selectedStream = (DatabaseStream)StreamsBox.SelectedItem;
if (nFiles > 0)
{
string type = Defaults.CML.StreamTypeNameFeature;
string name = featureName;
string ext = "stream";
double sr = frameStepToSampleRate(frameStep, stream.SampleRate);
DatabaseStream streamType = new DatabaseStream()
{
Name = name, Type = type, FileExt = ext, SampleRate = sr
};
DatabaseHandler.AddStream(streamType);
try
{
logTextBox.Text += handler.CMLExtractFeature(chain.Path, nParallel, tempInListPath, tempOutListPath, frameStep, leftContext, rightContext);
}
catch (Exception e) { MessageBox.Show("There was an error in the feature extaction pipeline: " + e); }
}
File.Delete(tempInListPath);
File.Delete(tempOutListPath);
GetStreams(selectedStream);
foreach (Chain item in ChainsBox.Items)
{
if (item.Name == selectedChain.Name)
{
ChainsBox.SelectedItem = item;
break;
}
}
}
private void UpdateFeatureName()
{
string name = "";
if (mode == Mode.MERGE)
{
var streams = StreamsBox.SelectedItems;
string streamList = "";
HashSet <string> sourceNames = new HashSet <string>();
HashSet <string> featureNames = new HashSet <string>();
double sampleRate = 0;
foreach (DatabaseStream stream in streams)
{
string[] tokens = stream.Name.Split(new char[] { '.' }, 2);
sourceNames.Add(tokens[0]);
if (tokens.Length > 1)
{
featureNames.Add(tokens[1]);
}
if (streamList == "")
{
sampleRate = stream.SampleRate;
streamList = stream.Name;
}
else
{
streamList += ";" + stream.Name;
}
}
string[] sourceArray;
sourceArray = sourceNames.ToArray();
Array.Sort(sourceArray);
sourceNames.Clear();
sourceNames.UnionWith(sourceArray);
string medias = "";
foreach (string media in sourceNames)
{
if (medias == "")
{
medias = media;
}
else
{
medias += "+" + media;
}
}
string[] featureStreams;
featureStreams = featureNames.ToArray();
Array.Sort(featureStreams);
featureNames.Clear();
featureNames.UnionWith(featureStreams);
string features = "";
foreach (string feature in featureNames)
{
if (features == "")
{
features = feature;
}
else
{
features += "+" + feature;
}
}
name = medias + "." + features;
}
else
{
if (StreamsBox.SelectedItem != null & ChainsBox.SelectedItem != null && StreamsBox.SelectedItem != null)
{
DatabaseStream stream = (DatabaseStream)StreamsBox.SelectedItem;
Chain chain = (Chain)ChainsBox.SelectedItem;
string leftContext = LeftContextTextBox.Text;
string frameStep = FrameStepTextBox.Text;
string rightContext = RightContextTextBox.Text;
string streamMeta = "[" + leftContext + "," + frameStep + "," + rightContext + "]";
name = stream.Name + "." + chain.Name + streamMeta;;
}
}
FeatureNameTextBox.Text = name;
}
private void Apply_Click(object sender, RoutedEventArgs e)
{
Trainer trainer = (Trainer)TrainerPathComboBox.SelectedItem;
bool force = mode == Mode.COMPLETE || ForceCheckBox.IsChecked.Value;
if (!File.Exists(trainer.Path))
{
MessageTools.Warning("file does not exist '" + trainer.Path + "'");
return;
}
string database = DatabaseHandler.DatabaseName;
DatabaseStream stream = (DatabaseStream)StreamsBox.SelectedItem;
string sessionList = "";
var sessions = SessionsBox.SelectedItems;
foreach (DatabaseSession session in sessions)
{
if (sessionList == "")
{
sessionList += session.Name;
}
else
{
sessionList += ";" + session.Name;
}
}
DatabaseScheme scheme = (DatabaseScheme)SchemesBox.SelectedItem;
string rolesList = "";
var roles = RolesBox.SelectedItems;
foreach (DatabaseRole role in roles)
{
if (rolesList == "")
{
rolesList += role.Name;
}
else
{
rolesList += ";" + role.Name;
}
}
DatabaseAnnotator annotator = (DatabaseAnnotator)AnnotatorsBox.SelectedItem;
string trainerLeftContext = LeftContextTextBox.Text;
string trainerRightContext = RightContextTextBox.Text;
string trainerBalance = ((ComboBoxItem)BalanceComboBox.SelectedItem).Content.ToString();
logTextBox.Text = "";
if (mode == Mode.TRAIN ||
mode == Mode.COMPLETE)
{
string streamName = "";
string[] streamParts = stream.Name.Split('.');
if (streamParts.Length <= 1)
{
streamName = stream.Name;
}
else
{
streamName = streamParts[1];
for (int i = 2; i < streamParts.Length; i++)
{
streamName += "." + streamParts[i];
}
}
string trainerDir = Properties.Settings.Default.CMLDirectory + "\\" +
Defaults.CML.ModelsFolderName + "\\" +
Defaults.CML.ModelsTrainerFolderName + "\\" +
scheme.Type.ToString().ToLower() + "\\" +
scheme.Name + "\\" +
stream.Type + "{" +
streamName + "}\\" +
trainer.Name + "\\";
Directory.CreateDirectory(trainerDir);
string trainerName = TrainerNameTextBox.Text == "" ? trainer.Name : TrainerNameTextBox.Text;
string trainerOutPath = mode == Mode.COMPLETE ? tempTrainerPath : trainerDir + trainerName;
if (force || !File.Exists(trainerOutPath + ".trainer"))
{
try
{
logTextBox.Text += handler.CMLTrainModel(trainer.Path,
trainerOutPath,
Properties.Settings.Default.DatabaseDirectory,
Properties.Settings.Default.DatabaseAddress,
Properties.Settings.Default.MongoDBUser,
MainHandler.Decode(Properties.Settings.Default.MongoDBPass),
database,
sessionList,
scheme.Name,
rolesList,
annotator.Name,
stream.Name,
trainerLeftContext,
trainerRightContext,
trainerBalance,
mode == Mode.COMPLETE);
}
catch (Exception ex)
{
logTextBox.Text += ex;
}
}
else
{
logTextBox.Text += "skip " + trainerOutPath + "\n";
}
}
if (mode == Mode.PREDICT ||
mode == Mode.COMPLETE)
{
if (true || force)
{
double confidence = -1.0;
if (ConfidenceCheckBox.IsChecked == true && ConfidenceTextBox.IsEnabled)
{
double.TryParse(ConfidenceTextBox.Text, out confidence);
Properties.Settings.Default.CMLDefaultConf = confidence;
}
double minGap = 0.0;
if (FillGapCheckBox.IsChecked == true && FillGapTextBox.IsEnabled)
{
double.TryParse(FillGapTextBox.Text, out minGap);
Properties.Settings.Default.CMLDefaultGap = minGap;
}
double minDur = 0.0;
if (RemoveLabelCheckBox.IsChecked == true && RemoveLabelTextBox.IsEnabled)
{
double.TryParse(RemoveLabelTextBox.Text, out minDur);
Properties.Settings.Default.CMLDefaultMinDur = minDur;
}
Properties.Settings.Default.Save();
try
{
logTextBox.Text += handler.CMLPredictAnnos(mode == Mode.COMPLETE ? tempTrainerPath : trainer.Path,
Properties.Settings.Default.DatabaseDirectory,
Properties.Settings.Default.DatabaseAddress,
Properties.Settings.Default.MongoDBUser,
MainHandler.Decode(Properties.Settings.Default.MongoDBPass),
database,
sessionList,
scheme.Name,
rolesList,
annotator.Name,
stream.Name,
trainerLeftContext,
trainerRightContext,
confidence,
minGap,
minDur,
mode == Mode.COMPLETE);
}
catch (Exception ex)
{
logTextBox.Text += ex;
}
}
}
if (mode == Mode.EVALUATE)
{
string evalOutPath = Properties.Settings.Default.CMLDirectory + "\\" + Path.GetFileNameWithoutExtension(Path.GetRandomFileName());
try
{
logTextBox.Text += handler.CMLEvaluateModel(evalOutPath,
trainer.Path,
Properties.Settings.Default.DatabaseDirectory,
Properties.Settings.Default.DatabaseAddress,
Properties.Settings.Default.MongoDBUser,
MainHandler.Decode(Properties.Settings.Default.MongoDBPass),
database,
sessionList,
scheme.Name,
rolesList,
annotator.Name,
stream.Name,
LosoCheckBox.IsChecked.Value);
if (File.Exists(evalOutPath))
{
ConfmatWindow confmat = new ConfmatWindow(evalOutPath);
confmat.ShowDialog();
File.Delete(evalOutPath);
}
}
catch (Exception ex)
{
logTextBox.Text += ex;
}
}
if (mode == Mode.COMPLETE)
{
handler.ReloadAnnoTierFromDatabase(AnnoTierStatic.Selected, false);
var dir = new DirectoryInfo(Path.GetDirectoryName(tempTrainerPath));
foreach (var file in dir.EnumerateFiles(Path.GetFileName(tempTrainerPath) + "*"))
{
file.Delete();
}
Close();
}
}
/*
* Test the external file database with or without environment.
* 1. Config and open the environment;
* 2. Verify the environment external file configs;
* 3. Config and open the database;
* 4. Verify the database external file configs;
* 5. Insert and verify some external file data by database methods;
* 6. Insert some external file data by cursor, update it and verify
* the update by database stream and cursor;
* 7. Verify the stats;
* 8. Close all handles.
* If "blobdbt" is true, set the data DatabaseEntry.ExternalFile as
* true, otherwise make the data DatabaseEntry reach the external file
* threshold in size.
*/
void TestBlobHashDatabase(uint env_threshold, string env_blobdir,
uint db_threshold, string db_blobdir, bool blobdbt)
{
if (env_threshold == 0 && db_threshold == 0)
{
return;
}
string hashDBName =
testHome + "/" + testName + ".db";
Configuration.ClearDir(testHome);
HashDatabaseConfig cfg = new HashDatabaseConfig();
cfg.Creation = CreatePolicy.ALWAYS;
string blrootdir = "__db_bl";
// Open the environment and verify the external file config.
if (env_threshold > 0)
{
DatabaseEnvironmentConfig envConfig =
new DatabaseEnvironmentConfig();
envConfig.AutoCommit = true;
envConfig.Create = true;
envConfig.UseMPool = true;
envConfig.UseLogging = true;
envConfig.UseTxns = true;
envConfig.UseLocking = true;
envConfig.ExternalFileThreshold = env_threshold;
if (env_blobdir != null)
{
envConfig.ExternalFileDir = env_blobdir;
blrootdir = env_blobdir;
}
DatabaseEnvironment env = DatabaseEnvironment.Open(
testHome, envConfig);
if (env_blobdir == null)
{
Assert.IsNull(env.ExternalFileDir);
}
else
{
Assert.AreEqual(0,
env.ExternalFileDir.CompareTo(env_blobdir));
}
Assert.AreEqual(env_threshold,
env.ExternalFileThreshold);
cfg.Env = env;
hashDBName = testName + ".db";
}
// Open the database and verify the external file config.
if (db_threshold > 0)
{
cfg.ExternalFileThreshold = db_threshold;
}
if (db_blobdir != null)
{
cfg.ExternalFileDir = db_blobdir;
/*
* The external file directory setting in the database
* is effective only when it is opened without
* an environment.
*/
if (cfg.Env == null)
{
blrootdir = db_blobdir;
}
}
HashDatabase db = HashDatabase.Open(hashDBName, cfg);
Assert.AreEqual(
db_threshold > 0 ? db_threshold : env_threshold,
db.ExternalFileThreshold);
if (db_blobdir == null && cfg.Env == null)
{
Assert.IsNull(db.ExternalFileDir);
}
else
{
Assert.AreEqual(0,
db.ExternalFileDir.CompareTo(blrootdir));
}
// Insert and verify some external file data by database
// methods.
string[] records = { "a", "b", "c", "d", "e", "f", "g", "h",
"i", "j", "k", "l", "m", "n", "o", "p","q","r", "s",
"t", "u", "v", "w", "x", "y", "z" };
DatabaseEntry kdbt = new DatabaseEntry();
DatabaseEntry ddbt = new DatabaseEntry();
byte[] kdata, ddata;
string str;
KeyValuePair <DatabaseEntry, DatabaseEntry> pair;
ddbt.ExternalFile = blobdbt;
Assert.AreEqual(blobdbt, ddbt.ExternalFile);
for (int i = 0; i < records.Length; i++)
{
kdata = BitConverter.GetBytes(i);
str = records[i];
if (!blobdbt)
{
for (int j = 0; j < db_threshold; j++)
{
str = str + records[i];
}
}
ddata = Encoding.ASCII.GetBytes(str);
kdbt.Data = kdata;
ddbt.Data = ddata;
db.Put(kdbt, ddbt);
try
{
pair = db.Get(kdbt);
}
catch (DatabaseException)
{
db.Close();
if (cfg.Env != null)
{
cfg.Env.Close();
}
throw new TestException();
}
Assert.AreEqual(ddata, pair.Value.Data);
}
/*
* Insert some external file data by cursor, update it and
* verify the update by database stream.
*/
kdata = BitConverter.GetBytes(records.Length);
ddata = Encoding.ASCII.GetBytes("abc");
kdbt.Data = kdata;
ddbt.Data = ddata;
ddbt.ExternalFile = true;
Assert.IsTrue(ddbt.ExternalFile);
pair =
new KeyValuePair <DatabaseEntry, DatabaseEntry>(kdbt, ddbt);
CursorConfig dbcConfig = new CursorConfig();
Transaction txn = null;
if (cfg.Env != null)
{
txn = cfg.Env.BeginTransaction();
}
HashCursor cursor = db.Cursor(dbcConfig, txn);
cursor.Add(pair);
DatabaseStreamConfig dbsc = new DatabaseStreamConfig();
dbsc.SyncPerWrite = true;
DatabaseStream dbs = cursor.DbStream(dbsc);
Assert.AreNotEqual(null, dbs);
Assert.IsFalse(dbs.GetConfig.ReadOnly);
Assert.IsTrue(dbs.GetConfig.SyncPerWrite);
Assert.AreEqual(3, dbs.Size());
DatabaseEntry sdbt = dbs.Read(0, 3);
Assert.IsNotNull(sdbt);
Assert.AreEqual(ddata, sdbt.Data);
sdbt = new DatabaseEntry(Encoding.ASCII.GetBytes("defg"));
Assert.IsTrue(dbs.Write(sdbt, 3));
Assert.AreEqual(7, dbs.Size());
sdbt = dbs.Read(0, 7);
Assert.IsNotNull(sdbt);
Assert.AreEqual(Encoding.ASCII.GetBytes("abcdefg"), sdbt.Data);
dbs.Close();
/*
* Verify the database stream can not write when it is
* configured to be read-only.
*/
dbsc.ReadOnly = true;
dbs = cursor.DbStream(dbsc);
Assert.IsTrue(dbs.GetConfig.ReadOnly);
try
{
Assert.IsFalse(dbs.Write(sdbt, 7));
throw new TestException();
}
catch (DatabaseException)
{
}
dbs.Close();
// Verify the update by cursor.
Assert.IsTrue(cursor.Move(kdbt, true));
pair = cursor.Current;
Assert.AreEqual(Encoding.ASCII.GetBytes("abcdefg"),
pair.Value.Data);
cursor.Close();
if (cfg.Env != null)
{
txn.Commit();
}
/*
* Verify the external files are created in the expected
* location.
* This part of test is disabled since BTreeDatabase.BlobSubDir
* is not exposed to users.
*/
//if (cfg.Env != null)
// blrootdir = testHome + "/" + blrootdir;
//string blobdir = blrootdir + "/" + db.BlobSubDir;
//Assert.AreEqual(records.Length + 1,
// Directory.GetFiles(blobdir, "__db.bl*").Length);
//Assert.AreEqual(1,
// Directory.GetFiles(blobdir, "__db_blob_meta.db").Length);
// Verify the stats.
HashStats st = db.Stats();
Assert.AreEqual(records.Length + 1, st.nExternalFiles);
// Close all handles.
db.Close();
if (cfg.Env != null)
{
cfg.Env.Close();
}
/*
* Remove the default external file directory
* when it is not under the test home.
*/
if (db_blobdir == null && cfg.Env == null)
{
Directory.Delete("__db_bl", true);
}
}
private void Merge()
{
string featureName = FeatureNameTextBox.Text;
if (featureName == "" || featureName.IndexOfAny(Path.GetInvalidFileNameChars()) >= 0)
{
MessageTools.Warning("not a valid feature name");
return;
}
string database = (string)DatabasesBox.SelectedItem;
var sessions = SessionsBox.SelectedItems;
var roles = RolesBox.SelectedItems;
var streams = StreamsBox.SelectedItems;
bool force = ForceCheckBox.IsChecked.Value;
string sessionList = "";
foreach (DatabaseSession session in sessions)
{
if (sessionList == "")
{
sessionList = session.Name;
}
else
{
sessionList += ";" + session.Name;
}
}
string roleList = "";
foreach (var role in roles)
{
if (roleList == "")
{
roleList = ((DatabaseRole)role).Name;
}
else
{
roleList += ";" + ((DatabaseRole)role).Name;
}
}
string[] streamsNames = new string[streams.Count];
int i = 0;
foreach (DatabaseStream stream in streams)
{
streamsNames[i++] = stream.Name;
}
Array.Sort(streamsNames);
string streamList = "";
double sampleRate = ((DatabaseStream)streams[0]).SampleRate;
foreach (string stream in streamsNames)
{
if (streamList == "")
{
streamList = stream;
}
else
{
streamList += ";" + stream;
}
}
string rootDir = Properties.Settings.Default.DatabaseDirectory + "\\" + database;
logTextBox.Text = handler.CMLMergeFeature(rootDir, sessionList, roleList, streamList, featureName, force);
string type = Defaults.CML.StreamTypeNameFeature;
string ext = "stream";
DatabaseStream streamType = new DatabaseStream()
{
Name = featureName, Type = type, FileExt = ext, SampleRate = sampleRate
};
DatabaseHandler.AddStream(streamType);
GetStreams(streamType);
}
