public void BatchSaveTest()
{
Dictionary <int, byte[]> data = new Dictionary <int, byte[]>();
Dictionary <int, IPtr> ptrs = new Dictionary <int, IPtr>();
using (storage = CreateNewStorage(true))
{
data[1] = CreateData(1, 100);
data[2] = CreateData(2, 200);
data[3] = CreateData(3, 300);
ptrs[1] = storage.Save(data[1]);
ptrs[2] = storage.Save(data[2]);
var p = storage.BatchSave(new Tuple <IPtr, byte[]>[]
{
new Tuple <IPtr, byte[]>(ptrs[1], data[1]),
new Tuple <IPtr, byte[]>(ptrs[2], data[2]),
new Tuple <IPtr, byte[]>(null, data[3]),
});
for (int i = 0; i < p.Length; ++i)
{
ptrs[i + 1] = p[i];
}
foreach (var key in data.Keys)
{
Microsoft.VisualStudio.TestTools.UnitTesting.Assert.IsTrue(
data[key].SequenceEqual(storage.Load(ptrs[key])));
}
}
}
static void Main(string[] args)
{
var fsStorage = new FsStorage();
ArtistNameExporter artistNameExporter = new ArtistNameExporter(fsStorage);
using (var fs = File.CreateText("artistNames.csv"))
{
fs.WriteLine("Id,Artist,Role");
int i = 1;
foreach (var artistWithRole in artistNameExporter.GetFullListOfArtistNames(@"c:\users\octo\Music\",
new[] { ".mp3", ".flac", ".wma", ".m4a" }))
{
Console.WriteLine("{2}. Writing: {0} as a {1}.", artistWithRole.Name, artistWithRole.Role, i);
fs.WriteLine($"art{i++},{artistWithRole.Name},{artistWithRole.Role}");
}
}
}
private FsIndexedStorage <int> CreateNewStorageWithLogging(bool clear)
{
if (clear)
{
operations = new List <string>();
Clear();
}
var db = new FsStorage(dir, prefix, maxDbFileLength: 100 * 1024);
var index = new FsIndex <int>(dir, prefix, FsStorage.CreatePtr, BitConverter.GetBytes, bytes => BitConverter.ToInt32(bytes, 0));
#if DEBUG
db.onReading += (ptr, bytes) => { Thread.Sleep(rnd.Next(200)); lock (operations) operations.Add(String.Format("[{0}] read from ptr {1} value {2}", Thread.CurrentThread.ManagedThreadId, ptr.ToString(), BitConverter.ToInt32(bytes, 0))); };
db.onWriting += (ptr, bytes) => { Thread.Sleep(rnd.Next(200)); lock (operations) operations.Add(String.Format("[{0}] write to ptr {1} value {2}", Thread.CurrentThread.ManagedThreadId, ptr.ToString(), BitConverter.ToInt32(bytes, 0))); };
index.onGet += (key, ptr) => { Thread.Sleep(rnd.Next(200)); lock (operations) operations.Add(String.Format("[{0}] get by key {1} ptr {2}", Thread.CurrentThread.ManagedThreadId, key, ptr != null ? ptr.ToString() : "")); };
index.onSet += (key, ptr) => { Thread.Sleep(rnd.Next(200)); lock (operations) operations.Add(String.Format("[{0}] set by key {1} ptr {2}", Thread.CurrentThread.ManagedThreadId, key, ptr.ToString())); };
index.onRem += (key, ptr) => { Thread.Sleep(rnd.Next(200)); lock (operations) operations.Add(String.Format("[{0}] rem key {1} with ptr {2}", Thread.CurrentThread.ManagedThreadId, key, ptr.ToString())); };
#endif
return(new FsIndexedStorage <int>(db, index));
}
static void Main(string[] args)
{
var result = Parser.Default.ParseArguments <Options>(args);
if (result.Tag == ParserResultType.Parsed)
{
var options = ((Parsed <Options>)result).Value;
Console.WriteLine("Executing with argument: >>>{0}<<<", options.RootPath);
var configSection = ((Tags2FilesConfigSection)ConfigurationManager.GetSection("tagsToFiles"));
var extensions =
configSection.Extensions
.Cast <ExtensionElement>().Select(e => e.Value).ToArray();
var individualRule = configSection.Rules
.Cast <RuleElement>().FirstOrDefault(r => r.Type == RuleType.Individual);
if (individualRule == null)
{
throw new ConfigurationErrorsException("no individual rule.");
}
var collectionRule = configSection.Rules
.Cast <RuleElement>().FirstOrDefault(r => r.Type == RuleType.Collection);
if (collectionRule == null)
{
throw new ConfigurationErrorsException("no collection rule.");
}
IStorage storage = new FsStorage();
storage.Directory.DirectoryRead += FileEnumerator_DirectoryRead;
storage.Directory.ExtensionsRead += FileEnumerator_ExtensionsRead;
T2FProcessor t2FProcessor = new T2FProcessor(storage,
new TagFileProcessor());
t2FProcessor.Start(options.RootPath, extensions, individualRule.Pattern,
collectionRule.Pattern, configSection.ActionType.Type);
return;
}
Console.WriteLine(CommandLine.Text.HelpText.AutoBuild(result));
}
public void SimpleSaveLoadTest()
{
using (storage = CreateNewStorage(true))
{
for (int i = 0; i < N; ++i)
{
int key = rnd.Next(100);
if (i < 200 || i % 2 == 0)
{
Store(key, rnd.Next());
}
else
{
Read(key);
}
if (i % 1000 == 0)
{
storage.Flush();
}
}
Read(200);
Store(200, 1240281);
Read(200);
Store(200, 291483);
Read(200);
}
using (storage = CreateNewStorage(false))
{
for (int i = 0; i < N; ++i)
{
int key = rnd.Next(100);
if (i < 200 || i % 2 == 0)
{
Store(key, rnd.Next());
}
else
{
Read(key);
}
if (i % 1000 == 0)
{
storage.Flush();
}
}
// test wrong ptr
/*
* IPtr wrong = new FsStorage.Ptr() {Capacity = 3600, FileNum = 0, Position = 140};
* control_index[1001] = wrong;
* control_storage[1001] = null;
* Read(1001);
*/
int big = 1024 * 1024 * 2 / 4;
Read(big);
Store(big, rnd.Next());
Read(big);
}
}
public void PerformanceBatchTest()
{
int N, NThreads, NBatch, NKeys;
bool SAVETEST = true, READTEST = true;
NKeys = 5000;
byte[][] data = new byte[NKeys][];
IPtr[] ptrs = new IPtr[NKeys];
N = 100000;
NThreads = 10;
NBatch = 5000;
int NReadThreads = 8, NWriteThreads = 2;
using (storage = CreateNewStorage(true))
{
for (int i = 0; i < data.Length; ++i)
{
data[i] = CreateData(i, 100 + rnd.Next(5000));
ptrs[i] = storage.Save(data[i]);
}
if (SAVETEST)
{
double[] KbytesSaved = Enumerable.Repeat(0.0, NThreads).ToArray();
Action <int> save = (tid) =>
{
for (int i = 0; i < N; i += NBatch)
{
Tuple <IPtr, byte[]>[] values = Enumerable.Repeat(0, NBatch).Select(_ => Tuple.Create(ptrs[rnd.Next(NKeys)], data[rnd.Next(NKeys)])).ToArray();
storage.BatchSave(values);
KbytesSaved[tid] += values.Sum(v => (long)v.Item2.Length) / 1024.0;
}
};
var tsave = MeasureTimeOfThredas(NThreads, save);
Console.WriteLine("" + (N * NThreads) + " save operations in " + NThreads + " threads elapsed: " +
tsave);
Console.WriteLine("Save Operations in 1 sec: " + ((double)N * NThreads / tsave.TotalSeconds).ToString("### ###"));
Console.WriteLine("KBytes saved in 1 sec: " +
(KbytesSaved.Sum() / tsave.TotalSeconds).ToString("### ### ### Kb"));
Console.WriteLine("Total KBytes saved: " + KbytesSaved.Sum());
}
if (READTEST)
{
double[] KbytesReaded = Enumerable.Repeat(0.0, NThreads).ToArray();
Action <int> read = (tid) =>
{
for (int i = 0; i < N; i += NBatch)
{
var keys = rnd.NextN(NKeys, NBatch).Select(key => ptrs[key]).ToArray();
var values = storage.BatchLoad(keys);
KbytesReaded[tid] += values.Sum(v => v != null ? (long)v.Item2.Length : (long)0) / 1024.0;
}
};
var tread = MeasureTimeOfThredas(NThreads, read);
Console.WriteLine("" + (N * NThreads) + " read operations in " + NThreads + " threads elapsed: " +
tread);
Console.WriteLine("Read Operations in 1 sec: " + ((double)N * NThreads / tread.TotalSeconds).ToString("### ###"));
Console.WriteLine("KBytes readed in 1 sec: " +
(KbytesReaded.Sum() / tread.TotalSeconds).ToString("### ### ##0 Kb"));
Console.WriteLine("Total KBytes readed: " + KbytesReaded.Sum());
}
if (SAVETEST && READTEST)
{
NThreads = NReadThreads + NWriteThreads;
double[] KbytesProcessed = Enumerable.Repeat(0.0, NThreads).ToArray();
Action <int> action = (tid) =>
{
for (int i = 0; i < N; i += NBatch)
{
Tuple <IPtr, byte[]>[] values;
if (tid < NWriteThreads)
{
var keys = rnd.NextN(NKeys, NBatch).Select(key => ptrs[key]).ToArray();
values = storage.BatchLoad(keys).ToArray();
}
else
{
values = Enumerable.Repeat(0, NBatch).Select(_ => Tuple.Create(ptrs[rnd.Next(NKeys)], data[rnd.Next(NKeys)])).ToArray();
storage.BatchSave(values);
}
KbytesProcessed[tid] += values.Sum(v => v != null ? (long)v.Item2.Length : (long)0) / 1024.0;
}
};
var time = MeasureTimeOfThredas(NThreads, action);
Console.WriteLine("" + (N * NReadThreads) + " read operations in " + NReadThreads + " threads and \n" +
"" + (N * NWriteThreads) + " write operations in " + NWriteThreads + " threads elapsed: " +
time);
Console.WriteLine("Read|Write Operations in 1 sec: " + ((double)N * NThreads / time.TotalSeconds).ToString("### ###"));
Console.WriteLine("KBytes readed|writed in 1 sec: " +
(KbytesProcessed.Sum() / time.TotalSeconds).ToString("### ### ### Kb"));
Console.WriteLine("Total KBytes processed: " + KbytesProcessed.Sum());
}
}
}
public TagsProvider(FsStorage fsStorage)
{
_fsStorage = fsStorage;
}