public static void SaveData(Schema s, String MemoryDatabaseFile, RelationVal Value, Firefly.Mapping.Binary.BinarySerializer bs)
{
var rvs = new RelationValueSerializer(s);
var Dir = FileNameHandling.GetFileDirectory(MemoryDatabaseFile);
if (Dir != "" && !Directory.Exists(Dir))
{
Directory.CreateDirectory(Dir);
}
using (var fs = Streams.CreateWritable(MemoryDatabaseFile))
{
fs.WriteSimpleString("KRUSDATA", 8);
fs.WriteUInt64(s.Hash());
var SchemaLengthPosition = fs.Position;
fs.WriteInt64(0);
var SchemaPosition = fs.Position;
bs.Write(fs, s);
var SchemaLength = fs.Position - SchemaPosition;
var DataLengthPosition = fs.Position;
fs.WriteInt64(0);
var DataPosition = fs.Position;
rvs.Write(fs, Value);
var DataLength = fs.Position - DataPosition;
fs.Position = SchemaLengthPosition;
fs.WriteInt64(SchemaLength);
fs.Position = DataLengthPosition;
fs.WriteInt64(DataLength);
}
}
public static KeyValuePair <Schema, RelationVal> LoadData(String MemoryDatabaseFile, Firefly.Mapping.Binary.BinarySerializer bs)
{
Schema s;
RelationVal Value;
using (var fs = Streams.OpenReadable(MemoryDatabaseFile))
{
if (fs.ReadSimpleString(8) != "KRUSDATA")
{
throw new InvalidOperationException("FileFormatMismatch");
}
fs.ReadUInt64();
var SchemaLength = fs.ReadInt64();
if (SchemaLength == 0)
{
throw new InvalidOperationException("FileContainsNoSchema");
}
var SchemaPosition = fs.Position;
s = bs.Read <Schema>(fs);
if (fs.Position != SchemaPosition + SchemaLength)
{
throw new InvalidOperationException();
}
var DataLength = fs.ReadInt64();
var DataPosition = fs.Position;
var rvs = new RelationValueSerializer(s);
Value = rvs.Read(fs);
if (fs.Position != DataPosition + DataLength)
{
throw new InvalidOperationException();
}
}
return(new KeyValuePair <Schema, RelationVal>(s, Value));
}
public static RelationVal LoadData(Schema s, String[] DataDirOrMemoryDatabaseFiles)
{
var Entities = s.Types.Where(t => t.OnEntity).Select(t => t.Entity).ToList();
var DuplicatedCollectionNames = Entities.GroupBy(e => e.CollectionName).Where(g => g.Count() > 1).Select(g => g.Key).ToList();
if (DuplicatedCollectionNames.Count > 0)
{
throw new InvalidOperationException("DuplicatedCollectionNames: {0}".Formats(String.Join(" ", DuplicatedCollectionNames)));
}
var CollectionNameToEntity = Entities.ToDictionary(e => e.CollectionName, StringComparer.OrdinalIgnoreCase);
var SchemaHash = s.Hash();
var Files = new Dictionary <String, Byte[]>();
var MemoryFiles = new List <RelationVal>();
foreach (var DataDirOrMemoryDatabaseFile in DataDirOrMemoryDatabaseFiles)
{
if (Directory.Exists(DataDirOrMemoryDatabaseFile))
{
foreach (var f in Directory.GetFiles(DataDirOrMemoryDatabaseFile, "*.tree", SearchOption.AllDirectories).OrderBy(ff => ff, StringComparer.OrdinalIgnoreCase))
{
Files.Add(f, File.ReadAllBytes(f));
}
}
else
{
RelationVal FileValue;
using (var fs = Streams.OpenReadable(DataDirOrMemoryDatabaseFile))
{
if (fs.ReadSimpleString(8) != "KRUSDATA")
{
throw new InvalidOperationException("FileFormatMismatch");
}
var Hash = fs.ReadUInt64();
if (Hash != SchemaHash)
{
throw new InvalidOperationException("DatabaseSchemaVersionMismatch");
}
var SchemaLength = fs.ReadInt64();
fs.Position += SchemaLength;
var DataLength = fs.ReadInt64();
var DataPosition = fs.Position;
var rvs = new RelationValueSerializer(s);
FileValue = rvs.Read(fs);
if (fs.Position != DataPosition + DataLength)
{
throw new InvalidOperationException();
}
}
MemoryFiles.Add(FileValue);
}
}
var Forests = Files.AsParallel().Select(p =>
{
using (var bas = new ByteArrayStream(p.Value))
{
using (var sr = Txt.CreateTextReader(bas.AsNewReading(), Firefly.TextEncoding.TextEncoding.Default))
{
return(TreeFile.ReadDirect(sr, p.Key, new TreeFormatParseSetting(), new TreeFormatEvaluateSetting()).Value.Nodes);
}
}
}).ToList();
var Tables = new Dictionary <String, List <Node> >();
foreach (var Forest in Forests)
{
foreach (var n in Forest)
{
if (!n.OnStem)
{
continue;
}
var Name = n.Stem.Name;
if (CollectionNameToEntity.ContainsKey(Name))
{
Name = CollectionNameToEntity[Name].Name;
}
if (!Tables.ContainsKey(Name))
{
Tables.Add(Name, new List <Node>());
}
var t = Tables[Name];
t.AddRange(n.Stem.Children);
}
}
var Missings = Entities.Select(e => e.Name).Except(Tables.Keys, StringComparer.OrdinalIgnoreCase).ToList();
foreach (var Name in Missings)
{
Tables.Add(Name, new List <Node>());
}
var rvts = new RelationValueTreeSerializer(s);
var Value = rvts.Read(Tables);
foreach (var FileValue in MemoryFiles)
{
for (int k = 0; k < FileValue.Tables.Count; k += 1)
{
Value.Tables[k].Rows.AddRange(FileValue.Tables[k].Rows);
}
}
return(Value);
}
public static void ApplySchemaDiff(String MemoryDatabaseFileOld, String MemoryDatabaseFileNew, String SchemaDiffFile, String MemoryDatabaseFileOutput)
{
var bs = Niveum.ObjectSchema.BinarySerializerWithString.Create();
Schema SchemaOld;
var SchemaNew = LoadSchema(MemoryDatabaseFileNew, bs);
var Loader = new RelationSchemaDiffLoader(SchemaNew);
Loader.LoadType(SchemaDiffFile);
var l = Loader.GetResult();
using (var fs = Streams.OpenReadable(MemoryDatabaseFileOld))
{
if (fs.ReadSimpleString(8) != "KRUSDATA")
{
throw new InvalidOperationException("FileFormatMismatch");
}
fs.ReadUInt64();
var OldSchemaLength = fs.ReadInt64();
if (OldSchemaLength == 0)
{
throw new InvalidOperationException("FileContainsNoSchema");
}
var OldSchemaPosition = fs.Position;
SchemaOld = bs.Read <Schema>(fs);
if (fs.Position != OldSchemaPosition + OldSchemaLength)
{
throw new InvalidOperationException();
}
fs.ReadInt64();
RelationSchemaDiffVerifier.Verifiy(SchemaOld, SchemaNew, l);
var orvs = new RelationValueSerializer(SchemaOld);
var nrvs = new RelationValueSerializer(SchemaNew);
var OldEntities = SchemaOld.Types.Where(t => t.OnEntity).Select(t => t.Entity).ToList();
var OldEntityNameToIndex = OldEntities.Select((e, i) => new KeyValuePair <String, int>(e.Name, i)).ToDictionary(p => p.Key, p => p.Value);
var NewEntities = SchemaNew.Types.Where(t => t.OnEntity).Select(t => t.Entity).ToList();
var dt = new RelationSchemaDiffTranslator(SchemaOld, SchemaNew, l);
var Dir = FileNameHandling.GetFileDirectory(MemoryDatabaseFileOutput);
if (Dir != "" && !Directory.Exists(Dir))
{
Directory.CreateDirectory(Dir);
}
var OldEntityCount = OldEntities.Count;
var Positions = new Dictionary <int, Int64>();
var CurrentEntityTableIndex = 0;
var CurrentCount = 0;
if (OldEntityCount > 0)
{
Positions.Add(0, fs.Position);
CurrentCount = fs.ReadInt32();
}
Action <int> AdvanceTo = EntityTableIndex =>
{
if (Positions.ContainsKey(EntityTableIndex))
{
fs.Position = Positions[EntityTableIndex];
CurrentEntityTableIndex = EntityTableIndex;
CurrentCount = fs.ReadInt32();
}
else
{
if ((EntityTableIndex < 0) || (EntityTableIndex >= OldEntityCount))
{
throw new InvalidOperationException();
}
while (CurrentEntityTableIndex < EntityTableIndex)
{
var rr = orvs.GetRowReader(OldEntities[CurrentEntityTableIndex]);
while (CurrentCount > 0)
{
rr(fs);
CurrentCount -= 1;
}
CurrentEntityTableIndex += 1;
if (CurrentEntityTableIndex >= OldEntityCount)
{
break;
}
Positions.Add(CurrentEntityTableIndex, fs.Position);
CurrentCount = fs.ReadInt32();
}
}
};
using (var fso = Streams.CreateResizable(MemoryDatabaseFileOutput))
{
fso.WriteSimpleString("KRUSDATA", 8);
fso.WriteUInt64(SchemaNew.Hash());
var SchemaLengthPosition = fso.Position;
fso.WriteInt64(0);
var SchemaPosition = fso.Position;
bs.Write(SchemaNew, fso);
var SchemaLength = fso.Position - SchemaPosition;
var DataLengthPosition = fso.Position;
fso.WriteInt64(0);
var DataPosition = fso.Position;
foreach (var ne in NewEntities)
{
var oOldEntityName = dt.GetOldEntityName(ne.Name);
if (oOldEntityName.OnNone)
{
fso.WriteInt32(0);
continue;
}
var Index = OldEntityNameToIndex[oOldEntityName.Some];
var rr = orvs.GetRowReader(OldEntities[Index]);
var rw = nrvs.GetRowWriter(ne);
var t = dt.GetTranslator(ne.Name);
AdvanceTo(Index);
fso.WriteInt32(CurrentCount);
while (CurrentCount > 0)
{
rw(fso, t(rr(fs)));
CurrentCount -= 1;
}
}
var DataLength = fso.Position - DataPosition;
fso.Position = SchemaLengthPosition;
fso.WriteInt64(SchemaLength);
fso.Position = DataLengthPosition;
fso.WriteInt64(DataLength);
}
}
}