public static void ExportCollection(String ConnectionString, String DataDir, List <String> ExportEntityNames)
{
var bs = Niveum.ObjectSchema.BinarySerializerWithString.Create();
var Pair = LoadData(ConnectionString, bs);
var s = Pair.Key;
var Value = Pair.Value;
var Entities = s.Types.Where(t => t.OnEntity).Select(t => t.Entity).ToList();
HashSet <String> ExportEntityNameSet = null;
if (ExportEntityNames.Count != 0)
{
ExportEntityNameSet = new HashSet <String>(ExportEntityNames.Distinct(StringComparer.OrdinalIgnoreCase), StringComparer.OrdinalIgnoreCase);
}
var Dir = FileNameHandling.GetAbsolutePath(DataDir, Environment.CurrentDirectory);
if (Dir != "" && !Directory.Exists(Dir))
{
Directory.CreateDirectory(Dir);
}
var rvts = new RelationValueTreeSerializer(s);
foreach (var p in Entities.ZipStrict(Value.Tables, (a, b) => new KeyValuePair <EntityDef, TableVal>(a, b)))
{
var e = p.Key;
var tv = p.Value;
if ((ExportEntityNameSet == null) || (ExportEntityNameSet.Contains(e.Name)))
{
var l = rvts.WriteTable(e, tv);
var t = RelationValueSyntaxTreeBuilder.BuildTable(e, l);
var TreeFilePath = FileNameHandling.GetPath(DataDir, e.Name + ".tree");
TreeFile.WriteRaw(TreeFilePath, new Syntax.Forest {
MultiNodesList = new List <Syntax.MultiNodes> {
t
}
});
}
}
}
public static Dictionary <String, TableVal> LoadPartialData(Schema s, String[] DataDirs, Firefly.Mapping.Binary.BinarySerializer bs)
{
var Entities = s.Types.Where(t => t.OnEntity).Select(t => t.Entity).ToList();
var DuplicatedCollectionNames = Entities.GroupBy(e => e.CollectionName).Where(g => g.Count() > 1).Select(g => g.Key).ToList();
if (DuplicatedCollectionNames.Count > 0)
{
throw new InvalidOperationException("DuplicatedCollectionNames: {0}".Formats(String.Join(" ", DuplicatedCollectionNames)));
}
var CollectionNameToEntity = Entities.ToDictionary(e => e.CollectionName, StringComparer.OrdinalIgnoreCase);
var EntityNameToEntity = Entities.ToDictionary(e => e.Name, StringComparer.OrdinalIgnoreCase);
var SchemaHash = s.Hash();
var Files = new Dictionary <String, Byte[]>();
foreach (var DataDirOrMemoryDatabaseFile in DataDirs)
{
foreach (var f in Directory.GetFiles(DataDirOrMemoryDatabaseFile, "*.tree", SearchOption.AllDirectories).OrderBy(ff => ff, StringComparer.OrdinalIgnoreCase))
{
Files.Add(f, File.ReadAllBytes(f));
}
}
var Forests = Files.AsParallel().Select(p =>
{
using (var bas = new ByteArrayStream(p.Value))
{
using (var sr = Txt.CreateTextReader(bas.AsNewReading(), Firefly.TextEncoding.TextEncoding.Default))
{
return(TreeFile.ReadDirect(sr, p.Key, new TreeFormatParseSetting(), new TreeFormatEvaluateSetting()).Value.Nodes);
}
}
}).ToList();
var Tables = new Dictionary <String, List <Node> >();
foreach (var Forest in Forests)
{
foreach (var n in Forest)
{
if (!n.OnStem)
{
continue;
}
var Name = n.Stem.Name;
if (CollectionNameToEntity.ContainsKey(Name))
{
Name = CollectionNameToEntity[Name].Name;
}
if (!Tables.ContainsKey(Name))
{
Tables.Add(Name, new List <Node>());
}
var t = Tables[Name];
t.AddRange(n.Stem.Children);
}
}
var rvts = new RelationValueTreeSerializer(s);
var TableValues = new Dictionary <String, TableVal>(StringComparer.OrdinalIgnoreCase);
foreach (var p in Tables)
{
var EntityName = p.Key;
var e = EntityNameToEntity[EntityName];
var tv = rvts.ReadTable(p.Value, e);
TableValues.Add(EntityName, tv);
}
return(TableValues);
}
public static RelationVal LoadData(Schema s, String[] DataDirOrMemoryDatabaseFiles)
{
var Entities = s.Types.Where(t => t.OnEntity).Select(t => t.Entity).ToList();
var DuplicatedCollectionNames = Entities.GroupBy(e => e.CollectionName).Where(g => g.Count() > 1).Select(g => g.Key).ToList();
if (DuplicatedCollectionNames.Count > 0)
{
throw new InvalidOperationException("DuplicatedCollectionNames: {0}".Formats(String.Join(" ", DuplicatedCollectionNames)));
}
var CollectionNameToEntity = Entities.ToDictionary(e => e.CollectionName, StringComparer.OrdinalIgnoreCase);
var SchemaHash = s.Hash();
var Files = new Dictionary <String, Byte[]>();
var MemoryFiles = new List <RelationVal>();
foreach (var DataDirOrMemoryDatabaseFile in DataDirOrMemoryDatabaseFiles)
{
if (Directory.Exists(DataDirOrMemoryDatabaseFile))
{
foreach (var f in Directory.GetFiles(DataDirOrMemoryDatabaseFile, "*.tree", SearchOption.AllDirectories).OrderBy(ff => ff, StringComparer.OrdinalIgnoreCase))
{
Files.Add(f, File.ReadAllBytes(f));
}
}
else
{
RelationVal FileValue;
using (var fs = Streams.OpenReadable(DataDirOrMemoryDatabaseFile))
{
if (fs.ReadSimpleString(8) != "KRUSDATA")
{
throw new InvalidOperationException("FileFormatMismatch");
}
var Hash = fs.ReadUInt64();
if (Hash != SchemaHash)
{
throw new InvalidOperationException("DatabaseSchemaVersionMismatch");
}
var SchemaLength = fs.ReadInt64();
fs.Position += SchemaLength;
var DataLength = fs.ReadInt64();
var DataPosition = fs.Position;
var rvs = new RelationValueSerializer(s);
FileValue = rvs.Read(fs);
if (fs.Position != DataPosition + DataLength)
{
throw new InvalidOperationException();
}
}
MemoryFiles.Add(FileValue);
}
}
var Forests = Files.AsParallel().Select(p =>
{
using (var bas = new ByteArrayStream(p.Value))
{
using (var sr = Txt.CreateTextReader(bas.AsNewReading(), Firefly.TextEncoding.TextEncoding.Default))
{
return(TreeFile.ReadDirect(sr, p.Key, new TreeFormatParseSetting(), new TreeFormatEvaluateSetting()).Value.Nodes);
}
}
}).ToList();
var Tables = new Dictionary <String, List <Node> >();
foreach (var Forest in Forests)
{
foreach (var n in Forest)
{
if (!n.OnStem)
{
continue;
}
var Name = n.Stem.Name;
if (CollectionNameToEntity.ContainsKey(Name))
{
Name = CollectionNameToEntity[Name].Name;
}
if (!Tables.ContainsKey(Name))
{
Tables.Add(Name, new List <Node>());
}
var t = Tables[Name];
t.AddRange(n.Stem.Children);
}
}
var Missings = Entities.Select(e => e.Name).Except(Tables.Keys, StringComparer.OrdinalIgnoreCase).ToList();
foreach (var Name in Missings)
{
Tables.Add(Name, new List <Node>());
}
var rvts = new RelationValueTreeSerializer(s);
var Value = rvts.Read(Tables);
foreach (var FileValue in MemoryFiles)
{
for (int k = 0; k < FileValue.Tables.Count; k += 1)
{
Value.Tables[k].Rows.AddRange(FileValue.Tables[k].Rows);
}
}
return(Value);
}