public void Import(Stream ingestion)
{
XElement result = XElement.Load(ingestion);
Ingestion ing = new Ingestion();
Import(ing.Deserialize(result));
}
private void AssertAggrigationsEqualIngestion(XElement xml, Ingestion ingestion)
{
Assert.AreEqual(xml.Name.LocalName, "aggregations");
int setCount = ingestion.Aggregations.Count;
foreach (XElement element in xml.Elements())
{
string name = element.Name.LocalName;
if (name == "collection")
{
--setCount;
string guid = element.Attribute("guid").Value;
Assert.IsTrue(ingestion.Aggregations.Select(m => m.Guid).Contains(guid));
}
else if (name == "item")
{
--setCount;
string guid = element.Attribute("guid").Value;
Assert.IsTrue(ingestion.Aggregations.Select(m => m.Guid).Contains(guid));
}
else
{
Assert.Fail("Unkown element \"{0}\" found.", name);
}
}
Assert.AreEqual(setCount, 0);
}
///<exception cref="IngestionException"/>
private async Task ExecuteAsyncHelper()
{
while (true)
{
try
{
await Ingestion.ExecuteCallAsync(this).ConfigureAwait(false);
return;
}
catch (IngestionException e)
{
if (!e.IsRecoverable || _retryCount >= _retryIntervals.Length)
{
throw;
}
MobileCenterLog.Warn(MobileCenterLog.LogTag, "Failed to execute service call", e);
}
await _retryIntervals[_retryCount++]().ConfigureAwait(false);
if (_tokenSource.Token.IsCancellationRequested)
{
throw new IngestionException("The operation has been cancelled");
}
}
}
public void Import(XElement ingestion)
{
Ingestion input = new Ingestion();
input.Deserialize(ingestion);
Import(input);
}
public void ExportPartialTest()
{
string result = "";
DatabaseHelper Dh = new DatabaseHelper(true);
// Build the Ingestion
CFCollection collection = Dh.Db.Collections.First();
IEnumerable <CFItem> items = Dh.Db.Items.Take(2);
IEnumerable <CFEntityType> entitytypes = Dh.Db.EntityTypes.Where(e => e.Id == collection.EntityType.Id || items.Select(i => i.EntityTypeId).Contains(e.Id)).Distinct();
IEnumerable <CFMetadataSet> metadatasets = Dh.Db.MetadataSets.Where(m => entitytypes.SelectMany(e => e.MetadataSets.Select(mm => mm.Id)).Contains(m.Id)).Distinct();
Ingestion ingestion = new Ingestion();
ingestion.MetadataSets.AddRange(metadatasets);
ingestion.EntityTypes.AddRange(entitytypes);
ingestion.Aggregations.Add(collection);
ingestion.Aggregations.AddRange(items);
CFItem[] itemArray = items.ToArray();
ingestion.Relationships.Add(new Relationship()
{
ParentRef = collection.Guid,
ChildRef = itemArray[0].Guid
});
ingestion.Relationships.Add(new Relationship()
{
IsMember = true,
IsRelated = false,
ParentRef = collection.Guid,
ChildRef = itemArray[1].Guid
});
ingestion.Relationships.Add(new Relationship()
{
IsRelated = true,
IsMember = false,
ParentRef = itemArray[0].Guid,
ChildRef = itemArray[1].Guid
});
// Export the injestion
using (MemoryStream stream = new MemoryStream())
{
Dh.Igs.Export(stream, ingestion);
stream.Position = 0;
using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
{
result = reader.ReadToEnd();
}
}
XElement testResult = XElement.Parse(result);
// Test result
AsserXmlEqualIngestion(testResult, ingestion);
}
public void Setup()
{
_formFile = Substitute.For <IFormFile>();
_saveFile = Substitute.For <IFile>();
_httpClientFactory = Substitute.For <IHttpClientFactory>();
_httpClient = Substitute.For <IHttpClient>();
_httpClientFactory.Create().Returns(_httpClient);
_sut = new Ingestion(_saveFile, _httpClientFactory);
}
protected override int Invoke(ProjectInfo project, IngestionIndex index)
{
InsertItems(
index,
Ingestion.EnumerateIngestionFiles(
project,
parseMetadataFromFile: !NoMetadata),
DryRun,
resetIndex: true);
return(0);
}
public void Export(Stream stream, Ingestion ingestion = null)
{
if (ingestion == null)
{
ingestion = Export();
}
XElement element = ingestion.Serialize();
element.Save(stream);
}
public void Import(Stream ingestion, int threads = 1)
{
#if DEBUG
Console.Error.WriteLine("Converting ingestion stream to Ingestion object.");
#endif
Ingestion ing = new Ingestion();
ing.Deserialize(ingestion);
Import(ing);
}
private static async Task Examples()
{
// Ingestion
await Ingestion.ImportAssets();
// Exporter
await Exporter.ExportEntities(Constants.EntityDefinitions.Asset.DefinitionName, 10);
// Policies
await Security.SetupUserGroupPolicies("M.SDK.Training", Constants.EntityDefinitions.Recipe.DefinitionName, Constants.Permissions.CookingPermission);
}
protected override int Invoke(ProjectInfo project, IngestionIndex index)
{
int totalItems = 0;
TimeSpan totalDuration = default;
long totalSize = 0;
IEnumerable <IngestionItem> YieldAndLog()
{
foreach (var item in Ingestion.EnumerateIngestionFiles(project, parseMetadataFromFile: !NoMetadata))
{
Log.Information(
"[{TotalItems}] {FilePath} {Duration} @ {Timestamp}",
++totalItems,
item.FilePath,
item.Duration,
item.Timestamp);
if (item.Duration.HasValue)
{
totalDuration += item.Duration.Value;
}
if (item.FileSize.HasValue)
{
totalSize += item.FileSize.Value;
}
yield return(item);
}
}
if (DryRun)
{
var enumerator = YieldAndLog().GetEnumerator();
while (enumerator.MoveNext())
{
;
}
}
else
{
index.Reset();
index.Insert(YieldAndLog());
}
Log.Information("Count = {TotalItems}", totalItems);
Log.Information("Duration = {TotalDuration}", totalDuration);
Log.Information("Size = {TotalSize} GB", totalSize / 1024.0 / 1024.0 / 1024.0);
return(0);
}
public async Task <Ingestion> CreateIngestionsAsync(IngestionViewModel viewModel)
{
var newIng = new Ingestion
{
Name = viewModel.Name,
OneDayFoodID = viewModel.OneDayFoodId
};
_context.Ingestion.Add(newIng);
_context.SaveChanges();
return(await
_context.Ingestion.SingleOrDefaultAsync(
t => t.Name == viewModel.Name& t.OneDayFoodID == viewModel.OneDayFoodId));
}
public virtual void Execute()
{
_tokenSource.Dispose();
_tokenSource = new CancellationTokenSource();
Ingestion.ExecuteCallAsync(this).ContinueWith(completedTask =>
{
if (completedTask.IsFaulted)
{
ServiceCallFailedCallback?.Invoke(completedTask.Exception?.InnerException as IngestionException);
}
else
{
ServiceCallSucceededCallback?.Invoke();
}
});
}
public AssemblyLoader(Config config, string baseFolder)
{
this.Config = config;
this.BaseFolder = baseFolder;
string[] runtimeAssemblies = Directory.GetFiles(RuntimeEnvironment.GetRuntimeDirectory(), "*.dll");
this.ContextAssemblies = new HashSet <string>(runtimeAssemblies);
Logger.Debug("Found {0} runtime assemblies", runtimeAssemblies.Length);
AddContextAssemblies();
this.Ingestion = new CSharpXmlIngestion();
this.LoadContext = new MetadataLoadContext(new PathAssemblyResolver(ContextAssemblies));
LoadAllAssemblies();
}
protected void DeleteButton_Click(Object sender, EventArgs e)
{
try
{
Guid argument = new Guid((sender as ImageButton).CommandArgument);
Ingestion logToDelete = MyDataContext.Default.Ingestions.Single(current => current.Guid == argument);
Models.PictureHandler.DeleteImage(logToDelete.PictureGuid);
MyDataContext.Default.DeleteObject(logToDelete);
MyDataContext.Default.SaveChanges();
}
catch (Exception ex)
{
this.Master.ShowError(ex);
}
}
protected void Page_PreRender(object sender, EventArgs e)
{
try
{
this.DateTextBox.Text = DateTime.Now.Date.ToShortDateString();
this.HourList.SelectedValue = DateTime.Now.GetInQuarters().Hour.ToString();
this.MinuteList.SelectedValue = DateTime.Now.GetInQuarters().Minute.ToString();
DateTime from = DateTime.Parse(this.FilterFrom.Text);
DateTime until = DateTime.Parse(this.FilterUntil.Text);
this.FeastRepeater.DataSource = Ingestion.LoadIngestionsPerDay(from, until);
this.FeastRepeater.DataBind();
}
catch (Exception ex)
{
this.Master.ShowError(ex);
}
}
protected void SaveButton_Click(Object sender, EventArgs e)
{
try
{
DateTime timeStamp = DateTime.Parse(this.DateTextBox.Text);
timeStamp = timeStamp.AddHours(this.HourList.SelectedValue.ToInt32());
timeStamp = timeStamp.AddMinutes(this.MinuteList.SelectedValue.ToInt32());
Ingestion newIngstion = new Ingestion(this.Session.GetCurrentUser());
newIngstion.Date = timeStamp;
newIngstion.PictureGuid = Guid.NewGuid();
MyDataContext.Default.Ingestions.AddObject(newIngstion);
MyDataContext.Default.SaveChanges();
newIngstion.SaveImage(this.PictureUpload.FileBytes);
}
catch (Exception ex)
{
this.Master.ShowError(ex);
}
}
private void AssertRelationshipsEqualIngestion(XElement xml, Ingestion ingestion)
{
Assert.AreEqual(xml.Name.LocalName, "relationships");
int setCount = ingestion.Relationships.Count;
foreach (XElement element in xml.Elements())
{
string name = element.Name.LocalName;
if (name == "relationship")
{
--setCount;
}
else
{
Assert.Fail("Unkown element \"{0}\" found.", name);
}
}
Assert.AreEqual(setCount, 0);
}
///<exception cref="IngestionException"/>
private async Task RunWithRetriesAsyncHelper()
{
while (true)
{
try
{
await Ingestion.ExecuteCallAsync(this).ConfigureAwait(false);
return;
}
catch (IngestionException e)
{
if (!e.IsRecoverable || _retryCount >= _retryIntervals.Length)
{
throw;
}
MobileCenterLog.Warn(MobileCenterLog.LogTag, "Failed to execute service call", e);
}
await _retryIntervals[_retryCount++]().ConfigureAwait(false);
_tokenSource.Token.ThrowIfCancellationRequested();
}
}
private void AssertEntityTypesXmlEqualIngestion(XElement xml, Ingestion ingestion)
{
Assert.AreEqual(xml.Name.LocalName, "entity-types");
int setCount = ingestion.EntityTypes.Count;
foreach (XElement element in xml.Elements())
{
string name = element.Name.LocalName;
if (name == "entity-type")
{
--setCount;
string id = element.Attribute("id").Value;
Assert.IsTrue(ingestion.EntityTypes.Select(m => m.Id.ToString()).Contains(id));
}
else
{
Assert.Fail("Unkown element \"{0}\" found.", name);
}
}
Assert.AreEqual(setCount, 0);
}
private void AssertMetadatasetsXmlEqualIngestion(XElement xml, Ingestion ingestion)
{
Assert.AreEqual(xml.Name.LocalName, "metadata-sets");
int setCount = ingestion.MetadataSets.Count;
foreach (XElement element in xml.Elements())
{
string name = element.Name.LocalName;
if (name == "metadata-set")
{
--setCount;
string guid = element.Attribute("guid").Value;
Assert.IsTrue(ingestion.MetadataSets.Select(m => m.Guid).Contains(guid));
}
else
{
Assert.Fail("Unkown element \"{0}\" found.", name);
}
}
Assert.AreEqual(setCount, 0);
}
protected void PictureRepeater_ItemDataBound(object sender, RepeaterItemEventArgs e)
{
try
{
Ingestion current = e.Item.DataItem as Ingestion;
if (current != null)
{
Image myImage = e.Item.FindControl("MyImage") as Image;
myImage.ImageUrl = current.GetPictureUrl();
Label timeLabel = e.Item.FindControl("TimeLabel") as Label;
timeLabel.Text = current.Date.ToShortTimeString();
ImageButton deleteButton = e.Item.FindControl("DeleteButton") as ImageButton;
deleteButton.CommandArgument = current.Guid.ToString();
}
}
catch (Exception ex)
{
this.Master.ShowError(ex);
}
}
public virtual async Task ExecuteAsync()
{
_tokenSource.Dispose();
_tokenSource = new CancellationTokenSource();
await Ingestion.ExecuteCallAsync(this).ConfigureAwait(false);
}
public Ingestion Export()
{
IEnumerable <CFCollection> collections = Db.Collections;
IEnumerable <CFItem> items = Db.Items;
IEnumerable <CFEntityType> entitytypes = Db.EntityTypes;
IEnumerable <CFMetadataSet> metadatasets = Db.MetadataSets;
//IEnumerable<Form> forms = Db.FormTemplates;
Ingestion ingestion = new Ingestion();
ingestion.MetadataSets.AddRange(metadatasets);
ingestion.EntityTypes.AddRange(entitytypes);
ingestion.Aggregations.AddRange(collections);
ingestion.Aggregations.AddRange(items);
//ingestion.Aggregations.AddRange(forms); //MR Feb 23 2018: Form is not an Aggregation object: ignore form now -- have to revisit this later
CFItem[] itemArray = items.ToArray();
//find all item member in each collection
foreach (CFCollection col in collections.ToList())
{
foreach (CFItem itm in items.ToList())
{
if (col.ChildMembers.Any(p => p.MappedGuid == itm.Guid))
{
ingestion.Relationships.Add(new Relationship
{
ParentRef = col.Guid,
ChildRef = itm.Guid,
IsMember = true,
IsRelated = false
});
}
}
}
//find all item member in each item
foreach (CFItem parentItem in items.ToList())
{
foreach (CFItem childItem in items.ToList())
{
if (parentItem.ChildMembers.Any(member => member.MappedGuid == childItem.Guid))
{
ingestion.Relationships.Add(new Relationship
{
ParentRef = parentItem.Guid,
ChildRef = childItem.Guid,
IsMember = true,
IsRelated = false
});
}
//find related items
if (parentItem.RelatedMembers.Any(member => member.MappedGuid == childItem.Guid))
{
ingestion.Relationships.Add(new Relationship
{
ParentRef = parentItem.Guid,
ChildRef = childItem.Guid,
IsMember = true,
IsRelated = true
});
}
}
}
return(ingestion);
}
public void Import(Ingestion ingestion)
{
//create new GUID and new EntityType-Id
Dictionary <string, string> GuidMap = new Dictionary <string, string>();
Dictionary <int, int> IdMap = new Dictionary <int, int>();
foreach (CFMetadataSet ms in ingestion.MetadataSets)
{
string oldId = ms.Guid;
string newGuid = NewGuid();
var meta = ms;
if (ingestion.Overwrite)
{
//use whatever GUID in the file
//check to make sure the GUID in not existing in the Db, if it's replace the one in the database
GuidMap.Add(oldId, oldId); //if overwrite is true use existing GUID
}
else
{
GuidMap.Add(oldId, newGuid);
}
//GuidMap = buildMapGuid(ingestion.Overwrite, oldId);
ms.Guid = GuidMap[oldId];
ms.MappedGuid = GuidMap[oldId];
ms.Serialize();
//meta.Serialize();
if (ingestion.Overwrite)
{
//check if the metadataset with this guid is existed in the database
//if yes, replace the one in th edatabase with this one from the file
CFMetadataSet metadataSet = Db.MetadataSets.Where(m => m.MappedGuid == ms.Guid).FirstOrDefault();
if (metadataSet != null)
{
metadataSet = ms;
Db.Entry(metadataSet).State = System.Data.Entity.EntityState.Modified;
}
else
{
Db.MetadataSets.Add(ms);
}
}
else
{
Db.MetadataSets.Add(ms);
}
}
Db.SaveChanges();
//add entityType
foreach (CFEntityType et in ingestion.EntityTypes)
{
int oldId = et.Id;
//I think below is not necessary since the DeserialiseEntytyType mtehod already grabbing the right metadataset
List <CFMetadataSet> newSets = new List <CFMetadataSet>();
foreach (CFMetadataSet set in et.MetadataSets)
{
string mGuid = set.Guid;
if (GuidMap.ContainsKey(mGuid))
{
mGuid = GuidMap[mGuid];
}
CFMetadataSet dbSet = Db.MetadataSets.Where(s => s.MappedGuid == mGuid).FirstOrDefault();
newSets.Add(dbSet);
}
et.MetadataSets.Clear();
et.MetadataSets = newSets;
foreach (CFEntityTypeAttributeMapping mapping in et.AttributeMappings)
{
string mGuid = mapping.MetadataSet.Guid;
if (GuidMap.ContainsKey(mGuid))
{
mGuid = GuidMap[mGuid];
}
mapping.MetadataSet = Db.MetadataSets.Where(m => m.MappedGuid == mGuid).FirstOrDefault();
}
CFEntityType newEt = null;
if (ingestion.Overwrite)
{
CFEntityType oldEt = Db.EntityTypes.Where(e => e.Name == et.Name).FirstOrDefault();
if (oldEt != null)
{
//modified it --replace with current value
oldEt = et;
Db.Entry(oldEt).State = System.Data.Entity.EntityState.Modified;
}
else
{
newEt = Db.EntityTypes.Add(et);
}
}
else
{
newEt = Db.EntityTypes.Add(et);
}
Db.SaveChanges();
IdMap.Add(oldId, newEt.Id);
}
//add aggregations
GuidMap.Clear();
foreach (var agg in ingestion.Aggregations)
{
string oldId = agg.Guid;
string newGuid = NewGuid();
if (ingestion.Overwrite)
{
//use whatever GUID in the file
//check to make sure the GUID in not existing in the Db, if it's replace the one in the database
GuidMap.Add(oldId, oldId); //if overwrite is true use existing GUID
}
else
{
agg.Guid = newGuid;
agg.MappedGuid = newGuid;
GuidMap.Add(oldId, newGuid);
}
//saving the aggregation object
if (ingestion.Overwrite)
{
CFAggregation _aggregation = Db.XmlModels.Where(a => a.MappedGuid == newGuid).FirstOrDefault() as CFAggregation;
if (_aggregation != null)
{
_aggregation = (CFAggregation)agg;
Db.Entry(_aggregation).State = System.Data.Entity.EntityState.Modified;
}
else
{
Type t = agg.GetType();
MethodInfo method = this.GetType().GetMethod("CreateAggregation");
MethodInfo genMethod = method.MakeGenericMethod(t);
var _agg = (CFAggregation)genMethod.Invoke(this, new object[] { agg });
Db.Entities.Add(_agg);
}
}
else
{
Type t = agg.GetType();
MethodInfo method = this.GetType().GetMethod("CreateAggregation");
MethodInfo genMethod = method.MakeGenericMethod(t);
var _agg = (CFAggregation)genMethod.Invoke(this, new object[] { agg });
Db.Entities.Add(_agg);
}
}
Db.SaveChanges();
foreach (var rel in ingestion.Relationships)
{
string pGuid = rel.ParentRef;
if (GuidMap.ContainsKey(pGuid))
{
pGuid = GuidMap[pGuid];
}
string cGuid = rel.ChildRef;
if (GuidMap.ContainsKey(cGuid))
{
cGuid = GuidMap[cGuid];
}
rel.ParentRef = pGuid;
rel.ChildRef = cGuid;
if (rel.IsMember)
{
AddMember(rel.ParentRef, rel.ChildRef);
}
if (rel.IsRelated)
{
//save it in Aggregation has related
AddRelatedMember(rel.ParentRef, rel.ChildRef);
}
}
}
protected override int Invoke(ProjectInfo project, IngestionIndex index)
{
if (InputFiles.Count > 0)
{
InsertItems(
index,
Ingestion.EnumerateIngestionFiles(
project,
parseMetadataFromFile:
!NoMetadata,
InputFiles),
DryRun);
return(0);
}
project = project.Evaluate(expandPaths: false);
if (project.Recordings is null || project.Recordings.Sources.Count == 0)
{
Console.Error.WriteLine("No recording sources are configured on this project.");
return(1);
}
InsertItems(
index,
FindNewIngestionItems(),
DryRun);
IEnumerable <IngestionItem> FindNewIngestionItems()
{
var newestTimestamp = index.SelectNewestTimestamp();
#nullable disable
foreach (var recordingSource in project.Recordings.Sources)
#nullable restore
{
if (!recordingSource.Schedule.HasValue)
{
continue;
}
// Unfortunately NCrontab doesn't work with DateTimeOffset, so normalize to UTC.
var expectedRecordingTimes = recordingSource
.Schedule
.Value
.GetNextOccurrences(
newestTimestamp.UtcDateTime,
DateTime.UtcNow);
var basePaths = new List <string>();
foreach (var expectedRecordingTime in expectedRecordingTimes)
{
if (Path.GetDirectoryName(
recordingSource.CreateOutputPath(
expectedRecordingTime)) is string expectedDirectory)
{
if (!basePaths.Contains(expectedDirectory))
{
basePaths.Add(expectedDirectory);
}
}
}
foreach (var ingestion in project.Ingestions)
{
foreach (var basePath in basePaths)
{
var adjustedIngestion = new IngestionInfo(
Path.GetFileName(ingestion.PathGlob),
basePath,
ingestion.PathFilter);
foreach (var item in Ingestion.EnumerateIngestionFiles(
adjustedIngestion,
parseMetadataFromFile: !NoMetadata,
basePath: ingestion.BasePath))
{
yield return(item.WithFilePath(Path.Combine(basePath, item.FilePath)));
}
}
}
}
}
protected override int Invoke(ProjectInfo project, IngestionIndex index)
{
index.Insert(Ingestion.EnumerateIngestionFiles(project, InputFiles));
return(0);
}
public void Import(Ingestion ingestion)
{
#if DEBUG
Console.Error.WriteLine("Starting ingestion of Ingestion object.");
#endif
//create new GUID and new EntityType-Id
Dictionary <string, string> GuidMap = new Dictionary <string, string>();
Dictionary <int, int> IdMap = new Dictionary <int, int>();
foreach (CFMetadataSet ms in ingestion.MetadataSets)
{
string oldId = ms.Guid;
string newGuid = NewGuid();
var meta = ms;
if (ingestion.Overwrite)
{
//use whatever GUID in the file
//check to make sure the GUID in not existing in the Db, if it's replace the one in the database
GuidMap.Add(oldId, oldId); //if overwrite is true use existing GUID
}
else
{
GuidMap.Add(oldId, newGuid);
}
//GuidMap = buildMapGuid(ingestion.Overwrite, oldId);
ms.Guid = GuidMap[oldId];
ms.MappedGuid = GuidMap[oldId];
ms.Serialize();
//meta.Serialize();
if (ingestion.Overwrite)
{
//check if the metadataset with this guid is existed in the database
//if yes, replace the one in th edatabase with this one from the file
CFMetadataSet metadataSet = Db.MetadataSets.Where(m => m.MappedGuid == ms.Guid).FirstOrDefault();
if (metadataSet != null)
{
metadataSet = ms;
Db.Entry(metadataSet).State = System.Data.Entity.EntityState.Modified;
}
else
{
Db.MetadataSets.Add(ms);
}
}
else
{
try
{
Db.MetadataSets.Add(ms);
}catch (ProviderIncompatibleException ex)
{
Console.Error.WriteLine("Current Connection String: " + Db.Database.Connection.ConnectionString);
throw ex;
}
}
}
Db.SaveChanges();
//add entityType
foreach (CFEntityType et in ingestion.EntityTypes)
{
int oldId = et.Id;
//I think below is not necessary since the DeserialiseEntytyType mtehod already grabbing the right metadataset
List <CFMetadataSet> newSets = new List <CFMetadataSet>();
foreach (CFMetadataSet set in et.MetadataSets)
{
string mGuid = set.Guid;
if (GuidMap.ContainsKey(mGuid))
{
mGuid = GuidMap[mGuid];
}
CFMetadataSet dbSet = Db.MetadataSets.Where(s => s.MappedGuid == mGuid).FirstOrDefault();
newSets.Add(dbSet);
}
et.MetadataSets.Clear();
et.MetadataSets = newSets;
foreach (CFEntityTypeAttributeMapping mapping in et.AttributeMappings)
{
string mGuid = mapping.MetadataSet.Guid;
if (GuidMap.ContainsKey(mGuid))
{
mGuid = GuidMap[mGuid];
}
mapping.MetadataSet = Db.MetadataSets.Where(m => m.MappedGuid == mGuid).FirstOrDefault();
}
CFEntityType newEt = null;
if (ingestion.Overwrite)
{
CFEntityType oldEt = Db.EntityTypes.Where(e => e.Name == et.Name).FirstOrDefault();
if (oldEt != null)
{
//modified it --replace with current value
oldEt = et;
Db.Entry(oldEt).State = System.Data.Entity.EntityState.Modified;
}
else
{
newEt = Db.EntityTypes.Add(et);
}
}
else
{
newEt = Db.EntityTypes.Add(et);
}
Db.SaveChanges();
IdMap.Add(oldId, newEt.Id);
}
//add aggregations
GuidMap.Clear();
int completed = 0;
int failed = 0;
IDictionary <Type, Func <IngestionService, CFXmlModel, CFAggregation> > createAggrigationCache = new Dictionary <Type, Func <IngestionService, CFXmlModel, CFAggregation> >();
Func <CFXmlModel, CFAggregation> getNewAggregation = agg =>
{
Type t = agg.GetType();
try
{
return(createAggrigationCache[t](this, agg));
}
catch (KeyNotFoundException ex)
{
MethodInfo method = this.GetType().GetMethod("CreateAggregation");
MethodInfo genMethod = method.MakeGenericMethod(t);
Func <IngestionService, CFXmlModel, CFAggregation> func = (Func <IngestionService, CFXmlModel, CFAggregation>)Delegate.CreateDelegate(typeof(Func <IngestionService, CFXmlModel, CFAggregation>), genMethod);
createAggrigationCache.Add(t, func);
Console.WriteLine("Exception : {0}", ex.Message);
return(func(this, agg));
}
};
ingestion.Aggregations.ForEach((agg) => {
Regex guidRegex = new Regex(@"(guid)=[""']?((?:.(?![""']?\s + (?:\S +)=|[> ""']))+.)[""']?");
string oldId = guidRegex.Match(agg.Content).Groups[2].Value;
string newGuid = NewGuid();
//saving the aggregation object
if (ingestion.Overwrite)
{
//use whatever GUID in the file
//check to make sure the GUID in not existing in the Db, if it's replace the one in the database
GuidMap.Add(oldId, oldId); //if overwrite is true use existing GUID
CFAggregation _aggregation = Db.XmlModels.Where(a => a.MappedGuid == oldId).FirstOrDefault() as CFAggregation;
if (_aggregation != null)
{
_aggregation = (CFAggregation)agg;
Db.Entry(_aggregation).State = System.Data.Entity.EntityState.Modified;
}
else
{
var _agg = getNewAggregation(agg);
Db.Entities.Add(_agg);
}
}
else
{
agg.Guid = newGuid;
agg.MappedGuid = newGuid;
GuidMap.Add(oldId, newGuid);
var _agg = getNewAggregation(agg);
try
{
Db.Entities.Add(_agg);
}catch (Exception ex)
{
#if DEBUG
Console.Error.WriteLine("{0} Error reading aggrigation: {1}", ex.Message, _agg.Name);
#endif
return(false);
}
}
return(true);
}, (successCount, failCount) =>
{
completed += successCount;
failed += failCount;
#if DEBUG
Console.Error.WriteLine("{0} Completed, {1} Failed", completed, failed);
#endif
Db.SaveChanges();
return(true);
});
Db.SaveChanges();
foreach (var rel in ingestion.Relationships)
{
string pGuid = rel.ParentRef;
if (GuidMap.ContainsKey(pGuid))
{
pGuid = GuidMap[pGuid];
}
string cGuid = rel.ChildRef;
if (GuidMap.ContainsKey(cGuid))
{
cGuid = GuidMap[cGuid];
}
rel.ParentRef = pGuid;
rel.ChildRef = cGuid;
if (rel.IsMember)
{
AddMember(rel.ParentRef, rel.ChildRef);
}
if (rel.IsRelated)
{
//save it in Aggregation has related
AddRelatedMember(rel.ParentRef, rel.ChildRef);
}
}
}
private void AsserXmlEqualIngestion(XElement xml, Ingestion ingestion)
{
Assert.AreEqual(xml.Name.LocalName, "ingestion");
Assert.AreEqual(bool.Parse(xml.Attribute("overwrite").Value), ingestion.Overwrite);
List <string> attributes = new List <string>()
{
"overwrite"
};
List <string> elements = new List <string>()
{
"metadata-sets", "entity-types", "aggregations", "relationships"
};
foreach (XAttribute attribute in xml.Attributes())
{
string name = attribute.Name.LocalName;
if (name == "overwrite")
{
Assert.AreEqual(bool.Parse(attribute.Value), ingestion.Overwrite);
attributes.Remove("overwrite");
}
else
{
Assert.Fail("Invalid attribute \"{0}\" found.", name);
}
}
foreach (XElement element in xml.Elements())
{
string name = element.Name.LocalName;
if (name == "metadata-sets")
{
AssertMetadatasetsXmlEqualIngestion(element, ingestion);
elements.Remove("metadata-sets");
}
else if (name == "entity-types")
{
AssertEntityTypesXmlEqualIngestion(element, ingestion);
elements.Remove("entity-types");
}
else if (name == "aggregations")
{
AssertAggrigationsEqualIngestion(element, ingestion);
elements.Remove("aggregations");
}
else if (name == "relationships")
{
AssertRelationshipsEqualIngestion(element, ingestion);
elements.Remove("relationships");
}
else
{
Assert.Fail("Unkown element \"{0}\" found.", name);
}
}
Assert.AreEqual(attributes.Count, 0);
Assert.AreEqual(elements.Count, 0);
}