public long GetRecordCountToScrub(ScrubRule scrubbingRule)
{
//var SourceCosmosDBSettings = CloneSettings.GetConfigurationSection("SourceCosmosDBSettings");
string DatabaseName = CloneSettings.TargetSettings.DatabaseName;
string CollectionName = CloneSettings.TargetSettings.CollectionName;
string totalCountQuery;
if (string.IsNullOrEmpty(scrubbingRule.FilterCondition))
{
totalCountQuery = "SELECT VALUE COUNT(1) FROM c";
}
else
{
totalCountQuery = "SELECT VALUE COUNT(1) FROM c" + " where " + scrubbingRule.FilterCondition;
}
FeedOptions queryOptions = new FeedOptions {
MaxItemCount = -1, EnableCrossPartitionQuery = true
};
long sourceTotalRecordCount;
using (var cosmosClient = GetTargetDocumentDbClient())
{
sourceTotalRecordCount = cosmosClient.CreateDocumentQuery <long>(
UriFactory.CreateDocumentCollectionUri(DatabaseName, CollectionName), totalCountQuery, queryOptions)
.AsEnumerable().First();
}
return(sourceTotalRecordCount);
}
private void CreateScrubRule(ScrubRule scrubRule = null)
{
WrapPanel parentStackPanelLeft = (WrapPanel)this.FindName("WrapPanel");
int newRuleIndex = newIndex(scrubRule);
AddRuleWithData(parentStackPanelLeft, newRuleIndex, scrubRule);
}
private void CreateScrubRule(ScrubRule scrubRule = null)
{
StackPanel parentStackPanelLeft = (StackPanel)this.FindName("RulesStackPanelLeft");
StackPanel parentStackPanelRight = (StackPanel)this.FindName("RulesStackPanelRight");
int newRuleIndex = newIndex(scrubRule);
if (newRuleIndex % 2 == 1)
{
AddRuleWithData(parentStackPanelLeft, newRuleIndex, scrubRule);
}
else
{
AddRuleWithData(parentStackPanelRight, newRuleIndex, scrubRule);
}
}
public static void TestCosmosScrubbing()
{
var tcs = new DataScrubMigrator();
// ScrubRule rule1 = new ScrubRule("c.id=\"a5e66b8b-a57c-4788-a194-58d3735a9854\"", "c.CompositeName", RuleType.Singlevalue, "Test overwritten Value xyz",1);
//ScrubRule rule2 = new ScrubRule("", "c.SuperKeys.KeyValue", RuleType.Shuffle, "",2);
//ScrubRule rule3 = new ScrubRule("", "c.EntityValue.Name", RuleType.Shuffle, "", 3);
//ScrubRule rule4 = new ScrubRule("", "c.EntityValue.Description", RuleType.Singlevalue, "OverWrite Filtered rule Description", 4);
//ScrubRule rule3 = new ScrubRule("c.id=\"1402e84d-e034-45f9-8064-3ab174119e4f\"", "c.EntityValue.Name", RuleType.Singlevalue,"Test overwritten EntityName", 3);
var scrubRules = new List <ScrubRule>();
//scrubRules.Add(rule2);
//scrubRules.Add(rule3);
//scrubRules.Add(rule4);
//ScrubRule rule5 = new ScrubRule("c.EntityType=\"External\"", "c.EmailAddress", RuleType.Singlevalue, "*****@*****.**", 4);
ScrubRule rule6 = new ScrubRule("c.id=\"2826d281-3a8b-4408-b064-efff26e26119\"", "c.EmailAddress", RuleType.SingleValue, "*****@*****.**", 4);
scrubRules.Add(rule6);
var documentMigrator = new CosmosCloneCommon.Migrator.DocumentMigrator();
documentMigrator.StartCopy(scrubRules).Wait();
//var result = tcs.StartScrub(scrubRules);
}
int newIndex(ScrubRule scrubRule = null)
{
int newRuleIndex = 0;
if (scrubRule == null)
{
newRuleIndex = RuleIndex++;
}
else
{
if (scrubRule.RuleId == 0)
{
newRuleIndex = RuleIndex++;
}
else
{
newRuleIndex = scrubRule.RuleId;
}
}
return(newRuleIndex);
}
public List <JToken> ScrubObjectList(List <string> srcList, ScrubRule scrubRule)
{
//var scrubbedObjects = new List<string>();
var scrubbedObjects = new List <JToken>();
var propNames = scrubRule.PropertyName.Split('.').ToList();
if (scrubRule.Type == RuleType.NullValue || scrubRule.Type == RuleType.SingleValue || scrubRule.Type == RuleType.PartialMaskFromLeft || scrubRule.Type == RuleType.PartialMaskFromRight)
{
foreach (var strObj in srcList)
{
try
{
JToken jToken = GetUpdatedJsonArrayValue((JToken)JObject.Parse(strObj), propNames, scrubRule.UpdateValue, scrubRule.Type);
scrubbedObjects.Add(jToken);
}
catch (Exception ex)
{
CloneLogger.LogInfo("Log failed");
CloneLogger.LogError(ex);
throw;
}
}
}
else if (scrubRule.Type == RuleType.Shuffle)
{
//get all similar values
var propertyValues = new List <JToken>();
foreach (var strObj in srcList)
{
try
{
List <JToken> jTokenList = new List <JToken>();
GetPropertyValues((JToken)JObject.Parse(strObj), propNames, ref jTokenList);
propertyValues.AddRange(jTokenList);
}
catch (Exception ex)
{
CloneLogger.LogInfo("Log failed");
CloneLogger.LogError(ex);
throw;
}
}
var shuffledTokens = RandomNumberGenerator.Shuffle(propertyValues);
var shuffledTokenQ = new Queue <JToken>(shuffledTokens);
foreach (var strObj in srcList)
{
try
{
JToken jToken = GetDocumentShuffledToken((JToken)JObject.Parse(strObj), propNames, ref shuffledTokenQ);
scrubbedObjects.Add(jToken);
}
catch (Exception ex)
{
CloneLogger.LogInfo("Log failed");
CloneLogger.LogError(ex);
throw;
}
}
}
else
{
foreach (var strObj in srcList)
{
scrubbedObjects.Add((JToken)strObj);
}
}
return(scrubbedObjects);
}
void AddRuleWithData(WrapPanel parentStackPanel, int ruleIndex, ScrubRule scrubRule = null)
{
StackPanel RuleHeadersp = new StackPanel();
RuleHeadersp.Orientation = Orientation.Horizontal;
TextBlock HeaderTB = new TextBlock();
HeaderTB.Text = "Rule " + (ruleIndex);
Label RuleIdLabel = new Label();
RuleIdLabel.Content = ruleIndex;
RuleIdLabel.Visibility = Visibility.Hidden;
Button DeleteBtn = new Button();
DeleteBtn.Name = "btnDelete_" + ruleIndex;
DeleteBtn.HorizontalAlignment = HorizontalAlignment.Right;
//DeleteBtn.Content = "Delete";
DeleteBtn.Margin = new Thickness(265, 0, 0, 0);
DeleteBtn.Click += DeleteBtn_Click;
DeleteBtn.Content = new Image
{
Source = new BitmapImage(new Uri("/Images/closeIcon.png", UriKind.Relative)),
VerticalAlignment = VerticalAlignment.Center
};
RuleHeadersp.Children.Add(HeaderTB);
RuleHeadersp.Children.Add(DeleteBtn);
Expander exp = new Expander();
exp.Header = RuleHeadersp;
exp.HorizontalAlignment = HorizontalAlignment.Left;
exp.Margin = new Thickness(20, 30, 0, 0);
exp.BorderThickness = new Thickness(1, 1, 1, 1);
exp.Width = 350;
exp.BorderBrush = Brushes.Gray;
exp.Background = Brushes.White;
exp.IsExpanded = true;
exp.Name = "RuleExpander_" + ruleIndex;
StackPanel sp = new StackPanel();
sp.Orientation = Orientation.Vertical;
sp.Margin = new Thickness(20, 5, 0, 0);
StackPanel filterSP = new StackPanel();
filterSP.Orientation = Orientation.Horizontal;
filterSP.Margin = new Thickness(0, 5, 0, 2);
StackPanel attributeSP = new StackPanel();
attributeSP.Orientation = Orientation.Horizontal;
attributeSP.Margin = new Thickness(0, 5, 0, 2);
StackPanel scrubTypeSP = new StackPanel();
scrubTypeSP.Orientation = Orientation.Horizontal;
scrubTypeSP.Margin = new Thickness(0, 5, 0, 2);
StackPanel findValueSP = new StackPanel();
findValueSP.Orientation = Orientation.Horizontal;
findValueSP.Margin = new Thickness(0, 5, 0, 2);
StackPanel scrubValueSP = new StackPanel();
scrubValueSP.Orientation = Orientation.Horizontal;
scrubValueSP.Margin = new Thickness(0, 5, 0, 2);
TextBlock FilterLabel = new TextBlock();
FilterLabel.VerticalAlignment = VerticalAlignment.Center;
FilterLabel.Margin = new Thickness(10, 0, 0, 0);
//FilterLabel.MaxWidth = 150;
Run runFilterLabel = new Run();
runFilterLabel.Text = "Filter Query";
runFilterLabel.FontSize = 15;
Run runFilterLabelHint = new Run();
runFilterLabelHint.Text = " \nEx: c.Type = \"document\"";
runFilterLabelHint.FontSize = 10;
FilterLabel.Inlines.Add(runFilterLabel);
FilterLabel.Inlines.Add(runFilterLabelHint);
TextBox FilterTB = new TextBox();
FilterTB.Name = "Filter" + ruleIndex;
FilterTB.Width = 150;
FilterTB.HorizontalContentAlignment = HorizontalAlignment.Left;
FilterTB.VerticalAlignment = VerticalAlignment.Center;
FilterTB.Margin = new Thickness(20, 0, 0, 0);
TextBlock AttributeScrubLabel = new TextBlock();
AttributeScrubLabel.VerticalAlignment = VerticalAlignment.Center;
AttributeScrubLabel.Margin = new Thickness(10, 0, 0, 0);
//AttributeScrubLabel.Width = 120;
Run runAttributeScrubLabel = new Run();
runAttributeScrubLabel.Text = "Attribute To Scrub";
runAttributeScrubLabel.FontSize = 15;
Run runAttributeScrubLabelHint = new Run();
runAttributeScrubLabelHint.Text = " \nEx: c.Person.Email";
runAttributeScrubLabelHint.FontSize = 10;
AttributeScrubLabel.Inlines.Add(runAttributeScrubLabel);
AttributeScrubLabel.Inlines.Add(runAttributeScrubLabelHint);
TextBox AttributeScrubTB = new TextBox();
AttributeScrubTB.Name = "ScrubAttribute" + ruleIndex;
AttributeScrubTB.Width = 150;
AttributeScrubTB.HorizontalContentAlignment = HorizontalAlignment.Left;
AttributeScrubTB.VerticalAlignment = VerticalAlignment.Center;
AttributeScrubTB.Margin = new Thickness(20, 0, 0, 0);
TextBlock ScrubTypeLabel = new TextBlock();
ScrubTypeLabel.Text = "Scrub Type";
ScrubTypeLabel.FontSize = 15;
ScrubTypeLabel.VerticalAlignment = VerticalAlignment.Center;
ScrubTypeLabel.Margin = new Thickness(10, 0, 0, 0);
ComboBox ScrubTypeCB = new ComboBox();
ScrubTypeCB.Name = "ScrubType" + ruleIndex;
ScrubTypeCB.Width = 150;
ScrubTypeCB.Margin = new Thickness(20, 0, 0, 0);
foreach (RuleType val in Enum.GetValues(typeof(RuleType)))
{
ScrubTypeCB.Items.Add(val.ToString());
}
ScrubTypeCB.SelectionChanged += new SelectionChangedEventHandler(scrubTypeComboBox_SelectedIndexChanged);
TextBlock FindValueLabel = new TextBlock();
FindValueLabel.Text = "Find";
FindValueLabel.FontSize = 15;
FindValueLabel.VerticalAlignment = VerticalAlignment.Center;
FindValueLabel.Margin = new Thickness(10, 0, 0, 5);
TextBox FindValueTB = new TextBox();
FindValueTB.Name = "FindValue" + ruleIndex;
FindValueTB.Width = 150;
FindValueTB.HorizontalContentAlignment = HorizontalAlignment.Left;
FindValueTB.VerticalAlignment = VerticalAlignment.Center;
FindValueTB.Margin = new Thickness(20, 0, 0, 0);
TextBlock ScrubValueLabel = new TextBlock();
ScrubValueLabel.Text = "Replace with";
ScrubValueLabel.FontSize = 15;
ScrubValueLabel.VerticalAlignment = VerticalAlignment.Center;
ScrubValueLabel.Margin = new Thickness(10, 0, 0, 5);
TextBox ScrubValueTB = new TextBox();
ScrubValueTB.Name = "ScrubValue" + ruleIndex;
ScrubValueTB.Width = 150;
ScrubValueTB.HorizontalContentAlignment = HorizontalAlignment.Left;
ScrubValueTB.VerticalAlignment = VerticalAlignment.Center;
ScrubValueTB.Margin = new Thickness(20, 0, 0, 0);
FilterLabel.Width = 120;
FilterTB.Width = 150;
AttributeScrubLabel.Width = 120;
AttributeScrubTB.Width = 150;
ScrubTypeLabel.Width = 120;
ScrubTypeCB.Width = 150;
ScrubValueLabel.Width = 120;
FindValueTB.Width = 150;
FindValueLabel.Width = 120;
filterSP.Children.Add(FilterLabel);
filterSP.Children.Add(FilterTB);
attributeSP.Children.Add(AttributeScrubLabel);
attributeSP.Children.Add(AttributeScrubTB);
scrubTypeSP.Children.Add(ScrubTypeLabel);
scrubTypeSP.Children.Add(ScrubTypeCB);
findValueSP.Children.Add(FindValueLabel);
findValueSP.Children.Add(FindValueTB);
findValueSP.Visibility = Visibility.Collapsed;
scrubValueSP.Children.Add(ScrubValueLabel);
scrubValueSP.Children.Add(ScrubValueTB);
scrubValueSP.Children.Add(RuleIdLabel);
scrubValueSP.Visibility = Visibility.Hidden;
sp.Children.Add(attributeSP);
sp.Children.Add(filterSP);
sp.Children.Add(scrubTypeSP);
sp.Children.Add(findValueSP);
sp.Children.Add(scrubValueSP);
exp.Content = sp;
if (scrubRule != null)
{
FilterTB.Text = scrubRule.FilterCondition;
AttributeScrubTB.Text = scrubRule.PropertyName;
if (scrubRule.Type != null)
{
ScrubTypeCB.SelectedIndex = (int)scrubRule.Type;
}
ScrubValueTB.Text = scrubRule.UpdateValue;
FindValueTB.Text = scrubRule.FindValue;
}
parentStackPanel.Children.Add(exp);
if (!SaveRuleButton.IsEnabled)
{
SaveRuleButton.IsEnabled = true;
}
}
public List <ScrubRule> getScrubRules()
{
//List<ScrubRule> sb = new List<ScrubRule>();
TextBox filterCondition = (TextBox)this.FindName("FilterCondition");
//sb.filterQuery = filterCondition.Text;
List <ScrubRule> srList = new List <ScrubRule>();
WrapPanel wrapPanel = (WrapPanel)this.FindName("WrapPanel");
foreach (UIElement SPUI in wrapPanel.Children)
{
Expander exp = (Expander)SPUI;
StackPanel lrsp = (StackPanel)exp.Content;
UIElementCollection uiElementsSP = lrsp.Children;
ScrubRule sr = new ScrubRule();
foreach (UIElement uiElementSP in uiElementsSP)
{
StackPanel tempSP = (StackPanel)uiElementSP;
UIElementCollection uiElements = tempSP.Children;
foreach (UIElement uiElement in uiElements)
{
if (uiElement.GetType().Name == "Label")
{
var ruleIdLabel = (Label)uiElement;
int ruleId;
if (int.TryParse(ruleIdLabel.Content.ToString(), out ruleId))
{
sr.RuleId = ruleId;
}
else
{
sr.RuleId = 0;
}
}
if (uiElement.GetType().Name == "TextBox")
{
TextBox tb = (TextBox)uiElement;
if (tb.Name.StartsWith("Filter"))
{
sr.FilterCondition = tb.Text.Trim();
}
else if (tb.Name.StartsWith("ScrubAttribute"))
{
sr.PropertyName = tb.Text.Trim();
}
else if (tb.Name.StartsWith("ScrubValue"))
{
sr.UpdateValue = tb.Text.Trim();
}
else if (tb.Name.StartsWith("FindValue"))
{
sr.FindValue = tb.Text.Trim();
}
}
if (uiElement.GetType().Name == "ComboBox")
{
ComboBox cb = (ComboBox)uiElement;
if (cb.Name.StartsWith("ScrubType"))
{
//sr.Type = (RuleType) Enum.Parse(typeof(RuleType), cb.Text);
RuleType rType;
if (Enum.TryParse <RuleType>(cb.Text, out rType))
{
sr.Type = rType;
}
else
{
sr.Type = null;
}
}
}
}
}
srList.Add(sr);
}
return(srList);
}
