protected void LoadCurrentMetrics() { // fill the metrics using mapping CurrentMetricsUnit = new MetricsUnit { GetEdgeField = GetEdgeField }; MetricsMappings.Apply(CurrentMetricsUnit); var signature = new Signature(); SignatureMappings.Apply(signature); // check if signature is already exists in delivery outputs var output = Delivery.Outputs.FirstOrDefault(x => x.Signature == signature.Value); // attach output to Metrics: take existing or create new if (output != null) { CurrentMetricsUnit.Output = output; } else { var deliveryOutput = new DeliveryOutput { Signature = signature.Value, TimePeriodStart = CurrentMetricsUnit.TimePeriodStart, TimePeriodEnd = CurrentMetricsUnit.TimePeriodEnd, Account = CurrentMetricsUnit.Account, Channel = CurrentMetricsUnit.Channel }; Delivery.Outputs.Add(deliveryOutput); CurrentMetricsUnit.Output = deliveryOutput; } }
public void ImportMetrics(DeliveryOutput targetOutput, MetricsUnit metrics) { EnsureBeginImport(); foreach (EdgeObject obj in metrics.GetObjectDimensions()) { } }
protected override void OnRead() { var ad = new Ad(); _adMappings.Apply(ad); this.ImportManager.ImportAd(ad); var metrics = new AdMetricsUnit(); metrics.Ad = ad; _metricsMappings.Apply(metrics); var signature = new Signature(); _signatureMappings.Apply(signature); //checking if signature is already exists in delivery outputs var outputs = from output in this.Delivery.Outputs where output.Signature.Equals(signature.ToString()) select output; DeliveryOutput op = outputs.FirstOrDefault <DeliveryOutput>(); if (op != null) { //Attaching output to Metrics (metrics as AdMetricsUnit).Output = op; } else { DeliveryOutput deliveryOutput = new DeliveryOutput() { Signature = signature.Value, TimePeriodStart = metrics.TimePeriodStart, TimePeriodEnd = metrics.TimePeriodEnd, Account = metrics.Ad.Account, Channel = metrics.Ad.Channel }; this.Delivery.Outputs.Add(deliveryOutput); //Attaching output to Metrics (metrics as AdMetricsUnit).Output = deliveryOutput; } this.ImportManager.ImportMetrics(metrics); }
protected override void OnRead() { var metrics = new GenericMetricsUnit(); _metricsMappings.Apply(metrics); //Writing to Log //if (metrics.Output.Checksum.Count() == 0) //{ // Edge.Core.Utilities.Log("Output checksum is empty",Core.Utilities.LogMessageType.Information); //} var signature = new Signature(); _signatureMappings.Apply(signature); //checking if signature is already exists in delivery outputs var outputs = from output in this.Delivery.Outputs where output.Signature.Equals(signature.ToString()) select output; DeliveryOutput op = outputs.FirstOrDefault <DeliveryOutput>(); if (op != null) { //Attaching output to Metrics (metrics as GenericMetricsUnit).Output = op; } else { DeliveryOutput deliveryOutput = new DeliveryOutput() { Signature = signature.Value, TimePeriodStart = metrics.TimePeriodStart, TimePeriodEnd = metrics.TimePeriodEnd, Account = metrics.Account, Channel = metrics.Channel }; this.Delivery.Outputs.Add(deliveryOutput); //Attaching output to Metrics (metrics as GenericMetricsUnit).Output = deliveryOutput; } this.ImportManager.ImportMetrics(metrics); }
protected override void OnRollbackOutput(DeliveryOutput output, int pass) { string guid = output.OutputID.ToString("N"); _rollbackCommand = _rollbackCommand ?? DataManager.CreateCommand(this.Options.SqlRollbackCommand, CommandType.StoredProcedure); _rollbackCommand.Connection = _sqlConnection; _rollbackCommand.Transaction = _rollbackTransaction; _rollbackCommand.Parameters["@DeliveryOutputID"].Value = guid; _rollbackCommand.Parameters["@TableName"].Value = output.Parameters[Consts.DeliveryOutputParameters.CommitTableName]; _rollbackCommand.ExecuteNonQuery(); // This is redundant (SP already does this) but to sync our objects in memory we do it here also output.Status = DeliveryOutputStatus.RolledBack; //For new db /*string guid = output.OutputID.ToString("N"); * if (output.Status == DeliveryOutputStatus.Staged) * { * _rollbackCommand = _rollbackCommand ?? DataManager.CreateCommand(this.Options.SqlRollbackCommand, CommandType.StoredProcedure); * _rollbackCommand.Connection = _sqlConnection; * _rollbackCommand.Transaction = _rollbackTransaction; * * _rollbackCommand.Parameters["@DeliveryOutputID"].Value = guid; * _rollbackCommand.Parameters["@TableName"].Value = output.Parameters[Consts.DeliveryHistoryParameters.CommitTableName]; * * _rollbackCommand.ExecuteNonQuery(); * output.Status=DeliveryOutputStatus.Canceled; * } * else if (output.Status == DeliveryOutputStatus.Committed) * { * output.Status = DeliveryOutputStatus.PendingRoleBack; * } * else * { * throw new Exception("It should not happend"); * } */ }
protected override void OnRollbackOutput(DeliveryOutput output, int pass) { string guid = output.OutputID.ToString("N"); if (output.Status == DeliveryOutputStatus.Staged) { _rollbackCommand = _rollbackCommand ?? SqlUtility.CreateCommand(Options.SqlRollbackCommand, CommandType.StoredProcedure); _rollbackCommand.Connection = _deliverySqlConnection; _rollbackCommand.Transaction = _rollbackTransaction; _rollbackCommand.Parameters["@DeliveryOutputID"].Value = guid; _rollbackCommand.Parameters["@StagingTable"].Value = output.Parameters[Consts.DeliveryHistoryParameters.StagingMetricsTableName]; _rollbackCommand.ExecuteNonQuery(); } else if (output.Status == DeliveryOutputStatus.Committed) { output.Status = DeliveryOutputStatus.RollbackPending; } else { throw new InvalidOperationException("Delivery output cannot be rolled back because it is neither staged nor committed."); } }
protected override void OnStage(Delivery delivery, int pass) { // get this from last 'Processed' history entry string measuresFieldNamesSQL = delivery.Parameters[Consts.DeliveryOutputParameters.MeasureFieldsSql].ToString(); string measuresNamesSQL = delivery.Parameters[Consts.DeliveryOutputParameters.MeasureNamesSql].ToString(); string tablePerfix = delivery.Parameters[Consts.DeliveryOutputParameters.TablePerfix].ToString(); string deliveryId = delivery.DeliveryID.ToString("N"); // ........................... // COMMIT data to OLTP _stageCommand = _stageCommand ?? DataManager.CreateCommand(this.Options.SqlStageCommand, CommandType.StoredProcedure); _stageCommand.Connection = _sqlConnection; _stageCommand.Transaction = _stageTransaction; _stageCommand.Parameters["@DeliveryFileName"].Size = 4000; _stageCommand.Parameters["@DeliveryFileName"].Value = tablePerfix; _stageCommand.Parameters["@CommitTableName"].Size = 4000; _stageCommand.Parameters["@CommitTableName"].Value = delivery.Parameters["CommitTableName"]; _stageCommand.Parameters["@MeasuresNamesSQL"].Size = 4000; _stageCommand.Parameters["@MeasuresNamesSQL"].Value = measuresNamesSQL; _stageCommand.Parameters["@MeasuresFieldNamesSQL"].Size = 4000; _stageCommand.Parameters["@MeasuresFieldNamesSQL"].Value = measuresFieldNamesSQL; _stageCommand.Parameters["@OutputIDsPerSignature"].Size = 8000; _stageCommand.Parameters["@OutputIDsPerSignature"].Direction = ParameterDirection.Output; _stageCommand.Parameters["@DeliveryID"].Size = 4000; _stageCommand.Parameters["@DeliveryID"].Value = deliveryId; try { _stageCommand.ExecuteNonQuery(); // _commitTransaction.Commit(); string outPutsIDsPerSignature = _stageCommand.Parameters["@OutputIDsPerSignature"].Value.ToString(); string[] existsOutPuts; if ((!string.IsNullOrEmpty(outPutsIDsPerSignature) && outPutsIDsPerSignature != "0")) { _stageTransaction.Rollback(); existsOutPuts = outPutsIDsPerSignature.Split(','); List <DeliveryOutput> outputs = new List <DeliveryOutput>(); foreach (string existOutput in existsOutPuts) { DeliveryOutput o = DeliveryOutput.Get(Guid.Parse(existOutput)); o.Parameters[Consts.DeliveryOutputParameters.CommitTableName] = delivery.Parameters["CommitTableName"]; outputs.Add(o); } throw new DeliveryConflictException(string.Format("DeliveryOutputs with the same signature are already committed in the database\n Deliveries:\n {0}:", outPutsIDsPerSignature)) { ConflictingOutputs = outputs.ToArray() }; } else { //already updated by sp, this is so we don't override it foreach (var output in delivery.Outputs) { output.Status = DeliveryOutputStatus.Staged; } } } finally { this.State = DeliveryImportManagerState.Idle; } }
protected override Core.Services.ServiceOutcome DoPipelineWork() { Dictionary <Consts.FileTypes, List <string> > filesByType = (Dictionary <Consts.FileTypes, List <string> >)Delivery.Parameters["FilesByType"]; StringBuilder warningsStr = new StringBuilder(); Dictionary <string, Campaign> campaignsData = new Dictionary <string, Campaign>(); Dictionary <string, Ad> ads = new Dictionary <string, Ad>(); Dictionary <string, List <Ad> > adsBycreatives = new Dictionary <string, List <Ad> >(); DeliveryOutput currentOutput = Delivery.Outputs.First(); currentOutput.Checksum = new Dictionary <string, double>(); using (this.ImportManager = new AdMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions() { MeasureOptions = MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice, MeasureOptionsOperator = OptionsOperator.Not, SegmentOptions = Data.Objects.SegmentOptions.All, SegmentOptionsOperator = OptionsOperator.And })) { this.ImportManager.BeginImport(this.Delivery); #region Campaigns List <string> campaignsFiles = filesByType[Consts.FileTypes.Campaigns]; foreach (var campaignFile in campaignsFiles) { DeliveryFile campaigns = this.Delivery.Files[campaignFile]; var campaignsReader = new JsonDynamicReader(campaigns.OpenContents(), "$.data[*].*"); using (campaignsReader) { while (campaignsReader.Read()) { Campaign camp = new Campaign() { Name = campaignsReader.Current.name, OriginalID = Convert.ToString(campaignsReader.Current.id), }; //string campaignStatus = campaignsReader.Current.campaign_status; //switch (campaignStatus) //{ // case "ACTIVE": // camp.Status = ObjectStatus.Active; // break; // case "PAUSED": // camp.Status = ObjectStatus.Paused; // break; // case "DELETED": // camp.Status = ObjectStatus.Deleted; // break; //} campaignsData.Add(camp.OriginalID, camp); } } } #endregion this.ReportProgress(0.1); #region adGroups And Targeting List <string> adGroupsFiles = filesByType[Consts.FileTypes.AdGroups]; foreach (var adGroup in adGroupsFiles) { DeliveryFile adGroups = this.Delivery.Files[adGroup]; var adGroupsReader = new JsonDynamicReader(FileManager.Open(adGroups.Location), "$.data[*].*"); using (adGroupsReader) { while (adGroupsReader.Read()) { Ad ad = new Ad(); ad.OriginalID = Convert.ToString(adGroupsReader.Current.id); ad.Segments = new Dictionary <Segment, SegmentObject>(); ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]] = campaignsData[Convert.ToString(adGroupsReader.Current.campaign_id)]; ad.Name = adGroupsReader.Current.name; ad.Channel = new Channel() { ID = 6 }; ad.Account = new Account() { ID = this.Delivery.Account.ID, OriginalID = this.Delivery.Account.OriginalID.ToString() }; if (Instance.Configuration.Options.ContainsKey("AutoAdGroupSegment") && Instance.Configuration.Options["AutoAdGroupSegment"].ToLower() == "true") { string[] delimiter = new string[1]; delimiter[0] = string.Empty; if (!Instance.Configuration.Options.ContainsKey("AdGroupDelimiter")) { Edge.Core.Utilities.Log.Write(string.Format("Facebook{0}", this), Core.Utilities.LogMessageType.Warning); } else { delimiter[0] = Instance.Configuration.Options["AdGroupDelimiter"]; } //Note for Ronen - this section if relevant for AD SETS ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.AdGroup]] = new AdGroup() { Campaign = (Campaign)ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]], Value = delimiter[0] == string.Empty ? ad.Name : ad.Name.Split(delimiter, StringSplitOptions.None)[0], OriginalID = delimiter[0] == string.Empty ? (ad.Name + ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]].OriginalID + ad.Account.ID) : (ad.Name.Split(delimiter, StringSplitOptions.None)[0] + ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]].OriginalID + ad.Account.ID) }; } else { ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.AdGroup]] = new AdGroup() { Value = ad.Name, OriginalID = ad.Name + ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]].OriginalID + ad.Account.ID }; } // adgroup targeting string age_min = string.Empty; if (((Dictionary <string, object>)adGroupsReader.Current.targeting).ContainsKey("age_min")) { age_min = adGroupsReader.Current.targeting["age_min"]; } if (!string.IsNullOrEmpty(age_min)) { AgeTarget ageTarget = new AgeTarget() { FromAge = int.Parse(age_min), ToAge = int.Parse(adGroupsReader.Current.targeting["age_max"]) }; ad.Targets.Add(ageTarget); } List <object> genders = null; if (((Dictionary <string, object>)adGroupsReader.Current.targeting).ContainsKey("genders")) { genders = adGroupsReader.Current.targeting["genders"]; } if (genders != null) { foreach (object gender in genders) { GenderTarget genderTarget = new GenderTarget(); if (gender.ToString() == "1") { genderTarget.Gender = Gender.Male; } else if (gender.ToString() == "2") { genderTarget.Gender = Gender.Female; } else { genderTarget.Gender = Gender.Unspecified; } genderTarget.OriginalID = gender.ToString(); ad.Targets.Add(genderTarget); } } if (adGroupsReader.Current.creative_ids != null) { foreach (string creative in adGroupsReader.Current.creative_ids) { if (!adsBycreatives.ContainsKey(creative)) { adsBycreatives.Add(creative, new List <Ad>()); } adsBycreatives[creative].Add(ad); } } ads.Add(ad.OriginalID, ad); } } } #endregion #region AdGroupStats start new import session //GetAdGroupStats #region for validation foreach (var measure in this.ImportManager.Measures) { if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired)) { if (!currentOutput.Checksum.ContainsKey(measure.Key)) { currentOutput.Checksum.Add(measure.Key, 0); //TODO : SHOULD BE NULL BUT SINCE CAN'T ADD NULLABLE ...TEMP } } } #endregion if (filesByType.ContainsKey(Consts.FileTypes.ConversionsStats)) { List <string> convStatsFiles = filesByType[Consts.FileTypes.ConversionsStats]; foreach (var convStat in convStatsFiles) { DeliveryFile conversionStatsFile = this.Delivery.Files[convStat]; //Getting Next conversion file URL var conversionStatsReader = new JsonDynamicReader(conversionStatsFile.OpenContents(), "$.data[*].*"); using (conversionStatsReader) { //Get Stats from conversion file } } } if (filesByType.ContainsKey(Consts.FileTypes.AdGroupStats)) { List <string> adGroupStatsFiles = filesByType[Consts.FileTypes.AdGroupStats]; foreach (var adGroupStat in adGroupStatsFiles) { DeliveryFile adGroupStats = this.Delivery.Files[adGroupStat]; var adGroupStatsReader = new JsonDynamicReader(adGroupStats.OpenContents(), "$.data[*].*"); using (adGroupStatsReader) { while (adGroupStatsReader.Read()) { AdMetricsUnit adMetricsUnit = new AdMetricsUnit(); adMetricsUnit.Output = currentOutput; adMetricsUnit.MeasureValues = new Dictionary <Measure, double>(); Ad tempAd; if (adGroupStatsReader.Current.adgroup_id != null) { if (ads.TryGetValue(adGroupStatsReader.Current.adgroup_id, out tempAd)) { adMetricsUnit.Ad = tempAd; //adMetricsUnit.PeriodStart = this.Delivery.TimePeriodDefinition.Start.ToDateTime(); //adMetricsUnit.PeriodEnd = this.Delivery.TimePeriodDefinition.End.ToDateTime(); // Common and Facebook specific meausures /* Sets totals for validations */ if (currentOutput.Checksum.ContainsKey(Measure.Common.Clicks)) { currentOutput.Checksum[Measure.Common.Clicks] += Convert.ToDouble(adGroupStatsReader.Current.clicks); } if (currentOutput.Checksum.ContainsKey(Measure.Common.Impressions)) { currentOutput.Checksum[Measure.Common.Impressions] += Convert.ToDouble(adGroupStatsReader.Current.impressions); } if (currentOutput.Checksum.ContainsKey(Measure.Common.Cost)) { currentOutput.Checksum[Measure.Common.Cost] += Convert.ToDouble(adGroupStatsReader.Current.spent) / 100d; } /* Sets measures values */ adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Clicks], Convert.ToInt64(adGroupStatsReader.Current.clicks)); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.UniqueClicks], Convert.ToInt64(adGroupStatsReader.Current.unique_clicks)); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Impressions], Convert.ToInt64(adGroupStatsReader.Current.impressions)); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.UniqueImpressions], Convert.ToInt64(adGroupStatsReader.Current.unique_impressions)); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Cost], Convert.ToDouble(Convert.ToDouble(adGroupStatsReader.Current.spent) / 100d)); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialImpressions], double.Parse(adGroupStatsReader.Current.social_impressions)); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialUniqueImpressions], double.Parse(adGroupStatsReader.Current.social_unique_impressions)); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialClicks], double.Parse(adGroupStatsReader.Current.social_clicks)); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialUniqueClicks], double.Parse(adGroupStatsReader.Current.social_unique_clicks)); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialCost], Convert.ToDouble(adGroupStatsReader.Current.social_spent) / 100d); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.Actions], 0); //adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.Connections], double.Parse(adGroupStatsReader.Current.connections)); this.ImportManager.ImportMetrics(adMetricsUnit); } else { warningsStr.AppendLine(string.Format("Ad {0} does not exist in the stats report delivery id: {1}", adGroupStatsReader.Current.id, this.Delivery.DeliveryID)); } } else { warningsStr.AppendLine("adGroupStatsReader.Current.id=null"); } } this.ReportProgress(0.4); } #endregion this.ReportProgress(0.6); #region Creatives List <string> creativeFiles = filesByType[Consts.FileTypes.Creatives]; Dictionary <string, string> usedCreatives = new Dictionary <string, string>(); foreach (string creative in creativeFiles) { DeliveryFile creativeFile = Delivery.Files[creative]; var adGroupCreativesReader = new JsonDynamicReader(creativeFile.OpenContents(), "$.data[*].*"); using (adGroupCreativesReader) { this.Mappings.OnFieldRequired = field => adGroupCreativesReader.Current[field]; while (adGroupCreativesReader.Read()) { List <Ad> adsByCreativeID = null; if (adsBycreatives.ContainsKey(adGroupCreativesReader.Current.id)) { if (!usedCreatives.ContainsKey(adGroupCreativesReader.Current.id)) { usedCreatives.Add(adGroupCreativesReader.Current.id, adGroupCreativesReader.Current.id); adsByCreativeID = adsBycreatives[adGroupCreativesReader.Current.id]; } } if (adsByCreativeID != null) { foreach (Ad ad in adsByCreativeID) { // Ronen , please see Naama's file of Ad types ad.DestinationUrl = adGroupCreativesReader.Current.object_url; /*Get Data from Mapping E.g Tracker*/ if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad))) { this.Mappings.Objects[typeof(Ad)].Apply(ad); } ad.Creatives = new List <Creative>(); switch ((string)adGroupCreativesReader.Current.type) { //case "8": // deprecated //case "9": // deprecated //case "10": // deprecated //case "16": // deprecated //case "17": // deprecated //case "19": // deprecated case "25": { TextCreative sponserStory = new TextCreative() { OriginalID = adGroupCreativesReader.Current.id, TextType = TextCreativeType.Title, Text = "Sponsored Story", }; ad.DestinationUrl = "Sponsored Story"; ad.Creatives.Add(sponserStory); break; } case "27": { TextCreative sponserStory = new TextCreative() { OriginalID = adGroupCreativesReader.Current.id, TextType = TextCreativeType.Title, Text = "Page Ads for a Page post" }; ad.DestinationUrl = "Page Ads for a Page post"; ad.Creatives.Add(sponserStory); break; } case "1": case "2": case "3": case "4": case "12": { ImageCreative ic = new ImageCreative() { ImageUrl = adGroupCreativesReader.Current.image_url, OriginalID = adGroupCreativesReader.Current.id //Name = adGroupCreativesReader.Current.name }; if (!string.IsNullOrEmpty(ic.ImageUrl)) { ad.Creatives.Add(ic); } TextCreative bc = new TextCreative() { OriginalID = adGroupCreativesReader.Current.id, TextType = TextCreativeType.Body, Text = adGroupCreativesReader.Current.body //Name = adGroupCreativesReader.Current.name }; if (!string.IsNullOrEmpty(bc.Text)) { ad.Creatives.Add(bc); } //bug creative type =9 story like TextCreative tc = new TextCreative() { OriginalID = adGroupCreativesReader.Current.id, TextType = TextCreativeType.Title, Text = adGroupCreativesReader.Current.title }; if (!string.IsNullOrEmpty(bc.Text)) { ad.Creatives.Add(tc); } break; } default: { TextCreative unknown = new TextCreative() { OriginalID = adGroupCreativesReader.Current.id, TextType = TextCreativeType.Title, Text = "UnKnown creative" }; ad.DestinationUrl = "UnKnown creative"; ad.Creatives.Add(unknown); break; } } this.ImportManager.ImportAd(ad); } } //TODO: REPORT PROGRESS 2 ReportProgress(PROGRESS) } } #endregion } } } currentOutput.Status = DeliveryOutputStatus.Imported; this.ImportManager.EndImport(); if (!string.IsNullOrEmpty(warningsStr.ToString())) { Log.Write(warningsStr.ToString(), LogMessageType.Warning); } } return(Core.Services.ServiceOutcome.Success); }
protected override IEnumerable <ValidationResult> Validate() { Channel channel = new Channel(); progress += 0.1; this.ReportProgress(progress); #region Getting Service option params //Getting Accounts list string[] accounts; if (this.Instance.AccountID == -1) { if (String.IsNullOrEmpty(this.Instance.Configuration.Options["AccountsList"])) { throw new Exception("Missing Configuration option AccountsList"); } accounts = this.Instance.Configuration.Options["AccountsList"].Split(','); } else { List <string> account = new List <string>() { this.Instance.AccountID.ToString() }; accounts = account.ToArray(); } //Getting Table string comparisonTable; if (String.IsNullOrEmpty(this.Instance.Configuration.Options["SourceTable"])) { throw new Exception("Missing Configuration option SourceTable"); } else { comparisonTable = this.Instance.Configuration.Options["SourceTable"]; } //Getting Channel List if (String.IsNullOrEmpty(this.Instance.Configuration.Options["ChannelList"])) { throw new Exception("Missing Configuration option ChannelList"); } string[] channels = this.Instance.Configuration.Options["ChannelList"].Split(','); //Getting TimePeriod DateTime fromDate, toDate; if ((String.IsNullOrEmpty(this.Instance.Configuration.Options["fromDate"])) && (String.IsNullOrEmpty(this.Instance.Configuration.Options["toDate"]))) { fromDate = this.TimePeriod.Start.ToDateTime(); toDate = this.TimePeriod.End.ToDateTime(); } else { fromDate = Convert.ToDateTime(this.Instance.Configuration.Options["fromDate"]); toDate = Convert.ToDateTime(this.Instance.Configuration.Options["toDate"]); } //Get Sub Channel (for cases such as Google Search and GDN) //string subChannel = String.Empty; //if (!String.IsNullOrEmpty(this.Instance.Configuration.Options["SubChannel"])) //{ // subChannel = this.Instance.Configuration.Options["SubChannel"].ToString(); //} #endregion if (this.Delivery == null || this.Delivery.DeliveryID.Equals(Guid.Empty)) { #region Creating Delivery Search List List <DeliveryOutput> deliveryOutputSearchList = new List <DeliveryOutput>(); while (fromDate <= toDate) { // {start: {base : '2009-01-01', h:0}, end: {base: '2009-01-01', h:'*'}} var subRange = new DateTimeRange() { Start = new DateTimeSpecification() { BaseDateTime = fromDate, Hour = new DateTimeTransformation() { Type = DateTimeTransformationType.Exact, Value = 0 }, }, End = new DateTimeSpecification() { BaseDateTime = fromDate, Hour = new DateTimeTransformation() { Type = DateTimeTransformationType.Max }, } }; foreach (var Channel in channels) { foreach (string account in accounts) { DeliveryOutput deliveryOutput = new DeliveryOutput(); deliveryOutput.Account = new Account() { ID = Convert.ToInt32(account) }; deliveryOutput.Channel = new Channel() { ID = Convert.ToInt32(Channel) }; deliveryOutput.TimePeriodStart = subRange.Start.ToDateTime(); deliveryOutput.TimePeriodEnd = subRange.End.ToDateTime(); deliveryOutputSearchList.Add(deliveryOutput); progress += 0.3 * ((1 - progress) / (channels.LongLength + accounts.LongLength)); this.ReportProgress(progress); } } fromDate = fromDate.AddDays(1); } #endregion foreach (DeliveryOutput deliveryOutput in deliveryOutputSearchList) { //Getting criterion matched deliveries DeliveryOutput[] outputToCheck = DeliveryOutput.GetByTimePeriod(deliveryOutput.TimePeriodStart, deliveryOutput.TimePeriodEnd, deliveryOutput.Channel, deliveryOutput.Account); bool foundCommited = false; progress += 0.3 * (1 - progress); this.ReportProgress(progress); //Get Relevant Outputs by source table var outputToCheck_withSubChannel = from o in outputToCheck where o.Parameters.ContainsKey("CommitTableName") && o.Parameters["CommitTableName"].Equals(comparisonTable.Split('.')[1]) select o; outputToCheck = outputToCheck_withSubChannel.ToArray <DeliveryOutput>(); foreach (DeliveryOutput output in outputToCheck) { // if (output.Parameters.ContainsKey("SubChannel") && output.Parameters["SubChannel"].Equals(subChannel)) if ((output.Status == DeliveryOutputStatus.Committed || output.Status == DeliveryOutputStatus.Staged) && output.Checksum != null && output.Checksum.Count > 0) { //Check Delivery data vs OLTP foundCommited = true; yield return(DeliveryDbCompare(output, output.Checksum, "OltpDB", comparisonTable)); } } //could not find deliveries by user criterions if (outputToCheck.Length == 0) { yield return(new ValidationResult() { ResultType = ValidationResultType.Error, AccountID = deliveryOutput.Account.ID, TargetPeriodStart = deliveryOutput.TimePeriodStart, TargetPeriodEnd = deliveryOutput.TimePeriodEnd, Message = "Cannot find outputs in DB", ChannelID = deliveryOutput.Channel.ID, CheckType = this.Instance.Configuration.Name }); } else if (!foundCommited) { yield return(new ValidationResult() { ResultType = ValidationResultType.Error, AccountID = deliveryOutput.Account.ID, TargetPeriodStart = deliveryOutput.TimePeriodStart, TargetPeriodEnd = deliveryOutput.TimePeriodEnd, Message = "Cannot find Commited output in DB", ChannelID = deliveryOutput.Channel.ID, CheckType = this.Instance.Configuration.Name }); } } // End of foreach } else { //Getting current Delivery totals foreach (DeliveryOutput output in this.Delivery.Outputs) { if ((output.Status == DeliveryOutputStatus.Committed || output.Status == DeliveryOutputStatus.Staged) && output.Checksum != null && output.Checksum.Count > 0) { yield return(DeliveryDbCompare(output, output.Checksum, "OltpDB", comparisonTable)); } } } }
protected override ServiceOutcome DoPipelineWork() { MappingContainer metricsUnitMapping; if (!Mappings.Objects.TryGetValue(typeof(MetricsUnit), out metricsUnitMapping)) { throw new MappingConfigurationException("Missing mapping definition for GenericMetricsUnit."); } _currentOutput = Delivery.Outputs.First(); _currentOutput.Checksum = new Dictionary <string, double>(); var columns = new Dictionary <string, int>(); foreach (var reportFile in Delivery.Files) { //Get Columns var reportReader = new JsonDynamicReader(reportFile.OpenContents(), "$.columnHeaders[*].*"); using (reportReader) { var colIndex = 0; while (reportReader.Read()) { columns.Add(reportReader.Current.name, colIndex); colIndex++; } } // get sample unit to create metrics table reportReader = new JsonDynamicReader(reportFile.OpenContents(), "$.rows[*].*"); using (reportReader) { Mappings.OnFieldRequired = field => reportReader.Current["array"][columns[field]]; reportReader.Read(); CurrentMetricsUnit = new MetricsUnit { GetEdgeField = GetEdgeField }; metricsUnitMapping.Apply(CurrentMetricsUnit); using (ImportManager = new MetricsDeliveryManager(InstanceID, EdgeTypes)) { ImportManager.BeginImport(Delivery, CurrentMetricsUnit); //Get values reportReader = new JsonDynamicReader(reportFile.OpenContents(), "$.rows[*].*"); using (reportReader) { Mappings.OnFieldRequired = field => reportReader.Current["array"][columns[field]]; while (reportReader.Read()) { CurrentMetricsUnit = new MetricsUnit { GetEdgeField = GetEdgeField }; metricsUnitMapping.Apply(CurrentMetricsUnit); ProcessMetrics(); } } // finish import (insert edge objects) ImportManager.EndImport(); } } } return(ServiceOutcome.Success); }
protected override Core.Services.ServiceOutcome DoPipelineWork() { Dictionary <Consts.FileTypes, List <string> > filesByType = (Dictionary <Consts.FileTypes, List <string> >)Delivery.Parameters["FilesByType"]; StringBuilder warningsStr = new StringBuilder(); Dictionary <string, Campaign> campaignsData = new Dictionary <string, Campaign>(); Dictionary <string, AdGroup> adGroupsData = new Dictionary <string, AdGroup>(); Dictionary <string, Ad> ads = new Dictionary <string, Ad>(); Dictionary <string, List <Ad> > adsBycreatives = new Dictionary <string, List <Ad> >(); var adStatIds = new Dictionary <string, string>(); var insertedAds = new Dictionary <string, string>(); var storyIds = new Dictionary <string, Dictionary <string, string> >(); var photoIds = new Dictionary <string, Dictionary <string, string> >(); DeliveryOutput currentOutput = Delivery.Outputs.First(); currentOutput.Checksum = new Dictionary <string, double>(); using (this.ImportManager = new AdMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions() { MeasureOptions = MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice, MeasureOptionsOperator = OptionsOperator.Not, SegmentOptions = Data.Objects.SegmentOptions.All, SegmentOptionsOperator = OptionsOperator.And })) { this.ImportManager.BeginImport(this.Delivery); #region AdSets if (filesByType.ContainsKey(Consts.FileTypes.CampaignGroups)) { List <string> campaignsFiles = filesByType[Consts.FileTypes.CampaignGroups]; foreach (var campaignFile in campaignsFiles) { DeliveryFile campaigns = this.Delivery.Files[campaignFile]; var campaignsReader = new JsonDynamicReader(campaigns.OpenContents(), "$.data[*].*"); using (campaignsReader) { while (campaignsReader.Read()) { Campaign camp = new Campaign() { Name = campaignsReader.Current.name, OriginalID = Convert.ToString(campaignsReader.Current.id), }; campaignsData.Add(camp.OriginalID, camp); } } } } #endregion #region AdGroups if (filesByType.ContainsKey(Consts.FileTypes.AdSets)) { List <string> adSetList = filesByType[Consts.FileTypes.AdSets]; foreach (var adSet in adSetList) { DeliveryFile adSetDF = this.Delivery.Files[adSet]; var adSetReader = new JsonDynamicReader(adSetDF.OpenContents(), "$.data[*].*"); using (adSetReader) { while (adSetReader.Read()) { var adGroupObj = new AdGroup() { Value = adSetReader.Current.name, OriginalID = Convert.ToString(adSetReader.Current.id), }; if (campaignsData.ContainsKey(adSetReader.Current.campaign_group_id)) { adGroupObj.Campaign = campaignsData[adSetReader.Current.campaign_group_id]; } adGroupsData.Add(adGroupObj.OriginalID, adGroupObj); } } } } #endregion #region adGroups And Targeting //****************************************************************************************************************************************************** if (filesByType.ContainsKey(Consts.FileTypes.AdGroups)) { List <string> adGroupsFiles = filesByType[Consts.FileTypes.AdGroups]; foreach (var adGroup in adGroupsFiles) { #region foreach DeliveryFile adGroups = this.Delivery.Files[adGroup]; var adGroupsReader = new JsonDynamicReader(FileManager.Open(adGroups.Location), "$.data[*].*"); using (adGroupsReader) { while (adGroupsReader.Read()) { var campaignId = Convert.ToString(adGroupsReader.Current.campaign_id); if (adGroupsData.ContainsKey(campaignId) && ((AdGroup)adGroupsData[campaignId]).Campaign != null) { Ad ad = new Ad(); ad.OriginalID = Convert.ToString(adGroupsReader.Current.id); ad.Segments = new Dictionary <Segment, SegmentObject>(); ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]] = ((AdGroup)adGroupsData[Convert.ToString(adGroupsReader.Current.campaign_id)]).Campaign; ad.Name = adGroupsReader.Current.name; ad.Channel = new Channel() { ID = 6 }; ad.Account = new Account() { ID = this.Delivery.Account.ID, OriginalID = this.Delivery.Account.OriginalID.ToString() }; if (adGroupsData.ContainsKey(adGroupsReader.Current.campaign_id)) { ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.AdGroup]] = adGroupsData[adGroupsReader.Current.campaign_id]; } // adgroup targeting string age_min = string.Empty; if (((Dictionary <string, object>)adGroupsReader.Current.targeting).ContainsKey("age_min")) { age_min = adGroupsReader.Current.targeting["age_min"]; } if (!string.IsNullOrEmpty(age_min)) { AgeTarget ageTarget = new AgeTarget() { FromAge = int.Parse(age_min), ToAge = int.Parse(adGroupsReader.Current.targeting["age_max"]) }; ad.Targets.Add(ageTarget); } List <object> genders = null; if (((Dictionary <string, object>)adGroupsReader.Current.targeting).ContainsKey("genders")) { genders = adGroupsReader.Current.targeting["genders"]; } if (genders != null) { foreach (object gender in genders) { GenderTarget genderTarget = new GenderTarget(); if (gender.ToString() == "1") { genderTarget.Gender = Gender.Male; } else if (gender.ToString() == "2") { genderTarget.Gender = Gender.Female; } else { genderTarget.Gender = Gender.Unspecified; } genderTarget.OriginalID = gender.ToString(); ad.Targets.Add(genderTarget); } } if (adGroupsReader.Current.creative_ids != null) { foreach (string creative in adGroupsReader.Current.creative_ids) { if (!adsBycreatives.ContainsKey(creative)) { adsBycreatives.Add(creative, new List <Ad>()); } adsBycreatives[creative].Add(ad); } } ads.Add(ad.OriginalID, ad); } } } #endregion } } //****************************************************************************************************************************************************** #endregion adGroups And Targeting //GetAdGroupStats #region for validation foreach (var measure in this.ImportManager.Measures) { if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired)) { if (!currentOutput.Checksum.ContainsKey(measure.Key)) { currentOutput.Checksum.Add(measure.Key, 0); //TODO : SHOULD BE NULL BUT SINCE CAN'T ADD NULLABLE ...TEMP } } } //************************************************************************** #endregion Dictionary <string, List <Dictionary <string, object> > > conversion_data = new Dictionary <string, List <Dictionary <string, object> > >(); if (filesByType.ContainsKey(Consts.FileTypes.ConversionsStats)) { #region Conversions List <string> conversionFiles = filesByType[Consts.FileTypes.ConversionsStats]; foreach (string conversionFile in conversionFiles) { DeliveryFile creativeFile = Delivery.Files[conversionFile]; var conversionCreativesReader = new JsonDynamicReader(creativeFile.OpenContents(), "$.data[*].*"); using (conversionCreativesReader) { this.Mappings.OnFieldRequired = field => conversionCreativesReader.Current[field]; while (conversionCreativesReader.Read()) { if (((Edge.Data.Pipeline.DynamicDictionaryObject)(conversionCreativesReader.Current)).Values == null) { break; // in case of empty data (e.g. data [] ) } if (!conversion_data.ContainsKey(conversionCreativesReader.Current.adgroup_id)) { Dictionary <string, string> adConversionData = new Dictionary <string, string>(); //string TotalConversionsManyPerClick, Leads, Signups, Purchases; List <object> actions = conversionCreativesReader.Current.actions; if (actions.Count > 0) { List <Dictionary <string, object> > action_list = actions.Select(s => (Dictionary <string, object>)s).ToList(); List <Dictionary <string, object> > action_items = action_list.Where(dict => dict.ContainsKey("action_type") && dict.ContainsKey("value") && ( dict["action_type"].ToString().Equals("offsite_conversion") || dict["action_type"].ToString().Equals("offsite_conversion.lead") || dict["action_type"].ToString().Equals("offsite_conversion.registration") || dict["action_type"].ToString().Equals("offsite_conversion.add_to_cart") || dict["action_type"].ToString().Equals("offsite_conversion.checkout") || dict["action_type"].ToString().Equals("offsite_conversion.key_page_view") || dict["action_type"].ToString().Equals("offsite_conversion.other") ) ).ToList(); if (action_items.Count > 0) { conversion_data.Add(conversionCreativesReader.Current.adgroup_id, action_items); } } } } //End of While read } // End of Using reader } //End Foreach conversion file #endregion } //End if contains conversion file #region AdGroupStats start new import session List <string> adGroupStatsFiles = filesByType[Consts.FileTypes.AdGroupStats]; foreach (var adGroupStat in adGroupStatsFiles) { DeliveryFile adGroupStats = this.Delivery.Files[adGroupStat]; var adGroupStatsReader = new JsonDynamicReader(adGroupStats.OpenContents(), "$.data[*].*"); using (adGroupStatsReader) { while (adGroupStatsReader.Read()) { #region Create Metrics AdMetricsUnit adMetricsUnit = new AdMetricsUnit(); adMetricsUnit.Output = currentOutput; adMetricsUnit.MeasureValues = new Dictionary <Measure, double>(); Ad tempAd; try { var x = adGroupStatsReader.Current.adgroup_id; } catch (Exception) { continue; } if (adGroupStatsReader.Current.adgroup_id != null) { adStatIds[adGroupStatsReader.Current.adgroup_id] = adGroupStatsReader.Current.adgroup_id; if (ads.TryGetValue(adGroupStatsReader.Current.adgroup_id, out tempAd)) { adMetricsUnit.Ad = tempAd; //adMetricsUnit.PeriodStart = this.Delivery.TimePeriodDefinition.Start.ToDateTime(); //adMetricsUnit.PeriodEnd = this.Delivery.TimePeriodDefinition.End.ToDateTime(); // Common and Facebook specific meausures /* Sets totals for validations */ if (currentOutput.Checksum.ContainsKey(Measure.Common.Clicks)) { currentOutput.Checksum[Measure.Common.Clicks] += Convert.ToDouble(adGroupStatsReader.Current.clicks); } if (currentOutput.Checksum.ContainsKey(Measure.Common.Impressions)) { currentOutput.Checksum[Measure.Common.Impressions] += Convert.ToDouble(adGroupStatsReader.Current.impressions); } if (currentOutput.Checksum.ContainsKey(Measure.Common.Cost)) { currentOutput.Checksum[Measure.Common.Cost] += Convert.ToDouble(adGroupStatsReader.Current.spent) / 100d; } /* Sets measures values */ adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Clicks], Convert.ToInt64(adGroupStatsReader.Current.clicks)); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.UniqueClicks], Convert.ToInt64(adGroupStatsReader.Current.unique_clicks)); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Impressions], Convert.ToInt64(adGroupStatsReader.Current.impressions)); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.UniqueImpressions], Convert.ToInt64(adGroupStatsReader.Current.unique_impressions)); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Cost], Convert.ToDouble(Convert.ToDouble(adGroupStatsReader.Current.spent) / 100d)); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialImpressions], double.Parse(adGroupStatsReader.Current.social_impressions)); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialUniqueImpressions], double.Parse(adGroupStatsReader.Current.social_unique_impressions)); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialClicks], double.Parse(adGroupStatsReader.Current.social_clicks)); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialUniqueClicks], double.Parse(adGroupStatsReader.Current.social_unique_clicks)); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialCost], Convert.ToDouble(adGroupStatsReader.Current.social_spent) / 100d); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.Actions], 0); //adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.Connections], double.Parse(adGroupStatsReader.Current.connections)); //Setting conversions from conversion files data List <Dictionary <string, object> > adgroup_conversion_data; if (conversion_data.TryGetValue(adGroupStatsReader.Current.adgroup_id, out adgroup_conversion_data)) { var TotalConversionsManyPerClick = from element in adgroup_conversion_data where element["action_type"].Equals("offsite_conversion") select element["value"]; var Leads = from element in adgroup_conversion_data where element["action_type"].Equals("offsite_conversion.lead") select element["value"]; var Signups = from element in adgroup_conversion_data where element["action_type"].Equals("offsite_conversion.registration") select element["value"]; //TO DO : Get field from Configuration var Purchases = from element in adgroup_conversion_data where element["action_type"].Equals(this.Delivery.Parameters["API_Purchases_Field_Name"]) select element["value"]; //add values to metrics unit adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.TotalConversionsManyPerClick], Convert.ToDouble(TotalConversionsManyPerClick.DefaultIfEmpty("0").FirstOrDefault())); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.Leads], Convert.ToDouble(Leads.DefaultIfEmpty("0").FirstOrDefault())); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.Signups], Convert.ToDouble(Signups.DefaultIfEmpty("0").FirstOrDefault())); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.Purchases], Convert.ToDouble(Purchases.DefaultIfEmpty("0").FirstOrDefault())); } this.ImportManager.ImportMetrics(adMetricsUnit); } else { warningsStr.AppendLine(string.Format("Ad {0} does not exist in the stats report delivery id: {1}", adGroupStatsReader.Current.id, this.Delivery.DeliveryID)); } } else { warningsStr.AppendLine("adGroupStatsReader.Current.id=null"); } #endregion } } } #endregion AdGroupStats start new import session #region Creatives List <string> creativeFiles = filesByType[Consts.FileTypes.Creatives]; Dictionary <string, string> usedCreatives = new Dictionary <string, string>(); foreach (string creative in creativeFiles) { DeliveryFile creativeFile = Delivery.Files[creative]; var adGroupCreativesReader = new JsonDynamicReader(creativeFile.OpenContents(), "$.data[*].*"); using (adGroupCreativesReader) { //this.Mappings.OnFieldRequired = field => if((field == "object_url" && adGroupCreativesReader.Current[field] != null) || field != "object_url")adGroupCreativesReader.Current[field]; this.Mappings.OnFieldRequired = field => adGroupCreativesReader.Current[field]; while (adGroupCreativesReader.Read()) { List <Ad> adsByCreativeID = null; if (adsBycreatives.ContainsKey(adGroupCreativesReader.Current.id)) { if (!usedCreatives.ContainsKey(adGroupCreativesReader.Current.id)) { usedCreatives.Add(adGroupCreativesReader.Current.id, adGroupCreativesReader.Current.id); adsByCreativeID = adsBycreatives[adGroupCreativesReader.Current.id]; } } if (adsByCreativeID != null) { foreach (Ad ad in adsByCreativeID) { if (!adStatIds.ContainsKey(ad.OriginalID)) { continue; } ad.Creatives = new List <Creative>(); if (!string.IsNullOrEmpty(adGroupCreativesReader.Current.object_type)) { string objectType = adGroupCreativesReader.Current.object_type; switch (objectType.ToUpper()) { case "SHARE": { #region Ads Type SHARE if (!string.IsNullOrEmpty(adGroupCreativesReader.Current.object_story_id)) { Dictionary <string, string> shareCreativeData; string object_story_id = adGroupCreativesReader.Current.object_story_id; if (storyIds.ContainsKey(object_story_id)) { shareCreativeData = storyIds[object_story_id]; } else { var accessToken = this.Instance.Configuration.Options[FacebookConfigurationOptions.Auth_AccessToken]; shareCreativeData = GetShareCreativeData(object_story_id, accessToken); } ad.DestinationUrl = shareCreativeData["link"]; if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad))) { this.Mappings.Objects[typeof(Ad)].Apply(ad); var trackersReadCommands = from n in this.Mappings.Objects[typeof(Ad)].ReadCommands where n.Field.Equals("link_url") select n; foreach (var command in trackersReadCommands) { string trackerValue = ApplyRegex(ad.DestinationUrl, command.RegexPattern); if (!String.IsNullOrEmpty(trackerValue)) { ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]].Value = trackerValue; ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]].OriginalID = trackerValue; } } } ad.Creatives.Add(GetTextCreative(shareCreativeData["text"], adGroupCreativesReader)); ad.Creatives.Add(GetBodyCreative(shareCreativeData["description"], adGroupCreativesReader)); ad.Creatives.Add(GetImageCreative(shareCreativeData["picture"], adGroupCreativesReader)); } #endregion break; } case "DOMAIN": { #region Ads Type DOMAIN if (!string.IsNullOrEmpty(adGroupCreativesReader.Current.object_url)) { if (Instance.Configuration.Options.ContainsKey(FacebookConfigurationOptions.AdGroupCreativeFields)) { ad.DestinationUrl = adGroupCreativesReader.Current.object_url; } } else if (!string.IsNullOrEmpty(adGroupCreativesReader.Current.link_url)) { ad.DestinationUrl = adGroupCreativesReader.Current.link_url; } else { ad.DestinationUrl = "UnKnown Url"; } /*Get Data from Mapping E.g Tracker*/ if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad))) { this.Mappings.Objects[typeof(Ad)].Apply(ad); } if (adGroupCreativesReader.Current.image_url != null) { CreateImageCreatives(ad, adGroupCreativesReader); } ad.Creatives.Add(GetTextCreative(adGroupCreativesReader)); ad.Creatives.Add(GetBodyCreative(adGroupCreativesReader)); #endregion break; } default: { #region Ads Type not handeled string adtype = objectType.ToUpper(); ad.DestinationUrl = adtype + " Ad Type"; /*Get Data from Mapping E.g Tracker*/ if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad))) { this.Mappings.Objects[typeof(Ad)].Apply(ad); } TextCreative unhandeledTextAd_title = new TextCreative() { OriginalID = ad.OriginalID, TextType = TextCreativeType.Title, Text = adtype + " Ad Type" }; ad.Creatives.Add(unhandeledTextAd_title); #endregion break; } } if (!insertedAds.ContainsKey(ad.OriginalID)) { insertedAds[ad.OriginalID] = ad.OriginalID; this.ImportManager.ImportAd(ad); } } } } //TODO: REPORT PROGRESS 2 ReportProgress(PROGRESS) } } #endregion } // End foreach creative currentOutput.Status = DeliveryOutputStatus.Imported; this.ImportManager.EndImport(); if (!string.IsNullOrEmpty(warningsStr.ToString())) { Log.Write(warningsStr.ToString(), LogMessageType.Warning); } } return(Core.Services.ServiceOutcome.Success); }
protected override ServiceOutcome DoPipelineWork() { currentOutput = Delivery.Outputs.First(); currentOutput.Checksum = new Dictionary <string, double>(); // TODO: add checks for delivery state string[] requiredHeaders; requiredHeaders = new string[1]; requiredHeaders[0] = WS.CampaignPerformanceReportColumn.CampaignName.ToString(); _campaignsCache = new Dictionary <string, Campaign>(); _adCache = new Dictionary <long, Ad>(); DeliveryFile adReport = this.Delivery.Files[Const.Files.AdReport]; DeliveryFile campaignReport = this.Delivery.Files[Const.Files.CampaignReport]; FileInfo campaignReportFileInfo = campaignReport.GetFileInfo(ArchiveType.Zip); string[] campaignReportSubFiles = campaignReportFileInfo.GetSubFiles(); var campaignReportReader = new CsvDynamicReader( campaignReport.OpenContents(subLocation: campaignReportSubFiles[0], archiveType: ArchiveType.Zip), requiredHeaders ); #region Reading campaigns file while (campaignReportReader.Read()) { string accountNameColVal = campaignReportReader.Current[WS.CampaignPerformanceReportColumn.AccountName.ToString()]; if (accountNameColVal.Trim() == string.Empty || accountNameColVal.Trim().Contains(endOfFileMicrosoftCorporation.Trim())) //end of file { break; } Campaign campaign = CreateCampaign(campaignReportReader.Current); _campaignsCache.Add(campaign.Name, campaign); } this.ReportProgress(0.2); #endregion #region Read the ad report, and build a lookup table for later using (this.ImportManager = new AdMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions() { MeasureOptions = MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice, MeasureOptionsOperator = OptionsOperator.Not, SegmentOptions = Data.Objects.SegmentOptions.All, SegmentOptionsOperator = OptionsOperator.And })) { ImportManager.BeginImport(this.Delivery); // create the ad report reader requiredHeaders = new string[1]; requiredHeaders[0] = WS.AdPerformanceReportColumn.AdId.ToString(); string[] adReportSubFiles = adReport.GetFileInfo(ArchiveType.Zip).GetSubFiles(); var adReportReader = new CsvDynamicReader( adReport.OpenContents(subLocation: adReportSubFiles[0], archiveType: ArchiveType.Zip), requiredHeaders); //read using (adReportReader) { while (adReportReader.Read()) { // create the ad this.Mappings.OnFieldRequired = field => adReportReader.Current[field]; string accountNameColVal = adReportReader.Current[WS.AdPerformanceReportColumn.AccountName.ToString()]; //end of file if (accountNameColVal.Trim() == string.Empty || accountNameColVal.Trim() == endOfFileMicrosoftCorporation.Trim()) { break; } Ad ad = CreateAd(adReportReader.Current, ImportManager); //TRACKER if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad))) { this.Mappings.Objects[typeof(Ad)].Apply(ad); } //ADDING CAMPAIGN ad.Segments[this.ImportManager. SegmentTypes[Segment.Common.Campaign]] = _campaignsCache[adReportReader.Current[WS.AdPerformanceReportColumn.CampaignName.ToString()]]; //CREATING Adgroup AdGroup adGroup = new AdGroup() { Value = adReportReader.Current[WS.AdPerformanceReportColumn.AdGroupName.ToString()], Campaign = (Campaign)ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]], OriginalID = adReportReader.Current[WS.AdPerformanceReportColumn.AdGroupId.ToString()] }; ad.Segments.Add(ImportManager.SegmentTypes[Segment.Common.AdGroup], adGroup); //CRAETING AD TYPE string adTypeKey = Convert.ToString(adReportReader.Current[WS.AdPerformanceReportColumn.AdType.ToString()]); ad.ExtraFields[AdType] = AdCenterAdTypeDic[adTypeKey]; ImportManager.ImportAd(ad); _adCache.Add(long.Parse(ad.OriginalID), ad); } } this.ReportProgress(0.7); #endregion // ............................................................... #region Read the keyword report, cross reference it with the ad data, and commit // Read the keyword report, cross reference it with the ad data, and commit // The name of the time period column is specified by the initializer, depending on the report DeliveryFile keywordReport = this.Delivery.Files[Const.Files.KeywordReport]; string timePeriodColumn = keywordReport.Parameters[Const.Parameters.TimePeriodColumnName] as string; // // create the keyword report reader requiredHeaders = new string[1]; requiredHeaders[0] = "Keyword"; string[] keywordReportSubFiles = keywordReport.GetFileInfo(ArchiveType.Zip).GetSubFiles(); var keywordReportReader = new CsvDynamicReader( keywordReport.OpenContents(subLocation: keywordReportSubFiles[0], archiveType: ArchiveType.Zip), requiredHeaders); //Added by Shay for validation foreach (KeyValuePair <string, Measure> measure in ImportManager.Measures) { if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired)) { currentOutput.Checksum.Add(measure.Key, 0); } } // read and save in transaction using (keywordReportReader) { while (keywordReportReader.Read()) { string GregorianDateColVal = keywordReportReader.Current.GregorianDate; if (GregorianDateColVal.Trim() == string.Empty || GregorianDateColVal.Trim() == endOfFileMicrosoftCorporation.Trim()) //end of file { break; } // get the unit from the keyword report, and add the missing ad data AdMetricsUnit unit = CreateMetrics(keywordReportReader.Current, timePeriodColumn, ImportManager); ImportManager.ImportMetrics(unit); // Validation information: currentOutput.Checksum[Measure.Common.Clicks] += unit.MeasureValues[ImportManager.Measures[Measure.Common.Clicks]]; currentOutput.Checksum[Measure.Common.Cost] += unit.MeasureValues[ImportManager.Measures[Measure.Common.Cost]]; currentOutput.Checksum[Measure.Common.Impressions] += unit.MeasureValues[ImportManager.Measures[Measure.Common.Impressions]]; } ImportManager.EndImport(); ReportProgress(1); } #endregion } return(ServiceOutcome.Success); }
protected override Core.Services.ServiceOutcome DoPipelineWork() { MappingContainer metricsUnitMapping; if (!this.Mappings.Objects.TryGetValue(typeof(GenericMetricsUnit), out metricsUnitMapping)) { throw new MappingConfigurationException("Missing mapping definition for GenericMetricsUnit."); } currentOutput = this.Delivery.Outputs.First(); currentOutput.Checksum = new Dictionary <string, double>(); foreach (var ReportFile in Delivery.Files) { bool isAttribute = Boolean.Parse(ReportFile.Parameters["Bo.IsAttribute"].ToString()); var ReportReader = new XmlDynamicReader (ReportFile.OpenContents(), ReportFile.Parameters["Bo.Xpath"].ToString()); using (this.ImportManager = new GenericMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions() { MeasureOptions = MeasureOptions.IsBackOffice, MeasureOptionsOperator = OptionsOperator.Or, SegmentOptions = Data.Objects.SegmentOptions.All, SegmentOptionsOperator = OptionsOperator.And })) { ImportManager.BeginImport(this.Delivery); //checksum init foreach (var m in ImportManager.Measures.Values) { if (m.Options.HasFlag(MeasureOptions.ValidationRequired)) { currentOutput.Checksum[m.Name] = 0; } } using (ReportReader) { dynamic readerHelper; while (ReportReader.Read()) { if (isAttribute) { readerHelper = ReportReader.Current.Attributes; } else { readerHelper = ReportReader.Current; } this.Mappings.OnFieldRequired = field => readerHelper[field]; GenericMetricsUnit genericMetricsUnit = new GenericMetricsUnit(); metricsUnitMapping.Apply(genericMetricsUnit); foreach (var m in genericMetricsUnit.MeasureValues) { if (m.Key.Options.HasFlag(MeasureOptions.ValidationRequired)) { if (!currentOutput.Checksum.ContainsKey(m.Key.Name)) { currentOutput.Checksum.Add(m.Key.Name, m.Value); } else { currentOutput.Checksum[m.Key.Name] += m.Value; } } } genericMetricsUnit.Output = currentOutput; ImportManager.ImportMetrics(genericMetricsUnit); } } ImportManager.EndImport(); } } return(Core.Services.ServiceOutcome.Success); }
protected override ValidationResult DeliveryDbCompare(DeliveryOutput deliveryOutput, Dictionary <string, double> totals, string DbConnectionStringName, string comparisonTable) { Dictionary <string, double> oltpTotals = new Dictionary <string, double>(); Dictionary <string, double> dwhTotals = new Dictionary <string, double>(); string dayCode = Convert.ToDateTime(Params["Date"]).ToString("yyyyMMdd"); #region Get Totals from Oltp using (SqlConnection sqlCon = new SqlConnection(AppSettings.GetConnectionString(this, "OltpDB"))) { sqlCon.Open(); SqlCommand sqlCommand = new SqlCommand( "SELECT SUM(cost),sum(imps),sum(clicks) from " + OltpTable + " where account_id = @Account_ID and Day_Code = @Daycode and Channel_ID = @Channel_ID" ); SqlParameter accountIdParam = new SqlParameter("@Account_ID", System.Data.SqlDbType.Int); SqlParameter daycodeParam = new SqlParameter("@Daycode", System.Data.SqlDbType.Int); SqlParameter channelIdParam = new SqlParameter("@Channel_ID", System.Data.SqlDbType.Int); accountIdParam.Value = Params["AccountID"]; daycodeParam.Value = dayCode; channelIdParam.Value = Params["ChannelID"]; sqlCommand.Parameters.Add(accountIdParam); sqlCommand.Parameters.Add(daycodeParam); sqlCommand.Parameters.Add(channelIdParam); sqlCommand.Connection = sqlCon; using (var _reader = sqlCommand.ExecuteReader()) { progress += 0.5 * (1 - progress); this.ReportProgress(progress); if (!_reader.IsClosed) { while (_reader.Read()) { if (!_reader[0].Equals(DBNull.Value)) { oltpTotals.Add("Cost", Convert.ToDouble(_reader[0])); oltpTotals.Add("Imps", Convert.ToDouble(_reader[1])); oltpTotals.Add("Clicks", Convert.ToDouble(_reader[2])); } } } } } #endregion #region Get Totals from DWH using (SqlConnection sqlCon = new SqlConnection(AppSettings.GetConnectionString(this, "DwhDB"))) { sqlCon.Open(); SqlCommand sqlCommand = new SqlCommand( "SELECT SUM(cost),sum(Impressions),sum(clicks) from " + DwhTabel + " where account_id = @Account_ID and Day_ID = @Daycode and Channel_ID = @Channel_ID" ); SqlParameter accountIdParam = new SqlParameter("@Account_ID", System.Data.SqlDbType.Int); SqlParameter daycodeParam = new SqlParameter("@Daycode", System.Data.SqlDbType.Int); SqlParameter channelIdParam = new SqlParameter("@Channel_ID", System.Data.SqlDbType.Int); accountIdParam.Value = Params["AccountID"]; daycodeParam.Value = dayCode; channelIdParam.Value = Params["ChannelID"]; sqlCommand.Parameters.Add(accountIdParam); sqlCommand.Parameters.Add(daycodeParam); sqlCommand.Parameters.Add(channelIdParam); sqlCommand.Connection = sqlCon; using (var _reader = sqlCommand.ExecuteReader()) { progress += 0.5 * (1 - progress); this.ReportProgress(progress); if (!_reader.IsClosed) { while (_reader.Read()) { if (!_reader[0].Equals(DBNull.Value)) { dwhTotals.Add("Cost", Convert.ToDouble(_reader[0])); dwhTotals.Add("Imps", Convert.ToDouble(_reader[1])); dwhTotals.Add("Clicks", Convert.ToDouble(_reader[2])); } } } } } #endregion #region Comparing totals results return(IsEqual(Params, oltpTotals, dwhTotals, "Oltp", "Dwh")); #endregion }
protected override ValidationResult DeliveryDbCompare(DeliveryOutput deliveryOutput, Dictionary <string, double> totals, string DbConnectionStringName, string comparisonTable) { string dayCode = deliveryOutput.TimePeriodStart.ToString("yyyyMMdd"); // Delivery Per Day = > TargetPeriod.Start = daycode Dictionary <string, Diffrence> diffs = new Dictionary <string, Diffrence>(); string checksumThreshold = Instance.Configuration.Options[Consts.ConfigurationOptions.ChecksumTheshold]; double allowedChecksumThreshold = checksumThreshold == null ? 0.01 : double.Parse(checksumThreshold); ValidationResult result; using (SqlConnection sqlCon = new SqlConnection(AppSettings.GetConnectionString(this, DbConnectionStringName))) { sqlCon.Open(); //Get measures need to be check StringBuilder sqlBuilder = new StringBuilder();; Dictionary <string, Measure> measures = Measure.GetMeasures(deliveryOutput.Account, deliveryOutput.Channel, sqlCon, MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice, OptionsOperator.Not); foreach (var measure in measures) { if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired)) { sqlBuilder.AppendFormat("{1}SUM([{0}]) as [{0}]", measure.Value.OltpName, sqlBuilder.Length > 0 ? ", " : null); } } sqlBuilder.Insert(0, "SELECT \n"); sqlBuilder.AppendFormat(" FROM {0}\n ", comparisonTable); sqlBuilder.Append("WHERE Account_ID=@accountID:int \nAND Day_Code=@daycode:int \nAND Channel_ID=@Channel_ID:int"); ///*******Work around*******************//////// if (deliveryOutput.Account.OriginalID != null) { sqlBuilder.Append(" \nand Account_ID_SRC=@OriginalID:nvarchar"); } SqlCommand sqlCommand = DataManager.CreateCommand(sqlBuilder.ToString()); sqlCommand.Parameters["@accountID"].Value = deliveryOutput.Account.ID; sqlCommand.Parameters["@daycode"].Value = dayCode;; sqlCommand.Parameters["@Channel_ID"].Value = deliveryOutput.Channel.ID; ///*******Work around*******************//////// if (deliveryOutput.Account.OriginalID != null) { sqlCommand.Parameters["@OriginalID"].Value = deliveryOutput.Account.OriginalID; } sqlCommand.Connection = sqlCon; using (var _reader = sqlCommand.ExecuteReader()) { if (!_reader.IsClosed) { if (_reader.Read()) { foreach (var measure in measures) { Diffrence difference = new Diffrence(); if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired)) { if (!_reader[measure.Value.OltpName].Equals(DBNull.Value)) { difference.Source = totals[measure.Value.Name]; difference.Target = Convert.ToDouble(_reader[measure.Value.OltpName]); difference.diff = Math.Abs((Convert.ToDouble(_reader[measure.Value.OltpName]) - totals[measure.Value.Name])); diffs[measure.Value.Name] = difference; } #region Scenario : data exists in delivery and not in DB else if (totals[measure.Value.Name] > 0) { return(new ValidationResult() { ResultType = ValidationResultType.Error, AccountID = deliveryOutput.Account.ID, DeliveryID = deliveryOutput.DeliveryID, TargetPeriodStart = deliveryOutput.TimePeriodStart, TargetPeriodEnd = deliveryOutput.TimePeriodEnd, Message = string.Format("Data exists in delivery but not in DB for Account {0} output {1}", deliveryOutput.Account.ID, deliveryOutput.OutputID.ToString("N")), ChannelID = deliveryOutput.Channel.ID, CheckType = this.Instance.Configuration.Name }); } else //diff is 0 { difference.Source = 0; difference.Target = 0; difference.diff = 0; diffs[measure.Value.Name] = difference; } #endregion } } StringBuilder errors = new StringBuilder(); foreach (KeyValuePair <string, Diffrence> diff in diffs) { if (diff.Value.diff / diff.Value.Source > allowedChecksumThreshold) { errors.AppendFormat("validation Error - differences has been found for Account-{0}-channel{1}-output{2}-measure{3} source{4} target {5}\n", deliveryOutput.Account.ID, deliveryOutput.Channel.ID, deliveryOutput.OutputID.ToString("N"), diff.Key, diff.Value.Source, diff.Value.Target); //TODO: USE } } if (errors.Length > 0) { return(new ValidationResult() { ResultType = ValidationResultType.Error, AccountID = deliveryOutput.Account.ID, DeliveryID = deliveryOutput.DeliveryID, TargetPeriodStart = deliveryOutput.TimePeriodStart, TargetPeriodEnd = deliveryOutput.TimePeriodEnd, Message = errors.ToString(), ChannelID = deliveryOutput.Channel.ID, CheckType = this.Instance.Configuration.Name }); } else { return(new ValidationResult() { ResultType = ValidationResultType.Information, AccountID = deliveryOutput.Account.ID, DeliveryID = deliveryOutput.DeliveryID, TargetPeriodStart = deliveryOutput.TimePeriodStart, TargetPeriodEnd = deliveryOutput.TimePeriodEnd, Message = "validation Success - Account Original ID: " + deliveryOutput.Account.OriginalID, ChannelID = deliveryOutput.Channel.ID, CheckType = this.Instance.Configuration.Name }); } } } // If reader is closed else { result = new ValidationResult() { ResultType = ValidationResultType.Error, AccountID = deliveryOutput.Account.ID, DeliveryID = deliveryOutput.DeliveryID, TargetPeriodStart = deliveryOutput.TimePeriodStart, TargetPeriodEnd = deliveryOutput.TimePeriodEnd, Message = "Cannot Read Data from DB connection closed - Account Original ID: " + deliveryOutput.Account.OriginalID, ChannelID = deliveryOutput.Channel.ID, CheckType = this.Instance.Configuration.Name } }; } //Could not find check scenario result = new ValidationResult() { ResultType = ValidationResultType.Error, AccountID = deliveryOutput.Account.ID, DeliveryID = deliveryOutput.DeliveryID, TargetPeriodStart = deliveryOutput.TimePeriodStart, TargetPeriodEnd = deliveryOutput.TimePeriodEnd, Message = "Could not find check scenario - Account Original ID: " + deliveryOutput.Account.OriginalID, ChannelID = deliveryOutput.Channel.ID, CheckType = this.Instance.Configuration.Name }; } return(result); } }
protected override ValidationResult DeliveryDbCompare(DeliveryOutput output, Dictionary <string, double> deliveryTotals, string DbConnectionStringName, string comparisonTable) { using (SqlConnection sqlCon = new SqlConnection(AppSettings.GetConnectionString(this, DbConnectionStringName))) { sqlCon.Open(); Dictionary <string, Measure> measurs = Measure.GetMeasures(output.Account, output.Channel, sqlCon, MeasureOptions.IsBackOffice, OptionsOperator.And); string dayCode = output.TimePeriodStart.ToString("yyyyMMdd"); // Delivery Per Day = > TargetPeriod.Start = daycode Dictionary <string, double> diff = new Dictionary <string, double>(); Dictionary <string, Measure> validationRequiredMeasure = new Dictionary <string, Measure>(); #region Creating Command txt /*==========================================================*/ StringBuilder command = new StringBuilder(); command.Append("Select "); foreach (var measure in measurs) { if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired)) { validationRequiredMeasure.Add(measure.Value.SourceName, measure.Value); command.Append(string.Format("SUM({0}) as {1}", measure.Value.OltpName, measure.Value.SourceName)); command.Append(","); } } command.Remove(command.Length - 1, 1); //remove last comma character command.Append(" from "); command.Append(comparisonTable); command.Append(" where Account_ID = @Account_ID and Day_ID = @Daycode and Channel_ID = @Channel_ID"); /*===========================================================*/ #endregion #region SQL Command /*==============================================================================*/ SqlCommand sqlCommand = DataManager.CreateCommand(command.ToString()); sqlCommand.Connection = sqlCon; SqlParameter accountIdParam = new SqlParameter("@Account_ID", System.Data.SqlDbType.Int); SqlParameter daycodeParam = new SqlParameter("@Daycode", System.Data.SqlDbType.Int); SqlParameter channelIdParam = new SqlParameter("@Channel_ID", System.Data.SqlDbType.Int); accountIdParam.Value = output.Account.ID; daycodeParam.Value = dayCode; channelIdParam.Value = output.Channel.ID; sqlCommand.Parameters.Add(accountIdParam); sqlCommand.Parameters.Add(daycodeParam); sqlCommand.Parameters.Add(channelIdParam); sqlCommand.Connection = sqlCon; /*==============================================================================*/ #endregion using (var _reader = sqlCommand.ExecuteReader()) { progress += 0.5 / 1 - progress; this.ReportProgress(progress); if (!_reader.IsClosed) { while (_reader.Read()) { if (!_reader[0].Equals(DBNull.Value)) { foreach (var measureItem in validationRequiredMeasure) { diff.Add(measureItem.Value.SourceName, Math.Abs(Convert.ToDouble(_reader[measureItem.Value.SourceName]) - deliveryTotals[measureItem.Value.SourceName])); } } #region Scenario : data exists in delivery and not in DB else { double sum = 0; foreach (var measure in validationRequiredMeasure) { sum += deliveryTotals[measure.Value.SourceName]; } if (sum > 0) { return new ValidationResult() { ResultType = ValidationResultType.Error, AccountID = output.Account.ID, TargetPeriodStart = output.TimePeriodStart, TargetPeriodEnd = output.TimePeriodEnd, Message = "Data exists in delivery but not in DB for Account ID: " + output.Account.ID, ChannelID = output.Channel.ID, CheckType = this.Instance.Configuration.Name } } ; } #endregion foreach (var item in diff) { #region Scenario: Found differences if (item.Value > ALLOWED_DIFF) { return new ValidationResult() { ResultType = ValidationResultType.Error, AccountID = output.Account.ID, DeliveryID = output.DeliveryID, TargetPeriodStart = output.TimePeriodStart, TargetPeriodEnd = output.TimePeriodEnd, Message = "validation Error - differences has been found - Account ID: " + output.Account.ID, ChannelID = output.Channel.ID, CheckType = this.Instance.Configuration.Name } } ; #endregion #region Scenario: Differences were not found else { return new ValidationResult() { ResultType = ValidationResultType.Information, AccountID = output.Account.ID, DeliveryID = output.DeliveryID, TargetPeriodStart = output.TimePeriodStart, TargetPeriodEnd = output.TimePeriodEnd, Message = "validation Success - Account ID: " + output.Account.ID, ChannelID = output.Channel.ID, CheckType = this.Instance.Configuration.Name } }; #endregion } } } // If reader is closed else { return new ValidationResult() { ResultType = ValidationResultType.Error, AccountID = output.Account.ID, DeliveryID = output.DeliveryID, TargetPeriodStart = output.TimePeriodStart, TargetPeriodEnd = output.TimePeriodEnd, Message = "Cannot Read Data from DB connection closed - Account Original ID: " + output.Account.OriginalID, ChannelID = output.Channel.ID, CheckType = this.Instance.Configuration.Name } }; } //Could not find check scenario return(new ValidationResult() { ResultType = ValidationResultType.Error, AccountID = output.Account.ID, DeliveryID = output.DeliveryID, TargetPeriodStart = output.TimePeriodStart, TargetPeriodEnd = output.TimePeriodEnd, Message = "Could not find check scenario - Account Original ID: " + output.Account.OriginalID, ChannelID = output.Channel.ID, CheckType = this.Instance.Configuration.Name }); } } }
protected override Core.Services.ServiceOutcome DoPipelineWork() { bool includeConversionTypes = Boolean.Parse(this.Delivery.Parameters["includeConversionTypes"].ToString()); bool includeDisplaytData = Boolean.Parse(this.Delivery.Parameters["includeDisplaytData"].ToString()); bool ConvertToUSD = Boolean.Parse(this.Delivery.Parameters["ConvertToUSD"].ToString()); //double ConstCurrencyRate = this.Delivery.Parameters.ContainsKey("ConstCurrencyRate") ? Convert.ToDouble(this.Delivery.Parameters["ConstCurrencyRate"]) : 1; //Status Members Dictionary <string, ObjectStatus> kwd_Status_Data = new Dictionary <string, ObjectStatus>(); Dictionary <string, ObjectStatus> placement_kwd_Status_Data = new Dictionary <string, ObjectStatus>(); Dictionary <Int64, ObjectStatus> adGroup_Status_Data = new Dictionary <Int64, ObjectStatus>(); Dictionary <Int64, ObjectStatus> ad_Status_Data = new Dictionary <Int64, ObjectStatus>(); Dictionary <Int64, ObjectStatus> campaign_Status_Data = new Dictionary <Int64, ObjectStatus>(); using (this.ImportManager = new AdMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions() { MeasureOptions = MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice, MeasureOptionsOperator = OptionsOperator.Not, SegmentOptions = Data.Objects.SegmentOptions.All, SegmentOptionsOperator = OptionsOperator.And })) { string[] requiredHeaders = new string[1]; requiredHeaders[0] = Const.AdPreRequiredHeader; #region Getting Keywords Data Dictionary <string, double> _totals = new Dictionary <string, double>(); DeliveryFile _keyWordsFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.KEYWORDS_PERFORMANCE_REPORT]]; requiredHeaders[0] = Const.AdPreRequiredHeader; var _keywordsReader = new CsvDynamicReader(_keyWordsFile.OpenContents(compression: FileCompression.Gzip), requiredHeaders); _keywordsReader.MatchExactColumns = false; Dictionary <string, KeywordTarget> _keywordsData = new Dictionary <string, KeywordTarget>(); this.ImportManager.BeginImport(this.Delivery); using (_keywordsReader) { while (_keywordsReader.Read()) { this.Mappings.OnFieldRequired = field => _keywordsReader.Current[field]; if (_keywordsReader.Current[Const.KeywordIdFieldName] == Const.EOF) { break; } KeywordPrimaryKey keywordPrimaryKey = new KeywordPrimaryKey() { KeywordId = Convert.ToInt64(_keywordsReader.Current[Const.KeywordIdFieldName]), AdgroupId = Convert.ToInt64(_keywordsReader.Current[Const.AdGroupIdFieldName]), CampaignId = Convert.ToInt64(_keywordsReader.Current[Const.CampaignIdFieldName]) }; KeywordTarget keyword = new KeywordTarget() { OriginalID = _keywordsReader.Current[Const.KeywordIdFieldName], Keyword = _keywordsReader.Current[Const.KeywordFieldName], //Status = kwd_Status_Data[keywordPrimaryKey.ToString()] }; keyword.QualityScore = Convert.ToString(_keywordsReader.Current[Const.QualityScoreFieldName]); string matchType = _keywordsReader.Current[Const.MatchTypeFieldName]; keyword.MatchType = (KeywordMatchType)Enum.Parse(typeof(KeywordMatchType), matchType, true); //Setting Tracker for Keyword if (!String.IsNullOrWhiteSpace(Convert.ToString(_keywordsReader.Current[Const.DestUrlFieldName]))) { keyword.DestinationUrl = Convert.ToString(_keywordsReader.Current[Const.DestUrlFieldName]); //setting kwd tracker //if (((String)(_keywordsReader.Current[Const.DestUrlFieldName])).IndexOf(Const.CreativeIDTrackingValue, StringComparison.OrdinalIgnoreCase) >= 0) if (Convert.ToBoolean(this.Delivery.Parameters["UseKwdTrackerAsAdTracker"])) { if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(KeywordTarget))) { this.Mappings.Objects[typeof(KeywordTarget)].Apply(keyword); } } } _keywordsData.Add(keywordPrimaryKey.ToString(), keyword); } } #endregion Dictionary <string, PlacementTarget> _placementsData = new Dictionary <string, PlacementTarget>(); #region Getting Placements Data string[] _placementsFileRequiredHeaders = new string[1]; _placementsFileRequiredHeaders[0] = Const.PlacementCriteriaID; DeliveryFile _PlacementsFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.PLACEMENT_PERFORMANCE_REPORT]]; var _PlacementsReader = new CsvDynamicReader(_PlacementsFile.OpenContents(compression: FileCompression.Gzip), _placementsFileRequiredHeaders); using (_PlacementsReader) { while (_PlacementsReader.Read()) { if (_PlacementsReader.Current[Const.PlacementCriteriaID] == Const.EOF) { break; } //Read data only if managed GDN, otherwise it is an automatic GDN so skip if (!((String)(_PlacementsReader.Current[Const.PlacementCriteriaID])).Trim().Equals("--")) { KeywordPrimaryKey placementPrimaryKey = new KeywordPrimaryKey() { KeywordId = Convert.ToInt64(_PlacementsReader.Current[Const.PlacementCriteriaID]), AdgroupId = Convert.ToInt64(_PlacementsReader.Current[Const.AdGroupIdFieldName]), CampaignId = Convert.ToInt64(_PlacementsReader.Current[Const.CampaignIdFieldName]) }; PlacementTarget placement = new PlacementTarget() { OriginalID = _PlacementsReader.Current[Const.PlacementCriteriaID], Placement = _PlacementsReader.Current[Const.PlacementFieldName], PlacementType = PlacementType.Managed, // Status = placement_kwd_Status_Data[placementPrimaryKey.ToString()] }; //Setting Tracker for placment if (!String.IsNullOrWhiteSpace(Convert.ToString(_PlacementsReader.Current[Const.DestUrlFieldName]))) { placement.DestinationUrl = Convert.ToString(_PlacementsReader.Current[Const.DestUrlFieldName]); } _placementsData.Add(placementPrimaryKey.ToString(), placement); } } } #endregion #region Getting Conversions Data //Get Ads Conversion ( for ex. signup , purchase ) DeliveryFile _conversionFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.AD_PERFORMANCE_REPORT] + "_Conv"]; var _conversionsReader = new CsvDynamicReader(_conversionFile.OpenContents(compression: FileCompression.Gzip), requiredHeaders); Dictionary <string, Dictionary <string, long> > importedAdsWithConv = new Dictionary <string, Dictionary <string, long> >(); using (_conversionsReader) { while (_conversionsReader.Read()) { if (_conversionsReader.Current[Const.AdIDFieldName] == Const.EOF) // if end of report { break; } string conversionKey = String.Format("{0}#{1}", _conversionsReader.Current[Const.AdIDFieldName], _conversionsReader.Current[Const.KeywordIdFieldName]); Dictionary <string, long> conversionDic = new Dictionary <string, long>(); if (!importedAdsWithConv.TryGetValue(conversionKey, out conversionDic)) { //ADD conversionKey to importedAdsWithConv //than add conversion field to importedAdsWithConv : <conversion name , conversion value> Dictionary <string, long> conversion = new Dictionary <string, long>(); conversion.Add(Convert.ToString(_conversionsReader.Current[Const.ConversionTrackingPurposeFieldName]), Convert.ToInt64(_conversionsReader.Current[Const.ConversionManyPerClickFieldName])); importedAdsWithConv.Add(conversionKey, conversion); } else // if Key exists { // if current add already has current conversion type than add value to the current type if (!conversionDic.ContainsKey(Convert.ToString(_conversionsReader.Current[Const.ConversionTrackingPurposeFieldName]))) { conversionDic.Add(Convert.ToString(_conversionsReader.Current[Const.ConversionTrackingPurposeFieldName]), Convert.ToInt64(_conversionsReader.Current[Const.ConversionManyPerClickFieldName])); } // else create new conversion type and add the value else { conversionDic[Convert.ToString(_conversionsReader.Current[Const.ConversionTrackingPurposeFieldName])] += Convert.ToInt64(_conversionsReader.Current[Const.ConversionManyPerClickFieldName]); } } } } #endregion #region Getting Ads Data DeliveryFile _adPerformanceFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.AD_PERFORMANCE_REPORT]]; var _adsReader = new CsvDynamicReader(_adPerformanceFile.OpenContents(compression: FileCompression.Gzip), requiredHeaders); Dictionary <string, Ad> importedAds = new Dictionary <string, Ad>(); //session.Begin(false); //this.ImportManager.BeginImport(this.Delivery); DeliveryOutput currentOutput = Delivery.Outputs.First(); foreach (KeyValuePair <string, Measure> measure in this.ImportManager.Measures) { if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired)) { _totals.Add(measure.Key, 0); } } using (_adsReader) { this.Mappings.OnFieldRequired = field => _adsReader.Current[field]; while (_adsReader.Read()) { string currencyCode = ((string)(_adsReader.Current.Currency)).ToUpper(); // Adding totals line for validation (checksum) if (_adsReader.Current[Const.AdIDFieldName] == Const.EOF) { foreach (KeyValuePair <string, Measure> measure in this.ImportManager.Measures) { if (!measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired)) { continue; } switch (measure.Key) { case Measure.Common.Clicks: _totals[Measure.Common.Clicks] = Convert.ToInt64(_adsReader.Current.Clicks); break; case Measure.Common.Cost: _totals[Measure.Common.Cost] = ConvertToUSD ? this.ConvertToUSD(this.Delivery.Parameters["CurrencyCode"].ToString().ToUpper(), (Convert.ToDouble(_adsReader.Current.Cost)) / 1000000) : (Convert.ToDouble(_adsReader.Current.Cost)) / 1000000; break; case Measure.Common.Impressions: _totals[Measure.Common.Impressions] = Convert.ToInt64(_adsReader.Current.Impressions); break; } } break; } AdMetricsUnit adMetricsUnit = new AdMetricsUnit(); adMetricsUnit.Output = currentOutput; Ad ad; #region Try Get SearchKWD /***********************************************************************************/ //Creating kwd primary key KeywordPrimaryKey kwdKey = new KeywordPrimaryKey() { AdgroupId = Convert.ToInt64(_adsReader.Current[Const.AdGroupIdFieldName]), KeywordId = Convert.ToInt64(_adsReader.Current[Const.KeywordIdFieldName]), CampaignId = Convert.ToInt64(_adsReader.Current[Const.CampaignIdFieldName]) }; //Check if keyword file contains this kwdkey and not a GDN Keyword String[] GdnKwdIds = this.Delivery.Parameters["KeywordContentId"].ToString().Split(','); bool IsSearchKwd = false; KeywordTarget kwd = null; PlacementTarget placement = null; if (!GdnKwdIds.Contains(kwdKey.KeywordId.ToString()) && _keywordsData.ContainsKey(kwdKey.ToString())) { kwd = new KeywordTarget(); try { kwd = _keywordsData[kwdKey.ToString()]; } catch (Exception) { //Creating KWD with OriginalID , since the KWD doesnt exists in KWD report. kwd = new KeywordTarget { OriginalID = Convert.ToString(_adsReader.Current[Const.KeywordIdFieldName]) }; } IsSearchKwd = true; } else { placement = new PlacementTarget(); try { placement = _placementsData[kwdKey.ToString()]; } catch (Exception) { placement.OriginalID = Convert.ToString(_adsReader.Current[Const.KeywordIdFieldName]); placement.PlacementType = PlacementType.Automatic; placement.Placement = Const.AutoDisplayNetworkName; } } /***********************************************************************************/ #endregion string adId = _adsReader.Current[Const.AdIDFieldName]; if (!importedAds.ContainsKey(adId)) { ad = new Ad(); ad.OriginalID = adId; ad.Channel = new Channel() { ID = 1 }; ad.Account = new Account { ID = this.Delivery.Account.ID, OriginalID = (String)_adPerformanceFile.Parameters["AdwordsClientID"] }; // ad.Status = ad_Status_Data[Convert.ToInt64(adId)]; #region Ad Type /****************************************************************/ string adTypeColumnValue = Convert.ToString(_adsReader.Current[Const.AdTypeFieldName]); string devicePreferenceColumnValue = Convert.ToString(_adsReader.Current[Const.AdDevicePreferenceFieldName]); if (!GoogleAdTypeDic.ContainsKey(adTypeColumnValue)) { continue; } string adTypeEdgeValue = GoogleAdTypeDic[adTypeColumnValue].ToString(); //EdgeAdType atv = (EdgeAdType)Enum.Parse(typeof(EdgeAdType), adTypeEdgeValue, true); //is mobile ad ? if (devicePreferenceColumnValue.Equals(Const.AdDevicePreferenceMobileFieldValue)) { string mobileValue = string.Format("Mobile {0}", Convert.ToString(_adsReader.Current[Const.AdTypeFieldName])); //Check if this mobile value exists on dictionary if (GoogleAdTypeDic.ContainsKey(mobileValue)) { adTypeEdgeValue = GoogleAdTypeDic[mobileValue].ToString(); } else { adTypeEdgeValue = GoogleAdTypeDic[Const.AdTypeValues.Mobile_ad].ToString(); } } ad.ExtraFields[AdType] = (int)(EdgeAdType)Enum.Parse(typeof(EdgeAdType), adTypeEdgeValue, true);; /****************************************************************/ #endregion //Creative ad.Creatives.Add(new TextCreative { TextType = TextCreativeType.DisplayUrl, Text = _adsReader.Current[Const.DisplayURLFieldName] }); #region Ad Tracker segment /******************************************************/ ////Setting Tracker for Ad if (!String.IsNullOrWhiteSpace(_adsReader.Current[Const.DestUrlFieldName])) { ad.DestinationUrl = _adsReader.Current[Const.DestUrlFieldName]; if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad))) { this.Mappings.Objects[typeof(Ad)].Apply(ad); } } //if Ad doesnt contains tracker than check for kwd tracker if (Convert.ToBoolean(this.Delivery.Parameters["UseKwdTrackerAsAdTracker"])) { if (kwd != null && kwd.Segments != null && kwd.Segments.ContainsKey(this.ImportManager.SegmentTypes[Segment.Common.Tracker])) { if (kwd.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]] != null && kwd.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]].Value != null) { SegmentObject tracker = kwd.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]]; //if value contains ADID than replace ADID with AD original id tracker.Value = tracker.Value.Replace("ADID", ad.OriginalID); ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]] = tracker; } } } /******************************************************/ #endregion #region Campaign /****************************************************************/ ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]] = new Campaign() { OriginalID = _adsReader.Current[Const.CampaignIdFieldName], Name = _adsReader.Current[Const.CampaignFieldName], //Status = campaign_Status_Data[Convert.ToInt64(_adsReader.Current[Const.CampaignIdFieldName])] }; /****************************************************************/ #endregion #region Image /****************************************************************/ //Image Type > Create Image if (String.Equals(Convert.ToString(_adsReader.Current[Const.AdTypeFieldName]), "Image ad")) { string adNameField = _adsReader.Current[Const.AdFieldName]; string[] imageParams = adNameField.Trim().Split(new Char[] { ':', ';' }); // Ad name: 468_60_Test7options_Romanian.swf; 468 x 60 ad.Name = imageParams[1].Trim(); ad.Creatives.Add(new ImageCreative() { ImageUrl = imageParams[1].Trim(), ImageSize = imageParams[2].Trim() }); } /****************************************************************/ #endregion #region Text od Display /****************************************************************/ else //Text ad or Display ad { ad.Name = _adsReader.Current[Const.AdFieldName]; ad.Creatives.Add(new TextCreative { TextType = TextCreativeType.Title, Text = _adsReader.Current.Ad, }); ad.Creatives.Add(new TextCreative { TextType = TextCreativeType.Body, Text = _adsReader.Current["Description line 1"], Text2 = _adsReader.Current["Description line 2"] }); } /****************************************************************/ #endregion #region Adgroup /****************************************************************/ //Insert Adgroup ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.AdGroup]] = new AdGroup() { Campaign = (Campaign)ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]], Value = _adsReader.Current[Const.AdGroupFieldName], OriginalID = _adsReader.Current[Const.AdGroupIdFieldName], // Status = adGroup_Status_Data[Convert.ToInt64(_adsReader.Current[Const.AdGroupIdFieldName])] }; /****************************************************************/ #endregion #region Network /****************************************************************/ //Insert Network Type Display Network / Search Network //string networkType = Convert.ToString(_adsReader.Current[Const.NetworkFieldName]); //if (networkType.Equals(Const.GoogleSearchNetwork)) // networkType = Const.SystemSearchNetwork; //else if (networkType.Equals(Const.GoogleDisplayNetwork)) // networkType = Const.SystemDisplayNetwork; //ad.ExtraFields[NetworkType] = networkType; /****************************************************************/ #endregion importedAds.Add(adId, ad); this.ImportManager.ImportAd(ad); } else { ad = importedAds[adId]; } adMetricsUnit.Ad = ad; //INSERTING SEARCH KEYWORD INTO METRICS if (IsSearchKwd & kwd != null) { adMetricsUnit.TargetDimensions = new List <Target>(); adMetricsUnit.TargetDimensions.Add(kwd); } //INSERTING GDN KEYWORD INTO METRICS else if (placement != null) { //INSERTING KEYWORD INTO METRICS adMetricsUnit.TargetDimensions = new List <Target>(); adMetricsUnit.TargetDimensions.Add(placement); } //INSERTING METRICS DATA adMetricsUnit.MeasureValues = new Dictionary <Measure, double>(); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Clicks], Convert.ToInt64(_adsReader.Current.Clicks)); //Currencies Conversion Support adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Cost], ConvertToUSD ? this.ConvertToUSD(currencyCode.ToUpper(), (Convert.ToDouble(_adsReader.Current.Cost)) / 1000000) : (Convert.ToDouble(_adsReader.Current.Cost)) / 1000000); if (ConvertToUSD) { adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.CostBeforeConversion], (Convert.ToDouble(_adsReader.Current.Cost)) / 1000000); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.USDConversionRate], Convert.ToDouble(this.ImportManager.CurrencyRates[currencyCode].RateValue)); } adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Impressions], Convert.ToInt64(_adsReader.Current.Impressions)); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.AveragePosition], Convert.ToDouble(_adsReader.Current[Const.AvgPositionFieldName])); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionOnePerClickFieldName]], Convert.ToDouble(_adsReader.Current[Const.ConversionOnePerClickFieldName])); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionManyPerClickFieldName]], Convert.ToDouble(_adsReader.Current[Const.ConversionManyPerClickFieldName])); //Inserting conversion values string conversionKey = String.Format("{0}#{1}", ad.OriginalID, _adsReader.Current[Const.KeywordIdFieldName]); Dictionary <string, long> conversionDic = new Dictionary <string, long>(); if (importedAdsWithConv.TryGetValue(conversionKey, out conversionDic)) { foreach (var pair in conversionDic) { if (GoogleMeasuresDic.ContainsKey(pair.Key)) { adMetricsUnit.MeasureValues[this.ImportManager.Measures[GoogleMeasuresDic[pair.Key]]] = pair.Value; } } } adMetricsUnit.Currency = new Currency { Code = Convert.ToString(_adsReader.Current.Currency) }; this.ImportManager.ImportMetrics(adMetricsUnit); } #endregion } #region Getting Sitelinks Data without metrics if (this.Delivery.Parameters.ContainsKey("AppendSitelinks") && Boolean.Parse(this.Delivery.Parameters["AppendSitelinks"].ToString())) { string[] sitelinksRequiredHeaders = new string[1]; sitelinksRequiredHeaders[0] = Const.PlaceholderFeedItemID; DeliveryFile _sitelinkPerformanceFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.PLACEHOLDER_FEED_ITEM_REPORT]]; var _sitelinkReader = new CsvDynamicReader(_sitelinkPerformanceFile.OpenContents(compression: FileCompression.Gzip), sitelinksRequiredHeaders); AdMetricsUnit siteLinkMetricsUnit = new AdMetricsUnit(); siteLinkMetricsUnit.Output = currentOutput; Ad sitelinkAd; using (_sitelinkReader) { this.Mappings.OnFieldRequired = field => _sitelinkReader.Current[field]; //to do : get site link tracker string[] sitelinkAttr; while (_sitelinkReader.Read()) { if (((String)_sitelinkReader.Current[Const.SiteLinkAttributeValuesHeader]).Equals(Const.Sitelink_EOF)) { break; } string sitelinkId = string.Format("{0}{1}{2}", _sitelinkReader.Current[Const.PlaceholderFeedItemID], _sitelinkReader.Current[Const.CampaignIdFieldName], _sitelinkReader.Current[Const.AdGroupIdFieldName]); sitelinkAd = new Ad(); sitelinkAd.OriginalID = sitelinkId; sitelinkAd.Channel = new Channel() { ID = 1 }; sitelinkAd.Account = new Account { ID = this.Delivery.Account.ID, OriginalID = (String)_adPerformanceFile.Parameters["AdwordsClientID"] }; //Creative sitelinkAttr = ((String)_sitelinkReader.Current[Const.SiteLinkAttributeValuesHeader]).Split(';'); bool legacy = sitelinkAttr.Count() != 4 ? true : false; string destUrl = string.Empty; bool destUrlParsingError = false; //Checking desturl errors ( we would like to insert only sitelinks that contains desturl ). try { destUrl = sitelinkAttr[1]; } catch (Exception e) { Log.Write("Error while trying to pars destination url from attribute field on sitelink report", e); destUrlParsingError = true; } if (!destUrlParsingError) { ////Setting Tracker for Ad if (!String.IsNullOrWhiteSpace(sitelinkAttr[1])) { sitelinkAd.DestinationUrl = sitelinkAttr[1]; if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad))) { this.Mappings.Objects[typeof(Ad)].Apply(sitelinkAd); } } sitelinkAd.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]] = new Campaign() { OriginalID = _sitelinkReader.Current[Const.CampaignIdFieldName], Name = _sitelinkReader.Current[Const.CampaignFieldName], }; //Insert Adgroup sitelinkAd.Segments[this.ImportManager.SegmentTypes[Segment.Common.AdGroup]] = new AdGroup() { Campaign = (Campaign)sitelinkAd.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]], Value = _sitelinkReader.Current[Const.AdGroupFieldName], OriginalID = _sitelinkReader.Current[Const.AdGroupIdFieldName], // Status = adGroup_Status_Data[Convert.ToInt64(_adsReader.Current[Const.AdGroupIdFieldName])] }; sitelinkAd.Creatives.Add(new TextCreative { TextType = TextCreativeType.DisplayUrl, Text = sitelinkAttr[1] }); if (!legacy) // only in case it doesnt contains legacy siteink { sitelinkAd.Creatives.Add(new TextCreative { TextType = TextCreativeType.Body, Text = sitelinkAttr[2], Text2 = sitelinkAttr[3] }); } sitelinkAd.Name = "[Sitelink] " + sitelinkAttr[0]; sitelinkAd.Creatives.Add(new TextCreative { TextType = TextCreativeType.Title, Text = "[Sitelink] " + sitelinkAttr[0] }); //Ad Type //Note: changed to "sitelink" following Amir request sitelinkAd.ExtraFields[AdType] = (int)(EdgeAdType.Sitelink); siteLinkMetricsUnit.Ad = sitelinkAd; //Setting Default Currency as USD following Amir's request from March 2014 siteLinkMetricsUnit.Currency = new Currency { Code = "USD" }; //INSERTING METRICS DATA siteLinkMetricsUnit.MeasureValues = new Dictionary <Measure, double>(); siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Clicks], 0); siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Cost], 0); siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Impressions], 0); siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.AveragePosition], 0); siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionOnePerClickFieldName]], 0); siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionManyPerClickFieldName]], 0); ImportManager.ImportMetrics(siteLinkMetricsUnit); ImportManager.ImportAd(sitelinkAd); } } } }// end if #endregion currentOutput.Checksum = _totals; this.ImportManager.EndImport(); } return(Core.Services.ServiceOutcome.Success); }
protected override Core.Services.ServiceOutcome DoPipelineWork() { string[] requiredHeaders = new string[] { Const.AutoPlacRequiredHeader }; DeliveryOutput currentOutput = Delivery.Outputs.First(); //Open Auto Plac file DeliveryFile _autoPlacFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.AUTOMATIC_PLACEMENTS_PERFORMANCE_REPORT]]; var _autoPlacReader = new CsvDynamicReader(_autoPlacFile.OpenContents(compression: FileCompression.Gzip), requiredHeaders); using (this.ImportManager = new GenericMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions() { MeasureOptions = MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice, MeasureOptionsOperator = OptionsOperator.Not, SegmentOptions = Data.Objects.SegmentOptions.All, SegmentOptionsOperator = OptionsOperator.And })) { Dictionary <string, double> _totals = new Dictionary <string, double>(); this.ImportManager.BeginImport(this.Delivery); //Intializing totals for validation foreach (KeyValuePair <string, Measure> measure in this.ImportManager.Measures) { if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired)) { _totals.Add(measure.Key, 0); } } using (_autoPlacReader) { this.Mappings.OnFieldRequired = field => _autoPlacReader.Current[field]; while (_autoPlacReader.Read()) { #region Setting Totals /*==================================================================================================================*/ // If end of file if (_autoPlacReader.Current[Const.CampaignIdFieldName] == Const.EOF) { //Setting totals for validation from totals line in adowrds file foreach (KeyValuePair <string, Measure> measure in this.ImportManager.Measures) { if (!measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired)) { continue; } switch (measure.Key) { case Measure.Common.Clicks: _totals[Measure.Common.Clicks] = Convert.ToInt64(_autoPlacReader.Current.Clicks); break; case Measure.Common.Cost: _totals[Measure.Common.Cost] = (Convert.ToDouble(_autoPlacReader.Current.Cost)) / 1000000; break; case Measure.Common.Impressions: _totals[Measure.Common.Impressions] = Convert.ToInt64(_autoPlacReader.Current.Impressions); break; } } break; } /*==================================================================================================================*/ #endregion GenericMetricsUnit autoPlacMetricsUnit = new GenericMetricsUnit(); autoPlacMetricsUnit.Output = currentOutput; autoPlacMetricsUnit.Channel = new Channel() { ID = 1 }; autoPlacMetricsUnit.Account = new Account { ID = this.Delivery.Account.ID, OriginalID = (String)_autoPlacFile.Parameters["AdwordsClientID"] }; autoPlacMetricsUnit.SegmentDimensions = new Dictionary <Segment, SegmentObject>(); //ADDING CAMPAIGN Campaign campaign = new Campaign() { OriginalID = _autoPlacReader.Current[Const.CampaignIdFieldName], Name = _autoPlacReader.Current[Const.CampaignFieldName], Status = ObjectStatusDic[((string)_autoPlacReader.Current[Const.CampaignStatusFieldName]).ToUpper()] }; autoPlacMetricsUnit.SegmentDimensions.Add(this.ImportManager.SegmentTypes[Segment.Common.Campaign], campaign); //ADDING ADGROUP AdGroup adgroup = new AdGroup() { Campaign = campaign, Value = _autoPlacReader.Current[Const.AdGroupFieldName], OriginalID = _autoPlacReader.Current[Const.AdGroupIdFieldName] }; autoPlacMetricsUnit.SegmentDimensions.Add(this.ImportManager.SegmentTypes[Segment.Common.AdGroup], adgroup); //INSERTING METRICS DATA autoPlacMetricsUnit.MeasureValues = new Dictionary <Measure, double>(); autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Clicks], Convert.ToInt64(_autoPlacReader.Current.Clicks)); autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Cost], (Convert.ToDouble(_autoPlacReader.Current.Cost)) / 1000000); autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Impressions], Convert.ToInt64(_autoPlacReader.Current.Impressions)); autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionOnePerClickFieldName]], Convert.ToDouble(_autoPlacReader.Current[Const.ConversionOnePerClickFieldName])); autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionManyPerClickFieldName]], Convert.ToDouble(_autoPlacReader.Current[Const.ConversionManyPerClickFieldName])); //CREATING PLACEMENT autoPlacMetricsUnit.TargetDimensions = new List <Target>(); PlacementTarget placement = new PlacementTarget() { Placement = _autoPlacReader.Current[Const.DomainFieldName], PlacementType = PlacementType.Automatic // Add status !!! }; autoPlacMetricsUnit.TargetDimensions.Add(placement); //SETTING TIME PERIOD autoPlacMetricsUnit.TimePeriodStart = this.Delivery.TimePeriodDefinition.Start.ToDateTime(); autoPlacMetricsUnit.TimePeriodEnd = this.Delivery.TimePeriodDefinition.End.ToDateTime(); this.ImportManager.ImportMetrics(autoPlacMetricsUnit); } } this.Delivery.Outputs.First().Checksum = _totals; this.ImportManager.EndImport(); } return(Core.Services.ServiceOutcome.Success); }
protected override Core.Services.ServiceOutcome DoPipelineWork() { MappingContainer metricsUnitMapping; if (!this.Mappings.Objects.TryGetValue(typeof(GenericMetricsUnit), out metricsUnitMapping)) { throw new MappingConfigurationException("Missing mapping definition for GenericMetricsUnit."); } currentOutput = this.Delivery.Outputs.First(); currentOutput.Checksum = new Dictionary <string, double>(); Dictionary <string, int> columns = new Dictionary <string, int>(); foreach (var ReportFile in Delivery.Files) { //Get Columns var reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.Gzip), "$.columnHeaders[*].*"); using (reportReader) { int colIndex = 0; while (reportReader.Read()) { columns.Add(reportReader.Current.name, colIndex); colIndex++; } ///sssss } using (this.ImportManager = new GenericMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions() { MeasureOptions = MeasureOptions.IsBackOffice, MeasureOptionsOperator = OptionsOperator.Or, SegmentOptions = Data.Objects.SegmentOptions.All, SegmentOptionsOperator = OptionsOperator.And })) { this.ImportManager.BeginImport(this.Delivery); Dictionary <string, GenericMetricsUnit> data = new Dictionary <string, GenericMetricsUnit>(); // Checksums _isChecksum = true; reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.Gzip), "$.totalsForAllResults.*"); using (reportReader) { this.Mappings.OnFieldRequired = field => reportReader.Current[field]; if (reportReader.Read()) { GenericMetricsUnit checksumUnit = new GenericMetricsUnit(); metricsUnitMapping.Apply(checksumUnit); foreach (var m in checksumUnit.MeasureValues) { if (m.Key.Options.HasFlag(MeasureOptions.ValidationRequired)) { currentOutput.Checksum.Add(m.Key.Name, m.Value); } } } } _isChecksum = false; //Get Valuees reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.Gzip), "$.rows[*].*"); using (reportReader) { this.Mappings.OnFieldRequired = field => reportReader.Current["array"][columns[field]]; while (reportReader.Read()) { GenericMetricsUnit tempUnit = new GenericMetricsUnit(); metricsUnitMapping.Apply(tempUnit); SegmentObject tracker = tempUnit.SegmentDimensions[ImportManager.SegmentTypes[Segment.Common.Tracker]]; GenericMetricsUnit existingUnit = null; // check if we already found a metrics unit with the same tracker if (!data.TryGetValue(tracker.Value, out existingUnit)) { tempUnit.Output = currentOutput; data.Add(tracker.Value, tempUnit); } else { // if tracker already exists, merge with existing values foreach (var m in tempUnit.MeasureValues) { if (!m.Key.Options.HasFlag(MeasureOptions.IsBackOffice)) { continue; } existingUnit.MeasureValues[m.Key] += m.Value; } } } } // Import all unique units per tracker foreach (GenericMetricsUnit metricsUnit in data.Values) { this.ImportManager.ImportMetrics(metricsUnit); } ImportManager.EndImport(); } } return(Core.Services.ServiceOutcome.Success); }
protected override ValidationResult DeliveryDbCompare(DeliveryOutput deliveryOutput, Dictionary <string, double> totals, string DbConnectionStringName, string comparisonTable) { double costDif = 0; double impsDif = 0; double clicksDif = 0; string dayCode = deliveryOutput.TimePeriodStart.ToString("yyyyMMdd"); // Delivery Per Day = > TargetPeriod.Start = daycode using (SqlConnection sqlCon = new SqlConnection(AppSettings.GetConnectionString(this, DbConnectionStringName))) { // This option is relevant only for account with one commited delivery. // for example it is not relevant for Easy forex. sqlCon.Open(); SqlCommand sqlCommand = DataManager.CreateCommand( "SELECT SUM(cost),sum(imps),sum(clicks) from " + comparisonTable + " where Account_ID = @Account_ID and Day_ID = @Daycode and Channel_ID = @Channel_ID" ); SqlParameter accountIdParam = new SqlParameter("@Account_ID", System.Data.SqlDbType.Int); SqlParameter daycodeParam = new SqlParameter("@Daycode", System.Data.SqlDbType.Int); SqlParameter channelIdParam = new SqlParameter("@Channel_ID", System.Data.SqlDbType.Int); accountIdParam.Value = deliveryOutput.Account.ID; daycodeParam.Value = dayCode; channelIdParam.Value = deliveryOutput.Channel.ID; sqlCommand.Parameters.Add(accountIdParam); sqlCommand.Parameters.Add(daycodeParam); sqlCommand.Parameters.Add(channelIdParam); sqlCommand.Connection = sqlCon; using (var _reader = sqlCommand.ExecuteReader()) { progress += 0.5 / 1 - progress; this.ReportProgress(progress); if (!_reader.IsClosed) { while (_reader.Read()) { if (!_reader[0].Equals(DBNull.Value)) { costDif = Math.Abs(Convert.ToUInt64(_reader[0]) - totals["Cost"]); clicksDif = Math.Abs(Convert.ToUInt64(_reader[2]) - totals["Clicks"]); impsDif = Math.Abs(Convert.ToUInt64(_reader[1]) - totals["Impressions"]); } #region Scenario : data exists in delivery and not in DB else if (totals["Cost"] > 0 || totals["Clicks"] > 0 || totals["Impressions"] > 0) { return new ValidationResult() { ResultType = ValidationResultType.Error, AccountID = deliveryOutput.Account.ID, TargetPeriodStart = deliveryOutput.TimePeriodStart, TargetPeriodEnd = deliveryOutput.TimePeriodEnd, Message = "Data exists in delivery but not in DB for Account Original ID: " + deliveryOutput.Account.OriginalID, ChannelID = deliveryOutput.Channel.ID, CheckType = this.Instance.Configuration.Name } } ; #endregion #region Scenario: Found differences if ((costDif != 0 && (costDif / totals["Cost"] > ALLOWED_DIFF)) || (clicksDif != 0 && (clicksDif / totals["Clicks"] > ALLOWED_DIFF)) || (impsDif != 0 && (impsDif / totals["Impressions"] > ALLOWED_DIFF))) { return new ValidationResult() { ResultType = ValidationResultType.Error, AccountID = deliveryOutput.Account.ID, DeliveryID = deliveryOutput.Delivery.DeliveryID, TargetPeriodStart = deliveryOutput.TimePeriodStart, TargetPeriodEnd = deliveryOutput.TimePeriodEnd, Message = "validation Error - differences has been found - Account Original ID: " + deliveryOutput.Account.OriginalID, ChannelID = deliveryOutput.Channel.ID, CheckType = this.Instance.Configuration.Name } } ; #endregion #region Scenario: Differences were not found else { return new ValidationResult() { ResultType = ValidationResultType.Information, AccountID = deliveryOutput.Account.ID, DeliveryID = deliveryOutput.Delivery.DeliveryID, TargetPeriodStart = deliveryOutput.TimePeriodStart, TargetPeriodEnd = deliveryOutput.TimePeriodEnd, Message = "validation Success - Account Original ID: " + deliveryOutput.Account.OriginalID, ChannelID = deliveryOutput.Channel.ID, CheckType = this.Instance.Configuration.Name } }; #endregion } } // If reader is closed else { return new ValidationResult() { ResultType = ValidationResultType.Error, AccountID = deliveryOutput.Account.ID, DeliveryID = deliveryOutput.Delivery.DeliveryID, TargetPeriodStart = deliveryOutput.TimePeriodStart, TargetPeriodEnd = deliveryOutput.TimePeriodEnd, Message = "Cannot Read Data from DB connection closed - Account Original ID: " + deliveryOutput.Account.OriginalID, ChannelID = deliveryOutput.Channel.ID, CheckType = this.Instance.Configuration.Name } }; } //Could not find check scenario return(new ValidationResult() { ResultType = ValidationResultType.Error, AccountID = deliveryOutput.Account.ID, DeliveryID = deliveryOutput.Delivery.DeliveryID, TargetPeriodStart = deliveryOutput.TimePeriodStart, TargetPeriodEnd = deliveryOutput.TimePeriodEnd, Message = "Could not find check scenario - Account Original ID: " + deliveryOutput.Account.OriginalID, ChannelID = deliveryOutput.Channel.ID, CheckType = this.Instance.Configuration.Name }); } } } }
abstract protected ValidationResult DeliveryDbCompare(DeliveryOutput deliveryOutput, Dictionary <string, double> totals, string DbConnectionStringName, string comparisonTable);
protected override Core.Services.ServiceOutcome DoPipelineWork() { MappingContainer metricsUnitMapping; if (!this.Mappings.Objects.TryGetValue(typeof(GenericMetricsUnit), out metricsUnitMapping)) { throw new MappingConfigurationException("Missing mapping definition for GenericMetricsUnit."); } currentOutput = this.Delivery.Outputs.First(); currentOutput.Checksum = new Dictionary <string, double>(); Dictionary <string, int> columns = new Dictionary <string, int>(); using (this.ImportManager = new GenericMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions() { MeasureOptions = MeasureOptions.IsBackOffice, MeasureOptionsOperator = OptionsOperator.Or, SegmentOptions = Data.Objects.SegmentOptions.All, SegmentOptionsOperator = OptionsOperator.And })) { this.ImportManager.BeginImport(this.Delivery); Dictionary <string, GenericMetricsUnit> data = new Dictionary <string, GenericMetricsUnit>(); foreach (var ReportFile in Delivery.Files) { //check number of recordes JsonDynamicReader reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.None), "$.totalSize"); int numOfRecordes = 0; if (reportReader.Read()) { numOfRecordes = int.Parse(reportReader.Current.totalSize); } reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.None), "$.nextRecordsUrl"); if (reportReader.Read()) { Log.Write(string.Format("Salesforce Attention - account {0} contains more than 1 file per query {1}", this.Delivery.Account.ID, ReportFile.Parameters["Query"].ToString()), LogMessageType.Warning); } if (numOfRecordes > 0) { //Get Values reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.None), "$.records[*].*"); using (reportReader) { this.Mappings.OnFieldRequired = field => { object value = new object(); string[] nestedFields; try { if (field.Contains('.')) { nestedFields = field.Split('.'); value = reportReader.Current[nestedFields[0]]; foreach (var item in nestedFields) { if (item.Equals(nestedFields[0])) { continue; } value = ((Dictionary <string, object>)value)[item]; } value = reportReader.Current[nestedFields[0]][nestedFields[1]]; Console.Write(value.ToString()); return(value); } else { Console.Write(value.ToString()); return(reportReader.Current[field]); } } catch (Exception ex) { throw new Exception(string.Format("Error while trying to map field {0} from mapper", field), ex); } }; while (reportReader.Read()) { GenericMetricsUnit metricsUnit = new GenericMetricsUnit(); metricsUnitMapping.Apply(metricsUnit); if (metricsUnit.MeasureValues != null) { SegmentObject tracker = metricsUnit.SegmentDimensions[ImportManager.SegmentTypes[Segment.Common.Tracker]]; GenericMetricsUnit importedUnit = null; // check if we already found a metrics unit with the same tracker if (!data.TryGetValue(tracker.Value, out importedUnit)) { metricsUnit.Output = currentOutput; data.Add(tracker.Value, metricsUnit); } else //Tracker already exists { // Merge captured measure with existing measures foreach (var capturedMeasure in metricsUnit.MeasureValues) { if (!capturedMeasure.Key.Options.HasFlag(MeasureOptions.IsBackOffice)) { continue; } //Measure already exists per tracker than aggregate: if (importedUnit.MeasureValues.ContainsKey(capturedMeasure.Key)) { importedUnit.MeasureValues[capturedMeasure.Key] += capturedMeasure.Value; } else { //Captured Measure doest exists with this tracker: importedUnit.MeasureValues.Add(capturedMeasure.Key, capturedMeasure.Value); } } } #region Validation // For validations foreach (var m in metricsUnit.MeasureValues) { if (m.Key.Options.HasFlag(MeasureOptions.ValidationRequired)) { if (!currentOutput.Checksum.ContainsKey(m.Key.Name)) { currentOutput.Checksum.Add(m.Key.Name, m.Value); } else { currentOutput.Checksum[m.Key.Name] += m.Value; } } } #endregion } } } } else { Log.Write("No Records Found in File " + ReportFile.Name, LogMessageType.Information); } } // Import all unique units per tracker foreach (GenericMetricsUnit metricsUnit in data.Values) { this.ImportManager.ImportMetrics(metricsUnit); } ImportManager.EndImport(); } return(Core.Services.ServiceOutcome.Success); }