protected override void OnRead() { var metrics = new GenericMetricsUnit(); _metricsMappings.Apply(metrics); //Writing to Log //if (metrics.Output.Checksum.Count() == 0) //{ // Edge.Core.Utilities.Log("Output checksum is empty",Core.Utilities.LogMessageType.Information); //} var signature = new Signature(); _signatureMappings.Apply(signature); //checking if signature is already exists in delivery outputs var outputs = from output in this.Delivery.Outputs where output.Signature.Equals(signature.ToString()) select output; DeliveryOutput op = outputs.FirstOrDefault <DeliveryOutput>(); if (op != null) { //Attaching output to Metrics (metrics as GenericMetricsUnit).Output = op; } else { DeliveryOutput deliveryOutput = new DeliveryOutput() { Signature = signature.Value, TimePeriodStart = metrics.TimePeriodStart, TimePeriodEnd = metrics.TimePeriodEnd, Account = metrics.Account, Channel = metrics.Channel }; this.Delivery.Outputs.Add(deliveryOutput); //Attaching output to Metrics (metrics as GenericMetricsUnit).Output = deliveryOutput; } this.ImportManager.ImportMetrics(metrics); }
protected override Core.Services.ServiceOutcome DoPipelineWork() { MappingContainer metricsUnitMapping; if (!this.Mappings.Objects.TryGetValue(typeof(GenericMetricsUnit), out metricsUnitMapping)) { throw new MappingConfigurationException("Missing mapping definition for GenericMetricsUnit."); } currentOutput = this.Delivery.Outputs.First(); currentOutput.Checksum = new Dictionary <string, double>(); Dictionary <string, int> columns = new Dictionary <string, int>(); foreach (var ReportFile in Delivery.Files) { //Get Columns var reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.Gzip), "$.columnHeaders[*].*"); using (reportReader) { int colIndex = 0; while (reportReader.Read()) { columns.Add(reportReader.Current.name, colIndex); colIndex++; } ///sssss } using (this.ImportManager = new GenericMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions() { MeasureOptions = MeasureOptions.IsBackOffice, MeasureOptionsOperator = OptionsOperator.Or, SegmentOptions = Data.Objects.SegmentOptions.All, SegmentOptionsOperator = OptionsOperator.And })) { this.ImportManager.BeginImport(this.Delivery); Dictionary <string, GenericMetricsUnit> data = new Dictionary <string, GenericMetricsUnit>(); // Checksums _isChecksum = true; reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.Gzip), "$.totalsForAllResults.*"); using (reportReader) { this.Mappings.OnFieldRequired = field => reportReader.Current[field]; if (reportReader.Read()) { GenericMetricsUnit checksumUnit = new GenericMetricsUnit(); metricsUnitMapping.Apply(checksumUnit); foreach (var m in checksumUnit.MeasureValues) { if (m.Key.Options.HasFlag(MeasureOptions.ValidationRequired)) { currentOutput.Checksum.Add(m.Key.Name, m.Value); } } } } _isChecksum = false; //Get Valuees reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.Gzip), "$.rows[*].*"); using (reportReader) { this.Mappings.OnFieldRequired = field => reportReader.Current["array"][columns[field]]; while (reportReader.Read()) { GenericMetricsUnit tempUnit = new GenericMetricsUnit(); metricsUnitMapping.Apply(tempUnit); SegmentObject tracker = tempUnit.SegmentDimensions[ImportManager.SegmentTypes[Segment.Common.Tracker]]; GenericMetricsUnit existingUnit = null; // check if we already found a metrics unit with the same tracker if (!data.TryGetValue(tracker.Value, out existingUnit)) { tempUnit.Output = currentOutput; data.Add(tracker.Value, tempUnit); } else { // if tracker already exists, merge with existing values foreach (var m in tempUnit.MeasureValues) { if (!m.Key.Options.HasFlag(MeasureOptions.IsBackOffice)) { continue; } existingUnit.MeasureValues[m.Key] += m.Value; } } } } // Import all unique units per tracker foreach (GenericMetricsUnit metricsUnit in data.Values) { this.ImportManager.ImportMetrics(metricsUnit); } ImportManager.EndImport(); } } return(Core.Services.ServiceOutcome.Success); }
protected override Core.Services.ServiceOutcome DoPipelineWork() { MappingContainer metricsUnitMapping; if (!this.Mappings.Objects.TryGetValue(typeof(GenericMetricsUnit), out metricsUnitMapping)) { throw new MappingConfigurationException("Missing mapping definition for GenericMetricsUnit."); } currentOutput = this.Delivery.Outputs.First(); currentOutput.Checksum = new Dictionary <string, double>(); foreach (var ReportFile in Delivery.Files) { bool isAttribute = Boolean.Parse(ReportFile.Parameters["Bo.IsAttribute"].ToString()); var ReportReader = new XmlDynamicReader (ReportFile.OpenContents(), ReportFile.Parameters["Bo.Xpath"].ToString()); using (this.ImportManager = new GenericMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions() { MeasureOptions = MeasureOptions.IsBackOffice, MeasureOptionsOperator = OptionsOperator.Or, SegmentOptions = Data.Objects.SegmentOptions.All, SegmentOptionsOperator = OptionsOperator.And })) { ImportManager.BeginImport(this.Delivery); //checksum init foreach (var m in ImportManager.Measures.Values) { if (m.Options.HasFlag(MeasureOptions.ValidationRequired)) { currentOutput.Checksum[m.Name] = 0; } } using (ReportReader) { dynamic readerHelper; while (ReportReader.Read()) { if (isAttribute) { readerHelper = ReportReader.Current.Attributes; } else { readerHelper = ReportReader.Current; } this.Mappings.OnFieldRequired = field => readerHelper[field]; GenericMetricsUnit genericMetricsUnit = new GenericMetricsUnit(); metricsUnitMapping.Apply(genericMetricsUnit); foreach (var m in genericMetricsUnit.MeasureValues) { if (m.Key.Options.HasFlag(MeasureOptions.ValidationRequired)) { if (!currentOutput.Checksum.ContainsKey(m.Key.Name)) { currentOutput.Checksum.Add(m.Key.Name, m.Value); } else { currentOutput.Checksum[m.Key.Name] += m.Value; } } } genericMetricsUnit.Output = currentOutput; ImportManager.ImportMetrics(genericMetricsUnit); } } ImportManager.EndImport(); } } return(Core.Services.ServiceOutcome.Success); }
protected override Core.Services.ServiceOutcome DoPipelineWork() { string[] requiredHeaders = new string[] { Const.AutoPlacRequiredHeader }; DeliveryOutput currentOutput = Delivery.Outputs.First(); //Open Auto Plac file DeliveryFile _autoPlacFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.AUTOMATIC_PLACEMENTS_PERFORMANCE_REPORT]]; var _autoPlacReader = new CsvDynamicReader(_autoPlacFile.OpenContents(compression: FileCompression.Gzip), requiredHeaders); using (this.ImportManager = new GenericMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions() { MeasureOptions = MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice, MeasureOptionsOperator = OptionsOperator.Not, SegmentOptions = Data.Objects.SegmentOptions.All, SegmentOptionsOperator = OptionsOperator.And })) { Dictionary <string, double> _totals = new Dictionary <string, double>(); this.ImportManager.BeginImport(this.Delivery); //Intializing totals for validation foreach (KeyValuePair <string, Measure> measure in this.ImportManager.Measures) { if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired)) { _totals.Add(measure.Key, 0); } } using (_autoPlacReader) { this.Mappings.OnFieldRequired = field => _autoPlacReader.Current[field]; while (_autoPlacReader.Read()) { #region Setting Totals /*==================================================================================================================*/ // If end of file if (_autoPlacReader.Current[Const.CampaignIdFieldName] == Const.EOF) { //Setting totals for validation from totals line in adowrds file foreach (KeyValuePair <string, Measure> measure in this.ImportManager.Measures) { if (!measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired)) { continue; } switch (measure.Key) { case Measure.Common.Clicks: _totals[Measure.Common.Clicks] = Convert.ToInt64(_autoPlacReader.Current.Clicks); break; case Measure.Common.Cost: _totals[Measure.Common.Cost] = (Convert.ToDouble(_autoPlacReader.Current.Cost)) / 1000000; break; case Measure.Common.Impressions: _totals[Measure.Common.Impressions] = Convert.ToInt64(_autoPlacReader.Current.Impressions); break; } } break; } /*==================================================================================================================*/ #endregion GenericMetricsUnit autoPlacMetricsUnit = new GenericMetricsUnit(); autoPlacMetricsUnit.Output = currentOutput; autoPlacMetricsUnit.Channel = new Channel() { ID = 1 }; autoPlacMetricsUnit.Account = new Account { ID = this.Delivery.Account.ID, OriginalID = (String)_autoPlacFile.Parameters["AdwordsClientID"] }; autoPlacMetricsUnit.SegmentDimensions = new Dictionary <Segment, SegmentObject>(); //ADDING CAMPAIGN Campaign campaign = new Campaign() { OriginalID = _autoPlacReader.Current[Const.CampaignIdFieldName], Name = _autoPlacReader.Current[Const.CampaignFieldName], Status = ObjectStatusDic[((string)_autoPlacReader.Current[Const.CampaignStatusFieldName]).ToUpper()] }; autoPlacMetricsUnit.SegmentDimensions.Add(this.ImportManager.SegmentTypes[Segment.Common.Campaign], campaign); //ADDING ADGROUP AdGroup adgroup = new AdGroup() { Campaign = campaign, Value = _autoPlacReader.Current[Const.AdGroupFieldName], OriginalID = _autoPlacReader.Current[Const.AdGroupIdFieldName] }; autoPlacMetricsUnit.SegmentDimensions.Add(this.ImportManager.SegmentTypes[Segment.Common.AdGroup], adgroup); //INSERTING METRICS DATA autoPlacMetricsUnit.MeasureValues = new Dictionary <Measure, double>(); autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Clicks], Convert.ToInt64(_autoPlacReader.Current.Clicks)); autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Cost], (Convert.ToDouble(_autoPlacReader.Current.Cost)) / 1000000); autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Impressions], Convert.ToInt64(_autoPlacReader.Current.Impressions)); autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionOnePerClickFieldName]], Convert.ToDouble(_autoPlacReader.Current[Const.ConversionOnePerClickFieldName])); autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionManyPerClickFieldName]], Convert.ToDouble(_autoPlacReader.Current[Const.ConversionManyPerClickFieldName])); //CREATING PLACEMENT autoPlacMetricsUnit.TargetDimensions = new List <Target>(); PlacementTarget placement = new PlacementTarget() { Placement = _autoPlacReader.Current[Const.DomainFieldName], PlacementType = PlacementType.Automatic // Add status !!! }; autoPlacMetricsUnit.TargetDimensions.Add(placement); //SETTING TIME PERIOD autoPlacMetricsUnit.TimePeriodStart = this.Delivery.TimePeriodDefinition.Start.ToDateTime(); autoPlacMetricsUnit.TimePeriodEnd = this.Delivery.TimePeriodDefinition.End.ToDateTime(); this.ImportManager.ImportMetrics(autoPlacMetricsUnit); } } this.Delivery.Outputs.First().Checksum = _totals; this.ImportManager.EndImport(); } return(Core.Services.ServiceOutcome.Success); }
protected override Core.Services.ServiceOutcome DoPipelineWork() { MappingContainer metricsUnitMapping; if (!this.Mappings.Objects.TryGetValue(typeof(GenericMetricsUnit), out metricsUnitMapping)) { throw new MappingConfigurationException("Missing mapping definition for GenericMetricsUnit."); } currentOutput = this.Delivery.Outputs.First(); currentOutput.Checksum = new Dictionary <string, double>(); Dictionary <string, int> columns = new Dictionary <string, int>(); using (this.ImportManager = new GenericMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions() { MeasureOptions = MeasureOptions.IsBackOffice, MeasureOptionsOperator = OptionsOperator.Or, SegmentOptions = Data.Objects.SegmentOptions.All, SegmentOptionsOperator = OptionsOperator.And })) { this.ImportManager.BeginImport(this.Delivery); Dictionary <string, GenericMetricsUnit> data = new Dictionary <string, GenericMetricsUnit>(); foreach (var ReportFile in Delivery.Files) { //check number of recordes JsonDynamicReader reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.None), "$.totalSize"); int numOfRecordes = 0; if (reportReader.Read()) { numOfRecordes = int.Parse(reportReader.Current.totalSize); } reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.None), "$.nextRecordsUrl"); if (reportReader.Read()) { Log.Write(string.Format("Salesforce Attention - account {0} contains more than 1 file per query {1}", this.Delivery.Account.ID, ReportFile.Parameters["Query"].ToString()), LogMessageType.Warning); } if (numOfRecordes > 0) { //Get Values reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.None), "$.records[*].*"); using (reportReader) { this.Mappings.OnFieldRequired = field => { object value = new object(); string[] nestedFields; try { if (field.Contains('.')) { nestedFields = field.Split('.'); value = reportReader.Current[nestedFields[0]]; foreach (var item in nestedFields) { if (item.Equals(nestedFields[0])) { continue; } value = ((Dictionary <string, object>)value)[item]; } value = reportReader.Current[nestedFields[0]][nestedFields[1]]; Console.Write(value.ToString()); return(value); } else { Console.Write(value.ToString()); return(reportReader.Current[field]); } } catch (Exception ex) { throw new Exception(string.Format("Error while trying to map field {0} from mapper", field), ex); } }; while (reportReader.Read()) { GenericMetricsUnit metricsUnit = new GenericMetricsUnit(); metricsUnitMapping.Apply(metricsUnit); if (metricsUnit.MeasureValues != null) { SegmentObject tracker = metricsUnit.SegmentDimensions[ImportManager.SegmentTypes[Segment.Common.Tracker]]; GenericMetricsUnit importedUnit = null; // check if we already found a metrics unit with the same tracker if (!data.TryGetValue(tracker.Value, out importedUnit)) { metricsUnit.Output = currentOutput; data.Add(tracker.Value, metricsUnit); } else //Tracker already exists { // Merge captured measure with existing measures foreach (var capturedMeasure in metricsUnit.MeasureValues) { if (!capturedMeasure.Key.Options.HasFlag(MeasureOptions.IsBackOffice)) { continue; } //Measure already exists per tracker than aggregate: if (importedUnit.MeasureValues.ContainsKey(capturedMeasure.Key)) { importedUnit.MeasureValues[capturedMeasure.Key] += capturedMeasure.Value; } else { //Captured Measure doest exists with this tracker: importedUnit.MeasureValues.Add(capturedMeasure.Key, capturedMeasure.Value); } } } #region Validation // For validations foreach (var m in metricsUnit.MeasureValues) { if (m.Key.Options.HasFlag(MeasureOptions.ValidationRequired)) { if (!currentOutput.Checksum.ContainsKey(m.Key.Name)) { currentOutput.Checksum.Add(m.Key.Name, m.Value); } else { currentOutput.Checksum[m.Key.Name] += m.Value; } } } #endregion } } } } else { Log.Write("No Records Found in File " + ReportFile.Name, LogMessageType.Information); } } // Import all unique units per tracker foreach (GenericMetricsUnit metricsUnit in data.Values) { this.ImportManager.ImportMetrics(metricsUnit); } ImportManager.EndImport(); } return(Core.Services.ServiceOutcome.Success); }