protected override ServiceOutcome DoPipelineWork() { if (Configuration.Parameters["AccountID"] != null) { int.TryParse(Configuration.Parameters["AccountID"].ToString(), out _accountId); } var checksumThreshold = Configuration.Parameters.Get <string>(Consts.ConfigurationOptions.ChecksumTheshold, false); var identityInDebug = Configuration.Parameters.ContainsKey("IdentityInDebug") && Configuration.Parameters.Get <bool>("IdentityInDebug", false); var options = new MetricsDeliveryManagerOptions { SqlTransformCommand = Configuration.Parameters.Get <string>(Consts.AppSettings.SqlTransformCommand), ChecksumThreshold = checksumThreshold == null ? 0.01 : double.Parse(checksumThreshold), IdentityInDebug = identityInDebug }; using (var importManager = new MetricsDeliveryManager(InstanceID, options: options)) { // TODO: need this? Only check tickets, don't check conflicts HandleConflicts(importManager, DeliveryConflictBehavior.Ignore, getBehaviorFromConfiguration: false); // perform transform Log(String.Format("Start transform deliver '{0}'", Delivery.DeliveryID), LogMessageType.Information); importManager.Transform(new[] { Delivery }); Log(String.Format("Finished transform deliver '{0}'", Delivery.DeliveryID), LogMessageType.Information); } return(ServiceOutcome.Success); }
protected override ServiceOutcome DoPipelineWork() { var checksumThreshold = Configuration.Parameters.Get <string>(Consts.ConfigurationOptions.ChecksumTheshold, false); var identityInDebug = Configuration.Parameters.ContainsKey("IdentityInDebug") && Configuration.Parameters.Get <bool>("IdentityInDebug", false); var identityConfig = Configuration.Parameters.ContainsKey("IdentityConfig") ? Configuration.Parameters.Get <string>("IdentityConfig") : null; var options = new MetricsDeliveryManagerOptions { SqlStageCommand = Configuration.Parameters.Get <string>(Consts.AppSettings.SqlStageCommand), SqlRollbackCommand = Configuration.Parameters.Get <string>(Consts.AppSettings.SqlRollbackCommand), ChecksumThreshold = checksumThreshold == null ? 0.01 : double.Parse(checksumThreshold), IdentityInDebug = identityInDebug, IdentityConfig = identityConfig }; using (var importManager = new MetricsDeliveryManager(InstanceID, options: options)) { var success = false; do { try { // perform staging Log("Staging: start", LogMessageType.Information); importManager.Stage(new[] { Delivery }); Log("Staging: end", LogMessageType.Information); success = true; } catch (DeliveryConflictException dceex) { // rollback in case of exception Log("Rollback: start", LogMessageType.Information); importManager.RollbackOutputs(dceex.ConflictingOutputs); Log("Rollback: end", LogMessageType.Information); } catch (Exception ex) { throw new Exception(String.Format("Delivery {0} failed during staging.", Delivery.DeliveryID), ex); } }while (!success); } return(ServiceOutcome.Success); }
protected override ServiceOutcome DoPipelineWork() { InitMappings(); LoadConfiguration(); // Import data using (ReaderAdapter) { using (ImportManager = new MetricsDeliveryManager(InstanceID, EdgeTypes, _importManagerOptions) { OnLog = Log }) { // create objects tables and metrics table according to sample metrics ImportManager.BeginImport(Delivery, GetSampleMetrics()); Log("ImportManager.BeginImport() executed successfully", LogMessageType.Debug); // open delivery file using (var stream = _deliveryFile.OpenContents(compression: _compression)) { ReaderAdapter.Init(stream, Configuration); // for each row in file read and import into metrics table var readSuccess = false; while (CheckEndOfFile()) { readSuccess = true; ProcessMetrics(); } if (!readSuccess) { Log("Could Not read data from file!, check file mapping and configuration", Core.Utilities.LogMessageType.Warning); } ImportManager.EndImport(); Log("ImportManager.EndImport() executed successfully", LogMessageType.Debug); } } } return(ServiceOutcome.Success); }
protected override ServiceOutcome DoPipelineWork() { // init mapping and load configuration Log("Starting Google.AdWords.CampaignCriterionProcessorService", LogMessageType.Debug); InitMappings(); Mappings.OnMappingApplied = SetEdgeType; if (!Mappings.Objects.TryGetValue(typeof(MetricsUnit), out MetricsMappings)) { throw new MappingConfigurationException("Missing mapping definition for MetricsUnit.", "Object"); } LoadConfiguration(); Progress = 0.1; // get nessesary EdgeFields var relationField = GetEdgeFieldByName("Relation"); var locationField = GetEdgeFieldByName("Location"); var languageField = GetEdgeFieldByName("Language"); var isLocationNegativeField = GetEdgeFieldByName("IsLocationNegative"); var isLanguageNegativeField = GetEdgeFieldByName("IsLanguageNegative"); using (ReaderAdapter) { using (var stream = _deliveryFile.OpenContents(compression: _compression)) { ReaderAdapter.Init(stream, Configuration); while (ReaderAdapter.Reader.Read()) { // load metrics unit which contains Relation object of Campaign-Language or Campaign-Location CurrentMetricsUnit = new MetricsUnit { GetEdgeField = GetEdgeField, Output = new DeliveryOutput() }; MetricsMappings.Apply(CurrentMetricsUnit); // insert loaded relation to List per Campaign var relation = CurrentMetricsUnit.Dimensions[relationField] as RelationObject; if (relation != null) { var campaign = relation.Object1 as Campaign; if (campaign != null) { if (!_campaignRelationMap.ContainsKey(campaign.OriginalID)) { _campaignRelationMap.Add(campaign.OriginalID, new List <RelationObject>()); } _campaignRelationMap[campaign.OriginalID].Add(relation); } } } } } Progress = 0.5; using (ImportManager = new MetricsDeliveryManager(InstanceID, EdgeTypes, _importManagerOptions) { OnLog = Log }) { // create object tables ImportManager.BeginImport(Delivery, null); // add objects to EdgeObjectsManager cache PrepareImportObjects(x => x is Location, locationField, isLocationNegativeField); PrepareImportObjects(x => x is Language, languageField, isLanguageNegativeField); // import objects ImportManager.EndImport(); } return(ServiceOutcome.Success); }
protected override ServiceOutcome DoPipelineWork() { Log("Starting Google.AdWords.ProcessorService", LogMessageType.Debug); InitMappings(); Mappings.OnMappingApplied = SetEdgeType; if (!Mappings.Objects.TryGetValue(typeof(KeywordTarget), out KeywordMappings)) { throw new MappingConfigurationException("Missing mapping definition for KeywordTarget.", "Object"); } if (!Mappings.Objects.TryGetValue(typeof(PlacementTarget), out PlacementMappings)) { throw new MappingConfigurationException("Missing mapping definition for PlacementTarget.", "Object"); } if (!Mappings.Objects.TryGetValue(typeof(MetricsUnit), out MetricsMappings)) { throw new MappingConfigurationException("Missing mapping definition for MetricsUnit.", "Object"); } if (!Mappings.Objects.TryGetValue(typeof(Signature), out SignatureMappings)) { throw new MappingConfigurationException("Missing mapping definition for Signature.", "Object"); } using (ImportManager = new MetricsDeliveryManager(InstanceID, EdgeTypes, new MetricsDeliveryManagerOptions())) //{ // MeasureOptions = MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice, // MeasureOptionsOperator = OptionsOperator.Not, // SegmentOptions = Data.Objects.SegmentOptions.All, // SegmentOptionsOperator = OptionsOperator.And //})) { ImportManager.BeginImport(Delivery, GetSampleMetrics()); Log("Objects and Metrics tables are created", LogMessageType.Debug); Progress = 0.1; var requiredHeaders = new[] { AdWordsConst.AdPreRequiredHeader }; // Getting Keywords Data Log("Start loading keywords", LogMessageType.Debug); LoadKeywords(Delivery.Files[GoogleStaticReportsNamesUtill.ReportNames[global::Google.Api.Ads.AdWords.v201302.ReportDefinitionReportType.KEYWORDS_PERFORMANCE_REPORT]], requiredHeaders); Log("Finished loading keywords", LogMessageType.Debug); Progress = 0.3; // Getting Placements Data Log("Start loading placements", LogMessageType.Debug); LoadPlacements(Delivery.Files[GoogleStaticReportsNamesUtill.ReportNames[global::Google.Api.Ads.AdWords.v201302.ReportDefinitionReportType.PLACEMENT_PERFORMANCE_REPORT]], requiredHeaders); Log("Finished loading placements", LogMessageType.Debug); // Getting Conversions Data ( for ex. signup , purchase ) Log("Start loading conversions", LogMessageType.Debug); LoadConversions(Delivery.Files[GoogleStaticReportsNamesUtill.ReportNames[global::Google.Api.Ads.AdWords.v201302.ReportDefinitionReportType.AD_PERFORMANCE_REPORT] + "_Conv"], requiredHeaders); Log("Finished loading conversions", LogMessageType.Debug); Progress = 0.4; #region Getting Ads Data and Import Metrics Log("Start loading Ads", LogMessageType.Debug); var adPerformanceFile = Delivery.Files[GoogleStaticReportsNamesUtill.ReportNames[global::Google.Api.Ads.AdWords.v201302.ReportDefinitionReportType.AD_PERFORMANCE_REPORT]]; _adsReader = new CsvDynamicReader(adPerformanceFile.OpenContents(compression: FileCompression.Gzip), requiredHeaders); Mappings.OnFieldRequired = field => _adsReader.Current[field]; _importedAds.Clear(); using (_adsReader) { Mappings.OnFieldRequired = fieldName => _adsReader.Current[fieldName]; while (_adsReader.Read() && _adsReader.Current[AdWordsConst.AdIDFieldName] != AdWordsConst.EOF) { ProcessMetrics(); // add Ad if it is not exists yet if (!_importedAds.ContainsKey(CurrentMetricsUnit.Ad.OriginalID)) { _importedAds.Add(CurrentMetricsUnit.Ad.OriginalID, CurrentMetricsUnit.Ad); } } //Progress = 0.8; //Log("Start importing objects", LogMessageType.Debug); ImportManager.EndImport(); //Log("Finished importing objects", LogMessageType.Debug); } #endregion } return(ServiceOutcome.Success); }
protected override ServiceOutcome DoPipelineWork() { Log("Starting SalesForce.ProcessorService", LogMessageType.Debug); InitMappings(); Mappings.OnMappingApplied = SetEdgeType; if (!Mappings.Objects.TryGetValue(typeof(MetricsUnit), out MetricsMappings)) { throw new MappingConfigurationException("Missing mapping definition for MetricsUnit.", "Object"); } if (!Mappings.Objects.TryGetValue(typeof(Signature), out SignatureMappings)) { throw new MappingConfigurationException("Missing mapping definition for Signature.", "Object"); } using (ImportManager = new MetricsDeliveryManager(InstanceID, EdgeTypes, new MetricsDeliveryManagerOptions())) { // create objects and metrics table according to the sample metrics ImportManager.BeginImport(Delivery, GetSampleMetrics()); Log("Objects and Metrics tables are created", LogMessageType.Debug); Progress = 0.1; var metricsMap = new Dictionary <string, MetricsUnit>(); foreach (var file in Delivery.Files) { using (var reader = new JsonDynamicReader(file.OpenContents(), "$.records[*].*")) { Mappings.OnFieldRequired = fieldName => reader.Current[fieldName]; Log(String.Format("Start reading metrics for file '{0}'", file.Name), LogMessageType.Debug); while (reader.Read()) { // load current metrics unit LoadCurrentMetrics(); if (CurrentMetricsUnit.Dimensions.ContainsKey(GetEdgeField("Tracker")) && CurrentMetricsUnit.Dimensions[GetEdgeField("Tracker")] is StringValue) { var tracker = CurrentMetricsUnit.Dimensions[GetEdgeField("Tracker")] as StringValue; // check if metrics unit already exists by tracker if (metricsMap.ContainsKey(tracker.Value)) { // summaries all measure values foreach (var measure in CurrentMetricsUnit.MeasureValues) { metricsMap[tracker.Value].MeasureValues[measure.Key] += measure.Value; } } else { // add new metrics unit by tracker metricsMap.Add(tracker.Value, CurrentMetricsUnit); } } else { Log(String.Format("Failed to get tracker in file '{0}'", file.Name), LogMessageType.Warning); } } } } Progress = 0.5; // import summarized metrics Log("Start importing summarizzed metrics", LogMessageType.Debug); foreach (var metrics in metricsMap) { ImportManager.ImportMetrics(metrics.Value); } Progress = 0.8; Log("Start importing objects", LogMessageType.Debug); ImportManager.EndImport(); } return(ServiceOutcome.Success); }
protected override ServiceOutcome DoPipelineWork() { MappingContainer metricsUnitMapping; if (!Mappings.Objects.TryGetValue(typeof(MetricsUnit), out metricsUnitMapping)) { throw new MappingConfigurationException("Missing mapping definition for GenericMetricsUnit."); } _currentOutput = Delivery.Outputs.First(); _currentOutput.Checksum = new Dictionary <string, double>(); var columns = new Dictionary <string, int>(); foreach (var reportFile in Delivery.Files) { //Get Columns var reportReader = new JsonDynamicReader(reportFile.OpenContents(), "$.columnHeaders[*].*"); using (reportReader) { var colIndex = 0; while (reportReader.Read()) { columns.Add(reportReader.Current.name, colIndex); colIndex++; } } // get sample unit to create metrics table reportReader = new JsonDynamicReader(reportFile.OpenContents(), "$.rows[*].*"); using (reportReader) { Mappings.OnFieldRequired = field => reportReader.Current["array"][columns[field]]; reportReader.Read(); CurrentMetricsUnit = new MetricsUnit { GetEdgeField = GetEdgeField }; metricsUnitMapping.Apply(CurrentMetricsUnit); using (ImportManager = new MetricsDeliveryManager(InstanceID, EdgeTypes)) { ImportManager.BeginImport(Delivery, CurrentMetricsUnit); //Get values reportReader = new JsonDynamicReader(reportFile.OpenContents(), "$.rows[*].*"); using (reportReader) { Mappings.OnFieldRequired = field => reportReader.Current["array"][columns[field]]; while (reportReader.Read()) { CurrentMetricsUnit = new MetricsUnit { GetEdgeField = GetEdgeField }; metricsUnitMapping.Apply(CurrentMetricsUnit); ProcessMetrics(); } } // finish import (insert edge objects) ImportManager.EndImport(); } } } return(ServiceOutcome.Success); }
protected override ServiceOutcome DoPipelineWork() { Log("Starting Facebook.GraphApi.ProcessorService", LogMessageType.Debug); InitMappings(); Mappings.OnMappingApplied = SetEdgeType; if (!Mappings.Objects.TryGetValue(typeof(Ad), out AdMappings)) { throw new MappingConfigurationException("Missing mapping definition for Ad.", "Object"); } if (!Mappings.Objects.TryGetValue(typeof(Campaign), out CampaignMappings)) { throw new MappingConfigurationException("Missing mapping definition for Campaign.", "Object"); } if (!Mappings.Objects.TryGetValue(typeof(CompositeCreative), out CreativeMappings)) { throw new MappingConfigurationException("Missing mapping definition for CompositeCreative.", "Object"); } if (!Mappings.Objects.TryGetValue(typeof(MetricsUnit), out MetricsMappings)) { throw new MappingConfigurationException("Missing mapping definition for MetricsUnit.", "Object"); } if (!Mappings.Objects.TryGetValue(typeof(Signature), out SignatureMappings)) { throw new MappingConfigurationException("Missing mapping definition for Signature.", "Object"); } using (ImportManager = new MetricsDeliveryManager(InstanceID, EdgeTypes, new MetricsDeliveryManagerOptions())) { // create objects and metrics table according to the sample metrics ImportManager.BeginImport(Delivery, GetSampleMetrics()); Log("Objects and Metrics tables are created", LogMessageType.Debug); Progress = 0.1; // load Campaigns, Creatives and Ads into local Cache ClearLocalCache(); var filesByType = Delivery.Parameters["FilesByType"] as IDictionary <Consts.FileTypes, List <string> >; LoadFiles(filesByType[Consts.FileTypes.Campaigns], LoadCampaigns); LoadFiles(filesByType[Consts.FileTypes.Creatives], LoadCreatives); LoadFiles(filesByType[Consts.FileTypes.AdGroups], LoadAds); Log("Campaigns, Creatives and Ads are loaded into local cache", LogMessageType.Debug); Progress = 0.3; // start processing metrics foreach (var filePath in filesByType[Consts.FileTypes.AdGroupStats]) { using (var reader = new JsonDynamicReader(Delivery.Files[filePath].OpenContents(), "$.data[*].*")) { Mappings.OnFieldRequired = fieldName => reader.Current[fieldName]; while (reader.Read()) { ProcessMetrics(); } } } Progress = 0.8; Log("Start importing objects", LogMessageType.Debug); ImportManager.EndImport(); Log("Finished importing objects", LogMessageType.Debug); } return(ServiceOutcome.Success); }