//public CurrencyImportManager ImportManager;
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
List <CurrencyRate> rates = new List <CurrencyRate>();
//TO DO : USE MAPPING CONFIGURATION FOR THIS SERVICE.
foreach (DeliveryFile ReportFile in this.Delivery.Files)
{
bool isAttribute = Boolean.Parse(ReportFile.Parameters["XML.IsAttribute"].ToString());
var ReportReader = new XmlDynamicReader
(ReportFile.OpenContents(), ReportFile.Parameters["XML.Path"].ToString());
using (ReportReader)
{
dynamic reader;
while (ReportReader.Read())
{
if (isAttribute)
{
reader = ReportReader.Current.Attributes;
}
else
{
reader = ReportReader.Current;
}
CurrencyRate currencyUnit = new CurrencyRate();
//Currency Code
List <object> CurrencyData = reader["field"];
currencyUnit.Currency.Code = (((XmlDynamicObject)CurrencyData[0]).InnerText.Split('/')).Count() > 1 ?((XmlDynamicObject)CurrencyData[0]).InnerText.Split('/')[1]: string.Empty;
if (string.IsNullOrEmpty(currencyUnit.Currency.Code))
{
continue;
}
//Currecy Date
currencyUnit.RateDate = DateTime.Now;
//Currency Rate
currencyUnit.RateValue = Convert.ToDecimal(((XmlDynamicObject)CurrencyData[1]).InnerText) == 0?0: 1 / Convert.ToDecimal(((XmlDynamicObject)CurrencyData[1]).InnerText);
rates.Add(currencyUnit);
}
}
CurrencyRate.SaveCurrencyRates(rates);
//ImportManager.EndImport();
//}
}
return(Core.Services.ServiceOutcome.Success);
}
private List <DeliveryFile> FetchNext(List <DeliveryFile> fetchFrom, int offset)
{
BatchDownloadOperation nextBatch = new BatchDownloadOperation();
List <DeliveryFile> nextRecordsFiles = new List <DeliveryFile>();
foreach (DeliveryFile ReportFile in fetchFrom)
{
//setting cuurent file has batched and batching next file
ReportFile.Parameters.Add("Batch", true);
string fileName = ReportFile.Name + "-" + offset;
JsonDynamicReader reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.None), "$.nextRecordsUrl");
string nextRecordPath;
if (reportReader.Read())
{
nextRecordPath = reportReader.Current.nextRecordsUrl;
DeliveryFile nextRecordFile = new DeliveryFile();
nextRecordFile.SourceUrl = ((Token)(ReportFile.Parameters["Token"])).instance_url + nextRecordPath;
HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(nextRecordFile.SourceUrl);
request.Headers.Add("Authorization: OAuth " + ((Token)(ReportFile.Parameters["Token"])).access_token);
//check if response contains more than one file
FileDownloadOperation fileDownloadOperation = nextRecordFile.Download(request);
nextBatch.Add(fileDownloadOperation);
nextRecordsFiles.Add(nextRecordFile);
}
}
if (nextRecordsFiles.Count > 0)
{
nextBatch.Start();
nextBatch.Wait();
nextBatch.EnsureSuccess();
foreach (DeliveryFile file in FetchNext(nextRecordsFiles, offset))
{
this.Delivery.Files.Add(file);
}
}
return(nextRecordsFiles);
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
MappingContainer metricsUnitMapping;
if (!this.Mappings.Objects.TryGetValue(typeof(GenericMetricsUnit), out metricsUnitMapping))
{
throw new MappingConfigurationException("Missing mapping definition for GenericMetricsUnit.");
}
currentOutput = this.Delivery.Outputs.First();
currentOutput.Checksum = new Dictionary <string, double>();
Dictionary <string, int> columns = new Dictionary <string, int>();
foreach (var ReportFile in Delivery.Files)
{
//Get Columns
var reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.Gzip), "$.columnHeaders[*].*");
using (reportReader)
{
int colIndex = 0;
while (reportReader.Read())
{
columns.Add(reportReader.Current.name, colIndex);
colIndex++;
}
///sssss
}
using (this.ImportManager = new GenericMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
MeasureOptions = MeasureOptions.IsBackOffice,
MeasureOptionsOperator = OptionsOperator.Or,
SegmentOptions = Data.Objects.SegmentOptions.All,
SegmentOptionsOperator = OptionsOperator.And
}))
{
this.ImportManager.BeginImport(this.Delivery);
Dictionary <string, GenericMetricsUnit> data = new Dictionary <string, GenericMetricsUnit>();
// Checksums
_isChecksum = true;
reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.Gzip), "$.totalsForAllResults.*");
using (reportReader)
{
this.Mappings.OnFieldRequired = field => reportReader.Current[field];
if (reportReader.Read())
{
GenericMetricsUnit checksumUnit = new GenericMetricsUnit();
metricsUnitMapping.Apply(checksumUnit);
foreach (var m in checksumUnit.MeasureValues)
{
if (m.Key.Options.HasFlag(MeasureOptions.ValidationRequired))
{
currentOutput.Checksum.Add(m.Key.Name, m.Value);
}
}
}
}
_isChecksum = false;
//Get Valuees
reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.Gzip), "$.rows[*].*");
using (reportReader)
{
this.Mappings.OnFieldRequired = field => reportReader.Current["array"][columns[field]];
while (reportReader.Read())
{
GenericMetricsUnit tempUnit = new GenericMetricsUnit();
metricsUnitMapping.Apply(tempUnit);
SegmentObject tracker = tempUnit.SegmentDimensions[ImportManager.SegmentTypes[Segment.Common.Tracker]];
GenericMetricsUnit existingUnit = null;
// check if we already found a metrics unit with the same tracker
if (!data.TryGetValue(tracker.Value, out existingUnit))
{
tempUnit.Output = currentOutput;
data.Add(tracker.Value, tempUnit);
}
else
{
// if tracker already exists, merge with existing values
foreach (var m in tempUnit.MeasureValues)
{
if (!m.Key.Options.HasFlag(MeasureOptions.IsBackOffice))
{
continue;
}
existingUnit.MeasureValues[m.Key] += m.Value;
}
}
}
}
// Import all unique units per tracker
foreach (GenericMetricsUnit metricsUnit in data.Values)
{
this.ImportManager.ImportMetrics(metricsUnit);
}
ImportManager.EndImport();
}
}
return(Core.Services.ServiceOutcome.Success);
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
MappingContainer metricsUnitMapping;
if (!this.Mappings.Objects.TryGetValue(typeof(GenericMetricsUnit), out metricsUnitMapping))
{
throw new MappingConfigurationException("Missing mapping definition for GenericMetricsUnit.");
}
currentOutput = this.Delivery.Outputs.First();
currentOutput.Checksum = new Dictionary <string, double>();
foreach (var ReportFile in Delivery.Files)
{
bool isAttribute = Boolean.Parse(ReportFile.Parameters["Bo.IsAttribute"].ToString());
var ReportReader = new XmlDynamicReader
(ReportFile.OpenContents(), ReportFile.Parameters["Bo.Xpath"].ToString());
using (this.ImportManager = new GenericMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
MeasureOptions = MeasureOptions.IsBackOffice,
MeasureOptionsOperator = OptionsOperator.Or,
SegmentOptions = Data.Objects.SegmentOptions.All,
SegmentOptionsOperator = OptionsOperator.And
}))
{
ImportManager.BeginImport(this.Delivery);
//checksum init
foreach (var m in ImportManager.Measures.Values)
{
if (m.Options.HasFlag(MeasureOptions.ValidationRequired))
{
currentOutput.Checksum[m.Name] = 0;
}
}
using (ReportReader)
{
dynamic readerHelper;
while (ReportReader.Read())
{
if (isAttribute)
{
readerHelper = ReportReader.Current.Attributes;
}
else
{
readerHelper = ReportReader.Current;
}
this.Mappings.OnFieldRequired = field => readerHelper[field];
GenericMetricsUnit genericMetricsUnit = new GenericMetricsUnit();
metricsUnitMapping.Apply(genericMetricsUnit);
foreach (var m in genericMetricsUnit.MeasureValues)
{
if (m.Key.Options.HasFlag(MeasureOptions.ValidationRequired))
{
if (!currentOutput.Checksum.ContainsKey(m.Key.Name))
{
currentOutput.Checksum.Add(m.Key.Name, m.Value);
}
else
{
currentOutput.Checksum[m.Key.Name] += m.Value;
}
}
}
genericMetricsUnit.Output = currentOutput;
ImportManager.ImportMetrics(genericMetricsUnit);
}
}
ImportManager.EndImport();
}
}
return(Core.Services.ServiceOutcome.Success);
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
MappingContainer metricsUnitMapping;
if (!this.Mappings.Objects.TryGetValue(typeof(GenericMetricsUnit), out metricsUnitMapping))
{
throw new MappingConfigurationException("Missing mapping definition for GenericMetricsUnit.");
}
currentOutput = this.Delivery.Outputs.First();
currentOutput.Checksum = new Dictionary <string, double>();
Dictionary <string, int> columns = new Dictionary <string, int>();
using (this.ImportManager = new GenericMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
MeasureOptions = MeasureOptions.IsBackOffice,
MeasureOptionsOperator = OptionsOperator.Or,
SegmentOptions = Data.Objects.SegmentOptions.All,
SegmentOptionsOperator = OptionsOperator.And
}))
{
this.ImportManager.BeginImport(this.Delivery);
Dictionary <string, GenericMetricsUnit> data = new Dictionary <string, GenericMetricsUnit>();
foreach (var ReportFile in Delivery.Files)
{
//check number of recordes
JsonDynamicReader reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.None), "$.totalSize");
int numOfRecordes = 0;
if (reportReader.Read())
{
numOfRecordes = int.Parse(reportReader.Current.totalSize);
}
reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.None), "$.nextRecordsUrl");
if (reportReader.Read())
{
Log.Write(string.Format("Salesforce Attention - account {0} contains more than 1 file per query {1}", this.Delivery.Account.ID, ReportFile.Parameters["Query"].ToString()), LogMessageType.Warning);
}
if (numOfRecordes > 0)
{
//Get Values
reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.None), "$.records[*].*");
using (reportReader)
{
this.Mappings.OnFieldRequired = field =>
{
object value = new object();
string[] nestedFields;
try
{
if (field.Contains('.'))
{
nestedFields = field.Split('.');
value = reportReader.Current[nestedFields[0]];
foreach (var item in nestedFields)
{
if (item.Equals(nestedFields[0]))
{
continue;
}
value = ((Dictionary <string, object>)value)[item];
}
value = reportReader.Current[nestedFields[0]][nestedFields[1]];
Console.Write(value.ToString());
return(value);
}
else
{
Console.Write(value.ToString());
return(reportReader.Current[field]);
}
}
catch (Exception ex)
{
throw new Exception(string.Format("Error while trying to map field {0} from mapper", field), ex);
}
};
while (reportReader.Read())
{
GenericMetricsUnit metricsUnit = new GenericMetricsUnit();
metricsUnitMapping.Apply(metricsUnit);
if (metricsUnit.MeasureValues != null)
{
SegmentObject tracker = metricsUnit.SegmentDimensions[ImportManager.SegmentTypes[Segment.Common.Tracker]];
GenericMetricsUnit importedUnit = null;
// check if we already found a metrics unit with the same tracker
if (!data.TryGetValue(tracker.Value, out importedUnit))
{
metricsUnit.Output = currentOutput;
data.Add(tracker.Value, metricsUnit);
}
else //Tracker already exists
{
// Merge captured measure with existing measures
foreach (var capturedMeasure in metricsUnit.MeasureValues)
{
if (!capturedMeasure.Key.Options.HasFlag(MeasureOptions.IsBackOffice))
{
continue;
}
//Measure already exists per tracker than aggregate:
if (importedUnit.MeasureValues.ContainsKey(capturedMeasure.Key))
{
importedUnit.MeasureValues[capturedMeasure.Key] += capturedMeasure.Value;
}
else
{
//Captured Measure doest exists with this tracker:
importedUnit.MeasureValues.Add(capturedMeasure.Key, capturedMeasure.Value);
}
}
}
#region Validation
// For validations
foreach (var m in metricsUnit.MeasureValues)
{
if (m.Key.Options.HasFlag(MeasureOptions.ValidationRequired))
{
if (!currentOutput.Checksum.ContainsKey(m.Key.Name))
{
currentOutput.Checksum.Add(m.Key.Name, m.Value);
}
else
{
currentOutput.Checksum[m.Key.Name] += m.Value;
}
}
}
#endregion
}
}
}
}
else
{
Log.Write("No Records Found in File " + ReportFile.Name, LogMessageType.Information);
}
}
// Import all unique units per tracker
foreach (GenericMetricsUnit metricsUnit in data.Values)
{
this.ImportManager.ImportMetrics(metricsUnit);
}
ImportManager.EndImport();
}
return(Core.Services.ServiceOutcome.Success);
}
//public CurrencyImportManager ImportManager;
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
List <CurrencyRate> rates = new List <CurrencyRate>();
//TO DO : USE MAPPING CONFIGURATION FOR THIS SERVICE.
//ImportManager = new CurrencyImportManager(this.Instance.InstanceID, null);
foreach (DeliveryFile ReportFile in this.Delivery.Files)
{
bool isAttribute = Boolean.Parse(ReportFile.Parameters["XML.IsAttribute"].ToString());
var ReportReader = new XmlDynamicReader
(ReportFile.OpenContents(), ReportFile.Parameters["XML.Path"].ToString());
//using (ImportManager)
//{
// ImportManager.BeginImport(this.Delivery);
//DeliveryOutput currentOutput = Delivery.Outputs.First();
using (ReportReader)
{
dynamic reader;
while (ReportReader.Read())
{
if (isAttribute)
{
reader = ReportReader.Current.Attributes;
}
else
{
reader = ReportReader.Current;
}
CurrencyRate currencyUnit = new CurrencyRate();
//Currency Code
currencyUnit.Currency.Code = Regex.Replace(Convert.ToString(reader["Symbol"]), "USD", string.Empty);
//Currecy Date
string[] date = (reader["Date"] as string).Split('/');
try { currencyUnit.RateDate = new DateTime(Int32.Parse(date[2]), Int32.Parse(date[0]), Int32.Parse(date[1])); }
catch (Exception ex)
{
throw new Exception(String.Format("Could not parse the date parts (y = '{0}', m = '{1}', d = '{2}'.", date[2], date[0], date[1]), ex);
}
//Currency Rate
double rate = Convert.ToDouble(reader["Last"]);
currencyUnit.RateValue = Convert.ToDecimal(rate.ToString("#0.0000"));
////Output
//currencyUnit.Output = new DeliveryOutput()
//{
// Signature = string.Format("{0}_{1}", currencyUnit.Currency.Code, currencyUnit.RateDate),
// TimePeriodStart = currencyUnit.DateCreated,
// TimePeriodEnd = currencyUnit.DateCreated,
//};
//this.Delivery.Outputs.Add(currencyUnit.Output);
//ImportManager.ImportCurrency(currencyUnit);
rates.Add(currencyUnit);
}
}
CurrencyRate.SaveCurrencyRates(rates);
//ImportManager.EndImport();
//}
}
return(Core.Services.ServiceOutcome.Success);
}