static void Main(string[] args)
{
var customActivity = new MongoDbDumpTransformActivity();
var config =
JObject.Parse(
File.ReadAllText(@"..\..\..\..\..\LocalSettings-DataFactory.json"));
var linkedServices = new List <LinkedService>()
{
new LinkedService("GHTorrentAzureStorage",
new LinkedServiceProperties(new CustomDataSourceLinkedService(JObject.Parse(String.Format("{{\"sasUri\": \"{0}\"}}", config["GHTorrentAzureStorage"][0]["value"].Value <string>())))))
, new LinkedService("GHInsightsAzureStorage",
new LinkedServiceProperties(new AzureStorageLinkedService(config["GHInsightsAzureStorage"][0]["value"].Value <string>())))
};
var mongoDbDump = new Dataset("MongoDbDump",
new DatasetProperties(new AzureBlobDataset()
{
FolderPath = "downloads/",
FileName = "mongo-dump-{Year}-{Month}-{Day}.tar.gz",
PartitionedBy = new List <Partition>()
{
{
new Partition()
{
Name = "Year",
Value = new DateTimePartitionValue()
{
Date = "SliceStart", Format = "yyyy"
}
}
},
{
new Partition()
{
Name = "Month",
Value = new DateTimePartitionValue()
{
Date = "SliceStart", Format = "MM"
}
}
},
{
new Partition()
{
Name = "Day",
Value = new DateTimePartitionValue()
{
Date = "SliceStart", Format = "dd"
}
}
}
}
},
new Availability("Daily", 1), "GHTorrentAzureStorage"));
var eventDetailRawFilesBlob = new Dataset("EventDetail",
new DatasetProperties(new AzureBlobDataset()
{
FolderPath = @"test/{EventName}/v1/{Year}/{Month}",
FileName = "{EventName}_{Year}_{Month}_{Day}.json.gz",
PartitionedBy = new List <Partition>()
{
{
new Partition()
{
Name = "Year",
Value = new DateTimePartitionValue()
{
Date = "SliceStart", Format = "yyyy"
}
}
},
{
new Partition()
{
Name = "Month",
Value = new DateTimePartitionValue()
{
Date = "SliceStart", Format = "MM"
}
}
},
{
new Partition()
{
Name = "Day",
Value = new DateTimePartitionValue()
{
Date = "SliceStart", Format = "dd"
}
}
}
}
},
new Availability("Daily", 1), "GHInsightsAzureStorage"));
var datasets = new List <Dataset>()
{
mongoDbDump, eventDetailRawFilesBlob
};
var activity = new Activity()
{
Description = "Fake Activity",
Inputs = new ActivityInput[] { new ActivityInput("MongoDbDump") },
LinkedServiceName = "BatchProcessor",
Name = "GHTorrentEventDetailPipeline",
Outputs = new ActivityOutput[] { new ActivityOutput("EventDetail") },
Policy =
new ActivityPolicy()
{
Concurrency = 4,
ExecutionPriorityOrder = "NewestFirst",
Retry = 3,
Timeout = TimeSpan.Parse("04:00:00")
},
Scheduler = new Scheduler("Day", 1),
TypeProperties = new DotNetActivity("DataFactoryLib.dll"
, "DataFactoryLib.MongoDbDumpTransformActivity"
, "datafactory/DataFactoryLib.zip"
, "GHInsightsAzureStorage")
{
ExtendedProperties = new Dictionary <string, string>()
{
{ "Year", "2015" },
{ "Month", "12" },
{ "Day", "02" }
}
}
};
IActivityLogger logger = new DebugLogger();
customActivity.Execute(linkedServices, datasets, activity, logger);
}
static void Main(string[] args)
{
var customActivity = new MongoDbDumpTransformActivity();
var config =
JObject.Parse(
File.ReadAllText(@"..\..\..\..\..\LocalSettings-DataFactory.json"));
var linkedServices = new List<LinkedService>()
{ new LinkedService("GHTorrentAzureStorage",
new LinkedServiceProperties(new CustomDataSourceLinkedService(JObject.Parse(String.Format("{{\"sasUri\": \"{0}\"}}", config["GHTorrentAzureStorage"][0]["value"].Value<string>())))))
,new LinkedService("GHInsightsAzureStorage",
new LinkedServiceProperties(new AzureStorageLinkedService(config["GHInsightsAzureStorage"][0]["value"].Value<string>())))
};
var mongoDbDump = new Dataset("MongoDbDump",
new DatasetProperties(new AzureBlobDataset()
{
FolderPath = "downloads/",
FileName = "mongo-dump-{Year}-{Month}-{Day}.tar.gz",
PartitionedBy = new List<Partition>()
{
{
new Partition()
{
Name = "Year",
Value = new DateTimePartitionValue() {Date = "SliceStart", Format = "yyyy"}
}
},
{
new Partition()
{
Name = "Month",
Value = new DateTimePartitionValue() {Date = "SliceStart", Format = "MM"}
}
},
{
new Partition()
{
Name = "Day",
Value = new DateTimePartitionValue() {Date = "SliceStart", Format = "dd"}
}
}
}
},
new Availability("Daily", 1), "GHTorrentAzureStorage"));
var eventDetailRawFilesBlob = new Dataset("EventDetail",
new DatasetProperties(new AzureBlobDataset()
{
FolderPath = @"test/{EventName}/v1/{Year}/{Month}",
FileName = "{EventName}_{Year}_{Month}_{Day}.json.gz",
PartitionedBy = new List<Partition>()
{
{
new Partition()
{
Name = "Year",
Value = new DateTimePartitionValue() {Date = "SliceStart", Format = "yyyy"}
}
},
{
new Partition()
{
Name = "Month",
Value = new DateTimePartitionValue() {Date = "SliceStart", Format = "MM"}
}
},
{
new Partition()
{
Name = "Day",
Value = new DateTimePartitionValue() {Date = "SliceStart", Format = "dd"}
}
}
}
},
new Availability("Daily", 1), "GHInsightsAzureStorage"));
var datasets = new List<Dataset>() { mongoDbDump, eventDetailRawFilesBlob };
var activity = new Activity()
{
Description = "Fake Activity",
Inputs = new ActivityInput[] {new ActivityInput("MongoDbDump")},
LinkedServiceName = "BatchProcessor",
Name = "GHTorrentEventDetailPipeline",
Outputs = new ActivityOutput[] {new ActivityOutput("EventDetail") },
Policy =
new ActivityPolicy()
{
Concurrency = 4,
ExecutionPriorityOrder = "NewestFirst",
Retry = 3,
Timeout = TimeSpan.Parse("04:00:00")
},
Scheduler = new Scheduler("Day", 1),
TypeProperties = new DotNetActivity("DataFactoryLib.dll"
, "DataFactoryLib.MongoDbDumpTransformActivity"
, "datafactory/DataFactoryLib.zip"
, "GHInsightsAzureStorage")
{
ExtendedProperties = new Dictionary<string, string>()
{
{"Year", "2015"},
{"Month", "12"},
{"Day", "02"}
}
}
};
IActivityLogger logger = new DebugLogger();
customActivity.Execute(linkedServices, datasets, activity, logger);
}