public async Task <ActionResult> Offline(string startTimeInclusive, string endTimeExclusive, string dataFormat = "json")
{
var token = Request.Headers["auth"];
if (token != ConfigurationManager.AppSettings[ApplicationMetadataStore.AKPassword])
{
throw new UnauthorizedAccessException();
}
var storageAccount = CloudStorageAccount.Parse(ConfigurationManager.AppSettings[ApplicationMetadataStore.AKConnectionString]);
var blobClient = storageAccount.CreateCloudBlobClient();
StreamWriter responseWriter = null;
switch (dataFormat)
{
case "json":
responseWriter = new StreamWriter(Response.OutputStream, Encoding.UTF8);
break;
case "vw":
var settingsBlobContainer = blobClient.GetContainerReference(ApplicationBlobConstants.SettingsContainerName);
var blob = settingsBlobContainer.GetBlockBlobReference(ApplicationBlobConstants.LatestClientSettingsBlobName);
if (!await blob.ExistsAsync())
{
return(new HttpStatusCodeResult(HttpStatusCode.InternalServerError, "Application settings blob not found."));
}
ApplicationClientMetadata clientMeta = JsonConvert.DeserializeObject <ApplicationClientMetadata>(await blob.DownloadTextAsync());
responseWriter = new VowpalWabbitStreamWriter(Response.OutputStream, Encoding.UTF8, clientMeta.TrainArguments);
break;
default:
return(new HttpStatusCodeResult(HttpStatusCode.BadRequest, "Unrecognized data format."));
}
using (responseWriter)
{
await AzureBlobDownloader.Download(
storageAccount,
DateTime.ParseExact(startTimeInclusive, "yyyyMMddHHmm", System.Globalization.CultureInfo.InvariantCulture),
DateTime.ParseExact(endTimeExclusive, "yyyyMMddHHmm", System.Globalization.CultureInfo.InvariantCulture),
responseWriter).ConfigureAwait(false);
}
return(new HttpStatusCodeResult(HttpStatusCode.OK));
}
static void Main(string[] args)
{
try
{
var stopwatch = Stopwatch.StartNew();
var storageAccount = new CloudStorageAccount(new StorageCredentials("storage name", "storage key"), false);
var outputDirectory = @"c:\temp\";
Directory.CreateDirectory(outputDirectory);
var startTimeInclusive = new DateTime(2016, 8, 11, 19, 0, 0);
var endTimeExclusive = new DateTime(2016, 8, 18, 0, 0, 0);
var outputFile = Path.Combine(outputDirectory, $"{startTimeInclusive:yyyy-MM-dd_HH}-{endTimeExclusive:yyyy-MM-dd_HH}.json");
// download and merge blob data
using (var writer = new StreamWriter(outputFile))
{
AzureBlobDownloader.Download(storageAccount, startTimeInclusive, endTimeExclusive, writer, outputDirectory).Wait();
}
// pre-process JSON
JsonTransform.TransformIgnoreProperties(outputFile, outputFile + ".small",
"Somefeatures");
outputFile += ".small";
// filter broken events
JsonTransform.TransformFixMarginal(outputFile,
numExpectedActions: 10, // examples with different number of actions are ignored
startingNamespace: 'G', // starting namespace of the marginal features, if more than one marginal features then the next letter is used, e.g. G for the first one, H for second, and so on.
marginalProperties: new TupleList <string, string>
{
// The property parent and name to create marginal features for
{ "DVideoFeatures", "VideoId" },
//{ "DVideoFeatures", "VideoTitle" }, // uncomment if more marginal features are needed
});
outputFile += ".fixed";
using (var reader = new StreamReader(outputFile))
using (var writer = new StreamWriter(new GZipStream(File.Create(outputFile + ".vw.gz"), CompressionLevel.Optimal)))
{
VowpalWabbitJsonToString.Convert(reader, writer);
}
var bags = new[] { 1, 2, 4, 6, 8, 10 }.Select(a => "--bag " + a);
var softmaxes = new[] { 0, 1, 2, 4, 8, 16, 32 }.Select(a => "--softmax --lambda " + a);
var epsilons = new[] { .33333f, .2f, .1f, .05f }.Select(a => "--epsilon " + a);
var arguments = Util.Expand(
epsilons.Union(bags).Union(softmaxes),
new[] { "--cb_type ips", "--cb_type mtr", "--cb_type dr" },
new[] { "-q AB -q UD" },
new[] { 0.005, 0.01, 0.02, 0.1 }.Select(l => string.Format(CultureInfo.InvariantCulture, "-l {0}", l))
)
.Select(a => $"--cb_explore_adf {a} --interact ud ")
.ToList();
var sep = "\t";
var historyFile = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "mwt.experiments");
using (var historyWriter = new StreamWriter(File.Open(historyFile, FileMode.OpenOrCreate)))
{
for (int i = 0; i < arguments.Count; i++)
{
var startTime = DateTime.UtcNow;
var outputPredictionFile = $"{outputFile}.prediction";
var outputPrediction2hFile = $"{outputFile}.{i + 1}.2h.prediction";
// VW training
OfflineTrainer.Train(arguments[i],
outputFile,
predictionFile: outputPrediction2hFile,
reloadInterval: TimeSpan.FromHours(2),
cacheFilePrefix: null); // null to use input file's name for cache, see the method documentation for more details
var metricResult = Metrics.Compute(outputFile, outputPredictionFile, outputPrediction2hFile);
historyWriter.WriteLine($"{startTime}{sep}{arguments[i]}{sep}{string.Join(sep, metricResult.Select(m => m.Name + sep + m.Value))}");
}
}
Console.WriteLine("\ndone " + stopwatch.Elapsed);
Console.WriteLine("Run information is added to: ", historyFile);
}
catch (Exception ex)
{
Console.WriteLine($"Exception: {ex.Message}. {ex.StackTrace}");
}
Console.ReadKey();
}