public async Task <string> FunctionHandler2(Vendors vendor, int buildingId, int chunkId, string prefix,
int attempt)
{
Dictionary <string, long> restorePoint = null;
while (true)
{
try
{
if (attempt > 55)
{
Console.WriteLine("Too many attempts");
return(null);
}
Settings.Initialize(buildingId, vendor);
Settings.Current.S3AwsAccessKeyId = "";
Settings.Current.S3AwsSecretAccessKey = "";
Settings.Current.Bucket = "";
Settings.Current.StorageType = S3StorageType.Parquet;
Settings.Current.CDMFolder = "cdm";
Settings.Current.TimeoutValue = 18000;
Settings.Current.WatchdogValue = 100000 * 1000;
Settings.Current.MinPersonToBuild = 100;
Settings.Current.MinPersonToSave = 100;
Console.WriteLine(
$"vendor={vendor};buildingId={buildingId};chunkId={chunkId};prefix={prefix};attempt={attempt}");
Initialize();
if (restorePoint != null)
{
_restorePoint = restorePoint;
}
_vocabulary.Attach();
var chunkBuilder = new LambdaChunkBuilder(CreatePersonBuilder);
_lastSavedPersonIdOutput = chunkBuilder.Process(chunkId, prefix, _restorePoint, attempt);
Console.WriteLine("DONE");
return("done");
}
catch (Exception e)
{
attempt++;
Console.WriteLine();
Console.WriteLine("Attempt: " + attempt);
Console.WriteLine();
restorePoint = new Dictionary <string, long>(_restorePoint);
}
}
}
/// <summary>
/// This method is called for every Lambda invocation. This method takes in an S3 event object and can be used
/// to respond to S3 notifications.
/// </summary>
/// <param name="evnt"></param>
/// <param name="context"></param>
/// <returns></returns>
public async Task <string> FunctionHandler(S3Event evnt, ILambdaContext context)
{
//return null;
var s3Event = evnt.Records?[0].S3;
if (s3Event == null)
{
return(null);
}
int chunkId = 0;
int buildingId = 0;
string prefix = String.Empty;
int attempt = 0;
Vendors vendor;
try
{
// 0 1 2 3 4 5
//vendor.buildingId.chunkId.prefix.attempt.txt
vendor = Enum.Parse <Vendors>(s3Event.Object.Key.Split('.')[0]);
buildingId = int.Parse(s3Event.Object.Key.Split('.')[1]);
chunkId = int.Parse(s3Event.Object.Key.Split('.')[2]);
prefix = s3Event.Object.Key.Split('.')[3].Trim();
if (s3Event.Object.Key.Split('.').Length == 6)
{
attempt = int.Parse(s3Event.Object.Key.Split('.')[4]);
}
Settings.Initialize(buildingId, vendor);
Settings.Current.TimeoutValue = 180;
Settings.Current.WatchdogValue = 10 * 1000;
Settings.Current.MinPersonToBuild = 100;
Settings.Current.MinPersonToSave = 100;
Settings.Current.S3AwsAccessKeyId = Environment.GetEnvironmentVariable("S3AwsAccessKeyId");
Settings.Current.S3AwsSecretAccessKey = Environment.GetEnvironmentVariable("S3AwsSecretAccessKey");
Settings.Current.Bucket = Environment.GetEnvironmentVariable("Bucket");
Settings.Current.CDMFolder = Environment.GetEnvironmentVariable("CDMFolder");
Settings.Current.StorageType = Enum.TryParse(Environment.GetEnvironmentVariable("StorageType"), true,
out S3StorageType storageType) ? storageType : S3StorageType.CSV;
//TODO different behavior(num of subChunks 256, 31...)
if (attempt > 55)
{
//Console.WriteLine($"*** too many attempt || chunkId={chunkId};prefix={prefix};attempt={attempt} - STARTED from PersonId={lastSavedPersonIdInput}");
return(null);
}
Console.WriteLine($"vendor={vendor};buildingId={buildingId};chunkId={chunkId};prefix={prefix};attempt={attempt}");
Console.WriteLine($"Bucket={Settings.Current.Bucket};CDMFolder={Settings.Current.CDMFolder};StorageType={Settings.Current.StorageType};");
Console.WriteLine($"TimeoutValue={Settings.Current.TimeoutValue}s;WatchdogValue={Settings.Current.WatchdogValue}ms;MinPersonToBuild={Settings.Current.MinPersonToBuild}; MinPersonToSave={Settings.Current.MinPersonToSave}");
Initialize();
GetRestorePoint(s3Event);
_vocabulary.Attach();
var chunkBuilder = new LambdaChunkBuilder(CreatePersonBuilder);
var attempt1 = attempt;
_lastSavedPersonIdOutput = chunkBuilder.Process(chunkId, prefix, _restorePoint, attempt1);
RemoveAttemptFile(s3Event);
if (_lastSavedPersonIdOutput.HasValue || (Settings.Current.WatchdogTimeout && chunkBuilder.TotalPersonConverted == 0))
{
attempt++;
CreateAttemptFile(s3Event, chunkId, prefix, attempt);
Console.WriteLine($"chunkId={chunkId};prefix={prefix} - FINISHED by timeout on PersonId={_lastSavedPersonIdOutput.Value}");
return("done");
}
Console.WriteLine($"chunkId={chunkId};prefix={prefix} - FINISHED, {s3Event.Object.Key} - removed");
return("done");
}
catch (Exception e)
{
Console.WriteLine($"WARN_EXC - FunctionHandler");
Console.WriteLine($"getting object {s3Event.Object.Key} from bucket {s3Event.Bucket.Name}. Make sure they exist and your bucket is in the same region as this function.");
Console.WriteLine(e.Message);
Console.WriteLine(e.StackTrace);
if (RemoveAttemptFile(s3Event))
{
attempt++;
if (!CreateAttemptFile(s3Event, chunkId, prefix, attempt))
{
Console.WriteLine($"Can't convert chunkId={chunkId} prefix={prefix} | CreateAttemptFile");
}
}
else
{
Console.WriteLine($"Can't convert chunkId={chunkId} prefix={prefix} | RemoveAttemptFile");
}
return("interrupted");
}
}
