public async Task <string> FunctionHandler2(Vendors vendor, int buildingId, int chunkId, string prefix, int attempt) { Dictionary <string, long> restorePoint = null; while (true) { try { if (attempt > 55) { Console.WriteLine("Too many attempts"); return(null); } Settings.Initialize(buildingId, vendor); Settings.Current.S3AwsAccessKeyId = ""; Settings.Current.S3AwsSecretAccessKey = ""; Settings.Current.Bucket = ""; Settings.Current.StorageType = S3StorageType.Parquet; Settings.Current.CDMFolder = "cdm"; Settings.Current.TimeoutValue = 18000; Settings.Current.WatchdogValue = 100000 * 1000; Settings.Current.MinPersonToBuild = 100; Settings.Current.MinPersonToSave = 100; Console.WriteLine( $"vendor={vendor};buildingId={buildingId};chunkId={chunkId};prefix={prefix};attempt={attempt}"); Initialize(); if (restorePoint != null) { _restorePoint = restorePoint; } _vocabulary.Attach(); var chunkBuilder = new LambdaChunkBuilder(CreatePersonBuilder); _lastSavedPersonIdOutput = chunkBuilder.Process(chunkId, prefix, _restorePoint, attempt); Console.WriteLine("DONE"); return("done"); } catch (Exception e) { attempt++; Console.WriteLine(); Console.WriteLine("Attempt: " + attempt); Console.WriteLine(); restorePoint = new Dictionary <string, long>(_restorePoint); } } }
/// <summary> /// This method is called for every Lambda invocation. This method takes in an S3 event object and can be used /// to respond to S3 notifications. /// </summary> /// <param name="evnt"></param> /// <param name="context"></param> /// <returns></returns> public async Task <string> FunctionHandler(S3Event evnt, ILambdaContext context) { //return null; var s3Event = evnt.Records?[0].S3; if (s3Event == null) { return(null); } int chunkId = 0; int buildingId = 0; string prefix = String.Empty; int attempt = 0; Vendors vendor; try { // 0 1 2 3 4 5 //vendor.buildingId.chunkId.prefix.attempt.txt vendor = Enum.Parse <Vendors>(s3Event.Object.Key.Split('.')[0]); buildingId = int.Parse(s3Event.Object.Key.Split('.')[1]); chunkId = int.Parse(s3Event.Object.Key.Split('.')[2]); prefix = s3Event.Object.Key.Split('.')[3].Trim(); if (s3Event.Object.Key.Split('.').Length == 6) { attempt = int.Parse(s3Event.Object.Key.Split('.')[4]); } Settings.Initialize(buildingId, vendor); Settings.Current.TimeoutValue = 180; Settings.Current.WatchdogValue = 10 * 1000; Settings.Current.MinPersonToBuild = 100; Settings.Current.MinPersonToSave = 100; Settings.Current.S3AwsAccessKeyId = Environment.GetEnvironmentVariable("S3AwsAccessKeyId"); Settings.Current.S3AwsSecretAccessKey = Environment.GetEnvironmentVariable("S3AwsSecretAccessKey"); Settings.Current.Bucket = Environment.GetEnvironmentVariable("Bucket"); Settings.Current.CDMFolder = Environment.GetEnvironmentVariable("CDMFolder"); Settings.Current.StorageType = Enum.TryParse(Environment.GetEnvironmentVariable("StorageType"), true, out S3StorageType storageType) ? storageType : S3StorageType.CSV; //TODO different behavior(num of subChunks 256, 31...) if (attempt > 55) { //Console.WriteLine($"*** too many attempt || chunkId={chunkId};prefix={prefix};attempt={attempt} - STARTED from PersonId={lastSavedPersonIdInput}"); return(null); } Console.WriteLine($"vendor={vendor};buildingId={buildingId};chunkId={chunkId};prefix={prefix};attempt={attempt}"); Console.WriteLine($"Bucket={Settings.Current.Bucket};CDMFolder={Settings.Current.CDMFolder};StorageType={Settings.Current.StorageType};"); Console.WriteLine($"TimeoutValue={Settings.Current.TimeoutValue}s;WatchdogValue={Settings.Current.WatchdogValue}ms;MinPersonToBuild={Settings.Current.MinPersonToBuild}; MinPersonToSave={Settings.Current.MinPersonToSave}"); Initialize(); GetRestorePoint(s3Event); _vocabulary.Attach(); var chunkBuilder = new LambdaChunkBuilder(CreatePersonBuilder); var attempt1 = attempt; _lastSavedPersonIdOutput = chunkBuilder.Process(chunkId, prefix, _restorePoint, attempt1); RemoveAttemptFile(s3Event); if (_lastSavedPersonIdOutput.HasValue || (Settings.Current.WatchdogTimeout && chunkBuilder.TotalPersonConverted == 0)) { attempt++; CreateAttemptFile(s3Event, chunkId, prefix, attempt); Console.WriteLine($"chunkId={chunkId};prefix={prefix} - FINISHED by timeout on PersonId={_lastSavedPersonIdOutput.Value}"); return("done"); } Console.WriteLine($"chunkId={chunkId};prefix={prefix} - FINISHED, {s3Event.Object.Key} - removed"); return("done"); } catch (Exception e) { Console.WriteLine($"WARN_EXC - FunctionHandler"); Console.WriteLine($"getting object {s3Event.Object.Key} from bucket {s3Event.Bucket.Name}. Make sure they exist and your bucket is in the same region as this function."); Console.WriteLine(e.Message); Console.WriteLine(e.StackTrace); if (RemoveAttemptFile(s3Event)) { attempt++; if (!CreateAttemptFile(s3Event, chunkId, prefix, attempt)) { Console.WriteLine($"Can't convert chunkId={chunkId} prefix={prefix} | CreateAttemptFile"); } } else { Console.WriteLine($"Can't convert chunkId={chunkId} prefix={prefix} | RemoveAttemptFile"); } return("interrupted"); } }