/// <summary>
/// Unmarshaller the response from the service to the response class.
/// </summary>
/// <param name="context"></param>
/// <returns></returns>
public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context)
{
CreateJobResponse response = new CreateJobResponse();
context.Read();
int targetDepth = context.CurrentDepth;
while (context.ReadAtDepth(targetDepth))
{
if (context.TestExpression("description", targetDepth))
{
var unmarshaller = StringUnmarshaller.Instance;
response.Description = unmarshaller.Unmarshall(context);
continue;
}
if (context.TestExpression("jobArn", targetDepth))
{
var unmarshaller = StringUnmarshaller.Instance;
response.JobArn = unmarshaller.Unmarshall(context);
continue;
}
if (context.TestExpression("jobId", targetDepth))
{
var unmarshaller = StringUnmarshaller.Instance;
response.JobId = unmarshaller.Unmarshall(context);
continue;
}
}
return(response);
}
public override AmazonWebServiceResponse Unmarshall(XmlUnmarshallerContext context)
{
CreateJobResponse response = new CreateJobResponse();
context.Read();
int targetDepth = context.CurrentDepth;
while (context.ReadAtDepth(targetDepth))
{
if (context.IsStartElement)
{
if (context.TestExpression("CreateJobResult", 2))
{
UnmarshallResult(context, response);
continue;
}
if (context.TestExpression("ResponseMetadata", 2))
{
response.ResponseMetadata = ResponseMetadataUnmarshaller.Instance.Unmarshall(context);
}
}
}
return(response);
}
/// <summary>
/// <para> This operation initiates the process of scheduling an upload or
/// download of your data. You include in the request a manifest that
/// describes the data transfer specifics. The response to the request
/// includes a job ID, which you can use in other operations, a signature
/// that you use to identify your storage device, and the address where
/// you should ship your storage device. </para>
/// </summary>
///
/// <param name="createJobRequest">Container for the necessary parameters
/// to execute the CreateJob service method on AmazonImportExport.</param>
///
/// <returns>The response from the CreateJob service method, as returned
/// by AmazonImportExport.</returns>
///
/// <exception cref="MalformedManifestException"/>
/// <exception cref="InvalidAddressException"/>
/// <exception cref="BucketPermissionException"/>
/// <exception cref="InvalidParameterException"/>
/// <exception cref="MultipleRegionsException"/>
/// <exception cref="MissingParameterException"/>
/// <exception cref="InvalidFileSystemException"/>
/// <exception cref="MissingCustomsException"/>
/// <exception cref="NoSuchBucketException"/>
/// <exception cref="InvalidAccessKeyIdException"/>
/// <exception cref="InvalidManifestFieldException"/>
/// <exception cref="InvalidCustomsException"/>
/// <exception cref="MissingManifestFieldException"/>
/// <exception cref="CreateJobQuotaExceededException"/>
public CreateJobResponse CreateJob(CreateJobRequest createJobRequest)
{
IRequest <CreateJobRequest> request = new CreateJobRequestMarshaller().Marshall(createJobRequest);
CreateJobResponse response = Invoke <CreateJobRequest, CreateJobResponse> (request, this.signer, CreateJobResponseUnmarshaller.GetInstance());
return(response);
}
public async Task <CreateJobResponse> CreateUploadJobAsync(Guid guid, string mark)
{
var result = new CreateJobResponse();
var body = this.CreateCreateUploadJobRequestBody(guid, UploadJobTypeEnum.ReviseInventoryStatus);
var headers = this.CreateCreateUploadJobRequestHeaders();
await ActionPolicies.GetAsync.Do(async() =>
{
var webRequest = await this.CreateEbayStandartPostRequestToBulkExchangeServerAsync(this._endPointBulkExhange, headers, body).ConfigureAwait(false);
using (var memStream = await this._webRequestServices.GetResponseStreamAsync(webRequest, mark, CancellationToken.None).ConfigureAwait(false))
{
var createJobResponseParsed = new EbayBulkCreateJobParser().Parse(memStream);
if (createJobResponseParsed != null)
{
if (createJobResponseParsed.Errors != null)
{
result.Errors = createJobResponseParsed.Errors;
return;
}
result.JobId = createJobResponseParsed.JobId;
}
}
});
return(result);
}
private static void UnmarshallResult(XmlUnmarshallerContext context, CreateJobResponse response)
{
int originalDepth = context.CurrentDepth;
int targetDepth = originalDepth + 1;
if (context.IsStartOfDocument)
{
targetDepth += 1;
}
while (context.Read())
{
if (context.IsStartElement || context.IsAttribute)
{
if (context.TestExpression("JobId", targetDepth))
{
var unmarshaller = StringUnmarshaller.Instance;
response.JobId = unmarshaller.Unmarshall(context);
continue;
}
}
else if (context.IsEndElement && context.CurrentDepth < originalDepth)
{
return;
}
}
return;
}
public void JobDeleteJobRequest()
{
CreateJobResponse createResponse = Zencoder.CreateJob("s3://bucket-name/file-name.avi", null, null, null, true, false);
Assert.IsTrue(createResponse.Success);
// TODO: Investigate whether Zencoder has truly deprecated this API operation.
// For now, just test for an InConflict status, because that's what it seems
// we should expect.
DeleteJobResponse deleteResponse = Zencoder.DeleteJob(createResponse.Id);
Assert.IsTrue(deleteResponse.InConflict);
AutoResetEvent[] handles = new AutoResetEvent[] { new AutoResetEvent(false) };
Zencoder.DeleteJob(
createResponse.Id,
r =>
{
Assert.IsTrue(r.InConflict);
handles[0].Set();
});
WaitHandle.WaitAll(handles);
}
public void JobJobProgressRequest()
{
Output output = new Output()
{
Label = "iPhone",
Url = "s3://output-bucket/output-file-1-name.mp4",
Width = 480,
Height = 320
};
CreateJobResponse createResponse = Zencoder.CreateJob("s3://bucket-name/file-name.avi", new Output[] { output });
Assert.IsTrue(createResponse.Success);
JobProgressResponse progressResponse = Zencoder.JobProgress(createResponse.Outputs.First().Id);
Assert.IsTrue(progressResponse.Success);
AutoResetEvent[] handles = new AutoResetEvent[] { new AutoResetEvent(false) };
Zencoder.JobProgress(
createResponse.Outputs.First().Id,
r =>
{
Assert.IsTrue(r.Success);
handles[0].Set();
});
WaitHandle.WaitAll(handles);
}
/// <summary>
/// Unmarshaller the response from the service to the response class.
/// </summary>
/// <param name="context"></param>
/// <returns></returns>
public override AmazonWebServiceResponse Unmarshall(XmlUnmarshallerContext context)
{
CreateJobResponse response = new CreateJobResponse();
UnmarshallResult(context, response);
return(response);
}
public void JobCreateJobResponseFromJson()
{
CreateJobResponse response = CreateJobResponse.FromJson(@"{""id"":""1234"",""outputs"":[{""id"":""4321""}]}");
Assert.AreEqual(1234, response.Id);
Assert.AreEqual(1, response.Outputs.Length);
Assert.AreEqual(4321, response.Outputs.First().Id);
}
public static CreateJobResponse Unmarshall(UnmarshallerContext context)
{
CreateJobResponse createJobResponse = new CreateJobResponse();
createJobResponse.HttpResponse = context.HttpResponse;
createJobResponse.RequestId = context.StringValue("CreateJob.RequestId");
return(createJobResponse);
}
private static void UnmarshallResult(XmlUnmarshallerContext context, CreateJobResponse response)
{
int originalDepth = context.CurrentDepth;
int targetDepth = originalDepth + 1;
if (context.IsStartOfDocument)
{
targetDepth += 2;
}
while (context.ReadAtDepth(originalDepth))
{
if (context.IsStartElement || context.IsAttribute)
{
if (context.TestExpression("ArtifactList/member", targetDepth))
{
var unmarshaller = ArtifactUnmarshaller.Instance;
var item = unmarshaller.Unmarshall(context);
response.ArtifactList.Add(item);
continue;
}
if (context.TestExpression("JobId", targetDepth))
{
var unmarshaller = StringUnmarshaller.Instance;
response.JobId = unmarshaller.Unmarshall(context);
continue;
}
if (context.TestExpression("JobType", targetDepth))
{
var unmarshaller = StringUnmarshaller.Instance;
response.JobType = unmarshaller.Unmarshall(context);
continue;
}
if (context.TestExpression("Signature", targetDepth))
{
var unmarshaller = StringUnmarshaller.Instance;
response.Signature = unmarshaller.Unmarshall(context);
continue;
}
if (context.TestExpression("SignatureFileContents", targetDepth))
{
var unmarshaller = StringUnmarshaller.Instance;
response.SignatureFileContents = unmarshaller.Unmarshall(context);
continue;
}
if (context.TestExpression("WarningMessage", targetDepth))
{
var unmarshaller = StringUnmarshaller.Instance;
response.WarningMessage = unmarshaller.Unmarshall(context);
continue;
}
}
}
return;
}
public void JobCancelJobRequest()
{
CreateJobResponse createResponse = Zencoder.CreateJob("s3://bucket-name/file-name.avi", null, null, null, true, false);
Assert.IsTrue(createResponse.Success);
CancelJobResponse cancelResponse = Zencoder.CancelJob(createResponse.Id);
Assert.IsTrue(cancelResponse.Success);
}
public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context)
{
CreateJobResponse response = new CreateJobResponse();
context.Read();
response.CreateJobResult = CreateJobResultUnmarshaller.GetInstance().Unmarshall(context);
return(response);
}
public static CreateJobResponse Unmarshall(UnmarshallerContext _ctx)
{
CreateJobResponse createJobResponse = new CreateJobResponse();
createJobResponse.HttpResponse = _ctx.HttpResponse;
createJobResponse.RequestId = _ctx.StringValue("CreateJob.RequestId");
createJobResponse.Success = _ctx.BooleanValue("CreateJob.Success");
createJobResponse.Code = _ctx.StringValue("CreateJob.Code");
createJobResponse.ErrorMessage = _ctx.StringValue("CreateJob.ErrorMessage");
createJobResponse.JobId = _ctx.StringValue("CreateJob.JobId");
return(createJobResponse);
}
public void OutputH264CreateJob()
{
Output output = new Output()
{
H264Level = H264Level.FourPointOne,
H264ReferenceFrames = 5,
H264Profile = H264Profile.High,
Tuning = Tuning.FastDecode
};
CreateJobResponse response = Zencoder.CreateJob("s3://bucket-name/file-name.avi", new Output[] { output });
Assert.IsTrue(response.Success);
}
public static CreateJobResponse Unmarshall(UnmarshallerContext _ctx)
{
CreateJobResponse createJobResponse = new CreateJobResponse();
createJobResponse.HttpResponse = _ctx.HttpResponse;
createJobResponse.RequestId = _ctx.StringValue("CreateJob.RequestId");
createJobResponse.Code = _ctx.IntegerValue("CreateJob.Code");
createJobResponse.Success = _ctx.BooleanValue("CreateJob.Success");
createJobResponse.Message = _ctx.StringValue("CreateJob.Message");
CreateJobResponse.CreateJob_Data data = new CreateJobResponse.CreateJob_Data();
data.JobId = _ctx.LongValue("CreateJob.Data.JobId");
createJobResponse.Data = data;
return(createJobResponse);
}
public void JobCreateJobRequest()
{
Output[] outputs = new Output[]
{
new Output()
{
Label = "iPhone",
Url = "s3://output-bucket/output-file-1-name.mp4",
Width = 480,
Height = 320
},
new Output()
{
Label = "WebHD",
Url = "s3://output-bucket/output-file-2-name.mp4",
Width = 1280,
Height = 720
}
};
CreateJobResponse response = Zencoder.CreateJob("s3://bucket-name/file-name.avi", outputs, null, null, true);
Assert.IsTrue(response.Success);
Assert.IsTrue(response.Id > 0);
Assert.AreEqual(outputs.Count(), response.Outputs.Count());
Assert.IsTrue(response.Outputs.First().Id > 0);
AutoResetEvent[] handles = new AutoResetEvent[] { new AutoResetEvent(false) };
Zencoder.CreateJob(
"s3://bucket-name/file-name.avi",
null,
3,
"asia",
true,
r =>
{
Assert.IsTrue(r.Success);
Assert.IsTrue(r.Id > 0);
Assert.IsTrue(r.Outputs.Count() > 0);
Assert.IsTrue(r.Outputs.First().Id > 0);
handles[0].Set();
});
WaitHandle.WaitAll(handles);
}
public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context)
{
CreateJobResponse response = new CreateJobResponse();
context.Read();
int targetDepth = context.CurrentDepth;
while (context.ReadAtDepth(targetDepth))
{
if (context.TestExpression("Job", targetDepth))
{
response.Job = JobUnmarshaller.GetInstance().Unmarshall(context);
continue;
}
}
return(response);
}
public void JobResubmitJobRequest()
{
CreateJobResponse createResponse = Zencoder.CreateJob("s3://bucket-name/file-name.avi", null, null, null, true);
Assert.IsTrue(createResponse.Success);
ResubmitJobResponse resubmitResponse = Zencoder.ResubmitJob(createResponse.Id);
Assert.IsTrue(resubmitResponse.Success);
AutoResetEvent[] handles = new AutoResetEvent[] { new AutoResetEvent(false) };
Zencoder.ResubmitJob(
createResponse.Id,
r =>
{
Assert.IsTrue(r.Success);
handles[0].Set();
});
WaitHandle.WaitAll(handles);
}
public void JobDeleteJobRequest()
{
CreateJobResponse createResponse = Zencoder.CreateJob("s3://bucket-name/file-name.avi", null, null, null, true);
Assert.IsTrue(createResponse.Success);
DeleteJobResponse deleteResponse = Zencoder.DeleteJob(createResponse.Id);
Assert.IsTrue(deleteResponse.Success);
AutoResetEvent[] handles = new AutoResetEvent[] { new AutoResetEvent(false) };
Zencoder.DeleteJob(
createResponse.Id,
r =>
{
Assert.IsTrue(r.StatusCode == HttpStatusCode.NotFound);
handles[0].Set();
});
WaitHandle.WaitAll(handles);
}
public void JobCancelJobRequestAsync()
{
CreateJobResponse createResponse = Zencoder.CreateJob("s3://bucket-name/file-name.avi", null, null, null, true, false);
Assert.IsTrue(createResponse.Success);
AutoResetEvent[] handles = new AutoResetEvent[] { new AutoResetEvent(false) };
CancelJobResponse asyncResponse = null;
Zencoder.CancelJob(
createResponse.Id,
r =>
{
asyncResponse = r;
handles[0].Set();
});
WaitHandle.WaitAll(handles);
Assert.IsNotNull(asyncResponse);
Assert.IsTrue(asyncResponse.Success);
}
private static void ConfigureCreateJobRequest(SimpleMapper mapper)
{
mapper.CreateMap <CreateJobRequest, IppRequestMessage>((src, map) =>
{
var dst = new IppRequestMessage {
IppOperation = IppOperation.CreateJob
};
mapper.Map <IIppPrinterRequest, IppRequestMessage>(src, dst);
if (src.NewJobAttributes != null)
{
map.Map(src.NewJobAttributes, dst);
}
return(dst);
});
mapper.CreateMap <IppResponseMessage, CreateJobResponse>((src, map) =>
{
var dst = new CreateJobResponse();
map.Map <IppResponseMessage, IIppJobResponse>(src, dst);
return(dst);
});
}
private static void UnmarshallResult(JsonUnmarshallerContext context, CreateJobResponse response)
{
int originalDepth = context.CurrentDepth;
int targetDepth = originalDepth + 1;
while (context.Read())
{
if (context.TestExpression("Job", targetDepth))
{
context.Read();
response.Job = JobUnmarshaller.GetInstance().Unmarshall(context);
continue;
}
if (context.CurrentDepth <= originalDepth)
{
return;
}
}
return;
}
public override AmazonWebServiceResponse Unmarshall(XmlUnmarshallerContext context)
{
CreateJobResponse response = new CreateJobResponse();
while (context.Read())
{
if (context.IsStartElement)
{
if (context.TestExpression("CreateJobResult", 2))
{
response.CreateJobResult = CreateJobResultUnmarshaller.GetInstance().Unmarshall(context);
continue;
}
if (context.TestExpression("ResponseMetadata", 2))
{
response.ResponseMetadata = ResponseMetadataUnmarshaller.GetInstance().Unmarshall(context);
}
}
}
return(response);
}
static async Task <string> CreateMediaConverterJobAsync(string fileName)
{
// TODO: bucket name to be change and regsion name also
string mediaConvertRole = "arn:aws:iam::026141093142:role/MediaConvert_Default_Role";
string fileInput = "s3://sachinmediaconverter/" + fileName;
string fileOutput = "s3://sachinmediaconverter/";
// Once you know what your customer endpoint is, set it here
string mediaConvertEndpoint = "";
// If we do not have our customer-specific endpoint
if (String.IsNullOrEmpty(mediaConvertEndpoint))
{
// Obtain the customer-specific MediaConvert endpoint
AmazonMediaConvertClient client = new AmazonMediaConvertClient(Amazon.RegionEndpoint.USEast2);
DescribeEndpointsRequest describeRequest = new DescribeEndpointsRequest();
DescribeEndpointsResponse describeResponse = await client.DescribeEndpointsAsync(describeRequest);
mediaConvertEndpoint = describeResponse.Endpoints[0].Url;
}
// Since we have a service url for MediaConvert, we do not
// need to set RegionEndpoint. If we do, the ServiceURL will
// be overwritten
AmazonMediaConvertConfig mcConfig = new AmazonMediaConvertConfig {
ServiceURL = mediaConvertEndpoint,
};
AmazonMediaConvertClient mcClient = new AmazonMediaConvertClient(mcConfig);
CreateJobRequest createJobRequest = new CreateJobRequest();
createJobRequest.Role = mediaConvertRole;
createJobRequest.UserMetadata.Add("Customer", "Amazon");
#region Create job settings
JobSettings jobSettings = new JobSettings();
jobSettings.AdAvailOffset = 0;
jobSettings.TimecodeConfig = new TimecodeConfig();
jobSettings.TimecodeConfig.Source = TimecodeSource.ZEROBASED;
createJobRequest.Settings = jobSettings;
#region OutputGroup 1
OutputGroup ofg = new OutputGroup();
ofg.Name = "File Group";
ofg.CustomName = "test";
ofg.OutputGroupSettings = new OutputGroupSettings();
ofg.OutputGroupSettings.Type = OutputGroupType.FILE_GROUP_SETTINGS;
ofg.OutputGroupSettings.FileGroupSettings = new FileGroupSettings();
ofg.OutputGroupSettings.FileGroupSettings.Destination = fileOutput;
Output output = new Output();
output.NameModifier = "_output";
output.Preset = "System-Broadcast_Xdcam_Mxf_Mpeg2_Wav_16x9_1280x720p_60Hz_50Mbps";
output.Extension = ".mp4";
#region VideoDescription
//VideoDescription vdes = new VideoDescription();
//output.VideoDescription = vdes;
//vdes.ScalingBehavior = ScalingBehavior.DEFAULT;
//vdes.TimecodeInsertion = VideoTimecodeInsertion.DISABLED;
//vdes.AntiAlias = AntiAlias.ENABLED;
//vdes.Sharpness = 50;
//vdes.AfdSignaling = AfdSignaling.NONE;
//vdes.DropFrameTimecode = DropFrameTimecode.ENABLED;
//vdes.RespondToAfd = RespondToAfd.NONE;
//vdes.ColorMetadata = ColorMetadata.INSERT;
//vdes.CodecSettings = new VideoCodecSettings();
//vdes.CodecSettings.Codec = VideoCodec.H_264;
//H264Settings h264 = new H264Settings();
//h264.InterlaceMode = H264InterlaceMode.PROGRESSIVE;
//h264.NumberReferenceFrames = 3;
//h264.Syntax = H264Syntax.DEFAULT;
//h264.Softness = 0;
//h264.GopClosedCadence = 1;
//h264.GopSize = 90;
//h264.Slices = 1;
//h264.GopBReference = H264GopBReference.DISABLED;
//h264.SlowPal = H264SlowPal.DISABLED;
//h264.SpatialAdaptiveQuantization = H264SpatialAdaptiveQuantization.ENABLED;
//h264.TemporalAdaptiveQuantization = H264TemporalAdaptiveQuantization.ENABLED;
//h264.FlickerAdaptiveQuantization = H264FlickerAdaptiveQuantization.DISABLED;
//h264.EntropyEncoding = H264EntropyEncoding.CABAC;
//h264.Bitrate = 5000000;
//h264.FramerateControl = H264FramerateControl.SPECIFIED;
//h264.RateControlMode = H264RateControlMode.CBR;
//h264.CodecProfile = H264CodecProfile.MAIN;
//h264.Telecine = H264Telecine.NONE;
//h264.MinIInterval = 0;
//h264.AdaptiveQuantization = H264AdaptiveQuantization.HIGH;
//h264.CodecLevel = H264CodecLevel.AUTO;
//h264.FieldEncoding = H264FieldEncoding.PAFF;
//h264.SceneChangeDetect = H264SceneChangeDetect.ENABLED;
//h264.QualityTuningLevel = H264QualityTuningLevel.SINGLE_PASS;
//h264.FramerateConversionAlgorithm = H264FramerateConversionAlgorithm.DUPLICATE_DROP;
//h264.UnregisteredSeiTimecode = H264UnregisteredSeiTimecode.DISABLED;
//h264.GopSizeUnits = H264GopSizeUnits.FRAMES;
//h264.ParControl = H264ParControl.SPECIFIED;
//h264.NumberBFramesBetweenReferenceFrames = 2;
//h264.RepeatPps = H264RepeatPps.DISABLED;
//h264.FramerateNumerator = 30;
//h264.FramerateDenominator = 1;
//h264.ParNumerator = 1;
//h264.ParDenominator = 1;
//output.VideoDescription.CodecSettings.H264Settings = h264;
#endregion VideoDescription
#region AudioDescription
//AudioDescription ades = new AudioDescription();
//ades.LanguageCodeControl = AudioLanguageCodeControl.FOLLOW_INPUT;
//// This name matches one specified in the Inputs below
//ades.AudioSourceName = "Audio Selector 1";
//ades.CodecSettings = new AudioCodecSettings();
//ades.CodecSettings.Codec = AudioCodec.AAC;
//AacSettings aac = new AacSettings();
//aac.AudioDescriptionBroadcasterMix = AacAudioDescriptionBroadcasterMix.NORMAL;
//aac.RateControlMode = AacRateControlMode.CBR;
//aac.CodecProfile = AacCodecProfile.LC;
//aac.CodingMode = AacCodingMode.CODING_MODE_2_0;
//aac.RawFormat = AacRawFormat.NONE;
//aac.SampleRate = 48000;
//aac.Specification = AacSpecification.MPEG4;
//aac.Bitrate = 64000;
//ades.CodecSettings.AacSettings = aac;
//output.AudioDescriptions.Add(ades);
#endregion AudioDescription
#region Mp4 Container
//output.ContainerSettings = new ContainerSettings();
//output.ContainerSettings.Container = ContainerType.MP4;
//Mp4Settings mp4 = new Mp4Settings();
//mp4.CslgAtom = Mp4CslgAtom.INCLUDE;
//mp4.FreeSpaceBox = Mp4FreeSpaceBox.EXCLUDE;
//mp4.MoovPlacement = Mp4MoovPlacement.PROGRESSIVE_DOWNLOAD;
//output.ContainerSettings.Mp4Settings = mp4;
#endregion Mp4 Container
ofg.Outputs.Add(output);
createJobRequest.Settings.OutputGroups.Add(ofg);
#endregion OutputGroup
#region OutputGroup 2
OutputGroup ofg1 = new OutputGroup();
ofg1.Name = "File Group";
ofg1.CustomName = "test1";
ofg1.OutputGroupSettings = new OutputGroupSettings();
ofg1.OutputGroupSettings.Type = OutputGroupType.FILE_GROUP_SETTINGS;
ofg1.OutputGroupSettings.FileGroupSettings = new FileGroupSettings();
ofg1.OutputGroupSettings.FileGroupSettings.Destination = fileOutput;
Output output1 = new Output();
output1.NameModifier = "_output1";
output1.Preset = "System-Broadcast_Xdcam_Mxf_Mpeg2_Wav_16x9_1280x720p_60Hz_50Mbps";
output1.Extension = ".mp4";
#region VideoDescription
//VideoDescription vdes = new VideoDescription();
//output.VideoDescription = vdes;
//vdes.ScalingBehavior = ScalingBehavior.DEFAULT;
//vdes.TimecodeInsertion = VideoTimecodeInsertion.DISABLED;
//vdes.AntiAlias = AntiAlias.ENABLED;
//vdes.Sharpness = 50;
//vdes.AfdSignaling = AfdSignaling.NONE;
//vdes.DropFrameTimecode = DropFrameTimecode.ENABLED;
//vdes.RespondToAfd = RespondToAfd.NONE;
//vdes.ColorMetadata = ColorMetadata.INSERT;
//vdes.CodecSettings = new VideoCodecSettings();
//vdes.CodecSettings.Codec = VideoCodec.H_264;
//H264Settings h264 = new H264Settings();
//h264.InterlaceMode = H264InterlaceMode.PROGRESSIVE;
//h264.NumberReferenceFrames = 3;
//h264.Syntax = H264Syntax.DEFAULT;
//h264.Softness = 0;
//h264.GopClosedCadence = 1;
//h264.GopSize = 90;
//h264.Slices = 1;
//h264.GopBReference = H264GopBReference.DISABLED;
//h264.SlowPal = H264SlowPal.DISABLED;
//h264.SpatialAdaptiveQuantization = H264SpatialAdaptiveQuantization.ENABLED;
//h264.TemporalAdaptiveQuantization = H264TemporalAdaptiveQuantization.ENABLED;
//h264.FlickerAdaptiveQuantization = H264FlickerAdaptiveQuantization.DISABLED;
//h264.EntropyEncoding = H264EntropyEncoding.CABAC;
//h264.Bitrate = 5000000;
//h264.FramerateControl = H264FramerateControl.SPECIFIED;
//h264.RateControlMode = H264RateControlMode.CBR;
//h264.CodecProfile = H264CodecProfile.MAIN;
//h264.Telecine = H264Telecine.NONE;
//h264.MinIInterval = 0;
//h264.AdaptiveQuantization = H264AdaptiveQuantization.HIGH;
//h264.CodecLevel = H264CodecLevel.AUTO;
//h264.FieldEncoding = H264FieldEncoding.PAFF;
//h264.SceneChangeDetect = H264SceneChangeDetect.ENABLED;
//h264.QualityTuningLevel = H264QualityTuningLevel.SINGLE_PASS;
//h264.FramerateConversionAlgorithm = H264FramerateConversionAlgorithm.DUPLICATE_DROP;
//h264.UnregisteredSeiTimecode = H264UnregisteredSeiTimecode.DISABLED;
//h264.GopSizeUnits = H264GopSizeUnits.FRAMES;
//h264.ParControl = H264ParControl.SPECIFIED;
//h264.NumberBFramesBetweenReferenceFrames = 2;
//h264.RepeatPps = H264RepeatPps.DISABLED;
//h264.FramerateNumerator = 30;
//h264.FramerateDenominator = 1;
//h264.ParNumerator = 1;
//h264.ParDenominator = 1;
//output.VideoDescription.CodecSettings.H264Settings = h264;
#endregion VideoDescription
#region AudioDescription
//AudioDescription ades = new AudioDescription();
//ades.LanguageCodeControl = AudioLanguageCodeControl.FOLLOW_INPUT;
//// This name matches one specified in the Inputs below
//ades.AudioSourceName = "Audio Selector 1";
//ades.CodecSettings = new AudioCodecSettings();
//ades.CodecSettings.Codec = AudioCodec.AAC;
//AacSettings aac = new AacSettings();
//aac.AudioDescriptionBroadcasterMix = AacAudioDescriptionBroadcasterMix.NORMAL;
//aac.RateControlMode = AacRateControlMode.CBR;
//aac.CodecProfile = AacCodecProfile.LC;
//aac.CodingMode = AacCodingMode.CODING_MODE_2_0;
//aac.RawFormat = AacRawFormat.NONE;
//aac.SampleRate = 48000;
//aac.Specification = AacSpecification.MPEG4;
//aac.Bitrate = 64000;
//ades.CodecSettings.AacSettings = aac;
//output.AudioDescriptions.Add(ades);
#endregion AudioDescription
#region Mp4 Container
//output.ContainerSettings = new ContainerSettings();
//output.ContainerSettings.Container = ContainerType.MP4;
//Mp4Settings mp4 = new Mp4Settings();
//mp4.CslgAtom = Mp4CslgAtom.INCLUDE;
//mp4.FreeSpaceBox = Mp4FreeSpaceBox.EXCLUDE;
//mp4.MoovPlacement = Mp4MoovPlacement.PROGRESSIVE_DOWNLOAD;
//output.ContainerSettings.Mp4Settings = mp4;
#endregion Mp4 Container
ofg1.Outputs.Add(output1);
createJobRequest.Settings.OutputGroups.Add(ofg1);
#endregion OutputGroup
#region Input
Input input = new Input();
input.FilterEnable = InputFilterEnable.AUTO;
input.PsiControl = InputPsiControl.USE_PSI;
input.FilterStrength = 0;
input.DeblockFilter = InputDeblockFilter.DISABLED;
input.DenoiseFilter = InputDenoiseFilter.DISABLED;
input.TimecodeSource = InputTimecodeSource.ZEROBASED;
input.InputScanType = InputScanType.AUTO;
input.FileInput = fileInput;
AudioSelector audsel = new AudioSelector();
audsel.Offset = 0;
audsel.DefaultSelection = AudioDefaultSelection.DEFAULT;
audsel.ProgramSelection = 1;
//audsel.SelectorType = AudioSelectorType.TRACK;
//audsel.Tracks.Add(1);
input.AudioSelectors.Add("Audio Selector 1", audsel);
input.VideoSelector = new VideoSelector();
input.VideoSelector.ColorSpace = ColorSpace.FOLLOW;
input.VideoSelector.Rotate = InputRotate.DEGREE_0;
input.VideoSelector.AlphaBehavior = AlphaBehavior.DISCARD;
createJobRequest.Settings.Inputs.Add(input);
#endregion Input
#endregion Create job settings
try {
CreateJobResponse createJobResponse = await mcClient.CreateJobAsync(createJobRequest);
return("Job Id: " + createJobResponse.Job.Id);
} catch (BadRequestException bre) {
// If the enpoint was bad
if (bre.Message.StartsWith("You must use the customer-"))
{
// The exception contains the correct endpoint; extract it
mediaConvertEndpoint = bre.Message.Split('\'')[1];
return(mediaConvertEndpoint);
// Code to retry query
}
}
return(null);
}
protected virtual void PerformProcessVideo(AmazonElasticTranscoderClient client, Asset asset)
{
base.ExecuteMethod("PerformProcessVideo", delegate()
{
try
{
string actualFile = asset.raw_url;
// strip bucket
int ix = actualFile.ToLower().IndexOf(this.AmazonBucket.ToLower());
if (ix > -1)
{
actualFile = actualFile.Substring(ix + this.AmazonBucket.Length).Trim('/');
}
// strip cloud front
if (!string.IsNullOrEmpty(this.AmazonCloudFrontUrl))
{
ix = actualFile.ToLower().IndexOf(this.AmazonCloudFrontUrl.ToLower());
if (ix > -1)
{
actualFile = actualFile.Substring(ix + this.AmazonCloudFrontUrl.Length).Trim('/');
}
ix = asset.raw_url.LastIndexOf('/');
}
string prefix = asset.raw_url.Substring(0, ix) + "/";
CreateJobResponse response = client.CreateJob(new CreateJobRequest()
{
PipelineId = this.AmazonPipeLineID,
OutputKeyPrefix = OUTPUT_PREFIX + asset.asset_id.ToString() + "/",
Input = new JobInput()
{
Key = actualFile,
AspectRatio = "auto",
Container = "auto",
FrameRate = "auto",
Interlaced = "auto",
Resolution = "auto"
},
Outputs = new List <CreateJobOutput>()
{
new CreateJobOutput()
{
Key = actualFile,
PresetId = this.AmazonPresetID,
ThumbnailPattern = "thumb_{count}",
Rotate = "auto"
}
}
});
if (response.Job != null)
{
HealthReporter.Current.UpdateMetric(HealthTrackType.Each, HealthReporter.VIDEO_TRANSCODE_QUEUE_SUCCESS, 0, 1);
this.API.Direct.Assets.UpdateEncodingInfo(asset.asset_id, response.Job.Id, true, EncoderStatus.queued.ToString(), "Amazon Queued on " + DateTime.UtcNow.ToString(), true);
}
else
{
HealthReporter.Current.UpdateMetric(HealthTrackType.Each, HealthReporter.VIDEO_TRANSCODE_QUEUE_FAILED, 0, 1);
this.API.Direct.Assets.UpdateEncodingInfo(asset.asset_id, string.Empty, false, EncoderStatus.raw.ToString(), "Amazon Queue Failed on " + DateTime.UtcNow.ToString(), true);
}
}
catch (Exception ex)
{
this.API.Direct.Assets.UpdateEncodingInfo(asset.asset_id, string.Empty, false, EncoderStatus.raw.ToString(), CoreUtility.FormatException(ex), true);
HealthReporter.Current.UpdateMetric(HealthTrackType.Each, HealthReporter.VIDEO_TRANSCODE_QUEUE_FAILED, 0, 1);
this.IFoundation.LogError(ex, "PerformProcessVideo");
}
});
}
static void Main(string[] args)
{
String mediaConvertRole = "arn:aws:iam::378090785532:role/TurnerMediaConvertRole";
String fileInput = "s3://mediaconvertin/B83UV-A-03-EL-COMANDANTE---EL-COMANDANTE-ae98f140.mov";
String fileOutput = "s3://mediaconvertout/Salida.mov";
AmazonMediaConvertClient client;
// Once you know what your customer endpoint is, set it here
String mediaConvertEndpoint = "";
// If we do not have our customer-specific endpoint
if (String.IsNullOrEmpty(mediaConvertEndpoint))
{
//Amazon.Runtime.AWSCredentials c = new Amazon.Runtime.AWSCredentials();
// Obtain the customer-specific MediaConvert endpoint
client = new
AmazonMediaConvertClient("AKIAJ35VKR7CZVBCL2GQ", "L2cnZrGPhvCD17X8hAbk4w2rd1sjPVtX8BE7OOmv", Amazon.RegionEndpoint.USEast1);
DescribeEndpointsRequest describeRequest = new
DescribeEndpointsRequest();
DescribeEndpointsResponse describeResponse =
client.DescribeEndpointsAsync(describeRequest).Result;
mediaConvertEndpoint = describeResponse.Endpoints[0].Url;
}
// Since we have a service url for MediaConvert, we do not
// need to set RegionEndpoint. If we do, the ServiceURL will
// be overwritten
client = new
AmazonMediaConvertClient("AKIAJ35VKR7CZVBCL2GQ", "L2cnZrGPhvCD17X8hAbk4w2rd1sjPVtX8BE7OOmv", Amazon.RegionEndpoint.USEast1);
NetworkCredential cred = new NetworkCredential("AKIAJ35VKR7CZVBCL2GQ", "L2cnZrGPhvCD17X8hAbk4w2rd1sjPVtX8BE7OOmv");
AmazonMediaConvertConfig mcConfig = new AmazonMediaConvertConfig
{
ServiceURL = mediaConvertEndpoint,
ProxyCredentials = cred
// ProxyCredentials = client
};
AmazonMediaConvertClient mcClient = new AmazonMediaConvertClient(mcConfig);
CreateJobRequest createJobRequest = new CreateJobRequest();
createJobRequest.Role = mediaConvertRole;
createJobRequest.UserMetadata.Add("Customer", "Amazon");
#region Create job settings
JobSettings jobSettings = new JobSettings();
jobSettings.AdAvailOffset = 0;
jobSettings.TimecodeConfig = new TimecodeConfig();
jobSettings.TimecodeConfig.Source = TimecodeSource.EMBEDDED;
createJobRequest.Settings = jobSettings;
#region OutputGroup
OutputGroup ofg = new OutputGroup();
ofg.Name = "File Group";
ofg.OutputGroupSettings = new OutputGroupSettings();
ofg.OutputGroupSettings.Type = OutputGroupType.FILE_GROUP_SETTINGS;
ofg.OutputGroupSettings.FileGroupSettings = new FileGroupSettings();
ofg.OutputGroupSettings.FileGroupSettings.Destination = fileOutput;
Output output = new Output();
output.NameModifier = "_1";
#region VideoDescription
VideoDescription vdes = new VideoDescription();
output.VideoDescription = vdes;
vdes.ScalingBehavior = ScalingBehavior.DEFAULT;
vdes.TimecodeInsertion = VideoTimecodeInsertion.DISABLED;
vdes.AntiAlias = AntiAlias.ENABLED;
vdes.Sharpness = 50;
vdes.AfdSignaling = AfdSignaling.NONE;
vdes.DropFrameTimecode = DropFrameTimecode.ENABLED;
vdes.RespondToAfd = RespondToAfd.NONE;
vdes.ColorMetadata = ColorMetadata.INSERT;
vdes.CodecSettings = new VideoCodecSettings();
vdes.CodecSettings.Codec = VideoCodec.H_264;
H264Settings h264 = new H264Settings();
h264.InterlaceMode = H264InterlaceMode.PROGRESSIVE;
h264.NumberReferenceFrames = 3;
h264.Syntax = H264Syntax.DEFAULT;
h264.Softness = 0;
h264.GopClosedCadence = 1;
h264.GopSize = 90;
h264.Slices = 1;
h264.GopBReference = H264GopBReference.DISABLED;
h264.SlowPal = H264SlowPal.DISABLED;
h264.SpatialAdaptiveQuantization =
H264SpatialAdaptiveQuantization.ENABLED;
h264.TemporalAdaptiveQuantization =
H264TemporalAdaptiveQuantization.ENABLED;
h264.FlickerAdaptiveQuantization =
H264FlickerAdaptiveQuantization.DISABLED;
h264.EntropyEncoding = H264EntropyEncoding.CABAC;
h264.Bitrate = 5000000;
h264.FramerateControl = H264FramerateControl.SPECIFIED;
h264.RateControlMode = H264RateControlMode.CBR;
h264.CodecProfile = H264CodecProfile.MAIN;
h264.Telecine = H264Telecine.NONE;
h264.MinIInterval = 0;
h264.AdaptiveQuantization = H264AdaptiveQuantization.HIGH;
h264.CodecLevel = H264CodecLevel.AUTO;
h264.FieldEncoding = H264FieldEncoding.PAFF;
h264.SceneChangeDetect = H264SceneChangeDetect.ENABLED;
h264.QualityTuningLevel = H264QualityTuningLevel.SINGLE_PASS;
h264.FramerateConversionAlgorithm =
H264FramerateConversionAlgorithm.DUPLICATE_DROP;
h264.UnregisteredSeiTimecode = H264UnregisteredSeiTimecode.DISABLED;
h264.GopSizeUnits = H264GopSizeUnits.FRAMES;
h264.ParControl = H264ParControl.SPECIFIED;
h264.NumberBFramesBetweenReferenceFrames = 2;
h264.RepeatPps = H264RepeatPps.DISABLED;
h264.FramerateNumerator = 30;
h264.FramerateDenominator = 1;
h264.ParNumerator = 1;
h264.ParDenominator = 1;
output.VideoDescription.CodecSettings.H264Settings = h264;
#endregion VideoDescription
#region AudioDescription
AudioDescription ades = new AudioDescription();
ades.LanguageCodeControl = AudioLanguageCodeControl.FOLLOW_INPUT;
// This name matches one specified in the Inputs below
ades.AudioSourceName = "Audio Selector 1";
ades.CodecSettings = new AudioCodecSettings();
ades.CodecSettings.Codec = AudioCodec.AAC;
AacSettings aac = new AacSettings();
aac.AudioDescriptionBroadcasterMix =
AacAudioDescriptionBroadcasterMix.NORMAL;
aac.RateControlMode = AacRateControlMode.CBR;
aac.CodecProfile = AacCodecProfile.LC;
aac.CodingMode = AacCodingMode.CODING_MODE_2_0;
aac.RawFormat = AacRawFormat.NONE;
aac.SampleRate = 48000;
aac.Specification = AacSpecification.MPEG4;
aac.Bitrate = 64000;
ades.CodecSettings.AacSettings = aac;
output.AudioDescriptions.Add(ades);
#endregion AudioDescription
#region Mp4 Container
output.ContainerSettings = new ContainerSettings();
output.ContainerSettings.Container = ContainerType.MP4;
Mp4Settings mp4 = new Mp4Settings();
mp4.CslgAtom = Mp4CslgAtom.INCLUDE;
mp4.FreeSpaceBox = Mp4FreeSpaceBox.EXCLUDE;
mp4.MoovPlacement = Mp4MoovPlacement.PROGRESSIVE_DOWNLOAD;
output.ContainerSettings.Mp4Settings = mp4;
#endregion Mp4 Container
ofg.Outputs.Add(output);
createJobRequest.Settings.OutputGroups.Add(ofg);
#endregion OutputGroup
#region Input
Input input = new Input();
input.FilterEnable = InputFilterEnable.AUTO;
input.PsiControl = InputPsiControl.USE_PSI;
input.FilterStrength = 0;
input.DeblockFilter = InputDeblockFilter.DISABLED;
input.DenoiseFilter = InputDenoiseFilter.DISABLED;
input.TimecodeSource = InputTimecodeSource.EMBEDDED;
input.FileInput = fileInput;
AudioSelector audsel = new AudioSelector();
audsel.Offset = 0;
audsel.DefaultSelection = AudioDefaultSelection.NOT_DEFAULT;
audsel.ProgramSelection = 1;
audsel.SelectorType = AudioSelectorType.TRACK;
audsel.Tracks.Add(1);
input.AudioSelectors.Add("Audio Selector 1", audsel);
input.VideoSelector = new VideoSelector();
input.VideoSelector.ColorSpace = ColorSpace.FOLLOW;
createJobRequest.Settings.Inputs.Add(input);
#endregion Input
#endregion Create job settings
try
{
CreateJobResponse createJobResponse =
mcClient.CreateJobAsync(createJobRequest).Result;
Console.WriteLine("Job Id: {0}", createJobResponse.Job.Id);
}
catch (BadRequestException bre)
{
// If the enpoint was bad
if (bre.Message.StartsWith("You must use the customer-"))
{
// The exception contains the correct endpoint; extract it
mediaConvertEndpoint = bre.Message.Split('\'')[1];
// Code to retry query
}
}
}
/// <summary>
/// Unmarshaller the response from the service to the response class.
/// </summary>
/// <param name="context"></param>
/// <returns></returns>
public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context)
{
CreateJobResponse response = new CreateJobResponse();
context.Read();
int targetDepth = context.CurrentDepth;
while (context.ReadAtDepth(targetDepth))
{
if (context.TestExpression("Arn", targetDepth))
{
var unmarshaller = StringUnmarshaller.Instance;
response.Arn = unmarshaller.Unmarshall(context);
continue;
}
if (context.TestExpression("CreatedAt", targetDepth))
{
var unmarshaller = DateTimeUnmarshaller.Instance;
response.CreatedAt = unmarshaller.Unmarshall(context);
continue;
}
if (context.TestExpression("Details", targetDepth))
{
var unmarshaller = ResponseDetailsUnmarshaller.Instance;
response.Details = unmarshaller.Unmarshall(context);
continue;
}
if (context.TestExpression("Errors", targetDepth))
{
var unmarshaller = new ListUnmarshaller <JobError, JobErrorUnmarshaller>(JobErrorUnmarshaller.Instance);
response.Errors = unmarshaller.Unmarshall(context);
continue;
}
if (context.TestExpression("Id", targetDepth))
{
var unmarshaller = StringUnmarshaller.Instance;
response.Id = unmarshaller.Unmarshall(context);
continue;
}
if (context.TestExpression("State", targetDepth))
{
var unmarshaller = StringUnmarshaller.Instance;
response.State = unmarshaller.Unmarshall(context);
continue;
}
if (context.TestExpression("Type", targetDepth))
{
var unmarshaller = StringUnmarshaller.Instance;
response.Type = unmarshaller.Unmarshall(context);
continue;
}
if (context.TestExpression("UpdatedAt", targetDepth))
{
var unmarshaller = DateTimeUnmarshaller.Instance;
response.UpdatedAt = unmarshaller.Unmarshall(context);
continue;
}
}
return(response);
}
public async Task FunctionHandler(S3Event s3event, ILambdaContext context)
{
if (s3event != null)
{
foreach (var record in s3event.Records)
{
context.Logger.LogLine(record.S3.Bucket.Name);
string bucket = record.S3.Bucket.Name.Substring(14).Replace('-', ' ');
context.Logger.LogLine(bucket);
if (LogContextOperations.IfInputExist(bucket))
{
LogInput retrievedLI = LogContextOperations.GetLogInputByName(bucket);
GlueConsolidatedEntity retrievedGCE = LogContextOperations.GetGlueConsolidatedEntity(retrievedLI.ID);
GlueDatabaseTable retrievedGDT = LogContextOperations.GetGlueDatabaseTable(retrievedLI.ID);
context.Logger.LogLine(retrievedLI.ID + " | " + retrievedLI.Name);
if (retrievedLI.InitialIngest == false && retrievedGCE == null)
{
context.Logger.LogLine("Log Input has not be crawled before and has no crawler");
CreateCrawlerResponse createCrawlerResponse = await GlueClient.CreateCrawlerAsync(new CreateCrawlerRequest
{
Name = retrievedLI.Name,
DatabaseName = LogContextOperations.GetGlueDatabase().Name,
Role = "GlueServiceRole",
SchemaChangePolicy = new SchemaChangePolicy
{
DeleteBehavior = DeleteBehavior.DEPRECATE_IN_DATABASE,
UpdateBehavior = UpdateBehavior.UPDATE_IN_DATABASE
},
Tags = new Dictionary <string, string>
{
{ "Project", "OSPJ" }
},
Targets = new CrawlerTargets
{
S3Targets = new List <S3Target>
{
new S3Target
{
Path = "s3://" + LogContextOperations.GetInputS3BucketName(retrievedLI.ID)
}
}
}
});
if (createCrawlerResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
context.Logger.LogLine("Crawler Created");
StartCrawlerResponse startCrawlerResponse = await GlueClient.StartCrawlerAsync(new StartCrawlerRequest
{
Name = retrievedLI.Name
});
if (startCrawlerResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
context.Logger.LogLine("Crawler Just Created Started");
LogContextOperations.AddGlueConsolidatedEntity(new GlueConsolidatedEntity
{
CrawlerName = retrievedLI.Name,
LinkedLogInputID = retrievedLI.ID
});
}
}
}
else if (retrievedLI.InitialIngest == false && retrievedGCE != null)
{
context.Logger.LogLine("Log Input has not be crawled before but has a crawler");
GetCrawlerResponse getCrawlerResponse = await GlueClient.GetCrawlerAsync(new GetCrawlerRequest
{
Name = retrievedGCE.CrawlerName
});
if (getCrawlerResponse.Crawler.State.Equals(CrawlerState.READY) && getCrawlerResponse.Crawler.LastCrawl == null)
{
StartCrawlerResponse startCrawlerResponse = await GlueClient.StartCrawlerAsync(new StartCrawlerRequest
{
Name = retrievedGCE.CrawlerName
});
if (startCrawlerResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
context.Logger.LogLine("Crawler Started");
}
}
else if (getCrawlerResponse.Crawler.State.Equals(CrawlerState.READY) && getCrawlerResponse.Crawler.LastCrawl != null)
{
context.Logger.LogLine("Log Input has been crawled before, has a crawler but not a job");
LogContextOperations.AddGlueDatabaseTable(new GlueDatabaseTable
{
LinkedDatabaseID = 1,
LinkedGlueConsolidatedInputEntityID = retrievedGCE.ID,
Name = getCrawlerResponse.Crawler.Targets.S3Targets[0].Path.Substring(5).Replace("-", "_")
});
GetTableResponse getTableResponse = await GlueClient.GetTableAsync(new GetTableRequest
{
DatabaseName = "master-database",
Name = getCrawlerResponse.Crawler.Targets.S3Targets[0].Path.Substring(5).Replace("-", "_")
});
if (getTableResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
UpdateTableResponse updateTableResponse = await GlueClient.UpdateTableAsync(new UpdateTableRequest
{
DatabaseName = getTableResponse.Table.DatabaseName,
TableInput = new TableInput
{
Name = getTableResponse.Table.Name,
Parameters = getTableResponse.Table.Parameters,
LastAccessTime = getTableResponse.Table.LastAccessTime,
LastAnalyzedTime = getTableResponse.Table.LastAnalyzedTime,
Owner = getTableResponse.Table.Owner,
StorageDescriptor = getTableResponse.Table.StorageDescriptor,
Retention = getTableResponse.Table.Retention,
TableType = getTableResponse.Table.TableType
}
});
if (updateTableResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
retrievedGDT = LogContextOperations.GetGlueDatabaseTable(retrievedLI.ID);
CreateJobRequest createJobRequest = new CreateJobRequest
{
Name = retrievedLI.Name,
DefaultArguments = new Dictionary <string, string>
{
{ "--enable-spark-ui", "true" },
{ "--spark-event-logs-path", "s3://aws-glue-spark-188363912800-ap-southeast-1" },
{ "--job-bookmark-option", "job-bookmark-enable" },
{ "--job-language", "python" },
{ "--TempDir", "s3://aws-glue-temporary-188363912800-ap-southeast-1/root" },
{ "--TABLE_NAME", retrievedGDT.Name }
},
MaxCapacity = 10.0,
Role = "GlueServiceRole",
Connections = new ConnectionsList
{
Connections = new List <string>
{
"SmartInsights"
}
},
Tags = new Dictionary <string, string>
{
{ "Project", "OSPJ" }
},
MaxRetries = 0,
GlueVersion = "1.0",
ExecutionProperty = new ExecutionProperty
{
MaxConcurrentRuns = 1
},
Timeout = 2880
};
if (retrievedLI.LogInputCategory.Equals(LogInputCategory.ApacheWebServer))
{
createJobRequest.Command = new JobCommand
{
PythonVersion = "3",
Name = "glueetl",
ScriptLocation = "s3://aws-glue-scripts-188363912800-ap-southeast-1/root/Apache CLF"
};
}
else if (retrievedLI.LogInputCategory.Equals(LogInputCategory.SquidProxy))
{
createJobRequest.Command = new JobCommand
{
PythonVersion = "3",
Name = "glueetl",
ScriptLocation = "s3://aws-glue-scripts-188363912800-ap-southeast-1/root/Cisco Squid Proxy"
};
}
else if (retrievedLI.LogInputCategory.Equals(LogInputCategory.SSH))
{
createJobRequest.Command = new JobCommand
{
PythonVersion = "3",
Name = "glueetl",
ScriptLocation = "s3://aws-glue-scripts-188363912800-ap-southeast-1/root/Splunk SSH"
};
}
else if (retrievedLI.LogInputCategory.Equals(LogInputCategory.WindowsEventLogs))
{
createJobRequest.Command = new JobCommand
{
PythonVersion = "3",
Name = "glueetl",
ScriptLocation = "s3://aws-glue-scripts-188363912800-ap-southeast-1/root/Windows Events"
};
}
CreateJobResponse createJobResponse = await GlueClient.CreateJobAsync(createJobRequest);
if (createJobResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
context.Logger.LogLine("Job Created");
StartJobRunResponse startJobRunResponse = await GlueClient.StartJobRunAsync(new StartJobRunRequest
{
JobName = createJobResponse.Name,
MaxCapacity = 10.0
});
if (startJobRunResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
context.Logger.LogLine("Job Just Created Started");
retrievedGCE.JobName = createJobResponse.Name;
LogContextOperations.UpdateGlueConsolidatedEntity(retrievedGCE);
LogContextOperations.UpdateInputIngestionStatus(bucket);
}
}
}
}
}
}
else if (retrievedLI.InitialIngest == true)
{
context.Logger.LogLine("Log Input has been crawled before");
context.Logger.LogLine(retrievedGCE.JobName);
if (retrievedGCE.JobName == null && retrievedGDT == null)
{
context.Logger.LogLine("Log Input has not be transferred over to RDS before due to no job");
GetCrawlerResponse getCrawlerResponse = await GlueClient.GetCrawlerAsync(new GetCrawlerRequest
{
Name = retrievedGCE.CrawlerName
});
if (getCrawlerResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
LogContextOperations.AddGlueDatabaseTable(new GlueDatabaseTable
{
LinkedDatabaseID = 1,
LinkedGlueConsolidatedInputEntityID = retrievedGCE.ID,
Name = getCrawlerResponse.Crawler.Targets.S3Targets[0].Path.Substring(5).Replace("-", "_")
});
GetTableResponse getTableResponse = await GlueClient.GetTableAsync(new GetTableRequest
{
DatabaseName = "master-database",
Name = getCrawlerResponse.Crawler.Targets.S3Targets[0].Path.Substring(5).Replace("-", "_")
});
if (getTableResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
UpdateTableResponse updateTableResponse = await GlueClient.UpdateTableAsync(new UpdateTableRequest
{
DatabaseName = getTableResponse.Table.DatabaseName,
TableInput = new TableInput
{
Name = getTableResponse.Table.Name,
Parameters = getTableResponse.Table.Parameters,
LastAccessTime = getTableResponse.Table.LastAccessTime,
LastAnalyzedTime = getTableResponse.Table.LastAnalyzedTime,
Owner = getTableResponse.Table.Owner,
StorageDescriptor = getTableResponse.Table.StorageDescriptor,
Retention = getTableResponse.Table.Retention,
TableType = getTableResponse.Table.TableType
}
});
if (updateTableResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
retrievedGDT = LogContextOperations.GetGlueDatabaseTable(retrievedLI.ID);
CreateJobRequest createJobRequest = new CreateJobRequest
{
Name = retrievedLI.Name,
DefaultArguments = new Dictionary <string, string>
{
{ "--enable-spark-ui", "true" },
{ "--spark-event-logs-path", "s3://aws-glue-spark-188363912800-ap-southeast-1" },
{ "--job-bookmark-option", "job-bookmark-enable" },
{ "--job-language", "python" },
{ "--TempDir", "s3://aws-glue-temporary-188363912800-ap-southeast-1/root" },
{ "--TABLE_NAME", retrievedGDT.Name }
},
MaxCapacity = 10.0,
Role = "GlueServiceRole",
Connections = new ConnectionsList
{
Connections = new List <string>
{
"SmartInsights"
}
},
Tags = new Dictionary <string, string>
{
{ "Project", "OSPJ" }
},
MaxRetries = 0,
GlueVersion = "1.0",
ExecutionProperty = new ExecutionProperty
{
MaxConcurrentRuns = 1
},
Timeout = 2880
};
if (retrievedLI.LogInputCategory.Equals(LogInputCategory.ApacheWebServer))
{
createJobRequest.Command = new JobCommand
{
PythonVersion = "3",
Name = "glueetl",
ScriptLocation = "s3://aws-glue-scripts-188363912800-ap-southeast-1/root/Apache CLF"
};
}
else if (retrievedLI.LogInputCategory.Equals(LogInputCategory.SquidProxy))
{
createJobRequest.Command = new JobCommand
{
PythonVersion = "3",
Name = "glueetl",
ScriptLocation = "s3://aws-glue-scripts-188363912800-ap-southeast-1/root/Cisco Squid Proxy"
};
}
else if (retrievedLI.LogInputCategory.Equals(LogInputCategory.SSH))
{
createJobRequest.Command = new JobCommand
{
PythonVersion = "3",
Name = "glueetl",
ScriptLocation = "s3://aws-glue-scripts-188363912800-ap-southeast-1/root/Splunk SSH"
};
}
else if (retrievedLI.LogInputCategory.Equals(LogInputCategory.WindowsEventLogs))
{
createJobRequest.Command = new JobCommand
{
PythonVersion = "3",
Name = "glueetl",
ScriptLocation = "s3://aws-glue-scripts-188363912800-ap-southeast-1/root/Windows Events"
};
}
CreateJobResponse createJobResponse = await GlueClient.CreateJobAsync(createJobRequest);
if (createJobResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
context.Logger.LogLine("Job Created");
StartJobRunResponse startJobRunResponse = await GlueClient.StartJobRunAsync(new StartJobRunRequest
{
JobName = createJobResponse.Name,
MaxCapacity = 10.0
});
if (startJobRunResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
context.Logger.LogLine("Job Just Created Started");
retrievedGCE.JobName = createJobResponse.Name;
LogContextOperations.UpdateGlueConsolidatedEntity(retrievedGCE);
LogContextOperations.UpdateInputIngestionStatus(bucket);
}
}
}
}
}
}
else
{
GetCrawlerResponse getCrawlerResponse = await GlueClient.GetCrawlerAsync(new GetCrawlerRequest
{
Name = retrievedGCE.CrawlerName
});
if (getCrawlerResponse.HttpStatusCode.Equals(HttpStatusCode.OK) && getCrawlerResponse.Crawler.State.Equals(CrawlerState.READY))
{
if ((getCrawlerResponse.Crawler.LastCrawl.StartTime.Hour < DateTime.Now.Hour && getCrawlerResponse.Crawler.LastCrawl.StartTime.Day == DateTime.Now.Day) || getCrawlerResponse.Crawler.LastCrawl.StartTime.Day != DateTime.Now.Day)
{
context.Logger.LogLine("Log Input has been transferred over to RDS before but time condition not met");
StartCrawlerResponse startCrawlerResponse = await GlueClient.StartCrawlerAsync(new StartCrawlerRequest
{
Name = retrievedGCE.CrawlerName
});
if (startCrawlerResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
context.Logger.LogLine("Crawler Started");
}
}
else
{
context.Logger.LogLine("Log Input has been transferred over to RDS before and time condition met");
GetTableResponse getTableResponse = await GlueClient.GetTableAsync(new GetTableRequest
{
DatabaseName = "master-database",
Name = getCrawlerResponse.Crawler.Targets.S3Targets[0].Path.Substring(5).Replace("-", "_")
});
if (getTableResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
UpdateTableResponse updateTableResponse = await GlueClient.UpdateTableAsync(new UpdateTableRequest
{
DatabaseName = getTableResponse.Table.DatabaseName,
TableInput = new TableInput
{
Name = getTableResponse.Table.Name,
Parameters = getTableResponse.Table.Parameters,
LastAccessTime = getTableResponse.Table.LastAccessTime,
LastAnalyzedTime = getTableResponse.Table.LastAnalyzedTime,
Owner = getTableResponse.Table.Owner,
StorageDescriptor = getTableResponse.Table.StorageDescriptor,
Retention = getTableResponse.Table.Retention,
TableType = getTableResponse.Table.TableType
}
});
if (updateTableResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
context.Logger.LogLine("Table updated before running job");
context.Logger.LogLine(retrievedGCE.JobName);
GetJobRunsResponse getJobRunsResponse = await GlueClient.GetJobRunsAsync(new GetJobRunsRequest
{
JobName = retrievedGCE.JobName
});
bool jobRunning = false;
context.Logger.LogLine(getJobRunsResponse.JobRuns.Count().ToString());
foreach (JobRun j in getJobRunsResponse.JobRuns)
{
context.Logger.LogLine(j.Id + " | " + j.JobRunState);
if (j.JobRunState.Equals(JobRunState.STARTING) || j.JobRunState.Equals(JobRunState.RUNNING) || j.JobRunState.Equals(JobRunState.STOPPING))
{
jobRunning = true;
break;
}
}
context.Logger.LogLine(jobRunning.ToString());
if (!jobRunning)
{
StartJobRunResponse startJobRunResponse = await GlueClient.StartJobRunAsync(new StartJobRunRequest
{
JobName = retrievedGCE.JobName,
MaxCapacity = 10.0
});
if (startJobRunResponse.HttpStatusCode.Equals(HttpStatusCode.OK))
{
context.Logger.LogLine("Job Started");
}
}
}
}
}
}
}
}
}
}
}
}
static async Task <string> CreateMediaConverterJobForDASHAsync(string fileName)
{
// TODO: bucket name to be change and regsion name also
string mediaConvertRole = "arn:aws:iam::026141093142:role/MediaConvert_Default_Role";
string fileInput = "s3://testbucketmediaconverter/" + fileName;
string fileOutput = "s3://testbucketmediaconverter/";
// Once you know what your customer endpoint is, set it here
string mediaConvertEndpoint = "";
// If we do not have our customer-specific endpoint
if (String.IsNullOrEmpty(mediaConvertEndpoint))
{
// Obtain the customer-specific MediaConvert endpoint
AmazonMediaConvertClient client = new AmazonMediaConvertClient(Amazon.RegionEndpoint.USEast2);
DescribeEndpointsRequest describeRequest = new DescribeEndpointsRequest();
DescribeEndpointsResponse describeResponse = await client.DescribeEndpointsAsync(describeRequest);
mediaConvertEndpoint = describeResponse.Endpoints[0].Url;
}
// Since we have a service url for MediaConvert, we do not
// need to set RegionEndpoint. If we do, the ServiceURL will
// be overwritten
AmazonMediaConvertConfig mcConfig = new AmazonMediaConvertConfig {
ServiceURL = mediaConvertEndpoint,
};
AmazonMediaConvertClient mcClient = new AmazonMediaConvertClient(mcConfig);
CreateJobRequest createJobRequest = new CreateJobRequest();
createJobRequest.Role = mediaConvertRole;
createJobRequest.UserMetadata.Add("Customer", "Amazon");
#region Create job settings
JobSettings jobSettings = new JobSettings();
jobSettings.AdAvailOffset = 0;
jobSettings.TimecodeConfig = new TimecodeConfig();
jobSettings.TimecodeConfig.Source = TimecodeSource.ZEROBASED;
createJobRequest.Settings = jobSettings;
#region OutputGroup 1
OutputGroup ofg = new OutputGroup();
ofg.Name = "DASH ISO";
ofg.CustomName = "test";
ofg.OutputGroupSettings = new OutputGroupSettings();
ofg.OutputGroupSettings.Type = OutputGroupType.DASH_ISO_GROUP_SETTINGS;
ofg.OutputGroupSettings.DashIsoGroupSettings = new DashIsoGroupSettings();
ofg.OutputGroupSettings.DashIsoGroupSettings.Destination = fileOutput;
ofg.OutputGroupSettings.DashIsoGroupSettings.SegmentLength = 30;
ofg.OutputGroupSettings.DashIsoGroupSettings.MinFinalSegmentLength = 0;
ofg.OutputGroupSettings.DashIsoGroupSettings.FragmentLength = 2;
ofg.OutputGroupSettings.DashIsoGroupSettings.SegmentControl = DashIsoSegmentControl.SINGLE_FILE;
ofg.OutputGroupSettings.DashIsoGroupSettings.MpdProfile = DashIsoMpdProfile.MAIN_PROFILE;
ofg.OutputGroupSettings.DashIsoGroupSettings.HbbtvCompliance = "NONE";
Output output = new Output();
output.NameModifier = "_output";
output.Preset = "System-Ott_Dash_Mp4_Avc_16x9_1280x720p_30Hz_3.5Mbps";
//output.Extension = ".mp4";
ofg.Outputs.Add(output);
createJobRequest.Settings.OutputGroups.Add(ofg);
#endregion OutputGroup
#region Input
Input input = new Input();
input.FilterEnable = InputFilterEnable.AUTO;
input.PsiControl = InputPsiControl.USE_PSI;
input.FilterStrength = 0;
input.DeblockFilter = InputDeblockFilter.DISABLED;
input.DenoiseFilter = InputDenoiseFilter.DISABLED;
input.TimecodeSource = InputTimecodeSource.ZEROBASED;
input.InputScanType = InputScanType.AUTO;
input.FileInput = fileInput;
AudioSelector audsel = new AudioSelector();
audsel.Offset = 0;
audsel.DefaultSelection = AudioDefaultSelection.DEFAULT;
audsel.ProgramSelection = 1;
//audsel.SelectorType = AudioSelectorType.TRACK;
//audsel.Tracks.Add(1);
input.AudioSelectors.Add("Audio Selector 1", audsel);
input.VideoSelector = new VideoSelector();
input.VideoSelector.ColorSpace = ColorSpace.FOLLOW;
input.VideoSelector.Rotate = InputRotate.DEGREE_0;
input.VideoSelector.AlphaBehavior = AlphaBehavior.DISCARD;
createJobRequest.Settings.Inputs.Add(input);
#endregion Input
#endregion Create job settings
try {
CreateJobResponse createJobResponse = await mcClient.CreateJobAsync(createJobRequest);
return("Job Id: " + createJobResponse.Job.Id);
} catch (BadRequestException bre) {
// If the enpoint was bad
if (bre.Message.StartsWith("You must use the customer-"))
{
// The exception contains the correct endpoint; extract it
mediaConvertEndpoint = bre.Message.Split('\'')[1];
return(mediaConvertEndpoint);
// Code to retry query
}
}
return(null);
}
