public Task<LiveOperationResult> UploadAsync(string path, IFileSource fileSource, OverwriteOption option, IBackgroundTransferProvider btu, CancellationToken ct, IProgress<LiveOperationProgress> progress)
{
if (string.IsNullOrEmpty(path))
{
throw new ArgumentException(
"path",
String.Format(CultureInfo.CurrentUICulture, ResourceHelper.GetString("UrlInvalid"),
"path"));
}
if (null == fileSource)
{
throw new ArgumentNullException(
"fileSource",
String.Format(CultureInfo.CurrentUICulture, ResourceHelper.GetString("InvalidNullParameter"),
"fileSource"));
}
if (null != fileSource && string.IsNullOrEmpty(fileSource.Filename))
{
throw new ArgumentException(
"fileName",
String.Format(CultureInfo.CurrentUICulture, ResourceHelper.GetString("InvalidNullOrEmptyParameter"),
"fileName"));
}
if (null == btu)
{
btu = new Microsoft.Live.Transfers.BasicTransferProvider();
}
ApiOperation op = btu.GetUploadOperation(this, this.GetResourceUri(path, ApiMethod.Upload), fileSource, option, progress, syncContext);
return this.ExecuteApiOperation(op, ct);
}
public static void Reduce(IFileSource<string, string> inputfolders, IFileDestination<string, string> outputfolder, IReducer reducer, int threads)
{
var threadpool = new ThreadPool(threads, "Reducer file threads");
Console.WriteLine("Preparing reducer threads.");
string inputfileid;
while (inputfolders.ReadNext(out inputfileid))
{
var ft = new ReduceFileThread();
ft.InputFolders = inputfolders;
ft.OutputFolder = outputfolder;
// ft.Writer = wrt;
ft.reducer = reducer;
ft.FileID = inputfileid;
threadpool.Queue(ft);
}
Console.WriteLine("Waiting for reducer threads to finish...");
var t1 = new Timing("Inner mapper");
threadpool.WaitAll();
t1.End();
Console.WriteLine("Reducer threads finished.");
GC.Collect();
}
public ForegroundUploadOperation(LiveConnectClient client, Uri url, string filename, IFileSource inputFile, OverwriteOption option, IProgress<LiveOperationProgress> progress, SynchronizationContextWrapper syncContext)
: base(client, url, ApiMethod.Upload, null, syncContext)
{
this.Filename = filename;
this.Progress = progress;
this.OverwriteOption = option;
this.FileSource = inputFile;
}
public void Dump(IFileSource<string, string> filesource, string title)
{
DumpHeader(title);
var fn = "";
while (filesource.ReadNext(out fn))
{
var reader = filesource.CreateStreamReader(fn);
string k, v;
while (reader.Read(out k, out v))
WriteLine(k + " => " + v);
}
DumpFooter(title);
}
internal MapAndReduceJob()
{
InputPartitioner = null;
Mapper = null;
Reducer = null;
ShufflePartitioner = null;
CombineOutput = false;
Input = null;
Output = null;
InputFitsInMemory = false;
MapperFitsInMemory = false;
ShufflerFitsInMemory = false;
ReducerFitsInMemory = false;
RunReducerAfterEveryMapper = false;
}
public Task<LiveDownloadOperationResult> DownloadAsync(string path, IFileSource destination, IBackgroundTransferProvider btu = null, IProgress<LiveOperationProgress> progress = null)
{
if (path == null)
{
throw new ArgumentNullException("path");
}
if (string.IsNullOrWhiteSpace(path))
{
throw new ArgumentException(
"path",
String.Format(CultureInfo.CurrentUICulture, ResourceHelper.GetString("UrlInvalid"), "path"));
}
if (null == destination)
{
throw new ArgumentNullException("destination",
String.Format(CultureInfo.CurrentUICulture, ResourceHelper.GetString("InvalidNullParameter"), "destination"));
}
return this.InternalDownloadAsync(path, destination, btu, new CancellationToken(false), progress);
}
public static void Map(IFileSource<string, string> inputfolders, IFileDestination<string, string> shuffleroutput, IMapper mapper, IReducer prereducer, int threads)
{
// var partitioner = new StandardKeyPartitioner(Count);
var threadpool = new ThreadPool(threads, "Mapper file threads");
var threadpool2 = new ThreadPool(threads, "Mapper threads");
Console.WriteLine("Preparing mappers...");
string inputfileid;
while (inputfolders.ReadNext(out inputfileid))
{
var ft = new MapFilesThread();
ft.InputFolders = inputfolders;
ft.OutputFolder = shuffleroutput;
ft.mapper = mapper;
ft.reducer = prereducer;
ft.FileIDs.Add(inputfileid);
ft.threadpool = threadpool;
threadpool.Queue(ft);
threadpool.Step();
}
Console.WriteLine("Waiting for mapper file threads to finish...");
var t1 = new Timing("Inner mapper");
threadpool.WaitAll();
t1.End();
Console.WriteLine("Mapper file threads finished.");
/*
Console.WriteLine("Waiting for mapper threads to finish...");
var t2 = new Timing("Inner mapper");
threadpool2.WaitAll();
t2.End();
Console.WriteLine("Mapper threads finished.");
*/
GC.Collect();
}
public static void Combine(IFileSource<string, string> inputs, IFileDestination<string, string> output)
{
var t0 = new Timing("Combining files");
using (var writer = output.CreateWriter())
{
var stp = new ThreadPool(5);
Console.WriteLine("Preparing combiners...");
string inputfile;
while (inputs.ReadNext(out inputfile))
{
stp.Queue(new CombinerThread
{
outputwriter = writer,
filesource = inputs,
FileId = inputfile
});
}
Console.WriteLine("Combining files...");
stp.WaitAll();
}
t0.End();
Console.WriteLine("Combining done.");
GC.Collect();
}
public static void Map(IFileSource<string, string> inputfolders, IFileDestination<string, string> shuffleroutput, IMapper mapper, int threads)
{
Map(inputfolders, shuffleroutput, mapper, null, threads);
}
public static void Map(IFileSource<string, string> inputfolders, IFileDestination<string, string> shuffleroutput, IMapper mapper, IReducer prereducer)
{
Map(inputfolders, shuffleroutput, mapper, prereducer, 50);
}
/// <inheritdoc /> public virtual async Task <SongInfo> ExtractAsyncTask(IFileSource source) => await ExtractAsyncTask(SongInfo.ReadFromFile(source.FileInfo.FullName));
public override void Initialize(IFileSource fileSource)
{
base.Initialize(fileSource);
_directions = MapboxAccess.Instance.Directions;
}
/// <inheritdoc /> public override IFileSource Execute(IFileSource fileSource) => Processor(fileSource);
public ApiOperation GetUploadOperation(LiveConnectClient client, Uri url, IFileSource inputFile, OverwriteOption option, IProgress<LiveOperationProgress> progress, SynchronizationContextWrapper syncContext)
{
return new UploadOperation(client, url, inputFile.Filename, inputFile.GetReadStream(), option, progress, syncContext);
}
internal Task<LiveDownloadOperationResult> InternalDownloadAsync(string path, IFileSource destination, IBackgroundTransferProvider btu, CancellationToken ct, IProgress<LiveOperationProgress> progress)
{
if (this.Session == null)
{
throw new LiveConnectException(ApiOperation.ApiClientErrorCode, ResourceHelper.GetString("UserNotLoggedIn"));
}
var tcs = new TaskCompletionSource<LiveDownloadOperationResult>();
var op = btu.GetDownloadOperation(this, this.GetResourceUri(path, ApiMethod.Download), destination, progress, syncContext);
op.OperationCompletedCallback = (LiveDownloadOperationResult result) =>
{
if (result.IsCancelled)
{
tcs.TrySetCanceled();
}
else if (result.Error != null)
{
tcs.TrySetException(result.Error);
}
else
{
tcs.TrySetResult(result);
}
};
ct.Register(op.Cancel);
op.Execute();
return tcs.Task;
}
private void ShowVedioControlForm(IFileSource iFileSource)
{
if (vedioControlForm == null)
vedioControlForm = new VideoControlForm();
vedioControlForm.FileSource = iFileSource;
vedioControlForm.Icon = this.Icon;
// vedioControlForm.Parent = this;
vedioControlForm.Show();
}
/// <inheritdoc /> public abstract IFileSource Execute(IFileSource fileSource);
public void InsertSequencePoint(IFileSource file, int line, int position)
{
}
public FileData(IFileSource file)
{
File = file;
}
public async Task SaveAsync(
IFileSource uploadedFile,
ImageAsset imageAsset,
string propertyName
)
{
Image imageFile = null;
IImageFormat imageFormat = null;
using (var inputSteam = await uploadedFile.OpenReadStreamAsync())
{
try
{
imageFile = Image.Load(inputSteam, out imageFormat);
}
catch (ArgumentException ex)
{
// We'll get an argument exception if the image file is invalid
// so lets check to see if we can identify if it is an invalid file type and show that error
// This might not always be the case since a file extension or mime type might not be supplied.
var ext = Path.GetExtension(uploadedFile.FileName);
if ((!string.IsNullOrEmpty(ext) && !ImageAssetConstants.PermittedImageTypes.ContainsKey(ext)) ||
(!string.IsNullOrEmpty(uploadedFile.MimeType) && !ImageAssetConstants.PermittedImageTypes.ContainsValue(uploadedFile.MimeType)))
{
throw new PropertyValidationException("The file is not a supported image type.", propertyName);
}
throw;
}
using (imageFile) // validate image file
{
ValidateImage(propertyName, imageFile, imageFormat);
var requiredReEncoding = true;
var fileExtension = "jpg";
var foundExtension = _permittedImageFileExtensions
.FirstOrDefault(e => imageFormat.FileExtensions.Contains(e));
if (foundExtension != null)
{
fileExtension = foundExtension;
requiredReEncoding = false;
}
imageAsset.WidthInPixels = imageFile.Width;
imageAsset.HeightInPixels = imageFile.Height;
imageAsset.FileExtension = fileExtension;
imageAsset.FileSizeInBytes = inputSteam.Length;
using (var scope = _transactionScopeManager.Create(_dbContext))
{
var fileName = Path.ChangeExtension(imageAsset.FileNameOnDisk, imageAsset.FileExtension);
if (requiredReEncoding)
{
// Convert the image to jpg
using (var outputStream = new MemoryStream())
{
if (requiredReEncoding)
{
imageFile.Save(outputStream, new JpegEncoder());
}
else
{
imageFile.Save(outputStream, imageFormat);
}
await _fileStoreService.CreateAsync(ASSET_FILE_CONTAINER_NAME, fileName, outputStream);
// recalculate size and save
imageAsset.FileSizeInBytes = outputStream.Length;
}
}
else
{
// Save the raw file directly
await _fileStoreService.CreateAsync(ASSET_FILE_CONTAINER_NAME, fileName, inputSteam);
}
await _dbContext.SaveChangesAsync();
await scope.CompleteAsync();
};
}
}
}
/// <summary>
/// Uploads a file into the OneDriveItem this command is called on. This command is only available on OneDriveItem instances that represents
/// a folder or album.
/// </summary>
/// <param name="sourceItem"></param>
/// <param name="option"></param>
/// <param name="btp"></param>
/// <param name="progress"></param>
/// <returns></returns>
public async Task<OneDriveItem> UploadFileAsync(IFileSource sourceItem, OverwriteOption option, IBackgroundTransferProvider btp = null, IProgress<LiveOperationProgress> progress = null)
{
if (this.ItemType != OneDriveItemType.Folder && this.ItemType != OneDriveItemType.Album)
{
throw new InvalidOperationException("Cannot upload files to items that do not represent a folder or album");
}
LiveOperationResult result = await Client.LiveClient.UploadAsync(
string.Format("/{0}", this.Identifier),
sourceItem,
option,
btp,
progress);
System.Diagnostics.Debug.WriteLine("UploadFile Result: {0}", result.RawResult);
FileUploadResult fur = FileUploadResult.FromJson(result.RawResult);
return await this.Client.GetItemFromIdentifier(fur.Identifier);
}
public List <IToken> GetTokens(IFileSource fileSource)
{
return(GetTokens(fileSource, new OffsetsStore()));
}
public Task<LiveOperationResult> UploadAsync(string path, IFileSource fileSource, OverwriteOption option, IBackgroundTransferProvider btu = null, IProgress<LiveOperationProgress> progress = null)
{
return this.UploadAsync(path, fileSource, option, btu, new CancellationToken(false), progress);
}
public static void Reduce(IFileSource<string, string> inputfolders, IFileDestination<string, string> outputfolder, IReducer reducer)
{
Reduce(inputfolders, outputfolder, reducer, 30);
}
public List <IToken> GetTokens(IFileSource fileSource, OffsetsStore offsetsStore)
{
offsetsStore = offsetsStore ?? new OffsetsStore();
return(IntGetTokens(fileSource.GetData(), fileSource, offsetsStore));
}
/// <summary>
/// Delete the passed file source.
/// </summary>
/// <param name="fileSoure">The file source that will be deleted.</param>
/// <returns>This method always returns <c>null</c>.</returns>
public override IFileSource Execute(IFileSource fileSoure)
{
File.Delete(fileSoure.FileInfo.FullName);
return(null);
}
public ForegroundUploadOperation(LiveConnectClient client, Uri url, string filename, IFileSource inputFile, OverwriteOption option, IProgress <LiveOperationProgress> progress, SynchronizationContextWrapper syncContext)
: base(client, url, ApiMethod.Upload, null, syncContext)
{
this.Filename = filename;
this.Progress = progress;
this.OverwriteOption = option;
this.FileSource = inputFile;
}
public FluentMapAndReduce Input(IFileSource<string, string> input)
{
job.Input = input;
return this;
}
/// <summary> Initializes a new instance of the <see cref="Geocoder" /> class. </summary>
/// <param name="fileSource"> Network access abstraction. </param>
public Geocoder(IFileSource fileSource)
{
this.fileSource = fileSource;
}
public void SetFileSource(IFileSource fileSource)
{
_fileSource = fileSource;
}
public DownloadOperation GetDownloadOperation(LiveConnectClient client, Uri url, IFileSource outputFile, IProgress<LiveOperationProgress> progress, SynchronizationContextWrapper syncContext)
{
// TODO: This is include since we aren't download to outputFile.
var downloadOp = new DownloadOperation(client, url, progress, syncContext);
return downloadOp;
}
