示例#1
0
 public static void UploadToDfs(Uri dfsDirectory, string localPath)
 {
     DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsDirectory.AbsoluteUri);
     try
     {
         if (dfsDirectory.Scheme == "hdfs")
         {
             using (var hdfs = new Microsoft.Research.Peloponnese.Hdfs.HdfsInstance(dfsDirectory))
             {
                 string dfsPath = dfsDirectory.AbsolutePath.TrimEnd('/') + "/" + Path.GetFileName(localPath);
                 DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsPath);
                 hdfs.UploadAll(localPath, dfsPath);
             }
         }
         else if (dfsDirectory.Scheme == "azureblob")
         {
             string account, key, container, blob;
             Microsoft.Research.Peloponnese.Azure.Utils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob);
             var    azure   = new Microsoft.Research.Peloponnese.Azure.AzureDfsClient(account, key, container);
             string dfsPath = blob.TrimEnd('/') + "/" + Path.GetFileName(localPath);
             Uri    dfsUri  = Microsoft.Research.Peloponnese.Azure.Utils.ToAzureUri(account, container, dfsPath, null, key);
             DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsUri.AbsoluteUri);
             azure.PutDfsFile(dfsUri, localPath);
         }
         else if (dfsDirectory.Scheme == "file")
         {
             string dstPath = Path.Combine(dfsDirectory.AbsolutePath, Path.GetFileName(localPath));
             File.Copy(localPath, dstPath);
         }
     }
     catch (Exception e)
     {
         DryadLogger.LogWarning("Failed to upload query plan: " + e.ToString());
     }
 }
示例#2
0
        public static void UploadToDfs(Uri dfsDirectory, string dfsName, string payload)
        {
            byte[] payloadBytes = System.Text.Encoding.UTF8.GetBytes(payload);

            DryadLogger.LogInformation("Uploading payload to " + dfsName + " at " + dfsDirectory.AbsoluteUri);
            try
            {
                if (dfsDirectory.Scheme == "hdfs")
                {
                    using (var hdfs = new Microsoft.Research.Peloponnese.Hdfs.HdfsInstance(dfsDirectory))
                    {
                        string dfsPath = dfsDirectory.AbsolutePath + dfsName;
                        hdfs.WriteAll(dfsPath, payloadBytes);
                    }
                }
                else if (dfsDirectory.Scheme == "azureblob")
                {
                    string account, key, container, blob;
                    Microsoft.Research.Peloponnese.Azure.Utils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob);
                    var    azure   = new Microsoft.Research.Peloponnese.Azure.AzureDfsClient(account, key, container);
                    string dfsPath = blob + dfsName;
                    Uri    dfsUri  = Microsoft.Research.Peloponnese.Azure.Utils.ToAzureUri(account, container, dfsPath, null, key);
                    azure.PutDfsFile(dfsUri, payloadBytes);
                }
                else if (dfsDirectory.Scheme == "file")
                {
                    string dstPath = Path.Combine(dfsDirectory.AbsolutePath, dfsName);
                    File.WriteAllBytes(dstPath, payloadBytes);
                }
            }
            catch (Exception e)
            {
                DryadLogger.LogWarning("Failed to upload final output: " + e.ToString());
            }
        }
示例#3
0
 public static void UploadToDfs(Uri dfsDirectory, string localPath)
 {
     DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsDirectory.AbsoluteUri);
     try
     {
         if (dfsDirectory.Scheme == "hdfs")
         {
             using (var hdfs = new Microsoft.Research.Peloponnese.Hdfs.HdfsInstance(dfsDirectory))
             {
                 string dfsPath = dfsDirectory.AbsolutePath.TrimEnd('/') + "/" + Path.GetFileName(localPath);
                 DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsPath);
                 hdfs.UploadAll(localPath, dfsPath);
             }
         }
         else if (dfsDirectory.Scheme == "azureblob")
         {
             string account, key, container, blob;
             Microsoft.Research.Peloponnese.Azure.Utils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob);
             var azure = new Microsoft.Research.Peloponnese.Azure.AzureDfsClient(account, key, container);
             string dfsPath = blob.TrimEnd('/') + "/" + Path.GetFileName(localPath);
             Uri dfsUri = Microsoft.Research.Peloponnese.Azure.Utils.ToAzureUri(account, container, dfsPath, null, key);
             DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsUri.AbsoluteUri);
             azure.PutDfsFile(dfsUri, localPath);
         }
         else if (dfsDirectory.Scheme == "file")
         {
             string dstPath = Path.Combine(dfsDirectory.AbsolutePath, Path.GetFileName(localPath));
             File.Copy(localPath, dstPath);
         }
     }
     catch (Exception e)
     {
         DryadLogger.LogWarning("Failed to upload query plan: " + e.ToString());
     }
 }
示例#4
0
        public static void UploadToDfs(Uri dfsDirectory, string dfsName, string payload)
        {
            byte[] payloadBytes = System.Text.Encoding.UTF8.GetBytes(payload);

            DryadLogger.LogInformation("Uploading payload to " + dfsName + " at " + dfsDirectory.AbsoluteUri);
            try
            {
                if (dfsDirectory.Scheme == "hdfs")
                {
                    using (var hdfs = new Microsoft.Research.Peloponnese.Hdfs.HdfsInstance(dfsDirectory))
                    {
                        string dfsPath = dfsDirectory.AbsolutePath + dfsName;
                        hdfs.WriteAll(dfsPath, payloadBytes);
                    }
                }
                else if (dfsDirectory.Scheme == "azureblob")
                {
                    string account, key, container, blob;
                    Microsoft.Research.Peloponnese.Azure.Utils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob);
                    var azure = new Microsoft.Research.Peloponnese.Azure.AzureDfsClient(account, key, container);
                    string dfsPath = blob + dfsName;
                    Uri dfsUri = Microsoft.Research.Peloponnese.Azure.Utils.ToAzureUri(account, container, dfsPath, null, key);
                    azure.PutDfsFile(dfsUri, payloadBytes);
                }
                else if (dfsDirectory.Scheme == "file")
                {
                    string dstPath = Path.Combine(dfsDirectory.AbsolutePath, dfsName);
                    File.WriteAllBytes(dstPath, payloadBytes);
                }
            }
            catch (Exception e)
            {
                DryadLogger.LogWarning("Failed to upload final output: " + e.ToString());
            }
        }