/// <summary>
/// Send an HTTP GET request to the specified URI and intepret the result as TResult.
/// </summary>
/// <typeparam name="TResult">The .NET class to use for response deserialization.</typeparam>
/// <param name="uri">A uniform resource identifier that is the target of this GET request.</param>
/// <param name="dataFormat">One of the data-interchange formats that Socrata supports. The default is JSON.</param>
/// <returns>The HTTP response, deserialized into an object of type <typeparamref name="TResult"/>.</returns>
internal TResult read <TResult>(Uri uri, SodaDataFormat dataFormat = SodaDataFormat.JSON)
where TResult : class
{
var request = new SodaRequest(uri, "GET", AppToken, Username, password, dataFormat, null, RequestTimeout);
return(request.ParseResponse <TResult>());
}
/// <summary>
/// Export the error rows (if present).
/// </summary>
/// <param name="filepath">The output file (csv)</param>
/// <param name="output">The specified transformed output</param>
public void ExportErrorRows(string filepath, AppliedTransform output)
{
if (String.IsNullOrEmpty(filepath))
{
throw new ArgumentException("Filepath must be specified.", "filepath");
}
if (output == null)
{
throw new ArgumentException("Applied Transform required.", "output");
}
if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password))
{
throw new InvalidOperationException("Write operations require an authenticated client.");
}
var endpoint = output.GetErrorRowEndpoint().Replace("{input_schema_id}", output.GetInputSchemaId()).Replace("{output_schema_id}", output.GetOutputSchemaId());
var errorRowsUri = SodaUri.ForErrorRows(Host, endpoint);
Debug.WriteLine(errorRowsUri.ToString());
var downloadRowsRequest = new SodaRequest(errorRowsUri, "GET", null, username, password, SodaDataFormat.CSV, "");
var result = downloadRowsRequest.ParseResponse <String>();
System.IO.File.WriteAllText(filepath, result);
}
/// <summary>
/// Await the completion of the update, optionally output the status.
/// </summary>
/// <param name="lambda">A lambda function for outputting status if desired.</param>
public void AwaitCompletion(Action <string> lambda)
{
string status = "";
Result r = null;
while (status != "successful" && status != "failure")
{
var jobRequest = new SodaRequest(revisionEndpoint, "GET", null, username, password, SodaDataFormat.JSON);
try
{
r = jobRequest.ParseResponse <Result>();
}
catch (WebException webException)
{
string message = webException.UnwrapExceptionMessage();
r = new Result()
{
Message = webException.Message, IsError = true, ErrorCode = message
};
}
status = r.Resource["task_sets"][0]["status"];
lambda(status);
System.Threading.Thread.Sleep(1000);
}
}
/// <summary>
/// Send an HTTP request of the specified method and interpret the result.
/// </summary>
/// <typeparam name="TPayload">The .NET class that represents the request payload.</typeparam>
/// <typeparam name="TResult">The .NET class to use for response deserialization.</typeparam>
/// <param name="uri">A uniform resource identifier that is the target of this GET request.</param>
/// <param name="method">One of POST, PUT, or DELETE</param>
/// <param name="payload">An object graph to serialize and send with the request.</param>
/// <returns>The HTTP response, deserialized into an object of type <typeparamref name="TResult"/>.</returns>
internal TResult write <TPayload, TResult>(Uri uri, string method, TPayload payload)
where TPayload : class
where TResult : class
{
var request = new SodaRequest(uri, method, AppToken, Username, password, SodaDataFormat.JSON, payload.ToJsonString(), RequestTimeout);
return(request.ParseResponse <TResult>());
}
/// <summary>
/// Get the specified source data.
/// </summary>
/// <param name="source">The result of the Source creation</param>
/// <returns>A <see cref="Source"/>The updated Source object</returns>
public Source GetSource(Source source)
{
if (source == null)
{
throw new ArgumentException("Source required.", "source");
}
var sourceUri = SodaUri.ForSource(Host, source.Self());
var sourceUpdateResponse = new SodaRequest(sourceUri, "GET", null, username, password, SodaDataFormat.JSON, "");
Result result = sourceUpdateResponse.ParseResponse <Result>();
return(new Source(result));
}
/// <summary>
/// Replace any existing rows with the payload data, using the specified resource identifier.
/// </summary>
/// <param name="payload">A string of serialized data.</param>
/// <param name="dataFormat">One of the data-interchange formats that Socrata supports, into which the payload has been serialized.</param>
/// <param name="resourceId">The identifier (4x4) for a resource on the Socrata host to target.</param>
/// <returns>A <see cref="SodaResult"/> indicating success or failure.</returns>
/// <exception cref="System.ArgumentOutOfRangeException">Thrown if the specified <paramref name="dataFormat"/> is equal to <see cref="SodaDataFormat.XML"/>.</exception>
/// <exception cref="System.ArgumentOutOfRangeException">Thrown if the specified <paramref name="resourceId"/> does not match the Socrata 4x4 pattern.</exception>
/// <exception cref="System.InvalidOperationException">Thrown if this SodaClient was initialized without authentication credentials.</exception>
public SodaResult Replace(string payload, SodaDataFormat dataFormat, string resourceId)
{
if (dataFormat == SodaDataFormat.XML)
{
throw new ArgumentOutOfRangeException("dataFormat", "XML is not a valid format for write operations.");
}
if (FourByFour.IsNotValid(resourceId))
{
throw new ArgumentOutOfRangeException("resourceId", "The provided resourceId is not a valid Socrata (4x4) resource identifier.");
}
if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password))
{
throw new InvalidOperationException("Write operations require an authenticated client.");
}
var uri = SodaUri.ForResourceAPI(Host, resourceId);
var request = new SodaRequest(uri, "PUT", AppToken, Username, password, dataFormat, payload);
SodaResult result = null;
try
{
if (dataFormat == SodaDataFormat.JSON)
{
result = request.ParseResponse <SodaResult>();
}
else if (dataFormat == SodaDataFormat.CSV)
{
string resultJson = request.ParseResponse <string>();
result = Newtonsoft.Json.JsonConvert.DeserializeObject <SodaResult>(resultJson);
}
}
catch (WebException webException)
{
string message = webException.UnwrapExceptionMessage();
result = new SodaResult()
{
Message = webException.Message, IsError = true, ErrorCode = message, Data = payload
};
}
catch (Exception ex)
{
result = new SodaResult()
{
Message = ex.Message, IsError = true, ErrorCode = ex.Message, Data = payload
};
}
return(result);
}
/// <summary>
/// Replace any existing rows with the payload data, using the specified resource identifier.
/// </summary>
/// <param name="method">One of update, replace, or delete</param>
/// <param name="resourceId">The identifier (4x4) for a resource on the Socrata host to target.</param>
/// <param name="permission">The permission level of the dataset, can be one of either "public" or "private".</param>
/// <returns>A <see cref="Revision"/> newly created Revision.</returns>
/// <exception cref="System.ArgumentOutOfRangeException">Thrown if the specified <paramref name="resourceId"/> does not match the Socrata 4x4 pattern.</exception>
public Revision CreateRevision(string method, string resourceId, string permission = "private")
{
if (String.IsNullOrEmpty(method))
{
throw new ArgumentException("Method must be specified either update, replace, or delete.", "method");
}
if (FourByFour.IsNotValid(resourceId))
{
throw new ArgumentOutOfRangeException("resourceId", "The provided resourceId is not a valid Socrata (4x4) resource identifier.");
}
if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password))
{
throw new InvalidOperationException("Write operations require an authenticated client.");
}
var revisionUri = SodaUri.ForRevision(Host, resourceId);
// Construct Revision Request body
Newtonsoft.Json.Linq.JObject payload = new Newtonsoft.Json.Linq.JObject();
Newtonsoft.Json.Linq.JObject action = new Newtonsoft.Json.Linq.JObject();
action["type"] = method;
action["permission"] = permission;
payload["action"] = action;
var request = new SodaRequest(revisionUri, "POST", null, username, password, SodaDataFormat.JSON, payload.ToString());
Result result = null;
try
{
result = request.ParseResponse <Result>();
}
catch (WebException webException)
{
string message = webException.UnwrapExceptionMessage();
result = new Result()
{
Message = webException.Message, IsError = true, ErrorCode = message, Data = payload
};
}
catch (Exception ex)
{
result = new Result()
{
Message = ex.Message, IsError = true, ErrorCode = ex.Message, Data = payload
};
}
return(new Revision(result));
}
/// <summary>
/// Create a new dataset with a given name and permission level.
/// </summary>
/// <param name="name">The dataset name</param>
/// <param name="permission">The permission level of the dataset, can be one of either "public" or "private".</param>
/// <returns>A <see cref="Revision"/> newly created Revision.</returns>
public Revision CreateDataset(string name, string permission = "private")
{
if (String.IsNullOrEmpty(name))
{
throw new ArgumentException("Dataset name required.", "name");
}
if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password))
{
throw new InvalidOperationException("Write operations require an authenticated client.");
}
var revisionUri = SodaUri.ForRevision(Host);
// Construct Revision Request body
Newtonsoft.Json.Linq.JObject payload = new Newtonsoft.Json.Linq.JObject();
Newtonsoft.Json.Linq.JObject metadata = new Newtonsoft.Json.Linq.JObject();
Newtonsoft.Json.Linq.JObject action = new Newtonsoft.Json.Linq.JObject();
metadata["name"] = name;
action["type"] = "replace";
action["permission"] = permission;
payload["action"] = action;
payload["metadata"] = metadata;
var request = new SodaRequest(revisionUri, "POST", null, username, password, SodaDataFormat.JSON, payload.ToString());
Result result = null;
try
{
result = request.ParseResponse <Result>();
}
catch (WebException webException)
{
string message = webException.UnwrapExceptionMessage();
result = new Result()
{
Message = webException.Message, IsError = true, ErrorCode = message, Data = payload
};
}
catch (Exception ex)
{
result = new Result()
{
Message = ex.Message, IsError = true, ErrorCode = ex.Message, Data = payload
};
}
return(new Revision(result));
}
/// <summary>
/// Creates the source for the specified revision.
/// </summary>
/// <param name="data">A string of serialized data.</param>
/// <param name="revision">The revision created as part of a create revision step.</param>
/// <param name="dataFormat">The format of the data.</param>
/// <param name="filename">The filename that should be associated with this upload.</param>
/// <returns>A <see cref="Source"/> indicating success or failure.</returns>
/// <exception cref="System.InvalidOperationException">Thrown if this SodaDSMAPIClient was initialized without authentication credentials.</exception>
public Source CreateSource(string data, Revision revision, SodaDataFormat dataFormat = SodaDataFormat.CSV, string filename = "NewUpload")
{
if (String.IsNullOrEmpty(data))
{
throw new ArgumentException("Data must be provided.", "data");
}
if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password))
{
throw new InvalidOperationException("Write operations require an authenticated client.");
}
if (revision == null)
{
throw new ArgumentException("Revision required.", "revision");
}
var sourceUri = SodaUri.ForSource(Host, revision.GetSourceEndpoint());
Debug.WriteLine(sourceUri.ToString());
revision.GetRevisionNumber();
// Construct Revision Request body
Newtonsoft.Json.Linq.JObject payload = new Newtonsoft.Json.Linq.JObject();
Newtonsoft.Json.Linq.JObject source_type = new Newtonsoft.Json.Linq.JObject();
Newtonsoft.Json.Linq.JObject parse_option = new Newtonsoft.Json.Linq.JObject();
source_type["type"] = "upload";
source_type["filename"] = filename;
parse_option["parse_source"] = true;
payload["source_type"] = source_type;
payload["parse_options"] = parse_option;
var createSourceRequest = new SodaRequest(sourceUri, "POST", null, username, password, SodaDataFormat.JSON, payload.ToString());
Result sourceOutput = createSourceRequest.ParseResponse <Result>();
string uploadDataPath = sourceOutput.Links["bytes"];
var uploadUri = SodaUri.ForUpload(Host, uploadDataPath);
Debug.WriteLine(uploadUri.ToString());
var fileUploadRequest = new SodaRequest(uploadUri, "POST", null, username, password, dataFormat, data);
fileUploadRequest.SetDataType(SodaDataFormat.JSON);
Result result = fileUploadRequest.ParseResponse <Result>();
return(new Source(result));
}
/// <summary>
/// Apply the source, transforms, and update to the specified dataset.
/// </summary>
/// <param name="outputSchema">A string of serialized data.</param>
/// <param name="revision">A string of serialized data.</param>
/// <returns>A <see cref="PipelineJob"/> for determining success or failure.</returns>
public PipelineJob Apply(AppliedTransform outputSchema, Revision revision)
{
if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password))
{
throw new InvalidOperationException("Write operations require an authenticated client.");
}
if (outputSchema == null || revision == null)
{
throw new InvalidOperationException("Both the output schema and revision are required.");
}
Newtonsoft.Json.Linq.JObject payload = new Newtonsoft.Json.Linq.JObject();
payload["output_schema_id"] = outputSchema.GetOutputSchemaId();
var uri = SodaUri.ForSource(Host, revision.GetApplyEndpoint());
Debug.WriteLine(uri.ToString());
var applyRequest = new SodaRequest(uri, "PUT", null, username, password, SodaDataFormat.JSON, payload.ToString());
Result result = null;
try
{
result = applyRequest.ParseResponse <Result>();
}
catch (WebException webException)
{
string message = webException.UnwrapExceptionMessage();
result = new Result()
{
Message = webException.Message, IsError = true, ErrorCode = message, Data = payload
};
}
catch (Exception ex)
{
result = new Result()
{
Message = ex.Message, IsError = true, ErrorCode = ex.Message, Data = payload
};
}
return(new PipelineJob(SodaUri.ForJob(Host, revision.getRevisionLink()), username, password));
}
/// <summary>
/// Delete a single row using the specified row identifier and the specified resource identifier.
/// </summary>
/// <param name="rowId">The identifier of the row to be deleted.</param>
/// <param name="resourceId">The identifier (4x4) for a resource on the Socrata host to target.</param>
/// <returns>A <see cref="SodaResult"/> indicating success or failure.</returns>
/// <exception cref="System.ArgumentException">Thrown if the specified <paramref name="rowId"/> is null or empty.</exception>
/// <exception cref="System.ArgumentOutOfRangeException">Thrown if the specified <paramref name="resourceId"/> does not match the Socrata 4x4 pattern.</exception>
/// <exception cref="System.InvalidOperationException">Thrown if this SodaClient was initialized without authentication credentials.</exception>
public SodaResult DeleteRow(string rowId, string resourceId)
{
if (String.IsNullOrEmpty(rowId))
{
throw new ArgumentException("Must specify the row to be deleted using its row identifier.", "rowId");
}
if (FourByFour.IsNotValid(resourceId))
{
throw new ArgumentOutOfRangeException("resourceId", "The provided resourceId is not a valid Socrata (4x4) resource identifier.");
}
if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password))
{
throw new InvalidOperationException("Write operations require an authenticated client.");
}
var uri = SodaUri.ForResourceAPI(Host, resourceId, rowId);
var request = new SodaRequest(uri, "DELETE", AppToken, Username, password);
return(request.ParseResponse <SodaResult>());
}