Data lake processor module implementation file. More...

#include "data_lake_processor.h"
#include "data_lake_object.h"
#include "data_lake_json.h"
#include "mcl_data_lake/mcl_data_lake_common.h"
#include "mcl_data_lake/mcl_data_lake_configuration.h"
#include "mcl_core/mcl_memory.h"
#include "mcl_core/mcl_string_util.h"

Include dependency graph for data_lake_processor.c:

Go to the source code of this file.

Macros
#define	SINGLE_OBJECT_SIZE 1

Functions
static mcl_error_t	_add_authorization_header (mcl_http_request_t request, const char access_token)

static data_lake_object_t **	_data_lake_object_list_to_array (mcl_list_t *object_list)

static mcl_error_t	_generate_upload_urls (data_lake_processor_t processor, data_lake_object_t object_array, mcl_size_t array_size, const char subtenant_id)

static void	_clear_signed_urls (data_lake_object_t **object_array, mcl_size_t array_size)

static mcl_bool_t	_check_url_if_it_is_for_azure_storage (data_lake_object_t *object)

mcl_error_t	data_lake_processor_generate_upload_url (data_lake_processor_t processor, data_lake_object_t object, const char *subtenant_id)

mcl_error_t	data_lake_processor_generate_upload_urls (data_lake_processor_t processor, mcl_list_t object_list, const char *subtenant_id)

mcl_error_t	data_lake_processor_upload (data_lake_processor_t processor, data_lake_object_t object)

Variables
static const char	bearer_format [] = "Bearer %s"

static const char	string_identifier [] = "%s"

static const char	application_json [] = "application/json"

static const char	content_type [] = "Content-Type"

static const char	azure_storage_url [] = "blob.core.windows.net"

static const char	azure_header_blob_type [] = "x-ms-blob-type"

static const char	azure_header_block_blob [] = "BlockBlob"

Detailed Description

Data lake processor module implementation file.

Copyright: Copyright (C) 2019 Siemens Aktiengesellschaft.
All rights reserved.

Definition in file data_lake_processor.c.

Macro Definition Documentation

#define SINGLE_OBJECT_SIZE 1

Definition at line 17 of file data_lake_processor.c.

Referenced by data_lake_processor_generate_upload_url().

Function Documentation

static mcl_error_t _add_authorization_header	(	mcl_http_request_t *	request,
		const char *	access_token
	)

static

Definition at line 168 of file data_lake_processor.c.

References bearer_format, MCL_DEBUG_ENTRY, MCL_DEBUG_LEAVE, MCL_FREE, mcl_http_request_add_header(), MCL_MALLOC, MCL_NULL, MCL_NULL_CHAR_SIZE, MCL_OK, MCL_OUT_OF_MEMORY, mcl_string_util_snprintf(), mcl_string_util_strlen(), and string_identifier.

Referenced by _generate_upload_urls().

Here is the call graph for this function:

Here is the caller graph for this function:

static mcl_bool_t _check_url_if_it_is_for_azure_storage ( data_lake_object_t * object )

static

Definition at line 339 of file data_lake_processor.c.

References azure_storage_url, MCL_DEBUG_ENTRY, MCL_DEBUG_LEAVE, MCL_FALSE, MCL_NULL_CHAR_SIZE, mcl_string_util_memcmp(), mcl_string_util_strlen(), MCL_TRUE, and data_lake_object_t::signed_url.

Referenced by data_lake_processor_upload().

Here is the call graph for this function:

Here is the caller graph for this function:

static void _clear_signed_urls	(	data_lake_object_t **	object_array,
		mcl_size_t	array_size
	)

static

Definition at line 297 of file data_lake_processor.c.

References MCL_DEBUG_ENTRY, MCL_DEBUG_LEAVE, and MCL_FREE.

Referenced by _generate_upload_urls().

Here is the caller graph for this function:

static data_lake_object_t ** _data_lake_object_list_to_array ( mcl_list_t * object_list )

static

Definition at line 310 of file data_lake_processor.c.

References mcl_list_t::count, mcl_list_node_t::data, MCL_DEBUG_ENTRY, MCL_DEBUG_LEAVE, mcl_list_next(), MCL_MALLOC, and MCL_NULL.

Referenced by data_lake_processor_generate_upload_urls().

Here is the call graph for this function:

Here is the caller graph for this function:

static mcl_error_t _generate_upload_urls	(	data_lake_processor_t *	processor,
		data_lake_object_t **	object_array,
		mcl_size_t	array_size,
		const char *	subtenant_id
	)

static

Definition at line 199 of file data_lake_processor.c.

References _add_authorization_header(), _clear_signed_urls(), data_lake_processor_t::access_token, application_json, data_lake_processor_t::client_id, content_type, data_lake_json_from_objects(), data_lake_json_match_signed_urls_with_objects(), data_lake_processor_t::http_client, MCL_DEBUG_ENTRY, MCL_DEBUG_LEAVE, MCL_ERROR, MCL_FREE, mcl_http_client_send(), MCL_HTTP_POST, mcl_http_request_add_header(), mcl_http_request_destroy(), mcl_http_request_initialize(), MCL_HTTP_REQUEST_PARAMETER_BODY, MCL_HTTP_REQUEST_PARAMETER_BODY_SIZE, MCL_HTTP_REQUEST_PARAMETER_METHOD, MCL_HTTP_REQUEST_PARAMETER_URL, mcl_http_request_set_parameter(), mcl_http_response_destroy(), mcl_http_response_get_status(), MCL_HTTP_STATUS_CODE_CREATED, MCL_NULL, MCL_OK, mcl_string_util_strlen(), mcl_http_response_t::payload, mcl_http_response_t::payload_size, mcl_http_response_t::status_code, and data_lake_processor_t::upload_url_generation_url.

Referenced by data_lake_processor_generate_upload_url(), and data_lake_processor_generate_upload_urls().

Here is the call graph for this function:

Here is the caller graph for this function:

mcl_error_t data_lake_processor_generate_upload_url	(	data_lake_processor_t *	processor,
		data_lake_object_t *	object,
		const char *	subtenant_id
	)

This function gets the signed URL to upload data to.

Parameters

[in]	processor	Data lake processor.
[in]	object	Data lake object.
[in]	subtenant_id	Subtenant ID.

Returns

MCL_OK in case of success.
MCL_DATA_LAKE_SIGNED_URL_GENERATION_FAIL if signed URL could not be retrieved for the object.
MCL_TRIGGERED_WITH_NULL in case processor or object is null.
MCL_OUT_OF_MEMORY in case there is not enough memory in the system to proceed.
MCL_INVALID_PARAMETER if path of object is not set.
MCL_COULD_NOT_RESOLVE_PROXY if the proxy host name can not be resolved.
MCL_COULD_NOT_RESOLVE_HOST if the remote host name can not be resolved.
MCL_SSL_HANDSHAKE_FAIL if a problem occurs during SSL handshake.
MCL_COULD_NOT_CONNECT if there is a connection error with the server.
MCL_SERVER_CERTIFICATE_NOT_VERIFIED if the server can not be verified.
MCL_IMPROPER_CERTIFICATE in case the certificate provided is in improper format and it can not be parsed.
#MCL_CA_CERTIFICATE_AUTHENTICATION_FAIL if the server certificate can not be authenticated by the root certificate.
MCL_NETWORK_SEND_FAIL in case of an error in sending data to network.
MCL_NETWORK_RECEIVE_FAIL in case of an error in receiving data from the network.
MCL_BAD_REQUEST if response status code of server is 400.
MCL_UNAUTHORIZED if response status code of server is 401.
MCL_SERVER_FAIL if response status code of server is 500.

Definition at line 43 of file data_lake_processor.c.

References _generate_upload_urls(), MCL_DEBUG_ENTRY, MCL_DEBUG_LEAVE, and SINGLE_OBJECT_SIZE.

Referenced by mcl_data_lake_generate_upload_url(), and mcl_data_lake_generate_upload_url_for_subtenant().

Here is the call graph for this function:

Here is the caller graph for this function:

mcl_error_t data_lake_processor_generate_upload_urls	(	data_lake_processor_t *	processor,
		mcl_list_t *	object_list,
		const char *	subtenant_id
	)

This function gets the signed URLs for a list of objects to upload data to.

Parameters

[in]	processor	Data lake processor.
[in]	object_list	List of data lake objects.
[in]	subtenant_id	Subtenant ID.

Returns

MCL_OK in case of success.
MCL_DATA_LAKE_SIGNED_URL_GENERATION_FAIL if signed URL could not be retrieved for all data lake objects with non-NULL paths.
MCL_TRIGGERED_WITH_NULL in case processor or object_list is null.
MCL_OUT_OF_MEMORY in case there is not enough memory in the system to proceed.
MCL_INVALID_PARAMETER if all paths are NULL.
MCL_COULD_NOT_RESOLVE_PROXY if the proxy host name can not be resolved.
MCL_COULD_NOT_RESOLVE_HOST if the remote host name can not be resolved.
MCL_SSL_HANDSHAKE_FAIL if a problem occurs during SSL handshake.
MCL_COULD_NOT_CONNECT if there is a connection error with the server.
MCL_SERVER_CERTIFICATE_NOT_VERIFIED if the server can not be verified.
MCL_IMPROPER_CERTIFICATE in case the certificate provided is in improper format and it can not be parsed.
#MCL_CA_CERTIFICATE_AUTHENTICATION_FAIL if the server certificate can not be authenticated by the root certificate.
MCL_NETWORK_SEND_FAIL in case of an error in sending data to network.
MCL_NETWORK_RECEIVE_FAIL in case of an error in receiving data from the network.
MCL_BAD_REQUEST if response status code of server is 400.
MCL_UNAUTHORIZED if response status code of server is 401.
MCL_SERVER_FAIL if response status code of server is 500.