完成HTTP解析请求侧解析基本流程

* 集成HTTP Parser,解析HTTP请求侧数据;
* 增加tfe_http.h中请求方法、应答状态的常量定义
* 变更tfe_http.h中HTTP头部标准定义,去掉非头部字段,增加TFE前缀避免冲突。
This commit is contained in:
Lu Qiuwen
2018-09-06 10:12:08 +08:00
parent 89b79eab60
commit e31ecbb8db
8 changed files with 723 additions and 120 deletions

View File

@@ -1,4 +1,8 @@
add_library(http src/http.cpp)
add_library(http src/http_entry.cpp src/http_half.cpp)
target_include_directories(http PRIVATE include/internal)
target_include_directories(http PUBLIC incluce/external)
target_link_libraries(http common)
target_link_libraries(http http-parser-static)
target_link_libraries(http libevent-static)

View File

@@ -0,0 +1,82 @@
#pragma once
extern "C"
{
#include <sys/queue.h>
}
#include <tfe_http.h>
#include <http_parser.h>
struct http_plugin
{
};
TAILQ_HEAD(hs_private_list, http_session_private);
struct http_session_private
{
struct tfe_http_session hs_public;
TAILQ_ENTRY(http_session_private) next;
struct http_connection_private * hc_private;
};
struct http_connection_private
{
/* ADDRESS */
struct tfe_stream_addr * layer_addr;
/* SESSION LIST, REQUEST-RESPONSE PAIRS */
struct hs_private_list hs_private_list;
/* IS PREEMPTED */
unsigned int is_preempted;
/* SESSION ID COUNTER */
unsigned int session_id_counter;
};
struct http_half_private
{
/* PUBLIC STRUCTURE */
struct tfe_http_half hf_public;
/* SESSION OF THIS REQUEST/RESPONSE */
struct http_session_private * session;
/* HTTP PARSER */
struct http_parser * parse_object;
/* HTTP PARSER SETTING */
struct http_parser_settings * parse_settings;
/* CURSOR RELATED TO DELAYED DATA */
size_t parse_cursor;
/* HTTP PARSER'S ERRNO */
enum http_errno parse_errno;
/* UNDERLAY BUFFER */
int method_or_status;
short major;
short minor;
struct evbuffer * evbuf_uri;
char * underlay_uri;
char * underlay_url;
struct evbuffer * evbuf_header_field;
struct evbuffer * evbuf_header_value;
struct evbuffer * evbuf_body;
/* STATUS */
bool finished;
};
static inline struct http_half_private * to_hf_request_private(struct http_session_private * hs_private)
{
struct tfe_http_half * hf_public = hs_private->hs_public.req;
return container_of(hf_public, struct http_half_private, hf_public);
}
static inline struct http_half_private * to_hf_response_private(struct http_session_private * hs_private)
{
struct tfe_http_half * hf_public = hs_private->hs_public.resp;
return container_of(hf_public, struct http_half_private, hf_public);
}
static inline struct tfe_http_half * to_hf_public(struct http_half_private * hf_private)
{
return &hf_private->hf_public;
}

View File

@@ -0,0 +1,18 @@
#pragma once
struct http_half_private * hf_private_create(tfe_http_direction ht_dir, short major, short minor);
void hf_private_destory(struct http_half_private * hf_private);
/** Parse the raw tcp input for HTTP half structure
* @param hf_private
* hf_private handler.
* @param data
* raw tcp data, used in the stream's callback.
* @param len
* raw tcp data length.
* @return
* 0 for need more data,
* 1 for reach the boundary, need to call http business callbacks,
* -1 for error.
*/
int hf_private_parse(struct http_half_private * hf_private, const unsigned char * data, size_t len);

View File

@@ -1,79 +0,0 @@
#include <MESA/MESA_list_queue.h>
#include <tfe_stream.h>
#include <tfe_utils.h>
#include <tfe_plugin.h>
#include <tfe_http.h>
#include <http_parser.h>
extern "C"
{
#include <sys/queue.h>
}
struct http_plugin
{
};
struct http_plugin __g_http_plugin;
struct http_plugin * g_http_plugin = &__g_http_plugin;
int http_plugin_init(struct tfe_proxy * proxy)
{
return 0;
}
void http_plugin_deinit(struct tfe_proxy * proxy)
{
return;
}
TAILQ_HEAD(http_session_private_list, http_session_private);
struct http_session_private
{
TAILQ_ENTRY(http_session_private) s_next;
struct tfe_http_session s_public;
};
struct http_connection_private
{
struct layer_addr * layer_addr;
struct http_session_private_list session_private_list;
};
struct http_half_private
{
struct tfe_http_half head;
};
int http_connection_entry_open(const struct tfe_stream * stream, unsigned int thread_id,
enum tfe_conn_dir dir, void ** pme)
{
return 0;
}
enum tfe_stream_action http_connection_entry_data(const struct tfe_stream * stream, unsigned int thread_id,
enum tfe_conn_dir dir, const unsigned char * data, size_t len, void ** pme)
{
return ACTION_FORWARD_DATA;
}
void http_connection_entry_close(const struct tfe_stream * stream, unsigned int thread_id,
enum tfe_stream_close_reason reason, void ** pme)
{
}
static struct tfe_plugin __http_plugin_info =
{
.symbol = "HTTP",
.type = TFE_PLUGIN_TYPE_PROTOCOL,
.proto = APP_PROTO_HTTP1,
.on_init = http_plugin_init,
.on_deinit = http_plugin_deinit,
.on_open = http_connection_entry_open,
.on_data = http_connection_entry_data,
.on_close = http_connection_entry_close
};
TFE_PLUGIN_REGISTER(HTTP, __http_plugin_info)

View File

@@ -0,0 +1,196 @@
#include <MESA/MESA_list_queue.h>
#include <tfe_stream.h>
#include <tfe_utils.h>
#include <tfe_plugin.h>
#include <tfe_stream.h>
#include <http_parser.h>
#include <malloc.h>
#include <http_common.h>
#include <http_half.h>
#include <assert.h>
struct http_plugin __g_http_plugin;
struct http_plugin * g_http_plugin = &__g_http_plugin;
int http_plugin_init(struct tfe_proxy * proxy)
{
return 0;
}
void http_plugin_deinit(struct tfe_proxy * proxy)
{
return;
}
struct http_session_private * hs_private_create( struct http_connection_private * hc_private,
struct http_half_private * hf_private_req, struct http_half_private * hf_private_resp)
{
struct http_session_private * __hs_private = ALLOC(struct http_session_private, 1);
/* HS-PUBLIC */
__hs_private->hs_public.major_version = 1;
__hs_private->hs_public.req = hf_private_req != NULL ? to_hf_public(hf_private_req) : NULL;
__hs_private->hs_public.resp = hf_private_req != NULL ? to_hf_public(hf_private_resp) : NULL;
__hs_private->hs_public.session_id = hc_private->session_id_counter++;
/* HS-PRIVATE*/
__hs_private->hc_private = hc_private;
return __hs_private;
}
void hs_private_destory(struct http_session_private * hs_private)
{
free(hs_private);
}
static void __SET_PME_HC_PRIVATE(struct http_connection_private * h_conn, void ** pme)
{
*pme = (void *) h_conn;
}
static struct http_connection_private * __GET_PME_HC_PRIVATE(void ** pme)
{
return (struct http_connection_private *) (*pme);
}
int http_connection_entry_open(const struct tfe_stream * stream, unsigned int thread_id,
enum tfe_conn_dir dir, void ** pme)
{
struct http_connection_private * ht_conn = ALLOC(struct http_connection_private, 1);
TAILQ_INIT(&ht_conn->hs_private_list);
__SET_PME_HC_PRIVATE(ht_conn, pme);
return 0;
}
enum tfe_stream_action __http_connection_entry_on_request(const struct tfe_stream * stream,
struct http_connection_private * hc_private, unsigned int thread_id, const unsigned char * data, size_t len)
{
struct http_session_private * hs_private = TAILQ_LAST(&hc_private->hs_private_list, hs_private_list);
struct http_half_private * hf_private_request = NULL;
int ret = 0;
/* There is no available in session list,
* that indicate all HTTP request has corresponding response */
if (hs_private == NULL)
{
goto __new_session;
}
/* The last request is finished, we need to create a new session,
* or proceed parse content for last request */
hf_private_request = to_hf_request_private(hs_private);
if (hf_private_request->finished)
{
goto __new_session;
}
goto __parse;
__new_session:
hf_private_request = hf_private_create(TFE_HTTP_REQUEST, 1, 0);
hs_private = hs_private_create(hc_private, hf_private_request, NULL);
TAILQ_INSERT_TAIL(&hc_private->hs_private_list, hs_private, next);
__parse:
ret = hf_private_parse(hf_private_request, data, len);
/* Need more data, no boundary touched */
if (ret == 0)
{
return ACTION_DEFER_DATA;
}
/* Some kind of error happened, write log and detach the stream */
if (ret == -1)
{
TFE_STREAM_LOG_ERROR(stream, "Failed at parsing stream as HTTP: %u, %s, %s",
hf_private_request->parse_errno, http_errno_name(hf_private_request->parse_errno),
http_errno_description(hf_private_request->parse_errno));
tfe_stream_detach(stream);
return ACTION_FORWARD_DATA;
}
assert(ret == 1);
/* Touch a boundary, such as the end of HTTP headers, bodys, et al.
* need to call user's cb */
size_t __forward_bytes = hf_private_request->parse_cursor;
tfe_stream_action_set_opt(stream, ACTION_OPT_FOWARD_BYTES, &__forward_bytes, sizeof(__forward_bytes));
return ACTION_FORWARD_DATA;
}
enum tfe_stream_action __http_connection_entry_on_response(const struct tfe_stream * stream,
struct http_connection_private * ht_conn, unsigned int thread_id, const unsigned char * data, size_t len)
{
return ACTION_FORWARD_DATA;
}
int __http_connection_identify(const struct tfe_stream * stream,
struct http_connection_private * ht_conn, const unsigned char * data, size_t len)
{
struct http_half_private * hf_private = hf_private_create(TFE_HTTP_RESPONSE, 1, 0);
int ret = hf_private_parse(hf_private, data, len);
hf_private_destory(hf_private);
return ret;
}
#define HTTP_INDENTIFY_LENGTH 8
enum tfe_stream_action http_connection_entry_data(const struct tfe_stream * stream, unsigned int thread_id,
enum tfe_conn_dir dir, const unsigned char * data, size_t len, void ** pme)
{
struct http_connection_private * ht_conn = __GET_PME_HC_PRIVATE(pme);
if (ht_conn->is_preempted == 0)
{
/* If the server push response before client send request, this must not be HTTP, detach the stream */
if (dir == CONN_DIR_UPSTREAM)
goto __detach;
/* Protocol Identification, we need 8 bytes at least to tell it is HTTP or not */
if (len < HTTP_INDENTIFY_LENGTH)
{
static const unsigned int __defer_bytes = HTTP_INDENTIFY_LENGTH;
tfe_stream_action_set_opt(stream, ACTION_OPT_DEFER_BYTES, (void *) &__defer_bytes, sizeof(__defer_bytes));
return ACTION_DEFER_DATA;
}
/* Now, we want to identify this stream */
int ret = __http_connection_identify(stream, ht_conn, data, len);
if (ret != 0) goto __detach;
/* This is HTTP, try to preempt the stream
* It may be failed because other plugin preempted before us */
ret = tfe_stream_preempt(stream);
if (ret != 0) goto __detach;
}
/* This stream has been preempt, this plugin try to parse it */
return (dir == CONN_DIR_DOWNSTREAM) ? __http_connection_entry_on_request(stream, ht_conn, thread_id,
data, len) : __http_connection_entry_on_response(stream, ht_conn, thread_id, data, len);
__detach:
tfe_stream_detach(stream);
return ACTION_FORWARD_DATA;
}
void http_connection_entry_close(const struct tfe_stream * stream, unsigned int thread_id,
enum tfe_stream_close_reason reason, void ** pme)
{
struct http_connection_private * __ht_conn = __GET_PME_HC_PRIVATE(pme);
}
static struct tfe_plugin __http_plugin_info =
{
.symbol = "HTTP",
.type = TFE_PLUGIN_TYPE_PROTOCOL,
.proto = APP_PROTO_HTTP1,
.on_init = http_plugin_init,
.on_deinit = http_plugin_deinit,
.on_open = http_connection_entry_open,
.on_data = http_connection_entry_data,
.on_close = http_connection_entry_close
};
TFE_PLUGIN_REGISTER(HTTP, __http_plugin_info)

View File

@@ -0,0 +1,325 @@
#include <stdlib.h>
#include <tfe_http.h>
#include <tfe_utils.h>
#include <http_common.h>
#include <assert.h>
#include <string.h>
#include <math.h>
#include <sys/param.h>
#include <event2/buffer.h>
#define __PARSER_TO_HF_PRIVATE(_parser) ((struct http_half_private *)(_parser->data))
static const char * __str_std_header_field_map[] =
{
[TFE_HTTP_UNKNOWN_FIELD] = NULL,
[TFE_HTTP_HOST] = "Host",
[TFE_HTTP_REFERER] = "Referer",
[TFE_HTTP_USER_AGENT] = "User-Agent",
[TFE_HTTP_COOKIE] = "Cookie",
[TFE_HTTP_PROXY_AUTHORIZATION] = "Proxy-Authorization",
[TFE_HTTP_AUTHORIZATION] = "Authorization",
[TFE_HTTP_LOCATION] = "Location",
[TFE_HTTP_SERVER] = "Server",
[TFE_HTTP_ETAG] = "Etag",
[TFE_HTTP_DATE] = "Date",
[TFE_HTTP_TRAILER] = "Trailer",
[TFE_HTTP_TRANSFER_ENCODING] = "Transfer-Encoding",
[TFE_HTTP_VIA] = "Via",
[TFE_HTTP_PRAGMA] = "Pragma",
[TFE_HTTP_CONNECTION] = "Connection",
[TFE_HTTP_CONT_ENCODING] = "Content-Encoding",
[TFE_HTTP_CONT_LANGUAGE] = "Content-Language",
[TFE_HTTP_CONT_LOCATION] = "Content-Location",
[TFE_HTTP_CONT_RANGE] = "Content-Range",
[TFE_HTTP_CONT_LENGTH] = "Content-Length",
[TFE_HTTP_CONT_TYPE] = "Content-Type",
[TFE_HTTP_CONT_DISPOSITION] = "Content-Disposition",
[TFE_HTTP_EXPIRES] = "Expires",
[TFE_HTTP_ACCEPT_ENCODING] = "Accept-Encoding"
};
static enum tfe_http_std_field __str_header_field_to_std_field_id(const char * str_field, size_t len)
{
/* TODO: store the header text in hash table or rbtree, or use AC multistring search algo. */
for (int i = 0; i < TFE_DIM(__str_std_header_field_map); i++)
{
const char * __std_header_field = __str_std_header_field_map[i];
if (__std_header_field == NULL)
continue;
size_t __compare_length = MIN(strlen(__std_header_field), len);
if (evutil_ascii_strncasecmp(__std_header_field, str_field, __compare_length) != 0)
continue;
return (enum tfe_http_std_field) i;
}
return TFE_HTTP_UNKNOWN_FIELD;
}
/* To flush header field and value which stash in evbuffer */
static void __http_half_header_kv_complete(struct http_half_private * hf_private)
{
size_t sz_evbuf_field = evbuffer_get_length(hf_private->evbuf_header_field);
size_t sz_evbuf_value = evbuffer_get_length(hf_private->evbuf_header_value);
enum tfe_http_std_field std_field_id;
struct http_field_name compact_field;
static const char __zero = 0;
const char * str_field;
const char * str_value;
/* No header field or length of header field is zero, ignore this field-value pair. */
if (evbuffer_get_length(hf_private->evbuf_header_field) == 0)
{
goto __clear_buffer;
}
/* Write a '\0' for evbuffers */
evbuffer_add(hf_private->evbuf_header_field, &__zero, sizeof(__zero));
evbuffer_add(hf_private->evbuf_header_value, &__zero, sizeof(__zero));
/* Convert evbuffer to const char * pair */
str_field = (const char *) evbuffer_pullup(hf_private->evbuf_header_field, sz_evbuf_field);
str_value = (const char *) evbuffer_pullup(hf_private->evbuf_header_value, sz_evbuf_value);
std_field_id = __str_header_field_to_std_field_id(str_field, sz_evbuf_field);
if (std_field_id != TFE_HTTP_UNKNOWN_FIELD)
{
compact_field.field_id = std_field_id;
compact_field.field_name = NULL;
}
else
{
compact_field.field_id = TFE_HTTP_UNKNOWN_FIELD;
compact_field.field_name = (char *) str_field;
}
tfe_http_field_write(&hf_private->hf_public, &compact_field, str_value);
goto __clear_buffer;
__clear_buffer:
evbuffer_drain(hf_private->evbuf_header_field, sz_evbuf_field);
evbuffer_drain(hf_private->evbuf_header_value, sz_evbuf_value);
}
void __hf_public_req_fill_from_private(struct http_half_private * hf_private, struct http_parser * parser)
{
struct tfe_http_half * hf_public = &hf_private->hf_public;
struct tfe_http_req_spec * hf_req_spec = &hf_public->req_spec;
/* accept-encoding, host is located in header's K-V structure */
hf_req_spec->method = parser->method;
const static struct http_field_name __accept_encoding_field_name =
{
.field_id = TFE_HTTP_ACCEPT_ENCODING,
.field_name = NULL
};
const static struct http_field_name __host_field_name =
{
.field_id = TFE_HTTP_HOST,
.field_name = NULL
};
hf_req_spec->accept_encoding = (char *)tfe_http_field_read(hf_public, &__accept_encoding_field_name);
hf_req_spec->host = (char *)tfe_http_field_read(hf_public, &__host_field_name);
/* uri is stored in underlay evbuffer, we need to append a terminal zero */
static const char __zero = 0;
evbuffer_add(hf_private->evbuf_uri, &__zero, sizeof(__zero));
hf_req_spec->uri = (char *)evbuffer_pullup(hf_private->evbuf_uri, evbuffer_get_length(hf_private->evbuf_uri));
/* TODO: URL
* url is more complex. need to review RFC */
hf_req_spec->url = NULL;
}
void __hf_public_resp_fill_from_private(struct http_half_private * hf_private, struct http_parser * parser)
{
struct tfe_http_half * hf_public = &hf_private->hf_public;
struct tfe_http_resp_spec * hf_resp_spec = &hf_public->resp_spec;
/* Status Code */
hf_resp_spec->resp_code = parser->status_code;
const static struct http_field_name __cont_encoding_field_name =
{
.field_id = TFE_HTTP_CONT_ENCODING,
.field_name = NULL
};
/* Content Encoding */
hf_resp_spec->content_encoding = (char *)tfe_http_field_read(hf_public, &__cont_encoding_field_name);
}
/* ==================================================================================================================
* REQUEST PARSER CALLBACKS
* ================================================================================================================== */
static int __parser_callback_on_message_begin(struct http_parser * parser)
{
struct http_half_private * hf_private = __PARSER_TO_HF_PRIVATE(parser);
enum tfe_http_direction direction = hf_private->hf_public.direction;
assert(hf_private->evbuf_uri == NULL && hf_private->evbuf_body == NULL);
assert(hf_private->evbuf_header_field == NULL && hf_private->evbuf_header_value == NULL);
hf_private->evbuf_uri = evbuffer_new();
hf_private->evbuf_header_field = evbuffer_new();
hf_private->evbuf_header_value = evbuffer_new();
hf_private->evbuf_body = evbuffer_new();
return 0;
}
static int __parser_callback_on_uri_field(struct http_parser * parser, const char * at, size_t length)
{
struct http_half_private * hf_private = __PARSER_TO_HF_PRIVATE(parser);
return evbuffer_add(hf_private->evbuf_uri, at, length);
}
static int __parser_callback_on_header_field(struct http_parser * parser, const char * at, size_t length)
{
struct http_half_private * hf_private = __PARSER_TO_HF_PRIVATE(parser);
/* Last field-value tuple doesn't push into hf_private, flush these */
if (evbuffer_get_length(hf_private->evbuf_header_field) != 0)
{
__http_half_header_kv_complete(hf_private);
}
return evbuffer_add(hf_private->evbuf_header_field, at, length);
}
static int __parser_callback_on_header_value(struct http_parser * parser, const char * at, size_t length)
{
struct http_half_private * hf_private = __PARSER_TO_HF_PRIVATE(parser);
return evbuffer_add(hf_private->evbuf_header_value, at, length);
}
static int __parser_callback_on_body(struct http_parser * parser, const char * at, size_t length)
{
struct http_half_private * hf_private = __PARSER_TO_HF_PRIVATE(parser);
return evbuffer_add(hf_private->evbuf_body, at, length);
}
static int __parser_callback_on_headers_complete(http_parser * parser)
{
struct http_half_private * hf_private = __PARSER_TO_HF_PRIVATE(parser);
enum tfe_http_direction hf_direction = hf_private->hf_public.direction;
if (evbuffer_get_length(hf_private->evbuf_header_field) != 0)
{
__http_half_header_kv_complete(hf_private);
}
struct tfe_http_half * hf_public = &hf_private->hf_public;
hf_public->major_version = parser->http_major;
hf_public->minor_version = parser->http_minor;
if (hf_direction == TFE_HTTP_REQUEST)
{
__hf_public_req_fill_from_private(hf_private, parser);
}
else
{
__hf_public_resp_fill_from_private(hf_private, parser);
}
http_parser_pause(parser, 1);
return 0;
}
static int __parser_callback_on_message_complete(http_parser * parser)
{
struct http_half_private * hf_private = __PARSER_TO_HF_PRIVATE(parser);
http_parser_pause(parser, 1);
return 0;
}
static http_parser_settings __http_half_parse_setting =
{
.on_message_begin = __parser_callback_on_message_begin,
.on_url = __parser_callback_on_uri_field,
.on_status = NULL,
.on_header_field = __parser_callback_on_header_field,
.on_header_value = __parser_callback_on_header_value,
.on_headers_complete = __parser_callback_on_headers_complete,
.on_body = __parser_callback_on_body,
.on_message_complete = __parser_callback_on_message_complete,
.on_chunk_header = NULL,
.on_chunk_complete = NULL
};
struct http_half_private * hf_private_create(tfe_http_direction ht_dir, short major, short minor)
{
struct http_half_private * hf_private = ALLOC(struct http_half_private, 1);
assert(hf_private != NULL && (major == 0 || major == 1) && (minor == 0 || minor == 1));
assert(ht_dir == TFE_HTTP_REQUEST || ht_dir == TFE_HTTP_RESPONSE);
/* PUBLIC */
hf_private->hf_public.major_version = major;
hf_private->hf_public.minor_version = minor;
hf_private->hf_public.direction = ht_dir;
/* PRIVATE */
hf_private->parse_object = (struct http_parser *) malloc(sizeof(struct http_parser));
assert(hf_private->parse_object != NULL);
if (ht_dir == TFE_HTTP_REQUEST)
{
http_parser_init(hf_private->parse_object, HTTP_REQUEST);
}
else
{
http_parser_init(hf_private->parse_object, HTTP_RESPONSE);
}
hf_private->parse_settings = &__http_half_parse_setting;
hf_private->parse_object->data = hf_private;
return hf_private;
}
void hf_private_destory(struct http_half_private * hf_private)
{
if (hf_private->parse_object != NULL) free(hf_private->parse_object);
free(hf_private);
}
int hf_private_parse(struct http_half_private * hf_private, const unsigned char * data, size_t len)
{
assert(hf_private->parse_cursor <= len);
int ret = 0;
/* Caculate the memory zones to scan. The zone from data to data + cursor has been scaned
* at last construct procedure, so we don't need to scan again. */
const char * __data_with_offset = (const char *) TFE_PTR_ADD(data, hf_private->parse_cursor);
size_t __len_with_offset = len - hf_private->parse_cursor;
/* Scan the memory zone */
size_t sz_parsed = http_parser_execute(hf_private->parse_object,
&__http_half_parse_setting, __data_with_offset, __len_with_offset);
/* The paused parsar indicate that some kind of boundary has been touched,
* we should return and call user's data callback. resume it to normal status */
if (sz_parsed && HTTP_PARSER_ERRNO(hf_private->parse_object) == HPE_PAUSED)
{
http_parser_pause(hf_private->parse_object, 0);
ret = 1;
goto __out;
}
/* Some kind of exception happend */
if (sz_parsed && HTTP_PARSER_ERRNO(hf_private->parse_object) > 0)
{
hf_private->parse_errno = HTTP_PARSER_ERRNO(hf_private->parse_object);
ret = -1;
goto __out;
}
__out:
hf_private->parse_cursor += sz_parsed;
return ret;
}