diff --git a/decoders/http/CMakeLists.txt b/decoders/http/CMakeLists.txt index bd45e6e..c242afe 100644 --- a/decoders/http/CMakeLists.txt +++ b/decoders/http/CMakeLists.txt @@ -1,8 +1,8 @@ include_directories(${CMAKE_SOURCE_DIR}/deps) -set(HTTP_SRC http_decoder.cpp http_decoder_utils.cpp http_decoder_half.cpp - http_decoder_table.cpp http_decoder_string.cpp http_content_decompress.cpp - http_decoder_result_queue.cpp http_decoder_stat.cpp http_decoder_tunnel.cpp) +set(HTTP_SRC http_decoder.c http_decoder_utils.c http_decoder_half.c + http_decoder_table.c http_decoder_string.c http_content_decompress.c + http_decoder_result_queue.c http_decoder_stat.c http_decoder_tunnel.c) add_library(http STATIC ${HTTP_SRC}) add_library(http_dyn SHARED ${HTTP_SRC}) diff --git a/decoders/http/http_content_decompress.cpp b/decoders/http/http_content_decompress.c similarity index 77% rename from decoders/http/http_content_decompress.cpp rename to decoders/http/http_content_decompress.c index cb90811..922c624 100644 --- a/decoders/http/http_content_decompress.cpp +++ b/decoders/http/http_content_decompress.c @@ -15,17 +15,22 @@ struct http_content_decompress size_t buffer_size; }; -enum http_content_encoding http_content_encoding_str2int(const char *content_encoding) +void http_content_decompress_ownership_borrow(struct http_content_decompress *decompress) { - if (strcasestr(content_encoding, "gzip") != NULL) + decompress->buffer = NULL; // ownership move to data->decompress_buffer_list, will be freed when message has been processed by all plugins +} + +enum http_content_encoding http_content_encoding_str2int(const char *content_encoding, size_t encoding_str_len) +{ + if (http_strncasecmp_safe("gzip", content_encoding, 4, encoding_str_len) == 0) { return HTTP_CONTENT_ENCODING_GZIP; } - if (strcasestr(content_encoding, "deflate") != NULL) + if (http_strncasecmp_safe("deflate", content_encoding, 7, encoding_str_len) == 0) { return HTTP_CONTENT_ENCODING_DEFLATE; } - if (strcasestr(content_encoding, "br") != NULL) + if (http_strncasecmp_safe("br", content_encoding, 2, encoding_str_len) == 0) { return HTTP_CONTENT_ENCODING_BR; } @@ -128,11 +133,12 @@ static int http_content_decompress_write_zlib(struct http_content_decompress *de z_stream *z_stream_ptr = decompress->z_stream_ptr; z_stream_ptr->avail_in = (unsigned int)indata_len; z_stream_ptr->next_in = (unsigned char *)indata; - z_stream_ptr->avail_out = (unsigned int)decompress->buffer_size; + z_stream_ptr->avail_out = (unsigned int)HTTP_DECOMPRESS_BUFFER_SIZE; z_stream_ptr->next_out = (unsigned char *)decompress->buffer; - *outdata = NULL; *outdata_len = 0; + size_t total_have = 0; + int no_buffer; do { @@ -142,37 +148,36 @@ static int http_content_decompress_write_zlib(struct http_content_decompress *de (void)inflateEnd(z_stream_ptr); return -1; } - - size_t have = decompress->buffer_size - z_stream_ptr->avail_out; + size_t have = HTTP_DECOMPRESS_BUFFER_SIZE - z_stream_ptr->avail_out; if (have > 0) { + total_have += have; if (0 == z_stream_ptr->avail_out) { - decompress->buffer_size += have; - decompress->buffer = REALLOC(char, decompress->buffer, - decompress->buffer_size); + decompress->buffer_size += HTTP_DECOMPRESS_BUFFER_SIZE; + decompress->buffer = REALLOC(char, decompress->buffer, decompress->buffer_size); + z_stream_ptr->avail_out = HTTP_DECOMPRESS_BUFFER_SIZE; + z_stream_ptr->next_out = (unsigned char *)decompress->buffer + total_have; *outdata = decompress->buffer; - *outdata_len = *outdata_len + have; - // http_decoder_log(DEBUG, "%s realloc outbuffer %zu bytes", - // http_content_encoding_int2str(decompress->encoding), - // decompress->buffer_size); - z_stream_ptr->avail_out = have; - z_stream_ptr->next_out = (unsigned char *)decompress->buffer + - (decompress->buffer_size - have); + *outdata_len = total_have; + no_buffer = 1; } else { *outdata = decompress->buffer; - *outdata_len = have; + *outdata_len = total_have; + no_buffer = 0; } } + else + { + break; + } if (Z_STREAM_END == ret) { break; } - } while (z_stream_ptr->avail_in != 0); - decompress->buffer = NULL; - decompress->buffer_size = 0; + } while (no_buffer == 1); return 0; } @@ -182,56 +187,52 @@ static int http_content_decompress_write_br(struct http_content_decompress *deco { size_t available_in = indata_len; const unsigned char *next_in = (const unsigned char *)indata; - size_t available_out = decompress->buffer_size; + size_t available_out = HTTP_DECOMPRESS_BUFFER_SIZE; unsigned char *next_out = (unsigned char *)decompress->buffer; *outdata = NULL; *outdata_len = 0; + size_t total_have = 0; + int no_buffer; - for (;;) + do { int ret = BrotliDecoderDecompressStream(decompress->br_state, &available_in, - &next_in, &available_out, &next_out, 0); - size_t have = decompress->buffer_size - available_out; + &next_in, &available_out, &next_out, NULL); + if (ret == BROTLI_DECODER_RESULT_ERROR) + { + // BrotliDecoderErrorCode errcode = BrotliDecoderGetErrorCode(decompress->br_state); + *outdata = NULL; + *outdata_len = 0; + return -1; + } + size_t have = HTTP_DECOMPRESS_BUFFER_SIZE - available_out; if (have > 0) { + total_have += have; if (0 == available_out) { - decompress->buffer_size += have; - decompress->buffer = REALLOC(char, decompress->buffer, - decompress->buffer_size); + decompress->buffer_size += HTTP_DECOMPRESS_BUFFER_SIZE; + decompress->buffer = REALLOC(char, decompress->buffer, decompress->buffer_size); + available_out = HTTP_DECOMPRESS_BUFFER_SIZE; + next_out = (unsigned char *)decompress->buffer + total_have; *outdata = decompress->buffer; - *outdata_len = *outdata_len + have; - available_out = have; - next_out = (unsigned char *)decompress->buffer + - (decompress->buffer_size - have); + *outdata_len = total_have; + no_buffer = 1; } else { *outdata = decompress->buffer; *outdata_len = have; + no_buffer = 0; } } - - if (ret == BROTLI_DECODER_RESULT_SUCCESS || - ret == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) + else { - decompress->buffer = NULL; - decompress->buffer_size = 0; - return 0; + break; } - - if (ret == BROTLI_DECODER_RESULT_ERROR) - { - // BrotliDecoderErrorCode errcode = - BrotliDecoderGetErrorCode(decompress->br_state); - *outdata = NULL; - *outdata_len = 0; - return -1; - } - - assert(ret == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT); - } + } while (no_buffer == 1); + return 0; } int http_content_decompress_write(struct http_content_decompress *decompress, @@ -243,7 +244,6 @@ int http_content_decompress_write(struct http_content_decompress *decompress, assert(indata_len > 0); assert(outdata); assert(outdata_len); - *outdata = NULL; *outdata_len = 0; diff --git a/decoders/http/http_content_decompress.h b/decoders/http/http_content_decompress.h index 7f2c221..2ac4689 100644 --- a/decoders/http/http_content_decompress.h +++ b/decoders/http/http_content_decompress.h @@ -17,7 +17,7 @@ extern "C" struct http_content_decompress; - enum http_content_encoding http_content_encoding_str2int(const char *content_encoding); + enum http_content_encoding http_content_encoding_str2int(const char *content_encoding, size_t encoding_str_len); const char *http_content_encoding_int2str(enum http_content_encoding content_encoding); @@ -30,7 +30,7 @@ extern "C" int http_content_decompress_write(struct http_content_decompress *decompress, const char *indata, size_t indata_len, char **outdata, size_t *outdata_len); - + void http_content_decompress_ownership_borrow(struct http_content_decompress *decompress); #ifdef __cplusplus } #endif diff --git a/decoders/http/http_decoder.cpp b/decoders/http/http_decoder.c similarity index 92% rename from decoders/http/http_decoder.cpp rename to decoders/http/http_decoder.c index f1c65f7..e30f459 100644 --- a/decoders/http/http_decoder.cpp +++ b/decoders/http/http_decoder.c @@ -159,9 +159,10 @@ static void http_event_handler(enum http_event event, struct http_decoder_half_d int tot_c2s_headers = http_half_data_get_total_parsed_header_count(half_data); http_decoder_stat_update(&httpd_env->hd_stat, thread_id, HTTP_C2S_HEADERS, tot_c2s_headers); - hstring tmp_url = {}; - http_half_data_get_url(half_data, &tmp_url); - http_decoder_stat_update(&httpd_env->hd_stat, thread_id, HTTP_URL_BYTES, tmp_url.iov_len); + const char *tmp_url = NULL; + size_t tmp_url_len = 0; + http_half_data_get_url(half_data, &tmp_url, &tmp_url_len); + http_decoder_stat_update(&httpd_env->hd_stat, thread_id, HTTP_URL_BYTES, tmp_url_len); } break; case HTTP_EVENT_REQ_BODY_BEGIN: @@ -172,7 +173,7 @@ static void http_event_handler(enum http_event event, struct http_decoder_half_d { hstring raw_body = {}; hstring decompress_body = {}; - http_decoder_half_data_get_raw_body(half_data, &raw_body); + http_decoder_half_data_get_raw_body(half_data, (const char **)&raw_body.iov_base, &raw_body.iov_len); http_half_get_lastest_decompress_buffer(half_data, &decompress_body); msg = http_body_message_new(HTTP_MESSAGE_REQ_BODY, queue, queue_idx, HTTP_REQUEST, &raw_body, &decompress_body); session_mq_publish_message(ev_ctx->ref_session, exdata->pub_topic_id, msg); @@ -303,7 +304,7 @@ static void http_event_handler(enum http_event event, struct http_decoder_half_d case HTTP_EVENT_RES_BODY_DATA: { hstring raw_body = {}; - http_decoder_half_data_get_raw_body(half_data, &raw_body); + http_decoder_half_data_get_raw_body(half_data, (const char **)&raw_body.iov_base, &raw_body.iov_len); hstring decompress_body = {}; http_half_get_lastest_decompress_buffer(half_data, &decompress_body); msg = http_body_message_new(HTTP_MESSAGE_RES_BODY, queue, queue_idx, HTTP_RESPONSE, &raw_body, &decompress_body); @@ -536,31 +537,31 @@ static int load_http_decoder_config(const char *cfg_path, return ret; } -static int http_msg_get_request_header(const struct http_message *msg, const hstring *key, - struct http_header *hdr_result) +static int http_msg_get_request_header(const struct http_message *msg, const char *name, size_t name_len, + struct http_header_field *hdr_result) { const struct http_decoder_half_data *req_data = msg->ref_queue->array[msg->queue_index].req_data; - return http_decoder_half_data_get_header(req_data, key, hdr_result); + return http_decoder_half_data_get_header(req_data, name, name_len, hdr_result); } -static int http_msg_get_response_header(const struct http_message *msg, const hstring *key, - struct http_header *hdr_result) +static int http_msg_get_response_header(const struct http_message *msg, const char *name, size_t name_len, + struct http_header_field *hdr_result) { const struct http_decoder_half_data *res_data = msg->ref_queue->array[msg->queue_index].res_data; - return http_decoder_half_data_get_header(res_data, key, hdr_result); + return http_decoder_half_data_get_header(res_data, name, name_len, hdr_result); } static int http_msg_request_header_next(const struct http_message *msg, - struct http_header *hdr) + struct http_header_field *hdr) { const struct http_decoder_half_data *req_data = msg->ref_queue->array[msg->queue_index].req_data; return http_decoder_half_data_iter_header((struct http_decoder_half_data *)req_data, hdr); } -static int http_msg_response_header_next(const struct http_message *msg, struct http_header *hdr) +static int http_msg_response_header_next(const struct http_message *msg, struct http_header_field *hdr) { const struct http_decoder_half_data *res_data = msg->ref_queue->array[msg->queue_index].res_data; @@ -656,7 +657,7 @@ extern "C" http_decoder_stat_update(&httpd_env->hd_stat, thread_id, HTTP_SESSION_FREE, 1); } - static void http_decoder_execute(struct session *sess, struct http_decoder_env *httpd_env, http_decoder_exdata *exdata, const char *payload, uint16_t payload_len) + static void http_decoder_execute(struct session *sess, struct http_decoder_env *httpd_env, struct http_decoder_exdata *exdata, const char *payload, uint16_t payload_len) { if (httpd_in_tunnel_transmitting(httpd_env, exdata)) { @@ -820,7 +821,7 @@ extern "C" static const struct http_topic_exdata_compose g_topic_exdata_compose[HTTPD_TOPIC_INDEX_MAX] = { {HTTPD_TOPIC_TCP_STREAM_INDEX, TOPIC_TCP_STREAM, http_decoder_tcp_stream_msg_cb, NULL, "HTTP_DECODER_EXDATA_BASEON_TCP_STREAM", httpd_ex_data_free_cb, -1, -1}, - {HTTPD_TOPIC_HTTP_MSG_INDEX, HTTP_DECODER_TOPIC, NULL, http_message_free, NULL, NULL, -1, -1}, + {HTTPD_TOPIC_HTTP_MSG_INDEX, HTTP_TOPIC, NULL, http_message_free, NULL, NULL, -1, -1}, {HTTPD_TOPIC_HTTP_TUNNEL_INDEX, HTTP_DECODER_TUNNEL_TOPIC, http_decoder_tunnel_msg_cb, http_message_free, "HTTP_DECODER_EXDATA_BASEON_HTTP_TUNNEL", httpd_ex_data_free_cb, -1, -1}, }; @@ -910,7 +911,7 @@ extern "C" _http_decoder_context_free(httpd_env); } - enum http_message_type http_message_type_get(const struct http_message *msg) + enum http_message_type http_message_get_type(const struct http_message *msg) { if (unlikely(NULL == msg)) { @@ -919,16 +920,16 @@ extern "C" return msg->type; } - void http_message_request_line_get0(const struct http_message *msg, + void http_message_get0_request_line(const struct http_message *msg, struct http_request_line *line) { if (unlikely(NULL == msg || msg->type != HTTP_MESSAGE_REQ_LINE)) { if (line) { - line->method.iov_base = NULL; - line->uri.iov_base = NULL; - line->version.iov_base = NULL; + line->method = NULL; + line->uri = NULL; + line->version = NULL; } return; } @@ -941,15 +942,15 @@ extern "C" http_decoder_half_data_get_request_line(req_data, line); } - void http_message_response_line_get0(const struct http_message *msg, + void http_message_get0_response_line(const struct http_message *msg, struct http_response_line *line) { if (unlikely(NULL == msg || msg->type != HTTP_MESSAGE_RES_LINE)) { if (line) { - line->version.iov_base = NULL; - line->status.iov_base = NULL; + line->version = NULL; + line->status = NULL; } return; } @@ -962,11 +963,11 @@ extern "C" http_decoder_half_data_get_response_line(res_data, line); } - void http_message_header_get0(const struct http_message *msg, const hstring *key, - struct http_header *hdr_result) + void http_message_get0_header(const struct http_message *msg, const char *name, size_t name_len, + struct http_header_field *hdr_result) { int ret = -1; - if (unlikely(NULL == msg || NULL == key)) + if (unlikely(NULL == msg || NULL == name || 0 == name_len)) { goto fail; } @@ -974,11 +975,11 @@ extern "C" assert(msg->queue_index < HD_RESULT_QUEUE_LEN); if (HTTP_MESSAGE_REQ_HEADER == msg->type) { - ret = http_msg_get_request_header(msg, key, hdr_result); + ret = http_msg_get_request_header(msg, name, name_len, hdr_result); } else if (HTTP_MESSAGE_RES_HEADER == msg->type) { - ret = http_msg_get_response_header(msg, key, hdr_result); + ret = http_msg_get_response_header(msg, name, name_len, hdr_result); } if (ret >= 0) { @@ -987,13 +988,13 @@ extern "C" fail: if (hdr_result) { - hdr_result->key.iov_base = NULL; - hdr_result->val.iov_base = NULL; + hdr_result->name = NULL; + hdr_result->value = NULL; } return; } - int http_message_header_next(const struct http_message *msg, struct http_header *header) + int http_message_get0_next_header(const struct http_message *msg, struct http_header_field *header) { int ret = 1; if (unlikely(NULL == msg)) @@ -1018,8 +1019,8 @@ extern "C" fail: if (header) { - header->key.iov_base = NULL; - header->val.iov_base = NULL; + header->name = NULL; + header->value = NULL; } return -1; } @@ -1047,7 +1048,7 @@ extern "C" return -1; } - void http_message_raw_body_get0(const struct http_message *msg, hstring *body) + void http_message_get0_uncompressed_body(const struct http_message *msg, const char **body_ptr, size_t *body_len) { if (unlikely(NULL == msg)) { @@ -1057,20 +1058,20 @@ extern "C" assert(msg->queue_index < HD_RESULT_QUEUE_LEN); if (msg->raw_payload.iov_base != NULL && msg->raw_payload.iov_len != 0) { - body->iov_base = msg->raw_payload.iov_base; - body->iov_len = msg->raw_payload.iov_len; + *body_ptr = msg->raw_payload.iov_base; + *body_len = msg->raw_payload.iov_len; return; } fail: - if (body) + if (body_ptr) { - body->iov_base = NULL; - body->iov_len = 0; + *body_ptr = NULL; + *body_len = 0; } return; } - void http_message_decompress_body_get0(const struct http_message *msg, hstring *decompress_body) + void http_message_get0_decompressed_body(const struct http_message *msg, const char **dec_body_ptr, size_t *dec_body_len) { enum http_content_encoding ecode = HTTP_CONTENT_ENCODING_NONE; struct http_decoder_half_data *ref_data = NULL; @@ -1082,12 +1083,12 @@ extern "C" assert(msg->queue_index < HD_RESULT_QUEUE_LEN); if (msg->decompress_payload.iov_base != NULL && msg->decompress_payload.iov_len != 0) { - decompress_body->iov_base = msg->decompress_payload.iov_base; - decompress_body->iov_len = msg->decompress_payload.iov_len; + *dec_body_ptr = msg->decompress_payload.iov_base; + *dec_body_len = msg->decompress_payload.iov_len; return; } /** - * @brief If the body hasn't been compressed, same as http_message_raw_body_get0(). + * @brief If the body hasn't been compressed, same as http_message_get0_uncompressed_body(). * */ @@ -1107,51 +1108,46 @@ extern "C" if (msg->raw_payload.iov_base != NULL && msg->raw_payload.iov_len != 0) { - decompress_body->iov_base = msg->raw_payload.iov_base; - decompress_body->iov_len = msg->raw_payload.iov_len; + *dec_body_ptr = msg->raw_payload.iov_base; + *dec_body_len = msg->raw_payload.iov_len; } return; fail: - if (decompress_body) + if (dec_body_ptr) { - decompress_body->iov_base = NULL; - decompress_body->iov_len = 0; + *dec_body_ptr = NULL; + *dec_body_len = 0; } return; } - void http_message_raw_url_get0(const struct http_message *msg, hstring *url) + void http_message_get0_raw_url(const struct http_message *msg, const char **url_val, size_t *url_len) { + struct http_decoder_half_data *req_data = NULL; if (unlikely(NULL == msg)) { - if (url) - { - url->iov_base = NULL; - url->iov_len = 0; - } - return; + goto fail; } assert(msg->ref_queue); assert(msg->queue_index < HD_RESULT_QUEUE_LEN); - struct http_decoder_half_data *req_data = - msg->ref_queue->array[msg->queue_index].req_data; + req_data = msg->ref_queue->array[msg->queue_index].req_data; - if (http_half_data_get_url(req_data, url) < 0) + if (http_half_data_get_url(req_data, url_val, url_len) < 0) { goto fail; } return; fail: - if (url) + if (url_val) { - url->iov_base = NULL; - url->iov_len = 0; + *url_val = NULL; + *url_len = 0; } return; } - +#if 0 void http_message_decoded_url_get0(const struct http_message *msg, struct iovec *url) { if (unlikely(NULL == msg)) @@ -1183,7 +1179,7 @@ extern "C" } return; } - +#endif int http_message_get_transaction_seq(const struct http_message *msg) { if (unlikely(NULL == msg)) diff --git a/decoders/http/http_decoder_half.cpp b/decoders/http/http_decoder_half.c similarity index 89% rename from decoders/http/http_decoder_half.cpp rename to decoders/http/http_decoder_half.c index a505fac..70991c6 100644 --- a/decoders/http/http_decoder_half.cpp +++ b/decoders/http/http_decoder_half.c @@ -33,7 +33,8 @@ struct http_decoder_half_data #endif int joint_url_complete; int url_is_encoded; - hstring joint_url; // http://[:]/? + // http://[:]/? + hstring joint_url; hstring decoded_url; long long transaction_index; }; @@ -67,7 +68,7 @@ static void printf_debug_info(const char *desc, const char *at, size_t length) { if (at) { - char *temp = safe_dup(at, length); + char *temp = http_safe_dup(at, length); printf("HTTP PARSER STAGE: %s: %s\n", desc, temp); FREE(temp); } @@ -131,7 +132,7 @@ static void http_decoder_half_data_decompress(struct http_decoder_half_data *dat } hstring raw_body = {}; - http_decoder_table_get_body(data->table, &raw_body); + http_decoder_table_get_body(data->table, (char **)&raw_body.iov_base, &raw_body.iov_len); if (raw_body.iov_base == NULL || raw_body.iov_len == 0) { return; @@ -163,6 +164,7 @@ static void http_decoder_half_data_decompress(struct http_decoder_half_data *dat decompress_buffer->iov.iov_base = local_outdata; decompress_buffer->iov.iov_len = local_outdata_len; DL_APPEND(data->decompress_buffer_list, decompress_buffer); + http_content_decompress_ownership_borrow(data->decompress); } } @@ -327,7 +329,7 @@ static int on_uri_complete(llhttp_t *http) http_decoder_table_commit(half->ref_data->table, HTTP_ITEM_URI); hstring uri_result = {}; - http_decoder_table_get_uri(half->ref_data->table, &uri_result); + http_decoder_table_get_uri(half->ref_data->table, (char **)&uri_result.iov_base, &uri_result.iov_len); assert(uri_result.iov_base); http_decoder_cached_portion_url(half, &uri_result); @@ -472,19 +474,11 @@ static int on_header_value_complete(llhttp_t *http) if (half->ref_data->content_encoding == HTTP_CONTENT_ENCODING_NONE) { - struct http_header http_hdr = {}; - hstring key = {.iov_base = (char *)"Content-Encoding", .iov_len = 16}; + struct http_header_field http_hdr = {}; - if (http_decoder_table_get_header(half->ref_data->table, &key, &http_hdr) == 0) + if (http_decoder_table_get_header(half->ref_data->table, (char *)"Content-Encoding", 16, &http_hdr) == 0) { - char encoding_str[MAX_ENCODING_STR_LEN + 1] = {0}; - size_t iov_len = http_hdr.val.iov_len; - if (iov_len > MAX_ENCODING_STR_LEN) - { - iov_len = MAX_ENCODING_STR_LEN; - } - memcpy(encoding_str, http_hdr.val.iov_base, iov_len); - half->ref_data->content_encoding = http_content_encoding_str2int(encoding_str); + half->ref_data->content_encoding = http_content_encoding_str2int(http_hdr.value, http_hdr.value_len); } } @@ -910,9 +904,9 @@ void http_decoder_half_data_free(nmx_pool_t *mempool, struct http_decoder_half_d int http_decoder_half_data_get_request_line(struct http_decoder_half_data *data, struct http_request_line *line) { - http_decoder_table_get_method(data->table, &line->method); - http_decoder_table_get_uri(data->table, &line->uri); - http_decoder_table_get_version(data->table, &line->version); + http_decoder_table_get_method(data->table, &line->method, &line->method_len); + http_decoder_table_get_uri(data->table, &line->uri, &line->uri_len); + http_decoder_table_get_version(data->table, &line->version, &line->version_len); line->major_version = data->major_version; line->minor_version = data->minor_version; @@ -923,8 +917,8 @@ int http_decoder_half_data_get_request_line(struct http_decoder_half_data *data, int http_decoder_half_data_get_response_line(struct http_decoder_half_data *data, struct http_response_line *line) { - http_decoder_table_get_version(data->table, &line->version); - http_decoder_table_get_status(data->table, &line->status); + http_decoder_table_get_version(data->table, &line->version, &line->version_len); + http_decoder_table_get_status(data->table, &line->status, &line->status_len); line->major_version = data->major_version; line->minor_version = data->minor_version; @@ -934,14 +928,14 @@ int http_decoder_half_data_get_response_line(struct http_decoder_half_data *data } int http_decoder_half_data_get_header(const struct http_decoder_half_data *data, - const hstring *key, - struct http_header *hdr_result) + const char *name, size_t name_len, + struct http_header_field *hdr_result) { - return http_decoder_table_get_header(data->table, key, hdr_result); + return http_decoder_table_get_header(data->table, name, name_len, hdr_result); } int http_decoder_half_data_iter_header(struct http_decoder_half_data *data, - struct http_header *header) + struct http_header_field *header) { return http_decoder_table_iter_header((struct http_decoder_table *)data->table, header); } @@ -964,13 +958,13 @@ int http_decoder_half_data_has_parsed_header(struct http_decoder_half_data *data return http_decoder_table_has_parsed_header(data->table); } -int http_decoder_half_data_get_raw_body(const struct http_decoder_half_data *data, hstring *body) +int http_decoder_half_data_get_raw_body(const struct http_decoder_half_data *data, const char **body, size_t *body_len) { if (NULL == data || NULL == body) { return -1; } - return http_decoder_table_get_body(data->table, body); + return http_decoder_table_get_body(data->table, (char **)body, body_len); } #if 0 int http_decoder_half_data_get_decompress_body(const struct http_decoder_half_data *data, hstring *body) @@ -996,34 +990,33 @@ void http_decoder_half_data_dump(struct http_decoder_half *half) http_decoder_table_dump(half->ref_data->table); } -static void using_session_addr_as_host(struct session *ref_session, - struct http_header *host_result, nmx_pool_t *mempool) +static void using_session_addr_as_host(struct session *ref_session, struct http_header_field *host_result, nmx_pool_t *mempool) { #if 1 // in native steallar, can't get the tuple4 from the session yet!!! struct httpd_session_addr ssaddr = {}; httpd_session_get_addr(ref_session, &ssaddr); if (ssaddr.ipver != 4 && ssaddr.ipver != 6) { - host_result->val.iov_base = MEMPOOL_CALLOC(mempool, char, 1); - sprintf((char *)host_result->val.iov_base, "%s", ""); - host_result->val.iov_len = strlen((char *)host_result->val.iov_base); + host_result->value = MEMPOOL_CALLOC(mempool, char, 1); + sprintf((char *)host_result->value, "%s", ""); + host_result->value_len = strlen((char *)host_result->value); return; } char ip_string_buf[INET6_ADDRSTRLEN]; if (4 == ssaddr.ipver) { - host_result->val.iov_base = MEMPOOL_CALLOC(mempool, char, (INET_ADDRSTRLEN + 7) /* "ip:port" max length */); + host_result->value = MEMPOOL_CALLOC(mempool, char, (INET_ADDRSTRLEN + 7) /* "ip:port" max length */); inet_ntop(AF_INET, &ssaddr.daddr4, ip_string_buf, INET_ADDRSTRLEN); - sprintf((char *)host_result->val.iov_base, "%s:%u", ip_string_buf, ntohs(ssaddr.dport)); - host_result->val.iov_len = strlen((char *)host_result->val.iov_base); + sprintf((char *)host_result->value, "%s:%u", ip_string_buf, ntohs(ssaddr.dport)); + host_result->value_len = strlen((char *)host_result->value); } else if (6 == ssaddr.ipver) { - host_result->val.iov_base = MEMPOOL_CALLOC(mempool, char, (INET6_ADDRSTRLEN + 7) /* "ip:port" max length */); + host_result->value = MEMPOOL_CALLOC(mempool, char, (INET6_ADDRSTRLEN + 7) /* "ip:port" max length */); inet_ntop(AF_INET6, &ssaddr.daddr6, ip_string_buf, INET6_ADDRSTRLEN); - sprintf((char *)host_result->val.iov_base, "%s:%u", ip_string_buf, ntohs(ssaddr.dport)); - host_result->val.iov_len = strlen((char *)host_result->val.iov_base); + sprintf((char *)host_result->value, "%s:%u", ip_string_buf, ntohs(ssaddr.dport)); + host_result->value_len = strlen((char *)host_result->value); } else { @@ -1036,19 +1029,19 @@ static void using_session_addr_as_host(struct session *ref_session, #endif } -void http_decoder_join_url(struct http_decoder_half_data *hfdata, nmx_pool_t *mempool, const struct http_header *host_hdr) +void http_decoder_join_url(struct http_decoder_half_data *hfdata, nmx_pool_t *mempool, const struct http_header_field *host_hdr) { int append_slash_len = 0; if ('/' != ((char *)hfdata->joint_url.iov_base)[0]) { append_slash_len = 1; } - int url_cache_str_len = host_hdr->val.iov_len + hfdata->joint_url.iov_len + append_slash_len; + int url_cache_str_len = host_hdr->value_len + hfdata->joint_url.iov_len + append_slash_len; char *url_cache_str = MEMPOOL_CALLOC(mempool, char, url_cache_str_len); char *ptr = url_cache_str; - memcpy(ptr, host_hdr->val.iov_base, host_hdr->val.iov_len); - ptr += host_hdr->val.iov_len; + memcpy(ptr, host_hdr->value, host_hdr->value_len); + ptr += host_hdr->value_len; if (append_slash_len) { *ptr = '/'; @@ -1059,19 +1052,6 @@ void http_decoder_join_url(struct http_decoder_half_data *hfdata, nmx_pool_t *me MEMPOOL_FREE(mempool, hfdata->joint_url.iov_base); // free the cached uri buffer hfdata->joint_url.iov_base = url_cache_str; hfdata->joint_url.iov_len = url_cache_str_len; - - hfdata->url_is_encoded = httpd_url_is_encoded(url_cache_str, url_cache_str_len); - if (hfdata->url_is_encoded) - { - hfdata->decoded_url.iov_base = MEMPOOL_CALLOC(mempool, char, url_cache_str_len); - httpd_url_decode(url_cache_str, url_cache_str_len, (char *)hfdata->decoded_url.iov_base, &hfdata->decoded_url.iov_len); - } - else - { - hfdata->decoded_url.iov_base = url_cache_str; - hfdata->decoded_url.iov_len = url_cache_str_len; - } - hfdata->joint_url_complete = 1; } @@ -1079,62 +1059,54 @@ void http_decoder_get_url(struct http_decoder_half_data *hfdata, nmx_pool_t *mem { struct http_request_line reqline = {}; http_decoder_half_data_get_request_line(hfdata, &reqline); - if (unlikely(strncasecmp_safe("CONNECT", (char *)reqline.method.iov_base, 7, reqline.method.iov_len) == 0)) + if (unlikely(http_strncasecmp_safe("CONNECT", (char *)reqline.method, 7, reqline.method_len) == 0)) { - hfdata->joint_url.iov_base = MEMPOOL_CALLOC(mempool, char, reqline.uri.iov_len + 1); - memcpy(hfdata->joint_url.iov_base, reqline.uri.iov_base, reqline.uri.iov_len); - hfdata->joint_url.iov_len = reqline.uri.iov_len; + hfdata->joint_url.iov_base = MEMPOOL_CALLOC(mempool, char, reqline.uri_len + 1); + memcpy(hfdata->joint_url.iov_base, reqline.uri, reqline.uri_len); + hfdata->joint_url.iov_len = reqline.uri_len; hfdata->joint_url_complete = 1; } } -int http_decoder_join_url_finally(struct http_event_context *ev_ctx, - struct http_decoder_half_data *hfdata, - nmx_pool_t *mempool) +int http_decoder_join_url_finally(struct http_event_context *ev_ctx, struct http_decoder_half_data *hfdata, nmx_pool_t *mempool) { - struct http_header addr_as_host = {}; - if (hfdata->joint_url_complete) { return 0; } - + struct http_header_field addr_as_host = {}; using_session_addr_as_host(ev_ctx->ref_session, &addr_as_host, mempool); http_decoder_join_url(hfdata, mempool, &addr_as_host); - MEMPOOL_FREE(mempool, addr_as_host.val.iov_base); // free session addr to host buffer + MEMPOOL_FREE(mempool, addr_as_host.value); // free session addr to host buffer return 1; } void http_decoder_get_host_feed_url(struct http_decoder_half *half) { - struct http_header host_result = {}; - hstring host_key = {(char *)"Host", 4}; - if (half->ref_data->joint_url_complete) { return; } - - int host_header_cnt = http_decoder_half_data_get_header(half->ref_data, &host_key, - &host_result); + struct http_header_field host_result = {}; + int host_header_cnt = http_decoder_half_data_get_header(half->ref_data, (char *)"Host", 4, &host_result); if (host_header_cnt < 0) { return; } - http_decoder_join_url(half->ref_data, half->http_ev_ctx->ref_mempool, &host_result); } -int http_half_data_get_url(struct http_decoder_half_data *res_data, hstring *url) +int http_half_data_get_url(struct http_decoder_half_data *res_data, const char **url_val, size_t *url_len) { if (0 == res_data->joint_url_complete) { return -1; } - url->iov_base = res_data->joint_url.iov_base; - url->iov_len = res_data->joint_url.iov_len; + *url_val = res_data->joint_url.iov_base; + *url_len = res_data->joint_url.iov_len; return 0; } +#if 0 int http_half_data_get_decode_url(struct http_decoder_half_data *res_data, hstring *url) { if (0 == res_data->joint_url_complete) @@ -1145,6 +1117,7 @@ int http_half_data_get_decode_url(struct http_decoder_half_data *res_data, hstri url->iov_len = res_data->decoded_url.iov_len; return 0; } +#endif int http_half_data_get_transaction_seq(struct http_decoder_half_data *hf_data) { diff --git a/decoders/http/http_decoder_half.h b/decoders/http/http_decoder_half.h index adf61d1..f525f78 100644 --- a/decoders/http/http_decoder_half.h +++ b/decoders/http/http_decoder_half.h @@ -6,6 +6,10 @@ #include "http_decoder_result_queue.h" #include +#ifndef hstring +#include +typedef struct iovec hstring; +#endif // only one http event is fired at a time enum http_event { @@ -38,6 +42,7 @@ struct http_event_context struct http_decoder_half; struct http_decoder_half_data; +struct http_decoder_env; typedef void http_event_cb(enum http_event event, struct http_decoder_half_data **data, struct http_event_context *ev_ctx, void *httpd_plugin_env); @@ -69,16 +74,16 @@ int http_decoder_half_data_get_response_line(struct http_decoder_half_data *data struct http_response_line *line); int http_decoder_half_data_get_header(const struct http_decoder_half_data *data, - const hstring *key, struct http_header *hdr_res); + const char *name, size_t name_len, struct http_header_field *hdr_res); int http_decoder_half_data_iter_header(struct http_decoder_half_data *data, - struct http_header *header); + struct http_header_field *header); int http_decoder_half_data_reset_header_iter(struct http_decoder_half_data *req_data); int http_decoder_half_data_has_parsed_header(struct http_decoder_half_data *data); -int http_decoder_half_data_get_raw_body(const struct http_decoder_half_data *data, hstring *body); +int http_decoder_half_data_get_raw_body(const struct http_decoder_half_data *data, const char **body, size_t *body_len); -int http_decoder_half_data_get_decompress_body(const struct http_decoder_half_data *data, hstring *body); +int http_decoder_half_data_get_decompress_body(const struct http_decoder_half_data *data, const char **body, size_t *body_len); void http_half_get_lastest_decompress_buffer(struct http_decoder_half_data *data, hstring *decompress_body); void http_half_decompress_buffer_free(struct http_decoder_half_data *data, hstring *decompress_body); void http_decoder_half_data_dump(struct http_decoder_half *half); @@ -88,11 +93,11 @@ void http_decoder_get_url(struct http_decoder_half_data *hfdata, nmx_pool_t *mem int http_half_data_get_decode_url(struct http_decoder_half_data *res_data, hstring *url); void http_decoder_join_url(struct http_decoder_half_data *hfdata, nmx_pool_t *mempool, - const struct http_header *host_hdr); + const struct http_header_field *host_hdr); int http_decoder_join_url_finally(struct http_event_context *ev_ctx, struct http_decoder_half_data *hfdata, nmx_pool_t *mempool); -int http_half_data_get_url(struct http_decoder_half_data *res_data, hstring *url); +int http_half_data_get_url(struct http_decoder_half_data *res_data, const char **url_val, size_t *url_len); int http_half_data_get_transaction_seq(struct http_decoder_half_data *hf_data); void http_half_data_update_commit_index(struct http_decoder_half_data *half_data); diff --git a/decoders/http/http_decoder_private.h b/decoders/http/http_decoder_private.h index f838b71..04130f0 100644 --- a/decoders/http/http_decoder_private.h +++ b/decoders/http/http_decoder_private.h @@ -5,7 +5,6 @@ #define __USE_MISC 1 #endif -#include #ifdef __cplusplus extern "C" { @@ -31,6 +30,11 @@ extern "C" #include "fieldstat/fieldstat_easy.h" #include "toml/toml.h" +#ifndef hstring +#include +typedef struct iovec hstring; +#endif + #ifndef likely #define likely(x) __builtin_expect((x), 1) #endif @@ -142,9 +146,9 @@ extern "C" struct http_message; struct http_message *http_message_new(enum http_message_type type, struct http_decoder_result_queue *queue, - int queue_index, unsigned char flow_type); + int queue_index, uint8_t flow_type); struct http_message *http_body_message_new(enum http_message_type type, struct http_decoder_result_queue *queue, - int queue_index, uint8_t flow_type, hstring *raw_payload); + int queue_index, uint8_t flow_type, hstring *raw_payload, hstring *decompress_payload); int http_topic_exdata_compose_get_index(const struct http_decoder_env *httpd_env, int by_topic_id); #ifdef __cplusplus } diff --git a/decoders/http/http_decoder_result_queue.cpp b/decoders/http/http_decoder_result_queue.c similarity index 100% rename from decoders/http/http_decoder_result_queue.cpp rename to decoders/http/http_decoder_result_queue.c diff --git a/decoders/http/http_decoder_stat.cpp b/decoders/http/http_decoder_stat.c similarity index 100% rename from decoders/http/http_decoder_stat.cpp rename to decoders/http/http_decoder_stat.c diff --git a/decoders/http/http_decoder_string.cpp b/decoders/http/http_decoder_string.c similarity index 92% rename from decoders/http/http_decoder_string.cpp rename to decoders/http/http_decoder_string.c index 88abd17..6fd5b04 100644 --- a/decoders/http/http_decoder_string.cpp +++ b/decoders/http/http_decoder_string.c @@ -246,24 +246,23 @@ enum string_state http_decoder_string_state(const struct http_decoder_string *rs return rstr->state; } -int http_decoder_string_get(const struct http_decoder_string *rstr, hstring *out) +int http_decoder_string_get(const struct http_decoder_string *rstr, char **name, size_t *name_len) { - if (NULL == rstr || NULL == out) + if (NULL == rstr || NULL == name || 0 == name_len) { return -1; } if (http_decoder_string_state(rstr) == STRING_STATE_COMMIT) { - out->iov_base = rstr->commit.iov_base; - out->iov_len = rstr->commit.iov_len; + *name = rstr->commit.iov_base; + *name_len = rstr->commit.iov_len; } else { - out->iov_base = NULL; - out->iov_len = 0; + *name = NULL; + *name_len = 0; } - return 0; } @@ -274,9 +273,9 @@ void http_decoder_string_dump(struct http_decoder_string *rstr, const char *desc return; } - char *refer_str = safe_dup((char *)rstr->refer.iov_base, rstr->refer.iov_len); - char *cache_str = safe_dup((char *)rstr->cache.iov_base, rstr->cache.iov_len); - char *commit_str = safe_dup((char *)rstr->commit.iov_base, rstr->commit.iov_len); + char *refer_str = http_safe_dup((char *)rstr->refer.iov_base, rstr->refer.iov_len); + char *cache_str = http_safe_dup((char *)rstr->cache.iov_base, rstr->cache.iov_len); + char *commit_str = http_safe_dup((char *)rstr->commit.iov_base, rstr->commit.iov_len); printf("%s: state: %s, refer: {len: %02zu, iov_base: %s}, cache: {len: %02zu, iov_base: %s}, commit: {len: %02zu, iov_base: %s}\n", desc, string_state_to_desc(rstr->state), diff --git a/decoders/http/http_decoder_string.h b/decoders/http/http_decoder_string.h index 006794c..83721e9 100644 --- a/decoders/http/http_decoder_string.h +++ b/decoders/http/http_decoder_string.h @@ -67,7 +67,7 @@ void http_decoder_string_reinit(struct http_decoder_string *rstr); enum string_state http_decoder_string_state(const struct http_decoder_string *rstr); -int http_decoder_string_get(const struct http_decoder_string *rstr, hstring *out); +int http_decoder_string_get(const struct http_decoder_string *rstr, char **name, size_t *name_len); void http_decoder_string_dump(struct http_decoder_string *rstr, const char *desc); \ No newline at end of file diff --git a/decoders/http/http_decoder_table.cpp b/decoders/http/http_decoder_table.c similarity index 91% rename from decoders/http/http_decoder_table.cpp rename to decoders/http/http_decoder_table.c index bdf15d0..c85b876 100644 --- a/decoders/http/http_decoder_table.cpp +++ b/decoders/http/http_decoder_table.c @@ -30,7 +30,7 @@ struct http_decoder_table size_t header_cnt; size_t header_index; // current parsing header size_t header_iter; // plugins iterate cursor - size_t commit_header_index; // pushed to plugins, whether has called http_message_header_next() + size_t commit_header_index; // pushed to plugins, whether has called http_message_get0_next_header() struct http_decoder_header *headers; }; @@ -410,58 +410,58 @@ void http_decoder_table_dump(struct http_decoder_table *table) } } -int http_decoder_table_get_uri(const struct http_decoder_table *table, hstring *out) +int http_decoder_table_get_uri(const struct http_decoder_table *table, char **out, size_t *out_len) { if (NULL == table || NULL == out) { return -1; } - return http_decoder_string_get(&table->uri, out); + return http_decoder_string_get(&table->uri, out, out_len); } -int http_decoder_table_get_method(const struct http_decoder_table *table, hstring *out) +int http_decoder_table_get_method(const struct http_decoder_table *table, char **out, size_t *out_len) { if (NULL == table || NULL == out) { return -1; } - return http_decoder_string_get(&table->method, out); + return http_decoder_string_get(&table->method, out, out_len); } -int http_decoder_table_get_status(const struct http_decoder_table *table, hstring *out) +int http_decoder_table_get_status(const struct http_decoder_table *table, char **out, size_t *out_len) { if (NULL == table || NULL == out) { return -1; } - return http_decoder_string_get(&table->status, out); + return http_decoder_string_get(&table->status, out, out_len); } -int http_decoder_table_get_version(const struct http_decoder_table *table, hstring *out) +int http_decoder_table_get_version(const struct http_decoder_table *table, char **out, size_t *out_len) { if (NULL == table || NULL == out) { return -1; } - return http_decoder_string_get(&table->version, out); + return http_decoder_string_get(&table->version, out, out_len); } -int http_decoder_table_get_body(const struct http_decoder_table *table, hstring *out) +int http_decoder_table_get_body(const struct http_decoder_table *table, char **out, size_t *out_len) { if (NULL == table || NULL == out) { return -1; } - return http_decoder_string_get(&table->body, out); + return http_decoder_string_get(&table->body, (char **)out, out_len); } -int http_decoder_table_get_header(const struct http_decoder_table *table, const hstring *key, - struct http_header *hdr_result) +int http_decoder_table_get_header(const struct http_decoder_table *table, const char *name, size_t name_len, + struct http_header_field *hdr_result) { for (size_t i = 0; i < table->header_cnt; i++) { const struct http_decoder_header *tmp_header = &table->headers[i]; - if (tmp_header->key.commit.iov_len != key->iov_len) + if (tmp_header->key.commit.iov_len != name_len) { continue; } @@ -470,13 +470,13 @@ int http_decoder_table_get_header(const struct http_decoder_table *table, const http_decoder_string_state(&tmp_header->val) == STRING_STATE_COMMIT) { hstring tmp_key; - http_decoder_string_get(&tmp_header->key, &tmp_key); + http_decoder_string_get(&tmp_header->key, (char **)&tmp_key.iov_base, &tmp_key.iov_len); - if (tmp_key.iov_len == key->iov_len && - (0 == strncasecmp((char *)tmp_key.iov_base, (char *)key->iov_base, key->iov_len))) + if (tmp_key.iov_len == name_len && + (0 == strncasecmp((char *)tmp_key.iov_base, name, name_len))) { - http_decoder_string_get(&tmp_header->key, &hdr_result->key); - http_decoder_string_get(&tmp_header->val, &hdr_result->val); + http_decoder_string_get(&tmp_header->key, &hdr_result->name, &hdr_result->name_len); + http_decoder_string_get(&tmp_header->val, &hdr_result->value, &hdr_result->value_len); return 0; } } @@ -484,7 +484,7 @@ int http_decoder_table_get_header(const struct http_decoder_table *table, const return -1; } -int http_decoder_table_iter_header(struct http_decoder_table *table, struct http_header *hdr) +int http_decoder_table_iter_header(struct http_decoder_table *table, struct http_header_field *hdr) { if (NULL == table || NULL == hdr) { @@ -502,17 +502,17 @@ int http_decoder_table_iter_header(struct http_decoder_table *table, struct http http_decoder_string_state(&tmp_header->val) == STRING_STATE_COMMIT) { - http_decoder_string_get(&tmp_header->key, &hdr->key); - http_decoder_string_get(&tmp_header->val, &hdr->val); + http_decoder_string_get(&tmp_header->key, &hdr->name, &hdr->name_len); + http_decoder_string_get(&tmp_header->val, &hdr->value, &hdr->value_len); table->header_iter++; return 1; } } - hdr->key.iov_base = NULL; - hdr->key.iov_len = 0; - hdr->val.iov_base = NULL; - hdr->val.iov_len = 0; + hdr->name = NULL; + hdr->name_len = 0; + hdr->value = NULL; + hdr->value_len = 0; return -1; } diff --git a/decoders/http/http_decoder_table.h b/decoders/http/http_decoder_table.h index d32f6be..9a8d948 100644 --- a/decoders/http/http_decoder_table.h +++ b/decoders/http/http_decoder_table.h @@ -36,22 +36,22 @@ void http_decoder_table_reinit(struct http_decoder_table *table); void http_decoder_table_dump(struct http_decoder_table *table); -int http_decoder_table_get_uri(const struct http_decoder_table *table, hstring *out); +int http_decoder_table_get_uri(const struct http_decoder_table *table, char **out, size_t *out_len); -int http_decoder_table_get_method(const struct http_decoder_table *table, hstring *out); +int http_decoder_table_get_method(const struct http_decoder_table *table, char **out, size_t *out_len); -int http_decoder_table_get_status(const struct http_decoder_table *table, hstring *out); +int http_decoder_table_get_status(const struct http_decoder_table *table, char **out, size_t *out_len); -int http_decoder_table_get_version(const struct http_decoder_table *table, hstring *out); +int http_decoder_table_get_version(const struct http_decoder_table *table, char **out, size_t *out_len); -int http_decoder_table_get_body(const struct http_decoder_table *table, hstring *out); +int http_decoder_table_get_body(const struct http_decoder_table *table, char **out, size_t *out_len); int http_decoder_table_get_header(const struct http_decoder_table *table, - const hstring *key, - struct http_header *hdr_res); + const char *name, size_t name_len, + struct http_header_field *hdr_res); int http_decoder_table_iter_header(struct http_decoder_table *table, - struct http_header *hdr); + struct http_header_field *hdr); int http_decoder_table_reset_header_iter(struct http_decoder_table *table); /** * @brief Is there a parsed header diff --git a/decoders/http/http_decoder_tunnel.cpp b/decoders/http/http_decoder_tunnel.c similarity index 89% rename from decoders/http/http_decoder_tunnel.cpp rename to decoders/http/http_decoder_tunnel.c index cf5a573..6396051 100644 --- a/decoders/http/http_decoder_tunnel.cpp +++ b/decoders/http/http_decoder_tunnel.c @@ -23,8 +23,8 @@ int httpd_tunnel_identify(struct http_decoder_env *httpd_env, int curdir, struct { struct http_request_line reqline = {}; http_decoder_half_data_get_request_line(hfdata, &reqline); - if (0 == strncasecmp_safe("CONNECT", (char *)reqline.method.iov_base, - 7, reqline.method.iov_len)) + if (0 == http_strncasecmp_safe("CONNECT", (char *)reqline.method, + 7, reqline.method_len)) { return 1; } @@ -33,8 +33,8 @@ int httpd_tunnel_identify(struct http_decoder_env *httpd_env, int curdir, struct { struct http_response_line resline = {}; http_decoder_half_data_get_response_line(hfdata, &resline); - if (resline.status_code == HTTP_STATUS_OK && 0 == strncasecmp_safe("Connection established", (char *)resline.status.iov_base, - strlen("Connection established"), resline.status.iov_len)) + if (resline.status_code == HTTP_STATUS_OK && 0 == http_strncasecmp_safe("Connection established", (char *)resline.status, + strlen("Connection established"), resline.status_len)) { return 1; } diff --git a/decoders/http/http_decoder_utils.cpp b/decoders/http/http_decoder_utils.c similarity index 93% rename from decoders/http/http_decoder_utils.cpp rename to decoders/http/http_decoder_utils.c index 909af13..10162d4 100644 --- a/decoders/http/http_decoder_utils.cpp +++ b/decoders/http/http_decoder_utils.c @@ -3,7 +3,7 @@ #include "stellar/http.h" #include "http_decoder_private.h" -char *safe_dup(const char *str, size_t len) +char *http_safe_dup(const char *str, size_t len) { if (str == NULL || len == 0) { @@ -14,7 +14,7 @@ char *safe_dup(const char *str, size_t len) return dup; } -int strncasecmp_safe(const char *fix_s1, const char *dyn_s2, size_t fix_n1, size_t dyn_n2) +int http_strncasecmp_safe(const char *fix_s1, const char *dyn_s2, size_t fix_n1, size_t dyn_n2) { if (fix_s1 == NULL || dyn_s2 == NULL) { @@ -187,11 +187,15 @@ static const unsigned char __g_httpd_hextable[] = { #include // https://github.com/curl/curl/blob/2e930c333658725657b94a923d175c6622e0f41d/lib/urlapi.c -void httpd_url_decode(const char *string, size_t length, char *ostring, size_t *olen) +// void httpd_url_decode(const char *string, size_t length, char *ostring, size_t *olen) +size_t http_url_decode(const char *string, size_t length, char *ostring, size_t olen) { - size_t alloc = length; char *ns = ostring; - + if (NULL == string || NULL == ostring || 0 == olen) + { + return 0; + } + size_t alloc = length; while (alloc) { unsigned char in = (unsigned char)*string; @@ -209,14 +213,12 @@ void httpd_url_decode(const char *string, size_t length, char *ostring, size_t * alloc--; } *ns++ = (char)in; + // if ((size_t)(ns - ostring) >= olen - 1) + // { + // return 1; + // } } - *ns = 0; /* terminate it */ - - if (olen) - /* store output size */ - *olen = ns - ostring; - - return; + return ns - ostring; } int httpd_url_is_encoded(const char *url, size_t len) diff --git a/decoders/http/http_decoder_utils.h b/decoders/http/http_decoder_utils.h index 54dcd04..01a32b2 100644 --- a/decoders/http/http_decoder_utils.h +++ b/decoders/http/http_decoder_utils.h @@ -2,7 +2,6 @@ #include #include -#include #ifdef __cplusplus extern "C" { @@ -17,8 +16,8 @@ extern "C" } #endif -char *safe_dup(const char *str, size_t len); -int strncasecmp_safe(const char *fix_s1, const char *dyn_s2, size_t fix_n1, size_t dyn_n2); +char *http_safe_dup(const char *str, size_t len); +int http_strncasecmp_safe(const char *fix_s1, const char *dyn_s2, size_t fix_n1, size_t dyn_n2); const char *http_message_type_to_string(enum http_message_type type); int http_message_type_is_req(struct session *sess, enum http_message_type msg_type); int http_event_is_req(enum http_event event); diff --git a/decoders/http/version.map b/decoders/http/version.map index 15d1d95..a64b729 100644 --- a/decoders/http/version.map +++ b/decoders/http/version.map @@ -5,7 +5,7 @@ global: http_decoder_init; http_decoder_exit; http_decoder_tcp_stream_msg_cb; - http_tunnel_message_*; + http_url_decode; }; local: *; }; \ No newline at end of file diff --git a/test/decoders/http/base64.c b/deps/base64/base64.c similarity index 100% rename from test/decoders/http/base64.c rename to deps/base64/base64.c diff --git a/test/decoders/http/base64.h b/deps/base64/base64.h similarity index 100% rename from test/decoders/http/base64.h rename to deps/base64/base64.h diff --git a/test/decoders/http/md5.c b/deps/md5/md5.c similarity index 100% rename from test/decoders/http/md5.c rename to deps/md5/md5.c diff --git a/test/decoders/http/md5.h b/deps/md5/md5.h similarity index 100% rename from test/decoders/http/md5.h rename to deps/md5/md5.h diff --git a/include/stellar/http.h b/include/stellar/http.h index 3c749ca..1cc5bec 100644 --- a/include/stellar/http.h +++ b/include/stellar/http.h @@ -1,12 +1,11 @@ #pragma once - +#include #ifdef __cplusplus extern "C" { #endif -#include -#define HTTP_DECODER_TOPIC "HTTP_DECODER_MESSAGE" +#define HTTP_TOPIC "HTTP_MESSAGE" struct http_message; enum http_message_type @@ -15,14 +14,14 @@ extern "C" HTTP_MESSAGE_REQ_LINE, HTTP_MESSAGE_REQ_HEADER, - HTTP_MESSAGE_REQ_HEADER_END, + HTTP_MESSAGE_REQ_HEADER_END, // todo, delete END, push all fileds at once HTTP_MESSAGE_REQ_BODY_START, HTTP_MESSAGE_REQ_BODY, HTTP_MESSAGE_REQ_BODY_END, HTTP_MESSAGE_RES_LINE, HTTP_MESSAGE_RES_HEADER, - HTTP_MESSAGE_RES_HEADER_END, + HTTP_MESSAGE_RES_HEADER_END, // todo, delete END, push all fileds at once HTTP_MESSAGE_RES_BODY_START, HTTP_MESSAGE_RES_BODY, HTTP_MESSAGE_RES_BODY_END, @@ -32,67 +31,77 @@ extern "C" HTTP_MESSAGE_MAX }; - struct http_header + struct http_header_field { - struct iovec key; - struct iovec val; + char *name; + size_t name_len; + char *value; + size_t value_len; }; struct http_request_line { - struct iovec method; - struct iovec uri; - struct iovec version; + char *method; + size_t method_len; + char *uri; + size_t uri_len; + char *version; + size_t version_len; + int major_version; int minor_version; }; struct http_response_line { - struct iovec version; - struct iovec status; + char *version; + size_t version_len; + char *status; + size_t status_len; int major_version; int minor_version; int status_code; }; - typedef struct iovec hstring; + enum http_message_type http_message_get_type(const struct http_message *msg); - enum http_message_type http_message_type_get(const struct http_message *msg); + void http_message_get0_request_line(const struct http_message *msg, struct http_request_line *line); - void http_message_request_line_get0(const struct http_message *msg, struct http_request_line *line); - - void http_message_response_line_get0(const struct http_message *msg, struct http_response_line *line); + void http_message_get0_response_line(const struct http_message *msg, struct http_response_line *line); /* * Pay attention: key->iov_base is case-insensitive. */ - void http_message_header_get0(const struct http_message *msg, const struct iovec *key, struct http_header *hdr_result); + void http_message_get0_header(const struct http_message *msg, const char *name, size_t name_len, struct http_header_field *field_result); /** * @brief loop reading all headers. * * @retval succeed( >= 0) failed(-1) */ - int http_message_header_next(const struct http_message *msg, struct http_header *header); + int http_message_get0_next_header(const struct http_message *msg, struct http_header_field *header); /** * @retval succeed( >= 0) failed(-1) */ - int http_message_reset_header_iter(struct http_message *msg); + int http_message_reset_header_iter(struct http_message *msg); // to do , obsoleted - void http_message_raw_body_get0(const struct http_message *msg, struct iovec *body); + void http_message_get0_uncompressed_body(const struct http_message *msg, const char **body, size_t *body_len); /** - * @brief If the body hasn't been compressed, same as http_message_raw_body_get0(). + * @brief If the body hasn't been compressed, same as http_message_get0_uncompressed_body(). * */ - void http_message_decompress_body_get0(const struct http_message *msg, struct iovec *body); + void http_message_get0_decompressed_body(const struct http_message *msg, const char **body, size_t *body_len); - // raw - void http_message_raw_url_get0(const struct http_message *msg, struct iovec *url); + void http_message_get0_raw_url(const struct http_message *msg, const char **url, size_t *url_len); - void http_message_decoded_url_get0(const struct http_message *msg, struct iovec *url); + /* + return value: + 0: failed + >1: success, length of decoded_url_buffer, not C string( no EOF with '\0' ) + */ + size_t http_url_decode(const char *raw_url, size_t raw_url_len, char *decoded_url_buffer, size_t decoded_url_buffer_len); /** * @retval succeed( >= 0) failed(-1) diff --git a/infra/version.map b/infra/version.map index abb7f9b..03ad01a 100644 --- a/infra/version.map +++ b/infra/version.map @@ -70,7 +70,7 @@ global: http_decoder_init; http_decoder_exit; http_decoder_tcp_stream_msg_cb; - http_tunnel_message_*; + http_url_decode; lpi_plugin_load; lpi_plugin_unload; diff --git a/test/decoders/http/CMakeLists.txt b/test/decoders/http/CMakeLists.txt index 766e49c..5011a95 100644 --- a/test/decoders/http/CMakeLists.txt +++ b/test/decoders/http/CMakeLists.txt @@ -1,27 +1,18 @@ -set(DECODER_NAME http) - aux_source_directory(${PROJECT_SOURCE_DIR}/deps/toml DEPS_SRC) +aux_source_directory(${PROJECT_SOURCE_DIR}/deps/md5 DEPS_SRC) -add_library(${DECODER_NAME}_test SHARED http_decoder_test_plug.cpp md5.c ${DEPS_SRC}) -add_dependencies(${DECODER_NAME}_test ${DECODER_NAME}) -target_link_libraries(${DECODER_NAME}_test cjson-static llhttp-static) -set_target_properties(${DECODER_NAME}_test PROPERTIES PREFIX "") - -# add_library(${DECODER_NAME}_perf SHARED http_decoder_perf_plug.cpp) -# add_dependencies(${DECODER_NAME}_perf ${DECODER_NAME}) -# set_target_properties(${DECODER_NAME}_perf PROPERTIES PREFIX "") - -set(TEST_RUN_DIR ${CMAKE_BINARY_DIR}/testing) +add_library(http_test SHARED http_test_plug.cpp ${DEPS_SRC}) +add_dependencies(http_test http) +target_link_libraries(http_test cjson-static llhttp-static) +set_target_properties(http_test PROPERTIES PREFIX "") include_directories(${CMAKE_SOURCE_DIR}/include) include_directories(${CMAKE_SOURCE_DIR}/deps) include_directories(${PROJECT_SOURCE_DIR}/decoders/http) include_directories(${PROJECT_SOURCE_DIR}/include/stellar) -aux_source_directory(${PROJECT_SOURCE_DIR}/deps/toml PERF_TEST_DEP_SRC) - -add_executable(gtest_http http_decoder_gtest.cpp ${PROJECT_SOURCE_DIR}/decoders/http/http_decoder_utils.cpp) -target_link_libraries(gtest_http gtest stellar_lib llhttp-static) +add_executable(gtest_http http_gtest.cpp ${PROJECT_SOURCE_DIR}/decoders/http/http_decoder_utils.c ${PROJECT_SOURCE_DIR}/decoders/http/http_content_decompress.c ${PROJECT_SOURCE_DIR}/deps/md5/md5.c) +target_link_libraries(gtest_http gtest stellar_lib llhttp-static z brotli-dec-static brotli-enc-static brotli-common-static libevent-static) add_executable(http_test_main plugin_test_main.cpp) set_target_properties(http_test_main diff --git a/benchmarks/json/http/http_6over4_single_trans.json b/test/decoders/http/benchmarks/json/http_6over4_single_trans.json similarity index 100% rename from benchmarks/json/http/http_6over4_single_trans.json rename to test/decoders/http/benchmarks/json/http_6over4_single_trans.json diff --git a/benchmarks/json/http/http_chn_encode_url.json b/test/decoders/http/benchmarks/json/http_chn_encode_url.json similarity index 100% rename from benchmarks/json/http/http_chn_encode_url.json rename to test/decoders/http/benchmarks/json/http_chn_encode_url.json diff --git a/benchmarks/json/http/http_chunked_res_gzip.json b/test/decoders/http/benchmarks/json/http_chunked_res_gzip.json similarity index 100% rename from benchmarks/json/http/http_chunked_res_gzip.json rename to test/decoders/http/benchmarks/json/http_chunked_res_gzip.json diff --git a/benchmarks/json/http/http_connect_flood.json b/test/decoders/http/benchmarks/json/http_connect_flood.json similarity index 100% rename from benchmarks/json/http/http_connect_flood.json rename to test/decoders/http/benchmarks/json/http_connect_flood.json diff --git a/benchmarks/json/http/http_get_encoded_uri.json b/test/decoders/http/benchmarks/json/http_get_encoded_uri.json similarity index 100% rename from benchmarks/json/http/http_get_encoded_uri.json rename to test/decoders/http/benchmarks/json/http_get_encoded_uri.json diff --git a/benchmarks/json/http/http_get_long_cookie.json b/test/decoders/http/benchmarks/json/http_get_long_cookie.json similarity index 100% rename from benchmarks/json/http/http_get_long_cookie.json rename to test/decoders/http/benchmarks/json/http_get_long_cookie.json diff --git a/benchmarks/json/http/http_get_malformed.json b/test/decoders/http/benchmarks/json/http_get_malformed.json similarity index 100% rename from benchmarks/json/http/http_get_malformed.json rename to test/decoders/http/benchmarks/json/http_get_malformed.json diff --git a/benchmarks/json/http/http_get_multi_trans.json b/test/decoders/http/benchmarks/json/http_get_multi_trans.json similarity index 100% rename from benchmarks/json/http/http_get_multi_trans.json rename to test/decoders/http/benchmarks/json/http_get_multi_trans.json diff --git a/benchmarks/json/http/http_get_req_pipeline.json b/test/decoders/http/benchmarks/json/http_get_req_pipeline.json similarity index 100% rename from benchmarks/json/http/http_get_req_pipeline.json rename to test/decoders/http/benchmarks/json/http_get_req_pipeline.json diff --git a/benchmarks/json/http/http_get_single_trans.json b/test/decoders/http/benchmarks/json/http_get_single_trans.json similarity index 100% rename from benchmarks/json/http/http_get_single_trans.json rename to test/decoders/http/benchmarks/json/http_get_single_trans.json diff --git a/benchmarks/json/http/http_gzip_out_of_order.json b/test/decoders/http/benchmarks/json/http_gzip_out_of_order.json similarity index 100% rename from benchmarks/json/http/http_gzip_out_of_order.json rename to test/decoders/http/benchmarks/json/http_gzip_out_of_order.json diff --git a/benchmarks/json/http/http_hdr_truncated_after_kv.json b/test/decoders/http/benchmarks/json/http_hdr_truncated_after_kv.json similarity index 100% rename from benchmarks/json/http/http_hdr_truncated_after_kv.json rename to test/decoders/http/benchmarks/json/http_hdr_truncated_after_kv.json diff --git a/benchmarks/json/http/http_hdr_truncated_in_kv.json b/test/decoders/http/benchmarks/json/http_hdr_truncated_in_kv.json similarity index 100% rename from benchmarks/json/http/http_hdr_truncated_in_kv.json rename to test/decoders/http/benchmarks/json/http_hdr_truncated_in_kv.json diff --git a/benchmarks/json/http/http_hdr_value_empty.json b/test/decoders/http/benchmarks/json/http_hdr_value_empty.json similarity index 100% rename from benchmarks/json/http/http_hdr_value_empty.json rename to test/decoders/http/benchmarks/json/http_hdr_value_empty.json diff --git a/benchmarks/json/http/http_hdrs_exceed_maximum.json b/test/decoders/http/benchmarks/json/http_hdrs_exceed_maximum.json similarity index 100% rename from benchmarks/json/http/http_hdrs_exceed_maximum.json rename to test/decoders/http/benchmarks/json/http_hdrs_exceed_maximum.json diff --git a/benchmarks/json/http/http_inner_tunnel_for_http.json b/test/decoders/http/benchmarks/json/http_inner_tunnel_for_http.json similarity index 100% rename from benchmarks/json/http/http_inner_tunnel_for_http.json rename to test/decoders/http/benchmarks/json/http_inner_tunnel_for_http.json diff --git a/benchmarks/json/http/http_inner_tunnel_for_pop3.json b/test/decoders/http/benchmarks/json/http_inner_tunnel_for_pop3.json similarity index 100% rename from benchmarks/json/http/http_inner_tunnel_for_pop3.json rename to test/decoders/http/benchmarks/json/http_inner_tunnel_for_pop3.json diff --git a/benchmarks/json/http/http_msg_type_state.json b/test/decoders/http/benchmarks/json/http_msg_type_state.json similarity index 100% rename from benchmarks/json/http/http_msg_type_state.json rename to test/decoders/http/benchmarks/json/http_msg_type_state.json diff --git a/benchmarks/json/http/http_msg_type_state_c2s.json b/test/decoders/http/benchmarks/json/http_msg_type_state_c2s.json similarity index 100% rename from benchmarks/json/http/http_msg_type_state_c2s.json rename to test/decoders/http/benchmarks/json/http_msg_type_state_c2s.json diff --git a/benchmarks/json/http/http_msg_type_state_exception_c2s.json b/test/decoders/http/benchmarks/json/http_msg_type_state_exception_c2s.json similarity index 100% rename from benchmarks/json/http/http_msg_type_state_exception_c2s.json rename to test/decoders/http/benchmarks/json/http_msg_type_state_exception_c2s.json diff --git a/benchmarks/json/http/http_msg_type_state_exception_s2c.json b/test/decoders/http/benchmarks/json/http_msg_type_state_exception_s2c.json similarity index 100% rename from benchmarks/json/http/http_msg_type_state_exception_s2c.json rename to test/decoders/http/benchmarks/json/http_msg_type_state_exception_s2c.json diff --git a/benchmarks/json/http/http_msg_type_state_pipeline.json b/test/decoders/http/benchmarks/json/http_msg_type_state_pipeline.json similarity index 100% rename from benchmarks/json/http/http_msg_type_state_pipeline.json rename to test/decoders/http/benchmarks/json/http_msg_type_state_pipeline.json diff --git a/benchmarks/json/http/http_msg_type_state_s2c.json b/test/decoders/http/benchmarks/json/http_msg_type_state_s2c.json similarity index 100% rename from benchmarks/json/http/http_msg_type_state_s2c.json rename to test/decoders/http/benchmarks/json/http_msg_type_state_s2c.json diff --git a/benchmarks/json/http/http_msg_type_state_tunnel.json b/test/decoders/http/benchmarks/json/http_msg_type_state_tunnel.json similarity index 100% rename from benchmarks/json/http/http_msg_type_state_tunnel.json rename to test/decoders/http/benchmarks/json/http_msg_type_state_tunnel.json diff --git a/benchmarks/json/http/http_msg_type_state_tunnel_c2s.json b/test/decoders/http/benchmarks/json/http_msg_type_state_tunnel_c2s.json similarity index 100% rename from benchmarks/json/http/http_msg_type_state_tunnel_c2s.json rename to test/decoders/http/benchmarks/json/http_msg_type_state_tunnel_c2s.json diff --git a/benchmarks/json/http/http_msg_type_state_tunnel_s2c.json b/test/decoders/http/benchmarks/json/http_msg_type_state_tunnel_s2c.json similarity index 100% rename from benchmarks/json/http/http_msg_type_state_tunnel_s2c.json rename to test/decoders/http/benchmarks/json/http_msg_type_state_tunnel_s2c.json diff --git a/benchmarks/json/http/http_multi_parse_error.json b/test/decoders/http/benchmarks/json/http_multi_parse_error.json similarity index 100% rename from benchmarks/json/http/http_multi_parse_error.json rename to test/decoders/http/benchmarks/json/http_multi_parse_error.json diff --git a/benchmarks/json/http/http_no_content_length.json b/test/decoders/http/benchmarks/json/http_no_content_length.json similarity index 100% rename from benchmarks/json/http/http_no_content_length.json rename to test/decoders/http/benchmarks/json/http_no_content_length.json diff --git a/benchmarks/json/http/http_out_of_order.json b/test/decoders/http/benchmarks/json/http_out_of_order.json similarity index 100% rename from benchmarks/json/http/http_out_of_order.json rename to test/decoders/http/benchmarks/json/http_out_of_order.json diff --git a/benchmarks/json/http/http_over_pppoe.json b/test/decoders/http/benchmarks/json/http_over_pppoe.json similarity index 100% rename from benchmarks/json/http/http_over_pppoe.json rename to test/decoders/http/benchmarks/json/http_over_pppoe.json diff --git a/benchmarks/json/http/http_over_tcp_keepalive.json b/test/decoders/http/benchmarks/json/http_over_tcp_keepalive.json similarity index 100% rename from benchmarks/json/http/http_over_tcp_keepalive.json rename to test/decoders/http/benchmarks/json/http_over_tcp_keepalive.json diff --git a/benchmarks/json/http/http_over_tls.json b/test/decoders/http/benchmarks/json/http_over_tls.json similarity index 100% rename from benchmarks/json/http/http_over_tls.json rename to test/decoders/http/benchmarks/json/http_over_tls.json diff --git a/test/decoders/http/benchmarks/json/http_pipeline_header_splitting.json b/test/decoders/http/benchmarks/json/http_pipeline_header_splitting.json new file mode 100644 index 0000000..bebfa94 --- /dev/null +++ b/test/decoders/http/benchmarks/json/http_pipeline_header_splitting.json @@ -0,0 +1,69 @@ +[ + { + "__X_HTTP_TUPLE4": "127.0.0.1:44980-127.0.0.1:8080-6-0" + }, + { + "__X_HTTP_TRANSACTION": "request", + "__X_HTTP_TRANSACTION_SEQ": 0, + "method": "GET", + "uri": "/111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111.html", + "req_version": "1.1", + "major_version": 1, + "minor_version": 1, + "Host": "127.0.0.1:8080 ", + "User-Agent": "curl/7.61.111", + "Accept": "*/*", + "-X-header1": "value111", + "-X-header2": "value222", + "-X-header3": "value333", + "-X-header4": "value444", + "-X-header5": "value5555", + "__X_HTTP_URL": "127.0.0.1:8080 /111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111.html" + }, + { + "__X_HTTP_TRANSACTION": "response", + "__X_HTTP_TRANSACTION_SEQ": 0, + "res_version": "1.1", + "res_status": "OK", + "major_version": 1, + "minor_version": 1, + "status_code": 200, + "Server": "BaseHTTP/0.6 Python/3.6.88", + "Date": "Sun, 01 Sep 2024 07:31:09 GMTT", + "Content-length": "88", + "__X_HTTP_RAW_PAYLOAD_MD5": "c396b63d897591e928fae959915e9ebc", + "__X_HTTP_DECOMPRESS_PAYLOAD_MD5": "c396b63d897591e928fae959915e9ebc" + }, + { + "__X_HTTP_TRANSACTION": "request", + "__X_HTTP_TRANSACTION_SEQ": 1, + "method": "GET", + "uri": "/2222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222.html", + "req_version": "1.1", + "major_version": 1, + "minor_version": 1, + "Host": "127.0.0.1:8080", + "User-Agent": "curl/7.61.1", + "Accept": "*/*", + "-X-header1": "value111", + "-X-header2": "value2222", + "-X-header3": "value3333", + "-X-header4": "value4444", + "-X-header5": "value55555", + "__X_HTTP_URL": "127.0.0.1:8080/2222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222.html" + }, + { + "__X_HTTP_TRANSACTION": "response", + "__X_HTTP_TRANSACTION_SEQ": 1, + "res_version": "1.1", + "res_status": "OK", + "major_version": 1, + "minor_version": 1, + "status_code": 200, + "Server": "BaseHTTP/0.6 Python/3.6.88", + "Date": "Sun, 01 Sep 2024 07:31:09 GMTT", + "Content-length": "89", + "__X_HTTP_RAW_PAYLOAD_MD5": "5e3c5a7de2696e52ef42d9ccfd8ce831", + "__X_HTTP_DECOMPRESS_PAYLOAD_MD5": "5e3c5a7de2696e52ef42d9ccfd8ce831" + } +] \ No newline at end of file diff --git a/benchmarks/json/http/http_post_multipart_form_data.json b/test/decoders/http/benchmarks/json/http_post_multipart_form_data.json similarity index 100% rename from benchmarks/json/http/http_post_multipart_form_data.json rename to test/decoders/http/benchmarks/json/http_post_multipart_form_data.json diff --git a/benchmarks/json/http/http_post_single_trans.json b/test/decoders/http/benchmarks/json/http_post_single_trans.json similarity index 100% rename from benchmarks/json/http/http_post_single_trans.json rename to test/decoders/http/benchmarks/json/http_post_single_trans.json diff --git a/benchmarks/json/http/http_req_1byte_sliding_window.json b/test/decoders/http/benchmarks/json/http_req_1byte_sliding_window.json similarity index 100% rename from benchmarks/json/http/http_req_1byte_sliding_window.json rename to test/decoders/http/benchmarks/json/http_req_1byte_sliding_window.json diff --git a/benchmarks/json/http/http_res_1byte_sliding_window.json b/test/decoders/http/benchmarks/json/http_res_1byte_sliding_window.json similarity index 100% rename from benchmarks/json/http/http_res_1byte_sliding_window.json rename to test/decoders/http/benchmarks/json/http_res_1byte_sliding_window.json diff --git a/benchmarks/json/http/http_res_gzip.json b/test/decoders/http/benchmarks/json/http_res_gzip.json similarity index 100% rename from benchmarks/json/http/http_res_gzip.json rename to test/decoders/http/benchmarks/json/http_res_gzip.json diff --git a/benchmarks/json/http/http_trans_pipeline.json b/test/decoders/http/benchmarks/json/http_trans_pipeline.json similarity index 100% rename from benchmarks/json/http/http_trans_pipeline.json rename to test/decoders/http/benchmarks/json/http_trans_pipeline.json diff --git a/benchmarks/json/http/http_tunnel_for_http.json b/test/decoders/http/benchmarks/json/http_tunnel_for_http.json similarity index 100% rename from benchmarks/json/http/http_tunnel_for_http.json rename to test/decoders/http/benchmarks/json/http_tunnel_for_http.json diff --git a/benchmarks/json/http/http_tunnel_for_http_c2s.json b/test/decoders/http/benchmarks/json/http_tunnel_for_http_c2s.json similarity index 100% rename from benchmarks/json/http/http_tunnel_for_http_c2s.json rename to test/decoders/http/benchmarks/json/http_tunnel_for_http_c2s.json diff --git a/benchmarks/json/http/http_tunnel_for_http_s2c.json b/test/decoders/http/benchmarks/json/http_tunnel_for_http_s2c.json similarity index 100% rename from benchmarks/json/http/http_tunnel_for_http_s2c.json rename to test/decoders/http/benchmarks/json/http_tunnel_for_http_s2c.json diff --git a/benchmarks/json/http/http_tunnel_for_pop3.json b/test/decoders/http/benchmarks/json/http_tunnel_for_pop3.json similarity index 100% rename from benchmarks/json/http/http_tunnel_for_pop3.json rename to test/decoders/http/benchmarks/json/http_tunnel_for_pop3.json diff --git a/benchmarks/json/http/http_tunnel_s2c_only_hdr.json b/test/decoders/http/benchmarks/json/http_tunnel_s2c_only_hdr.json similarity index 100% rename from benchmarks/json/http/http_tunnel_s2c_only_hdr.json rename to test/decoders/http/benchmarks/json/http_tunnel_s2c_only_hdr.json diff --git a/benchmarks/json/http/http_upgrade_http2.json b/test/decoders/http/benchmarks/json/http_upgrade_http2.json similarity index 100% rename from benchmarks/json/http/http_upgrade_http2.json rename to test/decoders/http/benchmarks/json/http_upgrade_http2.json diff --git a/benchmarks/json/http/http_upgrade_websocket.json b/test/decoders/http/benchmarks/json/http_upgrade_websocket.json similarity index 100% rename from benchmarks/json/http/http_upgrade_websocket.json rename to test/decoders/http/benchmarks/json/http_upgrade_websocket.json diff --git a/benchmarks/json/http/http_url_test_with_host.json b/test/decoders/http/benchmarks/json/http_url_test_with_host.json similarity index 100% rename from benchmarks/json/http/http_url_test_with_host.json rename to test/decoders/http/benchmarks/json/http_url_test_with_host.json diff --git a/benchmarks/json/http/http_url_test_without_host.json b/test/decoders/http/benchmarks/json/http_url_test_without_host.json similarity index 100% rename from benchmarks/json/http/http_url_test_without_host.json rename to test/decoders/http/benchmarks/json/http_url_test_without_host.json diff --git a/benchmarks/json/http/http_url_test_without_host_v6.json b/test/decoders/http/benchmarks/json/http_url_test_without_host_v6.json similarity index 100% rename from benchmarks/json/http/http_url_test_without_host_v6.json rename to test/decoders/http/benchmarks/json/http_url_test_without_host_v6.json diff --git a/benchmarks/json/http/http_zlib_deadlock.json b/test/decoders/http/benchmarks/json/http_zlib_deadlock.json similarity index 100% rename from benchmarks/json/http/http_zlib_deadlock.json rename to test/decoders/http/benchmarks/json/http_zlib_deadlock.json diff --git a/benchmarks/json/http/non_http.json b/test/decoders/http/benchmarks/json/non_http.json similarity index 100% rename from benchmarks/json/http/non_http.json rename to test/decoders/http/benchmarks/json/non_http.json diff --git a/benchmarks/pcap/http/http_6over4_single_trans.pcap b/test/decoders/http/benchmarks/pcap/http_6over4_single_trans.pcap similarity index 100% rename from benchmarks/pcap/http/http_6over4_single_trans.pcap rename to test/decoders/http/benchmarks/pcap/http_6over4_single_trans.pcap diff --git a/benchmarks/pcap/http/http_chn_encode_url.pcap b/test/decoders/http/benchmarks/pcap/http_chn_encode_url.pcap similarity index 100% rename from benchmarks/pcap/http/http_chn_encode_url.pcap rename to test/decoders/http/benchmarks/pcap/http_chn_encode_url.pcap diff --git a/benchmarks/pcap/http/http_chunked_res_gzip.pcap b/test/decoders/http/benchmarks/pcap/http_chunked_res_gzip.pcap similarity index 100% rename from benchmarks/pcap/http/http_chunked_res_gzip.pcap rename to test/decoders/http/benchmarks/pcap/http_chunked_res_gzip.pcap diff --git a/benchmarks/pcap/http/http_connect_flood.pcap b/test/decoders/http/benchmarks/pcap/http_connect_flood.pcap similarity index 100% rename from benchmarks/pcap/http/http_connect_flood.pcap rename to test/decoders/http/benchmarks/pcap/http_connect_flood.pcap diff --git a/benchmarks/pcap/http/http_error.pcap b/test/decoders/http/benchmarks/pcap/http_error.pcap similarity index 100% rename from benchmarks/pcap/http/http_error.pcap rename to test/decoders/http/benchmarks/pcap/http_error.pcap diff --git a/benchmarks/pcap/http/http_fin.pcap b/test/decoders/http/benchmarks/pcap/http_fin.pcap similarity index 100% rename from benchmarks/pcap/http/http_fin.pcap rename to test/decoders/http/benchmarks/pcap/http_fin.pcap diff --git a/benchmarks/pcap/http/http_get_encoded_uri.pcap b/test/decoders/http/benchmarks/pcap/http_get_encoded_uri.pcap similarity index 100% rename from benchmarks/pcap/http/http_get_encoded_uri.pcap rename to test/decoders/http/benchmarks/pcap/http_get_encoded_uri.pcap diff --git a/benchmarks/pcap/http/http_get_long_cookie.pcap b/test/decoders/http/benchmarks/pcap/http_get_long_cookie.pcap similarity index 100% rename from benchmarks/pcap/http/http_get_long_cookie.pcap rename to test/decoders/http/benchmarks/pcap/http_get_long_cookie.pcap diff --git a/benchmarks/pcap/http/http_get_malformed.pcap b/test/decoders/http/benchmarks/pcap/http_get_malformed.pcap similarity index 100% rename from benchmarks/pcap/http/http_get_malformed.pcap rename to test/decoders/http/benchmarks/pcap/http_get_malformed.pcap diff --git a/benchmarks/pcap/http/http_get_multi_trans.pcap b/test/decoders/http/benchmarks/pcap/http_get_multi_trans.pcap similarity index 100% rename from benchmarks/pcap/http/http_get_multi_trans.pcap rename to test/decoders/http/benchmarks/pcap/http_get_multi_trans.pcap diff --git a/benchmarks/pcap/http/http_get_req_pipeline.pcap b/test/decoders/http/benchmarks/pcap/http_get_req_pipeline.pcap similarity index 100% rename from benchmarks/pcap/http/http_get_req_pipeline.pcap rename to test/decoders/http/benchmarks/pcap/http_get_req_pipeline.pcap diff --git a/benchmarks/pcap/http/http_get_single_trans.pcap b/test/decoders/http/benchmarks/pcap/http_get_single_trans.pcap similarity index 100% rename from benchmarks/pcap/http/http_get_single_trans.pcap rename to test/decoders/http/benchmarks/pcap/http_get_single_trans.pcap diff --git a/benchmarks/pcap/http/http_gzip_out_of_order.pcap b/test/decoders/http/benchmarks/pcap/http_gzip_out_of_order.pcap similarity index 100% rename from benchmarks/pcap/http/http_gzip_out_of_order.pcap rename to test/decoders/http/benchmarks/pcap/http_gzip_out_of_order.pcap diff --git a/benchmarks/pcap/http/http_hdr_truncated_after_kv.pcap b/test/decoders/http/benchmarks/pcap/http_hdr_truncated_after_kv.pcap similarity index 100% rename from benchmarks/pcap/http/http_hdr_truncated_after_kv.pcap rename to test/decoders/http/benchmarks/pcap/http_hdr_truncated_after_kv.pcap diff --git a/benchmarks/pcap/http/http_hdr_truncated_in_kv.pcap b/test/decoders/http/benchmarks/pcap/http_hdr_truncated_in_kv.pcap similarity index 100% rename from benchmarks/pcap/http/http_hdr_truncated_in_kv.pcap rename to test/decoders/http/benchmarks/pcap/http_hdr_truncated_in_kv.pcap diff --git a/benchmarks/pcap/http/http_hdr_value_empty.pcap b/test/decoders/http/benchmarks/pcap/http_hdr_value_empty.pcap similarity index 100% rename from benchmarks/pcap/http/http_hdr_value_empty.pcap rename to test/decoders/http/benchmarks/pcap/http_hdr_value_empty.pcap diff --git a/benchmarks/pcap/http/http_hdrs_exceed_maximum.pcap b/test/decoders/http/benchmarks/pcap/http_hdrs_exceed_maximum.pcap similarity index 100% rename from benchmarks/pcap/http/http_hdrs_exceed_maximum.pcap rename to test/decoders/http/benchmarks/pcap/http_hdrs_exceed_maximum.pcap diff --git a/benchmarks/pcap/http/http_multi_parse_error.pcap b/test/decoders/http/benchmarks/pcap/http_multi_parse_error.pcap similarity index 100% rename from benchmarks/pcap/http/http_multi_parse_error.pcap rename to test/decoders/http/benchmarks/pcap/http_multi_parse_error.pcap diff --git a/benchmarks/pcap/http/http_no_content_length.pcap b/test/decoders/http/benchmarks/pcap/http_no_content_length.pcap similarity index 100% rename from benchmarks/pcap/http/http_no_content_length.pcap rename to test/decoders/http/benchmarks/pcap/http_no_content_length.pcap diff --git a/benchmarks/pcap/http/http_out_of_order.pcap b/test/decoders/http/benchmarks/pcap/http_out_of_order.pcap similarity index 100% rename from benchmarks/pcap/http/http_out_of_order.pcap rename to test/decoders/http/benchmarks/pcap/http_out_of_order.pcap diff --git a/benchmarks/pcap/http/http_over_pppoe.pcap b/test/decoders/http/benchmarks/pcap/http_over_pppoe.pcap similarity index 100% rename from benchmarks/pcap/http/http_over_pppoe.pcap rename to test/decoders/http/benchmarks/pcap/http_over_pppoe.pcap diff --git a/benchmarks/pcap/http/http_over_tcp_keepalive.pcap b/test/decoders/http/benchmarks/pcap/http_over_tcp_keepalive.pcap similarity index 100% rename from benchmarks/pcap/http/http_over_tcp_keepalive.pcap rename to test/decoders/http/benchmarks/pcap/http_over_tcp_keepalive.pcap diff --git a/benchmarks/pcap/http/http_over_tls.pcap b/test/decoders/http/benchmarks/pcap/http_over_tls.pcap similarity index 100% rename from benchmarks/pcap/http/http_over_tls.pcap rename to test/decoders/http/benchmarks/pcap/http_over_tls.pcap diff --git a/benchmarks/pcap/http/http_pipeline_C2S.pcap b/test/decoders/http/benchmarks/pcap/http_pipeline_C2S.pcap similarity index 100% rename from benchmarks/pcap/http/http_pipeline_C2S.pcap rename to test/decoders/http/benchmarks/pcap/http_pipeline_C2S.pcap diff --git a/benchmarks/pcap/http/http_pipeline_S2C.pcap b/test/decoders/http/benchmarks/pcap/http_pipeline_S2C.pcap similarity index 100% rename from benchmarks/pcap/http/http_pipeline_S2C.pcap rename to test/decoders/http/benchmarks/pcap/http_pipeline_S2C.pcap diff --git a/test/decoders/http/benchmarks/pcap/http_pipeline_header_splitting.pcap b/test/decoders/http/benchmarks/pcap/http_pipeline_header_splitting.pcap new file mode 100644 index 0000000..57acce5 Binary files /dev/null and b/test/decoders/http/benchmarks/pcap/http_pipeline_header_splitting.pcap differ diff --git a/benchmarks/pcap/http/http_post.pcap b/test/decoders/http/benchmarks/pcap/http_post.pcap similarity index 100% rename from benchmarks/pcap/http/http_post.pcap rename to test/decoders/http/benchmarks/pcap/http_post.pcap diff --git a/benchmarks/pcap/http/http_post_c2s.pcap b/test/decoders/http/benchmarks/pcap/http_post_c2s.pcap similarity index 100% rename from benchmarks/pcap/http/http_post_c2s.pcap rename to test/decoders/http/benchmarks/pcap/http_post_c2s.pcap diff --git a/benchmarks/pcap/http/http_post_multipart_form_data.pcap b/test/decoders/http/benchmarks/pcap/http_post_multipart_form_data.pcap similarity index 100% rename from benchmarks/pcap/http/http_post_multipart_form_data.pcap rename to test/decoders/http/benchmarks/pcap/http_post_multipart_form_data.pcap diff --git a/benchmarks/pcap/http/http_post_s2c.pcap b/test/decoders/http/benchmarks/pcap/http_post_s2c.pcap similarity index 100% rename from benchmarks/pcap/http/http_post_s2c.pcap rename to test/decoders/http/benchmarks/pcap/http_post_s2c.pcap diff --git a/benchmarks/pcap/http/http_post_single_trans.pcap b/test/decoders/http/benchmarks/pcap/http_post_single_trans.pcap similarity index 100% rename from benchmarks/pcap/http/http_post_single_trans.pcap rename to test/decoders/http/benchmarks/pcap/http_post_single_trans.pcap diff --git a/benchmarks/pcap/http/http_req_1byte_sliding_window.pcap b/test/decoders/http/benchmarks/pcap/http_req_1byte_sliding_window.pcap similarity index 100% rename from benchmarks/pcap/http/http_req_1byte_sliding_window.pcap rename to test/decoders/http/benchmarks/pcap/http_req_1byte_sliding_window.pcap diff --git a/benchmarks/pcap/http/http_res_1byte_sliding_window.pcap b/test/decoders/http/benchmarks/pcap/http_res_1byte_sliding_window.pcap similarity index 100% rename from benchmarks/pcap/http/http_res_1byte_sliding_window.pcap rename to test/decoders/http/benchmarks/pcap/http_res_1byte_sliding_window.pcap diff --git a/benchmarks/pcap/http/http_res_gzip.pcap b/test/decoders/http/benchmarks/pcap/http_res_gzip.pcap similarity index 100% rename from benchmarks/pcap/http/http_res_gzip.pcap rename to test/decoders/http/benchmarks/pcap/http_res_gzip.pcap diff --git a/benchmarks/pcap/http/http_session_exception_c2s.pcap b/test/decoders/http/benchmarks/pcap/http_session_exception_c2s.pcap similarity index 100% rename from benchmarks/pcap/http/http_session_exception_c2s.pcap rename to test/decoders/http/benchmarks/pcap/http_session_exception_c2s.pcap diff --git a/benchmarks/pcap/http/http_session_exception_s2c.pcap b/test/decoders/http/benchmarks/pcap/http_session_exception_s2c.pcap similarity index 100% rename from benchmarks/pcap/http/http_session_exception_s2c.pcap rename to test/decoders/http/benchmarks/pcap/http_session_exception_s2c.pcap diff --git a/benchmarks/pcap/http/http_trans_pipeline.pcap b/test/decoders/http/benchmarks/pcap/http_trans_pipeline.pcap similarity index 100% rename from benchmarks/pcap/http/http_trans_pipeline.pcap rename to test/decoders/http/benchmarks/pcap/http_trans_pipeline.pcap diff --git a/benchmarks/pcap/http/http_tunnel_for_http.pcap b/test/decoders/http/benchmarks/pcap/http_tunnel_for_http.pcap similarity index 100% rename from benchmarks/pcap/http/http_tunnel_for_http.pcap rename to test/decoders/http/benchmarks/pcap/http_tunnel_for_http.pcap diff --git a/benchmarks/pcap/http/http_tunnel_for_http_C2S.pcap b/test/decoders/http/benchmarks/pcap/http_tunnel_for_http_C2S.pcap similarity index 100% rename from benchmarks/pcap/http/http_tunnel_for_http_C2S.pcap rename to test/decoders/http/benchmarks/pcap/http_tunnel_for_http_C2S.pcap diff --git a/benchmarks/pcap/http/http_tunnel_for_http_S2C.pcap b/test/decoders/http/benchmarks/pcap/http_tunnel_for_http_S2C.pcap similarity index 100% rename from benchmarks/pcap/http/http_tunnel_for_http_S2C.pcap rename to test/decoders/http/benchmarks/pcap/http_tunnel_for_http_S2C.pcap diff --git a/benchmarks/pcap/http/http_tunnel_for_pop3.pcap b/test/decoders/http/benchmarks/pcap/http_tunnel_for_pop3.pcap similarity index 100% rename from benchmarks/pcap/http/http_tunnel_for_pop3.pcap rename to test/decoders/http/benchmarks/pcap/http_tunnel_for_pop3.pcap diff --git a/benchmarks/pcap/http/http_tunnel_s2c_only_hdr.pcap b/test/decoders/http/benchmarks/pcap/http_tunnel_s2c_only_hdr.pcap similarity index 100% rename from benchmarks/pcap/http/http_tunnel_s2c_only_hdr.pcap rename to test/decoders/http/benchmarks/pcap/http_tunnel_s2c_only_hdr.pcap diff --git a/benchmarks/pcap/http/http_upgrade_http2.pcap b/test/decoders/http/benchmarks/pcap/http_upgrade_http2.pcap similarity index 100% rename from benchmarks/pcap/http/http_upgrade_http2.pcap rename to test/decoders/http/benchmarks/pcap/http_upgrade_http2.pcap diff --git a/benchmarks/pcap/http/http_upgrade_websocket.pcap b/test/decoders/http/benchmarks/pcap/http_upgrade_websocket.pcap similarity index 100% rename from benchmarks/pcap/http/http_upgrade_websocket.pcap rename to test/decoders/http/benchmarks/pcap/http_upgrade_websocket.pcap diff --git a/benchmarks/pcap/http/http_url_test_with_host.pcap b/test/decoders/http/benchmarks/pcap/http_url_test_with_host.pcap similarity index 100% rename from benchmarks/pcap/http/http_url_test_with_host.pcap rename to test/decoders/http/benchmarks/pcap/http_url_test_with_host.pcap diff --git a/benchmarks/pcap/http/http_url_test_without_host.pcap b/test/decoders/http/benchmarks/pcap/http_url_test_without_host.pcap similarity index 100% rename from benchmarks/pcap/http/http_url_test_without_host.pcap rename to test/decoders/http/benchmarks/pcap/http_url_test_without_host.pcap diff --git a/benchmarks/pcap/http/http_url_test_without_host_v6.pcap b/test/decoders/http/benchmarks/pcap/http_url_test_without_host_v6.pcap similarity index 100% rename from benchmarks/pcap/http/http_url_test_without_host_v6.pcap rename to test/decoders/http/benchmarks/pcap/http_url_test_without_host_v6.pcap diff --git a/benchmarks/pcap/http/http_whitespace_before_header.pcap b/test/decoders/http/benchmarks/pcap/http_whitespace_before_header.pcap similarity index 100% rename from benchmarks/pcap/http/http_whitespace_before_header.pcap rename to test/decoders/http/benchmarks/pcap/http_whitespace_before_header.pcap diff --git a/benchmarks/pcap/http/http_zlib_deadlock.pcap b/test/decoders/http/benchmarks/pcap/http_zlib_deadlock.pcap similarity index 100% rename from benchmarks/pcap/http/http_zlib_deadlock.pcap rename to test/decoders/http/benchmarks/pcap/http_zlib_deadlock.pcap diff --git a/benchmarks/pcap/http/non_http.pcap b/test/decoders/http/benchmarks/pcap/non_http.pcap similarity index 100% rename from benchmarks/pcap/http/non_http.pcap rename to test/decoders/http/benchmarks/pcap/non_http.pcap diff --git a/test/decoders/http/http_decoder_gtest.cpp b/test/decoders/http/http_decoder_gtest.cpp deleted file mode 100644 index 68f275f..0000000 --- a/test/decoders/http/http_decoder_gtest.cpp +++ /dev/null @@ -1,64 +0,0 @@ -#include -#include -#include -#include "http.h" -#include "http_decoder_private.h" - -void httpd_url_decode(const char *string, size_t length, char *ostring, size_t *olen); -TEST(http_url_decoder, none) -{ - char decode_url_buf[2048]; - size_t decode_url_len = sizeof(decode_url_buf); - const char *encode_url = "https://docs.geedge.net/#all-updates"; - httpd_url_decode(encode_url, strlen(encode_url), decode_url_buf, &decode_url_len); - - EXPECT_STREQ("https://docs.geedge.net/#all-updates", decode_url_buf); - EXPECT_EQ(decode_url_len, strlen("https://docs.geedge.net/#all-updates")); -} - -TEST(http_url_decoder, simple) -{ - char decode_url_buf[2048]; - size_t decode_url_len = sizeof(decode_url_buf); - const char *encode_url = "http://a.b.cn/%A1%B2%C3%D4"; - httpd_url_decode(encode_url, strlen(encode_url), decode_url_buf, &decode_url_len); - - const unsigned char expect_result[] = - {0x68, 0x74, 0x74, 0x70, - 0x3A, 0x2F, 0x2F, 0x61, - 0x2E, 0x62, 0x2E, 0x63, - 0x6E, 0x2F, 0xA1, 0xB2, - 0xC3, 0xD4, 0x00 - }; - EXPECT_STREQ((char *)expect_result, decode_url_buf); - EXPECT_EQ(decode_url_len, sizeof(expect_result)-1); -} - -TEST(http_url_decoder, chinese1) -{ - char decode_url_buf[2048]; - size_t decode_url_len = sizeof(decode_url_buf); - const char *encode_url = "http://www.baidu.com/%E6%B5%8B%E8%AF%95%E4%B8%AD%E6%96%87%E8%A7%A3%E7%A0%81"; - httpd_url_decode(encode_url, strlen(encode_url), decode_url_buf, &decode_url_len); - - EXPECT_STREQ("http://www.baidu.com/\xE6\xB5\x8B\xE8\xAF\x95\xE4\xB8\xAD\xE6\x96\x87\xE8\xA7\xA3\xE7\xA0\x81", decode_url_buf); - EXPECT_EQ(decode_url_len, strlen("http://www.baidu.com/\xE6\xB5\x8B\xE8\xAF\x95\xE4\xB8\xAD\xE6\x96\x87\xE8\xA7\xA3\xE7\xA0\x81")); -} - -TEST(http_url_decoder, chinese2) -{ - char decode_url_buf[2048]; - size_t decode_url_len = sizeof(decode_url_buf); - const char *encode_url = "http%3A%2F%2Fwww.baidu.com%2F%E7%BC%96%E8%A7%A3%E7%A0%81%E6%B5%8B%E8%AF%95%E5%93%88%E5%93%88"; - httpd_url_decode(encode_url, strlen(encode_url), decode_url_buf, &decode_url_len); - - EXPECT_STREQ("http://www.baidu.com/\xE7\xBC\x96\xE8\xA7\xA3\xE7\xA0\x81\xE6\xB5\x8B\xE8\xAF\x95\xE5\x93\x88\xE5\x93\x88", decode_url_buf); - EXPECT_EQ(decode_url_len, strlen("http://www.baidu.com/\xE7\xBC\x96\xE8\xA7\xA3\xE7\xA0\x81\xE6\xB5\x8B\xE8\xAF\x95\xE5\x93\x88\xE5\x93\x88")); -} - - -int main(int argc, char const *argv[]) -{ - ::testing::InitGoogleTest(&argc, (char **)argv); - return RUN_ALL_TESTS(); -} diff --git a/test/decoders/http/http_decoder_perf_main.cpp b/test/decoders/http/http_decoder_perf_main.cpp deleted file mode 100644 index 7d314e9..0000000 --- a/test/decoders/http/http_decoder_perf_main.cpp +++ /dev/null @@ -1,226 +0,0 @@ -#ifdef __cplusplus -extern "C" -{ -#endif -#include -#include -#include -#include -#ifdef __cplusplus -} -#endif -#include -#include -#include -#include -#include -#include -#include "http_decoder_gtest.h" - -#define TIME_MEASURE 1 -#if TIME_MEASURE -#define TIME_START() struct timespec _start_time, _end_time; long long time_diff_ns; clock_gettime(CLOCK_REALTIME, &_start_time) -#define TIME_UPDATE() clock_gettime(CLOCK_REALTIME, &_start_time) -#define TIME_DIFF() \ - do { \ - clock_gettime(CLOCK_REALTIME, &_end_time); \ - if (_end_time.tv_sec == _start_time.tv_sec)\ - {\ - time_diff_ns = (_end_time.tv_nsec - _start_time.tv_nsec);\ - }else{\ - time_diff_ns = (_end_time.tv_sec * 1000 * 1000 * 1000 + _end_time.tv_nsec) - (_start_time.tv_sec * 1000 * 1000 * 1000 + _start_time.tv_nsec);\ - }\ - }while (0) -#else -#define TIME_START() -#define TIME_DIFF() -#endif - -struct test_packet{ - const char *payload; - size_t payload_len; - u_int8_t dir; -}; -extern "C" void *http_decoder_perf_plug_init(struct stellar *st); -extern "C" void *http_decoder_init(struct stellar *st); -extern "C" void http_decoder_exit(void *plugin_env); -extern "C" void *httpd_session_ctx_new_cb(struct session *sess, void *plugin_env); -extern "C" void _httpd_ex_data_free_cb(struct session *s, int idx,void *ex_data, void *arg); -extern "C" void http_decoder_tcp_stream_msg_cb(struct session *sess, int topic_id, const void *msg, void *no_use_ctx, void *plugin_env); -extern "C" void http_decoder_perf_entry(struct session *sess, int topic_id, const void *raw_msg, void *per_session_ctx, void *plugin_env); - -static struct fieldstat_easy *fs4_instance; -static int fs4_simple_id; -static int fs4_long_long_url_id; -static int fs4_frag_id; - -static void perf_test_loop(struct stellar *st, int tcp_stream_topic_id, struct test_packet *test_payload, int test_payload_index_max, int fs4_metric_id) -{ - int idx = 0; - struct session *sess; - TIME_START(); - sess = stellar_session_new(st, tcp_stream_topic_id, test_payload[idx].payload, test_payload[idx].payload_len, test_payload[idx].dir); - idx++; - TIME_DIFF(); - fieldstat_easy_histogram_record(fs4_instance, 0, fs4_metric_id, NULL, 0, time_diff_ns); - - for(; idx < test_payload_index_max; idx++){ - TIME_UPDATE(); - stellar_session_active(st, sess, tcp_stream_topic_id, test_payload[idx].payload, test_payload[idx].payload_len, test_payload[idx].dir); - TIME_DIFF(); - fieldstat_easy_histogram_record(fs4_instance, 0, fs4_metric_id, NULL, 0, time_diff_ns); - } - - TIME_UPDATE(); - stellar_session_close(st, sess, tcp_stream_topic_id); - TIME_DIFF(); - fieldstat_easy_histogram_record(fs4_instance, 0, fs4_metric_id, NULL, 0, time_diff_ns); -} - -#define SET_DATA_LENGTH_DIR(pkt_payload, cstr_in_heap, cur_dir, tmp_index) \ - pkt_payload.payload = cstr_in_heap; \ - pkt_payload.payload_len = strlen(pkt_payload.payload); \ - pkt_payload.dir = cur_dir;\ - tmp_index++; - -static void init_test_data_simple(struct test_packet *test_payload, int *index) -{ - int tmp_index = 0; - const char *c2s_payload = strdup("GET / HTTP/1.1\r\nHost: www.simple.com\r\nConnection: keep-alive\r\nCache-Control: max-age=0\r\nUpgrade-Insecure-Requests: 1\r\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8\r\nAccept-Encoding: gzip, deflate\r\nAccept-Language: zh-CN,zh;q=0.9\r\n\r\n"); - SET_DATA_LENGTH_DIR(test_payload[tmp_index], c2s_payload, FLOW_DIRECTION_C2S , tmp_index); - - const char *content = "Hello, http decoder perf test simple!!!"; - int content_length = strlen(content); - char *s2c_payload = (char *)malloc(1024); - snprintf(s2c_payload, 1024, "HTTP/1.1 200 OK\r\nServer: Apache-Coyote/1.1\r\nConnection: keep-alive\r\nDate: Sat, 01 May 2024 01:36:57 GMT\r\nContent-Type: text/html;charset=UTF-8\r\nContent-Length: %d\r\n\r\n%s", content_length, content); - - SET_DATA_LENGTH_DIR(test_payload[tmp_index], s2c_payload, FLOW_DIRECTION_S2C, tmp_index); - *index = tmp_index; -} - -static void init_test_data_frag(struct test_packet *test_payload, int *index) -{ - int tmp_index = 0; - /* c2s */ - SET_DATA_LENGTH_DIR(test_payload[tmp_index], strdup("GET / HTTP/1.1\r\n"), FLOW_DIRECTION_C2S , tmp_index); - SET_DATA_LENGTH_DIR(test_payload[tmp_index], strdup("Host: www.fragment.com\r\n"), FLOW_DIRECTION_C2S , tmp_index); - SET_DATA_LENGTH_DIR(test_payload[tmp_index], strdup("Cache-Control: max-age=0\r\n"), FLOW_DIRECTION_C2S , tmp_index); - SET_DATA_LENGTH_DIR(test_payload[tmp_index], strdup("Connection: keep-alive\r\n"), FLOW_DIRECTION_C2S , tmp_index); - SET_DATA_LENGTH_DIR(test_payload[tmp_index], strdup("Referer: http://fragment.com/register.jsp?redirect:http://aa.bb.cc.dd.com/?\r\n"), FLOW_DIRECTION_C2S , tmp_index); - SET_DATA_LENGTH_DIR(test_payload[tmp_index], strdup("User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299\r\n"), FLOW_DIRECTION_C2S , tmp_index); - SET_DATA_LENGTH_DIR(test_payload[tmp_index], strdup("Cookie: JSESSIONID=385C79E211D561C0CA13D90F150F603D34875GH87FSHG8S7RTHG74875GHS8R7THG87SRTH\r\n"), FLOW_DIRECTION_C2S , tmp_index); - SET_DATA_LENGTH_DIR(test_payload[tmp_index], strdup("Accept: */*\r\n"), FLOW_DIRECTION_C2S , tmp_index); - SET_DATA_LENGTH_DIR(test_payload[tmp_index], strdup("Accept-Encoding: gzip, deflate\r\n"), FLOW_DIRECTION_C2S , tmp_index); - SET_DATA_LENGTH_DIR(test_payload[tmp_index], strdup("\r\n"), FLOW_DIRECTION_C2S , tmp_index); //header EOF - - /* s2c */ - SET_DATA_LENGTH_DIR(test_payload[tmp_index], strdup("HTTP/1.1 200 OK\r\n"), FLOW_DIRECTION_S2C, tmp_index); - SET_DATA_LENGTH_DIR(test_payload[tmp_index], strdup("Server: Apache-Coyote/1.1\r\n"), FLOW_DIRECTION_S2C, tmp_index); - SET_DATA_LENGTH_DIR(test_payload[tmp_index], strdup("Connection: keep-alive\r\n"), FLOW_DIRECTION_S2C, tmp_index); - SET_DATA_LENGTH_DIR(test_payload[tmp_index], strdup("Content-Type: text/html;charset=UTF-8\r\n"), FLOW_DIRECTION_S2C, tmp_index); - SET_DATA_LENGTH_DIR(test_payload[tmp_index], strdup("Date: Sat, 01 May 2024 01:36:57 GMT\r\n"), FLOW_DIRECTION_S2C, tmp_index); - - char *cont_len_buf = (char *)malloc(1024); - const char *s2c_payload = strdup("Hello, http decoder perf test fragment!!!"); - int content_length = strlen(s2c_payload); - snprintf(cont_len_buf, 1024, "Content-Length: %d\r\n", content_length); - SET_DATA_LENGTH_DIR(test_payload[tmp_index], cont_len_buf, FLOW_DIRECTION_S2C, tmp_index); - - SET_DATA_LENGTH_DIR(test_payload[tmp_index], strdup("\r\n"), FLOW_DIRECTION_S2C, tmp_index); //header EOF - SET_DATA_LENGTH_DIR(test_payload[tmp_index], s2c_payload, FLOW_DIRECTION_S2C, tmp_index); - *index = tmp_index; -} - -static void init_test_data_long_long_url(struct test_packet *test_payload, int *index) -{ - int tmp_index = 0; - - const char *request_line_frag1 = strdup("GET /long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/\ -/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/\ -/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/\ -/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/\ -/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/\ -/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/\ -/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/\ -/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/"); - SET_DATA_LENGTH_DIR(test_payload[tmp_index], request_line_frag1, FLOW_DIRECTION_C2S , tmp_index); - - const char *request_line_frag2 = strdup("long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/\ -/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/\ -/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long/long.index.html HTTP/1.1\r\n"); - SET_DATA_LENGTH_DIR(test_payload[tmp_index], request_line_frag2, FLOW_DIRECTION_C2S , tmp_index); - - const char *request_line_frag3 = strdup("Host: www.long-long-url.com\r\nConnection: keep-alive\r\nCache-Control: max-age=0\r\nUpgrade-Insecure-Requests: 1\r\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8\r\nAccept-Encoding: gzip, deflate\r\nAccept-Language: zh-CN,zh;q=0.9\r\n\r\n"); - SET_DATA_LENGTH_DIR(test_payload[tmp_index], request_line_frag3, FLOW_DIRECTION_C2S , tmp_index); - - const char *content = "Hello, http decoder perf test long long url!!!"; - int content_length = strlen(content); - char *s2c_payload = (char *)malloc(1024); - snprintf(s2c_payload, 1024, "HTTP/1.1 200 OK\r\nServer: Apache-Coyote/1.1\r\nConnection: keep-alive\r\nDate: Sat, 01 May 2024 01:36:57 GMT\r\nContent-Type: text/html;charset=UTF-8\r\nContent-Length: %d\r\n\r\n%s", content_length, content); - - SET_DATA_LENGTH_DIR(test_payload[tmp_index], s2c_payload, FLOW_DIRECTION_S2C, tmp_index); - *index = tmp_index; -} - -static void test_payload_free(struct test_packet *test_payload, int test_payload_index_max) -{ - for(int i = 0; i < test_payload_index_max; i++){ - if(test_payload[i].payload){ - free((void *)test_payload[i].payload); - } - } -} - -static void perf_stat_init(void) -{ - fs4_instance = fieldstat_easy_new(1, "http_decoder_test", NULL, 0); - fieldstat_easy_enable_auto_output(fs4_instance, "./httpd_hisgram.json", 1); - - fs4_simple_id = fieldstat_easy_register_histogram(fs4_instance, "simple", 1, 99999999, 5); - fs4_frag_id = fieldstat_easy_register_histogram(fs4_instance, "frag", 1, 99999999, 5); - fs4_long_long_url_id = fieldstat_easy_register_histogram(fs4_instance, "long-url", 1, 99999999, 5); -} - -int main(int argc, char const *argv[]) -{ - struct test_packet test_payload_simple [4] = {}; - int payload_index_simple = 0; - struct test_packet test_payload_long_long_url [8] = {}; - int payload_index_long_long_url = 0; - struct test_packet test_payload_frag [32] = {}; - int payload_index_frag = 0; - - struct stellar *st = stellar_init(); - - int tcp_stream_topic_id = stellar_mq_get_topic_id(st, TOPIC_TCP_STREAM); - - if(stellar_load_plugin(st, http_decoder_init) < 0){ - fprintf(stderr, "load plugin 'http_decoder_init' failed\n"); - return -1; - } - - if(stellar_load_plugin(st, http_decoder_perf_plug_init) < 0){ - fprintf(stderr, "load plugin 'http_decoder_perf_plug_init' failed\n"); - return -1; - } - - perf_stat_init(); - init_test_data_simple(test_payload_simple, &payload_index_simple); - init_test_data_long_long_url(test_payload_long_long_url, &payload_index_long_long_url); - init_test_data_frag(test_payload_frag, &payload_index_frag); - - // while(1){ - for(int i = 0; i < 1000; i++){ - perf_test_loop(st, tcp_stream_topic_id,test_payload_simple, payload_index_simple, fs4_simple_id); - perf_test_loop(st, tcp_stream_topic_id,test_payload_long_long_url, payload_index_long_long_url, fs4_long_long_url_id); - perf_test_loop(st, tcp_stream_topic_id,test_payload_frag, payload_index_frag, fs4_frag_id); - } - - test_payload_free(test_payload_simple, sizeof(test_payload_simple)/sizeof(struct test_packet)) ; - test_payload_free(test_payload_long_long_url, sizeof(test_payload_long_long_url)/sizeof(struct test_packet)) ; - test_payload_free(test_payload_frag, sizeof(test_payload_frag)/sizeof(struct test_packet)) ; - stellar_destroy(st); - sleep(1); - fieldstat_easy_free(fs4_instance); - return 0; -} diff --git a/test/decoders/http/http_decoder_perf_plug.cpp b/test/decoders/http/http_decoder_perf_plug.cpp deleted file mode 100644 index 338acaa..0000000 --- a/test/decoders/http/http_decoder_perf_plug.cpp +++ /dev/null @@ -1,143 +0,0 @@ -#include "http.h" -#include -#include -#include -#include -#include - -#ifdef __cplusplus -extern "C" -{ -#include "http_decoder_private.h" -#include "cJSON.h" -} -#endif - -#define MAX_KEY_STR_LEN 2048 - -static int g_result_count = 0; -static int g_header_count = 1; -static int g_exdata_idx = -1; -static int g_topic_id = -1; -static int g_plugin_id = -1; - -#define DEBUG_PRINT(fmt, ...) //printf(fmt, ##__VA_ARGS__) - -extern "C" void http_decoder_perf_entry(struct session *sess, int topic_id, const void *raw_msg, void *per_session_ctx, void *plugin_env) -{ - struct http_request_line req_line = {0}; - struct http_response_line res_line = {0}; - struct http_header header = {0}; - hstring url = {}; - hstring body = {}; - struct http_message *msg = (struct http_message *)raw_msg; - enum http_message_type msg_type = http_message_type_get(msg); - void *ret1, *ret2; - - switch (msg_type) - { - case HTTP_MESSAGE_REQ_LINE: - DEBUG_PRINT("---------------------------------------------------------------\n"); - http_message_request_line_get0(msg, &req_line); - if (req_line.uri.iov_base) - { - DEBUG_PRINT("req_line.method.iov_base: %.*s\n", req_line.method.iov_len, req_line.method.iov_base); - ret1 = memmem(req_line.method.iov_base, req_line.method.iov_len, "PUT", 3); - DEBUG_PRINT("req_line.version.iov_base: %.*s\n", req_line.version.iov_len, req_line.version.iov_base); - } - break; - case HTTP_MESSAGE_REQ_HEADER: - while (http_message_header_next(msg, &header) >= 0) - { - ret1 = memmem(header.key.iov_base, header.key.iov_len, "key", 3); - ret2 = memmem(header.val.iov_base, header.val.iov_len, "val", 3); - DEBUG_PRINT("REQ header: %.*s : %.*s\n", (int)header.key.iov_len, header.key.iov_base, (int)header.val.iov_len, header.val.iov_base); - } - http_message_raw_url_get0(msg, &url); - if(url.iov_base && url.iov_len){ - DEBUG_PRINT("URL: %.*s\n", url.iov_len, url.iov_base); - } - break; - case HTTP_MESSAGE_REQ_BODY: - // http_message_get_request_raw_body(msg, &body); - // output_http_body(&body, 0); - http_message_decompress_body_get0(msg, &body); - // output_http_body(&body, 1); - ret1 = memmem(body.iov_base, body.iov_len, "", 7); - break; - case HTTP_MESSAGE_RES_LINE: - http_message_response_line_get0(msg, &res_line); - ret1 = memmem(res_line.status.iov_base, res_line.status.iov_len, "OK", 2); - DEBUG_PRINT("res_line.status.iov_base: %.*s\n", (int)res_line.status.iov_len, res_line.status.iov_base); - break; - case HTTP_MESSAGE_RES_HEADER: - while (http_message_header_next(msg, &header) >= 0) - { - ret1 = memmem(header.key.iov_base, header.key.iov_len, "key", 3); - ret2 = memmem(header.val.iov_base, header.val.iov_len, "val", 3); - DEBUG_PRINT("RES header: %.*s : %.*s\n", (int)header.key.iov_len, header.key.iov_base, (int)header.val.iov_len, header.val.iov_base); - } - break; - case HTTP_MESSAGE_RES_BODY_START: - case HTTP_MESSAGE_RES_BODY: - case HTTP_MESSAGE_RES_BODY_END: - http_message_raw_body_get0(msg, &body); - if(body.iov_base!=NULL && body.iov_len > 0){ - DEBUG_PRINT("res raw body: %.*s\n", body.iov_len, body.iov_base); - } - // output_http_body(&body, 0); - http_message_decompress_body_get0(msg, &body); - if(body.iov_base!=NULL && body.iov_len > 0){ - // output_http_body(&body, 1); - ret1 = memmem(body.iov_base, body.iov_len, "", 7); - DEBUG_PRINT("res unzip body: %.*s\n", body.iov_len, body.iov_base); - DEBUG_PRINT("---------------------------------------------------------------\n"); - } - break; - // to do: check payload - default: - break; - } - return; -} - -static on_session_msg_cb_func*g_entry_fun = &http_decoder_perf_entry; - -static void http_decoder_test_exdata_free(int idx, void *ex_ptr, void *arg) -{ - return; -} - -extern "C" void *http_decoder_perf_plug_init(struct stellar *st) -{ - g_plugin_id = stellar_session_plugin_register(st, NULL, NULL, NULL); - g_exdata_idx = stellar_exdata_new_index(st, "HTTP_DECODER_REQ_TEST", - http_decoder_test_exdata_free, - NULL); - if (g_exdata_idx < 0) - { - printf("[%s:%d]: can't get http_decoder exdata index !!!\n", __FUNCTION__, __LINE__); - exit(-1); - } - - g_topic_id = stellar_mq_get_topic_id(st, "HTTP_DECODER_MESSAGE"); - if (g_topic_id < 0) - { - printf("[%s:%d]: can't get http_decoder topic id !!!\n", __FUNCTION__, __LINE__); - exit(-1); - } - - stellar_session_mq_subscribe(st, g_topic_id, g_entry_fun, g_plugin_id); - // printf("http_decoder_test_init OK!\n"); - - return NULL; -} - -extern "C" void http_decoder_perf_plug_exit(void *test_ctx) -{ - if (test_ctx != NULL) - { - FREE(test_ctx); - } - printf("http_decoder_perf plug exit OK!\n"); -} diff --git a/test/decoders/http/http_gtest.cpp b/test/decoders/http/http_gtest.cpp new file mode 100644 index 0000000..4f99fb6 --- /dev/null +++ b/test/decoders/http/http_gtest.cpp @@ -0,0 +1,358 @@ +#include +#include +#include +#include "zlib.h" +#include "md5/md5.h" +#include "http.h" +#include "http_decoder_private.h" +#include "brotli/decode.h" +#include "brotli/encode.h" +#include "event2/buffer.h" + +#define ZIP_UNZIP_TEST_DATA_LEN (1024 * 1024) +void httpd_url_decode(const char *string, size_t length, char *ostring, size_t *olen); +TEST(http_url_decoder, none_encode) +{ + char decoded_url_buf[2048] = {}; + size_t decode_url_buf_len = sizeof(decoded_url_buf); + const char *encode_url = "https://docs.geedge.net/#all-updates"; + size_t decoded_url_len = http_url_decode(encode_url, strlen(encode_url), decoded_url_buf, decode_url_buf_len); + EXPECT_EQ(decoded_url_len, strlen("https://docs.geedge.net/#all-updates")); + EXPECT_STREQ("https://docs.geedge.net/#all-updates", decoded_url_buf); +} + +TEST(http_url_decoder, simple) +{ + char decoded_url_buf[2048] = {}; + size_t decoded_url_buf_len = sizeof(decoded_url_buf); + const char *encode_url = "http://a.b.cn/%A1%B2%C3%D4"; + size_t decoded_url_len = http_url_decode(encode_url, strlen(encode_url), decoded_url_buf, decoded_url_buf_len); + + const unsigned char expect_result[] = + {0x68, 0x74, 0x74, 0x70, + 0x3A, 0x2F, 0x2F, 0x61, + 0x2E, 0x62, 0x2E, 0x63, + 0x6E, 0x2F, 0xA1, 0xB2, + 0xC3, 0xD4, 0x00}; + EXPECT_EQ(decoded_url_len, sizeof(expect_result) - 1); + EXPECT_EQ(0, memcmp(expect_result, decoded_url_buf, decoded_url_len)); +} + +TEST(http_url_decoder, chinese1) +{ + char decoded_url_buf[2048] = {}; + size_t decoded_url_buf_len = sizeof(decoded_url_buf); + const char *encode_url = "http://www.baidu.com/%E6%B5%8B%E8%AF%95%E4%B8%AD%E6%96%87%E8%A7%A3%E7%A0%81"; + size_t decoded_url_len = http_url_decode(encode_url, strlen(encode_url), decoded_url_buf, decoded_url_buf_len); + + EXPECT_STREQ("http://www.baidu.com/\xE6\xB5\x8B\xE8\xAF\x95\xE4\xB8\xAD\xE6\x96\x87\xE8\xA7\xA3\xE7\xA0\x81", decoded_url_buf); + EXPECT_EQ(decoded_url_len, strlen("http://www.baidu.com/\xE6\xB5\x8B\xE8\xAF\x95\xE4\xB8\xAD\xE6\x96\x87\xE8\xA7\xA3\xE7\xA0\x81")); +} + +TEST(http_url_decoder, chinese2) +{ + char decoded_url_buf[2048]; + size_t decoded_url_buf_len = sizeof(decoded_url_buf); + const char *encode_url = "http%3A%2F%2Fwww.baidu.com%2F%E7%BC%96%E8%A7%A3%E7%A0%81%E6%B5%8B%E8%AF%95%E5%93%88%E5%93%88"; + size_t decoded_url_len = http_url_decode(encode_url, strlen(encode_url), decoded_url_buf, decoded_url_buf_len); + + EXPECT_EQ(0, memcmp("http://www.baidu.com/\xE7\xBC\x96\xE8\xA7\xA3\xE7\xA0\x81\xE6\xB5\x8B\xE8\xAF\x95\xE5\x93\x88\xE5\x93\x88", decoded_url_buf, decoded_url_len)); + EXPECT_EQ(decoded_url_len, strlen("http://www.baidu.com/\xE7\xBC\x96\xE8\xA7\xA3\xE7\xA0\x81\xE6\xB5\x8B\xE8\xAF\x95\xE5\x93\x88\xE5\x93\x88")); +} + +TEST(http, event_buffer) +{ + struct evbuffer *evbuf = evbuffer_new(); + + evbuffer_add(evbuf, "hello", 5); + + size_t len = evbuffer_get_length(evbuf); + EXPECT_EQ(len, 5); + char outbuf[16]; + len = evbuffer_copyout(evbuf, outbuf, sizeof(outbuf)); + EXPECT_EQ(len, 5); + EXPECT_EQ(0, memcmp(outbuf, "hello", 5)); + + evbuffer_add(evbuf, ",", 1); + evbuffer_add(evbuf, "world", 5); + + len = evbuffer_copyout(evbuf, outbuf, sizeof(outbuf)); + EXPECT_EQ(len, 11); + EXPECT_EQ(0, memcmp(outbuf, "hello,world", 11)); + + evbuffer_free(evbuf); +} + +static int http_compress_use_deflate(unsigned char *indata, size_t indata_len, unsigned char *zip_data, size_t *zip_data_len) +{ +#define ZIP_CHUNK 4096 +#define UZIP_CHUNK 16384 + unsigned have; + z_stream strm = {}; + + /* allocate deflate state */ + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + + int ret = deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -MAX_WBITS, 9, Z_DEFAULT_STRATEGY); /* no bad return value */ + if (ret != Z_OK) + { + return -1; + } + + strm.next_in = indata; + strm.avail_in = indata_len; + + size_t read_raw_data_len = 0; + size_t ziped_len = 0; + int flush_mode; + do + { + /* run deflate() on input until output buffer not full, finish + compression if all of source has been read in */ + strm.avail_in = MIN(indata_len - read_raw_data_len, ZIP_CHUNK); + strm.next_in = indata + read_raw_data_len; + flush_mode = (read_raw_data_len + ZIP_CHUNK < indata_len) ? Z_NO_FLUSH : Z_FINISH; + do + { + unsigned char local_chunk_buf[UZIP_CHUNK]; + strm.avail_out = UZIP_CHUNK; + strm.next_out = local_chunk_buf; + ret = deflate(&strm, flush_mode); + assert(ret != Z_STREAM_ERROR); + have = UZIP_CHUNK - strm.avail_out; + if (have > 0) + { + memcpy(zip_data + ziped_len, local_chunk_buf, have); + ziped_len += have; + } + } while (strm.avail_out == 0); + read_raw_data_len += ZIP_CHUNK; + } while (read_raw_data_len < indata_len); + + *zip_data_len = ziped_len; + assert(strm.avail_in == 0); /* all input will be used */ + /* done when last data in file processed */ + + /* clean up and return */ + (void)deflateEnd(&strm); + return Z_OK; +} + +static int http_compress_use_brotli(unsigned char *input_data, size_t input_length, unsigned char *compressed_data, size_t *compressed_data_len) +{ + // Allocate memory for the compressed data + size_t compressed_length = BrotliEncoderMaxCompressedSize(input_length); + + // Create a Brotli encoder state + BrotliEncoderState *encoder = BrotliEncoderCreateInstance(NULL, NULL, NULL); + if (!encoder) + { + fprintf(stderr, "Failed to create Brotli encoder.\n"); + return 1; + } + // Set up the Brotli encoder + BrotliEncoderSetParameter(encoder, BROTLI_PARAM_QUALITY, 11); + + // Compress the data + size_t available_in = input_length; + const uint8_t *next_in = (const uint8_t *)input_data; + size_t available_out = compressed_length; + uint8_t *next_out = compressed_data; + + BrotliEncoderOperation op = BROTLI_OPERATION_FINISH; + if (!BrotliEncoderCompressStream(encoder, op, &available_in, &next_in, &available_out, &next_out, NULL)) + { + fprintf(stderr, "Brotli compression failed.\n"); + BrotliEncoderDestroyInstance(encoder); + *compressed_data_len = 0; + return -1; + } + // Calculate actual compressed size + size_t actual_compressed_length = compressed_length - available_out; + // printf("Original size: %zu\n", input_length); + // printf("Compressed size: %zu\n", actual_compressed_length); + + // Clean up + BrotliEncoderDestroyInstance(encoder); + *compressed_data_len = actual_compressed_length; + return 0; +} + +/* High Compression Ratio */ +static unsigned char *http_build_ascii_text(size_t data_len) +{ + unsigned char *raw_data = (unsigned char *)malloc(data_len); + for (int i = 0; i < (int)data_len; i++) + { + raw_data[i] = 'A' + i % 26; + } + return raw_data; +} + +/* Low Compression Ratio */ +static unsigned char *http_build_random_content(size_t data_len) +{ + srand(12345678); + unsigned char *raw_data = (unsigned char *)malloc(data_len); + for (int i = 0; i < (int)data_len; i++) + { + raw_data[i] = (unsigned char)(rand() ^ i); + } + return raw_data; +} + +static int http_zip_unzip_test(unsigned char *raw_data, size_t raw_data_len, enum http_content_encoding encoding, int input_zip_block_size) +{ + unsigned char raw_data_md5sum[16], zip_unzip_data_md5sum[16]; + MD5_CTX raw_data_md5_ctx = {}, zip_unzip_data_md5_ctx = {}; + + MD5Init(&raw_data_md5_ctx); + MD5Init(&zip_unzip_data_md5_ctx); + + MD5Update(&raw_data_md5_ctx, raw_data, raw_data_len); + MD5Final(raw_data_md5sum, &raw_data_md5_ctx); + + unsigned char *zip_data = (unsigned char *)malloc(raw_data_len * 2); + size_t zip_data_len = raw_data_len * 2; + if (HTTP_CONTENT_ENCODING_DEFLATE == encoding) + { + http_compress_use_deflate(raw_data, raw_data_len, zip_data, &zip_data_len); + } + else + { + http_compress_use_brotli(raw_data, raw_data_len, zip_data, &zip_data_len); + } + + struct http_content_decompress *decompress_ins = http_content_decompress_create(encoding); + unsigned char *unzip_data; + size_t unzip_data_len; + + size_t total_read_zip_size = 0; + size_t total_unziped_data_len = 0; + while (total_read_zip_size < zip_data_len) + { + unzip_data = NULL; + unzip_data_len = 0; + int decompress_input_len = MIN((size_t)input_zip_block_size, zip_data_len - total_read_zip_size); + int ret = http_content_decompress_write(decompress_ins, (char *)zip_data + total_read_zip_size, decompress_input_len, (char **)&unzip_data, &unzip_data_len); + if (ret < 0) + { + goto fail; + } + if (unzip_data && unzip_data_len) + { + MD5Update(&zip_unzip_data_md5_ctx, unzip_data, unzip_data_len); + total_unziped_data_len += unzip_data_len; + } + total_read_zip_size += decompress_input_len; + } + + free(zip_data); + http_content_decompress_destroy(decompress_ins); + MD5Final(zip_unzip_data_md5sum, &zip_unzip_data_md5_ctx); + + if (total_unziped_data_len != raw_data_len) + { + // printf("ERROR: zip-unzip data len:%zu, raw data len:%zu\n", total_unziped_data_len, raw_data_len); + return -1; + } + return memcmp(raw_data_md5sum, zip_unzip_data_md5sum, 16); + +fail: + free(zip_data); + http_content_decompress_destroy(decompress_ins); + MD5Final(zip_unzip_data_md5sum, &zip_unzip_data_md5_ctx); + return -1; +} + +/* This test takes a long time! use gtest --gtest_also_run_disabled_tests to run this test if needs. */ +TEST(http, DISABLED_deflate_ascii) +{ + int ret; + unsigned char *raw_data = http_build_ascii_text(ZIP_UNZIP_TEST_DATA_LEN); + for (int block = 1; block <= 1460; block++) + { + if (block % 10 == 0) + { + printf("raw content: ascii text, encoding: deflate, block range: %d - %d\n", block, block + 9); + } + ret = http_zip_unzip_test(raw_data, ZIP_UNZIP_TEST_DATA_LEN, HTTP_CONTENT_ENCODING_DEFLATE, block); + if (ret != 0) + { + printf("raw content: ascii text, encoding: deflate, block size:%d, result = failed!\n", block); + } + EXPECT_EQ(0, ret); + } + free(raw_data); +} + +/* This test takes a long time! use gtest --gtest_also_run_disabled_tests to run this test if needs. */ +TEST(http, DISABLED_deflate_random) +{ + int ret; + unsigned char *raw_data = http_build_random_content(ZIP_UNZIP_TEST_DATA_LEN); + for (int block = 1; block <= 1460; block++) + { + if (block % 10 == 0) + { + printf("raw content: random binary, encoding: deflate, block range: %d - %d\n", block, block + 9); + } + ret = http_zip_unzip_test(raw_data, ZIP_UNZIP_TEST_DATA_LEN, HTTP_CONTENT_ENCODING_DEFLATE, block); + if (ret != 0) + { + printf("raw content: random binary, encoding: deflate, block size:%d, result = failed!\n", block); + } + EXPECT_EQ(0, ret); + } + free(raw_data); +} + +/* This test takes a long time! use gtest --gtest_also_run_disabled_tests to run this test if needs. */ +TEST(http, DISABLED_brotli_random) +{ + int ret; + unsigned char *raw_data = http_build_random_content(ZIP_UNZIP_TEST_DATA_LEN); + for (int block = 33; block <= 1460; block++) + { + if (block % 10 == 0) + { + printf("raw content: random binary, encoding: brotli, block range: %d - %d\n", block, block + 9); + } + ret = http_zip_unzip_test(raw_data, ZIP_UNZIP_TEST_DATA_LEN, HTTP_CONTENT_ENCODING_BR, block); + if (ret != 0) + { + printf("raw content: random binary, encoding: brotli, block size:%d, result = failed!\n", block); + } + EXPECT_EQ(0, ret); + } + free(raw_data); +} + +/* This test takes a long time! use gtest --gtest_also_run_disabled_tests to run this test if needs. */ +TEST(http, DISABLED_brotli_ascii) +{ + int ret; + unsigned char *raw_data = http_build_ascii_text(ZIP_UNZIP_TEST_DATA_LEN); + for (int block = 33; block <= 1460; block++) + { + if (block % 10 == 0) + { + printf("raw content: ascii text, encoding: brotli, block range: %d - %d\n", block, block + 9); + } + ret = http_zip_unzip_test(raw_data, ZIP_UNZIP_TEST_DATA_LEN, HTTP_CONTENT_ENCODING_BR, block); + if (ret != 0) + { + printf("raw content: ascii text, encoding: brotli, block size:%d, result = failed!\n", block); + } + EXPECT_EQ(0, ret); + } + free(raw_data); +} + +int main(int argc, char const *argv[]) +{ + ::testing::InitGoogleTest(&argc, (char **)argv); + return RUN_ALL_TESTS(); +} diff --git a/test/decoders/http/http_decoder_gtest.h b/test/decoders/http/http_gtest.h similarity index 97% rename from test/decoders/http/http_decoder_gtest.h rename to test/decoders/http/http_gtest.h index eabae77..362bd39 100644 --- a/test/decoders/http/http_decoder_gtest.h +++ b/test/decoders/http/http_gtest.h @@ -10,13 +10,10 @@ extern "C" #ifdef __cplusplus } #endif -#include "http.h" -#include "md5.h" #include #include #include #include -#include "cJSON.h" #define TRUE 1 #define FLASE 0 diff --git a/test/decoders/http/http_decoder_test_plug.cpp b/test/decoders/http/http_test_plug.cpp similarity index 89% rename from test/decoders/http/http_decoder_test_plug.cpp rename to test/decoders/http/http_test_plug.cpp index 86be09f..1370b70 100644 --- a/test/decoders/http/http_decoder_test_plug.cpp +++ b/test/decoders/http/http_test_plug.cpp @@ -11,8 +11,8 @@ extern "C" { #include "cJSON.h" -#include "http_decoder_gtest.h" -#include "md5.h" +#include "http_gtest.h" +#include "md5/md5.h" #include "toml/toml.h" int commit_test_result_json(cJSON *node, const char *name); @@ -92,7 +92,7 @@ void output_http_res_line(struct http_response_line *res_line) printf("res_status:%s\n", tmp_str); } -void output_http_header(struct http_header *header) +void output_http_header(struct http_header_field *header) { char tmp_key[MAX_KEY_STR_LEN] = {0}; char tmp_val[MAX_KEY_STR_LEN] = {0}; @@ -168,11 +168,9 @@ void transaction_index_to_json(cJSON *ctx, int transaction_index) void req_line_to_json(cJSON *ctx, struct http_request_line *req_line) { - http_field_to_json(ctx, "method", (char *)req_line->method.iov_base, - req_line->method.iov_len); - http_field_to_json(ctx, "uri", (char *)req_line->uri.iov_base, req_line->uri.iov_len); - http_field_to_json(ctx, "req_version", (char *)req_line->version.iov_base, - req_line->version.iov_len); + http_field_to_json(ctx, "method", (char *)req_line->method, req_line->method_len); + http_field_to_json(ctx, "uri", (char *)req_line->uri, req_line->uri_len); + http_field_to_json(ctx, "req_version", (char *)req_line->version, req_line->version_len); cJSON_AddNumberToObject(ctx, "major_version", req_line->major_version); cJSON_AddNumberToObject(ctx, "minor_version", req_line->minor_version); @@ -180,36 +178,34 @@ void req_line_to_json(cJSON *ctx, struct http_request_line *req_line) void res_line_to_json(cJSON *ctx, struct http_response_line *res_line) { - http_field_to_json(ctx, "res_version", (char *)res_line->version.iov_base, - res_line->version.iov_len); - http_field_to_json(ctx, "res_status", (char *)res_line->status.iov_base, - res_line->status.iov_len); + http_field_to_json(ctx, "res_version", (char *)res_line->version, res_line->version_len); + http_field_to_json(ctx, "res_status", (char *)res_line->status, res_line->status_len); cJSON_AddNumberToObject(ctx, "major_version", res_line->major_version); cJSON_AddNumberToObject(ctx, "minor_version", res_line->minor_version); cJSON_AddNumberToObject(ctx, "status_code", res_line->status_code); } -void http_header_to_json(cJSON *ctx, struct http_header *header) +void http_header_to_json(cJSON *ctx, struct http_header_field *header) { char key[MAX_KEY_STR_LEN] = {0}; - if ((header->key.iov_base == NULL) || (header->val.iov_base == NULL)) + if ((header->name == NULL) || (header->value == NULL)) { return; } - memcpy(key, header->key.iov_base, header->key.iov_len); + memcpy(key, header->name, header->name_len); if (cJSON_HasObjectItem(ctx, key) == FALSE) { - http_field_to_json(ctx, key, (char *)header->val.iov_base, header->val.iov_len); + http_field_to_json(ctx, key, (char *)header->value, header->value_len); } else { // ctx already has the key, so rename key by key%d - char new_key[header->val.iov_len + 64] = {0}; - snprintf(new_key,sizeof(new_key), "%s%d", key, g_header_count++); - http_field_to_json(ctx, new_key, (char *)header->val.iov_base, header->val.iov_len); + char new_key[header->value_len + 64] = {0}; + snprintf(new_key, sizeof(new_key), "%s%d", key, g_header_count++); + http_field_to_json(ctx, new_key, (char *)header->value, header->value_len); } } @@ -222,28 +218,31 @@ void http_url_add_to_json(cJSON *ctx, struct http_message *msg) return; } - http_message_raw_url_get0(msg, &raw_url_result); + http_message_get0_raw_url(msg, (const char **)&raw_url_result, &raw_url_result.iov_len); if (raw_url_result.iov_base == NULL) { return; } - struct http_header raw_url_header_result = {}; + struct http_header_field raw_url_header_result = {}; - raw_url_header_result.key.iov_base = (char *)GTEST_HTTP_URL_NAME; - raw_url_header_result.key.iov_len = strlen(GTEST_HTTP_URL_NAME); - raw_url_header_result.val = raw_url_result; + raw_url_header_result.name = (char *)GTEST_HTTP_URL_NAME; + raw_url_header_result.name_len = strlen(GTEST_HTTP_URL_NAME); + raw_url_header_result.value = (char *)raw_url_result.iov_base; + raw_url_header_result.value_len = raw_url_result.iov_len; http_header_to_json(ctx, &raw_url_header_result); - hstring decode_url_result = {}; - struct http_header decode_url_header_result = {}; - http_message_decoded_url_get0(msg, &decode_url_result); - if (decode_url_result.iov_len != raw_url_result.iov_len) + char decoded_url_buffer[raw_url_result.iov_len] = {0}; + size_t decode_url_len = http_url_decode((const char *)raw_url_result.iov_base, raw_url_result.iov_len, decoded_url_buffer, sizeof(decoded_url_buffer)); + + if (decode_url_len < raw_url_result.iov_len) { - decode_url_header_result.key.iov_base = (char *)"__X_HTTP_DECODED_URL"; - decode_url_header_result.key.iov_len = strlen("__X_HTTP_DECODED_URL"); - decode_url_header_result.val = decode_url_result; + struct http_header_field decode_url_header_result = {}; + decode_url_header_result.name = (char *)"__X_HTTP_DECODED_URL"; + decode_url_header_result.name_len = strlen("__X_HTTP_DECODED_URL"); + decode_url_header_result.value = decoded_url_buffer; + decode_url_header_result.value_len = decode_url_len; http_header_to_json(ctx, &decode_url_header_result); } } @@ -395,10 +394,10 @@ extern "C" void http_decoder_test_entry(struct session *sess, int topic_id, cons (void)plugin_env; struct http_request_line req_line = {}; struct http_response_line res_line = {}; - struct http_header header = {}; + struct http_header_field header = {}; hstring body = {}; struct http_message *msg = (struct http_message *)raw_msg; - enum http_message_type msg_type = http_message_type_get(msg); + enum http_message_type msg_type = http_message_get_type(msg); struct gtest_plug_exdata_t *gtest_plug_exdata = (struct gtest_plug_exdata_t *)session_exdata_get(sess, g_exdata_idx); if (NULL == gtest_plug_exdata) @@ -422,11 +421,11 @@ extern "C" void http_decoder_test_entry(struct session *sess, int topic_id, cons { case HTTP_MESSAGE_REQ_LINE: commit_last_half_flow_data(gtest_plug_exdata, msg, HTTP_TRANSACTION_REQ, 0); - http_message_request_line_get0(msg, &req_line); + http_message_get0_request_line(msg, &req_line); req_line_to_json(gtest_plug_exdata->result_jnode[HTTP_TRANSACTION_REQ], &req_line); break; case HTTP_MESSAGE_REQ_HEADER: - while (http_message_header_next(msg, &header) >= 0) + while (http_message_get0_next_header(msg, &header) >= 0) { http_header_to_json(gtest_plug_exdata->result_jnode[HTTP_TRANSACTION_REQ], &header); } @@ -438,13 +437,13 @@ extern "C" void http_decoder_test_entry(struct session *sess, int topic_id, cons case HTTP_MESSAGE_REQ_BODY_START: case HTTP_MESSAGE_REQ_BODY: memset(&body, 0, sizeof(hstring)); - http_message_raw_body_get0(msg, &body); + http_message_get0_uncompressed_body(msg, (const char **)&body.iov_base, &body.iov_len); if (body.iov_base && body.iov_len) { append_http_payload(sess, gtest_plug_exdata, PAYLOAD_RAW, &body, HTTP_TRANSACTION_REQ); } // output_http_body(&body, 0); - http_message_decompress_body_get0(msg, &body); + http_message_get0_decompressed_body(msg, (const char **)&body.iov_base, &body.iov_len); // output_http_body(&body, 1); if (body.iov_base && body.iov_len) { @@ -453,11 +452,11 @@ extern "C" void http_decoder_test_entry(struct session *sess, int topic_id, cons break; case HTTP_MESSAGE_RES_LINE: commit_last_half_flow_data(gtest_plug_exdata, msg, HTTP_TRANSACTION_RES, 0); - http_message_response_line_get0(msg, &res_line); + http_message_get0_response_line(msg, &res_line); res_line_to_json(gtest_plug_exdata->result_jnode[HTTP_TRANSACTION_RES], &res_line); break; case HTTP_MESSAGE_RES_HEADER: - while (http_message_header_next(msg, &header) >= 0) + while (http_message_get0_next_header(msg, &header) >= 0) { http_header_to_json(gtest_plug_exdata->result_jnode[HTTP_TRANSACTION_RES], &header); } @@ -466,14 +465,14 @@ extern "C" void http_decoder_test_entry(struct session *sess, int topic_id, cons case HTTP_MESSAGE_RES_BODY_START: case HTTP_MESSAGE_RES_BODY: memset(&body, 0, sizeof(hstring)); - http_message_raw_body_get0(msg, &body); + http_message_get0_uncompressed_body(msg, (const char **)&body.iov_base, &body.iov_len); if (body.iov_base && body.iov_len) { append_http_payload(sess, gtest_plug_exdata, PAYLOAD_RAW, &body, HTTP_TRANSACTION_RES); } // output_http_body(&body, 0); memset(&body, 0, sizeof(hstring)); - http_message_decompress_body_get0(msg, &body); + http_message_get0_decompressed_body(msg, (const char **)&body.iov_base, &body.iov_len); // output_http_body(&body, 1); if (body.iov_base && body.iov_len) { @@ -525,7 +524,7 @@ extern "C" void http_decoder_test_state_entry(struct session *sess, int topic_id char msg_index_name[64] = {}; char msg_index_value[64] = {}; cJSON *json_object = NULL; - enum http_message_type msg_type = http_message_type_get((struct http_message *)raw_msg); + enum http_message_type msg_type = http_message_get_type((struct http_message *)raw_msg); struct gtest_plug_exdata_t *gtest_plug_exdata = (struct gtest_plug_exdata_t *)session_exdata_get(sess, g_exdata_idx); if (NULL == gtest_plug_exdata) diff --git a/test/decoders/http/plugin_test_main.cpp b/test/decoders/http/plugin_test_main.cpp index 90b5bdf..9247df6 100644 --- a/test/decoders/http/plugin_test_main.cpp +++ b/test/decoders/http/plugin_test_main.cpp @@ -3,7 +3,7 @@ #include #include #include -#include "http_decoder_gtest.h" +#include "http_gtest.h" #ifdef __cplusplus extern "C" diff --git a/test/decoders/http/test_based_on_stellar/CMakeLists.txt b/test/decoders/http/test_based_on_stellar/CMakeLists.txt index fcc2c0e..bc08289 100644 --- a/test/decoders/http/test_based_on_stellar/CMakeLists.txt +++ b/test/decoders/http/test_based_on_stellar/CMakeLists.txt @@ -20,20 +20,20 @@ add_test(NAME HTTP_COPY_HTTP_GTEST_ENTRY_CONF COMMAND sh -c "mkdir -p ${TEST_RUN # update config files add_test(NAME HTTP_UPDATE_GTEST_PLUG_ENTRY COMMAND bash -c "sed -i 's/name=.*/name=\\x22http_decoder_test_entry\\x22/' ${TEST_RUN_DIR}/etc/http/gtest_entry.toml") -add_test(NAME HTTP_UPDATE_GTEST_PLUG_TOPIC COMMAND bash -c "sed -i 's/topic=.*/topic=\\x22HTTP_DECODER_MESSAGE\\x22/' ${TEST_RUN_DIR}/etc/http/gtest_entry.toml") +add_test(NAME HTTP_UPDATE_GTEST_PLUG_TOPIC COMMAND bash -c "sed -i 's/topic=.*/topic=\\x22HTTP_MESSAGE\\x22/' ${TEST_RUN_DIR}/etc/http/gtest_entry.toml") # update plugin to be tested -add_test(NAME HTTP_CP_DECODER_SO COMMAND sh -c "cp ${CMAKE_BINARY_DIR}/decoders/http/http_dyn.so ${TEST_RUN_DIR}/plugin/${DECODER_NAME}.so") -add_test(NAME HTTP_CP_DECODER_GTEST_SO COMMAND sh -c "cp ${CMAKE_BINARY_DIR}/test/decoders/http/${DECODER_NAME}_test.so ${TEST_RUN_DIR}/plugin/${DECODER_NAME}_test.so") +add_test(NAME HTTP_CP_DECODER_SO COMMAND sh -c "cp ${CMAKE_BINARY_DIR}/decoders/http/http_dyn.so ${TEST_RUN_DIR}/plugin/http.so") +add_test(NAME HTTP_CP_DECODER_GTEST_SO COMMAND sh -c "cp ${CMAKE_BINARY_DIR}/test/decoders/http/http_test.so ${TEST_RUN_DIR}/plugin/http_test.so") set_tests_properties(HTTP_MKDIR_METRIC HTTP_COPY_SPEC HTTP_COPY_HTTP_DECODER_CONF HTTP_COPY_HTTP_GTEST_ENTRY_CONF HTTP_COPY_CONF HTTP_COPY_LOG_CONF HTTP_CP_DECODER_SO HTTP_CP_DECODER_GTEST_SO HTTP_UPDATE_GTEST_PLUG_ENTRY HTTP_UPDATE_GTEST_PLUG_TOPIC - PROPERTIES FIXTURES_SETUP TestFixture) + PROPERTIES FIXTURES_SETUP HTTP_GTEST) -set(TEST_JSON_DIR ${PROJECT_SOURCE_DIR}/benchmarks/json/http) -set(TEST_PCAP_DIR ${PROJECT_SOURCE_DIR}/benchmarks/pcap/http) +set(TEST_JSON_DIR ${PROJECT_SOURCE_DIR}/test/decoders/http/benchmarks/json) +set(TEST_PCAP_DIR ${PROJECT_SOURCE_DIR}/test/decoders/http/benchmarks/pcap) # run tests add_test(NAME HTTP_GET_SINGLE_TRANS_TEST COMMAND sh -c "ln -sf ${TEST_PCAP_DIR}/http_get_single_trans.pcap ${TEST_RUN_DIR}/pcap/test.pcap; ./${TEST_MAIN} ${TEST_JSON_DIR}/http_get_single_trans.json" WORKING_DIRECTORY ${TEST_RUN_DIR}) @@ -70,6 +70,7 @@ add_test(NAME HTTP_CHN_ENCODE_URL COMMAND sh -c "ln -sf ${TEST_PCAP_DIR}/http_c add_test(NAME HTTP_ZLIB_DEADLOCK COMMAND sh -c "ln -sf ${TEST_PCAP_DIR}/http_zlib_deadlock.pcap ${TEST_RUN_DIR}/pcap/test.pcap; ./${TEST_MAIN} ${TEST_JSON_DIR}/http_zlib_deadlock.json" WORKING_DIRECTORY ${TEST_RUN_DIR}) add_test(NAME HTTP_OUT_OF_ORDER COMMAND sh -c "ln -sf ${TEST_PCAP_DIR}/http_out_of_order.pcap ${TEST_RUN_DIR}/pcap/test.pcap; ./${TEST_MAIN} ${TEST_JSON_DIR}/http_out_of_order.json" WORKING_DIRECTORY ${TEST_RUN_DIR}) add_test(NAME HTTP_GZIP_OUT_OF_ORDER COMMAND sh -c "ln -sf ${TEST_PCAP_DIR}/http_gzip_out_of_order.pcap ${TEST_RUN_DIR}/pcap/test.pcap; ./${TEST_MAIN} ${TEST_JSON_DIR}/http_gzip_out_of_order.json" WORKING_DIRECTORY ${TEST_RUN_DIR}) +add_test(NAME HTTP_HEADER_SPLITTING COMMAND sh -c "ln -sf ${TEST_PCAP_DIR}/http_pipeline_header_splitting.pcap ${TEST_RUN_DIR}/pcap/test.pcap; ./${TEST_MAIN} ${TEST_JSON_DIR}/http_pipeline_header_splitting.json" WORKING_DIRECTORY ${TEST_RUN_DIR}) set_tests_properties(HTTP_GET_SINGLE_TRANS_TEST HTTP_GET_MULTI_TRANS_TEST @@ -98,13 +99,14 @@ set_tests_properties(HTTP_GET_SINGLE_TRANS_TEST HTTP_ZLIB_DEADLOCK HTTP_OUT_OF_ORDER HTTP_GZIP_OUT_OF_ORDER - PROPERTIES FIXTURES_REQUIRED TestFixture) + HTTP_HEADER_SPLITTING + PROPERTIES FIXTURES_REQUIRED HTTP_GTEST) add_test(NAME HTTP_UPDATE_STATE_PLUG_ENTRY COMMAND bash -c "sed -i 's/name=.*/name=\\x22http_decoder_test_state_entry\\x22/' ${TEST_RUN_DIR}/etc/http/gtest_entry.toml") -add_test(NAME HTTP_UPDATE_STATE_PLUG_TOPIC COMMAND bash -c "sed -i 's/topic=.*/topic=\\x22HTTP_DECODER_MESSAGE\\x22/' ${TEST_RUN_DIR}/etc/http/gtest_entry.toml") +add_test(NAME HTTP_UPDATE_STATE_PLUG_TOPIC COMMAND bash -c "sed -i 's/topic=.*/topic=\\x22HTTP_MESSAGE\\x22/' ${TEST_RUN_DIR}/etc/http/gtest_entry.toml") set_tests_properties(HTTP_UPDATE_STATE_PLUG_ENTRY HTTP_UPDATE_STATE_PLUG_TOPIC HTTP_CP_DECODER_SO HTTP_CP_DECODER_GTEST_SO HTTP_MKDIR_METRIC - PROPERTIES FIXTURES_SETUP TestState) + PROPERTIES FIXTURES_SETUP HTTP_STATE_TEST) add_test(NAME HTTP_MSG_TYPE_STATE_TEST COMMAND sh -c "ln -sf ${TEST_PCAP_DIR}/http_post.pcap ${TEST_RUN_DIR}/pcap/test.pcap; ./${TEST_MAIN} ${TEST_JSON_DIR}/http_msg_type_state.json" WORKING_DIRECTORY ${TEST_RUN_DIR}) add_test(NAME HTTP_MSG_TYPE_STATE_C2S_TEST COMMAND sh -c "ln -sf ${TEST_PCAP_DIR}/http_post_c2s.pcap ${TEST_RUN_DIR}/pcap/test.pcap; ./${TEST_MAIN} ${TEST_JSON_DIR}/http_msg_type_state_c2s.json" WORKING_DIRECTORY ${TEST_RUN_DIR}) @@ -119,5 +121,5 @@ set_tests_properties(HTTP_MSG_TYPE_STATE_TEST HTTP_MSG_TYPE_STATE_PIPELINE_TEST HTTP_MSG_TYPE_STATE_SES_EXCEPTION_C2S_TEST HTTP_MSG_TYPE_STATE_SES_EXCEPTION_S2C_TEST - PROPERTIES FIXTURES_REQUIRED TestState) + PROPERTIES FIXTURES_REQUIRED HTTP_STATE_TEST) diff --git a/test/decoders/http/test_based_on_stellar/env/gtest_entry.toml b/test/decoders/http/test_based_on_stellar/env/gtest_entry.toml index 35e32ca..7491e9d 100644 --- a/test/decoders/http/test_based_on_stellar/env/gtest_entry.toml +++ b/test/decoders/http/test_based_on_stellar/env/gtest_entry.toml @@ -1,3 +1,3 @@ [entry] name="http_decoder_test_state_entry" -topic="HTTP_DECODER_MESSAGE" +topic="HTTP_MESSAGE" diff --git a/test/decoders/http/test_based_on_stellar/env/stellar.toml b/test/decoders/http/test_based_on_stellar/env/stellar.toml index d741b1c..58f5c94 100644 --- a/test/decoders/http/test_based_on_stellar/env/stellar.toml +++ b/test/decoders/http/test_based_on_stellar/env/stellar.toml @@ -9,20 +9,20 @@ pcap_path = "./pcap/test.pcap" nr_worker_thread = 1 # range: [1, 256] cpu_mask = [5, 6, 7, 8, 9, 10, 11, 12] - idle_yield_interval_ms = 900 # range: [0, 60000] (ms) + idle_yield_interval_ms = 90 # range: [0, 60000] (ms) [ip_reassembly] enable = 1 bucket_entries = 32 # range: [1, 4294967295] (must be power of 2) bucket_num = 1024 # range: [1, 4294967295] - ip_frag_timeout_ms = 10000 # range: [1, 60000] (ms) + ip_frag_timeout_ms = 1000 # range: [1, 60000] (ms) ip_frag_expire_polling_interval_ms = 0 # range: [0, 60000] (ms) ip_frag_expire_polling_limit = 1024 # range: [1, 1024] [session_manager] - tcp_session_max = 50000 - udp_session_max = 50000 + tcp_session_max = 500 + udp_session_max = 500 evict_old_on_tcp_table_limit = 1 # range: [0, 1] evict_old_on_udp_table_limit = 1 # range: [0, 1] @@ -31,35 +31,35 @@ expire_batch_max = 1024 # range: [1, 1024] [session_manager.tcp_timeout_ms] - init = 5000 # range: [1, 60000] (ms) - handshake = 5000 # range: [1, 60000] (ms) - data = 5000 # range: [1, 15999999000] (ms) - half_closed = 5000 # range: [1, 604800000] (ms) - time_wait = 5000 # range: [1, 600000] (ms) - discard_default = 10000 # range: [1, 15999999000] (ms) - unverified_rst = 5000 # range: [1, 600000] (ms) + init = 500 # range: [1, 60000] (ms) + handshake = 500 # range: [1, 60000] (ms) + data = 500 # range: [1, 15999999000] (ms) + half_closed = 500 # range: [1, 604800000] (ms) + time_wait = 500 # range: [1, 600000] (ms) + discard_default = 1000 # range: [1, 15999999000] (ms) + unverified_rst = 500 # range: [1, 600000] (ms) [session_manager.udp_timeout_ms] - data = 5000 # range: [1, 15999999000] (ms) - discard_default = 5000 # range: [1, 15999999000] (ms) + data = 500 # range: [1, 15999999000] (ms) + discard_default = 500 # range: [1, 15999999000] (ms) [session_manager.duplicated_packet_bloom_filter] - enable = 1 + enable = 0 capacity = 1000000 # range: [1, 4294967295] time_window_ms = 10000 # range: [1, 60000] (ms) error_rate = 0.00001 # range: [0.0, 1.0] [session_manager.evicted_session_bloom_filter] - enable = 1 # range: [0, 1] + enable = 0 # range: [0, 1] capacity = 1000000 # range: [1, 4294967295] time_window_ms = 10000 # range: [1, 60000] (ms) error_rate = 0.00001 # range: [0.0, 1.0] [session_manager.tcp_reassembly] enable = 1 # range: [0, 1] - timeout_ms = 10000 # range: [1, 60000] (ms) + timeout_ms = 100 # range: [1, 60000] (ms) buffered_segments_max = 256 # range: [2, 4096] per flow [schedule] - merge_stat_interval = 500 # range: [1, 60000] (ms) - output_stat_interval = 2000 # range: [1, 60000] (ms) + merge_stat_interval = 50 # range: [1, 60000] (ms) + output_stat_interval = 200 # range: [1, 60000] (ms) diff --git a/vendors/CMakeLists.txt b/vendors/CMakeLists.txt index deecc53..5ed2c1b 100644 --- a/vendors/CMakeLists.txt +++ b/vendors/CMakeLists.txt @@ -74,3 +74,27 @@ add_library(brotli-dec-static STATIC IMPORTED GLOBAL) add_dependencies(brotli-dec-static brotli) set_property(TARGET brotli-dec-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) set_property(TARGET brotli-dec-static PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib64/libbrotlidec-static.a) + +add_library(brotli-enc-static STATIC IMPORTED GLOBAL) +add_dependencies(brotli-enc-static brotli) +set_property(TARGET brotli-enc-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) +set_property(TARGET brotli-enc-static PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib64/libbrotlienc-static.a) + +# Libevent +ExternalProject_Add(libevent + PREFIX libevent + URL ${CMAKE_CURRENT_SOURCE_DIR}/libevent-2.1.12-stable.tar.gz + URL_MD5 b5333f021f880fe76490d8a799cd79f4 + CMAKE_ARGS -DEVENT__DISABLE_OPENSSL=ON -DEVENT__DISABLE_BENCHMARK=ON -DBUILD_TESTING=OFF -DEVENT__DISABLE_TESTS=ON -DEVENT__LIBRARY_TYPE=STATIC + -DCMAKE_INSTALL_PREFIX= + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} +) + +ExternalProject_Get_Property(libevent INSTALL_DIR) +file(MAKE_DIRECTORY ${INSTALL_DIR}/include) + +add_library(libevent-static STATIC IMPORTED GLOBAL) +add_dependencies(libevent-static libevent) +set_property(TARGET libevent-static PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libevent.a) +set_property(TARGET libevent-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) +set_property(TARGET libevent-static PROPERTY INTERFACE_LINK_LIBRARIES pthread) \ No newline at end of file diff --git a/vendors/libevent-2.1.12-stable.tar.gz b/vendors/libevent-2.1.12-stable.tar.gz new file mode 100644 index 0000000..4fcefff Binary files /dev/null and b/vendors/libevent-2.1.12-stable.tar.gz differ