rename *.cpp to *.c; renmae API. move benchmarks/* to test/decoder/http; fix TSG-22347

decoders/http/CMakeLists.txt

@@ -1,8 +1,8 @@
 include_directories(${CMAKE_SOURCE_DIR}/deps)
 
-set(HTTP_SRC http_decoder.cpp http_decoder_utils.cpp http_decoder_half.cpp
+set(HTTP_SRC http_decoder.c http_decoder_utils.c http_decoder_half.c
-             http_decoder_table.cpp http_decoder_string.cpp http_content_decompress.cpp
+             http_decoder_table.c http_decoder_string.c http_content_decompress.c
-             http_decoder_result_queue.cpp http_decoder_stat.cpp http_decoder_tunnel.cpp)
+             http_decoder_result_queue.c http_decoder_stat.c http_decoder_tunnel.c)
 
 add_library(http STATIC ${HTTP_SRC})
 add_library(http_dyn SHARED ${HTTP_SRC})

decoders/http/http_content_decompress.c

@@ -15,17 +15,22 @@ struct http_content_decompress
     size_t buffer_size;
 };
 
-enum http_content_encoding http_content_encoding_str2int(const char *content_encoding)
+void http_content_decompress_ownership_borrow(struct http_content_decompress *decompress)
 {
-    if (strcasestr(content_encoding, "gzip") != NULL)
+    decompress->buffer = NULL; // ownership move to data->decompress_buffer_list, will be freed when message has been processed by all plugins
+}
+
+enum http_content_encoding http_content_encoding_str2int(const char *content_encoding, size_t encoding_str_len)
+{
+    if (http_strncasecmp_safe("gzip", content_encoding, 4, encoding_str_len) == 0)
     {
         return HTTP_CONTENT_ENCODING_GZIP;
     }
-    if (strcasestr(content_encoding, "deflate") != NULL)
+    if (http_strncasecmp_safe("deflate", content_encoding, 7, encoding_str_len) == 0)
     {
         return HTTP_CONTENT_ENCODING_DEFLATE;
     }
-    if (strcasestr(content_encoding, "br") != NULL)
+    if (http_strncasecmp_safe("br", content_encoding, 2, encoding_str_len) == 0)
     {
         return HTTP_CONTENT_ENCODING_BR;
     }
@@ -128,11 +133,12 @@ static int http_content_decompress_write_zlib(struct http_content_decompress *de
     z_stream *z_stream_ptr = decompress->z_stream_ptr;
     z_stream_ptr->avail_in = (unsigned int)indata_len;
     z_stream_ptr->next_in = (unsigned char *)indata;
-    z_stream_ptr->avail_out = (unsigned int)decompress->buffer_size;
+    z_stream_ptr->avail_out = (unsigned int)HTTP_DECOMPRESS_BUFFER_SIZE;
     z_stream_ptr->next_out = (unsigned char *)decompress->buffer;
-
     *outdata = NULL;
     *outdata_len = 0;
+    size_t total_have = 0;
+    int no_buffer;
 
     do
     {
@@ -142,37 +148,36 @@ static int http_content_decompress_write_zlib(struct http_content_decompress *de
             (void)inflateEnd(z_stream_ptr);
             return -1;
         }
-
+        size_t have = HTTP_DECOMPRESS_BUFFER_SIZE - z_stream_ptr->avail_out;
-        size_t have = decompress->buffer_size - z_stream_ptr->avail_out;
         if (have > 0)
         {
+            total_have += have;
             if (0 == z_stream_ptr->avail_out)
             {
-                decompress->buffer_size += have;
+                decompress->buffer_size += HTTP_DECOMPRESS_BUFFER_SIZE;
-                decompress->buffer = REALLOC(char, decompress->buffer,
+                decompress->buffer = REALLOC(char, decompress->buffer, decompress->buffer_size);
-                                             decompress->buffer_size);
+                z_stream_ptr->avail_out = HTTP_DECOMPRESS_BUFFER_SIZE;
+                z_stream_ptr->next_out = (unsigned char *)decompress->buffer + total_have;
                 *outdata = decompress->buffer;
-                *outdata_len = *outdata_len + have;
+                *outdata_len = total_have;
-                // http_decoder_log(DEBUG, "%s realloc outbuffer %zu bytes",
+                no_buffer = 1;
-                //                  http_content_encoding_int2str(decompress->encoding),
-                //                  decompress->buffer_size);
-                z_stream_ptr->avail_out = have;
-                z_stream_ptr->next_out = (unsigned char *)decompress->buffer +
-                                         (decompress->buffer_size - have);
             }
             else
             {
                 *outdata = decompress->buffer;
-                *outdata_len = have;
+                *outdata_len = total_have;
+                no_buffer = 0;
             }
         }
+        else
+        {
+            break;
+        }
         if (Z_STREAM_END == ret)
         {
             break;
         }
-    } while (z_stream_ptr->avail_in != 0);
+    } while (no_buffer == 1);
-    decompress->buffer = NULL;
-    decompress->buffer_size = 0;
     return 0;
 }
 
@@ -182,56 +187,52 @@ static int http_content_decompress_write_br(struct http_content_decompress *deco
 {
     size_t available_in = indata_len;
     const unsigned char *next_in = (const unsigned char *)indata;
-    size_t available_out = decompress->buffer_size;
+    size_t available_out = HTTP_DECOMPRESS_BUFFER_SIZE;
     unsigned char *next_out = (unsigned char *)decompress->buffer;
 
     *outdata = NULL;
     *outdata_len = 0;
+    size_t total_have = 0;
+    int no_buffer;
 
-    for (;;)
+    do
     {
         int ret = BrotliDecoderDecompressStream(decompress->br_state, &available_in,
-                                                &next_in, &available_out, &next_out, 0);
+                                                &next_in, &available_out, &next_out, NULL);
-        size_t have = decompress->buffer_size - available_out;
+        if (ret == BROTLI_DECODER_RESULT_ERROR)
+        {
+            // BrotliDecoderErrorCode errcode = BrotliDecoderGetErrorCode(decompress->br_state);
+            *outdata = NULL;
+            *outdata_len = 0;
+            return -1;
+        }
+        size_t have = HTTP_DECOMPRESS_BUFFER_SIZE - available_out;
         if (have > 0)
         {
+            total_have += have;
             if (0 == available_out)
             {
-                decompress->buffer_size += have;
+                decompress->buffer_size += HTTP_DECOMPRESS_BUFFER_SIZE;
-                decompress->buffer = REALLOC(char, decompress->buffer,
+                decompress->buffer = REALLOC(char, decompress->buffer, decompress->buffer_size);
-                                             decompress->buffer_size);
+                available_out = HTTP_DECOMPRESS_BUFFER_SIZE;
+                next_out = (unsigned char *)decompress->buffer + total_have;
                 *outdata = decompress->buffer;
-                *outdata_len = *outdata_len + have;
+                *outdata_len = total_have;
-                available_out = have;
+                no_buffer = 1;
-                next_out = (unsigned char *)decompress->buffer +
-                           (decompress->buffer_size - have);
             }
             else
             {
                 *outdata = decompress->buffer;
                 *outdata_len = have;
+                no_buffer = 0;
             }
         }
-
+        else
-        if (ret == BROTLI_DECODER_RESULT_SUCCESS ||
-            ret == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT)
         {
-            decompress->buffer = NULL;
+            break;
-            decompress->buffer_size = 0;
+        }
+    } while (no_buffer == 1);
     return 0;
-        }
-
-        if (ret == BROTLI_DECODER_RESULT_ERROR)
-        {
-            // BrotliDecoderErrorCode errcode =
-            BrotliDecoderGetErrorCode(decompress->br_state);
-            *outdata = NULL;
-            *outdata_len = 0;
-            return -1;
-        }
-
-        assert(ret == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT);
-    }
 }
 
 int http_content_decompress_write(struct http_content_decompress *decompress,
@@ -243,7 +244,6 @@ int http_content_decompress_write(struct http_content_decompress *decompress,
     assert(indata_len > 0);
     assert(outdata);
     assert(outdata_len);
-
     *outdata = NULL;
     *outdata_len = 0;
 

decoders/http/http_content_decompress.h

@@ -17,7 +17,7 @@ extern "C"
 
     struct http_content_decompress;
 
-    enum http_content_encoding http_content_encoding_str2int(const char *content_encoding);
+    enum http_content_encoding http_content_encoding_str2int(const char *content_encoding, size_t encoding_str_len);
 
     const char *http_content_encoding_int2str(enum http_content_encoding content_encoding);
 
@@ -30,7 +30,7 @@ extern "C"
     int http_content_decompress_write(struct http_content_decompress *decompress,
                                       const char *indata, size_t indata_len,
                                       char **outdata, size_t *outdata_len);
-
+    void http_content_decompress_ownership_borrow(struct http_content_decompress *decompress);
 #ifdef __cplusplus
 }
 #endif

decoders/http/http_decoder.c

@@ -159,9 +159,10 @@ static void http_event_handler(enum http_event event, struct http_decoder_half_d
         int tot_c2s_headers = http_half_data_get_total_parsed_header_count(half_data);
         http_decoder_stat_update(&httpd_env->hd_stat, thread_id, HTTP_C2S_HEADERS, tot_c2s_headers);
 
-        hstring tmp_url = {};
+        const char *tmp_url = NULL;
-        http_half_data_get_url(half_data, &tmp_url);
+        size_t tmp_url_len = 0;
-        http_decoder_stat_update(&httpd_env->hd_stat, thread_id, HTTP_URL_BYTES, tmp_url.iov_len);
+        http_half_data_get_url(half_data, &tmp_url, &tmp_url_len);
+        http_decoder_stat_update(&httpd_env->hd_stat, thread_id, HTTP_URL_BYTES, tmp_url_len);
     }
     break;
     case HTTP_EVENT_REQ_BODY_BEGIN:
@@ -172,7 +173,7 @@ static void http_event_handler(enum http_event event, struct http_decoder_half_d
     {
         hstring raw_body = {};
         hstring decompress_body = {};
-        http_decoder_half_data_get_raw_body(half_data, &raw_body);
+        http_decoder_half_data_get_raw_body(half_data, (const char **)&raw_body.iov_base, &raw_body.iov_len);
         http_half_get_lastest_decompress_buffer(half_data, &decompress_body);
         msg = http_body_message_new(HTTP_MESSAGE_REQ_BODY, queue, queue_idx, HTTP_REQUEST, &raw_body, &decompress_body);
         session_mq_publish_message(ev_ctx->ref_session, exdata->pub_topic_id, msg);
@@ -303,7 +304,7 @@ static void http_event_handler(enum http_event event, struct http_decoder_half_d
     case HTTP_EVENT_RES_BODY_DATA:
     {
         hstring raw_body = {};
-        http_decoder_half_data_get_raw_body(half_data, &raw_body);
+        http_decoder_half_data_get_raw_body(half_data, (const char **)&raw_body.iov_base, &raw_body.iov_len);
         hstring decompress_body = {};
         http_half_get_lastest_decompress_buffer(half_data, &decompress_body);
         msg = http_body_message_new(HTTP_MESSAGE_RES_BODY, queue, queue_idx, HTTP_RESPONSE, &raw_body, &decompress_body);
@@ -536,31 +537,31 @@ static int load_http_decoder_config(const char *cfg_path,
     return ret;
 }
 
-static int http_msg_get_request_header(const struct http_message *msg, const hstring *key,
+static int http_msg_get_request_header(const struct http_message *msg, const char *name, size_t name_len,
-                                       struct http_header *hdr_result)
+                                       struct http_header_field *hdr_result)
 {
     const struct http_decoder_half_data *req_data =
         msg->ref_queue->array[msg->queue_index].req_data;
-    return http_decoder_half_data_get_header(req_data, key, hdr_result);
+    return http_decoder_half_data_get_header(req_data, name, name_len, hdr_result);
 }
 
-static int http_msg_get_response_header(const struct http_message *msg, const hstring *key,
+static int http_msg_get_response_header(const struct http_message *msg, const char *name, size_t name_len,
-                                        struct http_header *hdr_result)
+                                        struct http_header_field *hdr_result)
 {
     const struct http_decoder_half_data *res_data =
         msg->ref_queue->array[msg->queue_index].res_data;
-    return http_decoder_half_data_get_header(res_data, key, hdr_result);
+    return http_decoder_half_data_get_header(res_data, name, name_len, hdr_result);
 }
 
 static int http_msg_request_header_next(const struct http_message *msg,
-                                        struct http_header *hdr)
+                                        struct http_header_field *hdr)
 {
     const struct http_decoder_half_data *req_data =
         msg->ref_queue->array[msg->queue_index].req_data;
     return http_decoder_half_data_iter_header((struct http_decoder_half_data *)req_data, hdr);
 }
 
-static int http_msg_response_header_next(const struct http_message *msg, struct http_header *hdr)
+static int http_msg_response_header_next(const struct http_message *msg, struct http_header_field *hdr)
 {
     const struct http_decoder_half_data *res_data =
         msg->ref_queue->array[msg->queue_index].res_data;
@@ -656,7 +657,7 @@ extern "C"
         http_decoder_stat_update(&httpd_env->hd_stat, thread_id, HTTP_SESSION_FREE, 1);
     }
 
-    static void http_decoder_execute(struct session *sess, struct http_decoder_env *httpd_env, http_decoder_exdata *exdata, const char *payload, uint16_t payload_len)
+    static void http_decoder_execute(struct session *sess, struct http_decoder_env *httpd_env, struct http_decoder_exdata *exdata, const char *payload, uint16_t payload_len)
     {
         if (httpd_in_tunnel_transmitting(httpd_env, exdata))
         {
@@ -820,7 +821,7 @@ extern "C"
     static const struct http_topic_exdata_compose g_topic_exdata_compose[HTTPD_TOPIC_INDEX_MAX] =
         {
             {HTTPD_TOPIC_TCP_STREAM_INDEX, TOPIC_TCP_STREAM, http_decoder_tcp_stream_msg_cb, NULL, "HTTP_DECODER_EXDATA_BASEON_TCP_STREAM", httpd_ex_data_free_cb, -1, -1},
-            {HTTPD_TOPIC_HTTP_MSG_INDEX, HTTP_DECODER_TOPIC, NULL, http_message_free, NULL, NULL, -1, -1},
+            {HTTPD_TOPIC_HTTP_MSG_INDEX, HTTP_TOPIC, NULL, http_message_free, NULL, NULL, -1, -1},
             {HTTPD_TOPIC_HTTP_TUNNEL_INDEX, HTTP_DECODER_TUNNEL_TOPIC, http_decoder_tunnel_msg_cb, http_message_free, "HTTP_DECODER_EXDATA_BASEON_HTTP_TUNNEL", httpd_ex_data_free_cb, -1, -1},
     };
 
@@ -910,7 +911,7 @@ extern "C"
         _http_decoder_context_free(httpd_env);
     }
 
-    enum http_message_type http_message_type_get(const struct http_message *msg)
+    enum http_message_type http_message_get_type(const struct http_message *msg)
     {
         if (unlikely(NULL == msg))
         {
@@ -919,16 +920,16 @@ extern "C"
         return msg->type;
     }
 
-    void http_message_request_line_get0(const struct http_message *msg,
+    void http_message_get0_request_line(const struct http_message *msg,
                                         struct http_request_line *line)
     {
         if (unlikely(NULL == msg || msg->type != HTTP_MESSAGE_REQ_LINE))
         {
             if (line)
             {
-                line->method.iov_base = NULL;
+                line->method = NULL;
-                line->uri.iov_base = NULL;
+                line->uri = NULL;
-                line->version.iov_base = NULL;
+                line->version = NULL;
             }
             return;
         }
@@ -941,15 +942,15 @@ extern "C"
         http_decoder_half_data_get_request_line(req_data, line);
     }
 
-    void http_message_response_line_get0(const struct http_message *msg,
+    void http_message_get0_response_line(const struct http_message *msg,
                                          struct http_response_line *line)
     {
         if (unlikely(NULL == msg || msg->type != HTTP_MESSAGE_RES_LINE))
         {
             if (line)
             {
-                line->version.iov_base = NULL;
+                line->version = NULL;
-                line->status.iov_base = NULL;
+                line->status = NULL;
             }
             return;
         }
@@ -962,11 +963,11 @@ extern "C"
         http_decoder_half_data_get_response_line(res_data, line);
     }
 
-    void http_message_header_get0(const struct http_message *msg, const hstring *key,
+    void http_message_get0_header(const struct http_message *msg, const char *name, size_t name_len,
-                                  struct http_header *hdr_result)
+                                  struct http_header_field *hdr_result)
     {
         int ret = -1;
-        if (unlikely(NULL == msg || NULL == key))
+        if (unlikely(NULL == msg || NULL == name || 0 == name_len))
         {
             goto fail;
         }
@@ -974,11 +975,11 @@ extern "C"
         assert(msg->queue_index < HD_RESULT_QUEUE_LEN);
         if (HTTP_MESSAGE_REQ_HEADER == msg->type)
         {
-            ret = http_msg_get_request_header(msg, key, hdr_result);
+            ret = http_msg_get_request_header(msg, name, name_len, hdr_result);
         }
         else if (HTTP_MESSAGE_RES_HEADER == msg->type)
         {
-            ret = http_msg_get_response_header(msg, key, hdr_result);
+            ret = http_msg_get_response_header(msg, name, name_len, hdr_result);
         }
         if (ret >= 0)
         {
@@ -987,13 +988,13 @@ extern "C"
     fail:
         if (hdr_result)
         {
-            hdr_result->key.iov_base = NULL;
+            hdr_result->name = NULL;
-            hdr_result->val.iov_base = NULL;
+            hdr_result->value = NULL;
         }
         return;
     }
 
-    int http_message_header_next(const struct http_message *msg, struct http_header *header)
+    int http_message_get0_next_header(const struct http_message *msg, struct http_header_field *header)
     {
         int ret = 1;
         if (unlikely(NULL == msg))
@@ -1018,8 +1019,8 @@ extern "C"
     fail:
         if (header)
         {
-            header->key.iov_base = NULL;
+            header->name = NULL;
-            header->val.iov_base = NULL;
+            header->value = NULL;
         }
         return -1;
     }
@@ -1047,7 +1048,7 @@ extern "C"
         return -1;
     }
 
-    void http_message_raw_body_get0(const struct http_message *msg, hstring *body)
+    void http_message_get0_uncompressed_body(const struct http_message *msg, const char **body_ptr, size_t *body_len)
     {
         if (unlikely(NULL == msg))
         {
@@ -1057,20 +1058,20 @@ extern "C"
         assert(msg->queue_index < HD_RESULT_QUEUE_LEN);
         if (msg->raw_payload.iov_base != NULL && msg->raw_payload.iov_len != 0)
         {
-            body->iov_base = msg->raw_payload.iov_base;
+            *body_ptr = msg->raw_payload.iov_base;
-            body->iov_len = msg->raw_payload.iov_len;
+            *body_len = msg->raw_payload.iov_len;
             return;
         }
     fail:
-        if (body)
+        if (body_ptr)
         {
-            body->iov_base = NULL;
+            *body_ptr = NULL;
-            body->iov_len = 0;
+            *body_len = 0;
         }
         return;
     }
 
-    void http_message_decompress_body_get0(const struct http_message *msg, hstring *decompress_body)
+    void http_message_get0_decompressed_body(const struct http_message *msg, const char **dec_body_ptr, size_t *dec_body_len)
     {
         enum http_content_encoding ecode = HTTP_CONTENT_ENCODING_NONE;
         struct http_decoder_half_data *ref_data = NULL;
@@ -1082,12 +1083,12 @@ extern "C"
         assert(msg->queue_index < HD_RESULT_QUEUE_LEN);
         if (msg->decompress_payload.iov_base != NULL && msg->decompress_payload.iov_len != 0)
         {
-            decompress_body->iov_base = msg->decompress_payload.iov_base;
+            *dec_body_ptr = msg->decompress_payload.iov_base;
-            decompress_body->iov_len = msg->decompress_payload.iov_len;
+            *dec_body_len = msg->decompress_payload.iov_len;
             return;
         }
         /**
-         * @brief If the body hasn't been compressed, same as http_message_raw_body_get0().
+         * @brief If the body hasn't been compressed, same as http_message_get0_uncompressed_body().
          *
          */
 
@@ -1107,51 +1108,46 @@ extern "C"
 
         if (msg->raw_payload.iov_base != NULL && msg->raw_payload.iov_len != 0)
         {
-            decompress_body->iov_base = msg->raw_payload.iov_base;
+            *dec_body_ptr = msg->raw_payload.iov_base;
-            decompress_body->iov_len = msg->raw_payload.iov_len;
+            *dec_body_len = msg->raw_payload.iov_len;
         }
         return;
     fail:
-        if (decompress_body)
+        if (dec_body_ptr)
         {
-            decompress_body->iov_base = NULL;
+            *dec_body_ptr = NULL;
-            decompress_body->iov_len = 0;
+            *dec_body_len = 0;
         }
         return;
     }
 
-    void http_message_raw_url_get0(const struct http_message *msg, hstring *url)
+    void http_message_get0_raw_url(const struct http_message *msg, const char **url_val, size_t *url_len)
     {
+        struct http_decoder_half_data *req_data = NULL;
         if (unlikely(NULL == msg))
         {
-            if (url)
+            goto fail;
-            {
-                url->iov_base = NULL;
-                url->iov_len = 0;
-            }
-            return;
         }
         assert(msg->ref_queue);
         assert(msg->queue_index < HD_RESULT_QUEUE_LEN);
 
-        struct http_decoder_half_data *req_data =
+        req_data = msg->ref_queue->array[msg->queue_index].req_data;
-            msg->ref_queue->array[msg->queue_index].req_data;
 
-        if (http_half_data_get_url(req_data, url) < 0)
+        if (http_half_data_get_url(req_data, url_val, url_len) < 0)
         {
             goto fail;
         }
         return;
 
     fail:
-        if (url)
+        if (url_val)
         {
-            url->iov_base = NULL;
+            *url_val = NULL;
-            url->iov_len = 0;
+            *url_len = 0;
         }
         return;
     }
-
+#if 0
     void http_message_decoded_url_get0(const struct http_message *msg, struct iovec *url)
     {
         if (unlikely(NULL == msg))
@@ -1183,7 +1179,7 @@ extern "C"
         }
         return;
     }
-
+#endif
     int http_message_get_transaction_seq(const struct http_message *msg)
     {
         if (unlikely(NULL == msg))

decoders/http/http_decoder_half.c

@@ -33,7 +33,8 @@ struct http_decoder_half_data
 #endif
     int joint_url_complete;
     int url_is_encoded;
-    hstring joint_url; // http://<host>[:<port>]/<path>?<searchpart>
+    // http://<host>[:<port>]/<path>?<searchpart>
+    hstring joint_url;
     hstring decoded_url;
     long long transaction_index;
 };
@@ -67,7 +68,7 @@ static void printf_debug_info(const char *desc, const char *at, size_t length)
 {
     if (at)
     {
-        char *temp = safe_dup(at, length);
+        char *temp = http_safe_dup(at, length);
         printf("HTTP PARSER STAGE: %s: %s\n", desc, temp);
         FREE(temp);
     }
@@ -131,7 +132,7 @@ static void http_decoder_half_data_decompress(struct http_decoder_half_data *dat
     }
 
     hstring raw_body = {};
-    http_decoder_table_get_body(data->table, &raw_body);
+    http_decoder_table_get_body(data->table, (char **)&raw_body.iov_base, &raw_body.iov_len);
     if (raw_body.iov_base == NULL || raw_body.iov_len == 0)
     {
         return;
@@ -163,6 +164,7 @@ static void http_decoder_half_data_decompress(struct http_decoder_half_data *dat
         decompress_buffer->iov.iov_base = local_outdata;
         decompress_buffer->iov.iov_len = local_outdata_len;
         DL_APPEND(data->decompress_buffer_list, decompress_buffer);
+        http_content_decompress_ownership_borrow(data->decompress);
     }
 }
 
@@ -327,7 +329,7 @@ static int on_uri_complete(llhttp_t *http)
     http_decoder_table_commit(half->ref_data->table, HTTP_ITEM_URI);
 
     hstring uri_result = {};
-    http_decoder_table_get_uri(half->ref_data->table, &uri_result);
+    http_decoder_table_get_uri(half->ref_data->table, (char **)&uri_result.iov_base, &uri_result.iov_len);
     assert(uri_result.iov_base);
     http_decoder_cached_portion_url(half, &uri_result);
 
@@ -472,19 +474,11 @@ static int on_header_value_complete(llhttp_t *http)
 
     if (half->ref_data->content_encoding == HTTP_CONTENT_ENCODING_NONE)
     {
-        struct http_header http_hdr = {};
+        struct http_header_field http_hdr = {};
-        hstring key = {.iov_base = (char *)"Content-Encoding", .iov_len = 16};
 
-        if (http_decoder_table_get_header(half->ref_data->table, &key, &http_hdr) == 0)
+        if (http_decoder_table_get_header(half->ref_data->table, (char *)"Content-Encoding", 16, &http_hdr) == 0)
         {
-            char encoding_str[MAX_ENCODING_STR_LEN + 1] = {0};
+            half->ref_data->content_encoding = http_content_encoding_str2int(http_hdr.value, http_hdr.value_len);
-            size_t iov_len = http_hdr.val.iov_len;
-            if (iov_len > MAX_ENCODING_STR_LEN)
-            {
-                iov_len = MAX_ENCODING_STR_LEN;
-            }
-            memcpy(encoding_str, http_hdr.val.iov_base, iov_len);
-            half->ref_data->content_encoding = http_content_encoding_str2int(encoding_str);
         }
     }
 
@@ -910,9 +904,9 @@ void http_decoder_half_data_free(nmx_pool_t *mempool, struct http_decoder_half_d
 int http_decoder_half_data_get_request_line(struct http_decoder_half_data *data,
                                             struct http_request_line *line)
 {
-    http_decoder_table_get_method(data->table, &line->method);
+    http_decoder_table_get_method(data->table, &line->method, &line->method_len);
-    http_decoder_table_get_uri(data->table, &line->uri);
+    http_decoder_table_get_uri(data->table, &line->uri, &line->uri_len);
-    http_decoder_table_get_version(data->table, &line->version);
+    http_decoder_table_get_version(data->table, &line->version, &line->version_len);
 
     line->major_version = data->major_version;
     line->minor_version = data->minor_version;
@@ -923,8 +917,8 @@ int http_decoder_half_data_get_request_line(struct http_decoder_half_data *data,
 int http_decoder_half_data_get_response_line(struct http_decoder_half_data *data,
                                              struct http_response_line *line)
 {
-    http_decoder_table_get_version(data->table, &line->version);
+    http_decoder_table_get_version(data->table, &line->version, &line->version_len);
-    http_decoder_table_get_status(data->table, &line->status);
+    http_decoder_table_get_status(data->table, &line->status, &line->status_len);
 
     line->major_version = data->major_version;
     line->minor_version = data->minor_version;
@@ -934,14 +928,14 @@ int http_decoder_half_data_get_response_line(struct http_decoder_half_data *data
 }
 
 int http_decoder_half_data_get_header(const struct http_decoder_half_data *data,
-                                      const hstring *key,
+                                      const char *name, size_t name_len,
-                                      struct http_header *hdr_result)
+                                      struct http_header_field *hdr_result)
 {
-    return http_decoder_table_get_header(data->table, key, hdr_result);
+    return http_decoder_table_get_header(data->table, name, name_len, hdr_result);
 }
 
 int http_decoder_half_data_iter_header(struct http_decoder_half_data *data,
-                                       struct http_header *header)
+                                       struct http_header_field *header)
 {
     return http_decoder_table_iter_header((struct http_decoder_table *)data->table, header);
 }
@@ -964,13 +958,13 @@ int http_decoder_half_data_has_parsed_header(struct http_decoder_half_data *data
     return http_decoder_table_has_parsed_header(data->table);
 }
 
-int http_decoder_half_data_get_raw_body(const struct http_decoder_half_data *data, hstring *body)
+int http_decoder_half_data_get_raw_body(const struct http_decoder_half_data *data, const char **body, size_t *body_len)
 {
     if (NULL == data || NULL == body)
     {
         return -1;
     }
-    return http_decoder_table_get_body(data->table, body);
+    return http_decoder_table_get_body(data->table, (char **)body, body_len);
 }
 #if 0
 int http_decoder_half_data_get_decompress_body(const struct http_decoder_half_data *data, hstring *body)
@@ -996,34 +990,33 @@ void http_decoder_half_data_dump(struct http_decoder_half *half)
     http_decoder_table_dump(half->ref_data->table);
 }
 
-static void using_session_addr_as_host(struct session *ref_session,
+static void using_session_addr_as_host(struct session *ref_session, struct http_header_field *host_result, nmx_pool_t *mempool)
-                                       struct http_header *host_result, nmx_pool_t *mempool)
 {
 #if 1 // in native steallar, can't get the tuple4 from the session yet!!!
     struct httpd_session_addr ssaddr = {};
     httpd_session_get_addr(ref_session, &ssaddr);
     if (ssaddr.ipver != 4 && ssaddr.ipver != 6)
     {
-        host_result->val.iov_base = MEMPOOL_CALLOC(mempool, char, 1);
+        host_result->value = MEMPOOL_CALLOC(mempool, char, 1);
-        sprintf((char *)host_result->val.iov_base, "%s", "");
+        sprintf((char *)host_result->value, "%s", "");
-        host_result->val.iov_len = strlen((char *)host_result->val.iov_base);
+        host_result->value_len = strlen((char *)host_result->value);
         return;
     }
 
     char ip_string_buf[INET6_ADDRSTRLEN];
     if (4 == ssaddr.ipver)
     {
-        host_result->val.iov_base = MEMPOOL_CALLOC(mempool, char, (INET_ADDRSTRLEN + 7) /* "ip:port" max length */);
+        host_result->value = MEMPOOL_CALLOC(mempool, char, (INET_ADDRSTRLEN + 7) /* "ip:port" max length */);
         inet_ntop(AF_INET, &ssaddr.daddr4, ip_string_buf, INET_ADDRSTRLEN);
-        sprintf((char *)host_result->val.iov_base, "%s:%u", ip_string_buf, ntohs(ssaddr.dport));
+        sprintf((char *)host_result->value, "%s:%u", ip_string_buf, ntohs(ssaddr.dport));
-        host_result->val.iov_len = strlen((char *)host_result->val.iov_base);
+        host_result->value_len = strlen((char *)host_result->value);
     }
     else if (6 == ssaddr.ipver)
     {
-        host_result->val.iov_base = MEMPOOL_CALLOC(mempool, char, (INET6_ADDRSTRLEN + 7) /* "ip:port" max length */);
+        host_result->value = MEMPOOL_CALLOC(mempool, char, (INET6_ADDRSTRLEN + 7) /* "ip:port" max length */);
         inet_ntop(AF_INET6, &ssaddr.daddr6, ip_string_buf, INET6_ADDRSTRLEN);
-        sprintf((char *)host_result->val.iov_base, "%s:%u", ip_string_buf, ntohs(ssaddr.dport));
+        sprintf((char *)host_result->value, "%s:%u", ip_string_buf, ntohs(ssaddr.dport));
-        host_result->val.iov_len = strlen((char *)host_result->val.iov_base);
+        host_result->value_len = strlen((char *)host_result->value);
     }
     else
     {
@@ -1036,19 +1029,19 @@ static void using_session_addr_as_host(struct session *ref_session,
 #endif
 }
 
-void http_decoder_join_url(struct http_decoder_half_data *hfdata, nmx_pool_t *mempool, const struct http_header *host_hdr)
+void http_decoder_join_url(struct http_decoder_half_data *hfdata, nmx_pool_t *mempool, const struct http_header_field *host_hdr)
 {
     int append_slash_len = 0;
     if ('/' != ((char *)hfdata->joint_url.iov_base)[0])
     {
         append_slash_len = 1;
     }
-    int url_cache_str_len = host_hdr->val.iov_len + hfdata->joint_url.iov_len + append_slash_len;
+    int url_cache_str_len = host_hdr->value_len + hfdata->joint_url.iov_len + append_slash_len;
     char *url_cache_str = MEMPOOL_CALLOC(mempool, char, url_cache_str_len);
 
     char *ptr = url_cache_str;
-    memcpy(ptr, host_hdr->val.iov_base, host_hdr->val.iov_len);
+    memcpy(ptr, host_hdr->value, host_hdr->value_len);
-    ptr += host_hdr->val.iov_len;
+    ptr += host_hdr->value_len;
     if (append_slash_len)
     {
         *ptr = '/';
@@ -1059,19 +1052,6 @@ void http_decoder_join_url(struct http_decoder_half_data *hfdata, nmx_pool_t *me
     MEMPOOL_FREE(mempool, hfdata->joint_url.iov_base); // free the cached uri buffer
     hfdata->joint_url.iov_base = url_cache_str;
     hfdata->joint_url.iov_len = url_cache_str_len;
-
-    hfdata->url_is_encoded = httpd_url_is_encoded(url_cache_str, url_cache_str_len);
-    if (hfdata->url_is_encoded)
-    {
-        hfdata->decoded_url.iov_base = MEMPOOL_CALLOC(mempool, char, url_cache_str_len);
-        httpd_url_decode(url_cache_str, url_cache_str_len, (char *)hfdata->decoded_url.iov_base, &hfdata->decoded_url.iov_len);
-    }
-    else
-    {
-        hfdata->decoded_url.iov_base = url_cache_str;
-        hfdata->decoded_url.iov_len = url_cache_str_len;
-    }
-
     hfdata->joint_url_complete = 1;
 }
 
@@ -1079,62 +1059,54 @@ void http_decoder_get_url(struct http_decoder_half_data *hfdata, nmx_pool_t *mem
 {
     struct http_request_line reqline = {};
     http_decoder_half_data_get_request_line(hfdata, &reqline);
-    if (unlikely(strncasecmp_safe("CONNECT", (char *)reqline.method.iov_base, 7, reqline.method.iov_len) == 0))
+    if (unlikely(http_strncasecmp_safe("CONNECT", (char *)reqline.method, 7, reqline.method_len) == 0))
     {
-        hfdata->joint_url.iov_base = MEMPOOL_CALLOC(mempool, char, reqline.uri.iov_len + 1);
+        hfdata->joint_url.iov_base = MEMPOOL_CALLOC(mempool, char, reqline.uri_len + 1);
-        memcpy(hfdata->joint_url.iov_base, reqline.uri.iov_base, reqline.uri.iov_len);
+        memcpy(hfdata->joint_url.iov_base, reqline.uri, reqline.uri_len);
-        hfdata->joint_url.iov_len = reqline.uri.iov_len;
+        hfdata->joint_url.iov_len = reqline.uri_len;
         hfdata->joint_url_complete = 1;
     }
 }
 
-int http_decoder_join_url_finally(struct http_event_context *ev_ctx,
+int http_decoder_join_url_finally(struct http_event_context *ev_ctx, struct http_decoder_half_data *hfdata, nmx_pool_t *mempool)
-                                  struct http_decoder_half_data *hfdata,
-                                  nmx_pool_t *mempool)
 {
-    struct http_header addr_as_host = {};
-
     if (hfdata->joint_url_complete)
     {
         return 0;
     }
-
+    struct http_header_field addr_as_host = {};
     using_session_addr_as_host(ev_ctx->ref_session, &addr_as_host, mempool);
     http_decoder_join_url(hfdata, mempool, &addr_as_host);
-    MEMPOOL_FREE(mempool, addr_as_host.val.iov_base); // free session addr to host buffer
+    MEMPOOL_FREE(mempool, addr_as_host.value); // free session addr to host buffer
     return 1;
 }
 
 void http_decoder_get_host_feed_url(struct http_decoder_half *half)
 {
-    struct http_header host_result = {};
-    hstring host_key = {(char *)"Host", 4};
-
     if (half->ref_data->joint_url_complete)
     {
         return;
     }
-
+    struct http_header_field host_result = {};
-    int host_header_cnt = http_decoder_half_data_get_header(half->ref_data, &host_key,
+    int host_header_cnt = http_decoder_half_data_get_header(half->ref_data, (char *)"Host", 4, &host_result);
-                                                            &host_result);
     if (host_header_cnt < 0)
     {
         return;
     }
-
     http_decoder_join_url(half->ref_data, half->http_ev_ctx->ref_mempool, &host_result);
 }
 
-int http_half_data_get_url(struct http_decoder_half_data *res_data, hstring *url)
+int http_half_data_get_url(struct http_decoder_half_data *res_data, const char **url_val, size_t *url_len)
 {
     if (0 == res_data->joint_url_complete)
     {
         return -1;
     }
-    url->iov_base = res_data->joint_url.iov_base;
+    *url_val = res_data->joint_url.iov_base;
-    url->iov_len = res_data->joint_url.iov_len;
+    *url_len = res_data->joint_url.iov_len;
     return 0;
 }
+#if 0
 int http_half_data_get_decode_url(struct http_decoder_half_data *res_data, hstring *url)
 {
     if (0 == res_data->joint_url_complete)
@@ -1145,6 +1117,7 @@ int http_half_data_get_decode_url(struct http_decoder_half_data *res_data, hstri
     url->iov_len = res_data->decoded_url.iov_len;
     return 0;
 }
+#endif
 
 int http_half_data_get_transaction_seq(struct http_decoder_half_data *hf_data)
 {

decoders/http/http_decoder_half.h

@@ -6,6 +6,10 @@
 #include "http_decoder_result_queue.h"
 #include <llhttp.h>
 
+#ifndef hstring
+#include <bits/types/struct_iovec.h>
+typedef struct iovec hstring;
+#endif
 // only one http event is fired at a time
 enum http_event
 {
@@ -38,6 +42,7 @@ struct http_event_context
 
 struct http_decoder_half;
 struct http_decoder_half_data;
+struct http_decoder_env;
 
 typedef void http_event_cb(enum http_event event, struct http_decoder_half_data **data,
                            struct http_event_context *ev_ctx, void *httpd_plugin_env);
@@ -69,16 +74,16 @@ int http_decoder_half_data_get_response_line(struct http_decoder_half_data *data
                                              struct http_response_line *line);
 
 int http_decoder_half_data_get_header(const struct http_decoder_half_data *data,
-                                      const hstring *key, struct http_header *hdr_res);
+                                      const char *name, size_t name_len, struct http_header_field *hdr_res);
 
 int http_decoder_half_data_iter_header(struct http_decoder_half_data *data,
-                                       struct http_header *header);
+                                       struct http_header_field *header);
 int http_decoder_half_data_reset_header_iter(struct http_decoder_half_data *req_data);
 int http_decoder_half_data_has_parsed_header(struct http_decoder_half_data *data);
 
-int http_decoder_half_data_get_raw_body(const struct http_decoder_half_data *data, hstring *body);
+int http_decoder_half_data_get_raw_body(const struct http_decoder_half_data *data, const char **body, size_t *body_len);
 
-int http_decoder_half_data_get_decompress_body(const struct http_decoder_half_data *data, hstring *body);
+int http_decoder_half_data_get_decompress_body(const struct http_decoder_half_data *data, const char **body, size_t *body_len);
 void http_half_get_lastest_decompress_buffer(struct http_decoder_half_data *data, hstring *decompress_body);
 void http_half_decompress_buffer_free(struct http_decoder_half_data *data, hstring *decompress_body);
 void http_decoder_half_data_dump(struct http_decoder_half *half);
@@ -88,11 +93,11 @@ void http_decoder_get_url(struct http_decoder_half_data *hfdata, nmx_pool_t *mem
 int http_half_data_get_decode_url(struct http_decoder_half_data *res_data, hstring *url);
 void http_decoder_join_url(struct http_decoder_half_data *hfdata,
                            nmx_pool_t *mempool,
-                           const struct http_header *host_hdr);
+                           const struct http_header_field *host_hdr);
 int http_decoder_join_url_finally(struct http_event_context *ev_ctx,
                                   struct http_decoder_half_data *hfdata,
                                   nmx_pool_t *mempool);
-int http_half_data_get_url(struct http_decoder_half_data *res_data, hstring *url);
+int http_half_data_get_url(struct http_decoder_half_data *res_data, const char **url_val, size_t *url_len);
 int http_half_data_get_transaction_seq(struct http_decoder_half_data *hf_data);
 
 void http_half_data_update_commit_index(struct http_decoder_half_data *half_data);

decoders/http/http_decoder_private.h

@@ -5,7 +5,6 @@
 #define __USE_MISC 1
 #endif
 
-#include <cstddef>
 #ifdef __cplusplus
 extern "C"
 {
@@ -31,6 +30,11 @@ extern "C"
 #include "fieldstat/fieldstat_easy.h"
 #include "toml/toml.h"
 
+#ifndef hstring
+#include <bits/types/struct_iovec.h>
+typedef struct iovec hstring;
+#endif
+
 #ifndef likely
 #define likely(x) __builtin_expect((x), 1)
 #endif
@@ -142,9 +146,9 @@ extern "C"
     struct http_message;
 
     struct http_message *http_message_new(enum http_message_type type, struct http_decoder_result_queue *queue,
-                                          int queue_index, unsigned char flow_type);
+                                          int queue_index, uint8_t flow_type);
     struct http_message *http_body_message_new(enum http_message_type type, struct http_decoder_result_queue *queue,
-                                               int queue_index, uint8_t flow_type, hstring *raw_payload);
+                                               int queue_index, uint8_t flow_type, hstring *raw_payload, hstring *decompress_payload);
     int http_topic_exdata_compose_get_index(const struct http_decoder_env *httpd_env, int by_topic_id);
 #ifdef __cplusplus
 }

decoders/http/http_decoder_string.c

@@ -246,24 +246,23 @@ enum string_state http_decoder_string_state(const struct http_decoder_string *rs
     return rstr->state;
 }
 
-int http_decoder_string_get(const struct http_decoder_string *rstr, hstring *out)
+int http_decoder_string_get(const struct http_decoder_string *rstr, char **name, size_t *name_len)
 {
-    if (NULL == rstr || NULL == out)
+    if (NULL == rstr || NULL == name || 0 == name_len)
     {
         return -1;
     }
 
     if (http_decoder_string_state(rstr) == STRING_STATE_COMMIT)
     {
-        out->iov_base = rstr->commit.iov_base;
+        *name = rstr->commit.iov_base;
-        out->iov_len = rstr->commit.iov_len;
+        *name_len = rstr->commit.iov_len;
     }
     else
     {
-        out->iov_base = NULL;
+        *name = NULL;
-        out->iov_len = 0;
+        *name_len = 0;
     }
-
     return 0;
 }
 
@@ -274,9 +273,9 @@ void http_decoder_string_dump(struct http_decoder_string *rstr, const char *desc
         return;
     }
 
-    char *refer_str = safe_dup((char *)rstr->refer.iov_base, rstr->refer.iov_len);
+    char *refer_str = http_safe_dup((char *)rstr->refer.iov_base, rstr->refer.iov_len);
-    char *cache_str = safe_dup((char *)rstr->cache.iov_base, rstr->cache.iov_len);
+    char *cache_str = http_safe_dup((char *)rstr->cache.iov_base, rstr->cache.iov_len);
-    char *commit_str = safe_dup((char *)rstr->commit.iov_base, rstr->commit.iov_len);
+    char *commit_str = http_safe_dup((char *)rstr->commit.iov_base, rstr->commit.iov_len);
 
     printf("%s: state: %s, refer: {len: %02zu, iov_base: %s}, cache: {len: %02zu, iov_base: %s}, commit: {len: %02zu, iov_base: %s}\n",
            desc, string_state_to_desc(rstr->state),

decoders/http/http_decoder_string.h

@@ -67,7 +67,7 @@ void http_decoder_string_reinit(struct http_decoder_string *rstr);
 
 enum string_state http_decoder_string_state(const struct http_decoder_string *rstr);
 
-int http_decoder_string_get(const struct http_decoder_string *rstr, hstring *out);
+int http_decoder_string_get(const struct http_decoder_string *rstr, char **name, size_t *name_len);
 
 void http_decoder_string_dump(struct http_decoder_string *rstr, const char *desc);
  

decoders/http/http_decoder_table.c

@@ -30,7 +30,7 @@ struct http_decoder_table
     size_t header_cnt;
     size_t header_index;        // current parsing header
     size_t header_iter;         // plugins iterate cursor
-    size_t commit_header_index; // pushed to plugins, whether has called http_message_header_next()
+    size_t commit_header_index; // pushed to plugins, whether has called http_message_get0_next_header()
     struct http_decoder_header *headers;
 };
 
@@ -410,58 +410,58 @@ void http_decoder_table_dump(struct http_decoder_table *table)
     }
 }
 
-int http_decoder_table_get_uri(const struct http_decoder_table *table, hstring *out)
+int http_decoder_table_get_uri(const struct http_decoder_table *table, char **out, size_t *out_len)
 {
     if (NULL == table || NULL == out)
     {
         return -1;
     }
-    return http_decoder_string_get(&table->uri, out);
+    return http_decoder_string_get(&table->uri, out, out_len);
 }
 
-int http_decoder_table_get_method(const struct http_decoder_table *table, hstring *out)
+int http_decoder_table_get_method(const struct http_decoder_table *table, char **out, size_t *out_len)
 {
     if (NULL == table || NULL == out)
     {
         return -1;
     }
-    return http_decoder_string_get(&table->method, out);
+    return http_decoder_string_get(&table->method, out, out_len);
 }
 
-int http_decoder_table_get_status(const struct http_decoder_table *table, hstring *out)
+int http_decoder_table_get_status(const struct http_decoder_table *table, char **out, size_t *out_len)
 {
     if (NULL == table || NULL == out)
     {
         return -1;
     }
-    return http_decoder_string_get(&table->status, out);
+    return http_decoder_string_get(&table->status, out, out_len);
 }
 
-int http_decoder_table_get_version(const struct http_decoder_table *table, hstring *out)
+int http_decoder_table_get_version(const struct http_decoder_table *table, char **out, size_t *out_len)
 {
     if (NULL == table || NULL == out)
     {
         return -1;
     }
-    return http_decoder_string_get(&table->version, out);
+    return http_decoder_string_get(&table->version, out, out_len);
 }
 
-int http_decoder_table_get_body(const struct http_decoder_table *table, hstring *out)
+int http_decoder_table_get_body(const struct http_decoder_table *table, char **out, size_t *out_len)
 {
     if (NULL == table || NULL == out)
     {
         return -1;
     }
-    return http_decoder_string_get(&table->body, out);
+    return http_decoder_string_get(&table->body, (char **)out, out_len);
 }
 
-int http_decoder_table_get_header(const struct http_decoder_table *table, const hstring *key,
+int http_decoder_table_get_header(const struct http_decoder_table *table, const char *name, size_t name_len,
-                                  struct http_header *hdr_result)
+                                  struct http_header_field *hdr_result)
 {
     for (size_t i = 0; i < table->header_cnt; i++)
     {
         const struct http_decoder_header *tmp_header = &table->headers[i];
-        if (tmp_header->key.commit.iov_len != key->iov_len)
+        if (tmp_header->key.commit.iov_len != name_len)
         {
             continue;
         }
@@ -470,13 +470,13 @@ int http_decoder_table_get_header(const struct http_decoder_table *table, const
             http_decoder_string_state(&tmp_header->val) == STRING_STATE_COMMIT)
         {
             hstring tmp_key;
-            http_decoder_string_get(&tmp_header->key, &tmp_key);
+            http_decoder_string_get(&tmp_header->key, (char **)&tmp_key.iov_base, &tmp_key.iov_len);
 
-            if (tmp_key.iov_len == key->iov_len &&
+            if (tmp_key.iov_len == name_len &&
-                (0 == strncasecmp((char *)tmp_key.iov_base, (char *)key->iov_base, key->iov_len)))
+                (0 == strncasecmp((char *)tmp_key.iov_base, name, name_len)))
             {
-                http_decoder_string_get(&tmp_header->key, &hdr_result->key);
+                http_decoder_string_get(&tmp_header->key, &hdr_result->name, &hdr_result->name_len);
-                http_decoder_string_get(&tmp_header->val, &hdr_result->val);
+                http_decoder_string_get(&tmp_header->val, &hdr_result->value, &hdr_result->value_len);
                 return 0;
             }
         }
@@ -484,7 +484,7 @@ int http_decoder_table_get_header(const struct http_decoder_table *table, const
     return -1;
 }
 
-int http_decoder_table_iter_header(struct http_decoder_table *table, struct http_header *hdr)
+int http_decoder_table_iter_header(struct http_decoder_table *table, struct http_header_field *hdr)
 {
     if (NULL == table || NULL == hdr)
     {
@@ -502,17 +502,17 @@ int http_decoder_table_iter_header(struct http_decoder_table *table, struct http
             http_decoder_string_state(&tmp_header->val) == STRING_STATE_COMMIT)
         {
 
-            http_decoder_string_get(&tmp_header->key, &hdr->key);
+            http_decoder_string_get(&tmp_header->key, &hdr->name, &hdr->name_len);
-            http_decoder_string_get(&tmp_header->val, &hdr->val);
+            http_decoder_string_get(&tmp_header->val, &hdr->value, &hdr->value_len);
             table->header_iter++;
             return 1;
         }
     }
 
-    hdr->key.iov_base = NULL;
+    hdr->name = NULL;
-    hdr->key.iov_len = 0;
+    hdr->name_len = 0;
-    hdr->val.iov_base = NULL;
+    hdr->value = NULL;
-    hdr->val.iov_len = 0;
+    hdr->value_len = 0;
 
     return -1;
 }

decoders/http/http_decoder_table.h

@@ -36,22 +36,22 @@ void http_decoder_table_reinit(struct http_decoder_table *table);
 
 void http_decoder_table_dump(struct http_decoder_table *table);
 
-int http_decoder_table_get_uri(const struct http_decoder_table *table, hstring *out);
+int http_decoder_table_get_uri(const struct http_decoder_table *table, char **out, size_t *out_len);
 
-int http_decoder_table_get_method(const struct http_decoder_table *table, hstring *out);
+int http_decoder_table_get_method(const struct http_decoder_table *table, char **out, size_t *out_len);
 
-int http_decoder_table_get_status(const struct http_decoder_table *table, hstring *out);
+int http_decoder_table_get_status(const struct http_decoder_table *table, char **out, size_t *out_len);
 
-int http_decoder_table_get_version(const struct http_decoder_table *table, hstring *out);
+int http_decoder_table_get_version(const struct http_decoder_table *table, char **out, size_t *out_len);
 
-int http_decoder_table_get_body(const struct http_decoder_table *table, hstring *out);
+int http_decoder_table_get_body(const struct http_decoder_table *table, char **out, size_t *out_len);
 
 int http_decoder_table_get_header(const struct http_decoder_table *table,
-                                  const hstring *key,
+                                  const char *name, size_t name_len,
-                                  struct http_header *hdr_res);
+                                  struct http_header_field *hdr_res);
 
 int http_decoder_table_iter_header(struct http_decoder_table *table,
-                                   struct http_header *hdr);
+                                   struct http_header_field *hdr);
 int http_decoder_table_reset_header_iter(struct http_decoder_table *table);
 /**
  * @brief Is there a parsed header

decoders/http/http_decoder_tunnel.c

@@ -23,8 +23,8 @@ int httpd_tunnel_identify(struct http_decoder_env *httpd_env, int curdir, struct
     {
         struct http_request_line reqline = {};
         http_decoder_half_data_get_request_line(hfdata, &reqline);
-        if (0 == strncasecmp_safe("CONNECT", (char *)reqline.method.iov_base,
+        if (0 == http_strncasecmp_safe("CONNECT", (char *)reqline.method,
-                                  7, reqline.method.iov_len))
+                                  7, reqline.method_len))
         {
             return 1;
         }
@@ -33,8 +33,8 @@ int httpd_tunnel_identify(struct http_decoder_env *httpd_env, int curdir, struct
     {
         struct http_response_line resline = {};
         http_decoder_half_data_get_response_line(hfdata, &resline);
-        if (resline.status_code == HTTP_STATUS_OK && 0 == strncasecmp_safe("Connection established", (char *)resline.status.iov_base,
+        if (resline.status_code == HTTP_STATUS_OK && 0 == http_strncasecmp_safe("Connection established", (char *)resline.status,
-                                                                           strlen("Connection established"), resline.status.iov_len))
+                                                                           strlen("Connection established"), resline.status_len))
         {
             return 1;
         }

decoders/http/http_decoder_utils.c

@@ -3,7 +3,7 @@
 #include "stellar/http.h"
 #include "http_decoder_private.h"
 
-char *safe_dup(const char *str, size_t len)
+char *http_safe_dup(const char *str, size_t len)
 {
     if (str == NULL || len == 0)
     {
@@ -14,7 +14,7 @@ char *safe_dup(const char *str, size_t len)
     return dup;
 }
 
-int strncasecmp_safe(const char *fix_s1, const char *dyn_s2, size_t fix_n1, size_t dyn_n2)
+int http_strncasecmp_safe(const char *fix_s1, const char *dyn_s2, size_t fix_n1, size_t dyn_n2)
 {
     if (fix_s1 == NULL || dyn_s2 == NULL)
     {
@@ -187,11 +187,15 @@ static const unsigned char __g_httpd_hextable[] = {
 
 #include <ctype.h>
 // https://github.com/curl/curl/blob/2e930c333658725657b94a923d175c6622e0f41d/lib/urlapi.c
-void httpd_url_decode(const char *string, size_t length, char *ostring, size_t *olen)
+// void httpd_url_decode(const char *string, size_t length, char *ostring, size_t *olen)
+size_t http_url_decode(const char *string, size_t length, char *ostring, size_t olen)
 {
-    size_t alloc = length;
     char *ns = ostring;
-
+    if (NULL == string || NULL == ostring || 0 == olen)
+    {
+        return 0;
+    }
+    size_t alloc = length;
     while (alloc)
     {
         unsigned char in = (unsigned char)*string;
@@ -209,14 +213,12 @@ void httpd_url_decode(const char *string, size_t length, char *ostring, size_t *
             alloc--;
         }
         *ns++ = (char)in;
+        // if ((size_t)(ns - ostring) >= olen - 1)
+        // {
+        //     return 1;
+        // }
     }
-    *ns = 0; /* terminate it */
+    return ns - ostring;
-
-    if (olen)
-        /* store output size */
-        *olen = ns - ostring;
-
-    return;
 }
 
 int httpd_url_is_encoded(const char *url, size_t len)

decoders/http/http_decoder_utils.h

@@ -2,7 +2,6 @@
 
 #include <stdlib.h>
 #include <stdio.h>
-#include <cstddef>
 #ifdef __cplusplus
 extern "C"
 {
@@ -17,8 +16,8 @@ extern "C"
 }
 #endif
 
-char *safe_dup(const char *str, size_t len);
+char *http_safe_dup(const char *str, size_t len);
-int strncasecmp_safe(const char *fix_s1, const char *dyn_s2, size_t fix_n1, size_t dyn_n2);
+int http_strncasecmp_safe(const char *fix_s1, const char *dyn_s2, size_t fix_n1, size_t dyn_n2);
 const char *http_message_type_to_string(enum http_message_type type);
 int http_message_type_is_req(struct session *sess, enum http_message_type msg_type);
 int http_event_is_req(enum http_event event);

decoders/http/version.map

@@ -5,7 +5,7 @@ global:
 		http_decoder_init;
 		http_decoder_exit;
 		http_decoder_tcp_stream_msg_cb;
-		http_tunnel_message_*;
+		http_url_decode;
 	};
 local: *;
 };

include/stellar/http.h

@@ -1,12 +1,11 @@
 #pragma once
-
+#include <stddef.h>
 #ifdef __cplusplus
 extern "C"
 {
 #endif
-#include <bits/types/struct_iovec.h>
 
-#define HTTP_DECODER_TOPIC "HTTP_DECODER_MESSAGE"
+#define HTTP_TOPIC "HTTP_MESSAGE"
     struct http_message;
 
     enum http_message_type
@@ -15,14 +14,14 @@ extern "C"
 
         HTTP_MESSAGE_REQ_LINE,
         HTTP_MESSAGE_REQ_HEADER,
-        HTTP_MESSAGE_REQ_HEADER_END,
+        HTTP_MESSAGE_REQ_HEADER_END, // todo, delete END, push all fileds at once
         HTTP_MESSAGE_REQ_BODY_START,
         HTTP_MESSAGE_REQ_BODY,
         HTTP_MESSAGE_REQ_BODY_END,
 
         HTTP_MESSAGE_RES_LINE,
         HTTP_MESSAGE_RES_HEADER,
-        HTTP_MESSAGE_RES_HEADER_END,
+        HTTP_MESSAGE_RES_HEADER_END, // todo, delete END, push all fileds at once
         HTTP_MESSAGE_RES_BODY_START,
         HTTP_MESSAGE_RES_BODY,
         HTTP_MESSAGE_RES_BODY_END,
@@ -32,67 +31,77 @@ extern "C"
         HTTP_MESSAGE_MAX
     };
 
-    struct http_header
+    struct http_header_field
     {
-        struct iovec key;
+        char *name;
-        struct iovec val;
+        size_t name_len;
+        char *value;
+        size_t value_len;
     };
 
     struct http_request_line
     {
-        struct iovec method;
+        char *method;
-        struct iovec uri;
+        size_t method_len;
-        struct iovec version;
+        char *uri;
+        size_t uri_len;
+        char *version;
+        size_t version_len;
+
         int major_version;
         int minor_version;
     };
 
     struct http_response_line
     {
-        struct iovec version;
+        char *version;
-        struct iovec status;
+        size_t version_len;
+        char *status;
+        size_t status_len;
         int major_version;
         int minor_version;
         int status_code;
     };
 
-    typedef struct iovec hstring;
+    enum http_message_type http_message_get_type(const struct http_message *msg);
 
-    enum http_message_type http_message_type_get(const struct http_message *msg);
+    void http_message_get0_request_line(const struct http_message *msg, struct http_request_line *line);
 
-    void http_message_request_line_get0(const struct http_message *msg, struct http_request_line *line);
+    void http_message_get0_response_line(const struct http_message *msg, struct http_response_line *line);
-
-    void http_message_response_line_get0(const struct http_message *msg, struct http_response_line *line);
 
     /*
      *  Pay attention: key->iov_base is case-insensitive.
      */
-    void http_message_header_get0(const struct http_message *msg, const struct iovec *key, struct http_header *hdr_result);
+    void http_message_get0_header(const struct http_message *msg, const char *name, size_t name_len, struct http_header_field *field_result);
 
     /**
      * @brief loop reading all headers.
      *
      * @retval succeed( >= 0) failed(-1)
      */
-    int http_message_header_next(const struct http_message *msg, struct http_header *header);
+    int http_message_get0_next_header(const struct http_message *msg, struct http_header_field *header);
 
     /**
      * @retval succeed( >= 0) failed(-1)
      */
-    int http_message_reset_header_iter(struct http_message *msg);
+    int http_message_reset_header_iter(struct http_message *msg); // to do , obsoleted
 
-    void http_message_raw_body_get0(const struct http_message *msg, struct iovec *body);
+    void http_message_get0_uncompressed_body(const struct http_message *msg, const char **body, size_t *body_len);
 
     /**
-     * @brief If the body hasn't been compressed, same as http_message_raw_body_get0().
+     * @brief If the body hasn't been compressed, same as http_message_get0_uncompressed_body().
      *
      */
-    void http_message_decompress_body_get0(const struct http_message *msg, struct iovec *body);
+    void http_message_get0_decompressed_body(const struct http_message *msg, const char **body, size_t *body_len);
 
-    // raw
+    void http_message_get0_raw_url(const struct http_message *msg, const char **url, size_t *url_len);
-    void http_message_raw_url_get0(const struct http_message *msg, struct iovec *url);
 
-    void http_message_decoded_url_get0(const struct http_message *msg, struct iovec *url);
+    /*
+        return value:
+        0: failed
+        >1: success, length of decoded_url_buffer, not C string( no EOF with '\0' )
+    */
+    size_t http_url_decode(const char *raw_url, size_t raw_url_len, char *decoded_url_buffer, size_t decoded_url_buffer_len);
 
     /**
      * @retval succeed( >= 0) failed(-1)

infra/version.map

@@ -70,7 +70,7 @@ global:
     	http_decoder_init;
         http_decoder_exit;
         http_decoder_tcp_stream_msg_cb;
-        http_tunnel_message_*;
+		http_url_decode;
 
         lpi_plugin_load;
         lpi_plugin_unload;

test/decoders/http/CMakeLists.txt

@@ -1,27 +1,18 @@
-set(DECODER_NAME http)
-
 aux_source_directory(${PROJECT_SOURCE_DIR}/deps/toml DEPS_SRC)
+aux_source_directory(${PROJECT_SOURCE_DIR}/deps/md5 DEPS_SRC)
 
-add_library(${DECODER_NAME}_test SHARED http_decoder_test_plug.cpp md5.c ${DEPS_SRC})
+add_library(http_test SHARED http_test_plug.cpp ${DEPS_SRC})
-add_dependencies(${DECODER_NAME}_test ${DECODER_NAME})
+add_dependencies(http_test http)
-target_link_libraries(${DECODER_NAME}_test cjson-static llhttp-static)
+target_link_libraries(http_test cjson-static llhttp-static)
-set_target_properties(${DECODER_NAME}_test PROPERTIES PREFIX "")
+set_target_properties(http_test PROPERTIES PREFIX "")
-
-# add_library(${DECODER_NAME}_perf SHARED http_decoder_perf_plug.cpp)
-# add_dependencies(${DECODER_NAME}_perf ${DECODER_NAME})
-# set_target_properties(${DECODER_NAME}_perf PROPERTIES PREFIX "")
-
-set(TEST_RUN_DIR ${CMAKE_BINARY_DIR}/testing)
 
 include_directories(${CMAKE_SOURCE_DIR}/include)
 include_directories(${CMAKE_SOURCE_DIR}/deps)
 include_directories(${PROJECT_SOURCE_DIR}/decoders/http)
 include_directories(${PROJECT_SOURCE_DIR}/include/stellar)
 
-aux_source_directory(${PROJECT_SOURCE_DIR}/deps/toml PERF_TEST_DEP_SRC)
+add_executable(gtest_http http_gtest.cpp ${PROJECT_SOURCE_DIR}/decoders/http/http_decoder_utils.c ${PROJECT_SOURCE_DIR}/decoders/http/http_content_decompress.c ${PROJECT_SOURCE_DIR}/deps/md5/md5.c)
-
+target_link_libraries(gtest_http gtest stellar_lib llhttp-static z brotli-dec-static brotli-enc-static  brotli-common-static libevent-static)
-add_executable(gtest_http http_decoder_gtest.cpp ${PROJECT_SOURCE_DIR}/decoders/http/http_decoder_utils.cpp)
-target_link_libraries(gtest_http gtest stellar_lib llhttp-static)
 
 add_executable(http_test_main plugin_test_main.cpp)
 set_target_properties(http_test_main 

test/decoders/http/benchmarks/json/http_pipeline_header_splitting.json

@@ -0,0 +1,69 @@
+[
+    {
+        "__X_HTTP_TUPLE4": "127.0.0.1:44980-127.0.0.1:8080-6-0"
+    },
+    {
+        "__X_HTTP_TRANSACTION": "request",
+        "__X_HTTP_TRANSACTION_SEQ": 0,
+        "method": "GET",
+        "uri": "/111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111.html",
+        "req_version": "1.1",
+        "major_version": 1,
+        "minor_version": 1,
+        "Host": "127.0.0.1:8080 ",
+        "User-Agent": "curl/7.61.111",
+        "Accept": "*/*",
+        "-X-header1": "value111",
+        "-X-header2": "value222",
+        "-X-header3": "value333",
+        "-X-header4": "value444",
+        "-X-header5": "value5555",
+        "__X_HTTP_URL": "127.0.0.1:8080 /111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111.html"
+    },
+    {
+        "__X_HTTP_TRANSACTION": "response",
+        "__X_HTTP_TRANSACTION_SEQ": 0,
+        "res_version": "1.1",
+        "res_status": "OK",
+        "major_version": 1,
+        "minor_version": 1,
+        "status_code": 200,
+        "Server": "BaseHTTP/0.6 Python/3.6.88",
+        "Date": "Sun, 01 Sep 2024 07:31:09 GMTT",
+        "Content-length": "88",
+        "__X_HTTP_RAW_PAYLOAD_MD5": "c396b63d897591e928fae959915e9ebc",
+        "__X_HTTP_DECOMPRESS_PAYLOAD_MD5": "c396b63d897591e928fae959915e9ebc"
+    },
+    {
+        "__X_HTTP_TRANSACTION": "request",
+        "__X_HTTP_TRANSACTION_SEQ": 1,
+        "method": "GET",
+        "uri": "/2222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222.html",
+        "req_version": "1.1",
+        "major_version": 1,
+        "minor_version": 1,
+        "Host": "127.0.0.1:8080",
+        "User-Agent": "curl/7.61.1",
+        "Accept": "*/*",
+        "-X-header1": "value111",
+        "-X-header2": "value2222",
+        "-X-header3": "value3333",
+        "-X-header4": "value4444",
+        "-X-header5": "value55555",
+        "__X_HTTP_URL": "127.0.0.1:8080/2222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222.html"
+    },
+    {
+        "__X_HTTP_TRANSACTION": "response",
+        "__X_HTTP_TRANSACTION_SEQ": 1,
+        "res_version": "1.1",
+        "res_status": "OK",
+        "major_version": 1,
+        "minor_version": 1,
+        "status_code": 200,
+        "Server": "BaseHTTP/0.6 Python/3.6.88",
+        "Date": "Sun, 01 Sep 2024 07:31:09 GMTT",
+        "Content-length": "89",
+        "__X_HTTP_RAW_PAYLOAD_MD5": "5e3c5a7de2696e52ef42d9ccfd8ce831",
+        "__X_HTTP_DECOMPRESS_PAYLOAD_MD5": "5e3c5a7de2696e52ef42d9ccfd8ce831"
+    }
+]