gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加对BR压缩编码的支持及对应的单元测试用例

Browse Source
This commit is contained in:
luqiuwen
2018-12-09 18:02:43 +06:00
parent de92efb380
commit 1d89768160
9 changed files with 453 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

487
plugin/protocol/http/src/http_convert.cpp
View File

@@ -3,204 +3,391 @@
#include <http_half.h>
#include <http_convert.h>
#include <event2/buffer.h>
#include <brotli/decode.h>
#include <brotli/encode.h>
struct hf_content_uncompress
{
/* MODE AND CALLBACKS */
unsigned int content_encode;
hf_private_cb * data_cb;
void * data_cb_user;
/* MODE AND CALLBACKS */
unsigned int content_encode;
hf_private_cb * data_cb;
void * data_cb_user;
/* ZLIB STREAM */
z_stream * z_stream_ptr;
unsigned char * chunk;
size_t sz_chunk;
/* ZLIB STREAM */
z_stream * z_stream_ptr;
/* BR DECODER STATE */
BrotliDecoderState * brdec_state;
unsigned char * chunk;
size_t sz_chunk;
};
struct hf_content_compress
{
z_stream * z_stream_ptr;
unsigned int content_encode;
unsigned int content_encode;
z_stream * z_stream_ptr;
BrotliEncoderState * brenc_state;
};
void hf_content_uncompress_destroy(struct hf_content_uncompress * cv_object)
{
(void) inflateEnd(cv_object->z_stream_ptr);
free(cv_object->z_stream_ptr);
free(cv_object->chunk);
cv_object->z_stream_ptr = NULL;
free(cv_object);
if (cv_object->z_stream_ptr != NULL)
{
inflateEnd(cv_object->z_stream_ptr);
free(cv_object->z_stream_ptr);
cv_object->z_stream_ptr = NULL;
}
if (cv_object->brdec_state)
{
BrotliDecoderDestroyInstance(cv_object->brdec_state);
cv_object->brdec_state = NULL;
}
free(cv_object->chunk);
free(cv_object);
}
struct hf_content_uncompress * hf_content_uncompress_create(unsigned int content_encode,
hf_private_cb * data_cb, void * data_cb_user)
hf_private_cb * data_cb, void * data_cb_user)
{
struct hf_content_uncompress * cv_object = ALLOC(struct hf_content_uncompress, 1);
assert(data_cb != NULL);
struct hf_content_uncompress * cv_object = ALLOC(struct hf_content_uncompress, 1);
assert(data_cb != NULL);
cv_object->content_encode = content_encode;
cv_object->data_cb = data_cb;
cv_object->data_cb_user = data_cb_user;
cv_object->content_encode = content_encode;
cv_object->data_cb = data_cb;
cv_object->data_cb_user = data_cb_user;
/* ZSTREAM */
cv_object->z_stream_ptr = ALLOC(z_stream, 1);
cv_object->z_stream_ptr->zalloc = NULL;
cv_object->z_stream_ptr->zfree = NULL;
cv_object->z_stream_ptr->opaque = NULL;
cv_object->z_stream_ptr->avail_in = 0;
cv_object->z_stream_ptr->next_in = Z_NULL;
/* CHUNK, 4K */
/* CHUNK, 4K */
#define CHUNK_SIZE (1024 * 1024 * 4)
cv_object->chunk = (unsigned char *) malloc(CHUNK_SIZE);
cv_object->sz_chunk = CHUNK_SIZE;
cv_object->chunk = (unsigned char *) malloc(CHUNK_SIZE);
cv_object->sz_chunk = CHUNK_SIZE;
int ret = 0;
if (content_encode == HTTP_ACCEPT_ENCODING_GZIP)
{
ret = inflateInit2(cv_object->z_stream_ptr, MAX_WBITS + 16);
}
else if (content_encode == HTTP_ACCEPT_ENCODING_DEFLATE)
{
ret = inflateInit2(cv_object->z_stream_ptr, -MAX_WBITS);
}
/* ZSTREAM */
if (content_encode == HTTP_ACCEPT_ENCODING_GZIP || content_encode == HTTP_ACCEPT_ENCODING_DEFLATE)
{
cv_object->z_stream_ptr = ALLOC(z_stream, 1);
cv_object->z_stream_ptr->zalloc = NULL;
cv_object->z_stream_ptr->zfree = NULL;
cv_object->z_stream_ptr->opaque = NULL;
cv_object->z_stream_ptr->avail_in = 0;
cv_object->z_stream_ptr->next_in = Z_NULL;
if (ret != Z_OK) goto __errout;
return cv_object;
int ret = 0;
if (content_encode == HTTP_ACCEPT_ENCODING_GZIP)
{
ret = inflateInit2(cv_object->z_stream_ptr, MAX_WBITS + 16);
}
if (content_encode == HTTP_ACCEPT_ENCODING_DEFLATE)
{
ret = inflateInit2(cv_object->z_stream_ptr, -MAX_WBITS);
}
if (ret != Z_OK)
{
goto __errout;
}
}
if (content_encode == HTTP_ACCEPT_ENCODING_BR)
{
cv_object->brdec_state = BrotliDecoderCreateInstance(NULL, NULL, NULL);
if (unlikely(cv_object->brdec_state == NULL)) goto __errout;
}
return cv_object;
__errout:
free(cv_object->z_stream_ptr);
free(cv_object->chunk);
free(cv_object);
return NULL;
if (cv_object->z_stream_ptr != NULL)
{
free(cv_object->z_stream_ptr);
cv_object->z_stream_ptr = NULL;
}
if (cv_object->brdec_state != NULL)
{
BrotliDecoderDestroyInstance(cv_object->brdec_state);
cv_object->brdec_state = NULL;
}
free(cv_object->chunk);
free(cv_object);
return NULL;
}
static int __hf_content_uncompress_write_zlib(struct hf_content_uncompress * cv_object,
struct http_half_private * hf_private, tfe_http_event http_ev, const unsigned char * data, size_t datalen)
{
z_stream * z_stream_ptr = cv_object->z_stream_ptr;
z_stream_ptr->avail_in = (unsigned int) datalen;
z_stream_ptr->next_in = (unsigned char *) data;
if (z_stream_ptr->avail_in == 0)
{
(void) inflateEnd(z_stream_ptr);
return Z_ERRNO;
}
int ret = 0;
do
{
z_stream_ptr->avail_out = (unsigned int) cv_object->sz_chunk;
z_stream_ptr->next_out = cv_object->chunk;
ret = inflate(z_stream_ptr, Z_NO_FLUSH);
if (ret == Z_STREAM_ERROR || ret == Z_NEED_DICT ||
ret == Z_DATA_ERROR || ret == Z_MEM_ERROR)
{
goto __error;
}
unsigned int have = (unsigned int) cv_object->sz_chunk - z_stream_ptr->avail_out;
if (have > 0 && cv_object->data_cb != NULL)
{
cv_object->data_cb(hf_private, http_ev, cv_object->chunk, (size_t) have, cv_object->data_cb_user);
}
}
while (z_stream_ptr->avail_out == 0);
return ret;
__error:
(void) inflateEnd(z_stream_ptr);
return ret;
}
static int __hf_content_uncompress_write_br(struct hf_content_uncompress * cv_object,
struct http_half_private * hf_private, tfe_http_event http_ev, const unsigned char * data, size_t datalen)
{
size_t available_in = datalen;
const unsigned char * next_in = data;
size_t available_out;
unsigned char * next_out;
int ret;
for (;;)
{
available_out = cv_object->sz_chunk;
next_out = cv_object->chunk;
ret = BrotliDecoderDecompressStream(cv_object->brdec_state, &available_in, &next_in,
&available_out, &next_out, 0);
size_t have = (unsigned int) cv_object->sz_chunk - available_out;
if (have > 0 && cv_object->data_cb != NULL)
{
cv_object->data_cb(hf_private, http_ev, cv_object->chunk, have, cv_object->data_cb_user);
}
if (ret == BROTLI_DECODER_RESULT_SUCCESS || ret == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT)
{
return 1;
}
if (ret == BROTLI_DECODER_RESULT_ERROR)
{
BrotliDecoderErrorCode errcode = BrotliDecoderGetErrorCode(cv_object->brdec_state);
TFE_LOG_ERROR(g_http_plugin->logger, "BrotliDecoderDecompressStream() failed: errno = %d, %s",
errcode, BrotliDecoderErrorString(errcode)); return -1;
}
assert(ret == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT);
}
}
int hf_content_uncompress_write(struct hf_content_uncompress * cv_object,
struct http_half_private * hf_private, tfe_http_event http_ev, const unsigned char * data, size_t datalen)
struct http_half_private * hf_private, tfe_http_event http_ev, const unsigned char * data, size_t datalen)
{
z_stream * z_stream_ptr = cv_object->z_stream_ptr;
z_stream_ptr->avail_in = (unsigned int) datalen;
z_stream_ptr->next_in = (unsigned char *) data;
if (cv_object->content_encode == HTTP_ACCEPT_ENCODING_GZIP ||
cv_object->content_encode == HTTP_ACCEPT_ENCODING_DEFLATE)
{
return __hf_content_uncompress_write_zlib(cv_object, hf_private, http_ev, data, datalen);
}
if (z_stream_ptr->avail_in == 0)
{
(void) inflateEnd(z_stream_ptr);
return Z_ERRNO;
}
if (cv_object->content_encode == HTTP_ACCEPT_ENCODING_BR)
{
return __hf_content_uncompress_write_br(cv_object, hf_private, http_ev, data, datalen);
}
int ret = 0;
do
{
z_stream_ptr->avail_out = (unsigned int) cv_object->sz_chunk;
z_stream_ptr->next_out = cv_object->chunk;
ret = inflate(z_stream_ptr, Z_NO_FLUSH);
assert(ret != Z_STREAM_ERROR); /* state not clobbered */
if (ret == Z_NEED_DICT || ret == Z_DATA_ERROR || ret == Z_MEM_ERROR)
{
goto __error;
}
unsigned int have = (unsigned int) cv_object->sz_chunk - z_stream_ptr->avail_out;
if (have > 0 && cv_object->data_cb != NULL)
{
cv_object->data_cb(hf_private, http_ev, cv_object->chunk, (size_t) have, cv_object->data_cb_user);
}
} while (z_stream_ptr->avail_out == 0);
return ret;
__error:
(void) inflateEnd(z_stream_ptr);
return ret;
assert(0);
return -1;
}
struct hf_content_compress * hf_content_compress_create(unsigned int content_encode)
{
struct hf_content_compress * cv_object = ALLOC(struct hf_content_compress, 1);
cv_object->content_encode = content_encode;
struct hf_content_compress * cv_object = ALLOC(struct hf_content_compress, 1);
cv_object->content_encode = content_encode;
/* ZSTREAM */
cv_object->z_stream_ptr = ALLOC(z_stream, 1);
cv_object->z_stream_ptr->zalloc = NULL;
cv_object->z_stream_ptr->zfree = NULL;
cv_object->z_stream_ptr->opaque = NULL;
cv_object->z_stream_ptr->avail_in = 0;
cv_object->z_stream_ptr->next_in = Z_NULL;
if (cv_object->content_encode == HTTP_ACCEPT_ENCODING_GZIP ||
cv_object->content_encode == HTTP_ACCEPT_ENCODING_DEFLATE)
{
cv_object->z_stream_ptr = ALLOC(z_stream, 1);
cv_object->z_stream_ptr->zalloc = NULL;
cv_object->z_stream_ptr->zfree = NULL;
cv_object->z_stream_ptr->opaque = NULL;
cv_object->z_stream_ptr->avail_in = 0;
cv_object->z_stream_ptr->next_in = Z_NULL;
int __windows_bits = 0;
if (content_encode == HTTP_ACCEPT_ENCODING_GZIP)
{
__windows_bits = MAX_WBITS + 16;
}
int __windows_bits = 0;
if (content_encode == HTTP_ACCEPT_ENCODING_GZIP)
{
__windows_bits = MAX_WBITS + 16;
}
if (content_encode == HTTP_ACCEPT_ENCODING_DEFLATE)
{
__windows_bits = -MAX_WBITS;
}
if (content_encode == HTTP_ACCEPT_ENCODING_DEFLATE)
{
__windows_bits = -MAX_WBITS;
}
int ret = deflateInit2(cv_object->z_stream_ptr, Z_DEFAULT_COMPRESSION,
Z_DEFLATED, __windows_bits, 8, Z_DEFAULT_STRATEGY);
int ret = deflateInit2(cv_object->z_stream_ptr, Z_DEFAULT_COMPRESSION,
Z_DEFLATED, __windows_bits, 8, Z_DEFAULT_STRATEGY);
if (ret != Z_OK) goto __errout;
return cv_object;
if (ret != Z_OK) goto __errout;
}
if (cv_object->content_encode == HTTP_ACCEPT_ENCODING_BR)
{
cv_object->brenc_state = BrotliEncoderCreateInstance(NULL, NULL, NULL);
if (unlikely(cv_object->brenc_state == NULL)) goto __errout;
}
return cv_object;
__errout:
free(cv_object->z_stream_ptr);
free(cv_object);
return NULL;
if (cv_object->z_stream_ptr)
{
free(&cv_object->z_stream_ptr);
cv_object->z_stream_ptr = NULL;
}
if (cv_object->brenc_state)
{
BrotliEncoderDestroyInstance(cv_object->brenc_state);
cv_object->brenc_state = NULL;
}
free(cv_object);
return NULL;
}
#define SZ_IOVEC 2
#define SZ_RESERVE_SPACE 512
static int __hf_content_compress_write_zlib(struct hf_content_compress * cv_object,
const unsigned char * in_data, size_t sz_in_data, struct evbuffer * out_ev_buf, int end)
{
struct evbuffer_iovec v[SZ_IOVEC];
/* Reserve the space, because the length of the compressed data will be short
* than uncompressed data in usually, we set the reserve space as much as sz_in_data */
size_t __sz_reserve_space = sz_in_data > SZ_RESERVE_SPACE ? sz_in_data : SZ_RESERVE_SPACE;
int iov_count = evbuffer_reserve_space(out_ev_buf, __sz_reserve_space, v, SZ_IOVEC);
if (iov_count < 1 || iov_count > SZ_IOVEC) return -1;
z_stream * z = cv_object->z_stream_ptr;
z->next_in = (unsigned char *) in_data;
z->avail_in = (unsigned int) sz_in_data;
unsigned int iov_offset = 0;
z->next_out = (unsigned char *) v[iov_offset].iov_base;
z->avail_out = (unsigned int) v[iov_offset].iov_len;
int flush = end ? Z_FINISH : Z_NO_FLUSH;
int ret = 0;
do
{
ret = deflate(z, flush);
assert(ret != Z_STREAM_ERROR);
assert(iov_offset < SZ_IOVEC);
if (z->avail_out == 0 || z->avail_in == 0)
{
unsigned int len = (unsigned int) v[iov_offset].iov_len - z->avail_out;
v[iov_offset].iov_len = (size_t) len;
iov_offset++;
z->next_out = (unsigned char *) v[iov_offset].iov_base;
z->avail_out = (unsigned int) v[iov_offset].iov_len;
}
}
while (z->avail_in > 0);
assert(end == 0 || ret == Z_STREAM_END);
(void) ret;
return evbuffer_commit_space(out_ev_buf, v, iov_count);
}
static int __hf_content_compress_write_br(struct hf_content_compress * cv_object,
const unsigned char * in_data, size_t sz_in_data, struct evbuffer * out_ev_buf, int end)
{
struct evbuffer_iovec v[SZ_IOVEC];
/* Reserve the space, because the length of the compressed data will be short
* than uncompressed data in usually, we set the reserve space as much as sz_in_data */
size_t __sz_reserve_space = sz_in_data > SZ_RESERVE_SPACE ? sz_in_data : SZ_RESERVE_SPACE;
int iov_count = evbuffer_reserve_space(out_ev_buf, __sz_reserve_space, v, SZ_IOVEC);
if (iov_count < 1 || iov_count > SZ_IOVEC) return -1;
const unsigned char * next_in = in_data;
size_t avail_in = sz_in_data;
unsigned int iov_offset = 0;
unsigned char * next_out = (unsigned char *)v[iov_offset].iov_base;
size_t avail_out = v[iov_offset].iov_len;
enum BrotliEncoderOperation op = end ? BROTLI_OPERATION_FINISH : BROTLI_OPERATION_PROCESS;
int ret = 0;
do
{
ret = BrotliEncoderCompressStream(cv_object->brenc_state, op,
&avail_in, &next_in, &avail_out, &next_out, NULL);
if(unlikely(ret == BROTLI_FALSE))
{
TFE_LOG_ERROR(g_http_plugin->logger, "BrotliEncoderCompressStream() error.");
return ret;
}
assert(iov_offset < SZ_IOVEC);
if (avail_out == 0 || avail_in == 0)
{
size_t len = v[iov_offset].iov_len - avail_out;
v[iov_offset].iov_len = len;
iov_offset++;
next_out = (unsigned char *) v[iov_offset].iov_base;
avail_out = (unsigned int) v[iov_offset].iov_len;
}
}
while (avail_in > 0);
(void) ret;
return evbuffer_commit_space(out_ev_buf, v, iov_count);
}
int hf_content_compress_write(struct hf_content_compress * cv_object,
const unsigned char * in_data, size_t sz_in_data, struct evbuffer * out_ev_buf, int end)
const unsigned char * in_data, size_t sz_in_data, struct evbuffer * out_ev_buf, int end)
{
#define SZ_IOVEC 2
struct evbuffer_iovec v[SZ_IOVEC];
if (cv_object->content_encode == HTTP_ACCEPT_ENCODING_GZIP ||
cv_object->content_encode == HTTP_ACCEPT_ENCODING_DEFLATE)
{
return __hf_content_compress_write_zlib(cv_object, in_data, sz_in_data, out_ev_buf, end);
}
/* Reserve the space, because the length of the compressed data will be short
* than uncompressed data in usually, we set the reserve space as much as sz_in_data */
size_t __sz_reserve_space = sz_in_data > 512 ? sz_in_data : 512;
int iov_count = evbuffer_reserve_space(out_ev_buf, __sz_reserve_space, v, SZ_IOVEC);
if (iov_count < 1 || iov_count > SZ_IOVEC) return -1;
if (cv_object->content_encode == HTTP_ACCEPT_ENCODING_BR)
{
return __hf_content_compress_write_br(cv_object, in_data, sz_in_data, out_ev_buf, end);
}
z_stream * z = cv_object->z_stream_ptr;
z->next_in = (unsigned char *) in_data;
z->avail_in = (unsigned int) sz_in_data;
unsigned int iov_offset = 0;
z->next_out = (unsigned char *) v[iov_offset].iov_base;
z->avail_out = (unsigned int) v[iov_offset].iov_len;
int flush = end ? Z_FINISH : Z_NO_FLUSH;
int ret = 0;
do
{
ret = deflate(z, flush);
assert(ret != Z_STREAM_ERROR);
assert(iov_offset < SZ_IOVEC);
if (z->avail_out == 0 || z->avail_in == 0)
{
unsigned int len = (unsigned int) v[iov_offset].iov_len - z->avail_out;
v[iov_offset].iov_len = (size_t) len;
iov_offset++;
z->next_out = (unsigned char *) v[iov_offset].iov_base;
z->avail_out = (unsigned int) v[iov_offset].iov_len;
}
} while (z->avail_in > 0);
assert(end == 0 || ret == Z_STREAM_END);
(void)ret;
return evbuffer_commit_space(out_ev_buf, v, iov_count);
assert(0);
return -1;
}
void hf_content_compress_destroy(hf_content_compress * cv_object)
{
(void) deflateEnd(cv_object->z_stream_ptr);
free(cv_object->z_stream_ptr);
cv_object->z_stream_ptr = NULL;
free(cv_object);
(void) deflateEnd(cv_object->z_stream_ptr);
free(cv_object->z_stream_ptr);
cv_object->z_stream_ptr = NULL;
free(cv_object);
}

6
plugin/protocol/http/src/http_entry.cpp
View File

@@ -89,8 +89,8 @@ int http_plugin_init(struct tfe_proxy * proxy)
plugin_ctx->gc_event_hs_private[thread_id] = gc_event;
}
plugin_ctx->access_logger = MESA_create_runtime_log_handle("log/http.log", RLOG_LV_INFO);
assert(plugin_ctx->access_logger != NULL);
plugin_ctx->logger = MESA_create_runtime_log_handle("log/http.log", RLOG_LV_INFO);
assert(plugin_ctx->logger != NULL);
return 0;
}
@@ -438,7 +438,7 @@ enum tfe_stream_action http_connection_entry(const struct tfe_stream * stream, e
/* Some kind of error happened, write log and detach the stream */
if (ret == -1)
{
TFE_LOG_ERROR(g_http_plugin->access_logger, "%s: Failed at parsing stream as HTTP, %u, %s, %s",
TFE_LOG_ERROR(g_http_plugin->logger, "%s: Failed at parsing stream as HTTP, %u, %s, %s",
stream->str_stream_info, hf_private_in->parse_errno, http_errno_name(hf_private_in->parse_errno),
http_errno_description(hf_private_in->parse_errno));

10
plugin/protocol/http/src/http_half.cpp
View File

@@ -97,6 +97,11 @@ uint16_t __hf_content_encoding_parse(const char * str_content_encoding)
return HTTP_ACCEPT_ENCODING_X_BZIP2;
}
if(strcasestr(str_content_encoding, "br") != NULL)
{
return HTTP_ACCEPT_ENCODING_BR;
}
return HTTP_ACCEPT_ENCODING_NONE;
}
@@ -109,6 +114,7 @@ const char * __hf_content_encoding_to_str(unsigned int encode)
case HTTP_ACCEPT_ENCODING_DEFLATE: return "deflate";
case HTTP_ACCEPT_ENCODING_BZIP2: return "bzip2";
case HTTP_ACCEPT_ENCODING_X_BZIP2: return "x-bzip2";
case HTTP_ACCEPT_ENCODING_BR: return "br";
default: return "";
}
}
@@ -431,7 +437,7 @@ static int __parser_callback_on_body(struct http_parser * parser, const char * a
const char * __str_stream = hf_private->session->hc_private->stream->str_stream_info;
const char * __str_url = hf_private->session->hs_public.req->req_spec.url;
TFE_LOG_ERROR(g_http_plugin->access_logger, "%s %s %s, ret = %d. ", __str_stream, __str_url, __what, ret);
TFE_LOG_ERROR(g_http_plugin->logger, "%s %s %s, ret = %d. ", __str_stream, __str_url, __what, ret);
hf_private->is_passthrough = true; return -1;
}
}
@@ -1200,7 +1206,7 @@ void __write_access_log(struct http_session_private * hs_private)
request->major, request->minor, __str_resp_code, __str_cont_type, __str_cont_encoding,
__str_upgrade, __str_req_passthrough, __str_resp_passthrough, __str_user_req, __str_user_resp, __str_suspend);
TFE_LOG_INFO(g_http_plugin->access_logger, "%s", __access_log);
TFE_LOG_INFO(g_http_plugin->logger, "%s", __access_log);
free(__access_log);
}