gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加GZIP压缩功能及对应的单元测试

Browse Source
This commit is contained in:
Lu Qiuwen
2018-09-28 15:05:54 +08:00
parent 5bf18848c1
commit a14b665f90
5 changed files with 208 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
plugin/protocol/http/include/internal/http_convert.h
View File

@@ -2,18 +2,20 @@
#include <tfe_http.h>
#include <http_half.h>
struct hf_content_converter;
struct hf_content_uncompress;
struct hf_content_compress;
enum hf_content_conv_work_mode
{
HF_CONTENT_CONV_COMPRASS,
HF_CONTENT_CONV_UNCOMPRASS
};
int hf_content_converter_write(struct hf_content_converter * cv_object,
int hf_content_uncompress_write(struct hf_content_uncompress * cv_object,
struct http_half_private * hf_private, tfe_http_event http_ev, const unsigned char * data, size_t datalen);
void hf_content_converter_destroy(struct hf_content_converter * cv_object);
void hf_content_uncompress_destroy(struct hf_content_uncompress * cv_object);
struct hf_content_converter * hf_content_converter_create(enum hf_content_conv_work_mode mode,
unsigned int content_encode, hf_private_cb * data_cb, void * data_cb_user);
struct hf_content_uncompress * hf_content_uncompress_create(unsigned int content_encode, hf_private_cb * data_cb,
void * data_cb_user);
struct hf_content_compress * hf_content_compress_create(unsigned int content_encode);
int hf_content_compress_write(struct hf_content_compress * cv_object,
const unsigned char * in_data, size_t sz_in_data, struct evbuffer * out_ev_buf, int end);
void hf_content_compress_destroy(hf_content_compress * cv_object);

2
plugin/protocol/http/include/internal/http_half.h
View File

@@ -66,7 +66,7 @@ struct http_half_private
/* Content-Encoding */
uint16_t content_encoding;
struct hf_content_converter * cv_unpress_object;
struct hf_content_uncompress * cv_uncompress_object;
/* Header Parser */
struct evbuffer * evbuf_header_field;

129
plugin/protocol/http/src/http_convert.cpp
View File

@@ -2,11 +2,11 @@
#include <http_common.h>
#include <http_half.h>
#include <http_convert.h>
#include <event2/buffer.h>
struct hf_content_converter
struct hf_content_uncompress
{
/* MODE AND CALLBACKS */
enum hf_content_conv_work_mode mode;
unsigned int content_encode;
hf_private_cb * data_cb;
void * data_cb_user;
@@ -17,31 +17,26 @@ struct hf_content_converter
size_t sz_chunk;
};
void hf_content_converter_destroy(struct hf_content_converter * cv_object)
struct hf_content_compress
{
if (cv_object->z_stream_ptr && cv_object->mode == HF_CONTENT_CONV_COMPRASS)
{
(void)deflateEnd(cv_object->z_stream_ptr);
free(cv_object->z_stream_ptr);
}
if (cv_object->z_stream_ptr && cv_object->mode == HF_CONTENT_CONV_UNCOMPRASS)
{
(void)inflateEnd(cv_object->z_stream_ptr);
free(cv_object->z_stream_ptr);
}
z_stream * z_stream_ptr;
unsigned int content_encode;
};
void hf_content_uncompress_destroy(struct hf_content_uncompress * cv_object)
{
(void) inflateEnd(cv_object->z_stream_ptr);
free(cv_object->z_stream_ptr);
cv_object->z_stream_ptr = NULL;
free(cv_object);
}
struct hf_content_converter * hf_content_converter_create(enum hf_content_conv_work_mode mode,
unsigned int content_encode, hf_private_cb * data_cb, void * data_cb_user)
struct hf_content_uncompress * hf_content_uncompress_create(unsigned int content_encode,
hf_private_cb * data_cb, void * data_cb_user)
{
struct hf_content_converter * cv_object = ALLOC(struct hf_content_converter, 1);
struct hf_content_uncompress * cv_object = ALLOC(struct hf_content_uncompress, 1);
assert(data_cb != NULL);
cv_object->mode = mode;
cv_object->content_encode = content_encode;
cv_object->data_cb = data_cb;
cv_object->data_cb_user = data_cb_user;
@@ -55,8 +50,8 @@ struct hf_content_converter * hf_content_converter_create(enum hf_content_conv_w
cv_object->z_stream_ptr->next_in = Z_NULL;
/* CHUNK, 4K */
#define CHUNK_SIZE (1024 * 1024 * 4)
cv_object->chunk = (unsigned char *)malloc(CHUNK_SIZE);
#define CHUNK_SIZE (1024 * 1024 * 4)
cv_object->chunk = (unsigned char *) malloc(CHUNK_SIZE);
cv_object->sz_chunk = CHUNK_SIZE;
int ret = 0;
@@ -79,16 +74,16 @@ __errout:
return NULL;
}
int hf_content_converter_write(struct hf_content_converter * cv_object,
int hf_content_uncompress_write(struct hf_content_uncompress * cv_object,
struct http_half_private * hf_private, tfe_http_event http_ev, const unsigned char * data, size_t datalen)
{
z_stream * z_stream_ptr = cv_object->z_stream_ptr;
z_stream_ptr->avail_in = (unsigned int)datalen;
z_stream_ptr->next_in = (unsigned char *)data;
z_stream_ptr->avail_in = (unsigned int) datalen;
z_stream_ptr->next_in = (unsigned char *) data;
if (z_stream_ptr->avail_in == 0)
{
(void)inflateEnd(z_stream_ptr);
(void) inflateEnd(z_stream_ptr);
return Z_ERRNO;
}
@@ -116,6 +111,90 @@ int hf_content_converter_write(struct hf_content_converter * cv_object,
return ret;
__error:
(void)inflateEnd(z_stream_ptr);
(void) inflateEnd(z_stream_ptr);
return ret;
}
struct hf_content_compress * hf_content_compress_create(unsigned int content_encode)
{
struct hf_content_compress * cv_object = ALLOC(struct hf_content_compress, 1);
cv_object->content_encode = content_encode;
/* ZSTREAM */
cv_object->z_stream_ptr = ALLOC(z_stream, 1);
cv_object->z_stream_ptr->zalloc = NULL;
cv_object->z_stream_ptr->zfree = NULL;
cv_object->z_stream_ptr->opaque = NULL;
cv_object->z_stream_ptr->avail_in = 0;
cv_object->z_stream_ptr->next_in = Z_NULL;
int __windows_bits = 0;
if (content_encode == HTTP_ACCEPT_ENCODING_GZIP)
{
__windows_bits = MAX_WBITS + 16;
}
if (content_encode == HTTP_ACCEPT_ENCODING_DEFLATE)
{
__windows_bits = -MAX_WBITS;
}
int ret = deflateInit2(cv_object->z_stream_ptr, Z_DEFAULT_COMPRESSION,
Z_DEFLATED, __windows_bits, 8, Z_DEFAULT_STRATEGY);
if (ret != Z_OK) goto __errout;
return cv_object;
__errout:
free(cv_object->z_stream_ptr);
free(cv_object);
return NULL;
}
int hf_content_compress_write(struct hf_content_compress * cv_object,
const unsigned char * in_data, size_t sz_in_data, struct evbuffer * out_ev_buf, int end)
{
#define SZ_IOVEC 2
struct evbuffer_iovec v[SZ_IOVEC];
/* Reserve the space, because the length of the compressed data will be short
* than uncompressed data in usually, we set the reserve space as much as sz_in_data */
size_t __sz_reserve_space = sz_in_data > 512 ? sz_in_data : 512;
int iov_count = evbuffer_reserve_space(out_ev_buf, __sz_reserve_space, v, SZ_IOVEC);
if (iov_count < 1 || iov_count > SZ_IOVEC) return -1;
z_stream * z = cv_object->z_stream_ptr;
z->next_in = (unsigned char *) in_data;
z->avail_in = (unsigned int) sz_in_data;
unsigned int iov_offset = 0;
z->next_out = (unsigned char *) v[iov_offset].iov_base;
z->avail_out = (unsigned int) v[iov_offset].iov_len;
int flush = end ? Z_FINISH : Z_NO_FLUSH;
int ret = 0;
do
{
ret = deflate(z, flush);
assert(ret != Z_STREAM_ERROR);
assert(iov_offset < SZ_IOVEC);
if (z->avail_out == 0 || z->avail_in == 0)
{
unsigned int len = (unsigned int) v[iov_offset].iov_len - z->avail_out;
v[iov_offset].iov_len = (size_t) len;
iov_offset++;
z->next_out = (unsigned char *) v[iov_offset].iov_base;
z->avail_out = (unsigned int) v[iov_offset].iov_len;
}
} while (z->avail_in > 0);
assert(end == 0 || ret == Z_STREAM_END);
return evbuffer_commit_space(out_ev_buf, v, iov_count);
}
void hf_content_compress_destroy(hf_content_compress * cv_object)
{
return;
}

10
plugin/protocol/http/src/http_half.cpp
View File

@@ -332,10 +332,10 @@ static int __parser_callback_on_body(struct http_parser * parser, const char * a
/* Create ungzip context */
if (hf_private->content_encoding != HTTP_ACCEPT_ENCODING_NONE)
{
hf_private->cv_unpress_object = hf_content_converter_create(HF_CONTENT_CONV_UNCOMPRASS,
hf_private->cv_uncompress_object = hf_content_uncompress_create(
hf_private->content_encoding, hf_private->event_cb, hf_private->event_cb_user);
if (unlikely(hf_private->cv_unpress_object == NULL)) assert(0);
if (unlikely(hf_private->cv_uncompress_object == NULL)) assert(0);
}
hf_private->event_cb(hf_private, ev_body_begin, NULL, parser->content_length, hf_private->event_cb_user);
@@ -345,10 +345,10 @@ static int __parser_callback_on_body(struct http_parser * parser, const char * a
int ret = 0;
if (hf_private->event_cb && length != 0)
{
if (hf_private->cv_unpress_object != NULL)
if (hf_private->cv_uncompress_object != NULL)
{
ret = hf_content_converter_write(hf_private->cv_unpress_object, hf_private, ev_body_cont,
(const unsigned char *)at, length);
ret = hf_content_uncompress_write(hf_private->cv_uncompress_object, hf_private, ev_body_cont,
(const unsigned char *) at, length);
}
else
{

96
plugin/protocol/http/test/test_http_convert.cpp
View File

@@ -1,6 +1,7 @@
#include <gtest/gtest.h>
#include <http_convert.h>
#include <event.h>
unsigned char __64x[] = {
0x58, 0x58, 0x58, 0x58, 0x58, 0x58, 0x58, 0x58, 0x58, 0x58, 0x58, 0x58,
@@ -157,7 +158,7 @@ protected:
__ctx->data = new unsigned char[2048];
__ctx->sz_data = 0;
cv_object = hf_content_converter_create(HF_CONTENT_CONV_UNCOMPRASS,
cv_object = hf_content_uncompress_create(
HTTP_ACCEPT_ENCODING_GZIP, __gzip_64x_data_callback, __ctx);
ASSERT_TRUE(cv_object != NULL);
@@ -167,18 +168,18 @@ protected:
{
delete[] __ctx->data;
delete __ctx;
hf_content_converter_destroy(cv_object);
hf_content_uncompress_destroy(cv_object);
}
protected:
struct callback_ctx * __ctx{};
struct hf_content_converter * cv_object{};
struct hf_content_uncompress * cv_object{};
};
TEST_F(HttpConvertUncompress, Gzip64x)
{
int ret = hf_content_converter_write(cv_object, NULL, EV_HTTP_REQ_BODY_CONT,
(const unsigned char *)__64x_gz, (size_t)__64x_gz_len);
int ret = hf_content_uncompress_write(cv_object, NULL, EV_HTTP_REQ_BODY_CONT,
(const unsigned char *) __64x_gz, (size_t) __64x_gz_len);
EXPECT_EQ(ret, 1);
EXPECT_EQ(__ctx->sz_data, __64x_len);
@@ -187,8 +188,8 @@ TEST_F(HttpConvertUncompress, Gzip64x)
TEST_F(HttpConvertUncompress, GzipMonkey)
{
int ret = hf_content_converter_write(cv_object, NULL, EV_HTTP_REQ_BODY_CONT,
(const unsigned char *)monkey_gz, (size_t)monkey_gz_len);
int ret = hf_content_uncompress_write(cv_object, NULL, EV_HTTP_REQ_BODY_CONT,
(const unsigned char *) monkey_gz, (size_t) monkey_gz_len);
EXPECT_EQ(ret, 1);
EXPECT_EQ(__ctx->sz_data, monkey_len);
@@ -200,18 +201,91 @@ TEST_F(HttpConvertUncompress, GzipMonkeyFrag)
unsigned frag_length = monkey_gz_len / 2;
int ret = 0;
ret = hf_content_converter_write(cv_object, NULL, EV_HTTP_REQ_BODY_CONT,
(const unsigned char *)monkey_gz, frag_length);
ret = hf_content_uncompress_write(cv_object, NULL, EV_HTTP_REQ_BODY_CONT,
(const unsigned char *) monkey_gz, frag_length);
EXPECT_EQ(ret, 0);
ret = hf_content_converter_write(cv_object, NULL, EV_HTTP_REQ_BODY_CONT,
(const unsigned char *)monkey_gz + frag_length, monkey_gz_len - frag_length);
ret = hf_content_uncompress_write(cv_object, NULL, EV_HTTP_REQ_BODY_CONT,
(const unsigned char *) monkey_gz + frag_length, monkey_gz_len - frag_length);
EXPECT_EQ(ret, 1);
EXPECT_EQ(__ctx->sz_data, monkey_len);
EXPECT_EQ(memcmp(__ctx->data, monkey, monkey_len), 0);
}
class HttpConvertCompress : public ::testing::Test
{
protected:
static constexpr unsigned int __encode = HTTP_ACCEPT_ENCODING_GZIP;
void SetUp() override
{
cv_compress_object = hf_content_compress_create(__encode);
cv_uncompress_object = hf_content_uncompress_create(__encode, __gzip_uncompress_callback, &uncompress_buf);
ASSERT_TRUE(cv_compress_object != NULL);
}
void TearDown() override
{
hf_content_compress_destroy(cv_compress_object);
evbuffer_free(buf);
}
protected:
struct hf_content_compress * cv_compress_object{};
struct hf_content_uncompress * cv_uncompress_object{};
struct evbuffer * buf{evbuffer_new()};
std::vector<char> uncompress_buf;
protected:
static int __gzip_uncompress_callback(struct http_half_private * hf_private,
tfe_http_event ev, const unsigned char * data, size_t len, void * user)
{
auto * __uncompress_buf = static_cast<std::vector<char> *>(user);
__uncompress_buf->insert(__uncompress_buf->end(), data, data + len);
}
};
TEST_F(HttpConvertCompress, MonkeyToGzip)
{
int ret = hf_content_compress_write(cv_compress_object, monkey, sizeof(monkey), buf, 1);
ASSERT_EQ(ret, 0);
unsigned char * __raw_buf_ptr = evbuffer_pullup(buf, -1);
size_t __raw_buf_length = evbuffer_get_length(buf);
ret = hf_content_uncompress_write(cv_uncompress_object, NULL, EV_HTTP_REQ_BODY_CONT,
__raw_buf_ptr, __raw_buf_length);
ASSERT_TRUE(__raw_buf_ptr != NULL);
ASSERT_EQ(ret, 1);
EXPECT_EQ(uncompress_buf.size(), sizeof(monkey));
EXPECT_EQ(memcmp(uncompress_buf.data(), monkey, sizeof(monkey)), 0);
}
TEST_F(HttpConvertCompress, MonkeyToGzipStrem)
{
unsigned frag_length = sizeof(monkey) / 2;
/* First frag */
int ret = hf_content_compress_write(cv_compress_object, monkey, frag_length, buf, 0);
ASSERT_EQ(ret, 0);
/* Last frag */
ret = hf_content_compress_write(cv_compress_object, monkey + frag_length, sizeof(monkey) - frag_length, buf, 1);
ASSERT_EQ(ret, 0);
unsigned char * __raw_buf_ptr = evbuffer_pullup(buf, -1);
size_t __raw_buf_length = evbuffer_get_length(buf);
ret = hf_content_uncompress_write(cv_uncompress_object, NULL, EV_HTTP_REQ_BODY_CONT,
__raw_buf_ptr, __raw_buf_length);
ASSERT_TRUE(__raw_buf_ptr != NULL);
ASSERT_EQ(ret, 1);
EXPECT_EQ(uncompress_buf.size(), sizeof(monkey));
EXPECT_EQ(memcmp(uncompress_buf.data(), monkey, sizeof(monkey)), 0);
}
void tfe_stream_write_access_log(const struct tfe_stream * stream, int level, const char * fmt, ...)
{
return;