diff --git a/plugin/protocol/http2/CMakeLists.txt b/plugin/protocol/http2/CMakeLists.txt index bf14e6b..637267c 100644 --- a/plugin/protocol/http2/CMakeLists.txt +++ b/plugin/protocol/http2/CMakeLists.txt @@ -8,7 +8,7 @@ target_link_libraries(http2 nghttp2-static) target_link_libraries(http2 libevent-static) target_link_libraries(http2 z) -target_link_libraries(http2 brotlienc-static brotlidec-static) +target_link_libraries(http2 brotlienc-static brotlidec-static zstd-static) ### UNITTEST CASE add_executable(test-http2-stream test/test_http2_stream.cpp) diff --git a/plugin/protocol/http2/include/internal/http2_common.h b/plugin/protocol/http2/include/internal/http2_common.h index d34c26c..a0dd572 100644 --- a/plugin/protocol/http2/include/internal/http2_common.h +++ b/plugin/protocol/http2/include/internal/http2_common.h @@ -13,6 +13,8 @@ #include #include #include +#include +#include typedef struct RTLogInit2Data_ { @@ -72,25 +74,23 @@ http2_header_str_to_val(const char *str, size_t slen, const char * map[], unsign #define HTTP2_CONTENT_ENCODING_X_GZIP BV(5) #define HTTP2_CONTENT_ENCODING_X_BZIP2 BV(6) #define HTTP2_CONTENT_ENCODING_BR BV(7) +#define HTTP2_CONTENT_ENCODING_ZSTD BV(8) -struct z_stream_st{ +struct http2_codec_ctx +{ z_stream zst; BrotliDecoderState *brdec_state; BrotliEncoderState *brenc_state; + ZSTD_DCtx* dctx; + ZSTD_CCtx* cctx; }; RTLogInit2Data *logger(); Http2Plugin *http2_plugin(); -int inflate_read(const uint8_t *source,int len,char **dest, int *outlen, - struct z_stream_st **strm, int encode); - -int deflate_write(struct z_stream_st **strm, const uint8_t *source, - int slen, struct evbuffer * evbuf, int gzip, int end); - -void inflate_finished(struct z_stream_st **strm); - -void deflate_finished(struct z_stream_st **strm); - +int http2_decompress_stream(const uint8_t *source, int len, char **dest, int *outlen, struct http2_codec_ctx **codec_ctx, int encode); +int http2_compress_stream(struct http2_codec_ctx **codec_ctx, const uint8_t *source, int slen, struct evbuffer * evbuf, int encode, int mode); +void http2_compress_finished(struct http2_codec_ctx **codec_ctx); +void http2_decompress_finished(struct http2_codec_ctx **codec_ctx); #endif diff --git a/plugin/protocol/http2/include/internal/http2_stream.h b/plugin/protocol/http2/include/internal/http2_stream.h index e8548b7..821732a 100644 --- a/plugin/protocol/http2/include/internal/http2_stream.h +++ b/plugin/protocol/http2/include/internal/http2_stream.h @@ -27,11 +27,11 @@ enum h2_read_state struct tfe_h2_payload { - int gzip; + int encode_type; uint8_t flags; ssize_t padlen; - struct z_stream_st *inflate; - struct z_stream_st *deflate; + struct http2_codec_ctx *inflate; + struct http2_codec_ctx *deflate; struct evbuffer * evbuf_body; }; diff --git a/plugin/protocol/http2/src/http2_common.cpp b/plugin/protocol/http2/src/http2_common.cpp index 89ae7f4..fb313f4 100644 --- a/plugin/protocol/http2/src/http2_common.cpp +++ b/plugin/protocol/http2/src/http2_common.cpp @@ -136,43 +136,97 @@ str_to_val(const char *val, const struct value_string *vs) return str_to_val_idx(val, vs); } -int inflate_init(struct z_stream_st **strm, int gzip) +static int gzip_decompress_init(struct http2_codec_ctx **codec_ctx, int encode) { - if (*strm != NULL) + if (*codec_ctx != NULL) + { return Z_OK; - - *strm = ALLOC(struct z_stream_st, 1); - assert(*strm); - + } + + *codec_ctx = ALLOC(struct http2_codec_ctx, 1); + if(*codec_ctx == NULL) + { + *codec_ctx=NULL; + return -1; + } /* ZSTREAM */ - (*strm)->zst.zalloc = NULL; - (*strm)->zst.zfree = NULL; - (*strm)->zst.opaque = NULL; - (*strm)->zst.avail_in = 0; - (*strm)->zst.next_in = Z_NULL; + (*codec_ctx)->zst.zalloc = NULL; + (*codec_ctx)->zst.zfree = NULL; + (*codec_ctx)->zst.opaque = NULL; + (*codec_ctx)->zst.avail_in = 0; + (*codec_ctx)->zst.next_in = Z_NULL; // Z_OK stand for 0; Z_ERRNO stand for -1. int ret = Z_ERRNO; - if (gzip == HTTP2_CONTENT_ENCODING_GZIP) - ret = inflateInit2(&((*strm)->zst), MAX_WBITS + 16); - else if (gzip == HTTP2_CONTENT_ENCODING_DEFLATE) - ret = inflateInit2(&((*strm)->zst), -MAX_WBITS); - else if (gzip == HTTP2_CONTENT_ENCODING_BR){ - (*strm)->brdec_state = BrotliDecoderCreateInstance(NULL, NULL, NULL); - if ((*strm)->brdec_state != NULL) - ret = Z_OK; + if (encode == HTTP2_CONTENT_ENCODING_GZIP) + { + ret = inflateInit2(&((*codec_ctx)->zst), MAX_WBITS + 16); + } + if (encode == HTTP2_CONTENT_ENCODING_DEFLATE) + { + ret = inflateInit2(&((*codec_ctx)->zst), -MAX_WBITS); } - - if (ret != Z_OK) - FREE(strm); - return ret; } -static int inflate_br_read(struct z_stream_st **strm, const uint8_t *source, int len, - char **dest, int *outlen) +static int gzip_decompress_decode(struct http2_codec_ctx **strm, char **dest, int *outlen) { -#define CHUNK (1024 * 1024 * 4) + #define CHUNK (1024 * 1024 * 4) + int ret = -1; + unsigned have; + unsigned char out[CHUNK]; + int totalsize = 0; + + /* run inflate() on input until output buffer not full */ + do { + (*strm)->zst.avail_out = CHUNK; + (*strm)->zst.next_out = out; + ret = inflate(&((*strm)->zst), Z_NO_FLUSH); + switch (ret) { + case Z_STREAM_ERROR: + ret = Z_STREAM_ERROR; + case Z_NEED_DICT: + ret = Z_DATA_ERROR; /* and fall through */ + case Z_DATA_ERROR: + case Z_MEM_ERROR: + inflateEnd(&((*strm)->zst)); + return ret; + } + have = CHUNK - (*strm)->zst.avail_out; + totalsize += have; + *dest = (char *)realloc(*dest,totalsize); + memcpy(*dest + totalsize - have,out,have); + *outlen = have; + } while ((*strm)->zst.avail_out == 0); + return ret; +} + +static int brotli_decompress_init(struct http2_codec_ctx **codec_ctx) +{ + int ret = Z_ERRNO; + + if (*codec_ctx != NULL) + { + return Z_OK; + } + + *codec_ctx = ALLOC(struct http2_codec_ctx, 1); + if(*codec_ctx == NULL) + { + *codec_ctx=NULL; + return -1; + } + (*codec_ctx)->brdec_state = BrotliDecoderCreateInstance(NULL, NULL, NULL); + if ((*codec_ctx)->brdec_state != NULL) + { + return Z_OK; + } + return ret; +} + +static int brotli_decompress_decode(struct http2_codec_ctx **strm, const uint8_t *source, int len, char **dest, int *outlen) +{ + #define CHUNK (1024 * 1024 * 4) unsigned char out[CHUNK]; int totalsize = 0 ,ret = -1; size_t available_out; @@ -211,109 +265,171 @@ finish: return ret; } -static int inflate_gzip_read(struct z_stream_st **strm, char **dest, int *outlen) +static int zstd_decompress_init(struct http2_codec_ctx **codec_ctx) { - #define CHUNK (1024 * 1024 * 4) - int ret = -1; - unsigned have; - unsigned char out[CHUNK]; - int totalsize = 0; - - /* run inflate() on input until output buffer not full */ - do { - (*strm)->zst.avail_out = CHUNK; - (*strm)->zst.next_out = out; - ret = inflate(&((*strm)->zst), Z_NO_FLUSH); - switch (ret) { - case Z_STREAM_ERROR: - ret = Z_STREAM_ERROR; - case Z_NEED_DICT: - ret = Z_DATA_ERROR; /* and fall through */ - case Z_DATA_ERROR: - case Z_MEM_ERROR: - inflateEnd(&((*strm)->zst)); - return ret; - } - have = CHUNK - (*strm)->zst.avail_out; - totalsize += have; - *dest = (char *)realloc(*dest,totalsize); - memcpy(*dest + totalsize - have,out,have); - *outlen = have; - } while ((*strm)->zst.avail_out == 0); - - return ret; -} - -int inflate_read(const uint8_t *source,int len,char **dest, int *outlen, - struct z_stream_st **strm, int encode) -{ - int ret = -1; - - ret = inflate_init(strm, encode); - if (ret != Z_OK){ - return ret; - } - - if (encode == HTTP2_CONTENT_ENCODING_GZIP || - encode == HTTP2_CONTENT_ENCODING_DEFLATE){ - (*strm)->zst.avail_in = len; - (*strm)->zst.next_in = (Bytef *)source; - ret = inflate_gzip_read(strm, dest, outlen); - } - if (encode == HTTP2_CONTENT_ENCODING_BR){ - ret = inflate_br_read(strm, source, len, dest, outlen); - } - return ret; -} - -int deflate_init(struct z_stream_st **strm, int gzip) -{ - if (*strm != NULL) + int ret = Z_ERRNO; + + if (*codec_ctx != NULL) + { return Z_OK; - - int ret = 0; // 0 stand for Z_OK - - *strm = ALLOC(struct z_stream_st, 1); - assert(*strm); - - if (gzip == HTTP2_CONTENT_ENCODING_GZIP || - gzip == HTTP2_CONTENT_ENCODING_DEFLATE){ - (*strm)->zst.zalloc = (alloc_func)0; - (*strm)->zst.zfree = (free_func)0; - (*strm)->zst.opaque = (voidpf)0; - (*strm)->zst.avail_in = 0; - (*strm)->zst.next_in = Z_NULL; - - int wbits = 0; - if (gzip == HTTP2_CONTENT_ENCODING_GZIP){ - wbits = MAX_WBITS + 16; - } - if (gzip == HTTP2_CONTENT_ENCODING_DEFLATE){ - wbits = -MAX_WBITS; - } - ret = deflateInit2(&((*strm)->zst), Z_DEFAULT_COMPRESSION, - Z_DEFLATED, wbits, 8, Z_DEFAULT_STRATEGY); - } - if (gzip == HTTP2_CONTENT_ENCODING_BR){ - (*strm)->brenc_state = BrotliEncoderCreateInstance(NULL, NULL, NULL); - if ( (*strm)->brenc_state == NULL) - ret = -1; - - BrotliEncoderSetParameter((*strm)->brenc_state, BROTLI_PARAM_QUALITY, 3); + + *codec_ctx = ALLOC(struct http2_codec_ctx, 1); + if(*codec_ctx == NULL) + { + *codec_ctx=NULL; + return -1; + } + (*codec_ctx)->dctx = ZSTD_createDCtx(); + if((*codec_ctx)->dctx != NULL) + { + ret = Z_OK; } - if (ret != Z_OK) - FREE(strm); - return ret; } -static int deflate_br_write(struct z_stream_st **strm, - const unsigned char * source, size_t slen, - struct evbuffer * evbuf, int end) +static int zstd_decompress_decode(struct http2_codec_ctx **strm, const uint8_t *source, int len, char **dest, int *outlen) { - struct evbuffer_iovec v[1]; + int chunk_sz = 1024 * 1024 * 4; + unsigned char out[chunk_sz]; + int totalsize = 0 ,ret = -1; + size_t available_out; + unsigned char * next_out=NULL; + size_t available_in = len; + const unsigned char * next_in = source; + + ZSTD_inBuffer input = {next_in, available_in, 0 }; + for ( ; ; ){ + available_out = CHUNK; + next_out = out; + ZSTD_outBuffer output = {next_out, available_out, 0 }; + + ret = ZSTD_decompressStream((*strm)->dctx, &output, &input); + //printf("error name = %s\n", ZSTD_getErrorName(ret)); + + size_t have = CHUNK - (output.size - output.pos); + if (have > 0) + { + totalsize += have; + *dest = (char *)realloc(*dest,totalsize); + memcpy(*dest + totalsize - have, out, have); + *outlen = have; + } + if(ret >= 0) + { + ret=1; + goto finish; + } + if(ret < 0) + { + ret = -1; + goto finish; + } + } +finish: + return ret; +} + +static int gzip_compress_init(struct http2_codec_ctx **codec_ctx, int encode) +{ + if (*codec_ctx != NULL) + { + return Z_OK; + } + + *codec_ctx = ALLOC(struct http2_codec_ctx, 1); + if(*codec_ctx == NULL) + { + *codec_ctx=NULL; + return -1; + } + (*codec_ctx)->zst.zalloc = (alloc_func)0; + (*codec_ctx)->zst.zfree = (free_func)0; + (*codec_ctx)->zst.opaque = (voidpf)0; + (*codec_ctx)->zst.avail_in = 0; + (*codec_ctx)->zst.next_in = Z_NULL; + + int wbits = 0; + if (encode == HTTP2_CONTENT_ENCODING_GZIP) + { + wbits = MAX_WBITS + 16; + } + if (encode == HTTP2_CONTENT_ENCODING_DEFLATE) + { + wbits = -MAX_WBITS; + } + return deflateInit2(&((*codec_ctx)->zst), Z_DEFAULT_COMPRESSION, Z_DEFLATED, wbits, 8, Z_DEFAULT_STRATEGY); +} + +static int gzip_compress_encode(struct http2_codec_ctx **strm, const uint8_t *source, int slen, struct evbuffer * evbuf, int end) +{ +#define SZ_IOVEC 2 + int ret = 0; + unsigned int i = 0; + struct evbuffer_iovec io[SZ_IOVEC]; + + size_t max = slen > 8192 ? slen : 8192; + int iov_count = evbuffer_reserve_space(evbuf, max, io, SZ_IOVEC); + if (iov_count < 1 || iov_count > SZ_IOVEC) + { + return -1; + } + + (*strm)->zst.next_in = (unsigned char *) source; + (*strm)->zst.avail_in = (unsigned int) slen; + (*strm)->zst.next_out = (unsigned char *) io[i].iov_base; + (*strm)->zst.avail_out = (unsigned int) io[i].iov_len; + + int flush = end ? Z_FINISH : Z_NO_FLUSH; + do + { + ret = deflate(&((*strm)->zst), flush); + assert(ret != Z_STREAM_ERROR); + assert(i < SZ_IOVEC); + + if ((*strm)->zst.avail_out == 0 || (*strm)->zst.avail_in == 0) + { + unsigned int len = (unsigned int) io[i].iov_len - (*strm)->zst.avail_out; + io[i].iov_len = (size_t) len; + + i++; + (*strm)->zst.next_out = (unsigned char *) io[i].iov_base; + (*strm)->zst.avail_out = (unsigned int) io[i].iov_len; + } + } while ((*strm)->zst.avail_in > 0); + + assert(end == 0 || ret == Z_STREAM_END); + + (void)ret; + return evbuffer_commit_space(evbuf, io, iov_count); +} + +static int brotli_compress_init(struct http2_codec_ctx **codec_ctx) +{ + if (*codec_ctx != NULL) + { + return Z_OK; + } + + *codec_ctx = ALLOC(struct http2_codec_ctx, 1); + if(*codec_ctx == NULL) + { + *codec_ctx=NULL; + return -1; + } + (*codec_ctx)->brenc_state = BrotliEncoderCreateInstance(NULL, NULL, NULL); + if ((*codec_ctx)->brenc_state == NULL) + { + return -1; + } + BrotliEncoderSetParameter((*codec_ctx)->brenc_state, BROTLI_PARAM_QUALITY, 3); + return 0; +} + +static int brotli_compress_encode(struct http2_codec_ctx **strm, const uint8_t *source, int slen, struct evbuffer * evbuf, int end) +{ +struct evbuffer_iovec v[1]; size_t __sz_reserve_chunk = slen > 8192 ? slen : 8192; int iov_count = evbuffer_reserve_space(evbuf, __sz_reserve_chunk, v, 1); if (iov_count != 1) return -1; @@ -351,105 +467,247 @@ static int deflate_br_write(struct z_stream_st **strm, avail_out = (unsigned int) v[0].iov_len; } } - } - while (avail_in > 0); + }while (avail_in > 0); return 0; - } -static int defalta_gzip_write(struct z_stream_st **strm, const uint8_t *source, int slen, - struct evbuffer * evbuf, int end) +static int zstd_compress_init(struct http2_codec_ctx **codec_ctx) { -#define SZ_IOVEC 2 - int ret = 0; - unsigned int i = 0; - struct evbuffer_iovec io[SZ_IOVEC]; - - size_t max = slen > 8192 ? slen : 8192; - int iov_count = evbuffer_reserve_space(evbuf, max, io, SZ_IOVEC); - if (iov_count < 1 || iov_count > SZ_IOVEC) - return -1; - - (*strm)->zst.next_in = (unsigned char *) source; - (*strm)->zst.avail_in = (unsigned int) slen; - (*strm)->zst.next_out = (unsigned char *) io[i].iov_base; - (*strm)->zst.avail_out = (unsigned int) io[i].iov_len; - - int flush = end ? Z_FINISH : Z_NO_FLUSH; - do + if (*codec_ctx != NULL) { - ret = deflate(&((*strm)->zst), flush); - assert(ret != Z_STREAM_ERROR); - assert(i < SZ_IOVEC); - - if ((*strm)->zst.avail_out == 0 || (*strm)->zst.avail_in == 0) - { - unsigned int len = (unsigned int) io[i].iov_len - (*strm)->zst.avail_out; - io[i].iov_len = (size_t) len; - - i++; - (*strm)->zst.next_out = (unsigned char *) io[i].iov_base; - (*strm)->zst.avail_out = (unsigned int) io[i].iov_len; - } - } while ((*strm)->zst.avail_in > 0); - - assert(end == 0 || ret == Z_STREAM_END); - - (void)ret; - return evbuffer_commit_space(evbuf, io, iov_count); + return Z_OK; + } + + *codec_ctx = ALLOC(struct http2_codec_ctx, 1); + if(*codec_ctx == NULL) + { + *codec_ctx=NULL; + return -1; + } + (*codec_ctx)->cctx = ZSTD_createCCtx(); + if((*codec_ctx)->cctx == NULL) + { + return -1; + } + ZSTD_CCtx_setParameter((*codec_ctx)->cctx, ZSTD_c_compressionLevel, 1); + ZSTD_CCtx_setParameter((*codec_ctx)->cctx, ZSTD_c_checksumFlag, 1); + return 0; } -int deflate_write(struct z_stream_st **strm, const uint8_t *source, - int slen, struct evbuffer * evbuf, int gzip, int end) +static int zstd_compress_encode(struct http2_codec_ctx **strm, const uint8_t *source, int slen, struct evbuffer * evbuf, int end) { - int ret = 0; + size_t toRead = 0; + struct evbuffer_iovec v[1] = {0}; + ZSTD_outBuffer output = {0}; + const unsigned char *next_in = NULL; + size_t remaining=0; - ret = deflate_init(strm, gzip); - if (ret != Z_OK){ - return ret; + size_t const buffInSize = slen; + size_t const buffOutSize = ZSTD_CStreamOutSize(); + + //printf("slen = %d\n", slen); + size_t __sz_reserve_chunk = (size_t)slen > buffOutSize ? (size_t)slen : buffOutSize; + + int iov_count = evbuffer_reserve_space(evbuf, __sz_reserve_chunk, v, 1); + if (iov_count != 1) + { + return -1; } - if (gzip == HTTP2_CONTENT_ENCODING_GZIP || - gzip == HTTP2_CONTENT_ENCODING_DEFLATE){ - ret = defalta_gzip_write(strm, source, slen, evbuf, end); - } - if (gzip == HTTP2_CONTENT_ENCODING_BR){ - ret = deflate_br_write(strm, source, slen, evbuf, end); - } + unsigned char * next_out = (unsigned char *)v[0].iov_base; + size_t avail_out = v[0].iov_len; + output = (ZSTD_outBuffer){next_out, avail_out, 0}; + ZSTD_EndDirective const mode = end ? ZSTD_e_end : ZSTD_e_continue; + //ZSTD_EndDirective const mode = end ? ZSTD_e_flush : ZSTD_e_continue; + if (source != NULL) + { + next_in = source; + while (next_in < source + slen) + { + toRead = (next_in + buffInSize < source + slen) ? buffInSize : (source + slen - next_in); + ZSTD_inBuffer input = {next_in, toRead, 0}; + + if (output.pos == output.size && output.pos != 0 && output.size != 0) + { + if (evbuffer_reserve_space(evbuf, __sz_reserve_chunk, v, 1) == -1) + { + return -1; + } + next_out = (unsigned char *)v[0].iov_base; + avail_out = v[0].iov_len; + output = (ZSTD_outBuffer){next_out, avail_out, 0}; + } + remaining = ZSTD_compressStream2((*strm)->cctx, &output, &input, mode); + if (ZSTD_isError(remaining)) + { + return -1; + } + if (output.pos > 0) + { + v[0].iov_len = output.pos; + if (evbuffer_commit_space(evbuf, v, 1) == -1) + { + return -1; + } + output.pos = 0; + } + next_in += toRead; + } + } + else + { + ZSTD_inBuffer input = {source, (size_t)slen, 0}; + remaining = ZSTD_compressStream2((*strm)->cctx, &output, &input, mode); + if (ZSTD_isError(remaining)) + { + return -1; + } + if (output.pos > 0) + { + v[0].iov_len = output.pos; + if (evbuffer_commit_space(evbuf, v, 1) == -1) + { + return -1; + } + } + } + return 0; +} + +int http2_decompress_stream(const uint8_t *source, int len, char **dest, int *outlen, struct http2_codec_ctx **codec_ctx, int encode) +{ + int ret = -1; // Z_OK stand for 0; Z_ERRNO stand for -1. + + switch(encode) + { + case HTTP2_CONTENT_ENCODING_GZIP: + case HTTP2_CONTENT_ENCODING_DEFLATE: + ret = gzip_decompress_init(codec_ctx, encode); + if(ret != Z_OK) + { + goto finish; + } + (*codec_ctx)->zst.avail_in = len; + (*codec_ctx)->zst.next_in = (Bytef *)source; + ret = gzip_decompress_decode(codec_ctx, dest, outlen); + break; + case HTTP2_CONTENT_ENCODING_BR: + ret = brotli_decompress_init(codec_ctx); + if(ret != Z_OK) + { + goto finish; + } + ret =brotli_decompress_decode(codec_ctx, source, len, dest, outlen); + break; + case HTTP2_CONTENT_ENCODING_ZSTD: + ret = zstd_decompress_init(codec_ctx); + if(ret != Z_OK) + { + goto finish; + } + ret = zstd_decompress_decode(codec_ctx, source, len, dest, outlen);; + break; + default: + break; + } + return ret; +finish: + if(*codec_ctx) + { + FREE(codec_ctx); + } return ret; } -void inflate_finished(struct z_stream_st **strm) +int http2_compress_stream(struct http2_codec_ctx **codec_ctx, const uint8_t *source, int slen, struct evbuffer * evbuf, int encode, int mode) { - if (*strm != NULL){ - if ((*strm)->brdec_state) + int ret = -1; // Z_OK stand for 0; Z_ERRNO stand for -1. + + switch(encode) + { + case HTTP2_CONTENT_ENCODING_GZIP: + case HTTP2_CONTENT_ENCODING_DEFLATE: + ret = gzip_compress_init(codec_ctx, encode); + if(ret != Z_OK) + { + goto finish; + } + ret = gzip_compress_encode(codec_ctx, source, slen, evbuf, mode); + break; + case HTTP2_CONTENT_ENCODING_BR: + ret = brotli_compress_init(codec_ctx); + if(ret != Z_OK) + { + goto finish; + } + ret = brotli_compress_encode(codec_ctx, source, slen, evbuf, mode); + break; + case HTTP2_CONTENT_ENCODING_ZSTD: + ret = zstd_compress_init(codec_ctx); + if(ret != Z_OK) + { + goto finish; + } + ret = zstd_compress_encode(codec_ctx, source, slen, evbuf, mode); + break; + default: + break; + } + return ret; + +finish: + if(*codec_ctx) + { + FREE(codec_ctx); + } + return ret; +} + +void http2_decompress_finished(struct http2_codec_ctx **codec_ctx) +{ + if (*codec_ctx != NULL) + { + if ((*codec_ctx)->brdec_state) { - BrotliDecoderDestroyInstance((*strm)->brdec_state); - (*strm)->brdec_state = NULL; + BrotliDecoderDestroyInstance((*codec_ctx)->brdec_state); + (*codec_ctx)->brdec_state = NULL; goto finish; } - (void)inflateEnd(&((*strm)->zst)); + if ((*codec_ctx)->dctx) + { + ZSTD_freeDCtx((*codec_ctx)->dctx); + (*codec_ctx)->dctx = NULL; + goto finish; + } + (void)inflateEnd(&((*codec_ctx)->zst)); finish: - free(*strm); - *strm = NULL; + free(*codec_ctx); + *codec_ctx = NULL; } } -void deflate_finished(struct z_stream_st **strm) + +void http2_compress_finished(struct http2_codec_ctx **codec_ctx) { - if (*strm != NULL) + if (*codec_ctx != NULL) { - if ((*strm)->brenc_state) + if ((*codec_ctx)->brenc_state) { - BrotliEncoderDestroyInstance((*strm)->brenc_state); - (*strm)->brenc_state = NULL; + BrotliEncoderDestroyInstance((*codec_ctx)->brenc_state); + (*codec_ctx)->brenc_state = NULL; goto finish; } - (void) deflateEnd(&((*strm)->zst)); + if ((*codec_ctx)->cctx) + { + ZSTD_freeCCtx((*codec_ctx)->cctx); + (*codec_ctx)->cctx = NULL; + goto finish; + } + (void) deflateEnd(&((*codec_ctx)->zst)); finish: - free(*strm); - *strm = NULL; + free(*codec_ctx); + *codec_ctx = NULL; } } diff --git a/plugin/protocol/http2/src/http2_stream.cpp b/plugin/protocol/http2/src/http2_stream.cpp index 15f32fa..a8a89fc 100644 --- a/plugin/protocol/http2/src/http2_stream.cpp +++ b/plugin/protocol/http2/src/http2_stream.cpp @@ -142,6 +142,7 @@ const char * method_idx_to_str(int encode) case HTTP2_CONTENT_ENCODING_BZIP2: return "bzip2"; case HTTP2_CONTENT_ENCODING_X_BZIP2: return "x-bzip2"; case HTTP2_CONTENT_ENCODING_BR: return "br"; + case HTTP2_CONTENT_ENCODING_ZSTD: return "zstd"; default: return ""; } } @@ -162,9 +163,13 @@ static int method_to_str_idx(const char * method) if (strcasestr(method, "x-bzip2") != NULL) return HTTP2_CONTENT_ENCODING_X_BZIP2; + if (strcasestr(method, "br") != NULL) return HTTP2_CONTENT_ENCODING_BR; + if (strcasestr(method, "zstd") != NULL) + return HTTP2_CONTENT_ENCODING_ZSTD; + return HTTP2_CONTENT_ENCODING_NONE; } @@ -352,8 +357,8 @@ static int h2_half_ops_append_body(struct tfe_http_half * half, char * buff, siz struct tfe_h2_payload *body = &resp->h2_payload; if (buff == NULL || size == 0){ - if (body->gzip != HTTP2_CONTENT_ENCODING_NONE){ - xret = deflate_write(&body->deflate, NULL, 0, resp->h2_payload.evbuf_body, body->gzip, 1); + if (body->encode_type != HTTP2_CONTENT_ENCODING_NONE){ + xret = http2_compress_stream(&body->deflate, NULL, 0, resp->h2_payload.evbuf_body, body->encode_type, 1); } resp->message_state = H2_READ_STATE_COMPLETE; goto finish; @@ -363,9 +368,9 @@ static int h2_half_ops_append_body(struct tfe_http_half * half, char * buff, siz resp->h2_payload.evbuf_body = evbuffer_new(); } - if (body->gzip != HTTP2_CONTENT_ENCODING_NONE){ - xret = deflate_write(&body->deflate, (const uint8_t *)buff, size, - resp->h2_payload.evbuf_body, body->gzip, 0); + if (body->encode_type != HTTP2_CONTENT_ENCODING_NONE){ + xret = http2_compress_stream(&body->deflate, (const uint8_t *)buff, size, + resp->h2_payload.evbuf_body, body->encode_type, 0); }else{ xret = evbuffer_add(resp->h2_payload.evbuf_body, buff, size); } @@ -401,8 +406,8 @@ void delete_stream_half_data(struct tfe_h2_half_private **data, struct tfe_h2_payload *body = &((*data)->h2_payload); - inflate_finished(&body->inflate); - deflate_finished(&body->deflate); + http2_compress_finished(&body->inflate); + http2_decompress_finished(&body->deflate); if (body->evbuf_body && body_flag){ evbuffer_free(body->evbuf_body); body->evbuf_body = NULL; @@ -449,13 +454,13 @@ int h2_half_ops_body_begin(struct tfe_http_half * half, int by_stream) if (by_stream) { if (body->inflate){ - inflate_finished(&body->inflate); + http2_compress_finished(&body->inflate); } if (body->deflate){ - deflate_finished(&body->deflate); + http2_decompress_finished(&body->deflate); } - body->gzip = HTTP2_CONTENT_ENCODING_NONE; + body->encode_type = HTTP2_CONTENT_ENCODING_NONE; resp->message_state = H2_READ_STATE_READING; resp->by_stream = by_stream; } @@ -476,10 +481,10 @@ int h2_half_ops_body_data(struct tfe_http_half * h2_response, const unsigned cha struct tfe_h2_half_private * h2_resp_priv = nghttp2_to_half_private(h2_response); struct tfe_h2_payload *body = &h2_resp_priv->h2_payload; - if (body->gzip != HTTP2_CONTENT_ENCODING_NONE){ + if (body->encode_type != HTTP2_CONTENT_ENCODING_NONE){ - xret = deflate_write(&body->deflate, (const uint8_t *)data, sz_data, - h2_resp_priv->h2_payload.evbuf_body, body->gzip, 0); + xret = http2_compress_stream(&body->deflate, (const uint8_t *)data, sz_data, + h2_resp_priv->h2_payload.evbuf_body, body->encode_type, 0); }else{ xret = evbuffer_add(h2_resp_priv->h2_payload.evbuf_body, data, sz_data); } @@ -583,7 +588,7 @@ static struct tfe_h2_half_private* tfe_half_private_init(enum tfe_http_direction half_private->h2_payload.inflate = NULL; half_private->h2_payload.deflate = NULL; half_private->h2_payload.evbuf_body = evbuffer_new(); - half_private->h2_payload.gzip = HTTP2_CONTENT_ENCODING_NONE; + half_private->h2_payload.encode_type = HTTP2_CONTENT_ENCODING_NONE; half_private->h2_payload.padlen = 0; half_private->stream_id = stream_id; @@ -613,7 +618,7 @@ struct tfe_http_half * h2_ops_response_create(struct tfe_http_session * session, resp->method_or_status = resp_code; if (stream->resp) - resp->h2_payload.gzip = stream->resp->h2_payload.gzip; + resp->h2_payload.encode_type = stream->resp->h2_payload.encode_type; return &resp->half_public; } @@ -759,10 +764,10 @@ static enum tfe_stream_action http2_frame_submit_built_resp(struct tfe_h2_stream tfe_http_field_write(&pangu_resp->half_public, &cont_field, NULL); tfe_http_field_write(&pangu_resp->half_public, &cont_field, str_sz_evbuf_body); - if (body->gzip != HTTP2_CONTENT_ENCODING_NONE) + if (body->encode_type != HTTP2_CONTENT_ENCODING_NONE) { const static struct http_field_name encoding_field = {TFE_HTTP_CONT_ENCODING, NULL}; - const char *content_encoding = method_idx_to_str(body->gzip); + const char *content_encoding = method_idx_to_str(body->encode_type); tfe_http_field_write(&pangu_resp->half_public, &encoding_field, NULL); tfe_http_field_write(&pangu_resp->half_public, &encoding_field, content_encoding); } @@ -1859,7 +1864,7 @@ static int http2_fill_up_header(nghttp2_session *ngh2_session, const nghttp2_fra } if (field.field_id == TFE_HTTP_CONT_ENCODING) { - half->h2_payload.gzip = method_to_str_idx((const char *)value); + half->h2_payload.encode_type = method_to_str_idx((const char *)value); } h2_header = &half->header; tfe_h2_header_add_field(h2_header, &field, (const char *)value, 1); @@ -1892,7 +1897,7 @@ static int http2_fill_up_promise(nghttp2_session *ngh2_session, const nghttp2_fr } if (field.field_id == TFE_HTTP_CONT_ENCODING) { - resp->h2_payload.gzip = method_to_str_idx((const char *)value); + resp->h2_payload.encode_type = method_to_str_idx((const char *)value); } headers = &resp->promised; tfe_h2_header_add_field(headers, &field, (const char *)value, 1); @@ -2062,9 +2067,9 @@ static int http2_client_on_data_chunk_recv(nghttp2_session *session, uint8_t fla } evbuffer_add(resp->h2_payload.evbuf_body, input, input_len); - if (resp->h2_payload.gzip != HTTP2_CONTENT_ENCODING_NONE) + if (resp->h2_payload.encode_type != HTTP2_CONTENT_ENCODING_NONE) { - ret = inflate_read(input, input_len, &uncompr, &uncompr_len, &resp->h2_payload.inflate, resp->h2_payload.gzip); + ret = http2_decompress_stream(input, input_len, &uncompr, &uncompr_len, &resp->h2_payload.inflate, resp->h2_payload.encode_type); if (((ret == Z_STREAM_END) || (ret == Z_OK)) && uncompr_len > 0) { input = (const uint8_t*)uncompr; @@ -2395,9 +2400,9 @@ static int http2_server_on_data_chunk_recv(nghttp2_session *session, uint8_t fla req->h2_payload.flags = flags; evbuffer_add(req->h2_payload.evbuf_body, input, input_len); - if (req->h2_payload.gzip != HTTP2_CONTENT_ENCODING_NONE){ - ret = inflate_read(input, input_len, &uncompr, &uncompr_len, - &req->h2_payload.inflate, req->h2_payload.gzip); + if (req->h2_payload.encode_type != HTTP2_CONTENT_ENCODING_NONE){ + ret = http2_decompress_stream(input, input_len, &uncompr, &uncompr_len, + &req->h2_payload.inflate, req->h2_payload.encode_type); if (((ret == Z_STREAM_END) || (ret == Z_OK)) && uncompr > 0){ input = (const uint8_t*)uncompr; input_len = uncompr_len; diff --git a/plugin/protocol/http2/test/test_http2_stream.cpp b/plugin/protocol/http2/test/test_http2_stream.cpp index 64be21b..98dc6f2 100644 --- a/plugin/protocol/http2/test/test_http2_stream.cpp +++ b/plugin/protocol/http2/test/test_http2_stream.cpp @@ -337,13 +337,13 @@ TEST(UI_TEST_INFLATE_GZIP, inflate_01) { char *uncompr = NULL; int ret = 0, uncompr_len = 0; - struct z_stream_st *inflate = NULL; - ret = inflate_read(ut_gip_01, sizeof(ut_gip_01), &uncompr, &uncompr_len, &inflate, 2); + struct http2_codec_ctx *inflate = NULL; + ret = http2_decompress_stream(ut_gip_01, sizeof(ut_gip_01), &uncompr, &uncompr_len, &inflate, 2); EXPECT_EQ(ret, Z_STREAM_END); EXPECT_EQ(uncompr_len, sizeof(ut_ungip_01)); EXPECT_EQ(memcmp(uncompr, ut_ungip_01, sizeof(ut_ungip_01)), 0); - inflate_finished(&inflate); + http2_decompress_finished(&inflate); free(uncompr); uncompr = NULL; @@ -351,17 +351,17 @@ TEST(UI_TEST_INFLATE_GZIP, inflate_01) TEST(UI_TEST_INFLATE_GZIP, inflate_02) { - struct z_stream_st *inflate = NULL; + struct http2_codec_ctx *inflate = NULL; int ret = 0, half = 0; int size = sizeof(ut_gip_01); half = size / 2; char *uncompr1 = NULL; int uncompr_len1 = 0; - ret = inflate_read(ut_gip_01, half, &uncompr1, &uncompr_len1, &inflate, 2); + ret = http2_decompress_stream(ut_gip_01, half, &uncompr1, &uncompr_len1, &inflate, 2); EXPECT_EQ(ret, Z_OK); char *uncompr2 = NULL; int uncompr_len2 = 0; - ret = inflate_read(ut_gip_01 + half, size - half, &uncompr2, &uncompr_len2, &inflate, 2); + ret = http2_decompress_stream(ut_gip_01 + half, size - half, &uncompr2, &uncompr_len2, &inflate, 2); EXPECT_EQ(ret, Z_STREAM_END); char uncompr[1024] = {0}; @@ -370,7 +370,7 @@ TEST(UI_TEST_INFLATE_GZIP, inflate_02) EXPECT_EQ(uncompr_len1 + uncompr_len2, sizeof(ut_ungip_01)); EXPECT_EQ(memcmp(uncompr, ut_ungip_01, sizeof(ut_ungip_01)), 0); - inflate_finished(&inflate); + http2_decompress_finished(&inflate); free(uncompr1); uncompr1 = NULL; free(uncompr2); @@ -384,23 +384,23 @@ TEST(UI_TEST_DEFLATE_GZIP, deflate_01) struct evbuffer * buf = evbuffer_new(); int size = sizeof(ut_ungip_01); - struct z_stream_st *deflate = NULL; - ret = deflate_write(&deflate, ut_ungip_01, size, buf, 2, 1); + struct http2_codec_ctx *deflate = NULL; + ret = http2_compress_stream(&deflate, ut_ungip_01, size, buf, 2, 1); EXPECT_EQ(ret, Z_OK); - deflate_finished(&deflate); + http2_compress_finished(&deflate); dest = evbuffer_pullup(buf, -1); dlen = evbuffer_get_length(buf); char *uncompr = NULL; int uncompr_len = 0; - struct z_stream_st *inflate = NULL; - ret = inflate_read(dest, dlen, &uncompr, &uncompr_len, &inflate, 2); + struct http2_codec_ctx *inflate = NULL; + ret = http2_decompress_stream(dest, dlen, &uncompr, &uncompr_len, &inflate, 2); EXPECT_EQ(ret, Z_STREAM_END); EXPECT_EQ(uncompr_len, sizeof(ut_ungip_01)); EXPECT_EQ(memcmp(uncompr, ut_ungip_01, sizeof(ut_ungip_01)), 0); - inflate_finished(&inflate); + http2_decompress_finished(&inflate); free(uncompr); uncompr = NULL; evbuffer_free(buf); @@ -415,31 +415,31 @@ TEST(UI_TEST_DEFLATE_GZIP, deflate_02) half = size / 2; struct evbuffer * buf = evbuffer_new(); - struct z_stream_st *deflate = NULL; + struct http2_codec_ctx *deflate = NULL; /* First frag */ - ret = deflate_write(&deflate, ut_ungip_01, half, buf, 2, 0); + ret = http2_compress_stream(&deflate, ut_ungip_01, half, buf, 2, 0); EXPECT_EQ(ret, Z_OK); /* Last frag */ - ret = deflate_write(&deflate, ut_ungip_01 + half, size - half, buf, 2, 0); + ret = http2_compress_stream(&deflate, ut_ungip_01 + half, size - half, buf, 2, 0); EXPECT_EQ(ret, Z_OK); /* End frag */ - ret = deflate_write(&deflate, NULL, 0, buf, 2, 1); + ret = http2_compress_stream(&deflate, NULL, 0, buf, 2, 1); EXPECT_EQ(ret, Z_OK); - deflate_finished(&deflate); + http2_compress_finished(&deflate); dest = evbuffer_pullup(buf, -1); dlen = evbuffer_get_length(buf); char *uncompr = NULL; int uncompr_len = 0; - struct z_stream_st *inflate = NULL; - ret = inflate_read(dest, dlen, &uncompr, &uncompr_len, &inflate, 2); + struct http2_codec_ctx *inflate = NULL; + ret = http2_decompress_stream(dest, dlen, &uncompr, &uncompr_len, &inflate, 2); EXPECT_EQ(ret, Z_STREAM_END); EXPECT_EQ(uncompr_len, sizeof(ut_ungip_01)); EXPECT_EQ(memcmp(uncompr, ut_ungip_01, sizeof(ut_ungip_01)), 0); - inflate_finished(&inflate); + http2_decompress_finished(&inflate); free(uncompr); uncompr = NULL; @@ -453,22 +453,22 @@ TEST(UI_TEST_DEFLATE_BR, deflate_01) struct evbuffer * buf = evbuffer_new(); int size = sizeof(ut_ungip_01); - struct z_stream_st *deflate = NULL; - ret = deflate_write(&deflate, ut_ungip_01, size, buf, HTTP2_CONTENT_ENCODING_BR, 1); + struct http2_codec_ctx *deflate = NULL; + ret = http2_compress_stream(&deflate, ut_ungip_01, size, buf, HTTP2_CONTENT_ENCODING_BR, 1); EXPECT_EQ(ret, Z_OK); - deflate_finished(&deflate); + http2_compress_finished(&deflate); dest = evbuffer_pullup(buf, -1); dlen = evbuffer_get_length(buf); char *uncompr = NULL; int uncompr_len = 0; - struct z_stream_st *inflate = NULL; - ret = inflate_read(dest, dlen, &uncompr, &uncompr_len, &inflate, HTTP2_CONTENT_ENCODING_BR); + struct http2_codec_ctx *inflate = NULL; + ret = http2_decompress_stream(dest, dlen, &uncompr, &uncompr_len, &inflate, HTTP2_CONTENT_ENCODING_BR); EXPECT_EQ(ret, Z_STREAM_END); EXPECT_EQ(uncompr_len, sizeof(ut_ungip_01)); EXPECT_EQ(memcmp(uncompr, ut_ungip_01, sizeof(ut_ungip_01)), 0); - inflate_finished(&inflate); + http2_decompress_finished(&inflate); free(uncompr); uncompr = NULL; @@ -484,37 +484,272 @@ TEST(UI_TEST_DEFLATE_BR, deflate_02) half = size / 2; struct evbuffer * buf = evbuffer_new(); - struct z_stream_st *deflate = NULL; + struct http2_codec_ctx *deflate = NULL; /* First frag */ - ret = deflate_write(&deflate, ut_ungip_01, half, buf, HTTP2_CONTENT_ENCODING_BR, 0); + ret = http2_compress_stream(&deflate, ut_ungip_01, half, buf, HTTP2_CONTENT_ENCODING_BR, 0); EXPECT_EQ(ret, Z_OK); /* Last frag */ - ret = deflate_write(&deflate, ut_ungip_01 + half, size - half, buf, HTTP2_CONTENT_ENCODING_BR, 0); + ret = http2_compress_stream(&deflate, ut_ungip_01 + half, size - half, buf, HTTP2_CONTENT_ENCODING_BR, 0); EXPECT_EQ(ret, Z_OK); /* End frag */ - ret = deflate_write(&deflate, NULL, 0, buf, HTTP2_CONTENT_ENCODING_BR, 1); + ret = http2_compress_stream(&deflate, NULL, 0, buf, HTTP2_CONTENT_ENCODING_BR, 1); EXPECT_EQ(ret, Z_OK); - deflate_finished(&deflate); + http2_compress_finished(&deflate); dest = evbuffer_pullup(buf, -1); dlen = evbuffer_get_length(buf); char *uncompr = NULL; int uncompr_len = 0; - struct z_stream_st *inflate = NULL; - ret = inflate_read(dest, dlen, &uncompr, &uncompr_len, &inflate, HTTP2_CONTENT_ENCODING_BR); + struct http2_codec_ctx *inflate = NULL; + ret = http2_decompress_stream(dest, dlen, &uncompr, &uncompr_len, &inflate, HTTP2_CONTENT_ENCODING_BR); EXPECT_EQ(ret, Z_STREAM_END); EXPECT_EQ(uncompr_len, sizeof(ut_ungip_01)); EXPECT_EQ(memcmp(uncompr, ut_ungip_01, sizeof(ut_ungip_01)), 0); - inflate_finished(&inflate); + http2_decompress_finished(&inflate); free(uncompr); uncompr = NULL; evbuffer_free(buf); } +TEST(UI_TEST_INFLATE_ZSTD, Decompress_Facebook_Manifest) +{ + char *output = NULL; + int ret = 0, output_len = 0; + struct http2_codec_ctx *codec_ctx = NULL; + ret = http2_decompress_stream(facebook_response_zstd_body, sizeof(facebook_response_zstd_body), &output, &output_len, &codec_ctx, HTTP2_CONTENT_ENCODING_ZSTD); + printf("output = %s\n", output); + EXPECT_EQ(ret, 1); + EXPECT_EQ(output_len, strlen(facebook_response_text_body)); + EXPECT_EQ(memcmp(output, facebook_response_text_body, strlen(facebook_response_text_body)), 0); + http2_decompress_finished(&codec_ctx); + free(output); + output = NULL; +} + +#if 0 +TEST(UI_TEST_INFLATE_ZSTD, Decompress_Facebook_Index) +{ + char *output = NULL; + int ret = 0, output_len = 0; + struct http2_codec_ctx *codec_ctx = NULL; + ret = http2_decompress_stream(facebook_index_zstd_body, sizeof(facebook_index_zstd_body), &output, &output_len, &codec_ctx, HTTP2_CONTENT_ENCODING_ZSTD); + printf("output = %s\n", output); + EXPECT_EQ(ret, 1); + //EXPECT_EQ(output_len, strlen(facebook_response_text_body)); + //EXPECT_EQ(memcmp(output, facebook_response_text_body, strlen(facebook_response_text_body)), 0); + http2_compress_finished(&codec_ctx); + free(output); + output = NULL; +} +#endif + +TEST(UI_TEST_INFLATE_ZSTD, Compress_AND_Decompress_Facebook_Manifest) +{ + unsigned char *encode_str = NULL; + int ret = 0, encode_len = 0; + struct evbuffer *encode_buf = evbuffer_new(); + int input_len = strlen(facebook_response_text_body); + + struct http2_codec_ctx *codec_ctx_ecode = NULL; + ret = http2_compress_stream(&codec_ctx_ecode, (const uint8_t *)facebook_response_text_body, input_len, encode_buf, HTTP2_CONTENT_ENCODING_ZSTD, 1); + EXPECT_EQ(ret, Z_OK); + http2_compress_finished(&codec_ctx_ecode); + + encode_str = evbuffer_pullup(encode_buf, -1); + encode_len = evbuffer_get_length(encode_buf) + 1; + + char *output = NULL; + int output_len = 0; + struct http2_codec_ctx *codec_ctx_decode = NULL; + ret = http2_decompress_stream(encode_str, encode_len, &output, &output_len, &codec_ctx_decode, HTTP2_CONTENT_ENCODING_ZSTD); + printf("output = %s\n", output); + + EXPECT_EQ(ret, 1); + EXPECT_EQ(output_len, strlen(facebook_response_text_body)); + EXPECT_EQ(memcmp(output, facebook_response_text_body, strlen(facebook_response_text_body)), 0); + http2_decompress_finished(&codec_ctx_decode); + + free(output); + output = NULL; + evbuffer_free(encode_buf); +} + +TEST(UI_TEST_INFLATE_ZSTD, Compress_Facebook_Manifest_MORE_TIMES) +{ + unsigned char *encode_str = NULL; + int ret = 0, half_len = 0, encode_len = 0; + + int text_len = strlen(facebook_response_text_body); + half_len = text_len / 2; + + struct evbuffer *encode_buf = evbuffer_new(); + struct http2_codec_ctx *codec_ctx = NULL; + /* First frag */ + ret = http2_compress_stream(&codec_ctx, (const uint8_t *)facebook_response_text_body, half_len, encode_buf, HTTP2_CONTENT_ENCODING_ZSTD, 0); + EXPECT_EQ(ret, 0); + /* Last frag */ + ret = http2_compress_stream(&codec_ctx, (const uint8_t *)facebook_response_text_body + half_len, text_len - half_len, encode_buf, HTTP2_CONTENT_ENCODING_ZSTD, 0); + EXPECT_EQ(ret, 0); + /* End frag */ + ret = http2_compress_stream(&codec_ctx, NULL, 0, encode_buf, HTTP2_CONTENT_ENCODING_ZSTD, 1); + EXPECT_EQ(ret, 0); + http2_compress_finished(&codec_ctx); + + encode_str = evbuffer_pullup(encode_buf, -1); + encode_len = evbuffer_get_length(encode_buf); + printf("encode_len = %d\n", encode_len); + + char *output = NULL; + int output_len = 0; + ret = http2_decompress_stream(encode_str, encode_len, &output, &output_len, &codec_ctx, HTTP2_CONTENT_ENCODING_ZSTD); + printf("output = %s, ret =%d\n", output, ret); + EXPECT_EQ(ret, 1); + EXPECT_EQ(output_len, strlen(facebook_response_text_body)); + EXPECT_EQ(memcmp(output, facebook_response_text_body, strlen(facebook_response_text_body)), 0); + http2_decompress_finished(&codec_ctx); + + free(output); + output = NULL; + evbuffer_free(encode_buf); +} + +static int read_file(const char *filename, char **input) +{ + FILE* fp=NULL; + struct stat file_info; + stat(filename, &file_info); + size_t input_sz=file_info.st_size; + + fp=fopen(filename,"r"); + if(fp==NULL) + { + return 0; + } + *input=(char*)malloc(input_sz); + fread(*input,1,input_sz,fp); + fclose(fp); + return input_sz; +} + +static int write_file(const char *outfile, char *output, int out_len) +{ + FILE* fp=NULL; + fp=fopen(outfile,"wb"); + if(fp==NULL) + { + return 0; + } + size_t writtenSize = fwrite(output, 1, out_len, fp); + fclose(fp); + return writtenSize; +} + +TEST(UI_TEST_INFLATE_ZSTD, Compress_Manual_Html_Simple) +{ + int ret=0, encode_len=0; + char *input=0; size_t input_sz=0; + unsigned char *encode_str = NULL; + + struct http2_codec_ctx *codec_ctx = NULL; + + struct evbuffer *encode_buf = evbuffer_new(); + + const char* filename="./zstd_manual_simple.html"; + const char *outfile="./zstd_manual_simple_test.html"; + input_sz = read_file(filename, &input); + if(input_sz !=0 || input != NULL) + { + ret = http2_compress_stream(&codec_ctx, (const uint8_t *)input, input_sz, encode_buf, HTTP2_CONTENT_ENCODING_ZSTD, 1); + EXPECT_EQ(ret, 0); + http2_compress_finished(&codec_ctx); + + encode_str = evbuffer_pullup(encode_buf, -1); + encode_len = evbuffer_get_length(encode_buf); + printf("encode_len = %d\n", encode_len); + + char *output = NULL; + int output_len = 0; + ret = http2_decompress_stream(encode_str, encode_len, &output, &output_len, &codec_ctx, HTTP2_CONTENT_ENCODING_ZSTD); + printf("output_len = %d, ret =%d\n", output_len, ret); + write_file(outfile, output, output_len); + http2_decompress_finished(&codec_ctx); + + free(input); + free(output); + evbuffer_free(encode_buf); + } +} + +TEST(UI_TEST_INFLATE_ZSTD, Compress_Manual_Html_Large) +{ + int ret=0, encode_len=0; + char *input=0; size_t input_sz=0; + unsigned char *encode_str = NULL; + + struct http2_codec_ctx *codec_ctx = NULL; + + struct evbuffer *encode_buf = evbuffer_new(); + + const char *inputfile="./zstd_manual_large.html"; + const char *outfile="./zstd_manual_large_test.html"; + input_sz = read_file(inputfile, &input); + if(input_sz !=0 || input != NULL) + { + ret = http2_compress_stream(&codec_ctx, (const uint8_t *)input, input_sz, encode_buf, HTTP2_CONTENT_ENCODING_ZSTD, 1); + EXPECT_EQ(ret, 0); + http2_compress_finished(&codec_ctx); + + encode_str = evbuffer_pullup(encode_buf, -1); + encode_len = evbuffer_get_length(encode_buf); + printf("encode_len = %d\n", encode_len); + + char *output = NULL; + int output_len = 0; + ret = http2_decompress_stream(encode_str, encode_len, &output, &output_len, &codec_ctx, HTTP2_CONTENT_ENCODING_ZSTD); + printf("output_len = %d, ret =%d\n", output_len, ret); + write_file(outfile, output, output_len); + http2_decompress_finished(&codec_ctx); + + free(input); + free(output); + evbuffer_free(encode_buf); + } +} + +#if 0 +TEST(UI_TEST_INFLATE_ZSTD, Decompress_Facebook_Manifest_Half) +{ + struct http2_codec_ctx *codec_ctx = NULL; + int ret = 0, half = 0; + int size = sizeof(facebook_response_zstd_body); + half = size / 2; + + char *output1 = NULL; int output_len1 = 0; + ret = http2_decompress_stream(ut_gip_01, half, &output1, &output_len1, &codec_ctx, HTTP2_CONTENT_ENCODING_ZSTD); + EXPECT_EQ(ret, 1); + + char *output2 = NULL; int output_len2 = 0; + ret = http2_decompress_stream(ut_gip_01 + half, size - half, &output2, &output_len2, &codec_ctx, HTTP2_CONTENT_ENCODING_ZSTD); + EXPECT_EQ(ret, 1); + + char output[4096] = {0}; + memcpy(output, output1, output_len1); + memcpy(output + output_len1, output2, output_len2); + EXPECT_EQ(output_len1 + output_len2, strlen(facebook_response_text_body)); + EXPECT_EQ(memcmp(output, facebook_response_text_body, strlen(facebook_response_text_body)), 0); + + http2_compress_finished(&codec_ctx); + free(output1); + output1 = NULL; + free(output2); + output2 = NULL; +} +#endif + int main(int argc, char ** argv) { ::testing::InitGoogleTest(&argc, argv); diff --git a/plugin/protocol/http2/test/test_http2_stream.h b/plugin/protocol/http2/test/test_http2_stream.h index 7691d33..ae0c5e5 100644 --- a/plugin/protocol/http2/test/test_http2_stream.h +++ b/plugin/protocol/http2/test/test_http2_stream.h @@ -2279,6 +2279,51 @@ unsigned char ut_ungip_01[] = { 0x6c, 0x66, 0x6a }; +unsigned char facebook_response_zstd_body[] = { + 0x28, 0xb5, 0x2f, 0xfd, 0x00, 0x58, 0xf4, 0x0f, 0x00, 0xd2, 0xe1, 0x63, 0x26, 0xf0, 0x90, 0x56, + 0x07, 0xe8, 0x95, 0xd9, 0x18, 0xd7, 0xb5, 0x8a, 0xe1, 0x4b, 0x6b, 0x8b, 0x6b, 0x60, 0xf8, 0xf1, + 0xa4, 0x4c, 0x2d, 0x2b, 0xf3, 0x0e, 0x2d, 0x62, 0x1f, 0xbf, 0x39, 0x9e, 0xb5, 0x16, 0x87, 0xc5, + 0x61, 0x5a, 0x02, 0x9b, 0xb6, 0x90, 0x2f, 0x4c, 0x59, 0x44, 0x6e, 0x75, 0xf6, 0x7d, 0x90, 0x25, + 0x5f, 0xc6, 0x70, 0x2e, 0x4e, 0x9c, 0xed, 0xea, 0xeb, 0xb1, 0xce, 0xcb, 0x65, 0x48, 0x99, 0xeb, + 0xcb, 0xb0, 0xeb, 0x95, 0xc5, 0x86, 0x35, 0x66, 0x36, 0x3a, 0x7c, 0xd9, 0x5d, 0x5d, 0x13, 0x23, + 0xc6, 0x1d, 0xd2, 0xd8, 0xf7, 0x01, 0x40, 0xe2, 0xeb, 0xb0, 0x2d, 0x0c, 0xaf, 0x4e, 0x40, 0x91, + 0x64, 0x24, 0x03, 0x92, 0x09, 0x01, 0x08, 0xba, 0x90, 0xc9, 0x98, 0x60, 0x46, 0x84, 0xb8, 0x73, + 0xb2, 0x57, 0x16, 0x9b, 0xc7, 0x03, 0x5a, 0xef, 0xcb, 0x8e, 0x62, 0xae, 0x2e, 0xa7, 0xbd, 0x78, + 0x22, 0x39, 0xdf, 0xe5, 0xf6, 0x20, 0xa5, 0xcc, 0x2c, 0x97, 0xaf, 0x4b, 0x7c, 0xe5, 0x7c, 0x97, + 0xbe, 0xda, 0x6c, 0xbb, 0xdb, 0x92, 0x7b, 0x1b, 0x57, 0x16, 0xd0, 0x24, 0x03, 0x81, 0x26, 0x19, + 0x91, 0x8b, 0x08, 0x48, 0x81, 0x42, 0xd3, 0xd0, 0x0a, 0x4e, 0x6d, 0x2e, 0x49, 0xfa, 0x44, 0x2d, + 0x71, 0x7c, 0x5f, 0x6f, 0xaa, 0xb9, 0x57, 0x87, 0x14, 0x40, 0x19, 0x48, 0x01, 0xf4, 0x75, 0xdb, + 0x1c, 0xcf, 0x5e, 0x5d, 0x92, 0xd4, 0x41, 0x70, 0xe0, 0x48, 0x40, 0x60, 0x80, 0x14, 0x28, 0x2a, + 0xfd, 0x44, 0xf5, 0x89, 0x1e, 0x9a, 0x27, 0x8a, 0x68, 0x3c, 0x11, 0x9d, 0xa8, 0xd3, 0xaa, 0xad, + 0x4f, 0x74, 0x5b, 0x3a, 0xd2, 0x35, 0xcb, 0xc5, 0xc9, 0x64, 0x9c, 0xf6, 0x2e, 0x5f, 0xfb, 0x44, + 0x4f, 0xb4, 0x2c, 0xbd, 0x77, 0xfa, 0x7a, 0xab, 0xf6, 0x1d, 0x15, 0xeb, 0x56, 0xce, 0xf6, 0xae, + 0x0e, 0xa9, 0x50, 0x28, 0x04, 0x50, 0xc7, 0x9e, 0xee, 0xd5, 0x2b, 0x01, 0x5f, 0xaf, 0x13, 0xce, + 0xc6, 0x9c, 0xc4, 0xcc, 0x2a, 0x61, 0x34, 0xb6, 0x5c, 0xfd, 0x61, 0x04, 0x20, 0xc7, 0x27, 0x5f, + 0xfc, 0xba, 0x86, 0x21, 0xb5, 0xcf, 0x5d, 0x9d, 0x37, 0xa8, 0xec, 0xae, 0xcd, 0x76, 0x9c, 0x93, + 0xc1, 0xa5, 0xeb, 0xc1, 0x13, 0x7d, 0x1d, 0x56, 0x0c, 0xbb, 0x95, 0xaf, 0xbd, 0x3a, 0xc6, 0x9c, + 0x96, 0x6e, 0x5c, 0xf9, 0x7a, 0x47, 0x96, 0xec, 0xd6, 0x39, 0xd9, 0xab, 0xcd, 0x00, 0x25, 0xb2, + 0x3e, 0x5d, 0xf7, 0x36, 0x98, 0xeb, 0x5a, 0xfb, 0xd2, 0x95, 0xbe, 0x63, 0x1d, 0xb6, 0x64, 0xca, + 0xda, 0x3d, 0xcb, 0x5a, 0x72, 0x7d, 0xd5, 0x98, 0x7a, 0xd7, 0x35, 0x50, 0x4e, 0xd6, 0xd0, 0xf2, + 0xec, 0xd3, 0x60, 0xb5, 0x2d, 0x46, 0x5d, 0x32, 0x89, 0x48, 0x20, 0x02, 0xca, 0xe4, 0x31, 0x81, + 0xbe, 0xee, 0x1e, 0xab, 0xed, 0x64, 0x5b, 0x16, 0x9b, 0xf4, 0x8e, 0x28, 0x20, 0x60, 0x02, 0x09, + 0xa6, 0x8a, 0xd4, 0x03, 0xda, 0xa3, 0xc4, 0x0f, 0xc9, 0xe8, 0x6d, 0x60, 0x00, 0x65, 0xd0, 0x10, + 0x95, 0x26, 0x20, 0x11, 0xc2, 0xf4, 0x22, 0x75, 0xf7, 0x36, 0xcc, 0xb2, 0xc2, 0xc7, 0x8e, 0xd0, + 0x76, 0x23, 0x47, 0x0f, 0x90, 0x99, 0x7a, 0xf9, 0x3c, 0x52, 0x0b, 0xe2, 0x25, 0x1f, 0x21, 0x4e, + 0x1f, 0x6f, 0x01, 0x80, 0xe2, 0x14, 0x6d, 0x43, 0x3d, 0xd9, 0x8a, 0xcf, 0x39, 0x04, 0xc8, 0x18, + 0xb9, 0xf8, 0x29, 0xb9, 0x1e, 0x08, 0xb6, 0xb0, 0x2b, 0xb6, 0x06, 0x9b, 0x76, 0x2e, 0xa8, 0x3e, + 0xdc, 0x1f, 0x0e, 0xeb, 0xe2, 0x76, 0xb4, 0xac, 0xb0, 0x5b, 0x0e, 0xda, 0x00, 0xe3, 0xc3, 0x08, + 0xb1, 0x95, 0xfc, 0x1c, 0xda, 0x97, 0x07, 0x01, 0x00, 0x00 +}; + +const char *facebook_response_text_body = "{\"gcm_sender_id\":\"15057814354\",\"gcm_user_visible_only\":true,\"edge_side_panel\":\ +{\"preferred_width\":376},\"short_name\":\"Facebook\",\"name\":\"Facebook\",\"start_url\":\"\\/?ref=homescreenpwa\",\ +\"display\":\"minimal-ui\",\"background_color\":\"#FFFFFF\",\"theme_color\":\"#1877F2\",\"icons\":[{\"src\":\"https:\\/\\/static.xx.fbcdn.net\\/rsrc.php\\/v3\\/y0\\/r\\/eFZD1KABzRA.png\",\ +\"sizes\":\"192x192\",\"type\":\"image\\/png\"},{\"src\":\"https:\\/\\/static.xx.fbcdn.net\\/rsrc.php\\/v3\\/yd\\/r\\/DeNyZD1Vj3q.png\",\"sizes\":\"512x512\",\"type\":\"image\\/png\"}],\ +\"widgets\":[{\"name\":\"Facebook\",\"description\":\"Facebook\",\"short_name\":\"Facebook\",\"tag\":\"fb_widget\",\"type\":\"application\\/json\",\"update\":100,\"icons\"\ +:[{\"src\":\"https:\\/\\/static.xx.fbcdn.net\\/rsrc.php\\/v3\\/y0\\/r\\/eFZD1KABzRA.png\",\"sizes\":\"192x192\"}],\"screenshots\":[{\"src\":\"https:\\/\\/static.xx.fbcdn.net\\/rs\ +rc.php\\/v3\\/yT\\/r\\/mqlhqxHpT-X.png\",\"sizes\":\"464x478\",\"label\":\"Widget Screenshot\"}],\"template\":\"dummy\",\"data\":\"\\/dummy.json\",\"ms_ac_template\":\"\\/dummy.json\"}],\ +\"related_applications\":[{\"platform\":\"play\",\"id\":\"com.facebook.katana\"},{\"platform\":\"play\",\"id\":\"com.facebook.lite\"},{\"platform\":\"play\",\"id\":\"com.facebook.orca\"},\ +{\"platform\":\"play\",\"id\":\"com.facebook.mlite\"}],\"prefer_related_applications\":false}"; #endif diff --git a/vendor/CMakeLists.txt b/vendor/CMakeLists.txt index a122088..14922fc 100644 --- a/vendor/CMakeLists.txt +++ b/vendor/CMakeLists.txt @@ -343,6 +343,23 @@ add_dependencies(libnetfilter_queue-static libnetfilter_queue) set_property(TARGET libnetfilter_queue-static PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libnetfilter_queue.a) set_property(TARGET libnetfilter_queue-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) +### zstd +ExternalProject_Add(zstd PREFIX zstd + URL ${CMAKE_CURRENT_SOURCE_DIR}/zstd-1.5.6.tar.gz + URL_MD5 5a473726b3445d0e5d6296afd1ab6854 + CONFIGURE_COMMAND "" + BUILD_COMMAND make + INSTALL_COMMAND make install prefix= + BUILD_IN_SOURCE 1) + +ExternalProject_Get_Property(zstd INSTALL_DIR) +file(MAKE_DIRECTORY ${INSTALL_DIR}/include) + +add_library(zstd-static STATIC IMPORTED GLOBAL) +add_dependencies(zstd-static zstd) +set_property(TARGET zstd-static PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libzstd.a) +set_property(TARGET zstd-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) + ### gperftools #ExternalProject_Add(gperftools # PREFIX gperftools diff --git a/vendor/zstd-1.5.6.tar.gz b/vendor/zstd-1.5.6.tar.gz new file mode 100644 index 0000000..7e63317 Binary files /dev/null and b/vendor/zstd-1.5.6.tar.gz differ