diff --git a/cache/src/tango_cache_pending.cpp b/cache/src/tango_cache_pending.cpp index 2c7ea19..ab9b69b 100644 --- a/cache/src/tango_cache_pending.cpp +++ b/cache/src/tango_cache_pending.cpp @@ -284,15 +284,14 @@ enum cache_pending_action tfe_cache_put_pending(const struct tfe_http_half *resp memset(freshness,0,sizeof(struct response_freshness)); if(response->resp_spec.resp_code!=TFE_HTTP_STATUS_OK || NULL!=tfe_http_std_field_read(response, TFE_HTTP_CONT_RANGE) //NOT upload response with content-range - || NULL==response->resp_spec.content_length) - { - return FORBIDDEN; - } - value=tfe_http_std_field_read(response, TFE_HTTP_SET_COOKIE); - if(value!=NULL) + || NULL==response->resp_spec.content_length + || NULL!=tfe_http_std_field_read(response, TFE_HTTP_AUTHORIZATION) + || NULL!=tfe_http_nonstd_field_read(response, "WWW-Authenticate") + || NULL!=tfe_http_std_field_read(response, TFE_HTTP_SET_COOKIE)) { return FORBIDDEN; } + value = tfe_http_std_field_read(response, TFE_HTTP_PRAGMA); if (value != NULL) { diff --git a/conf/pangu/table_info.conf b/conf/pangu/table_info.conf index 043accc..7c5a34d 100644 --- a/conf/pangu/table_info.conf +++ b/conf/pangu/table_info.conf @@ -11,15 +11,21 @@ #id name type # #For plugin table -#id name type valid_column +#id name type json_descr # #For expr/expr_plus Table #id name type src_charset dst_charset do_merge cross_cache quick_mode 0 PXY_CTRL_COMPILE compile escape -- 1 PXY_CTRL_GROUP group -- 2 PXY_CTRL_IP ip --- -3 PXY_CTRL_HTTP_URL expr UTF8 GBK/BIG5/UNICODE/UTF8/url_encode_gb2312/url_encode_utf8 yes 128 quickoff -4 PXY_CTRL_HTTP_REQ_HDR expr_plus UTF8 GBK/BIG5/UNICODE/UTF8/url_encode_gb2312/url_encode_utf8 yes 128 quickoff -5 PXY_CTRL_HTTP_REQ_BODY expr UTF8 GBK/BIG5/UNICODE/UTF8/url_encode_gb2312/url_encode_utf8 yes 128 quickoff -6 PXY_CTRL_HTTP_RES_HDR expr_plus UTF8 GBK/BIG5/UNICODE/UTF8/url_encode_gb2312/url_encode_utf8 yes 128 quickoff -7 PXY_CTRL_HTTP_RES_BODY expr UTF8 GBK/BIG5/UNICODE/UTF8/url_encode_gb2312/url_encode_utf8 yes 128 quickoff +3 PXY_CTRL_HTTP_URL expr UTF8 GBK/UNICODE/UTF8/url_encode_gb2312/url_encode_utf8 yes 0 quickoff +4 PXY_CTRL_HTTP_REQ_HDR expr_plus UTF8 UTF8 yes 0 quickoff +5 PXY_CTRL_HTTP_REQ_BODY expr UTF8 GBK/BIG5/UNICODE/UTF8 yes 128 quickoff +6 PXY_CTRL_HTTP_RES_HDR expr_plus UTF8 UTF8 UTF8 yes 0 quickoff +7 PXY_CTRL_HTTP_RES_BODY expr UTF8 GBK/BIG5/UNICODE/UTF8 yes 128 quickoff +8 PXY_CACHE_COMPILE compile escape -- +9 PXY_CACHE_GROUP group -- +10 PXY_CACHE_HTTP_URL expr UTF8 UTF8 yes 0 quickoff +11 PXY_CACHE_HTTP_COOKIE expr UTF8 UTF8 yes 0 quickoff +12 PXY_OBJ_TRUSTED_CA_CERT plugin {"valid":4,"foreign":"3"} +13 PXY_OBJ_TRUSTED_CA_CRL plugin {"valid":4,"foreign":"3"} \ No newline at end of file diff --git a/plugin/business/pangu-http/CMakeLists.txt b/plugin/business/pangu-http/CMakeLists.txt index 2b93b94..a11a0fe 100644 --- a/plugin/business/pangu-http/CMakeLists.txt +++ b/plugin/business/pangu-http/CMakeLists.txt @@ -1,6 +1,6 @@ add_library(pangu-http src/pangu_logger.cpp src/pangu_http.cpp src/pattern_replace.cpp src/pangu_web_cache.cpp) target_link_libraries(pangu-http PUBLIC common http tango-cache-client) -target_link_libraries(pangu-http PUBLIC librdkafka ctemplate-static cjson pcre2-static pthread) +target_link_libraries(pangu-http PUBLIC librdkafka ctemplate-static cjson pcre2-static libdablooms pthread) target_link_libraries(pangu-http PUBLIC maatframe) add_executable(test_pattern_replace src/test_pattern_replace.cpp src/pattern_replace.cpp) diff --git a/plugin/business/pangu-http/src/pangu_http.cpp b/plugin/business/pangu-http/src/pangu_http.cpp index d631749..38ae4d9 100644 --- a/plugin/business/pangu-http/src/pangu_http.cpp +++ b/plugin/business/pangu-http/src/pangu_http.cpp @@ -85,6 +85,8 @@ struct pangu_rt int fs_id[__PG_STAT_MAX]; struct event_base* gc_evbase; struct event* gcev; + + int ca_store_reseting; }; struct pangu_rt * g_pangu_rt; @@ -206,11 +208,111 @@ static void pangu_http_stat_init(struct pangu_rt * pangu_runtime) return; } +void trusted_CA_update_start_cb(int update_type, void* u_para) +{ + if(update_type==MAAT_RULE_UPDATE_TYPE_FULL) + { + if(g_pangu_rt->ca_store_reseting==0) + { + tfe_proxy_ssl_reset_trust_ca(); + TFE_LOG_INFO(g_pangu_rt->local_logger, "Trusted CA Store Reset Start."); + } + g_pangu_rt->ca_store_reseting++; + } + +} +void trusted_CA_update_cert_cb(int table_id, const char* table_line, void* u_para) +{ + int ret=0, cfg_id=0, is_valid=0; + char cert_name[128]={0}, cert_file[1024]={0}; + ret=sscanf(table_line, "%d\t%s\t%s\t%d", &cfg_id, cert_name, cert_file, &is_valid); + if(ret!=4) + { + TFE_LOG_ERROR(g_pangu_rt->local_logger, "Trusted CA Store parse cert config failed: %s", table_line); + return; + } + if(is_valid==1) + { + ret=tfe_proxy_ssl_add_trust_ca(cert_file); + if(ret<0) + { + TFE_LOG_ERROR(g_pangu_rt->local_logger, "Trusted CA Store add cert failed %d:%s:%s", cfg_id, cert_name, cert_file); + } + else + { + TFE_LOG_INFO(g_pangu_rt->local_logger, "Trusted CA Store add cert success %d:%s:%s", cfg_id, cert_name, cert_file); + } + } + else + { + ret=tfe_proxy_ssl_del_trust_ca(cert_file); + if(ret<0) + { + TFE_LOG_ERROR(g_pangu_rt->local_logger, "Trusted CA Store del cert failed %d:%s:%s", cfg_id, cert_name, cert_file); + } + else + { + TFE_LOG_INFO(g_pangu_rt->local_logger, "Trusted CA Store del cert success %d:%s:%s", cfg_id, cert_name, cert_file); + } + } + return; +} +void trusted_CA_update_crl_cb(int table_id,const char* table_line,void* u_para) +{ + int ret=0, crl_id=0, cert_id=0, is_valid=0; + char crl_file[1024]={0}; + ret=sscanf(table_line, "%d\t%d\t%s\t%d", &crl_id, &cert_id, crl_file, &is_valid); + if(ret!=4) + { + TFE_LOG_ERROR(g_pangu_rt->local_logger, "Trusted CA Store parse crl config failed: %s", table_line); + return; + } + if(is_valid==1) + { + ret=tfe_proxy_ssl_add_crl(crl_file); + if(ret<0) + { + TFE_LOG_ERROR(g_pangu_rt->local_logger, "Trusted CA Store add crl failed %d:%s:%s", crl_id, cert_id, crl_file); + } + else + { + TFE_LOG_INFO(g_pangu_rt->local_logger, "Trusted CA Store add crl success %d:%d:%s", crl_id, cert_id, crl_file); + } + } + else + { + ret=tfe_proxy_ssl_del_crl(crl_file); + if(ret<0) + { + TFE_LOG_ERROR(g_pangu_rt->local_logger, "Trusted CA Store del crl failed %d:%s:%s", crl_id, cert_id, crl_file); + } + else + { + TFE_LOG_INFO(g_pangu_rt->local_logger, "Trusted CA Store del crl success %d:%d:%s", crl_id, cert_id, crl_file); + } + } + return; +} +void trusted_CA_update_finish_cb(void* u_para) +{ + int is_last_updating_table=0, ret=0; + if(g_pangu_rt->ca_store_reseting>0) + { + g_pangu_rt->ca_store_reseting--; + if(g_pangu_rt->ca_store_reseting==0) + { + TFE_LOG_INFO(g_pangu_rt->local_logger, "Trusted CA Store Reset Finish."); + } + } +} + int pangu_http_init(struct tfe_proxy * proxy) { const char * profile = "./pangu_conf/pangu_pxy.conf"; - const char * logfile = "./log/pangu_pxy.log"; + const char * logfile = "./log/pangu_pxy.log"; + int table_id=0; + g_pangu_rt = ALLOC(struct pangu_rt, 1); g_pangu_rt->thread_num = tfe_proxy_get_work_thread_count(); g_pangu_rt->gc_evbase=tfe_proxy_get_gc_evbase(); @@ -266,7 +368,8 @@ int pangu_http_init(struct tfe_proxy * proxy) MESA_load_profile_int_def(profile, "TANGO_CACHE", "enable_cache", &(g_pangu_rt->cache_enabled), 1); if(g_pangu_rt->cache_enabled) { - g_pangu_rt->cache = create_web_cache_handle(profile, "TANGO_CACHE", g_pangu_rt->gc_evbase, g_pangu_rt->local_logger); + g_pangu_rt->cache = create_web_cache_handle(profile, "TANGO_CACHE", g_pangu_rt->gc_evbase, + g_pangu_rt->maat, g_pangu_rt->local_logger); if(!g_pangu_rt->cache) { TFE_LOG_INFO(NULL, "Tango Cache init failed."); @@ -274,6 +377,30 @@ int pangu_http_init(struct tfe_proxy * proxy) } TFE_LOG_INFO(NULL, "Tango Cache Enabled."); } + table_id=Maat_table_register(g_pangu_rt->maat, "PXY_OBJ_TRUSTED_CA_CERT"); + if(table_id<0) + { + TFE_LOG_INFO(NULL, "Pangu HTTP register table PXY_OBJ_TRUSTED_CA_CERT failed."); + goto error_out; + } + Maat_table_callback_register(g_pangu_rt->maat, table_id, + trusted_CA_update_start_cb, + trusted_CA_update_cert_cb, + trusted_CA_update_finish_cb, + g_pangu_rt); + + table_id=Maat_table_register(g_pangu_rt->maat, "PXY_OBJ_TRUSTED_CA_CRL"); + if(table_id<0) + { + TFE_LOG_INFO(NULL, "Pangu HTTP register table PXY_OBJ_TRUSTED_CA_CRL failed."); + goto error_out; + } + Maat_table_callback_register(g_pangu_rt->maat, table_id, + trusted_CA_update_start_cb, + trusted_CA_update_crl_cb, + trusted_CA_update_finish_cb, + g_pangu_rt); + TFE_LOG_INFO(NULL, "Pangu HTTP init success."); return 0; @@ -297,9 +424,9 @@ struct pangu_http_ctx int magic_num; enum pangu_action action; char * action_para; - scan_status_t mid; + scan_status_t scan_mid; stream_para_t sp; - + struct cache_mid* cmid; struct Maat_rule_t * enforce_rules; size_t n_enforce; char * enforce_para; @@ -345,7 +472,7 @@ static struct pangu_http_ctx * pangu_http_ctx_new(unsigned int thread_id) { struct pangu_http_ctx * ctx = ALLOC(struct pangu_http_ctx, 1); ctx->magic_num=HTTP_CTX_MAGIC_NUM; - ctx->mid = NULL; + ctx->scan_mid = NULL; ctx->thread_id = (int) thread_id; return ctx; } @@ -360,8 +487,8 @@ static void pangu_http_ctx_free(struct pangu_http_ctx * ctx) } FREE(&ctx->enforce_rules); FREE(&ctx->enforce_para); - Maat_clean_status(&(ctx->mid)); - ctx->mid = NULL; + Maat_clean_status(&(ctx->scan_mid)); + ctx->scan_mid = NULL; if(ctx->sp) { @@ -749,7 +876,7 @@ enum pangu_action http_scan(const struct tfe_http_session * session, enum tfe_ht int str_url_length = (int) (strlen(session->req->req_spec.url)); scan_ret = Maat_full_scan_string(g_pangu_rt->maat, g_pangu_rt->scan_table_id[PXY_CTRL_HTTP_URL], - CHARSET_UTF8, str_url, str_url_length, result, NULL, MAX_SCAN_RESULT, &(ctx->mid), ctx->thread_id); + CHARSET_UTF8, str_url, str_url_length, result, NULL, MAX_SCAN_RESULT, &(ctx->scan_mid), ctx->thread_id); if (scan_ret > 0) { @@ -770,13 +897,13 @@ enum pangu_action http_scan(const struct tfe_http_session * session, enum tfe_ht } const char * str_field_name = http_field_to_string(&field_name); - scan_ret = Maat_set_scan_status(g_pangu_rt->maat, &(ctx->mid), MAAT_SET_SCAN_DISTRICT, + scan_ret = Maat_set_scan_status(g_pangu_rt->maat, &(ctx->scan_mid), MAAT_SET_SCAN_DISTRICT, str_field_name, strlen(str_field_name)); assert(scan_ret == 0); scan_ret = Maat_full_scan_string(g_pangu_rt->maat, table_id, CHARSET_UTF8, field_val, strlen(field_val), - result + hit_cnt, NULL, MAX_SCAN_RESULT - hit_cnt, &(ctx->mid), ctx->thread_id); + result + hit_cnt, NULL, MAX_SCAN_RESULT - hit_cnt, &(ctx->scan_mid), ctx->thread_id); if (scan_ret > 0) { hit_cnt += scan_ret; @@ -795,7 +922,7 @@ enum pangu_action http_scan(const struct tfe_http_session * session, enum tfe_ht if (body_frag != NULL) { scan_ret = Maat_stream_scan_string(&(ctx->sp), CHARSET_UTF8, (const char *) body_frag, (int) frag_size, - result + hit_cnt, NULL, MAX_SCAN_RESULT - hit_cnt, &(ctx->mid)); + result + hit_cnt, NULL, MAX_SCAN_RESULT - hit_cnt, &(ctx->scan_mid)); if (scan_ret > 0) { hit_cnt += scan_ret; @@ -1057,7 +1184,7 @@ void cache_pending(const struct tfe_http_session * session, unsigned int thread_ enum cache_pending_result ret; ctx->f_cache_pending=future_create("cache_pend", cache_pending_on_succ, cache_pending_on_fail, ctx); ctx->ref_session=tfe_http_session_allow_write(session); - ctx->pending_result=web_cache_async_pending(g_pangu_rt->cache, thread_id, session->req, ctx->f_cache_pending); + ctx->pending_result=web_cache_async_pending(g_pangu_rt->cache, thread_id, session->req, &(ctx->cmid), ctx->f_cache_pending); switch(ctx->pending_result) { case PENDING_RESULT_REVALIDATE: @@ -1077,7 +1204,7 @@ void cache_pending(const struct tfe_http_session * session, unsigned int thread_ void cache_query(const struct tfe_http_session * session, unsigned int thread_id, struct pangu_http_ctx * ctx) { ctx->f_cache_query=future_create("cache_get", cache_query_on_succ, cache_query_on_fail, ctx); - int ret=web_cache_async_query(g_pangu_rt->cache, thread_id, session->req, ctx->f_cache_query); + int ret=web_cache_async_query(g_pangu_rt->cache, thread_id, session->req, &(ctx->cmid), ctx->f_cache_query); if(ret==0) { ctx->ref_session=tfe_http_session_allow_write(session); @@ -1097,7 +1224,7 @@ void cache_update(const struct tfe_http_session * session, enum tfe_http_event e if(events & EV_HTTP_RESP_BODY_BEGIN) { - ctx->cache_update_ctx=web_cache_update_start(g_pangu_rt->cache, thread_id, session); + ctx->cache_update_ctx=web_cache_update_start(g_pangu_rt->cache, thread_id, session, &(ctx->cmid)); } if(events & EV_HTTP_RESP_BODY_CONT && ctx->cache_update_ctx!=NULL) { @@ -1124,7 +1251,7 @@ void pangu_on_http_begin(const struct tfe_stream * stream, ctx = pangu_http_ctx_new(thread_id); addr_tfe2sapp(stream->addr, &sapp_addr); hit_cnt = Maat_scan_proto_addr(g_pangu_rt->maat, g_pangu_rt->scan_table_id[PXY_CTRL_IP], &sapp_addr, 0, - result, MAX_SCAN_RESULT, &(ctx->mid), (int) thread_id); + result, MAX_SCAN_RESULT, &(ctx->scan_mid), (int) thread_id); if (hit_cnt > 0) { @@ -1174,6 +1301,7 @@ void pangu_on_http_end(const struct tfe_stream * stream, { ATOMIC_INC(&(g_pangu_rt->stat_val[STAT_ACTION_REPLACE])); } + cache_mid_clear(&(ctx->cmid)); pangu_http_ctx_free(ctx); *pme = NULL; return; diff --git a/plugin/business/pangu-http/src/pangu_web_cache.cpp b/plugin/business/pangu-http/src/pangu_web_cache.cpp index 1468805..1e01395 100644 --- a/plugin/business/pangu-http/src/pangu_web_cache.cpp +++ b/plugin/business/pangu-http/src/pangu_web_cache.cpp @@ -9,11 +9,19 @@ #include #include -#include #include #include +extern "C" +{ +#include +} + +#include + +#include + enum cache_stat_field { STAT_CACHE_QUERY, @@ -29,6 +37,7 @@ enum cache_stat_field STAT_CACHE_QUERYING, STAT_CACHE_PENDING, STAT_CACHE_UPLOAD_CNT, + STAT_CACHE_UPLOAD_BYPASS, STAT_CACHE_UPLOAD_OVERRIDE, STAT_CACHE_UPLOAD_FORBIDEN, STAT_CACHE_UPLOAD_ABANDON, @@ -44,6 +53,33 @@ enum cache_stat_field STAT_CACHE_OVERRIDE_UPLOAD_OBJ_SIZE, __CACHE_STAT_MAX }; + +struct cache_key_descr +{ + int is_not_empty; + size_t qs_num; + char** ignore_qs; + char* include_cookie; +}; +struct cache_param +{ + int ref_cnt; + struct cache_key_descr cache_key; + + char no_revalidate; + char cache_dyn_url; + char cache_cookied_cont; + char ignore_req_nocache; + char ignore_res_nocache; + char force_caching; + + int min_use; + time_t pinning_time_sec; + time_t inactive_time_sec; + long max_cache_size; + long max_cache_obj_size; + pthread_mutex_t lock; +}; struct cache_handle { unsigned int thread_count; @@ -62,11 +98,16 @@ struct cache_handle struct event_base* gc_evbase; struct event* gcev; + + int cache_policy_enabled; //otherwise use default cache policy struct cache_param default_cache_policy; Maat_feather_t ref_feather; int cache_param_idx; int table_url_constraint; int table_cookie_constraint; + size_t cache_key_bloom_size; + int cache_key_bloom_life; + counting_bloom_t **cache_key_bloom; void* logger; }; struct cache_update_context @@ -74,7 +115,7 @@ struct cache_update_context struct cache_handle* ref_cache_handle; struct tango_cache_ctx * write_ctx; }; -static void cache_gc_cb(evutil_socket_t fd, short what, void * arg) +static void web_cache_stat_cb(evutil_socket_t fd, short what, void * arg) { struct cache_handle* cache=(struct cache_handle *)arg; struct cache_statistics client_stat_sum, client_stat; @@ -169,6 +210,7 @@ void cache_stat_init(struct cache_handle* cache) set_stat_spec(&spec[STAT_CACHE_QUERYING], "getting",FS_STYLE_STATUS, FS_CALC_CURRENT); set_stat_spec(&spec[STAT_CACHE_PENDING], "pending",FS_STYLE_STATUS, FS_CALC_CURRENT); set_stat_spec(&spec[STAT_CACHE_UPLOAD_CNT], "cache_put",FS_STYLE_FIELD, FS_CALC_CURRENT); + set_stat_spec(&spec[STAT_CACHE_UPLOAD_BYPASS], "cache_bypass",FS_STYLE_FIELD, FS_CALC_CURRENT); set_stat_spec(&spec[STAT_CACHE_UPLOAD_OVERRIDE], "or_put",FS_STYLE_FIELD, FS_CALC_CURRENT); set_stat_spec(&spec[STAT_CACHE_UPLOAD_FORBIDEN], "put_forbid",FS_STYLE_FIELD, FS_CALC_CURRENT); set_stat_spec(&spec[STAT_CACHE_UPLOAD_ABANDON], "put_abandon",FS_STYLE_FIELD, FS_CALC_CURRENT); @@ -214,44 +256,170 @@ void cache_stat_init(struct cache_handle* cache) FS_start(cache->fs_handle); struct timeval gc_delay = {0, 500*1000}; //Microseconds, we set 500 miliseconds here. - cache->gcev = event_new(cache->gc_evbase, -1, EV_PERSIST, cache_gc_cb, cache); + cache->gcev = event_new(cache->gc_evbase, -1, EV_PERSIST, web_cache_stat_cb, cache); evtimer_add(cache->gcev, &gc_delay); return; } -struct cache_key_descr -{ - size_t qs_num; - char** ignore_qs; - char* include_hdrs; - char* include_cookie; -}; -struct cache_param -{ - struct cache_key_descr key_descr; - - char revalidate; - char cache_dyn_url; - char cache_cookied_cont; - char ignore_req_nocache; - char ignore_res_nocache; - char force_caching; - - int min_use; - int pinning_time_sec; - int inactive_time_sec; - int max_cache_size_mb; - int max_cache_obj_size_mb; -}; -int time_secs_interval(const char* str){} -int storage_mb_size(const char* str){} -int is_dynamic_url(const char* url){} +time_t time_unit_sec(const char* str) +{ + time_t value=0; + sscanf(str, "%ld", &value); + switch(str[strlen(str)-1]) + { + case 's': + break; + case 'm': + value*=60; + break; + case 'h': + value*=3600; + break; + case 'd': + value*=24*3600; + break; + default: + break; + } + return value; +} +size_t storage_unit_byte(const char* str) +{ + size_t value=0; + sscanf(str, "%ld", &value); + switch(str[strlen(str)-1]) + { + case 'b': + break; + case 'k': + value*=1024; + break; + case 'm': + value*=1024*1024; + break; + case 'g': + value*=1024*1024*1024; + break; + case 't': + if(value<1024) + { +#pragma GCC diagnostic ignored "-Woverflow" + value*=1024*1024*1024*1024; + + } + break; + default: + break; + } + return value; +} + + +//A URL is considered dynamic if it ends in “.asp(x)” or contains a question mark (?), a semicolon (;), or “cgi”. +char is_dynamic_url(const char* url) +{ + if(strchr(url, '?') + || strchr(url, ';') + || strstr(url, "cgi") + || 0==strcmp(url+strlen(url)-3,"asp") + || 0==strcmp(url+strlen(url)-4, "aspx")) + { + return 1; + } + return 0; +} +char * cookie_scanvalue(const char * key, const char * qs, char * val, size_t val_len) +{ + int i=0, key_len=0; + + key_len = strlen(key); + while(qs[0] != '\0') + { + if ( strncmp(key, qs, key_len) == 0 ) + break; + qs += strcspn(qs, ";") + 1; + } + + if ( qs[0] == '\0' ) return NULL; + + qs += strcspn(qs, "=;"); + if ( qs[0] == '=' ) + { + qs++; + i = strcspn(qs, "=;"); + strncpy(val, qs, (val_len-1)<(i+1) ? (val_len-1) : (i+1)); + } + else + { + if ( val_len > 0 ) + val[0] = '\0'; + } + + return val; +} + + +char* get_cache_key(const struct tfe_http_half * request, const struct cache_key_descr* desc) +{ + if(desc==NULL|| !desc->is_not_empty) + { + return NULL; + } + int i=0, shall_ignore=0; + + char *token=NULL,*sub_token=NULL,*saveptr; + char* url=tfe_strdup(request->req_spec.url); + const char* cookie=NULL; + char cookie_val[256]={0}; //most 256 bytes for cookie key + size_t key_size=strlen(url)+sizeof(cookie_val); + char* cache_key=ALLOC(char, key_size); + char* query_string=strchr(url, '?'); + if(query_string!=NULL && desc->qs_num>0) + { + query_string++; + for (token = url; ; token= NULL) + { + sub_token= strtok_r(token,"&", &saveptr); + if (sub_token == NULL) + break; + shall_ignore=0; + for(i=0; iqs_num; i++) + { + if(0==strncasecmp(sub_token, desc->ignore_qs[i], strlen(desc->ignore_qs[i]))) + { + shall_ignore=1; + break; + } + } + if(!shall_ignore) + { + strncat(cache_key, sub_token, key_size); + } + } + } + else + { + strncat(cache_key, url, key_size); + } + if(desc->include_cookie && (cookie=tfe_http_std_field_read(request, TFE_HTTP_COOKIE))!=NULL) + { + cookie_scanvalue(desc->include_cookie, cookie, cookie_val, sizeof(cookie_val)); + if(strlen(cookie_val)>0) + { + strncat(cache_key, cookie_val, key_size); + } + } + FREE(&(url)); + +} + void cache_param_new(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, MAAT_RULE_EX_DATA* ad, long argl, void *argp) { struct cache_handle* cache=(struct cache_handle*) argp; int i=0; + size_t len=0; *ad=NULL; if(rule->serv_def_lendefault_cache_policy; + param->ref_cnt=1; + pthread_mutex_init(&(param->lock), NULL); key_desc=cJSON_GetObjectItem(json,"cache_key"); if(key_desc && key_desc->type==cJSON_Object) { + param->cache_key.is_not_empty=1; qs=cJSON_GetObjectItem(json,"ignore_qs"); if(qs && qs->type==cJSON_Array) { - param->key_descr.qs_num=cJSON_GetArraySize(qs); - param->key_descr.ignore_qs=ALLOC(char*, param->key_descr.qs_num); - for(i=0; ikey_descr.qs_num; i++) + param->cache_key.qs_num=cJSON_GetArraySize(qs); + param->cache_key.ignore_qs=ALLOC(char*, param->cache_key.qs_num); + for(i=0; icache_key.qs_num; i++) { item=cJSON_GetArrayItem(item, i); - param->key_descr.ignore_qs[i]=tfe_strdup(qs->valuestring); + len=strlen(qs->valuestring)+2; + param->cache_key.ignore_qs[i]=ALLOC(char, len); + strncat(param->cache_key.ignore_qs[i], qs->valuestring, len); + strncat(param->cache_key.ignore_qs[i], "=", len); } - } - } - item=cJSON_GetObjectItem(key_desc,"hdrs"); - if(item && item->type==cJSON_String) param->key_descr.include_hdrs=tfe_strdup(item->valuestring); + } + item=cJSON_GetObjectItem(key_desc,"cookie"); + if(item && item->type==cJSON_String) param->cache_key.include_cookie=tfe_strdup(param->cache_key.include_cookie); + + } - item=cJSON_GetObjectItem(key_desc,"cookie"); - if(item && item->type==cJSON_String) param->key_descr.include_cookie=tfe_strdup(item->valuestring); - item=cJSON_GetObjectItem(json,"revalidate"); - if(item && item->type==cJSON_Number) param->revalidate=item->valueint; + item=cJSON_GetObjectItem(json,"no_revalidate"); + if(item && item->type==cJSON_Number) param->no_revalidate=item->valueint; item=cJSON_GetObjectItem(json,"cache_dyn_url"); if(item && item->type==cJSON_Number) param->cache_dyn_url=item->valueint; @@ -310,16 +484,16 @@ void cache_param_new(int idx, const struct Maat_rule_t* rule, const char* srv_de if(item && item->type==cJSON_Number) param->min_use=item->valueint; item=cJSON_GetObjectItem(json,"pinning_time"); - if(item && item->type==cJSON_String) param->pinning_time_sec=time_secs_interval(item->valuestring); + if(item && item->type==cJSON_String) param->pinning_time_sec=time_unit_sec(item->valuestring); item=cJSON_GetObjectItem(json,"inactive_time"); - if(item && item->type==cJSON_String) param->inactive_time_sec=time_secs_interval(item->valuestring); + if(item && item->type==cJSON_String) param->inactive_time_sec=time_unit_sec(item->valuestring); item=cJSON_GetObjectItem(json,"max_cache_size"); - if(item && item->type==cJSON_String) param->max_cache_size_mb=storage_mb_size(item->valuestring); + if(item && item->type==cJSON_String) param->max_cache_size=storage_unit_byte(item->valuestring); item=cJSON_GetObjectItem(json,"max_cache_obj_size"); - if(item && item->type==cJSON_String) param->max_cache_obj_size_mb=storage_mb_size(item->valuestring); + if(item && item->type==cJSON_String) param->max_cache_obj_size=storage_unit_byte(item->valuestring); cJSON_Delete(json); return; @@ -332,28 +506,73 @@ void cache_param_free(int idx, const struct Maat_rule_t* rule, const char* srv_d return; } struct cache_param* param=(struct cache_param*)*ad; - for(i=0; ikey_descr.qs_num; i++) - { - FREE(&(param->key_descr.ignore_qs[i])); + pthread_mutex_lock(&(param->lock)); + param->ref_cnt--; + if(param->ref_cnt>0) + { + pthread_mutex_unlock(&(param->lock)); + return; } - FREE(&(param->key_descr.ignore_qs)); - FREE(&(param->key_descr.include_cookie)); - FREE(&(param->key_descr.include_hdrs)); + pthread_mutex_unlock(&(param->lock)); + pthread_mutex_destroy(&(param->lock)); + for(i=0; icache_key.qs_num; i++) + { + FREE(&(param->cache_key.ignore_qs[i])); + } + FREE(&(param->cache_key.ignore_qs)); + FREE(&(param->cache_key.include_cookie)); FREE(&(param)); return; } +void cache_param_dup(int idx, MAAT_RULE_EX_DATA *to, MAAT_RULE_EX_DATA *from, long argl, void *argp) +{ + struct cache_param* from_param=*((struct cache_param**)from); + pthread_mutex_lock(&(from_param->lock)); + from_param->ref_cnt++; + pthread_mutex_unlock(&(from_param->lock)); + *((struct cache_param**)to)=from_param; + return; +} + +static void cache_key_bloom_gc_cb(evutil_socket_t fd, short what, void * arg) +{ + counting_bloom_t* old_bloom=*((counting_bloom_t**)arg), *new_bloom=NULL; + + new_bloom=new_counting_bloom(old_bloom->capacity, old_bloom->error_rate, NULL); + free_counting_bloom(old_bloom); + *((counting_bloom_t**)arg)=old_bloom; + return; +} struct cache_handle* create_web_cache_handle(const char* profile_path, const char* section, struct event_base* gc_evbase, Maat_feather_t feather, void *logger) { struct cache_handle* cache=ALLOC(struct cache_handle, 1); int temp=0; + struct event* ev=NULL; cache->logger=logger; cache->thread_count=tfe_proxy_get_work_thread_count(); cache->clients=ALLOC(struct tango_cache_instance *, cache->thread_count); + cache->cache_key_bloom=ALLOC(counting_bloom_t*, cache->thread_count); + MESA_load_profile_int_def(profile_path, section, "cache_policy_enabled", + &(cache->cache_policy_enabled), 1); + + + MESA_load_profile_int_def(profile_path, section, "cache_key_bloom_size", + (int*)&(cache->cache_key_bloom_size), 16*1000*1000); + MESA_load_profile_int_def(profile_path, section, "cache_key_bloom_life", + &(cache->cache_key_bloom_life), 30*60); + struct timeval gc_refresh_delay = {cache->cache_key_bloom_life, 0}; int i=0; for(i=0; ithread_count; i++) { + if(cache->cache_policy_enabled) + { + cache->cache_key_bloom[i]=new_counting_bloom(cache->cache_key_bloom_size, 0.01, NULL); + ev = event_new(tfe_proxy_get_work_thread_evbase(i), -1, EV_PERSIST, cache_key_bloom_gc_cb, &(cache->cache_key_bloom[i])); + evtimer_add(ev, &gc_refresh_delay); + } + cache->clients[i]=tango_cache_instance_new(tfe_proxy_get_work_thread_evbase(i), profile_path, section, logger); if(cache->clients[i]==NULL) { @@ -369,29 +588,32 @@ struct cache_handle* create_web_cache_handle(const char* profile_path, const cha MESA_load_profile_int_def(profile_path, section, "cache_undefined_obj", &(cache->cache_undefined_obj_enabled), 1); MESA_load_profile_int_def(profile_path, section, "cached_undefined_obj_minimum_size", &(temp), 100*1024); cache->cache_undefined_obj_min_size=temp; - MESA_load_profile_int_def(profile_path, section, "cache_minimum_time_override", &(cache->minimum_cache_seconds), 60*5); cache->gc_evbase=gc_evbase; - cache->default_cache_policy.key_descr.qs_num=0; - cache->default_cache_policy.revalidate=1; + cache->default_cache_policy.cache_key.qs_num=0; + cache->default_cache_policy.no_revalidate=0; cache->default_cache_policy.cache_dyn_url=0; cache->default_cache_policy.cache_cookied_cont=0; cache->default_cache_policy.ignore_req_nocache=0; cache->default_cache_policy.ignore_res_nocache=0; cache->default_cache_policy.force_caching=0; - cache->default_cache_policy.min_use=1; + cache->default_cache_policy.min_use=0; cache->default_cache_policy.pinning_time_sec=0; cache->default_cache_policy.inactive_time_sec=0; - cache->default_cache_policy.max_cache_size_mb=0; - cache->default_cache_policy.max_cache_obj_size_mb=1024;//<1GB by default + cache->default_cache_policy.max_cache_size=0; + cache->default_cache_policy.max_cache_obj_size=1024*1024*1024;//<1GB by default - cache->table_url_constraint=Maat_table_register(feather, "PXY_CACHE_HTTP_URL"); - cache->table_cookie_constraint=Maat_table_register(feather, "PXY_CACHE_HTTP_COOKIE"); - - cache->cache_param_idx=Maat_rule_get_ex_new_index(feather, "PXY_CACHE_COMPILE", - cache_param_new, cache_param_free, 0, cache); - + if(cache->cache_policy_enabled) + { + cache->table_url_constraint=Maat_table_register(feather, "PXY_CACHE_HTTP_URL"); + cache->table_cookie_constraint=Maat_table_register(feather, "PXY_CACHE_HTTP_COOKIE"); + + cache->cache_param_idx=Maat_rule_get_ex_new_index(feather, "PXY_CACHE_COMPILE", + cache_param_new, cache_param_free, cache_param_dup, + 0, cache); + cache->ref_feather=feather; + } cache_stat_init(cache); return cache; error_out: @@ -545,7 +767,6 @@ struct cache_pending_context { enum cache_pending_result status; int is_undefined_obj; - struct request_freshness req_fresshness; char* req_if_none_match, *req_if_modified_since; const struct tfe_http_half * request; @@ -566,7 +787,10 @@ void cache_pending_ctx_free_cb(void* p) if(ctx->f_tango_cache_fetch) { future_destroy(ctx->f_tango_cache_fetch); - } + } + FREE(&(ctx->cached_obj_meta.etag)); + FREE(&(ctx->cached_obj_meta.last_modified)); + FREE(&(ctx->cached_obj_meta.content_type)); free(ctx); return; } @@ -631,50 +855,91 @@ static void cache_query_meta_on_fail(enum e_future_error err, const char * what, } struct cache_mid { + enum cache_pending_result result; + struct request_freshness req_fresshness; + char shall_bypass; + char is_using_exception_param; + char is_dyn_url; + char has_cookie; + char use_cnt; + int cfg_id; + char* cache_key; struct cache_param* param; }; +void cache_mid_clear(struct cache_mid **mid) +{ + if(*mid==NULL) + { + return; + } + if((*mid)->is_using_exception_param) + { + cache_param_free(0, NULL, NULL, (void**)&((*mid)->param), 0, NULL); + } + FREE(&((*mid)->cache_key)); + FREE(mid); + return; +} + +#define CACHE_ACTION_BYPASS 0x80 enum cache_pending_result web_cache_async_pending(struct cache_handle* handle, unsigned int thread_id, const struct tfe_http_half * request, struct cache_mid** mid, struct future* f_revalidate) { - struct request_freshness req_fresshness={0,0}; enum cache_pending_result result=PENDING_RESULT_FOBIDDEN; int is_undefined_obj=0; struct Maat_rule_t cache_policy; - const struct cache_param* param=&(handle->default_cache_policy); + struct cache_param* param=&(handle->default_cache_policy); MAAT_RULE_EX_DATA ex_data=NULL; scan_status_t scan_mid=NULL; int ret=0; const char* cookie=NULL; - ret=Maat_full_scan_string(handle->ref_feather, handle->table_url_constraint, CHARSET_UTF8, - request->req_spec.url, strlen(request->req_spec.url), - &cache_policy, NULL, 1, &scan_mid, thread_id); - + struct cache_mid* _mid=ALLOC(struct cache_mid, 1); + *mid=_mid; cookie=tfe_http_std_field_read(request, TFE_HTTP_COOKIE); - if(cookie && ret<=0) + if(cookie) { - ret=Maat_full_scan_string(handle->ref_feather, handle->table_cookie_constraint, CHARSET_UTF8, - cookie, strlen(cookie), + _mid->has_cookie=1; + } + _mid->is_dyn_url=is_dynamic_url(request->req_spec.url); + if(handle->cache_policy_enabled) + { + ret=Maat_full_scan_string(handle->ref_feather, handle->table_url_constraint, CHARSET_UTF8, + request->req_spec.url, strlen(request->req_spec.url), &cache_policy, NULL, 1, &scan_mid, thread_id); - } - Maat_clean_status(&scan_mid); + - if(ret>0) - { - ex_data=Maat_rule_get_ex_data(handle->ref_feather, &cache_policy, handle->cache_param_idx); - if(ex_data!=NULL) + if(cookie && ret<=0) { - param=(const struct cache_param*)ex_data; + ret=Maat_full_scan_string(handle->ref_feather, handle->table_cookie_constraint, CHARSET_UTF8, + cookie, strlen(cookie), + &cache_policy, NULL, 1, &scan_mid, thread_id); + } + Maat_clean_status(&scan_mid); + + if(ret>0) + { + ex_data=Maat_rule_get_ex_data(handle->ref_feather, &cache_policy, handle->cache_param_idx); + if(ex_data!=NULL) + { + param=(struct cache_param*)ex_data; + _mid->is_using_exception_param=1; + _mid->param=param; + } + if(cache_policy.action==CACHE_ACTION_BYPASS) + { + _mid->shall_bypass=1; + } + _mid->cfg_id=cache_policy.config_id; + } + if(_mid->shall_bypass || + (!param->cache_dyn_url && _mid->is_dyn_url && param->cache_key.qs_num==0) || + (param->cache_cookied_cont && _mid->has_cookie) ) + { + _mid->result=PENDING_RESULT_FOBIDDEN; + return _mid->result; } } - if(param->cache_dyn_url==0 && is_dynamic_url(request->req_spec.url)) - { - return FORBIDDEN; - } - if(param->cache_cookied_cont==0 && cookie!=NULL) - { - return FORBIDDEN; - } - enum cache_pending_action get_action=tfe_cache_get_pending(request, &(req_fresshness)); + enum cache_pending_action get_action=tfe_cache_get_pending(request, &(_mid->req_fresshness)); switch(get_action) { case UNDEFINED: @@ -705,51 +970,68 @@ enum cache_pending_result web_cache_async_pending(struct cache_handle* handle, u result=PENDING_RESULT_ALLOWED; break; default: - result=PENDING_RESULT_REVALIDATE; + if(param->no_revalidate) + { + result=PENDING_RESULT_ALLOWED; + } + else + { + result=PENDING_RESULT_REVALIDATE; + } break; } if(result!=PENDING_RESULT_REVALIDATE) { - return result; + _mid->result=result; + return _mid->result; } struct tango_cache_meta_get meta; memset(&meta, 0, sizeof(meta)); - meta.url=request->req_spec.url; - meta.get=req_fresshness; + if(param->cache_key.is_not_empty) + { + _mid->cache_key=get_cache_key(request, &(param->cache_key)); + meta.url = _mid->cache_key; + } + else + { + meta.url = request->req_spec.url; + } + meta.get = _mid->req_fresshness; struct promise* p=future_to_promise(f_revalidate); struct cache_pending_context* ctx=ALLOC(struct cache_pending_context, 1); ctx->status=PENDING_RESULT_FOBIDDEN; ctx->ref_handle=handle; - ctx->req_fresshness=req_fresshness; ctx->url=tfe_strdup(request->req_spec.url); + ctx->req_if_modified_since=tfe_strdup(tfe_http_std_field_read(request, TFE_HTTP_IF_MODIFIED_SINCE)); ctx->req_if_none_match=tfe_strdup(tfe_http_std_field_read(request, TFE_HTTP_IF_NONE_MATCH)); promise_set_ctx(p, ctx, cache_pending_ctx_free_cb); ATOMIC_INC(&(handle->stat_val[STAT_CACHE_PENDING])); ctx->f_tango_cache_fetch=future_create("_cache_pend", cache_query_meta_on_succ, cache_query_meta_on_fail, p); - int ret=tango_cache_head_object(handle->clients[thread_id], ctx->f_tango_cache_fetch, &meta); + ret=tango_cache_head_object(handle->clients[thread_id], ctx->f_tango_cache_fetch, &meta); if(ret<0) { cache_pending_ctx_free_cb(ctx); - return PENDING_RESULT_FOBIDDEN; + _mid->result=PENDING_RESULT_FOBIDDEN; + return _mid->result; } - assert(ret==0); - return PENDING_RESULT_REVALIDATE; + _mid->result=PENDING_RESULT_REVALIDATE; + + return _mid->result; } int web_cache_async_query(struct cache_handle* handle, unsigned int thread_id, - const struct tfe_http_half * request, struct future* f) + const struct tfe_http_half * request, struct cache_mid** mid, struct future* f) { struct request_freshness req_fresshness; enum cache_pending_action get_action; struct cache_query_context* query_ctx=NULL; struct promise* p=NULL; struct future* _f=NULL; - - get_action=tfe_cache_get_pending(request, &req_fresshness); - assert(get_action!=FORBIDDEN); + struct cache_mid* _mid=*mid; + assert(_mid->result!=PENDING_RESULT_FOBIDDEN); if(ATOMIC_READ(&(handle->stat_val[STAT_CACHE_QUERYING])) > ATOMIC_READ(&(handle->put_concurrency_max))) { @@ -760,6 +1042,7 @@ int web_cache_async_query(struct cache_handle* handle, unsigned int thread_id, struct tango_cache_meta_get meta; memset(&meta, 0, sizeof(meta)); meta.url=request->req_spec.url; + meta.get=_mid->req_fresshness; memcpy(&(meta.get), &req_fresshness, sizeof(meta.get)); query_ctx=ALLOC(struct cache_query_context, 1); query_ctx->ref_handle=handle; @@ -805,17 +1088,28 @@ static void wrap_cache_update_on_fail(enum e_future_error err, const char * what } struct cache_update_context* web_cache_update_start(struct cache_handle* handle, unsigned int thread_id, - const struct tfe_http_session * session) + const struct tfe_http_session * session, struct cache_mid **mid) { struct cache_update_context* update_ctx=NULL; struct response_freshness resp_freshness; enum cache_pending_action put_action; struct tango_cache_ctx *write_ctx=NULL; - char cont_len_str[TFE_STRING_MAX]={0}, user_tag_str[TFE_STRING_MAX]={0}; + char cont_type_str[TFE_STRING_MAX]={0}, user_tag_str[TFE_STRING_MAX]={0}; const char* value=NULL; char *tmp=NULL; int i=0, is_undefined_obj=0; size_t content_len=0; + const struct cache_param* param=NULL; + struct cache_mid* _mid=*mid; + + if(_mid!=NULL && _mid->is_using_exception_param) + { + param=_mid->param; + } + else + { + param=&(handle->default_cache_policy); + } if(session->resp->resp_spec.content_length) { sscanf(session->resp->resp_spec.content_length, "%lu", &content_len); @@ -824,19 +1118,24 @@ struct cache_update_context* web_cache_update_start(struct cache_handle* handle, put_action=tfe_cache_put_pending(session->resp, &resp_freshness); switch(put_action){ case FORBIDDEN: - ATOMIC_INC(&(handle->stat_val[STAT_CACHE_UPLOAD_FORBIDEN])); - TFE_LOG_DEBUG(handle->logger, "cache update forbiden: %s", session->req->req_spec.url); - return NULL; - case REVALIDATE: - case ALLOWED: - break; - case UNDEFINED: - if(handle->cache_undefined_obj_enabled && content_lencache_undefined_obj_min_size) + if(!(param->ignore_res_nocache || param->force_caching)) { + ATOMIC_INC(&(handle->stat_val[STAT_CACHE_UPLOAD_FORBIDEN])); TFE_LOG_DEBUG(handle->logger, "cache update forbiden: %s", session->req->req_spec.url); + return NULL; + } + break; + case REVALIDATE: + case ALLOWED: + case UNDEFINED: + if(_mid->shall_bypass + || content_len > param->max_cache_obj_size + || (!param->cache_cookied_cont && _mid->has_cookie) ) + { + ATOMIC_INC(&(handle->stat_val[STAT_CACHE_UPLOAD_BYPASS])); + TFE_LOG_DEBUG(handle->logger, "cache update bypass: %d : %s", _mid->cfg_id, session->req->req_spec.url); return NULL; } - is_undefined_obj=1; break; default: assert(0); @@ -847,15 +1146,37 @@ struct cache_update_context* web_cache_update_start(struct cache_handle* handle, ATOMIC_INC(&(handle->stat_val[STAT_CACHE_UPLOAD_ABANDON])); return NULL; } + const char* key=NULL; + size_t key_len=0; + if(param->min_use>0) + { + if(_mid->cache_key) + { + key=_mid->cache_key; + key_len=strlen(_mid->cache_key); + } + else + { + key=session->req->req_spec.url; + key_len=strlen(session->req->req_spec.url); + } + _mid->use_cnt=counting_bloom_check(handle->cache_key_bloom[thread_id], key, key_len); + + if(_mid->use_cntmin_use) + { + counting_bloom_add(handle->cache_key_bloom[thread_id], key, key_len); + return NULL; + } + } ATOMIC_INC(&(handle->stat_val[STAT_CACHE_UPLOADING])); struct tango_cache_meta_put meta; memset(&meta, 0, sizeof(meta)); meta.url=session->req->req_spec.url; - i=0; + i=0; - snprintf(cont_len_str, sizeof(cont_len_str), "content-type:%s",session->resp->resp_spec.content_type); - meta.std_hdr[i]=cont_len_str; + snprintf(cont_type_str, sizeof(cont_type_str), "content-type:%s",session->resp->resp_spec.content_type); + meta.std_hdr[i]=cont_type_str; i++; const char* etag=tfe_http_std_field_read(session->resp, TFE_HTTP_ETAG); const char* last_modified=tfe_http_std_field_read(session->resp, TFE_HTTP_LAST_MODIFIED); @@ -868,7 +1189,8 @@ struct cache_update_context* web_cache_update_start(struct cache_handle* handle, meta.usertag_len=strlen(user_tag_str)+1; } meta.put=resp_freshness; - + meta.put.timeout=MAX(param->pinning_time_sec, resp_freshness.timeout); + struct wrap_cache_put_ctx* _cache_put_ctx=ALLOC(struct wrap_cache_put_ctx, 1); _cache_put_ctx->url=tfe_strdup(session->req->req_spec.url); _cache_put_ctx->start=time(NULL); diff --git a/plugin/business/pangu-http/src/pangu_web_cache.h b/plugin/business/pangu-http/src/pangu_web_cache.h index db0a234..0f81852 100644 --- a/plugin/business/pangu-http/src/pangu_web_cache.h +++ b/plugin/business/pangu-http/src/pangu_web_cache.h @@ -2,6 +2,7 @@ #include #include #include +#include enum cache_query_status { @@ -13,7 +14,9 @@ enum cache_query_status WEB_CACHE_HIT }; struct cache_handle; -struct cache_handle* create_web_cache_handle(const char* profile_path, const char* section, struct event_base* gc_evbase, void *logger); +struct cache_handle* create_web_cache_handle(const char* profile_path, const char* section, + struct event_base* gc_evbase, Maat_feather_t feather, void *logger); + struct cached_meta { size_t content_length; @@ -25,7 +28,7 @@ const struct cached_meta* cache_query_result_read_meta(future_result_t * result) size_t cache_query_result_get_data(future_result_t * result, const unsigned char** pp_data); int web_cache_async_query(struct cache_handle* handle, unsigned int thread_id, - const struct tfe_http_half * request, struct future* f); + const struct tfe_http_half * request, struct cache_mid** mid, struct future* f); enum cache_query_result_type @@ -51,8 +54,9 @@ struct cache_mid; const struct cached_meta* cache_pending_result_read_meta(future_result_t * result); enum cache_pending_result web_cache_async_pending(struct cache_handle* handle, unsigned int thread_id, - const struct tfe_http_half * request, struct cache_mid **mid, struct future* f_revalidate); -void cache_mid_free(struct cache_mid **mid); + const struct tfe_http_half * request, struct cache_mid** mid, struct future* f_revalidate); + +void cache_mid_clear(struct cache_mid **mid); diff --git a/vendor/CMakeLists.txt b/vendor/CMakeLists.txt index 970d191..cd9923c 100644 --- a/vendor/CMakeLists.txt +++ b/vendor/CMakeLists.txt @@ -248,6 +248,7 @@ add_dependencies(libcurl-static libcurl) set_property(TARGET libcurl-static PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libcurl.a) set_property(TARGET libcurl-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) +### hiredis ExternalProject_Add(hiredis PREFIX hiredis URL ${CMAKE_CURRENT_SOURCE_DIR}/hiredis-0.14.0.zip URL_MD5 376af92277701fae52a8c917c3ce3044 @@ -264,3 +265,20 @@ add_library(hiredis-static STATIC IMPORTED GLOBAL) add_dependencies(libcurl-static hiredis) set_property(TARGET hiredis-static PROPERTY IMPORTED_LOCATION ${SOURCE_DIR}/libhiredis.a) set_property(TARGET hiredis-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${HIREDIS_INCLUDE_DIRECTORIES}) + +### dablooms +ExternalProject_Add(dablooms PREFIX dablooms + URL ${CMAKE_CURRENT_SOURCE_DIR}/dablooms-0.9.1.tar.gz + URL_MD5 0c725d3066d279299438fc9b00d492a5 + CONFIGURE_COMMAND "" + BUILD_COMMAND make + INSTALL_COMMAND make install prefix= + BUILD_IN_SOURCE 1) + +ExternalProject_Get_Property(dablooms INSTALL_DIR) +file(MAKE_DIRECTORY ${INSTALL_DIR}/include) + +add_library(libdablooms SHARED IMPORTED GLOBAL) +add_dependencies(libdablooms dablooms) +set_property(TARGET libdablooms PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libdablooms.a) +set_property(TARGET libdablooms PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) diff --git a/vendor/dablooms-0.9.1.tar.gz b/vendor/dablooms-0.9.1.tar.gz new file mode 100644 index 0000000..f24fefb Binary files /dev/null and b/vendor/dablooms-0.9.1.tar.gz differ