From 0fd3b3880eb20652f862f1a4e69dea78fb2379f1 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Sun, 11 Nov 2018 13:03:54 +0800 Subject: [PATCH] =?UTF-8?q?#68=20=E5=AF=B9=E6=8E=A5=E7=95=8C=E9=9D=A2?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=8F=AF=E4=BF=A1=E8=AF=81=E4=B9=A6=E7=9A=84?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E3=80=82=E4=BF=AE=E5=A4=8D=E9=83=A8=E5=88=86?= =?UTF-8?q?=E5=87=BD=E6=95=B0=E7=9A=84bug=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- conf/pangu/pangu_ctrl.json | 106 ++++++++++++- conf/pangu/table_info.conf | 4 +- plugin/business/pangu-http/src/pangu_http.cpp | 5 +- .../pangu-http/src/pangu_web_cache.cpp | 142 +++++++++++------- 4 files changed, 196 insertions(+), 61 deletions(-) diff --git a/conf/pangu/pangu_ctrl.json b/conf/pangu/pangu_ctrl.json index fa3587a..9f20c5d 100644 --- a/conf/pangu/pangu_ctrl.json +++ b/conf/pangu/pangu_ctrl.json @@ -1,4 +1,6 @@ { + "compile_table": "PXY_CTRL_COMPILE", + "group_table": "PXY_CTRL_GROUP", "rules": [ { "compile_id": 125, @@ -52,7 +54,7 @@ } ] }, - { + { "compile_id": 127, "service": 1, "action": 80, @@ -77,7 +79,7 @@ ] } ] - }, + }, { "compile_id": 130, "service": 1, @@ -103,7 +105,7 @@ ] } ] - }, + }, { "compile_id": 131, "service": 1, @@ -129,7 +131,101 @@ ] } ] + }, + { + "compile_id": 133, + "service": 1, + "action": 48, + "do_blacklist": 1, + "do_log": 1, + "effective_range": 0, + "user_region": "{\"cache_key\":{\"ignore_qs\":[\"sqp\",\"rs\"],\"cookie\":\"jsession_id\"},\"no_revalidate\":0,\"cache_dyn_url\":1,\"cache_cookied_cont\":1,\"ignore_srv_nocache\":1,\"ignore_cli_nocache\":1,\"force_caching\":1,\"min_use\":1,\"pinning_time\":\"60m\",\"max_cache_size\":\"1t\",\"inactive_time\":\"1h\"}", + "is_valid": "yes", + "table_name":"PXY_CACHE_COMPILE", + "groups": [ + { + "regions": [ + { + "table_name": "PXY_CACHE_HTTP_URL", + "table_type": "string", + "table_content": { + "keywords": "i.ytimg.com", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 134, + "service": 1, + "action": 128, + "do_blacklist": 1, + "do_log": 1, + "effective_range": 0, + "user_region": "{}", + "is_valid": "yes", + "table_name":"PXY_CACHE_COMPILE", + "groups": [ + { + "regions": [ + { + "table_name": "PXY_CACHE_HTTP_URL", + "table_type": "string", + "table_content": { + "keywords": "mesalab.com", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 135, + "service": 1, + "action": 48, + "do_blacklist": 1, + "do_log": 1, + "effective_range": 0, + "user_region": "{\"cache_key\":{\"cookie\":\"MUID\"},\"cache_dyn_url\":1,\"cache_cookied_cont\":1}", + "is_valid": "yes", + "table_name":"PXY_CACHE_COMPILE", + "groups": [ + { + "regions": [ + { + "table_name": "PXY_CACHE_HTTP_URL", + "table_type": "string", + "table_content": { + "keywords": "cn.bing.com", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + } + ], + "plugin_table": [ + { + "table_name": "PXY_OBJ_TRUSTED_CA_CERT", + "table_content": [ + "1\tDigiCertSHA2SecureServerCA\t./pangu_files/DigiCertSHA2SecureServerCA.pem\t1" + ] + }, + { + "table_name": "PXY_OBJ_TRUSTED_CA_CRL", + "table_content": [ + "1\t1\t./pangu_files/ssca-sha2-g5.crl\t1" + ] } - - ] + ] } diff --git a/conf/pangu/table_info.conf b/conf/pangu/table_info.conf index 7c5a34d..5c196c6 100644 --- a/conf/pangu/table_info.conf +++ b/conf/pangu/table_info.conf @@ -20,9 +20,9 @@ 2 PXY_CTRL_IP ip --- 3 PXY_CTRL_HTTP_URL expr UTF8 GBK/UNICODE/UTF8/url_encode_gb2312/url_encode_utf8 yes 0 quickoff 4 PXY_CTRL_HTTP_REQ_HDR expr_plus UTF8 UTF8 yes 0 quickoff -5 PXY_CTRL_HTTP_REQ_BODY expr UTF8 GBK/BIG5/UNICODE/UTF8 yes 128 quickoff +5 PXY_CTRL_HTTP_REQ_BODY expr UTF8 GBK/UNICODE/UTF8 yes 128 quickoff 6 PXY_CTRL_HTTP_RES_HDR expr_plus UTF8 UTF8 UTF8 yes 0 quickoff -7 PXY_CTRL_HTTP_RES_BODY expr UTF8 GBK/BIG5/UNICODE/UTF8 yes 128 quickoff +7 PXY_CTRL_HTTP_RES_BODY expr UTF8 GBK/UNICODE/UTF8 yes 128 quickoff 8 PXY_CACHE_COMPILE compile escape -- 9 PXY_CACHE_GROUP group -- 10 PXY_CACHE_HTTP_URL expr UTF8 UTF8 yes 0 quickoff diff --git a/plugin/business/pangu-http/src/pangu_http.cpp b/plugin/business/pangu-http/src/pangu_http.cpp index 38ae4d9..f455b86 100644 --- a/plugin/business/pangu-http/src/pangu_http.cpp +++ b/plugin/business/pangu-http/src/pangu_http.cpp @@ -133,7 +133,8 @@ static Maat_feather_t create_maat_feather(const char * profile, const char * sec case MAAT_INPUT_JSON: Maat_set_feather_opt(target, MAAT_OPT_JSON_FILE_PATH, json_cfg_file, strlen(json_cfg_file) + 1); break; - case MAAT_INPUT_REDIS: Maat_set_feather_opt(target, MAAT_OPT_REDIS_IP, redis_server, strlen(redis_server) + 1); + case MAAT_INPUT_REDIS: + Maat_set_feather_opt(target, MAAT_OPT_REDIS_IP, redis_server, strlen(redis_server) + 1); Maat_set_feather_opt(target, MAAT_OPT_REDIS_PORT, &redis_port, sizeof(redis_port)); Maat_set_feather_opt(target, MAAT_OPT_REDIS_INDEX, &redis_db_idx, sizeof(redis_db_idx)); break; @@ -145,6 +146,8 @@ static Maat_feather_t create_maat_feather(const char * profile, const char * sec break; } + + Maat_set_feather_opt(target, MAAT_OPT_FOREIGN_CONT_DIR, "./pangu_files", strlen("./pangu_files")+1); if (maat_stat_on) { Maat_set_feather_opt(target, MAAT_OPT_STAT_FILE_PATH, maat_stat_file, strlen(maat_stat_file) + 1); diff --git a/plugin/business/pangu-http/src/pangu_web_cache.cpp b/plugin/business/pangu-http/src/pangu_web_cache.cpp index 1e01395..129ffe1 100644 --- a/plugin/business/pangu-http/src/pangu_web_cache.cpp +++ b/plugin/business/pangu-http/src/pangu_web_cache.cpp @@ -64,7 +64,7 @@ struct cache_key_descr struct cache_param { int ref_cnt; - struct cache_key_descr cache_key; + struct cache_key_descr key_descr; char no_revalidate; char cache_dyn_url; @@ -80,6 +80,14 @@ struct cache_param long max_cache_obj_size; pthread_mutex_t lock; }; +struct cache_bloom +{ + int thread_id; + size_t size; + double error_rate; + char filename[TFE_PATH_MAX]; + counting_bloom_t *bloom; +}; struct cache_handle { unsigned int thread_count; @@ -105,9 +113,10 @@ struct cache_handle int cache_param_idx; int table_url_constraint; int table_cookie_constraint; - size_t cache_key_bloom_size; + int cache_key_bloom_life; - counting_bloom_t **cache_key_bloom; + size_t cache_key_bloom_size; + struct cache_bloom *cache_key_bloom; void* logger; }; struct cache_update_context @@ -277,7 +286,7 @@ time_t time_unit_sec(const char* str) value*=3600; break; case 'd': - value*=24*3600; + value*=((size_t)24*3600); break; default: break; @@ -296,18 +305,22 @@ size_t storage_unit_byte(const char* str) value*=1024; break; case 'm': - value*=1024*1024; + value*=((size_t)1024*1024); break; case 'g': - value*=1024*1024*1024; + value*=((size_t)1024*1024*1024); break; case 't': if(value<1024) { #pragma GCC diagnostic ignored "-Woverflow" - value*=1024*1024*1024*1024; + value*=((size_t)1024*1024*1024*1024); } + else //maximum 1PB + { + value=(size_t)1024*(1024*1024*1024*1024); + } break; default: break; @@ -374,28 +387,38 @@ char* get_cache_key(const struct tfe_http_half * request, const struct cache_key char cookie_val[256]={0}; //most 256 bytes for cookie key size_t key_size=strlen(url)+sizeof(cookie_val); char* cache_key=ALLOC(char, key_size); - char* query_string=strchr(url, '?'); - if(query_string!=NULL && desc->qs_num>0) + char* query_string=NULL; + + if(desc->qs_num>0) { - query_string++; - for (token = url; ; token= NULL) + query_string=strchr(url, '?'); + if(query_string!=NULL) { - sub_token= strtok_r(token,"&", &saveptr); - if (sub_token == NULL) - break; - shall_ignore=0; - for(i=0; iqs_num; i++) + strncat(cache_key, url, MIN(query_string-url,key_size)); + query_string++; + for (token = query_string; ; token= NULL) { - if(0==strncasecmp(sub_token, desc->ignore_qs[i], strlen(desc->ignore_qs[i]))) - { - shall_ignore=1; + sub_token= strtok_r(token,"&", &saveptr); + if (sub_token == NULL) break; + shall_ignore=0; + for(i=0; iqs_num; i++) + { + if(0==strncasecmp(sub_token, desc->ignore_qs[i], strlen(desc->ignore_qs[i]))) + { + shall_ignore=1; + break; + } + } + if(!shall_ignore) + { + strncat(cache_key, sub_token, key_size); } } - if(!shall_ignore) - { - strncat(cache_key, sub_token, key_size); - } + } + else + { + strncat(cache_key, url, key_size); } } else @@ -411,7 +434,7 @@ char* get_cache_key(const struct tfe_http_half * request, const struct cache_key } } FREE(&(url)); - + return cache_key; } void cache_param_new(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, @@ -440,23 +463,23 @@ void cache_param_new(int idx, const struct Maat_rule_t* rule, const char* srv_de key_desc=cJSON_GetObjectItem(json,"cache_key"); if(key_desc && key_desc->type==cJSON_Object) { - param->cache_key.is_not_empty=1; - qs=cJSON_GetObjectItem(json,"ignore_qs"); + param->key_descr.is_not_empty=1; + qs=cJSON_GetObjectItem(key_desc,"ignore_qs"); if(qs && qs->type==cJSON_Array) { - param->cache_key.qs_num=cJSON_GetArraySize(qs); - param->cache_key.ignore_qs=ALLOC(char*, param->cache_key.qs_num); - for(i=0; icache_key.qs_num; i++) + param->key_descr.qs_num=cJSON_GetArraySize(qs); + param->key_descr.ignore_qs=ALLOC(char*, param->key_descr.qs_num); + for(i=0; ikey_descr.qs_num; i++) { - item=cJSON_GetArrayItem(item, i); - len=strlen(qs->valuestring)+2; - param->cache_key.ignore_qs[i]=ALLOC(char, len); - strncat(param->cache_key.ignore_qs[i], qs->valuestring, len); - strncat(param->cache_key.ignore_qs[i], "=", len); + item=cJSON_GetArrayItem(qs, i); + len=strlen(item->valuestring)+2; + param->key_descr.ignore_qs[i]=ALLOC(char, len); + strncat(param->key_descr.ignore_qs[i], item->valuestring, len); + strncat(param->key_descr.ignore_qs[i], "=", len); } } item=cJSON_GetObjectItem(key_desc,"cookie"); - if(item && item->type==cJSON_String) param->cache_key.include_cookie=tfe_strdup(param->cache_key.include_cookie); + if(item && item->type==cJSON_String) param->key_descr.include_cookie=tfe_strdup(param->key_descr.include_cookie); } @@ -496,6 +519,7 @@ void cache_param_new(int idx, const struct Maat_rule_t* rule, const char* srv_de if(item && item->type==cJSON_String) param->max_cache_obj_size=storage_unit_byte(item->valuestring); cJSON_Delete(json); + *ad=param; return; } void cache_param_free(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, MAAT_RULE_EX_DATA* ad, long argl, void *argp) @@ -515,12 +539,12 @@ void cache_param_free(int idx, const struct Maat_rule_t* rule, const char* srv_d } pthread_mutex_unlock(&(param->lock)); pthread_mutex_destroy(&(param->lock)); - for(i=0; icache_key.qs_num; i++) + for(i=0; ikey_descr.qs_num; i++) { - FREE(&(param->cache_key.ignore_qs[i])); + FREE(&(param->key_descr.ignore_qs[i])); } - FREE(&(param->cache_key.ignore_qs)); - FREE(&(param->cache_key.include_cookie)); + FREE(&(param->key_descr.ignore_qs)); + FREE(&(param->key_descr.include_cookie)); FREE(&(param)); return; } @@ -536,11 +560,12 @@ void cache_param_dup(int idx, MAAT_RULE_EX_DATA *to, MAAT_RULE_EX_DATA *from, lo static void cache_key_bloom_gc_cb(evutil_socket_t fd, short what, void * arg) { - counting_bloom_t* old_bloom=*((counting_bloom_t**)arg), *new_bloom=NULL; + struct cache_bloom* p_bloom= (struct cache_bloom*) arg; + counting_bloom_t* new_bloom=NULL; - new_bloom=new_counting_bloom(old_bloom->capacity, old_bloom->error_rate, NULL); - free_counting_bloom(old_bloom); - *((counting_bloom_t**)arg)=old_bloom; + new_bloom=new_counting_bloom(p_bloom->size, p_bloom->error_rate, p_bloom->filename); + free_counting_bloom(p_bloom->bloom); + p_bloom->bloom=new_bloom; return; } @@ -553,7 +578,8 @@ struct cache_handle* create_web_cache_handle(const char* profile_path, const cha cache->logger=logger; cache->thread_count=tfe_proxy_get_work_thread_count(); cache->clients=ALLOC(struct tango_cache_instance *, cache->thread_count); - cache->cache_key_bloom=ALLOC(counting_bloom_t*, cache->thread_count); + cache->cache_key_bloom=ALLOC(struct cache_bloom, cache->thread_count); + struct cache_bloom* p_bloom=NULL; MESA_load_profile_int_def(profile_path, section, "cache_policy_enabled", &(cache->cache_policy_enabled), 1); @@ -562,14 +588,24 @@ struct cache_handle* create_web_cache_handle(const char* profile_path, const cha (int*)&(cache->cache_key_bloom_size), 16*1000*1000); MESA_load_profile_int_def(profile_path, section, "cache_key_bloom_life", &(cache->cache_key_bloom_life), 30*60); + char bloom_filename[TFE_PATH_MAX]{0}; struct timeval gc_refresh_delay = {cache->cache_key_bloom_life, 0}; int i=0; for(i=0; ithread_count; i++) { if(cache->cache_policy_enabled) { - cache->cache_key_bloom[i]=new_counting_bloom(cache->cache_key_bloom_size, 0.01, NULL); - ev = event_new(tfe_proxy_get_work_thread_evbase(i), -1, EV_PERSIST, cache_key_bloom_gc_cb, &(cache->cache_key_bloom[i])); + p_bloom=cache->cache_key_bloom+i; + p_bloom->thread_id=i; + p_bloom->size=cache->cache_key_bloom_size; + p_bloom->error_rate=0.01; + snprintf(p_bloom->filename, sizeof(p_bloom->filename), "/tmp/pangu_cache_blooms.%d", i); + p_bloom->bloom=new_counting_bloom(p_bloom->size, p_bloom->error_rate, p_bloom->filename); + if(p_bloom->bloom==NULL) + { + goto error_out; + } + ev = event_new(tfe_proxy_get_work_thread_evbase(i), -1, EV_PERSIST, cache_key_bloom_gc_cb, p_bloom); evtimer_add(ev, &gc_refresh_delay); } @@ -591,7 +627,7 @@ struct cache_handle* create_web_cache_handle(const char* profile_path, const cha cache->gc_evbase=gc_evbase; - cache->default_cache_policy.cache_key.qs_num=0; + cache->default_cache_policy.key_descr.qs_num=0; cache->default_cache_policy.no_revalidate=0; cache->default_cache_policy.cache_dyn_url=0; cache->default_cache_policy.cache_cookied_cont=0; @@ -932,7 +968,7 @@ enum cache_pending_result web_cache_async_pending(struct cache_handle* handle, u _mid->cfg_id=cache_policy.config_id; } if(_mid->shall_bypass || - (!param->cache_dyn_url && _mid->is_dyn_url && param->cache_key.qs_num==0) || + (!param->cache_dyn_url && _mid->is_dyn_url && param->key_descr.qs_num==0) || (param->cache_cookied_cont && _mid->has_cookie) ) { _mid->result=PENDING_RESULT_FOBIDDEN; @@ -988,9 +1024,9 @@ enum cache_pending_result web_cache_async_pending(struct cache_handle* handle, u struct tango_cache_meta_get meta; memset(&meta, 0, sizeof(meta)); - if(param->cache_key.is_not_empty) + if(param->key_descr.is_not_empty) { - _mid->cache_key=get_cache_key(request, &(param->cache_key)); + _mid->cache_key=get_cache_key(request, &(param->key_descr)); meta.url = _mid->cache_key; } else @@ -1083,7 +1119,7 @@ static void wrap_cache_update_on_succ(future_result_t * result, void * user) static void wrap_cache_update_on_fail(enum e_future_error err, const char * what, void * user) { struct wrap_cache_put_ctx* ctx=(struct wrap_cache_put_ctx*)user; - TFE_LOG_DEBUG(ctx->ref_handle->logger, "cache upload failed: %s elapse: %d", ctx->url, time(NULL)-ctx->start); + TFE_LOG_DEBUG(ctx->ref_handle->logger, "cache upload failed: %s %s lapse: %d", ctx->url, what, time(NULL)-ctx->start); wrap_cache_put_ctx_free(ctx); } @@ -1160,11 +1196,11 @@ struct cache_update_context* web_cache_update_start(struct cache_handle* handle, key=session->req->req_spec.url; key_len=strlen(session->req->req_spec.url); } - _mid->use_cnt=counting_bloom_check(handle->cache_key_bloom[thread_id], key, key_len); + _mid->use_cnt=counting_bloom_check(handle->cache_key_bloom[thread_id].bloom, key, key_len); if(_mid->use_cntmin_use) { - counting_bloom_add(handle->cache_key_bloom[thread_id], key, key_len); + counting_bloom_add(handle->cache_key_bloom[thread_id].bloom, key, key_len); return NULL; } }