gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

#68 对接界面增加可信证书的接口。修复部分函数的bug。

Browse Source
This commit is contained in:
zhengchao
2018-11-11 13:03:54 +08:00
parent 62d9eb0bef
commit 0fd3b3880e
4 changed files with 196 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

106
conf/pangu/pangu_ctrl.json
View File

@@ -1,4 +1,6 @@
{ {
"compile_table": "PXY_CTRL_COMPILE",
"group_table": "PXY_CTRL_GROUP",
"rules": [ "rules": [
{ {
"compile_id": 125, "compile_id": 125,
@@ -52,7 +54,7 @@
} }
] ]
}, },
{ {
"compile_id": 127, "compile_id": 127,
"service": 1, "service": 1,
"action": 80, "action": 80,
@@ -77,7 +79,7 @@
] ]
} }
] ]
}, },
{ {
"compile_id": 130, "compile_id": 130,
"service": 1, "service": 1,
@@ -103,7 +105,7 @@
] ]
} }
] ]
}, },
{ {
"compile_id": 131, "compile_id": 131,
"service": 1, "service": 1,
@@ -129,7 +131,101 @@
] ]
} }
] ]
},
{
"compile_id": 133,
"service": 1,
"action": 48,
"do_blacklist": 1,
"do_log": 1,
"effective_range": 0,
"user_region": "{\"cache_key\":{\"ignore_qs\":[\"sqp\",\"rs\"],\"cookie\":\"jsession_id\"},\"no_revalidate\":0,\"cache_dyn_url\":1,\"cache_cookied_cont\":1,\"ignore_srv_nocache\":1,\"ignore_cli_nocache\":1,\"force_caching\":1,\"min_use\":1,\"pinning_time\":\"60m\",\"max_cache_size\":\"1t\",\"inactive_time\":\"1h\"}",
"is_valid": "yes",
"table_name":"PXY_CACHE_COMPILE",
"groups": [
{
"regions": [
{
"table_name": "PXY_CACHE_HTTP_URL",
"table_type": "string",
"table_content": {
"keywords": "i.ytimg.com",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
},
{
"compile_id": 134,
"service": 1,
"action": 128,
"do_blacklist": 1,
"do_log": 1,
"effective_range": 0,
"user_region": "{}",
"is_valid": "yes",
"table_name":"PXY_CACHE_COMPILE",
"groups": [
{
"regions": [
{
"table_name": "PXY_CACHE_HTTP_URL",
"table_type": "string",
"table_content": {
"keywords": "mesalab.com",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
},
{
"compile_id": 135,
"service": 1,
"action": 48,
"do_blacklist": 1,
"do_log": 1,
"effective_range": 0,
"user_region": "{\"cache_key\":{\"cookie\":\"MUID\"},\"cache_dyn_url\":1,\"cache_cookied_cont\":1}",
"is_valid": "yes",
"table_name":"PXY_CACHE_COMPILE",
"groups": [
{
"regions": [
{
"table_name": "PXY_CACHE_HTTP_URL",
"table_type": "string",
"table_content": {
"keywords": "cn.bing.com",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
}
],
"plugin_table": [
{
"table_name": "PXY_OBJ_TRUSTED_CA_CERT",
"table_content": [
"1\tDigiCertSHA2SecureServerCA\t./pangu_files/DigiCertSHA2SecureServerCA.pem\t1"
]
},
{
"table_name": "PXY_OBJ_TRUSTED_CA_CRL",
"table_content": [
"1\t1\t./pangu_files/ssca-sha2-g5.crl\t1"
]
} }
]
]
} }

4
conf/pangu/table_info.conf
View File

@@ -20,9 +20,9 @@
2 PXY_CTRL_IP ip --- 2 PXY_CTRL_IP ip ---
3 PXY_CTRL_HTTP_URL expr UTF8 GBK/UNICODE/UTF8/url_encode_gb2312/url_encode_utf8 yes 0 quickoff 3 PXY_CTRL_HTTP_URL expr UTF8 GBK/UNICODE/UTF8/url_encode_gb2312/url_encode_utf8 yes 0 quickoff
4 PXY_CTRL_HTTP_REQ_HDR expr_plus UTF8 UTF8 yes 0 quickoff 4 PXY_CTRL_HTTP_REQ_HDR expr_plus UTF8 UTF8 yes 0 quickoff
5 PXY_CTRL_HTTP_REQ_BODY expr UTF8 GBK/BIG5/UNICODE/UTF8 yes 128 quickoff 5 PXY_CTRL_HTTP_REQ_BODY expr UTF8 GBK/UNICODE/UTF8 yes 128 quickoff
6 PXY_CTRL_HTTP_RES_HDR expr_plus UTF8 UTF8 UTF8 yes 0 quickoff 6 PXY_CTRL_HTTP_RES_HDR expr_plus UTF8 UTF8 UTF8 yes 0 quickoff
7 PXY_CTRL_HTTP_RES_BODY expr UTF8 GBK/BIG5/UNICODE/UTF8 yes 128 quickoff 7 PXY_CTRL_HTTP_RES_BODY expr UTF8 GBK/UNICODE/UTF8 yes 128 quickoff
8 PXY_CACHE_COMPILE compile escape -- 8 PXY_CACHE_COMPILE compile escape --
9 PXY_CACHE_GROUP group -- 9 PXY_CACHE_GROUP group --
10 PXY_CACHE_HTTP_URL expr UTF8 UTF8 yes 0 quickoff 10 PXY_CACHE_HTTP_URL expr UTF8 UTF8 yes 0 quickoff

5
plugin/business/pangu-http/src/pangu_http.cpp
View File

@@ -133,7 +133,8 @@ static Maat_feather_t create_maat_feather(const char * profile, const char * sec
case MAAT_INPUT_JSON: case MAAT_INPUT_JSON:
Maat_set_feather_opt(target, MAAT_OPT_JSON_FILE_PATH, json_cfg_file, strlen(json_cfg_file) + 1); Maat_set_feather_opt(target, MAAT_OPT_JSON_FILE_PATH, json_cfg_file, strlen(json_cfg_file) + 1);
break; break;
case MAAT_INPUT_REDIS: Maat_set_feather_opt(target, MAAT_OPT_REDIS_IP, redis_server, strlen(redis_server) + 1); case MAAT_INPUT_REDIS:
Maat_set_feather_opt(target, MAAT_OPT_REDIS_IP, redis_server, strlen(redis_server) + 1);
Maat_set_feather_opt(target, MAAT_OPT_REDIS_PORT, &redis_port, sizeof(redis_port)); Maat_set_feather_opt(target, MAAT_OPT_REDIS_PORT, &redis_port, sizeof(redis_port));
Maat_set_feather_opt(target, MAAT_OPT_REDIS_INDEX, &redis_db_idx, sizeof(redis_db_idx)); Maat_set_feather_opt(target, MAAT_OPT_REDIS_INDEX, &redis_db_idx, sizeof(redis_db_idx));
break; break;
@@ -145,6 +146,8 @@ static Maat_feather_t create_maat_feather(const char * profile, const char * sec
break; break;
} }
Maat_set_feather_opt(target, MAAT_OPT_FOREIGN_CONT_DIR, "./pangu_files", strlen("./pangu_files")+1);
if (maat_stat_on) if (maat_stat_on)
{ {
Maat_set_feather_opt(target, MAAT_OPT_STAT_FILE_PATH, maat_stat_file, strlen(maat_stat_file) + 1); Maat_set_feather_opt(target, MAAT_OPT_STAT_FILE_PATH, maat_stat_file, strlen(maat_stat_file) + 1);

142
plugin/business/pangu-http/src/pangu_web_cache.cpp
View File

@@ -64,7 +64,7 @@ struct cache_key_descr
struct cache_param struct cache_param
{ {
int ref_cnt; int ref_cnt;
struct cache_key_descr cache_key; struct cache_key_descr key_descr;
char no_revalidate; char no_revalidate;
char cache_dyn_url; char cache_dyn_url;
@@ -80,6 +80,14 @@ struct cache_param
long max_cache_obj_size; long max_cache_obj_size;
pthread_mutex_t lock; pthread_mutex_t lock;
}; };
struct cache_bloom
{
int thread_id;
size_t size;
double error_rate;
char filename[TFE_PATH_MAX];
counting_bloom_t *bloom;
};
struct cache_handle struct cache_handle
{ {
unsigned int thread_count; unsigned int thread_count;
@@ -105,9 +113,10 @@ struct cache_handle
int cache_param_idx; int cache_param_idx;
int table_url_constraint; int table_url_constraint;
int table_cookie_constraint; int table_cookie_constraint;
size_t cache_key_bloom_size;
int cache_key_bloom_life; int cache_key_bloom_life;
counting_bloom_t **cache_key_bloom; size_t cache_key_bloom_size;
struct cache_bloom *cache_key_bloom;
void* logger; void* logger;
}; };
struct cache_update_context struct cache_update_context
@@ -277,7 +286,7 @@ time_t time_unit_sec(const char* str)
value*=3600; value*=3600;
break; break;
case 'd': case 'd':
value*=24*3600; value*=((size_t)24*3600);
break; break;
default: default:
break; break;
@@ -296,18 +305,22 @@ size_t storage_unit_byte(const char* str)
value*=1024; value*=1024;
break; break;
case 'm': case 'm':
value*=1024*1024; value*=((size_t)1024*1024);
break; break;
case 'g': case 'g':
value*=1024*1024*1024; value*=((size_t)1024*1024*1024);
break; break;
case 't': case 't':
if(value<1024) if(value<1024)
{ {
#pragma GCC diagnostic ignored "-Woverflow" #pragma GCC diagnostic ignored "-Woverflow"
value*=1024*1024*1024*1024; value*=((size_t)1024*1024*1024*1024);
} }
else //maximum 1PB
{
value=(size_t)1024*(1024*1024*1024*1024);
}
break; break;
default: default:
break; break;
@@ -374,28 +387,38 @@ char* get_cache_key(const struct tfe_http_half * request, const struct cache_key
char cookie_val[256]={0}; //most 256 bytes for cookie key char cookie_val[256]={0}; //most 256 bytes for cookie key
size_t key_size=strlen(url)+sizeof(cookie_val); size_t key_size=strlen(url)+sizeof(cookie_val);
char* cache_key=ALLOC(char, key_size); char* cache_key=ALLOC(char, key_size);
char* query_string=strchr(url, '?'); char* query_string=NULL;
if(query_string!=NULL && desc->qs_num>0)
if(desc->qs_num>0)
{ {
query_string++; query_string=strchr(url, '?');
for (token = url; ; token= NULL) if(query_string!=NULL)
{ {
sub_token= strtok_r(token,"&", &saveptr); strncat(cache_key, url, MIN(query_string-url,key_size));
if (sub_token == NULL) query_string++;
break; for (token = query_string; ; token= NULL)
shall_ignore=0;
for(i=0; i<desc->qs_num; i++)
{ {
if(0==strncasecmp(sub_token, desc->ignore_qs[i], strlen(desc->ignore_qs[i]))) sub_token= strtok_r(token,"&", &saveptr);
{ if (sub_token == NULL)
shall_ignore=1;
break; break;
shall_ignore=0;
for(i=0; i<desc->qs_num; i++)
{
if(0==strncasecmp(sub_token, desc->ignore_qs[i], strlen(desc->ignore_qs[i])))
{
shall_ignore=1;
break;
}
}
if(!shall_ignore)
{
strncat(cache_key, sub_token, key_size);
} }
} }
if(!shall_ignore) }
{ else
strncat(cache_key, sub_token, key_size); {
} strncat(cache_key, url, key_size);
} }
} }
else else
@@ -411,7 +434,7 @@ char* get_cache_key(const struct tfe_http_half * request, const struct cache_key
} }
} }
FREE(&(url)); FREE(&(url));
return cache_key;
} }
void cache_param_new(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, void cache_param_new(int idx, const struct Maat_rule_t* rule, const char* srv_def_large,
@@ -440,23 +463,23 @@ void cache_param_new(int idx, const struct Maat_rule_t* rule, const char* srv_de
key_desc=cJSON_GetObjectItem(json,"cache_key"); key_desc=cJSON_GetObjectItem(json,"cache_key");
if(key_desc && key_desc->type==cJSON_Object) if(key_desc && key_desc->type==cJSON_Object)
{ {
param->cache_key.is_not_empty=1; param->key_descr.is_not_empty=1;
qs=cJSON_GetObjectItem(json,"ignore_qs"); qs=cJSON_GetObjectItem(key_desc,"ignore_qs");
if(qs && qs->type==cJSON_Array) if(qs && qs->type==cJSON_Array)
{ {
param->cache_key.qs_num=cJSON_GetArraySize(qs); param->key_descr.qs_num=cJSON_GetArraySize(qs);
param->cache_key.ignore_qs=ALLOC(char*, param->cache_key.qs_num); param->key_descr.ignore_qs=ALLOC(char*, param->key_descr.qs_num);
for(i=0; i<param->cache_key.qs_num; i++) for(i=0; i<param->key_descr.qs_num; i++)
{ {
item=cJSON_GetArrayItem(item, i); item=cJSON_GetArrayItem(qs, i);
len=strlen(qs->valuestring)+2; len=strlen(item->valuestring)+2;
param->cache_key.ignore_qs[i]=ALLOC(char, len); param->key_descr.ignore_qs[i]=ALLOC(char, len);
strncat(param->cache_key.ignore_qs[i], qs->valuestring, len); strncat(param->key_descr.ignore_qs[i], item->valuestring, len);
strncat(param->cache_key.ignore_qs[i], "=", len); strncat(param->key_descr.ignore_qs[i], "=", len);
} }
} }
item=cJSON_GetObjectItem(key_desc,"cookie"); item=cJSON_GetObjectItem(key_desc,"cookie");
if(item && item->type==cJSON_String) param->cache_key.include_cookie=tfe_strdup(param->cache_key.include_cookie); if(item && item->type==cJSON_String) param->key_descr.include_cookie=tfe_strdup(param->key_descr.include_cookie);
} }
@@ -496,6 +519,7 @@ void cache_param_new(int idx, const struct Maat_rule_t* rule, const char* srv_de
if(item && item->type==cJSON_String) param->max_cache_obj_size=storage_unit_byte(item->valuestring); if(item && item->type==cJSON_String) param->max_cache_obj_size=storage_unit_byte(item->valuestring);
cJSON_Delete(json); cJSON_Delete(json);
*ad=param;
return; return;
} }
void cache_param_free(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, MAAT_RULE_EX_DATA* ad, long argl, void *argp) void cache_param_free(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, MAAT_RULE_EX_DATA* ad, long argl, void *argp)
@@ -515,12 +539,12 @@ void cache_param_free(int idx, const struct Maat_rule_t* rule, const char* srv_d
} }
pthread_mutex_unlock(&(param->lock)); pthread_mutex_unlock(&(param->lock));
pthread_mutex_destroy(&(param->lock)); pthread_mutex_destroy(&(param->lock));
for(i=0; i<param->cache_key.qs_num; i++) for(i=0; i<param->key_descr.qs_num; i++)
{ {
FREE(&(param->cache_key.ignore_qs[i])); FREE(&(param->key_descr.ignore_qs[i]));
} }
FREE(&(param->cache_key.ignore_qs)); FREE(&(param->key_descr.ignore_qs));
FREE(&(param->cache_key.include_cookie)); FREE(&(param->key_descr.include_cookie));
FREE(&(param)); FREE(&(param));
return; return;
} }
@@ -536,11 +560,12 @@ void cache_param_dup(int idx, MAAT_RULE_EX_DATA *to, MAAT_RULE_EX_DATA *from, lo
static void cache_key_bloom_gc_cb(evutil_socket_t fd, short what, void * arg) static void cache_key_bloom_gc_cb(evutil_socket_t fd, short what, void * arg)
{ {
counting_bloom_t* old_bloom=*((counting_bloom_t**)arg), *new_bloom=NULL; struct cache_bloom* p_bloom= (struct cache_bloom*) arg;
counting_bloom_t* new_bloom=NULL;
new_bloom=new_counting_bloom(old_bloom->capacity, old_bloom->error_rate, NULL); new_bloom=new_counting_bloom(p_bloom->size, p_bloom->error_rate, p_bloom->filename);
free_counting_bloom(old_bloom); free_counting_bloom(p_bloom->bloom);
*((counting_bloom_t**)arg)=old_bloom; p_bloom->bloom=new_bloom;
return; return;
} }
@@ -553,7 +578,8 @@ struct cache_handle* create_web_cache_handle(const char* profile_path, const cha
cache->logger=logger; cache->logger=logger;
cache->thread_count=tfe_proxy_get_work_thread_count(); cache->thread_count=tfe_proxy_get_work_thread_count();
cache->clients=ALLOC(struct tango_cache_instance *, cache->thread_count); cache->clients=ALLOC(struct tango_cache_instance *, cache->thread_count);
cache->cache_key_bloom=ALLOC(counting_bloom_t*, cache->thread_count); cache->cache_key_bloom=ALLOC(struct cache_bloom, cache->thread_count);
struct cache_bloom* p_bloom=NULL;
MESA_load_profile_int_def(profile_path, section, "cache_policy_enabled", MESA_load_profile_int_def(profile_path, section, "cache_policy_enabled",
&(cache->cache_policy_enabled), 1); &(cache->cache_policy_enabled), 1);
@@ -562,14 +588,24 @@ struct cache_handle* create_web_cache_handle(const char* profile_path, const cha
(int*)&(cache->cache_key_bloom_size), 16*1000*1000); (int*)&(cache->cache_key_bloom_size), 16*1000*1000);
MESA_load_profile_int_def(profile_path, section, "cache_key_bloom_life", MESA_load_profile_int_def(profile_path, section, "cache_key_bloom_life",
&(cache->cache_key_bloom_life), 30*60); &(cache->cache_key_bloom_life), 30*60);
char bloom_filename[TFE_PATH_MAX]{0};
struct timeval gc_refresh_delay = {cache->cache_key_bloom_life, 0}; struct timeval gc_refresh_delay = {cache->cache_key_bloom_life, 0};
int i=0; int i=0;
for(i=0; i<cache->thread_count; i++) for(i=0; i<cache->thread_count; i++)
{ {
if(cache->cache_policy_enabled) if(cache->cache_policy_enabled)
{ {
cache->cache_key_bloom[i]=new_counting_bloom(cache->cache_key_bloom_size, 0.01, NULL); p_bloom=cache->cache_key_bloom+i;
ev = event_new(tfe_proxy_get_work_thread_evbase(i), -1, EV_PERSIST, cache_key_bloom_gc_cb, &(cache->cache_key_bloom[i])); p_bloom->thread_id=i;
p_bloom->size=cache->cache_key_bloom_size;
p_bloom->error_rate=0.01;
snprintf(p_bloom->filename, sizeof(p_bloom->filename), "/tmp/pangu_cache_blooms.%d", i);
p_bloom->bloom=new_counting_bloom(p_bloom->size, p_bloom->error_rate, p_bloom->filename);
if(p_bloom->bloom==NULL)
{
goto error_out;
}
ev = event_new(tfe_proxy_get_work_thread_evbase(i), -1, EV_PERSIST, cache_key_bloom_gc_cb, p_bloom);
evtimer_add(ev, &gc_refresh_delay); evtimer_add(ev, &gc_refresh_delay);
} }
@@ -591,7 +627,7 @@ struct cache_handle* create_web_cache_handle(const char* profile_path, const cha
cache->gc_evbase=gc_evbase; cache->gc_evbase=gc_evbase;
cache->default_cache_policy.cache_key.qs_num=0; cache->default_cache_policy.key_descr.qs_num=0;
cache->default_cache_policy.no_revalidate=0; cache->default_cache_policy.no_revalidate=0;
cache->default_cache_policy.cache_dyn_url=0; cache->default_cache_policy.cache_dyn_url=0;
cache->default_cache_policy.cache_cookied_cont=0; cache->default_cache_policy.cache_cookied_cont=0;
@@ -932,7 +968,7 @@ enum cache_pending_result web_cache_async_pending(struct cache_handle* handle, u
_mid->cfg_id=cache_policy.config_id; _mid->cfg_id=cache_policy.config_id;
} }
if(_mid->shall_bypass || if(_mid->shall_bypass ||
(!param->cache_dyn_url && _mid->is_dyn_url && param->cache_key.qs_num==0) || (!param->cache_dyn_url && _mid->is_dyn_url && param->key_descr.qs_num==0) ||
(param->cache_cookied_cont && _mid->has_cookie) ) (param->cache_cookied_cont && _mid->has_cookie) )
{ {
_mid->result=PENDING_RESULT_FOBIDDEN; _mid->result=PENDING_RESULT_FOBIDDEN;
@@ -988,9 +1024,9 @@ enum cache_pending_result web_cache_async_pending(struct cache_handle* handle, u
struct tango_cache_meta_get meta; struct tango_cache_meta_get meta;
memset(&meta, 0, sizeof(meta)); memset(&meta, 0, sizeof(meta));
if(param->cache_key.is_not_empty) if(param->key_descr.is_not_empty)
{ {
_mid->cache_key=get_cache_key(request, &(param->cache_key)); _mid->cache_key=get_cache_key(request, &(param->key_descr));
meta.url = _mid->cache_key; meta.url = _mid->cache_key;
} }
else else
@@ -1083,7 +1119,7 @@ static void wrap_cache_update_on_succ(future_result_t * result, void * user)
static void wrap_cache_update_on_fail(enum e_future_error err, const char * what, void * user) static void wrap_cache_update_on_fail(enum e_future_error err, const char * what, void * user)
{ {
struct wrap_cache_put_ctx* ctx=(struct wrap_cache_put_ctx*)user; struct wrap_cache_put_ctx* ctx=(struct wrap_cache_put_ctx*)user;
TFE_LOG_DEBUG(ctx->ref_handle->logger, "cache upload failed: %s elapse: %d", ctx->url, time(NULL)-ctx->start); TFE_LOG_DEBUG(ctx->ref_handle->logger, "cache upload failed: %s %s lapse: %d", ctx->url, what, time(NULL)-ctx->start);
wrap_cache_put_ctx_free(ctx); wrap_cache_put_ctx_free(ctx);
} }
@@ -1160,11 +1196,11 @@ struct cache_update_context* web_cache_update_start(struct cache_handle* handle,
key=session->req->req_spec.url; key=session->req->req_spec.url;
key_len=strlen(session->req->req_spec.url); key_len=strlen(session->req->req_spec.url);
} }
_mid->use_cnt=counting_bloom_check(handle->cache_key_bloom[thread_id], key, key_len); _mid->use_cnt=counting_bloom_check(handle->cache_key_bloom[thread_id].bloom, key, key_len);
if(_mid->use_cnt<param->min_use) if(_mid->use_cnt<param->min_use)
{ {
counting_bloom_add(handle->cache_key_bloom[thread_id], key, key_len); counting_bloom_add(handle->cache_key_bloom[thread_id].bloom, key, key_len);
return NULL; return NULL;
} }
} }