gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

#68 对接界面增加可信证书的接口。修复部分函数的bug。

Browse Source
This commit is contained in:
zhengchao
2018-11-11 13:03:54 +08:00
parent 62d9eb0bef
commit 0fd3b3880e
4 changed files with 196 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
conf/pangu/pangu_ctrl.json
View File

@@ -1,4 +1,6 @@
{
"compile_table": "PXY_CTRL_COMPILE",
"group_table": "PXY_CTRL_GROUP",
"rules": [
{
"compile_id": 125,
@@ -129,7 +131,101 @@
]
}
]
},
{
"compile_id": 133,
"service": 1,
"action": 48,
"do_blacklist": 1,
"do_log": 1,
"effective_range": 0,
"user_region": "{\"cache_key\":{\"ignore_qs\":[\"sqp\",\"rs\"],\"cookie\":\"jsession_id\"},\"no_revalidate\":0,\"cache_dyn_url\":1,\"cache_cookied_cont\":1,\"ignore_srv_nocache\":1,\"ignore_cli_nocache\":1,\"force_caching\":1,\"min_use\":1,\"pinning_time\":\"60m\",\"max_cache_size\":\"1t\",\"inactive_time\":\"1h\"}",
"is_valid": "yes",
"table_name":"PXY_CACHE_COMPILE",
"groups": [
{
"regions": [
{
"table_name": "PXY_CACHE_HTTP_URL",
"table_type": "string",
"table_content": {
"keywords": "i.ytimg.com",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
},
{
"compile_id": 134,
"service": 1,
"action": 128,
"do_blacklist": 1,
"do_log": 1,
"effective_range": 0,
"user_region": "{}",
"is_valid": "yes",
"table_name":"PXY_CACHE_COMPILE",
"groups": [
{
"regions": [
{
"table_name": "PXY_CACHE_HTTP_URL",
"table_type": "string",
"table_content": {
"keywords": "mesalab.com",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
},
{
"compile_id": 135,
"service": 1,
"action": 48,
"do_blacklist": 1,
"do_log": 1,
"effective_range": 0,
"user_region": "{\"cache_key\":{\"cookie\":\"MUID\"},\"cache_dyn_url\":1,\"cache_cookied_cont\":1}",
"is_valid": "yes",
"table_name":"PXY_CACHE_COMPILE",
"groups": [
{
"regions": [
{
"table_name": "PXY_CACHE_HTTP_URL",
"table_type": "string",
"table_content": {
"keywords": "cn.bing.com",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
}
],
"plugin_table": [
{
"table_name": "PXY_OBJ_TRUSTED_CA_CERT",
"table_content": [
"1\tDigiCertSHA2SecureServerCA\t./pangu_files/DigiCertSHA2SecureServerCA.pem\t1"
]
},
{
"table_name": "PXY_OBJ_TRUSTED_CA_CRL",
"table_content": [
"1\t1\t./pangu_files/ssca-sha2-g5.crl\t1"
]
}
]
}

4
conf/pangu/table_info.conf
View File

@@ -20,9 +20,9 @@
2 PXY_CTRL_IP ip ---
3 PXY_CTRL_HTTP_URL expr UTF8 GBK/UNICODE/UTF8/url_encode_gb2312/url_encode_utf8 yes 0 quickoff
4 PXY_CTRL_HTTP_REQ_HDR expr_plus UTF8 UTF8 yes 0 quickoff
5 PXY_CTRL_HTTP_REQ_BODY expr UTF8 GBK/BIG5/UNICODE/UTF8 yes 128 quickoff
5 PXY_CTRL_HTTP_REQ_BODY expr UTF8 GBK/UNICODE/UTF8 yes 128 quickoff
6 PXY_CTRL_HTTP_RES_HDR expr_plus UTF8 UTF8 UTF8 yes 0 quickoff
7 PXY_CTRL_HTTP_RES_BODY expr UTF8 GBK/BIG5/UNICODE/UTF8 yes 128 quickoff
7 PXY_CTRL_HTTP_RES_BODY expr UTF8 GBK/UNICODE/UTF8 yes 128 quickoff
8 PXY_CACHE_COMPILE compile escape --
9 PXY_CACHE_GROUP group --
10 PXY_CACHE_HTTP_URL expr UTF8 UTF8 yes 0 quickoff

5
plugin/business/pangu-http/src/pangu_http.cpp
View File

@@ -133,7 +133,8 @@ static Maat_feather_t create_maat_feather(const char * profile, const char * sec
case MAAT_INPUT_JSON:
Maat_set_feather_opt(target, MAAT_OPT_JSON_FILE_PATH, json_cfg_file, strlen(json_cfg_file) + 1);
break;
case MAAT_INPUT_REDIS: Maat_set_feather_opt(target, MAAT_OPT_REDIS_IP, redis_server, strlen(redis_server) + 1);
case MAAT_INPUT_REDIS:
Maat_set_feather_opt(target, MAAT_OPT_REDIS_IP, redis_server, strlen(redis_server) + 1);
Maat_set_feather_opt(target, MAAT_OPT_REDIS_PORT, &redis_port, sizeof(redis_port));
Maat_set_feather_opt(target, MAAT_OPT_REDIS_INDEX, &redis_db_idx, sizeof(redis_db_idx));
break;
@@ -145,6 +146,8 @@ static Maat_feather_t create_maat_feather(const char * profile, const char * sec
break;
}
Maat_set_feather_opt(target, MAAT_OPT_FOREIGN_CONT_DIR, "./pangu_files", strlen("./pangu_files")+1);
if (maat_stat_on)
{
Maat_set_feather_opt(target, MAAT_OPT_STAT_FILE_PATH, maat_stat_file, strlen(maat_stat_file) + 1);

116
plugin/business/pangu-http/src/pangu_web_cache.cpp
View File

@@ -64,7 +64,7 @@ struct cache_key_descr
struct cache_param
{
int ref_cnt;
struct cache_key_descr cache_key;
struct cache_key_descr key_descr;
char no_revalidate;
char cache_dyn_url;
@@ -80,6 +80,14 @@ struct cache_param
long max_cache_obj_size;
pthread_mutex_t lock;
};
struct cache_bloom
{
int thread_id;
size_t size;
double error_rate;
char filename[TFE_PATH_MAX];
counting_bloom_t *bloom;
};
struct cache_handle
{
unsigned int thread_count;
@@ -105,9 +113,10 @@ struct cache_handle
int cache_param_idx;
int table_url_constraint;
int table_cookie_constraint;
size_t cache_key_bloom_size;
int cache_key_bloom_life;
counting_bloom_t **cache_key_bloom;
size_t cache_key_bloom_size;
struct cache_bloom *cache_key_bloom;
void* logger;
};
struct cache_update_context
@@ -277,7 +286,7 @@ time_t time_unit_sec(const char* str)
value*=3600;
break;
case 'd':
value*=24*3600;
value*=((size_t)24*3600);
break;
default:
break;
@@ -296,18 +305,22 @@ size_t storage_unit_byte(const char* str)
value*=1024;
break;
case 'm':
value*=1024*1024;
value*=((size_t)1024*1024);
break;
case 'g':
value*=1024*1024*1024;
value*=((size_t)1024*1024*1024);
break;
case 't':
if(value<1024)
{
#pragma GCC diagnostic ignored "-Woverflow"
value*=1024*1024*1024*1024;
value*=((size_t)1024*1024*1024*1024);
}
else //maximum 1PB
{
value=(size_t)1024*(1024*1024*1024*1024);
}
break;
default:
break;
@@ -374,11 +387,16 @@ char* get_cache_key(const struct tfe_http_half * request, const struct cache_key
char cookie_val[256]={0}; //most 256 bytes for cookie key
size_t key_size=strlen(url)+sizeof(cookie_val);
char* cache_key=ALLOC(char, key_size);
char* query_string=strchr(url, '?');
if(query_string!=NULL && desc->qs_num>0)
char* query_string=NULL;
if(desc->qs_num>0)
{
query_string=strchr(url, '?');
if(query_string!=NULL)
{
strncat(cache_key, url, MIN(query_string-url,key_size));
query_string++;
for (token = url; ; token= NULL)
for (token = query_string; ; token= NULL)
{
sub_token= strtok_r(token,"&", &saveptr);
if (sub_token == NULL)
@@ -402,6 +420,11 @@ char* get_cache_key(const struct tfe_http_half * request, const struct cache_key
{
strncat(cache_key, url, key_size);
}
}
else
{
strncat(cache_key, url, key_size);
}
if(desc->include_cookie && (cookie=tfe_http_std_field_read(request, TFE_HTTP_COOKIE))!=NULL)
{
cookie_scanvalue(desc->include_cookie, cookie, cookie_val, sizeof(cookie_val));
@@ -411,7 +434,7 @@ char* get_cache_key(const struct tfe_http_half * request, const struct cache_key
}
}
FREE(&(url));
return cache_key;
}
void cache_param_new(int idx, const struct Maat_rule_t* rule, const char* srv_def_large,
@@ -440,23 +463,23 @@ void cache_param_new(int idx, const struct Maat_rule_t* rule, const char* srv_de
key_desc=cJSON_GetObjectItem(json,"cache_key");
if(key_desc && key_desc->type==cJSON_Object)
{
param->cache_key.is_not_empty=1;
qs=cJSON_GetObjectItem(json,"ignore_qs");
param->key_descr.is_not_empty=1;
qs=cJSON_GetObjectItem(key_desc,"ignore_qs");
if(qs && qs->type==cJSON_Array)
{
param->cache_key.qs_num=cJSON_GetArraySize(qs);
param->cache_key.ignore_qs=ALLOC(char*, param->cache_key.qs_num);
for(i=0; i<param->cache_key.qs_num; i++)
param->key_descr.qs_num=cJSON_GetArraySize(qs);
param->key_descr.ignore_qs=ALLOC(char*, param->key_descr.qs_num);
for(i=0; i<param->key_descr.qs_num; i++)
{
item=cJSON_GetArrayItem(item, i);
len=strlen(qs->valuestring)+2;
param->cache_key.ignore_qs[i]=ALLOC(char, len);
strncat(param->cache_key.ignore_qs[i], qs->valuestring, len);
strncat(param->cache_key.ignore_qs[i], "=", len);
item=cJSON_GetArrayItem(qs, i);
len=strlen(item->valuestring)+2;
param->key_descr.ignore_qs[i]=ALLOC(char, len);
strncat(param->key_descr.ignore_qs[i], item->valuestring, len);
strncat(param->key_descr.ignore_qs[i], "=", len);
}
}
item=cJSON_GetObjectItem(key_desc,"cookie");
if(item && item->type==cJSON_String) param->cache_key.include_cookie=tfe_strdup(param->cache_key.include_cookie);
if(item && item->type==cJSON_String) param->key_descr.include_cookie=tfe_strdup(param->key_descr.include_cookie);
}
@@ -496,6 +519,7 @@ void cache_param_new(int idx, const struct Maat_rule_t* rule, const char* srv_de
if(item && item->type==cJSON_String) param->max_cache_obj_size=storage_unit_byte(item->valuestring);
cJSON_Delete(json);
*ad=param;
return;
}
void cache_param_free(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, MAAT_RULE_EX_DATA* ad, long argl, void *argp)
@@ -515,12 +539,12 @@ void cache_param_free(int idx, const struct Maat_rule_t* rule, const char* srv_d
}
pthread_mutex_unlock(&(param->lock));
pthread_mutex_destroy(&(param->lock));
for(i=0; i<param->cache_key.qs_num; i++)
for(i=0; i<param->key_descr.qs_num; i++)
{
FREE(&(param->cache_key.ignore_qs[i]));
FREE(&(param->key_descr.ignore_qs[i]));
}
FREE(&(param->cache_key.ignore_qs));
FREE(&(param->cache_key.include_cookie));
FREE(&(param->key_descr.ignore_qs));
FREE(&(param->key_descr.include_cookie));
FREE(&(param));
return;
}
@@ -536,11 +560,12 @@ void cache_param_dup(int idx, MAAT_RULE_EX_DATA *to, MAAT_RULE_EX_DATA *from, lo
static void cache_key_bloom_gc_cb(evutil_socket_t fd, short what, void * arg)
{
counting_bloom_t* old_bloom=*((counting_bloom_t**)arg), *new_bloom=NULL;
struct cache_bloom* p_bloom= (struct cache_bloom*) arg;
counting_bloom_t* new_bloom=NULL;
new_bloom=new_counting_bloom(old_bloom->capacity, old_bloom->error_rate, NULL);
free_counting_bloom(old_bloom);
*((counting_bloom_t**)arg)=old_bloom;
new_bloom=new_counting_bloom(p_bloom->size, p_bloom->error_rate, p_bloom->filename);
free_counting_bloom(p_bloom->bloom);
p_bloom->bloom=new_bloom;
return;
}
@@ -553,7 +578,8 @@ struct cache_handle* create_web_cache_handle(const char* profile_path, const cha
cache->logger=logger;
cache->thread_count=tfe_proxy_get_work_thread_count();
cache->clients=ALLOC(struct tango_cache_instance *, cache->thread_count);
cache->cache_key_bloom=ALLOC(counting_bloom_t*, cache->thread_count);
cache->cache_key_bloom=ALLOC(struct cache_bloom, cache->thread_count);
struct cache_bloom* p_bloom=NULL;
MESA_load_profile_int_def(profile_path, section, "cache_policy_enabled",
&(cache->cache_policy_enabled), 1);
@@ -562,14 +588,24 @@ struct cache_handle* create_web_cache_handle(const char* profile_path, const cha
(int*)&(cache->cache_key_bloom_size), 16*1000*1000);
MESA_load_profile_int_def(profile_path, section, "cache_key_bloom_life",
&(cache->cache_key_bloom_life), 30*60);
char bloom_filename[TFE_PATH_MAX]{0};
struct timeval gc_refresh_delay = {cache->cache_key_bloom_life, 0};
int i=0;
for(i=0; i<cache->thread_count; i++)
{
if(cache->cache_policy_enabled)
{
cache->cache_key_bloom[i]=new_counting_bloom(cache->cache_key_bloom_size, 0.01, NULL);
ev = event_new(tfe_proxy_get_work_thread_evbase(i), -1, EV_PERSIST, cache_key_bloom_gc_cb, &(cache->cache_key_bloom[i]));
p_bloom=cache->cache_key_bloom+i;
p_bloom->thread_id=i;
p_bloom->size=cache->cache_key_bloom_size;
p_bloom->error_rate=0.01;
snprintf(p_bloom->filename, sizeof(p_bloom->filename), "/tmp/pangu_cache_blooms.%d", i);
p_bloom->bloom=new_counting_bloom(p_bloom->size, p_bloom->error_rate, p_bloom->filename);
if(p_bloom->bloom==NULL)
{
goto error_out;
}
ev = event_new(tfe_proxy_get_work_thread_evbase(i), -1, EV_PERSIST, cache_key_bloom_gc_cb, p_bloom);
evtimer_add(ev, &gc_refresh_delay);
}
@@ -591,7 +627,7 @@ struct cache_handle* create_web_cache_handle(const char* profile_path, const cha
cache->gc_evbase=gc_evbase;
cache->default_cache_policy.cache_key.qs_num=0;
cache->default_cache_policy.key_descr.qs_num=0;
cache->default_cache_policy.no_revalidate=0;
cache->default_cache_policy.cache_dyn_url=0;
cache->default_cache_policy.cache_cookied_cont=0;
@@ -932,7 +968,7 @@ enum cache_pending_result web_cache_async_pending(struct cache_handle* handle, u
_mid->cfg_id=cache_policy.config_id;
}
if(_mid->shall_bypass ||
(!param->cache_dyn_url && _mid->is_dyn_url && param->cache_key.qs_num==0) ||
(!param->cache_dyn_url && _mid->is_dyn_url && param->key_descr.qs_num==0) ||
(param->cache_cookied_cont && _mid->has_cookie) )
{
_mid->result=PENDING_RESULT_FOBIDDEN;
@@ -988,9 +1024,9 @@ enum cache_pending_result web_cache_async_pending(struct cache_handle* handle, u
struct tango_cache_meta_get meta;
memset(&meta, 0, sizeof(meta));
if(param->cache_key.is_not_empty)
if(param->key_descr.is_not_empty)
{
_mid->cache_key=get_cache_key(request, &(param->cache_key));
_mid->cache_key=get_cache_key(request, &(param->key_descr));
meta.url = _mid->cache_key;
}
else
@@ -1083,7 +1119,7 @@ static void wrap_cache_update_on_succ(future_result_t * result, void * user)
static void wrap_cache_update_on_fail(enum e_future_error err, const char * what, void * user)
{
struct wrap_cache_put_ctx* ctx=(struct wrap_cache_put_ctx*)user;
TFE_LOG_DEBUG(ctx->ref_handle->logger, "cache upload failed: %s elapse: %d", ctx->url, time(NULL)-ctx->start);
TFE_LOG_DEBUG(ctx->ref_handle->logger, "cache upload failed: %s %s lapse: %d", ctx->url, what, time(NULL)-ctx->start);
wrap_cache_put_ctx_free(ctx);
}
@@ -1160,11 +1196,11 @@ struct cache_update_context* web_cache_update_start(struct cache_handle* handle,
key=session->req->req_spec.url;
key_len=strlen(session->req->req_spec.url);
}
_mid->use_cnt=counting_bloom_check(handle->cache_key_bloom[thread_id], key, key_len);
_mid->use_cnt=counting_bloom_check(handle->cache_key_bloom[thread_id].bloom, key, key_len);
if(_mid->use_cnt<param->min_use)
{
counting_bloom_add(handle->cache_key_bloom[thread_id], key, key_len);
counting_bloom_add(handle->cache_key_bloom[thread_id].bloom, key, key_len);
return NULL;
}
}