gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

完成cache参数解析部分的编码。

Browse Source
This commit is contained in:
zhengchao
2018-11-07 21:05:21 +08:00
parent 8bfecea928
commit d6862e4ef7
4 changed files with 221 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
cache/src/tango_cache_pending.cpp vendored
View File

@@ -115,7 +115,9 @@ enum cache_pending_action tfe_cache_get_pending(const struct tfe_http_half *requ
{
return FORBIDDEN;
}
if(NULL!=tfe_http_std_field_read(request, TFE_HTTP_CONT_RANGE))
if(NULL!=tfe_http_std_field_read(request, TFE_HTTP_CONT_RANGE) ||
NULL!=tfe_http_std_field_read(request, TFE_HTTP_AUTHORIZATION)||
NULL!=tfe_http_nonstd_field_read(request, "WWW-Authenticate"))
{
return FORBIDDEN;
}
@@ -306,7 +308,7 @@ enum cache_pending_action tfe_cache_put_pending(const struct tfe_http_half *resp
else
{
value = tfe_http_std_field_read(response, TFE_HTTP_EXPIRES);
if (value != NULL)
if (value != NULL && 0!= read_GMT_time(value))
{
res = ALLOWED;
}

7
common/include/tfe_http.h
View File

@@ -315,6 +315,13 @@ static inline const char * tfe_http_std_field_read(const struct tfe_http_half *
tmp_name.field_name=NULL;
return tfe_http_field_read(half, &tmp_name);
}
static inline const char * tfe_http_nonstd_field_read(const struct tfe_http_half * half, const char* field)
{
struct http_field_name tmp_name;
tmp_name.field_id=TFE_HTTP_UNKNOWN_FIELD;
tmp_name.field_name=field;
return tfe_http_field_read(half, &tmp_name);
}
static inline int tfe_http_field_write(struct tfe_http_half * half,
const struct http_field_name * name, const char * value)

213
plugin/business/pangu-http/src/pangu_web_cache.cpp
View File

@@ -9,6 +9,7 @@
#include <MESA/MESA_prof_load.h>
#include <MESA/field_stat2.h>
#include <Maat_rule.h>
#include <event2/event.h>
#include <event2/buffer.h>
@@ -16,7 +17,7 @@
enum cache_stat_field
{
STAT_CACHE_QUERY,
STAT_CACHE_QUERY_NOT_APPLICABLE,
STAT_CACHE_QUERY_FORBIDDEN,
STAT_CACHE_QUERY_VERIFY,
STAT_CACHE_QUERY_HIT,
STAT_CACHE_QUERY_BYTES,
@@ -61,6 +62,11 @@ struct cache_handle
struct event_base* gc_evbase;
struct event* gcev;
struct cache_param default_cache_policy;
Maat_feather_t ref_feather;
int cache_param_idx;
int table_url_constraint;
int table_cookie_constraint;
void* logger;
};
struct cache_update_context
@@ -151,7 +157,7 @@ void cache_stat_init(struct cache_handle* cache)
struct cache_stat_sepc spec[__CACHE_STAT_MAX];
set_stat_spec(&spec[STAT_CACHE_QUERY], "cache_get",FS_STYLE_FIELD, FS_CALC_CURRENT);
set_stat_spec(&spec[STAT_CACHE_QUERY_NOT_APPLICABLE], "get_forbid",FS_STYLE_FIELD, FS_CALC_CURRENT);
set_stat_spec(&spec[STAT_CACHE_QUERY_FORBIDDEN], "get_forbid",FS_STYLE_FIELD, FS_CALC_CURRENT);
set_stat_spec(&spec[STAT_CACHE_QUERY_VERIFY], "get_verify",FS_STYLE_FIELD, FS_CALC_CURRENT);
set_stat_spec(&spec[STAT_CACHE_QUERY_HIT], "hit_num",FS_STYLE_FIELD, FS_CALC_CURRENT);
set_stat_spec(&spec[STAT_CACHE_QUERY_BYTES], "downloaded(MB)",FS_STYLE_FIELD, FS_CALC_CURRENT);
@@ -212,7 +218,133 @@ void cache_stat_init(struct cache_handle* cache)
evtimer_add(cache->gcev, &gc_delay);
return;
}
struct cache_handle* create_web_cache_handle(const char* profile_path, const char* section, struct event_base* gc_evbase, void *logger)
struct cache_key_descr
{
size_t qs_num;
char** ignore_qs;
char* include_hdrs;
char* include_cookie;
};
struct cache_param
{
struct cache_key_descr key_descr;
char revalidate;
char cache_dyn_url;
char cache_cookied_cont;
char ignore_req_nocache;
char ignore_res_nocache;
char force_caching;
int min_use;
int pinning_time_sec;
int inactive_time_sec;
int max_cache_size_mb;
int max_cache_obj_size_mb;
};
int time_secs_interval(const char* str){}
int storage_mb_size(const char* str){}
int is_dynamic_url(const char* url){}
void cache_param_new(int idx, const struct Maat_rule_t* rule, const char* srv_def_large,
MAAT_RULE_EX_DATA* ad, long argl, void *argp)
{
struct cache_handle* cache=(struct cache_handle*) argp;
int i=0;
*ad=NULL;
if(rule->serv_def_len<strlen("{}")+1)
{
return;
}
cJSON *json=NULL, *key_desc=NULL, *qs=NULL,*item=NULL;
json=cJSON_Parse(srv_def_large);
if(json==NULL)
{
TFE_LOG_ERROR(cache->logger, "invalid cache parameter: id = %d", rule->config_id);
return;
}
struct cache_param* param=ALLOC(struct cache_param, 1);
*param=cache->default_cache_policy;
key_desc=cJSON_GetObjectItem(json,"cache_key");
if(key_desc && key_desc->type==cJSON_Object)
{
qs=cJSON_GetObjectItem(json,"ignore_qs");
if(qs && qs->type==cJSON_Array)
{
param->key_descr.qs_num=cJSON_GetArraySize(qs);
param->key_descr.ignore_qs=ALLOC(char*, param->key_descr.qs_num);
for(i=0; i<param->key_descr.qs_num; i++)
{
item=cJSON_GetArrayItem(item, i);
param->key_descr.ignore_qs[i]=tfe_strdup(qs->valuestring);
}
}
}
item=cJSON_GetObjectItem(key_desc,"hdrs");
if(item && item->type==cJSON_String) param->key_descr.include_hdrs=tfe_strdup(item->valuestring);
item=cJSON_GetObjectItem(key_desc,"cookie");
if(item && item->type==cJSON_String) param->key_descr.include_cookie=tfe_strdup(item->valuestring);
item=cJSON_GetObjectItem(json,"revalidate");
if(item && item->type==cJSON_Number) param->revalidate=item->valueint;
item=cJSON_GetObjectItem(json,"cache_dyn_url");
if(item && item->type==cJSON_Number) param->cache_dyn_url=item->valueint;
item=cJSON_GetObjectItem(json,"cache_cookied_cont");
if(item && item->type==cJSON_Number) param->cache_cookied_cont=item->valueint;
item=cJSON_GetObjectItem(json,"ignore_req_nocache");
if(item && item->type==cJSON_Number) param->ignore_req_nocache=item->valueint;
item=cJSON_GetObjectItem(json,"ignore_res_nocache");
if(item && item->type==cJSON_Number) param->ignore_res_nocache=item->valueint;
item=cJSON_GetObjectItem(json,"force_caching");
if(item && item->type==cJSON_Number) param->force_caching=item->valueint;
item=cJSON_GetObjectItem(json,"min_use");
if(item && item->type==cJSON_Number) param->min_use=item->valueint;
item=cJSON_GetObjectItem(json,"pinning_time");
if(item && item->type==cJSON_String) param->pinning_time_sec=time_secs_interval(item->valuestring);
item=cJSON_GetObjectItem(json,"inactive_time");
if(item && item->type==cJSON_String) param->inactive_time_sec=time_secs_interval(item->valuestring);
item=cJSON_GetObjectItem(json,"max_cache_size");
if(item && item->type==cJSON_String) param->max_cache_size_mb=storage_mb_size(item->valuestring);
item=cJSON_GetObjectItem(json,"max_cache_obj_size");
if(item && item->type==cJSON_String) param->max_cache_obj_size_mb=storage_mb_size(item->valuestring);
cJSON_Delete(json);
return;
}
void cache_param_free(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, MAAT_RULE_EX_DATA* ad, long argl, void *argp)
{
int i=0;
if(*ad==NULL)
{
return;
}
struct cache_param* param=(struct cache_param*)*ad;
for(i=0; i<param->key_descr.qs_num; i++)
{
FREE(&(param->key_descr.ignore_qs[i]));
}
FREE(&(param->key_descr.ignore_qs));
FREE(&(param->key_descr.include_cookie));
FREE(&(param->key_descr.include_hdrs));
FREE(&(param));
return;
}
struct cache_handle* create_web_cache_handle(const char* profile_path, const char* section,
struct event_base* gc_evbase, Maat_feather_t feather, void *logger)
{
struct cache_handle* cache=ALLOC(struct cache_handle, 1);
int temp=0;
@@ -240,6 +372,26 @@ struct cache_handle* create_web_cache_handle(const char* profile_path, const cha
MESA_load_profile_int_def(profile_path, section, "cache_minimum_time_override", &(cache->minimum_cache_seconds), 60*5);
cache->gc_evbase=gc_evbase;
cache->default_cache_policy.key_descr.qs_num=0;
cache->default_cache_policy.revalidate=1;
cache->default_cache_policy.cache_dyn_url=0;
cache->default_cache_policy.cache_cookied_cont=0;
cache->default_cache_policy.ignore_req_nocache=0;
cache->default_cache_policy.ignore_res_nocache=0;
cache->default_cache_policy.force_caching=0;
cache->default_cache_policy.min_use=1;
cache->default_cache_policy.pinning_time_sec=0;
cache->default_cache_policy.inactive_time_sec=0;
cache->default_cache_policy.max_cache_size_mb=0;
cache->default_cache_policy.max_cache_obj_size_mb=1024;//<1GB by default
cache->table_url_constraint=Maat_table_register(feather, "PXY_CACHE_HTTP_URL");
cache->table_cookie_constraint=Maat_table_register(feather, "PXY_CACHE_HTTP_COOKIE");
cache->cache_param_idx=Maat_rule_get_ex_new_index(feather, "PXY_CACHE_COMPILE",
cache_param_new, cache_param_free, 0, cache);
cache_stat_init(cache);
return cache;
error_out:
@@ -477,20 +629,58 @@ static void cache_query_meta_on_fail(enum e_future_error err, const char * what,
cache_pending_ctx_free_cb(ctx);
return;
}
struct cache_mid
{
struct cache_param* param;
};
enum cache_pending_result web_cache_async_pending(struct cache_handle* handle, unsigned int thread_id,
const struct tfe_http_half * request, struct future* f_revalidate)
const struct tfe_http_half * request, struct cache_mid** mid, struct future* f_revalidate)
{
struct request_freshness req_fresshness={0,0};
enum cache_pending_result result=PENDING_RESULT_FOBIDDEN;
int is_undefined_obj=0;
struct Maat_rule_t cache_policy;
const struct cache_param* param=&(handle->default_cache_policy);
MAAT_RULE_EX_DATA ex_data=NULL;
scan_status_t scan_mid=NULL;
int ret=0;
const char* cookie=NULL;
ret=Maat_full_scan_string(handle->ref_feather, handle->table_url_constraint, CHARSET_UTF8,
request->req_spec.url, strlen(request->req_spec.url),
&cache_policy, NULL, 1, &scan_mid, thread_id);
cookie=tfe_http_std_field_read(request, TFE_HTTP_COOKIE);
if(cookie && ret<=0)
{
ret=Maat_full_scan_string(handle->ref_feather, handle->table_cookie_constraint, CHARSET_UTF8,
cookie, strlen(cookie),
&cache_policy, NULL, 1, &scan_mid, thread_id);
}
Maat_clean_status(&scan_mid);
if(ret>0)
{
ex_data=Maat_rule_get_ex_data(handle->ref_feather, &cache_policy, handle->cache_param_idx);
if(ex_data!=NULL)
{
param=(const struct cache_param*)ex_data;
}
}
if(param->cache_dyn_url==0 && is_dynamic_url(request->req_spec.url))
{
return FORBIDDEN;
}
if(param->cache_cookied_cont==0 && cookie!=NULL)
{
return FORBIDDEN;
}
enum cache_pending_action get_action=tfe_cache_get_pending(request, &(req_fresshness));
switch(get_action)
{
case UNDEFINED:
if(!handle->query_undefined_obj_enabled)
{
ATOMIC_INC(&(handle->stat_val[STAT_CACHE_QUERY_NOT_APPLICABLE]));
ATOMIC_INC(&(handle->stat_val[STAT_CACHE_QUERY_FORBIDDEN]));
result=PENDING_RESULT_FOBIDDEN;
}
else
@@ -501,8 +691,15 @@ enum cache_pending_result web_cache_async_pending(struct cache_handle* handle, u
}
break;
case FORBIDDEN:
ATOMIC_INC(&(handle->stat_val[STAT_CACHE_QUERY_NOT_APPLICABLE]));
result=PENDING_RESULT_FOBIDDEN;
if(param->ignore_req_nocache || param->force_caching)
{
result=PENDING_RESULT_ALLOWED;
}
else
{
ATOMIC_INC(&(handle->stat_val[STAT_CACHE_QUERY_FORBIDDEN]));
result=PENDING_RESULT_FOBIDDEN;
}
break;
case ALLOWED:
result=PENDING_RESULT_ALLOWED;

8
plugin/business/pangu-http/src/pangu_web_cache.h
View File

@@ -47,16 +47,18 @@ enum cache_pending_result
PENDING_RESULT_ALLOWED,
PENDING_RESULT_MISS
};
struct cache_mid;
const struct cached_meta* cache_pending_result_read_meta(future_result_t * result);
enum cache_pending_result web_cache_async_pending(struct cache_handle* handle, unsigned int thread_id,
const struct tfe_http_half * request, struct future* f_revalidate);
const struct tfe_http_half * request, struct cache_mid **mid, struct future* f_revalidate);
void cache_mid_free(struct cache_mid **mid);
struct cache_update_context;
struct cache_update_context* web_cache_update_start(struct cache_handle* handle, unsigned int thread_id,
const struct tfe_http_session * session);
const struct tfe_http_session * session, struct cache_mid **mid);
void web_cache_update(struct cache_update_context* ctx, const unsigned char * body_frag, size_t frag_size);
void web_cache_update_end(struct cache_update_context* ctx);