diff --git a/cache/src/tango_cache_pending.cpp b/cache/src/tango_cache_pending.cpp index 6a732f1..2c7ea19 100644 --- a/cache/src/tango_cache_pending.cpp +++ b/cache/src/tango_cache_pending.cpp @@ -115,7 +115,9 @@ enum cache_pending_action tfe_cache_get_pending(const struct tfe_http_half *requ { return FORBIDDEN; } - if(NULL!=tfe_http_std_field_read(request, TFE_HTTP_CONT_RANGE)) + if(NULL!=tfe_http_std_field_read(request, TFE_HTTP_CONT_RANGE) || + NULL!=tfe_http_std_field_read(request, TFE_HTTP_AUTHORIZATION)|| + NULL!=tfe_http_nonstd_field_read(request, "WWW-Authenticate")) { return FORBIDDEN; } @@ -306,7 +308,7 @@ enum cache_pending_action tfe_cache_put_pending(const struct tfe_http_half *resp else { value = tfe_http_std_field_read(response, TFE_HTTP_EXPIRES); - if (value != NULL) + if (value != NULL && 0!= read_GMT_time(value)) { res = ALLOWED; } diff --git a/common/include/tfe_http.h b/common/include/tfe_http.h index 65f47ad..0384e13 100644 --- a/common/include/tfe_http.h +++ b/common/include/tfe_http.h @@ -315,6 +315,13 @@ static inline const char * tfe_http_std_field_read(const struct tfe_http_half * tmp_name.field_name=NULL; return tfe_http_field_read(half, &tmp_name); } +static inline const char * tfe_http_nonstd_field_read(const struct tfe_http_half * half, const char* field) +{ + struct http_field_name tmp_name; + tmp_name.field_id=TFE_HTTP_UNKNOWN_FIELD; + tmp_name.field_name=field; + return tfe_http_field_read(half, &tmp_name); +} static inline int tfe_http_field_write(struct tfe_http_half * half, const struct http_field_name * name, const char * value) diff --git a/plugin/business/pangu-http/src/pangu_web_cache.cpp b/plugin/business/pangu-http/src/pangu_web_cache.cpp index e82919f..1468805 100644 --- a/plugin/business/pangu-http/src/pangu_web_cache.cpp +++ b/plugin/business/pangu-http/src/pangu_web_cache.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include @@ -16,7 +17,7 @@ enum cache_stat_field { STAT_CACHE_QUERY, - STAT_CACHE_QUERY_NOT_APPLICABLE, + STAT_CACHE_QUERY_FORBIDDEN, STAT_CACHE_QUERY_VERIFY, STAT_CACHE_QUERY_HIT, STAT_CACHE_QUERY_BYTES, @@ -61,6 +62,11 @@ struct cache_handle struct event_base* gc_evbase; struct event* gcev; + struct cache_param default_cache_policy; + Maat_feather_t ref_feather; + int cache_param_idx; + int table_url_constraint; + int table_cookie_constraint; void* logger; }; struct cache_update_context @@ -151,7 +157,7 @@ void cache_stat_init(struct cache_handle* cache) struct cache_stat_sepc spec[__CACHE_STAT_MAX]; set_stat_spec(&spec[STAT_CACHE_QUERY], "cache_get",FS_STYLE_FIELD, FS_CALC_CURRENT); - set_stat_spec(&spec[STAT_CACHE_QUERY_NOT_APPLICABLE], "get_forbid",FS_STYLE_FIELD, FS_CALC_CURRENT); + set_stat_spec(&spec[STAT_CACHE_QUERY_FORBIDDEN], "get_forbid",FS_STYLE_FIELD, FS_CALC_CURRENT); set_stat_spec(&spec[STAT_CACHE_QUERY_VERIFY], "get_verify",FS_STYLE_FIELD, FS_CALC_CURRENT); set_stat_spec(&spec[STAT_CACHE_QUERY_HIT], "hit_num",FS_STYLE_FIELD, FS_CALC_CURRENT); set_stat_spec(&spec[STAT_CACHE_QUERY_BYTES], "downloaded(MB)",FS_STYLE_FIELD, FS_CALC_CURRENT); @@ -212,7 +218,133 @@ void cache_stat_init(struct cache_handle* cache) evtimer_add(cache->gcev, &gc_delay); return; } -struct cache_handle* create_web_cache_handle(const char* profile_path, const char* section, struct event_base* gc_evbase, void *logger) +struct cache_key_descr +{ + size_t qs_num; + char** ignore_qs; + char* include_hdrs; + char* include_cookie; +}; +struct cache_param +{ + struct cache_key_descr key_descr; + + char revalidate; + char cache_dyn_url; + char cache_cookied_cont; + char ignore_req_nocache; + char ignore_res_nocache; + char force_caching; + + int min_use; + int pinning_time_sec; + int inactive_time_sec; + int max_cache_size_mb; + int max_cache_obj_size_mb; +}; + + +int time_secs_interval(const char* str){} +int storage_mb_size(const char* str){} +int is_dynamic_url(const char* url){} +void cache_param_new(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, + MAAT_RULE_EX_DATA* ad, long argl, void *argp) +{ + struct cache_handle* cache=(struct cache_handle*) argp; + int i=0; + *ad=NULL; + if(rule->serv_def_lenlogger, "invalid cache parameter: id = %d", rule->config_id); + return; + } + struct cache_param* param=ALLOC(struct cache_param, 1); + *param=cache->default_cache_policy; + key_desc=cJSON_GetObjectItem(json,"cache_key"); + if(key_desc && key_desc->type==cJSON_Object) + { + qs=cJSON_GetObjectItem(json,"ignore_qs"); + if(qs && qs->type==cJSON_Array) + { + param->key_descr.qs_num=cJSON_GetArraySize(qs); + param->key_descr.ignore_qs=ALLOC(char*, param->key_descr.qs_num); + for(i=0; ikey_descr.qs_num; i++) + { + item=cJSON_GetArrayItem(item, i); + param->key_descr.ignore_qs[i]=tfe_strdup(qs->valuestring); + } + } + } + item=cJSON_GetObjectItem(key_desc,"hdrs"); + if(item && item->type==cJSON_String) param->key_descr.include_hdrs=tfe_strdup(item->valuestring); + + item=cJSON_GetObjectItem(key_desc,"cookie"); + if(item && item->type==cJSON_String) param->key_descr.include_cookie=tfe_strdup(item->valuestring); + + + item=cJSON_GetObjectItem(json,"revalidate"); + if(item && item->type==cJSON_Number) param->revalidate=item->valueint; + + item=cJSON_GetObjectItem(json,"cache_dyn_url"); + if(item && item->type==cJSON_Number) param->cache_dyn_url=item->valueint; + + item=cJSON_GetObjectItem(json,"cache_cookied_cont"); + if(item && item->type==cJSON_Number) param->cache_cookied_cont=item->valueint; + + item=cJSON_GetObjectItem(json,"ignore_req_nocache"); + if(item && item->type==cJSON_Number) param->ignore_req_nocache=item->valueint; + + item=cJSON_GetObjectItem(json,"ignore_res_nocache"); + if(item && item->type==cJSON_Number) param->ignore_res_nocache=item->valueint; + + item=cJSON_GetObjectItem(json,"force_caching"); + if(item && item->type==cJSON_Number) param->force_caching=item->valueint; + + item=cJSON_GetObjectItem(json,"min_use"); + if(item && item->type==cJSON_Number) param->min_use=item->valueint; + + item=cJSON_GetObjectItem(json,"pinning_time"); + if(item && item->type==cJSON_String) param->pinning_time_sec=time_secs_interval(item->valuestring); + + item=cJSON_GetObjectItem(json,"inactive_time"); + if(item && item->type==cJSON_String) param->inactive_time_sec=time_secs_interval(item->valuestring); + + item=cJSON_GetObjectItem(json,"max_cache_size"); + if(item && item->type==cJSON_String) param->max_cache_size_mb=storage_mb_size(item->valuestring); + + item=cJSON_GetObjectItem(json,"max_cache_obj_size"); + if(item && item->type==cJSON_String) param->max_cache_obj_size_mb=storage_mb_size(item->valuestring); + + cJSON_Delete(json); + return; +} +void cache_param_free(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, MAAT_RULE_EX_DATA* ad, long argl, void *argp) +{ + int i=0; + if(*ad==NULL) + { + return; + } + struct cache_param* param=(struct cache_param*)*ad; + for(i=0; ikey_descr.qs_num; i++) + { + FREE(&(param->key_descr.ignore_qs[i])); + } + FREE(&(param->key_descr.ignore_qs)); + FREE(&(param->key_descr.include_cookie)); + FREE(&(param->key_descr.include_hdrs)); + FREE(&(param)); + return; +} + +struct cache_handle* create_web_cache_handle(const char* profile_path, const char* section, + struct event_base* gc_evbase, Maat_feather_t feather, void *logger) { struct cache_handle* cache=ALLOC(struct cache_handle, 1); int temp=0; @@ -240,6 +372,26 @@ struct cache_handle* create_web_cache_handle(const char* profile_path, const cha MESA_load_profile_int_def(profile_path, section, "cache_minimum_time_override", &(cache->minimum_cache_seconds), 60*5); cache->gc_evbase=gc_evbase; + + cache->default_cache_policy.key_descr.qs_num=0; + cache->default_cache_policy.revalidate=1; + cache->default_cache_policy.cache_dyn_url=0; + cache->default_cache_policy.cache_cookied_cont=0; + cache->default_cache_policy.ignore_req_nocache=0; + cache->default_cache_policy.ignore_res_nocache=0; + cache->default_cache_policy.force_caching=0; + cache->default_cache_policy.min_use=1; + cache->default_cache_policy.pinning_time_sec=0; + cache->default_cache_policy.inactive_time_sec=0; + cache->default_cache_policy.max_cache_size_mb=0; + cache->default_cache_policy.max_cache_obj_size_mb=1024;//<1GB by default + + cache->table_url_constraint=Maat_table_register(feather, "PXY_CACHE_HTTP_URL"); + cache->table_cookie_constraint=Maat_table_register(feather, "PXY_CACHE_HTTP_COOKIE"); + + cache->cache_param_idx=Maat_rule_get_ex_new_index(feather, "PXY_CACHE_COMPILE", + cache_param_new, cache_param_free, 0, cache); + cache_stat_init(cache); return cache; error_out: @@ -477,20 +629,58 @@ static void cache_query_meta_on_fail(enum e_future_error err, const char * what, cache_pending_ctx_free_cb(ctx); return; } - +struct cache_mid +{ + struct cache_param* param; +}; enum cache_pending_result web_cache_async_pending(struct cache_handle* handle, unsigned int thread_id, - const struct tfe_http_half * request, struct future* f_revalidate) + const struct tfe_http_half * request, struct cache_mid** mid, struct future* f_revalidate) { struct request_freshness req_fresshness={0,0}; enum cache_pending_result result=PENDING_RESULT_FOBIDDEN; int is_undefined_obj=0; + struct Maat_rule_t cache_policy; + const struct cache_param* param=&(handle->default_cache_policy); + MAAT_RULE_EX_DATA ex_data=NULL; + scan_status_t scan_mid=NULL; + int ret=0; + const char* cookie=NULL; + ret=Maat_full_scan_string(handle->ref_feather, handle->table_url_constraint, CHARSET_UTF8, + request->req_spec.url, strlen(request->req_spec.url), + &cache_policy, NULL, 1, &scan_mid, thread_id); + + cookie=tfe_http_std_field_read(request, TFE_HTTP_COOKIE); + if(cookie && ret<=0) + { + ret=Maat_full_scan_string(handle->ref_feather, handle->table_cookie_constraint, CHARSET_UTF8, + cookie, strlen(cookie), + &cache_policy, NULL, 1, &scan_mid, thread_id); + } + Maat_clean_status(&scan_mid); + + if(ret>0) + { + ex_data=Maat_rule_get_ex_data(handle->ref_feather, &cache_policy, handle->cache_param_idx); + if(ex_data!=NULL) + { + param=(const struct cache_param*)ex_data; + } + } + if(param->cache_dyn_url==0 && is_dynamic_url(request->req_spec.url)) + { + return FORBIDDEN; + } + if(param->cache_cookied_cont==0 && cookie!=NULL) + { + return FORBIDDEN; + } enum cache_pending_action get_action=tfe_cache_get_pending(request, &(req_fresshness)); switch(get_action) { case UNDEFINED: if(!handle->query_undefined_obj_enabled) { - ATOMIC_INC(&(handle->stat_val[STAT_CACHE_QUERY_NOT_APPLICABLE])); + ATOMIC_INC(&(handle->stat_val[STAT_CACHE_QUERY_FORBIDDEN])); result=PENDING_RESULT_FOBIDDEN; } else @@ -501,8 +691,15 @@ enum cache_pending_result web_cache_async_pending(struct cache_handle* handle, u } break; case FORBIDDEN: - ATOMIC_INC(&(handle->stat_val[STAT_CACHE_QUERY_NOT_APPLICABLE])); - result=PENDING_RESULT_FOBIDDEN; + if(param->ignore_req_nocache || param->force_caching) + { + result=PENDING_RESULT_ALLOWED; + } + else + { + ATOMIC_INC(&(handle->stat_val[STAT_CACHE_QUERY_FORBIDDEN])); + result=PENDING_RESULT_FOBIDDEN; + } break; case ALLOWED: result=PENDING_RESULT_ALLOWED; diff --git a/plugin/business/pangu-http/src/pangu_web_cache.h b/plugin/business/pangu-http/src/pangu_web_cache.h index cb0227f..db0a234 100644 --- a/plugin/business/pangu-http/src/pangu_web_cache.h +++ b/plugin/business/pangu-http/src/pangu_web_cache.h @@ -47,16 +47,18 @@ enum cache_pending_result PENDING_RESULT_ALLOWED, PENDING_RESULT_MISS }; +struct cache_mid; + const struct cached_meta* cache_pending_result_read_meta(future_result_t * result); enum cache_pending_result web_cache_async_pending(struct cache_handle* handle, unsigned int thread_id, - const struct tfe_http_half * request, struct future* f_revalidate); - + const struct tfe_http_half * request, struct cache_mid **mid, struct future* f_revalidate); +void cache_mid_free(struct cache_mid **mid); struct cache_update_context; struct cache_update_context* web_cache_update_start(struct cache_handle* handle, unsigned int thread_id, - const struct tfe_http_session * session); + const struct tfe_http_session * session, struct cache_mid **mid); void web_cache_update(struct cache_update_context* ctx, const unsigned char * body_frag, size_t frag_size); void web_cache_update_end(struct cache_update_context* ctx);