commit d962394a02426072737d50318e71e2286023fab6 Author: git commit -m first Date: Thu Aug 29 10:14:11 2019 +0800 post all srccode diff --git a/bin/http_stat_header.so b/bin/http_stat_header.so new file mode 100755 index 0000000..79bacd9 Binary files /dev/null and b/bin/http_stat_header.so differ diff --git a/conf/http_stat_header.conf b/conf/http_stat_header.conf new file mode 100644 index 0000000..46c38f8 --- /dev/null +++ b/conf/http_stat_header.conf @@ -0,0 +1,7 @@ +[LOG] +log_level=30 +log_path=./log/http_statistic_runtime +fs_path=./log/http_statistic_fs + + + diff --git a/conf/http_stat_header.inf b/conf/http_stat_header.inf new file mode 100644 index 0000000..eba9fe3 --- /dev/null +++ b/conf/http_stat_header.inf @@ -0,0 +1,9 @@ +[PLUGINFO] +PLUGNAME=STHH +SO_PATH=./plug/business/http_stat_header/http_stat_header.so +INIT_FUNC=HTTP_STAT_HEADER_INIT +DESTROY_FUNC=HTTP_STAT_HEADER_DESTROY + +[HTTP] +FUNC_FLAG=HTTP_CONT_ENCODING,HTTP_CONT_TYPE,HTTP_URI +FUNC_NAME=HTTP_STAT_HEADER_ENTRY diff --git a/src/Makefile b/src/Makefile new file mode 100644 index 0000000..0c71b18 --- /dev/null +++ b/src/Makefile @@ -0,0 +1,23 @@ +CC = g++ +CFLAGS = -Wall -g -fPIC -shared -g3 -gdwarf-2 +INC = -I./inc/ -I/opt/MESA/include +LIBS = -L/opt/MESA/lib -lMESA_handle_logger -lMESA_prof_load -lMESA_field_stat2 +OBJS = http_stat_header.o http_header_v1.o +TARGET = http_stat_header.so + +.cpp.o: + $(CC) -c $(INC) $(CFLAGS) $< + +all: $(TARGET) CP + +$(TARGET): $(OBJS) + $(CC) -o $@ $^ $(LIBS) $(CFLAGS) + +CP: + cp $(TARGET) ../bin/ + +.PHONY:clean + +clean: + -rm -rf $(OBJS) $(TARGET) + diff --git a/src/http_header_v1.cpp b/src/http_header_v1.cpp new file mode 100644 index 0000000..7680bc8 --- /dev/null +++ b/src/http_header_v1.cpp @@ -0,0 +1,189 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "http_header_v1.hpp" + +BaseFS::~BaseFS() +{ + FS_stop(&fsstat_handle); +} + +BaseFS::BaseFS(const char * header_name, const char *logdir) +{ + char fsfile[256], commd[256]; + int value; + + if(0 != access(logdir, 0)) + { + snprintf(commd, 256, "mkdir -p %s", logdir); + system(commd); + } + snprintf(fsfile, 256, "%s/%s.txt", logdir, header_name); + fsstat_handle = FS_create_handle(); + FS_set_para(fsstat_handle, OUTPUT_DEVICE, fsfile, strlen(fsfile)+1); + value = 1; + FS_set_para(fsstat_handle, PRINT_MODE, &(value), sizeof(value)); + value = 1; + FS_set_para(fsstat_handle, STAT_CYCLE, &(value), sizeof(value)); + //value = 1; + //FS_set_para(fsstat_handle, CREATE_THREAD, &value, sizeof(value)); + value = 1; + FS_set_para(fsstat_handle, FLUSH_BY_DATE, &value, sizeof(value)); + char appname[64]; + FS_set_para(fsstat_handle, APP_NAME, appname, strlen(appname)+1); + char dst_ip[16] = "127.0.0.1"; + int dst_port = 80; + FS_set_para(fsstat_handle, STATS_SERVER_IP, dst_ip, strlen(dst_ip)+1); + FS_set_para(fsstat_handle, STATS_SERVER_PORT, &dst_port, sizeof(dst_port)); +} + +void BaseFS::start(void) +{ + FS_start(fsstat_handle); +} + +void BaseFS::passive_output(void) +{ + FS_passive_output(fsstat_handle); +} + +ContentTable::~ContentTable() +{ + FS_stop(&fsstat_handle); +} + +ContentTable::ContentTable(const char * header_name, const char *logdir):BaseFS(header_name, logdir) +{ + char total_num[64], no_head_num[64], head_num[64]; + total_status_id = 0; + no_header_id = 0; + num_column_id = 0; + ratio_column_id = 0; + header_total = 0; + no_header_num = 0; + + snprintf(total_num, 64, "%s_total", header_name); + snprintf(no_head_num, 64, "no_%s", header_name); + snprintf(head_num, 64, "%s_num", header_name); + if(0 != pthread_mutex_init(&mutex_lock, NULL)) + printf("init mutex error!!!!!!!!\n"); + + total_status_id = FS_register(fsstat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, total_num); + no_header_id = FS_register(fsstat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, no_head_num); + num_column_id = FS_register(fsstat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT, head_num); + ratio_column_id = FS_register(fsstat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT, "RATIO/milli"); +} + +int ContentTable::operate(enum field_op op, string line_name, long long value) +{ + map::iterator iter; + int line_id; + int ret = 0; + http_header_t hht; + string::size_type idx; + iter = dynamic_line_id.find(line_name); + if(iter == dynamic_line_id.end()) + { + pthread_mutex_lock(&mutex_lock); + if(iter == dynamic_line_id.end()) //防止多个线程竞争锁导致的dynamic_line_id不是最新多加一层判断 + { + line_id = FS_register(fsstat_handle, FS_STYLE_LINE, FS_CALC_CURRENT, line_name.c_str()); + hht.header_id = line_id; + hht.header_num = 1; + dynamic_line_id[line_name] = hht; + } + else + { + iter = dynamic_line_id.find(line_name); + if(iter == dynamic_line_id.end()) + { + line_id = FS_register(fsstat_handle, FS_STYLE_LINE, FS_CALC_CURRENT, line_name.c_str()); + hht.header_id = line_id; + hht.header_num = 1; + dynamic_line_id[line_name] = hht; + } + else + { + line_id = iter->second.header_id; + __sync_add_and_fetch(&(iter->second.header_num), 1); + } + } + pthread_mutex_unlock(&mutex_lock); + } + else + { + line_id = iter->second.header_id; + __sync_add_and_fetch(&(iter->second.header_num), 1); + } + __sync_add_and_fetch(&header_total, 1); + return ret; +} + +void ContentTable::operate_ratio() +{ + map::iterator iter; + http_header_t hht; + long long ratio = 0; + pthread_mutex_lock(&mutex_lock); + FS_operate(fsstat_handle, total_status_id, 0, FS_OP_SET, header_total); + FS_operate(fsstat_handle, no_header_id, 0, FS_OP_SET, no_header_num); + + iter = dynamic_line_id.begin(); + while(iter != dynamic_line_id.end()) + { + hht = iter->second; + ratio = (hht.header_num*1000)/header_total; + FS_operate(fsstat_handle, hht.header_id, num_column_id, FS_OP_SET, hht.header_num); + FS_operate(fsstat_handle, hht.header_id, ratio_column_id, FS_OP_SET, ratio); + iter ++; + } + FS_passive_output(fsstat_handle); + pthread_mutex_unlock(&mutex_lock); +} + +long long ContentTable::set_nocontent_num() +{ + return __sync_add_and_fetch(&no_header_num, 1); +} + +long long ContentTable::get_head_total() +{ + return header_total; +} + +ContentHistogram::~ContentHistogram() +{ + FS_stop(&fsstat_handle); +} + +ContentHistogram::ContentHistogram(const char * header_name, const char *logdir):BaseFS(header_name, logdir) +{ + if(0 != pthread_mutex_init(&mutex_lock, NULL)) + printf("init mutex error!!!!!!!!\n"); + const char* histogram_format="0.25,0.5,0.75,0.9,0.99000"; + FS_set_para(fsstat_handle, HISTOGRAM_GLOBAL_BINS, histogram_format, strlen(histogram_format)+10); + histogram_id = FS_register_histogram(fsstat_handle, FS_CALC_CURRENT, "CONTENT_LENGHT" ,1,9999999999,3); +} + +int ContentHistogram::content_lenght_operate(long long value) +{ + int ret = 0; + ret = FS_operate(fsstat_handle, histogram_id, 0, FS_OP_SET, value); + //FS_passive_output(fsstat_handle); + return ret; +} + + + diff --git a/src/http_header_v1.hpp b/src/http_header_v1.hpp new file mode 100644 index 0000000..6b4f5af --- /dev/null +++ b/src/http_header_v1.hpp @@ -0,0 +1,62 @@ +#ifndef __HTTP_HEADER_V1__ +#define __HTTP_HEADER_V1__ + +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +typedef struct _http_header_info +{ + int header_id; + long long header_num; +}http_header_t; + +class BaseFS +{ +public: + screen_stat_handle_t fsstat_handle; +public: + BaseFS(const char * header_name, const char *logdir); + ~BaseFS(); + + void start(void); + void passive_output(void); +}; + +class ContentTable : public BaseFS +{ +private: + map dynamic_line_id; + int total_status_id; + int no_header_id; + int num_column_id; + int ratio_column_id; + long long header_total; + long long no_header_num; + pthread_mutex_t mutex_lock; +public: + ContentTable(const char * header_name, const char *logdir); + ~ContentTable(); + int operate(enum field_op op, string line_name, long long value); + void operate_ratio(); + long long set_nocontent_num(); + long long get_head_total(); +}; + +class ContentHistogram : public BaseFS +{ +private: + int histogram_id; + pthread_mutex_t mutex_lock; +public: + ContentHistogram(const char * header_name, const char *logdir); + ~ContentHistogram(); + int content_lenght_operate(long long value); +}; +#endif diff --git a/src/http_stat_header.cpp b/src/http_stat_header.cpp new file mode 100644 index 0000000..d724199 --- /dev/null +++ b/src/http_stat_header.cpp @@ -0,0 +1,255 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "http_stat_header.h" +#include "http_header_v1.hpp" +#define HTTP_LOG_DEBUG 0 +g_env_conf_t g_env_conf_info; +long long bflag = 0; +int init_pmeinfo(content_header_flag **flag_pme, int thread_seq) +{ + content_header_flag* pme = (content_header_flag*)dictator_malloc(thread_seq, sizeof(content_header_flag)); + memset(pme, 0, sizeof(content_header_flag)); + *flag_pme = pme; + return 0; +} + +void clear_pmeinfo(content_header_flag *service_pme, int thread_seq) +{ + if(service_pme!=NULL) + { + dictator_free(thread_seq, service_pme); + } + service_pme = NULL; +} + +int checkout_head_val(std::string & line_name, int flag) +{ + int i = 0; + string tmp = line_name; + string::size_type idx; + char special_ch[14] = {'|',':','.',' ','<','>','[',']','#','!','@','\n','\r','\t'}; + for(i = 0; i < 14; i++) + { + idx = line_name.find(special_ch[i]); + if(idx != string::npos) + { + replace(line_name.begin(), line_name.end(), special_ch[i], '*'); + } + } + transform(line_name.begin(), line_name.end(), line_name.begin(), (int (*)(int))tolower); + if(flag == 1) + MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_INFO, "HTTP_CONT_TYPE", "src header value[%s], dest header value[%s]", tmp.c_str(), line_name.c_str()); + else if(2 == flag) + MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_INFO, "HTTP_CONT_ENCODING", "src header value[%s], dest header value[%s]", tmp.c_str(), line_name.c_str()); + else + MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_INFO, "HTTP_OTHER", "src header value[%s], dest header value[%s]", tmp.c_str(), line_name.c_str()); + + return 1; +} + + +void deal_with_data(stSessionInfo * session_info, int thread_seq, struct streaminfo *a_stream, content_header_flag * flag_pme) +{ + /*if(a_stream->curdir == DIR_C2S) + { + MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_DEBUG, "FLAG", "dir error"); + return; + }*/ + int ret = 0; + long long len = 0; + string line_name; + char content_len[64] = {'0'}; + switch(session_info->prot_flag) + { + case HTTP_CONT_TYPE: + line_name = string((char *)session_info->buf, session_info->buflen); + if(0 == checkout_head_val(line_name, 1)) + { + break; + } + g_env_conf_info.cont_type_fs->operate(FS_OP_ADD, line_name, 1); + flag_pme->type_flag = 1; + break; + case HTTP_CONT_ENCODING: + line_name = string((char *)session_info->buf, session_info->buflen); + if(0 == checkout_head_val(line_name, 2)) + { + break; + } + g_env_conf_info.cont_encoding_fs->operate(FS_OP_ADD, line_name,1); + flag_pme->encoding_flag = 1; + break; + case HTTP_CONT_LENGTH: + memcpy(content_len,(char *)session_info->buf, session_info->buflen); + len = atoll(content_len); + ret = g_env_conf_info.cont_lenght_fs->content_lenght_operate(len); + MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_DEBUG, "HTTP_CONT_LENGTH", "content_length[%s], buflen:%d, addr[%s], ret = [%d]", content_len, session_info->buflen, printaddr(&(a_stream->addr), thread_seq), ret); + break; + case HTTP_RES_LINE: + flag_pme->uri_flag = 1; + break; + default: + MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_DEBUG, "ABNORMAL_HTTP", "buf:%s", (char *)session_info->buf); + break; + } +} + +void print_addr(stSessionInfo * session_info, int thread_seq, struct streaminfo *a_stream, content_header_flag * flag_pme) +{ + char p[1024] = {'\0'}; + if(session_info && session_info->buf) + { + memcpy(p, (char *)session_info->buf, session_info->buflen); +#ifdef HTTP_LOG_DEBUG + MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_DEBUG, "TEST", "[%p],[%lld],flag[%d]", flag_pme, flag_pme->bflag, session_info->session_state); +#endif + MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_DEBUG, "SESSION_INFO", "buf:[%s],buflen:[%d],addr:[%s]", p,session_info->buflen, printaddr(&(a_stream->addr), thread_seq)); + } + else + { +#ifdef HTTP_LOG_DEBUG + MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_DEBUG, "TEST", "[%p],[%lld],flag[%d]", flag_pme, flag_pme->bflag, session_info->session_state); +#endif + MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_DEBUG, "SESSION_INFO", "addr:[%s]", printaddr(&(a_stream->addr), thread_seq)); + } + return; +} + +char HTTP_STAT_HEADER_ENTRY(stSessionInfo * session_info,void **pme, int thread_seq,struct streaminfo *a_stream,void *a_packet) +{ + MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_DEBUG, "HTTP_STAT_HEADER_ENTRY","HTTP_STAT_HEADER_ENTRY:[%d]", session_info->session_state); + content_header_flag * flag_pme = (content_header_flag *)*pme; + if(NULL==session_info) + { + return PROT_STATE_DROPME; + } + + if(NULL == flag_pme) + { + if(init_pmeinfo(&flag_pme, thread_seq)<0) + { + MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_DEBUG, "INIT", "error"); + return PROT_STATE_DROPME; + } + flag_pme->bflag = (++bflag); + *pme = flag_pme; + } +#ifdef HTTP_LOG_DEBUG + if(session_info->session_state&SESSION_STATE_PENDING) + { + __sync_add_and_fetch(&(g_env_conf_info.pend_sum), 1); + MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_DEBUG, "s-sum1", "[%p],[%lld],pend_sum:%lld, addr:[%s],state[%d]",flag_pme,flag_pme->bflag,g_env_conf_info.pend_sum, printaddr(&(a_stream->addr), thread_seq), session_info->session_state); + } +#endif + if(session_info->buflen < 200) + { + print_addr(session_info, thread_seq, a_stream, flag_pme); + deal_with_data(session_info, thread_seq, a_stream, flag_pme); + } + else + { + MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_FATAL, "HTTP_HEADER_ERROR", "The length of header:[%s]>200,[%d] bytes. Not do statistics! [%s]", session_info->buf, session_info->buflen, printaddr(&(a_stream->addr), thread_seq)); + } + if(session_info->session_state&SESSION_STATE_CLOSE) + { +#ifdef HTTP_LOG_DEBUG + __sync_add_and_fetch(&(g_env_conf_info.close_sum), 1); + MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_DEBUG, "s-sum2", "[%p],[%lld],close_sum:%lld, addr:[%s]",flag_pme,flag_pme->bflag,g_env_conf_info.close_sum, printaddr(&(a_stream->addr), thread_seq)); +#endif + if(1 == flag_pme->uri_flag && 1 != flag_pme->type_flag) + { + MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_DEBUG, "NO_TYPE", "[%p],[%lld]", flag_pme, flag_pme->bflag); + g_env_conf_info.cont_type_fs->set_nocontent_num(); + } + if(1 == flag_pme->uri_flag && 1 != flag_pme->encoding_flag) + { + MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_DEBUG, "NO_ENCODING", "[%p], [%lld]", flag_pme, flag_pme->bflag); + g_env_conf_info.cont_encoding_fs->set_nocontent_num(); + } + clear_pmeinfo((content_header_flag*)*pme, thread_seq); +#ifdef HTTP_LOG_DEBUG + MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_DEBUG, "sum", "pend_sum:%lld, close_sum:%lld", g_env_conf_info.pend_sum, g_env_conf_info.close_sum); +#endif + //return PROT_STATE_DROPME; + } + return PROT_STATE_GIVEME; +} + + +int http_service_readconf(const char* filename) +{ + MESA_load_profile_short_def(filename, "LOG", "log_level", (short*)&(g_env_conf_info.log_level), 30); + MESA_load_profile_string_def(filename, "LOG", "log_path", g_env_conf_info.log_path, sizeof(g_env_conf_info.log_path), DEFAULT_LOG_PATH); + MESA_load_profile_string_def(filename, "LOG", "fs_path", g_env_conf_info.fs_path, sizeof(g_env_conf_info.fs_path),DEFAULT_FS_PATH); + g_env_conf_info.logger = MESA_create_runtime_log_handle(g_env_conf_info.log_path, g_env_conf_info.log_level); + if(NULL == g_env_conf_info.logger) + { + printf("http_stat_header.so get log handle error!\n"); + return -1; + } + return 0; +} + +void * update_ratio(void * arg) +{ + while(1) + { + sleep(10); + g_env_conf_info.cont_type_fs->operate_ratio(); + g_env_conf_info.cont_encoding_fs->operate_ratio(); + g_env_conf_info.cont_lenght_fs->passive_output(); + } + return NULL; +} + +int HTTP_STAT_HEADER_INIT() +{ + pthread_t thread_tid; + pthread_attr_t attr; + memset(&g_env_conf_info, 0, sizeof(g_env_conf_t)); + if(0 != http_service_readconf(CONF_PATH)) + { + printf("http_stat_header.so http_service.conf read error!\n"); + return -1; + } + g_env_conf_info.cont_type_fs = new ContentTable(CONT_TYPE, g_env_conf_info.fs_path); + g_env_conf_info.cont_type_fs->start(); + g_env_conf_info.cont_encoding_fs = new ContentTable(CONT_ENCODING, g_env_conf_info.fs_path); + g_env_conf_info.cont_encoding_fs->start(); + g_env_conf_info.cont_lenght_fs = new ContentHistogram(CONT_LENGHT, g_env_conf_info.fs_path); + g_env_conf_info.cont_lenght_fs->start(); + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + if(0 != pthread_create(&thread_tid, &attr, update_ratio, NULL)) + { + printf("create thread update_ratio failed!\n"); + return -1; + } + return 0; +} + +void HTTP_STAT_HEADER_DESTROY() +{ + return; +} + diff --git a/src/http_stat_header.h b/src/http_stat_header.h new file mode 100644 index 0000000..a7dd42a --- /dev/null +++ b/src/http_stat_header.h @@ -0,0 +1,48 @@ +#ifndef HTTP_STAT_HEADER_ +#define HTTP_STAT_HEADER_ +#include "http_header_v1.hpp" + +#define CONF_PATH "./conf/http_stat_header.conf" +#define CONT_TYPE "content_type" +#define CONT_ENCODING "content_encoding" +#define CONT_LENGHT "content_lenght" +#define DEFAULT_LOG_PATH "./log/http_statistic_runtime" +#define DEFAULT_FS_PATH "./log/http_statistic_fs" + +typedef struct _content_header_flag +{ + int type_flag; + int encoding_flag; + int uri_flag; + long long bflag; +}content_header_flag; + +typedef struct _g_env_conf_t +{ + void * logger; + char log_path[512]; + short log_level; + char fs_path[512]; + screen_stat_handle_t cont_type_handle; + screen_stat_handle_t cont_encoding_handle; + ContentTable * cont_type_fs; + ContentTable * cont_encoding_fs; + ContentHistogram * cont_lenght_fs; + long long pend_sum; + long long close_sum; + long long data_sum; + +}g_env_conf_t; + +#ifdef __cplusplus +extern "C" { +#endif +char HTTP_STAT_HEADER_ENTRY(stSessionInfo * session_info,void **pme, int thread_seq,struct streaminfo *a_stream,void *a_packet); +int HTTP_STAT_HEADER_INIT(); +void HTTP_STAT_HEADER_DESTROY(); + +#ifdef __cplusplus +} +#endif + +#endif