gfwleak/wangyouzhan-http-statistic-fs
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

post all srccode

Browse Source
This commit is contained in:
git commit -m first
2019-08-29 10:14:11 +08:00
commit d962394a02
8 changed files with 593 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
bin/http_stat_header.so Executable file
View File

Binary file not shown.

7
conf/http_stat_header.conf Normal file
View File

@@ -0,0 +1,7 @@
[LOG]
log_level=30
log_path=./log/http_statistic_runtime
fs_path=./log/http_statistic_fs

9
conf/http_stat_header.inf Normal file
View File

@@ -0,0 +1,9 @@
[PLUGINFO]
PLUGNAME=STHH
SO_PATH=./plug/business/http_stat_header/http_stat_header.so
INIT_FUNC=HTTP_STAT_HEADER_INIT
DESTROY_FUNC=HTTP_STAT_HEADER_DESTROY
[HTTP]
FUNC_FLAG=HTTP_CONT_ENCODING,HTTP_CONT_TYPE,HTTP_URI
FUNC_NAME=HTTP_STAT_HEADER_ENTRY

23
src/Makefile Normal file
View File

@@ -0,0 +1,23 @@
CC = g++
CFLAGS = -Wall -g -fPIC -shared -g3 -gdwarf-2
INC = -I./inc/ -I/opt/MESA/include
LIBS = -L/opt/MESA/lib -lMESA_handle_logger -lMESA_prof_load -lMESA_field_stat2
OBJS = http_stat_header.o http_header_v1.o
TARGET = http_stat_header.so
.cpp.o:
$(CC) -c $(INC) $(CFLAGS) $<
all: $(TARGET) CP
$(TARGET): $(OBJS)
$(CC) -o $@ $^ $(LIBS) $(CFLAGS)
CP:
cp $(TARGET) ../bin/
.PHONY:clean
clean:
-rm -rf $(OBJS) $(TARGET)

189
src/http_header_v1.cpp Normal file
View File

@@ -0,0 +1,189 @@
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <algorithm>
#include <sys/stat.h>
#include <sys/types.h>
#include <MESA/MESA_htable.h>
#include <MESA/MESA_handle_logger.h>
#include <MESA/MESA_prof_load.h>
#include <MESA/field_stat2.h>
#include <MESA/MESA_list_queue.h>
#include "http_header_v1.hpp"
BaseFS::~BaseFS()
{
FS_stop(&fsstat_handle);
}
BaseFS::BaseFS(const char * header_name, const char *logdir)
{
char fsfile[256], commd[256];
int value;
if(0 != access(logdir, 0))
{
snprintf(commd, 256, "mkdir -p %s", logdir);
system(commd);
}
snprintf(fsfile, 256, "%s/%s.txt", logdir, header_name);
fsstat_handle = FS_create_handle();
FS_set_para(fsstat_handle, OUTPUT_DEVICE, fsfile, strlen(fsfile)+1);
value = 1;
FS_set_para(fsstat_handle, PRINT_MODE, &(value), sizeof(value));
value = 1;
FS_set_para(fsstat_handle, STAT_CYCLE, &(value), sizeof(value));
//value = 1;
//FS_set_para(fsstat_handle, CREATE_THREAD, &value, sizeof(value));
value = 1;
FS_set_para(fsstat_handle, FLUSH_BY_DATE, &value, sizeof(value));
char appname[64];
FS_set_para(fsstat_handle, APP_NAME, appname, strlen(appname)+1);
char dst_ip[16] = "127.0.0.1";
int dst_port = 80;
FS_set_para(fsstat_handle, STATS_SERVER_IP, dst_ip, strlen(dst_ip)+1);
FS_set_para(fsstat_handle, STATS_SERVER_PORT, &dst_port, sizeof(dst_port));
}
void BaseFS::start(void)
{
FS_start(fsstat_handle);
}
void BaseFS::passive_output(void)
{
FS_passive_output(fsstat_handle);
}
ContentTable::~ContentTable()
{
FS_stop(&fsstat_handle);
}
ContentTable::ContentTable(const char * header_name, const char *logdir):BaseFS(header_name, logdir)
{
char total_num[64], no_head_num[64], head_num[64];
total_status_id = 0;
no_header_id = 0;
num_column_id = 0;
ratio_column_id = 0;
header_total = 0;
no_header_num = 0;
snprintf(total_num, 64, "%s_total", header_name);
snprintf(no_head_num, 64, "no_%s", header_name);
snprintf(head_num, 64, "%s_num", header_name);
if(0 != pthread_mutex_init(&mutex_lock, NULL))
printf("init mutex error!!!!!!!!\n");
total_status_id = FS_register(fsstat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, total_num);
no_header_id = FS_register(fsstat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, no_head_num);
num_column_id = FS_register(fsstat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT, head_num);
ratio_column_id = FS_register(fsstat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT, "RATIO/milli");
}
int ContentTable::operate(enum field_op op, string line_name, long long value)
{
map<string, http_header_t>::iterator iter;
int line_id;
int ret = 0;
http_header_t hht;
string::size_type idx;
iter = dynamic_line_id.find(line_name);
if(iter == dynamic_line_id.end())
{
pthread_mutex_lock(&mutex_lock);
if(iter == dynamic_line_id.end()) //防止多个线程竞争锁导致的dynamic_line_id不是最新多加一层判断
{
line_id = FS_register(fsstat_handle, FS_STYLE_LINE, FS_CALC_CURRENT, line_name.c_str());
hht.header_id = line_id;
hht.header_num = 1;
dynamic_line_id[line_name] = hht;
}
else
{
iter = dynamic_line_id.find(line_name);
if(iter == dynamic_line_id.end())
{
line_id = FS_register(fsstat_handle, FS_STYLE_LINE, FS_CALC_CURRENT, line_name.c_str());
hht.header_id = line_id;
hht.header_num = 1;
dynamic_line_id[line_name] = hht;
}
else
{
line_id = iter->second.header_id;
__sync_add_and_fetch(&(iter->second.header_num), 1);
}
}
pthread_mutex_unlock(&mutex_lock);
}
else
{
line_id = iter->second.header_id;
__sync_add_and_fetch(&(iter->second.header_num), 1);
}
__sync_add_and_fetch(&header_total, 1);
return ret;
}
void ContentTable::operate_ratio()
{
map<string, http_header_t>::iterator iter;
http_header_t hht;
long long ratio = 0;
pthread_mutex_lock(&mutex_lock);
FS_operate(fsstat_handle, total_status_id, 0, FS_OP_SET, header_total);
FS_operate(fsstat_handle, no_header_id, 0, FS_OP_SET, no_header_num);
iter = dynamic_line_id.begin();
while(iter != dynamic_line_id.end())
{
hht = iter->second;
ratio = (hht.header_num*1000)/header_total;
FS_operate(fsstat_handle, hht.header_id, num_column_id, FS_OP_SET, hht.header_num);
FS_operate(fsstat_handle, hht.header_id, ratio_column_id, FS_OP_SET, ratio);
iter ++;
}
FS_passive_output(fsstat_handle);
pthread_mutex_unlock(&mutex_lock);
}
long long ContentTable::set_nocontent_num()
{
return __sync_add_and_fetch(&no_header_num, 1);
}
long long ContentTable::get_head_total()
{
return header_total;
}
ContentHistogram::~ContentHistogram()
{
FS_stop(&fsstat_handle);
}
ContentHistogram::ContentHistogram(const char * header_name, const char *logdir):BaseFS(header_name, logdir)
{
if(0 != pthread_mutex_init(&mutex_lock, NULL))
printf("init mutex error!!!!!!!!\n");
const char* histogram_format="0.25,0.5,0.75,0.9,0.99000";
FS_set_para(fsstat_handle, HISTOGRAM_GLOBAL_BINS, histogram_format, strlen(histogram_format)+10);
histogram_id = FS_register_histogram(fsstat_handle, FS_CALC_CURRENT, "CONTENT_LENGHT" ,1,9999999999,3);
}
int ContentHistogram::content_lenght_operate(long long value)
{
int ret = 0;
ret = FS_operate(fsstat_handle, histogram_id, 0, FS_OP_SET, value);
//FS_passive_output(fsstat_handle);
return ret;
}

62
src/http_header_v1.hpp Normal file
View File

@@ -0,0 +1,62 @@
#ifndef __HTTP_HEADER_V1__
#define __HTTP_HEADER_V1__
#include <sys/types.h>
#include <MESA/field_stat2.h>
#include <iostream>
#include <map>
#include <string>
#include <algorithm>
#include <iostream>
using namespace std;
typedef struct _http_header_info
{
int header_id;
long long header_num;
}http_header_t;
class BaseFS
{
public:
screen_stat_handle_t fsstat_handle;
public:
BaseFS(const char * header_name, const char *logdir);
~BaseFS();
void start(void);
void passive_output(void);
};
class ContentTable : public BaseFS
{
private:
map<string, http_header_t> dynamic_line_id;
int total_status_id;
int no_header_id;
int num_column_id;
int ratio_column_id;
long long header_total;
long long no_header_num;
pthread_mutex_t mutex_lock;
public:
ContentTable(const char * header_name, const char *logdir);
~ContentTable();
int operate(enum field_op op, string line_name, long long value);
void operate_ratio();
long long set_nocontent_num();
long long get_head_total();
};
class ContentHistogram : public BaseFS
{
private:
int histogram_id;
pthread_mutex_t mutex_lock;
public:
ContentHistogram(const char * header_name, const char *logdir);
~ContentHistogram();
int content_lenght_operate(long long value);
};
#endif

255
src/http_stat_header.cpp Normal file
View File

@@ -0,0 +1,255 @@
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <syslog.h>
#include <signal.h>
#include <error.h>
#include <getopt.h>
#include <unistd.h>
#include <assert.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/syslog.h>
#include<algorithm>
#include <MESA/MESA_htable.h>
#include <MESA/MESA_handle_logger.h>
#include <MESA/MESA_prof_load.h>
#include <MESA/field_stat2.h>
#include <MESA/MESA_list_queue.h>
#include <MESA/stream.h>
#include <MESA/http.h>
#include "http_stat_header.h"
#include "http_header_v1.hpp"
#define HTTP_LOG_DEBUG 0
g_env_conf_t g_env_conf_info;
long long bflag = 0;
int init_pmeinfo(content_header_flag **flag_pme, int thread_seq)
{
content_header_flag* pme = (content_header_flag*)dictator_malloc(thread_seq, sizeof(content_header_flag));
memset(pme, 0, sizeof(content_header_flag));
*flag_pme = pme;
return 0;
}
void clear_pmeinfo(content_header_flag *service_pme, int thread_seq)
{
if(service_pme!=NULL)
{
dictator_free(thread_seq, service_pme);
}
service_pme = NULL;
}
int checkout_head_val(std::string & line_name, int flag)
{
int i = 0;
string tmp = line_name;
string::size_type idx;
char special_ch[14] = {'|',':','.',' ','<','>','[',']','#','!','@','\n','\r','\t'};
for(i = 0; i < 14; i++)
{
idx = line_name.find(special_ch[i]);
if(idx != string::npos)
{
replace(line_name.begin(), line_name.end(), special_ch[i], '*');
}
}
transform(line_name.begin(), line_name.end(), line_name.begin(), (int (*)(int))tolower);
if(flag == 1)
MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_INFO, "HTTP_CONT_TYPE", "src header value[%s], dest header value[%s]", tmp.c_str(), line_name.c_str());
else if(2 == flag)
MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_INFO, "HTTP_CONT_ENCODING", "src header value[%s], dest header value[%s]", tmp.c_str(), line_name.c_str());
else
MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_INFO, "HTTP_OTHER", "src header value[%s], dest header value[%s]", tmp.c_str(), line_name.c_str());
return 1;
}
void deal_with_data(stSessionInfo * session_info, int thread_seq, struct streaminfo *a_stream, content_header_flag * flag_pme)
{
/*if(a_stream->curdir == DIR_C2S)
{
MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_DEBUG, "FLAG", "dir error");
return;
}*/
int ret = 0;
long long len = 0;
string line_name;
char content_len[64] = {'0'};
switch(session_info->prot_flag)
{
case HTTP_CONT_TYPE:
line_name = string((char *)session_info->buf, session_info->buflen);
if(0 == checkout_head_val(line_name, 1))
{
break;
}
g_env_conf_info.cont_type_fs->operate(FS_OP_ADD, line_name, 1);
flag_pme->type_flag = 1;
break;
case HTTP_CONT_ENCODING:
line_name = string((char *)session_info->buf, session_info->buflen);
if(0 == checkout_head_val(line_name, 2))
{
break;
}
g_env_conf_info.cont_encoding_fs->operate(FS_OP_ADD, line_name,1);
flag_pme->encoding_flag = 1;
break;
case HTTP_CONT_LENGTH:
memcpy(content_len,(char *)session_info->buf, session_info->buflen);
len = atoll(content_len);
ret = g_env_conf_info.cont_lenght_fs->content_lenght_operate(len);
MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_DEBUG, "HTTP_CONT_LENGTH", "content_length[%s], buflen:%d, addr[%s], ret = [%d]", content_len, session_info->buflen, printaddr(&(a_stream->addr), thread_seq), ret);
break;
case HTTP_RES_LINE:
flag_pme->uri_flag = 1;
break;
default:
MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_DEBUG, "ABNORMAL_HTTP", "buf:%s", (char *)session_info->buf);
break;
}
}
void print_addr(stSessionInfo * session_info, int thread_seq, struct streaminfo *a_stream, content_header_flag * flag_pme)
{
char p[1024] = {'\0'};
if(session_info && session_info->buf)
{
memcpy(p, (char *)session_info->buf, session_info->buflen);
#ifdef HTTP_LOG_DEBUG
MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_DEBUG, "TEST", "[%p],[%lld],flag[%d]", flag_pme, flag_pme->bflag, session_info->session_state);
#endif
MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_DEBUG, "SESSION_INFO", "buf:[%s],buflen:[%d],addr:[%s]", p,session_info->buflen, printaddr(&(a_stream->addr), thread_seq));
}
else
{
#ifdef HTTP_LOG_DEBUG
MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_DEBUG, "TEST", "[%p],[%lld],flag[%d]", flag_pme, flag_pme->bflag, session_info->session_state);
#endif
MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_DEBUG, "SESSION_INFO", "addr:[%s]", printaddr(&(a_stream->addr), thread_seq));
}
return;
}
char HTTP_STAT_HEADER_ENTRY(stSessionInfo * session_info,void **pme, int thread_seq,struct streaminfo *a_stream,void *a_packet)
{
MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_DEBUG, "HTTP_STAT_HEADER_ENTRY","HTTP_STAT_HEADER_ENTRY:[%d]", session_info->session_state);
content_header_flag * flag_pme = (content_header_flag *)*pme;
if(NULL==session_info)
{
return PROT_STATE_DROPME;
}
if(NULL == flag_pme)
{
if(init_pmeinfo(&flag_pme, thread_seq)<0)
{
MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_DEBUG, "INIT", "error");
return PROT_STATE_DROPME;
}
flag_pme->bflag = (++bflag);
*pme = flag_pme;
}
#ifdef HTTP_LOG_DEBUG
if(session_info->session_state&SESSION_STATE_PENDING)
{
__sync_add_and_fetch(&(g_env_conf_info.pend_sum), 1);
MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_DEBUG, "s-sum1", "[%p],[%lld],pend_sum:%lld, addr:[%s],state[%d]",flag_pme,flag_pme->bflag,g_env_conf_info.pend_sum, printaddr(&(a_stream->addr), thread_seq), session_info->session_state);
}
#endif
if(session_info->buflen < 200)
{
print_addr(session_info, thread_seq, a_stream, flag_pme);
deal_with_data(session_info, thread_seq, a_stream, flag_pme);
}
else
{
MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_FATAL, "HTTP_HEADER_ERROR", "The length of header:[%s]>200,[%d] bytes. Not do statistics! [%s]", session_info->buf, session_info->buflen, printaddr(&(a_stream->addr), thread_seq));
}
if(session_info->session_state&SESSION_STATE_CLOSE)
{
#ifdef HTTP_LOG_DEBUG
__sync_add_and_fetch(&(g_env_conf_info.close_sum), 1);
MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_DEBUG, "s-sum2", "[%p],[%lld],close_sum:%lld, addr:[%s]",flag_pme,flag_pme->bflag,g_env_conf_info.close_sum, printaddr(&(a_stream->addr), thread_seq));
#endif
if(1 == flag_pme->uri_flag && 1 != flag_pme->type_flag)
{
MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_DEBUG, "NO_TYPE", "[%p],[%lld]", flag_pme, flag_pme->bflag);
g_env_conf_info.cont_type_fs->set_nocontent_num();
}
if(1 == flag_pme->uri_flag && 1 != flag_pme->encoding_flag)
{
MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_DEBUG, "NO_ENCODING", "[%p], [%lld]", flag_pme, flag_pme->bflag);
g_env_conf_info.cont_encoding_fs->set_nocontent_num();
}
clear_pmeinfo((content_header_flag*)*pme, thread_seq);
#ifdef HTTP_LOG_DEBUG
MESA_handle_runtime_log(g_env_conf_info.logger, RLOG_LV_DEBUG, "sum", "pend_sum:%lld, close_sum:%lld", g_env_conf_info.pend_sum, g_env_conf_info.close_sum);
#endif
//return PROT_STATE_DROPME;
}
return PROT_STATE_GIVEME;
}
int http_service_readconf(const char* filename)
{
MESA_load_profile_short_def(filename, "LOG", "log_level", (short*)&(g_env_conf_info.log_level), 30);
MESA_load_profile_string_def(filename, "LOG", "log_path", g_env_conf_info.log_path, sizeof(g_env_conf_info.log_path), DEFAULT_LOG_PATH);
MESA_load_profile_string_def(filename, "LOG", "fs_path", g_env_conf_info.fs_path, sizeof(g_env_conf_info.fs_path),DEFAULT_FS_PATH);
g_env_conf_info.logger = MESA_create_runtime_log_handle(g_env_conf_info.log_path, g_env_conf_info.log_level);
if(NULL == g_env_conf_info.logger)
{
printf("http_stat_header.so get log handle error!\n");
return -1;
}
return 0;
}
void * update_ratio(void * arg)
{
while(1)
{
sleep(10);
g_env_conf_info.cont_type_fs->operate_ratio();
g_env_conf_info.cont_encoding_fs->operate_ratio();
g_env_conf_info.cont_lenght_fs->passive_output();
}
return NULL;
}
int HTTP_STAT_HEADER_INIT()
{
pthread_t thread_tid;
pthread_attr_t attr;
memset(&g_env_conf_info, 0, sizeof(g_env_conf_t));
if(0 != http_service_readconf(CONF_PATH))
{
printf("http_stat_header.so http_service.conf read error!\n");
return -1;
}
g_env_conf_info.cont_type_fs = new ContentTable(CONT_TYPE, g_env_conf_info.fs_path);
g_env_conf_info.cont_type_fs->start();
g_env_conf_info.cont_encoding_fs = new ContentTable(CONT_ENCODING, g_env_conf_info.fs_path);
g_env_conf_info.cont_encoding_fs->start();
g_env_conf_info.cont_lenght_fs = new ContentHistogram(CONT_LENGHT, g_env_conf_info.fs_path);
g_env_conf_info.cont_lenght_fs->start();
pthread_attr_init(&attr);
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
if(0 != pthread_create(&thread_tid, &attr, update_ratio, NULL))
{
printf("create thread update_ratio failed!\n");
return -1;
}
return 0;
}
void HTTP_STAT_HEADER_DESTROY()
{
return;
}

48
src/http_stat_header.h Normal file
View File

@@ -0,0 +1,48 @@
#ifndef HTTP_STAT_HEADER_
#define HTTP_STAT_HEADER_
#include "http_header_v1.hpp"
#define CONF_PATH "./conf/http_stat_header.conf"
#define CONT_TYPE "content_type"
#define CONT_ENCODING "content_encoding"
#define CONT_LENGHT "content_lenght"
#define DEFAULT_LOG_PATH "./log/http_statistic_runtime"
#define DEFAULT_FS_PATH "./log/http_statistic_fs"
typedef struct _content_header_flag
{
int type_flag;
int encoding_flag;
int uri_flag;
long long bflag;
}content_header_flag;
typedef struct _g_env_conf_t
{
void * logger;
char log_path[512];
short log_level;
char fs_path[512];
screen_stat_handle_t cont_type_handle;
screen_stat_handle_t cont_encoding_handle;
ContentTable * cont_type_fs;
ContentTable * cont_encoding_fs;
ContentHistogram * cont_lenght_fs;
long long pend_sum;
long long close_sum;
long long data_sum;
}g_env_conf_t;
#ifdef __cplusplus
extern "C" {
#endif
char HTTP_STAT_HEADER_ENTRY(stSessionInfo * session_info,void **pme, int thread_seq,struct streaminfo *a_stream,void *a_packet);
int HTTP_STAT_HEADER_INIT();
void HTTP_STAT_HEADER_DESTROY();
#ifdef __cplusplus
}
#endif
#endif