#167 增加已拦截连接数、字节数、PASSTHROUGH连接数统计并调整FieldStat状态文件的输出位置。

common/src/tfe_future.cpp

@@ -101,7 +101,7 @@ void future_promise_library_init(const char* profile)
 	g_FP_instance.name_table=htable;
 
 	screen_stat_handle_t fs=NULL;
-	const char* stat_path="./future.fieldstat";
+	const char* stat_path="log/future.fs2";
 	const char* app_name="FP";
 	fs=FS_create_handle();
 	FS_set_para(fs, APP_NAME, app_name, strlen(app_name)+1);

conf/pangu/pangu_pxy.conf

@@ -33,16 +33,28 @@ cache_store_object_way=2
 redis_cache_object_size=1024000
 #If CACHE_STORE_OBJECT_WAY is not 0, we will use redis to store meta and object.
 redis_cluster_addrs=10.4.20.211:9001,10.4.20.212:9001,10.4.20.213:9001,10.4.20.214:9001,10.4.20.215:9001,10.4.20.216:9001,10.4.20.217:9001,10.4.20.218:9001
+
 #Configs of WiredLB for Minios load balancer.
-#WIREDLB_OVERRIDE=1
+wiredlb_override=1
 wiredlb_health_port=42310
+wiredlb_topic=MinioFileLog
+wiredlb_datacenter=k18consul-tse
+wiredlb_health_port=52102
+wiredlb_group=FileLog
+
+log_fsstat_appname=tango_log_file
+log_fsstat_filepath=./log/tango_log_file.fs2
+log_fsstat_interval=10
+log_fsstat_trig=1
+log_fsstat_dst_ip=10.4.20.202
+log_fsstat_dst_port=8125
 
 [maat]
 # 0:json 1: redis 2: iris
 maat_input_mode=1
 table_info=resource/pangu/table_info.conf
 json_cfg_file=resource/
-stat_file=log/pangu_scan.status
+stat_file=log/pangu_scan.fs2
 full_cfg_dir=pangu_policy/full/index/
 inc_cfg_dir=pangu_policy/inc/index/
 maat_redis_server=10.4.34.4

platform/include/internal/platform.h

@@ -91,6 +91,7 @@ struct tfe_stream_private
 		struct ssl_stream * ssl_upstream;
 	};
 
+	uint8_t is_first_call_rxcb;
 	uint8_t is_plugin_opened;
 	int calling_idx;
 

platform/include/internal/proxy.h

@@ -17,10 +17,7 @@ enum TFE_STAT_FIELD
 	/* FDs */
 	STAT_FD_OPEN_BY_KNI_ACCEPT,
 	STAT_FD_CLOSE_BY_KNI_ACCEPT_FAIL,
-	/* FDs */
+	STAT_FD_CLOSE,
-	STAT_FD_INSTANT_CLOSE,
-	STAT_FD_DEFER_CLOSE_IN_QUEUE,
-	STAT_FD_DEFER_CLOSE_SUCCESS,
 
 	/* Stream */
 	STAT_STREAM_OPEN,
@@ -31,14 +28,16 @@ enum TFE_STAT_FIELD
 	STAT_STREAM_CLS_UP_ERR,
 	STAT_STREAM_CLS_KILL,
 
-	/* Stream Protocol */
+	/* Action */
+	STAT_STREAM_INTERCEPT,
+	STAT_STREAM_BYPASS,
+	STAT_STREAM_INCPT_BYTES,
+	STAT_STREAM_INCPT_DOWN_BYTES,
+	STAT_STREAM_INCPT_UP_BYTES,
+
+	/* Protocol */
 	STAT_STREAM_TCP_PLAIN,
 	STAT_STREAM_TCP_SSL,
-
-	/* RX DATA */
-	STAT_STREAM_DOWN_RX_BYTES,
-	STAT_STREAM_UP_RX_BYTES,
-
 	TFE_STAT_MAX
 };
 

platform/src/proxy.cpp

@@ -307,9 +307,7 @@ static const char * __str_stat_spec_map[] =
 		[STAT_SIGPIPE] = "SIGPIPE",
 		[STAT_FD_OPEN_BY_KNI_ACCEPT] = "fd_rx",
 		[STAT_FD_CLOSE_BY_KNI_ACCEPT_FAIL] = "fd_rx_err",
-		[STAT_FD_INSTANT_CLOSE] = "fd_inst_cls",
+		[STAT_FD_CLOSE] = "fd_inst_cls",
-		[STAT_FD_DEFER_CLOSE_IN_QUEUE] = "fd_dfr_cls",
-		[STAT_FD_DEFER_CLOSE_SUCCESS] = "fd_dfr_clsd",
 		[STAT_STREAM_OPEN] = "stm_open",
 		[STAT_STREAM_CLS] =  "stm_cls",
 		[STAT_STREAM_CLS_DOWN_EOF] = "dstm_eof",
@@ -317,16 +315,19 @@ static const char * __str_stat_spec_map[] =
 		[STAT_STREAM_CLS_DOWN_ERR] = "dstm_err",
 		[STAT_STREAM_CLS_UP_ERR] = "ustm_err",
 		[STAT_STREAM_CLS_KILL] = "stm_kill",
+		[STAT_STREAM_INTERCEPT] = "stm_incpt",
+		[STAT_STREAM_BYPASS] = "stm_byp",
+		[STAT_STREAM_INCPT_BYTES] = "stm_incpt_B",
+		[STAT_STREAM_INCPT_DOWN_BYTES] = "dstm_incpt_B",
+		[STAT_STREAM_INCPT_UP_BYTES] = "ustm_incpt_B",
 		[STAT_STREAM_TCP_PLAIN] = "plain",
-		[STAT_STREAM_TCP_SSL] = "SSL",
+		[STAT_STREAM_TCP_SSL] = "ssl",
-		[STAT_STREAM_DOWN_RX_BYTES] = "dstm_bytes",
-		[STAT_STREAM_UP_RX_BYTES] = "ustm_bytes",
 		[TFE_STAT_MAX] = NULL
 	};
 
 int tfe_stat_init(struct tfe_proxy * proxy, const char * profile)
 {
-	static const char * fieldstat_output = "./tfe.fieldstat";
+	static const char * fieldstat_output = "log/tfe.fs2";
 	static const char * app_name = "tfe3a";
 
 	int value = 0, i = 0;

platform/src/ssl_stream.cpp

@@ -1971,145 +1971,6 @@ void ssl_async_downstream_create(struct future * f, struct ssl_mgr * mgr, struct
 	return;
 }
 
-/*
- * Cleanly shut down an SSL socket.  Libevent currently has no support for
- * cleanly shutting down an SSL socket so we work around that by using a
- * low-level event.  This works for recent versions of OpenSSL.  OpenSSL
- * with the older SSL_shutdown() semantics, not exposing WANT_READ/WRITE
- * may or may not work.
- */
-UNUSED static struct ssl_shutdown_ctx * ssl_shutdown_ctx_new(struct ssl_stream * s_stream, struct event_base * evbase)
-{
-	struct ssl_shutdown_ctx * ctx = ALLOC(struct ssl_shutdown_ctx, 1);
-	ctx->evbase = evbase;
-	ctx->s_stream = s_stream;
-	ctx->ev = NULL;
-	ctx->mgr = s_stream->mgr;
-	ctx->dir = s_stream->dir;
-	ctx->retries = 0;
-	ctx->dir==CONN_DIR_DOWNSTREAM ? ATOMIC_INC(&(ctx->mgr->stat_val[SSL_DOWN_CLOSING]))
-										: ATOMIC_INC(&(ctx->mgr->stat_val[SSL_UP_CLOSING]));
-
-	return ctx;
-}
-
-static void ssl_shutdown_ctx_free(struct ssl_shutdown_ctx * ctx)
-{
-	ctx->dir==CONN_DIR_DOWNSTREAM ? ATOMIC_DEC(&(ctx->mgr->stat_val[SSL_DOWN_CLOSING]))
-								 : ATOMIC_DEC(&(ctx->mgr->stat_val[SSL_UP_CLOSING]));
-	memset(ctx, 0, sizeof(struct ssl_shutdown_ctx));
-	free(ctx);
-}
-
-/*
- * The shutdown socket event handler.  This is either
- * scheduled as a timeout-only event, or as a fd read or
- * fd write event, depending on whether SSL_shutdown()
- * indicates it needs read or write on the socket.
- */
-static void pxy_ssl_shutdown_cb(evutil_socket_t fd, short what, void * arg)
-{
-	struct ssl_shutdown_ctx * ctx = (struct ssl_shutdown_ctx *) arg;
-	struct timeval retry_delay = {0, 100};
-
-	void * logger = ctx->s_stream->mgr->logger;
-	struct ssl_mgr* mgr=ctx->s_stream->mgr;
-	short want = 0;
-	int rv = 0, sslerr = 0;
-	if (ctx->ev)
-	{
-		event_free(ctx->ev);
-		ctx->ev = NULL;
-	}
-
-	if(what == 0)
-	{
-		TFE_PROXY_STAT_INCREASE(STAT_FD_DEFER_CLOSE_IN_QUEUE, 1);
-	}
-
-	/*
-	 * Use the new (post-2008) semantics for SSL_shutdown() on a
-	 * non-blocking socket.  SSL_shutdown() returns -1 and WANT_READ
-	 * if the other end's close notify was not received yet, and
-	 * WANT_WRITE it could not write our own close notify.
-	 *
-	 * This is a good collection of recent and relevant documents:
-	 * http://bugs.python.org/issue8108
-	 */
-	if(what == EV_TIMEOUT)
-	{
-		SSL_set_shutdown(ctx->s_stream->ssl, SSL_RECEIVED_SHUTDOWN);
-	}
-	rv = SSL_shutdown(ctx->s_stream->ssl);
-
-	if (rv == 1)
-		goto complete;
-
-	if (rv != -1)
-	{
-		goto retry;
-	}
-
-	switch ((sslerr = SSL_get_error(ctx->s_stream->ssl, rv)))
-	{
-		case SSL_ERROR_WANT_READ: want = EV_READ;
-			goto retry;
-		case SSL_ERROR_WANT_WRITE: want = EV_WRITE;
-			goto retry;
-        case SSL_ERROR_ZERO_RETURN:
-		case SSL_ERROR_SYSCALL:
-		case SSL_ERROR_SSL: goto complete;
-		default: TFE_LOG_ERROR(logger, "Unhandled SSL_shutdown() "
-									   "error %i. Closing fd.\n", sslerr);
-			goto complete;
-	}
-
-	goto complete;
-
-retry:
-	if (ctx->retries++ >= MAX_NET_RETRIES)
-	{
-	/*
-		struct tfe_stream_addr* addr=tfe_stream_addr_create_by_fd(fd, ctx->s_stream->dir);
-		char* addr_string=tfe_stream_addr_to_str(addr);
-		TFE_LOG_ERROR(logger, "Failed to shutdown %s SSL connection cleanly: %s "
-							  "Max retries reached. Closing fd %d.",
-							  tfe_stream_conn_dir_to_str(ctx->s_stream->dir),
-							  addr_string, fd);
-		tfe_stream_addr_free(addr);
-		free(addr_string);
-		*/
-		if(ctx->s_stream->dir==CONN_DIR_DOWNSTREAM)
-		{
-			ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_DIRTY_CLOSED]));
-		}
-		else
-		{
-			ATOMIC_INC(&(mgr->stat_val[SSL_UP_DIRTY_CLOSED]));
-		}
-		goto complete;
-	}
-
-	ctx->ev = event_new(ctx->evbase, fd, want, pxy_ssl_shutdown_cb, ctx);
-
-	if (ctx->ev)
-	{
-		event_add(ctx->ev, &retry_delay);
-	}
-	else
-	{
-		TFE_LOG_ERROR(logger, "Failed to shutdown SSL connection cleanly: "
-						  "Cannot create event. Closing fd %d.", fd);
-	}
-	return;
-
-complete:
-	TFE_PROXY_STAT_INCREASE(STAT_FD_DEFER_CLOSE_SUCCESS, 1);
-	ssl_stream_free(ctx->s_stream);
-	evutil_closesocket(fd);
-	ssl_shutdown_ctx_free(ctx);
-}
-
 /*
  * Cleanly shutdown an SSL session on file descriptor fd using low-level
  * file descriptor readiness events on event base evbase.

platform/src/tcp_stream.cpp

@@ -422,7 +422,7 @@ static void __conn_private_destory(struct tfe_conn_private * conn)
 
 	free(conn);
 	(void)ret;
-	TFE_PROXY_STAT_INCREASE(STAT_FD_INSTANT_CLOSE, 1);
+	TFE_PROXY_STAT_INCREASE(STAT_FD_CLOSE, 1);
 }
 
 static void __conn_private_destory_with_ssl(struct event_base * ev_base,
@@ -445,6 +445,12 @@ static void __stream_bev_passthrough_readcb(struct bufferevent * bev, void * arg
 		return;
 	}
 
+	if (_stream->is_first_call_rxcb == 0)
+	{
+		TFE_PROXY_STAT_INCREASE(STAT_STREAM_BYPASS, 1);
+		_stream->is_first_call_rxcb = 1;
+	}
+
 	struct evbuffer * __output_buffer = bufferevent_get_output(peer_conn->bev);
 	evbuffer_add_buffer(__output_buffer, __input_buffer);
 }
@@ -589,6 +595,12 @@ static void __stream_bev_readcb(struct bufferevent * bev, void * arg)
 		return;
 	}
 
+	if (_stream->is_first_call_rxcb == 0)
+	{
+		TFE_PROXY_STAT_INCREASE(STAT_STREAM_INTERCEPT, 1);
+		_stream->is_first_call_rxcb = 1;
+	}
+
 	struct evbuffer * outbuf = bufferevent_get_output(peer_conn->bev);
 	assert(inbuf != NULL && outbuf != NULL);
 
@@ -673,15 +685,18 @@ static void __stream_bev_readcb(struct bufferevent * bev, void * arg)
 
 	if (dir == CONN_DIR_DOWNSTREAM)
 	{
-		TFE_PROXY_STAT_INCREASE(STAT_STREAM_DOWN_RX_BYTES, rx_offset_increase);
+		TFE_PROXY_STAT_INCREASE(STAT_STREAM_INCPT_DOWN_BYTES, rx_offset_increase);
 		_stream->downstream_rx_offset += rx_offset_increase;
 	}
 	else
 	{
-		TFE_PROXY_STAT_INCREASE(STAT_STREAM_UP_RX_BYTES, rx_offset_increase);
+		TFE_PROXY_STAT_INCREASE(STAT_STREAM_INCPT_UP_BYTES, rx_offset_increase);
 		_stream->upstream_rx_offset += rx_offset_increase;
 	}
 
+	/* Total Bytes */
+	TFE_PROXY_STAT_INCREASE(STAT_STREAM_INCPT_BYTES, rx_offset_increase);
+
 	if(_stream->need_to_be_kill)
 	{
 		const static struct linger sl{.l_onoff = 1, .l_linger = 0};
@@ -997,18 +1012,18 @@ void ssl_upstream_create_on_success(future_result_t * result, void * user)
 	enum ssl_stream_action ssl_action = ssl_upstream_create_result_release_action(result);
 	if (SSL_ACTION_PASSTHROUGH == ssl_action)
 	{
+		_stream->tcp_passthough = true;
 		_stream->conn_upstream = __conn_private_create_by_fd(_stream, _stream->defer_fd_upstream);
 		_stream->conn_downstream = __conn_private_create_by_fd(_stream, _stream->defer_fd_downstream);
 		__conn_private_enable(_stream->conn_downstream);
 		__conn_private_enable(_stream->conn_upstream);
 
-		_stream->tcp_passthough = 1;
 		_stream->defer_fd_downstream = 0;
 		_stream->defer_fd_upstream = 0;
 	}
 	else if (SSL_ACTION_SHUTDOWN == ssl_action)
 	{
-		tfe_stream_destory(_stream);
+		return tfe_stream_destory(_stream);
 	}
 	else
 	{
@@ -1232,7 +1247,8 @@ void __stream_fd_option_setup(struct tfe_stream_private * _stream, evutil_socket
 	struct tfe_proxy_tcp_options * tcp_options = &_stream->proxy_ref->tcp_options;
 
 	/* Make it non-blocking */
-	evutil_make_socket_nonblocking(fd);
+	int ret = evutil_make_socket_nonblocking(fd);
+	assert(ret >= 0);
 
 	/* Recv Buffer */
 	if (tcp_options->sz_rcv_buffer >= 0)
@@ -1307,6 +1323,8 @@ void __stream_fd_option_setup(struct tfe_stream_private * _stream, evutil_socket
 		TFE_LOG_ERROR(g_default_logger, "%s: Failed at setup FD's ttl option, ttl = %d, fd = %d",
 			stream->str_stream_info, __ttl, fd);
 	}
+
+	(void)ret;
 }
 
 int tfe_stream_init_by_fds(struct tfe_stream * stream, evutil_socket_t fd_downstream, evutil_socket_t fd_upstream)
@@ -1325,8 +1343,7 @@ int tfe_stream_init_by_fds(struct tfe_stream * stream, evutil_socket_t fd_downst
 	if (unlikely(_stream->head.addr == NULL))
 	{
 		TFE_LOG_ERROR(_stream->stream_logger, "Failed to create address from fd %d, %d, terminate fds.",
-			fd_downstream, fd_upstream);
+			fd_downstream, fd_upstream); goto __errout;
-		goto __errout;
 	}
 
 	_stream->str_stream_addr = tfe_stream_addr_to_str(_stream->head.addr);

plugin/business/pangu-http/src/pangu_web_cache.cpp

@@ -223,7 +223,7 @@ static void set_stat_spec(struct cache_stat_sepc* spec, const char* name, enum f
 void cache_stat_init(struct cache_handle* cache,
 const char* statsd_server_ip, int statsd_server_port, const char*histogram_bins)
 {
-	const char* fieldstat_output="./cache.fieldstat";
+	const char* fieldstat_output="log/cache.fs2";
 	const char* app_name="tfe_cache";
 
 	int value=0, i=0;