1. 将trylock替换为lock 2. 修改测试程序

.gitlab-ci.yml

@@ -3,6 +3,7 @@ variables:
   GIT_STRATEGY: "clone"
   BUILD_PADDING_PREFIX: /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/
   INSTALL_PREFIX: "/opt/MESA/lib/"
+  INSTALL_DEPENDENCY_LIBRARY:  ""
 
 stages:
 - build
@@ -49,6 +50,10 @@ develop_build_debug:
   variables:
     BUILD_TYPE: Debug
     PACKAGE: 1
+    UPLOAD_RPM: 1
+    ASAN_OPTION: ADDRESS
+    PULP3_REPO_NAME: framework-testing-x86_64.el7
+    PULP3_DIST_NAME: framework-testing-x86_64.el7
   artifacts:
     name: "MESA_handle_logger-$CI_COMMIT_REF_NAME-debug"
     paths:
@@ -64,6 +69,10 @@ develop_build_release:
   variables:
     BUILD_TYPE: RelWithDebInfo
     PACKAGE: 1
+    UPLOAD_RPM: 1
+    ASAN_OPTION: "OFF"
+    PULP3_REPO_NAME: framework-testing-x86_64.el7
+    PULP3_DIST_NAME: framework-testing-x86_64.el7
   artifacts:
     name: "MESA_handle_logger-$CI_COMMIT_REF_NAME-release"
     paths:
@@ -74,12 +83,30 @@ develop_build_release:
     - /^master.*$/i
 
 
+release_build_debug:
+  stage: build
+  variables:
+    BUILD_TYPE:  Debug
+    PACKAGE: 1
+    UPLOAD_RPM: 1
+    PULP3_REPO_NAME: framework-stable-x86_64.el7
+    PULP3_DIST_NAME: framework-stable-x86_64.el7
+  extends: .build_by_travis
+  artifacts:
+    name: "MESA_handle_logger-$CI_COMMIT_REF_NAME-release"
+    paths:
+      - build/*.rpm
+  only:
+    - tags
+
 release_build_release:
   stage: build
   variables:
     BUILD_TYPE: RelWithDebInfo
     PACKAGE: 1
-    UPLOAD: 1
+    UPLOAD_RPM: 1
+    UPLOAD_SYMBOL_FILES: 1
+    SYMBOL_TARGET: libMESA_handle_logger
     PULP3_REPO_NAME: framework-stable-x86_64.el7
     PULP3_DIST_NAME: framework-stable-x86_64.el7
   extends: .build_by_travis
@@ -88,58 +115,4 @@ release_build_release:
     paths:
       - build/*.rpm
   only:
-    - tags
-  except:
-    - /^v[0-9].*-testing$/i    
-
-release_build_release_devel:
-  stage: build
-  variables:
-    BUILD_TYPE: RelWithDebInfo
-    ENABLE_DEVEL_SWITCH: "ON"
-    PACKAGE: 1
-    UPLOAD: 1
-    PULP3_REPO_NAME: framework-stable-x86_64.el7
-    PULP3_DIST_NAME: framework-stable-x86_64.el7
-  extends: .build_by_travis
-  artifacts:
-    name: "MESA_handle_logger-$CI_COMMIT_REF_NAME-release"
-    paths:
-      - build/*.rpm
-  only:
-    - tags
-  except:
-    - /^v[0-9].*-testing$/i    
-
-testing_build_release:
-  stage: build
-  variables:
-    BUILD_TYPE: RelWithDebInfo
-    PACKAGE: 1
-    UPLOAD: 1
-    PULP3_REPO_NAME: framework-testing-x86_64.el7
-    PULP3_DIST_NAME: framework-testing-x86_64.el7
-  extends: .build_by_travis
-  artifacts:
-    name: "MESA_handle_logger-$CI_COMMIT_REF_NAME-release"
-    paths:
-      - build/*.rpm
-  only:
-    - /^v[0-9].*-testing$/i   
-
-testing_build_release_devel:
-  stage: build
-  variables:
-    BUILD_TYPE: RelWithDebInfo
-    ENABLE_DEVEL_SWITCH: "ON"
-    PACKAGE: 1
-    UPLOAD: 1
-    PULP3_REPO_NAME: framework-testing-x86_64.el7
-    PULP3_DIST_NAME: framework-testing-x86_64.el7
-  extends: .build_by_travis
-  artifacts:
-    name: "MESA_handle_logger-$CI_COMMIT_REF_NAME-release"
-    paths:
-      - build/*.rpm
-  only:
-    - /^v[0-9].*-testing$/i  
+    - tags

ci/travis.sh

@@ -33,7 +33,14 @@ env | sort
 : "${COMPILER_IS_GNUCXX:=OFF}"
 
 # Install dependency from YUM
-#yum install -y mrzcpd framework numactl-devel zlib-devel librdkafka-devel systemd-devel
+if [ -n "${INSTALL_DEPENDENCY_LIBRARY}" ]; then
+      yum install -y $INSTALL_DEPENDENCY_LIBRARY
+fi
+
+if [ $ASAN_OPTION ];then
+	source /opt/rh/devtoolset-7/enable
+fi
+
 mkdir build || true
 cd build
 
@@ -41,6 +48,8 @@ cmake3 -DCMAKE_CXX_FLAGS=$CXX_FLAGS \
       -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
       -DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX \
       -DENABLE_DEVEL=$ENABLE_DEVEL_SWITCH \
+      -DASAN_OPTION=$ASAN_OPTION \
+      -DVERSION_DAILY_BUILD=$TESTING_VERSION_BUILD \
       ..
 
 make
@@ -49,13 +58,14 @@ if [ -n "${PACKAGE}" ]; then
 	make package
 fi
 
-if [ -n "${UPLOAD}" ]; then	
+if [ -n "${UPLOAD_RPM}" ]; then	
 	cp ~/rpm_upload_tools.py ./
 	python3 rpm_upload_tools.py ${PULP3_REPO_NAME} ${PULP3_DIST_NAME} *.rpm
 fi
 
-#if [ -n "${UPLOAD_SYMBOL_FILES}" ]; then
-#  rpm -i tfe*debuginfo*.rpm
-#  cp /usr/lib/debug/opt/tsg/tfe/bin/tfe.debug /tmp/tfe.debuginfo.${CI_COMMIT_SHORT_SHA}
-#  sentry-cli upload-dif -t elf /tmp/tfe.debuginfo.${CI_COMMIT_SHORT_SHA}
-#fi
+if [ -n "${UPLOAD_SYMBOL_FILES}" ]; then
+	rpm -i $SYMBOL_TARGET*debuginfo*.rpm
+	_symbol_file=`find /usr/lib/debug/ -name "$SYMBOL_TARGET*.so*.debug"`
+	cp $_symbol_file ${_symbol_file}info.${CI_COMMIT_SHORT_SHA}
+	sentry-cli upload-dif -t elf ${_symbol_file}info.${CI_COMMIT_SHORT_SHA}
+fi

demo/test_handle_logger.c

@@ -16,11 +16,12 @@ void *test_handle = NULL;
 int g_mode = 0;
 int g_log_num = 0;
 int g_thread_num = 0;
+int g_log_print_way = 0;
 const char *g_zlog_conf = NULL;
 volatile long g_start_time = 0;
 volatile long g_end_time = 0;
 
-void call_logger(int log_num, int thread_num)
+void call_logger(int log_num, int thread_num, int way)
 {
 		int i = 0;
 		struct timespec start, end;
@@ -40,12 +41,25 @@ void call_logger(int log_num, int thread_num)
 		}	
 		for(i = 0; i < log_num; i++)
 		{
-				MESA_handle_runtime_log(sample_handle, RLOG_LV_DEBUG, "sample", "sample_handle MESA_handle_runtime_log, i = %d, thread_num = %d", i, thread_num);
-				//sleep(1);
+			switch (way)
+			{
+			case 1:
+				MESA_handle_runtime_log(sample_handle, RLOG_LV_DEBUG, "sample", "sample_handle RUNTIEM_LOG test, i = %d, thread_num = %d", i, thread_num);
 				MESA_handle_runtime_log(test_handle, RLOG_LV_INFO, "test", "test_handle MESA_handle_runtime_log, i = %d, thread_num = %d", i, thread_num);
-				//MESA_HANDLE_RUNTIME_LOG(sample_handle, RLOG_LV_FATAL, "sample", "sample_handle RUNTIEM_LOG test, i = %d, thread_num = %d", i, thread_num);
-				//sleep(1);
-				//MESA_HANDLE_RUNTIME_LOG(test_handle, RLOG_LV_FATAL, "test", "test_handle RUNTIEM_LOG test, i = %d, thread_num = %d", i, thread_num);
+				break;
+			case 2:
+				MESA_HANDLE_RUNTIME_LOG_RATELIMIT(sample_handle, RLOG_LV_DEBUG, "sample", "sample_handle RUNTIEM_LOG test, i = %d, thread_num = %d", i, thread_num);
+				MESA_HANDLE_RUNTIME_LOG_RATELIMIT(test_handle, RLOG_LV_INFO, "test", "test_handle RUNTIEM_LOG test, i = %d, thread_num = %d", i, thread_num);
+				break;
+			case 3:
+				MESA_HANDLE_RUNTIME_LOG(sample_handle, RLOG_LV_DEBUG, "sample", "sample_handle RUNTIEM_LOG test, i = %d, thread_num = %d", i, thread_num);
+				MESA_HANDLE_RUNTIME_LOG(test_handle, RLOG_LV_INFO, "test", "test_handle RUNTIEM_LOG test, i = %d, thread_num = %d", i, thread_num);
+				break;
+			default:
+				break;
+			}
+			// if (i % 10000 == 0)
+			// 	sleep(1);
 		}
 		clock_gettime(CLOCK_MONOTONIC, &end);
 		end_time = end.tv_sec*1000000 + end.tv_nsec/1000;
@@ -70,7 +84,7 @@ void *thread_logger(void *arg)
 {
 		int thread_num = (int)(unsigned long long)arg;
 		printf("thread %d created! \n", thread_num);
-		call_logger(g_log_num, thread_num);
+		call_logger(g_log_num, thread_num, g_log_print_way);
 		printf("thread %d finished! \n", thread_num);
 		return NULL;
 }
@@ -96,20 +110,39 @@ void sig_hup_handler(int sig)
 	MESA_handle_runtime_log_reconstruction(g_zlog_conf);
 }
 
+void sig_user1_handler(int sig)
+{
+	printf("SIGUSR1 recviced! set interval:3 burst:5\n");
+	MESA_set_runtime_log_handle_ratelimit(sample_handle, 3, 5);
+}
+
+void sig_user2_handler(int sig)
+{
+	printf("SIGUSR2 recviced! set interval:5 burst:3\n");
+	MESA_set_runtime_log_handle_ratelimit(sample_handle, 5, 3);
+}
+
+void sig_min_handler(int sig)
+{
+	printf("SIGRTMIN recviced! unlimit\n");
+	MESA_set_runtime_log_handle_ratelimit(sample_handle, 0, 3);
+}
+
 int main(int argc, char ** args)
 {
 	pthread_t t[MAX_THREAD_NUM];
 	int i = 0;
 
-	if (argc != 5)
+	if (argc != 6)
 	{
-		printf("Usage: ./($app) $mode[1 or 2] $zlog_conf_path $thread_num $log_num \n");
+		printf("Usage: ./($app) $mode[1 or 2] $zlog_conf_path $thread_num $log_num $log_print_way[1 2 3]\n");
 		return -1;
 	}
 	g_mode = atoi(args[1]);
 	g_zlog_conf = args[2];
 	g_thread_num = atoi(args[3]);
 	g_log_num = atoi(args[4]);
+	g_log_print_way = atoi(args[5]);
 
 	if(g_thread_num <= 0 || g_log_num <= 0)
 	{
@@ -144,6 +177,9 @@ int main(int argc, char ** args)
 		}
 		signal(SIGINT, sig_int_handler);
 		signal(SIGHUP, sig_hup_handler);
+		signal(SIGUSR1, sig_user1_handler);
+		signal(SIGUSR2, sig_user2_handler);
+		signal(SIGRTMIN, sig_min_handler);
 		while(1)
 		;
 		//MESA_destroy_runtime_log_handle(sample_handle);

inc/MESA_handle_logger.h

@@ -14,10 +14,21 @@ extern "C"
 {
 #endif
 
+#include <pthread.h>
+
 #define RLOG_LV_DEBUG       10
 #define RLOG_LV_INFO        20
 #define RLOG_LV_FATAL       30
 
+typedef struct log_ratelimit_s
+{
+    int		interval;
+    int		burst;
+    int		printed;
+    int		missed;
+    long	begin;
+    pthread_mutex_t mutex;
+}log_ratelimit_t;
 
 int MESA_handle_runtime_log_creation(const char *conf_path);
 int MESA_handle_runtime_log_reconstruction(const char *conf_path);
@@ -27,6 +38,15 @@ void MESA_handle_runtime_log_destruction();
     MESA_handle_runtime_log((handle), (lv), (mod), "file %s, line %d, " fmt, \
                        __FILE__, __LINE__, ##args)
 
+#define MESA_HANDLE_RUNTIME_LOG_RATELIMIT(handle, lv, mod, fmt, args...)    \
+({                                  \
+    static log_ratelimit_t _rs = {  \
+        .mutex = PTHREAD_MUTEX_INITIALIZER,  \
+    };   \
+    if (MESA_runtime_log_ratelimit((handle), (&_rs), (lv), (mod), (__func__)))   \
+        MESA_handle_runtime_log((handle), (lv), (mod), fmt, ##args);    \
+})
+
     /*
  * name: MESA_create_runtime_log_handle
  * functionality: get runtime_log handle;
@@ -63,6 +83,18 @@ void MESA_handle_runtime_log(void *handle, int level, const char *module, const
  */
 void MESA_destroy_runtime_log_handle(void *handle);
 
+/*
+ * name: MESA_set_runtime_log_handle_ratelimit
+ * functionality: set not more than @burst in every @interval when appends log message
+ * params:
+ *  handle: runtime log handle which is going to be released;
+ *  interval: interval time(second). if set '0', it means unlimit.
+ *  burst: in interval time max count
+ * returns:
+ *  none;
+ */
+void MESA_set_runtime_log_handle_ratelimit(void *handle, int interval, int burst);
+
 #ifdef __cplusplus
 }
 #endif

src/MESA_handle_logger.c

@@ -9,6 +9,9 @@
 #include <time.h>
 
 #define MAX_HANDLE_LOG_PATH 4096
+#define DEFAULT_BURST       10
+#define DEFAULT_INTERVAL    5
+
 static int g_zlog_inited = 0;
 static int g_zlog_conf_fp = -1;
 static char global_conf_filepath[MAX_HANDLE_LOG_PATH] = "";
@@ -16,6 +19,9 @@ static char tmp_conf_filepath[MAX_HANDLE_LOG_PATH] = "";
 
 typedef struct log_handle_t
 {
+    int interval;
+    int burst;
+    pthread_mutex_t mutex;
     int runtime_log_level;
     zlog_category_t *zc;
     const char *global_conf_path;
@@ -47,7 +53,6 @@ static __attribute__((__used__)) const char * GIT_VERSION_UNKNOWN = NULL;
 }
 #endif
 
-
 static int create_dir(const char *dir_path, int path_len)
 {
     if(dir_path == NULL)
@@ -156,13 +161,67 @@ static void snapshot_handle_info(const char *handle_name, const char *log_path,
 
     if(g_zlog_inited == 0)
     {
-        printf("[snapshot_handle_info], Nobody call MESA_handle_runtime_log_creation before, Using (%s) to init zlog, handle_name(%s), log_path(%s), log_level(%d)!!!\n", tmp_conf_filepath, handle_name, log_path, level);
+        fprintf(stderr,"[snapshot_handle_info], Nobody call MESA_handle_runtime_log_creation before, Using (%s) to init zlog, handle_name(%s), log_path(%s), log_level(%d)!!!\n", tmp_conf_filepath, handle_name, log_path, level);
         zlog_init(tmp_conf_filepath);
         zlog_reload(NULL);
     }
     return;
 }
 
+static void log_update_time(log_handle_t *p_handle, log_ratelimit_t *rs, int level, const char *module, const char *func)
+{
+    struct timespec current = {0, 0};
+    clock_gettime(CLOCK_MONOTONIC, &current);
+    if (((long)(rs->begin + rs->interval - current.tv_sec)) < 0)
+    {
+        if (rs->missed)
+        {
+            zlog(p_handle->zc, p_handle->runtime_log_file, strlen(p_handle->runtime_log_file), module, strlen(module), __LINE__, level, "%s: %d callbacks suppressed\n", func, rs->missed);
+            rs->missed = 0;
+		}
+        rs->begin = current.tv_sec;
+        rs->printed = 0;
+    }
+    return ;
+}
+
+int MESA_runtime_log_ratelimit(void *handle, log_ratelimit_t *rs, int level, const char *module, const char *func)
+{
+    int ret = 0;
+    struct timespec timer = {0, 0};
+    log_handle_t *p_handle = (log_handle_t *)handle;
+
+    if (pthread_mutex_lock(&rs->mutex) != 0)return 0;
+
+    if (!rs->begin)
+    {
+        clock_gettime(CLOCK_MONOTONIC, &timer);
+        rs->begin = timer.tv_sec;
+    }
+    rs->burst = p_handle->burst;
+    rs->interval = p_handle->interval;
+    if (!rs->interval)
+    {
+        pthread_mutex_unlock(&rs->mutex);
+        return 1;
+    }
+
+    log_update_time(p_handle, rs, level, module, func);
+
+    if (rs->burst && rs->burst > rs->printed)
+    {
+        rs->printed++;
+        ret = 1;
+	}
+    else
+    {
+        rs->missed++;
+        ret = 0;
+    }
+    pthread_mutex_unlock(&rs->mutex);
+    return ret;
+}
+
 void *MESA_create_runtime_log_handle(const char *file_path, int level)
 {
 	if(file_path == NULL)
@@ -193,19 +252,25 @@ void *MESA_create_runtime_log_handle(const char *file_path, int level)
     zc = zlog_get_category(p_name);
     if (!zc)
     {
-        printf("[MESA_create_runtime_log_handle], get zlog category (%s) in global_conf_filepath(%s) fail\n", p_name, global_conf_filepath);
+        fprintf(stderr,"[MESA_create_runtime_log_handle], get zlog category (%s) in global_conf_filepath(%s) fail\n", p_name, global_conf_filepath);
     }
     p_handle = (log_handle_t *)calloc(sizeof(log_handle_t), 1);
     strncpy(p_handle->runtime_log_file, file_path, sizeof(p_handle->runtime_log_file) - 1);
     p_handle->runtime_log_level = level;
     p_handle->zc = zc;
+    p_handle->burst = DEFAULT_BURST;
+    p_handle->interval = DEFAULT_INTERVAL;
+    pthread_mutex_init(&p_handle->mutex, NULL);
+    
     return (void *)p_handle;
 }
 
 void MESA_destroy_runtime_log_handle(void *handle)
 {
+    log_handle_t *p_handle = (log_handle_t *)handle;
     if(handle != NULL)
     {
+        pthread_mutex_destroy(&p_handle->mutex);
         free(handle);
         handle = NULL;
     }
@@ -213,23 +278,34 @@ void MESA_destroy_runtime_log_handle(void *handle)
     return;
 }
 
-
 void MESA_handle_runtime_log(void *handle, int level, const char *module, const char *fmt, ...)
 {
-    
+    va_list ap;
     log_handle_t *p_handle = (log_handle_t *)handle;
 
     if(p_handle == NULL || p_handle->runtime_log_file == NULL)return;
 
     if(p_handle->zc == NULL)return;
 
-    va_list ap;
     va_start(ap, fmt);
     vzlog(p_handle->zc, p_handle->runtime_log_file, strlen(p_handle->runtime_log_file), module, strlen(module), __LINE__, level, fmt, ap);
     va_end(ap);
+    
     return ;
 }
 
+void MESA_set_runtime_log_handle_ratelimit(void *handle, int interval, int burst)
+{
+    log_handle_t *p_handle = (log_handle_t *)handle;
+
+    if ((p_handle == NULL) || (interval < 0) || (burst < 0))return;
+
+    pthread_mutex_lock(&p_handle->mutex);
+    p_handle->interval = interval;
+    p_handle->burst = burst;
+    pthread_mutex_unlock(&p_handle->mutex);
+    return;
+}
 
 int MESA_handle_runtime_log_creation(const char *conf_path)
 {
@@ -237,11 +313,11 @@ int MESA_handle_runtime_log_creation(const char *conf_path)
     int rc = 0;
     if (conf_path == NULL || (access(conf_path, R_OK) != 0))
     {
-        printf("[MESA_handle_runtime_log_creation],  PATH (%s) unable to access, will output log to STDOUT or Using ZLOG_CONF_PATH (%s) !\n", conf_path, env);
+        fprintf(stderr,"[MESA_handle_runtime_log_creation],  PATH (%s) unable to access, will output log to STDOUT or Using ZLOG_CONF_PATH (%s) !\n", conf_path, env);
         rc = zlog_init(NULL);
         if (rc)
         {
-            printf("[MESA_handle_runtime_log_creation], INIT zlog by (NULL) or (%s) failed !!! \n", env);
+            fprintf(stderr,"[MESA_handle_runtime_log_creation], INIT zlog by (NULL) or (%s) failed !!! \n", env);
             return -1;
         }
         else
@@ -257,11 +333,11 @@ int MESA_handle_runtime_log_creation(const char *conf_path)
         rc = zlog_init(conf_path);
         if (rc)
         {
-            printf("[MESA_handle_runtime_log_creation], INIT zlog by (%s) failed , using [NULL] retry !\n", conf_path);
+            fprintf(stderr,"[MESA_handle_runtime_log_creation], INIT zlog by (%s) failed , using [NULL] retry !\n", conf_path);
             rc = zlog_init(NULL);
             if (rc)
             {
-                printf("[MESA_handle_runtime_log_creation], INIT zlog by (NULL) or (%s) failed \n", env);
+                fprintf(stderr,"[MESA_handle_runtime_log_creation], INIT zlog by (NULL) or (%s) failed \n", env);
                 return -1;
             }
             else
@@ -279,7 +355,7 @@ int MESA_handle_runtime_log_creation(const char *conf_path)
         }
         
     }
-    printf("[MESA_handle_runtime_log_creation], INIT zlog finish, Using (%s). \n", global_conf_filepath);
+    fprintf(stderr,"[MESA_handle_runtime_log_creation], INIT zlog finish, Using (%s). \n", global_conf_filepath);
     g_zlog_inited = 1;
     zlog_profile();