gfwleak/doris-doris-dispatch
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加HTTP Post上传配置接口，支持主从双机备份与同步（冷备）

Browse Source
This commit is contained in:
linuxrc@163.com
2021-08-25 18:40:20 +08:00
parent 67bafbefc9
commit 1aca701f12
28 changed files with 3295 additions and 648 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
server/doris_server_receive.h
View File

@@ -4,8 +4,23 @@
#include <stdio.h>
#include <sys/queue.h>
#include <event.h>
#include <openssl/md5.h>
#include <cjson/cJSON.h>
#include <evhttp.h>
#include "doris_producer_client.h"
#if(__GNUC__ * 100 + __GNUC_MINOR__ * 10 + __GNUC_PATCHLEVEL__ >= 410)
#define atomic_inc(x) __sync_add_and_fetch((x),1)
#define atomic_dec(x) __sync_sub_and_fetch((x),1)
#define atomic_add(x,y) __sync_add_and_fetch((x),(y))
#define atomic_sub(x,y) __sync_sub_and_fetch((x),(y))
#define atomic_read(x) __sync_add_and_fetch((x),0)
#define atomic_set(x,y) __sync_lock_test_and_set((x),y)
#else
#error "GCC version should be 4.1.2 later"
#endif
#include <map>
using namespace std;
@@ -20,8 +35,10 @@ enum DORIS_SERVER_FS_FILED
DRS_FSSTAT_CLIENT_META_REQ,
DRS_FSSTAT_SEND_META_NONEW,
DRS_FSSTAT_CLIENT_FILE_REQ,
DRS_FSSTAT_SEND_FILES,
DRS_FSSTAT_SEND_FILE_BYTES,
DRS_FSSTAT_SEND_FILE_RES_404,
DRS_FSSTAT_VERSION_EXPIRES,
DRS_FSSTAT_FIELD_NUM,
};
@@ -31,8 +48,8 @@ enum DORIS_SERVER_FS_COLUMN
DRS_FSCLM_RECV_FULL_VER=0,
DRS_FSCLM_RECV_INC_VER,
DRS_FSCLM_RECV_FILES,
DRS_FSCLM_POST_ON_THE_WAY,
DRS_FSCLM_SEND_META_RES,
DRS_FSCLM_SEND_FILE_RES,
DRS_FSCLM_CUR_FULL_VERSION,
DRS_FSCLM_CUR_INC_VERSION,
DRS_FSCLM_CONFIG_TOTAL_NUM,
@@ -58,6 +75,14 @@ struct cont_frag_node
TAILQ_ENTRY(cont_frag_node) frag_node;
};
struct internal_tablemeta
{
char tablename[64];
char filename[64];
size_t offset;
int32_t islast;
};
struct table_list_node
{
char tablename[64];
@@ -65,20 +90,49 @@ struct table_list_node
size_t filesize;
size_t cur_totallen;
/*this part for http post server*/
bool onceupload;
int32_t finished;
MD5_CTX md5ctx;
char tmppath[256];
char filename[128];
char fragmd5[36];
u_int32_t cfgnum;
char *fragcontent;
size_t fragsize;
cJSON *table_meta;
FILE *fp_cfg_file;
struct cont_frag_node *cur_frag;
TAILQ_HEAD(__table_cont_node, cont_frag_node) frag_head;
TAILQ_ENTRY(table_list_node) table_node;
};
struct doris_business;
struct version_list_node
{
int64_t version;
char *metacont;
int32_t metalen;
int16_t cfg_type; //1-full, 2-inc
int16_t cont_in_disk;
int8_t cont_in_disk;
int8_t version_finished;
cJSON *metajson, *arrayjson;
cJSON *table_meta;
/*this part for http post server*/
FILE *fp_idx_file;
char token[64];
char tmp_index_path[256]; //incĿ¼<C4BF>µ<EFBFBD><C2B5><EFBFBD><EFBFBD><EFBFBD>ȫ<EFBFBD><C8AB><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʱÿ<CAB1><C3BF><EFBFBD>汾<EFBFBD><E6B1BE>һ<EFBFBD><D2BB>
int16_t retry_times;
int16_t syncing;
int32_t total_cfgs;
struct doris_business *business;
struct evhttp_request *req;
struct doris_upload_ctx *synctx;
struct table_list_node *cur_table;
struct event timer_expire;
TAILQ_HEAD(__table_list_node, table_list_node) table_head;
TAILQ_ENTRY(version_list_node) version_node;
};
@@ -95,35 +149,21 @@ struct version_list_handle
struct version_list_handle *config_version_handle_new(void);
struct doris_business;
struct confile_save
{
struct event_base *evbase;
struct doris_business *business;
int64_t version;
int32_t source_from;
int32_t type;
int64_t version_cfgnum;
char inc_index_path[256]; //incĿ¼<C4BF>µ<EFBFBD><C2B5><EFBFBD><EFBFBD><EFBFBD>ȫ<EFBFBD><C8AB>
char tmp_index_path[256]; //incĿ¼<C4BF>µ<EFBFBD><C2B5><EFBFBD><EFBFBD><EFBFBD>ȫ<EFBFBD><C8AB>
char full_index_path[256]; //fullĿ¼<C4BF>µ<EFBFBD>ȫ<EFBFBD><C8AB>
char cfg_file_path[256];
FILE *fp_cfg_file;
FILE *fp_idx_file;
struct version_list_node *cur_vernode;
struct table_list_node *cur_table;
struct cont_frag_node *cur_frag;
};
struct common_timer_event
{
struct event timer_event;
void *data;
};
struct doris_business;
struct bufferevent *doris_https_bufferevent_cb(struct event_base *evabse, void *arg);
struct doris_business *lookup_bizstruct_from_name(const struct evkeyvalq *params);
void business_set_sync_peer_abnormal(struct doris_business *business);
void business_resume_sync_peer_normal(struct doris_business *business);
void* thread_doris_client_recv_cfg(void *arg);
void* thread_index_file_recv_cfg(void *arg);
void* thread_http_post_recv_cfg(void *arg);
#endif