gfwleak/doris-doris-dispatch
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加HTTP Post上传配置接口，支持主从双机备份与同步（冷备）

Browse Source
This commit is contained in:
linuxrc@163.com
2021-08-25 18:40:20 +08:00
parent 67bafbefc9
commit 1aca701f12
28 changed files with 3295 additions and 648 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
include/doris_client.h
View File

@@ -1,79 +0,0 @@
#ifndef __DORIS_CLIENT_H__
#define __DORIS_CLIENT_H__
#include <event.h>
#include <cjson/cJSON.h>
enum FSSTAT_DORIS_FILED_ITEMS
{
DRS_FS_FILED_REQ_FAIL=0,
DRS_FS_FILED_REQ_META,
DRS_FS_FILED_RES_META,
DRS_FS_FILED_RES_NOMETA,
DRS_FS_FILED_REQ_FILES,
DRS_FS_FILED_RES_FILES,
DRS_FS_FILED_RES_FRAGS,
DRS_FS_FILED_RES_FRAGERR,
DRS_FS_FILED_RES_BYTES,
DRS_FS_FILED_RES_VERERR,
DRS_FS_FILED_BACKUP1_REQ,
DRS_FS_FILED_BACKUP2_REQ,
FSSTAT_DORIS_FILED_NUM,
};
enum FSSTAT_DORIS_STATUS_ITEMS
{
DRS_FS_STAT_MST_CNN_SRV=0,
DRS_FS_STAT_MST_FAIL_SRV,
DRS_FS_STAT_BCK1_CNN_SRV,
DRS_FS_STAT_BCK1_FAIL_SRV,
DRS_FS_STAT_BCK2_CNN_SRV,
DRS_FS_STAT_BCK2_FAIL_SRV,
DRS_FS_STAT_MEMORY_USED,
DRS_FS_STAT_HTTP_SESSIONS,
FSSTAT_DORIS_STATUS_NUM,
};
struct doris_statistics
{
long long field[FSSTAT_DORIS_FILED_NUM];
long long status[FSSTAT_DORIS_STATUS_NUM];
};
struct doris_arguments
{
char bizname[32];
int64_t current_version; //<2F><>ǰ<EFBFBD>ѻ<EFBFBD>ȡ<EFBFBD><C8A1><EFBFBD>ϵ<EFBFBD><CFB5><EFBFBD><EFBFBD>°汾<C2B0>ţ<EFBFBD><C5A3><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>һ<EFBFBD><D2BB><EFBFBD>汾ȡ<E6B1BE><C8A1><EFBFBD><EFBFBD>
int32_t judian_id;
};
struct tablemeta
{
const char *tablename; //<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʽmaat<61><74><EFBFBD>䣻<EFBFBD><E4A3BB><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʽΪ<CABD>ļ<EFBFBD><C4BC><EFBFBD>
const char *filename; //<2F><><EFBFBD><EFBFBD>ʱҪ<CAB1><D2AA><EFBFBD><EFBFBD><EFBFBD>ļ<EFBFBD><C4BC><EFBFBD>
const char *userregion;
size_t size;
u_int32_t cfgnum;
};
struct doris_instance;
struct doris_callbacks
{
void *userdata;
void (*version_start)(struct doris_instance *instance, cJSON *meta, void *userdata); //meta<74><61><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>汾<EFBFBD><E6B1BE><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ڶ<EFBFBD><DAB6><EFBFBD>Ч
void (*cfgfile_start)(struct doris_instance *instance, const struct tablemeta *meta, const char *unused, void *userdata);
void (*cfgfile_update)(struct doris_instance *instance, const char *data, size_t len, void *userdata);
void (*cfgfile_finish)(struct doris_instance *instance, const char *md5, void *userdata);
void (*version_error)(struct doris_instance *instance, void *userdata); //<2F><><EFBFBD><EFBFBD><EFBFBD>ļ<EFBFBD>ʧ<EFBFBD>ܣ<EFBFBD><DCA3>ð汾<C3B0><E6B1BE>Ҫ<EFBFBD>ع<EFBFBD>
void (*version_finish)(struct doris_instance *instance, void *userdata);
};
struct doris_parameter *doris_parameter_new(const char *confile, struct event_base *manage_evbase, void *runtimelog);
struct doris_instance *doris_instance_new(struct doris_parameter *param, struct event_base *worker_evbase,
struct doris_callbacks *cbs, struct doris_arguments *args, void *runtimelog);
#endif

86
include/doris_consumer_client.h Normal file
View File

@@ -0,0 +1,86 @@
#ifndef __DORIS_CONSUMER_CLIENT_H__
#define __DORIS_CONSUMER_CLIENT_H__
#include <event.h>
#include <cjson/cJSON.h>
enum FSSTAT_DORIS_FILED_ITEMS
{
DRS_FS_FILED_REQ_FAIL=0,
DRS_FS_FILED_REQ_META,
DRS_FS_FILED_RES_META,
DRS_FS_FILED_RES_NOMETA,
DRS_FS_FILED_REQ_FILES,
DRS_FS_FILED_RES_FILES,
DRS_FS_FILED_RES_FRAGS,
DRS_FS_FILED_RES_FRAGERR,
DRS_FS_FILED_RES_BYTES,
DRS_FS_FILED_RES_VERERR,
DRS_FS_FILED_BACKUP1_REQ,
DRS_FS_FILED_BACKUP2_REQ,
FSSTAT_DORIS_FILED_NUM,
};
enum FSSTAT_DORIS_STATUS_ITEMS
{
DRS_FS_STAT_MST_CNN_SRV=0,
DRS_FS_STAT_MST_FAIL_SRV,
DRS_FS_STAT_BCK1_CNN_SRV,
DRS_FS_STAT_BCK1_FAIL_SRV,
DRS_FS_STAT_BCK2_CNN_SRV,
DRS_FS_STAT_BCK2_FAIL_SRV,
DRS_FS_STAT_MEMORY_USED,
DRS_FS_STAT_HTTP_SESSIONS,
FSSTAT_DORIS_STATUS_NUM,
};
struct doris_csum_statistics
{
long long field[FSSTAT_DORIS_FILED_NUM];
long long status[FSSTAT_DORIS_STATUS_NUM];
};
struct doris_arguments
{
char bizname[32];
int64_t current_version; //<2F><>ǰ<EFBFBD>ѻ<EFBFBD>ȡ<EFBFBD><C8A1><EFBFBD>ϵ<EFBFBD><CFB5><EFBFBD><EFBFBD>°汾<C2B0>ţ<EFBFBD><C5A3><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>һ<EFBFBD><D2BB><EFBFBD>汾ȡ<E6B1BE><C8A1><EFBFBD><EFBFBD>
int32_t judian_id;
};
struct tablemeta
{
const char *tablename; //<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʽmaat<61><74><EFBFBD>䣻<EFBFBD><E4A3BB><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʽΪ<CABD>ļ<EFBFBD><C4BC><EFBFBD>
const char *filename; //<2F><><EFBFBD><EFBFBD>ʱҪ<CAB1><D2AA><EFBFBD><EFBFBD><EFBFBD>ļ<EFBFBD><C4BC><EFBFBD>
const char *userregion;
size_t size;
u_int32_t cfgnum;
};
struct doris_csum_param;
struct doris_csum_instance;
struct doris_callbacks
{
void *userdata;
void (*version_start)(struct doris_csum_instance *instance, cJSON *meta, void *userdata); //meta<74><61><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>汾<EFBFBD><E6B1BE><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ڶ<EFBFBD><DAB6><EFBFBD>Ч
void (*cfgfile_start)(struct doris_csum_instance *instance, const struct tablemeta *meta, const char *unused, void *userdata);
void (*cfgfile_update)(struct doris_csum_instance *instance, const char *data, size_t len, void *userdata);
void (*cfgfile_finish)(struct doris_csum_instance *instance, const char *md5, void *userdata);
void (*version_error)(struct doris_csum_instance *instance, void *userdata); //<2F><><EFBFBD><EFBFBD><EFBFBD>ļ<EFBFBD>ʧ<EFBFBD>ܣ<EFBFBD><DCA3>ð汾<C3B0><E6B1BE>Ҫ<EFBFBD>ع<EFBFBD>
void (*version_finish)(struct doris_csum_instance *instance, void *userdata);
void (*version_updated)(struct doris_csum_instance *instance, void *userdata); //<2F><>ʱû<CAB1><C3BB><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
};
struct doris_csum_param *doris_csum_parameter_new(const char *confile, struct event_base *manage_evbase, void *runtimelog);
void doris_csum_parameter_destroy(struct doris_csum_param *param);
u_int32_t doris_csum_param_get_refernces(struct doris_csum_param *param);
struct doris_csum_instance *doris_csum_instance_new(struct doris_csum_param *param, struct event_base *worker_evbase,
struct doris_callbacks *cbs, struct doris_arguments *args, void *runtimelog);
struct doris_csum_param *doris_csum_instance_get_param(struct doris_csum_instance *instance);
void doris_csum_instance_destroy(struct doris_csum_instance *instance);
#endif

80
include/doris_producer_client.h Normal file
View File

@@ -0,0 +1,80 @@
#ifndef __DORIS_PRODUCER_CLIENT_H__
#define __DORIS_PRODUCER_CLIENT_H__
enum FSSTAT_DRS_PROD_FILED_ITEMS
{
DRS_FSPRD_FILED_VERSTART=0,
DRS_FSPRD_FILED_VEREND,
DRS_FSPRD_FILED_VERCANCEL,
DRS_FSPRD_FILED_FILEFRAG,
DRS_FSPRD_FILED_FILEONCE,
FSSTAT_DORIS_PRD_FILED_NUM,
};
enum FSSTAT_DRS_PROD_STATUS_ITEMS
{
DRS_FSPRD_STAT_CNNED_SERVERS=0,
DRS_FSPRD_STAT_FAILED_SERVERS,
DRS_FSPRD_STAT_MEMORY_USED,
DRS_FSPRD_STAT_HTTP_SESSIONS,
DRS_FSPRD_STAT_REQ_SESSIONS,
FSSTAT_DORIS_PRD_STATUS_NUM,
};
struct doris_prod_statistics
{
long long field[FSSTAT_DORIS_PRD_FILED_NUM];
long long status[FSSTAT_DORIS_PRD_STATUS_NUM];
};
struct doris_prod_param;
struct doris_prod_instance;
struct doris_upload_ctx;
struct doris_prod_param *doris_prod_parameter_new(const char *confile, struct event_base *manage_evbase, void *runtimelog);
struct doris_prod_instance *doris_prod_instance_new(struct doris_prod_param *param, struct event_base *worker_evbase, void *runtimelog);
enum PROD_VERSTART_RES
{
VERSTART_RES_OK=0,
VERSTART_RES_BUSY, //<2F><>Ӧ<EFBFBD><D3A6>300<30><30><EFBFBD><EFBFBD><EFBFBD><EFBFBD>server<65><72><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>start<72><74><EFBFBD><EFBFBD>;<EFBFBD>汾
VERSTART_RES_ERROR, //ȡ<><C8A1><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ӧ<EFBFBD><D3A6><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ӧ<EFBFBD><D3A6>400<30><30>Ҫô<D2AA>Ƿ<EFBFBD><C7B7><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ҫô<D2AA><C3B4>δ<EFBFBD><CEB4><EFBFBD><EFBFBD>start<72>İ汾<C4B0><E6B1BE>
VERSTART_CURL_ERROR, //curlԭ<6C><D4AD>δ<EFBFBD><CEB4><EFBFBD><EFBFBD>ȡ<EFBFBD><C8A1><EFBFBD><EFBFBD>Ӧ
};
/*cfgtype: 1-ȫ<><C8AB><EFBFBD><EFBFBD>2-<2D><><EFBFBD><EFBFBD>*/
struct doris_upload_ctx *doris_prod_upload_ctx_new(struct doris_prod_instance *instance,const char *business, int32_t cfgtype);
void doris_prod_upload_ctx_destroy(struct doris_upload_ctx *ctx);
int32_t doris_prod_version_start(struct doris_upload_ctx *ctx);
int32_t doris_prod_version_start_with_cb(struct doris_upload_ctx *ctx,
void (*verstart_cb)(enum PROD_VERSTART_RES result, const char *body, void *userdata), void *userdata);
enum PROD_VEROP_RES
{
VERSIONOP_RES_OK=0,
VERSIONOP_RES_ERROR,
VERSIONOP_CURL_ERROR,
};
struct table_meta
{
const char *tablename; //<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʽmaat<61><74><EFBFBD>䣻<EFBFBD><E4A3BB><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʽΪ<CABD>ļ<EFBFBD><C4BC><EFBFBD>
const char *filename; //<2F><><EFBFBD><EFBFBD>ʱҪ<CAB1><D2AA><EFBFBD><EFBFBD><EFBFBD>ļ<EFBFBD><C4BC><EFBFBD>
const char *userregion;
const char *md5;
u_int32_t cfgnum;
};
int32_t doris_prod_upload_once_with_cb(struct doris_upload_ctx *ctx, char *data, size_t size,
struct table_meta *meta, void (*upfrag_cb)(enum PROD_VEROP_RES result, void *userdata), void *userdata);
int32_t doris_prod_upload_frag_with_cb(struct doris_upload_ctx *ctx, char *data, size_t size, size_t offset,
bool last, struct table_meta *meta, void (*upfrag_cb)(enum PROD_VEROP_RES result, void *userdata), void *userdata);
int32_t doris_prod_version_end(struct doris_upload_ctx *ctx, void (*verend_cb)(enum PROD_VEROP_RES result, int64_t version, void *userdata), void *userdata);
int32_t doris_prod_version_cancel(struct doris_upload_ctx *ctx, void (*vercancel_cb)(enum PROD_VEROP_RES result, void *userdata), void *userdata);
#endif