gfwleak/doris-doris-dispatch
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

支持文件接口用户自定义信息；

Browse Source
This commit is contained in:
linuxrc@163.com
2021-07-27 16:25:13 +08:00
parent 5217e9188e
commit b32381f7f5
11 changed files with 76 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
client/doris_client_fetch.cpp
View File

@@ -60,9 +60,12 @@ void easy_string_savedata(struct easy_string *estr, const char *data, size_t len
void doris_confile_ctx_reset(struct doris_confile_ctx *ctx)
{
struct doris_http_ctx *httpctx=ctx->httpctx;
memset(ctx, 0, sizeof(struct doris_confile_ctx));
ctx->httpctx = httpctx;
//<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ֲ<EFBFBD><D6B2><EFBFBD>
ctx->res_code = 0;
ctx->contlength = 0;
ctx->contl_start = 0;
ctx->contl_end = 0;
ctx->contl_total = 0;
}
void doris_confile_ctx_destry(struct doris_confile_ctx *ctx)
@@ -104,6 +107,9 @@ void doris_fetch_next_confile_meta(struct doris_instance *instance)
sub = cJSON_GetObjectItem(cur_a_item, "cfg_num");
instance->curmeta.cfg_num = sub->valueint;
sub = cJSON_GetObjectItem(cur_a_item, "user_region");
instance->curmeta.user_region = (sub==NULL)?NULL:sub->valuestring;
if(NULL != (sub = cJSON_GetObjectItem(cur_a_item, "md5")))
{
instance->curmeta.validate_md5 = 1;
@@ -137,8 +143,8 @@ void doris_http_confile_header_cb(const char *start, size_t bytes, CURLcode code
instance->retry_times = 0;
if(instance->curmeta.curoffset == 0)
{
instance->cbs.cfgfile_start(instance, instance->curmeta.table_name,
instance->curmeta.size, instance->curmeta.cfg_num, instance->cbs.userdata);
instance->cbs.cfgfile_start(instance, instance->curmeta.table_name, instance->curmeta.size,
instance->curmeta.cfg_num, instance->curmeta.user_region, instance->cbs.userdata);
MD5_Init(&instance->ctx.md5ctx);
}
}
@@ -305,7 +311,7 @@ void doris_http_fetch_confile(struct doris_instance *instance)
else
{
instance->statistic.field[DRS_FS_FILED_REQ_FILES] += 1;
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_INFO, "Launch confile %s GET, req_version=%lu, %s",
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_DEBUG, "Launch confile %s GET, req_version=%lu, %s",
instance->curmeta.table_name, instance->req_version, range);
}
}
@@ -324,8 +330,8 @@ void doris_http_meta_header_cb(const char *ptr, size_t bytes, CURLcode code, lon
if(res_code != 200)
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_DEBUG, "No new meta found, cur_version=%lu, req_version=%lu, curlcode = %d",
instance->cur_version, instance->req_version, code);
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_DEBUG, "No new meta found, server: %s, cur_version=%lu, req_version=%lu, curlcode = %d",
instance->ctx.server, instance->cur_version, instance->req_version, code);
}
}
@@ -348,8 +354,8 @@ void doris_http_meta_done_cb(CURLcode res, long res_code, const char *err, void
if(res!=CURLE_OK)
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Request meta failed, cur_version=%lu, req_version=%lu, curlcode = %d, error: %s",
instance->cur_version, instance->req_version, res_code, err);
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Request meta failed, server: %s, cur_version=%lu, req_version=%lu, curlcode = %d, error: %s",
instance->ctx.server, instance->cur_version, instance->req_version, res_code, err);
goto out_error;
}
@@ -361,15 +367,16 @@ void doris_http_meta_done_cb(CURLcode res, long res_code, const char *err, void
instance->meta = cJSON_Parse(instance->estr.buff);
if(instance->meta == NULL)
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Parse meta failed, req_version=%lu, invalid json: %s", instance->req_version, instance->estr.buff);
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Parse meta failed, server: %s, req_version=%lu, invalid json: %s",
instance->ctx.server, instance->req_version, instance->estr.buff);
goto out_error;
}
sub = cJSON_GetObjectItem(instance->meta, "version");
instance->new_version = sub->valuedouble;
instance->req_version = instance->new_version;
instance->statistic.field[DRS_FS_FILED_RES_META] += 1;
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_INFO, "NEW_META found, cur_version=%lu, newjson: %s",
instance->cur_version, instance->estr.buff);
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_INFO, "NEW_META found, server: %s, cur_version=%lu, newjson: %s",
instance->ctx.server, instance->cur_version, instance->estr.buff);
instance->cbs.version_start(instance, instance->meta, instance->cbs.userdata);
instance->array = cJSON_GetObjectItem(instance->meta, "configs");
@@ -395,7 +402,8 @@ static void doris_http_fetch_meta(struct doris_instance *instance)
struct doris_http_callback curlcbs;
char metauri[128];
balance_seed = (((u_int64_t)rand()) << 32) | rand();
balance_seed = (((u_int64_t)rand()&0xFFFF) << 48) | (((u_int64_t)rand()&0xFFFF) << 32) |
(((u_int64_t)rand()&0xFFFF) << 16) | ((u_int64_t)rand()&0xFFFF);
memset(&curlcbs, 0, sizeof(struct doris_http_callback));
curlcbs.header_cb = doris_http_meta_header_cb;
@@ -405,16 +413,16 @@ static void doris_http_fetch_meta(struct doris_instance *instance)
instance->array_index = 0;
instance->cur_httpins = instance->httpins_master;
instance->ctx.httpctx = doris_http_ctx_new(instance->cur_httpins, &curlcbs, balance_seed);
instance->ctx.httpctx = doris_http_ctx_new(instance->cur_httpins, &curlcbs, balance_seed, instance->ctx.server, 64);
if(instance->ctx.httpctx==NULL && instance->httpins_backup1!=NULL)
{
instance->cur_httpins = instance->httpins_backup1;
instance->ctx.httpctx = doris_http_ctx_new(instance->cur_httpins, &curlcbs, balance_seed);
instance->ctx.httpctx = doris_http_ctx_new(instance->cur_httpins, &curlcbs, balance_seed, instance->ctx.server, 64);
}
if(instance->ctx.httpctx==NULL && instance->httpins_backup2!=NULL)
{
instance->cur_httpins = instance->httpins_backup2;
instance->ctx.httpctx = doris_http_ctx_new(instance->cur_httpins, &curlcbs, balance_seed);
instance->ctx.httpctx = doris_http_ctx_new(instance->cur_httpins, &curlcbs, balance_seed, instance->ctx.server, 64);
}
instance->req_version = instance->cur_version + 1; //ֻ<>а汾<D0B0><E6B1BE><EFBFBD>³ɹ<C2B3><C9B9><EFBFBD><EFBFBD><EFBFBD>cur_version<6F>Ż<EFBFBD><C5BB><EFBFBD><EFBFBD><EFBFBD>
@@ -616,7 +624,7 @@ struct doris_instance *doris_instance_new(struct doris_parameter *param, struct
{
return NULL;
}
srand(time(NULL));
srand((int64_t)param);
if(param->param_backup1 != NULL)
{