gfwleak/doris-doris-dispatch
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

支持文件接口用户自定义信息；

Browse Source
This commit is contained in:
linuxrc@163.com
2021-07-27 16:25:13 +08:00
parent 5217e9188e
commit b32381f7f5
11 changed files with 76 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
client/doris_client_fetch.cpp
View File

@@ -60,9 +60,12 @@ void easy_string_savedata(struct easy_string *estr, const char *data, size_t len
void doris_confile_ctx_reset(struct doris_confile_ctx *ctx)
{
struct doris_http_ctx *httpctx=ctx->httpctx;
memset(ctx, 0, sizeof(struct doris_confile_ctx));
ctx->httpctx = httpctx;
//<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ֲ<EFBFBD><D6B2><EFBFBD>
ctx->res_code = 0;
ctx->contlength = 0;
ctx->contl_start = 0;
ctx->contl_end = 0;
ctx->contl_total = 0;
}
void doris_confile_ctx_destry(struct doris_confile_ctx *ctx)
@@ -104,6 +107,9 @@ void doris_fetch_next_confile_meta(struct doris_instance *instance)
sub = cJSON_GetObjectItem(cur_a_item, "cfg_num");
instance->curmeta.cfg_num = sub->valueint;
sub = cJSON_GetObjectItem(cur_a_item, "user_region");
instance->curmeta.user_region = (sub==NULL)?NULL:sub->valuestring;
if(NULL != (sub = cJSON_GetObjectItem(cur_a_item, "md5")))
{
instance->curmeta.validate_md5 = 1;
@@ -137,8 +143,8 @@ void doris_http_confile_header_cb(const char *start, size_t bytes, CURLcode code
instance->retry_times = 0;
if(instance->curmeta.curoffset == 0)
{
instance->cbs.cfgfile_start(instance, instance->curmeta.table_name,
instance->curmeta.size, instance->curmeta.cfg_num, instance->cbs.userdata);
instance->cbs.cfgfile_start(instance, instance->curmeta.table_name, instance->curmeta.size,
instance->curmeta.cfg_num, instance->curmeta.user_region, instance->cbs.userdata);
MD5_Init(&instance->ctx.md5ctx);
}
}
@@ -305,7 +311,7 @@ void doris_http_fetch_confile(struct doris_instance *instance)
else
{
instance->statistic.field[DRS_FS_FILED_REQ_FILES] += 1;
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_INFO, "Launch confile %s GET, req_version=%lu, %s",
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_DEBUG, "Launch confile %s GET, req_version=%lu, %s",
instance->curmeta.table_name, instance->req_version, range);
}
}
@@ -324,8 +330,8 @@ void doris_http_meta_header_cb(const char *ptr, size_t bytes, CURLcode code, lon
if(res_code != 200)
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_DEBUG, "No new meta found, cur_version=%lu, req_version=%lu, curlcode = %d",
instance->cur_version, instance->req_version, code);
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_DEBUG, "No new meta found, server: %s, cur_version=%lu, req_version=%lu, curlcode = %d",
instance->ctx.server, instance->cur_version, instance->req_version, code);
}
}
@@ -348,8 +354,8 @@ void doris_http_meta_done_cb(CURLcode res, long res_code, const char *err, void
if(res!=CURLE_OK)
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Request meta failed, cur_version=%lu, req_version=%lu, curlcode = %d, error: %s",
instance->cur_version, instance->req_version, res_code, err);
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Request meta failed, server: %s, cur_version=%lu, req_version=%lu, curlcode = %d, error: %s",
instance->ctx.server, instance->cur_version, instance->req_version, res_code, err);
goto out_error;
}
@@ -361,15 +367,16 @@ void doris_http_meta_done_cb(CURLcode res, long res_code, const char *err, void
instance->meta = cJSON_Parse(instance->estr.buff);
if(instance->meta == NULL)
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Parse meta failed, req_version=%lu, invalid json: %s", instance->req_version, instance->estr.buff);
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Parse meta failed, server: %s, req_version=%lu, invalid json: %s",
instance->ctx.server, instance->req_version, instance->estr.buff);
goto out_error;
}
sub = cJSON_GetObjectItem(instance->meta, "version");
instance->new_version = sub->valuedouble;
instance->req_version = instance->new_version;
instance->statistic.field[DRS_FS_FILED_RES_META] += 1;
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_INFO, "NEW_META found, cur_version=%lu, newjson: %s",
instance->cur_version, instance->estr.buff);
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_INFO, "NEW_META found, server: %s, cur_version=%lu, newjson: %s",
instance->ctx.server, instance->cur_version, instance->estr.buff);
instance->cbs.version_start(instance, instance->meta, instance->cbs.userdata);
instance->array = cJSON_GetObjectItem(instance->meta, "configs");
@@ -395,7 +402,8 @@ static void doris_http_fetch_meta(struct doris_instance *instance)
struct doris_http_callback curlcbs;
char metauri[128];
balance_seed = (((u_int64_t)rand()) << 32) | rand();
balance_seed = (((u_int64_t)rand()&0xFFFF) << 48) | (((u_int64_t)rand()&0xFFFF) << 32) |
(((u_int64_t)rand()&0xFFFF) << 16) | ((u_int64_t)rand()&0xFFFF);
memset(&curlcbs, 0, sizeof(struct doris_http_callback));
curlcbs.header_cb = doris_http_meta_header_cb;
@@ -405,16 +413,16 @@ static void doris_http_fetch_meta(struct doris_instance *instance)
instance->array_index = 0;
instance->cur_httpins = instance->httpins_master;
instance->ctx.httpctx = doris_http_ctx_new(instance->cur_httpins, &curlcbs, balance_seed);
instance->ctx.httpctx = doris_http_ctx_new(instance->cur_httpins, &curlcbs, balance_seed, instance->ctx.server, 64);
if(instance->ctx.httpctx==NULL && instance->httpins_backup1!=NULL)
{
instance->cur_httpins = instance->httpins_backup1;
instance->ctx.httpctx = doris_http_ctx_new(instance->cur_httpins, &curlcbs, balance_seed);
instance->ctx.httpctx = doris_http_ctx_new(instance->cur_httpins, &curlcbs, balance_seed, instance->ctx.server, 64);
}
if(instance->ctx.httpctx==NULL && instance->httpins_backup2!=NULL)
{
instance->cur_httpins = instance->httpins_backup2;
instance->ctx.httpctx = doris_http_ctx_new(instance->cur_httpins, &curlcbs, balance_seed);
instance->ctx.httpctx = doris_http_ctx_new(instance->cur_httpins, &curlcbs, balance_seed, instance->ctx.server, 64);
}
instance->req_version = instance->cur_version + 1; //ֻ<>а汾<D0B0><E6B1BE><EFBFBD>³ɹ<C2B3><C9B9><EFBFBD><EFBFBD><EFBFBD>cur_version<6F>Ż<EFBFBD><C5BB><EFBFBD><EFBFBD><EFBFBD>
@@ -616,7 +624,7 @@ struct doris_instance *doris_instance_new(struct doris_parameter *param, struct
{
return NULL;
}
srand(time(NULL));
srand((int64_t)param);
if(param->param_backup1 != NULL)
{

2
client/doris_client_fetch.h
View File

@@ -52,6 +52,7 @@ struct fetch_file_meta
u_int32_t cfg_num;
u_int32_t validate_md5;
char md5str[36];
const char *user_region;
};
struct md5_long
@@ -68,6 +69,7 @@ union doris_md5
struct doris_confile_ctx
{
struct doris_http_ctx *httpctx;
char server[64];
MD5_CTX md5ctx;
long res_code;

4
client/doris_client_transfer.cpp
View File

@@ -49,7 +49,8 @@ void doris_http_ctx_destroy(struct doris_http_ctx *ctx)
free(ctx);
}
struct doris_http_ctx *doris_http_ctx_new(struct doris_http_instance *instance, struct doris_http_callback *cb, u_int64_t balance_seed)
struct doris_http_ctx *doris_http_ctx_new(struct doris_http_instance *instance,
struct doris_http_callback *cb, u_int64_t balance_seed, char *host, int32_t size)
{
struct doris_http_ctx *ctx;
struct doris_curl_multihd *multidata;
@@ -61,6 +62,7 @@ struct doris_http_ctx *doris_http_ctx_new(struct doris_http_instance *instance,
}
assert(instance->server_hosts->find(result.bucket_id) != instance->server_hosts->end());
multidata = instance->server_hosts->find(result.bucket_id)->second;
snprintf(host, size, multidata->host->srvaddr);
ctx = (struct doris_http_ctx *)calloc(1, sizeof(struct doris_http_ctx));
ctx->instance = instance;

2
client/doris_client_transfer.h
View File

@@ -44,7 +44,7 @@ struct doris_curl_multihd
};
struct doris_curl_multihd *doris_initialize_multihd_for_host(struct doris_http_instance *instance, struct dst_host_cnn_balance *host);
struct doris_http_ctx *doris_http_ctx_new(struct doris_http_instance *instance, struct doris_http_callback *cb, u_int64_t balance_seed);
struct doris_http_ctx *doris_http_ctx_new(struct doris_http_instance *instance, struct doris_http_callback *cb, u_int64_t balance_seed, char *host, int32_t size);
void doris_http_ctx_destroy(struct doris_http_ctx *ctx);
void doris_http_ctx_reset(struct doris_http_ctx *ctx, struct doris_http_callback *cb);

6
client/nirvana_conhash.cpp
View File

@@ -145,7 +145,7 @@ static u_int64_t bucket_gen_uniq_point(struct ch_bucket_inner *inner_bucket, u_i
u_int32_t hash, i=0;
seed = (((u_int64_t)cur_point_index)<<32) | inner_bucket->bucket.bucket_id;
hash = murmurhash2(&seed, sizeof(u_int64_t), 23068673);
hash = murmurhash2(&seed, sizeof(u_int64_t), 515880193);
x = (((u_int64_t)hash)<<32) | inner_bucket->bucket.bucket_id;
while(i != cur_point_index)
@@ -155,7 +155,7 @@ static u_int64_t bucket_gen_uniq_point(struct ch_bucket_inner *inner_bucket, u_i
if(x == inner_bucket->point_array[i].point_val) //<2F><>ͻ
{
seed = (((u_int64_t)hash)<<32) | inner_bucket->bucket.bucket_id;
hash = murmurhash2(&seed, sizeof(u_int64_t), 23068673);
hash = murmurhash2(&seed, sizeof(u_int64_t), 515880193);
x = (((u_int64_t)hash)<<32) | inner_bucket->bucket.bucket_id;
i = 0;
break;
@@ -434,7 +434,7 @@ enum CONHASH_ERRCODE conhash_lookup_bucket(struct consistent_hash *ch, const voi
return CONHASH_NO_VALID_BUCKETS;
}
hash = MurmurHash64A(key, len, 23068673);
hash = MurmurHash64A(key, len, 515880193);
idx = search_up_bound(hash, ch->point_array, ch->point_num, sizeof(struct ch_point), offsetof(struct ch_point, point_val));
ch->point_array[idx].hit_cnt++;
bucket_index = ch->point_array[idx].bucket_index;