gfwleak/doris-doris-dispatch
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

适配非行列式透传落地文件名

Browse Source
This commit is contained in:
linuxrc@163.com
2021-08-03 10:49:52 +08:00
parent b32381f7f5
commit dcca411ddc
7 changed files with 80 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
client/doris_client_fetch.cpp
View File

@@ -99,16 +99,18 @@ void doris_fetch_next_confile_meta(struct doris_instance *instance)
instance->array_index++;
sub = cJSON_GetObjectItem(cur_a_item, "tablename");
instance->curmeta.table_name = sub->valuestring;
instance->curmeta.meta.tablename = sub->valuestring;
sub = cJSON_GetObjectItem(cur_a_item, "filename");
instance->curmeta.meta.filename = sub->valuestring;
sub = cJSON_GetObjectItem(cur_a_item, "size");
instance->curmeta.size = sub->valuedouble;
instance->curmeta.meta.size = sub->valuedouble;
sub = cJSON_GetObjectItem(cur_a_item, "cfg_num");
instance->curmeta.cfg_num = sub->valueint;
instance->curmeta.meta.cfgnum = sub->valueint;
sub = cJSON_GetObjectItem(cur_a_item, "user_region");
instance->curmeta.user_region = (sub==NULL)?NULL:sub->valuestring;
instance->curmeta.meta.userregion = (sub==NULL)?NULL:sub->valuestring;
if(NULL != (sub = cJSON_GetObjectItem(cur_a_item, "md5")))
{
@@ -137,14 +139,13 @@ void doris_http_confile_header_cb(const char *start, size_t bytes, CURLcode code
if(res_code != 200 && res_code!=206)
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_DEBUG, "Fetch confile %s failed, req_version=%lu, curlcode = %d",
instance->curmeta.table_name, instance->req_version, code);
instance->curmeta.meta.tablename, instance->req_version, code);
return;
}
instance->retry_times = 0;
if(instance->curmeta.curoffset == 0)
{
instance->cbs.cfgfile_start(instance, instance->curmeta.table_name, instance->curmeta.size,
instance->curmeta.cfg_num, instance->curmeta.user_region, instance->cbs.userdata);
instance->cbs.cfgfile_start(instance, &instance->curmeta.meta, NULL, instance->cbs.userdata);
MD5_Init(&instance->ctx.md5ctx);
}
}
@@ -170,7 +171,7 @@ void doris_http_confile_header_cb(const char *start, size_t bytes, CURLcode code
memcpy(buffer, start+13, bytes-13);
buffer[bytes-13] = '\0';
ret = sscanf(buffer, "%*[^0-9]%lu-%lu/%lu", &instance->ctx.contl_start, &instance->ctx.contl_end, &instance->ctx.contl_total);
assert(ret == 3 && instance->ctx.contl_total == instance->curmeta.size && instance->ctx.contl_start==instance->curmeta.curoffset);
assert(ret == 3 && instance->ctx.contl_total == instance->curmeta.meta.size && instance->ctx.contl_start==instance->curmeta.curoffset);
}
break;
default: break;
@@ -202,7 +203,7 @@ void doris_http_confile_done_cb(CURLcode res, long res_code, const char *err, vo
if(res!=CURLE_OK)
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Fetch confile %s failed, req_version=%lu, curlcode = %d, error: %s",
instance->curmeta.table_name, instance->req_version, res_code, err);
instance->curmeta.meta.tablename, instance->req_version, res_code, err);
goto out_error;
}
@@ -214,28 +215,28 @@ void doris_http_confile_done_cb(CURLcode res, long res_code, const char *err, vo
if(instance->ctx.contl_total != 0)
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_DEBUG, "Fetch confile %s success, req_version=%lu, Content-Range: %lu-%lu/%lu",
instance->curmeta.table_name, instance->req_version, instance->ctx.contl_start, instance->ctx.contl_end, instance->ctx.contl_total);
instance->curmeta.meta.tablename, instance->req_version, instance->ctx.contl_start, instance->ctx.contl_end, instance->ctx.contl_total);
}
else
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_DEBUG, "Fetch confile %s success, req_version=%lu, Content-Length: %lu/%lu",
instance->curmeta.table_name, instance->req_version, instance->ctx.contlength, instance->curmeta.size);
instance->curmeta.meta.tablename, instance->req_version, instance->ctx.contlength, instance->curmeta.meta.size);
}
instance->statistic.field[DRS_FS_FILED_RES_FRAGS] += 1;
if(instance->curmeta.curoffset >= instance->curmeta.size) //<2F><><EFBFBD>ļ<EFBFBD><C4BC><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
if(instance->curmeta.curoffset >= instance->curmeta.meta.size) //<2F><><EFBFBD>ļ<EFBFBD><C4BC><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
{
doris_md5_final_string(&instance->ctx.md5ctx, md5buffer, 64);
if(instance->curmeta.validate_md5 && strcasecmp(instance->curmeta.md5str, md5buffer))
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Fetch confile %s over, version=%lu, md5 validate fail, real: %s, expect: %s",
instance->curmeta.table_name, instance->req_version, md5buffer, instance->curmeta.md5str);
instance->curmeta.meta.tablename, instance->req_version, md5buffer, instance->curmeta.md5str);
direct_fail=true;goto out_md5;
}
else
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_INFO, "Fetch confile %s.%010lu over, md5: %s",
instance->curmeta.table_name, instance->req_version, md5buffer);
instance->curmeta.meta.tablename, instance->req_version, md5buffer);
}
instance->statistic.field[DRS_FS_FILED_RES_FILES] += 1;
instance->cbs.cfgfile_finish(instance, md5buffer, instance->cbs.userdata);
@@ -296,13 +297,13 @@ void doris_http_fetch_confile(struct doris_instance *instance)
doris_http_ctx_reset(instance->ctx.httpctx, &curlcbs);
//<2F><><EFBFBD><EFBFBD><EFBFBD>ļ<EFBFBD><C4BC>ֶ<EFBFBD><D6B6><EFBFBD><EFBFBD>أ<EFBFBD><D8A3>ϴ<EFBFBD>δ<EFBFBD><CEB4><EFBFBD>ɵļ<C9B5><C4BC><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
if((instance->curmeta.size > instance->param->fetch_frag_size) || instance->curmeta.curoffset!=0)
if((instance->curmeta.meta.size > instance->param->fetch_frag_size) || instance->curmeta.curoffset!=0)
{
sprintf(range, "Range: bytes=%lu-%lu", instance->curmeta.curoffset, instance->curmeta.curoffset + instance->param->fetch_frag_size - 1);
doris_http_ctx_add_header(instance->ctx.httpctx, range);
}
snprintf(metauri, 128, "configfile?tablename=%s&version=%lu&business=%s", instance->curmeta.table_name, instance->req_version, instance->args.bizname);
snprintf(metauri, 128, "configfile?tablename=%s&version=%lu&business=%s", instance->curmeta.meta.tablename, instance->req_version, instance->args.bizname);
if(doris_http_launch_get_request(instance->ctx.httpctx, metauri))
{
instance->statistic.field[DRS_FS_FILED_REQ_FAIL] += 1;
@@ -312,7 +313,7 @@ void doris_http_fetch_confile(struct doris_instance *instance)
{
instance->statistic.field[DRS_FS_FILED_REQ_FILES] += 1;
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_DEBUG, "Launch confile %s GET, req_version=%lu, %s",
instance->curmeta.table_name, instance->req_version, range);
instance->curmeta.meta.tablename, instance->req_version, range);
}
}
@@ -351,6 +352,7 @@ void doris_http_meta_done_cb(CURLcode res, long res_code, const char *err, void
{
struct doris_instance *instance = (struct doris_instance *)userp;
cJSON *sub;
int64_t new_version;
if(res!=CURLE_OK)
{
@@ -372,7 +374,16 @@ void doris_http_meta_done_cb(CURLcode res, long res_code, const char *err, void
goto out_error;
}
sub = cJSON_GetObjectItem(instance->meta, "version");
instance->new_version = sub->valuedouble;
new_version = sub->valuedouble;
if(new_version <= instance->cur_version)
{
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "An older version received, abandon it. server: %s, cur_version=%lu, invalid json: %s",
instance->ctx.server, instance->cur_version, instance->estr.buff);
cJSON_Delete(instance->meta);
instance->meta = NULL;
goto out_error;
}
instance->new_version = new_version;
instance->req_version = instance->new_version;
instance->statistic.field[DRS_FS_FILED_RES_META] += 1;
MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_INFO, "NEW_META found, server: %s, cur_version=%lu, newjson: %s",