gfwleak/linxin-coredump-tools
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1.修改kubectl debug 调用时切换工作目录 2.增加debug时参数能指定交互方式

Browse Source
This commit is contained in:
linxin
2023-04-27 17:20:57 +08:00
parent 00294bc452
commit cb077615ff
7 changed files with 1486 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
.gitlab-ci.yml Normal file
View File

@@ -0,0 +1,67 @@
variables:
GIT_STRATEGY: "clone"
BUILD_BASED_IMAGE_CENTOS7: "git.mesalab.cn:7443/mesa_platform/build-env:master"
BUILD_BASED_IMAGE_ROCKYLINUX8: "git.mesalab.cn:7443/mesa_platform/build-env:rockylinux"
stages:
- prebuild
- build-centos7
- build-rockylinux8
.build_rpm:
variables:
BUILD_PADDING_PREFIX: /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/
TESTING_VERSION_BUILD: 0
before_script:
- mkdir -p $BUILD_PADDING_PREFIX/$CI_PROJECT_NAMESPACE/
- ln -s $CI_PROJECT_DIR $BUILD_PADDING_PREFIX/$CI_PROJECT_PATH
- cd $BUILD_PADDING_PREFIX/$CI_PROJECT_PATH
- chmod +x ./ci/travis.sh
script:
- yum makecache
- ./ci/travis.sh
tags:
- share
feature_branch_build_rockylinux8:
image: $BUILD_BASED_IMAGE_ROCKYLINUX8
stage: build-rockylinux8
variables:
PACKAGE: 1
LIBYAML_BUILD: 1
TESTING_VERSION_BUILD: 1
extends: .build_rpm
except:
- tags
- /^dev-.*$/i
- /^rel-.*$/i
- /^master.*$/i
develop_build_rockylinux8:
image: $BUILD_BASED_IMAGE_ROCKYLINUX8
stage: build-rockylinux8
variables:
PACKAGE: 1
UPLOAD: 1
LIBYAML_BUILD: 1
TESTING_VERSION_BUILD: 1
PULP3_RPM_REPO_NAME: tsg-testing-x86_64.el8
PULP3_RPM_DIST_NAME: tsg-testing-x86_64.el8
extends: .build_rpm
only:
- /^dev-.*$/i
- /^rel-.*$/i
- /^master.*$/i
release_build_rockylinux8:
image: $BUILD_BASED_IMAGE_ROCKYLINUX8
stage: build-rockylinux8
variables:
PACKAGE: 1
UPLOAD: 1
LIBYAML_BUILD: 1
PULP3_RPM_REPO_NAME: tsg-stable-x86_64.el8
PULP3_RPM_DIST_NAME: tsg-stable-x86_64.el8
extends: .build_rpm
only:
- tags

1268
autorevision.sh Normal file
View File

File diff suppressed because it is too large Load Diff

63
ci/travis.sh Normal file
View File

@@ -0,0 +1,63 @@
#!/usr/bin/env sh
echo $CPLUS_INCLUDE_PATH
set -evx
chmod +x ci/get-nprocessors.sh
. ci/get-nprocessors.sh
# if possible, ask for the precise number of processors,
# otherwise take 2 processors as reasonable default; see
# https://docs.travis-ci.com/user/speeding-up-the-build/#Makefile-optimization
if [ -x /usr/bin/getconf ]; then
NPROCESSORS=$(/usr/bin/getconf _NPROCESSORS_ONLN)
else
NPROCESSORS=2
fi
# as of 2017-09-04 Travis CI reports 32 processors, but GCC build
# crashes if parallelized too much (maybe memory consumption problem),
# so limit to 4 processors for the time being.
if [ $NPROCESSORS -gt 4 ] ; then
echo "$0:Note: Limiting processors to use by make from $NPROCESSORS to 4."
NPROCESSORS=4
fi
# Tell make to use the processors. No preceding '-' required.
MAKEFLAGS="j${NPROCESSORS}"
export MAKEFLAGS
env | sort
# Set default values to OFF for these variables if not specified.
: "${NO_EXCEPTION:=OFF}"
: "${NO_RTTI:=OFF}"
: "${COMPILER_IS_GNUCXX:=OFF}"
yum install -y cligen-devel clixon-devel
if [ -n "${LIBYAML_BUILD}" ]; then
git clone https://github.com/yaml/libyaml.git
cd libyaml
sed -i 's/^add_library.*$/add_library(yaml SHARED ${SRCS})/g' CMakeLists.txt
cmake3 .
make && make install
cd ..
fi
mkdir build || true
cd build
cmake3 -DVERSION_DAILY_BUILD=$TESTING_VERSION_BUILD ..
if [ -n "${PACKAGE}" ]; then
make package
fi
ls -halt ./*.rpm
if [ -n "${UPLOAD}" ]; then
cp ~/rpm_upload_tools.py ./
python3 rpm_upload_tools.py ${PULP3_RPM_REPO_NAME} ${PULP3_RPM_DIST_NAME} *.rpm
fi

105
coredump-ctl/coredump-ctl.go
View File

@@ -30,25 +30,40 @@ func terminalSize() (width, height int, err error) {
}
return width, height, nil
}
func debug(config types.Coredump_config) error {
func debug(config types.Coredump_config, command string) error {
// 使用kubectl命令执行debug操作
if config.Image_id != "" {
if config.Hostname == "" {
return errors.New("hostname must not be empty")
}
cmd := exec.Command("kubectl", "debug", "node/"+config.Hostname, "-it", "--image="+config.Image_id, "--image-pull-policy=Never", "-- bash")
cmd.Stdin = os.Stdin
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
fmt.Println(cmd.String())
input := fmt.Sprintf("you can exec gdb %s /host%s get stack info", config.Process_exe_path, config.Storage)
fmt.Println(input)
if err := cmd.Start(); err != nil {
return err
}
err := cmd.Wait()
if err != nil {
return err
if command == "gdb" {
chmod := "cd" + filepath.Dir(config.Process_exe_path)
cmd := exec.Command("kubectl", "debug", "node/"+config.Hostname, "-it", "--image="+config.Image_id, "--image-pull-policy=Never", "--", "gdb", config.Process_exe_path, "/host"+config.Storage, "-ex", chmod)
cmd.Stdin = os.Stdin
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
fmt.Println(cmd.String())
if err := cmd.Start(); err != nil {
return err
}
err := cmd.Wait()
if err != nil {
return err
}
} else {
cmd := exec.Command("kubectl", "debug", "node/"+config.Hostname, "-it", "--image="+config.Image_id, "--image-pull-policy=Never", "--", command)
cmd.Stdin = os.Stdin
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
fmt.Println(cmd.String())
fmt.Println("exe=", config.Process_exe_path, "coredump=/host", config.Storage)
if err := cmd.Start(); err != nil {
return err
}
err := cmd.Wait()
if err != nil {
return err
}
}
} else { // 使用gdb命令进行命令行交互分析
cmd := exec.Command("gdb", config.Process_exe_path, config.Storage)
@@ -93,25 +108,28 @@ func WalkDirectory(dir string) {
fmt.Printf("Error walking directory %s: %v\n", dir, err)
}
}
func list() {
func list(pid string) {
defaultWidth := 80
if width, _, err := terminalSize(); err == nil {
defaultWidth = int(width)
}
// 输出表头信息
fmt.Printf("%-6s %-6s %-6s %-15s %-20s %-15s %-10s %-20s %-20s\n",
"PID", "UID", "GID", "SIGNATURE", "TIMESTAMP", "FILE", "HOSTNAME", "IMAGE_NAME", "storge")
fmt.Printf("%-8s %-3s %-3s %-3s %-20s %-30s %-30s %-100s %-50s\n",
"PID", "UID", "GID", "SIG", "TIMESTAMP", "EXE", "HOSTNAME", "IMAGE", "STORGE")
fmt.Println(strings.Repeat("-", defaultWidth))
total := 0
// 输出配置信息
for _, c := range configs {
processExe := filepath.Base(c.Process_exe_path)
if pid != "" && strings.Compare(c.Initial_ns_pid, pid) != 0 {
continue
}
coreTime := time.Unix(c.Timestamp, 0).Format("2006-01-02 15:04:05")
fmt.Printf("%-6s %-6s %-6s %-15d %-20s %-15s %-10s %-20s %-20s\n",
c.Process_ns_pid, c.UID, c.GID, c.Signal, coreTime, processExe, c.Hostname, c.Image_id, c.Storage)
fmt.Printf("%-8s %-3s %-3s %-6d %-20s %-30s %-30s %-100s %-50s\n",
c.Initial_ns_pid, c.UID, c.GID, c.Signal, coreTime, c.Process_exe_path, c.Hostname, c.Image_id, c.Storage)
total += 1
}
fmt.Println()
fmt.Println("Total", len(configs), "coredumps")
fmt.Println("Total", total, "coredumps")
}
// Help 打印使用帮助
@@ -120,15 +138,22 @@ func Help() {
fmt.Println("list options:")
fmt.Println(" -dir string")
fmt.Println(" Directory path")
fmt.Println(" -p string")
fmt.Println(" Pid to matching")
fmt.Println(" -command string")
fmt.Println(" exe command when attach to pod")
}
func main() {
listCmd := flag.NewFlagSet("list", flag.ExitOnError)
debugCmd := flag.NewFlagSet("debug", flag.ExitOnError)
helpCmd := flag.NewFlagSet("help", flag.ExitOnError)
dirPath := flag.String("dir", "", "Directory path")
var dirPath, pid, command string
listCmd.StringVar(&pid, "p", "", "Pid to matching")
debugCmd.StringVar(&pid, "p", "", "Pid to matching")
listCmd.StringVar(&dirPath, "dir", "", "Directory path")
debugCmd.StringVar(&dirPath, "dir", "", "Directory path")
debugCmd.StringVar(&command, "command", "", "")
if len(os.Args) == 1 {
helpCmd.Usage()
os.Exit(1)
@@ -146,21 +171,31 @@ func main() {
os.Exit(1)
}
if listCmd.Parsed() {
if *dirPath == "" {
*dirPath = "/var/tmp"
if dirPath == "" {
dirPath = "/var/tmp"
}
WalkDirectory(*dirPath)
list()
WalkDirectory(dirPath)
list(pid)
return
}
if debugCmd.Parsed() {
if *dirPath == "" {
*dirPath = "/var/tmp"
if dirPath == "" {
dirPath = "/var/tmp"
}
WalkDirectory(*dirPath)
fmt.Println(configs[0])
if err := debug(configs[0]); err != nil {
fmt.Println(err)
if command == "" {
command = "gdb"
}
WalkDirectory(dirPath)
for _, config := range configs {
fmt.Println(config)
if strings.Compare(config.Initial_ns_pid, pid) == 0 || pid == "" {
err := debug(config, command)
if err != nil {
fmt.Println(err)
} else {
break
}
}
}
return
}

19
coredump-handler/coredump-handler.go
View File

@@ -55,8 +55,8 @@ func isDiskSufficient(pipe_config types.Pipeconfig) (bool, error) {
return true, nil
}
func createCoreDumpDir(pipe_config *types.Pipeconfig, args types.Coredump_config) error {
pipe_config.File_base_path = fmt.Sprintf("%s/%s_%s_%d", pipe_config.File_base_path, args.Initial_ns_pid, args.Process_ns_pid, args.Timestamp)
dirName := pipe_config.File_base_path
pipe_config.Storage = fmt.Sprintf("%s/%s_%s_%d_coredump", pipe_config.Storage, args.Initial_ns_pid, args.Process_ns_pid, args.Timestamp)
dirName := pipe_config.Storage
if _, err := os.Stat(dirName); os.IsNotExist(err) {
// 目录不存在，创建目录
if err := os.MkdirAll(dirName, os.ModePerm); err != nil {
@@ -212,9 +212,9 @@ func main() {
journal.Print(journal.PriErr, err.Error())
return
}
coredump_config.Storage = pipe_config.File_base_path
coredump_config.Storage = pipe_config.Storage
//切换至存储coredump目录
err = changeDirectory(pipe_config.File_base_path)
err = changeDirectory(pipe_config.Storage)
if err != nil {
journal.Print(journal.PriErr, err.Error())
return
@@ -228,19 +228,22 @@ func main() {
journal.Print(journal.PriInfo, err.Error())
}
}
if coredump_config.Image_id == "" {
coredump_config.Image_id = "NULL"
}
//根据配置项选择存储coredump文件方式
if pipe_config.Save_model == 0 {
if pipe_config.Compress == false {
err = writeCoreDumpToFile(coredump_config)
if err != nil {
journal.Print(journal.PriErr, err.Error())
}
coredump_config.Storage = fmt.Sprintf("%s/%s_%s_%d_coredump.info", pipe_config.File_base_path, coredump_config.Initial_ns_pid, coredump_config.Process_ns_pid, coredump_config.Timestamp)
} else if pipe_config.Save_model == 1 {
coredump_config.Storage = fmt.Sprintf("%s/%s_%s_%d_coredump.info", pipe_config.Storage, coredump_config.Initial_ns_pid, coredump_config.Process_ns_pid, coredump_config.Timestamp)
} else if pipe_config.Compress == true {
err = compress(coredump_config)
if err != nil {
journal.Print(journal.PriErr, err.Error())
}
coredump_config.Storage = fmt.Sprintf("%s/%s_%s_%d_coredump.info.zip", pipe_config.File_base_path, coredump_config.Initial_ns_pid, coredump_config.Process_ns_pid, coredump_config.Timestamp)
coredump_config.Storage = fmt.Sprintf("%s/%s_%s_%d_coredump.info.zip", pipe_config.Storage, coredump_config.Initial_ns_pid, coredump_config.Process_ns_pid, coredump_config.Timestamp)
}
//将image name写入coredump config
err = writeCoreConfig(coredump_config)

2
coredump-handler/coredump-handler_test.go
View File

@@ -53,7 +53,7 @@ func TestIsDiskSufficient(t *testing.T) {
func TestCreateCoreDumpDir(t *testing.T) {
pipe_config := types.Pipeconfig{
File_base_path: "/tmp",
Storage: "/tmp",
}
err := createCoreDumpDir(&pipe_config, coredump_config)
if err != nil {

11
types/types.go
View File

@@ -7,17 +7,18 @@ type Coreconfig struct {
Socket_path string
}
type Pipeconfig struct {
Save_model int //0为文件保存 1为压缩保存 2为minidump保存
File_base_path string
Total_file_mem_limit string
Containered_sock_path string
Compress bool //是否压缩
Storage_type int //是否存储为minidump格式
Storage string //存储位置
Total_file_mem_limit string //磁盘空间限制
Containered_sock_path string //ctr sock地址
}
type Coredump_config struct {
Initial_ns_pid string
Process_ns_pid string
Process_exe_path string
Timestamp int64
Corepipe_config_path string
Corepipe_config_path string `json:"-"`
Image_id string
UID string
GID string