linxin-coredump-tools/coredump-handler/coredump-handler.go

package main

//#cgo CXXFLAGS: -std=c++11
// #cgo CFLAGS: -I .
// #cgo LDFLAGS: /opt/tsg/coredump/bin/coredump_handler_wrapper.so
//#include "coredump_handler_wrapper.h"
import "C"

import (
	"archive/zip"
	"bufio"
	"context"
	"coredump-tools/config"
	"coredump-tools/types"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"math"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	"github.com/containerd/containerd"
	"github.com/containerd/containerd/namespaces"
	"github.com/coreos/go-systemd/v22/journal"
	"github.com/shirou/gopsutil/disk"
)

var coredump_config types.Coredump_config
var percent int64
var sysrootPath string

func argsJudge() error {

	if coredump_config.Initial_ns_pid == "" ||
		coredump_config.Process_ns_pid == "" ||
		coredump_config.Corepipe_config_path == "" ||
		coredump_config.Timestamp == 0 ||
		coredump_config.Process_exe_path == "" ||
		coredump_config.GID == "" ||
		coredump_config.Hostname == "" ||
		coredump_config.UID == "" ||
		coredump_config.Signal == -1 {
		err := fmt.Sprintf("Failed to initialize command line parameters. -P=%s -p=%s -E=%s -configpath=%s -t=%d -g=%s -h=%s -s=%d -u=%s",
			coredump_config.Initial_ns_pid,
			coredump_config.Process_ns_pid,
			coredump_config.Process_exe_path,
			coredump_config.Corepipe_config_path,
			coredump_config.Timestamp,
			coredump_config.GID,
			coredump_config.Hostname,
			coredump_config.Signal,
			coredump_config.UID)
		return errors.New(err)
	}
	return nil
}

// judge disk's space is sufficient or not
func isDiskSufficient(pipe_config types.Pipeconfig) (bool, string, error) {
	wd := pipe_config.Storage
	diskinfo, err := disk.Usage(wd)
	if err != nil {
		return false, "", err
	}
	usage := int64(math.Floor(diskinfo.UsedPercent + 0.5))
	info := fmt.Sprintf("usage:%d,limited:%d", usage, percent)
	if usage >= percent {
		return false, info, nil
	}
	return true, info, nil
}

// create dir to storage coredump file and coredump info
func createCoreDumpDir(pipe_config *types.Pipeconfig, args types.Coredump_config) error {
	pipe_config.Storage = fmt.Sprintf("%s/coredump_%s_%s_%d", pipe_config.Storage, args.Initial_ns_pid, args.Process_ns_pid, args.Timestamp)
	dirName := pipe_config.Storage
	if _, err := os.Stat(dirName); os.IsNotExist(err) {
		if err := os.MkdirAll(dirName, os.ModePerm); err != nil {
			return err
		}
	} else {
		return errors.New("directory already exists")
	}
	return nil
}

// chmod to storage dir
func changeDirectory(dir string) error {
	if err := os.Chdir(dir); err != nil {
		return err
	}
	return nil
}

// get container id by search /proc dir
func getContainerId(pid string) (string, error) {
	cgroup_path := fmt.Sprintf("/proc/%s/cgroup", pid)
	content, err := os.ReadFile(cgroup_path)
	if err != nil {
		return "", err
	}
	re := regexp.MustCompile(`([a-f\d]{64})`)
	match := re.FindStringSubmatch(string(content))
	if len(match) < 2 {
		return "", errors.New("failed to extract container ID from cgroup file")
	}
	containerID := match[1]
	return containerID, nil
}

// get container info by container id
func getContainerInfo(container_id string, sock_path string) (string, string, error) {
	// connect to containerd daemon
	client, err := containerd.New(sock_path)
	if err != nil {
		return "", "", err
	}
	defer client.Close()
	// according to container id to get container info
	ctx := namespaces.WithNamespace(context.Background(), "k8s.io")
	container, err := client.LoadContainer(ctx, container_id)
	if err != nil {
		return "", "", err
	}
	imageRef, err := container.Image(ctx)
	if err != nil {
		return "", "", err
	}
	label, err := container.Labels(ctx)
	if err != nil {
		return "", "", err
	}
	pod_name, ok := label["io.kubernetes.pod.name"]
	if !ok {
		return imageRef.Name(), "", nil
	}
	return imageRef.Name(), pod_name, nil
}

// write coredump info to file
func writeCoreConfig(config types.Coredump_config) error {
	filename := fmt.Sprintf("%s_%s_%d.info", config.Initial_ns_pid, config.Process_ns_pid, config.Timestamp)
	file, err := os.Create(filename)
	if err != nil {
		return err
	}
	encoder := json.NewEncoder(file)
	err = encoder.Encode(&config)
	if err != nil {
		return err
	}
	return nil
}

// write coredump to file
func writeCoreDumpToFile(config types.Coredump_config, pipe_config types.Pipeconfig) error {
	filename := fmt.Sprintf("%s_%s_%d.coredump", config.Initial_ns_pid, config.Process_ns_pid, config.Timestamp)
	file, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer file.Close()
	buf := make([]byte, 10*1024*1024)
	for {
		n, err := os.Stdin.Read(buf)
		if err != nil && err != io.EOF {
			return err
		}
		if n == 0 {
			break
		}
		_, err = file.Write(buf[:n])
		if err != nil {
			return err
		}
		flag, _, err := isDiskSufficient(pipe_config)
		if err != nil {
			journal.Print(journal.PriErr, "Can't judge disk's space is sufficient or not. "+err.Error())
			return err
		}
		if !flag {
			return errors.New("Disk space exceeds limit when writing coredump!")
		}
	}
	return nil
}

func HandleCrash(pid int, procfsDir string, mdFilename string) bool {
	journal.Print(journal.PriInfo, "start convert to minidump")
	return bool(C.HandleCrash(C.pid_t(pid), C.CString(procfsDir), C.CString(mdFilename)))
}

func writeMiniDumpToFile(config types.Coredump_config) error {
	filename := fmt.Sprintf("%s/%s_%s_%d.minidump", config.Storage, config.Initial_ns_pid, config.Process_ns_pid, config.Timestamp)
	pid, err := strconv.Atoi(config.Initial_ns_pid)
	if err != nil {
		return err
	}
	flag := HandleCrash(pid, "/proc/"+config.Initial_ns_pid, filename)
	if !flag {
		return errors.New("can not convert to minidump")
	}
	journal.Print(journal.PriInfo, "end convert to minidump")
	return nil
}

func compress(config types.Coredump_config) error {
	// Create a new zip archive.
	filename := fmt.Sprintf("%s_%s_%d.coredump", config.Initial_ns_pid, config.Process_ns_pid, config.Timestamp)
	zipfile, err := os.Create(filename + ".zip")
	if err != nil {
		return err
	}
	defer zipfile.Close()

	// Create a new zip writer.
	zipwriter := zip.NewWriter(zipfile)
	defer zipwriter.Close()

	// Create a zip file header.
	header := &zip.FileHeader{
		Name:   filename,
		Method: zip.Deflate,
	}

	// Write the header to the zip file.
	writer, err := zipwriter.CreateHeader(header)
	if err != nil {
		return err
	}

	// Set the block size to 1 megabyte.
	const blockSize = 1024 * 1024

	// Read and write the input in blocks.
	buf := make([]byte, blockSize)
	for {
		n, err := os.Stdin.Read(buf)
		if err != nil && err != io.EOF {
			return err
		}
		if n == 0 {
			break
		}
		_, err = writer.Write(buf[:n])
		if err != nil {
			return err
		}
	}

	return nil
}

func getBinaryFileFromMaps(mapsFile string) ([]string, error) {
	file, err := os.Open(mapsFile)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	binaryFileSet := make(map[string]struct{})
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line := scanner.Text()
		parts := strings.Fields(line)
		if len(parts) > 5 && !strings.Contains(parts[5], "[") && strings.Contains(parts[1], "x") {
			binaryFileSet[parts[5]] = struct{}{}
		}
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}

	var binaryFiles []string
	for binaryFile := range binaryFileSet {
		binaryFiles = append(binaryFiles, binaryFile)
	}

	return binaryFiles, nil
}

func getSymlinkFile(binaryFiles []string) []string {
	Files := make(map[string]bool)
	for i := 0; i < len(binaryFiles); i++ {
		exePath := binaryFiles[i]
		Files[exePath] = true
		exeDir := filepath.Dir(exePath)
		exePath = filepath.Base(exePath)
		files, err := os.ReadDir(sysrootPath + "/root" + exeDir)
		if err != nil {
			journal.Print(journal.PriErr, "read dir %s err: %v\n", exeDir, err)
			continue
		}
		for _, file := range files {
			if file.Type()&os.ModeSymlink != 0 {
				linkPath := filepath.Join(exeDir, file.Name())
				targetPath, err := os.Readlink(sysrootPath + "/root" + linkPath)
				if err != nil {
					journal.Print(journal.PriErr, "read %s err: %v\n", linkPath, err)
					continue
				}

				if targetPath == exePath {
					if _, ok := Files[linkPath]; ok {
						continue
					}
					binaryFiles = append(binaryFiles, linkPath)
				}
			}
		}
	}
	return binaryFiles
}

func getLdconf(binaryFiles []string) []string {
	binaryFiles = append(binaryFiles, "/etc/ld.so.conf")
	ldConfDir := "/etc/ld.so.conf.d"

	// 检查目录是否存在
	_, err := os.Stat(sysrootPath + "/root" + ldConfDir)
	if os.IsNotExist(err) {
		journal.Print(journal.PriErr, "directory %s is not exist", ldConfDir)
		return binaryFiles
	}

	// 读取目录下的所有文件
	files, err := os.ReadDir(sysrootPath + "/root" + ldConfDir)
	if err != nil {
		journal.Print(journal.PriErr, "can not read directory %s: %v", ldConfDir, err)
		return binaryFiles
	}
	for _, file := range files {
		if !file.IsDir() {
			filePath := filepath.Join(ldConfDir, file.Name())
			binaryFiles = append(binaryFiles, filePath)
		}
	}
	return binaryFiles
}

func addFileToZip(zipWriter *zip.Writer, filename string, pipe_config types.Pipeconfig) error {
	binaryFileAbsPath := fmt.Sprintf("%s/root%s", sysrootPath, filename)
	fileToZip, err := os.Open(binaryFileAbsPath)
	if err != nil {
		return err
	}
	defer fileToZip.Close()

	info, err := fileToZip.Stat()
	if err != nil {
		return err
	}

	header, err := zip.FileInfoHeader(info)
	if err != nil {
		return err
	}
	header.Name = filename
	header.Method = zip.Deflate

	writer, err := zipWriter.CreateHeader(header)
	if err != nil {
		return err
	}

	_, err = io.Copy(writer, fileToZip)
	if err != nil {
		return err
	}
	flag, _, err := isDiskSufficient(pipe_config)
	if err != nil {
		journal.Print(journal.PriErr, "Can't judge disk's space is sufficient or not. "+err.Error())
		return err
	}
	if !flag {
		return errors.New("Disk space exceeds limit when writing coredump!")
	}
	return nil
}

func writeBinaryFile(pipe_config types.Pipeconfig, filePath string) error {
	mapsFile := fmt.Sprintf("%s/maps", sysrootPath)
	binaryFiles, err := getBinaryFileFromMaps(mapsFile)
	if err != nil {
		return err
	}
	binaryFiles = getSymlinkFile(binaryFiles)
	binaryFiles = getLdconf(binaryFiles)
	zipFile, err := os.Create(filePath)
	if err != nil {
		return err
	}
	defer zipFile.Close()

	zipWriter := zip.NewWriter(zipFile)
	defer zipWriter.Close()

	for _, file := range binaryFiles {
		addFileToZip(zipWriter, file, pipe_config)
	}

	journal.Print(journal.PriInfo, "Tar file created successfully")
	return nil
}

func init() {
	flag.StringVar(&coredump_config.Initial_ns_pid, "P", "", "initial ns pid")
	flag.StringVar(&coredump_config.Process_ns_pid, "p", "", "process ns pid")
	flag.StringVar(&coredump_config.Process_exe_path, "E", "", "pathname of executable process")
	flag.StringVar(&coredump_config.Corepipe_config_path, "C", "", "configfile's path")
	flag.Int64Var(&coredump_config.Timestamp, "t", 0, "the time of `coredump")
	flag.StringVar(&coredump_config.GID, "g", "", "Numeric real GID of dumped process.")
	flag.IntVar(&coredump_config.Signal, "s", -1, "Number of signal causing dump")
	flag.StringVar(&coredump_config.UID, "u", "", "Numeric real UID of dumped process.")
}

func start() {
	var err error
	coredump_config.Hostname, err = os.Hostname()
	if err != nil {
		journal.Print(journal.PriErr, err.Error())
	}
	info := fmt.Sprintf("initialize command line parameters. -P=%s -p=%s -E=%s -C=%s -t=%d -g=%s -h=%s -s=%d -u=%s", coredump_config.Initial_ns_pid, coredump_config.Process_ns_pid, coredump_config.Process_exe_path, coredump_config.Corepipe_config_path, coredump_config.Timestamp, coredump_config.GID, coredump_config.Hostname, coredump_config.Signal, coredump_config.UID)
	journal.Print(journal.PriInfo, info)
	coredump_config.Process_exe_path = strings.Replace(coredump_config.Process_exe_path, "!", "/", -1)
	//Judge agrs is correct or not.
	sysrootPath = fmt.Sprintf("/proc/%s", coredump_config.Initial_ns_pid)
	err = argsJudge()
	if err != nil {
		journal.Print(journal.PriErr, err.Error())
		return
	}
	//read config file and judge config info is correct or not.
	pipe_config, err := config.PipeInit(coredump_config.Corepipe_config_path)
	if err != nil {
		journal.Print(journal.PriErr, err.Error())
		return
	}
	if pipe_config.Storage_type == 0 {
		return
	}
	//create source dir
	if _, err := os.Stat(pipe_config.Storage); os.IsNotExist(err) {
		if err := os.MkdirAll(pipe_config.Storage, os.ModePerm); err != nil {
			journal.Print(journal.PriErr, err.Error())
			return
		}
	}
	percent, err = strconv.ParseInt(pipe_config.Total_file_mem_limit[:len(pipe_config.Total_file_mem_limit)-1], 10, 32)
	if err != nil {
		journal.Print(journal.PriErr, err.Error())
		return
	}
	//judge disk usage
	flag, info, err := isDiskSufficient(pipe_config)
	if err != nil {
		journal.Print(journal.PriErr, err.Error())
		return
	}
	if info != "" {
		journal.Print(journal.PriInfo, info)
	}
	if !flag {
		journal.Print(journal.PriErr, "Disk space exceeds limit")
		return
	}
	//create dir to storage coredump
	err = createCoreDumpDir(&pipe_config, coredump_config)
	if err != nil {
		journal.Print(journal.PriErr, err.Error())
		return
	}
	coredump_config.Storage = pipe_config.Storage
	//chmod to coredump dir
	err = changeDirectory(pipe_config.Storage)
	if err != nil {
		journal.Print(journal.PriErr, err.Error())
		return
	}
	//find coredump process's container id
	container_id, err := getContainerId(coredump_config.Initial_ns_pid)
	//find image name
	if err == nil && len(container_id) != 0 {
		coredump_config.Container_id = container_id
		coredump_config.Image_name, coredump_config.Pod_name, err = getContainerInfo(container_id, pipe_config.Containered_sock_path)
		if err != nil {
			journal.Print(journal.PriInfo, err.Error())
		}
	}
	if coredump_config.Container_id == "" {
		coredump_config.Container_id = "NULL"
	}
	if coredump_config.Image_name == "" {
		coredump_config.Image_name = "NULL"
	}
	if coredump_config.Pod_name == "" {
		coredump_config.Pod_name = "NULL"
	}
	//write coredump file
	if !pipe_config.Compress {
		if pipe_config.Storage_type == 1 {
			err = writeCoreDumpToFile(coredump_config, pipe_config)
			if err != nil {
				journal.Print(journal.PriErr, err.Error())
			}
			coredump_config.Storage = fmt.Sprintf("%s/%s_%s_%d.coredump", pipe_config.Storage, coredump_config.Initial_ns_pid, coredump_config.Process_ns_pid, coredump_config.Timestamp)
		} else if pipe_config.Storage_type == 2 {
			err = writeMiniDumpToFile(coredump_config)
			if err != nil {
				journal.Print(journal.PriErr, err.Error())
			}
			coredump_config.Storage = fmt.Sprintf("%s/%s_%s_%d.minidump", pipe_config.Storage, coredump_config.Initial_ns_pid, coredump_config.Process_ns_pid, coredump_config.Timestamp)
		}
	} else {
		err = compress(coredump_config)
		if err != nil {
			journal.Print(journal.PriErr, err.Error())
		}
		coredump_config.Storage = fmt.Sprintf("%s/%s_%s_%d.coredump.zip", pipe_config.Storage, coredump_config.Initial_ns_pid, coredump_config.Process_ns_pid, coredump_config.Timestamp)
	}
	coredump_config.Binary_file = fmt.Sprintf("%s/%s_%s_%d.Binary_file.zip", pipe_config.Storage, coredump_config.Initial_ns_pid, coredump_config.Process_ns_pid, coredump_config.Timestamp)
	writeBinaryFile(pipe_config, coredump_config.Binary_file)
	//write coredump info
	err = writeCoreConfig(coredump_config)
	if err != nil {
		journal.Print(journal.PriErr, err.Error())
	}
	flag, info, err = isDiskSufficient(pipe_config)
	if err != nil {
		journal.Print(journal.PriErr, "Can't judge disk's space is sufficient or not. "+err.Error())
		return
	}
	if info != "" {
		journal.Print(journal.PriInfo, info)
	}
	if !flag {
		journal.Print(journal.PriErr, "Disk space exceeds limit after write coredump!")
		err := os.RemoveAll(pipe_config.Storage)
		if err != nil {
			journal.Print(journal.PriErr, err.Error())
			return
		}
		journal.Print(journal.PriInfo, err.Error())
		return
	}
}

func main() {
	info := fmt.Sprintf("start handle coredump")
	journal.Print(journal.PriInfo, info)
	flag.Parse()
	start()
	journal.Print(journal.PriInfo, "finish to write coredump")
}