533 lines
13 KiB
Go
533 lines
13 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"coredump-tools/types"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/alexeyco/simpletable"
|
|
"github.com/google/uuid"
|
|
"github.com/urfave/cli/v2"
|
|
"golang.org/x/term"
|
|
v1 "k8s.io/api/core/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/client-go/kubernetes"
|
|
"k8s.io/client-go/kubernetes/scheme"
|
|
"k8s.io/client-go/rest"
|
|
"k8s.io/client-go/tools/clientcmd"
|
|
"k8s.io/client-go/tools/remotecommand"
|
|
)
|
|
|
|
var configs []types.Coredump_config
|
|
|
|
var (
|
|
pid string
|
|
dirPath string
|
|
command string
|
|
prestartPath string
|
|
)
|
|
|
|
// WalkDirectory search file with suffix name ".info"
|
|
|
|
func WalkDirectory(dir string) {
|
|
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if info.IsDir() && strings.HasPrefix(info.Name(), "coredump_") {
|
|
err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if !info.IsDir() && strings.HasSuffix(info.Name(), ".info") {
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
var config types.Coredump_config
|
|
err = json.Unmarshal(data, &config)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if config.Container_id == "" {
|
|
config.Container_id = "NULL"
|
|
}
|
|
if config.Image_name == "" {
|
|
config.Image_name = "NULL"
|
|
}
|
|
if _, err := os.Stat(config.Storage); err != nil {
|
|
config.Storage = "MISS"
|
|
}
|
|
configs = append(configs, config)
|
|
}
|
|
|
|
return nil
|
|
})
|
|
|
|
if err != nil {
|
|
fmt.Printf("Error walking directory %s: %v\n", path, err)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
})
|
|
sort.Slice(configs, func(i, j int) bool {
|
|
return configs[i].Timestamp < configs[j].Timestamp
|
|
})
|
|
if err != nil {
|
|
fmt.Printf("Error walking directory %s: %v\n", dir, err)
|
|
}
|
|
}
|
|
func list(pid string) {
|
|
table := simpletable.New()
|
|
|
|
table.Header = &simpletable.Header{
|
|
Cells: []*simpletable.Cell{
|
|
{Text: "PID"},
|
|
{Text: "EXE"},
|
|
{Text: "IMAGE"},
|
|
{Text: "TIMESTAMP"},
|
|
},
|
|
}
|
|
total := 0
|
|
// output the config's info
|
|
for _, c := range configs {
|
|
if pid != "" && strings.Compare(c.Initial_ns_pid, pid) != 0 {
|
|
continue
|
|
}
|
|
coreTime := time.Unix(c.Timestamp, 0).Format("2006-01-02 15:04:05")
|
|
r := []*simpletable.Cell{
|
|
{Text: c.Initial_ns_pid},
|
|
{Text: c.Process_exe_path},
|
|
{Text: c.Image_name},
|
|
{Text: coreTime},
|
|
}
|
|
table.Body.Cells = append(table.Body.Cells, r)
|
|
total += 1
|
|
}
|
|
fmt.Println(table.String())
|
|
fmt.Println("Total", total, "coredumps")
|
|
}
|
|
|
|
func info(pid string) {
|
|
table := simpletable.New()
|
|
|
|
table.Header = &simpletable.Header{
|
|
Cells: []*simpletable.Cell{
|
|
{Text: "PID"},
|
|
{Text: "UID"},
|
|
{Text: "GID"},
|
|
{Text: "SIG"},
|
|
{Text: "EXE"},
|
|
{Text: "POD"},
|
|
{Text: "IMAGE"},
|
|
{Text: "HOSTNAME"},
|
|
{Text: "STORAGE"},
|
|
{Text: "TIMESTAMP"},
|
|
},
|
|
}
|
|
total := 0
|
|
// output the config's info
|
|
for _, c := range configs {
|
|
if pid != "" && strings.Compare(c.Initial_ns_pid, pid) != 0 {
|
|
continue
|
|
}
|
|
coreTime := time.Unix(c.Timestamp, 0).Format("2006-01-02 15:04:05")
|
|
r := []*simpletable.Cell{
|
|
{Text: c.Initial_ns_pid},
|
|
{Text: c.UID},
|
|
{Text: c.GID},
|
|
{Text: strconv.Itoa(c.Signal)},
|
|
{Text: c.Process_exe_path},
|
|
{Text: c.Pod_name},
|
|
{Text: c.Image_name},
|
|
{Text: c.Hostname},
|
|
{Text: c.Storage},
|
|
{Text: coreTime},
|
|
}
|
|
table.Body.Cells = append(table.Body.Cells, r)
|
|
total += 1
|
|
}
|
|
fmt.Println(table.String())
|
|
fmt.Println("Total", total, "coredumps")
|
|
}
|
|
|
|
func debug(config types.Coredump_config, command, prestartPath string) error {
|
|
if strings.HasSuffix(config.Storage, ".minidump") {
|
|
corefile := strings.Replace(config.Storage, ".minidump", ".coredump", -1)
|
|
cmd := exec.Command("minidump-2-core", "-o", corefile, config.Storage)
|
|
var stderr bytes.Buffer
|
|
cmd.Stderr = &stderr
|
|
err := cmd.Run()
|
|
if err != nil {
|
|
return (errors.New(err.Error() + ":" + stderr.String()))
|
|
}
|
|
config.Storage = corefile
|
|
defer os.Remove(corefile)
|
|
}
|
|
if config.Image_name != "NULL" {
|
|
kubeconfig := os.Getenv("KUBECONFIG")
|
|
if kubeconfig == "" {
|
|
kubeconfig = os.Getenv("HOME") + "/.kube/config"
|
|
}
|
|
|
|
// Creates the kubernetes client using the specified kubeconfig
|
|
conf, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
clientset, err := kubernetes.NewForConfig(conf)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
podName, err := debugInpod(conf, clientset, config, command, prestartPath)
|
|
if err != nil {
|
|
fmt.Println(err)
|
|
}
|
|
// Delete the Pod
|
|
fmt.Printf("Deleting Pod %q...\n", podName)
|
|
err = clientset.CoreV1().Pods("default").Delete(context.Background(), podName, metav1.DeleteOptions{})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("Deleted Pod %q.\n", podName)
|
|
return nil
|
|
} else { // using gdb to debug the coredump file
|
|
cmd := exec.Command("gdb", config.Process_exe_path, config.Storage, "-cd", filepath.Dir(config.Process_exe_path))
|
|
cmd.Stdin = os.Stdin
|
|
cmd.Stdout = os.Stdout
|
|
cmd.Stderr = os.Stderr
|
|
fmt.Println(cmd.String())
|
|
if err := cmd.Run(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
func debugInpod(conf *rest.Config, clientset *kubernetes.Clientset, config types.Coredump_config, command, prestartPath string) (string, error) {
|
|
// Define the Pod object
|
|
id := uuid.New()
|
|
fmt.Println(id.String())
|
|
podName := fmt.Sprintf("coredump-debug-%s", id.String())
|
|
containerName := "debug"
|
|
pod := &v1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: podName,
|
|
Namespace: "default",
|
|
},
|
|
Spec: v1.PodSpec{
|
|
Containers: []v1.Container{
|
|
{
|
|
Name: "debug",
|
|
Image: config.Image_name,
|
|
ImagePullPolicy: "IfNotPresent",
|
|
Command: []string{
|
|
"tail",
|
|
"-f",
|
|
},
|
|
VolumeMounts: []v1.VolumeMount{
|
|
{
|
|
Name: "host-dir",
|
|
MountPath: "/host",
|
|
},
|
|
{
|
|
Name: "lib-debuginfo-dir",
|
|
MountPath: "/usr/lib/debug",
|
|
},
|
|
{
|
|
Name: "src-debuginfo-dir",
|
|
MountPath: "/usr/src/debug",
|
|
},
|
|
{
|
|
Name: "mrzcpd",
|
|
MountPath: "/opt/tsg/mrzcpd",
|
|
},
|
|
},
|
|
SecurityContext: &v1.SecurityContext{
|
|
Privileged: &[]bool{true}[0],
|
|
},
|
|
},
|
|
},
|
|
Volumes: []v1.Volume{
|
|
{
|
|
Name: "host-dir",
|
|
VolumeSource: v1.VolumeSource{
|
|
HostPath: &v1.HostPathVolumeSource{
|
|
Path: "/",
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "lib-debuginfo-dir",
|
|
VolumeSource: v1.VolumeSource{
|
|
HostPath: &v1.HostPathVolumeSource{
|
|
Path: "/usr/lib/debug",
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "src-debuginfo-dir",
|
|
VolumeSource: v1.VolumeSource{
|
|
HostPath: &v1.HostPathVolumeSource{
|
|
Path: "/usr/src/debug",
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "mrzcpd",
|
|
VolumeSource: v1.VolumeSource{
|
|
HostPath: &v1.HostPathVolumeSource{
|
|
Path: "/opt/tsg/mrzcpd",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
RestartPolicy: v1.RestartPolicyNever,
|
|
},
|
|
}
|
|
if prestartPath != "" {
|
|
prestartVolumeMount := v1.VolumeMount{
|
|
Name: "prestart-dir",
|
|
MountPath: "/prestart.sh",
|
|
}
|
|
prestartVolume := v1.Volume{
|
|
Name: "prestart-dir",
|
|
VolumeSource: v1.VolumeSource{
|
|
HostPath: &v1.HostPathVolumeSource{
|
|
Path: prestartPath,
|
|
},
|
|
},
|
|
}
|
|
pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, prestartVolumeMount)
|
|
pod.Spec.Volumes = append(pod.Spec.Volumes, prestartVolume)
|
|
pod.Spec.Containers[0].Command = []string{
|
|
"sh",
|
|
"-c",
|
|
"chmod +x /prestart.sh && /prestart.sh && tail -f",
|
|
}
|
|
}
|
|
// Create the Pod
|
|
fmt.Println("Creating Pod...")
|
|
fmt.Printf("Creating Pod %q...\n", podName)
|
|
result, err := clientset.CoreV1().Pods("default").Create(context.Background(), pod, metav1.CreateOptions{})
|
|
if err != nil {
|
|
return podName, err
|
|
}
|
|
fmt.Printf("Created Pod %q.\n", result.GetObjectMeta().GetName())
|
|
|
|
// Wait for the Pod to be running and ready
|
|
fmt.Printf("Waiting for Pod %q to be ready...\n", podName)
|
|
ready := false
|
|
for i := 0; i < 10; i++ {
|
|
result, err := clientset.CoreV1().Pods("default").Get(context.Background(), podName, metav1.GetOptions{})
|
|
if err != nil {
|
|
return podName, err
|
|
}
|
|
status := result.Status
|
|
if status.Phase == v1.PodRunning && len(status.ContainerStatuses) > 0 && status.ContainerStatuses[0].Ready {
|
|
ready = true
|
|
break
|
|
}
|
|
time.Sleep(3 * time.Second)
|
|
}
|
|
if !ready {
|
|
return podName, errors.New("create pod timeout")
|
|
}
|
|
// Disable canonical mode and enable echo mode.
|
|
oldState, err := term.MakeRaw(int(os.Stdin.Fd()))
|
|
if err != nil {
|
|
return podName, err
|
|
}
|
|
defer term.Restore(int(os.Stdin.Fd()), oldState)
|
|
// Create exec request
|
|
var cmd []string
|
|
if command == "gdb" {
|
|
cmd = []string{"gdb", config.Process_exe_path, "/host" + config.Storage, "-cd", filepath.Dir(config.Process_exe_path)}
|
|
} else {
|
|
cmd = []string{command}
|
|
}
|
|
req := clientset.CoreV1().RESTClient().Post().Resource("pods").
|
|
Name(podName).Namespace("default").
|
|
SubResource("exec").
|
|
VersionedParams(&v1.PodExecOptions{
|
|
Container: containerName,
|
|
Command: cmd,
|
|
Stdin: true,
|
|
Stdout: true,
|
|
Stderr: true,
|
|
TTY: true,
|
|
}, scheme.ParameterCodec)
|
|
|
|
// Create exec executor
|
|
executor, err := remotecommand.NewSPDYExecutor(conf, "POST", req.URL())
|
|
if err != nil {
|
|
return podName, err
|
|
}
|
|
|
|
// Start exec
|
|
err = executor.StreamWithContext(context.Background(), remotecommand.StreamOptions{
|
|
Stdin: os.Stdin,
|
|
Stdout: os.Stdout,
|
|
Stderr: os.Stderr,
|
|
Tty: true,
|
|
})
|
|
if err != nil {
|
|
return podName, err
|
|
}
|
|
return podName, nil
|
|
}
|
|
|
|
func command_init() cli.App {
|
|
return cli.App{
|
|
Name: "coredump",
|
|
Usage: "Manage coredump files in Kubernetes clusters",
|
|
Commands: []*cli.Command{
|
|
{
|
|
Name: "list",
|
|
Aliases: []string{"ls"},
|
|
Usage: "List all coredump files",
|
|
Flags: []cli.Flag{
|
|
&cli.StringFlag{
|
|
Name: "pid",
|
|
Aliases: []string{"p"},
|
|
Usage: "Pid to match",
|
|
Value: "",
|
|
Destination: &pid,
|
|
},
|
|
&cli.StringFlag{
|
|
Name: "dir",
|
|
Aliases: []string{"d"},
|
|
Usage: "Coredump directory path(default: /var/lib/coredump)",
|
|
Value: "/var/lib/coredump",
|
|
Destination: &dirPath,
|
|
},
|
|
},
|
|
Action: func(c *cli.Context) error {
|
|
WalkDirectory(dirPath)
|
|
list(pid)
|
|
return nil
|
|
},
|
|
},
|
|
{
|
|
Name: "info",
|
|
Aliases: []string{"i"},
|
|
Usage: "Display all coredump files's info",
|
|
Flags: []cli.Flag{
|
|
&cli.StringFlag{
|
|
Name: "pid",
|
|
Aliases: []string{"p"},
|
|
Usage: "Pid to match",
|
|
Value: "",
|
|
Destination: &pid,
|
|
},
|
|
&cli.StringFlag{
|
|
Name: "dir",
|
|
Aliases: []string{"d"},
|
|
Usage: "Coredump directory path(default: /var/lib/coredump)",
|
|
Value: "/var/lib/coredump",
|
|
Destination: &dirPath,
|
|
},
|
|
},
|
|
Action: func(c *cli.Context) error {
|
|
WalkDirectory(dirPath)
|
|
info(pid)
|
|
return nil
|
|
},
|
|
},
|
|
{
|
|
Name: "debug",
|
|
Usage: "Start a debugging session for a coredump",
|
|
Flags: []cli.Flag{
|
|
&cli.StringFlag{
|
|
Name: "pid",
|
|
Aliases: []string{"p"},
|
|
Usage: "Pid to match",
|
|
Value: "",
|
|
Destination: &pid,
|
|
},
|
|
&cli.StringFlag{
|
|
Name: "dir",
|
|
Aliases: []string{"d"},
|
|
Usage: "Coredump directory path(default: /var/lib/coredump)",
|
|
Value: "/var/lib/coredump",
|
|
Destination: &dirPath,
|
|
},
|
|
&cli.StringFlag{
|
|
Name: "command",
|
|
Aliases: []string{"c"},
|
|
Usage: "Debugger command (default: gdb)",
|
|
Value: "gdb",
|
|
Destination: &command,
|
|
},
|
|
&cli.StringFlag{
|
|
Name: "prestart",
|
|
Aliases: []string{"P"},
|
|
Usage: "Prestart script path just used in Pod's coredump debug",
|
|
Value: "",
|
|
Destination: &prestartPath,
|
|
},
|
|
},
|
|
Action: func(c *cli.Context) error {
|
|
if prestartPath != "" {
|
|
_, err := os.Stat(prestartPath)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
fmt.Println("prestartfile is not exist!")
|
|
} else {
|
|
fmt.Println(err)
|
|
}
|
|
return nil
|
|
}
|
|
}
|
|
WalkDirectory(dirPath)
|
|
for _, config := range configs {
|
|
if strings.Compare(config.Initial_ns_pid, pid) == 0 || pid == "" {
|
|
if prestartPath != "" && config.Image_name == "NULL" {
|
|
fmt.Println("Coredump is not generate in Pod.Prestart just use in Pod's coredump debug!")
|
|
return nil
|
|
}
|
|
err := debug(config, command, prestartPath)
|
|
if err != nil {
|
|
fmt.Println(err)
|
|
} else {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
},
|
|
{
|
|
Name: "help",
|
|
Usage: "Show help message",
|
|
Action: func(c *cli.Context) error {
|
|
cli.ShowAppHelp(c)
|
|
return nil
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
func main() {
|
|
app := command_init()
|
|
err := app.Run(os.Args)
|
|
if err != nil {
|
|
fmt.Println(err)
|
|
}
|
|
}
|