379 lines
9.8 KiB
Go
379 lines
9.8 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"coredump-tools/types"
|
|
"crypto/rand"
|
|
"encoding/json"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"math/big"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"golang.org/x/crypto/ssh/terminal"
|
|
v1 "k8s.io/api/core/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/client-go/kubernetes"
|
|
"k8s.io/client-go/kubernetes/scheme"
|
|
"k8s.io/client-go/rest"
|
|
"k8s.io/client-go/tools/clientcmd"
|
|
"k8s.io/client-go/tools/remotecommand"
|
|
)
|
|
|
|
var configs []types.Coredump_config
|
|
|
|
func terminalSize() (width, height int, err error) {
|
|
file, err := os.OpenFile("/dev/tty", os.O_WRONLY, 0)
|
|
if err != nil {
|
|
return 0, 0, err
|
|
}
|
|
defer file.Close()
|
|
|
|
width, height, err = terminal.GetSize(int(file.Fd()))
|
|
if err != nil {
|
|
return 0, 0, err
|
|
}
|
|
return width, height, nil
|
|
}
|
|
func debug(config types.Coredump_config, command string) error {
|
|
// 使用kubectl命令执行debug操作
|
|
if config.Image_name != "" {
|
|
// // 定义要运行的 kubectl 命令
|
|
// command := "kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name} {end}'"
|
|
|
|
// // 在 shell 中运行 kubectl 命令并读取输出
|
|
// cmd := exec.Command("bash", "-c", command)
|
|
// output, err := cmd.Output()
|
|
// if err != nil {
|
|
// return err
|
|
// }
|
|
|
|
// // 将输出字符串按空格拆分为字符串切片
|
|
// nodes := strings.Split(strings.TrimSpace(string(output)), " ")
|
|
// if len(nodes) == 0 {
|
|
// return errors.New("the node list is empty")
|
|
// }
|
|
// if command == "gdb" {
|
|
// chmod := "cd" + filepath.Dir(config.Process_exe_path)
|
|
// cmd := exec.Command("kubectl", "debug", "node/"+nodes[0], "-it", "--image="+config.Image_id, "--image-pull-policy=Never", "--", "gdb", config.Process_exe_path, "/host"+config.Storage, "-ex", chmod)
|
|
// cmd.Stdin = os.Stdin
|
|
// cmd.Stdout = os.Stdout
|
|
// cmd.Stderr = os.Stderr
|
|
// fmt.Println(cmd.String())
|
|
// if err := cmd.Start(); err != nil {
|
|
// return err
|
|
// }
|
|
// err := cmd.Wait()
|
|
// if err != nil {
|
|
// return err
|
|
// }
|
|
// } else {
|
|
// cmd := exec.Command("kubectl", "debug", "node/"+config.Hostname, "-it", "--image="+config.Image_id, "--image-pull-policy=Never", "--", command)
|
|
// cmd.Stdin = os.Stdin
|
|
// cmd.Stdout = os.Stdout
|
|
// cmd.Stderr = os.Stderr
|
|
// fmt.Println(cmd.String())
|
|
// fmt.Println("exe=", config.Process_exe_path, "coredump=/host", config.Storage)
|
|
// if err := cmd.Start(); err != nil {
|
|
// return err
|
|
// }
|
|
// err := cmd.Wait()
|
|
// if err != nil {
|
|
// return err
|
|
// }
|
|
// }
|
|
kubeconfig := os.Getenv("KUBECONFIG")
|
|
if kubeconfig == "" {
|
|
kubeconfig = os.Getenv("HOME") + "/.kube/config"
|
|
}
|
|
|
|
// Creates the kubernetes client using the specified kubeconfig
|
|
conf, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
clientset, err := kubernetes.NewForConfig(conf)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
podName, err := debugInpod(conf, clientset, config, command)
|
|
if err != nil {
|
|
fmt.Println(err)
|
|
}
|
|
// Delete the Pod
|
|
fmt.Printf("Deleting Pod %q...\n", podName)
|
|
err = clientset.CoreV1().Pods("default").Delete(context.Background(), podName, metav1.DeleteOptions{})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("Deleted Pod %q.\n", podName)
|
|
return nil
|
|
} else { // 使用gdb命令进行命令行交互分析
|
|
cmd := exec.Command("gdb", config.Process_exe_path, config.Storage)
|
|
cmd.Stdin = os.Stdin
|
|
cmd.Stdout = os.Stdout
|
|
cmd.Stderr = os.Stderr
|
|
fmt.Println(cmd.String())
|
|
if err := cmd.Run(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
func generateRandomString(length int) (string, error) {
|
|
const charset = "abcdefghijklmnopqrstuvwxyz0123456789"
|
|
bytes := make([]byte, length)
|
|
max := big.NewInt(int64(len(charset)))
|
|
for i := 0; i < length; i++ {
|
|
n, err := rand.Int(rand.Reader, max)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
bytes[i] = charset[n.Int64()]
|
|
}
|
|
return string(bytes), nil
|
|
}
|
|
func debugInpod(conf *rest.Config, clientset *kubernetes.Clientset, config types.Coredump_config, command string) (string, error) {
|
|
// Define the Pod object
|
|
randword, err := generateRandomString(5)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
podName := fmt.Sprintf("node-debug-%s-%d", randword, time.Now().Unix())
|
|
containerName := "debug"
|
|
pod := &v1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: podName,
|
|
Namespace: "default",
|
|
},
|
|
Spec: v1.PodSpec{
|
|
Containers: []v1.Container{
|
|
{
|
|
Name: "debug",
|
|
Image: config.Image_name,
|
|
ImagePullPolicy: "IfNotPresent",
|
|
Command: []string{
|
|
"tail",
|
|
"-f",
|
|
},
|
|
VolumeMounts: []v1.VolumeMount{
|
|
{
|
|
Name: "host-dir",
|
|
MountPath: "/host",
|
|
},
|
|
},
|
|
SecurityContext: &v1.SecurityContext{
|
|
Privileged: &[]bool{true}[0],
|
|
},
|
|
},
|
|
},
|
|
Volumes: []v1.Volume{
|
|
{
|
|
Name: "host-dir",
|
|
VolumeSource: v1.VolumeSource{
|
|
HostPath: &v1.HostPathVolumeSource{
|
|
Path: "/",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
RestartPolicy: v1.RestartPolicyNever,
|
|
},
|
|
}
|
|
|
|
// Create the Pod
|
|
fmt.Println("Creating Pod...")
|
|
fmt.Printf("Creating Pod %q...\n", podName)
|
|
result, err := clientset.CoreV1().Pods("default").Create(context.Background(), pod, metav1.CreateOptions{})
|
|
if err != nil {
|
|
return podName, err
|
|
}
|
|
fmt.Printf("Created Pod %q.\n", result.GetObjectMeta().GetName())
|
|
|
|
// Wait for the Pod to be running and ready
|
|
fmt.Printf("Waiting for Pod %q to be ready...\n", podName)
|
|
ready := false
|
|
for i := 0; i < 10; i++ {
|
|
result, err := clientset.CoreV1().Pods("default").Get(context.Background(), podName, metav1.GetOptions{})
|
|
if err != nil {
|
|
return podName, err
|
|
}
|
|
status := result.Status
|
|
if status.Phase == v1.PodRunning && len(status.ContainerStatuses) > 0 && status.ContainerStatuses[0].Ready {
|
|
ready = true
|
|
break
|
|
}
|
|
time.Sleep(2 * time.Second)
|
|
}
|
|
if !ready {
|
|
return podName, errors.New("create pod timeout")
|
|
}
|
|
// Exec into the container
|
|
// Create exec request
|
|
var cmd []string
|
|
if command == "gdb" {
|
|
chmod := "cd" + filepath.Dir(config.Process_exe_path)
|
|
cmd = []string{"gdb", config.Process_exe_path, "/host" + config.Storage, "-ex", chmod}
|
|
} else {
|
|
cmd = []string{command}
|
|
}
|
|
req := clientset.CoreV1().RESTClient().Post().Resource("pods").
|
|
Name(podName).Namespace("default").
|
|
SubResource("exec").
|
|
VersionedParams(&v1.PodExecOptions{
|
|
Container: containerName,
|
|
Command: cmd,
|
|
Stdin: true,
|
|
Stdout: true,
|
|
Stderr: true,
|
|
TTY: true,
|
|
}, scheme.ParameterCodec)
|
|
|
|
// Create exec executor
|
|
executor, err := remotecommand.NewSPDYExecutor(conf, "POST", req.URL())
|
|
if err != nil {
|
|
return podName, err
|
|
}
|
|
|
|
// Start exec
|
|
err = executor.StreamWithContext(context.Background(), remotecommand.StreamOptions{
|
|
Stdin: os.Stdin,
|
|
Stdout: os.Stdout,
|
|
Stderr: os.Stderr,
|
|
Tty: true,
|
|
})
|
|
if err != nil {
|
|
return podName, err
|
|
}
|
|
return podName, nil
|
|
}
|
|
|
|
// WalkDirectory 遍历文件目录并处理后缀名为.config的文件
|
|
func WalkDirectory(dir string) {
|
|
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if !info.IsDir() {
|
|
if strings.HasSuffix(info.Name(), ".config") {
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
var config types.Coredump_config
|
|
err = json.Unmarshal(data, &config)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
configs = append(configs, config)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
})
|
|
|
|
if err != nil {
|
|
fmt.Printf("Error walking directory %s: %v\n", dir, err)
|
|
}
|
|
}
|
|
func list(pid string) {
|
|
defaultWidth := 80
|
|
if width, _, err := terminalSize(); err == nil {
|
|
defaultWidth = int(width)
|
|
}
|
|
// 输出表头信息
|
|
fmt.Printf("%-8s %-3s %-3s %-3s %-20s %-30s %-30s %-100s %-50s\n",
|
|
"PID", "UID", "GID", "SIG", "TIMESTAMP", "EXE", "HOSTNAME", "IMAGE", "STORGE")
|
|
fmt.Println(strings.Repeat("-", defaultWidth))
|
|
total := 0
|
|
// 输出配置信息
|
|
for _, c := range configs {
|
|
if pid != "" && strings.Compare(c.Initial_ns_pid, pid) != 0 {
|
|
continue
|
|
}
|
|
coreTime := time.Unix(c.Timestamp, 0).Format("2006-01-02 15:04:05")
|
|
fmt.Printf("%-8s %-3s %-3s %-6d %-20s %-30s %-30s %-100s %-50s\n",
|
|
c.Initial_ns_pid, c.UID, c.GID, c.Signal, coreTime, c.Process_exe_path, c.Hostname, c.Image_name, c.Storage)
|
|
total += 1
|
|
}
|
|
fmt.Println()
|
|
fmt.Println("Total", total, "coredumps")
|
|
}
|
|
|
|
// Help 打印使用帮助
|
|
func Help() {
|
|
fmt.Println("Usage: coredump-ctl [list | debug | help] [options] ")
|
|
fmt.Println("list options:")
|
|
fmt.Println(" -dir string")
|
|
fmt.Println(" Directory path")
|
|
fmt.Println(" -p string")
|
|
fmt.Println(" Pid to matching")
|
|
fmt.Println(" -command string")
|
|
fmt.Println(" exe command when attach to pod")
|
|
}
|
|
|
|
func main() {
|
|
listCmd := flag.NewFlagSet("list", flag.ExitOnError)
|
|
debugCmd := flag.NewFlagSet("debug", flag.ExitOnError)
|
|
helpCmd := flag.NewFlagSet("help", flag.ExitOnError)
|
|
var dirPath, pid, command string
|
|
listCmd.StringVar(&pid, "p", "", "Pid to matching")
|
|
debugCmd.StringVar(&pid, "p", "", "Pid to matching")
|
|
listCmd.StringVar(&dirPath, "dir", "", "Directory path")
|
|
debugCmd.StringVar(&dirPath, "dir", "", "Directory path")
|
|
debugCmd.StringVar(&command, "command", "", "")
|
|
if len(os.Args) == 1 {
|
|
helpCmd.Usage()
|
|
os.Exit(1)
|
|
}
|
|
switch os.Args[1] {
|
|
case "list":
|
|
listCmd.Parse(os.Args[2:])
|
|
case "debug":
|
|
debugCmd.Parse(os.Args[2:])
|
|
case "help":
|
|
helpCmd.Parse(os.Args[2:])
|
|
Help()
|
|
default:
|
|
fmt.Printf("%q is not a valid command.\n", os.Args[1])
|
|
os.Exit(1)
|
|
}
|
|
if listCmd.Parsed() {
|
|
if dirPath == "" {
|
|
dirPath = "/var/tmp"
|
|
}
|
|
WalkDirectory(dirPath)
|
|
list(pid)
|
|
return
|
|
}
|
|
if debugCmd.Parsed() {
|
|
if dirPath == "" {
|
|
dirPath = "/var/tmp"
|
|
}
|
|
if command == "" {
|
|
command = "gdb"
|
|
}
|
|
WalkDirectory(dirPath)
|
|
for _, config := range configs {
|
|
fmt.Println(config)
|
|
if strings.Compare(config.Initial_ns_pid, pid) == 0 || pid == "" {
|
|
err := debug(config, command)
|
|
if err != nil {
|
|
fmt.Println(err)
|
|
} else {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
return
|
|
}
|
|
}
|