k8s 上简单实现deviceplugin demo
This commit is contained in:
linxin
171
device.go
Normal file
171
device.go
Normal file
@@ -0,0 +1,171 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
|
"os"
|
||||||
|
"path"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// SysfsDevices = "/sys/bus/pci/devices"
|
||||||
|
SysfsDevices = "/root/demo"
|
||||||
|
MgmtPrefix = "/dev/xclmgmt"
|
||||||
|
UserPrefix = "/dev/dri"
|
||||||
|
QdmaPrefix = "/dev/xfpga"
|
||||||
|
QDMASTR = "dma.qdma.u"
|
||||||
|
UserPFKeyword = "drm"
|
||||||
|
DRMSTR = "renderD"
|
||||||
|
ROMSTR = "rom"
|
||||||
|
SNSTR = "xmc.u."
|
||||||
|
DSAverFile = "VBNV"
|
||||||
|
DSAtsFile = "timestamp"
|
||||||
|
InstanceFile = "instance"
|
||||||
|
MgmtFile = "mgmt_pf"
|
||||||
|
UserFile = "user_pf"
|
||||||
|
VendorFile = "vendor"
|
||||||
|
DeviceFile = "device"
|
||||||
|
SNFile = "serial_num"
|
||||||
|
VtShell = "xilinx_u30"
|
||||||
|
U30CommonShell = "ama_u30"
|
||||||
|
XilinxVendorID = "0x10ee"
|
||||||
|
ADVANTECH_ID = "0x13fe"
|
||||||
|
AWS_ID = "0x1d0f"
|
||||||
|
AristaVendorID = "0x3475"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Pairs struct {
|
||||||
|
Mgmt string
|
||||||
|
User string
|
||||||
|
Qdma string
|
||||||
|
}
|
||||||
|
|
||||||
|
type Device struct {
|
||||||
|
index string
|
||||||
|
deviceID string //devid of the user pf
|
||||||
|
Healthy string
|
||||||
|
SN string
|
||||||
|
}
|
||||||
|
|
||||||
|
func GetInstance(DBDF string) (string, error) {
|
||||||
|
strArray := strings.Split(DBDF, ":")
|
||||||
|
domain, err := strconv.ParseUint(strArray[0], 16, 16)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("strconv failed: %s\n", strArray[0])
|
||||||
|
}
|
||||||
|
bus, err := strconv.ParseUint(strArray[1], 16, 8)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("strconv failed: %s\n", strArray[1])
|
||||||
|
}
|
||||||
|
strArray = strings.Split(strArray[2], ".")
|
||||||
|
dev, err := strconv.ParseUint(strArray[0], 16, 8)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("strconv failed: %s\n", strArray[0])
|
||||||
|
}
|
||||||
|
fc, err := strconv.ParseUint(strArray[1], 16, 8)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("strconv failed: %s\n", strArray[1])
|
||||||
|
}
|
||||||
|
ret := domain*65536 + bus*256 + dev*8 + fc
|
||||||
|
return strconv.FormatUint(ret, 10), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func GetFileNameFromPrefix(dir string, prefix string) (string, error) {
|
||||||
|
userFiles, err := ioutil.ReadDir(dir)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("Can't read folder %s", dir)
|
||||||
|
}
|
||||||
|
for _, userFile := range userFiles {
|
||||||
|
fname := userFile.Name()
|
||||||
|
|
||||||
|
if !strings.HasPrefix(fname, prefix) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return fname, nil
|
||||||
|
}
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func GetFileContent(file string) (string, error) {
|
||||||
|
if buf, err := ioutil.ReadFile(file); err != nil {
|
||||||
|
return "", fmt.Errorf("Can't read file %s", file)
|
||||||
|
} else {
|
||||||
|
ret := strings.Trim(string(buf), "\n")
|
||||||
|
return ret, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func FileExist(fname string) bool {
|
||||||
|
if _, err := os.Stat(fname); err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func IsMgmtPf(pciID string) bool {
|
||||||
|
fname := path.Join(SysfsDevices, pciID, MgmtFile)
|
||||||
|
return FileExist(fname)
|
||||||
|
}
|
||||||
|
|
||||||
|
func IsUserPf(pciID string) bool {
|
||||||
|
fname := path.Join(SysfsDevices, pciID, UserFile)
|
||||||
|
return FileExist(fname)
|
||||||
|
}
|
||||||
|
|
||||||
|
func GetDevices() ([]Device, error) {
|
||||||
|
var devices []Device
|
||||||
|
pciFiles, err := ioutil.ReadDir(SysfsDevices)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("Can't read folder %s", SysfsDevices)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, pciFile := range pciFiles {
|
||||||
|
pciID := pciFile.Name()
|
||||||
|
// get device id
|
||||||
|
fname := path.Join(SysfsDevices, pciID, DeviceFile)
|
||||||
|
content, err := GetFileContent(fname)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
devid := content
|
||||||
|
fname = path.Join(SysfsDevices, pciID, "SN")
|
||||||
|
content, err = GetFileContent(fname)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
sn := content
|
||||||
|
healthy := pluginapi.Healthy
|
||||||
|
devices = append(devices, Device{
|
||||||
|
index: strconv.Itoa(len(devices) + 1),
|
||||||
|
deviceID: devid,
|
||||||
|
Healthy: healthy,
|
||||||
|
SN: sn,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return devices, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// func main() {
|
||||||
|
// devices, err := GetDevices()
|
||||||
|
// if err != nil {
|
||||||
|
// fmt.Printf("%s !!!\n", err)
|
||||||
|
// return
|
||||||
|
// }
|
||||||
|
// //SNFolder, err := GetFileNameFromPrefix(path.Join(SysfsDevices, "0000:e3:00.1"), SNSTR)
|
||||||
|
// //fname := path.Join(SysfsDevices, "0000:e3:00.1", SNFolder, SNFile)
|
||||||
|
// //content, err := GetFileContent(fname)
|
||||||
|
// //SN := content
|
||||||
|
// //fmt.Printf("SN: %v \n", SN)
|
||||||
|
// for _, device := range devices {
|
||||||
|
// fmt.Printf("Device: %v \n", device)
|
||||||
|
// fmt.Printf("ID: %s \n\n", device.deviceID)
|
||||||
|
// fmt.Printf("SN: %s \n\n", device.SN)
|
||||||
|
// fmt.Printf("Heathy: %s \n\n", device.Healthy)
|
||||||
|
// }
|
||||||
|
// }
|
||||||
20
go.mod
Normal file
20
go.mod
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
module deviceplugindemo
|
||||||
|
|
||||||
|
go 1.20
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/fsnotify/fsnotify v1.6.0
|
||||||
|
github.com/sirupsen/logrus v1.9.0
|
||||||
|
golang.org/x/net v0.8.0
|
||||||
|
google.golang.org/grpc v1.53.0
|
||||||
|
k8s.io/kubelet v0.26.2
|
||||||
|
)
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/gogo/protobuf v1.3.2 // indirect
|
||||||
|
github.com/golang/protobuf v1.5.2 // indirect
|
||||||
|
golang.org/x/sys v0.6.0 // indirect
|
||||||
|
golang.org/x/text v0.8.0 // indirect
|
||||||
|
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
|
||||||
|
google.golang.org/protobuf v1.28.1 // indirect
|
||||||
|
)
|
||||||
69
go.sum
Normal file
69
go.sum
Normal file
@@ -0,0 +1,69 @@
|
|||||||
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
|
||||||
|
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
|
||||||
|
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
||||||
|
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||||
|
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||||
|
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
|
||||||
|
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||||
|
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||||
|
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||||
|
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||||
|
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
|
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
|
||||||
|
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||||
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
|
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||||
|
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||||
|
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||||
|
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||||
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
|
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||||
|
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||||
|
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||||
|
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||||
|
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||||
|
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
|
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
|
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||||
|
golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
|
||||||
|
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
|
||||||
|
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
|
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
|
||||||
|
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
|
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||||
|
golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
|
||||||
|
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||||
|
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||||
|
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||||
|
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||||
|
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||||
|
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
|
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
|
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
|
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
|
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w=
|
||||||
|
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
|
||||||
|
google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc=
|
||||||
|
google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw=
|
||||||
|
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||||
|
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||||
|
google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
|
||||||
|
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
|
||||||
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
|
||||||
|
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
|
k8s.io/kubelet v0.26.2 h1:egg7YfhCpH9wvLwQdL2Mzuy4/kC6hO91azY0jgdYPWA=
|
||||||
|
k8s.io/kubelet v0.26.2/go.mod h1:IXthU5hcJQE6+K33LuaYYO0wUcYO8glhl/ip1Hzux44=
|
||||||
70
main.go
Normal file
70
main.go
Normal file
@@ -0,0 +1,70 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"flag"
|
||||||
|
"os"
|
||||||
|
"syscall"
|
||||||
|
|
||||||
|
"github.com/fsnotify/fsnotify"
|
||||||
|
log "github.com/sirupsen/logrus"
|
||||||
|
pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
// Parse command-line arguments
|
||||||
|
flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
|
||||||
|
flagLogLevel := flag.String("log-level", "info", "Define the logging level: error, info, debug.")
|
||||||
|
flag.Parse()
|
||||||
|
|
||||||
|
switch *flagLogLevel {
|
||||||
|
case "debug":
|
||||||
|
log.SetLevel(log.DebugLevel)
|
||||||
|
case "info":
|
||||||
|
log.SetLevel(log.InfoLevel)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Println("Starting FS watcher.")
|
||||||
|
watcher, err := newFSWatcher(pluginapi.DevicePluginPath)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("Failed to created FS watcher: %s.", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
defer watcher.Close()
|
||||||
|
|
||||||
|
log.Println("Starting OS watcher.")
|
||||||
|
sigs := newOSWatcher(syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
|
||||||
|
|
||||||
|
restart := true
|
||||||
|
var devicePlugin *FPGADevicePlugin
|
||||||
|
L:
|
||||||
|
for {
|
||||||
|
if restart {
|
||||||
|
devicePlugin = NewFPGADevicePlugin()
|
||||||
|
restart = false
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case update := <-devicePlugin.updateChan:
|
||||||
|
devicePlugin.checkDeviceUpdate(update)
|
||||||
|
|
||||||
|
case event := <-watcher.Events:
|
||||||
|
if event.Name == pluginapi.KubeletSocket && event.Op&fsnotify.Create == fsnotify.Create {
|
||||||
|
log.Printf("inotify: %s created, restarting.", pluginapi.KubeletSocket)
|
||||||
|
restart = true
|
||||||
|
}
|
||||||
|
|
||||||
|
case err := <-watcher.Errors:
|
||||||
|
log.Printf("inotify: %s", err)
|
||||||
|
|
||||||
|
case s := <-sigs:
|
||||||
|
switch s {
|
||||||
|
case syscall.SIGHUP:
|
||||||
|
log.Println("Received SIGHUP, restarting.")
|
||||||
|
restart = true
|
||||||
|
default:
|
||||||
|
log.Printf("Received signal \"%v\", shutting down.", s)
|
||||||
|
break L
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
385
server.go
Normal file
385
server.go
Normal file
@@ -0,0 +1,385 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"path"
|
||||||
|
"reflect"
|
||||||
|
_ "runtime/debug"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
log "github.com/sirupsen/logrus"
|
||||||
|
"golang.org/x/net/context"
|
||||||
|
"google.golang.org/grpc"
|
||||||
|
pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
resourceNamePrefix = "deviceplugindemo/"
|
||||||
|
serverSockPath = pluginapi.DevicePluginPath
|
||||||
|
// AWS_SN = "F1-Node"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FPGADevicePluginServer implements the Kubernetes device plugin API
|
||||||
|
type FPGADevicePluginServer struct {
|
||||||
|
devType string
|
||||||
|
devices map[string]Device
|
||||||
|
socket string
|
||||||
|
stop chan interface{}
|
||||||
|
update chan map[string]Device
|
||||||
|
|
||||||
|
server *grpc.Server
|
||||||
|
}
|
||||||
|
|
||||||
|
type FPGADevicePlugin struct {
|
||||||
|
devices map[string]map[string]Device
|
||||||
|
servers map[string]*FPGADevicePluginServer
|
||||||
|
updateChan chan map[string]map[string]Device
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewFPGADevicePlugin returns an initialized FPGADevicePlugin
|
||||||
|
func NewFPGADevicePlugin() *FPGADevicePlugin {
|
||||||
|
log.Debugf("create FPGA device plugin")
|
||||||
|
updateChan := make(chan map[string]map[string]Device)
|
||||||
|
plugin := FPGADevicePlugin{
|
||||||
|
devices: make(map[string]map[string]Device),
|
||||||
|
servers: make(map[string]*FPGADevicePluginServer),
|
||||||
|
updateChan: updateChan,
|
||||||
|
}
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
for {
|
||||||
|
devices, err := GetDevices()
|
||||||
|
if err != nil {
|
||||||
|
time.Sleep(75 * time.Second)
|
||||||
|
devices, err = GetDevices()
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("Error to get FPGA devices: %v", err)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
devMap := make(map[string]map[string]Device)
|
||||||
|
for _, device := range devices {
|
||||||
|
|
||||||
|
DSAtype := device.index
|
||||||
|
id := device.deviceID
|
||||||
|
if subMap, ok := devMap[DSAtype]; ok {
|
||||||
|
subMap = devMap[DSAtype]
|
||||||
|
subMap[id] = device
|
||||||
|
} else {
|
||||||
|
subMap = make(map[string]Device)
|
||||||
|
devMap[DSAtype] = subMap
|
||||||
|
subMap[id] = device
|
||||||
|
}
|
||||||
|
}
|
||||||
|
updateChan <- devMap
|
||||||
|
time.Sleep(5 * time.Second)
|
||||||
|
}
|
||||||
|
close(updateChan)
|
||||||
|
}()
|
||||||
|
|
||||||
|
return &plugin
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *FPGADevicePlugin) checkDeviceUpdate(n map[string]map[string]Device) {
|
||||||
|
added := make(map[string]map[string]Device)
|
||||||
|
updated := make(map[string]map[string]Device)
|
||||||
|
removed := make(map[string]map[string]Device)
|
||||||
|
|
||||||
|
for oDevType, oDevices := range m.devices {
|
||||||
|
if nDevices, ok := n[oDevType]; ok {
|
||||||
|
if !reflect.DeepEqual(oDevices, nDevices) {
|
||||||
|
updated[oDevType] = nDevices
|
||||||
|
}
|
||||||
|
delete(n, oDevType)
|
||||||
|
} else {
|
||||||
|
removed[oDevType] = oDevices
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for nDevType, nDevices := range n {
|
||||||
|
added[nDevType] = nDevices
|
||||||
|
}
|
||||||
|
|
||||||
|
//create new server for added devices
|
||||||
|
for aDevType, aDevices := range added {
|
||||||
|
devicePluginServer := m.NewFPGADevicePluginServer(aDevType, aDevices)
|
||||||
|
m.devices[aDevType] = aDevices
|
||||||
|
m.servers[aDevType] = devicePluginServer
|
||||||
|
go func(aDevType string, aDevices map[string]Device, name string) {
|
||||||
|
if err := m.servers[aDevType].Serve(name); err != nil {
|
||||||
|
log.Println("Could not contact Kubelet, Exit. Did you enable the device plugin feature gate?")
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
m.servers[aDevType].update <- aDevices
|
||||||
|
}(aDevType, aDevices, resourceNamePrefix+aDevType)
|
||||||
|
}
|
||||||
|
|
||||||
|
//stop server for removed devices
|
||||||
|
for rDevType, rDevices := range removed {
|
||||||
|
log.Debugf("Remove device %v", rDevices)
|
||||||
|
m.servers[rDevType].Stop()
|
||||||
|
delete(m.servers, rDevType)
|
||||||
|
delete(m.devices, rDevType)
|
||||||
|
}
|
||||||
|
|
||||||
|
//send update for updated devices
|
||||||
|
for uDevType, uDevices := range updated {
|
||||||
|
m.devices[uDevType] = uDevices
|
||||||
|
m.servers[uDevType].update <- uDevices
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewFPGADevicePluginServer returns an initialized FPGADevicePluginServer
|
||||||
|
func (m *FPGADevicePlugin) NewFPGADevicePluginServer(devType string, devices map[string]Device) *FPGADevicePluginServer {
|
||||||
|
return &FPGADevicePluginServer{
|
||||||
|
devType: devType,
|
||||||
|
devices: devices,
|
||||||
|
socket: path.Join(serverSockPath, devType+"-demodevice.sock"),
|
||||||
|
stop: make(chan interface{}),
|
||||||
|
update: make(chan map[string]Device, 1),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitForServer checks if grpc server is alive
|
||||||
|
// by making grpc blocking connection to the server socket
|
||||||
|
func waitForServer(socket string, timeout time.Duration) error {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||||
|
defer cancel()
|
||||||
|
conn, err := grpc.DialContext(ctx, socket, grpc.WithInsecure(), grpc.WithBlock(),
|
||||||
|
grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
|
||||||
|
return net.DialTimeout("unix", addr, timeout)
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
if conn != nil {
|
||||||
|
conn.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
fmt.Errorf("Failed dial context at %s", socket)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *FPGADevicePluginServer) deviceExists(id string) bool {
|
||||||
|
for k, _ := range m.devices {
|
||||||
|
if k == id {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *FPGADevicePluginServer) PreStartContainer(ctx context.Context, rqt *pluginapi.PreStartContainerRequest) (*pluginapi.PreStartContainerResponse, error) {
|
||||||
|
return nil, fmt.Errorf("PreStartContainer() should not be called")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *FPGADevicePluginServer) GetDevicePluginOptions(ctx context.Context, empty *pluginapi.Empty) (*pluginapi.DevicePluginOptions, error) {
|
||||||
|
fmt.Println("GetDevicePluginOptions: return empty options")
|
||||||
|
return new(pluginapi.DevicePluginOptions), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start starts the gRPC server of the device plugin
|
||||||
|
func (m *FPGADevicePluginServer) Start() error {
|
||||||
|
err := m.cleanup()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
sock, err := net.Listen("unix", m.socket)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
m.server = grpc.NewServer()
|
||||||
|
pluginapi.RegisterDevicePluginServer(m.server, m)
|
||||||
|
|
||||||
|
go m.server.Serve(sock)
|
||||||
|
|
||||||
|
// Wait for the server to start
|
||||||
|
if err = waitForServer(m.socket, 10*time.Second); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop stops the gRPC server
|
||||||
|
func (m *FPGADevicePluginServer) Stop() error {
|
||||||
|
if m.server == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
m.server.Stop()
|
||||||
|
m.server = nil
|
||||||
|
close(m.stop)
|
||||||
|
close(m.update)
|
||||||
|
|
||||||
|
return m.cleanup()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *FPGADevicePluginServer) cleanup() error {
|
||||||
|
if err := os.Remove(m.socket); err != nil && !os.IsNotExist(err) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Register registers the device plugin for the given resourceName with Kubelet.
|
||||||
|
func (m *FPGADevicePluginServer) Register(kubeletEndpoint, resourceName string) error {
|
||||||
|
conn, err := grpc.Dial(kubeletEndpoint, grpc.WithInsecure(),
|
||||||
|
grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
|
||||||
|
return net.DialTimeout("unix", addr, timeout)
|
||||||
|
}))
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
log.Debugf("Cann't connect to kubelet service")
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
|
||||||
|
client := pluginapi.NewRegistrationClient(conn)
|
||||||
|
reqt := &pluginapi.RegisterRequest{
|
||||||
|
Version: pluginapi.Version,
|
||||||
|
Endpoint: path.Base(m.socket),
|
||||||
|
ResourceName: resourceName,
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = client.Register(context.Background(), reqt)
|
||||||
|
if err != nil {
|
||||||
|
log.Debugf("Cann't register to kubelet service")
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// func IsContain(items []string, item string) bool {
|
||||||
|
// AWS_SN := "F1-Node"
|
||||||
|
// for _, eachItem := range items {
|
||||||
|
// if eachItem == item && strings.EqualFold(item, AWS_SN) != true {
|
||||||
|
// return true
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// return false
|
||||||
|
// }
|
||||||
|
func (m *FPGADevicePluginServer) sendDevices(s pluginapi.DevicePlugin_ListAndWatchServer) error {
|
||||||
|
resp := new(pluginapi.ListAndWatchResponse)
|
||||||
|
|
||||||
|
check_range := m.devices
|
||||||
|
SerialNums := []string{}
|
||||||
|
for _, device := range check_range {
|
||||||
|
if device.SN == "" {
|
||||||
|
log.Printf("Error, Device %v has empty Serial number", device.deviceID)
|
||||||
|
} else {
|
||||||
|
SerialNums = append(SerialNums, device.SN)
|
||||||
|
tem := &pluginapi.Device{
|
||||||
|
ID: device.deviceID,
|
||||||
|
Health: device.Healthy,
|
||||||
|
}
|
||||||
|
resp.Devices = append(resp.Devices, tem)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
log.Printf("Check SeialNums arry: %v", SerialNums)
|
||||||
|
log.Printf("Sending %d device(s) %v to kubelet", len(resp.Devices), resp.Devices)
|
||||||
|
if err := s.Send(resp); err != nil {
|
||||||
|
m.Stop()
|
||||||
|
log.Debugf("Cannot update device list")
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListAndWatch lists devices and update that list according to the health status
|
||||||
|
func (m *FPGADevicePluginServer) ListAndWatch(e *pluginapi.Empty, s pluginapi.DevicePlugin_ListAndWatchServer) error {
|
||||||
|
log.Debugf("In ListAndWatch(%s): stream: %v", m.devType, s)
|
||||||
|
//debug.PrintStack()
|
||||||
|
for m.devices = range m.update {
|
||||||
|
if err := m.sendDevices(s); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Allocate which return list of devices.
|
||||||
|
func (m *FPGADevicePluginServer) Allocate(ctx context.Context, req *pluginapi.AllocateRequest) (*pluginapi.AllocateResponse, error) {
|
||||||
|
log.Debugf("In Allocate()")
|
||||||
|
response := new(pluginapi.AllocateResponse)
|
||||||
|
for _, creq := range req.ContainerRequests {
|
||||||
|
log.Debugf("Request IDs: %v", creq.DevicesIDs)
|
||||||
|
|
||||||
|
cres := new(pluginapi.ContainerAllocateResponse)
|
||||||
|
|
||||||
|
// Check same serial number devices, devices with same serail number "F1-node" will be marked as independent devices
|
||||||
|
deviceIDs_arry := creq.DevicesIDs
|
||||||
|
|
||||||
|
for _, id := range deviceIDs_arry {
|
||||||
|
log.Printf("Receiving request %s", id)
|
||||||
|
dev, ok := m.devices[id]
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("Invalid allocation request with non-existing device %s", id)
|
||||||
|
}
|
||||||
|
if !m.deviceExists(id) {
|
||||||
|
return nil, fmt.Errorf("invalid allocation request: unknown device: %s", id)
|
||||||
|
}
|
||||||
|
fname := path.Join(SysfsDevices, dev.deviceID, DeviceFile)
|
||||||
|
cres.Mounts = append(cres.Mounts, &pluginapi.Mount{
|
||||||
|
HostPath: fname,
|
||||||
|
ContainerPath: fname,
|
||||||
|
ReadOnly: false,
|
||||||
|
})
|
||||||
|
response.ContainerResponses = append(response.ContainerResponses, cres)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return response, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Serve starts the gRPC server and register the device plugin to Kubelet
|
||||||
|
func (m *FPGADevicePluginServer) Serve(resourceName string) error {
|
||||||
|
log.Debugf("In Serve(%s)", m.socket)
|
||||||
|
err := m.Start()
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("Could not start device plugin: %v", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
log.Infof("Starting to serve on %s", m.socket)
|
||||||
|
|
||||||
|
err = m.Register(pluginapi.KubeletSocket, resourceName)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("Could not register device plugin: %v", err)
|
||||||
|
m.Stop()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
log.Infof("Registered device plugin with Kubelet %s", resourceName)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *FPGADevicePluginServer) GetPreferredAllocation(ctx context.Context, req *pluginapi.PreferredAllocationRequest) (*pluginapi.PreferredAllocationResponse, error) {
|
||||||
|
response := new(pluginapi.PreferredAllocationResponse)
|
||||||
|
for _, creq := range req.ContainerRequests {
|
||||||
|
log.Debugf("Request IDs: %v", creq.AvailableDeviceIDs)
|
||||||
|
|
||||||
|
cres := new(pluginapi.ContainerPreferredAllocationResponse)
|
||||||
|
|
||||||
|
// Check same serial number devices, devices with same serail number "F1-node" will be marked as independent devices
|
||||||
|
deviceIDs_arry := creq.AvailableDeviceIDs
|
||||||
|
|
||||||
|
for _, id := range deviceIDs_arry {
|
||||||
|
log.Printf("Receiving request %s", id)
|
||||||
|
dev, ok := m.devices[id]
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("Invalid allocation request with non-existing device %s", id)
|
||||||
|
}
|
||||||
|
if !m.deviceExists(id) {
|
||||||
|
return nil, fmt.Errorf("invalid allocation request: unknown device: %s", id)
|
||||||
|
}
|
||||||
|
fname := path.Join(SysfsDevices, dev.deviceID, DeviceFile)
|
||||||
|
cres.DeviceIDs = append(cres.DeviceIDs, fname)
|
||||||
|
response.ContainerResponses = append(response.ContainerResponses, cres)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return response, nil
|
||||||
|
}
|
||||||
32
watch.go
Normal file
32
watch.go
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
|
||||||
|
"github.com/fsnotify/fsnotify"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newFSWatcher(files ...string) (*fsnotify.Watcher, error) {
|
||||||
|
watcher, err := fsnotify.NewWatcher()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, f := range files {
|
||||||
|
err = watcher.Add(f)
|
||||||
|
if err != nil {
|
||||||
|
watcher.Close()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return watcher, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func newOSWatcher(sigs ...os.Signal) chan os.Signal {
|
||||||
|
sigChan := make(chan os.Signal, 1)
|
||||||
|
signal.Notify(sigChan, sigs...)
|
||||||
|
|
||||||
|
return sigChan
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user