2026-08-20 15:01:15 +08:00
|
|
|
|
package gpu
|
|
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
|
"context"
|
|
|
|
|
|
"errors"
|
|
|
|
|
|
"fmt"
|
|
|
|
|
|
"strings"
|
|
|
|
|
|
"sync"
|
|
|
|
|
|
|
|
|
|
|
|
"github.com/1Panel-dev/1Panel/agent/utils/cmd"
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
type provider interface {
|
|
|
|
|
|
LoadInfo(context.Context) (*Info, error)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
type Client struct {
|
|
|
|
|
|
providers []provider
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
type providerResult struct {
|
2026-11-12 23:27:43 +08:00
|
|
|
|
vendor string
|
|
|
|
|
|
info *Info
|
|
|
|
|
|
err error
|
2026-08-20 15:01:15 +08:00
|
|
|
|
}
|
|
|
|
|
|
|
2026-11-06 23:16:10 +08:00
|
|
|
|
func ManagementCommands() map[string]string {
|
|
|
|
|
|
commands := make(map[string]string)
|
2026-08-20 15:01:15 +08:00
|
|
|
|
if cmd.Which(nvidiaSMICommand) {
|
2026-11-06 23:16:10 +08:00
|
|
|
|
commands["nvidia"] = nvidiaSMICommand
|
2026-08-20 15:01:15 +08:00
|
|
|
|
}
|
|
|
|
|
|
if command, ok := findAMDSMI(); ok {
|
2026-11-06 23:16:10 +08:00
|
|
|
|
commands["amd"] = command
|
|
|
|
|
|
}
|
|
|
|
|
|
return commands
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func New() (bool, Client) {
|
|
|
|
|
|
client := Client{}
|
|
|
|
|
|
commands := ManagementCommands()
|
|
|
|
|
|
if commands["nvidia"] != "" {
|
|
|
|
|
|
client.providers = append(client.providers, nvidiaSMI{})
|
|
|
|
|
|
}
|
|
|
|
|
|
if command := commands["amd"]; command != "" {
|
2026-08-20 15:01:15 +08:00
|
|
|
|
client.providers = append(client.providers, amdSMI{command: command})
|
|
|
|
|
|
}
|
|
|
|
|
|
return len(client.providers) > 0, client
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func (c Client) LoadInfo() (*Info, error) {
|
|
|
|
|
|
return c.LoadInfoContext(context.Background())
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func (c Client) LoadInfoContext(ctx context.Context) (*Info, error) {
|
|
|
|
|
|
results := make([]providerResult, len(c.providers))
|
|
|
|
|
|
var wg sync.WaitGroup
|
|
|
|
|
|
for index, item := range c.providers {
|
|
|
|
|
|
wg.Add(1)
|
|
|
|
|
|
go func() {
|
|
|
|
|
|
defer wg.Done()
|
2026-11-12 23:27:43 +08:00
|
|
|
|
switch item.(type) {
|
|
|
|
|
|
case nvidiaSMI:
|
|
|
|
|
|
results[index].vendor = "nvidia"
|
|
|
|
|
|
case amdSMI:
|
|
|
|
|
|
results[index].vendor = "amd"
|
|
|
|
|
|
}
|
2026-08-20 15:01:15 +08:00
|
|
|
|
results[index].info, results[index].err = item.LoadInfo(ctx)
|
|
|
|
|
|
}()
|
|
|
|
|
|
}
|
|
|
|
|
|
wg.Wait()
|
|
|
|
|
|
|
2026-11-12 23:27:43 +08:00
|
|
|
|
merged := &Info{VendorWarnings: map[string][]string{}}
|
2026-08-20 15:01:15 +08:00
|
|
|
|
var (
|
|
|
|
|
|
errs []error
|
|
|
|
|
|
types []string
|
|
|
|
|
|
driverVersions []string
|
|
|
|
|
|
)
|
|
|
|
|
|
for _, result := range results {
|
|
|
|
|
|
if result.err != nil {
|
|
|
|
|
|
errs = append(errs, result.err)
|
2026-11-12 23:27:43 +08:00
|
|
|
|
merged.VendorWarnings[result.vendor] = append(merged.VendorWarnings[result.vendor], result.err.Error())
|
2026-08-20 15:01:15 +08:00
|
|
|
|
continue
|
|
|
|
|
|
}
|
|
|
|
|
|
if result.info == nil {
|
|
|
|
|
|
continue
|
|
|
|
|
|
}
|
2026-11-12 23:27:43 +08:00
|
|
|
|
if len(result.info.Warnings) > 0 {
|
|
|
|
|
|
merged.VendorWarnings[result.vendor] = append(merged.VendorWarnings[result.vendor], result.info.Warnings...)
|
|
|
|
|
|
}
|
2026-08-20 15:01:15 +08:00
|
|
|
|
if result.info.Type != "" {
|
|
|
|
|
|
types = append(types, result.info.Type)
|
|
|
|
|
|
}
|
|
|
|
|
|
if result.info.DriverVersion != "" {
|
|
|
|
|
|
driverVersions = append(driverVersions, formatDriverVersion(result.info.Type, result.info.DriverVersion))
|
|
|
|
|
|
}
|
|
|
|
|
|
if result.info.CudaVersion != "" {
|
|
|
|
|
|
merged.CudaVersion = result.info.CudaVersion
|
|
|
|
|
|
}
|
|
|
|
|
|
for _, device := range result.info.Devices {
|
|
|
|
|
|
if device.Type == "" {
|
|
|
|
|
|
device.Type = result.info.Type
|
|
|
|
|
|
}
|
|
|
|
|
|
merged.Devices = append(merged.Devices, device)
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
merged.Type = mergeTypes(types)
|
|
|
|
|
|
merged.DriverVersion = mergeDriverVersions(driverVersions)
|
|
|
|
|
|
if len(merged.Devices) == 0 && len(errs) > 0 {
|
2026-11-12 23:27:43 +08:00
|
|
|
|
return merged, fmt.Errorf("calling GPU monitoring tools failed: %w", errors.Join(errs...))
|
2026-09-28 18:32:12 +08:00
|
|
|
|
}
|
2026-08-20 15:01:15 +08:00
|
|
|
|
return merged, nil
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func formatDriverVersion(deviceType, version string) string {
|
|
|
|
|
|
if deviceType == "" {
|
|
|
|
|
|
return version
|
|
|
|
|
|
}
|
|
|
|
|
|
return fmt.Sprintf("%s: %s", strings.ToUpper(deviceType), version)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func mergeTypes(types []string) string {
|
|
|
|
|
|
if len(types) == 1 {
|
|
|
|
|
|
return types[0]
|
|
|
|
|
|
}
|
|
|
|
|
|
if len(types) > 1 {
|
|
|
|
|
|
return "mixed"
|
|
|
|
|
|
}
|
|
|
|
|
|
return ""
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func mergeDriverVersions(versions []string) string {
|
|
|
|
|
|
if len(versions) == 1 {
|
|
|
|
|
|
parts := strings.SplitN(versions[0], ": ", 2)
|
|
|
|
|
|
return parts[len(parts)-1]
|
|
|
|
|
|
}
|
|
|
|
|
|
return strings.Join(versions, ";")
|
|
|
|
|
|
}
|