| 475 | } |
| 476 | |
| 477 | func getWorkloadConfig(ctx context.Context) (string, error) { |
| 478 | // check if default workload is overridden by flag |
| 479 | if isValidWorkloadConfig(defaultGPUWorkloadConfigFlag) { |
| 480 | defaultGPUWorkloadConfig = defaultGPUWorkloadConfigFlag |
| 481 | } |
| 482 | |
| 483 | kubeConfig, err := rest.InClusterConfig() |
| 484 | if err != nil { |
| 485 | return "", fmt.Errorf("error getting cluster config - %s", err.Error()) |
| 486 | } |
| 487 | |
| 488 | kubeClient, err := kubernetes.NewForConfig(kubeConfig) |
| 489 | if err != nil { |
| 490 | return "", fmt.Errorf("error getting k8s client - %w", err) |
| 491 | } |
| 492 | |
| 493 | node, err := getNode(ctx, kubeClient) |
| 494 | if err != nil { |
| 495 | return "", fmt.Errorf("error getting node labels - %w", err) |
| 496 | } |
| 497 | |
| 498 | labels := node.GetLabels() |
| 499 | value, ok := labels[gpuWorkloadConfigLabelKey] |
| 500 | if !ok { |
| 501 | log.Infof("No %s label found; using default workload config: %s", gpuWorkloadConfigLabelKey, defaultGPUWorkloadConfig) |
| 502 | return defaultGPUWorkloadConfig, nil |
| 503 | } |
| 504 | if !isValidWorkloadConfig(value) { |
| 505 | log.Warnf("%s is an invalid workload config; using default workload config: %s", value, defaultGPUWorkloadConfig) |
| 506 | return defaultGPUWorkloadConfig, nil |
| 507 | } |
| 508 | return value, nil |
| 509 | } |
| 510 | |
| 511 | func start(ctx context.Context, cli *cli.Command) error { |
| 512 | // if cleanup is requested, delete all existing status files(default) |