Method InitializeExecutor

provider/spark.go:944–967

(store SparkFileStore)

}

func (s *SparkGenericExecutor) InitializeExecutor(store SparkFileStore) error {
	s.logger.Info("Uploading PySpark script to filestore")
	// We can't use CreateFilePath here because it calls Validate under the hood,
	// which will always fail given it's a local file without a valid scheme or bucket, for example.
	sparkLocalScriptPath := &filestore.LocalFilepath{}
	if err := sparkLocalScriptPath.SetKey(config.GetSparkLocalScriptPath()); err != nil {
		return fmt.Errorf("could not create local script path: %v", err)
	}

	sparkRemoteScriptPath, err := store.CreateFilePath(config.GetSparkRemoteScriptPath())
	if err != nil {
		return fmt.Errorf("could not create file path: %v", err)
	}

	err = readAndUploadFile(sparkLocalScriptPath, sparkRemoteScriptPath, store)
	if err != nil {
		return fmt.Errorf("could not upload '%s' to '%s': %v", sparkLocalScriptPath.Key(), sparkRemoteScriptPath.ToURI(), err)
	}
	scriptExists, err := store.Exists(sparkRemoteScriptPath)
	if err != nil || !scriptExists {
		return fmt.Errorf("could not upload spark script: Path: %s, Error: %v", sparkRemoteScriptPath.ToURI(), err)
	}
	return nil
}

func (s *SparkGenericExecutor) getYarnCommand(args string) (string, error) {
	configDir, err := os.MkdirTemp("", "hadoop-conf")