datasources

package
v0.1.6 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 10, 2025 License: Apache-2.0 Imports: 25 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var (
	SupportedTypesString = []string{string(TypeS3), string(TypeGit), string(TypeHTTP), string(TypeConda), string(TypeHuggingFace), string(TypeModelScope)}
	SupportedTypes       = []Type{TypeS3, TypeGit, TypeHTTP, TypeConda, TypeHuggingFace, TypeModelScope}
)

Functions

This section is empty.

Types

type CondaLoader 

type CondaLoader struct {
	Options Options
	// contains filtered or unexported fields
}

func NewCondaLoader 

func NewCondaLoader(datasourceOption map[string]string, options Options, secrets Secrets) (*CondaLoader, error)

func (*CondaLoader) Sync 

func (l *CondaLoader) Sync(_ string, _ string) error

Workflow overview:

  • conda --version
  • conda info
  • conda env list
  • conda config --show-sources
  • conda config --set show_channel_urls yes
  • conda config --prepend pkgs_dirs /opt/baize-runtime-env/conda/pkgs
  • conda config --prepend envs_dirs /opt/baize-runtime-env/conda/envs

if environment.yml exists:

  • read environment.yml
  • normalize environment.yml
  • conda env create -f environment.yml
  • conda clean --all

else:

  • create a normalized environment.yml
  • conda env create -f environment.yml
  • conda clean --all

if requirements.txt exists:

  • pip install -r requirements.txt

finalize the conda environment:

  • mv /opt/baize-runtime-env/conda/pkgs ${mount-root}/conda/pkgs
  • mv /opt/baize-runtime-env/conda/envs ${mount-root}/conda/envs

type CondaLoaderOptions 

type CondaLoaderOptions struct {
	Name                    string `json:"name"`
	PythonVersion           string `json:"pythonVersion"`
	PipIndexURL             string `json:"pipIndexUrl"`
	PipExtraIndexURL        string `json:"pipExtraIndexUrl"`
	CondaEnvironmentYmlPath string `json:"condaEnvironmentYmlPath"`
	PipRequirementsTxtPath  string `json:"pipRequirementsTxtPath"`
	CondaPrefixDir          string `json:"condaPrefixDir"`
	// contains filtered or unexported fields
}

type GitLoader 

type GitLoader struct {
	Options Options
	// contains filtered or unexported fields
}

func NewGitLoader 

func NewGitLoader(datasourceOption map[string]string, options Options, secrets Secrets) (*GitLoader, error)

func (*GitLoader) Sync 

func (d *GitLoader) Sync(fromURI string, toPath string) error

type GitLoaderOptions 

type GitLoaderOptions struct {
	Branch     string `json:"branch"`
	Commit     string `json:"commit"`
	Depth      string `json:"depth"`
	Submodules string `json:"submodules"`
	// contains filtered or unexported fields
}

type HTTPLoader 

type HTTPLoader struct {
	Options Options
	// contains filtered or unexported fields
}

func NewHTTPLoader 

func NewHTTPLoader(datasourceOptions map[string]string, options Options, secrets Secrets) (*HTTPLoader, error)

func (*HTTPLoader) Sync 

func (d *HTTPLoader) Sync(fromURI string, toPath string) error

type HTTPLoaderOptions 

type HTTPLoaderOptions struct {
	SyncMode string `json:"syncMode"`
	// contains filtered or unexported fields
}

type HuggingFaceLoader 

type HuggingFaceLoader struct {
	Options Options
	// contains filtered or unexported fields
}

func NewHuggingFaceLoader 

func NewHuggingFaceLoader(datasourceOptions map[string]string, options Options, secrets Secrets) (*HuggingFaceLoader, error)

func (*HuggingFaceLoader) Sync 

func (d *HuggingFaceLoader) Sync(fromURI string, toPath string) error

type HuggingFaceLoaderOptions 

type HuggingFaceLoaderOptions struct {
	Revision string `json:"revision"`
	RepoType string `json:"repoType"`
	Endpoint string `json:"endpoint"`
	Offline  bool   `json:"offline"`
	Include  string `json:"include"`
	Exclude  string `json:"exclude"`
	// contains filtered or unexported fields
}

type Loader 

type Loader interface {
	Sync(fromURI string, toPath string) error
}

type ModelScopeLoader 

type ModelScopeLoader struct {
	Options Options
	// contains filtered or unexported fields
}

func NewModelScopeLoader 

func NewModelScopeLoader(datasourceOptions map[string]string, options Options, secrets Secrets) (*ModelScopeLoader, error)

func (*ModelScopeLoader) Sync 

func (d *ModelScopeLoader) Sync(fromURI string, toPath string) error

type ModelScopeLoaderOptions 

type ModelScopeLoaderOptions struct {
	Revision string `json:"revision"`
	RepoType string `json:"repoType"`
	Include  string `json:"include"`
	Exclude  string `json:"exclude"`
	// contains filtered or unexported fields
}

type Options 

type Options struct {
	// Primary arguments
	Type Type
	URI  string

	// --options flags
	Path string
	Mode os.FileMode
	UID  int
	GID  int
	Root string
}

type PixiLoader added in v0.1.3 

type PixiLoader struct {
	Options Options
}

func NewPixiLoader added in v0.1.3 

func NewPixiLoader(datasourceOption map[string]string, options Options, secrets Secrets) (*PixiLoader, error)

func (*PixiLoader) Sync added in v0.1.3 

func (l *PixiLoader) Sync(fromURI string, toPath string) error

type S3Loader 

type S3Loader struct {
	Options Options
	// contains filtered or unexported fields
}

func NewS3Loader 

func NewS3Loader(datasourceOptions map[string]string, options Options, secrets Secrets) (*S3Loader, error)

func (*S3Loader) Sync 

func (d *S3Loader) Sync(fromURI string, toPath string) error

type S3LoaderOptions 

type S3LoaderOptions struct {
	Provider string `json:"provider"`
	Region   string `json:"region"`
	Endpoint string `json:"endpoint"`
	SyncMode string `json:"syncMode"`
	// contains filtered or unexported fields
}

type SecretKey 

type SecretKey string
const (
	SecretKeyUsername             SecretKey = "username"
	SecretKeyPassword             SecretKey = "password"
	SecretKeyPrivateKey           SecretKey = "ssh-privatekey"
	SecretKeyPrivateKeyPassphrase SecretKey = "ssh-privatekey-passphrase" // #nosec G101
	SecretKeyToken                SecretKey = "token"
	SecretKeyAccessKey            SecretKey = "access-key"
	SecretKeySecretKey            SecretKey = "secret-key"
)

type Secrets 

type Secrets struct {
	Username string `json:"-"`
	Password string `json:"-"`

	SSHPrivateKey           string `json:"-"`
	SSHPrivateKeyPassphrase string `json:"-"`

	Token string `json:"-"`

	AKSKAccessKeyID     string `json:"-"`
	AKSKSecretAccessKey string `json:"-"`
}

func ReadAndParseSecrets 

func ReadAndParseSecrets(name string) (Secrets, error)

type Type 

type Type string
const (
	TypeS3          Type = "S3"
	TypeGit         Type = "GIT"
	TypeHTTP        Type = "HTTP"
	TypeConda       Type = "CONDA"
	TypeHuggingFace Type = "HUGGING_FACE"
	TypeModelScope  Type = "MODEL_SCOPE"
)

Source Files

View all Source files

Directories

Path Synopsis
fake
Code generated by counterfeiter.
 Code generated by counterfeiter.
fake
Code generated by counterfeiter.
 Code generated by counterfeiter.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.