filia

type Crawler struct {
	Settings
	Queue CrawlerQueue
	Set   set.Set

	Output chan Document
	ErrC   chan error
	CloseC chan struct{}
}

type CrawlerQueue interface {
	// Send sends the list of urls in given order to the queue
	Send(urls ...string)
	// Recv receives one url from the queue and returns it. It may block.
	Recv() (url string)
}

type Decoder interface {
	Decode(doc *Document, rc io.ReadCloser) error
}

type Document struct {
	URL         *url.URL
	Type        DocumentType
	ContentType string
	Time        time.Time
	Size        int64

	Links []string

	Title        string
	Version      string
	Album        string
	Artist       string
	Performer    string
	Copyright    string
	License      string
	Organisation string
	Genre        string
	Date         string
	ISRC         string
	Author       string
	Description  string

	Content string

	NoIndex  bool
	NoFollow bool
}

type DocumentType int

type FTPProto struct {
	// contains filtered or unexported fields
}

type FileProto struct {
}

type GzipDecoder struct {
	Tar TarDecoder
}

type HTMLDecoder struct{}

type HTTPProto struct {
	Client http.Client
}

type ImageDecoder struct{}

type MediaDecoder struct{}

type PDFDecoder struct{}

type Proto interface {
	Get(url *url.URL) (doc Document, body io.ReadCloser, err error)
}

type SFTPProto struct {
	// contains filtered or unexported fields
}

type Settings struct {
	Protos   map[string]Proto
	Decoders map[string]Decoder
}

type StdCrawlerQueue chan string

type TarDecoder struct{}

type ZIPDecoder struct{}