youcrawl

type ChannelPipeline struct {
	ChannelMapping sync.Map
}

type ChannelPipelineToken interface {
	GetToken() string
}

type Context struct {
	Request     *http.Request
	Response    *http.Response
	Item        interface{}
	GlobalStore GlobalStore
	Pool        TaskPool
	Cookie      *cookiejar.Jar
	Doc         *goquery.Document
}

type CookieMiddleware struct {
	Store  CookieStore
	GetKey func(c *http.Client, r *http.Request, ctx *Context) string
}

type CookieMiddlewareOption struct {
	GetKey func(c *http.Client, r *http.Request, ctx *Context) string
}

type CookieStore interface {
	GetCookie(key string) *cookiejar.Jar
	SetCookie(key string, jar *cookiejar.Jar)
	GetOrCreate(key string) *cookiejar.Jar
}

type DefaultCookieStore struct {
	sync.Map
}

type DefaultItem struct {
	Store map[string]interface{}
}

type DelayMiddleware struct {
	Min   int
	Max   int
	Fixed int
}

type Engine struct {
	sync.Mutex
	*EngineOption
	// dispatch task
	Pool        TaskPool
	Parsers     []HTMLParser
	Middlewares []Middleware
	Pipelines   []Pipeline
	GlobalStore GlobalStore
	PostProcess []PostProcess
	Plugins     []Plugin
	// receive signal: force stop pool
	InterruptChan chan struct{}
	// receive signal: stop pool when all task has done
	StopPoolChan chan struct{}
}

type EngineOption struct {
	// max running in same time
	MaxRequest int

	// true for:
	// keep running until manually stopped
	Daemon bool
}

type GlobalStore interface {
	Init() error
	SetValue(key string, value interface{})
	GetValue(key string) interface{}
	GetOrCreate(key string, value interface{}) interface{}
}

type GlobalStorePipeline struct {
}

type HTMLParser func(ctx *Context) error

type ImageDownloadItem struct {
	Urls []string
}

type ImageDownloadPipeline struct {
	// get store folder
	//
	//./download/image by default
	GetStoreFileFolder func(item interface{}, store GlobalStore) string
	// get save filename
	//
	// same name with image,by default
	GetSaveFileName func(item interface{}, store GlobalStore, rawURL string) string
	// get urls
	//
	//if the type of Item is ImageDownloadItem, no need to specify
	GetUrls func(item interface{}, store GlobalStore) []string
	// maximum number of concurrent downloads
	MaxDownload int
	// request middlewares to use
	Middlewares []Middleware
	// call on each image downloaded complete
	OnImageDownloadComplete func(item interface{}, store GlobalStore, url string, downloadFilePath string)
	// call on all image download, regardless of whether all image download is successful
	OnDone func(item interface{}, store GlobalStore)
}

type MemoryGlobalStore struct {
	sync.Map
	Content map[string]interface{}
}

type Middleware interface {
	// before request call
	Process(c *http.Client, r *http.Request, ctx *Context)
	// after request call
	RequestCallback(c *http.Client, r *http.Request, ctx *Context)
}

type OutputCSVPostProcess struct {
	// contains filtered or unexported fields
}

type OutputCSVPostProcessOption struct {
	// output path.
	// if not provided,use `./output.csv` as default value
	OutputPath string
	// with header.
	// default : false
	WithHeader bool
	// key to write
	// if not provided,will write all key
	Keys []string
	// key to csv column name.
	// if not provide,use key name as csv column name
	KeysMapping map[string]string
	// if value not exist in item.
	// by default,use empty string
	NotExistValue string
}

type OutputJsonPostProcess struct {
	StorePath string
	GetData   func(store GlobalStore) interface{}
}

type Pipeline interface {
	Process(item interface{}, store GlobalStore) error
}

type Plugin interface {
	Run(e *Engine)
}

type PostProcess interface {
	Process(store GlobalStore) error
}

type ProxyMiddleware struct {
	List []string
}

type ProxyMiddlewareOption struct {
	// set proxy list,
	// if both ProxyList and ProxyFilePath are provided,combine tow list
	ProxyList []string
	// read proxy from file,use `./proxy.txt` by default,
	// if both ProxyList and ProxyFilePath are provided,combine tow list
	ProxyFilePath string
}

type RequestPool struct {
	Tasks         []Task
	Total         int
	CompleteCount int
	NextTask      *Task
	GetTaskChan   chan *Task
	DoneChan      chan struct{}
	CompleteChan  chan *Task
	PreventStop   bool
	Store         GlobalStore
	sync.RWMutex
}

type RequestPoolOption struct {
	UseCookie   bool
	PreventStop bool
}

type StatusOutputPlugin struct {
	// disable log output
	LogOutput bool
}

type Task struct {
	ID        string
	Url       string
	Context   Context
	Requested bool
	Completed bool
}

type TaskPool interface {
	AddURLs(urls ...string)
	AddTasks(task ...*Task)
	GetOneTask(e *Engine) <-chan *Task
	GetUnRequestedTask() (target *Task)
	OnTaskDone(task *Task)
	GetDoneChan() chan struct{}
	Close()
	SetPrevent(isPrevent bool)
	GetTotal() (int, error)
	GetUnRequestCount() (int, error)
	GetCompleteCount() (int, error)
}

type UserAgentMiddleware struct {
	List []string
}

type UserAgentMiddlewareOption struct {
	// set user agent list,
	// if both UserAgentList and UserAgentFilePath are provided,combine tow list
	UserAgentList []string
	// read useragent from file,use `./ua.txt` by default,
	// if both UserAgentList and UserAgentFilePath are provided,combine tow list
	UserAgentFilePath string
}