crawler

type Channels map[int]chan Page

type Config struct {
	StartURL               string
	AllowedDomains         []string // Domains to stay within
	UserAgents             []string
	CrawlDelay             time.Duration // Delay between requests to the same domain
	MaxDepth               int           // Maximum crawl depth
	MaxRetries             int           // Max retries for a failed request
	RequestTimeout         time.Duration
	QueueIdleTimeout       time.Duration
	ProxyURL               string // e.g., "http://user:pass@host:port"
	RobotsUserAgent        string // User agent to use for robots.txt checks
	ConcurrentRequests     int    // Number of concurrent fetch workers
	Channels               Channels
	Headers                map[string]string
	LanguageCode           string
	Filters                []func(*Page, *Config) bool
	MaxIdleConnsPerHost    int
	MaxIdleConns           int
	Proxies                []string
	RequireHeadless        bool
	ProcessSitemaps        bool
	DisableUserAgentHeader bool
}

type Crawler struct {
	// contains filtered or unexported fields
}

type Headless struct {
}

type Page struct {
	URL  *url.URL      // Page url
	Resp *PageResponse // Page response as returned from the GET request
	Body string        // Response body string
}

type PageResponse struct {
	StatusCode int
	Headers    map[string]any
	Body       io.Reader
}