forked from rannes.dev/sw-jobs-go
set job scraping limit to 20 jobs
This commit is contained in:
parent
20511db91f
commit
5fbdd9706e
13
scraper.go
13
scraper.go
@ -34,6 +34,7 @@ var (
|
||||
jobs []job
|
||||
lastFetch time.Time
|
||||
cacheTTL = time.Minute * 5
|
||||
jobLimit = 20
|
||||
)
|
||||
|
||||
func skillChecker(description string) skills {
|
||||
@ -63,8 +64,16 @@ func fetchData() error {
|
||||
excluded := []string{"senior", "lead"}
|
||||
// Instantiate a new collector to visit the job details page
|
||||
detailsCollector := c.Clone()
|
||||
|
||||
// Limit the number of jobs to fetch
|
||||
jobCount := 0
|
||||
|
||||
// On every <div> element with class "card__content attribute call callback
|
||||
c.OnHTML("div[class=card__content]", func(e *colly.HTMLElement) {
|
||||
// Return if the job limit has been reached
|
||||
if jobCount >= jobLimit {
|
||||
return
|
||||
}
|
||||
// Get the title and ensure it doesn't contain any excluded words
|
||||
title := e.ChildText("span.card-job-find-list__position")
|
||||
for _, excludedWord := range excluded {
|
||||
@ -83,6 +92,9 @@ func fetchData() error {
|
||||
})
|
||||
|
||||
detailsCollector.OnHTML("div.view-job-details", func(e *colly.HTMLElement) {
|
||||
if jobCount >= jobLimit {
|
||||
return
|
||||
}
|
||||
// Get logo and trim the url
|
||||
|
||||
logo := e.ChildAttr("div.media-item__image", "style")
|
||||
@ -102,6 +114,7 @@ func fetchData() error {
|
||||
Skills: skillChecker(e.ChildText("content.text-block__content > span")),
|
||||
}
|
||||
jobs = append(jobs, jobDetails)
|
||||
jobCount++
|
||||
})
|
||||
// Handle pagination
|
||||
c.OnHTML("a.page-link", func(e *colly.HTMLElement) {
|
||||
|
Loading…
Reference in New Issue
Block a user