rannes.dev/sw-jobs-go-lambda
1
0
Fork 0
forked from rannes.dev/sw-jobs-go
Code Pull Requests Activity

set job scraping limit to 20 jobs

Browse Source
This commit is contained in:
christian 2024-06-08 17:04:46 +02:00
parent 20511db91f
commit 5fbdd9706e
1 changed files with 13 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
scraper.go
View File

@ -34,6 +34,7 @@ var (
jobs []job jobs []job
lastFetch time.Time lastFetch time.Time
cacheTTL = time.Minute * 5 cacheTTL = time.Minute * 5
jobLimit = 20
) )
func skillChecker(description string) skills { func skillChecker(description string) skills {
@ -63,8 +64,16 @@ func fetchData() error {
excluded := []string{"senior", "lead"} excluded := []string{"senior", "lead"}
// Instantiate a new collector to visit the job details page // Instantiate a new collector to visit the job details page
detailsCollector := c.Clone() detailsCollector := c.Clone()
// Limit the number of jobs to fetch
jobCount := 0
// On every <div> element with class "card__content attribute call callback // On every <div> element with class "card__content attribute call callback
c.OnHTML("div[class=card__content]", func(e *colly.HTMLElement) { c.OnHTML("div[class=card__content]", func(e *colly.HTMLElement) {
// Return if the job limit has been reached
if jobCount >= jobLimit {
return
}
// Get the title and ensure it doesn't contain any excluded words // Get the title and ensure it doesn't contain any excluded words
title := e.ChildText("span.card-job-find-list__position") title := e.ChildText("span.card-job-find-list__position")
for _, excludedWord := range excluded { for _, excludedWord := range excluded {
@ -83,6 +92,9 @@ func fetchData() error {
}) })
detailsCollector.OnHTML("div.view-job-details", func(e *colly.HTMLElement) { detailsCollector.OnHTML("div.view-job-details", func(e *colly.HTMLElement) {
if jobCount >= jobLimit {
return
}
// Get logo and trim the url // Get logo and trim the url
logo := e.ChildAttr("div.media-item__image", "style") logo := e.ChildAttr("div.media-item__image", "style")
@ -102,6 +114,7 @@ func fetchData() error {
Skills: skillChecker(e.ChildText("content.text-block__content > span")), Skills: skillChecker(e.ChildText("content.text-block__content > span")),
} }
jobs = append(jobs, jobDetails) jobs = append(jobs, jobDetails)
jobCount++
}) })
// Handle pagination // Handle pagination
c.OnHTML("a.page-link", func(e *colly.HTMLElement) { c.OnHTML("a.page-link", func(e *colly.HTMLElement) {