diff --git a/scraper.go b/scraper.go
index 6bc9922..db9bd1d 100644
--- a/scraper.go
+++ b/scraper.go
@@ -34,6 +34,7 @@ var (
jobs []job
lastFetch time.Time
cacheTTL = time.Minute * 5
+ jobLimit = 20
)
func skillChecker(description string) skills {
@@ -63,8 +64,16 @@ func fetchData() error {
excluded := []string{"senior", "lead"}
// Instantiate a new collector to visit the job details page
detailsCollector := c.Clone()
+
+ // Limit the number of jobs to fetch
+ jobCount := 0
+
// On every
element with class "card__content attribute call callback
c.OnHTML("div[class=card__content]", func(e *colly.HTMLElement) {
+ // Return if the job limit has been reached
+ if jobCount >= jobLimit {
+ return
+ }
// Get the title and ensure it doesn't contain any excluded words
title := e.ChildText("span.card-job-find-list__position")
for _, excludedWord := range excluded {
@@ -83,6 +92,9 @@ func fetchData() error {
})
detailsCollector.OnHTML("div.view-job-details", func(e *colly.HTMLElement) {
+ if jobCount >= jobLimit {
+ return
+ }
// Get logo and trim the url
logo := e.ChildAttr("div.media-item__image", "style")
@@ -102,6 +114,7 @@ func fetchData() error {
Skills: skillChecker(e.ChildText("content.text-block__content > span")),
}
jobs = append(jobs, jobDetails)
+ jobCount++
})
// Handle pagination
c.OnHTML("a.page-link", func(e *colly.HTMLElement) {