From 5fbdd9706e4188f44398e96d47f8da9b0f7032da Mon Sep 17 00:00:00 2001
From: christian <christian@rannes.com>
Date: Sat, 8 Jun 2024 17:04:46 +0200
Subject: [PATCH] set job scraping limit to 20 jobs

---
 scraper.go | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/scraper.go b/scraper.go
index 6bc9922..db9bd1d 100644
--- a/scraper.go
+++ b/scraper.go
@@ -34,6 +34,7 @@ var (
 	jobs      []job
 	lastFetch time.Time
 	cacheTTL  = time.Minute * 5
+	jobLimit  = 20
 )
 
 func skillChecker(description string) skills {
@@ -63,8 +64,16 @@ func fetchData() error {
 	excluded := []string{"senior", "lead"}
 	// Instantiate a new collector to visit the job details page
 	detailsCollector := c.Clone()
+
+	// Limit the number of jobs to fetch
+	jobCount := 0
+
 	// On every <div> element with class "card__content attribute call callback
 	c.OnHTML("div[class=card__content]", func(e *colly.HTMLElement) {
+		// Return if the job limit has been reached
+		if jobCount >= jobLimit {
+			return
+		}
 		// Get the title and ensure it doesn't contain any excluded words
 		title := e.ChildText("span.card-job-find-list__position")
 		for _, excludedWord := range excluded {
@@ -83,6 +92,9 @@ func fetchData() error {
 	})
 
 	detailsCollector.OnHTML("div.view-job-details", func(e *colly.HTMLElement) {
+		if jobCount >= jobLimit {
+			return
+		}
 		// Get logo and trim the url
 
 		logo := e.ChildAttr("div.media-item__image", "style")
@@ -102,6 +114,7 @@ func fetchData() error {
 			Skills:      skillChecker(e.ChildText("content.text-block__content > span")),
 		}
 		jobs = append(jobs, jobDetails)
+		jobCount++
 	})
 	// Handle pagination
 	c.OnHTML("a.page-link", func(e *colly.HTMLElement) {