diff --git a/main.go b/main.go
index 9edba70..a3b703a 100644
--- a/main.go
+++ b/main.go
@@ -91,7 +91,7 @@ func scrapeHub() {
jobs []job
jobCount int
fName = "thehub.json"
- maxJobs = 30
+ maxJobs = 20
baseUrl = "https://thehub.io"
searchString = "https://thehub.io/jobs?roles=frontenddeveloper&roles=fullstackdeveloper&roles=backenddeveloper&roles=devops&paid=true&countryCode=DK&sorting=newJobs"
file, err = os.Create(fName)
@@ -106,13 +106,13 @@ func scrapeHub() {
c := colly.NewCollector(
// visit only the hub
colly.AllowedDomains("www.thehub.io", "thehub.io"),
-
- // Cache responses to prevent multiple requests
- colly.CacheDir("./thehub_cache"),
)
// Instantiate a new collector to visit the job details page
- detailsCollector := c.Clone()
+ detailsCollector := colly.NewCollector(
+ colly.AllowedDomains("www.thehub.io", "thehub.io"),
+ colly.CacheDir("./thehub_cache"),
+ )
// On every
element with class "card__content attribute call callback
c.OnHTML("div[class=card__content]", func(e *colly.HTMLElement) {
if jobCount >= maxJobs {
@@ -192,7 +192,7 @@ func scrapeItJobBank() {
jobs []job
jobCount int
fName = "it-jobbank.json"
- maxJobs = 30
+ maxJobs = 20
baseUrl = "https://www.it-jobbank.dk"
searchString = "https://www.it-jobbank.dk/jobsoegning/udvikling"
file, err = os.Create(fName)
@@ -208,11 +208,15 @@ func scrapeItJobBank() {
colly.AllowedDomains("www.it-jobbank.dk", "it-jobbank.dk"),
// Cache responses to prevent multiple requests
- colly.CacheDir("./itjobbank_cache"),
+ // colly.CacheDir("./itjobbank_cache"),
)
// Instantiate a new collector to visit the job details page
- detailsCollector := c.Clone()
+ detailsCollector := colly.NewCollector(
+ colly.AllowedDomains("www.it-jobbank.dk", "it-jobbank.dk"),
+ // Cache responses to prevent multiple requests
+ colly.CacheDir("./itjobbank_cache"))
+
// On every
element with class "card__content attribute call callback
c.OnHTML("div[class=result]", func(e *colly.HTMLElement) {
if jobCount >= maxJobs {