rannes.dev/sw-jobs-scraper
1
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

moved cache to details collector

Browse Source
This commit is contained in:
christian 2024-06-19 20:26:36 +02:00
parent 333739450f
commit cd675c0d6a
1 changed files with 12 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
main.go
View File

@ -91,7 +91,7 @@ func scrapeHub() {
jobs []job jobs []job
jobCount int jobCount int
fName = "thehub.json" fName = "thehub.json"
maxJobs = 30 maxJobs = 20
baseUrl = "https://thehub.io" baseUrl = "https://thehub.io"
searchString = "https://thehub.io/jobs?roles=frontenddeveloper&roles=fullstackdeveloper&roles=backenddeveloper&roles=devops&paid=true&countryCode=DK&sorting=newJobs" searchString = "https://thehub.io/jobs?roles=frontenddeveloper&roles=fullstackdeveloper&roles=backenddeveloper&roles=devops&paid=true&countryCode=DK&sorting=newJobs"
file, err = os.Create(fName) file, err = os.Create(fName)
@ -106,13 +106,13 @@ func scrapeHub() {
c := colly.NewCollector( c := colly.NewCollector(
// visit only the hub // visit only the hub
colly.AllowedDomains("www.thehub.io", "thehub.io"), colly.AllowedDomains("www.thehub.io", "thehub.io"),
// Cache responses to prevent multiple requests
colly.CacheDir("./thehub_cache"),
) )
// Instantiate a new collector to visit the job details page // Instantiate a new collector to visit the job details page
detailsCollector := c.Clone() detailsCollector := colly.NewCollector(
colly.AllowedDomains("www.thehub.io", "thehub.io"),
colly.CacheDir("./thehub_cache"),
)
// On every <div> element with class "card__content attribute call callback // On every <div> element with class "card__content attribute call callback
c.OnHTML("div[class=card__content]", func(e *colly.HTMLElement) { c.OnHTML("div[class=card__content]", func(e *colly.HTMLElement) {
if jobCount >= maxJobs { if jobCount >= maxJobs {
@ -192,7 +192,7 @@ func scrapeItJobBank() {
jobs []job jobs []job
jobCount int jobCount int
fName = "it-jobbank.json" fName = "it-jobbank.json"
maxJobs = 30 maxJobs = 20
baseUrl = "https://www.it-jobbank.dk" baseUrl = "https://www.it-jobbank.dk"
searchString = "https://www.it-jobbank.dk/jobsoegning/udvikling" searchString = "https://www.it-jobbank.dk/jobsoegning/udvikling"
file, err = os.Create(fName) file, err = os.Create(fName)
@ -208,11 +208,15 @@ func scrapeItJobBank() {
colly.AllowedDomains("www.it-jobbank.dk", "it-jobbank.dk"), colly.AllowedDomains("www.it-jobbank.dk", "it-jobbank.dk"),
// Cache responses to prevent multiple requests // Cache responses to prevent multiple requests
colly.CacheDir("./itjobbank_cache"), // colly.CacheDir("./itjobbank_cache"),
) )
// Instantiate a new collector to visit the job details page // Instantiate a new collector to visit the job details page
detailsCollector := c.Clone() detailsCollector := colly.NewCollector(
colly.AllowedDomains("www.it-jobbank.dk", "it-jobbank.dk"),
// Cache responses to prevent multiple requests
colly.CacheDir("./itjobbank_cache"))
// On every <div> element with class "card__content attribute call callback // On every <div> element with class "card__content attribute call callback
c.OnHTML("div[class=result]", func(e *colly.HTMLElement) { c.OnHTML("div[class=result]", func(e *colly.HTMLElement) {
if jobCount >= maxJobs { if jobCount >= maxJobs {