rannes.dev/sw-jobs-go
1
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixed issue where it would write the jobs twice

Browse Source
This commit is contained in:
christian 2024-06-08 21:03:19 +02:00
parent 9fe9efbd18
commit 38023c1aa5
1 changed files with 27 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
scraper.go
View File

@ -40,13 +40,27 @@ func skillChecker(description string) skills {
Typescript: strings.Contains(description, "TypeScript"), Typescript: strings.Contains(description, "TypeScript"),
} }
} }
func jobsToJson(file *os.File) {
// Encode jobs slice to JSON
encoder := json.NewEncoder(file)
encoder.SetIndent("", " ") // Pretty-print with indentation
if err := encoder.Encode(jobs); err != nil {
log.Fatalf("Cannot write to file %q: %s", fName, err)
}
fmt.Println("Job details successfully written to", fName)
}
// Slice to store job details // Slice to store job details
var jobs []job var (
jobs []job
jobCount int
maxJobs int = 30
fName string = "jobs.json"
)
func main() { func main() {
fName := "jobs.json"
file, err := os.Create(fName) file, err := os.Create(fName)
if err != nil { if err != nil {
log.Fatalf("Cannot create file %q: %s", fName, err) log.Fatalf("Cannot create file %q: %s", fName, err)
@ -69,6 +83,8 @@ func main() {
detailsCollector := c.Clone() detailsCollector := c.Clone()
// On every <div> element with class "card__content attribute call callback // On every <div> element with class "card__content attribute call callback
c.OnHTML("div[class=card__content]", func(e *colly.HTMLElement) { c.OnHTML("div[class=card__content]", func(e *colly.HTMLElement) {
//ensure only scrape the amount of jobs specified
// Get the title and ensure it doesn't contain any excluded words // Get the title and ensure it doesn't contain any excluded words
title := e.ChildText("span.card-job-find-list__position") title := e.ChildText("span.card-job-find-list__position")
for _, excludedWord := range excluded { for _, excludedWord := range excluded {
@ -80,15 +96,16 @@ func main() {
fullLink := baseUrl + link fullLink := baseUrl + link
detailsCollector.Visit(fullLink) detailsCollector.Visit(fullLink)
}) })
detailsCollector.OnRequest(func(r *colly.Request) { detailsCollector.OnRequest(func(r *colly.Request) {
fmt.Println("Visiting", r.URL.String()) fmt.Println("Visiting", r.URL.String())
}) })
detailsCollector.OnHTML("div.view-job-details", func(e *colly.HTMLElement) { detailsCollector.OnHTML("div[class='view-job-details']", func(e *colly.HTMLElement) {
// Get logo and trim the url
// Get logo and trim the url
logo := e.ChildAttr("div.media-item__image", "style") logo := e.ChildAttr("div.media-item__image", "style")
cutLeft := "background-image:url(" cutLeft := "background-image:url("
cutRight := ");" cutRight := ");"
@ -106,6 +123,12 @@ func main() {
Skills: skillChecker(e.ChildText("content.text-block__content > span")), Skills: skillChecker(e.ChildText("content.text-block__content > span")),
} }
jobs = append(jobs, jobDetails) jobs = append(jobs, jobDetails)
jobCount++
fmt.Println("Scraped job", jobCount)
if jobCount == maxJobs {
jobsToJson(file)
os.Exit(0)
}
}) })
// Handle pagination // Handle pagination
c.OnHTML("a.page-link", func(e *colly.HTMLElement) { c.OnHTML("a.page-link", func(e *colly.HTMLElement) {
@ -119,12 +142,4 @@ func main() {
c.Visit(searchString) c.Visit(searchString)
// Encode jobs slice to JSON
encoder := json.NewEncoder(file)
encoder.SetIndent("", " ") // Pretty-print with indentation
if err := encoder.Encode(jobs); err != nil {
log.Fatalf("Cannot write to file %q: %s", fName, err)
}
fmt.Println("Job details successfully written to", fName)
} }