.gitignore

@@ -1,4 +1,2 @@
 /thehub_cache
-/thehub.json
+/jobs.json
-/itjobbank_cache
-/it-jobbank.json

main.go

@@ -6,7 +6,6 @@ import (
 	"log"
 	"os"
 	"strings"
-	"time"
 
 	"github.com/gocolly/colly"
 )
@@ -20,8 +19,6 @@ type job struct {
 	Description string `json:"description"`
 	Link        string `json:"link"`
 	Skills      skills `json:"skills"`
-	FirstSeen   time.Time `json:"firstSeen"`
-	Source      string    `json:"source"`
 }
 
 type skills struct {
@@ -34,9 +31,6 @@ type skills struct {
 	Tailwind   bool `json:"tailwind"`
 }
 
-// Utility functions
-
-// Checks if a string contains any of the given keywords
 func skillChecker(description string) skills {
 	return skills{
 		React:      strings.Contains(description, "React"),
@@ -48,9 +42,7 @@ func skillChecker(description string) skills {
 		Tailwind:   strings.Contains(description, "Tailwind"),
 	}
 }
-
+func jobsToJson(file *os.File) {
-// Converts job struct to json
-func jobsToJson(file *os.File, jobs []job, fName string) {
 	// Encode jobs slice to JSON
 	encoder := json.NewEncoder(file)
 	encoder.SetIndent("", "  ") // Pretty-print with indentation
@@ -61,47 +53,24 @@ func jobsToJson(file *os.File, jobs []job, fName string) {
 	fmt.Println("Job details successfully written to", fName)
 }
 
-func checkIfPaid(description string) {
-	for _, keyword := range unpaidKeywords {
-		if strings.Contains(strings.ToLower(description), keyword) {
-			return
-		}
-	}
-}
-
-func checkIfStudent(description string) string {
-	for _, keyword := range studentKeywords {
-		if strings.Contains(strings.ToLower(description), keyword) {
-			return "student"
-		}
-	}
-	return "full time"
-}
-
 // Slice to store job details
-var (
-	excluded        = []string{"senior", "lead", "founder", "cto", "vp of", "erfaren", "arkitekt", "architect", "manager", "ulønnet", "unpaid", "praktik"}
-	unpaidKeywords  = []string{"unpaid", "praktik", "ulønnet"}
-	studentKeywords = []string{"studerende", "studenter", "student", "medhjælper"}
-)
-
-func scrapeHub() {
-	// declare and initialize variables
 var (
 	jobs     []job
 	jobCount int
-		fName        = "thehub.json"
+	maxJobs  int    = 30
-		maxJobs      = 30
+	fName    string = "jobs.json"
-		baseUrl      = "https://thehub.io"
+	excluded        = []string{"senior", "lead", "founder", "cto", "vp of"}
-		searchString = "https://thehub.io/jobs?roles=frontenddeveloper&roles=fullstackdeveloper&roles=backenddeveloper&roles=devops&paid=true&countryCode=DK&sorting=newJobs"
-		file, err    = os.Create(fName)
 )
 
+func scrapeJobs() {
+
+	file, err := os.Create(fName)
 	if err != nil {
 		log.Fatalf("Cannot create file %q: %s", fName, err)
 	}
 	defer file.Close()
-
+	baseUrl := "https://thehub.io"
+	searchString := "https://thehub.io/jobs?roles=frontenddeveloper&roles=fullstackdeveloper&roles=backenddeveloper&roles=devops&paid=true&countryCode=DK&sorting=newJobs"
 	// Instantiate default collector
 	c := colly.NewCollector(
 		// visit only the hub
@@ -115,9 +84,7 @@ func scrapeHub() {
 	detailsCollector := c.Clone()
 	// On every <div> element with class "card__content attribute call callback
 	c.OnHTML("div[class=card__content]", func(e *colly.HTMLElement) {
-		if jobCount >= maxJobs {
+		//ensure only scrape the amount of jobs specified
-			return
-		}
 
 		// Get the title and ensure it doesn't contain any excluded words
 		title := e.ChildText("span.card-job-find-list__position")
@@ -145,14 +112,14 @@ func scrapeHub() {
 		cutRight := ");"
 		trimmedLogo := strings.Trim(logo, cutLeft+cutRight)
 
-		// Get the HTML of the description and check to see if it's paid
+		// Get job description
+		// Get the HTML of the description
 		descriptionHTML, err := e.DOM.Find("content.text-block__content > span").Html()
 		if err != nil {
 			log.Printf("Error getting HTML of description: %s", err)
 			return
 		}
-
+		// Get company name
-		// fill in the job struct
 		jobDetails := job{
 			Title:       e.ChildText("h2[class=view-job-details__title]"),
 			Logo:        trimmedLogo,
@@ -162,15 +129,13 @@ func scrapeHub() {
 			Description: descriptionHTML,
 			Link:        e.Request.URL.String(),
 			Skills:      skillChecker(e.ChildText("content.text-block__content > span")),
-			FirstSeen:   time.Now(),
-			Source:      baseUrl,
 		}
 		jobs = append(jobs, jobDetails)
 		jobCount++
 		fmt.Println("Scraped job", jobCount)
-		if jobCount >= maxJobs {
+		if jobCount == maxJobs {
-			jobsToJson(file, jobs, fName)
+			jobsToJson(file)
-			return
+			os.Exit(0)
 		}
 	})
 	// Handle pagination
@@ -178,6 +143,7 @@ func scrapeHub() {
 		nextPage := e.Attr("href")
 		if nextPage != "" {
 			fullNextPage := baseUrl + nextPage
+			fmt.Println("Visiting next page:", fullNextPage)
 			e.Request.Visit(fullNextPage)
 		}
 	})
@@ -186,103 +152,6 @@ func scrapeHub() {
 
 }
 
-func scrapeItJobBank() {
-	// declare and initialize variables
-	var (
-		jobs         []job
-		jobCount     int
-		fName        = "it-jobbank.json"
-		maxJobs      = 30
-		baseUrl      = "https://www.it-jobbank.dk"
-		searchString = "https://www.it-jobbank.dk/jobsoegning"
-		file, err    = os.Create(fName)
-	)
-	if err != nil {
-		log.Fatalf("Cannot create file %q: %s", fName, err)
-	}
-	defer file.Close()
-
-	// Instantiate default collector
-	c := colly.NewCollector(
-		// visit only the hub
-		colly.AllowedDomains("www.it-jobbank.dk", "it-jobbank.dk"),
-
-		// Cache responses to prevent multiple requests
-		colly.CacheDir("./itjobbank_cache"),
-	)
-
-	// Instantiate a new collector to visit the job details page
-	detailsCollector := c.Clone()
-	// On every <div> element with class "card__content attribute call callback
-	c.OnHTML("div[class=result]", func(e *colly.HTMLElement) {
-		if jobCount >= maxJobs {
-			return
-		}
-		// Get the title and ensure it doesn't contain any excluded words
-		title := e.ChildText("h3.job-title > a")
-		for _, excludedWord := range excluded {
-			if strings.Contains(strings.ToLower(title), excludedWord) {
-				return
-			}
-		}
-		fullLink := e.ChildAttr("h3.job-title > a", "href")
-
-		detailsCollector.Visit(fullLink)
-	})
-
-	detailsCollector.OnRequest(func(r *colly.Request) {
-		fmt.Println("Visiting", r.URL.String())
-	})
-
-	detailsCollector.OnHTML("section > div", func(e *colly.HTMLElement) {
-		// get the description as html
-		descriptionHTML, err := e.DOM.Find("div[id=job_ad]").Html()
-		if err != nil {
-			log.Printf("Error getting HTML of description: %s", err)
-			return
-		}
-		// Check if the job is paid
-		checkIfPaid(descriptionHTML)
-		// fill in the job struct
-		title := e.ChildText("h1.title")
-		if title == "" {
-			title = e.ChildText("h1[id=jobtitle]")
-		}
-		jobDetails := job{
-			Title:       title,
-			Logo:        baseUrl + e.ChildAttr("div.companmony-logo > img", "src"),
-			Company:     e.ChildText("p.published"),
-			Location:    e.ChildText("div.job-location > p.caption"),
-			Type:        checkIfStudent(descriptionHTML),
-			Description: descriptionHTML,
-			Link:        e.Request.URL.String(),
-			Skills:      skillChecker(descriptionHTML),
-			FirstSeen:   time.Now(),
-			Source:      baseUrl,
-		}
-		jobs = append(jobs, jobDetails)
-		jobCount++
-		fmt.Println("Scraped job", jobCount)
-		if jobCount >= maxJobs {
-			jobsToJson(file, jobs, fName)
-			return
-		}
-	})
-	// Handle pagination
-	c.OnHTML("a.page-link", func(e *colly.HTMLElement) {
-		if jobCount >= maxJobs {
-			return
-		}
-		nextPage := e.Attr("href")
-		if nextPage != "" {
-			e.Request.Visit(nextPage)
-		}
-	})
-
-	c.Visit(searchString)
-}
-
 func main() {
-	scrapeHub()
+	scrapeJobs()
-	scrapeItJobBank()
 }