readme markdown

.gitignore

@@ -1,4 +1 @@
-/thehub_cache
+/lambda-package
-/thehub.json
-/itjobbank_cache
-/it-jobbank.json

build.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Set variables
+PACKAGE_DIR="./lambda-package"
+BUILD_FILE="bootstrap"
+ZIP_FILE="lambda-deployment.zip"
+SOURCE_FILE="main.go"
+
+# Delete the content of the lambda-package directory
+rm -rf $PACKAGE_DIR/*
+echo "Deleted the content of $PACKAGE_DIR"
+
+# Set environment variables and build the Go project
+GOOS=linux GOARCH=arm64 go build -o $BUILD_FILE -tags lambda.norpc $SOURCE_FILE
+echo "Built the Go project with GOOS=linux and GOARCH=arm64"
+
+# Move the build file to the lambda-package directory
+mv $BUILD_FILE $PACKAGE_DIR/
+echo "Moved the build file to $PACKAGE_DIR"
+
+# Change directory to lambda-package
+cd $PACKAGE_DIR
+
+# Zip the contents of lambda-package into lambda-deployment.zip
+zip -r $ZIP_FILE *
+echo "Zipped the contents of $PACKAGE_DIR into $ZIP_FILE"
+
+# Return to the original directory
+cd -
+
+echo "Script completed successfully"

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/antchfx/htmlquery v1.3.1 // indirect
 	github.com/antchfx/xmlquery v1.4.0 // indirect
 	github.com/antchfx/xpath v1.3.0 // indirect
+	github.com/aws/aws-lambda-go v1.47.0 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gocolly/colly v1.2.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect

go.sum

@@ -8,6 +8,8 @@ github.com/antchfx/xmlquery v1.4.0 h1:xg2HkfcRK2TeTbdb0m1jxCYnvsPaGY/oeZWTGqX/0h
 github.com/antchfx/xmlquery v1.4.0/go.mod h1:Ax2aeaeDjfIw3CwXKDQ0GkwZ6QlxoChlIBP+mGnDFjI=
 github.com/antchfx/xpath v1.3.0 h1:nTMlzGAK3IJ0bPpME2urTuFL76o4A96iYvoKFHRXJgc=
 github.com/antchfx/xpath v1.3.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
+github.com/aws/aws-lambda-go v1.47.0 h1:0H8s0vumYx/YKs4sE7YM0ktwL2eWse+kfopsRI1sXVI=
+github.com/aws/aws-lambda-go v1.47.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=

main.go

@@ -1,13 +1,12 @@
 package main
 
 import (
-	"encoding/json"
+	"context"
 	"fmt"
-	"log"
-	"os"
 	"strings"
 	"time"
 
+	"github.com/aws/aws-lambda-go/lambda"
 	"github.com/gocolly/colly"
 )
 
@@ -20,8 +19,6 @@ type job struct {
 	Description string `json:"description"`
 	Link        string `json:"link"`
 	Skills      skills `json:"skills"`
-	Scraped     string `json:"scraped"`
-	Source      string `json:"source"`
 }
 
 type skills struct {
@@ -31,94 +28,52 @@ type skills struct {
 	Svelte     bool `json:"svelte"`
 	Nextjs     bool `json:"nextjs"`
 	Typescript bool `json:"typescript"`
-	Tailwind   bool `json:"tailwind"`
 }
 
-// Utility functions
+var (
+	jobs      []job
+	lastFetch time.Time
+	cacheTTL  = time.Minute * 5
+	jobLimit  = 20
+)
 
-// Checks if a string contains any of the given keywords
 func skillChecker(description string) skills {
 	return skills{
 		React:      strings.Contains(description, "React"),
 		Python:     strings.Contains(description, "Python"),
-		Golang:     strings.Contains(description, "Golang"),
+		Golang:     strings.Contains(description, "Go"),
 		Svelte:     strings.Contains(description, "Svelte"),
 		Nextjs:     strings.Contains(description, "Next.js"),
 		Typescript: strings.Contains(description, "TypeScript"),
-		Tailwind:   strings.Contains(description, "Tailwind"),
 	}
 }
 
-// Converts job struct to json
+func fetchData() error {
-func jobsToJson(file *os.File, jobs []job, fName string) {
-	// Encode jobs slice to JSON
-	encoder := json.NewEncoder(file)
-	encoder.SetIndent("", "  ") // Pretty-print with indentation
-	if err := encoder.Encode(jobs); err != nil {
-		log.Fatalf("Cannot write to file %q: %s", fName, err)
-	}
-
-	fmt.Println("Job details successfully written to", fName)
-}
-
-func checkIfPaid(description string) {
-	for _, keyword := range unpaidKeywords {
-		if strings.Contains(strings.ToLower(description), keyword) {
-			return
-		}
-	}
-}
-
-func checkIfStudent(description string) string {
-	for _, keyword := range studentKeywords {
-		if strings.Contains(strings.ToLower(description), keyword) {
-			return "student"
-		}
-	}
-	return "full time"
-}
-
-// Slice to store job details
-var (
-	excluded        = []string{"senior", "lead", "founder", "cto", "vp of", "erfaren", "arkitekt", "architect", "manager", "ulønnet", "unpaid", "praktik", "cyber", "leder", "sikkerhed", "supporter"}
-	unpaidKeywords  = []string{"unpaid", "praktik", "ulønnet"}
-	studentKeywords = []string{"studerende", "studenter", "student", "medhjælper"}
-)
-
-func scrapeHub() {
-	// declare and initialize variables
-	var (
-		jobs         []job
-		jobCount     int
-		fName        = "thehub.json"
-		maxJobs      = 20
-		baseUrl      = "https://thehub.io"
-		searchString = "https://thehub.io/jobs?roles=frontenddeveloper&roles=fullstackdeveloper&roles=backenddeveloper&roles=devops&paid=true&countryCode=DK&sorting=newJobs"
-		file, err    = os.Create(fName)
-	)
-
-	if err != nil {
-		log.Fatalf("Cannot create file %q: %s", fName, err)
-	}
-	defer file.Close()
 
+	baseUrl := "https://thehub.io"
 	// Instantiate default collector
 	c := colly.NewCollector(
 		// visit only the hub
 		colly.AllowedDomains("www.thehub.io", "thehub.io"),
+
+		// Cache responses to prevent multiple requests
+		colly.CacheDir("./tmp"),
 	)
 
+	// Slice of excluded words in the job titles
+	excluded := []string{"senior", "lead"}
 	// Instantiate a new collector to visit the job details page
-	detailsCollector := colly.NewCollector(
+	detailsCollector := c.Clone()
-		colly.AllowedDomains("www.thehub.io", "thehub.io"),
+
-		colly.CacheDir("./thehub_cache"),
+	// Limit the number of jobs to fetch
-	)
+	jobCount := 0
+
 	// On every <div> element with class "card__content attribute call callback
 	c.OnHTML("div[class=card__content]", func(e *colly.HTMLElement) {
-		if jobCount >= maxJobs {
+		// Return if the job limit has been reached
+		if jobCount >= jobLimit {
 			return
 		}
-
 		// Get the title and ensure it doesn't contain any excluded words
 		title := e.ChildText("span.card-job-find-list__position")
 		for _, excludedWord := range excluded {
@@ -130,163 +85,72 @@ func scrapeHub() {
 		fullLink := baseUrl + link
 
 		detailsCollector.Visit(fullLink)
-
 	})
 
 	detailsCollector.OnRequest(func(r *colly.Request) {
 		fmt.Println("Visiting", r.URL.String())
 	})
 
-	detailsCollector.OnHTML("div[class='view-job-details']", func(e *colly.HTMLElement) {
+	detailsCollector.OnHTML("div.view-job-details", func(e *colly.HTMLElement) {
-
+		if jobCount >= jobLimit {
+			return
+		}
 		// Get logo and trim the url
+
 		logo := e.ChildAttr("div.media-item__image", "style")
 		cutLeft := "background-image:url("
 		cutRight := ");"
 		trimmedLogo := strings.Trim(logo, cutLeft+cutRight)
 
-		// Get the HTML of the description and check to see if it's paid
+		// Get company name
-		descriptionHTML, err := e.DOM.Find("content.text-block__content > span").Html()
-		if err != nil {
-			log.Printf("Error getting HTML of description: %s", err)
-			return
-		}
-
-		// fill in the job struct
 		jobDetails := job{
 			Title:       e.ChildText("h2[class=view-job-details__title]"),
 			Logo:        trimmedLogo,
 			Company:     e.ChildText(".bullet-inline-list > a:first-child"),
 			Location:    e.ChildText(".bullet-inline-list > a:nth-child(2)"),
 			Type:        e.ChildText(".bullet-inline-list > a:nth-child(3)"),
-			Description: descriptionHTML,
+			Description: e.ChildText("content.text-block__content > span"),
 			Link:        e.Request.URL.String(),
 			Skills:      skillChecker(e.ChildText("content.text-block__content > span")),
-			Scraped:     time.Now().String(),
-			Source:      baseUrl,
 		}
 		jobs = append(jobs, jobDetails)
 		jobCount++
-		fmt.Println("Scraped job", jobCount)
-		if jobCount >= maxJobs {
-			jobsToJson(file, jobs, fName)
-			return
-		}
 	})
 	// Handle pagination
 	c.OnHTML("a.page-link", func(e *colly.HTMLElement) {
 		nextPage := e.Attr("href")
 		if nextPage != "" {
 			fullNextPage := baseUrl + nextPage
+			fmt.Println("Visiting next page:", fullNextPage)
 			e.Request.Visit(fullNextPage)
 		}
 	})
-
+	// Visit the initial URL to start scraping
-	c.Visit(searchString)
+	err := c.Visit("https://thehub.io/jobs?roles=frontenddeveloper&roles=fullstackdeveloper&roles=backenddeveloper&search=developer&paid=true&countryCode=DK&sorting=newJobs")
-
+	if err != nil {
+		return err
+	}
+	return nil
 }
 
-func scrapeItJobBank() {
+func handler(ctx context.Context) ([]job, error) {
-	// declare and initialize variables
+	// Check if cache is valid
-	var (
+	if time.Since(lastFetch) < cacheTTL && len(jobs) > 0 {
-		jobs         []job
+		return jobs, nil
-		jobCount     int
+	}
-		fName        = "it-jobbank.json"
+
-		maxJobs      = 20
+	// Fetch new data
-		baseUrl      = "https://www.it-jobbank.dk"
+	err := fetchData()
-		searchString = "https://www.it-jobbank.dk/jobsoegning/udvikling"
-		file, err    = os.Create(fName)
-	)
 	if err != nil {
-		log.Fatalf("Cannot create file %q: %s", fName, err)
+		return nil, err
 	}
-	defer file.Close()
 
-	// Instantiate default collector
+	// Update cache timestamp
-	c := colly.NewCollector(
+	lastFetch = time.Now()
-		// visit only the hub
-		colly.AllowedDomains("www.it-jobbank.dk", "it-jobbank.dk"),
 
-		// Cache responses to prevent multiple requests
+	return jobs, nil
-		// colly.CacheDir("./itjobbank_cache"),
-	)
-
-	// Instantiate a new collector to visit the job details page
-	detailsCollector := colly.NewCollector(
-		colly.AllowedDomains("www.it-jobbank.dk", "it-jobbank.dk"),
-		// Cache responses to prevent multiple requests
-		colly.CacheDir("./itjobbank_cache"))
-
-	// On every <div> element with class "card__content attribute call callback
-	c.OnHTML("div[class=result]", func(e *colly.HTMLElement) {
-		if jobCount >= maxJobs {
-			return
-		}
-		// Get the title and ensure it doesn't contain any excluded words
-		title := e.ChildText("h3.job-title > a")
-		for _, excludedWord := range excluded {
-			if strings.Contains(strings.ToLower(title), excludedWord) {
-				return
-			}
-		}
-		fullLink := e.ChildAttr("h3.job-title > a", "href")
-
-		detailsCollector.Visit(fullLink)
-	})
-
-	detailsCollector.OnRequest(func(r *colly.Request) {
-		fmt.Println("Visiting", r.URL.String())
-	})
-
-	detailsCollector.OnHTML("section > div", func(e *colly.HTMLElement) {
-		// get the description as html
-		descriptionHTML, err := e.DOM.Find("div[id=job_ad]").Html()
-		if err != nil {
-			log.Printf("Error getting HTML of description: %s", err)
-			return
-		}
-		// Check if the job is paid
-		checkIfPaid(descriptionHTML)
-		// fill in the job struct
-		title := e.ChildText("h1.title")
-		if title == "" {
-			title = e.ChildText("h1[id=jobtitle]")
-		}
-		jobDetails := job{
-			Title:       title,
-			Logo:        baseUrl + e.ChildAttr("div.company-logo > img", "src"),
-			Company:     e.ChildText("p.published"),
-			Location:    e.ChildText("div.job-location > p.caption"),
-			Type:        checkIfStudent(descriptionHTML),
-			Description: descriptionHTML,
-			Link:        e.Request.URL.String(),
-			Skills:      skillChecker(descriptionHTML),
-			Scraped:     time.Now().String(),
-			Source:      baseUrl,
-		}
-		jobs = append(jobs, jobDetails)
-		jobCount++
-		fmt.Println("Scraped job", jobCount)
-		if jobCount >= maxJobs {
-			jobsToJson(file, jobs, fName)
-			return
-		}
-	})
-	// Handle pagination
-	c.OnHTML("a.page-link", func(e *colly.HTMLElement) {
-		if jobCount >= maxJobs {
-			return
-		}
-		nextPage := e.Attr("href")
-		if nextPage != "" {
-			e.Request.Visit(nextPage)
-		}
-	})
-
-	c.Visit(searchString)
 }
 
 func main() {
-	scrapeHub()
+	lambda.Start(handler)
-	scrapeItJobBank()
 }

readme.md

@@ -1,18 +1,13 @@
-# IT jobs scraper
+# The Hub Scraper
 
-This is a simple scraper that extracts job details from the [The Hub](https://thehub.io) website and itjobbank.
+deprecated as lambda was a bad solution for this, without setting up dynamodb, api etc. This will go live in a ec2 so it can write to local storage instead of running on demand.
 
+Go is fast but free tier lambda is not and I am not yet a smart man.
 
-## Filtering
+<del>This is  a simple scraper that extracts job details from the [The Hub](https://thehub.io) website.</del>
 
-The scraper filters out a list of keywords like senior, architect etc. as I wrote it for entry and mid level roles. It also filters out unpaid form the hub, and keyword based from itjobbank.
+<del>It's a fork of the original [The Hub Scraper](https://gitea.rannes.dev/rannes.dev/sw-jobs-go) by [Rannes](https://gitea.rannes.dev/rannes.dev).</del>
 
-## Usage
+<del>## Usage</del>
 
-To run the scraper, simply execute the following command:
+<del>To run the scraper zip it deploy it to AWS Lambda and then call the function. </del>
-
-```bash
-go run scraper.go
-```
-
-The scraper will create a `thehub.json` and `itjobbank.json` file in the current directory, which contains a list of job details in JSON format. It caches the pages, so very light on resources and requests.

run-scrapers.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-cd /home/admin/sw-jobs-go
-go run go-scraper >> scraper.log 2>&1