Compare commits

..

8 Commits
main ... main

Author SHA1 Message Date
3199099481 readme markdown 2024-06-08 18:24:54 +00:00
32c5fc7291 Update readme.md 2024-06-08 18:18:02 +00:00
9222b9915d added bash script to build and zip 2024-06-08 19:50:48 +02:00
186db79995 i alrady hate lambda 2024-06-08 19:35:01 +02:00
36a2f49dcf updated readme 2024-06-08 17:06:33 +02:00
5fbdd9706e set job scraping limit to 20 jobs 2024-06-08 17:04:46 +02:00
20511db91f added in memory caching for 5 minutes 2024-06-08 16:56:35 +02:00
c20690f323 added aws lambda integrations 2024-06-08 16:48:00 +02:00
7 changed files with 92 additions and 205 deletions

5
.gitignore vendored
View File

@ -1,4 +1 @@
/thehub_cache
/thehub.json
/itjobbank_cache
/it-jobbank.json
/lambda-package

31
build.sh Executable file
View File

@ -0,0 +1,31 @@
#!/bin/bash
# Set variables
PACKAGE_DIR="./lambda-package"
BUILD_FILE="bootstrap"
ZIP_FILE="lambda-deployment.zip"
SOURCE_FILE="main.go"
# Delete the content of the lambda-package directory
rm -rf $PACKAGE_DIR/*
echo "Deleted the content of $PACKAGE_DIR"
# Set environment variables and build the Go project
GOOS=linux GOARCH=arm64 go build -o $BUILD_FILE -tags lambda.norpc $SOURCE_FILE
echo "Built the Go project with GOOS=linux and GOARCH=arm64"
# Move the build file to the lambda-package directory
mv $BUILD_FILE $PACKAGE_DIR/
echo "Moved the build file to $PACKAGE_DIR"
# Change directory to lambda-package
cd $PACKAGE_DIR
# Zip the contents of lambda-package into lambda-deployment.zip
zip -r $ZIP_FILE *
echo "Zipped the contents of $PACKAGE_DIR into $ZIP_FILE"
# Return to the original directory
cd -
echo "Script completed successfully"

1
go.mod
View File

@ -8,6 +8,7 @@ require (
github.com/antchfx/htmlquery v1.3.1 // indirect
github.com/antchfx/xmlquery v1.4.0 // indirect
github.com/antchfx/xpath v1.3.0 // indirect
github.com/aws/aws-lambda-go v1.47.0 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/gocolly/colly v1.2.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect

2
go.sum
View File

@ -8,6 +8,8 @@ github.com/antchfx/xmlquery v1.4.0 h1:xg2HkfcRK2TeTbdb0m1jxCYnvsPaGY/oeZWTGqX/0h
github.com/antchfx/xmlquery v1.4.0/go.mod h1:Ax2aeaeDjfIw3CwXKDQ0GkwZ6QlxoChlIBP+mGnDFjI=
github.com/antchfx/xpath v1.3.0 h1:nTMlzGAK3IJ0bPpME2urTuFL76o4A96iYvoKFHRXJgc=
github.com/antchfx/xpath v1.3.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
github.com/aws/aws-lambda-go v1.47.0 h1:0H8s0vumYx/YKs4sE7YM0ktwL2eWse+kfopsRI1sXVI=
github.com/aws/aws-lambda-go v1.47.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=

236
main.go
View File

@ -1,13 +1,12 @@
package main
import (
"encoding/json"
"context"
"fmt"
"log"
"os"
"strings"
"time"
"github.com/aws/aws-lambda-go/lambda"
"github.com/gocolly/colly"
)
@ -20,8 +19,6 @@ type job struct {
Description string `json:"description"`
Link string `json:"link"`
Skills skills `json:"skills"`
Scraped string `json:"scraped"`
Source string `json:"source"`
}
type skills struct {
@ -31,94 +28,52 @@ type skills struct {
Svelte bool `json:"svelte"`
Nextjs bool `json:"nextjs"`
Typescript bool `json:"typescript"`
Tailwind bool `json:"tailwind"`
}
// Utility functions
var (
jobs []job
lastFetch time.Time
cacheTTL = time.Minute * 5
jobLimit = 20
)
// Checks if a string contains any of the given keywords
func skillChecker(description string) skills {
return skills{
React: strings.Contains(description, "React"),
Python: strings.Contains(description, "Python"),
Golang: strings.Contains(description, "Golang"),
Golang: strings.Contains(description, "Go"),
Svelte: strings.Contains(description, "Svelte"),
Nextjs: strings.Contains(description, "Next.js"),
Typescript: strings.Contains(description, "TypeScript"),
Tailwind: strings.Contains(description, "Tailwind"),
}
}
// Converts job struct to json
func jobsToJson(file *os.File, jobs []job, fName string) {
// Encode jobs slice to JSON
encoder := json.NewEncoder(file)
encoder.SetIndent("", " ") // Pretty-print with indentation
if err := encoder.Encode(jobs); err != nil {
log.Fatalf("Cannot write to file %q: %s", fName, err)
}
fmt.Println("Job details successfully written to", fName)
}
func checkIfPaid(description string) {
for _, keyword := range unpaidKeywords {
if strings.Contains(strings.ToLower(description), keyword) {
return
}
}
}
func checkIfStudent(description string) string {
for _, keyword := range studentKeywords {
if strings.Contains(strings.ToLower(description), keyword) {
return "student"
}
}
return "full time"
}
// Slice to store job details
var (
excluded = []string{"senior", "lead", "founder", "cto", "vp of", "erfaren", "arkitekt", "architect", "manager", "ulønnet", "unpaid", "praktik", "cyber", "leder", "sikkerhed", "supporter"}
unpaidKeywords = []string{"unpaid", "praktik", "ulønnet"}
studentKeywords = []string{"studerende", "studenter", "student", "medhjælper"}
)
func scrapeHub() {
// declare and initialize variables
var (
jobs []job
jobCount int
fName = "thehub.json"
maxJobs = 20
baseUrl = "https://thehub.io"
searchString = "https://thehub.io/jobs?roles=frontenddeveloper&roles=fullstackdeveloper&roles=backenddeveloper&roles=devops&paid=true&countryCode=DK&sorting=newJobs"
file, err = os.Create(fName)
)
if err != nil {
log.Fatalf("Cannot create file %q: %s", fName, err)
}
defer file.Close()
func fetchData() error {
baseUrl := "https://thehub.io"
// Instantiate default collector
c := colly.NewCollector(
// visit only the hub
colly.AllowedDomains("www.thehub.io", "thehub.io"),
// Cache responses to prevent multiple requests
colly.CacheDir("./tmp"),
)
// Slice of excluded words in the job titles
excluded := []string{"senior", "lead"}
// Instantiate a new collector to visit the job details page
detailsCollector := colly.NewCollector(
colly.AllowedDomains("www.thehub.io", "thehub.io"),
colly.CacheDir("./thehub_cache"),
)
detailsCollector := c.Clone()
// Limit the number of jobs to fetch
jobCount := 0
// On every <div> element with class "card__content attribute call callback
c.OnHTML("div[class=card__content]", func(e *colly.HTMLElement) {
if jobCount >= maxJobs {
// Return if the job limit has been reached
if jobCount >= jobLimit {
return
}
// Get the title and ensure it doesn't contain any excluded words
title := e.ChildText("span.card-job-find-list__position")
for _, excludedWord := range excluded {
@ -130,163 +85,72 @@ func scrapeHub() {
fullLink := baseUrl + link
detailsCollector.Visit(fullLink)
})
detailsCollector.OnRequest(func(r *colly.Request) {
fmt.Println("Visiting", r.URL.String())
})
detailsCollector.OnHTML("div[class='view-job-details']", func(e *colly.HTMLElement) {
detailsCollector.OnHTML("div.view-job-details", func(e *colly.HTMLElement) {
if jobCount >= jobLimit {
return
}
// Get logo and trim the url
logo := e.ChildAttr("div.media-item__image", "style")
cutLeft := "background-image:url("
cutRight := ");"
trimmedLogo := strings.Trim(logo, cutLeft+cutRight)
// Get the HTML of the description and check to see if it's paid
descriptionHTML, err := e.DOM.Find("content.text-block__content > span").Html()
if err != nil {
log.Printf("Error getting HTML of description: %s", err)
return
}
// fill in the job struct
// Get company name
jobDetails := job{
Title: e.ChildText("h2[class=view-job-details__title]"),
Logo: trimmedLogo,
Company: e.ChildText(".bullet-inline-list > a:first-child"),
Location: e.ChildText(".bullet-inline-list > a:nth-child(2)"),
Type: e.ChildText(".bullet-inline-list > a:nth-child(3)"),
Description: descriptionHTML,
Description: e.ChildText("content.text-block__content > span"),
Link: e.Request.URL.String(),
Skills: skillChecker(e.ChildText("content.text-block__content > span")),
Scraped: time.Now().String(),
Source: baseUrl,
}
jobs = append(jobs, jobDetails)
jobCount++
fmt.Println("Scraped job", jobCount)
if jobCount >= maxJobs {
jobsToJson(file, jobs, fName)
return
}
})
// Handle pagination
c.OnHTML("a.page-link", func(e *colly.HTMLElement) {
nextPage := e.Attr("href")
if nextPage != "" {
fullNextPage := baseUrl + nextPage
fmt.Println("Visiting next page:", fullNextPage)
e.Request.Visit(fullNextPage)
}
})
c.Visit(searchString)
// Visit the initial URL to start scraping
err := c.Visit("https://thehub.io/jobs?roles=frontenddeveloper&roles=fullstackdeveloper&roles=backenddeveloper&search=developer&paid=true&countryCode=DK&sorting=newJobs")
if err != nil {
return err
}
return nil
}
func scrapeItJobBank() {
// declare and initialize variables
var (
jobs []job
jobCount int
fName = "it-jobbank.json"
maxJobs = 20
baseUrl = "https://www.it-jobbank.dk"
searchString = "https://www.it-jobbank.dk/jobsoegning/udvikling"
file, err = os.Create(fName)
)
if err != nil {
log.Fatalf("Cannot create file %q: %s", fName, err)
func handler(ctx context.Context) ([]job, error) {
// Check if cache is valid
if time.Since(lastFetch) < cacheTTL && len(jobs) > 0 {
return jobs, nil
}
defer file.Close()
// Instantiate default collector
c := colly.NewCollector(
// visit only the hub
colly.AllowedDomains("www.it-jobbank.dk", "it-jobbank.dk"),
// Fetch new data
err := fetchData()
if err != nil {
return nil, err
}
// Cache responses to prevent multiple requests
// colly.CacheDir("./itjobbank_cache"),
)
// Update cache timestamp
lastFetch = time.Now()
// Instantiate a new collector to visit the job details page
detailsCollector := colly.NewCollector(
colly.AllowedDomains("www.it-jobbank.dk", "it-jobbank.dk"),
// Cache responses to prevent multiple requests
colly.CacheDir("./itjobbank_cache"))
// On every <div> element with class "card__content attribute call callback
c.OnHTML("div[class=result]", func(e *colly.HTMLElement) {
if jobCount >= maxJobs {
return
}
// Get the title and ensure it doesn't contain any excluded words
title := e.ChildText("h3.job-title > a")
for _, excludedWord := range excluded {
if strings.Contains(strings.ToLower(title), excludedWord) {
return
}
}
fullLink := e.ChildAttr("h3.job-title > a", "href")
detailsCollector.Visit(fullLink)
})
detailsCollector.OnRequest(func(r *colly.Request) {
fmt.Println("Visiting", r.URL.String())
})
detailsCollector.OnHTML("section > div", func(e *colly.HTMLElement) {
// get the description as html
descriptionHTML, err := e.DOM.Find("div[id=job_ad]").Html()
if err != nil {
log.Printf("Error getting HTML of description: %s", err)
return
}
// Check if the job is paid
checkIfPaid(descriptionHTML)
// fill in the job struct
title := e.ChildText("h1.title")
if title == "" {
title = e.ChildText("h1[id=jobtitle]")
}
jobDetails := job{
Title: title,
Logo: baseUrl + e.ChildAttr("div.company-logo > img", "src"),
Company: e.ChildText("p.published"),
Location: e.ChildText("div.job-location > p.caption"),
Type: checkIfStudent(descriptionHTML),
Description: descriptionHTML,
Link: e.Request.URL.String(),
Skills: skillChecker(descriptionHTML),
Scraped: time.Now().String(),
Source: baseUrl,
}
jobs = append(jobs, jobDetails)
jobCount++
fmt.Println("Scraped job", jobCount)
if jobCount >= maxJobs {
jobsToJson(file, jobs, fName)
return
}
})
// Handle pagination
c.OnHTML("a.page-link", func(e *colly.HTMLElement) {
if jobCount >= maxJobs {
return
}
nextPage := e.Attr("href")
if nextPage != "" {
e.Request.Visit(nextPage)
}
})
c.Visit(searchString)
return jobs, nil
}
func main() {
scrapeHub()
scrapeItJobBank()
lambda.Start(handler)
}

View File

@ -1,18 +1,13 @@
# IT jobs scraper
# The Hub Scraper
This is a simple scraper that extracts job details from the [The Hub](https://thehub.io) website and itjobbank.
deprecated as lambda was a bad solution for this, without setting up dynamodb, api etc. This will go live in a ec2 so it can write to local storage instead of running on demand.
Go is fast but free tier lambda is not and I am not yet a smart man.
## Filtering
<del>This is a simple scraper that extracts job details from the [The Hub](https://thehub.io) website.</del>
The scraper filters out a list of keywords like senior, architect etc. as I wrote it for entry and mid level roles. It also filters out unpaid form the hub, and keyword based from itjobbank.
<del>It's a fork of the original [The Hub Scraper](https://gitea.rannes.dev/rannes.dev/sw-jobs-go) by [Rannes](https://gitea.rannes.dev/rannes.dev).</del>
## Usage
<del>## Usage</del>
To run the scraper, simply execute the following command:
```bash
go run scraper.go
```
The scraper will create a `thehub.json` and `itjobbank.json` file in the current directory, which contains a list of job details in JSON format. It caches the pages, so very light on resources and requests.
<del>To run the scraper zip it deploy it to AWS Lambda and then call the function. </del>

View File

@ -1,3 +0,0 @@
#!/bin/bash
cd /home/admin/sw-jobs-go
go run go-scraper >> scraper.log 2>&1