rannes.dev/sw-jobs-go-lambda
1
0
Fork 0
forked from rannes.dev/sw-jobs-go
Code Pull Requests Activity

added aws lambda integrations

Browse Source
This commit is contained in:
christian 2024-06-08 16:48:00 +02:00
parent 9fe9efbd18
commit c20690f323
3 changed files with 11 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
go.mod
View File

@ -8,6 +8,7 @@ require (
github.com/antchfx/htmlquery v1.3.1 // indirect github.com/antchfx/htmlquery v1.3.1 // indirect
github.com/antchfx/xmlquery v1.4.0 // indirect github.com/antchfx/xmlquery v1.4.0 // indirect
github.com/antchfx/xpath v1.3.0 // indirect github.com/antchfx/xpath v1.3.0 // indirect
github.com/aws/aws-lambda-go v1.47.0 // indirect
github.com/gobwas/glob v0.2.3 // indirect github.com/gobwas/glob v0.2.3 // indirect
github.com/gocolly/colly v1.2.0 // indirect github.com/gocolly/colly v1.2.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect

2
go.sum
View File

@ -8,6 +8,8 @@ github.com/antchfx/xmlquery v1.4.0 h1:xg2HkfcRK2TeTbdb0m1jxCYnvsPaGY/oeZWTGqX/0h
github.com/antchfx/xmlquery v1.4.0/go.mod h1:Ax2aeaeDjfIw3CwXKDQ0GkwZ6QlxoChlIBP+mGnDFjI= github.com/antchfx/xmlquery v1.4.0/go.mod h1:Ax2aeaeDjfIw3CwXKDQ0GkwZ6QlxoChlIBP+mGnDFjI=
github.com/antchfx/xpath v1.3.0 h1:nTMlzGAK3IJ0bPpME2urTuFL76o4A96iYvoKFHRXJgc= github.com/antchfx/xpath v1.3.0 h1:nTMlzGAK3IJ0bPpME2urTuFL76o4A96iYvoKFHRXJgc=
github.com/antchfx/xpath v1.3.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/antchfx/xpath v1.3.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
github.com/aws/aws-lambda-go v1.47.0 h1:0H8s0vumYx/YKs4sE7YM0ktwL2eWse+kfopsRI1sXVI=
github.com/aws/aws-lambda-go v1.47.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=

27
scraper.go
View File

@ -1,12 +1,11 @@
package main package main
import ( import (
"encoding/json" "context"
"fmt" "fmt"
"log"
"os"
"strings" "strings"
"github.com/aws/aws-lambda-go/lambda"
"github.com/gocolly/colly" "github.com/gocolly/colly"
) )
@ -44,14 +43,8 @@ func skillChecker(description string) skills {
// Slice to store job details // Slice to store job details
var jobs []job var jobs []job
func main() { func handler(ctx context.Context) ([]job, error) {
fName := "jobs.json"
file, err := os.Create(fName)
if err != nil {
log.Fatalf("Cannot create file %q: %s", fName, err)
}
defer file.Close()
baseUrl := "https://thehub.io" baseUrl := "https://thehub.io"
searchString := "https://thehub.io/jobs?roles=frontenddeveloper&roles=fullstackdeveloper&roles=backenddeveloper&roles=devops&paid=true&countryCode=DK&sorting=newJobs" searchString := "https://thehub.io/jobs?roles=frontenddeveloper&roles=fullstackdeveloper&roles=backenddeveloper&roles=devops&paid=true&countryCode=DK&sorting=newJobs"
// Instantiate default collector // Instantiate default collector
@ -118,13 +111,9 @@ func main() {
}) })
c.Visit(searchString) c.Visit(searchString)
return jobs, nil
// Encode jobs slice to JSON }
encoder := json.NewEncoder(file)
encoder.SetIndent("", " ") // Pretty-print with indentation func main() {
if err := encoder.Encode(jobs); err != nil { lambda.Start(handler)
log.Fatalf("Cannot write to file %q: %s", fName, err)
}
fmt.Println("Job details successfully written to", fName)
} }