added in memory caching for 5 minutes

This commit is contained in:
christian 2024-06-08 16:56:35 +02:00
parent c20690f323
commit 20511db91f

View File

@ -4,6 +4,7 @@ import (
"context" "context"
"fmt" "fmt"
"strings" "strings"
"time"
"github.com/aws/aws-lambda-go/lambda" "github.com/aws/aws-lambda-go/lambda"
"github.com/gocolly/colly" "github.com/gocolly/colly"
@ -29,6 +30,12 @@ type skills struct {
Typescript bool `json:"typescript"` Typescript bool `json:"typescript"`
} }
var (
jobs []job
lastFetch time.Time
cacheTTL = time.Minute * 5
)
func skillChecker(description string) skills { func skillChecker(description string) skills {
return skills{ return skills{
React: strings.Contains(description, "React"), React: strings.Contains(description, "React"),
@ -40,13 +47,9 @@ func skillChecker(description string) skills {
} }
} }
// Slice to store job details func fetchData() error {
var jobs []job
func handler(ctx context.Context) ([]job, error) {
baseUrl := "https://thehub.io" baseUrl := "https://thehub.io"
searchString := "https://thehub.io/jobs?roles=frontenddeveloper&roles=fullstackdeveloper&roles=backenddeveloper&roles=devops&paid=true&countryCode=DK&sorting=newJobs"
// Instantiate default collector // Instantiate default collector
c := colly.NewCollector( c := colly.NewCollector(
// visit only the hub // visit only the hub
@ -109,8 +112,29 @@ func handler(ctx context.Context) ([]job, error) {
e.Request.Visit(fullNextPage) e.Request.Visit(fullNextPage)
} }
}) })
// Visit the initial URL to start scraping
err := c.Visit("https://thehub.io/jobs?roles=frontenddeveloper&roles=fullstackdeveloper&roles=backenddeveloper&search=developer&paid=true&countryCode=DK&sorting=newJobs")
if err != nil {
return err
}
return nil
}
func handler(ctx context.Context) ([]job, error) {
// Check if cache is valid
if time.Since(lastFetch) < cacheTTL && len(jobs) > 0 {
return jobs, nil
}
// Fetch new data
err := fetchData()
if err != nil {
return nil, err
}
// Update cache timestamp
lastFetch = time.Now()
c.Visit(searchString)
return jobs, nil return jobs, nil
} }