dockerfile and crontab

scrapers/Dockerfile

@@ -1,36 +1,26 @@
-# Use an official Python runtime as a parent image
+# Use the official Python image from the Docker Hub
-FROM python:slim
+FROM python:3.9-slim
 
-# Set the working directory in the container
+# Set the working directory inside the container
 WORKDIR /app
 
-# Ensure cron is installed
-RUN apt-get update && apt-get install -y cron && rm -rf /var/lib/apt/lists/*
-
-# Install any needed packages specified in requirements.txt
-COPY requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt
-
 # Copy the current directory contents into the container at /app
 COPY . /app
 
-# Copy the crontab file to the cron.d directory
+# Install the required Python packages
-COPY crontab /etc/cron.d/scraper-cron
+RUN pip install requests beautifulsoup4
+
+# Copy and add a cron job to run the script every hour
+COPY cronjob /etc/cron.d/jobindex-cron
 
 # Give execution rights on the cron job
-RUN chmod 0644 /etc/cron.d/scraper-cron
+RUN chmod 0644 /etc/cron.d/jobindex-cron
 
 # Apply cron job
-RUN crontab /etc/cron.d/scraper-cron
+RUN crontab /etc/cron.d/jobindex-cron
 
 # Create the log file to be able to run tail
 RUN touch /var/log/cron.log
 
-# Copy the script to run the scraper
-COPY run_scraper.sh /usr/local/bin/run_scraper.sh
-
-# Grant execution rights to the script
-RUN chmod +x /usr/local/bin/run_scraper.sh
-
 # Run the command on container startup
-CMD ["cron", "-f"]
+CMD cron && tail -f /var/log/cron.log

scrapers/crontab

@@ -1,2 +1,2 @@
-# Run the scraper every hour
+# Run the jobindex.py script every hour
-0 * * * * root /usr/local/bin/run_scraper.sh >> /var/log/cron.log 2>&1
+0 * * * * root python /app/jobindex.py >> /var/log/cron.log 2>&1

scrapers/run_scraper.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-cd /app
-python job_scraper.py