Ofure Ikheloa
c370de83d5
- Introduced RedisManager class in scraper.py for centralized Redis operations including job tracking and caching. - Enhanced job scraping logic in MultiPlatformJobScraper to support multiple platforms (Ashby, Lever, Greenhouse). - Updated browser initialization and context management to ensure better resource handling. - Improved error handling and logging throughout the scraping process. - Added SSL connection parameters management in a new ssl_connection.py module for RabbitMQ connections. - Refactored sender.py to utilize RedisManager for job deduplication and improved logging mechanisms. - Enhanced CSV processing logic in sender.py with better validation and error handling. - Updated connection parameters for RabbitMQ to support SSL configurations based on environment variables.
51 lines
2.1 KiB
Python
51 lines
2.1 KiB
Python
|
|
import os
|
|
import json
|
|
|
|
from dotenv import load_dotenv
|
|
|
|
# Load environment variables from .env file
|
|
load_dotenv()
|
|
directory = "C:/Users/OfuRich/Downloads"
|
|
|
|
# LLM Agent Configuration
|
|
DEEPSEEK_API_KEY = os.getenv("DEEPSEEK_API_KEY")
|
|
if not DEEPSEEK_API_KEY:
|
|
raise ValueError("DEEPSEEK_API_KEY environment variable not set in .env file")
|
|
|
|
|
|
def load_spoof_config():
|
|
"""Load spoof data from JSON config file. Falls back to defaults if missing."""
|
|
config_path = os.path.join(os.path.dirname(__file__), "spoof_config.json")
|
|
|
|
if os.path.exists(config_path):
|
|
with open(config_path, "r", encoding="utf-8") as f:
|
|
return json.load(f)
|
|
else:
|
|
# Generate default config file on first run
|
|
default_config = {
|
|
"renderers": {
|
|
"windows": [
|
|
"ANGLE (Intel, Intel(R) UHD Graphics 630 (0x00003E9B) Direct3D11 vs_5_0 ps_5_0, D3D11)",
|
|
"ANGLE (Intel, Intel(R) UHD Graphics (0x00009A49) Direct3D11 vs_5_0 ps_5_0, D3D11)",
|
|
"ANGLE (Intel(R) Iris(TM) Graphics 540 Direct3D11 vs_5_0 ps_5_0)",
|
|
"ANGLE (Intel, Intel(R) UHD Graphics 620 (0x00005916) Direct3D11 vs_5_0 ps_5_0, D3D11)",
|
|
"ANGLE (Intel, Intel(R) HD Graphics 530 (0x0000191B) Direct3D11 vs_5_0 ps_5_0, D3D11)",
|
|
"ANGLE (Intel, Intel(R) UHD Graphics 600 (0x00003180) Direct3D11 vs_5_0 ps_5_0, D3D11)",
|
|
"ANGLE (Intel, Intel(R) Iris(R) Xe Graphics (0x00009A49) Direct3D11 vs_5_0 ps_5_0, D3D11)",
|
|
],
|
|
"macos": [
|
|
"Intel HD Graphics 530 OpenGL Engine",
|
|
"Intel Iris Graphics 6100 OpenGL Engine",
|
|
"Intel UHD Graphics 630 OpenGL Engine",
|
|
"Intel HD Graphics 4000 OpenGL Engine",
|
|
"Intel Iris Pro OpenGL Engine",
|
|
"Intel UHD Graphics 617 OpenGL Engine",
|
|
]
|
|
},
|
|
"vendors": ["Intel Inc.", "Intel", "Intel Corporation"]
|
|
}
|
|
with open(config_path, "w", encoding="utf-8") as f:
|
|
json.dump(default_config, f, indent=2)
|
|
return default_config
|