Compare commits
No commits in common. "2b1387b3e6aa2bb1023cc1112029c55ebd21c452" and "91047cfc5c266adb6cf848ac6672ff3755288e51" have entirely different histories.
2b1387b3e6
...
91047cfc5c
8
.env
8
.env
@ -1,8 +0,0 @@
|
|||||||
SCRAPING_USERNAME="anointedsaviour1@gmail.com"
|
|
||||||
SCRAPING_PASSWORD="PeaceIkheloa2478#"
|
|
||||||
DEEPSEEK_API_KEY=sk-90ce747579f6469ea88a97e0168b7a34
|
|
||||||
DB_URL="jdbc:postgresql://aws-0-us-west-1.pooler.supabase.com:5432/postgres"
|
|
||||||
DB_USERNAME="postgres.gezjetnnesuwczhzoqll"
|
|
||||||
DB_PASSWORD="Vx3UbzWzxoVRUqQH"
|
|
||||||
DB_PORT=5432
|
|
||||||
DB_HOST="aws-0-us-west-1.pooler.supabase.com"
|
|
||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -1,234 +0,0 @@
|
|||||||
{
|
|
||||||
"cookies": [
|
|
||||||
{
|
|
||||||
"name": "lang",
|
|
||||||
"value": "v=2&lang=en-us",
|
|
||||||
"domain": ".linkedin.com",
|
|
||||||
"path": "/",
|
|
||||||
"expires": -1,
|
|
||||||
"httpOnly": false,
|
|
||||||
"secure": true,
|
|
||||||
"sameSite": "None"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "JSESSIONID",
|
|
||||||
"value": "\"ajax:6414932927153560871\"",
|
|
||||||
"domain": ".www.linkedin.com",
|
|
||||||
"path": "/",
|
|
||||||
"expires": 1772715301.1096,
|
|
||||||
"httpOnly": false,
|
|
||||||
"secure": true,
|
|
||||||
"sameSite": "None"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "bcookie",
|
|
||||||
"value": "\"v=2&ee0384d8-ad76-4350-8b62-ce9c3c0dca68\"",
|
|
||||||
"domain": ".linkedin.com",
|
|
||||||
"path": "/",
|
|
||||||
"expires": 1796475301.109999,
|
|
||||||
"httpOnly": false,
|
|
||||||
"secure": true,
|
|
||||||
"sameSite": "None"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "bscookie",
|
|
||||||
"value": "\"v=1&20251205125436e9a215cc-6d86-4537-8ae5-9aa27ad3c276AQHN0Y58UVRbAbemMKEQOFuWwpkYRav8\"",
|
|
||||||
"domain": ".www.linkedin.com",
|
|
||||||
"path": "/",
|
|
||||||
"expires": 1796475301.11051,
|
|
||||||
"httpOnly": true,
|
|
||||||
"secure": true,
|
|
||||||
"sameSite": "None"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "demdex",
|
|
||||||
"value": "49748263616183205111016013239368887912",
|
|
||||||
"domain": ".demdex.net",
|
|
||||||
"path": "/",
|
|
||||||
"expires": 1780491300.716252,
|
|
||||||
"httpOnly": false,
|
|
||||||
"secure": true,
|
|
||||||
"sameSite": "None"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "AMCVS_14215E3D5995C57C0A495C55%40AdobeOrg",
|
|
||||||
"value": "1",
|
|
||||||
"domain": ".linkedin.com",
|
|
||||||
"path": "/",
|
|
||||||
"expires": -1,
|
|
||||||
"httpOnly": false,
|
|
||||||
"secure": false,
|
|
||||||
"sameSite": "Lax"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "AMCV_14215E3D5995C57C0A495C55%40AdobeOrg",
|
|
||||||
"value": "-637568504%7CMCIDTS%7C20428%7CMCMID%7C50274389017273793801069241612151807395%7CMCAAMLH-1765544084%7C6%7CMCAAMB-1765544084%7C6G1ynYcLPuiQxYZrsz_pkqfLG9yMXBpb2zX5dvJdYQJzPXImdj0y%7CMCOPTOUT-1764946484s%7CNONE%7CvVersion%7C5.1.1",
|
|
||||||
"domain": ".linkedin.com",
|
|
||||||
"path": "/",
|
|
||||||
"expires": 1780491284,
|
|
||||||
"httpOnly": false,
|
|
||||||
"secure": false,
|
|
||||||
"sameSite": "Lax"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "pxcts",
|
|
||||||
"value": "92c3afda-d1d9-11f0-ab95-05d3c3f5ff56",
|
|
||||||
"domain": ".protechts.net",
|
|
||||||
"path": "/",
|
|
||||||
"expires": -1,
|
|
||||||
"httpOnly": false,
|
|
||||||
"secure": true,
|
|
||||||
"sameSite": "None",
|
|
||||||
"partitionKey": "https://linkedin.com",
|
|
||||||
"_crHasCrossSiteAncestor": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "_pxvid",
|
|
||||||
"value": "92c3a852-d1d9-11f0-ab95-34d9ccdf3684",
|
|
||||||
"domain": ".protechts.net",
|
|
||||||
"path": "/",
|
|
||||||
"expires": 1796475285,
|
|
||||||
"httpOnly": false,
|
|
||||||
"secure": true,
|
|
||||||
"sameSite": "None",
|
|
||||||
"partitionKey": "https://linkedin.com",
|
|
||||||
"_crHasCrossSiteAncestor": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "aam_uuid",
|
|
||||||
"value": "49748263616183205111016013239368887912",
|
|
||||||
"domain": ".linkedin.com",
|
|
||||||
"path": "/",
|
|
||||||
"expires": 1767531300,
|
|
||||||
"httpOnly": false,
|
|
||||||
"secure": false,
|
|
||||||
"sameSite": "Lax"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "_px3",
|
|
||||||
"value": "d3478ad10e0c90ce1ec8710e203504a13b98e740eed4acc00058c5af37bdf6f1:6vaI70cC/08uf41yqbmObw429PGWFWDTe98EpIWZFxQnFmQLqub8QMuavlytWphyaYfGCtOZxjDOC7611wbB6A==:1000:QxNXY91nXcQqWjnpV++FKIDtw9j06fm5+j3sb4/B5LvS6javeNtDJvOb9Z0nCKS7P6dzphqMElZ/2Ngbqs6e/5AJSh/q2zDwWADFc+RA3iTnr6BFtad6lcjMeAxtXCZtj57Vzozuh33QiRACo4VN54GJjHm30CT2Z8ZF9WeSDpUg8lwvuPJQmHMM+tmAsDPItkYdAtIDsKaul6t5/DOKnOhobjv1P73mZls/jX9d1+UE5gDr9VJk6+knLjyTTOO5t2bBXBgHJD3Ho6v8UeXJUjia5HoR3yaPSSMihu7HDpuZQ9LtwJU9sFI51H3u6qwdNQ4kIwl3yUj+DNDP90KCog==",
|
|
||||||
"domain": ".protechts.net",
|
|
||||||
"path": "/",
|
|
||||||
"expires": 1764939616,
|
|
||||||
"httpOnly": false,
|
|
||||||
"secure": true,
|
|
||||||
"sameSite": "None",
|
|
||||||
"partitionKey": "https://linkedin.com",
|
|
||||||
"_crHasCrossSiteAncestor": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "dextp",
|
|
||||||
"value": "771-1-1764939287775|1957-1-1764939288438",
|
|
||||||
"domain": ".demdex.net",
|
|
||||||
"path": "/",
|
|
||||||
"expires": 1780491288,
|
|
||||||
"httpOnly": false,
|
|
||||||
"secure": true,
|
|
||||||
"sameSite": "None"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "IDE",
|
|
||||||
"value": "AHWqTUkENM8ygqsXxJbYJhMCvsKQDpH0IQqLDSIYObecLYncoySBAITeoYbdR5npYVY",
|
|
||||||
"domain": ".doubleclick.net",
|
|
||||||
"path": "/",
|
|
||||||
"expires": 1799499291.016462,
|
|
||||||
"httpOnly": true,
|
|
||||||
"secure": true,
|
|
||||||
"sameSite": "None"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "MUID",
|
|
||||||
"value": "185C50F0360265D73A484649371F649B",
|
|
||||||
"domain": ".bing.com",
|
|
||||||
"path": "/",
|
|
||||||
"expires": 1798635291.421754,
|
|
||||||
"httpOnly": false,
|
|
||||||
"secure": true,
|
|
||||||
"sameSite": "None"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "MR",
|
|
||||||
"value": "0",
|
|
||||||
"domain": ".c.bing.com",
|
|
||||||
"path": "/",
|
|
||||||
"expires": 1765544091.422484,
|
|
||||||
"httpOnly": false,
|
|
||||||
"secure": true,
|
|
||||||
"sameSite": "None"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "dpm",
|
|
||||||
"value": "49748263616183205111016013239368887912",
|
|
||||||
"domain": ".dpm.demdex.net",
|
|
||||||
"path": "/",
|
|
||||||
"expires": 1780491292.29327,
|
|
||||||
"httpOnly": false,
|
|
||||||
"secure": true,
|
|
||||||
"sameSite": "None"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "li_rm",
|
|
||||||
"value": "AQG0wJrw9Q7tHgAAAZrulIAKlkxOAQyD0mcwHIsk3nC_rddnGBe7IhOMG-80h9zLtbB5WtwsFLZLzEi8EuVOpSIKfmzeBFrWJJzuu3-QSR-gNkdY4-Dp7tO2n-3o1tDzKZyGxC1uav9-VUMTabBuUVYZLn-drmWjnyJHDjkJFyVQI1z2lVHwIdzMtwo2w5KjYZ6lGH4YrniyRR_RsqpFHAQMiilcXkfS6Ky7ygw2MtJsNp00JuqUvFlNucrUStI0rjCVg4pJLYJ7ChLkKWnJ3g6-gzxQDpRzVtLxYYAJqu0ti8i7yN9hVY00eTJ-IkuXn4-OqB-jxcNqF8WwEPlvXg",
|
|
||||||
"domain": ".www.linkedin.com",
|
|
||||||
"path": "/",
|
|
||||||
"expires": 1796475301.106047,
|
|
||||||
"httpOnly": true,
|
|
||||||
"secure": true,
|
|
||||||
"sameSite": "None"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "li_at",
|
|
||||||
"value": "AQEDAVg_uJwBA20PAAABmu6Uf8MAAAGbEqEDw00AEaEvjJb7Pym0Oe837aCfwe0a6pwKOGXmFMV7gDRPPlYcrtc1XJ30XSgrjTnCR_gSX2aQ3DyR2xxYfCXxRrt-tBGThsqSmWddnC_fDRn2Srfw08cp",
|
|
||||||
"domain": ".www.linkedin.com",
|
|
||||||
"path": "/",
|
|
||||||
"expires": 1796475301.10862,
|
|
||||||
"httpOnly": true,
|
|
||||||
"secure": true,
|
|
||||||
"sameSite": "None"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "liap",
|
|
||||||
"value": "true",
|
|
||||||
"domain": ".linkedin.com",
|
|
||||||
"path": "/",
|
|
||||||
"expires": 1772715301.109226,
|
|
||||||
"httpOnly": false,
|
|
||||||
"secure": true,
|
|
||||||
"sameSite": "None"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "lidc",
|
|
||||||
"value": "\"b=TB36:s=T:r=T:a=T:p=T:g=22353:u=19:x=1:i=1764939301:t=1765009011:v=2:sig=AQFS_83h3lEfSOyPub4QelcjkyE2wWuH\"",
|
|
||||||
"domain": ".linkedin.com",
|
|
||||||
"path": "/",
|
|
||||||
"expires": 1765009011.625832,
|
|
||||||
"httpOnly": false,
|
|
||||||
"secure": true,
|
|
||||||
"sameSite": "None"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"origins": [
|
|
||||||
{
|
|
||||||
"origin": "https://li.protechts.net",
|
|
||||||
"localStorage": [
|
|
||||||
{
|
|
||||||
"name": "PXdOjV695v_px-ff",
|
|
||||||
"value": "eyJjYyI6eyJ0dGwiOjE3NjQ5MzkzNDUsInZhbCI6IlUyRnRaVk5wZEdVOVRtOXVaVHNnVTJWamRYSmxPeUJRWVhKMGFYUnBiMjVsWkRzZyJ9LCJ1aWk0Ijp7InR0bCI6MTc2NDkzOTM0NSwidmFsIjoiMSJ9fQ=="
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "PXdOjV695v_px_hvd",
|
|
||||||
"value": "0299e0542146573e19d2d7ceffa18989"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"origin": "https://www.google.com",
|
|
||||||
"localStorage": [
|
|
||||||
{
|
|
||||||
"name": "rc::a",
|
|
||||||
"value": "MWFhd2QwcmxwdDM0OQ=="
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
@ -1,145 +0,0 @@
|
|||||||
{
|
|
||||||
"cookies": [
|
|
||||||
{
|
|
||||||
"name": "lang",
|
|
||||||
"value": "v=2&lang=en-us",
|
|
||||||
"domain": ".linkedin.com",
|
|
||||||
"path": "/",
|
|
||||||
"expires": -1,
|
|
||||||
"httpOnly": false,
|
|
||||||
"secure": true,
|
|
||||||
"sameSite": "None"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "JSESSIONID",
|
|
||||||
"value": "\"ajax:6777866845446095826\"",
|
|
||||||
"domain": ".www.linkedin.com",
|
|
||||||
"path": "/",
|
|
||||||
"expires": 1772720419.575331,
|
|
||||||
"httpOnly": false,
|
|
||||||
"secure": true,
|
|
||||||
"sameSite": "None"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "bcookie",
|
|
||||||
"value": "\"v=2&2df8d7d1-ef66-4bbf-85cc-a8614a897f1c\"",
|
|
||||||
"domain": ".linkedin.com",
|
|
||||||
"path": "/",
|
|
||||||
"expires": 1796480420.575445,
|
|
||||||
"httpOnly": false,
|
|
||||||
"secure": true,
|
|
||||||
"sameSite": "None"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "bscookie",
|
|
||||||
"value": "\"v=1&2025120514195423370970-fabd-4cef-8386-f69c9534587fAQFBudf2MvybzP4QbiS6i2YswxKMJAFM\"",
|
|
||||||
"domain": ".www.linkedin.com",
|
|
||||||
"path": "/",
|
|
||||||
"expires": 1796480420.5756,
|
|
||||||
"httpOnly": true,
|
|
||||||
"secure": true,
|
|
||||||
"sameSite": "None"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "pxcts",
|
|
||||||
"value": "8625c7b1-d1e5-11f0-a953-66ded73d25b7",
|
|
||||||
"domain": ".protechts.net",
|
|
||||||
"path": "/",
|
|
||||||
"expires": -1,
|
|
||||||
"httpOnly": false,
|
|
||||||
"secure": true,
|
|
||||||
"sameSite": "None",
|
|
||||||
"partitionKey": "https://linkedin.com",
|
|
||||||
"_crHasCrossSiteAncestor": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "_pxvid",
|
|
||||||
"value": "8625c03d-d1e5-11f0-a953-6c8ab4db491e",
|
|
||||||
"domain": ".protechts.net",
|
|
||||||
"path": "/",
|
|
||||||
"expires": 1796480417,
|
|
||||||
"httpOnly": false,
|
|
||||||
"secure": true,
|
|
||||||
"sameSite": "None",
|
|
||||||
"partitionKey": "https://linkedin.com",
|
|
||||||
"_crHasCrossSiteAncestor": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "_px3",
|
|
||||||
"value": "eb9292f5a5f06e41b2f1589cbebc1e97882ab64df00d17000363fee910ea27be:nN2jjOmx6vJSNiO7suSMlMp14Ab4eM8YUloxFbReL9QieTXyUOWFCsIpO/w9BX1BmRPFJSNCGNJTC9utoa8vIA==:1000:SUukEtPVwtzWQqC21G4pB7fBHjdvse0/91EAK7cRiPD9Eq1pAATWEZ5EQqgGvRcg2VNhwTVGGnQ6ZqlS/REI0snwhD67rUw6+daqPepuYYEr0D47LoOxuEDzsBxT1nsYt/0ynAL0rPcU0vVczda4NxCZTdowLCmPYtnu0aagh1VhUvs6FABtH+yuNTuCeXwrp9OB3SAZrQpgREvBiOUVzuZI3N6H6LIR8vsRvY9c/SLiqB6yVqw5Xwd2hTR0VO1wUrEMVEueaYF+PG2ZDzMco5vsxbX/0ICINNmTwaPYtgaMQXrjo04ZXxhxRHCCNFxuoms3LqURwsYsYRGOtZpCZg==",
|
|
||||||
"domain": ".protechts.net",
|
|
||||||
"path": "/",
|
|
||||||
"expires": 1764944749,
|
|
||||||
"httpOnly": false,
|
|
||||||
"secure": true,
|
|
||||||
"sameSite": "None",
|
|
||||||
"partitionKey": "https://linkedin.com",
|
|
||||||
"_crHasCrossSiteAncestor": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "li_rm",
|
|
||||||
"value": "AQGJlIJqdL0mRQAAAZru4pmGT5x3o3THOy44-WDYOMl9t7HuU3IMEH2uMAyJLFcIC0IWp8mJvRBsLflUN0RNlA-rpcQX1dyOp9z-Eh9KWtyAk0ug9ZsKXMTVjqNJEHr4lOZ6AW2Dv3U1ILkNxYgb93u-jhrhyPUHBxiGI5_PWRk1zrkVGTibU0yzNKx3_EuzhB7nqsy0QMZB4TX-YE4NMb98lW5BXapVlQbnQVnSKp3P1sZvSjV1y1ombKQLQVaGY83cZikYDX_MxjnsPjb29GMuKNYS2EgB3aUiB1xnKKNN8WLEMSxRZLeG_eZnsSq8_qwlVIiGZf-4zU3-LyZS2A",
|
|
||||||
"domain": ".www.linkedin.com",
|
|
||||||
"path": "/",
|
|
||||||
"expires": 1796480419.574544,
|
|
||||||
"httpOnly": true,
|
|
||||||
"secure": true,
|
|
||||||
"sameSite": "None"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "liap",
|
|
||||||
"value": "true",
|
|
||||||
"domain": ".linkedin.com",
|
|
||||||
"path": "/",
|
|
||||||
"expires": 1772720419.575047,
|
|
||||||
"httpOnly": false,
|
|
||||||
"secure": true,
|
|
||||||
"sameSite": "None"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "li_at",
|
|
||||||
"value": "AQEDAVg_uJwFQozZAAABmu7imUcAAAGbEu8dR00AcgTBwgeJGqGuVHbQTogwJnKFd-szlMOfyA0ypZl1-yCPUBdHWXg59-PaswPdZgKcZ8rkH7y6v8U2x3ae1d0HTl44qR8Q78j5vaQuq4Y50_3kIT9S",
|
|
||||||
"domain": ".www.linkedin.com",
|
|
||||||
"path": "/",
|
|
||||||
"expires": 1796480419.575119,
|
|
||||||
"httpOnly": true,
|
|
||||||
"secure": true,
|
|
||||||
"sameSite": "None"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "lidc",
|
|
||||||
"value": "\"b=TB36:s=T:r=T:a=T:p=T:g=22353:u=19:x=1:i=1764944419:t=1765009011:v=2:sig=AQFlU75-P8j2IFgpOIjlSQVv9VvOpv6N\"",
|
|
||||||
"domain": ".linkedin.com",
|
|
||||||
"path": "/",
|
|
||||||
"expires": 1765009012.154817,
|
|
||||||
"httpOnly": false,
|
|
||||||
"secure": true,
|
|
||||||
"sameSite": "None"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "NID",
|
|
||||||
"value": "527=M-TAjjAaIklAJBHud6rpRzfdFLJgP6psEofReYwdSuHVwJJKK1cVDYyJbwmY20FvN0-bdr8NvT4vsFQTf3Fi_exVewHqWSyCLwru5tIa2rLZeXbMJc-UutLTrWG32_ENjhR7mlLeKQSct7UM3NzwdaaKa0HTK3_rmti2NFNzlMgTrjEgBp3hbh4B9Xz-H9GM1a4n0vNW4HBC814",
|
|
||||||
"domain": ".google.com",
|
|
||||||
"path": "/",
|
|
||||||
"expires": 1780755621.359423,
|
|
||||||
"httpOnly": true,
|
|
||||||
"secure": true,
|
|
||||||
"sameSite": "None"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"origins": [
|
|
||||||
{
|
|
||||||
"origin": "https://li.protechts.net",
|
|
||||||
"localStorage": [
|
|
||||||
{
|
|
||||||
"name": "PXdOjV695v_px-ff",
|
|
||||||
"value": "eyJjYyI6eyJ0dGwiOjE3NjQ5NDQ0NzgsInZhbCI6IlUyRnRaVk5wZEdVOVRtOXVaVHNnVTJWamRYSmxPeUJRWVhKMGFYUnBiMjVsWkRzZyJ9LCJ1aWk0Ijp7InR0bCI6MTc2NDk0NDQ3OCwidmFsIjoiMSJ9fQ=="
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "PXdOjV695v_px_hvd",
|
|
||||||
"value": "6c37c585348c7200743dd5dcaabffe0d"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
@ -1 +0,0 @@
|
|||||||
{"success_rate": 1.0, "captcha_count": 0, "cloudflare_count": 0, "avg_response_time": 10.0, "failed_domains": {}}
|
|
||||||
@ -1 +0,0 @@
|
|||||||
{"success_rate": 0.9, "captcha_count": 0, "cloudflare_count": 0, "avg_response_time": 10.0, "failed_domains": {}}
|
|
||||||
@ -1,4 +1,3 @@
|
|||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import random
|
import random
|
||||||
from typing import Optional, Dict
|
from typing import Optional, Dict
|
||||||
@ -8,8 +7,6 @@ from llm_agent import LLMJobRefiner
|
|||||||
import re
|
import re
|
||||||
from fetcher import StealthyFetcher
|
from fetcher import StealthyFetcher
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
import json
|
|
||||||
import redis
|
|
||||||
|
|
||||||
|
|
||||||
class LinkedInJobScraper:
|
class LinkedInJobScraper:
|
||||||
@ -26,8 +23,6 @@ class LinkedInJobScraper:
|
|||||||
self.user_request = user_request
|
self.user_request = user_request
|
||||||
self._init_db()
|
self._init_db()
|
||||||
self.llm_agent = LLMJobRefiner()
|
self.llm_agent = LLMJobRefiner()
|
||||||
# Initialize Redis connection
|
|
||||||
self.redis_client = redis.Redis(host='localhost', port=6379, db=0, decode_responses=True)
|
|
||||||
|
|
||||||
def _init_db(self):
|
def _init_db(self):
|
||||||
# This method is kept for backward compatibility but LLMJobRefiner handles PostgreSQL now
|
# This method is kept for backward compatibility but LLMJobRefiner handles PostgreSQL now
|
||||||
@ -194,82 +189,6 @@ class LinkedInJobScraper:
|
|||||||
print("🔚 No new jobs loaded. Stopping scroll.")
|
print("🔚 No new jobs loaded. Stopping scroll.")
|
||||||
break
|
break
|
||||||
|
|
||||||
async def _extract_job_posted_date(self, page) -> str:
|
|
||||||
"""
|
|
||||||
Extract the job posted date from LinkedIn job page
|
|
||||||
Returns date in MM/DD/YY format
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
# Try multiple selectors for the posted date
|
|
||||||
selectors = [
|
|
||||||
"span[class*='posted-date']",
|
|
||||||
"span:has-text('ago')",
|
|
||||||
"span:has-text('Posted')",
|
|
||||||
"span.job-details-jobs-unified-top-card__job-insight-view-model-secondary"
|
|
||||||
]
|
|
||||||
|
|
||||||
for selector in selectors:
|
|
||||||
date_element = await page.query_selector(selector)
|
|
||||||
if date_element:
|
|
||||||
date_text = await date_element.inner_text()
|
|
||||||
if date_text:
|
|
||||||
# Clean the text
|
|
||||||
date_text = date_text.strip()
|
|
||||||
|
|
||||||
# Check if it contains "ago" (e.g., "2 hours ago", "1 day ago")
|
|
||||||
if "ago" in date_text.lower():
|
|
||||||
# Use current date since it's relative
|
|
||||||
current_date = datetime.now()
|
|
||||||
return current_date.strftime("%m/%d/%y")
|
|
||||||
elif "Posted" in date_text:
|
|
||||||
# Extract date from "Posted X days ago" or similar
|
|
||||||
current_date = datetime.now()
|
|
||||||
return current_date.strftime("%m/%d/%y")
|
|
||||||
else:
|
|
||||||
# Try to parse actual date formats
|
|
||||||
# Common LinkedIn format: "Mar 15, 2025"
|
|
||||||
import re
|
|
||||||
date_match = re.search(r'([A-Za-z]+)\s+(\d{1,2}),\s+(\d{4})', date_text)
|
|
||||||
if date_match:
|
|
||||||
month_name = date_match.group(1)
|
|
||||||
day = date_match.group(2)
|
|
||||||
year = date_match.group(3)
|
|
||||||
|
|
||||||
# Convert month name to number
|
|
||||||
months = {
|
|
||||||
'Jan': '01', 'Feb': '02', 'Mar': '03', 'Apr': '04',
|
|
||||||
'May': '05', 'Jun': '06', 'Jul': '07', 'Aug': '08',
|
|
||||||
'Sep': '09', 'Oct': '10', 'Nov': '11', 'Dec': '12'
|
|
||||||
}
|
|
||||||
|
|
||||||
month_num = months.get(month_name[:3], '01')
|
|
||||||
return f"{month_num}/{day.zfill(2)}/{year[-2:]}"
|
|
||||||
|
|
||||||
# If no date found, use current date
|
|
||||||
current_date = datetime.now()
|
|
||||||
return current_date.strftime("%m/%d/%y")
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
print(f" ⚠️ Error extracting posted date: {str(e)}")
|
|
||||||
# Return current date as fallback
|
|
||||||
current_date = datetime.now()
|
|
||||||
return current_date.strftime("%m/%d/%y")
|
|
||||||
|
|
||||||
async def _add_job_to_redis_cache(self, job_url: str, job_id: str, error_type: str):
|
|
||||||
"""Add failed job to Redis cache for later retry"""
|
|
||||||
try:
|
|
||||||
job_data = {
|
|
||||||
"job_url": job_url,
|
|
||||||
"job_id": job_id,
|
|
||||||
"error_type": error_type,
|
|
||||||
"timestamp": datetime.now().isoformat()
|
|
||||||
}
|
|
||||||
# Use job_id as the key to avoid duplicates
|
|
||||||
self.redis_client.hset("failed_jobs", job_id, json.dumps(job_data))
|
|
||||||
print(f" 📦 Added failed job to Redis cache: {job_id} (Error: {error_type})")
|
|
||||||
except Exception as e:
|
|
||||||
print(f" ❌ Failed to add job to Redis cache: {str(e)}")
|
|
||||||
|
|
||||||
async def scrape_jobs(
|
async def scrape_jobs(
|
||||||
self,
|
self,
|
||||||
search_keywords: Optional[str],
|
search_keywords: Optional[str],
|
||||||
@ -389,7 +308,7 @@ class LinkedInJobScraper:
|
|||||||
print(f" ➕ Found {initial_jobs} initial job(s) (total: {len(all_job_links)})")
|
print(f" ➕ Found {initial_jobs} initial job(s) (total: {len(all_job_links)})")
|
||||||
|
|
||||||
iteration = 1
|
iteration = 1
|
||||||
while iteration <= 5: # Fixed the condition - was "iteration >= 5" which never runs
|
while True and iteration >= 5:
|
||||||
print(f"🔄 Iteration {iteration}: Checking for new jobs...")
|
print(f"🔄 Iteration {iteration}: Checking for new jobs...")
|
||||||
|
|
||||||
prev_job_count = len(all_job_links)
|
prev_job_count = len(all_job_links)
|
||||||
@ -434,14 +353,9 @@ class LinkedInJobScraper:
|
|||||||
job_page = await fetcher.fetch_url(full_url, wait_for_selector="h1.t-24")
|
job_page = await fetcher.fetch_url(full_url, wait_for_selector="h1.t-24")
|
||||||
if not job_page:
|
if not job_page:
|
||||||
print(f" ❌ Failed to fetch job page {full_url} after retries.")
|
print(f" ❌ Failed to fetch job page {full_url} after retries.")
|
||||||
await self._add_job_to_redis_cache(full_url, full_url.split("/")[-2] if "/jobs/view/" in full_url else "unknown", "fetch_failure")
|
|
||||||
self.engine.report_outcome("fetch_failure", url=full_url)
|
self.engine.report_outcome("fetch_failure", url=full_url)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Extract posted date from the job page
|
|
||||||
posted_date = await self._extract_job_posted_date(job_page)
|
|
||||||
print(f" 📅 Posted date extracted: {posted_date}")
|
|
||||||
|
|
||||||
apply_btn = None
|
apply_btn = None
|
||||||
apply_selectors = [
|
apply_selectors = [
|
||||||
"button[aria-label*='Apply']",
|
"button[aria-label*='Apply']",
|
||||||
@ -503,8 +417,7 @@ class LinkedInJobScraper:
|
|||||||
"page_content": page_content,
|
"page_content": page_content,
|
||||||
"url": final_url,
|
"url": final_url,
|
||||||
"job_id": job_id,
|
"job_id": job_id,
|
||||||
"search_keywords": search_keywords,
|
"search_keywords": search_keywords
|
||||||
"posted_date": posted_date # Add the posted date to raw data
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# LLM agent is now fully responsible for extraction and validation
|
# LLM agent is now fully responsible for extraction and validation
|
||||||
@ -524,24 +437,18 @@ class LinkedInJobScraper:
|
|||||||
|
|
||||||
refined_data['scraped_at'] = datetime.now().isoformat()
|
refined_data['scraped_at'] = datetime.now().isoformat()
|
||||||
refined_data['category'] = clean_keywords
|
refined_data['category'] = clean_keywords
|
||||||
refined_data['posted_date'] = posted_date # Add posted date to refined data
|
|
||||||
await self.llm_agent.save_job_data(refined_data, search_keywords)
|
await self.llm_agent.save_job_data(refined_data, search_keywords)
|
||||||
scraped_count += 1
|
scraped_count += 1
|
||||||
print(f" ✅ Scraped and refined: {refined_data['title'][:50]}...")
|
print(f" ✅ Scraped and refined: {refined_data['title'][:50]}...")
|
||||||
self.engine.report_outcome("success", url=raw_data["url"])
|
self.engine.report_outcome("success", url=raw_data["url"])
|
||||||
else:
|
else:
|
||||||
print(f" 🟡 Could not extract meaningful data from: {final_url}")
|
print(f" 🟡 Could not extract meaningful data from: {final_url}")
|
||||||
await self._add_job_to_redis_cache(final_url, job_id, "llm_failure")
|
|
||||||
self.engine.report_outcome("llm_failure", url=raw_data["url"])
|
self.engine.report_outcome("llm_failure", url=raw_data["url"])
|
||||||
|
|
||||||
await job_page.close()
|
await job_page.close()
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
error_msg = str(e)[:100]
|
print(f" ⚠️ Failed on job {idx+1}: {str(e)[:100]}")
|
||||||
print(f" ⚠️ Failed on job {idx+1}: {error_msg}")
|
|
||||||
job_id = full_url.split("/")[-2] if "/jobs/view/" in full_url else "unknown" if 'full_url' in locals() else "unknown"
|
|
||||||
job_url = full_url if 'full_url' in locals() else "unknown"
|
|
||||||
await self._add_job_to_redis_cache(job_url, job_id, f"exception: {error_msg}")
|
|
||||||
if 'job_page' in locals() and job_page:
|
if 'job_page' in locals() and job_page:
|
||||||
await job_page.close()
|
await job_page.close()
|
||||||
continue
|
continue
|
||||||
|
|||||||
@ -1,504 +0,0 @@
|
|||||||
# LinkedIn Jobs - 2025-12-05 14:04:45
|
|
||||||
|
|
||||||
## Job: Machine Learning Engineer
|
|
||||||
|
|
||||||
- **Keyword**: Machine Learning Engineer location:New York
|
|
||||||
- **Company**: The Arena
|
|
||||||
- **Location**: Lagos, Lagos State, Nigeria
|
|
||||||
- **Nature of Work**: Not specified in the provided content. Could not infer from keywords like 'remote', 'onsite', or 'hybrid'.
|
|
||||||
- **Salary Range**: Not specified in the provided content.
|
|
||||||
- **Job ID**: 4325564279
|
|
||||||
- **Category**: Machine Learning Engineer
|
|
||||||
- **Scraped At**: 2025-12-05T14:04:43.210072
|
|
||||||
- **URL**: <https://www.linkedin.com/jobs/view/4325564279/?eBP=NOT_ELIGIBLE_FOR_CHARGING&refId=KZdFT%2FkXGUGDBr1Ru66VSg%3D%3D&trackingId=O4Oht8qideqoj%2FUHqgXQKg%3D%3D&trk=flagship3_search_srp_jobs>
|
|
||||||
|
|
||||||
### Description
|
|
||||||
|
|
||||||
The job posting is for a Machine Learning Engineer position. The content appears to be from a LinkedIn job application dialog, showing contact information collection for the applicant Ofure Ikheloa. The main job details beyond the title and company are not fully visible in the provided content snippet, which focuses on the application form's contact info section.
|
|
||||||
|
|
||||||
### Requirements
|
|
||||||
|
|
||||||
Specific requirements are not detailed in the provided content snippet. The visible section is part of the application form for collecting the candidate's contact information.
|
|
||||||
|
|
||||||
### Qualifications
|
|
||||||
|
|
||||||
Specific qualifications are not detailed in the provided content snippet. The visible section is part of the application form.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Job: Junior Software Engineer (Fresh Graduates)
|
|
||||||
|
|
||||||
- **Keyword**: Machine Learning Engineer location:New York
|
|
||||||
- **Company**: Clarvos
|
|
||||||
- **Location**: Lagos, Lagos State, Nigeria
|
|
||||||
- **Nature of Work**: N/A
|
|
||||||
- **Salary Range**: N/A
|
|
||||||
- **Job ID**: 4348455050
|
|
||||||
- **Category**: Machine Learning Engineer
|
|
||||||
- **Scraped At**: 2025-12-05T14:06:23.354232
|
|
||||||
- **URL**: <https://www.linkedin.com/jobs/view/4348455050/?eBP=NOT_ELIGIBLE_FOR_CHARGING&refId=KZdFT%2FkXGUGDBr1Ru66VSg%3D%3D&trackingId=ztbzkgV%2BpdyOIs92zSWwNQ%3D%3D&trk=flagship3_search_srp_jobs>
|
|
||||||
|
|
||||||
### Description
|
|
||||||
|
|
||||||
The job posting is for a Junior Software Engineer position targeted at fresh graduates. The role appears to be with Clarvos, based in Lagos, Nigeria. The provided content shows an application form with contact information fields, indicating this is an active job application page on LinkedIn.
|
|
||||||
|
|
||||||
### Requirements
|
|
||||||
|
|
||||||
The specific requirements are not detailed in the provided content snippet. However, being a Junior Software Engineer role for fresh graduates typically requires foundational programming knowledge, problem-solving skills, and a willingness to learn.
|
|
||||||
|
|
||||||
### Qualifications
|
|
||||||
|
|
||||||
The role is explicitly for 'Fresh Graduates,' indicating that a recent bachelor's degree in Computer Science, Software Engineering, or a related field is the primary qualification. Specific educational requirements are not listed in the provided text.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Job: Machine Learning Engineer - Search
|
|
||||||
|
|
||||||
- **Keyword**: Machine Learning Engineer location:New York
|
|
||||||
- **Company**: Shopify
|
|
||||||
- **Location**: Remote - Americas
|
|
||||||
- **Nature of Work**: Remote
|
|
||||||
- **Salary Range**: N/A
|
|
||||||
- **Job ID**: unknown
|
|
||||||
- **Category**: Machine Learning Engineer
|
|
||||||
- **Scraped At**: 2025-12-05T14:37:51.551952
|
|
||||||
- **URL**: <https://www.shopify.com/careers/machine-learning-engineer-search_c15b011d-bfe1-4eae-af45-9f3955ce408d?utm_source=linkedin>
|
|
||||||
|
|
||||||
### Description
|
|
||||||
|
|
||||||
About the role Every day, millions of people search for products across Shopify's ecosystem. That's not just queries—that's dreams, businesses, and livelihoods riding on whether someone finds the perfect vintage jacket or the exact drill bit they need. As a Machine Learning Engineer specializing in Search Recommendations, you'll be the one making that magic happen. With a search index unifying over a billion products, you're tackling one of the hardest search problems at unprecedented scale. We're building cutting-edge product search from the ground up using the latest LLM advances and vector matching technologies to create search experiences that truly understand what people are looking for.
|
|
||||||
|
|
||||||
### Requirements
|
|
||||||
|
|
||||||
Key Responsibilities: Design and implement AI-powered features to enhance search recommendations and personalization Collaborate with data scientists and engineers to productionize data products through rigorous experimentation and metrics analysis Build and maintain robust, scalable data pipelines for search and recommendation systems Develop comprehensive tools for evaluation and relevance engineering, following high-quality software engineering practices Mentor engineers and data scientists while fostering a culture of innovation and technical excellence
|
|
||||||
|
|
||||||
### Qualifications
|
|
||||||
|
|
||||||
Qualifications: Expertise in relevance engineering and recommendation systems, with hands-on experience in Elasticsearch, Solr, or vector databases Strong proficiency in Python with solid object-oriented programming skills Proven ability to write optimized, low-latency code for high-performance systems Experience deploying machine learning, NLP, or generative AI products at scale (strong plus) Familiarity with statistical methods and exposure to Ruby, Rails, or Rust (advantageous) Track record of shipping ML solutions that real users depend on
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Job: Data/ML Engineer
|
|
||||||
|
|
||||||
- **Keyword**: Machine Learning Engineer location:New York
|
|
||||||
- **Company**: Sailplan
|
|
||||||
- **Location**: Fort Lauderdale
|
|
||||||
- **Nature of Work**: This position may be located remotely or from our Headquarters in Miami / Fort Lauderdale, Florida, as determined on a case by case basis. Remote candidates must be US citizens located in the United States or Canada. Remote candidates are expected to travel to office periodically as necessary.
|
|
||||||
- **Salary Range**: N/A
|
|
||||||
- **Job ID**: unknown
|
|
||||||
- **Category**: Machine Learning Engineer
|
|
||||||
- **Scraped At**: 2025-12-05T14:40:02.469176
|
|
||||||
- **URL**: <https://www.wiraa.com/job-description/usAD0C4BA9641769424C946CED3DC727D9?source=Linkedin>
|
|
||||||
|
|
||||||
### Description
|
|
||||||
|
|
||||||
SailPlan is a cutting-edge technology company that is dedicated to transforming the future of maritime transportation. SailPlan offers a range of innovative solutions and services that enable its clients to optimize their operations and reduce their environmental impact. SailPlan works with some of the most important names in the shipping industry to deliver a cleaner future for the world. SailPlan’s team comprises of experts with a diverse range of skills and experience, including naval architects, data scientists, and software engineers. The company’s collaborative and dynamic work culture fosters innovation and creativity, allowing the team to develop cutting-edge solutions that drive the industry forward. By combining state-of-the-art technology and a commitment to sustainability, SailPlan is leading the way towards a greener and more efficient maritime industry. At SailPlan, you will be part of a fast-growing team, will wear many hats and have ownership over building key aspects of our platform. You will work within a collaborative environment to build the next generation of technology for the maritime industry. If you think you have the right stuff, we are looking for YOU.
|
|
||||||
|
|
||||||
### Requirements
|
|
||||||
|
|
||||||
We are seeking an experienced Data Engineer to focus on productionalizing machine learning models within Google Cloud Platform (GCP) while collaborating on ETL design and planning. This role will be responsible for architecting, implementing, and maintaining the infrastructure needed for model training, retraining, and deployment, ensuring high-quality ML performance monitoring and external output serving. The ideal candidate has expertise in ML model operationalization, cloud architecture, and data pipeline orchestration, working closely with data scientists, cloud engineers, and analysts to bridge the gap between data engineering and MLOps, while also ensuring that model outputs are seamlessly integrated into analytics and decision-making systems.
|
|
||||||
|
|
||||||
### Qualifications
|
|
||||||
|
|
||||||
Core Requirements and Qualifications
|
|
||||||
Select and implement the appropriate GCP services for scalable ML workflows, including model training, retraining, and deployment
|
|
||||||
Develop automated monitoring and performance tracking for deployed models, surfacing quality metrics internally and ensuring external services receive high-quality outputs
|
|
||||||
Optimize model deployment pipelines to ensure efficient versioning, retraining triggers, and drift detection
|
|
||||||
Collaboration on ETL Development: Work alongside data engineers to design and optimize data pipelines that support machine learning models
|
|
||||||
Ensure seamless integration between data ingestion, transformations, and ML pipelines, leveraging BigQuery and DBT
|
|
||||||
Coordinate with sensor and instrumentation engineers to facilitate the ingestion of real-time sensor data for predictive modeling
|
|
||||||
Architect and implement CI/CD pipelines for ML models, enabling automated deployment, testing, and rollback strategies
|
|
||||||
Design cloud infrastructure that supports scalable and cost-efficient ML model training in production environments
|
|
||||||
Implement logging, alerting, and monitoring to proactively identify issues with models and data pipelines
|
|
||||||
Ensure ML model outputs are easily accessible and consumable by analytics, dashboards, and external services
|
|
||||||
Work closely with data analysts and cloud engineers to optimize Looker integrations and visualization pipelines for ML-driven insights
|
|
||||||
Maintain and document model lifecycle processes, ensuring clarity and reproducibility across the team
|
|
||||||
Required Skills
|
|
||||||
Bachelor’s or Master’s degree in Computer Science, Software Engineering, or a related field
|
|
||||||
Strong experience in MLOps and machine learning model operationalization, particularly within GCP
|
|
||||||
Proficiency in SQL and Python, with experience in data manipulation, feature engineering, and ML model deployment
|
|
||||||
Hands-on experience with CI/CD pipelines, version control (Git), and infrastructure-as-code (Terraform, Cloud Build)
|
|
||||||
Experience working with data pipelines (specifically time-series data) and collaborating with data engineers to support ML workflows
|
|
||||||
Excellent problem-solving skills and a proactive, collaborative mindset
|
|
||||||
Preferred Qualifications
|
|
||||||
Familiarity with the maritime/shipping domain, including knowledge of sensor data and operational challenges
|
|
||||||
Experience with dbt and BigQuery for efficient data transformation
|
|
||||||
Knowledge of LookML and Looker dashboards, especially for surfacing ML insights
|
|
||||||
Experience working with real-time streaming and high-fidelity time series data
|
|
||||||
Understanding of data governance, security, and compliance best practices in cloud-based environments
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Job: AI Engineer (m/f/d)
|
|
||||||
|
|
||||||
- **Keyword**: Machine Learning Engineer location:New York
|
|
||||||
- **Company**: myneva
|
|
||||||
- **Location**: Portugal, remote
|
|
||||||
- **Nature of Work**: Permanent employee, Full-time · Portugal, remote
|
|
||||||
- **Salary Range**: N/A
|
|
||||||
- **Job ID**: unknown
|
|
||||||
- **Category**: Machine Learning Engineer
|
|
||||||
- **Scraped At**: 2025-12-05T14:42:45.137786
|
|
||||||
- **URL**: <https://myneva-group.jobs.personio.de/job/2443218?language=en&src=752617&_pc=752617#apply>
|
|
||||||
|
|
||||||
### Description
|
|
||||||
|
|
||||||
Your mission We are looking for a polyglot AI Engineer, located in the mainland Portugal, who is as comfortable optimizing backend concurrency in Go as they are building RAG pipelines in Python and designing interfaces in TypeScript. In this role, you will wear two hats. First, you will build the core infrastructure of our AI Platform, ensuring high availability and low latency for model inference. Second, you will design and deploy autonomous AI Agents, for our product landscape and services. If you are a builder who believes that AI is more than just prompts, it’s about systems, integration, and architecture then we want to hear from you. 1. AI Platform Development (Go & Python): Architect and build scalable microservices in Go (Golang) to handle high-throughput requests for our AI services. Design efficient APIs (gRPC/REST) that serve as the bridge between our core application and AI models. Optimize inference latency and manage model serving infrastructure. 2. AI Agent Engineering (Python & TypeScript): Develop autonomous agents using Python frameworks (e.g., LangChain, LlamaIndex, or custom solutions) to automate internal business processes (e.g., data entry, customer support triage, financial reporting). Implement "Tool Use" and "Function Calling" to allow LLMs to interact with third-party APIs and our internal databases. Build the integration layer and user interfaces in TypeScript (Node.js/Next.js) to allow non-technical staff to interact with these agents.
|
|
||||||
|
|
||||||
### Requirements
|
|
||||||
|
|
||||||
Your profile Polyglot Proficiency: You have production-level experience in Go, Python, and TypeScript (at least 2 of the 3 languages). You know which tool to use for which job. AI/LLM Experience: You have built applications utilizing OpenAI API, Anthropic, or open-source models (Llama 3, Mistral). You understand context windows, token limits, and prompt engineering. Systems Thinking: You understand distributed systems, concurrency, and how to deploy AI in a way that doesn't break production. Agentic Workflows: Experience building multi-step reasoning agents (e.g., "Plan-and-Execute" patterns). Database Skills: Proficiency with SQL (Postgres) and Vector Stores.
|
|
||||||
|
|
||||||
### Qualifications
|
|
||||||
|
|
||||||
Nice to Haves (Bonus Points) Experience fine-tuning open-source models on custom datasets. Knowledge of temporal/orchestration frameworks for managing long-running agent workflows. Experience with container orchestration (Kubernetes/Docker). Background in DevOps or MLOps (MLflow, weights & biases).
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Job: ML/AI Engineer
|
|
||||||
|
|
||||||
- **Keyword**: Machine Learning Engineer location:New York
|
|
||||||
- **Company**: Deltek, Inc
|
|
||||||
- **Location**: US Remote
|
|
||||||
- **Nature of Work**: Remote
|
|
||||||
- **Salary Range**: The U.S. salary range for this position is $57000.00 - $99750.00.
|
|
||||||
- **Job ID**: unknown
|
|
||||||
- **Category**: Machine Learning Engineer
|
|
||||||
- **Scraped At**: 2025-12-05T14:46:54.116677
|
|
||||||
- **URL**: <https://sjobs.brassring.com/TGnewUI/Search/home/HomeWithPreLoad?PageType=JobDetails&partnerid=25397&siteid=5259&jobId=621951#jobDetails=621951_5259>
|
|
||||||
|
|
||||||
### Description
|
|
||||||
|
|
||||||
Position Responsibilities Develop and deploy machine learning models for classification, regression, forecasting, and NLP tasks using production-grade code and best practices Build data pipelines for ML model training and inference; work with structured and unstructured data from multiple enterprise systems Implement model training workflows including data preprocessing, feature engineering, hyperparameter tuning, and model evaluation Create production-ready ML services with RESTful APIs that can be consumed by web and mobile applications; ensure proper error handling, logging, and monitoring Work with large-scale datasets from enterprise ERP systems; process time-series data, transactional data, and unstructured documents Collaborate with data scientists to productionize research models; optimize models for latency, throughput, and cost Participate in code reviews and contribute to team's ML engineering practices; document solutions and share knowledge with team members Support deployed models including troubleshooting, performance optimization, and implementing improvements based on production metrics
|
|
||||||
|
|
||||||
### Requirements
|
|
||||||
|
|
||||||
Qualifications 2-4 years of ML engineering experience with hands-on model development and production deployment Strong Python programming : Experience with scikit-learn, pandas, numpy; familiarity with PyTorch or TensorFlow ML fundamentals : Solid understanding of supervised/unsupervised learning, model evaluation, cross-validation, and feature engineering API development : Experience building RESTful APIs (Flask, FastAPI, or similar); understanding of microservices architecture Data processing : SQL proficiency; experience with data pipelines, ETL processes, and working with databases (PostgreSQL, MySQL, or similar) Cloud platforms : Working knowledge of AWS, Azure, or GCP; experience with cloud storage, compute, and managed ML services Version control and collaboration : Git workflows, agile methodologies, working in cross-functional teams Bonus : Exposure to NLP techniques, LLMs, embedding models, or vector databases; experience in B2B SaaS environments Education : BS in Computer Science, Data Science, Mathematics, or related technical field
|
|
||||||
|
|
||||||
### Qualifications
|
|
||||||
|
|
||||||
N/A
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Job: Machine Learning Engineer
|
|
||||||
|
|
||||||
- **Keyword**: Machine Learning Engineer location:New York
|
|
||||||
- **Company**: PhysicsX
|
|
||||||
- **Location**: New York, United States
|
|
||||||
- **Nature of Work**: Hybrid setup – enjoy our Manhattan office while keeping remote flexibility.
|
|
||||||
- **Salary Range**: $120,000 - 240,000 depending on experience
|
|
||||||
- **Job ID**: unknown
|
|
||||||
- **Category**: Machine Learning Engineer
|
|
||||||
- **Scraped At**: 2025-12-05T14:49:29.919057
|
|
||||||
- **URL**: <https://job-boards.eu.greenhouse.io/physicsx/jobs/4644841101?gh_src=6d71ons2teu>
|
|
||||||
|
|
||||||
### Description
|
|
||||||
|
|
||||||
What you will do
|
|
||||||
Work closely with our simulation engineers, data scientists and customers to develop an understanding of the physics and engineering challenges we are solving
|
|
||||||
Design, build and test data pipelines for machine learning that are reliable, scalable and easily deployable
|
|
||||||
Explore and manipulate 3D point cloud & mesh data
|
|
||||||
Own the delivery of technical workstreams
|
|
||||||
Create analytics environments and resources in the cloud or on premise, spanning data engineering and science
|
|
||||||
Identify the best libraries, frameworks and tools for a given task, make product design decisions to set us up for success
|
|
||||||
Work at the intersection of data science and software engineering to translate the results of our R&D and projects into re-usable libraries, tooling and products
|
|
||||||
Continuously apply and improve engineering best practices and standards and coach your colleagues in their adoption
|
|
||||||
|
|
||||||
### Requirements
|
|
||||||
|
|
||||||
What you bring to the table
|
|
||||||
Experience applying Machine learning methods (including 3D graph/point cloud deep learning methods) to real-world engineering applications, with a focus on driving measurable impact in industry settings.
|
|
||||||
Experience in ML/Computational Statistics/Modelling use-cases in industrial settings (for example supply chain optimisation or manufacturing processes) is encouraged.
|
|
||||||
A track record of scoping and delivering projects in a customer facing role
|
|
||||||
2+ years’ experience in a data-driven role, with exposure to software engineering concepts and best practices (e.g., versioning, testing, CI/CD, API design, MLOps)
|
|
||||||
Building machine learning models and pipelines in Python, using common libraries and frameworks (e.g., TensorFlow, MLFlow)
|
|
||||||
Distributed computing frameworks (e.g., Spark, Dask)
|
|
||||||
Cloud platforms (e.g., AWS, Azure, GCP) and HP computing
|
|
||||||
Containerization and orchestration (Docker, Kubernetes)
|
|
||||||
Strong problem-solving skills and the ability to analyse issues, identify causes, and recommend solutions quickly
|
|
||||||
Excellent collaboration and communication skills - with teams and customers alike
|
|
||||||
A background in Physics, Engineering, or equivalent
|
|
||||||
|
|
||||||
### Qualifications
|
|
||||||
|
|
||||||
N/A
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Job: DevOps Engineer
|
|
||||||
|
|
||||||
- **Keyword**: DevOps Engineer location:New York
|
|
||||||
- **Company**: micro1
|
|
||||||
- **Location**: N/A
|
|
||||||
- **Nature of Work**: N/A
|
|
||||||
- **Salary Range**: N/A
|
|
||||||
- **Job ID**: 4342293908
|
|
||||||
- **Category**: DevOps Engineer
|
|
||||||
- **Scraped At**: 2025-12-05T15:46:55.076540
|
|
||||||
- **URL**: <https://www.linkedin.com/jobs/view/4342293908/?eBP=NOT_ELIGIBLE_FOR_CHARGING&refId=8EZEulwt3rTN7TqojhbDIQ%3D%3D&trackingId=8OTJNgCFiTsOg9ssPeenJQ%3D%3D&trk=flagship3_search_srp_jobs>
|
|
||||||
|
|
||||||
### Description
|
|
||||||
|
|
||||||
N/A
|
|
||||||
|
|
||||||
### Requirements
|
|
||||||
|
|
||||||
N/A
|
|
||||||
|
|
||||||
### Qualifications
|
|
||||||
|
|
||||||
N/A
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Job: DevOps Engineer
|
|
||||||
|
|
||||||
- **Keyword**: DevOps Engineer location:New York
|
|
||||||
- **Company**: Utah Tech Labs
|
|
||||||
- **Location**: North America + 1 more
|
|
||||||
- **Nature of Work**: Freelance
|
|
||||||
- **Salary Range**: $16 – $20/hr
|
|
||||||
- **Job ID**: 4325775293
|
|
||||||
- **Category**: DevOps Engineer
|
|
||||||
- **Scraped At**: 2025-12-05T15:51:10.659992
|
|
||||||
- **URL**: <https://app.usebraintrust.com/jobs/16561/?gh_src=06bf3def4us&utm_channel=jobboard&utm_source=linkedin>
|
|
||||||
|
|
||||||
### Description
|
|
||||||
|
|
||||||
We are looking for a skilled DevOps Engineer to help build the foundational infrastructure, CI/CD pipelines, and operational standards for our engineering organization. This role is hands-on and ideal for someone experienced in supporting large-scale applications across modern frontend, backend, and cloud technologies.
|
|
||||||
|
|
||||||
Responsibilities
|
|
||||||
Build and maintain AWS infrastructure using Terraform (IaC).
|
|
||||||
Set up CI/CD pipelines for Angular, React, Flutter, Python (FastAPI/Django), and Node.js (Express/TypeScript) .
|
|
||||||
Implement automated unit, integration, and regression testing pipelines.
|
|
||||||
Establish logging, monitoring, and alerting (CloudWatch, ELK/OpenSearch, Datadog, etc.).
|
|
||||||
Define and enforce data security , IAM, secrets management, and encryption best practices.
|
|
||||||
Optimize SQL/NoSQL databases, performance tuning, backups, and restore workflows.
|
|
||||||
Create SOPs, reusable infrastructure templates, and DevOps best-practice documentation.
|
|
||||||
|
|
||||||
### Requirements
|
|
||||||
|
|
||||||
2–3+ years DevOps experience , including work on larger, production-grade projects.
|
|
||||||
Strong experience with AWS (EC2, ECS/EKS, RDS, S3, CloudFront, IAM, VPC).
|
|
||||||
Strong proficiency with Terraform and infrastructure-as-code workflows.
|
|
||||||
Hands-on experience building CI/CD pipelines for modern frontend and backend frameworks.
|
|
||||||
Strong understanding of data security , network configuration, and secure deployment.
|
|
||||||
Familiarity with application logging, monitoring, and distributed tracing.
|
|
||||||
Experience optimizing relational and non-relational databases.
|
|
||||||
Deliverables include SOPs, infrastructure templates, CI/CD pipelines, and automated testing frameworks.
|
|
||||||
|
|
||||||
### Qualifications
|
|
||||||
|
|
||||||
N/A
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Job: DevOps Engineer
|
|
||||||
|
|
||||||
- **Keyword**: DevOps Engineer location:New York
|
|
||||||
- **Company**: Core4ce
|
|
||||||
- **Location**: Remote (Worldwide)
|
|
||||||
- **Nature of Work**: Remote
|
|
||||||
- **Salary Range**: N/A
|
|
||||||
- **Job ID**: 4342211043
|
|
||||||
- **Category**: DevOps Engineer
|
|
||||||
- **Scraped At**: 2025-12-05T15:52:59.449387
|
|
||||||
- **URL**: <https://jobs.silkroad.com/Core4ce/Careers/jobs/1026?src=LinkedIn>
|
|
||||||
|
|
||||||
### Description
|
|
||||||
|
|
||||||
As a DevOps Engineer, you will play a pivotal role in designing, implementing, and maintaining the infrastructure and tools necessary to support continuous integration, continuous deployment, and automated operations. You will collaborate with cross-functional teams to streamline development processes, improve system reliability, and enhance overall productivity.
|
|
||||||
|
|
||||||
### Requirements
|
|
||||||
|
|
||||||
Responsibilities: Design and implement scalable, reliable, and secure DevOps solutions that meet the needs of the organization. Lead the development and implementation of CI/CD pipelines to automate software delivery processes. Architect and manage cloud infrastructure, ensuring optimal performance, cost efficiency, and scalability. Collaborate with development, operations, and quality assurance teams to integrate automated testing and monitoring into the CI/CD pipeline. Establish and enforce DevOps best practices, standards, and guidelines across the organization. Identify opportunities for process improvement and efficiency gains within the software development lifecycle. Provide technical guidance and mentorship to junior team members on DevOps tools, practices, and methodologies. Conduct research and evaluation of new tools, technologies, and methodologies to improve DevOps processes. Troubleshoot and resolve infrastructure and deployment issues in production and non-production environments. Ensure compliance with security, privacy, and regulatory requirements in all DevOps activities.
|
|
||||||
|
|
||||||
### Qualifications
|
|
||||||
|
|
||||||
Qualifications: 5+ years of experience in DevSecOps, specifically within DoD environments. Ability to obtain and maintain a Secret clearance. Ability to obtain and maintain a DoD 8570 IAT Level II certification. Proficient in Kubernetes, data pipeline management, and containerization technologies. Knowledge of continuous integration and continuous deployment (CI/CD) tools like Jenkins, GitLab CI, JIRA, or CircleCI. Familiarity with security scanning tools and vulnerability assessment tools (e.g., SonarQube, Fortify, Nessus). Experience with AWS cloud platforms and their native DevSecOps tools. Ability to analyze and improve existing DevSecOps processes for efficiency and security. Familiarity with regulatory compliance standards relevant to the DoD (e.g., NIST, FedRAMP). Proactive approach to identifying and mitigating security risks in the software development lifecycle. Certifications relevant to DevSecOps such as Certified Kubernetes Administrator (CKA), AWS Certified DevOps Engineer, or similar credentials Preferred Qualifications: Bachelor's degree in computer science, Cybersecurity, Information Technology, or related field, with an emphasis on security. Experience with Query Development and Optimization Perform defect fixes and minor feature extensions (adding new data elements to reports, adding or modifying current reporting filters, moving, adding or changing how data is displayed in a profile page for an individual service member, integrating new data sources into the database) Experience with Query Development and Optimization Code and update SSIS packages to support data extraction On-going enhancements to support policy changes such as updates to existing business logic that computes Platform, Administrative,
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Job: DevOps Engineer
|
|
||||||
|
|
||||||
- **Keyword**: DevOps Engineer location:New York
|
|
||||||
- **Company**: Verana Health
|
|
||||||
- **Location**: N/A
|
|
||||||
- **Nature of Work**: Remote
|
|
||||||
- **Salary Range**: National Pay Range $148,000 - $175,000 USD
|
|
||||||
- **Job ID**: 4256494004
|
|
||||||
- **Category**: DevOps Engineer
|
|
||||||
- **Scraped At**: 2025-12-05T15:55:17.680202
|
|
||||||
- **URL**: <https://job-boards.greenhouse.io/veranahealth/jobs/8054444002>
|
|
||||||
|
|
||||||
### Description
|
|
||||||
|
|
||||||
Accelerate Real-World Insights Through Cloud Innovation Verana Health is a digital health company that harnesses exclusive real-world data to deliver quality insights that accelerate drug development and improve patient care. Backed by leading investors such as Johnson & Johnson Innovation – JJDC, Inc., Novo Growth, GV (formerly Google Ventures), and more, we are transforming how medical research is conducted. Our mission-driven team is committed to making a tangible difference in patients' lives through technology and data science. You will report to the Director of Engineering and work with engineering, data science, and IT teams. Your contributions will ensure the seamless operation of our cloud infrastructure, enabling faster, safer, and more reliable delivery of our data-driven solutions. This role is critical to Verana Health's ability to innovate and scale its impact on patient care. This is a remote position. Why This Role Matters As a DevOps Engineer at Verana Health, you will help ensure the reliability, security, and scalability of our cloud infrastructure. Your work will directly help deliver critical data analytics and research tools used by healthcare professionals and researchers worldwide. What You Get to Do Architect, deploy, and maintain cloud-based infrastructure using AWS, with a focus on automation, security, and scalability. Develop and optimize CI/CD pipelines to accelerate software delivery and improve operational efficiency. Collaborate with cross-functional teams (engineering, data science, QA) to support their DevOps needs and drive continuous improvement. Implement and enforce best practices for authorization, authentication, and compliance across AWS services. Monitor system performance, troubleshoot issues, and ensure high availability of critical applications and databases. Document and refine DevOps processes to foster knowledge sharing and operational excellence. Support database management, server administration (Linux/Windows), and infrastructure orchestration using tools like Docker, Kubernetes, and Terraform. Contribute to a culture of innovation, learning, and growth within the technology team.
|
|
||||||
|
|
||||||
### Requirements
|
|
||||||
|
|
||||||
Skills and Experience that Will Help You Succeed Essential Requirements: Bachelor's degree in computer science, software engineering, or a related scientific discipline. 5+ years of professional experience in DevOps, cloud engineering, or software development. Expertise in AWS services, including IAM, VPC, EC2, S3, and cloud security best practices. Hands-on experience with CI/CD tools, containerization (Docker, Kubernetes), and infrastructure as code (Terraform, CloudFormation). Proficiency in scripting (Bash, Python) and version control (GitLab, GitHub). Experience with Linux and Windows server administration, database management, and Databricks. Desirable Skills: Exposure to healthcare or clinical research environments. Experience mentoring or guiding junior team members. Continuous learning and process improvement. Must-Haves for the Role Expertise in AWS cloud infrastructure and security. Experience with CI/CD, containerization, and infrastructure as code. Strong scripting and automation skills.
|
|
||||||
|
|
||||||
### Qualifications
|
|
||||||
|
|
||||||
N/A
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Job: DevOps Engineer
|
|
||||||
|
|
||||||
- **Keyword**: DevOps Engineer location:New York
|
|
||||||
- **Company**: Machinify
|
|
||||||
- **Location**: California Office - Roseville, CA
|
|
||||||
- **Nature of Work**: Remote - Remote
|
|
||||||
- **Salary Range**: $150,000 - $180,000 USD
|
|
||||||
- **Job ID**: 4325826034
|
|
||||||
- **Category**: DevOps Engineer
|
|
||||||
- **Scraped At**: 2025-12-05T16:01:49.663716
|
|
||||||
- **URL**: <https://www.remotehunter.com/apply-with-ai/47671ed0-5c8f-4f9e-9958-ceef96a2bb13?utm_medium=job_posting&utm_source=linkedin&utm_campaign=devops_engineer_remote&utm_category=devops_engineer&utm_term=dev_ops_engineer>
|
|
||||||
|
|
||||||
### Description
|
|
||||||
|
|
||||||
Who We Are Machinify is a leading healthcare intelligence company with expertise across the payment continuum, delivering unmatched value, transparency, and efficiency to health plan clients across the country. Deployed by over 60 health plans, including many of the top 20, and representing more than 160 million lives, Machinify brings together a fully configurable and content-rich, AI-powered platform along with best-in-class expertise. We’re constantly reimagining what’s possible in our industry, creating disruptively simple, powerfully clear ways to maximize financial outcomes and drive down healthcare costs. Location: This role is fully remote About the Opportunity Machinify is seeking a DevOps Engineer to create, maintain and automate AI/ML cloud technologies with a focus on the continued migration from VMs into Kubernetes for all applicable technology solutions supporting the Machinify Cloud and to do so with an eye to the best future implementation while solving the problems of today. We do everything at big data scale, high uptime and with an eye to incredible customer experience. Machinify’s healthcare customers are at the center of everything we do, so we employ innovative thinkers to solve issues our customers don't even have yet and do so with operational excellence. Those innovative thinkers, our people, are the core of what makes Machinify differentiated in Healthcare. Everyone has a voice. Our teams are diverse and thrive on trust. We are humans who understand our customer and work collaboratively to deliver value and make a difference. The DevOps Engineer will provide technical leadership in significant technical, automation, programming, system administration, operational, and software configuration management through partnering with DevOps and Engineering as a whole. This vital team member will have responsibility for design, engineering, development and integration within production and pre-production environments, and will also be responsible for the programs configuration management, including the planning, design, engineering, implementation and execution of successful build and deployment of code updates to each upstream environment along with planning and implementing the configuration management of all underlying technologies. What you’ll do: Facilitate the movement of VM technologies into Kubernetes through migration or replacement. Automate everything. Nothing should require manual intervention. We routinely redeploy from the ground up to ensure automation is up to date Architect solutions to achieve a high level of performance, reliability, scalability, and security Create, maintain and troubleshoot distributed compute AI/ML technologies running in the Cloud Collaborate with a great team and learn from each other Change the way healthcare companies manage their business Be challenged when faced with solving complex problems Bring a passion for improving the lives of others by making their jobs easier and more productive Be responsible and accountable for everything you build and support Communicate effectively with other engineers in the same team, with other teams and with various other stakeholders such as product managers Operate in an Agile development environment
|
|
||||||
|
|
||||||
### Requirements
|
|
||||||
|
|
||||||
N/A
|
|
||||||
|
|
||||||
### Qualifications
|
|
||||||
|
|
||||||
Experience in the migration of VM Technologies into a Kubernetes Environment 5+ years of production support preferably in a Cloud Environment (AWS, Azure or GCP) Degree in Computer Science or equivalent work experience Extremely logical with the ability to solve problems in creative and efficient ways Knowledge / Experience in the following areas Containerization with Kubernetes Scripting (python, shell etc) Crossplane / Terraform (Infrastructure as code) Linux (CentOS/RHEL) Spark / Machine Learning running in the Cloud Frameworks for distributed machine-learning / AI, such as Azure OpenAI, AWS Bedrock or things like Tensorflow and MxNet. Good understanding of Operations security practices Working in / creating compliant environments such as Hi-Trust / SOC2 etc Continuous Integration/Continuous Deployment frameworks. Citus (Distributed Postgres a plus) You are scrappy, fast, adaptable and ambitious Critical thinking and problem solving skills
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Job: DevOps Engineer
|
|
||||||
|
|
||||||
- **Keyword**: DevOps Engineer location:New York
|
|
||||||
- **Company**: NV5
|
|
||||||
- **Location**: N/A
|
|
||||||
- **Nature of Work**: N/A
|
|
||||||
- **Salary Range**: N/A
|
|
||||||
- **Job ID**: 4325301423
|
|
||||||
- **Category**: DevOps Engineer
|
|
||||||
- **Scraped At**: 2025-12-05T16:03:56.440663
|
|
||||||
- **URL**: <https://careers-nv5.icims.com/jobs/11535/devops-engineer/login?mobile=false&width=1369&height=500&bga=true&needsRedirect=false&jan1offset=60&jun1offset=60>
|
|
||||||
|
|
||||||
### Description
|
|
||||||
|
|
||||||
N/A
|
|
||||||
|
|
||||||
### Requirements
|
|
||||||
|
|
||||||
N/A
|
|
||||||
|
|
||||||
### Qualifications
|
|
||||||
|
|
||||||
N/A
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Job: Data Scientist (Remote - US)
|
|
||||||
|
|
||||||
- **Keyword**: Data Scientist location:New York
|
|
||||||
- **Company**: Jobgether
|
|
||||||
- **Location**: US
|
|
||||||
- **Nature of Work**: Remote
|
|
||||||
- **Salary Range**: N/A
|
|
||||||
- **Job ID**: 4342169293
|
|
||||||
- **Category**: Data Scientist
|
|
||||||
- **Scraped At**: 2025-12-05T16:48:07.597147
|
|
||||||
- **URL**: <https://jobs.lever.co/jobgether/1e254d1b-5d4f-4060-8eb7-705a9ae77646/apply?source=LinkedIn>
|
|
||||||
|
|
||||||
### Description
|
|
||||||
|
|
||||||
N/A
|
|
||||||
|
|
||||||
### Requirements
|
|
||||||
|
|
||||||
N/A
|
|
||||||
|
|
||||||
### Qualifications
|
|
||||||
|
|
||||||
N/A
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Job: Data Scientist (Kaggle-Grandmaster)
|
|
||||||
|
|
||||||
- **Keyword**: Data Scientist location:New York
|
|
||||||
- **Company**: Mercor
|
|
||||||
- **Location**: N/A
|
|
||||||
- **Nature of Work**: Remote
|
|
||||||
- **Salary Range**: $56-$77 / hr
|
|
||||||
- **Job ID**: 4342199585
|
|
||||||
- **Category**: Data Scientist
|
|
||||||
- **Scraped At**: 2025-12-05T16:50:50.900310
|
|
||||||
- **URL**: <https://work.mercor.com/jobs/list_AAABmuPnQVAFcCPPhAJMHJKY/data-scientist-kaggle-grandmaster?referralCode=d12bb6d7-56b2-4c5d-b2aa-751065941704&utm_source=referral&utm_medium=share&utm_campaign=job_referral>
|
|
||||||
|
|
||||||
### Description
|
|
||||||
|
|
||||||
Role Description Mercor is hiring on behalf of a leading AI research lab to bring on a highly skilled Data Scientist with a Kaggle Grandmaster profile. In this role, you will transform complex datasets into actionable insights, high-performing models, and scalable analytical workflows. You will work closely with researchers and engineers to design rigorous experiments, build advanced statistical and ML models, and develop data-driven frameworks to support product and research decisions. What You’ll Do Analyze large, complex datasets to uncover patterns, develop insights, and inform modeling direction Build predictive models, statistical analyses, and machine learning pipelines across tabular, time-series, NLP, or multimodal data Design and implement robust validation strategies, experiment frameworks, and analytical methodologies Develop automated data workflows, feature pipelines, and reproducible research environments Conduct exploratory data analysis (EDA), hypothesis testing, and model-driven investigations to support research and product teams Translate modeling outcomes into clear recommendations for engineering, product, and leadership teams Collaborate with ML engineers to productionize models and ensure data workflows operate reliably at scale Present findings through well-structured dashboards, reports, and documentation
|
|
||||||
|
|
||||||
### Requirements
|
|
||||||
|
|
||||||
N/A
|
|
||||||
|
|
||||||
### Qualifications
|
|
||||||
|
|
||||||
Qualifications Kaggle Competitions Grandmaster or comparable achievement: top-tier rankings, multiple medals, or exceptional competition performance 3–5+ years of experience in data science or applied analytics Strong proficiency in Python and data tools (Pandas, NumPy, Polars, scikit-learn, etc.) Experience building ML models end-to-end: feature engineering, training, evaluation, and deployment Solid understanding of statistical methods, experiment design, and causal or quasi-experimental analysis Familiarity with modern data stacks: SQL, distributed datasets, dashboards, and experiment tracking tools Excellent communication skills with the ability to clearly present analytical insights Nice to Have Strong contributions across multiple Kaggle tracks (Notebooks, Datasets, Discussions, Code) Experience in an AI lab, fintech, product analytics, or ML-focused organization Knowledge of LLMs, embeddings, and modern ML techniques for text, images, and multimodal data Experience working with big data ecosystems (Spark, Ray, Snowflake, BigQuery, etc.) Familiarity with statistical modeling frameworks such as Bayesian methods or probabilistic programming
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Job: Data Engineer
|
|
||||||
|
|
||||||
- **Keyword**: Data Scientist location:New York
|
|
||||||
- **Company**: Tithe.ly
|
|
||||||
- **Location**: Lagos, Lagos State, Nigeria
|
|
||||||
- **Nature of Work**: N/A
|
|
||||||
- **Salary Range**: N/A
|
|
||||||
- **Job ID**: 4339262912
|
|
||||||
- **Category**: Data Scientist
|
|
||||||
- **Scraped At**: 2025-12-05T16:53:19.169174
|
|
||||||
- **URL**: <https://www.linkedin.com/jobs/view/4339262912/?eBP=NOT_ELIGIBLE_FOR_CHARGING&refId=90LYJbgaP%2FlZrP8CD4Vdzg%3D%3D&trackingId=rDpj810DVM4VmjwzqTCxcQ%3D%3D&trk=flagship3_search_srp_jobs>
|
|
||||||
|
|
||||||
### Description
|
|
||||||
|
|
||||||
N/A
|
|
||||||
|
|
||||||
### Requirements
|
|
||||||
|
|
||||||
N/A
|
|
||||||
|
|
||||||
### Qualifications
|
|
||||||
|
|
||||||
N/A
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Job: Data Engineering Scientist
|
|
||||||
|
|
||||||
- **Keyword**: Data Scientist location:New York
|
|
||||||
- **Company**: Birdy Grey
|
|
||||||
- **Location**: United States
|
|
||||||
- **Nature of Work**: Remote
|
|
||||||
- **Salary Range**: N/A
|
|
||||||
- **Job ID**: 4322321659
|
|
||||||
- **Category**: Data Scientist
|
|
||||||
- **Scraped At**: 2025-12-05T16:57:17.957749
|
|
||||||
- **URL**: <https://job-boards.greenhouse.io/birdygrey/jobs/4970284007?gh_src=daef57357us>
|
|
||||||
|
|
||||||
### Description
|
|
||||||
|
|
||||||
THE COMPANY: BIRDY GREY Birdy Grey is a direct-to-consumer brand whose mission is to celebrate friendships during one of the most important milestones in a person’s life: their wedding. Founded in 2017 by best friends Grace Lee (Founder & Chief Creative Officer) and Monica Ashauer (Co-Founder & Chief Strategy Officer), Birdy Grey offers affordable bridesmaid dresses starting at just $89, groomsmen suits starting at $199, plus fun gifts and accessories for everyone in the wedding party. Since day one, we've dressed over 2 million bridesmaids and we're proud to be a trusted resource for brides and grooms on their most cherished day. POSITION: Data Engineering Scientist REPORTS TO: Director, Data & Analytics LOCATION: Remote Headquartered in Los Angeles, CA with an office in New York, NY, Birdy Grey supports remote work for eligible roles. We ask that all employees travel to either office once a quarter. This role is not eligible for visa sponsorship. #LI-Remote We're looking for a Data Engineering Scientist who thrives on variety. This isn't a role where you'll specialize in one narrow area, you'll be building pipelines, analyzing data, creating dashboards, and developing models. If you are energized by wearing multiple hats, statistically rigorous, hungry to learn new things, and eager to observe the direct impact of your work, this role is for you. SCOPE OF RESPONSIBILITIES Data Science (30-40%) Lead the application of statistical and machine learning methodologies (using Python and relevant frameworks) to solve core business problems, focusing on predictive and prescriptive outcomes Co-design rigorous A/B and multivariate experiments, ensuring statistical validity to accurately measure the impact of product and business changes Identify opportunities where machine learning or advanced analytics can add value Prototype data-driven solutions to business problems Data Engineering (20-30%) Architect and Deploy robust data pipelines to collect, transform, and load data from various sources Design and optimize data storage solutions and database schemas Ensure data quality, reliability, and accessibility across the organization Automate repetitive data processes and workflows Establish and enforce data governance and security standards within our Cloud environment Data Analytics (30-40%) Serve as the embedded strategic data partner for key business teams, translating complex challenges into measurable analytical projects Design and manage high-impact, self-service business intelligence assets (primarily in Looker) that accelerate organizational decision velocity Conduct ad-hoc analyses to uncover insights and opportunities Translate complex findings into clear, actionable recommendations
|
|
||||||
|
|
||||||
### Requirements
|
|
||||||
|
|
||||||
THE RIGHT CANDIDATE: QUALIFICATIONS & PERSONAL ATTRIBUTES EDUCATION: Bachelor’s Degree Required EXPERIENCE / REQUIREMENTS: 5+ years of hands-on experience in a data science, data engineering, data analyst, analytics engineering, or ML role Expert SQL skills. Must be adept at designing, optimizing, and tuning complex queries, stored procedures, and scripts, specifically utilizing analytic window functions, CTEs, and advanced join techniques Expertise with Python (preferably) or R for data analysis and automation Expertise with at least one data visualization tool, preferably Looker Experience with Cloud data platforms, DevOps, MLOps (AWS, GCP, Azure) Knowledge of data orchestration tools (Airflow, Prefect, dbt, Dagster, etc.) Experience with ML frameworks (scikit-learn, TensorFlow, PyTorch) Experience with version control (Git) and software engineering best practices Familiarity with marketing analytics, retail, and econometric principles Understanding of basic statistics and when to apply different analytical approaches Ability to communicate technical concepts to non-technical stakeholders Comfortable with ambiguity and figuring things out independently NICE TO HAVES: Start-up experience Interest in bridal and fashion Experience with ticketing systems and change management processes Interest in increasing productivity via Automation & AI Start-up or D2C/e-commerce experience
|
|
||||||
|
|
||||||
### Qualifications
|
|
||||||
|
|
||||||
N/A
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
85
llm_agent.py
85
llm_agent.py
@ -75,7 +75,6 @@ class LLMJobRefiner:
|
|||||||
url TEXT,
|
url TEXT,
|
||||||
category TEXT,
|
category TEXT,
|
||||||
scraped_at TIMESTAMP,
|
scraped_at TIMESTAMP,
|
||||||
posted_date TEXT,
|
|
||||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||||
)
|
)
|
||||||
''')
|
''')
|
||||||
@ -88,7 +87,6 @@ class LLMJobRefiner:
|
|||||||
|
|
||||||
cursor.execute('CREATE INDEX IF NOT EXISTS idx_job_id ON jobs(job_id)')
|
cursor.execute('CREATE INDEX IF NOT EXISTS idx_job_id ON jobs(job_id)')
|
||||||
cursor.execute('CREATE INDEX IF NOT EXISTS idx_category ON jobs(category)')
|
cursor.execute('CREATE INDEX IF NOT EXISTS idx_category ON jobs(category)')
|
||||||
cursor.execute('CREATE INDEX IF NOT EXISTS idx_posted_date ON jobs(posted_date)')
|
|
||||||
|
|
||||||
conn.commit()
|
conn.commit()
|
||||||
cursor.close()
|
cursor.close()
|
||||||
@ -139,31 +137,46 @@ class LLMJobRefiner:
|
|||||||
return ""
|
return ""
|
||||||
|
|
||||||
async def refine_job_data(self, raw_data: Dict[str, Any], target_field: str) -> Dict[str, Any]:
|
async def refine_job_data(self, raw_data: Dict[str, Any], target_field: str) -> Dict[str, Any]:
|
||||||
|
# Clean the raw HTML content for better LLM processing
|
||||||
page_content = raw_data.get('page_content', '')
|
page_content = raw_data.get('page_content', '')
|
||||||
cleaned_content = self._clean_html_for_llm(page_content)
|
cleaned_content = self._clean_html_for_llm(page_content)
|
||||||
|
|
||||||
|
# Get job_id and url from raw data
|
||||||
job_id = raw_data.get('job_id', 'unknown')
|
job_id = raw_data.get('job_id', 'unknown')
|
||||||
url = raw_data.get('url', 'N/A')
|
url = raw_data.get('url', 'N/A')
|
||||||
posted_date = raw_data.get('posted_date', datetime.now().strftime("%m/%d/%y"))
|
|
||||||
|
|
||||||
prompt = f"""
|
prompt = f"""
|
||||||
You are a job posting data extractor.
|
You are a job posting data extractor with two modes:
|
||||||
|
|
||||||
EXTRACT EXACT TEXT - DO NOT SUMMARIZE, PARAPHRASE, OR INVENT.
|
PRIMARY MODE (PREFERRED):
|
||||||
|
- Extract EXACT text as it appears on the page for all fields
|
||||||
|
- DO NOT summarize, paraphrase, or interpret
|
||||||
|
- Copy verbatim content including original wording and formatting
|
||||||
|
|
||||||
For these critical fields, follow these rules:
|
FALLBACK MODE (ONLY IF FIELD IS MISSING):
|
||||||
- description: Extract ALL job description text. If ANY job details exist (duties, responsibilities, overview), include them. Only use "Not provided" if absolutely no description exists.
|
- If a field is NOT explicitly stated anywhere in the content, you MAY infer it using clear contextual clues
|
||||||
- requirements: Extract ALL requirements text. If ANY requirements exist (skills, experience, education needed), include them. Only use "Not provided" if none exist.
|
- Inference rules:
|
||||||
- qualifications: Extract ALL qualifications text. If ANY qualifications exist, include them. Only use "Not provided" if none exist.
|
* company_name: Look for patterns like "at [Company]", "Join [Company]", "[Company] is hiring"
|
||||||
|
* nature_of_work: Look for "remote", "onsite", "hybrid", "work from home", "office-based"
|
||||||
|
* location: Extract city/state/country mentions near job title or details
|
||||||
|
* title: Use the largest/primary heading if no explicit "job title" label exists
|
||||||
|
|
||||||
REQUIRED FIELDS (must have valid values, never "N/A"):
|
REQUIRED FIELDS (must always have a value):
|
||||||
- title, company_name, job_id, url
|
- title: Exact job title or best inference
|
||||||
|
- company_name: Exact company name or best inference
|
||||||
|
- job_id: Use provided: {job_id}
|
||||||
|
- url: Use provided: {url}
|
||||||
|
|
||||||
OPTIONAL FIELDS (can be "Not provided"):
|
OPTIONAL FIELDS (use exact text or "N/A" if not present and not inferable):
|
||||||
- location, salary_range, nature_of_work
|
- location
|
||||||
|
- description
|
||||||
|
- requirements
|
||||||
|
- qualifications
|
||||||
|
- salary_range
|
||||||
|
- nature_of_work
|
||||||
|
|
||||||
Page Content:
|
Page Content:
|
||||||
{cleaned_content}
|
{cleaned_content}
|
||||||
|
|
||||||
Response format (ONLY return this JSON):
|
Response format (ONLY return this JSON):
|
||||||
{{
|
{{
|
||||||
"title": "...",
|
"title": "...",
|
||||||
@ -186,35 +199,15 @@ class LLMJobRefiner:
|
|||||||
)
|
)
|
||||||
refined_data = self._parse_llm_response(response_text)
|
refined_data = self._parse_llm_response(response_text)
|
||||||
|
|
||||||
if not refined_data:
|
# Final validation - ensure required fields are present and meaningful
|
||||||
return None
|
if refined_data:
|
||||||
|
required_fields = ['title', 'company_name', 'job_id', 'url']
|
||||||
|
for field in required_fields:
|
||||||
|
if not refined_data.get(field) or refined_data[field] in ["N/A", "", "Unknown", "Company", "Job"]:
|
||||||
|
return None # LLM failed to extract properly
|
||||||
|
|
||||||
# Validate required fields
|
return refined_data
|
||||||
required_fields = ['title', 'company_name', 'job_id', 'url']
|
return None
|
||||||
for field in required_fields:
|
|
||||||
if not refined_data.get(field) or refined_data[field].strip() in ["N/A", "", "Unknown", "Company", "Job"]:
|
|
||||||
return None
|
|
||||||
|
|
||||||
# CRITICAL: Validate content fields - check if they SHOULD exist
|
|
||||||
content_fields = ['description', 'requirements', 'qualifications']
|
|
||||||
cleaned_original = cleaned_content.lower()
|
|
||||||
|
|
||||||
# Simple heuristic: if page contains job-related keywords, content fields should NOT be "Not provided"
|
|
||||||
job_indicators = ['responsibilit', 'duties', 'require', 'qualifi', 'skill', 'experienc', 'educat', 'degree', 'bachelor', 'master']
|
|
||||||
has_job_content = any(indicator in cleaned_original for indicator in job_indicators)
|
|
||||||
|
|
||||||
if has_job_content:
|
|
||||||
for field in content_fields:
|
|
||||||
value = refined_data.get(field, "").strip()
|
|
||||||
if value in ["Not provided", "N/A", ""]:
|
|
||||||
# LLM failed to extract existing content
|
|
||||||
print(f" ⚠️ LLM returned '{value}' for {field} but job content appears present")
|
|
||||||
return None
|
|
||||||
|
|
||||||
# Add the posted_date to the refined data
|
|
||||||
refined_data['posted_date'] = posted_date
|
|
||||||
|
|
||||||
return refined_data
|
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"LLM refinement failed: {str(e)}")
|
print(f"LLM refinement failed: {str(e)}")
|
||||||
@ -251,8 +244,8 @@ class LLMJobRefiner:
|
|||||||
cursor.execute('''
|
cursor.execute('''
|
||||||
INSERT INTO jobs
|
INSERT INTO jobs
|
||||||
(title, company_name, location, description, requirements,
|
(title, company_name, location, description, requirements,
|
||||||
qualifications, salary_range, nature_of_work, job_id, url, category, scraped_at, posted_date)
|
qualifications, salary_range, nature_of_work, job_id, url, category, scraped_at)
|
||||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||||
ON CONFLICT (job_id) DO NOTHING
|
ON CONFLICT (job_id) DO NOTHING
|
||||||
''', (
|
''', (
|
||||||
job_data.get("title", "N/A"),
|
job_data.get("title", "N/A"),
|
||||||
@ -266,8 +259,7 @@ class LLMJobRefiner:
|
|||||||
job_data.get("job_id", "N/A"),
|
job_data.get("job_id", "N/A"),
|
||||||
job_data.get("url", "N/A"),
|
job_data.get("url", "N/A"),
|
||||||
job_data.get("category", "N/A"),
|
job_data.get("category", "N/A"),
|
||||||
job_data.get("scraped_at"),
|
job_data.get("scraped_at")
|
||||||
job_data.get("posted_date", "N/A")
|
|
||||||
))
|
))
|
||||||
|
|
||||||
conn.commit()
|
conn.commit()
|
||||||
@ -294,7 +286,6 @@ class LLMJobRefiner:
|
|||||||
f.write(f"- **Nature of Work**: {job_data.get('nature_of_work', 'N/A')}\n")
|
f.write(f"- **Nature of Work**: {job_data.get('nature_of_work', 'N/A')}\n")
|
||||||
f.write(f"- **Salary Range**: {job_data.get('salary_range', 'N/A')}\n")
|
f.write(f"- **Salary Range**: {job_data.get('salary_range', 'N/A')}\n")
|
||||||
f.write(f"- **Job ID**: {job_data.get('job_id', 'N/A')}\n")
|
f.write(f"- **Job ID**: {job_data.get('job_id', 'N/A')}\n")
|
||||||
f.write(f"- **Posted Date**: {job_data.get('posted_date', 'N/A')}\n")
|
|
||||||
f.write(f"- **Category**: {job_data.get('category', 'N/A')}\n")
|
f.write(f"- **Category**: {job_data.get('category', 'N/A')}\n")
|
||||||
f.write(f"- **Scraped At**: {job_data.get('scraped_at', 'N/A')}\n")
|
f.write(f"- **Scraped At**: {job_data.get('scraped_at', 'N/A')}\n")
|
||||||
f.write(f"- **URL**: <{job_data.get('url', 'N/A')}>\n\n")
|
f.write(f"- **URL**: <{job_data.get('url', 'N/A')}>\n\n")
|
||||||
|
|||||||
@ -1,26 +0,0 @@
|
|||||||
{
|
|
||||||
"renderers": {
|
|
||||||
"windows": [
|
|
||||||
"ANGLE (Intel, Intel(R) UHD Graphics 630 (0x00003E9B) Direct3D11 vs_5_0 ps_5_0, D3D11)",
|
|
||||||
"ANGLE (Intel, Intel(R) UHD Graphics (0x00009A49) Direct3D11 vs_5_0 ps_5_0, D3D11)",
|
|
||||||
"ANGLE (Intel(R) Iris(TM) Graphics 540 Direct3D11 vs_5_0 ps_5_0)",
|
|
||||||
"ANGLE (Intel, Intel(R) UHD Graphics 620 (0x00005916) Direct3D11 vs_5_0 ps_5_0, D3D11)",
|
|
||||||
"ANGLE (Intel, Intel(R) HD Graphics 530 (0x0000191B) Direct3D11 vs_5_0 ps_5_0, D3D11)",
|
|
||||||
"ANGLE (Intel, Intel(R) UHD Graphics 600 (0x00003180) Direct3D11 vs_5_0 ps_5_0, D3D11)",
|
|
||||||
"ANGLE (Intel, Intel(R) Iris(R) Xe Graphics (0x00009A49) Direct3D11 vs_5_0 ps_5_0, D3D11)"
|
|
||||||
],
|
|
||||||
"macos": [
|
|
||||||
"Intel HD Graphics 530 OpenGL Engine",
|
|
||||||
"Intel Iris Graphics 6100 OpenGL Engine",
|
|
||||||
"Intel UHD Graphics 630 OpenGL Engine",
|
|
||||||
"Intel HD Graphics 4000 OpenGL Engine",
|
|
||||||
"Intel Iris Pro OpenGL Engine",
|
|
||||||
"Intel UHD Graphics 617 OpenGL Engine"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"vendors": [
|
|
||||||
"Intel Inc.",
|
|
||||||
"Intel",
|
|
||||||
"Intel Corporation"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
348
tr.py
348
tr.py
@ -1,348 +0,0 @@
|
|||||||
To adapt your existing **LinkedIn job scraper** to scrape **Amazon job listings** instead, you need to **completely replace LinkedIn-specific logic** with Amazon-specific logic, because:
|
|
||||||
|
|
||||||
- Amazon jobs are hosted on a **different domain**: `https://www.amazon.jobs`
|
|
||||||
- The **DOM structure**, job URLs, pagination, selectors, and flow are **entirely different**
|
|
||||||
- **No login is required** — Amazon job listings are public
|
|
||||||
- Amazon uses **infinite scroll with API pagination**, not traditional “Next” buttons
|
|
||||||
- Job detail pages are **self-contained** — no external apply redirects like LinkedIn
|
|
||||||
|
|
||||||
Below is the **fully modified `job_scraper2.py`** (renamed internally to `AmazonJobScraper`) that scrapes Amazon jobs using the same engine architecture but adapted for Amazon’s site.
|
|
||||||
|
|
||||||
> ✅ **You should rename the file to `amazon_job_scraper.py`** and update `amazon_main.py` accordingly.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### ✅ Modified `job_scraper2.py` → **Amazon Job Scraper**
|
|
||||||
|
|
||||||
```python
|
|
||||||
# job_scraper2.py (now for Amazon)
|
|
||||||
import asyncio
|
|
||||||
import random
|
|
||||||
import re
|
|
||||||
from typing import Optional, Dict
|
|
||||||
from playwright.async_api import async_playwright, TimeoutError as PlaywrightTimeoutError
|
|
||||||
from browserforge.injectors.playwright import AsyncNewContext
|
|
||||||
from llm_agent import LLMJobRefiner
|
|
||||||
from fetcher import StealthyFetcher
|
|
||||||
from datetime import datetime
|
|
||||||
|
|
||||||
|
|
||||||
class AmazonJobScraper:
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
engine,
|
|
||||||
db_path: str = "amazon_jobs.db",
|
|
||||||
human_speed: float = 1.0,
|
|
||||||
user_request: str = "Extract all standard job details"
|
|
||||||
):
|
|
||||||
self.engine = engine
|
|
||||||
self.db_path = db_path
|
|
||||||
self.human_speed = human_speed
|
|
||||||
self.user_request = user_request
|
|
||||||
self.llm_agent = LLMJobRefiner()
|
|
||||||
|
|
||||||
async def _human_click(self, page, element, wait_after: bool = True):
|
|
||||||
if not element:
|
|
||||||
return False
|
|
||||||
await element.scroll_into_view_if_needed()
|
|
||||||
await asyncio.sleep(random.uniform(0.3, 0.8) * self.human_speed)
|
|
||||||
try:
|
|
||||||
await element.click()
|
|
||||||
if wait_after:
|
|
||||||
await asyncio.sleep(random.uniform(1.0, 2.0) * self.human_speed)
|
|
||||||
return True
|
|
||||||
except:
|
|
||||||
return False
|
|
||||||
|
|
||||||
def _extract_location_from_keywords(self, search_keywords: str) -> str:
|
|
||||||
location_match = re.search(r'location:\s*([^,]+)', search_keywords, re.IGNORECASE)
|
|
||||||
return location_match.group(1).strip() if location_match else ""
|
|
||||||
|
|
||||||
def _build_amazon_search_url(self, keywords: str) -> str:
|
|
||||||
clean_keywords = re.sub(r'location:\s*[^,]+', '', keywords, flags=re.IGNORECASE).strip()
|
|
||||||
location = self._extract_location_from_keywords(keywords)
|
|
||||||
|
|
||||||
base_url = "https://www.amazon.jobs/en/search?"
|
|
||||||
params = []
|
|
||||||
|
|
||||||
if clean_keywords:
|
|
||||||
params.append(f"base_query={clean_keywords.replace(' ', '+')}")
|
|
||||||
if location:
|
|
||||||
params.append(f"loc_query={location.replace(' ', '+')}")
|
|
||||||
params.append("offset=0")
|
|
||||||
params.append("result_limit=10")
|
|
||||||
|
|
||||||
return base_url + "&".join(params)
|
|
||||||
|
|
||||||
async def _extract_page_content_for_llm(self, page) -> str:
|
|
||||||
await asyncio.sleep(2 * self.human_speed)
|
|
||||||
await self.engine._human_like_scroll(page)
|
|
||||||
await asyncio.sleep(2 * self.human_speed)
|
|
||||||
return await page.content()
|
|
||||||
|
|
||||||
async def _scrape_job_links_from_page(self, page, seen_job_ids, all_job_links):
|
|
||||||
job_cards = await page.query_selector_all('div.job-tile a[href^="/en/jobs/"]')
|
|
||||||
new_jobs = 0
|
|
||||||
for card in job_cards:
|
|
||||||
href = await card.get_attribute("href")
|
|
||||||
if not href:
|
|
||||||
continue
|
|
||||||
full_url = f"https://www.amazon.jobs{href}" if href.startswith("/") else href
|
|
||||||
job_id = href.split("/")[-1] if href.split("/")[-1] else "unknown"
|
|
||||||
|
|
||||||
if job_id in seen_job_ids:
|
|
||||||
continue
|
|
||||||
|
|
||||||
title_element = await card.query_selector('h3')
|
|
||||||
title = await title_element.inner_text() if title_element else "Unknown Title"
|
|
||||||
|
|
||||||
seen_job_ids.add(job_id)
|
|
||||||
all_job_links.append((full_url, title))
|
|
||||||
new_jobs += 1
|
|
||||||
|
|
||||||
return new_jobs
|
|
||||||
|
|
||||||
async def _scroll_and_collect_jobs(self, page, seen_job_ids, all_job_links, max_pages=5):
|
|
||||||
offset = 0
|
|
||||||
jobs_per_page = 10
|
|
||||||
for page_num in range(max_pages):
|
|
||||||
print(f"📄 Fetching Amazon job page {page_num + 1} (offset: {offset})")
|
|
||||||
current_url = page.url
|
|
||||||
if "offset=" in current_url:
|
|
||||||
base_url = current_url.split("offset=")[0]
|
|
||||||
new_url = base_url + f"offset={offset}&result_limit={jobs_per_page}"
|
|
||||||
else:
|
|
||||||
new_url = current_url + f"&offset={offset}&result_limit={jobs_per_page}"
|
|
||||||
|
|
||||||
await page.goto(new_url, wait_until='domcontentloaded', timeout=120000)
|
|
||||||
await asyncio.sleep(random.uniform(3.0, 5.0) * self.human_speed)
|
|
||||||
|
|
||||||
new_jobs = await self._scrape_job_links_from_page(page, seen_job_ids, all_job_links)
|
|
||||||
print(f" ➕ Found {new_jobs} new job(s) on page {page_num + 1} (total: {len(all_job_links)})")
|
|
||||||
|
|
||||||
if new_jobs == 0 and page_num > 0:
|
|
||||||
print("🔚 No new jobs — stopping pagination.")
|
|
||||||
break
|
|
||||||
|
|
||||||
offset += jobs_per_page
|
|
||||||
|
|
||||||
async def scrape_jobs(
|
|
||||||
self,
|
|
||||||
search_keywords: Optional[str],
|
|
||||||
max_pages: int = 5,
|
|
||||||
credentials: Optional[Dict] = None # Not used for Amazon
|
|
||||||
):
|
|
||||||
search_url = self._build_amazon_search_url(search_keywords)
|
|
||||||
print(f"🔍 Amazon search URL: {search_url}")
|
|
||||||
|
|
||||||
profile = self.engine._select_profile()
|
|
||||||
renderer = random.choice(self.engine.common_renderers[self.engine.os])
|
|
||||||
vendor = random.choice(self.engine.common_vendors)
|
|
||||||
spoof_script = self.engine._get_spoof_script(renderer, vendor)
|
|
||||||
|
|
||||||
async with async_playwright() as pw:
|
|
||||||
browser = await pw.chromium.launch(
|
|
||||||
headless=False,
|
|
||||||
args=['--disable-blink-features=AutomationControlled']
|
|
||||||
)
|
|
||||||
context = await AsyncNewContext(browser, fingerprint=profile)
|
|
||||||
|
|
||||||
await context.add_init_script(f"""
|
|
||||||
Object.defineProperty(navigator, 'hardwareConcurrency', {{ get: () => {profile.navigator.hardwareConcurrency} }});
|
|
||||||
Object.defineProperty(navigator, 'deviceMemory', {{ get: () => {profile.navigator.deviceMemory} }});
|
|
||||||
Object.defineProperty(navigator, 'platform', {{ get: () => '{profile.navigator.platform}' }});
|
|
||||||
""")
|
|
||||||
await context.add_init_script(spoof_script)
|
|
||||||
|
|
||||||
page = await context.new_page()
|
|
||||||
temp_fetcher = StealthyFetcher(self.engine, browser, context)
|
|
||||||
|
|
||||||
# Amazon doesn't require login
|
|
||||||
print("🌐 Navigating to Amazon Jobs (no login required)...")
|
|
||||||
await page.goto(search_url, wait_until='domcontentloaded', timeout=120000)
|
|
||||||
await asyncio.sleep(random.uniform(3.0, 5.0) * self.human_speed)
|
|
||||||
|
|
||||||
# Protection check
|
|
||||||
protection_type = await temp_fetcher._detect_protection(page)
|
|
||||||
if protection_type:
|
|
||||||
print(f"🛡️ Protection detected: {protection_type}")
|
|
||||||
content_accessible = await temp_fetcher._is_content_accessible(page)
|
|
||||||
if not content_accessible:
|
|
||||||
handled = await self.engine._handle_cloudflare(page) if protection_type == "cloudflare" else False
|
|
||||||
if not handled:
|
|
||||||
await browser.close()
|
|
||||||
self.engine.report_outcome("protection_block")
|
|
||||||
return
|
|
||||||
else:
|
|
||||||
print("✅ Protection present but content accessible.")
|
|
||||||
|
|
||||||
all_job_links = []
|
|
||||||
seen_job_ids = set()
|
|
||||||
|
|
||||||
print("🔄 Collecting job links via pagination...")
|
|
||||||
await self._scroll_and_collect_jobs(page, seen_job_ids, all_job_links, max_pages=max_pages)
|
|
||||||
|
|
||||||
print(f"✅ Collected {len(all_job_links)} unique Amazon job links.")
|
|
||||||
|
|
||||||
scraped_count = 0
|
|
||||||
for idx, (job_url, title) in enumerate(all_job_links):
|
|
||||||
try:
|
|
||||||
print(f" → Opening job {idx+1}/{len(all_job_links)}: {job_url}")
|
|
||||||
fetcher = StealthyFetcher(self.engine, browser, context)
|
|
||||||
job_page = await fetcher.fetch_url(job_url, wait_for_selector="h1.job-title")
|
|
||||||
|
|
||||||
if not job_page:
|
|
||||||
print(f" ❌ Failed to fetch job page: {job_url}")
|
|
||||||
self.engine.report_outcome("fetch_failure", url=job_url)
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Extract raw HTML for LLM
|
|
||||||
await self.engine._human_like_scroll(job_page)
|
|
||||||
await asyncio.sleep(2 * self.human_speed)
|
|
||||||
page_content = await self._extract_page_content_for_llm(job_page)
|
|
||||||
|
|
||||||
job_id = job_url.split("/")[-1] if job_url.split("/")[-1] else "unknown"
|
|
||||||
|
|
||||||
raw_data = {
|
|
||||||
"page_content": page_content,
|
|
||||||
"url": job_url,
|
|
||||||
"job_id": job_id,
|
|
||||||
"search_keywords": search_keywords
|
|
||||||
}
|
|
||||||
|
|
||||||
refined_data = await self.llm_agent.refine_job_data(raw_data, self.user_request)
|
|
||||||
|
|
||||||
if refined_data and refined_data.get("title", "N/A") != "N/A":
|
|
||||||
# Ensure compulsory fields
|
|
||||||
compulsory_fields = ['company_name', 'job_id', 'url']
|
|
||||||
for field in compulsory_fields:
|
|
||||||
if not refined_data.get(field) or refined_data[field] in ["N/A", "", "Unknown"]:
|
|
||||||
if field == 'job_id':
|
|
||||||
refined_data[field] = job_id
|
|
||||||
elif field == 'url':
|
|
||||||
refined_data[field] = job_url
|
|
||||||
elif field == 'company_name':
|
|
||||||
refined_data[field] = "Amazon"
|
|
||||||
|
|
||||||
refined_data['scraped_at'] = datetime.now().isoformat()
|
|
||||||
refined_data['category'] = re.sub(r'location:\s*[^,]+', '', search_keywords, flags=re.IGNORECASE).strip()
|
|
||||||
await self.llm_agent.save_job_data(refined_data, search_keywords)
|
|
||||||
scraped_count += 1
|
|
||||||
print(f" ✅ Scraped and refined: {refined_data['title'][:50]}...")
|
|
||||||
self.engine.report_outcome("success", url=job_url)
|
|
||||||
else:
|
|
||||||
print(f" 🟡 LLM could not extract valid data from: {job_url}")
|
|
||||||
self.engine.report_outcome("llm_failure", url=job_url)
|
|
||||||
|
|
||||||
await job_page.close()
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
print(f" ⚠️ Failed on job {idx+1}: {str(e)[:100]}")
|
|
||||||
if 'job_page' in locals() and job_page:
|
|
||||||
await job_page.close()
|
|
||||||
continue
|
|
||||||
|
|
||||||
await browser.close()
|
|
||||||
|
|
||||||
if scraped_count > 0:
|
|
||||||
self.engine.report_outcome("success")
|
|
||||||
print(f"✅ Completed! Processed {scraped_count} Amazon jobs for '{search_keywords}'.")
|
|
||||||
else:
|
|
||||||
self.engine.report_outcome("no_jobs")
|
|
||||||
print("⚠️ No Amazon jobs processed successfully.")
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### ✅ Modified `linkedin_main.py` → **amazon_main.py**
|
|
||||||
|
|
||||||
```python
|
|
||||||
# amazon_main.py
|
|
||||||
from scraping_engine import FingerprintScrapingEngine
|
|
||||||
from job_scraper2 import AmazonJobScraper # Updated class name
|
|
||||||
import os
|
|
||||||
from dotenv import load_dotenv
|
|
||||||
import asyncio
|
|
||||||
import random
|
|
||||||
import time
|
|
||||||
|
|
||||||
load_dotenv()
|
|
||||||
|
|
||||||
async def main():
|
|
||||||
engine = FingerprintScrapingEngine(
|
|
||||||
seed="amazon_job_scraping_12",
|
|
||||||
target_os="windows",
|
|
||||||
db_path="amazon_jobs.db",
|
|
||||||
markdown_path="amazon_jobs.md"
|
|
||||||
)
|
|
||||||
|
|
||||||
scraper = AmazonJobScraper(
|
|
||||||
engine,
|
|
||||||
human_speed=1.4,
|
|
||||||
user_request="Extract title, company, location, description, basic qualifications, preferred qualifications, job ID, and job type (full-time, part-time, etc.)"
|
|
||||||
)
|
|
||||||
|
|
||||||
job_titles = [
|
|
||||||
"Software Development Engineer",
|
|
||||||
"Data Scientist",
|
|
||||||
"Product Manager",
|
|
||||||
"UX Designer",
|
|
||||||
"Solutions Architect",
|
|
||||||
"Machine Learning Engineer",
|
|
||||||
"Frontend Engineer",
|
|
||||||
"Backend Engineer",
|
|
||||||
"Full Stack Engineer",
|
|
||||||
"Data Engineer"
|
|
||||||
]
|
|
||||||
|
|
||||||
fixed_location = "United States" # Amazon uses country/region, not city
|
|
||||||
|
|
||||||
while True:
|
|
||||||
random.shuffle(job_titles)
|
|
||||||
for job_title in job_titles:
|
|
||||||
search_keywords = f"{job_title} location:{fixed_location}"
|
|
||||||
print(f"\n{'='*60}")
|
|
||||||
print(f"Starting Amazon scrape for: {search_keywords}")
|
|
||||||
print(f"{'='*60}")
|
|
||||||
|
|
||||||
await scraper.scrape_jobs(
|
|
||||||
search_keywords=search_keywords,
|
|
||||||
max_pages=3 # Amazon loads 10 per page; 3 pages = ~30 jobs
|
|
||||||
)
|
|
||||||
|
|
||||||
print(f"\n✅ Completed scraping for: {job_title}")
|
|
||||||
print(f"⏳ Waiting 90 seconds before next job title...")
|
|
||||||
time.sleep(90)
|
|
||||||
|
|
||||||
print(f"\n✅ Completed full cycle. Restarting...")
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
asyncio.run(main())
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 🔑 Key Changes Summary
|
|
||||||
|
|
||||||
| Feature | LinkedIn | Amazon |
|
|
||||||
|-------|--------|--------|
|
|
||||||
| **Login** | Required | ❌ Not needed |
|
|
||||||
| **Job URL** | `/jobs/view/123` | `/en/jobs/123` |
|
|
||||||
| **Pagination** | “Next” button or infinite scroll | API-style `offset=0&result_limit=10` |
|
|
||||||
| **Apply Flow** | Modal or external redirect | All details on-page |
|
|
||||||
| **Location** | City/state (e.g., "New York") | Country/region (e.g., "United States") |
|
|
||||||
| **Selectors** | Complex job cards | Simple `div.job-tile a` |
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### ✅ Next Steps
|
|
||||||
|
|
||||||
1. **Rename files**:
|
|
||||||
- `job_scraper2.py` → keep name but now contains `AmazonJobScraper`
|
|
||||||
- `linkedin_main.py` → `amazon_main.py`
|
|
||||||
|
|
||||||
2. **Update `.env`** — credentials are no longer needed, but you can remove them.
|
|
||||||
|
|
||||||
3. **Test** with a single job title first before running the full loop.
|
|
||||||
|
|
||||||
Let me know if you also want to adjust the `LLMJobRefiner` prompt for Amazon’s job description format!
|
|
||||||
Loading…
x
Reference in New Issue
Block a user