ccppi/job-scrapper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

getting upstream

Browse Source
This commit is contained in:
ccppi
2024-08-22 11:12:11 +02:00
parent f3ab4db625
commit 850538a92f
3 changed files with 53 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
lib/login.py
View File

@@ -1,4 +1,5 @@
import requests
from requests_html import HTMLSession
from helpers import *
def login(entry):
user = entry.user
@@ -17,7 +18,7 @@ def login(entry):
"Origin": "https://www.jobagent.ch",
"Connection": "keep-alive",
"Referer": "https://jobagent.ch",
"Cookie": "datadome=BbGio7V9YBqYELb~B2a7DqE9Zr5EWb315OypbcxGQOFKbhkJR48etFSooYwtnKF2sK5leCh7Q_0o6W5YMwl0qEH~Fw3BU0m~48MgrkuaxO3Z1s5MTqCnTZVW3PcQv7KM; _uc=ad_storage=granted:analytics_storage=granted; _gcl_au=1.1.1328234550.1724056973.1502078804.1724062620.1724062680; _ga=GA1.1.1840632117.1724056971; _ga_T0E2JNNRW2=GS1.1.1724062555.3.1.1724062683.0.1.2098134382; JSESSIONID=AB8CC398C2576A6A87C53A74CCD8F7F5; _pin_unauth=dWlkPU56Y3pabU14WW1JdFptTTFNeTAwTkdFMkxUbGlZV0V0TWprNVkyTXpZemd4WldNNA; _fbp=fb.1.1724056975123.543612652217076856; _clck=16bp9by%7C2%7Cfog%7C0%7C1692; _clsk=1o7y6b9%7C1724062683361%7C9%7C1%7Cu.clarity.ms%2Fcollect; _rm=ai53eXNzJTQwa29sYWJub3cuY2g6MTcyNjY1NDY4MTA0NDpTSEEyNTY6ZGRkMmZhYTRjZWY3MWZkZDU1M2VlMTI4ZjYzOGY1NmFiYmRkNjNiMmI3ZjE1NWRhNmU3YzcwZWU1NjQ2Mjc0Mw; _uetsid=0737af805e0711efbe7bdd027b00b063; _uetvid=0737b3005e0711efb7c7035382896421",
#"Cookie": "datadome=BbGio7V9YBqYELb~B2a7DqE9Zr5EWb315OypbcxGQOFKbhkJR48etFSooYwtnKF2sK5leCh7Q_0o6W5YMwl0qEH~Fw3BU0m~48MgrkuaxO3Z1s5MTqCnTZVW3PcQv7KM; _uc=ad_storage=granted:analytics_storage=granted; _gcl_au=1.1.1328234550.1724056973.1502078804.1724062620.1724062680; _ga=GA1.1.1840632117.1724056971; _ga_T0E2JNNRW2=GS1.1.1724062555.3.1.1724062683.0.1.2098134382; JSESSIONID=AB8CC398C2576A6A87C53A74CCD8F7F5; _pin_unauth=dWlkPU56Y3pabU14WW1JdFptTTFNeTAwTkdFMkxUbGlZV0V0TWprNVkyTXpZemd4WldNNA; _fbp=fb.1.1724056975123.543612652217076856; _clck=16bp9by%7C2%7Cfog%7C0%7C1692; _clsk=1o7y6b9%7C1724062683361%7C9%7C1%7Cu.clarity.ms%2Fcollect; _rm=ai53eXNzJTQwa29sYWJub3cuY2g6MTcyNjY1NDY4MTA0NDpTSEEyNTY6ZGRkMmZhYTRjZWY3MWZkZDU1M2VlMTI4ZjYzOGY1NmFiYmRkNjNiMmI3ZjE1NWRhNmU3YzcwZWU1NjQ2Mjc0Mw; _uetsid=0737af805e0711efbe7bdd027b00b063; _uetvid=0737b3005e0711efb7c7035382896421",
# "Upgrade-Insecure-Requests": "1",
# "Sec-Fetch-Dest": "document",
# "Sec-Fetch-Mode": "navigate",
@@ -29,11 +30,40 @@ def login(entry):
payload = {"redirectUrl":"","email":user,"password":pw}
resp = session.post(loginurl,data=payload)
print(payload)
print("response from login attempt",resp)
if resp.url == 'https://www.jobagent.ch/user/login?error':
print("Error on login")
return -1
checkBlockers(session,resp)
r = session.get(scrapurl)
print(session.headers)
print("response:",r)
return session
#solveCaptcha when :
#string "captcha" is in response
#search for <iframe
#get src tag
#open a webbrowser to solve the captcha
#somehow getting the cookie maype?
def solveCaptcha(session,resp):
found = 0
if "captcha" or "Enable JavaScript" in resp :
#soup = BeautifulSoup(resp,"html.parser")
#result = soup.find("iframe")
#while found==0:
# if "captcha" in resp:#result:
print("captcha link!! found:")
found=1
#else:
# result.find_next()
print("exit loop")
print("response:",resp)
#if found:
#print("captchaurl:", result["src"])
#x = input("continue")
#else:
# print("did not recognise a captcha")
def checkBlockers(session,resp):
print("response from login attempt",resp)
if resp:
if resp.url == 'https://www.jobagent.ch/user/login?error':
print("Error on login")
return -1
solveCaptcha(session,resp)