From 35cc5d3bdea09691505c4c127c7e9f31c425dd0b Mon Sep 17 00:00:00 2001 From: queukat Date: Fri, 6 Sep 2024 20:06:30 +0300 Subject: [PATCH] resolve issues --- src/gpt.py | 96 +++++++++++++++++++++++++++++++----- src/linkedIn_easy_applier.py | 8 +-- src/linkedIn_job_manager.py | 57 +++++++++++++++------ 3 files changed, 126 insertions(+), 35 deletions(-) diff --git a/src/gpt.py b/src/gpt.py index 1f6a163..9495e2a 100644 --- a/src/gpt.py +++ b/src/gpt.py @@ -3,6 +3,8 @@ import os import re import textwrap import time +from abc import ABC, abstractmethod +from typing import Dict, List, Union from datetime import datetime from functools import wraps from pathlib import Path @@ -17,15 +19,73 @@ from langchain_core.output_parsers import StrOutputParser from langchain_core.prompt_values import StringPromptValue from langchain_core.prompts import ChatPromptTemplate from langchain_openai import ChatOpenAI +from Levenshtein import distance import src.strings as strings from src.utils import logger load_dotenv() +class AIModel(ABC): + @abstractmethod + def invoke(self, prompt: str) -> str: + pass + +class OpenAIModel(AIModel): + def __init__(self, api_key: str, llm_model: str, llm_api_url: str): + from langchain_openai import ChatOpenAI + self.model = ChatOpenAI(model_name=llm_model, openai_api_key=api_key, + temperature=0.4, base_url=llm_api_url) + + def invoke(self, prompt: str) -> str: + print("invoke in openai") + response = self.model.invoke(prompt) + return response + +class ClaudeModel(AIModel): + def __init__(self, api_key: str, llm_model: str, llm_api_url: str): + from langchain_anthropic import ChatAnthropic + self.model = ChatAnthropic(model=llm_model, api_key=api_key, + temperature=0.4, base_url=llm_api_url) + + def invoke(self, prompt: str) -> str: + response = self.model.invoke(prompt) + return response + +class OllamaModel(AIModel): + def __init__(self, api_key: str, llm_model: str, llm_api_url: str): + from langchain_ollama import ChatOllama + self.model = ChatOllama(model=llm_model, base_url=llm_api_url) + + def invoke(self, prompt: str) -> str: + response = self.model.invoke(prompt) + return response + +class AIAdapter: + def __init__(self, config: dict, api_key: str): + self.model = self._create_model(config, api_key) + + def _create_model(self, config: dict, api_key: str) -> AIModel: + llm_model_type = config['llm_model_type'] + llm_model = config['llm_model'] + llm_api_url = config['llm_api_url'] + print('Using {0} with {1} from {2}'.format(llm_model_type, llm_model, llm_api_url)) + + if llm_model_type == "openai": + return OpenAIModel(api_key, llm_model, llm_api_url) + elif llm_model_type == "claude": + return ClaudeModel(api_key, llm_model, llm_api_url) + elif llm_model_type == "ollama": + return OllamaModel(api_key, llm_model, llm_api_url) + else: + raise ValueError(f"Unsupported model type: {model_type}") + + def invoke(self, prompt: str) -> str: + return self.model.invoke(prompt) + class LLMLogger: - def __init__(self, llm: ChatOpenAI): + def __init__(self, llm: Union[OpenAIModel, OllamaModel, ClaudeModel]): logger.debug("Initializing LLMLogger with LLM: %s", llm) self.llm = llm logger.debug("LLMLogger successfully initialized with LLM: %s", llm) @@ -48,6 +108,7 @@ class LLMLogger: prompts = prompts.text logger.debug("Prompts converted to text: %s", prompts) elif isinstance(prompts, Dict): + # Convert prompts to a dictionary if they are not in the expected format logger.debug("Prompts are of type Dict") try: prompts = { @@ -76,7 +137,7 @@ class LLMLogger: except Exception as e: logger.error("Error obtaining current time: %s", str(e)) raise - + # Extract token usage details from the response try: token_usage = parsed_reply["usage_metadata"] output_tokens = token_usage["output_tokens"] @@ -86,14 +147,14 @@ class LLMLogger: except KeyError as e: logger.error("KeyError in parsed_reply structure: %s", str(e)) raise - + # Extract model details from the response try: model_name = parsed_reply["response_metadata"]["model_name"] logger.debug("Model name: %s", model_name) except KeyError as e: logger.error("KeyError in response_metadata: %s", str(e)) raise - + # Calculate the total cost of the API call try: prompt_price_per_token = 0.00000015 completion_price_per_token = 0.0000006 @@ -108,7 +169,7 @@ class LLMLogger: "model": model_name, "time": current_time, "prompts": prompts, - "replies": parsed_reply["content"], # Контент ответа + "replies": parsed_reply["content"], # Response content "total_tokens": total_tokens, "input_tokens": input_tokens, "output_tokens": output_tokens, @@ -118,7 +179,7 @@ class LLMLogger: except KeyError as e: logger.error("Error creating log entry: missing key %s in parsed_reply", str(e)) raise - + # Write the log entry to the log file in JSON format try: with open(calls_log, "a", encoding="utf-8") as f: json_string = json.dumps(log_entry, ensure_ascii=False, indent=4) @@ -130,17 +191,18 @@ class LLMLogger: class LoggerChatModel: - def __init__(self, llm: ChatOpenAI): + def __init__(self, llm: Union[OpenAIModel, OllamaModel, ClaudeModel]): logger.debug("Initializing LoggerChatModel with LLM: %s", llm) self.llm = llm logger.debug("LoggerChatModel successfully initialized with LLM: %s", llm) def __call__(self, messages: List[Dict[str, str]]) -> str: + # Call the LLM with the provided messages and log the response. logger.debug("Entering __call__ method with messages: %s", messages) while True: try: logger.debug("Attempting to call the LLM with messages") - reply = self.llm(messages) # Вызов LLM + reply = self.llm(messages) logger.debug("LLM response received: %s", reply) parsed_reply = self.parse_llmresult(reply) @@ -180,6 +242,8 @@ class LoggerChatModel: continue def parse_llmresult(self, llmresult: AIMessage) -> Dict[str, Dict]: + # Parse the LLM result into a structured format. + logger.debug("Parsing LLM result: %s", llmresult) try: @@ -218,10 +282,9 @@ class LoggerChatModel: class GPTAnswerer: - def __init__(self, openai_api_key): - self.llm_cheap = LoggerChatModel( - ChatOpenAI(model_name="gpt-4o-mini", openai_api_key=openai_api_key, temperature=0.4) - ) + def __init__(self, config, llm_api_key): + self.ai_adapter = AIAdapter(config, llm_api_key) + self.llm_cheap = LoggerChatModel(self.ai_adapter) logger.debug("GPTAnswerer initialized with API key") @property @@ -246,6 +309,7 @@ class GPTAnswerer: @staticmethod def _preprocess_template_string(template: str) -> str: + # Preprocess a template string to remove unnecessary indentation. logger.debug("Preprocessing template string") return textwrap.dedent(template) @@ -279,6 +343,7 @@ class GPTAnswerer: return prompt | self.llm_cheap | StrOutputParser() def answer_question_textual_wide_range(self, question: str) -> str: + # Define chains for each section of the resume logger.debug("Answering textual question: %s", question) chains = { "personal_information": self._create_chain(strings.personal_information_template), @@ -387,7 +452,11 @@ class GPTAnswerer: chain = prompt | self.llm_cheap | StrOutputParser() output = chain.invoke({"question": question}) logger.debug("Section determined from question: %s", output) - section_name = output.lower().replace(" ", "_") + match = re.search(r"(Personal information|Self Identification|Legal Authorization|Work Preferences|Education Details|Experience Details|Projects|Availability|Salary Expectations|Certifications|Languages|Interests|Cover letter)", output, re.IGNORECASE) + if not match: + raise ValueError("Could not extract section name from the response.") + + section_name = match.group(1).lower().replace(" ", "_") if section_name == "cover_letter": chain = chains.get(section_name) output = chain.invoke({"resume": self.resume, "job_description": self.job_description}) @@ -442,6 +511,7 @@ class GPTAnswerer: return best_option def resume_or_cover(self, phrase: str) -> str: + # Define the prompt template logger.debug("Determining if phrase refers to resume or cover letter: %s", phrase) prompt_template = """ Given the following phrase, respond with only 'resume' if the phrase is about a resume, or 'cover' if it's about a cover letter. If the phrase contains only the word 'upload', consider it as 'cover'. Do not provide any additional information or explanations. diff --git a/src/linkedIn_easy_applier.py b/src/linkedIn_easy_applier.py index 71139bd..d3f9bbc 100644 --- a/src/linkedIn_easy_applier.py +++ b/src/linkedIn_easy_applier.py @@ -8,7 +8,6 @@ import time import traceback from datetime import date from typing import List, Optional, Any, Tuple - from httpx import HTTPStatusError from openai import RateLimitError from reportlab.lib.pagesizes import letter @@ -175,7 +174,6 @@ class LinkedInEasyApplier: except Exception as e: logger.warning(f"Failed to click 'Easy Apply' button using {method['description']} on attempt {attempt + 1}: {e}") - # Обновление страницы после первой неудачной попытки if attempt == 0: logger.debug("Refreshing page to retry finding 'Easy Apply' button") self.driver.refresh() @@ -530,8 +528,7 @@ class LinkedInEasyApplier: self._enter_text(text_field, existing_answer['answer']) logger.debug("Entered existing textbox answer.") - # Нажать "Вниз" и "Enter" для выбора первого элемента в выпадающем списке - time.sleep(1) # Ожидание появления выпадающего списка + time.sleep(1) text_field.send_keys(Keys.ARROW_DOWN) text_field.send_keys(Keys.ENTER) logger.debug("Selected first option from the dropdown.") @@ -541,8 +538,7 @@ class LinkedInEasyApplier: self._enter_text(text_field, answer) logger.debug("Entered new textbox answer and saved it to JSON.") - # Нажать "Вниз" и "Enter" для выбора первого элемента в выпадающем списке - time.sleep(1) # Ожидание появления выпадающего списка + time.sleep(1) text_field.send_keys(Keys.ARROW_DOWN) text_field.send_keys(Keys.ENTER) logger.debug("Selected first option from the dropdown.") diff --git a/src/linkedIn_job_manager.py b/src/linkedIn_job_manager.py index 9f9c66d..82602dc 100644 --- a/src/linkedIn_job_manager.py +++ b/src/linkedIn_job_manager.py @@ -47,6 +47,7 @@ class LinkedInJobManager: self.title_blacklist = parameters.get('titleBlacklist', []) or [] self.positions = parameters.get('positions', []) self.locations = parameters.get('locations', []) + self.apply_once_at_company = parameters.get('applyOnceAtCompany', False) self.base_search_url = self.get_base_search_url(parameters) self.seen_jobs = [] resume_path = parameters.get('uploads', {}).get('resume', None) @@ -86,7 +87,6 @@ class LinkedInJobManager: time.sleep(random.uniform(1.5, 3.5)) utils.printyellow("Starting the application process for this page...") - # Проверка на наличие вакансий на странице try: jobs = self.get_jobs_from_page() if not jobs: @@ -94,7 +94,7 @@ class LinkedInJobManager: break except Exception as e: logger.error(f"Failed to retrieve jobs: {e}") - break # Выходим из цикла, если не удалось получить вакансии + break try: self.apply_jobs() @@ -136,40 +136,32 @@ class LinkedInJobManager: def get_jobs_from_page(self): - """ - Функция для получения списка вакансий на текущей странице. - Если вакансии не найдены, возвращает пустой список. - """ try: - # Проверка на отсутствие вакансий no_jobs_element = self.driver.find_element(By.CLASS_NAME, 'jobs-search-two-pane__no-results-banner--expand') if 'No matching jobs found' in no_jobs_element.text or 'unfortunately, things aren' in self.driver.page_source.lower(): utils.printyellow("No matching jobs found on this page.") logger.debug("No matching jobs found on this page, skipping.") - return [] # Возвращаем пустой список, если нет вакансий + return [] except NoSuchElementException: - pass # Если элемент не найден, продолжаем поиск вакансий + pass - # Поиск контейнера результатов с вакансиями try: job_results = self.driver.find_element(By.CLASS_NAME, "jobs-search-results-list") utils.scroll_slow(self.driver, job_results) utils.scroll_slow(self.driver, job_results, step=300, reverse=True) - # Поиск элементов списка вакансий job_list_elements = self.driver.find_elements(By.CLASS_NAME, 'scaffold-layout__list-container')[0].find_elements(By.CLASS_NAME, 'jobs-search-results__list-item') if not job_list_elements: utils.printyellow("No job class elements found on page.") logger.debug("No job class elements found on page, skipping.") return [] - # Возвращаем список найденных вакансий return job_list_elements except NoSuchElementException: logger.debug("No job results found on the page.") - return [] # Если не найден контейнер с результатами, возвращаем пустой список + return [] except Exception as e: logger.error(f"Error while fetching job elements: {e}") @@ -181,9 +173,9 @@ class LinkedInJobManager: if 'No matching jobs found' in no_jobs_element.text or 'unfortunately, things aren' in self.driver.page_source.lower(): utils.printyellow("No matching jobs found on this page, moving to next.") logger.debug("No matching jobs found on this page, skipping") - return # Выход из метода, если нет больше подходящих вакансий + return except NoSuchElementException: - pass # Если элемент не найден, просто продолжаем + pass job_results = self.driver.find_element(By.CLASS_NAME, "jobs-search-results-list") utils.scroll_slow(self.driver, job_results) @@ -192,7 +184,7 @@ class LinkedInJobManager: if not job_list_elements: utils.printyellow("No job class elements found on page, moving to next page.") logger.debug("No job class elements found on page, skipping") - return # Выход из метода, если нет вакансий на странице + return job_list = [Job(*self.extract_job_information_from_tile(job_element)) for job_element in job_list_elements] for job in job_list: if self.is_blacklisted(job.title, job.company, job.link): @@ -200,6 +192,12 @@ class LinkedInJobManager: logger.debug("Job blacklisted: %s at %s", job.title, job.company) self.write_to_file(job, "skipped") continue + if self.is_already_applied_to_job(job.title, job.company, job.link): + self.write_to_file(job, "skipped") + continue + if self.is_already_applied_to_company(job.company): + self.write_to_file(job, "skipped") + continue try: if job.apply_method not in {"Continue", "Applied", "Apply"}: self.easy_applier_component.job_apply(job) @@ -301,6 +299,33 @@ class LinkedInJobManager: title_blacklisted = any(word in job_title_words for word in self.title_blacklist) company_blacklisted = company.strip().lower() in (word.strip().lower() for word in self.company_blacklist) link_seen = link in self.seen_jobs + is_blacklisted = title_blacklisted or company_blacklisted or link_seen logger.debug("Job blacklisted status: %s", is_blacklisted) return is_blacklisted + + + def is_already_applied_to_job(self, job_title, company, link): + link_seen = link in self.seen_jobs + if link_seen: + utils.printyellow(f"Already applied to job: {job_title} at {company}, skipping...") + return link_seen + + def is_already_applied_to_company(self, company): + if not self.apply_once_at_company: + return False + + output_files = ["success.json"] + for file_name in output_files: + file_path = self.output_file_directory / file_name + if file_path.exists(): + with open(file_path, 'r', encoding='utf-8') as f: + try: + existing_data = json.load(f) + for applied_job in existing_data: + if applied_job['company'].strip().lower() == company.strip().lower(): + utils.printyellow(f"Already applied at {company} (once per company policy), skipping...") + return True + except json.JSONDecodeError: + continue + return False