From d964f599bed339bee2404898516fd20ba581df6c Mon Sep 17 00:00:00 2001 From: user Date: Sat, 31 Aug 2024 20:19:33 +0200 Subject: [PATCH 01/11] Added support for Ollama running locally or publicly hosted api --- README.md | 8 +++++++- data_folder/secrets.yaml | 3 ++- data_folder_example/secrets.yaml | 3 ++- main.py | 12 ++++++------ src/gpt.py | 25 +++++++++++++++++-------- 5 files changed, 34 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 15fe7b8..de60c0e 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,13 @@ This file contains sensitive information. Never share or commit this file to ver - Replace with your OpenAI API key for GPT integration - To obtain an API key, follow the tutorial at: https://medium.com/@lorenzozar/how-to-get-your-own-openai-api-key-f4d44e60c327 - Note: You need to add credit to your OpenAI account to use the API. You can add credit by visiting the [OpenAI billing dashboard](https://platform.openai.com/account/billing). - +- `openai_api_free_hosted_url`: + - Optional paramter, if you want to use freely hosted GPT model, set `openai_api_key: "freehosted"` and `openai_api_free_hosted_url` with the URL of the endpoint +- Ollama local support + - If you want to use Ollama which is deployed locally, leave `openai_api_key` blank. + - To setup Ollama to run locally follow the instructions here: [Ollama installation](https://github.com/ollama/ollama). + - Download mistral model by pulling mistral:v0.3 + ### 2. config.yaml diff --git a/data_folder/secrets.yaml b/data_folder/secrets.yaml index ad24cd8..9d0bfb3 100644 --- a/data_folder/secrets.yaml +++ b/data_folder/secrets.yaml @@ -1,3 +1,4 @@ email: myemaillinkedin@gmail.com password: ImpossiblePassowrd10 -openai_api_key: sk-11KRr4uuTwpRGfeRTfj1T9BlbkFJjP8QTrswHU1yGruru2FR \ No newline at end of file +openai_api_key: sk-11KRr4uuTwpRGfeRTfj1T9BlbkFJjP8QTrswHU1yGruru2FR +openai_api_free_hosted_url: https://api.pawan.krd/cosmosrp/v1 \ No newline at end of file diff --git a/data_folder_example/secrets.yaml b/data_folder_example/secrets.yaml index ad24cd8..9d0bfb3 100644 --- a/data_folder_example/secrets.yaml +++ b/data_folder_example/secrets.yaml @@ -1,3 +1,4 @@ email: myemaillinkedin@gmail.com password: ImpossiblePassowrd10 -openai_api_key: sk-11KRr4uuTwpRGfeRTfj1T9BlbkFJjP8QTrswHU1yGruru2FR \ No newline at end of file +openai_api_key: sk-11KRr4uuTwpRGfeRTfj1T9BlbkFJjP8QTrswHU1yGruru2FR +openai_api_free_hosted_url: https://api.pawan.krd/cosmosrp/v1 \ No newline at end of file diff --git a/main.py b/main.py index 9685677..82ed33f 100644 --- a/main.py +++ b/main.py @@ -101,7 +101,7 @@ class ConfigValidator: @staticmethod def validate_secrets(secrets_yaml_path: Path) -> tuple: secrets = ConfigValidator.validate_yaml_file(secrets_yaml_path) - mandatory_secrets = ['email', 'password', 'openai_api_key'] + mandatory_secrets = ['email', 'password'] for secret in mandatory_secrets: if secret not in secrets: @@ -114,7 +114,7 @@ class ConfigValidator: if not secrets['openai_api_key']: raise ConfigError(f"OpenAI API key cannot be empty in secrets file {secrets_yaml_path}.") - return secrets['email'], str(secrets['password']), secrets['openai_api_key'] + return secrets['email'], str(secrets['password']), secrets['openai_api_key'], secrets['openai_api_free_hosted_url'] class FileManager: @staticmethod @@ -158,7 +158,7 @@ def init_browser() -> webdriver.Chrome: except Exception as e: raise RuntimeError(f"Failed to initialize browser: {str(e)}") -def create_and_run_bot(email: str, password: str, parameters: dict, openai_api_key: str): +def create_and_run_bot(email, password, parameters, openai_api_key, openai_api_free_hosted_url): try: style_manager = StyleManager() resume_generator = ResumeGenerator() @@ -175,7 +175,7 @@ def create_and_run_bot(email: str, password: str, parameters: dict, openai_api_k browser = init_browser() login_component = LinkedInAuthenticator(browser) apply_component = LinkedInJobManager(browser) - gpt_answerer_component = GPTAnswerer(openai_api_key) + gpt_answerer_component = GPTAnswerer(openai_api_key, openai_api_free_hosted_url) bot = LinkedInBotFacade(login_component, apply_component) bot.set_secrets(email, password) bot.set_job_application_profile_and_resume(job_application_profile_object, resume_object) @@ -197,12 +197,12 @@ def main(resume: Path = None): secrets_file, config_file, plain_text_resume_file, output_folder = FileManager.validate_data_folder(data_folder) parameters = ConfigValidator.validate_config(config_file) - email, password, openai_api_key = ConfigValidator.validate_secrets(secrets_file) + email, password, openai_api_key, openai_api_free_hosted_url = ConfigValidator.validate_secrets(secrets_file) parameters['uploads'] = FileManager.file_paths_to_dict(resume, plain_text_resume_file) parameters['outputFileDirectory'] = output_folder - create_and_run_bot(email, password, parameters, openai_api_key) + create_and_run_bot(email, password, parameters, openai_api_key, openai_api_free_hosted_url) except ConfigError as ce: print(f"Configuration error: {str(ce)}") print("Refer to the configuration guide for troubleshooting: https://github.com/feder-cr/LinkedIn_AIHawk_automatic_job_application/blob/main/readme.md#configuration") diff --git a/src/gpt.py b/src/gpt.py index 371c0c2..684a4de 100644 --- a/src/gpt.py +++ b/src/gpt.py @@ -3,7 +3,7 @@ import os import re import textwrap from datetime import datetime -from typing import Dict, List +from typing import Dict, List, Union from pathlib import Path from dotenv import load_dotenv from langchain_core.messages.ai import AIMessage @@ -11,6 +11,7 @@ from langchain_core.output_parsers import StrOutputParser from langchain_core.prompt_values import StringPromptValue from langchain_core.prompts import ChatPromptTemplate from langchain_openai import ChatOpenAI +from langchain_ollama import ChatOllama from Levenshtein import distance import src.strings as strings @@ -20,7 +21,7 @@ load_dotenv() class LLMLogger: - def __init__(self, llm: ChatOpenAI): + def __init__(self, llm: Union[ChatOpenAI, ChatOllama]): self.llm = llm @staticmethod @@ -78,12 +79,12 @@ class LLMLogger: class LoggerChatModel: - def __init__(self, llm: ChatOpenAI): + def __init__(self, llm: Union[ChatOpenAI, ChatOllama]): self.llm = llm def __call__(self, messages: List[Dict[str, str]]) -> str: # Call the LLM with the provided messages and log the response. - reply = self.llm(messages) + reply = self.llm.invoke(messages) parsed_reply = self.parse_llmresult(reply) LLMLogger.log_request(prompts=messages, parsed_reply=parsed_reply) return reply @@ -113,10 +114,18 @@ class LoggerChatModel: class GPTAnswerer: - def __init__(self, openai_api_key): - self.llm_cheap = LoggerChatModel( - ChatOpenAI(model_name="gpt-4o-mini", openai_api_key=openai_api_key, temperature=0.4) - ) + def __init__(self, openai_api_key, openai_api_free_hosted_url): + if openai_api_key == "": + print('Using locally hosted mistral:v0.3') + self.llm_model = ChatOllama(model = "mistral:v0.3", temperature = 0.4, num_predict = 256) + elif openai_api_key == "freehosted": + print('Using free hosted gpt-4o-mini') + self.llm_model = ChatOpenAI(model_name="gpt-4o-mini", openai_api_key="anything", temperature=0.4, + base_url=openai_api_free_hosted_url) + else: + print("Using gpt-4o-mini") + self.llm_model = ChatOpenAI(model_name="gpt-4o-mini", openai_api_key=openai_api_key, temperature=0.4) + self.llm_cheap = LoggerChatModel(self.llm_model) @property def job_description(self): return self.job.description From ca4f56833a2834d068dc8cf0cc62ebd9f80d0873 Mon Sep 17 00:00:00 2001 From: user Date: Sat, 31 Aug 2024 22:58:40 +0200 Subject: [PATCH 02/11] Added support for ollama endpoint and defining the LLM model in the config --- README.md | 17 +++++++++-------- data_folder/config.yaml | 6 +++++- data_folder/secrets.yaml | 3 +-- data_folder_example/config.yaml | 4 ++++ data_folder_example/secrets.yaml | 3 +-- main.py | 15 ++++++--------- src/gpt.py | 23 ++++++++++++----------- 7 files changed, 38 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index de60c0e..b396b3f 100644 --- a/README.md +++ b/README.md @@ -148,17 +148,10 @@ This file contains sensitive information. Never share or commit this file to ver - Replace with your LinkedIn account email address - `password: [Your LinkedIn password]` - Replace with your LinkedIn account password -- `openai_api_key: [Your OpenAI API key]` +- `llm_api_key: [Your OpenAI or Ollama API key]` - Replace with your OpenAI API key for GPT integration - To obtain an API key, follow the tutorial at: https://medium.com/@lorenzozar/how-to-get-your-own-openai-api-key-f4d44e60c327 - Note: You need to add credit to your OpenAI account to use the API. You can add credit by visiting the [OpenAI billing dashboard](https://platform.openai.com/account/billing). -- `openai_api_free_hosted_url`: - - Optional paramter, if you want to use freely hosted GPT model, set `openai_api_key: "freehosted"` and `openai_api_free_hosted_url` with the URL of the endpoint -- Ollama local support - - If you want to use Ollama which is deployed locally, leave `openai_api_key` blank. - - To setup Ollama to run locally follow the instructions here: [Ollama installation](https://github.com/ollama/ollama). - - Download mistral model by pulling mistral:v0.3 - ### 2. config.yaml @@ -217,6 +210,14 @@ This file defines your job search parameters and bot behavior. Each section cont - Sales - Marketing ``` +- `llm_model_type`: + - Choose the model type, supported: openai / ollama +- `llm_model`: + - Choose the LLM model, currently supported: + - openai: gpt-4o + - ollama: llama2, mistral:v0.3 +- `llm_api_url`: + - Link of the API endpoint for the LLM model ### 3. plain_text_resume.yaml diff --git a/data_folder/config.yaml b/data_folder/config.yaml index 58a6f1c..e28e58a 100644 --- a/data_folder/config.yaml +++ b/data_folder/config.yaml @@ -39,4 +39,8 @@ companyBlacklist: titleBlacklist: - word1 - - word2 \ No newline at end of file + - word2 + +llm_model_type: [openai / ollama] +llm_model: ['gpt-4o' / 'mistral:v0.3'] +llm_api_url: [https://api.pawan.krd/cosmosrp/v1', http://127.0.0.1:11434/] \ No newline at end of file diff --git a/data_folder/secrets.yaml b/data_folder/secrets.yaml index 9d0bfb3..c218803 100644 --- a/data_folder/secrets.yaml +++ b/data_folder/secrets.yaml @@ -1,4 +1,3 @@ email: myemaillinkedin@gmail.com password: ImpossiblePassowrd10 -openai_api_key: sk-11KRr4uuTwpRGfeRTfj1T9BlbkFJjP8QTrswHU1yGruru2FR -openai_api_free_hosted_url: https://api.pawan.krd/cosmosrp/v1 \ No newline at end of file +llm_api_key: 'sk-11KRr4uuTwpRGfeRTfj1T9BlbkFJjP8QTrswHU1yGruru2FR' \ No newline at end of file diff --git a/data_folder_example/config.yaml b/data_folder_example/config.yaml index 6e362be..5f0a83f 100644 --- a/data_folder_example/config.yaml +++ b/data_folder_example/config.yaml @@ -37,3 +37,7 @@ companyBlacklist: - Crossover titleBlacklist: + +llm_model_type: openai +llm_model: 'gpt-4o' +llm_api_url: https://api.pawan.krd/cosmosrp/v1' \ No newline at end of file diff --git a/data_folder_example/secrets.yaml b/data_folder_example/secrets.yaml index 9d0bfb3..c218803 100644 --- a/data_folder_example/secrets.yaml +++ b/data_folder_example/secrets.yaml @@ -1,4 +1,3 @@ email: myemaillinkedin@gmail.com password: ImpossiblePassowrd10 -openai_api_key: sk-11KRr4uuTwpRGfeRTfj1T9BlbkFJjP8QTrswHU1yGruru2FR -openai_api_free_hosted_url: https://api.pawan.krd/cosmosrp/v1 \ No newline at end of file +llm_api_key: 'sk-11KRr4uuTwpRGfeRTfj1T9BlbkFJjP8QTrswHU1yGruru2FR' \ No newline at end of file diff --git a/main.py b/main.py index 82ed33f..af1f5b7 100644 --- a/main.py +++ b/main.py @@ -111,10 +111,7 @@ class ConfigValidator: raise ConfigError(f"Invalid email format in secrets file {secrets_yaml_path}.") if not secrets['password']: raise ConfigError(f"Password cannot be empty in secrets file {secrets_yaml_path}.") - if not secrets['openai_api_key']: - raise ConfigError(f"OpenAI API key cannot be empty in secrets file {secrets_yaml_path}.") - - return secrets['email'], str(secrets['password']), secrets['openai_api_key'], secrets['openai_api_free_hosted_url'] + return secrets['email'], str(secrets['password']), secrets['llm_api_key'] class FileManager: @staticmethod @@ -158,14 +155,14 @@ def init_browser() -> webdriver.Chrome: except Exception as e: raise RuntimeError(f"Failed to initialize browser: {str(e)}") -def create_and_run_bot(email, password, parameters, openai_api_key, openai_api_free_hosted_url): +def create_and_run_bot(email, password, parameters, llm_api_key): try: style_manager = StyleManager() resume_generator = ResumeGenerator() with open(parameters['uploads']['plainTextResume'], "r") as file: plain_text_resume = file.read() resume_object = Resume(plain_text_resume) - resume_generator_manager = FacadeManager(openai_api_key, style_manager, resume_generator, resume_object, Path("data_folder/output")) + resume_generator_manager = FacadeManager(llm_api_key, style_manager, resume_generator, resume_object, Path("data_folder/output")) os.system('cls' if os.name == 'nt' else 'clear') resume_generator_manager.choose_style() os.system('cls' if os.name == 'nt' else 'clear') @@ -175,7 +172,7 @@ def create_and_run_bot(email, password, parameters, openai_api_key, openai_api_f browser = init_browser() login_component = LinkedInAuthenticator(browser) apply_component = LinkedInJobManager(browser) - gpt_answerer_component = GPTAnswerer(openai_api_key, openai_api_free_hosted_url) + gpt_answerer_component = GPTAnswerer(parameters, llm_api_key) bot = LinkedInBotFacade(login_component, apply_component) bot.set_secrets(email, password) bot.set_job_application_profile_and_resume(job_application_profile_object, resume_object) @@ -197,12 +194,12 @@ def main(resume: Path = None): secrets_file, config_file, plain_text_resume_file, output_folder = FileManager.validate_data_folder(data_folder) parameters = ConfigValidator.validate_config(config_file) - email, password, openai_api_key, openai_api_free_hosted_url = ConfigValidator.validate_secrets(secrets_file) + email, password, llm_api_key = ConfigValidator.validate_secrets(secrets_file) parameters['uploads'] = FileManager.file_paths_to_dict(resume, plain_text_resume_file) parameters['outputFileDirectory'] = output_folder - create_and_run_bot(email, password, parameters, openai_api_key, openai_api_free_hosted_url) + create_and_run_bot(email, password, parameters, llm_api_key) except ConfigError as ce: print(f"Configuration error: {str(ce)}") print("Refer to the configuration guide for troubleshooting: https://github.com/feder-cr/LinkedIn_AIHawk_automatic_job_application/blob/main/readme.md#configuration") diff --git a/src/gpt.py b/src/gpt.py index 684a4de..bc71e8b 100644 --- a/src/gpt.py +++ b/src/gpt.py @@ -114,17 +114,18 @@ class LoggerChatModel: class GPTAnswerer: - def __init__(self, openai_api_key, openai_api_free_hosted_url): - if openai_api_key == "": - print('Using locally hosted mistral:v0.3') - self.llm_model = ChatOllama(model = "mistral:v0.3", temperature = 0.4, num_predict = 256) - elif openai_api_key == "freehosted": - print('Using free hosted gpt-4o-mini') - self.llm_model = ChatOpenAI(model_name="gpt-4o-mini", openai_api_key="anything", temperature=0.4, - base_url=openai_api_free_hosted_url) - else: - print("Using gpt-4o-mini") - self.llm_model = ChatOpenAI(model_name="gpt-4o-mini", openai_api_key=openai_api_key, temperature=0.4) + def __init__(self, config, llm_api_key): + llm_model_type = config['llm_model_type'] + llm_model = config['llm_model'] + llm_api_url = config['llm_api_url'] + + print('Using {0} with {1} from {2}'.format(llm_model_type, llm_model, llm_api_url)) + + if llm_model_type == "ollama": + self.llm_model = ChatOllama(model=llm_model, temperature = 0.4, base_url=llm_api_url) + elif llm_model_type == "openai": + self.llm_model = ChatOpenAI(model_name=llm_model, openai_api_key=llm_api_key, temperature=0.4, + base_url=llm_api_url) self.llm_cheap = LoggerChatModel(self.llm_model) @property def job_description(self): From a9d9e13474b29b05e9f2c537b3dd34b091bb33c5 Mon Sep 17 00:00:00 2001 From: user Date: Sun, 1 Sep 2024 11:59:43 +0200 Subject: [PATCH 03/11] Adapter class added --- README.md | 3 +- data_folder/config.yaml | 8 ++--- src/gpt.py | 79 +++++++++++++++++++++++++++++++++-------- 3 files changed, 70 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index b396b3f..fc4f060 100644 --- a/README.md +++ b/README.md @@ -211,11 +211,12 @@ This file defines your job search parameters and bot behavior. Each section cont - Marketing ``` - `llm_model_type`: - - Choose the model type, supported: openai / ollama + - Choose the model type, supported: openai / ollama / claude - `llm_model`: - Choose the LLM model, currently supported: - openai: gpt-4o - ollama: llama2, mistral:v0.3 + - claude: any model - `llm_api_url`: - Link of the API endpoint for the LLM model diff --git a/data_folder/config.yaml b/data_folder/config.yaml index e28e58a..60647c0 100644 --- a/data_folder/config.yaml +++ b/data_folder/config.yaml @@ -40,7 +40,7 @@ companyBlacklist: titleBlacklist: - word1 - word2 - -llm_model_type: [openai / ollama] -llm_model: ['gpt-4o' / 'mistral:v0.3'] -llm_api_url: [https://api.pawan.krd/cosmosrp/v1', http://127.0.0.1:11434/] \ No newline at end of file + +llm_model_type: [openai / ollama / claude] +llm_model: ['gpt-4o' / 'mistral:v0.3' / anymodel] +llm_api_url: [https://api.pawan.krd/cosmosrp/v1' / http://127.0.0.1:11434/] \ No newline at end of file diff --git a/src/gpt.py b/src/gpt.py index bc71e8b..b4b66bc 100644 --- a/src/gpt.py +++ b/src/gpt.py @@ -3,6 +3,7 @@ import os import re import textwrap from datetime import datetime +from abc import ABC, abstractmethod from typing import Dict, List, Union from pathlib import Path from dotenv import load_dotenv @@ -11,17 +12,75 @@ from langchain_core.output_parsers import StrOutputParser from langchain_core.prompt_values import StringPromptValue from langchain_core.prompts import ChatPromptTemplate from langchain_openai import ChatOpenAI -from langchain_ollama import ChatOllama from Levenshtein import distance import src.strings as strings load_dotenv() +class AIModel(ABC): + @abstractmethod + def generate_response(self, prompt: str) -> str: + pass + +class OpenAIModel(AIModel): + def __init__(self, api_key: str, llm_model: str, llm_api_url: str): + from langchain_openai import ChatOpenAI + self.model = ChatOpenAI(model_name=llm_model, openai_api_key=api_key, + temperature=0.4, base_url=llm_api_url) + + def generate_response(self, prompt: str) -> str: + response = self.model.invoke(prompt) + return response.content + +class ClaudeModel(AIModel): + def __init__(self, api_key: str, llm_model: str, llm_api_url: str): + from anthropic import Anthropic + self.client = Anthropic(api_key=api_key) + + def generate_response(self, prompt: str) -> str: + formatted_prompt = f"\n\nHuman: {prompt}\n\nAssistant:" + response = self.client.completions.create( + model="claude-2", + prompt=formatted_prompt, + max_tokens_to_sample=300 + ) + return response.completion.strip() + +class OllamaModel(AIModel): + def __init__(self, api_key: str, llm_model: str, llm_api_url: str): + from langchain_ollama import ChatOllama + self.model = ChatOllama(model=llm_model, base_url=llm_api_url) + + def generate_response(self, prompt: str) -> str: + response = self.model.invoke(prompt) + return response.content + +class AIAdapter: + def __init__(self, config: dict, api_key: str): + self.model = self._create_model(config, api_key) + + def _create_model(self, config: dict, api_key: str) -> AIModel: + llm_model_type = config['llm_model_type'] + llm_model = config['llm_model'] + llm_api_url = config['llm_api_url'] + print('Using {0} with {1} from {2}'.format(llm_model_type, llm_model, llm_api_url)) + + if llm_model_type == "openai": + return OpenAIModel(api_key, llm_model, llm_api_url) + elif llm_model_type == "claude": + return ClaudeModel(api_key, llm_model, llm_api_url) + elif llm_model_type == "ollama": + return OllamaModel(api_key, llm_model, llm_api_url) + else: + raise ValueError(f"Unsupported model type: {model_type}") + + def generate_response(self, prompt: str) -> str: + return self.model.generate_response(prompt) class LLMLogger: - def __init__(self, llm: Union[ChatOpenAI, ChatOllama]): + def __init__(self, llm: Union[OpenAIModel, OllamaModel, ClaudeModel]): self.llm = llm @staticmethod @@ -79,7 +138,7 @@ class LLMLogger: class LoggerChatModel: - def __init__(self, llm: Union[ChatOpenAI, ChatOllama]): + def __init__(self, llm: Union[OpenAIModel, OllamaModel, ClaudeModel]): self.llm = llm def __call__(self, messages: List[Dict[str, str]]) -> str: @@ -115,18 +174,8 @@ class LoggerChatModel: class GPTAnswerer: def __init__(self, config, llm_api_key): - llm_model_type = config['llm_model_type'] - llm_model = config['llm_model'] - llm_api_url = config['llm_api_url'] - - print('Using {0} with {1} from {2}'.format(llm_model_type, llm_model, llm_api_url)) - - if llm_model_type == "ollama": - self.llm_model = ChatOllama(model=llm_model, temperature = 0.4, base_url=llm_api_url) - elif llm_model_type == "openai": - self.llm_model = ChatOpenAI(model_name=llm_model, openai_api_key=llm_api_key, temperature=0.4, - base_url=llm_api_url) - self.llm_cheap = LoggerChatModel(self.llm_model) + self.ai_adapter = AIAdapter(config, llm_api_key) + self.llm_cheap = LoggerChatModel(self.ai_adapter) @property def job_description(self): return self.job.description From 6b4ebe1bd0546771b2e105788ab6a9f759377633 Mon Sep 17 00:00:00 2001 From: user Date: Sun, 1 Sep 2024 12:01:39 +0200 Subject: [PATCH 04/11] config.yaml corrected --- data_folder/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_folder/config.yaml b/data_folder/config.yaml index 60647c0..f38ce82 100644 --- a/data_folder/config.yaml +++ b/data_folder/config.yaml @@ -42,5 +42,5 @@ titleBlacklist: - word2 llm_model_type: [openai / ollama / claude] -llm_model: ['gpt-4o' / 'mistral:v0.3' / anymodel] +llm_model: [gpt-4o / mistral:v0.3 / anymodel] llm_api_url: [https://api.pawan.krd/cosmosrp/v1' / http://127.0.0.1:11434/] \ No newline at end of file From 9ed0c3ea6385d16177703fb3fe9ed45d9fb5cc5d Mon Sep 17 00:00:00 2001 From: Earl Perry Date: Sun, 1 Sep 2024 09:01:52 -0400 Subject: [PATCH 05/11] Instead of adding a documentation section, improved trouble shooting guide, with current issues, solutions, and addtional resources --- README.md | 85 ++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 75 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 15fe7b8..adb4eb1 100644 --- a/README.md +++ b/README.md @@ -501,19 +501,84 @@ Using this folder as a guide can be particularly helpful for: python main.py --resume /path/to/your/resume.pdf ``` -## Documentation -TODO ): +### Troubleshooting Common Issues -## Troubleshooting +#### 1. OpenAI API Rate Limit Errors + +**Error Message:** + +openai.RateLimitError: Error code: 429 - {'error': {'message': 'You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.', 'type': 'insufficient_quota', 'param': None, 'code': 'insufficient_quota'}} + +**Solution:** +- Check your OpenAI API billing settings at https://platform.openai.com/account/billing +- Ensure you have added a valid payment method to your OpenAI account +- Note that ChatGPT Plus subscription is different from API access +- If you've recently added funds or upgraded, wait 12-24 hours for changes to take effect +- Free tier has a 3 RPM limit; spend at least $5 on API usage to increase + +#### 2. LinkedIn Easy Apply Button Not Found + +**Error Message:** + +Exception: No clickable 'Easy Apply' button found + +**Solution:** +- Ensure that you're logged into LinkedIn properly +- Check if the job listings you're targeting actually have the "Easy Apply" option +- Verify that your search parameters in the `config.yaml` file are correct and returning jobs with the "Easy Apply" button +- Try increasing the wait time for page loading in the script to ensure all elements are loaded before searching for the button + +#### 3. Incorrect Information in Job Applications + +**Issue:** Bot provides inaccurate data for experience, CTC, and notice period + +**Solution:** +- Update prompts for professional experience specificity +- Add fields in `config.yaml` for current CTC, expected CTC, and notice period +- Modify bot logic to use these new config fields + +#### 4. YAML Configuration Errors + +**Error Message:** + +yaml.scanner.ScannerError: while scanning a simple key + +**Solution:** +- Copy example `config.yaml` and modify gradually +- Ensure proper YAML indentation and spacing +- Use a YAML validator tool +- Avoid unnecessary special characters or quotes + +#### 5. Bot Logs In But Doesn't Apply to Jobs + +**Issue:** Bot searches for jobs but continues scrolling without applying + +**Solution:** +- Check for security checks or CAPTCHAs +- Verify `config.yaml` job search parameters +- Ensure your LinkedIn profile meets job requirements +- Review console output for error messages + +### General Troubleshooting Tips + +- Use the latest version of the script +- Verify all dependencies are installed and updated +- Check internet connection stability +- Use VPNs cautiously to avoid triggering LinkedIn security +- Clear browser cache and cookies if issues persist + +For further assistance, please create an issue on the [GitHub repository](https://github.com/feder-cr/LinkedIn_AIHawk_automatic_job_application/issues) with detailed information about your problem, including error messages and your configuration (with sensitive information removed). + +### Additional Resources + +- [Video Tutorial: How to set up LinkedIn_AIHawk](https://youtu.be/gdW9wogHEUM) +- [OpenAI API Documentation](https://platform.openai.com/docs/) +- [LinkedIn Developer Documentation](https://developer.linkedin.com/) +- [Lang Chain Developer Documentation](https://python.langchain.com/v0.2/docs/integrations/components/) + +Remember to always use LinkedIn_AIHawk responsibly and in compliance with LinkedIn's terms of service. -- **Carefully read logs and output :** Most of the errors are verbosely reflected just watch the output and try to find the root couse. -- **If nothing works by unknown reason:** Use tested OS. Reboot and/or update OS. Use new clean venv. Try update Python to the tested version. -- **ChromeDriver Issues:** Ensure ChromeDriver is compatible with your installed Chrome version. -- **Missing Files:** Verify that all necessary files are present in the data folder. -- **Invalid YAML:** Check your YAML files for syntax errors . Try to use external YAML validators e.g. https://www.yamllint.com/ -- **OpenAI endpoint isues**: Try to check possible limits\blocking at their side - If you encounter any issues, you can open an issue on [GitHub](https://github.com/feder-cr/linkedIn_auto_jobs_applier_with_AI/issues). Please add valuable details to the subject and to the description. If you need new feature then please reflect this. I'll be more than happy to assist you! From 777d1fa4a6cd94ecf6805c671eacda85bec1d28f Mon Sep 17 00:00:00 2001 From: user Date: Sun, 1 Sep 2024 19:33:42 +0200 Subject: [PATCH 06/11] Claude support with langchain antrhopic --- data_folder/config.yaml | 4 ++-- requirements.txt | Bin 674 -> 766 bytes src/gpt.py | 31 ++++++++++++++----------------- 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/data_folder/config.yaml b/data_folder/config.yaml index f38ce82..d26a6cc 100644 --- a/data_folder/config.yaml +++ b/data_folder/config.yaml @@ -42,5 +42,5 @@ titleBlacklist: - word2 llm_model_type: [openai / ollama / claude] -llm_model: [gpt-4o / mistral:v0.3 / anymodel] -llm_api_url: [https://api.pawan.krd/cosmosrp/v1' / http://127.0.0.1:11434/] \ No newline at end of file +llm_model: [gpt-4o / mistral:v0.3 / claude-3-5-sonnet-20240620] +llm_api_url: [https://api.pawan.krd/cosmosrp/v1', http://127.0.0.1:11434/, https://api.anthropic.com/v1/messages] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index aef4baed9ae147969488a70f26fd5aae9edacb36..0edf5dfd3cbcff11444a30bf3b3f85ae1798dc54 100644 GIT binary patch delta 57 zcmZ3)`j2(OEy;X_93V_&$OU3sAT(gmV=x3_XG3844IO8Im_1 He8>m@q~8vv delta 11 Scmeyzx`=hct<7ePCl~=A_youR diff --git a/src/gpt.py b/src/gpt.py index b4b66bc..22c7d6c 100644 --- a/src/gpt.py +++ b/src/gpt.py @@ -20,7 +20,7 @@ load_dotenv() class AIModel(ABC): @abstractmethod - def generate_response(self, prompt: str) -> str: + def invoke(self, prompt: str) -> str: pass class OpenAIModel(AIModel): @@ -29,32 +29,29 @@ class OpenAIModel(AIModel): self.model = ChatOpenAI(model_name=llm_model, openai_api_key=api_key, temperature=0.4, base_url=llm_api_url) - def generate_response(self, prompt: str) -> str: + def invoke(self, prompt: str) -> str: + print("invoke in openai") response = self.model.invoke(prompt) - return response.content + return response class ClaudeModel(AIModel): def __init__(self, api_key: str, llm_model: str, llm_api_url: str): - from anthropic import Anthropic - self.client = Anthropic(api_key=api_key) + from langchain_anthropic import ChatAnthropic + self.model = ChatAnthropic(model=llm_model, api_key=api_key, + temperature=0.4, base_url=llm_api_url) - def generate_response(self, prompt: str) -> str: - formatted_prompt = f"\n\nHuman: {prompt}\n\nAssistant:" - response = self.client.completions.create( - model="claude-2", - prompt=formatted_prompt, - max_tokens_to_sample=300 - ) - return response.completion.strip() + def invoke(self, prompt: str) -> str: + response = self.model.invoke(prompt) + return response class OllamaModel(AIModel): def __init__(self, api_key: str, llm_model: str, llm_api_url: str): from langchain_ollama import ChatOllama self.model = ChatOllama(model=llm_model, base_url=llm_api_url) - def generate_response(self, prompt: str) -> str: + def invoke(self, prompt: str) -> str: response = self.model.invoke(prompt) - return response.content + return response class AIAdapter: def __init__(self, config: dict, api_key: str): @@ -75,8 +72,8 @@ class AIAdapter: else: raise ValueError(f"Unsupported model type: {model_type}") - def generate_response(self, prompt: str) -> str: - return self.model.generate_response(prompt) + def invoke(self, prompt: str) -> str: + return self.model.invoke(prompt) class LLMLogger: From 28b8fa37469fab8b09b454911f967c8e5be0a825 Mon Sep 17 00:00:00 2001 From: user Date: Sun, 1 Sep 2024 21:15:03 +0200 Subject: [PATCH 07/11] README updated --- README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fc4f060..cba2849 100644 --- a/README.md +++ b/README.md @@ -210,7 +210,9 @@ This file defines your job search parameters and bot behavior. Each section cont - Sales - Marketing ``` -- `llm_model_type`: +#### 2.1 config.yaml - Customize LLM model endpoint + +- `llm_model_type`: - Choose the model type, supported: openai / ollama / claude - `llm_model`: - Choose the LLM model, currently supported: @@ -219,7 +221,11 @@ This file defines your job search parameters and bot behavior. Each section cont - claude: any model - `llm_api_url`: - Link of the API endpoint for the LLM model - + - openai: https://api.pawan.krd/cosmosrp/v1 + - ollama: http://127.0.0.1:11434/ + - claude: https://api.anthropic.com/v1 + - Note: To run local Ollama, follow the guidelines here: [Guide to Ollama deployment](https://github.com/ollama/ollama) + ### 3. plain_text_resume.yaml This file contains your resume information in a structured format. Fill it out with your personal details, education, work experience, and skills. This information is used to auto-fill application forms and generate customized resumes. From d1d9e9f6a3eedbe4a0399f632559a4a332b740e7 Mon Sep 17 00:00:00 2001 From: user Date: Sun, 1 Sep 2024 21:16:02 +0200 Subject: [PATCH 08/11] config yaml file with default openai gpt freely hosted --- data_folder/config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data_folder/config.yaml b/data_folder/config.yaml index d26a6cc..bfbcd82 100644 --- a/data_folder/config.yaml +++ b/data_folder/config.yaml @@ -41,6 +41,6 @@ titleBlacklist: - word1 - word2 -llm_model_type: [openai / ollama / claude] -llm_model: [gpt-4o / mistral:v0.3 / claude-3-5-sonnet-20240620] -llm_api_url: [https://api.pawan.krd/cosmosrp/v1', http://127.0.0.1:11434/, https://api.anthropic.com/v1/messages] \ No newline at end of file +llm_model_type: openai +llm_model: gpt-4o +llm_api_url: https://api.pawan.krd/cosmosrp/v1 \ No newline at end of file From c0f186ba844f7b12ea515675709c4977bc817e32 Mon Sep 17 00:00:00 2001 From: Manu Altieri Date: Sun, 1 Sep 2024 22:48:01 +0200 Subject: [PATCH 09/11] Fixed easy apply search, implemented method to extract required fields from API with new endpoint, Todo; parse result json to return a dict that contains title, optionUrn and possible choices for the value if present --- src/linkedin-api.py | 76 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 73 insertions(+), 3 deletions(-) diff --git a/src/linkedin-api.py b/src/linkedin-api.py index f1395f8..6bb2c12 100644 --- a/src/linkedin-api.py +++ b/src/linkedin-api.py @@ -1,12 +1,19 @@ from typing import Dict, List from linkedin_api import Linkedin from typing import Optional, Union, Literal -from urllib.parse import urlencode +from urllib.parse import quote, urlencode +import logging +import json + +# set log to all debug +logging.basicConfig(level=logging.INFO) + class LinkedInEvolvedAPI(Linkedin): def __init__(self, username, password): super().__init__(username, password) + def search_jobs( self, keywords: Optional[str] = None, @@ -106,7 +113,7 @@ class LinkedInEvolvedAPI(Linkedin): if remote: query["selectedFilters"]["workplaceType"] = f"List({','.join(remote)})" if easy_apply: - query["selectedFilters"]["easyApply"] = "List(true)" + query["selectedFilters"]["applyWithLinkedin"] = "List(true)" query["selectedFilters"]["timePostedRange"] = f"List(r{listed_at})" query["spellCorrectionEnabled"] = "true" @@ -160,9 +167,72 @@ class LinkedInEvolvedAPI(Linkedin): self.logger.debug(f"results grew to {len(results)}") return results - + def get_fields_for_easy_apply(self,job_id:str) -> List[Dict]: + """Get fields needed for easy apply jobs. + + :param job_id: Job ID + :type job_id: str + :return: Fields + :rtype: dict + """ + + cookies = self.client.session.cookies.get_dict() + cookie_str = "; ".join([f"{k}={v}" for k, v in cookies.items()]) + + headers: Dict[str, str] = self._headers() + + headers['User-Agent'] = headers['user-agent'] + headers['Accept-Language'] = headers['accept-language'] + headers["Accept"] = "application/vnd.linkedin.normalized+json+2.1" + headers["csrf-token"] = cookies["JSESSIONID"].replace('"', "") + headers["Cookie"] = cookie_str + headers["Connection"] = "keep-alive" + + headers.pop("user-agent") + headers.pop("accept-language") + default_params = { + "decorationId": "com.linkedin.voyager.dash.deco.jobs.OnsiteApplyApplication-67", + "jobPostingUrn": f"urn:li:fsd_jobPosting:{job_id}", + "q": "jobPosting", + } + + default_params = urlencode(default_params) + res = self._fetch( + f"/voyagerJobsDashOnsiteApplyApplication?{default_params}", + headers=headers, + cookies=cookies, + ) + + try: + data = res.json() + except ValueError: + self.logger.error("Failed to parse JSON response") + return [] + + form_components = [] + + for item in data.get("included", []): + if 'formComponent' in item: + title = item['title']['text'] + form_components.append({title: item['formComponent']}) + + + + + def get_cookies_hitting_url(self, url: str): + res = self._fetch(url,base_request=False) + return res.headers + +## EXAMPLE USAGE +#if __name__ == "__main__": +# api: LinkedInEvolvedAPI = LinkedInEvolvedAPI("", "") +# jobs = api.search_jobs(keywords="Python", location_name="Italy", limit=1, easy_apply=True) +# for job in jobs: +# job_id: str = job["job_id"] +# fields = api.get_fields_for_easy_apply(job_id) + \ No newline at end of file From d76a35fa7ca0d53f93035ca8ed5e2f314b5b424d Mon Sep 17 00:00:00 2001 From: Manu Altieri Date: Sun, 1 Sep 2024 22:53:05 +0200 Subject: [PATCH 10/11] Fixed easy apply search, implemented method to extract required fields from API with new endpoint, Todo; parse result json to return a dict that contains title, optionUrn and possible choices for the value if present --- src/linkedin-api.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/linkedin-api.py b/src/linkedin-api.py index 6bb2c12..8b642ce 100644 --- a/src/linkedin-api.py +++ b/src/linkedin-api.py @@ -222,10 +222,6 @@ class LinkedInEvolvedAPI(Linkedin): - def get_cookies_hitting_url(self, url: str): - res = self._fetch(url,base_request=False) - return res.headers - ## EXAMPLE USAGE #if __name__ == "__main__": # api: LinkedInEvolvedAPI = LinkedInEvolvedAPI("", "") From d2372523fa876c3abbceffd9c7b371a3afb10535 Mon Sep 17 00:00:00 2001 From: Manu Altieri Date: Mon, 2 Sep 2024 11:10:18 +0200 Subject: [PATCH 11/11] fixed the parsing of the required fields for apply --- src/linkedin-api.py | 66 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 50 insertions(+), 16 deletions(-) diff --git a/src/linkedin-api.py b/src/linkedin-api.py index 8b642ce..a34e381 100644 --- a/src/linkedin-api.py +++ b/src/linkedin-api.py @@ -13,7 +13,6 @@ class LinkedInEvolvedAPI(Linkedin): def __init__(self, username, password): super().__init__(username, password) - def search_jobs( self, keywords: Optional[str] = None, @@ -182,16 +181,12 @@ class LinkedInEvolvedAPI(Linkedin): headers: Dict[str, str] = self._headers() - headers['User-Agent'] = headers['user-agent'] - headers['Accept-Language'] = headers['accept-language'] + headers["Accept"] = "application/vnd.linkedin.normalized+json+2.1" headers["csrf-token"] = cookies["JSESSIONID"].replace('"', "") headers["Cookie"] = cookie_str headers["Connection"] = "keep-alive" - headers.pop("user-agent") - headers.pop("accept-language") - default_params = { "decorationId": "com.linkedin.voyager.dash.deco.jobs.OnsiteApplyApplication-67", @@ -206,6 +201,16 @@ class LinkedInEvolvedAPI(Linkedin): cookies=cookies, ) + match res.status_code: + case 200: + pass + case 409: + self.logger.error("Failed to fetch fields for easy apply job because already applied to this job!") + return [] + case _: + self.logger.error("Failed to fetch fields for easy apply job") + return [] + try: data = res.json() except ValueError: @@ -216,19 +221,48 @@ class LinkedInEvolvedAPI(Linkedin): for item in data.get("included", []): if 'formComponent' in item: - title = item['title']['text'] - form_components.append({title: item['formComponent']}) + urn = item['urn'] + try: + title = item['title']['text'] + except TypeError: + title = urn + + form_component_type = list(item['formComponent'].keys())[0] + form_component_details = item['formComponent'][form_component_type] + + component_info = { + 'title': title, + 'urn': urn, + 'formComponentType': form_component_type, + } + + if 'textSelectableOptions' in form_component_details: + options = [ + opt['optionText']['text'] for opt in form_component_details['textSelectableOptions'] + ] + component_info['selectableOptions'] = options + elif 'selectableOptions' in form_component_details: + options = [ + opt['textSelectableOption']['optionText']['text'] + for opt in form_component_details['selectableOptions'] + ] + component_info['selectableOptions'] = options + + form_components.append(component_info) + return form_components - - ## EXAMPLE USAGE -#if __name__ == "__main__": -# api: LinkedInEvolvedAPI = LinkedInEvolvedAPI("", "") -# jobs = api.search_jobs(keywords="Python", location_name="Italy", limit=1, easy_apply=True) -# for job in jobs: -# job_id: str = job["job_id"] -# fields = api.get_fields_for_easy_apply(job_id) +if __name__ == "__main__": + api: LinkedInEvolvedAPI = LinkedInEvolvedAPI(username="", password="") + jobs = api.search_jobs(keywords="Frontend Developer", location_name="Italia", limit=5, easy_apply=True, offset=1) + for job in jobs: + job_id: str = job["job_id"] + + fields = api.get_fields_for_easy_apply(job_id) + for field in fields: + print(field) + \ No newline at end of file