From 3bbf895f5e7fd3e38e6d1a80357d0fd849431c58 Mon Sep 17 00:00:00 2001 From: wehub-resource-sync Date: Mon, 13 Jul 2026 12:29:44 +0800 Subject: [PATCH] chore: import upstream snapshot with attribution --- .env.example | 11 + .gitignore | 226 +++ .python-version | 1 + CONTRIBUTING.md | 69 + LICENSE | 21 + README.md | 284 ++++ README.wehub.md | 7 + config.py | 6 + evaluator.py | 91 ++ github.py | 496 ++++++ llm_utils.py | 62 + models.py | 391 +++++ pdf.py | 323 ++++ prompt.py | 67 + prompts/template_manager.py | 90 ++ prompts/templates/awards.jinja | 20 + prompts/templates/basics.jinja | 55 + prompts/templates/education.jinja | 23 + .../templates/github_project_selection.jinja | 100 ++ prompts/templates/projects.jinja | 21 + .../resume_evaluation_criteria.jinja | 185 +++ .../resume_evaluation_system_message.jinja | 49 + prompts/templates/skills.jinja | 20 + prompts/templates/system_message.jinja | 5 + prompts/templates/work.jinja | 36 + pymupdf_rag.py | 1377 +++++++++++++++++ requirements.txt | 9 + score.py | 377 +++++ transform.py | 938 +++++++++++ 29 files changed, 5360 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 .python-version create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE create mode 100644 README.md create mode 100644 README.wehub.md create mode 100644 config.py create mode 100644 evaluator.py create mode 100644 github.py create mode 100644 llm_utils.py create mode 100644 models.py create mode 100644 pdf.py create mode 100644 prompt.py create mode 100644 prompts/template_manager.py create mode 100644 prompts/templates/awards.jinja create mode 100644 prompts/templates/basics.jinja create mode 100644 prompts/templates/education.jinja create mode 100644 prompts/templates/github_project_selection.jinja create mode 100644 prompts/templates/projects.jinja create mode 100644 prompts/templates/resume_evaluation_criteria.jinja create mode 100644 prompts/templates/resume_evaluation_system_message.jinja create mode 100644 prompts/templates/skills.jinja create mode 100644 prompts/templates/system_message.jinja create mode 100644 prompts/templates/work.jinja create mode 100644 pymupdf_rag.py create mode 100644 requirements.txt create mode 100644 score.py create mode 100644 transform.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..cae00ff --- /dev/null +++ b/.env.example @@ -0,0 +1,11 @@ +# LLM Provider Configuration +# Options: "ollama" or "gemini" +LLM_PROVIDER=ollama + +# Default model to use +# For Ollama: "gemma3:4b", "qwen3:4b", "mistral:7b", etc. +# For Gemini: "gemini-2.5-pro", "gemini-2.5-flash", etc. +DEFAULT_MODEL=gemma3:4b + +# Google Gemini API Key (required if using Gemini provider) +GEMINI_API_KEY=your_gemini_api_key_here diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a2e75f9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,226 @@ +# Project Specific Files + +**/*.pyc +resume/*.pdf +run/*.pdf +test_*.py +cache/ +resume_evaluations.csv +greenhouse_resumes/* + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[codz] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py.cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +# Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +# poetry.lock +# poetry.toml + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. +# https://pdm-project.org/en/latest/usage/project/#working-with-version-control +# pdm.lock +# pdm.toml +.pdm-python +.pdm-build/ + +# pixi +# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. +# pixi.lock +# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one +# in the .venv directory. It is recommended not to include this directory in version control. +.pixi + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# Redis +*.rdb +*.aof +*.pid + +# RabbitMQ +mnesia/ +rabbitmq/ +rabbitmq-data/ + +# ActiveMQ +activemq-data/ + +# SageMath parsed files +*.sage.py + +# Environments +.env +.envrc +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +# .idea/ + +# Abstra +# Abstra is an AI-powered process automation framework. +# Ignore directories containing user credentials, local state, and settings. +# Learn more at https://abstra.io/docs +.abstra/ + +# Visual Studio Code +# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore +# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore +# and can be added to the global gitignore or merged into this file. However, if you prefer, +# you could uncomment the following to ignore the entire vscode folder +# .vscode/ + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc + +# Marimo +marimo/_static/ +marimo/_lsp/ +__marimo__/ + +# Streamlit +.streamlit/secrets.toml \ No newline at end of file diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..fa74ffc --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.11.13 \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..e997880 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,69 @@ +# Contributing Guidelines + +Thanks for your interest in improving this project. Contributions are welcome, including documentation updates, bug reports, feature requests, and code changes. + +## Reporting Bugs + +1. Check that the bug has not already been reported: https://github.com/interviewstreet/hiring-agent/issues +2. Open a new bug report: https://github.com/interviewstreet/hiring-agent/issues/new +3. Please include: + - Clear description of the issue and expected behavior + - Environment info: OS, Python version, hiring-agent commit or version + - Steps to reproduce + - Relevant logs or stack traces + +> [!TIP] +> If the bug involves PDF parsing or model output, attach a minimal PDF and the exact model you used. + +## Feature Requests + +1. Check for existing requests: https://github.com/interviewstreet/hiring-agent/issues +2. Open a new feature request: https://github.com/interviewstreet/hiring-agent/issues/new +3. Describe the problem, the proposed solution, and any alternatives you considered. + +## Contributing Code + +1. Pick an issue from https://github.com/interviewstreet/hiring-agent/issues or open one first. +2. Comment that you are working on it to avoid duplicate efforts. +3. Fork the repo, then create a feature branch for your change. + +### Development + +1. Fork and clone your fork. +2. Create a fresh branch per change. Avoid pushing changes to the default branch of your fork. +3. Set up the environment and run the pipeline locally to validate changes +4. Run the CLI on a small sample to verify behavior: + +### Coding Style + +* Use [Black](https://black.readthedocs.io/en/stable/) for formatting. +* Keep functions short and focused. Prefer pure helpers for prompt assembly and transformations. + +### Prompts and Providers + +* Keep prompts declarative and provider agnostic. +* Avoid model specific tokens or formatting that only one provider supports. +* Changes to prompts should include short before and after examples in the pull request description. + +### Tests and Smoke Checks + +* Validate changes with a couple of real resumes under different providers when possible: + + * One run with Ollama using the default local model. + * One run with Gemini if you have an API key. +* Add or adjust small smoke tests that exercise each stage with minimal inputs: + + * PDF to Markdown + * Section extraction to JSON Resume + * GitHub enrichment on a known username + * Evaluation to JSON with the required fields + + +### Commit Messages + +* Use clear, imperative subjects, for example: `fix: handle en dash date ranges in work parser`. +* Reference the issue number when applicable. + +## Code of Conduct + +Be respectful and collaborative. If you see unacceptable behavior, report it through an issue or contact the maintainers. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c9faf69 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 HackerRank + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..0396ef4 --- /dev/null +++ b/README.md @@ -0,0 +1,284 @@ +# Hiring Agent + +

Resume-to-Score pipeline that extracts structured data from PDFs, enriches with GitHub signals, and outputs a fair, explainable evaluation.

+ +

+ + Python + + + License: MIT + + + Code style: Black + +

+ +--- + +## Contents + +- [Overview](#overview) +- [Architecture](#architecture) +- [Installation and Setup](#installation-and-setup) + - [Prerequisites](#prerequisites) + - [Quick setup with pip](#quick-setup-with-pip) + - [Ollama models](#ollama-models) +- [Configuration](#configuration) +- [How it works](#how-it-works) +- [CLI usage](#cli-usage) +- [Directory layout](#directory-layout) +- [Provider details](#provider-details) +- [Contributing](#contributing) +- [License](#license) + +--- + +## Overview + +Hiring Agent parses a resume PDF to Markdown, extracts sectioned JSON using a local or hosted LLM, augments the data with GitHub profile and repository signals, then produces an objective evaluation with category scores, evidence, bonus points, and deductions. You can run fully local with Ollama or use Google Gemini. + +--- + +## Architecture + + + + + + +
+ +**Flow** + +1. `pymupdf_rag.py` converts PDF pages to Markdown-like text. +2. `pdf.py` calls the LLM per section using Jinja templates under `prompts/templates`. +3. `github.py` fetches profile and repos, classifies projects, and asks the LLM to select the top 7. +4. `evaluator.py` runs a strict-scored evaluation with fairness constraints. +5. `score.py` orchestrates everything end to end and writes CSV when development mode is on. + + + +**Key modules** + +- `models.py` + Pydantic schemas and LLM provider interfaces. + +- `llm_utils.py` + Provider initialization and response cleanup. + +- `transform.py` + Normalization from loose LLM JSON to JSON Resume style. + +- `prompts/` + All Jinja templates for extraction and scoring. + +
+ +--- + +## Installation and Setup + +### Prerequisites + +- **Python 3.11+** + + The repository pins `.python-version` to 3.11.13. + +- **One LLM backend** (either of them) + + - **Ollama** for local models + Install from the [official site](https://ollama.com/), then run `ollama serve`. + - **Google Gemini** if you have an API key, get it from [here](https://aistudio.google.com/api-keys). + +### Quick setup with pip + +```bash +$ git clone https://github.com/interviewstreet/hiring-agent +$ cd hiring-agent + +$ python -m venv .venv +# Linux or macOS +$ source .venv/bin/activate +# Windows +# .venv\Scripts\activate + +$ pip install -r requirements.txt +``` + +### Ollama Models + +Pull the model you want to use. For example: + +```bash +$ ollama pull gemma3:4b +``` + +If you want different results, you can pull other models such as: + +```bash +# For higher system configuration +$ ollama pull gemma3:12b + +# For lower system configuration +$ ollama pull gemma3:1b +``` + +--- + +## Configuration + +Copy the template and set your environment variables. + +```bash +$ cp .env.example .env +``` + +**Environment variables** + +| Variable | Values | Description | +| ---------------- | ------------------------------------------- | ---------------------------------------------------------------------- | +| `LLM_PROVIDER` | `ollama` or `gemini` | Chooses provider. Defaults to Ollama. | +| `DEFAULT_MODEL` | for example `gemma3:4b` or `gemini-2.5-pro` | Model name passed to the provider. | +| `GEMINI_API_KEY` | string | Required when `LLM_PROVIDER=gemini`. | +| `GITHUB_TOKEN` | optional | Inherits from your shell environment, improves GitHub API rate limits. | + +Provider mapping lives in `prompt.py` and `models.py`. The `config.py` file has a single flag: + +```python +# config.py +DEVELOPMENT_MODE = True # enables caching and CSV export +``` + +You can leave it on during iteration. See the next section for details. + +--- + +## How it works + +
+1) PDF extraction + +- `pymupdf_rag.py` and `pdf.py` read the PDF using PyMuPDF and convert pages to Markdown-like text. +- The `to_markdown` routine handles headings, links, tables, and basic formatting. + +
+ +
+2) Section parsing with templates + +- `prompts/templates/*.jinja` define strict instructions for each section + Basics, Work, Education, Skills, Projects, Awards. +- `pdf.PDFHandler` calls the LLM per section and assembles a `JSONResume` object (see `models.py`). + +
+ +
+3) GitHub enrichment + +- `github.py` extracts a username from the resume profiles, fetches profile and repos, and classifies each project. +- It asks the LLM to select exactly 7 unique projects with a minimum author commit threshold, favoring meaningful contributions. + +
+ +
+4) Evaluation + +- `evaluator.py` uses templates that encode fairness and scoring rules. +- Scores include `open_source`, `self_projects`, `production`, and `technical_skills`, plus bonus and deductions, then an explanation for evidence. + +
+ +
+5) Output and CSV export + +- `score.py` prints a readable summary to stdout. +- When `DEVELOPMENT_MODE=True` it creates or appends a `resume_evaluations.csv` with key fields, and caches intermediate JSON under `cache/`. + +
+ +--- + +## CLI usage + +### End to end scoring + +Provide a path to a resume PDF. + +```bash +$ python score.py /path/to/resume.pdf +``` + +What happens: + +1. If development mode is on, the PDF extraction result is cached to `cache/resumecache_.json`. +2. If a GitHub profile is found in the resume, repositories are fetched and cached to `cache/githubcache_.json`. +3. The evaluator prints a report and, in development mode, appends a CSV row to `resume_evaluations.csv`. + +--- + +## Directory layout + +```text +. +├── .env.example +├── .python-version +├── config.py +├── evaluator.py +├── github.py +├── llm_utils.py +├── models.py +├── pdf.py +├── prompt.py +├── prompts/ +│ ├── template_manager.py +│ └── templates/ +│ ├── awards.jinja +│ ├── basics.jinja +│ ├── education.jinja +│ ├── github_project_selection.jinja +│ ├── projects.jinja +│ ├── resume_evaluation_criteria.jinja +│ ├── resume_evaluation_system_message.jinja +│ ├── skills.jinja +│ ├── system_message.jinja +│ └── work.jinja +├── pymupdf_rag.py +├── requirements.txt +├── score.py +└── transform.py +``` + +--- + +## Provider details + +### Ollama + +- Set `LLM_PROVIDER=ollama` +- Set `DEFAULT_MODEL` to any pulled model, for example `gemma3:4b` +- The provider wrapper in `models.OllamaProvider` calls `ollama.chat` + +### Gemini + +- Set `LLM_PROVIDER=gemini` +- Set `DEFAULT_MODEL` to a supported Gemini model, for example `gemini-2.0-flash` +- Provide `GEMINI_API_KEY` +- The wrapper in `models.GeminiProvider` adapts responses to a unified format + +--- + +## Contributing + +Please read the [CONTRIBUTING.md](./CONTRIBUTING.md) for detailed guidelines on filing issues, proposing changes, and submitting pull requests. Key principles include: + +- Keep prompts declarative and provider-agnostic. +- Validate changes with a couple of real resumes under different providers. +- Add or adjust unit-free smoke tests that call each stage with minimal inputs. + +--- + + +## License + +[MIT](https://github.com/interviewstreet/hiring-agent/blob/master/LICENSE) © HackerRank diff --git a/README.wehub.md b/README.wehub.md new file mode 100644 index 0000000..ed4492b --- /dev/null +++ b/README.wehub.md @@ -0,0 +1,7 @@ +# WeHub 来源说明 + +- 原始项目：`interviewstreet/hiring-agent` +- 原始仓库：https://github.com/interviewstreet/hiring-agent +- 导入方式：上游默认分支的最新快照 +- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准 +- 本文件仅用于记录来源，不代表 WeHub 是原项目作者 diff --git a/config.py b/config.py new file mode 100644 index 0000000..38d21a9 --- /dev/null +++ b/config.py @@ -0,0 +1,6 @@ +""" +Configuration settings for the hiring agent application. +""" + +# Global development mode flag +DEVELOPMENT_MODE = True diff --git a/evaluator.py b/evaluator.py new file mode 100644 index 0000000..1f9e91f --- /dev/null +++ b/evaluator.py @@ -0,0 +1,91 @@ +from typing import Dict, List, Optional, Tuple, Any +from pydantic import BaseModel, Field, field_validator +from models import JSONResume, EvaluationData +from llm_utils import initialize_llm_provider, extract_json_from_response +import logging +import json +import re + +MAX_BONUS_POINTS = 20 +MIN_FINAL_SCORE = -20 +MAX_FINAL_SCORE = 120 + +from prompt import ( + DEFAULT_MODEL, + MODEL_PARAMETERS, + MODEL_PROVIDER_MAPPING, + GEMINI_API_KEY, +) +from prompts.template_manager import TemplateManager + +logger = logging.getLogger(__name__) + + +class ResumeEvaluator: + def __init__(self, model_name: str = DEFAULT_MODEL, model_params: dict = None): + if not model_name: + raise ValueError("Model name cannot be empty") + + self.model_name = model_name + self.model_params = model_params or MODEL_PARAMETERS.get( + model_name, {"temperature": 0.5, "top_p": 0.9} + ) + self.template_manager = TemplateManager() + self._initialize_llm_provider() + + def _initialize_llm_provider(self): + """Initialize the appropriate LLM provider based on the model.""" + self.provider = initialize_llm_provider(self.model_name) + + def _load_evaluation_prompt(self, resume_text: str) -> str: + criteria_template = self.template_manager.render_template( + "resume_evaluation_criteria", text_content=resume_text + ) + if criteria_template is None: + raise ValueError("Failed to load resume evaluation criteria template") + return criteria_template + + def evaluate_resume(self, resume_text: str) -> EvaluationData: + self._last_resume_text = resume_text + full_prompt = self._load_evaluation_prompt(resume_text) + # logger.info(f"🔤 Evaluation prompt being sent: {full_prompt}") + try: + system_message = self.template_manager.render_template( + "resume_evaluation_system_message" + ) + if system_message is None: + raise ValueError( + "Failed to load resume evaluation system message template" + ) + + # Prepare chat parameters + chat_params = { + "model": self.model_name, + "messages": [ + {"role": "system", "content": system_message}, + {"role": "user", "content": full_prompt}, + ], + "options": { + "stream": False, + "temperature": self.model_params.get("temperature", 0.5), + "top_p": self.model_params.get("top_p", 0.9), + }, + } + + # Add format parameter for structured output + kwargs = {"format": EvaluationData.model_json_schema()} + # Use the appropriate provider to make the API call + response = self.provider.chat(**chat_params, **kwargs) + + response_text = response["message"]["content"] + response_text = extract_json_from_response(response_text) + logger.error(f"🔤 Prompt response: {response_text}") + + evaluation_dict = json.loads(response_text) + evaluation_data = EvaluationData(**evaluation_dict) + + return evaluation_data + + except Exception as e: + logger.error(f"Error evaluating resume: {str(e)}") + raise diff --git a/github.py b/github.py new file mode 100644 index 0000000..0e19568 --- /dev/null +++ b/github.py @@ -0,0 +1,496 @@ +import os +import re +import json +import requests +import datetime +import time +from pathlib import Path + +from typing import Dict, List, Optional, Any +from models import GitHubProfile +from pdf import logger +from prompts.template_manager import TemplateManager +from prompt import DEFAULT_MODEL, MODEL_PARAMETERS +from llm_utils import initialize_llm_provider, extract_json_from_response +from config import DEVELOPMENT_MODE + + +def _create_cache_filename(api_url: str, params: dict = None) -> str: + url_parts = api_url.replace("https://api.github.com/", "").replace("/", "_") + + if params: + param_str = "_".join([f"{k}_{v}" for k, v in sorted(params.items())]) + filename = f"cache/gh_githubcache_{url_parts}_{param_str}.json" + else: + filename = f"cache/gh_githubcache_{url_parts}.json" + return filename + + +def _fetch_github_api(api_url, params=None): + headers = {} + github_token = os.environ.get("GITHUB_TOKEN") + if github_token: + headers["Authorization"] = f"token {github_token}" + + cache_filename = _create_cache_filename(api_url, params) + if DEVELOPMENT_MODE and os.path.exists(cache_filename): + print(f"Loading cached GitHub data from {cache_filename}") + try: + cached_data = json.loads(Path(cache_filename).read_text(encoding="utf-8")) + if not cached_data: + raise ValueError("Cached data is empty") + return 200, cached_data + except Exception as e: + print(f"⚠️ Warning: Error reading cache file {cache_filename}: {e}") + try: + os.remove(cache_filename) + except Exception as delete_err: + print( + f"Failed to delete invalid cache file {cache_filename}: {delete_err}" + ) + + response = requests.get(api_url, params, timeout=10, headers=headers) + status_code = response.status_code + + # Check GitHub rate limit headers + rate_limit_remaining = response.headers.get("X-RateLimit-Remaining") + rate_limit_limit = response.headers.get("X-RateLimit-Limit") + rate_limit_reset = response.headers.get("X-RateLimit-Reset") + logger.info( + f"{rate_limit_remaining}/{rate_limit_limit}. Reset at {rate_limit_reset}" + ) + + if rate_limit_remaining is not None and rate_limit_limit is not None: + remaining = int(rate_limit_remaining) + limit = int(rate_limit_limit) + + # Log rate limit information and handle proactively + if remaining < 10 and rate_limit_reset: + reset_timestamp = int(rate_limit_reset) + current_timestamp = int(time.time()) + wait_seconds = ( + max(0, reset_timestamp - current_timestamp) + 5 + ) # Add 5 second buffer + reset_time = datetime.datetime.fromtimestamp(reset_timestamp) + + # Cap maximum wait time at 1 hour + max_wait = 3600 + if wait_seconds > max_wait: + print( + f"⚠️ Rate limit reset time is too far in the future ({wait_seconds}s). Capping wait to {max_wait}s" + ) + wait_seconds = max_wait + + logger.error( + f"⚠️ GitHub API rate limit low: {remaining}/{limit} requests remaining. Resets at {reset_time}" + ) + print( + f"💡 Tip: Set GITHUB_TOKEN environment variable to increase rate limits (60/hour → 5000/hour)" + ) + + if wait_seconds > 0: + logger.info( + f"⏳ Proactively sleeping for {wait_seconds} seconds until rate limit resets..." + ) + time.sleep(wait_seconds) + print(f"✅ Rate limit should be reset now. Continuing...") + elif remaining < 100: + logger.info( + f"ℹ️ GitHub API rate limit: {remaining}/{limit} requests remaining" + ) + + data = response.json() if response.status_code == 200 else {} + + if DEVELOPMENT_MODE and status_code == 200: + try: + os.makedirs("cache", exist_ok=True) + Path(cache_filename).write_text( + json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8" + ) + except Exception as e: + logger.error(f"Error caching GitHub data to {cache_filename}: {e}") + + return status_code, data + + +def extract_github_username(github_url: str) -> Optional[str]: + if not github_url: + return None + + github_url = github_url.replace(" ", "") + github_url = github_url.strip() + + patterns = [ + r"https?://github\.com/([^/]+)", + r"github\.com/([^/]+)", + r"@([^/]+)", + r"^([a-zA-Z0-9-]+)$", + ] + + for pattern in patterns: + match = re.search(pattern, github_url) + if match: + username = match.group(1) + # Remove query parameters if present (e.g., "?tab=repositories") + if "?" in username: + username = username.split("?", 1)[0] + return username + return None + + +def fetch_github_profile(github_url: str) -> Optional[GitHubProfile]: + try: + username = extract_github_username(github_url) + logger.info(f"{username}") + if not username: + print(f"Could not extract username from: {github_url}") + return None + + api_url = f"https://api.github.com/users/{username}" + + status_code, data = _fetch_github_api(api_url) + + if status_code == 200: + profile = GitHubProfile( + username=username, + name=data.get("name"), + bio=data.get("bio"), + location=data.get("location"), + company=data.get("company"), + public_repos=data.get("public_repos"), + followers=data.get("followers"), + following=data.get("following"), + created_at=data.get("created_at"), + updated_at=data.get("updated_at"), + avatar_url=data.get("avatar_url"), + blog=data.get("blog"), + twitter_username=data.get("twitter_username"), + hireable=data.get("hireable"), + ) + + return profile + elif status_code == 404: + print(f"GitHub user not found: {username}") + return None + else: + print(f"GitHub API error: {status_code} - {data}") + return None + + except requests.exceptions.RequestException as e: + print(f"Error fetching GitHub profile: {e}") + return None + except Exception as e: + print(f"Unexpected error fetching GitHub profile: {e}") + return None + + +def fetch_contributions_count(owner: str, contributors_data): + user_contributions = 0 + total_contributions = 0 + + for contributor in contributors_data: + if isinstance(contributor, dict): + contributions = contributor.get("contributions", 0) + total_contributions += contributions + + if contributor.get("login", "").lower() == owner.lower(): + user_contributions = contributions + + return user_contributions, total_contributions + + +def fetch_repo_contributors(owner: str, repo_name: str) -> list[dict]: + try: + api_url = f"https://api.github.com/repos/{owner}/{repo_name}/contributors" + + status_code, contributors_data = _fetch_github_api(api_url) + + if status_code == 200: + return contributors_data + else: + return [] + + except Exception as e: + logger.error(f"Error fetching contributors for {owner}/{repo_name}: {e}") + return [] + + +def fetch_all_github_repos(github_url: str, max_repos: int = 100) -> List[Dict]: + try: + username = extract_github_username(github_url) + if not username: + print(f"Could not extract username from: {github_url}") + return [] + + api_url = f"https://api.github.com/users/{username}/repos" + + params = {"sort": "updated", "per_page": min(max_repos, 100), "type": "all"} + + status_code, repos_data = _fetch_github_api(api_url, params=params) + + if status_code == 200: + projects = [] + for repo in repos_data: + if repo.get("fork") and repo.get("forks_count", 0) < 5: + continue + + repo_name = repo.get("name") + + contributors_data = fetch_repo_contributors(username, repo_name) + contributor_count = len(contributors_data) + + user_contributions, total_contributions = fetch_contributions_count( + username, contributors_data + ) + + project_type = ( + "open_source" if contributor_count > 1 else "self_project" + ) + + project = { + "name": repo.get("name"), + "description": repo.get("description"), + "github_url": repo.get("html_url"), + "live_url": repo.get("homepage") if repo.get("homepage") else None, + "technologies": ( + [repo.get("language")] if repo.get("language") else [] + ), + "project_type": project_type, + "contributor_count": contributor_count, + "author_commit_count": user_contributions, + "total_commit_count": total_contributions, + "github_details": { + "stars": repo.get("stargazers_count", 0), + "forks": repo.get("forks_count", 0), + "language": repo.get("language"), + "description": repo.get("description"), + "created_at": repo.get("created_at"), + "updated_at": repo.get("updated_at"), + "topics": repo.get("topics", []), + "open_issues": repo.get("open_issues_count", 0), + "size": repo.get("size", 0), + "fork": repo.get("fork", False), + "archived": repo.get("archived", False), + "default_branch": repo.get("default_branch"), + "contributors": contributor_count, + }, + } + projects.append(project) + + projects.sort(key=lambda x: x["github_details"]["stars"], reverse=True) + + open_source_count = sum( + 1 for p in projects if p["project_type"] == "open_source" + ) + self_project_count = sum( + 1 for p in projects if p["project_type"] == "self_project" + ) + + print(f"✅ Found {len(projects)} repositories") + print( + f"📊 Project classification: {open_source_count} open source, {self_project_count} self projects" + ) + return projects + + elif status_code == 404: + print(f"GitHub user not found: {username}") + return [] + else: + print(f"GitHub API error: {status_code} - {repos_data}") + return [] + + except requests.exceptions.RequestException as e: + print(f"Error fetching GitHub repositories: {e}") + return [] + except Exception as e: + print(f"Unexpected error fetching GitHub repositories: {e}") + return [] + + +def generate_profile_json(profile: GitHubProfile) -> Dict: + if not profile: + return {} + + profile_data = { + "username": profile.username, + "name": profile.name, + "bio": profile.bio, + "location": profile.location, + "company": profile.company, + "public_repos": profile.public_repos, + "followers": profile.followers, + "following": profile.following, + "created_at": profile.created_at, + "updated_at": profile.updated_at, + "avatar_url": profile.avatar_url, + "blog": profile.blog, + "twitter_username": profile.twitter_username, + "hireable": profile.hireable, + } + + return profile_data + + +def generate_projects_json(projects: List[Dict]) -> List[Dict]: + if not projects: + return [] + + try: + projects_data = [] + for project in projects: + if project.get("author_commit_count") == 0: + continue + + project_data = { + "name": project.get("name"), + "description": project.get("description"), + "github_url": project.get("github_url"), + "live_url": project.get("live_url"), + "technologies": project.get("technologies", []), + "project_type": project.get("project_type", "self_project"), + "contributor_count": project.get("contributor_count", 1), + "author_commit_count": project.get("author_commit_count", 0), + "total_commit_count": project.get("total_commit_count", 0), + "github_details": project.get("github_details", {}), + } + projects_data.append(project_data) + + projects_json = json.dumps(projects_data, indent=2) + + template_manager = TemplateManager() + prompt = template_manager.render_template( + "github_project_selection", projects_data=projects_json + ) + + print( + f"🤖 Using LLM to select top 5 projects from {len(projects)} repositories..." + ) + + # Initialize the LLM provider + provider = initialize_llm_provider(DEFAULT_MODEL) + + # Get model parameters + model_params = MODEL_PARAMETERS.get( + DEFAULT_MODEL, {"temperature": 0.1, "top_p": 0.9} + ) + + # Prepare chat parameters + chat_params = { + "model": DEFAULT_MODEL, + "messages": [ + { + "role": "system", + "content": "You are an expert technical recruiter analyzing GitHub repositories to identify the most impressive projects. CRITICAL: You must select exactly 7 UNIQUE projects - no duplicates allowed. Each project must be different from the others.", + }, + {"role": "user", "content": prompt}, + ], + "options": model_params, + } + + # Call the LLM provider + response = provider.chat(**chat_params) + + response_text = response["message"]["content"] + + try: + response_text = response_text.strip() + response_text = extract_json_from_response(response_text) + + selected_projects = json.loads(response_text) + + unique_projects = [] + seen_names = set() + + for project in selected_projects: + project_name = project.get("name", "") + if project_name and project_name not in seen_names: + unique_projects.append(project) + seen_names.add(project_name) + + if len(unique_projects) < 7: + print( + f"⚠️ LLM selected {len(selected_projects)} projects but {len(unique_projects)} are unique" + ) + + for project in projects_data: + if len(unique_projects) >= 7: + break + project_name = project.get("name", "") + if project_name and project_name not in seen_names: + unique_projects.append(project) + seen_names.add(project_name) + + project_names = ", ".join( + [proj.get("name", "N/A") for proj in unique_projects] + ) + print( + f"✅ LLM selected {len(unique_projects)} unique top projects: {project_names}" + ) + return unique_projects + + except json.JSONDecodeError as e: + print(f"ERROR: Error parsing LLM response: {e}") + print(f"ERROR: Raw response: {response_text}") + + print("🔄 Falling back to first 7 projects") + return projects_data[:7] + + except Exception as e: + print(f"Error using LLM for project selection: {e}") + print("🔄 Falling back to first 7 projects") + + projects_data = [] + for project in projects[:7]: + project_data = { + "name": project.get("name"), + "description": project.get("description"), + "github_url": project.get("github_url"), + "live_url": project.get("live_url"), + "technologies": project.get("technologies", []), + "project_type": project.get("project_type", "self_project"), + "contributor_count": project.get("contributor_count", 1), + "github_details": project.get("github_details", {}), + } + projects_data.append(project_data) + + return projects_data + + +def fetch_and_display_github_info(github_url: str) -> Dict: + logger.info(f"{github_url}") + github_profile = fetch_github_profile(github_url) + if not github_profile: + print("\n❌ Failed to fetch GitHub profile details.") + return {} + + print("🔍 Fetching all repository details...") + projects = fetch_all_github_repos(github_url) + + if not projects: + print("\n❌ No repositories found or failed to fetch repository details.") + + profile_json = generate_profile_json(github_profile) + projects_json = generate_projects_json(projects) + + result = { + "profile": profile_json, + "projects": projects_json, + "total_projects": len(projects_json), + } + + return result + + +def main(github_url): + result = fetch_and_display_github_info(github_url) + print("\n" + "=" * 60) + print("JSON DATA OUTPUT") + print("=" * 60) + print(json.dumps(result, indent=2, ensure_ascii=False)) + print("=" * 60) + + return result + + +if __name__ == "__main__": + main("https://github.com/PavitKaur05") diff --git a/llm_utils.py b/llm_utils.py new file mode 100644 index 0000000..7e1d96d --- /dev/null +++ b/llm_utils.py @@ -0,0 +1,62 @@ +""" +Utility functions for LLM providers. +""" + +import logging +from typing import Any, Dict, Optional +from models import ModelProvider, OllamaProvider, GeminiProvider +from prompt import MODEL_PROVIDER_MAPPING, GEMINI_API_KEY + +logger = logging.getLogger(__name__) + + +def extract_json_from_response(response_text: str) -> str: + """ + Extract JSON content from markdown code blocks. + + Args: + response_text: Text that may contain JSON wrapped in markdown code blocks + + Returns: + Text with markdown code block syntax removed + """ + + response_text = response_text.strip() + if "" in response_text: + think_start = response_text.find("") + think_end = response_text.find("") + if think_start != -1 and think_end != -1: + response_text = response_text[:think_start] + response_text[think_end + 8 :] + + # Remove leading ```json if present + if response_text.startswith("```json"): + response_text = response_text[7:] + # Remove trailing ``` if present + if response_text.endswith("```"): + response_text = response_text[:-3] + return response_text + + +def initialize_llm_provider(model_name: str) -> Any: + """ + Initialize the appropriate LLM provider based on the model name. + + Args: + model_name: The name of the model to use + + Returns: + An initialized LLM provider (either OllamaProvider or GeminiProvider) + """ + # Default to Ollama provider + provider = OllamaProvider() + # If using Gemini and API key is available, use Gemini provider + model_provider = MODEL_PROVIDER_MAPPING.get(model_name, ModelProvider.OLLAMA) + if model_provider == ModelProvider.GEMINI: + if not GEMINI_API_KEY: + logger.warning("⚠️ Gemini API key not found. Falling back to Ollama.") + else: + logger.info(f"🔄 Using Google Gemini API provider with model {model_name}") + provider = GeminiProvider(api_key=GEMINI_API_KEY) + else: + logger.info(f"🔄 Using Ollama provider with model {model_name}") + return provider diff --git a/models.py b/models.py new file mode 100644 index 0000000..e714600 --- /dev/null +++ b/models.py @@ -0,0 +1,391 @@ +from typing import List, Optional, Dict, Tuple, Any, Protocol, runtime_checkable +from pydantic import BaseModel, Field, field_validator +from enum import Enum + + +class ModelProvider(Enum): + """Enum for supported model providers.""" + + OLLAMA = "ollama" + GEMINI = "gemini" + + +@runtime_checkable +class LLMProvider(Protocol): + """Protocol for LLM providers.""" + + def chat( + self, + model: str, + messages: List[Dict[str, str]], + options: Dict[str, Any] = None, + **kwargs + ) -> Dict[str, Any]: + """Send a chat request to the LLM provider.""" + ... + + +class Location(BaseModel): + """Location information for JSON Resume format.""" + + address: Optional[str] = None + postalCode: Optional[str] = None + city: Optional[str] = None + countryCode: Optional[str] = None + region: Optional[str] = None + + +class Profile(BaseModel): + """Social profile information for JSON Resume format.""" + + network: Optional[str] = None + username: Optional[str] = None + url: str + + +class Basics(BaseModel): + """Basic information for JSON Resume format.""" + + name: str + email: Optional[str] = None + phone: Optional[str] = None + url: Optional[str] = None + summary: Optional[str] = None + location: Optional[Location] = None + profiles: Optional[List[Profile]] = None + + +class Work(BaseModel): + """Work experience for JSON Resume format.""" + + name: Optional[str] = None + position: Optional[str] = None + url: Optional[str] = None + startDate: Optional[str] = None + endDate: Optional[str] = None + summary: Optional[str] = None + highlights: Optional[List[str]] = None + + +class Volunteer(BaseModel): + """Volunteer experience for JSON Resume format.""" + + organization: Optional[str] = None + position: Optional[str] = None + url: Optional[str] = None + startDate: Optional[str] = None + endDate: Optional[str] = None + summary: Optional[str] = None + highlights: Optional[List[str]] = None + + +class Education(BaseModel): + """Education information for JSON Resume format.""" + + institution: Optional[str] = None + url: Optional[str] = None + area: Optional[str] = None + studyType: Optional[str] = None + startDate: Optional[str] = None + endDate: Optional[str] = None + score: Optional[str] = None + courses: Optional[List[str]] = None + + +class Award(BaseModel): + """Award information for JSON Resume format.""" + + title: Optional[str] = None + date: Optional[str] = None + awarder: Optional[str] = None + summary: Optional[str] = None + + +class Certificate(BaseModel): + """Certificate information for JSON Resume format.""" + + name: Optional[str] = None + date: Optional[str] = None + issuer: Optional[str] = None + url: Optional[str] = None + + +class Publication(BaseModel): + """Publication information for JSON Resume format.""" + + name: Optional[str] = None + publisher: Optional[str] = None + releaseDate: Optional[str] = None + url: Optional[str] = None + summary: Optional[str] = None + + +class Skill(BaseModel): + """Skill information for JSON Resume format.""" + + name: Optional[str] = None + level: Optional[str] = None + keywords: Optional[List[str]] = None + + +class Language(BaseModel): + """Language information for JSON Resume format.""" + + language: Optional[str] = None + fluency: Optional[str] = None + + +class Interest(BaseModel): + """Interest information for JSON Resume format.""" + + name: Optional[str] = None + keywords: Optional[List[str]] = None + + +class Reference(BaseModel): + """Reference information for JSON Resume format.""" + + name: Optional[str] = None + reference: Optional[str] = None + + +class Project(BaseModel): + """Project information for JSON Resume format.""" + + name: Optional[str] = None + startDate: Optional[str] = None + endDate: Optional[str] = None + description: Optional[str] = None + highlights: Optional[List[str]] = None + url: Optional[str] = None + technologies: Optional[List[str]] = None + skills: Optional[List[str]] = None + + +class BasicsSection(BaseModel): + """Basics section containing basic information.""" + + basics: Optional[Basics] = None + + +class WorkSection(BaseModel): + """Work section containing a list of work experiences.""" + + work: Optional[List[Work]] = None + + +class EducationSection(BaseModel): + """Education section containing a list of education entries.""" + + education: Optional[List[Education]] = None + + +class SkillsSection(BaseModel): + """Skills section containing a list of skill categories.""" + + skills: Optional[List[Skill]] = None + + +class ProjectsSection(BaseModel): + """Projects section containing a list of projects.""" + + projects: Optional[List[Project]] = None + + +class AwardsSection(BaseModel): + """Awards section containing a list of awards.""" + + awards: Optional[List[Award]] = None + + +class JSONResume(BaseModel): + """Complete JSON Resume format model.""" + + basics: Optional[Basics] = None + work: Optional[List[Work]] = None + volunteer: Optional[List[Volunteer]] = None + education: Optional[List[Education]] = None + awards: Optional[List[Award]] = None + certificates: Optional[List[Certificate]] = None + publications: Optional[List[Publication]] = None + skills: Optional[List[Skill]] = None + languages: Optional[List[Language]] = None + interests: Optional[List[Interest]] = None + references: Optional[List[Reference]] = None + projects: Optional[List[Project]] = None + + +class CategoryScore(BaseModel): + score: float = Field(ge=0, description="Score achieved in this category") + max: int = Field(gt=0, description="Maximum possible score") + evidence: str = Field(min_length=1, description="Evidence supporting the score") + + +class Scores(BaseModel): + open_source: CategoryScore + self_projects: CategoryScore + production: CategoryScore + technical_skills: CategoryScore + + +class BonusPoints(BaseModel): + total: float = Field(ge=0, le=20, description="Total bonus points") + breakdown: str = Field(description="Breakdown of bonus points") + + +class Deductions(BaseModel): + total: float = Field( + ge=0, + description="Total deduction points (stored as positive, applied as negative)", + ) + reasons: str = Field(description="Reasons for deductions") + + +class EvaluationData(BaseModel): + scores: Scores + bonus_points: BonusPoints + deductions: Deductions + key_strengths: List[str] = Field(min_items=1, max_items=5) + areas_for_improvement: List[str] = Field(min_items=1, max_items=5) + + +class GitHubProfile(BaseModel): + """Pydantic model for GitHub profile data.""" + + username: str + name: Optional[str] = None + bio: Optional[str] = None + location: Optional[str] = None + company: Optional[str] = None + public_repos: Optional[int] = None + followers: Optional[int] = None + following: Optional[int] = None + created_at: Optional[str] = None + updated_at: Optional[str] = None + avatar_url: Optional[str] = None + blog: Optional[str] = None + twitter_username: Optional[str] = None + hireable: Optional[bool] = None + + +class OllamaProvider: + """Ollama LLM provider implementation.""" + + def __init__(self): + import ollama + + self.client = ollama + + def chat( + self, + model: str, + messages: List[Dict[str, str]], + options: Dict[str, Any] = None, + **kwargs + ) -> Dict[str, Any]: + """Send a chat request to Ollama.""" + + ollama_options = options.copy() if options else {} + + # remove steam from ollama options + ollama_options.pop("stream", None) + + # Add num_ctx 32K context window to options + ollama_options["num_ctx"] = 32768 + + # convert to chat params + chat_params = { + "model": model, + "messages": messages, + "options": ollama_options, + } + + # add it to top level + if "stream" in kwargs: + chat_params["stream"] = kwargs["stream"] + + if "format" in kwargs: + chat_params["format"] = kwargs["format"] + + return self.client.chat(**chat_params) + + +class GeminiProvider: + """Google Gemini API provider implementation.""" + + def __init__(self, api_key: str): + import google.generativeai as genai + + genai.configure(api_key=api_key) + self.client = genai + + def chat( + self, + model: str, + messages: List[Dict[str, str]], + options: Dict[str, Any] = None, + **kwargs + ) -> Dict[str, Any]: + """Send a chat request to Google Gemini API.""" + import re + import time + import random + from google.api_core.exceptions import ResourceExhausted + + MAX_RETRIES = 5 + BASE_DELAY = 10.0 # seconds — base for exponential backoff + MAX_DELAY = 120.0 # cap so we never wait more than 2 minutes + + # Map options to Gemini parameters + generation_config = {} + if options: + if "temperature" in options: + generation_config["temperature"] = options["temperature"] + if "top_p" in options: + generation_config["top_p"] = options["top_p"] + + # Create a Gemini model + gemini_model = self.client.GenerativeModel( + model_name=model, generation_config=generation_config + ) + + # Convert messages to Gemini format + gemini_messages = [] + for msg in messages: + role = "user" if msg["role"] == "user" else "model" + gemini_messages.append({"role": role, "parts": [msg["content"]]}) + + for attempt in range(MAX_RETRIES): + try: + # Send the chat request + response = gemini_model.generate_content(gemini_messages) + + # Convert Gemini response to Ollama-like format for compatibility + return {"message": {"role": "assistant", "content": response.text}} + + except ResourceExhausted as e: + if attempt == MAX_RETRIES - 1: + # All retries exhausted — re-raise the original exception. + # This surfaces unrecoverable quota errors (RPD, TPM, etc.) + # instead of silently failing or returning bad data. + raise + + # Parse the API-suggested retry delay from the error message + match = re.search(r"retry[_ ]in\s+([\d.]+)s", str(e), re.IGNORECASE) + api_hint = float(match.group(1)) if match else None + + # Exponential backoff: BASE_DELAY * 2^attempt, capped at MAX_DELAY + exp_delay = min(BASE_DELAY * (2 ** attempt), MAX_DELAY) + + # Prefer the API hint when it is shorter than our computed delay + delay = api_hint if (api_hint and api_hint < exp_delay) else exp_delay + + # Add ±20% randomized jitter to avoid thundering herd + sleep_time = round(delay * random.uniform(0.8, 1.2), 2) + + print( + f"[GeminiProvider] Rate limit hit " + f"(attempt {attempt + 1}/{MAX_RETRIES}). " + f"Retrying in {sleep_time}s..." + ) + time.sleep(sleep_time) diff --git a/pdf.py b/pdf.py new file mode 100644 index 0000000..f793c50 --- /dev/null +++ b/pdf.py @@ -0,0 +1,323 @@ +import os +import sys +import json +import time +import logging +import pymupdf + +from models import ( + JSONResume, + Basics, + Work, + Education, + Skill, + Project, + Award, + BasicsSection, + WorkSection, + EducationSection, + SkillsSection, + ProjectsSection, + AwardsSection, +) +from llm_utils import initialize_llm_provider, extract_json_from_response +from pymupdf_rag import to_markdown +from typing import List, Optional, Dict, Any +from prompt import ( + DEFAULT_MODEL, + MODEL_PARAMETERS, + MODEL_PROVIDER_MAPPING, + GEMINI_API_KEY, +) +from prompts.template_manager import TemplateManager +from transform import transform_parsed_data + +logger = logging.getLogger(__name__) + + +class PDFHandler: + def __init__(self): + self.template_manager = TemplateManager() + self._initialize_llm_provider() + + def _initialize_llm_provider(self): + """Initialize the appropriate LLM provider based on the model.""" + self.provider = initialize_llm_provider(DEFAULT_MODEL) + + def extract_text_from_pdf(self, pdf_path: str) -> Optional[str]: + try: + if not os.path.exists(pdf_path): + raise FileNotFoundError(f"PDF file not found: {pdf_path}") + + with pymupdf.open(pdf_path) as doc: + pages = range(doc.page_count) + resume_text = to_markdown( + doc, + pages=pages, + ) + logger.debug( + f"Extracted text from PDF: {len(resume_text) if resume_text else 0} characters" + ) + return resume_text + except Exception as e: + logger.error(f"An error occurred while reading the PDF: {e}") + return None + + def _call_llm_for_section( + self, section_name: str, text_content: str, prompt: str, return_model=None + ) -> Optional[Dict]: + try: + start_time = time.time() + logger.debug( + f"🔄 Extracting {section_name} section using {DEFAULT_MODEL}..." + ) + + model_params = MODEL_PARAMETERS.get( + DEFAULT_MODEL, {"temperature": 0.1, "top_p": 0.9} + ) + + section_system_message = self.template_manager.render_template( + "system_message", section_name_param=section_name + ) + if not section_system_message: + logger.error( + f"❌ Failed to render system message template for {section_name}" + ) + return None + + chat_params = { + "model": DEFAULT_MODEL, + "messages": [ + {"role": "system", "content": section_system_message}, + {"role": "user", "content": prompt}, + ], + "options": { + "stream": False, + "temperature": model_params["temperature"], + "top_p": model_params["top_p"], + }, + } + + kwargs = {} + if return_model: + kwargs["format"] = return_model.model_json_schema() + + # Use the appropriate provider to make the API call + response = self.provider.chat(**chat_params, **kwargs) + + response_text = response["message"]["content"] + + try: + response_text = extract_json_from_response(response_text) + json_start = response_text.find("{") + json_end = response_text.rfind("}") + if json_start != -1 and json_end != -1: + response_text = response_text[json_start : json_end + 1] + parsed_data = json.loads(response_text) + logger.debug(f"✅ Successfully extracted {section_name} section") + + transformed_data = transform_parsed_data(parsed_data) + end_time = time.time() + total_time = end_time - start_time + logger.debug( + f"⏱️ Total time for separate section extraction: {total_time:.2f} seconds" + ) + + return transformed_data + except json.JSONDecodeError as e: + logger.error(f"❌ Error parsing JSON for {section_name} section: {e}") + logger.error(f"Raw response: {response_text}") + return None + + except Exception as e: + logger.error(f"❌ Error calling LLM for {section_name} section: {e}") + return None + + def extract_basics_section(self, resume_text: str) -> Optional[Dict]: + prompt = self.template_manager.render_template( + "basics", text_content=resume_text + ) + if not prompt: + logger.error("❌ Failed to render basics template") + return None + return self._call_llm_for_section("basics", resume_text, prompt, BasicsSection) + + def extract_work_section(self, resume_text: str) -> Optional[Dict]: + prompt = self.template_manager.render_template("work", text_content=resume_text) + if not prompt: + logger.error("❌ Failed to render work template") + return None + return self._call_llm_for_section("work", resume_text, prompt, WorkSection) + + def extract_education_section(self, resume_text: str) -> Optional[Dict]: + prompt = self.template_manager.render_template( + "education", text_content=resume_text + ) + if not prompt: + logger.error("❌ Failed to render education template") + return None + return self._call_llm_for_section( + "education", resume_text, prompt, EducationSection + ) + + def extract_skills_section(self, resume_text: str) -> Optional[Dict]: + prompt = self.template_manager.render_template( + "skills", text_content=resume_text + ) + if not prompt: + logger.error("❌ Failed to render skills template") + return None + return self._call_llm_for_section("skills", resume_text, prompt, SkillsSection) + + def extract_projects_section(self, resume_text: str) -> Optional[Dict]: + prompt = self.template_manager.render_template( + "projects", text_content=resume_text + ) + if not prompt: + logger.error("❌ Failed to render projects template") + return None + return self._call_llm_for_section( + "projects", resume_text, prompt, ProjectsSection + ) + + def extract_awards_section(self, resume_text: str) -> Optional[Dict]: + prompt = self.template_manager.render_template( + "awards", text_content=resume_text + ) + if not prompt: + logger.error("❌ Failed to render awards template") + return None + return self._call_llm_for_section("awards", resume_text, prompt, AwardsSection) + + def extract_json_from_text(self, resume_text: str) -> Optional[JSONResume]: + try: + return self._extract_all_sections_separately(resume_text) + except Exception as e: + logger.error(f"Error calling Ollama: {e}") + return None + + def extract_json_from_pdf(self, pdf_path: str) -> Optional[JSONResume]: + try: + logger.debug(f"📄 Extracting text from PDF: {pdf_path}") + text_content = self.extract_text_from_pdf(pdf_path) + + if not text_content: + logger.error("❌ Failed to extract text from PDF") + return None + + logger.debug( + f"✅ Successfully extracted {len(text_content)} characters from PDF" + ) + + logger.debug("🔄 Extracting all sections separately...") + return self._extract_all_sections_separately(text_content) + + except Exception as e: + logger.error(f"❌ Error during PDF to JSON extraction: {e}") + return None + + def _extract_section_data( + self, text_content: str, section_name: str, return_model=None + ) -> Optional[Dict]: + section_extractors = { + "basics": self.extract_basics_section, + "work": self.extract_work_section, + "education": self.extract_education_section, + "skills": self.extract_skills_section, + "projects": self.extract_projects_section, + "awards": self.extract_awards_section, + } + + if section_name not in section_extractors: + logger.error(f"❌ Invalid section name: {section_name}") + logger.error(f"Valid sections: {list(section_extractors.keys())}") + return None + + return section_extractors[section_name](text_content) + + def _extract_single_section( + self, text_content: str, section_name: str, return_model=None + ) -> Optional[Dict]: + section_data = self._extract_section_data( + text_content, section_name, return_model + ) + if section_data: + complete_resume = { + "basics": None, + "work": None, + "volunteer": None, + "education": None, + "awards": None, + "certificates": None, + "publications": None, + "skills": None, + "languages": None, + "interests": None, + "references": None, + "projects": None, + "meta": None, + } + + complete_resume.update(section_data) + return complete_resume + + return None + + def _extract_all_sections_separately( + self, text_content: str + ) -> Optional[JSONResume]: + start_time = time.time() + + sections = ["basics", "work", "education", "skills", "projects", "awards"] + + complete_resume = { + "basics": None, + "work": None, + "volunteer": None, + "education": None, + "awards": None, + "certificates": None, + "publications": None, + "skills": None, + "languages": None, + "interests": None, + "references": None, + "projects": None, + "meta": None, + } + + for section_name in sections: + section_data = self._extract_section_data(text_content, section_name) + + if section_data: + complete_resume.update(section_data) + logger.debug(f"✅ Successfully extracted {section_name} section") + else: + logger.error( + f"⚠️ Failed to extract {section_name} section. Aborting extraction to prevent partial/invalid resume data." + ) + return None + + try: + if complete_resume.get("basics") and isinstance( + complete_resume["basics"], dict + ): + try: + complete_resume["basics"] = Basics(**complete_resume["basics"]) + except Exception as e: + logger.error(f"❌ Error creating Basics object: {e}") + complete_resume["basics"] = None + + json_resume = JSONResume(**complete_resume) + + end_time = time.time() + total_time = end_time - start_time + logger.info( + f"⏱️ Total time for separate section extraction: {total_time:.2f} seconds" + ) + + return json_resume + + except Exception as e: + logger.error(f"❌ Error creating JSONResume object: {e}") + return None diff --git a/prompt.py b/prompt.py new file mode 100644 index 0000000..108e970 --- /dev/null +++ b/prompt.py @@ -0,0 +1,67 @@ +""" +Prompts for Resume Evaluation System + +This module contains all the prompts used by the resume evaluation system. +Centralizing prompts here makes them easier to maintain and update. +""" + +import os +from dotenv import load_dotenv +from models import ModelProvider + +# Load environment variables +load_dotenv() + +# Constants +DEFAULT_MODEL_NAME = "gemma3:4b" +DEFAULT_PROVIDER = ModelProvider.OLLAMA + +# Get model and provider from environment or use defaults +DEFAULT_MODEL = os.getenv("DEFAULT_MODEL", DEFAULT_MODEL_NAME) +PROVIDER = os.getenv("LLM_PROVIDER", DEFAULT_PROVIDER.value) + +# Validate provider +if PROVIDER not in [p.value for p in ModelProvider]: + PROVIDER = DEFAULT_PROVIDER.value + +# Model-specific parameters +MODEL_PARAMETERS = { + # Ollama models + "qwen3:1.7b": {"temperature": 0.0, "top_p": 0.9}, + "gemma3:1b": {"temperature": 0.0, "top_p": 0.9}, + "qwen3:4b": {"temperature": 0.1, "top_p": 0.4}, + "gemma3:4b": {"temperature": 0.1, "top_p": 0.9}, + "gemma3:12b": {"temperature": 0.1, "top_p": 0.9}, + "mistral:7b": {"temperature": 0.1, "top_p": 0.9}, + # Google Gemini models + "gemini-2.0-flash": {"temperature": 0.1, "top_p": 0.9}, + "gemini-2.0-flash-lite": {"temperature": 0.1, "top_p": 0.9}, + "gemini-2.5-pro": {"temperature": 0.1, "top_p": 0.9}, + "gemini-2.5-flash": {"temperature": 0.1, "top_p": 0.9}, + "gemini-2.5-flash-lite": {"temperature": 0.1, "top_p": 0.9}, + "gemini-3.5-flash": {"temperature": 0.1, "top_p": 0.9}, + "gemini-3.1-flash-lite": {"temperature": 0.1, "top_p": 0.9}, +} + +# Model provider mapping +# Maps model names to their provider +MODEL_PROVIDER_MAPPING = { + # Ollama models + "qwen3:1.7b": ModelProvider.OLLAMA, + "gemma3:1b": ModelProvider.OLLAMA, + "qwen3:4b": ModelProvider.OLLAMA, + "gemma3:4b": ModelProvider.OLLAMA, + "gemma3:12b": ModelProvider.OLLAMA, + "mistral:7b": ModelProvider.OLLAMA, + # Google Gemini models + "gemini-2.0-flash": ModelProvider.GEMINI, + "gemini-2.0-flash-lite": ModelProvider.GEMINI, + "gemini-2.5-flash": ModelProvider.GEMINI, + "gemini-2.5-flash-lite": ModelProvider.GEMINI, + "gemini-2.5-pro": ModelProvider.GEMINI, + "gemini-3.5-flash": ModelProvider.GEMINI, + "gemini-3.1-flash-lite": ModelProvider.GEMINI, +} + +# Get API keys from environment +GEMINI_API_KEY = os.getenv("GEMINI_API_KEY", "") diff --git a/prompts/template_manager.py b/prompts/template_manager.py new file mode 100644 index 0000000..b68f680 --- /dev/null +++ b/prompts/template_manager.py @@ -0,0 +1,90 @@ +""" +Template Manager for Section Extraction + +This module provides functionality to load and render Jinja templates for +section-specific resume extraction prompts. +""" + +import os +from typing import Dict, Optional +from jinja2 import Environment, FileSystemLoader, Template + + +class TemplateManager: + """ + Manages Jinja templates for section-specific resume extraction. + + This class provides functionality to load and render templates for + different resume sections (basics, work, education, skills, projects, awards). + """ + + def __init__(self, template_dir: str = "prompts/templates"): + """ + Initialize the template manager. + + Args: + template_dir (str): Directory containing Jinja templates + """ + self.template_dir = template_dir + self.env = Environment( + loader=FileSystemLoader(template_dir), trim_blocks=True, lstrip_blocks=True + ) + self._templates: Dict[str, Template] = {} + self._load_templates() + + def _load_templates(self): + """Load all available templates.""" + template_files = { + "basics": "basics.jinja", + "work": "work.jinja", + "education": "education.jinja", + "skills": "skills.jinja", + "projects": "projects.jinja", + "awards": "awards.jinja", + "system_message": "system_message.jinja", + "github_project_selection": "github_project_selection.jinja", + "resume_evaluation_criteria": "resume_evaluation_criteria.jinja", + "resume_evaluation_system_message": "resume_evaluation_system_message.jinja", + } + + for section_name, filename in template_files.items(): + try: + template_path = os.path.join(self.template_dir, filename) + if os.path.exists(template_path): + self._templates[section_name] = self.env.get_template(filename) + else: + print(f"⚠️ Template file not found: {template_path}") + except Exception as e: + print(f"❌ Error loading template {filename}: {e}") + + def get_available_sections(self) -> list: + """ + Get list of available section names. + + Returns: + list: List of available section names + """ + return list(self._templates.keys()) + + def render_template(self, section_name: str, **kwargs) -> Optional[str]: + """ + Render a template for a specific section. + + Args: + section_name (str): Name of the section (basics, work, education, etc.) + **kwargs: Template variables (e.g., text_content) + + Returns: + Optional[str]: Rendered template string, or None if template not found + """ + if section_name not in self._templates: + print(f"❌ Template not found for section: {section_name}") + print(f"Available sections: {self.get_available_sections()}") + return None + + try: + template = self._templates[section_name] + return template.render(**kwargs) + except Exception as e: + print(f"❌ Error rendering template for {section_name}: {e}") + return None diff --git a/prompts/templates/awards.jinja b/prompts/templates/awards.jinja new file mode 100644 index 0000000..aacaa97 --- /dev/null +++ b/prompts/templates/awards.jinja @@ -0,0 +1,20 @@ +Extract ONLY the awards and honors information from this resume. + +--- The input markdown starts here --- + +{{ text_content }} + +--- The input markdown ends here --- + +Return ONLY a JSON object with this structure: +{ + "awards": [ + { + "title": "Award name", + "date": "Award date (YYYY-MM)", + "awarder": "Awarding organization" + } + ] +} + +**IMPORTANT**: Return ONLY valid JSON. Do not include any explanatory text. \ No newline at end of file diff --git a/prompts/templates/basics.jinja b/prompts/templates/basics.jinja new file mode 100644 index 0000000..d02320c --- /dev/null +++ b/prompts/templates/basics.jinja @@ -0,0 +1,55 @@ +Extract ONLY the basic information (name, email, phone, location, profiles) from this resume. + +--- The input resume markdown starts here --- + +{{ text_content }} + +--- The input resume markdown ends here --- + +Return ONLY a JSON object with this structure: +{ + "basics": { + "name": "Full name", + "email": "Email address", + "phone": "Phone number", + "url": null, + "summary": null, + "location": { + "city": "City", + "countryCode": "Country code" + }, + "profiles": [ + { + "network": "Platform name", + "url": "Full URL", + "username": "Username from URL" + } + ] + } +} + +**IMPORTANT**: If there is any About Me or Summary section, add that to the summary section of the basics + +**CRITICAL**: For profiles section: +- ONLY extract URLs that are EXPLICITLY present in the resume markdown +- Look for URLs in markdown format [text](url) or plain URLs +- DO NOT create any URLs that are not in the original resume +- DO NOT add generic platform URLs like "https://github.com" or "https://linkedin.com" unless they are actually present in the resume +- If no URLs are found in the resume, return an empty profiles array: "profiles": [] +- If a URL is present but you cannot determine the network or username, still include it with the available information +- If the URL is of the format github.io or personal domain, mark the network as "Portfolio" +- If there is any link at the header of the resume or Portfolio link, add that as url in the basics + +**EXAMPLES OF WHAT NOT TO DO:** +- Do NOT add "https://github.com" if GitHub is not mentioned in the resume +- Do NOT add "https://linkedin.com" if LinkedIn is not mentioned in the resume +- Do NOT add "https://leetcode.com" if LeetCode is not mentioned in the resume +- Do NOT add "https://stackoverflow.com" if Stack Overflow is not mentioned in the resume + +**EXAMPLES OF WHAT TO DO:** +- If resume contains "GitHub: https://github.com/username", extract it +- If resume contains "[My Portfolio](https://example.com)", extract it +- If resume contains "LinkedIn: linkedin.com/in/username", extract it + +**IMPORTANT**: Return ONLY valid JSON. Do not include any explanatory text. + diff --git a/prompts/templates/education.jinja b/prompts/templates/education.jinja new file mode 100644 index 0000000..62fd095 --- /dev/null +++ b/prompts/templates/education.jinja @@ -0,0 +1,23 @@ +Extract ONLY the education information from this resume. + +--- The input markdown starts here --- + +{{ text_content }} + +--- The input markdown ends here --- + +Return ONLY a JSON object with this structure: +{ + "education": [ + { + "institution": "School/University name", + "area": "Field of study", + "studyType": "Degree type", + "startDate": "Start date (YYYY-MM)", + "endDate": "End date (YYYY-MM)", + "score": "GPA/Percentage" + } + ] +} + +**IMPORTANT**: Return ONLY valid JSON. Do not include any explanatory text. \ No newline at end of file diff --git a/prompts/templates/github_project_selection.jinja b/prompts/templates/github_project_selection.jinja new file mode 100644 index 0000000..d25799e --- /dev/null +++ b/prompts/templates/github_project_selection.jinja @@ -0,0 +1,100 @@ +You are an expert technical recruiter analyzing GitHub repositories to identify the most impressive and relevant projects for a software engineering position. + +**ABSOLUTE REQUIREMENT**: You must ONLY select projects where the author_commit_count is 4 or higher. Projects with 1, 2, or 3 commits indicate minimal involvement and should NEVER be selected. + +Given a list of GitHub repositories, select the TOP 7 most impressive projects that would be most relevant for evaluating a candidate's technical skills and experience. + +**IMPORTANT: Contributions to Popular Open Source Projects** +- **HIGH PRIORITY**: Contributions to well-known, popular open source projects (1000+ stars) are extremely valuable, even if the contribution is small +- Popular projects include: React, Vue, Angular, Node.js, Express, Django, Flask, TensorFlow, PyTorch, Kubernetes, Docker, VS Code, etc. +- A small contribution to a popular project (bug fix, documentation, feature) is often more impressive than a complete personal project +- Look for repositories that are forks of popular projects where the candidate has made meaningful contributions +- Consider the impact and reach of the project, not just the size of the contribution + +**Selection Criteria (in order of importance):** +1. **Author Contribution Level**: Projects where the candidate has made significant contributions (high author_commit_count) - HIGHEST PRIORITY +2. **Popular Open Source Contributions**: Contributions to well-known projects (1000+ stars) - HIGH PRIORITY +3. **Technical Complexity**: Projects that demonstrate advanced programming concepts, architecture, or problem-solving +4. **Real-world Impact**: Projects with actual users, deployments, or practical applications +5. **Code Quality**: Well-documented, maintained, and professional code +6. **Community Engagement**: Projects with stars, forks, or community contributions +7. **Technology Stack**: Projects using modern, relevant technologies +8. **Originality**: Unique projects rather than tutorial-based or classroom assignments + +**Projects to PRIORITIZE:** +- Projects with high author_commit_count (15+ commits) - indicates substantial involvement and deep engagement +- Projects with moderate author_commit_count (5-14 commits) - shows meaningful contribution +- Contributions to popular open source projects (React, Vue, Angular, Node.js, Express, Django, Flask, TensorFlow, PyTorch, Kubernetes, Docker, VS Code, etc.) +- Forks of popular projects with meaningful contributions +- Projects with significant community adoption (100+ stars) +- Projects that solve real-world problems +- Well-documented and maintained projects + +**Projects to AVOID:** +- Projects with very low author_commit_count (1-3 commits) - these indicate minimal involvement and should be avoided +- Simple tutorial projects (e.g., "Hello World", "HELLO-WORLD", basic calculators) - unless they're contributions to popular projects +- Classroom assignments with generic names +- Projects with very low stars/forks and no meaningful activity +- Very old projects with no recent activity +- Personal projects with no real-world impact (unless they demonstrate exceptional technical complexity) + +**Repository Data:** +{{ projects_data }} + +**FILTERING STEP:** +1. First, sort all projects by author_commit_count in descending order (highest to lowest) +2. Filter out all projects with author_commit_count less than 4 +3. Only consider projects where the candidate has made at least 4 commits +4. Start selection from the top of the sorted list (highest commit counts first) + +**CRITICAL REQUIREMENTS:** +- Select exactly 7 UNIQUE projects (no duplicates) if 7 or more qualifying projects exist +- If fewer than 7 qualifying projects exist, select ALL of them (do not pad with additional projects) +- Do not select the same repository multiple times +- Ensure all selected projects are distinct and represent different aspects of the candidate's skills +- **HARD REQUIREMENT**: Only select projects where author_commit_count is 4 or higher +- **HARD REQUIREMENT**: Do NOT select any project with author_commit_count of 1, 2, or 3 +- **HARD REQUIREMENT**: Start selection from projects with the highest author_commit_count + + +Select exactly 7 unique projects that best represent the candidate's technical abilities. + +**CRITICAL: Minimum Contribution Threshold** +- Only select projects where author_commit_count is 4 or higher +- Projects with 1-3 commits indicate minimal involvement and should be excluded +- Prioritize projects with 15+ commits (substantial involvement) +- Then prioritize projects with 5-14 commits (meaningful contribution) + +**Available High-Contribution Projects:** +Look for projects with author_commit_count of 15 or higher first, then projects with 5-14 commits. Start your selection from projects with the highest commit counts. + +Prioritize contributions to popular open source projects over personal projects. Respond with a JSON array containing only the selected project objects: + +[ + { + "name": "Project name", + "description": "Project description", + "github_url": "GitHub URL", + "live_url": "Live URL if available", + "technologies": ["tech1", "tech2"], + "reason_for_project_selection": "Reason why this project is selected", + "author_commit_count": 0, + "total_commit_count": 0, + "github_details": { + "stars": 0, + "forks": 0, + "language": "Primary language", + "description": "Description", + "created_at": "Creation date", + "updated_at": "Last updated date", + "topics": ["topic1", "topic2"], + "open_issues": 0, + "size": 0, + "fork": false, + "archived": false, + "default_branch": "main" + } + } +] + +Respond only with valid JSON, no additional text. \ No newline at end of file diff --git a/prompts/templates/projects.jinja b/prompts/templates/projects.jinja new file mode 100644 index 0000000..feea98a --- /dev/null +++ b/prompts/templates/projects.jinja @@ -0,0 +1,21 @@ +Extract ONLY the projects information from this resume. + +--- The input markdown starts here --- + +{{ text_content }} + +--- The input markdown ends here --- + +Return ONLY a JSON object with this structure: +{ + "projects": [ + { + "name": "Project name", + "description": "Project description", + "url": "Project URL", + "technologies": ["Tech 1", "Tech 2"] + } + ] +} + +**IMPORTANT**: Return ONLY valid JSON. Do not include any explanatory text. \ No newline at end of file diff --git a/prompts/templates/resume_evaluation_criteria.jinja b/prompts/templates/resume_evaluation_criteria.jinja new file mode 100644 index 0000000..45c0daf --- /dev/null +++ b/prompts/templates/resume_evaluation_criteria.jinja @@ -0,0 +1,185 @@ +You are evaluating a resume for a Software Intern position at HackerRank. Analyze the resume data and provide scores based on these criteria: + +**MANDATORY: You MUST always fill ALL FOUR categories: open_source, self_projects, production, technical_skills.** + +## CRITICAL FAIRNESS REQUIREMENTS +**SCORES MUST NEVER DEPEND ON:** +- Candidate's name, gender, or personal demographic information +- College, university, or educational institution name +- CGPA, GPA, or academic grades +- City, location, or geographical information +- Any personal characteristics unrelated to technical skills and experience + +**EVALUATION MUST BE BASED ONLY ON:** +- Technical skills and programming languages +- Project complexity and real-world impact +- Open source contributions and community involvement +- Work experience and production-level contributions +- Technical communication and documentation abilities +- Problem-solving and algorithmic thinking demonstrated in projects + +## PROGRAM DISTINCTIONS +- "Google Summer of Code (GSoC)" and "Girl Script Summer of Code" are COMPLETELY DIFFERENT programs +- NEVER use "GSoC" as shorthand for "Girl Script Summer of Code" +- When you see "Girl Script Summer of Code" in the resume, refer to it as "Girl Script Summer of Code" +- When you see "Google Summer of Code" in the resume, refer to it as "Google Summer of Code (GSoC)" + +## ANALYSIS INSTRUCTIONS +- Analyze the structured resume data (basics, work, volunteer, projects, skills, etc.) +- Use GitHub data (if provided in === GITHUB DATA === section) as additional context +- Use blog data (if provided in === BLOG DATA === section) for technical communication assessment + +## SCORING CRITERIA + +### Open Source (0-35 points) +**HIGH SCORES (25-35 points):** +- Contributions to popular open source projects (1000+ stars) +- Significant contributions to well-known projects +- Google Summer of Code (GSoC) participation +- Substantial community involvement + +**MEDIUM SCORES (15-24 points):** +- Contributions to smaller open source projects +- Active GitHub presence with meaningful contributions to other repositories +- Participation in open source programs + +**LOW SCORES (5-10 points):** +- Only personal GitHub repositories with no contributions to other projects +- Minimal open source activity +- Basic GitHub presence +- **CRITICAL**: Hacktoberfest participation alone (without evidence of contributions to significant projects) should receive 3-5 points maximum + +**VERY LOW SCORES (0-4 points):** +- No GitHub presence +- Only very basic personal repositories +- Repositories that are clearly tutorial-based with no community involvement + +**CRITICAL RULES:** +- Having personal GitHub repositories does NOT constitute open source contribution +- True open source contribution means contributing to OTHER people's projects +- When GitHub data shows all projects are 'self_project' type, open source score MUST be 10 points or less + +### Self Projects (0-30 points) +**HIGH SCORES (20-30 points):** +- Complex projects with real-world impact +- Advanced architecture, multiple technologies +- User adoption or contributions to popular open source projects + +**MEDIUM SCORES (10-19 points):** +- Projects with some complexity, good documentation +- Multiple features or moderate technical challenge + +**LOW SCORES (1-9 points):** +- Simple tutorial projects (todo lists, calculators, basic CRUD apps, weather apps, note-taking apps, recipe apps, exercise apps) +- Basic sentiment analysis using standard libraries (NLTK, scikit-learn) +- Classroom assignments or projects with minimal technical complexity + +**ZERO SCORES (0 points):** +- No projects or only extremely basic projects that demonstrate no technical skills + +**PROJECT LINK REQUIREMENTS:** +- **NO LINKS**: Projects without URLs, GitHub links, or live demos should receive 30-50% lower scores +- **INACTIVE LINKS**: Projects with broken links should receive 20-30% lower scores +- **LIVE DEMO BONUS**: Projects with working live demos should receive 10-20% higher scores + +### Production (0-25 points) +- Analyze the 'work' and 'volunteer' sections for real-world, internship, or production experience +- **SPECIAL CONSIDERATION**: Give extra points for founder roles, co-founder positions, or early-stage engineer roles (first 10-20 employees) at startups + +### Technical Skills (0-10 points) +- Analyze the 'skills', 'languages', and evidence of technical breadth or problem-solving in projects, work, or competitions + +## PROJECT COMPLEXITY ASSESSMENT + +**Simple/Basic Projects (Low Impact):** +- Todo list applications, calculators, basic CRUD applications +- Weather apps using public APIs, note-taking applications +- Simple portfolio websites, basic form applications +- "Hello World" applications, classroom assignment projects +- Tutorial-based projects, recipe sharing applications +- Exercise/health apps using public APIs +- Basic sentiment analysis using standard libraries +- Simple e-commerce applications, basic social media clones + +**Complex/Advanced Projects (High Impact):** +- Full-stack applications with multiple features +- Projects with user authentication and databases +- Machine learning or AI applications +- Real-time applications (chat, streaming, etc.) +- Mobile applications with native features +- Projects with microservices architecture +- Contributions to popular open source projects +- Projects with significant user adoption +- Projects solving real-world problems +- Projects demonstrating advanced algorithms or data structures + +## BONUS POINTS (Maximum total: 20 points) +- +5 points for Google Summer of Code (GSoC) participation +- +3 points for Girl Script Summer of Code participation +- +3-5 points for startup founder/co-founder experience +- +2-3 points for early-stage engineer experience (first 10-20 employees at a startup) +- +2 points for portfolio website (GitHub URL in basics.url) +- +1 point for LinkedIn profile +- +1-3 points for high-quality technical blogs (if blog data provided) + +**CRITICAL**: The total bonus points cannot exceed 20 points under any circumstances. + +## DEDUCTIONS +**For Simple Projects:** +- -2 to -5 points if resume contains only simple tutorial projects +- -1 to -3 points for each simple project beyond the first one +- -1 point for projects with generic names like "Calculator", "Todo App", "Weather App" +- -2 points if all projects are classroom assignments or tutorial-based + +**For Projects Without Links:** +- -3 to -5 points for each project without any GitHub link, live demo, or active URL +- -2 to -3 points for each project with only GitHub link but no live demo +- -1 to -2 points for each project with broken or inactive links + +**CRITICAL ENFORCEMENT:** +- When GitHub data shows all projects are 'self_project' type, apply 3-5 point deductions for lack of true open source contributions +- For candidates with only personal GitHub repositories, open source score should NEVER exceed 10 points +- For candidates with only tutorial-based projects, self_projects score should NEVER exceed 15 points + +## CRITICAL REQUIREMENTS +1. You MUST respond with ONLY the JSON structure below - no summary, no other fields +2. You MUST fill ALL FOUR score categories: open_source, self_projects, production, technical_skills +3. You MUST provide evidence for each score +4. You MUST NOT add any other fields like "summary", "skills", "experience", etc. +5. You MUST NOT change the field names or structure + +**IMPORTANT LIST CONSTRAINTS:** +- key_strengths: Provide 1-5 items (maximum 5 key strengths) +- areas_for_improvement: Provide 1-3 items (maximum 3 areas for improvement) + +**IMPORTANT SCORE CONSTRAINTS:** +- Evidence fields cannot be empty string +- All category scores must be >= 0 (cannot be negative) +- **CATEGORY SCORE LIMITS** (CANNOT be exceeded under any circumstances): + - open_source: 0-35 points (maximum 35) + - self_projects: 0-30 points (maximum 30) + - production: 0-25 points (maximum 25) + - technical_skills: 0-10 points (maximum 10) +- Bonus points total must be <= 20 (maximum 20 points) +- **OVERALL SCORE LIMIT**: The total score (categories + bonus - deductions) cannot exceed 120 points + +**DO NOT RETURN A RESUME SUMMARY. RETURN ONLY THE SCORING EVALUATION IN THE SPECIFIED JSON FORMAT.** + +Analyze the following resume and provide a JSON response with this EXACT structure (all fields are required): + +{ + "scores": { + "open_source": {"score": 0, "max": 35, "evidence": "string"}, + "self_projects": {"score": 0, "max": 30, "evidence": "string"}, + "production": {"score": 0, "max": 25, "evidence": "string"}, + "technical_skills": {"score": 0, "max": 10, "evidence": "string"} + }, + "bonus_points": {"total": 0, "breakdown": "string"}, + "deductions": {"total": 0, "reasons": "string"}, + "key_strengths": ["strength1", "strength2", "strength3", "strength4", "strength5"], + "areas_for_improvement": ["improvement1", "improvement2", "improvement3"] +} + +Resume to evaluate: + +{{ text_content }} \ No newline at end of file diff --git a/prompts/templates/resume_evaluation_system_message.jinja b/prompts/templates/resume_evaluation_system_message.jinja new file mode 100644 index 0000000..eb68c0f --- /dev/null +++ b/prompts/templates/resume_evaluation_system_message.jinja @@ -0,0 +1,49 @@ +You are an expert technical recruiter evaluating resumes. Provide accurate, objective evaluations based on the given criteria. + +**CRITICAL: You are NOT writing a resume summary. You are SCORING a resume for a job application.** + +**CRITICAL FAIRNESS REQUIREMENTS:** +**SCORES MUST NEVER DEPEND ON THE FOLLOWING FACTORS:** +- Candidate's name, gender, or any personal demographic information +- College, university, or educational institution name +- CGPA, GPA, or academic grades +- City, location, or geographical information +- Any personal characteristics unrelated to technical skills and experience + +**EVALUATION MUST BE BASED ONLY ON:** +- Technical skills and programming languages +- Project complexity and real-world impact +- Open source contributions and community involvement +- Work experience and production-level contributions +- Technical communication and documentation abilities +- Problem-solving and algorithmic thinking demonstrated in projects + +**MANDATORY: You MUST always fill ALL FOUR categories: open_source, self_projects, production, technical_skills.** + +- For open_source: Analyze all open source contributions, GitHub/GitLab activity, and community involvement. Look for Google Summer of Code (GSoC) and Girl Script Summer of Code participation. **CRITICAL**: Having personal GitHub repositories does NOT constitute open source contribution. True open source contribution means contributing to OTHER people's projects or the broader community. Personal repositories should receive low scores (5-10 points) unless they demonstrate exceptional complexity or community impact. **CRITICAL**: Hacktoberfest participation alone (without evidence of contributions to significant projects) should receive 5-8 points maximum. **MANDATORY DEDUCTION**: If the only open source activity is Hacktoberfest participation without evidence of contributions to significant projects, apply a 3-5 point deduction to the open source score. **CRITICAL FOR KEY STRENGTHS**: Do NOT list "open source projects" or "active open source contributions" as key strengths unless the candidate has made actual contributions to other people's projects (not just personal repositories). **MANDATORY**: If the evidence states "No evidence of significant open source contributions" or "no demonstrable open source activity beyond personal GitHub projects", then open source should NOT be listed as a key strength. **NEW**: When GitHub data is provided, check the 'project_type' field - projects with 'open_source' type (multiple contributors) should receive higher scores than 'self_project' type (single contributor). + +- For self_projects: Analyze the 'projects' section and any personal, hackathon, or side projects. **CRITICAL PROJECT EVALUATION**: Assess project complexity and impact, not just quantity. Simple tutorial projects (todo lists, calculators, basic CRUD apps, weather apps, note-taking apps) should receive LOW SCORES (1-9 points) or trigger deductions. **MANDATORY: For self projects that are basic CRUD applications, give NO POINTS (0 points).** Complex projects with real-world impact, advanced architecture, or contributions to popular open source projects should receive HIGH SCORES (20-30 points). Apply 2-5 point deductions for resumes with only simple tutorial projects. **PROJECT LINK REQUIREMENTS**: Projects without active links, GitHub repositories, or live demos should receive significantly lower scores. Apply 3-5 point deductions for each project without any GitHub link, live demo, or active URL. Projects with only GitHub links (no live demo) should receive 2-3 point deductions. Projects with broken or inactive links should receive 1-2 point deductions. Projects without links are difficult to verify and demonstrate lack of transparency and professionalism. + +- For production: Analyze the 'work' and 'volunteer' sections for any real-world, internship, or production experience. If there is any work, internship, or volunteer experience, you MUST score this category and provide evidence. **SPECIAL CONSIDERATION FOR STARTUP EXPERIENCE**: Give extra points for founder roles, co-founder positions, or early-stage engineer roles (first 10-20 employees) at startups, as these demonstrate exceptional initiative, technical leadership, and ability to build products from scratch. + +- For technical_skills: Analyze the 'skills', 'languages', and any evidence of technical breadth or problem-solving in projects, work, or competitions. You MUST score this category and provide evidence. + +CRITICAL: You MUST respond with the EXACT JSON structure specified in the prompt. Do not change category names, add extra fields, or modify the structure. The response must include ALL required fields: candidate_name, scores (with open_source, self_projects, production, technical_skills), bonus_points, deductions, key_strengths, areas_for_improvement. + +**IMPORTANT LIST CONSTRAINTS:** +- key_strengths: Provide 1-5 items (maximum 5 key strengths) +- areas_for_improvement: Provide 1-3 items (maximum 3 areas for improvement) + +**IMPORTANT SCORE CONSTRAINTS:** +- Evidence fields cannot be empty string +- All category scores must be >= 0 (cannot be negative) +- **CATEGORY SCORE LIMITS** (CANNOT be exceeded under any circumstances): + - open_source: 0-35 points (maximum 35) + - self_projects: 0-30 points (maximum 30) + - production: 0-25 points (maximum 25) + - technical_skills: 0-10 points (maximum 10) +- Bonus points total must be <= 20 (maximum 20 points) +- **CRITICAL**: The total bonus points cannot exceed 20 points under any circumstances +- **OVERALL SCORE LIMIT**: The total score (categories + bonus - deductions) cannot exceed 120 points + +IMPORTANT: Always check the structured 'profiles' section in the resume data before applying deductions for missing GitHub/portfolio. Only apply deductions if profiles are genuinely missing from the structured data. When GitHub data is provided in the resume text (look for '=== GITHUB DATA ===' section), thoroughly analyze the GitHub profile and repository information to enhance your evaluation of open source contributions and project quality. **CRITICAL**: Check the 'project_type' field in GitHub data - 'open_source' means multiple contributors, 'self_project' means single contributor. Self projects should receive low open source scores. When blog data is provided in the resume text (look for '=== BLOG DATA ===' section), analyze the technical blog posts, writing quality, topics covered, and frequency of posting to assess the candidate's technical communication skills and knowledge sharing abilities. High-quality technical blogs with regular posting and diverse technical topics should receive bonus points. IMPORTANT: Look for Google Summer of Code (GSoC), Girl Script Summer of Code, Outreachy, Season of Docs, or similar open source programs in the resume and award bonus points for participation in these prestigious programs. **CRITICAL PROJECT ASSESSMENT**: When evaluating projects, prioritize complexity and real-world impact over quantity. Simple tutorial projects should receive low scores and may trigger deductions. A single complex project is worth more than multiple simple ones. **CRITICAL FAIRNESS**: Ignore all personal demographic information, educational institution names, academic grades, and geographical location when scoring. Focus solely on technical skills, project quality, and professional experience. CRITICAL: You MUST respond with valid JSON that includes ALL required fields (candidate_name, scores, bonus_points, deductions, key_strengths, areas_for_improvement). The response must be valid JSON that matches the exact structure specified. Do not omit any fields or add extra fields. **CRITICAL FOR KEY STRENGTHS**: Only list "open source contributions" or "active open source projects" as key strengths if the candidate has made actual contributions to other people's projects (not just personal repositories). Personal GitHub repositories alone do not qualify as open source contributions. **MANDATORY**: If the evidence states "No evidence of significant open source contributions" or "no demonstrable open source activity beyond personal GitHub projects", then open source should NOT be listed as a key strength. \ No newline at end of file diff --git a/prompts/templates/skills.jinja b/prompts/templates/skills.jinja new file mode 100644 index 0000000..a94bb3a --- /dev/null +++ b/prompts/templates/skills.jinja @@ -0,0 +1,20 @@ +Extract ONLY the skills information from this resume. + +--- The input markdown starts here --- + +{{ text_content }} + +--- The input markdown ends here --- + +Return ONLY a JSON object with this structure: +{ + "skills": [ + { + "name": "Skill category", + "level": null, + "keywords": ["Skill 1", "Skill 2"] + } + ] +} + +**IMPORTANT**: Return ONLY valid JSON. Do not include any explanatory text. \ No newline at end of file diff --git a/prompts/templates/system_message.jinja b/prompts/templates/system_message.jinja new file mode 100644 index 0000000..e588771 --- /dev/null +++ b/prompts/templates/system_message.jinja @@ -0,0 +1,5 @@ +You are an expert resume parser. Extract ONLY the {{ section_name_param }} section from resumes and format it according to the JSON Resume specification. + +**CRITICAL: You must respond with ONLY valid JSON. Do not include any explanatory text, thinking process, markdown formatting, or tags. Return ONLY the JSON object.** + +Return ONLY the {{ section_name_param }} section in JSON format. \ No newline at end of file diff --git a/prompts/templates/work.jinja b/prompts/templates/work.jinja new file mode 100644 index 0000000..44d6b61 --- /dev/null +++ b/prompts/templates/work.jinja @@ -0,0 +1,36 @@ +Extract ONLY the work experience from this resume. + +--- The input markdown starts here --- + +{{ text_content }} + +--- The input markdown ends here --- + +Return ONLY a JSON object with this structure: +{ + "work": [ + { + "name": "Company name", + "position": "Job title", + "startDate": "Start date (YYYY-MM)", + "endDate": "End date (YYYY-MM) or 'Present'", + "summary": "Job description", + "highlights": ["Achievement 1", "Achievement 2"] + } + ] +} + +**IMPORTANT**: + +For date extraction, follow these rules carefully: + +1. Look for date ranges in the work experience section. These often appear as "Start – End" or "Start - End" format. +2. Date ranges can be separated by any of these characters: hyphen "-", en dash "–", em dash "—", or the word "to". +3. Carefully extract both the start and end dates from these ranges. +4. If a position is current/ongoing, the end date may be indicated by words like "Present", "Current", "Now", "Ongoing", etc. +5. Look for dates near position titles or company names +6. Dates might be formatted as "Month Year" or "MM/YYYY" or other variations +7. If dates appear as "Month Year - Month Year", extract both parts +8. If only years are provided (e.g., "2019-2021"), use the year information only + +**IMPORTANT**: Return ONLY valid JSON. Do not include any explanatory text. \ No newline at end of file diff --git a/pymupdf_rag.py b/pymupdf_rag.py new file mode 100644 index 0000000..0489715 --- /dev/null +++ b/pymupdf_rag.py @@ -0,0 +1,1377 @@ +""" +This script accepts a PDF document filename and converts it to a text file +in Markdown format, compatible with the GitHub standard. + +It must be invoked with the filename like this: + +python pymupdf_rag.py input.pdf [-pages PAGES] + +The "PAGES" parameter is a string (containing no spaces) of comma-separated +page numbers to consider. Each item is either a single page number or a +number range "m-n". Use "N" to address the document's last page number. +Example: "-pages 2-15,40,43-N" + +It will produce a markdown text file called "input.md". + +Text will be sorted in Western reading order. Any table will be included in +the text in markdwn format as well. + +Dependencies +------------- +PyMuPDF v1.25.5 or later + +Copyright and License +---------------------- +Copyright (C) 2024-2025 Artifex Software, Inc. + +PyMuPDF4LLM is free software: you can redistribute it and/or modify it under the +terms of the GNU Affero General Public License as published by the Free +Software Foundation, either version 3 of the License, or (at your option) +any later version. + +Alternative licensing terms are available from the licensor. +For commercial licensing, see or contact +Artifex Software, Inc., 39 Mesa Street, Suite 108A, San Francisco, +CA 94129, USA, for further information. +""" + +import os +import string +from binascii import b2a_base64 +import pymupdf +from pymupdf import mupdf +from pymupdf4llm.helpers.get_text_lines import get_raw_lines, is_white +from pymupdf4llm.helpers.multi_column import column_boxes +from dataclasses import dataclass +from collections import defaultdict + +pymupdf.TOOLS.unset_quad_corrections(True) + +# Characters recognized as bullets when starting a line. +bullet = tuple( + [ + "- ", + "* ", + "> ", + chr(0xB6), + chr(0xB7), + chr(8224), + chr(8225), + chr(8226), + chr(0xF0A7), + chr(0xF0B7), + ] + + list(map(chr, range(9632, 9680))) +) + +GRAPHICS_TEXT = "\n![](%s)\n" + + +class IdentifyHeaders: + """Compute data for identifying header text. + + All non-white text from all selected pages is extracted and its font size + noted as a rounded value. + The most frequent font size (and all smaller ones) is taken as body text + font size. + Larger font sizes are mapped to strings of multiples of '#', the header + tag in Markdown, which in turn is Markdown's representation of HTML's + header tags

to

. + Larger font sizes than body text but smaller than the
font size are + represented as
. + """ + + def __init__( + self, + doc: str, + pages: list = None, + body_limit: float = 12, # force this to be body text + max_levels: int = 6, # accept this many header levels + ): + """Read all text and make a dictionary of fontsizes. + + Args: + doc: PDF document or filename + pages: consider these page numbers only + body_limit: treat text with larger font size as a header + """ + if not isinstance(max_levels, int) or max_levels not in range(1, 7): + raise ValueError("max_levels must be an integer between 1 and 6") + if isinstance(doc, pymupdf.Document): + mydoc = doc + else: + mydoc = pymupdf.open(doc) + + if pages is None: # use all pages if omitted + pages = range(mydoc.page_count) + + fontsizes = defaultdict(int) + for pno in pages: + page = mydoc.load_page(pno) + blocks = page.get_text("dict", flags=pymupdf.TEXTFLAGS_TEXT)["blocks"] + for span in [ # look at all non-empty horizontal spans + s + for b in blocks + for l in b["lines"] + for s in l["spans"] + if not is_white(s["text"]) + ]: + fontsz = round(span["size"]) # # compute rounded fontsize + fontsizes[fontsz] += len(span["text"].strip()) # add character count + + if mydoc != doc: + # if opened here, close it now + mydoc.close() + + # maps a fontsize to a string of multiple # header tag characters + self.header_id = {} + + # If not provided, choose the most frequent font size as body text. + # If no text at all on all pages, just use body_limit. + # In any case all fonts not exceeding + temp = sorted( + [(k, v) for k, v in fontsizes.items()], key=lambda i: (i[1], i[0]) + ) + if temp: + # most frequent font size + self.body_limit = max(body_limit, temp[-1][0]) + else: + self.body_limit = body_limit + + # identify up to 6 font sizes as header candidates + sizes = sorted( + [f for f in fontsizes.keys() if f > self.body_limit], + reverse=True, + )[:max_levels] + + # make the header tag dictionary + for i, size in enumerate(sizes, start=1): + self.header_id[size] = "#" * i + " " + if self.header_id.keys(): + self.body_limit = min(self.header_id.keys()) - 1 + + def get_header_id(self, span: dict, page=None) -> str: + """Return appropriate markdown header prefix. + + Given a text span from a "dict"/"rawdict" extraction, determine the + markdown header prefix string of 0 to n concatenated '#' characters. + """ + fontsize = round(span["size"]) # compute fontsize + if fontsize <= self.body_limit: + return "" + hdr_id = self.header_id.get(fontsize, "") + return hdr_id + + +class TocHeaders: + """Compute data for identifying header text. + + This is an alternative to IdentifyHeaders. Instead of running through the + full document to identify font sizes, it uses the document's Table Of + Contents (TOC) to identify headers on pages. + Like IdentifyHeaders, this also is no guarantee to find headers, but it + represents a good chance for appropriately built documents. In such cases, + this method can be very much faster and more accurate, because we can + directly use the hierarchy level of TOC items to ientify the header level. + Examples where this works very well are the Adobe PDF documents. + """ + + def __init__(self, doc: str): + """Read and store the TOC of the document.""" + if isinstance(doc, pymupdf.Document): + mydoc = doc + else: + mydoc = pymupdf.open(doc) + + self.TOC = doc.get_toc() + if mydoc != doc: + # if opened here, close it now + mydoc.close() + + def get_header_id(self, span: dict, page=None) -> str: + """Return appropriate markdown header prefix. + + Given a text span from a "dict"/"rawdict" extraction, determine the + markdown header prefix string of 0 to n concatenated '#' characters. + """ + if not page: + return "" + # check if this page has TOC entries with an actual title + my_toc = [t for t in self.TOC if t[1] and t[-1] == page.number + 1] + if not my_toc: # no TOC items present on this page + return "" + # Check if the span matches a TOC entry. This must be done in the + # most forgiving way: exact matches are rare animals. + text = span["text"].strip() # remove leading and trailing whitespace + for t in my_toc: + title = t[1].strip() # title of TOC entry + lvl = t[0] # level of TOC entry + if text.startswith(title) or title.startswith(text): + # found a match: return the header tag + return "#" * lvl + " " + return "" + + +# store relevant parameters here +@dataclass +class Parameters: + pass + + +def refine_boxes(boxes, enlarge=0): + """Join any rectangles with a pairwise non-empty overlap. + + Accepts and returns a list of Rect items. + Note that rectangles that only "touch" each other (common point or edge) + are not considered as overlapping. + Use a positive "enlarge" parameter to enlarge rectangle by these many + points in every direction. + + TODO: Consider using a sweeping line algorithm for this. + """ + delta = (-enlarge, -enlarge, enlarge, enlarge) + new_rects = [] + # list of all vector graphic rectangles + prects = boxes[:] + + while prects: # the algorithm will empty this list + r = +prects[0] + delta # copy of first rectangle + repeat = True # initialize condition + while repeat: + repeat = False # set false as default + for i in range(len(prects) - 1, 0, -1): # from back to front + if r.intersects(prects[i].irect): # enlarge first rect with this + r |= prects[i] + del prects[i] # delete this rect + repeat = True # indicate must try again + + # first rect now includes all overlaps + new_rects.append(r) + del prects[0] + + new_rects = sorted(set(new_rects), key=lambda r: (r.x0, r.y0)) + return new_rects + + +def is_significant(box, paths): + """Check whether the rectangle "box" contains 'signifiant' drawings. + + This means that some path is contained in the "interior" of box. + To this end, we build a sub-box of 90% of the original box and check + whether this still contains drawing paths. + """ + if box.width > box.height: + d = box.width * 0.025 + else: + d = box.height * 0.025 + nbox = box + (d, d, -d, -d) # nbox covers 90% of box interior + # paths contained in, but not equal to box: + my_paths = [p for p in paths if p["rect"] in box and p["rect"] != box] + widths = set(round(p["rect"].width) for p in my_paths) | {round(box.width)} + heights = set(round(p["rect"].height) for p in my_paths) | {round(box.height)} + if len(widths) == 1 or len(heights) == 1: + return False # all paths are horizontal or vertical lines / rectangles + for p in my_paths: + rect = p["rect"] + if ( + not (rect & nbox).is_empty and not p["rect"].is_empty + ): # intersects interior: significant! + return True + # Remaining case: a horizontal or vertical line + # horizontal line: + if ( + 1 + and rect.y0 == rect.y1 + and nbox.y0 <= rect.y0 <= nbox.y1 + and rect.x0 < nbox.x1 + and rect.x1 > nbox.x0 + ): + pass # return True + # vertical line + if ( + 1 + and rect.x0 == rect.x1 + and nbox.x0 <= rect.x0 <= nbox.x1 + and rect.y0 < nbox.y1 + and rect.y1 > nbox.y0 + ): + pass # return True + return False + + +def to_markdown( + doc, + *, + pages=None, + hdr_info=None, + write_images=False, + embed_images=False, + ignore_images=False, + ignore_graphics=False, + detect_bg_color=True, + image_path="", + image_format="png", + image_size_limit=0.05, + filename=None, + force_text=True, + page_chunks=False, + page_separators=False, + margins=0, + dpi=150, + page_width=612, + page_height=None, + table_strategy="lines_strict", + graphics_limit=None, + fontsize_limit=3, + ignore_code=False, + extract_words=False, + show_progress=False, + use_glyphs=False, + ignore_alpha=False, +) -> str: + """Process the document and return the text of the selected pages. + + Args: + doc: pymupdf.Document or string. + pages: list of page numbers to consider (0-based). + hdr_info: callable or object having method 'get_hdr_info'. + write_images: (bool) save images / graphics as files. + embed_images: (bool) embed images in markdown text (base64 encoded) + image_path: (str) store images in this folder. + image_format: (str) use this image format. Choose a supported one. + force_text: (bool) output text despite of image background. + page_chunks: (bool) whether to segment output by page. + page_separators: (bool) whether to include page separators in output. + margins: omit content overlapping margin areas. + dpi: (int) desired resolution for generated images. + page_width: (float) assumption if page layout is variable. + page_height: (float) assumption if page layout is variable. + table_strategy: choose table detection strategy + graphics_limit: (int) if vector graphics count exceeds this, ignore all. + ignore_code: (bool) suppress code-like formatting (mono-space fonts) + extract_words: (bool, False) include "words"-like output in page chunks + show_progress: (bool, False) print progress as each page is processed. + use_glyphs: (bool, False) replace the Invalid Unicode by glyph numbers. + ignore_alpha: (bool, True) ignore text with alpha = 0 (transparent). + + """ + if write_images is False and embed_images is False and force_text is False: + raise ValueError("Image and text on images cannot both be suppressed.") + if embed_images is True: + write_images = False + image_path = "" + if not 0 <= image_size_limit < 1: + raise ValueError("'image_size_limit' must be non-negative and less than 1.") + DPI = dpi + IGNORE_CODE = ignore_code + IMG_EXTENSION = image_format + EXTRACT_WORDS = extract_words + if EXTRACT_WORDS is True: + page_chunks = True + ignore_code = True + IMG_PATH = image_path + if IMG_PATH and write_images is True and not os.path.exists(IMG_PATH): + os.mkdir(IMG_PATH) + + if not isinstance(doc, pymupdf.Document): + doc = pymupdf.open(doc) + + FILENAME = doc.name if filename is None else filename + GRAPHICS_LIMIT = graphics_limit + FONTSIZE_LIMIT = fontsize_limit + IGNORE_IMAGES = ignore_images + IGNORE_GRAPHICS = ignore_graphics + DETECT_BG_COLOR = detect_bg_color + if doc.is_form_pdf or (doc.is_pdf and doc.has_annots()): + doc.bake() + + # for reflowable documents allow making 1 page for the whole document + if doc.is_reflowable: + if hasattr(page_height, "__float__"): + # accept user page dimensions + doc.layout(width=page_width, height=page_height) + else: + # no page height limit given: make 1 page for whole document + doc.layout(width=page_width, height=792) + page_count = doc.page_count + height = 792 * page_count # height that covers full document + doc.layout(width=page_width, height=height) + + if pages is None: # use all pages if no selection given + pages = list(range(doc.page_count)) + + if hasattr(margins, "__float__"): + margins = [margins] * 4 + if len(margins) == 2: + margins = (0, margins[0], 0, margins[1]) + if len(margins) != 4: + raise ValueError("margins must be one, two or four floats") + elif not all(hasattr(m, "__float__") for m in margins): + raise ValueError("margin values must be floats") + + # If "hdr_info" is not an object with a method "get_header_id", scan the + # document and use font sizes as header level indicators. + if callable(hdr_info): + get_header_id = hdr_info + elif hasattr(hdr_info, "get_header_id") and callable(hdr_info.get_header_id): + get_header_id = hdr_info.get_header_id + elif hdr_info is False: + get_header_id = lambda s, page=None: "" + else: + hdr_info = IdentifyHeaders(doc) + get_header_id = hdr_info.get_header_id + + def max_header_id(spans, page): + hdr_ids = sorted( + [l for l in set([len(get_header_id(s, page=page)) for s in spans]) if l > 0] + ) + if not hdr_ids: + return "" + return "#" * (hdr_ids[0] - 1) + " " + + def resolve_links(links, span): + """Accept a span and return a markdown link string. + + Args: + links: a list as returned by page.get_links() + span: a span dictionary as returned by page.get_text("dict") + + Returns: + None or a string representing the link in MD format. + """ + bbox = pymupdf.Rect(span["bbox"]) # span bbox + span_text = span["text"].strip() + + # Find all links that overlap with this span + overlapping_links = [] + for link in links: + hot = link["from"] # the hot area of the link + # Check if the link area intersects with the span bbox + if bbox.intersects(hot): + overlapping_links.append(link) + + if not overlapping_links: + return None + + # If only one link, return simple format + if len(overlapping_links) == 1: + link = overlapping_links[0] + # Check if this looks like a partial URL (starts with http or contains domain parts) + if span_text.startswith("http"): + # Use the full link URL as the display text + return f'[{link["uri"]}]({link["uri"]})' + else: + return f'[{span_text}]({link["uri"]})' + + # Multiple links found - need to split the text + return _resolve_multiple_links(span_text, overlapping_links, bbox) + + def _resolve_multiple_links(span_text, links, span_bbox): + """Resolve multiple links within a single span text. + + Args: + span_text: The text content of the span + links: List of overlapping links + span_bbox: The bounding box of the span + + Returns: + str: Markdown formatted text with multiple links + """ + # Common patterns for multiple links + if "|" in span_text: + # Split by pipe separator + parts = [part.strip() for part in span_text.split("|")] + if len(parts) == len(links): + # Perfect match - each part corresponds to a link + result_parts = [] + for i, (part, link) in enumerate(zip(parts, links)): + result_parts.append(f'[{part}]({link["uri"]})') + return " | ".join(result_parts) + elif len(parts) >= len(links): + # More parts than links - try to match intelligently + return _match_parts_to_links(parts, links, span_bbox) + + # Try to identify individual words that should be linked + words = span_text.split() + if len(words) >= len(links): + return _match_words_to_links(words, links, span_bbox) + + # Fallback: return the first link with the full text + return f'[{span_text}]({links[0]["uri"]})' + + def _match_parts_to_links(parts, links, span_bbox): + """Match parts of text to specific links based on content and position.""" + # Common platform names to match + platform_keywords = { + "github": ["github", "git"], + "linkedin": ["linkedin", "linked"], + "hackerrank": ["hackerrank", "hacker"], + "twitter": ["twitter", "tweet"], + "portfolio": ["portfolio", "site", "website"], + "behance": ["behance"], + "dribbble": ["dribbble"], + "leetcode": ["leetcode", "leet"], + "stackoverflow": ["stackoverflow", "stack"], + } + + result_parts = [] + used_links = set() + + for part in parts: + part_lower = part.lower() + matched_link = None + + # Try to match by platform keywords + for platform, keywords in platform_keywords.items(): + if any(keyword in part_lower for keyword in keywords): + # Find corresponding link + for i, link in enumerate(links): + if i not in used_links: + uri = link.get("uri", "").lower() + if platform in uri: + matched_link = link + used_links.add(i) + break + if matched_link: + break + + # If no keyword match, try to find the best remaining link + if not matched_link and links: + for i, link in enumerate(links): + if i not in used_links: + matched_link = link + used_links.add(i) + break + + if matched_link: + result_parts.append(f'[{part}]({matched_link["uri"]})') + else: + result_parts.append(part) + + return " | ".join(result_parts) + + def _match_words_to_links(words, links, span_bbox): + """Match individual words to links based on position and content.""" + # Simple heuristic: distribute links evenly among words + if len(words) == len(links): + result_parts = [] + for word, link in zip(words, links): + result_parts.append(f'[{word}]({link["uri"]})') + return " ".join(result_parts) + + # If more words than links, try to match by content + return _match_parts_to_links(words, links, span_bbox) + + def save_image(parms, rect, i): + """Optionally render the rect part of a page. + + We will ignore images that are empty or that have an edge smaller + than x% of the corresponding page edge.""" + page = parms.page + if ( + rect.width < page.rect.width * image_size_limit + or rect.height < page.rect.height * image_size_limit + ): + return "" + if write_images is True or embed_images is True: + pix = page.get_pixmap(clip=rect, dpi=DPI) + else: + return "" + if pix.height <= 0 or pix.width <= 0: + return "" + + if write_images is True: + filename = os.path.basename(parms.filename).replace(" ", "-") + image_filename = os.path.join( + IMG_PATH, f"{filename}-{page.number}-{i}.{IMG_EXTENSION}" + ) + pix.save(image_filename) + return image_filename.replace("\\", "/") + elif embed_images is True: + # make a base64 encoded string of the image + data = b2a_base64(pix.tobytes(IMG_EXTENSION)).decode() + data = f"data:image/{IMG_EXTENSION};base64," + data + return data + return "" + + def write_text( + parms, + clip: pymupdf.Rect, + tables=True, + images=True, + force_text=force_text, + ): + """Output the text found inside the given clip. + + This is an alternative for plain text in that it outputs + text enriched with markdown styling. + The logic is capable of recognizing headers, body text, code blocks, + inline code, bold, italic and bold-italic styling. + There is also some effort for list supported (ordered / unordered) in + that typical characters are replaced by respective markdown characters. + + 'tables'/'images' indicate whether this execution should output these + objects. + """ + + if clip is None: + clip = parms.clip + out_string = "" + + # This is a list of tuples (linerect, spanlist) + nlines = get_raw_lines( + parms.textpage, + clip=clip, + tolerance=3, + ignore_invisible=not parms.accept_invisible, + ) + nlines = [ + l for l in nlines if not intersects_rects(l[0], parms.tab_rects.values()) + ] + + parms.line_rects.extend([l[0] for l in nlines]) # store line rectangles + + prev_lrect = None # previous line rectangle + prev_bno = -1 # previous block number of line + code = False # mode indicator: outputting code + prev_hdr_string = None + + for lrect, spans in nlines: + # there may be tables or images inside the text block: skip them + if intersects_rects(lrect, parms.img_rects): + continue + + # ------------------------------------------------------------ + # Pick up tables ABOVE this text block + # ------------------------------------------------------------ + if tables: + tab_candidates = [ + (i, tab_rect) + for i, tab_rect in parms.tab_rects.items() + if tab_rect.y1 <= lrect.y0 + and i not in parms.written_tables + and ( + 0 + or lrect.x0 <= tab_rect.x0 < lrect.x1 + or lrect.x0 < tab_rect.x1 <= lrect.x1 + or tab_rect.x0 <= lrect.x0 < lrect.x1 <= tab_rect.x1 + ) + ] + for i, _ in tab_candidates: + out_string += "\n" + parms.tabs[i].to_markdown(clean=False) + "\n" + if EXTRACT_WORDS: + # for "words" extraction, add table cells as line rects + cells = sorted( + set( + [ + pymupdf.Rect(c) + for c in parms.tabs[i].header.cells + + parms.tabs[i].cells + if c is not None + ] + ), + key=lambda c: (c.y1, c.x0), + ) + parms.line_rects.extend(cells) + parms.written_tables.append(i) + prev_hdr_string = None + + # ------------------------------------------------------------ + # Pick up images / graphics ABOVE this text block + # ------------------------------------------------------------ + if images: + for i in range(len(parms.img_rects)): + if i in parms.written_images: + continue + r = parms.img_rects[i] + if r.y1 <= lrect.y0 and ( + 0 + or lrect.x0 <= r.x0 < lrect.x1 + or lrect.x0 < r.x1 <= lrect.x1 + or r.x0 <= lrect.x0 < lrect.x1 <= r.x1 + ): + pathname = save_image(parms, r, i) + if pathname: + out_string += GRAPHICS_TEXT % pathname + + # recursive invocation + if force_text is True: + img_txt = write_text( + parms, + r, + tables=False, + images=False, + force_text=True, + ) + + if not is_white(img_txt): + out_string += img_txt + parms.written_images.append(i) + prev_hdr_string = None + + parms.line_rects.append(lrect) + # if line rect is far away from the previous one, add a line break + if ( + len(parms.line_rects) > 1 + and lrect.y1 - parms.line_rects[-2].y1 > lrect.height * 1.5 + ): + out_string += "\n" + # make text string for the full line + text = " ".join([s["text"] for s in spans]).strip() + + # full line strikeout? + all_strikeout = all([s["char_flags"] & 1 for s in spans]) + # full line italic? + all_italic = all([s["flags"] & 2 for s in spans]) + # full line bold? + all_bold = all([(s["flags"] & 16) or (s["char_flags"] & 8) for s in spans]) + # full line mono-spaced? + all_mono = all([s["flags"] & 8 for s in spans]) + + # if line is a header, this will return multiple "#" characters, + # otherwise an empty string + hdr_string = max_header_id(spans, page=parms.page) # a header? + + if hdr_string: # if a header line, process it specially + # Check if any spans in this heading have links + has_links = False + for s in spans: + if resolve_links(parms.links, s): + has_links = True + break + + if has_links: + # Process heading with links span-by-span + header_text = "" + for s in spans: + # Apply heading formatting to each span + span_text = s["text"].strip() + + # Apply font formatting + if all_mono: + span_text = "`" + span_text + "`" + if all_italic: + span_text = "_" + span_text + "_" + if all_bold: + span_text = "**" + span_text + "**" + if all_strikeout: + span_text = "~~" + span_text + "~~" + + # Resolve links for this span + ltext = resolve_links(parms.links, s) + if ltext: + header_text += ltext + " " + else: + header_text += span_text + " " + + # Add the heading prefix and output + if hdr_string != prev_hdr_string: + out_string += hdr_string + header_text.strip() + "\n" + else: + # intercept if header text has been broken in multiple lines + while out_string.endswith("\n"): + out_string = out_string[:-1] + out_string += " " + header_text.strip() + "\n" + prev_hdr_string = hdr_string + continue + else: + # No links in heading, process as before + if all_mono: + text = "`" + text + "`" + if all_italic: + text = "_" + text + "_" + if all_bold: + text = "**" + text + "**" + if all_strikeout: + text = "~~" + text + "~~" + if hdr_string != prev_hdr_string: + out_string += hdr_string + text + "\n" + else: + # intercept if header text has been broken in multiple lines + while out_string.endswith("\n"): + out_string = out_string[:-1] + out_string += " " + text + "\n" + prev_hdr_string = hdr_string + continue + + prev_hdr_string = hdr_string + + # start or extend a code block + if all_mono and not IGNORE_CODE: + if not code: # if not already in code output mode: + out_string += "```\n" # switch on "code" mode + code = True + # compute approx. distance from left - assuming a width + # of 0.5*fontsize. + delta = int((lrect.x0 - clip.x0) / (spans[0]["size"] * 0.5)) + indent = " " * delta + + out_string += indent + text + "\n" + continue # done with this line + + if code and not all_mono: + out_string += "```\n" # switch off code mode + code = False + + span0 = spans[0] + bno = span0["block"] # block number of line + if bno != prev_bno: + out_string += "\n" + prev_bno = bno + + if ( # check if we need another line break + prev_lrect + and lrect.y1 - prev_lrect.y1 > lrect.height * 1.5 + or span0["text"].startswith("[") + or span0["text"].startswith(bullet) + or span0["flags"] & 1 # superscript? + ): + out_string += "\n" + prev_lrect = lrect + + # this line is not all-mono, so switch off "code" mode + if code: # in code output mode? + out_string += "```\n" # switch of code mode + code = False + + for i, s in enumerate(spans): # iterate spans of the line + # decode font properties + mono = s["flags"] & 8 + bold = s["flags"] & 16 or s["char_flags"] & 8 + italic = s["flags"] & 2 + strikeout = s["char_flags"] & 1 + + prefix = "" + suffix = "" + if mono: + prefix = "`" + prefix + suffix += "`" + if bold: + prefix = "**" + prefix + suffix += "**" + if italic: + prefix = "_" + prefix + suffix += "_" + if strikeout: + prefix = "~~" + prefix + suffix += "~~" + + # convert intersecting link to markdown syntax + ltext = resolve_links(parms.links, s) + if ltext: + text = f"{hdr_string}{prefix}{ltext}{suffix} " + else: + text = f"{hdr_string}{prefix}{s['text'].strip()}{suffix} " + if text.startswith(bullet): + text = "- " + text[1:] + text = text.replace(" ", " ") + dist = span0["bbox"][0] - clip.x0 + cwidth = (span0["bbox"][2] - span0["bbox"][0]) / len(span0["text"]) + if cwidth == 0.0: + cwidth = span0["size"] * 0.5 + text = " " * int(round(dist / cwidth)) + text + + out_string += text + if not code: + out_string += "\n" + out_string += "\n" + if code: + out_string += "```\n" # switch of code mode + code = False + out_string += "\n\n" + return ( + out_string.replace(" \n", "\n").replace(" ", " ").replace("\n\n\n", "\n\n") + ) + + def is_in_rects(rect, rect_list): + """Check if rect is contained in a rect of the list.""" + for i, r in enumerate(rect_list, start=1): + if rect in r: + return i + return 0 + + def intersects_rects(rect, rect_list): + """Check if middle of rect is contained in a rect of the list.""" + delta = (-1, -1, 1, 1) # enlarge rect_list members somewhat by this + enlarged = rect + delta + abs_enlarged = abs(enlarged) * 0.5 + for i, r in enumerate(rect_list, start=1): + if abs(enlarged & r) > abs_enlarged: + return i + return 0 + + def output_tables(parms, text_rect): + """Output tables above given text rectangle.""" + this_md = "" # markdown string for table(s) content + if text_rect is not None: # select tables above the text block + for i, trect in sorted( + [j for j in parms.tab_rects.items() if j[1].y1 <= text_rect.y0], + key=lambda j: (j[1].y1, j[1].x0), + ): + if i in parms.written_tables: + continue + this_md += parms.tabs[i].to_markdown(clean=False) + "\n" + if EXTRACT_WORDS: + # for "words" extraction, add table cells as line rects + cells = sorted( + set( + [ + pymupdf.Rect(c) + for c in parms.tabs[i].header.cells + + parms.tabs[i].cells + if c is not None + ] + ), + key=lambda c: (c.y1, c.x0), + ) + parms.line_rects.extend(cells) + parms.written_tables.append(i) # do not touch this table twice + + else: # output all remaining tables + for i, trect in parms.tab_rects.items(): + if i in parms.written_tables: + continue + this_md += parms.tabs[i].to_markdown(clean=False) + "\n" + if EXTRACT_WORDS: + # for "words" extraction, add table cells as line rects + cells = sorted( + set( + [ + pymupdf.Rect(c) + for c in parms.tabs[i].header.cells + + parms.tabs[i].cells + if c is not None + ] + ), + key=lambda c: (c.y1, c.x0), + ) + parms.line_rects.extend(cells) + parms.written_tables.append(i) # do not touch this table twice + return this_md + + def output_images(parms, text_rect, force_text): + """Output images and graphics above text rectangle.""" + if not parms.img_rects: + return "" + this_md = "" # markdown string + if text_rect is not None: # select images above the text block + for i, img_rect in enumerate(parms.img_rects): + if img_rect.y0 > text_rect.y0: + continue + if img_rect.x0 >= text_rect.x1 or img_rect.x1 <= text_rect.x0: + continue + if i in parms.written_images: + continue + pathname = save_image(parms, img_rect, i) + parms.written_images.append(i) # do not touch this image twice + if pathname: + this_md += GRAPHICS_TEXT % pathname + if force_text: + img_txt = write_text( + parms, + img_rect, + tables=False, # we have no tables here + images=False, # we have no other images here + force_text=True, + ) + if not is_white(img_txt): # was there text at all? + this_md += img_txt + else: # output all remaining images + for i, img_rect in enumerate(parms.img_rects): + if i in parms.written_images: + continue + pathname = save_image(parms, img_rect, i) + parms.written_images.append(i) # do not touch this image twice + if pathname: + this_md += GRAPHICS_TEXT % pathname + if force_text: + img_txt = write_text( + parms, + img_rect, + tables=False, # we have no tables here + images=False, # we have no other images here + force_text=True, + ) + if not is_white(img_txt): + this_md += img_txt + + return this_md + + def page_is_ocr(page): + """Check if page exclusivley contains OCR text. + + For this to be true, all text must be written as "ignore-text". + """ + try: + text_types = set([b[0] for b in page.get_bboxlog() if "text" in b[0]]) + if text_types == {"ignore-text"}: + return True + except: + pass + return False + + def get_bg_color(page): + """Determine the background color of the page. + + The function returns a PDF RGB color triple or None. + We check the color of 10 x 10 pixel areas in the four corners of the + page. If they are unicolor and of the same color, we assume this to + be the background color. + """ + pix = page.get_pixmap( + clip=(page.rect.x0, page.rect.y0, page.rect.x0 + 10, page.rect.y0 + 10) + ) + if not pix.samples or not pix.is_unicolor: + return None + pixel_ul = pix.pixel(0, 0) # upper left color + pix = page.get_pixmap( + clip=(page.rect.x1 - 10, page.rect.y0, page.rect.x1, page.rect.y0 + 10) + ) + if not pix.samples or not pix.is_unicolor: + return None + pixel_ur = pix.pixel(0, 0) # upper right color + if not pixel_ul == pixel_ur: + return None + pix = page.get_pixmap( + clip=(page.rect.x0, page.rect.y1 - 10, page.rect.x0 + 10, page.rect.y1) + ) + if not pix.samples or not pix.is_unicolor: + return None + pixel_ll = pix.pixel(0, 0) # lower left color + if not pixel_ul == pixel_ll: + return None + pix = page.get_pixmap( + clip=(page.rect.x1 - 10, page.rect.y1 - 10, page.rect.x1, page.rect.y1) + ) + if not pix.samples or not pix.is_unicolor: + return None + pixel_lr = pix.pixel(0, 0) # lower right color + if not pixel_ul == pixel_lr: + return None + return (pixel_ul[0] / 255, pixel_ul[1] / 255, pixel_ul[2] / 255) + + def get_metadata(doc, pno): + meta = doc.metadata.copy() + meta["file_path"] = FILENAME + meta["page_count"] = doc.page_count + meta["page"] = pno + 1 + return meta + + def sort_words(words: list) -> list: + """Reorder words in lines. + + The argument list must be presorted by bottom, then left coordinates. + + Words with similar top / bottom coordinates are assumed to belong to + the same line and will be sorted left to right within that line. + """ + if not words: + return [] + nwords = [] + line = [words[0]] + lrect = pymupdf.Rect(words[0][:4]) + for w in words[1:]: + if abs(w[1] - lrect.y0) <= 3 or abs(w[3] - lrect.y1) <= 3: + line.append(w) + lrect |= w[:4] + else: + line.sort(key=lambda w: w[0]) + nwords.extend(line) + line = [w] + lrect = pymupdf.Rect(w[:4]) + line.sort(key=lambda w: w[0]) + nwords.extend(line) + return nwords + + def get_page_output( + doc, pno, margins, textflags, FILENAME, IGNORE_IMAGES, IGNORE_GRAPHICS + ): + """Process one page. + + Args: + doc: pymupdf.Document + pno: 0-based page number + textflags: text extraction flag bits + + Returns: + Markdown string of page content and image, table and vector + graphics information. + """ + page = doc[pno] + page.remove_rotation() # make sure we work on rotation=0 + parms = Parameters() # all page information + parms.page = page + parms.filename = FILENAME + parms.md_string = "" + parms.images = [] + parms.tables = [] + parms.graphics = [] + parms.words = [] + parms.line_rects = [] + parms.accept_invisible = ( + page_is_ocr(page) or ignore_alpha + ) # accept invisible text + + # determine background color + parms.bg_color = None if not DETECT_BG_COLOR else get_bg_color(page) + + left, top, right, bottom = margins + parms.clip = page.rect + (left, top, -right, -bottom) + + # extract external links on page + parms.links = [l for l in page.get_links() if l["kind"] == pymupdf.LINK_URI] + + # extract annotation rectangles on page + parms.annot_rects = [a.rect for a in page.annots()] + + # make a TextPage for all later extractions + parms.textpage = page.get_textpage(flags=textflags, clip=parms.clip) + + # extract images on page + if not IGNORE_IMAGES: + img_info = page.get_image_info() + else: + img_info = [] + for i in range(len(img_info)): + img_info[i]["bbox"] = pymupdf.Rect(img_info[i]["bbox"]) + + # filter out images that are too small or outside the clip + img_info = [ + i + for i in img_info + if i["bbox"].width >= image_size_limit * parms.clip.width + and i["bbox"].height >= image_size_limit * parms.clip.height + and i["bbox"].intersects(parms.clip) + and i["bbox"].width > 3 + and i["bbox"].height > 3 + ] + + # sort descending by image area size + img_info.sort(key=lambda i: abs(i["bbox"]), reverse=True) + + # subset of images truly inside the clip + if img_info: + img_max_size = abs(parms.clip) * 0.9 + sane = [i for i in img_info if abs(i["bbox"] & parms.clip) < img_max_size] + if len(sane) < len(img_info): # found some + img_info = sane # use those images instead + # output full page image + name = save_image(parms, parms.clip, "full") + if name: + parms.md_string += GRAPHICS_TEXT % name + + img_info = img_info[:30] # only accept the largest up to 30 images + # run from back to front (= small to large) + for i in range(len(img_info) - 1, 0, -1): + r = img_info[i]["bbox"] + if r.is_empty: + del img_info[i] + continue + for j in range(i): # image areas larger than r + if r in img_info[j]["bbox"]: + del img_info[i] # contained in some larger image + break + parms.images = img_info + + parms.img_rects = [i["bbox"] for i in parms.images] + + # catch too-many-graphics situation + graphics_count = len([b for b in page.get_bboxlog() if "path" in b[0]]) + if GRAPHICS_LIMIT and graphics_count > GRAPHICS_LIMIT: + IGNORE_GRAPHICS = True + + # Locate all tables on page + parms.written_tables = [] # stores already written tables + omitted_table_rects = [] + parms.tabs = [] + if IGNORE_GRAPHICS or not table_strategy: + # do not try to extract tables + pass + else: + tabs = page.find_tables(clip=parms.clip, strategy=table_strategy) + for t in tabs.tables: + # remove tables with too few rows or columns + if t.row_count < 2 or t.col_count < 2: + omitted_table_rects.append(pymupdf.Rect(t.bbox)) + continue + parms.tabs.append(t) + parms.tabs.sort(key=lambda t: (t.bbox[0], t.bbox[1])) + + # Make a list of table boundary boxes. + # Must include the header bbox (which may exist outside tab.bbox) + tab_rects = {} + for i, t in enumerate(parms.tabs): + tab_rects[i] = pymupdf.Rect(t.bbox) | pymupdf.Rect(t.header.bbox) + tab_dict = { + "bbox": tuple(tab_rects[i]), + "rows": t.row_count, + "columns": t.col_count, + } + parms.tables.append(tab_dict) + parms.tab_rects = tab_rects + # list of table rectangles + parms.tab_rects0 = list(tab_rects.values()) + + # Select paths not intersecting any table. + # Ignore full page graphics. + # Ignore fill paths having the background color. + if not IGNORE_GRAPHICS: + paths = [ + p + for p in page.get_drawings() + if p["rect"] in parms.clip + and p["rect"].width < parms.clip.width + and p["rect"].height < parms.clip.height + and (p["rect"].width > 3 or p["rect"].height > 3) + and not (p["type"] == "f" and p["fill"] == parms.bg_color) + and not intersects_rects(p["rect"], parms.tab_rects0) + and not intersects_rects(p["rect"], parms.annot_rects) + ] + else: + paths = [] + # catch too-many-graphics situation + if GRAPHICS_LIMIT and len(paths) > GRAPHICS_LIMIT: + paths = [] + + # We also ignore vector graphics that only represent + # "text emphasizing sugar". + vg_clusters0 = [] # worthwhile vector graphics go here + + # walk through all vector graphics outside any table + clusters = page.cluster_drawings(drawings=paths) + for bbox in clusters: + if is_significant(bbox, paths): + vg_clusters0.append(bbox) + + # remove paths that are not in some relevant graphic + parms.actual_paths = [p for p in paths if is_in_rects(p["rect"], vg_clusters0)] + + # also add image rectangles to the list and vice versa + vg_clusters0.extend(parms.img_rects) + parms.img_rects.extend(vg_clusters0) + parms.img_rects = sorted(set(parms.img_rects), key=lambda r: (r.y1, r.x0)) + parms.written_images = [] + # these may no longer be pairwise disjoint: + # remove area overlaps by joining into larger rects + parms.vg_clusters0 = refine_boxes(vg_clusters0) + + parms.vg_clusters = dict((i, r) for i, r in enumerate(parms.vg_clusters0)) + # identify text bboxes on page, avoiding tables, images and graphics + text_rects = column_boxes( + parms.page, + paths=parms.actual_paths, + no_image_text=not force_text, + textpage=parms.textpage, + avoid=parms.tab_rects0 + parms.vg_clusters0, + footer_margin=margins[3], + header_margin=margins[1], + ignore_images=IGNORE_IMAGES, + ) + + """ + ------------------------------------------------------------------ + Extract markdown text iterating over text rectangles. + We also output any tables. They may live above, below or inside + the text rectangles. + ------------------------------------------------------------------ + """ + for text_rect in text_rects: + # output tables above this rectangle + parms.md_string += output_tables(parms, text_rect) + parms.md_string += output_images(parms, text_rect, force_text) + + # output text inside this rectangle + parms.md_string += write_text( + parms, + text_rect, + force_text=force_text, + images=True, + tables=True, + ) + + parms.md_string = parms.md_string.replace(" ,", ",").replace("-\n", "") + + # write any remaining tables and images + parms.md_string += output_tables(parms, None) + parms.md_string += output_images(parms, None, force_text) + + while parms.md_string.startswith("\n"): + parms.md_string = parms.md_string[1:] + parms.md_string = parms.md_string.replace(chr(0), chr(0xFFFD)) + + if EXTRACT_WORDS is True: + # output words in sequence compliant with Markdown text + rawwords = parms.textpage.extractWORDS() + rawwords.sort(key=lambda w: (w[3], w[0])) + + words = [] + for lrect in parms.line_rects: + lwords = [] + for w in rawwords: + wrect = pymupdf.Rect(w[:4]) + if wrect in lrect: + lwords.append(w) + words.extend(sort_words(lwords)) + + # remove word duplicates without spoiling the sequence + # duplicates may occur for multiple reasons + nwords = [] # words w/o duplicates + for w in words: + if w not in nwords: + nwords.append(w) + words = nwords + + else: + words = [] + parms.words = words + if page_separators: + # add page separators to output + parms.md_string += f"\n\n--- end of page={parms.page.number} ---\n\n" + return parms + + if page_chunks is False: + document_output = "" + else: + document_output = [] + + # read the Table of Contents + toc = doc.get_toc() + + # Text extraction flags: + # omit clipped text, collect styles, use accurate bounding boxes + textflags = ( + 0 + | mupdf.FZ_STEXT_CLIP + | mupdf.FZ_STEXT_ACCURATE_BBOXES + # | mupdf.FZ_STEXT_IGNORE_ACTUALTEXT + | 32768 # mupdf.FZ_STEXT_COLLECT_STYLES + ) + # optionally replace 0xFFFD by glyph number + if use_glyphs: + textflags |= mupdf.FZ_STEXT_USE_GID_FOR_UNKNOWN_UNICODE + + for pno in pages: + parms = get_page_output( + doc, pno, margins, textflags, FILENAME, IGNORE_IMAGES, IGNORE_GRAPHICS + ) + if page_chunks is False: + document_output += parms.md_string + else: + # build subet of TOC for this page + page_tocs = [t for t in toc if t[-1] == pno + 1] + + metadata = get_metadata(doc, pno) + document_output.append( + { + "metadata": metadata, + "toc_items": page_tocs, + "tables": parms.tables, + "images": parms.images, + "graphics": parms.graphics, + "text": parms.md_string, + "words": parms.words, + } + ) + del parms + + return document_output diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..df14bd5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +PyMuPDF==1.26.3 +ollama==0.5.1 +pydantic==2.11.7 +requests==2.32.4 +pymupdf4llm==0.0.27 +Jinja2==3.1.6 +google-generativeai==0.4.0 +python-dotenv==1.0.1 +black==25.9.0 \ No newline at end of file diff --git a/score.py b/score.py new file mode 100644 index 0000000..21fd06c --- /dev/null +++ b/score.py @@ -0,0 +1,377 @@ +import os +import sys +import json +import logging +import csv +from pdf import PDFHandler +from github import fetch_and_display_github_info +from models import JSONResume, EvaluationData +from typing import List, Optional, Dict +from evaluator import ResumeEvaluator +from pathlib import Path +from prompt import DEFAULT_MODEL, MODEL_PARAMETERS +from transform import ( + transform_evaluation_response, + convert_json_resume_to_text, + convert_github_data_to_text, + convert_blog_data_to_text, +) +from config import DEVELOPMENT_MODE + +logger = logging.getLogger(__name__) + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)5s - %(lineno)5d - %(funcName)33s - %(levelname)5s - %(message)s", +) + + +def print_evaluation_results( + evaluation: EvaluationData, candidate_name: str = "Candidate" +): + """Print evaluation results in a readable format.""" + print("\n" + "=" * 80) + print(f"📊 RESUME EVALUATION RESULTS FOR: {candidate_name}") + print("=" * 80) + + if not evaluation: + print("❌ No evaluation data available") + return + + # Calculate overall score + total_score = 0 + max_score = 0 + + if hasattr(evaluation, "scores") and evaluation.scores: + for category_name, category_data in evaluation.scores.model_dump().items(): + category_score = min(category_data["score"], category_data["max"]) + total_score += category_score + max_score += category_data["max"] + + # Log warning if score was capped + if category_score < category_data["score"]: + print( + f"⚠️ Warning: {category_name} score capped from {category_data['score']} to {category_score} (max: {category_data['max']})" + ) + + # Add bonus points + if hasattr(evaluation, "bonus_points") and evaluation.bonus_points: + total_score += evaluation.bonus_points.total + + # Subtract deductions + if hasattr(evaluation, "deductions") and evaluation.deductions: + total_score -= evaluation.deductions.total + + # Ensure total score doesn't exceed maximum possible score + max_possible_score = max_score + 20 # 120 (100 categories + 20 bonus) + if total_score > max_possible_score: + total_score = max_possible_score + print(f"⚠️ Warning: Total score capped at maximum possible value") + + # Overall Score + print(f"\n🎯 OVERALL SCORE: {total_score:.1f}/{max_score}") + + # Detailed Scores + print("\n📈 DETAILED SCORES:") + print("-" * 60) + + if hasattr(evaluation, "scores") and evaluation.scores: + # Define category maximums + category_maxes = { + "open_source": 35, + "self_projects": 30, + "production": 25, + "technical_skills": 10, + } + + # Open Source + if hasattr(evaluation.scores, "open_source") and evaluation.scores.open_source: + os_score = evaluation.scores.open_source + capped_score = min(os_score.score, category_maxes["open_source"]) + print(f"🌐 Open Source: {capped_score}/{os_score.max}") + print(f" Evidence: {os_score.evidence}") + print() + + # Self Projects + if ( + hasattr(evaluation.scores, "self_projects") + and evaluation.scores.self_projects + ): + sp_score = evaluation.scores.self_projects + capped_score = min(sp_score.score, category_maxes["self_projects"]) + print(f"🚀 Self Projects: {capped_score}/{sp_score.max}") + print(f" Evidence: {sp_score.evidence}") + print() + + # Production Experience + if hasattr(evaluation.scores, "production") and evaluation.scores.production: + prod_score = evaluation.scores.production + capped_score = min(prod_score.score, category_maxes["production"]) + print(f"🏢 Production Experience: {capped_score}/{prod_score.max}") + print(f" Evidence: {prod_score.evidence}") + print() + + # Technical Skills + if ( + hasattr(evaluation.scores, "technical_skills") + and evaluation.scores.technical_skills + ): + tech_score = evaluation.scores.technical_skills + capped_score = min(tech_score.score, category_maxes["technical_skills"]) + print(f"💻 Technical Skills: {capped_score}/{tech_score.max}") + print(f" Evidence: {tech_score.evidence}") + print() + + # Bonus Points + if hasattr(evaluation, "bonus_points") and evaluation.bonus_points: + print(f"\n⭐ BONUS POINTS: {evaluation.bonus_points.total}") + print("-" * 30) + print(f" {evaluation.bonus_points.breakdown}") + + # Deductions + if ( + hasattr(evaluation, "deductions") + and evaluation.deductions + and evaluation.deductions.total > 0 + ): + print(f"\n⚠️ DEDUCTIONS: -{evaluation.deductions.total}") + print("-" * 30) + if evaluation.deductions.reasons: + print(f" {evaluation.deductions.reasons}") + + # Key Strengths + if hasattr(evaluation, "key_strengths") and evaluation.key_strengths: + print(f"\n✅ KEY STRENGTHS:") + print("-" * 30) + for i, strength in enumerate(evaluation.key_strengths, 1): + print(f" {i}. {strength}") + + # Areas for Improvement + if ( + hasattr(evaluation, "areas_for_improvement") + and evaluation.areas_for_improvement + ): + print(f"\n🔧 AREAS FOR IMPROVEMENT:") + print("-" * 30) + for i, area in enumerate(evaluation.areas_for_improvement, 1): + print(f" {i}. {area}") + + print("\n" + "=" * 80) + + +def _evaluate_resume( + resume_data: JSONResume, github_data: dict = None, blog_data: dict = None +) -> Optional[EvaluationData]: + """Evaluate the resume using AI and display results.""" + + model_params = MODEL_PARAMETERS.get(DEFAULT_MODEL) + evaluator = ResumeEvaluator(model_name=DEFAULT_MODEL, model_params=model_params) + + # Convert JSON resume data to text + resume_text = convert_json_resume_to_text(resume_data) + + # Add GitHub data if available + if github_data: + github_text = convert_github_data_to_text(github_data) + resume_text += github_text + + # Add blog data if available + if blog_data: + blog_text = convert_blog_data_to_text(blog_data) + resume_text += blog_text + + # Evaluate the enhanced resume + evaluation_result = evaluator.evaluate_resume(resume_text) + + # print(evaluation_result) + + return evaluation_result + + +def is_valid_resume_data(resume_data: JSONResume) -> bool: + """Check if the resume data has at least some extracted core content.""" + if not resume_data: + return False + core_sections = [ + resume_data.basics, + resume_data.work, + resume_data.education, + resume_data.skills, + resume_data.projects, + ] + return any(section is not None for section in core_sections) + + +def find_profile(profiles, network): + if not profiles: + return None + return next( + (p for p in profiles if p.network and p.network.lower() == network.lower()), + None, + ) + + +def main(pdf_path): + # Create cache filename based on PDF path + cache_filename = ( + f"cache/resumecache_{os.path.basename(pdf_path).replace('.pdf', '')}.json" + ) + github_cache_filename = ( + f"cache/githubcache_{os.path.basename(pdf_path).replace('.pdf', '')}.json" + ) + + resume_data = None + cache_loaded = False + + # Check if cache exists and we're in development mode + if DEVELOPMENT_MODE and os.path.exists(cache_filename): + print(f"Loading cached data from {cache_filename}") + try: + cached_data = json.loads(Path(cache_filename).read_text(encoding="utf-8")) + loaded_resume = JSONResume(**cached_data) + if not is_valid_resume_data(loaded_resume): + raise ValueError("Cached resume data contains no core content") + resume_data = loaded_resume + cache_loaded = True + except Exception as e: + print(f"⚠️ Warning: Invalid cache file {cache_filename}: {e}") + print("Ignoring cache and reprocessing PDF...") + try: + os.remove(cache_filename) + except Exception as delete_err: + print( + f"Failed to delete invalid cache file {cache_filename}: {delete_err}" + ) + + if not cache_loaded: + logger.debug( + f"Extracting data from PDF" + + (" and caching to " + cache_filename if DEVELOPMENT_MODE else "") + ) + pdf_handler = PDFHandler() + resume_data = pdf_handler.extract_json_from_pdf(pdf_path) + + if resume_data == None: + return None + + if DEVELOPMENT_MODE: + if is_valid_resume_data(resume_data): + os.makedirs(os.path.dirname(cache_filename), exist_ok=True) + Path(cache_filename).write_text( + json.dumps(resume_data.model_dump(), indent=2, ensure_ascii=False), + encoding="utf-8", + ) + else: + logger.warning( + "Newly extracted resume data is empty/invalid. Skipping cache write." + ) + + # Check if cache exists and we're in development mode + github_data = {} + github_cache_loaded = False + if DEVELOPMENT_MODE and os.path.exists(github_cache_filename): + print(f"Loading cached data from {github_cache_filename}") + try: + loaded_github = json.loads( + Path(github_cache_filename).read_text(encoding="utf-8") + ) + if ( + not isinstance(loaded_github, dict) + or not loaded_github + or "profile" not in loaded_github + ): + raise ValueError("Cached GitHub data is invalid or empty") + github_data = loaded_github + github_cache_loaded = True + except Exception as e: + print(f"⚠️ Warning: Invalid GitHub cache file {github_cache_filename}: {e}") + print("Ignoring GitHub cache and refetching...") + try: + os.remove(github_cache_filename) + except Exception as delete_err: + print( + f"Failed to delete invalid GitHub cache file {github_cache_filename}: {delete_err}" + ) + + if not github_cache_loaded: + # Add validation to handle None values + profiles = [] + if resume_data and hasattr(resume_data, "basics") and resume_data.basics: + profiles = resume_data.basics.profiles or [] + github_profile = find_profile(profiles, "Github") + + if github_profile: + print( + f"Fetching GitHub data" + + ( + " and caching to " + github_cache_filename + if DEVELOPMENT_MODE + else "" + ) + ) + github_data = fetch_and_display_github_info(github_profile.url) + + if ( + DEVELOPMENT_MODE + and github_data + and isinstance(github_data, dict) + and "profile" in github_data + ): + os.makedirs(os.path.dirname(github_cache_filename), exist_ok=True) + Path(github_cache_filename).write_text( + json.dumps(github_data, indent=2, ensure_ascii=False), + encoding="utf-8", + ) + + score = _evaluate_resume(resume_data, github_data) + + # Get candidate name for display + candidate_name = os.path.basename(pdf_path).replace(".pdf", "") + if ( + resume_data + and hasattr(resume_data, "basics") + and resume_data.basics + and resume_data.basics.name + ): + candidate_name = resume_data.basics.name + + # Print evaluation results in readable format + print_evaluation_results(score, candidate_name) + + if DEVELOPMENT_MODE: + csv_row = transform_evaluation_response( + file_name=os.path.basename(pdf_path), + evaluation=score, + resume_data=resume_data, + github_data=github_data, + ) + + # Write CSV row to file + csv_path = "resume_evaluations.csv" + file_exists = os.path.exists(csv_path) + + with open(csv_path, "a", newline="", encoding="utf-8") as csvfile: + fieldnames = list(csv_row.keys()) + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + + # Write headers if file doesn't exist + if not file_exists: + writer.writeheader() + + # Write the row + writer.writerow(csv_row) + + return score + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("Usage: python score.py ") + exit(1) + pdf_path = sys.argv[1] + + if not os.path.exists(pdf_path): + print(f"Error: File '{pdf_path}' does not exist.") + exit(1) + + main(pdf_path) diff --git a/transform.py b/transform.py new file mode 100644 index 0000000..25eab1d --- /dev/null +++ b/transform.py @@ -0,0 +1,938 @@ +from typing import Dict, List, Optional +import pdb +from models import JSONResume + + +def transform_parsed_data(parsed_data: Dict) -> Dict: + try: + if isinstance(parsed_data, dict): + if "basics" in parsed_data and len(parsed_data) > 1: + transformed = { + "basics": transform_basics(parsed_data.get("basics", {})), + "work": transform_work_experience( + parsed_data.get( + "work_experience", + parsed_data.get("work", parsed_data.get("experience", [])), + ) + ), + "volunteer": transform_organizations( + parsed_data.get("organizations", []) + ), + "education": transform_education(parsed_data.get("education", [])), + "awards": transform_achievements( + parsed_data.get( + "achievements", + parsed_data.get( + "awards", parsed_data.get("honors_and_awards", []) + ), + ) + ), + "certificates": parsed_data.get("certificates", []), + "publications": parsed_data.get("publications", []), + "skills": transform_skills_comprehensive(parsed_data), + "languages": parsed_data.get("languages", []), + "interests": parsed_data.get("interests", []), + "references": parsed_data.get("references", []), + "projects": transform_projects_comprehensive(parsed_data), + "meta": parsed_data.get("meta", {}), + } + else: + if "basics" in parsed_data: + basics_data = parsed_data.get("basics", parsed_data) + transformed = {"basics": transform_basics(basics_data)} + elif ( + "work" in parsed_data + or "work_experience" in parsed_data + or "experience" in parsed_data + ): + work_data = parsed_data.get( + "work", + parsed_data.get( + "work_experience", parsed_data.get("experience", []) + ), + ) + transformed = {"work": transform_work_experience(work_data)} + elif "education" in parsed_data: + transformed = { + "education": transform_education( + parsed_data.get("education", []) + ) + } + elif ( + "skills" in parsed_data + or "librariesFrameworks" in parsed_data + or "toolsPlatforms" in parsed_data + or "databases" in parsed_data + ): + transformed = { + "skills": transform_skills_comprehensive(parsed_data) + } + elif "projects" in parsed_data or "projectsOpenSource" in parsed_data: + transformed = { + "projects": transform_projects_comprehensive(parsed_data) + } + elif ( + "awards" in parsed_data + or "achievements" in parsed_data + or "honors_and_awards" in parsed_data + ): + awards_data = parsed_data.get( + "awards", + parsed_data.get( + "achievements", parsed_data.get("honors_and_awards", []) + ), + ) + transformed = {"awards": transform_achievements(awards_data)} + else: + transformed = parsed_data + + return transformed + else: + return parsed_data + + except Exception as e: + print(f"Error transforming parsed data: {e}") + return parsed_data + + +def extract_domain_from_url(url: str) -> str: + try: + if "://" in url: + url = url.split("://")[1] + domain = url.split("/")[0] + if domain.startswith("www."): + domain = domain[4:] + return domain + except Exception: + return "" + + +def get_network_name(domain: str) -> str: + domain_mapping = { + "github.com": "GitHub", + "linkedin.com": "LinkedIn", + "leetcode.com": "LeetCode", + "stackoverflow.com": "Stack Overflow", + "hackerrank.com": "HackerRank", + "behance.net": "Behance", + "dev.to": "DEV Community", + "twitter.com": "X", + "x.com": "X", + } + return domain_mapping.get(domain, "") + + +def transform_basics(basics_data: Dict) -> Dict: + if not isinstance(basics_data, dict): + return basics_data + + profiles = basics_data.get("profiles", []) + + transformed_profiles = [] + if isinstance(profiles, list): + for i, profile in enumerate(profiles): + if isinstance(profile, dict): + transformed_profile = profile.copy() + url = transformed_profile.get("url", "") + network = transformed_profile.get("network") + + if url and network is None: + domain = extract_domain_from_url(url) + network_name = get_network_name(domain) + + if network_name: + transformed_profile["network"] = network_name + username = extract_username_from_url(url, domain) + if username: + transformed_profile["username"] = username + transformed_profiles.append(transformed_profile) + + basics_data["profiles"] = transformed_profiles + return basics_data + + +def extract_username_from_url(url: str, domain: str) -> str: + try: + path = url.split(domain)[1] if domain in url else "" + if not path: + return "" + path = path.lstrip("/") + + parts = [part for part in path.split("/") if part] + + if parts: + if domain == "linkedin.com": + return parts[1] + elif domain == "stackoverflow.com": + return parts[2] + else: + return parts[0] + return "" + except Exception: + return "" + + +def transform_work_experience(work_list: List) -> List[Dict]: + transformed = [] + for item in work_list: + if isinstance(item, dict): + description = item.get("description", "") + if isinstance(description, list): + description = " ".join(description) + + # Try to parse from 'startDate' if it contains a date range + start_date_input = item.get("startDate", "") + if start_date_input and any( + month in start_date_input + for month in [ + "Jan", + "Feb", + "Mar", + "Apr", + "May", + "Jun", + "Jul", + "Aug", + "Sep", + "Oct", + "Nov", + "Dec", + ] + ): + start_date, end_date = parse_date_range(start_date_input) + else: + # Use existing startDate and endDate values + start_date = item.get("startDate") + end_date = item.get("endDate") + + transformed.append( + { + "name": item.get("name", ""), + "position": item.get( + "position", item.get("type", item.get("title", "")) + ), + "url": item.get("url", None), + "startDate": start_date, + "endDate": end_date, + "summary": item.get("summary", description), + "highlights": item.get("highlights", []), + } + ) + return transformed + + +def transform_organizations(org_list: List) -> List[Dict]: + transformed = [] + for item in org_list: + if isinstance(item, dict): + transformed.append( + { + "organization": item.get("name", ""), + "position": item.get("role", ""), + "url": item.get("url", None), + "startDate": None, + "endDate": "Present", + "summary": None, + "highlights": [], + } + ) + return transformed + + +def transform_education(edu_list: List) -> List[Dict]: + transformed = [] + for item in edu_list: + if isinstance(item, dict): + if "degree" in item: + score = item.get("gpa", item.get("percentage", None)) + if score is not None: + score = str(score) + + start_date, end_date = parse_date_range(item.get("years", "")) + transformed.append( + { + "institution": item.get("institution", ""), + "url": item.get("url", None), + "area": ( + item.get("degree", "").split(", ")[-1] + if "," in item.get("degree", "") + else None + ), + "studyType": ( + item.get("degree", "").split(", ")[0] + if "," in item.get("degree", "") + else item.get("degree", "") + ), + "startDate": start_date, + "endDate": end_date, + "score": score, + "courses": [], + } + ) + else: + transformed.append(item) + return transformed + + +def transform_achievements(achievements_list: List) -> List[Dict]: + transformed = [] + for item in achievements_list: + if isinstance(item, dict): + title = item.get("title", item.get("name", "")) + awarder = item.get("awarder", item.get("organization", "")) + summary = item.get("summary", item.get("description", None)) + + transformed.append( + { + "title": title, + "date": f"{item.get('year', '')}-01" if item.get("year") else None, + "awarder": awarder, + "summary": summary, + } + ) + return transformed + + +def transform_skills(skills_list: List) -> List[Dict]: + transformed = [] + for item in skills_list: + if isinstance(item, dict): + if "category" in item: + transformed.append( + { + "name": item.get("category", ""), + "level": None, + "keywords": item.get("keywords", []), + } + ) + else: + transformed.append(item) + return transformed + + +def transform_projects(projects_list: List) -> List[Dict]: + transformed = [] + for item in projects_list: + if isinstance(item, dict): + skills = [] + project_name = item.get("name", "") + if "|" in project_name: + name_parts = project_name.split("|") + if len(name_parts) > 1: + skills_part = name_parts[1].strip() + skills = [skill.strip() for skill in skills_part.split(",")] + item["name"] = name_parts[0].strip() + + technologies = item.get("technologies", []) + if isinstance(technologies, str): + technologies = [tech.strip() for tech in technologies.split(",")] + + if not skills and technologies: + skills = technologies + + transformed.append( + { + "name": item.get("name", ""), + "startDate": None, + "endDate": None, + "description": item.get("description", ""), + "highlights": [item.get("type", "")] if item.get("type") else [], + "url": item.get("url", None), + "technologies": technologies, + "skills": skills, + } + ) + return transformed + + +def transform_skills_comprehensive(parsed_data: Dict) -> List[Dict]: + skills = [] + + if "skills" in parsed_data and isinstance(parsed_data["skills"], list): + if parsed_data["skills"] and isinstance(parsed_data["skills"][0], str): + skills.append( + { + "name": "Programming Languages", + "level": None, + "keywords": parsed_data["skills"], + } + ) + else: + skills.extend(transform_skills(parsed_data["skills"])) + + skill_categories = { + "librariesFrameworks": "Libraries/Frameworks", + "toolsPlatforms": "Tools/Platforms", + "databases": "Databases", + } + + for field, category_name in skill_categories.items(): + if field in parsed_data and isinstance(parsed_data[field], list): + skills.append( + {"name": category_name, "level": None, "keywords": parsed_data[field]} + ) + + return skills + + +def transform_projects_comprehensive(parsed_data: Dict) -> List[Dict]: + projects = [] + + if "projects" in parsed_data: + projects.extend(transform_projects(parsed_data["projects"])) + + if "projectsOpenSource" in parsed_data: + for item in parsed_data["projectsOpenSource"]: + if isinstance(item, dict): + skills = [] + project_name = item.get("name", "") + if "|" in project_name: + name_parts = project_name.split("|") + if len(name_parts) > 1: + skills_part = name_parts[1].strip() + skills = [skill.strip() for skill in skills_part.split(",")] + item["name"] = name_parts[0].strip() + + projects.append( + { + "name": item.get("name", ""), + "startDate": None, + "endDate": None, + "description": item.get("summary", ""), + "highlights": [], + "url": item.get("url", None), + "technologies": item.get("technologies", []), + "skills": skills, + } + ) + + return projects + + +def parse_date_range(date_range: str) -> tuple: + """ + Parse date range and return both start and end dates. + For format like "Jan-Mar 2021", returns ("Jan 2021", "Mar 2021") + """ + if not date_range: + return None, None + + # Handle "onwards" case + if "onwards" in date_range: + # Extract the start date from "onwards" format + start_part = date_range.replace("onwards", "").strip() + if start_part: + return start_part, "Present" + return None, "Present" + + # Handle format like "Jan-Mar 2021" + if " " in date_range and any( + month in date_range + for month in [ + "Jan", + "Feb", + "Mar", + "Apr", + "May", + "Jun", + "Jul", + "Aug", + "Sep", + "Oct", + "Nov", + "Dec", + ] + ): + parts = date_range.split(" ") + if len(parts) >= 2: + year = parts[-1] + month_map = { + "Jan": "Jan", + "Feb": "Feb", + "Mar": "Mar", + "Apr": "Apr", + "May": "May", + "Jun": "Jun", + "Jul": "Jul", + "Aug": "Aug", + "Sep": "Sep", + "Oct": "Oct", + "Nov": "Nov", + "Dec": "Dec", + } + + # Check if it's a range like "Jan-Mar 2021" + if "-" in parts[0] and len(parts[0].split("-")) == 2: + start_month, end_month = parts[0].split("-") + start_date = f"{month_map.get(start_month, start_month)} {year}" + end_date = f"{month_map.get(end_month, end_month)} {year}" + return start_date, end_date + else: + # Single month format like "Jan 2021" + month = month_map.get(parts[0], parts[0]) + start_date = f"{month} {year}" + return start_date, None + + # Handle year range like "2020-2021" + if "-" in date_range and len(date_range.split("-")) == 2: + start_year, end_year = date_range.split("-") + start_date = f"{start_year}-01" + end_date = f"{end_year}-12" + return start_date, end_date + + return None, None + + +def fetch_profile(profiles, network_names, prefix): + """Helper function to extract profile information for a given network.""" + for network in network_names: + profile = next( + (p for p in profiles if p.network and p.network.lower() == network.lower()), + None, + ) + if profile: + return profile + + +def transform_evaluation_response( + file_name=None, resume_data=None, github_data=None, evaluation=None +): + """ + Transform the three inputs (resume_data, github_data, evaluation) into the most important columns as a CSV row. + + Args: + resume_data: JSONResume object containing parsed resume data + github_data: dict containing GitHub profile data + evaluation: EvaluationData object containing evaluation results + + Returns: + dict: Dictionary with the most important columns for CSV output + """ + csv_row = {} + + csv_row["file_name"] = file_name + + # Extract basic information from resume_data + if resume_data and hasattr(resume_data, "basics") and resume_data.basics: + basics = resume_data.basics + csv_row["name"] = basics.name if basics.name else "" + csv_row["email"] = basics.email if basics.email else "" + csv_row["phone"] = basics.phone if basics.phone else "" + csv_row["location"] = ( + f"{basics.location.city}, {basics.location.region}" + if basics.location + else "" + ) + csv_row["summary"] = basics.summary if basics.summary else "" + + # Extract all profile information + if basics.profiles: + # Extract profiles for each platform + github_profile = fetch_profile(basics.profiles, ["github"], "github") + linkedin_profile = fetch_profile(basics.profiles, ["linkedin"], "linkedin") + twitter_profile = fetch_profile( + basics.profiles, ["twitter", "x"], "twitter" + ) + dev_profile = fetch_profile( + basics.profiles, ["dev community", "dev"], "dev" + ) + behance_profile = fetch_profile(basics.profiles, ["behance"], "behance") + + # Add GitHub profile columns + if github_profile: + csv_row["github_url"] = github_profile.url + csv_row["github_username"] = ( + github_profile.username if github_profile.username else "" + ) + else: + csv_row["github_url"] = "" + csv_row["github_username"] = "" + + # Add LinkedIn profile columns + if linkedin_profile: + csv_row["linkedin_url"] = linkedin_profile.url + csv_row["linkedin_username"] = ( + linkedin_profile.username if linkedin_profile.username else "" + ) + else: + csv_row["linkedin_url"] = "" + csv_row["linkedin_username"] = "" + + # Add Twitter/X profile columns + if twitter_profile: + csv_row["twitter_url"] = twitter_profile.url + csv_row["twitter_username"] = ( + twitter_profile.username if twitter_profile.username else "" + ) + else: + csv_row["twitter_url"] = "" + csv_row["twitter_username"] = "" + + # Add DEV Community profile columns + if dev_profile: + csv_row["dev_url"] = dev_profile.url + csv_row["dev_username"] = ( + dev_profile.username if dev_profile.username else "" + ) + else: + csv_row["dev_url"] = "" + csv_row["dev_username"] = "" + + # Add Behance profile columns + if behance_profile: + csv_row["behance_url"] = behance_profile.url + csv_row["behance_username"] = ( + behance_profile.username if behance_profile.username else "" + ) + else: + csv_row["behance_url"] = "" + csv_row["behance_username"] = "" + else: + # Initialize empty profile columns + for prefix in ["github", "linkedin", "twitter", "dev", "behance"]: + csv_row[f"{prefix}_url"] = "" + csv_row[f"{prefix}_username"] = "" + + # Extract work experience summary + if resume_data and hasattr(resume_data, "work") and resume_data.work: + work_experience = resume_data.work + csv_row["total_work_experience"] = len(work_experience) + + # Get most recent position + if work_experience: + latest_work = work_experience[0] # Assuming sorted by date + csv_row["current_position"] = ( + latest_work.position if latest_work.position else "" + ) + csv_row["current_company"] = latest_work.name if latest_work.name else "" + else: + csv_row["current_position"] = "" + csv_row["current_company"] = "" + else: + csv_row["total_work_experience"] = 0 + csv_row["current_position"] = "" + csv_row["current_company"] = "" + + # Extract education summary + if resume_data and hasattr(resume_data, "education") and resume_data.education: + education = resume_data.education + csv_row["total_education"] = len(education) + + # Get highest education level + if education: + highest_edu = education[0] # Assuming sorted by date + csv_row["highest_degree"] = ( + highest_edu.studyType if highest_edu.studyType else "" + ) + csv_row["institution"] = ( + highest_edu.institution if highest_edu.institution else "" + ) + else: + csv_row["highest_degree"] = "" + csv_row["institution"] = "" + else: + csv_row["total_education"] = 0 + csv_row["highest_degree"] = "" + csv_row["institution"] = "" + + # Extract skills summary + if resume_data and hasattr(resume_data, "skills") and resume_data.skills: + skills = resume_data.skills + all_skills = [] + for skill_category in skills: + if skill_category.keywords: + all_skills.extend(skill_category.keywords) + csv_row["total_skills"] = len(all_skills) + csv_row["skills_list"] = ", ".join(all_skills[:10]) # Top 10 skills + else: + csv_row["total_skills"] = 0 + csv_row["skills_list"] = "" + + # Extract projects summary + if resume_data and hasattr(resume_data, "projects") and resume_data.projects: + projects = resume_data.projects + csv_row["total_projects"] = len(projects) + else: + csv_row["total_projects"] = 0 + + # Extract GitHub data + if github_data: + csv_row["github_repos"] = github_data.get("public_repos", 0) + csv_row["github_followers"] = github_data.get("followers", 0) + csv_row["github_following"] = github_data.get("following", 0) + csv_row["github_created_at"] = github_data.get("created_at", "") + csv_row["github_bio"] = github_data.get("bio", "") + else: + csv_row["github_repos"] = 0 + csv_row["github_followers"] = 0 + csv_row["github_following"] = 0 + csv_row["github_created_at"] = "" + csv_row["github_bio"] = "" + + # Extract evaluation scores + if evaluation and hasattr(evaluation, "scores"): + scores = evaluation.scores + + csv_row["open_source_score"] = scores.open_source.score + csv_row["open_source_max"] = scores.open_source.max + + csv_row["self_projects_score"] = scores.self_projects.score + csv_row["self_projects_max"] = scores.self_projects.max + + csv_row["production_score"] = scores.production.score + csv_row["production_max"] = scores.production.max + + csv_row["technical_skills_score"] = scores.technical_skills.score + csv_row["technical_skills_max"] = scores.technical_skills.max + + total_score = ( + scores.open_source.score + + scores.self_projects.score + + scores.production.score + + scores.technical_skills.score + ) + total_max = ( + scores.open_source.max + + scores.self_projects.max + + scores.production.max + + scores.technical_skills.max + ) + + csv_row["total_score"] = total_score + csv_row["total_max"] = total_max + else: + csv_row["open_source_score"] = "N/A" + csv_row["open_source_max"] = "N/A" + csv_row["self_projects_score"] = "N/A" + csv_row["self_projects_max"] = "N/A" + csv_row["production_score"] = "N/A" + csv_row["production_max"] = "N/A" + csv_row["technical_skills_score"] = "N/A" + csv_row["technical_skills_max"] = "N/A" + csv_row["total_score"] = "N/A" + csv_row["total_max"] = "N/A" + + # Extract bonus points and deductions + if evaluation and hasattr(evaluation, "bonus_points"): + csv_row["bonus_points"] = evaluation.bonus_points.total + csv_row["bonus_breakdown"] = evaluation.bonus_points.breakdown + else: + csv_row["bonus_points"] = 0 + csv_row["bonus_breakdown"] = "" + + if evaluation and hasattr(evaluation, "deductions"): + csv_row["deductions"] = evaluation.deductions.total + csv_row["deduction_reasons"] = evaluation.deductions.reasons + else: + csv_row["deductions"] = 0 + csv_row["deduction_reasons"] = "" + + # Extract key strengths and areas for improvement + if evaluation and hasattr(evaluation, "key_strengths"): + csv_row["key_strengths"] = "; ".join(evaluation.key_strengths) + else: + csv_row["key_strengths"] = "" + + if evaluation and hasattr(evaluation, "areas_for_improvement"): + csv_row["areas_for_improvement"] = "; ".join(evaluation.areas_for_improvement) + else: + csv_row["areas_for_improvement"] = "" + + return csv_row + + +def convert_json_resume_to_text(resume_data: JSONResume) -> str: + text_parts = [] + + if resume_data.basics: + basics = resume_data.basics + text_parts.append("=== BASIC INFORMATION ===") + text_parts.append(f"Name: {basics.name or 'Not provided'}") + text_parts.append(f"Email: {basics.email or 'Not provided'}") + text_parts.append(f"Phone: {basics.phone or 'Not provided'}") + text_parts.append(f"Website: {basics.url or 'Not provided'}") + + if basics.summary: + text_parts.append(f"Summary: {basics.summary}") + + if basics.location: + loc = basics.location + location_parts = [] + if loc.address: + location_parts.append(loc.address) + if loc.city: + location_parts.append(loc.city) + if loc.region: + location_parts.append(loc.region) + if loc.postalCode: + location_parts.append(loc.postalCode) + if loc.countryCode: + location_parts.append(loc.countryCode) + + if location_parts: + text_parts.append(f"Location: {', '.join(location_parts)}") + + if basics.profiles: + text_parts.append("Profiles:") + for profile in basics.profiles: + text_parts.append( + f" - {profile.network}: {profile.username} ({profile.url})" + ) + + if resume_data.work: + text_parts.append("\n=== WORK EXPERIENCE ===") + for i, work in enumerate(resume_data.work, 1): + text_parts.append(f"{i}. {work.position} at {work.name}") + text_parts.append(f" Period: {work.startDate} - {work.endDate}") + if work.url: + text_parts.append(f" Website: {work.url}") + if work.summary: + text_parts.append(f" Description: {work.summary}") + if work.highlights: + text_parts.append(" Key Achievements:") + for highlight in work.highlights: + text_parts.append(f" • {highlight}") + + if resume_data.education: + text_parts.append("\n=== EDUCATION ===") + for i, edu in enumerate(resume_data.education, 1): + text_parts.append(f"{i}. {edu.studyType} in {edu.area}") + text_parts.append(f" Institution: {edu.institution}") + text_parts.append(f" Period: {edu.startDate} - {edu.endDate}") + if edu.score: + text_parts.append(f" Score: {edu.score}") + if edu.url: + text_parts.append(f" Website: {edu.url}") + if edu.courses: + text_parts.append(f" Courses: {', '.join(edu.courses)}") + + if resume_data.skills: + text_parts.append("\n=== SKILLS ===") + for skill in resume_data.skills: + text_parts.append(f"• {skill.name}") + if skill.level: + text_parts.append(f" Level: {skill.level}") + if skill.keywords: + text_parts.append(f" Keywords: {', '.join(skill.keywords)}") + + if resume_data.projects: + text_parts.append("\n=== PROJECTS ===") + for i, project in enumerate(resume_data.projects, 1): + text_parts.append(f"{i}. {project.name}") + if project.startDate and project.endDate: + text_parts.append(f" Period: {project.startDate} - {project.endDate}") + if project.description: + text_parts.append(f" Description: {project.description}") + if project.url: + text_parts.append(f" URL: {project.url}") + if project.highlights: + text_parts.append(" Highlights:") + for highlight in project.highlights: + text_parts.append(f" • {highlight}") + + if resume_data.awards: + text_parts.append("\n=== AWARDS ===") + for award in resume_data.awards: + text_parts.append(f"• {award.title} - {award.awarder} ({award.date})") + if award.summary: + text_parts.append(f" {award.summary}") + + if resume_data.certificates: + text_parts.append("\n=== CERTIFICATES ===") + for cert in resume_data.certificates: + text_parts.append(f"• {cert.name} - {cert.issuer} ({cert.date})") + if cert.url: + text_parts.append(f" URL: {cert.url}") + + if resume_data.publications: + text_parts.append("\n=== PUBLICATIONS ===") + for pub in resume_data.publications: + text_parts.append(f"• {pub.name} - {pub.publisher} ({pub.releaseDate})") + if pub.url: + text_parts.append(f" URL: {pub.url}") + if pub.summary: + text_parts.append(f" {pub.summary}") + + if resume_data.languages: + text_parts.append("\n=== LANGUAGES ===") + for lang in resume_data.languages: + text_parts.append(f"• {lang.language} - {lang.fluency}") + + if resume_data.interests: + text_parts.append("\n=== INTERESTS ===") + for interest in resume_data.interests: + text_parts.append(f"• {interest.name}") + if interest.keywords: + text_parts.append(f" Keywords: {', '.join(interest.keywords)}") + + if resume_data.references: + text_parts.append("\n=== REFERENCES ===") + for ref in resume_data.references: + text_parts.append(f"• {ref.name}") + if ref.reference: + text_parts.append(f" {ref.reference}") + + if resume_data.volunteer: + text_parts.append("\n=== VOLUNTEER EXPERIENCE ===") + for volunteer in resume_data.volunteer: + text_parts.append(f"• {volunteer.position} at {volunteer.organization}") + text_parts.append(f" Period: {volunteer.startDate} - {volunteer.endDate}") + if volunteer.url: + text_parts.append(f" Website: {volunteer.url}") + if volunteer.summary: + text_parts.append(f" Description: {volunteer.summary}") + if volunteer.highlights: + text_parts.append(" Highlights:") + for highlight in volunteer.highlights: + text_parts.append(f" • {highlight}") + + return "\n".join(text_parts) + + +def convert_github_data_to_text(github_data: dict) -> str: + github_text = "\n\n=== GITHUB DATA ===\n" + + if "profile" in github_data: + profile = github_data["profile"] + github_text += f"GitHub Profile:\n" + github_text += f"- Username: {profile.get('username', 'N/A')}\n" + github_text += f"- Name: {profile.get('name', 'N/A')}\n" + github_text += f"- Bio: {profile.get('bio', 'N/A')}\n" + github_text += f"- Public Repositories: {profile.get('public_repos', 'N/A')}\n" + github_text += f"- Followers: {profile.get('followers', 'N/A')}\n" + github_text += f"- Following: {profile.get('following', 'N/A')}\n" + github_text += f"- Account Created: {profile.get('created_at', 'N/A')}\n" + github_text += f"- Last Updated: {profile.get('updated_at', 'N/A')}\n" + + if "projects" in github_data: + projects = github_data["projects"] + github_text += f"\nGitHub Projects ({len(projects)} total):\n" + for i, project in enumerate(projects[:10], 1): + github_text += f"{i}. {project.get('name', 'N/A')}\n" + github_text += f" Description: {project.get('description', 'N/A')}\n" + github_text += f" URL: {project.get('github_url', 'N/A')}\n" + if "github_details" in project: + details = project["github_details"] + github_text += f" Stars: {details.get('stars', 'N/A')}\n" + github_text += f" Forks: {details.get('forks', 'N/A')}\n" + github_text += f" Language: {details.get('language', 'N/A')}\n" + github_text += "\n" + + return github_text + + +def convert_blog_data_to_text(blog_data: dict) -> str: + blog_text = "\n\n=== BLOG DATA ===\n" + blog_text += f"Total Blogs Found: {blog_data.get('total_blogs', 'N/A')}\n" + blog_text += f"Blog Score: {blog_data.get('blog_score', 'N/A')}/10.0\n" + blog_text += f"Blog Details: {blog_data.get('blog_details', 'N/A')}\n" + + if "blogs" in blog_data: + blog_text += "\nBlog URLs Found:\n" + for i, blog in enumerate(blog_data["blogs"][:5], 1): + blog_text += f"{i}. {blog.get('url', 'N/A')}\n" + blog_text += f" Score: {blog.get('score', 'N/A')}/10.0\n" + blog_text += f" Details: {blog.get('details', 'N/A')}\n" + blog_text += "\n" + + return blog_text