chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,2 @@
|
||||
# Auto detect text files and perform LF normalization
|
||||
* text=auto
|
||||
@@ -0,0 +1,15 @@
|
||||
# These are supported funding model platforms
|
||||
|
||||
github: ScrapeGraphAI
|
||||
patreon: # Replace with a single Patreon username
|
||||
open_collective: scrapegraphai
|
||||
ko_fi: # Replace with a single Ko-fi username
|
||||
tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
|
||||
community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
|
||||
liberapay: # Replace with a single Liberapay username
|
||||
issuehunt: # Replace with a single IssueHunt username
|
||||
lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
|
||||
polar: # Replace with a single Polar username
|
||||
buy_me_a_coffee: # Replace with a single Buy Me a Coffee username
|
||||
thanks_dev: # Replace with a single thanks.dev username
|
||||
custom:
|
||||
@@ -0,0 +1,38 @@
|
||||
---
|
||||
name: Bug report
|
||||
about: Create a report to help us improve
|
||||
title: ''
|
||||
labels: ''
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Describe the bug**
|
||||
A clear and concise description of what the bug is.
|
||||
|
||||
**To Reproduce**
|
||||
Steps to reproduce the behavior:
|
||||
1. Go to '...'
|
||||
2. Click on '....'
|
||||
3. Scroll down to '....'
|
||||
4. See error
|
||||
|
||||
**Expected behavior**
|
||||
A clear and concise description of what you expected to happen.
|
||||
|
||||
**Screenshots**
|
||||
If applicable, add screenshots to help explain your problem.
|
||||
|
||||
**Desktop (please complete the following information):**
|
||||
- OS: [e.g. iOS]
|
||||
- Browser [e.g. chrome, safari]
|
||||
- Version [e.g. 22]
|
||||
|
||||
**Smartphone (please complete the following information):**
|
||||
- Device: [e.g. iPhone6]
|
||||
- OS: [e.g. iOS8.1]
|
||||
- Browser [e.g. stock browser, safari]
|
||||
- Version [e.g. 22]
|
||||
|
||||
**Additional context**
|
||||
Add any other context about the problem here.
|
||||
@@ -0,0 +1,8 @@
|
||||
---
|
||||
name: Custom issue template
|
||||
about: Describe this issue template's purpose here.
|
||||
title: ''
|
||||
labels: ''
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
@@ -0,0 +1,20 @@
|
||||
---
|
||||
name: Feature request
|
||||
about: Suggest an idea for this project
|
||||
title: ''
|
||||
labels: ''
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Is your feature request related to a problem? Please describe.**
|
||||
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
|
||||
|
||||
**Describe the solution you'd like**
|
||||
A clear and concise description of what you want to happen.
|
||||
|
||||
**Describe alternatives you've considered**
|
||||
A clear and concise description of any alternative solutions or features you've considered.
|
||||
|
||||
**Additional context**
|
||||
Add any other context or screenshots about the feature request here.
|
||||
@@ -0,0 +1,63 @@
|
||||
# For most projects, this workflow file will not need changing; you simply need
|
||||
# to commit it to your repository.
|
||||
#
|
||||
# You may wish to alter this file to override the set of languages analyzed,
|
||||
# or to provide custom queries or build logic.
|
||||
#
|
||||
# ******** NOTE ********
|
||||
# We have attempted to detect the languages in your repository. Please check
|
||||
# the `language` matrix defined below to confirm you have the correct set of
|
||||
# supported CodeQL languages.
|
||||
#
|
||||
name: "CodeQL"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "main" ]
|
||||
pull_request:
|
||||
branches: [ "main" ]
|
||||
schedule:
|
||||
- cron: '42 19 * * 5'
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: Analyze
|
||||
# Runner size impacts CodeQL analysis time. To learn more, please see:
|
||||
# - https://gh.io/recommended-hardware-resources-for-running-codeql
|
||||
# - https://gh.io/supported-runners-and-hardware-resources
|
||||
# - https://gh.io/using-larger-runners
|
||||
# Consider using larger runners for possible analysis time improvements.
|
||||
runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
|
||||
timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
|
||||
permissions:
|
||||
# required for all workflows
|
||||
security-events: write
|
||||
|
||||
# only required for workflows in private repositories
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
language: [ 'python' ]
|
||||
# CodeQL supports [ 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' ]
|
||||
# Use only 'java-kotlin' to analyze code written in Java, Kotlin or both
|
||||
# Use only 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
|
||||
# Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v3
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
# If you wish to specify custom queries, you can do so here or in a config file.
|
||||
# By default, queries listed here will override any specified in a config file.
|
||||
# Prefix the list here with "+" to use these queries and those in the config file.
|
||||
|
||||
# For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
|
||||
# queries: security-extended,security-and-quality
|
||||
@@ -0,0 +1,39 @@
|
||||
# Dependency Review Action
|
||||
#
|
||||
# This Action will scan dependency manifest files that change as part of a Pull Request,
|
||||
# surfacing known-vulnerable versions of the packages declared or updated in the PR.
|
||||
# Once installed, if the workflow run is marked as required, PRs introducing known-vulnerable
|
||||
# packages will be blocked from merging.
|
||||
#
|
||||
# Source repository: https://github.com/actions/dependency-review-action
|
||||
# Public documentation: https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/about-dependency-review#dependency-review-enforcement
|
||||
name: 'Dependency review'
|
||||
on:
|
||||
pull_request:
|
||||
branches: [ "main" ]
|
||||
|
||||
# If using a dependency submission action in this workflow this permission will need to be set to:
|
||||
#
|
||||
# permissions:
|
||||
# contents: write
|
||||
#
|
||||
# https://docs.github.com/en/enterprise-cloud@latest/code-security/supply-chain-security/understanding-your-software-supply-chain/using-the-dependency-submission-api
|
||||
permissions:
|
||||
contents: read
|
||||
# Write permissions for pull-requests are required for using the `comment-summary-in-pr` option, comment out if you aren't using this option
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
dependency-review:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: 'Checkout repository'
|
||||
uses: actions/checkout@v4
|
||||
- name: 'Dependency Review'
|
||||
uses: actions/dependency-review-action@v4
|
||||
# Commonly enabled options, see https://github.com/actions/dependency-review-action#configuration-options for all available options.
|
||||
with:
|
||||
comment-summary-in-pr: always
|
||||
# fail-on-severity: moderate
|
||||
# deny-licenses: GPL-1.0-or-later, LGPL-2.0-or-later
|
||||
# retry-on-snapshot-warnings: true
|
||||
@@ -0,0 +1,101 @@
|
||||
name: Release
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- pre/*
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Install git
|
||||
run: |
|
||||
sudo apt update
|
||||
sudo apt install -y git
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.10'
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v3
|
||||
|
||||
- name: Install Node Env
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 20
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4.1.1
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Build and validate package
|
||||
run: |
|
||||
uv venv
|
||||
. .venv/bin/activate
|
||||
uv pip install --upgrade setuptools wheel hatchling
|
||||
uv sync --frozen
|
||||
uv pip install -e .
|
||||
uv build
|
||||
uv pip install --upgrade pkginfo==1.12.0 twine==6.0.1 # Upgrade pkginfo and install twine
|
||||
python -m twine check dist/*
|
||||
|
||||
- name: Debug Dist Directory
|
||||
run: ls -al dist
|
||||
|
||||
- name: Cache build
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ./dist
|
||||
key: ${{ runner.os }}-build-${{ github.sha }}
|
||||
|
||||
release:
|
||||
name: Release
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
environment: development
|
||||
if: >
|
||||
github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/pre/beta') ||
|
||||
(github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged &&
|
||||
(github.event.pull_request.base.ref == 'main' || github.event.pull_request.base.ref == 'pre/beta'))
|
||||
permissions:
|
||||
contents: write
|
||||
issues: write
|
||||
pull-requests: write
|
||||
id-token: write
|
||||
steps:
|
||||
- name: Checkout repo
|
||||
uses: actions/checkout@v4.1.1
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Restore build artifacts
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ./dist
|
||||
key: ${{ runner.os }}-build-${{ github.sha }}
|
||||
|
||||
- name: Clean stale dist artifacts
|
||||
run: rm -rf ./dist/* || true
|
||||
|
||||
- name: Semantic Release
|
||||
uses: cycjimmy/semantic-release-action@v4.1.0
|
||||
with:
|
||||
semantic_version: 23
|
||||
extra_plugins: |
|
||||
semantic-release-pypi@3
|
||||
@semantic-release/git
|
||||
@semantic-release/commit-analyzer@12
|
||||
@semantic-release/release-notes-generator@13
|
||||
@semantic-release/github@10
|
||||
@semantic-release/changelog@6
|
||||
conventional-changelog-conventionalcommits@7
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
|
||||
@@ -0,0 +1,49 @@
|
||||
name: Test Suite
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, pre/beta, dev]
|
||||
pull_request:
|
||||
branches: [main, pre/beta]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
unit-tests:
|
||||
name: Unit Tests
|
||||
runs-on: ubuntu-latest
|
||||
# Hard cap so a hanging test can never burn the full 6h runner budget again.
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v4
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync
|
||||
|
||||
# Run only the deterministic, mock-based unit suites. The provider graph
|
||||
# tests (tests/graphs/*), integration tests (tests/integration/*) and the
|
||||
# browser/network suites (e.g. tests/test_chromium.py) require real API
|
||||
# keys or live network/browsers, so in CI they either fail or hang
|
||||
# indefinitely. They should be re-added here once mocked or gated behind
|
||||
# markers + secrets.
|
||||
- name: Run unit tests
|
||||
run: >
|
||||
uv run pytest
|
||||
tests/test_batch_api.py
|
||||
tests/test_csv_scraper_multi_graph.py
|
||||
tests/test_depth_search_graph.py
|
||||
tests/test_json_scraper_graph.py
|
||||
tests/test_minimax_models.py
|
||||
tests/test_scrape_do.py
|
||||
tests/test_search_graph.py
|
||||
tests/utils/convert_to_md_test.py
|
||||
tests/utils/parse_state_keys_test.py
|
||||
+197
@@ -0,0 +1,197 @@
|
||||
.DS_Store
|
||||
.DS_Store?
|
||||
._*
|
||||
|
||||
# Byte-compiled / optimized / DLL files
|
||||
**/__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
dist/
|
||||
*.egg-info/
|
||||
*.egg
|
||||
MANIFEST
|
||||
*.python-version
|
||||
|
||||
docs/build/
|
||||
docs/source/_templates/
|
||||
docs/source/_static/
|
||||
.env
|
||||
venv/
|
||||
.venv/
|
||||
.vscode/
|
||||
.conda/
|
||||
|
||||
# exclude pdf, mp3
|
||||
*.pdf
|
||||
*.mp3
|
||||
*.sqlite
|
||||
*.google-cookie
|
||||
*.python-version
|
||||
examples/graph_examples/ScrapeGraphAI_generated_graph
|
||||
examples/**/result.csv
|
||||
examples/**/result.json
|
||||
main.py
|
||||
lib/
|
||||
*.html
|
||||
.idea
|
||||
|
||||
# extras
|
||||
cache/
|
||||
run_smart_scraper.py
|
||||
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
.ruff_cache/
|
||||
cover/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
.pybuilder/
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
.python-version
|
||||
|
||||
# pipenv
|
||||
Pipfile.lock
|
||||
|
||||
# poetry
|
||||
poetry.lock
|
||||
|
||||
# pdm
|
||||
pdm.lock
|
||||
.pdm.toml
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# pytype static type analyzer
|
||||
.pytype/
|
||||
|
||||
# Cython debug symbols
|
||||
cython_debug/
|
||||
|
||||
# PyCharm
|
||||
.idea/
|
||||
|
||||
# VS Code
|
||||
.vscode/
|
||||
|
||||
# macOS
|
||||
.DS_Store
|
||||
|
||||
dev.ipynb
|
||||
|
||||
# CodeBeaver reports and artifacts
|
||||
.codebeaver
|
||||
@@ -0,0 +1,23 @@
|
||||
repos:
|
||||
- repo: https://github.com/psf/black
|
||||
rev: 24.8.0
|
||||
hooks:
|
||||
- id: black
|
||||
|
||||
- repo: https://github.com/charliermarsh/ruff-pre-commit
|
||||
rev: v0.6.9
|
||||
hooks:
|
||||
- id: ruff
|
||||
|
||||
- repo: https://github.com/pycqa/isort
|
||||
rev: 5.13.2
|
||||
hooks:
|
||||
- id: isort
|
||||
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v4.6.0
|
||||
hooks:
|
||||
- id: trailing-whitespace
|
||||
- id: end-of-file-fixer
|
||||
- id: check-yaml
|
||||
exclude: mkdocs.yml
|
||||
@@ -0,0 +1,55 @@
|
||||
plugins:
|
||||
- - "@semantic-release/commit-analyzer"
|
||||
- preset: conventionalcommits
|
||||
- - "@semantic-release/release-notes-generator"
|
||||
- writerOpts:
|
||||
commitsSort:
|
||||
- subject
|
||||
- scope
|
||||
preset: conventionalcommits
|
||||
presetConfig:
|
||||
types:
|
||||
- type: feat
|
||||
section: Features
|
||||
- type: fix
|
||||
section: Bug Fixes
|
||||
- type: chore
|
||||
section: chore
|
||||
- type: docs
|
||||
section: Docs
|
||||
- type: style
|
||||
hidden: true
|
||||
- type: refactor
|
||||
section: Refactor
|
||||
- type: perf
|
||||
section: Perf
|
||||
- type: test
|
||||
section: Test
|
||||
- type: build
|
||||
section: Build
|
||||
- type: ci
|
||||
section: CI
|
||||
- "@semantic-release/changelog"
|
||||
- "semantic-release-pypi"
|
||||
- "@semantic-release/github"
|
||||
- - "@semantic-release/git"
|
||||
- assets:
|
||||
- CHANGELOG.md
|
||||
- pyproject.toml
|
||||
message: |-
|
||||
ci(release): ${nextRelease.version} [skip ci]
|
||||
|
||||
${nextRelease.notes}
|
||||
branches:
|
||||
#child branches coming from tagged version for bugfix (1.1.x) or new features (1.x)
|
||||
#maintenance branch
|
||||
- name: "+([0-9])?(.{+([0-9]),x}).x"
|
||||
channel: "stable"
|
||||
#release a production version when merging towards main
|
||||
- name: "main"
|
||||
channel: "stable"
|
||||
#prerelease branch
|
||||
- name: "pre/beta"
|
||||
channel: "dev"
|
||||
prerelease: "beta"
|
||||
debug: true
|
||||
@@ -0,0 +1,11 @@
|
||||
This file marks that commits have been rewritten to follow Conventional Commits format.
|
||||
|
||||
Original commits:
|
||||
- 9439fe5: Fix langchain import issues blocking tests
|
||||
- 323f26a: Add comprehensive timeout feature documentation
|
||||
|
||||
Rewritten as:
|
||||
- 8c9cb8b: fix(imports): update deprecated langchain imports to langchain_core
|
||||
- 4c764bc: docs(timeout): add comprehensive timeout configuration guide
|
||||
|
||||
These follow the semantic-release convention configured in .releaserc.yml
|
||||
+4906
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,128 @@
|
||||
# Contributor Covenant Code of Conduct
|
||||
|
||||
## Our Pledge
|
||||
|
||||
We as members, contributors, and leaders pledge to make participation in our
|
||||
community a harassment-free experience for everyone, regardless of age, body
|
||||
size, visible or invisible disability, ethnicity, sex characteristics, gender
|
||||
identity and expression, level of experience, education, socio-economic status,
|
||||
nationality, personal appearance, race, religion, or sexual identity
|
||||
and orientation.
|
||||
|
||||
We pledge to act and interact in ways that contribute to an open, welcoming,
|
||||
diverse, inclusive, and healthy community.
|
||||
|
||||
## Our Standards
|
||||
|
||||
Examples of behavior that contributes to a positive environment for our
|
||||
community include:
|
||||
|
||||
* Demonstrating empathy and kindness toward other people
|
||||
* Being respectful of differing opinions, viewpoints, and experiences
|
||||
* Giving and gracefully accepting constructive feedback
|
||||
* Accepting responsibility and apologizing to those affected by our mistakes,
|
||||
and learning from the experience
|
||||
* Focusing on what is best not just for us as individuals, but for the
|
||||
overall community
|
||||
|
||||
Examples of unacceptable behavior include:
|
||||
|
||||
* The use of sexualized language or imagery, and sexual attention or
|
||||
advances of any kind
|
||||
* Trolling, insulting or derogatory comments, and personal or political attacks
|
||||
* Public or private harassment
|
||||
* Publishing others' private information, such as a physical or email
|
||||
address, without their explicit permission
|
||||
* Other conduct which could reasonably be considered inappropriate in a
|
||||
professional setting
|
||||
|
||||
## Enforcement Responsibilities
|
||||
|
||||
Community leaders are responsible for clarifying and enforcing our standards of
|
||||
acceptable behavior and will take appropriate and fair corrective action in
|
||||
response to any behavior that they deem inappropriate, threatening, offensive,
|
||||
or harmful.
|
||||
|
||||
Community leaders have the right and responsibility to remove, edit, or reject
|
||||
comments, commits, code, wiki edits, issues, and other contributions that are
|
||||
not aligned to this Code of Conduct, and will communicate reasons for moderation
|
||||
decisions when appropriate.
|
||||
|
||||
## Scope
|
||||
|
||||
This Code of Conduct applies within all community spaces, and also applies when
|
||||
an individual is officially representing the community in public spaces.
|
||||
Examples of representing our community include using an official e-mail address,
|
||||
posting via an official social media account, or acting as an appointed
|
||||
representative at an online or offline event.
|
||||
|
||||
## Enforcement
|
||||
|
||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be
|
||||
reported to the community leaders responsible for enforcement at
|
||||
mvincig11@gmail.com.
|
||||
All complaints will be reviewed and investigated promptly and fairly.
|
||||
|
||||
All community leaders are obligated to respect the privacy and security of the
|
||||
reporter of any incident.
|
||||
|
||||
## Enforcement Guidelines
|
||||
|
||||
Community leaders will follow these Community Impact Guidelines in determining
|
||||
the consequences for any action they deem in violation of this Code of Conduct:
|
||||
|
||||
### 1. Correction
|
||||
|
||||
**Community Impact**: Use of inappropriate language or other behavior deemed
|
||||
unprofessional or unwelcome in the community.
|
||||
|
||||
**Consequence**: A private, written warning from community leaders, providing
|
||||
clarity around the nature of the violation and an explanation of why the
|
||||
behavior was inappropriate. A public apology may be requested.
|
||||
|
||||
### 2. Warning
|
||||
|
||||
**Community Impact**: A violation through a single incident or series
|
||||
of actions.
|
||||
|
||||
**Consequence**: A warning with consequences for continued behavior. No
|
||||
interaction with the people involved, including unsolicited interaction with
|
||||
those enforcing the Code of Conduct, for a specified period of time. This
|
||||
includes avoiding interactions in community spaces as well as external channels
|
||||
like social media. Violating these terms may lead to a temporary or
|
||||
permanent ban.
|
||||
|
||||
### 3. Temporary Ban
|
||||
|
||||
**Community Impact**: A serious violation of community standards, including
|
||||
sustained inappropriate behavior.
|
||||
|
||||
**Consequence**: A temporary ban from any sort of interaction or public
|
||||
communication with the community for a specified period of time. No public or
|
||||
private interaction with the people involved, including unsolicited interaction
|
||||
with those enforcing the Code of Conduct, is allowed during this period.
|
||||
Violating these terms may lead to a permanent ban.
|
||||
|
||||
### 4. Permanent Ban
|
||||
|
||||
**Community Impact**: Demonstrating a pattern of violation of community
|
||||
standards, including sustained inappropriate behavior, harassment of an
|
||||
individual, or aggression toward or disparagement of classes of individuals.
|
||||
|
||||
**Consequence**: A permanent ban from any sort of public interaction within
|
||||
the community.
|
||||
|
||||
## Attribution
|
||||
|
||||
This Code of Conduct is adapted from the [Contributor Covenant][homepage],
|
||||
version 2.0, available at
|
||||
https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
|
||||
|
||||
Community Impact Guidelines were inspired by [Mozilla's code of conduct
|
||||
enforcement ladder](https://github.com/mozilla/diversity).
|
||||
|
||||
[homepage]: https://www.contributor-covenant.org
|
||||
|
||||
For answers to common questions about this code of conduct, see the FAQ at
|
||||
https://www.contributor-covenant.org/faq. Translations are available at
|
||||
https://www.contributor-covenant.org/translations.
|
||||
@@ -0,0 +1,44 @@
|
||||
# Contributing to ScrapeGraphAI 🚀
|
||||
|
||||
Hey there! Thanks for checking out **ScrapeGraphAI**! We're excited to have you here! 🎉
|
||||
|
||||
## Quick Start Guide 🏃♂️
|
||||
|
||||
1. Fork the repository from the **pre/beta branch** 🍴
|
||||
2. Clone your fork locally 💻
|
||||
3. Install uv (if you haven't):
|
||||
```bash
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
```
|
||||
4. Run `uv sync` (creates virtual env & installs dependencies) ⚡
|
||||
5. Run `uv run pre-commit install` 🔧
|
||||
6. Make your awesome changes ✨
|
||||
7. Test thoroughly 🧪
|
||||
8. Push & open a PR to the pre/beta branch 🎯
|
||||
|
||||
## Contribution Guidelines 📝
|
||||
|
||||
Keep it clean and simple:
|
||||
- Follow our code style (PEP 8 & Google Python Style) 🎨
|
||||
- Document your changes clearly 📚
|
||||
- Use these commit prefixes for your final PR commit:
|
||||
```
|
||||
feat: ✨ New feature
|
||||
fix: 🐛 Bug fix
|
||||
docs: 📚 Documentation
|
||||
style: 💅 Code style
|
||||
refactor: ♻️ Code changes
|
||||
test: 🧪 Testing
|
||||
perf: ⚡ Performance
|
||||
```
|
||||
- Be nice to others! 💝
|
||||
|
||||
## Need Help? 🤔
|
||||
|
||||
Found a bug or have a cool idea? Open an issue and let's chat! 💬
|
||||
|
||||
## License 📜
|
||||
|
||||
MIT Licensed. See [LICENSE](LICENSE) file for details.
|
||||
|
||||
Let's build something amazing together! 🌟
|
||||
@@ -0,0 +1,9 @@
|
||||
FROM python:3.11-slim
|
||||
|
||||
RUN apt-get update && apt-get upgrade -y && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN pip install --no-cache-dir scrapegraphai
|
||||
RUN pip install --no-cache-dir scrapegraphai[burr]
|
||||
|
||||
RUN python3 -m playwright install-deps
|
||||
RUN python3 -m playwright install
|
||||
@@ -0,0 +1,7 @@
|
||||
Copyright 2024 Scrapgraph-ai team
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
@@ -0,0 +1,49 @@
|
||||
# Makefile for Project Automation
|
||||
|
||||
.PHONY: install lint type-check test build all clean
|
||||
|
||||
# Variables
|
||||
PACKAGE_NAME = scrapegraphai
|
||||
TEST_DIR = tests
|
||||
|
||||
# Default target
|
||||
all: lint type-check test
|
||||
|
||||
# Install project dependencies
|
||||
install:
|
||||
uv sync
|
||||
uv run pre-commit install
|
||||
|
||||
# Linting and Formatting Checks
|
||||
lint:
|
||||
uv run ruff check $(PACKAGE_NAME) $(TEST_DIR)
|
||||
uv run black --check $(PACKAGE_NAME) $(TEST_DIR)
|
||||
uv run isort --check-only $(PACKAGE_NAME) $(TEST_DIR)
|
||||
|
||||
# Type Checking with MyPy
|
||||
type-check:
|
||||
uv run mypy $(PACKAGE_NAME) $(TEST_DIR)
|
||||
|
||||
# Run Tests with Coverage
|
||||
test:
|
||||
uv run pytest --cov=$(PACKAGE_NAME) --cov-report=xml $(TEST_DIR)/
|
||||
|
||||
# Run Pre-Commit Hooks
|
||||
pre-commit:
|
||||
uv run pre-commit run --all-files
|
||||
|
||||
# Clean Up Generated Files
|
||||
clean:
|
||||
rm -rf dist/
|
||||
rm -rf build/
|
||||
rm -rf *.egg-info
|
||||
rm -rf htmlcov/
|
||||
rm -rf .mypy_cache/
|
||||
rm -rf .pytest_cache/
|
||||
rm -rf .ruff_cache/
|
||||
rm -rf .uv/
|
||||
rm -rf .venv/
|
||||
|
||||
# Build the Package
|
||||
build:
|
||||
uv build --no-sources
|
||||
@@ -0,0 +1,7 @@
|
||||
This PR adds a null check for document.body before referencing document.body.scrollHeight. The motivation is that in some cases (such as non-standard DOM structures or scripts running before the DOM is fully loaded), document.body can be null, which would previously have caused runtime errors.
|
||||
|
||||
The fix is appropriate and covers a genuine bug that may be encountered in edge cases. The solution is concise and maintains safety without introducing unnecessary complexity. Labeling the PR as bug and size:XS is accurate. No other unintended changes observed.
|
||||
|
||||
If not already done, consider adding a simple test or log to ensure scrollHeight is accessed only if document.body exists, even for future contributors. Otherwise, this looks good!
|
||||
|
||||
✅ LGTM! Thanks for improving the robustness of the codebase.
|
||||
@@ -0,0 +1,254 @@
|
||||
## 🚀 **Looking for an even faster and simpler way to scrape at scale (only 5 lines of code)?** Check out our enhanced version at [**ScrapeGraphAI.com**](https://scrapegraphai.com/?utm_source=github&utm_medium=readme&utm_campaign=oss_cta&ut#m_content=top_banner)! 🚀
|
||||
|
||||
---
|
||||
|
||||
# 🕷️ ScrapeGraphAI: You Only Scrape Once
|
||||
|
||||
<p align="center">
|
||||
<a href="https://scrapegraphai.com">
|
||||
<img src="media/banner.png" alt="ScrapeGraphAI" style="width: 100%;">
|
||||
</a>
|
||||
</p>
|
||||
|
||||
[English](README.md) | [中文](docs/chinese.md) | [日本語](docs/japanese.md)
|
||||
| [한국어](docs/korean.md)
|
||||
| [Русский](docs/russian.md) | [Türkçe](docs/turkish.md)
|
||||
| [Deutsch](docs/german.md)
|
||||
| [Español](docs/spanish.md)
|
||||
| [français](docs/french.md)
|
||||
| [Português](docs/portuguese.md)
|
||||
| [Italiano](docs/italian.md)
|
||||
|
||||
[](https://pepy.tech/projects/scrapegraphai)
|
||||
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
[](https://discord.gg/gkxQDAjfeX)
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/15078" target="_blank"><img src="https://trendshift.io/api/badge/repositories/15078" alt="ScrapeGraphAI%2FScrapegraph-ai | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
<p align="center">
|
||||
|
||||
[ScrapeGraphAI](https://scrapegraphai.com) is a *web scraping* python library that uses LLM and direct graph logic to create scraping pipelines for websites and local documents (XML, HTML, JSON, Markdown, etc.).
|
||||
|
||||
Just say which information you want to extract and the library will do it for you!
|
||||
|
||||
## 🚀 Integrations
|
||||
ScrapeGraphAI offers seamless integration with popular frameworks and tools to enhance your scraping capabilities. Whether you're building with Python or Node.js, using LLM frameworks, or working with no-code platforms, we've got you covered with our comprehensive integration options..
|
||||
|
||||
<p align="center">
|
||||
<a href="https://scrapegraphai.com">
|
||||
<img src="https://raw.githubusercontent.com/ScrapeGraphAI/.github/main/profile/assets/api_banner.png" alt="Web data extraction at scale? Try ScrapeGraphAI cloud" style="width: 100%;">
|
||||
</a>
|
||||
</p>
|
||||
|
||||
You can find more informations at the following [link](https://scrapegraphai.com)
|
||||
|
||||
**Integrations**:
|
||||
- **API**: [Documentation](https://docs.scrapegraphai.com/introduction)
|
||||
- **SDKs**: [Python](https://docs.scrapegraphai.com/sdks/python), [Node](https://docs.scrapegraphai.com/sdks/javascript)
|
||||
- **LLM Frameworks**: [Langchain](https://docs.scrapegraphai.com/integrations/langchain), [Llama Index](https://docs.scrapegraphai.com/integrations/llamaindex), [Crew.ai](https://docs.scrapegraphai.com/integrations/crewai), [Agno](https://docs.scrapegraphai.com/integrations/agno), [CamelAI](https://github.com/camel-ai/camel)
|
||||
- **Low-code Frameworks**: [Pipedream](https://pipedream.com/apps/scrapegraphai), [Bubble](https://bubble.io/plugin/scrapegraphai-1745408893195x213542371433906180), [Zapier](https://zapier.com/apps/scrapegraphai/integrations), [n8n](http://localhost:5001/dashboard), [Dify](https://dify.ai), [Toolhouse](https://app.toolhouse.ai/mcp-servers/scrapegraph_smartscraper)
|
||||
- **MCP server**: [Link](https://smithery.ai/server/@ScrapeGraphAI/scrapegraph-mcp)
|
||||
|
||||
|
||||
## 🚀 Quick install
|
||||
|
||||
The reference page for Scrapegraph-ai is available on the official page of PyPI: [pypi](https://pypi.org/project/scrapegraphai/).
|
||||
|
||||
```bash
|
||||
pip install scrapegraphai
|
||||
|
||||
# IMPORTANT (for fetching websites content)
|
||||
playwright install
|
||||
```
|
||||
|
||||
**Note**: it is recommended to install the library in a virtual environment to avoid conflicts with other libraries 🐱
|
||||
|
||||
|
||||
## 💻 Usage
|
||||
There are multiple standard scraping pipelines that can be used to extract information from a website (or local file).
|
||||
|
||||
The most common one is the `SmartScraperGraph`, which extracts information from a single page given a user prompt and a source URL.
|
||||
|
||||
|
||||
```python
|
||||
from scrapegraphai.graphs import SmartScraperGraph
|
||||
|
||||
# Define the configuration for the scraping pipeline
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"model": "ollama/llama3.2",
|
||||
"model_tokens": 8192,
|
||||
"format": "json",
|
||||
},
|
||||
"verbose": True,
|
||||
"headless": False,
|
||||
}
|
||||
|
||||
# Create the SmartScraperGraph instance
|
||||
smart_scraper_graph = SmartScraperGraph(
|
||||
prompt="Extract useful information from the webpage, including a description of what the company does, founders and social media links",
|
||||
source="https://scrapegraphai.com/",
|
||||
config=graph_config
|
||||
)
|
||||
|
||||
# Run the pipeline
|
||||
result = smart_scraper_graph.run()
|
||||
|
||||
import json
|
||||
print(json.dumps(result, indent=4))
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
> For OpenAI and other models you just need to change the llm config!
|
||||
> ```python
|
||||
>graph_config = {
|
||||
> "llm": {
|
||||
> "api_key": "YOUR_OPENAI_API_KEY",
|
||||
> "model": "openai/gpt-4o-mini",
|
||||
> },
|
||||
> "verbose": True,
|
||||
> "headless": False,
|
||||
>}
|
||||
>```
|
||||
|
||||
|
||||
The output will be a dictionary like the following:
|
||||
|
||||
```python
|
||||
{
|
||||
"description": "ScrapeGraphAI transforms websites into clean, organized data for AI agents and data analytics. It offers an AI-powered API for effortless and cost-effective data extraction.",
|
||||
"founders": [
|
||||
{
|
||||
"name": "",
|
||||
"role": "Founder & Technical Lead",
|
||||
"linkedin": "https://www.linkedin.com/in/perinim/"
|
||||
},
|
||||
{
|
||||
"name": "Marco Vinciguerra",
|
||||
"role": "Founder & Software Engineer",
|
||||
"linkedin": "https://www.linkedin.com/in/marco-vinciguerra-7ba365242/"
|
||||
},
|
||||
{
|
||||
"name": "Lorenzo Padoan",
|
||||
"role": "Founder & Product Engineer",
|
||||
"linkedin": "https://www.linkedin.com/in/lorenzo-padoan-4521a2154/"
|
||||
}
|
||||
],
|
||||
"social_media_links": {
|
||||
"linkedin": "https://www.linkedin.com/company/101881123",
|
||||
"twitter": "https://x.com/scrapegraphai",
|
||||
"github": "https://github.com/ScrapeGraphAI/Scrapegraph-ai"
|
||||
}
|
||||
}
|
||||
```
|
||||
There are other pipelines that can be used to extract information from multiple pages, generate Python scripts, or even generate audio files.
|
||||
|
||||
| Pipeline Name | Description |
|
||||
|-------------------------|------------------------------------------------------------------------------------------------------------------|
|
||||
| SmartScraperGraph | Single-page scraper that only needs a user prompt and an input source. |
|
||||
| SearchGraph | Multi-page scraper that extracts information from the top n search results of a search engine. |
|
||||
| SpeechGraph | Single-page scraper that extracts information from a website and generates an audio file. |
|
||||
| ScriptCreatorGraph | Single-page scraper that extracts information from a website and generates a Python script. |
|
||||
| SmartScraperMultiGraph | Multi-page scraper that extracts information from multiple pages given a single prompt and a list of sources. |
|
||||
| ScriptCreatorMultiGraph | Multi-page scraper that generates a Python script for extracting information from multiple pages and sources. |
|
||||
|
||||
For each of these graphs there is the multi version. It allows to make calls of the LLM in parallel.
|
||||
|
||||
It is possible to use different LLM through APIs, such as **OpenAI**, **Groq**, **Azure**, **Gemini**, **MiniMax** and more, or local models using **Ollama**.
|
||||
|
||||
Remember to have [Ollama](https://ollama.com/) installed and download the models using the **ollama pull** command, if you want to use local models.
|
||||
|
||||
|
||||
## 📖 Documentation
|
||||
|
||||
[](https://colab.research.google.com/drive/1sEZBonBMGP44CtO6GQTwAlL0BGJXjtfd?usp=sharing)
|
||||
|
||||
The documentation for ScrapeGraphAI can be found [here](https://docs.scrapegraphai.com/introduction).
|
||||
## 🆚 Open Source vs Managed API
|
||||
|
||||
ScrapeGraphAI comes in two flavours: **this open-source library**, which you run yourself, and the **managed cloud API** (used via the [Python](https://github.com/ScrapeGraphAI/scrapegraph-py) and [JS/TS](https://github.com/ScrapeGraphAI/scrapegraph-js) SDKs). This table explains the difference so you can pick the right one.
|
||||
|
||||
| | Open Source (`scrapegraphai`) | Managed API (`scrapegraph-py` / `scrapegraph-js`) |
|
||||
|---|---|---|
|
||||
| **What it is** | A Python library you run yourself | A hosted cloud service you call via SDK |
|
||||
| **Where it runs** | Your own infrastructure (self-hosted) | ScrapeGraphAI cloud |
|
||||
| **LLM** | Bring your own (OpenAI, Groq, Gemini, Azure, local via Ollama) | Managed for you |
|
||||
| **Browser / JS rendering** | You configure it (Playwright) | Managed (stealth, `auto`/`fast`/`js` modes) |
|
||||
| **Proxies & anti-bot** | Your responsibility | Included |
|
||||
| **Scaling & maintenance** | Your responsibility | Fully managed |
|
||||
| **Cost model** | LLM tokens + your own infra | Pay-as-you-go credits |
|
||||
| **Auth** | Your own LLM keys | `SGAI_API_KEY` |
|
||||
| **Capabilities** | Graph pipelines (SmartScraper, Search, Speech, ScriptCreator…) | Scrape, Extract, Search, Crawl, Monitor, History |
|
||||
| **Setup effort** | More configuration | Minimal — API key + one call |
|
||||
| **License** | MIT | SDK is MIT; the API service is paid |
|
||||
|
||||
**Choose the open-source library** if you want full control, on-prem/self-hosted data, local LLMs (Ollama), or fine-grained cost tuning — and you're happy to manage browsers, proxies and scaling yourself.
|
||||
|
||||
**Choose the managed API** if you want zero infrastructure, managed JS rendering & anti-bot, built-in **Crawl** and scheduled **Monitor** jobs, and the fastest path to production — billed per credit.
|
||||
|
||||
- Open-source library: https://github.com/ScrapeGraphAI/Scrapegraph-ai
|
||||
- Python SDK: https://github.com/ScrapeGraphAI/scrapegraph-py
|
||||
- JS/TS SDK: https://github.com/ScrapeGraphAI/scrapegraph-js
|
||||
- API docs: https://docs.scrapegraphai.com/introduction
|
||||
|
||||
## 🤝 Contributing
|
||||
|
||||
Feel free to contribute and join our Discord server to discuss with us improvements and give us suggestions!
|
||||
|
||||
Please see the [contributing guidelines](https://github.com/ScrapeGraphAI/Scrapegraph-ai/blob/main/CONTRIBUTING.md).
|
||||
|
||||
[](https://discord.gg/uJN7TYcpNa)
|
||||
[](https://www.linkedin.com/company/scrapegraphai/)
|
||||
[](https://twitter.com/scrapegraphai)
|
||||
|
||||
## 🔗 ScrapeGraph API & SDKs
|
||||
If you are looking for a quick solution to integrate ScrapeGraph in your system, check out our powerful API [here!](https://scrapegraphai.com)
|
||||
|
||||
[](https://scrapegraphai.com)
|
||||
|
||||
We offer SDKs in both Python and Node.js, making it easy to integrate into your projects. Check them out below:
|
||||
|
||||
| SDK | Language | GitHub Link |
|
||||
|-----------|----------|-----------------------------------------------------------------------------|
|
||||
| Python SDK | Python | [scrapegraph-py](https://docs.scrapegraphai.com/sdks/python) |
|
||||
| Node.js SDK | Node.js | [scrapegraph-js](https://docs.scrapegraphai.com/sdks/javascript) |
|
||||
|
||||
The Official API Documentation can be found [here](https://docs.scrapegraphai.com/introduction).
|
||||
|
||||
## 📈 Telemetry
|
||||
We collect anonymous usage metrics to enhance our package's quality and user experience. The data helps us prioritize improvements and ensure compatibility. If you wish to opt-out, set the environment variable SCRAPEGRAPHAI_TELEMETRY_ENABLED=false. For more information, please refer to the documentation [here](https://docs.scrapegraphai.com/introduction).
|
||||
|
||||
## ❤️ Contributors
|
||||
[](https://github.com/ScrapeGraphAI/Scrapegraph-ai/graphs/contributors)
|
||||
|
||||
## 🎓 Citations
|
||||
If you have used our library for research purposes please quote us with the following reference:
|
||||
```text
|
||||
@misc{scrapegraph-ai,
|
||||
author = {Lorenzo Padoan, Marco Vinciguerra},
|
||||
title = {Scrapegraph-ai},
|
||||
year = {2024},
|
||||
url = {https://github.com/ScrapeGraphAI/Scrapegraph-ai},
|
||||
note = {A Python library for scraping leveraging large language models}
|
||||
}
|
||||
```
|
||||
## Authors
|
||||
|
||||
| | Contact Info |
|
||||
|--------------------|----------------------|
|
||||
| Marco Vinciguerra | [](https://www.linkedin.com/in/marco-vinciguerra-7ba365242/) |
|
||||
| Lorenzo Padoan | [](https://www.linkedin.com/in/lorenzo-padoan-4521a2154/) |
|
||||
|
||||
## 📜 License
|
||||
|
||||
ScrapeGraphAI is licensed under the MIT License. See the [LICENSE](https://github.com/ScrapeGraphAI/Scrapegraph-ai/blob/main/LICENSE) file for more information.
|
||||
|
||||
## Acknowledgements
|
||||
|
||||
- We would like to thank all the contributors to the project and the open-source community for their support.
|
||||
- ScrapeGraphAI is meant to be used for data exploration and research purposes only. We are not responsible for any misuse of the library.
|
||||
|
||||
Made with ❤️ by [ScrapeGraph AI](https://scrapegraphai.com)
|
||||
|
||||
[Scarf tracking](https://static.scarf.sh/a.png?x-pxid=102d4b8c-cd6a-4b9e-9a16-d6d141b9212d)
|
||||
@@ -0,0 +1,7 @@
|
||||
# WeHub 来源说明
|
||||
|
||||
- 原始项目：`ScrapeGraphAI/Scrapegraph-ai`
|
||||
- 原始仓库：https://github.com/ScrapeGraphAI/Scrapegraph-ai
|
||||
- 导入方式：上游默认分支的最新快照
|
||||
- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准
|
||||
- 本文件仅用于记录来源，不代表 WeHub 是原项目作者
|
||||
@@ -0,0 +1,5 @@
|
||||
# Security Policy
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
For reporting a vulnerability contact directly mvincig11@gmail.com
|
||||
@@ -0,0 +1,123 @@
|
||||
# Semantic Commit Format for This PR
|
||||
|
||||
## Current Situation
|
||||
|
||||
This PR contains commits that need to be rewritten to follow Conventional Commits format for semantic-release compatibility.
|
||||
|
||||
**Note:** The timeout documentation is marked as `feat(timeout)` (not `docs`) because it exposes a user-facing feature. Even though the implementation existed, this PR makes the feature discoverable and usable by users through documentation, which warrants a feature-level semantic version bump.
|
||||
|
||||
## Commits to Rewrite
|
||||
|
||||
### Commit 1: 9439fe5
|
||||
**Current:** `Fix langchain import issues blocking tests`
|
||||
|
||||
**Should be:**
|
||||
```
|
||||
fix(imports): update deprecated langchain imports to langchain_core
|
||||
|
||||
Update imports from deprecated langchain.prompts to langchain_core.prompts
|
||||
across 20 files to fix test suite import errors. These changes address
|
||||
breaking API changes in newer langchain versions.
|
||||
|
||||
Fixes #1015
|
||||
```
|
||||
|
||||
**Type:** `fix` - Bug fix for test import errors
|
||||
**Scope:** `imports` - Changes affect import statements
|
||||
|
||||
---
|
||||
|
||||
### Commit 2: 323f26a
|
||||
**Current:** `Add comprehensive timeout feature documentation`
|
||||
|
||||
**Should be:**
|
||||
```
|
||||
feat(timeout): add configurable timeout support for FetchNode
|
||||
|
||||
Add comprehensive documentation for the timeout configuration feature:
|
||||
- Configuration examples with different timeout values
|
||||
- Use cases for HTTP requests, PDF parsing, and ChromiumLoader
|
||||
- Graph integration examples
|
||||
- Best practices and troubleshooting guide
|
||||
|
||||
The timeout feature enables users to control execution time for blocking
|
||||
operations (HTTP requests, PDF parsing, ChromiumLoader) to prevent
|
||||
indefinite hangs. Configurable via node_config with 30s default.
|
||||
|
||||
Fixes #1015
|
||||
```
|
||||
|
||||
**Type:** `feat` - New feature documentation/exposure to users
|
||||
**Scope:** `timeout` - Timeout configuration feature
|
||||
|
||||
---
|
||||
|
||||
## How to Apply (For Maintainer)
|
||||
|
||||
Since automated tools can't force-push to rewrite history, the maintainer needs to manually rewrite these commits:
|
||||
|
||||
### Option 1: Interactive Rebase
|
||||
```bash
|
||||
git rebase -i 6d13212
|
||||
# Mark commits 9439fe5 and 323f26a as 'reword'
|
||||
# Update commit messages with semantic format above
|
||||
# Force push: git push --force-with-lease
|
||||
```
|
||||
|
||||
### Option 2: Squash and Rewrite
|
||||
```bash
|
||||
# Reset to initial commit
|
||||
git reset --soft 6d13212
|
||||
|
||||
# Stage import fixes
|
||||
git add scrapegraphai/
|
||||
|
||||
# Commit with semantic message
|
||||
git commit -m "fix(imports): update deprecated langchain imports to langchain_core
|
||||
|
||||
Update imports from deprecated langchain.prompts to langchain_core.prompts
|
||||
across 20 files to fix test suite import errors. These changes address
|
||||
breaking API changes in newer langchain versions.
|
||||
|
||||
Fixes #1015"
|
||||
|
||||
# Stage documentation
|
||||
git add docs/
|
||||
|
||||
# Commit with semantic message
|
||||
git commit -m "feat(timeout): add configurable timeout support for FetchNode
|
||||
|
||||
Add comprehensive documentation for the timeout configuration feature:
|
||||
- Configuration examples with different timeout values
|
||||
- Use cases for HTTP requests, PDF parsing, and ChromiumLoader
|
||||
- Graph integration examples
|
||||
- Best practices and troubleshooting guide
|
||||
|
||||
The timeout feature enables users to control execution time for blocking
|
||||
operations (HTTP requests, PDF parsing, ChromiumLoader) to prevent
|
||||
indefinite hangs. Configurable via node_config with 30s default.
|
||||
|
||||
Fixes #1015"
|
||||
|
||||
# Force push
|
||||
git push --force-with-lease origin copilot/add-timeout-to-fetch-node
|
||||
```
|
||||
|
||||
## Semantic Release Configuration
|
||||
|
||||
This repository uses `@semantic-release/commit-analyzer` with `conventionalcommits` preset (see `.releaserc.yml`).
|
||||
|
||||
Valid types for this repo:
|
||||
- `feat`: New features → Minor version bump
|
||||
- `fix`: Bug fixes → Patch version bump
|
||||
- `docs`: Documentation changes → No version bump (shown in changelog)
|
||||
- `chore`: Maintenance tasks
|
||||
- `refactor`: Code refactoring
|
||||
- `perf`: Performance improvements
|
||||
- `test`: Test changes
|
||||
|
||||
## References
|
||||
|
||||
- [Conventional Commits](https://www.conventionalcommits.org/)
|
||||
- [Semantic Release](https://semantic-release.gitbook.io/)
|
||||
- Repository config: `.releaserc.yml`
|
||||
@@ -0,0 +1,327 @@
|
||||
# Enhanced Testing Infrastructure - Implementation Summary
|
||||
|
||||
## Overview
|
||||
|
||||
A comprehensive testing infrastructure has been implemented for ScrapeGraphAI with support for unit tests, integration tests, performance benchmarking, and automated CI/CD pipelines.
|
||||
|
||||
## What Was Added
|
||||
|
||||
### 1. Core Testing Configuration
|
||||
|
||||
#### `pytest.ini`
|
||||
- Complete pytest configuration with coverage tracking
|
||||
- Custom markers for test categorization (integration, slow, benchmark, etc.)
|
||||
- Code coverage settings with HTML/XML reports
|
||||
- Test discovery patterns and exclusions
|
||||
|
||||
#### `tests/conftest.py`
|
||||
- Shared fixtures for all LLM providers (OpenAI, Ollama, Anthropic, Groq, Azure, Gemini)
|
||||
- Mock LLM and embedder fixtures for unit testing
|
||||
- Test data fixtures (HTML, JSON, XML, CSV)
|
||||
- Temporary file fixtures
|
||||
- Performance tracking fixtures
|
||||
- Custom pytest hooks and CLI options
|
||||
- Automatic test filtering based on markers
|
||||
|
||||
### 2. Mock HTTP Server (`tests/fixtures/mock_server/`)
|
||||
|
||||
A fully functional HTTP server for consistent testing without external dependencies:
|
||||
|
||||
**Features:**
|
||||
- Static HTML pages (home, products, projects)
|
||||
- JSON/XML/CSV API endpoints
|
||||
- Slow response simulation
|
||||
- Error condition testing (404, 500)
|
||||
- Rate limiting simulation
|
||||
- Dynamic content generation
|
||||
- Pagination support
|
||||
- Thread-safe operation
|
||||
|
||||
**Endpoints:**
|
||||
- `/` - Home page
|
||||
- `/products` - Product listings with prices and stock status
|
||||
- `/projects` - Project listings with descriptions
|
||||
- `/api/data.json` - JSON data endpoint
|
||||
- `/api/data.xml` - XML data endpoint
|
||||
- `/api/data.csv` - CSV data endpoint
|
||||
- `/slow` - 2-second delay simulation
|
||||
- `/error/404` - 404 error page
|
||||
- `/error/500` - 500 error page
|
||||
- `/rate-limited` - Rate limit testing (5 requests max)
|
||||
- `/dynamic` - Dynamically generated content
|
||||
- `/pagination?page=N` - Paginated content
|
||||
|
||||
### 3. Performance Benchmarking (`tests/fixtures/benchmarking.py`)
|
||||
|
||||
**Components:**
|
||||
- `BenchmarkResult` - Individual test result tracking
|
||||
- `BenchmarkSummary` - Statistical analysis across multiple runs
|
||||
- `BenchmarkTracker` - Result collection and reporting
|
||||
- `benchmark()` - Decorator/function for benchmarking
|
||||
- Baseline comparison utilities
|
||||
- Performance regression detection
|
||||
|
||||
**Metrics Tracked:**
|
||||
- Execution time (mean, median, std dev, min, max)
|
||||
- Memory usage
|
||||
- Token usage
|
||||
- API call counts
|
||||
- Success rates
|
||||
|
||||
**Features:**
|
||||
- JSON export of results
|
||||
- Human-readable reports
|
||||
- Warmup runs support
|
||||
- Multiple test runs with statistics
|
||||
- Baseline comparison for regression detection
|
||||
|
||||
### 4. Test Utilities (`tests/fixtures/helpers.py`)
|
||||
|
||||
**Assertion Helpers:**
|
||||
- `assert_valid_scrape_result()` - Validate scraping results
|
||||
- `assert_execution_info_valid()` - Validate execution metadata
|
||||
- `assert_response_time_acceptable()` - Performance assertions
|
||||
- `assert_no_errors_in_result()` - Error detection
|
||||
|
||||
**Mock Response Builders:**
|
||||
- `create_mock_llm_response()` - Generate mock LLM responses
|
||||
- `create_mock_graph_result()` - Mock graph execution results
|
||||
|
||||
**Data Generators:**
|
||||
- `generate_test_html()` - Customizable HTML generation
|
||||
- `generate_test_json()` - Test JSON data
|
||||
- `generate_test_csv()` - Test CSV data
|
||||
|
||||
**Validation Utilities:**
|
||||
- `validate_schema_match()` - Pydantic schema validation
|
||||
- `validate_extracted_fields()` - Field extraction validation
|
||||
|
||||
**Additional Utilities:**
|
||||
- `RateLimitHelper` - Rate limiting testing
|
||||
- `retry_with_backoff()` - Retry logic with exponential backoff
|
||||
- `compare_results()` - Result comparison
|
||||
- `fuzzy_match_strings()` - Fuzzy string matching
|
||||
- File loading and saving utilities
|
||||
|
||||
### 5. Integration Test Suite
|
||||
|
||||
#### `tests/integration/test_smart_scraper_integration.py`
|
||||
- SmartScraperGraph with multiple LLM providers
|
||||
- Schema-based scraping tests
|
||||
- Timeout handling tests
|
||||
- Error condition tests (404, 500)
|
||||
- Performance benchmarks
|
||||
- Real website testing support
|
||||
|
||||
#### `tests/integration/test_multi_graph_integration.py`
|
||||
- SmartScraperMultiGraph tests
|
||||
- Concurrent scraping tests
|
||||
- Performance benchmarks for multi-page scraping
|
||||
- SearchGraph integration tests
|
||||
|
||||
#### `tests/integration/test_file_formats_integration.py`
|
||||
- JSONScraperGraph tests (files and URLs)
|
||||
- XMLScraperGraph tests (files and URLs)
|
||||
- CSVScraperGraph tests (files and URLs)
|
||||
- Performance benchmarks for file format scrapers
|
||||
|
||||
### 6. GitHub Actions Workflow (`.github/workflows/test-suite.yml`)
|
||||
|
||||
**Jobs:**
|
||||
|
||||
1. **Unit Tests**
|
||||
- Matrix: Ubuntu, macOS, Windows
|
||||
- Python versions: 3.10, 3.11, 3.12
|
||||
- Coverage reporting to Codecov
|
||||
- Fast execution without external dependencies
|
||||
|
||||
2. **Integration Tests**
|
||||
- Test groups: smart-scraper, multi-graph, file-formats
|
||||
- Real LLM provider testing (with API keys)
|
||||
- Artifact uploads for test results
|
||||
|
||||
3. **Performance Benchmarks**
|
||||
- Track execution time and resource usage
|
||||
- Save results as artifacts
|
||||
- Compare against baseline (on PRs)
|
||||
|
||||
4. **Code Quality**
|
||||
- Ruff linting
|
||||
- Black formatting check
|
||||
- isort import sorting check
|
||||
- mypy type checking
|
||||
|
||||
5. **Test Coverage Report**
|
||||
- Aggregate coverage from all jobs
|
||||
- PR comments with coverage changes
|
||||
|
||||
6. **Test Summary**
|
||||
- Overall test status reporting
|
||||
|
||||
**Triggers:**
|
||||
- Push to main, pre/beta, dev branches
|
||||
- Pull requests to main, pre/beta
|
||||
- Manual workflow dispatch
|
||||
|
||||
### 7. Documentation
|
||||
|
||||
#### `tests/README_TESTING.md`
|
||||
Comprehensive guide covering:
|
||||
- Test organization structure
|
||||
- Running different test types
|
||||
- Using fixtures and markers
|
||||
- Performance benchmarking
|
||||
- Mock server usage
|
||||
- Environment variables
|
||||
- Writing new tests (with templates)
|
||||
- Best practices
|
||||
- Troubleshooting
|
||||
|
||||
## Key Features
|
||||
|
||||
### Multi-Provider Support
|
||||
Test compatibility across all supported LLM providers:
|
||||
- OpenAI (GPT-3.5, GPT-4)
|
||||
- Ollama (local models)
|
||||
- Anthropic Claude
|
||||
- Groq
|
||||
- Azure OpenAI
|
||||
- Google Gemini
|
||||
|
||||
### Test Markers
|
||||
Organized test categorization:
|
||||
- `@pytest.mark.unit` - Fast unit tests
|
||||
- `@pytest.mark.integration` - Integration tests
|
||||
- `@pytest.mark.slow` - Long-running tests
|
||||
- `@pytest.mark.benchmark` - Performance tests
|
||||
- `@pytest.mark.requires_api_key` - Needs API credentials
|
||||
|
||||
### Flexible Test Execution
|
||||
```bash
|
||||
# Unit tests only
|
||||
pytest -m "unit or not integration"
|
||||
|
||||
# Integration tests
|
||||
pytest --integration
|
||||
|
||||
# Performance benchmarks
|
||||
pytest --benchmark -m benchmark
|
||||
|
||||
# Slow tests
|
||||
pytest --slow
|
||||
|
||||
# With coverage
|
||||
pytest --cov=scrapegraphai --cov-report=html
|
||||
```
|
||||
|
||||
### Mock Server Benefits
|
||||
- No external dependencies for basic tests
|
||||
- Consistent, reproducible test conditions
|
||||
- Simulate error conditions and edge cases
|
||||
- Test rate limiting and timeouts
|
||||
- Fast test execution
|
||||
|
||||
### Performance Tracking
|
||||
- Automatic tracking of execution time
|
||||
- Token usage monitoring
|
||||
- API call counting
|
||||
- Regression detection
|
||||
- Baseline comparison
|
||||
|
||||
## Usage Examples
|
||||
|
||||
### Basic Unit Test
|
||||
```python
|
||||
def test_with_mock(mock_llm_model):
|
||||
"""Fast test with mocked LLM."""
|
||||
result = some_function(mock_llm_model)
|
||||
assert result is not None
|
||||
```
|
||||
|
||||
### Integration Test
|
||||
```python
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.requires_api_key
|
||||
def test_real_scraping(openai_config, mock_server):
|
||||
"""Test with real LLM and mock server."""
|
||||
url = mock_server.get_url("/products")
|
||||
scraper = SmartScraperGraph(
|
||||
prompt="Extract products",
|
||||
source=url,
|
||||
config=openai_config
|
||||
)
|
||||
result = scraper.run()
|
||||
assert_valid_scrape_result(result)
|
||||
```
|
||||
|
||||
### Performance Benchmark
|
||||
```python
|
||||
@pytest.mark.benchmark
|
||||
def test_performance(benchmark_tracker, openai_config):
|
||||
"""Benchmark scraping performance."""
|
||||
import time
|
||||
|
||||
start = time.perf_counter()
|
||||
# Run operation
|
||||
end = time.perf_counter()
|
||||
|
||||
benchmark_tracker.record(BenchmarkResult(
|
||||
test_name="my_test",
|
||||
execution_time=end - start,
|
||||
success=True
|
||||
))
|
||||
```
|
||||
|
||||
## Benefits
|
||||
|
||||
1. **Comprehensive Coverage**: Unit, integration, and performance tests
|
||||
2. **Fast Feedback**: Quick unit tests with extensive mocking
|
||||
3. **Real-World Testing**: Integration tests with actual LLM providers
|
||||
4. **Performance Monitoring**: Track and prevent performance regressions
|
||||
5. **CI/CD Ready**: Automated testing in GitHub Actions
|
||||
6. **Developer Friendly**: Clear documentation and templates
|
||||
7. **Flexible Execution**: Run specific test subsets easily
|
||||
8. **Cross-Platform**: Tested on Linux, macOS, Windows
|
||||
9. **Multi-Python**: Support for Python 3.10, 3.11, 3.12
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. **Add more integration tests** for additional graph types
|
||||
2. **Expand mock server** with more realistic scenarios
|
||||
3. **Add visual regression testing** for screenshot comparisons
|
||||
4. **Implement mutation testing** for test quality
|
||||
5. **Add property-based testing** with Hypothesis
|
||||
6. **Create performance dashboards** for trend visualization
|
||||
7. **Add load testing** for concurrent scraping scenarios
|
||||
|
||||
## Files Created/Modified
|
||||
|
||||
**New Files:**
|
||||
- `pytest.ini` - Pytest configuration
|
||||
- `tests/conftest.py` - Shared fixtures
|
||||
- `tests/fixtures/mock_server/server.py` - Mock HTTP server
|
||||
- `tests/fixtures/benchmarking.py` - Performance framework
|
||||
- `tests/fixtures/helpers.py` - Test utilities
|
||||
- `tests/integration/test_smart_scraper_integration.py`
|
||||
- `tests/integration/test_multi_graph_integration.py`
|
||||
- `tests/integration/test_file_formats_integration.py`
|
||||
- `.github/workflows/test-suite.yml` - CI/CD workflow
|
||||
- `tests/README_TESTING.md` - Testing documentation
|
||||
- `TESTING_INFRASTRUCTURE.md` - This file
|
||||
|
||||
**Directories Created:**
|
||||
- `tests/fixtures/`
|
||||
- `tests/fixtures/mock_server/`
|
||||
- `tests/integration/`
|
||||
- `benchmark_results/` (auto-created when running benchmarks)
|
||||
|
||||
## Contributing
|
||||
|
||||
When adding new tests:
|
||||
1. Use appropriate fixtures from conftest.py
|
||||
2. Add proper markers (@pytest.mark.*)
|
||||
3. Follow existing test structure
|
||||
4. Update documentation as needed
|
||||
5. Ensure tests pass in CI
|
||||
|
||||
For questions or issues with the testing infrastructure, please open an issue on GitHub.
|
||||
@@ -0,0 +1,14 @@
|
||||
cff-version: 0.0.1
|
||||
message: "If you use Scrapegraph-ai in your research, please cite it using these metadata."
|
||||
authors:
|
||||
- family-names: Perini
|
||||
given-names: Marco
|
||||
- family-names: Padoan
|
||||
given-names: Lorenzo
|
||||
- family-names: Vinciguerra
|
||||
given-names: Marco
|
||||
title: Scrapegraph-ai
|
||||
version: v0.0.10
|
||||
date-released: 2024-1-10
|
||||
url: https://github.com/VinciGit00/Scrapegraph-ai
|
||||
license: MIT
|
||||
@@ -0,0 +1,2 @@
|
||||
from: pytest
|
||||
setup_commands: ['@merge', 'pip install -q selenium', 'pip install -q playwright', 'playwright install']
|
||||
@@ -0,0 +1,13 @@
|
||||
version: '3.8'
|
||||
services:
|
||||
ollama:
|
||||
image: ollama/ollama
|
||||
container_name: ollama
|
||||
ports:
|
||||
- "11434:11434"
|
||||
volumes:
|
||||
- ollama_volume:/root/.ollama
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
ollama_volume:
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 48 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 336 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 59 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 121 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 59 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 85 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.0 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 52 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 72 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 57 KiB |
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 82 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 38 KiB |
@@ -0,0 +1,15 @@
|
||||
<svg width="147" height="120" viewBox="0 0 147 120" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<g clip-path="url(#clip0_118_212)">
|
||||
<path d="M138.095 66.4673L145.459 55.2755C147.475 52.1999 146.621 48.0649 143.545 46.0316L137.941 42.358C133.789 39.6241 129.176 37.6933 124.323 36.6339L124.648 23.8189C124.733 20.3674 122.17 17.4114 118.736 17.0184C118.513 17.0013 117.3 16.8646 115.557 16.8646C113.148 16.8646 109.97 17.1209 106.877 18.1119C103.358 12.7809 98.5391 8.26997 92.9346 5.12601C87.0568 1.77702 80.2392 -0.0170898 73.2507 -0.0170898C66.6211 -0.0170898 60.1452 1.60615 54.4895 4.66467C48.4237 7.87697 43.3661 12.5587 39.6753 18.1119C36.5997 17.1209 33.4216 16.8646 31.0124 16.8646C29.2695 16.8646 28.0564 16.9842 27.8342 17.0184C24.3998 17.4114 21.8368 20.3503 21.9222 23.8189L22.2469 36.6168C17.3943 37.6762 12.7638 39.607 8.61169 42.3409L3.00725 46.0145C-0.0683634 48.0308 -0.922699 52.1657 1.09353 55.2584L8.45791 66.4502C9.14138 67.4754 10.0811 68.2956 11.1747 68.8082C6.21955 75.5745 3.29772 83.8958 3.29772 92.8492V106.091C3.29772 109.782 6.28789 112.755 9.96154 112.755H23.3575C24.9295 112.755 27.4412 112.055 28.8936 109.816C29.0474 110.124 29.2183 110.414 29.3891 110.722L32.6698 116.566C33.5412 118.103 34.9765 119.248 36.6852 119.727C37.2832 119.897 37.8812 119.983 38.4793 119.983C39.607 119.983 40.7347 119.692 41.7428 119.129L53.4301 112.567C56.6424 110.756 57.7873 106.707 55.9761 103.494L53.3105 98.7441C58.6074 103.58 65.6301 106.433 73.2849 106.433C80.9398 106.433 87.7915 103.648 93.0542 98.915L90.4741 103.494C90.2008 103.99 89.9786 104.519 89.8419 105.066C89.5344 106.228 88.9022 110.261 93.5327 112.858L94.8483 113.592L98.8637 115.848L104.707 119.129C105.733 119.71 106.86 119.983 107.971 119.983C110.312 119.983 112.567 118.753 113.797 116.583L117.625 109.765C118.821 111.576 120.871 112.772 123.195 112.772H136.591C138.898 112.772 143.255 111.252 143.255 105.528V99.4789V92.8492C143.255 83.9128 140.333 75.5745 135.378 68.8082C136.454 68.2956 137.394 67.5096 138.095 66.4502V66.4673Z" fill="#AC6DFF"/>
|
||||
<path d="M105.22 58.8809C122.768 60.282 136.574 74.9424 136.574 92.8493V106.091H123.178V92.8493C123.178 82.2726 115.233 73.5584 104.998 72.2939C108.825 75.5746 111.935 79.8804 113.814 85.2628C116.275 92.2683 115.438 100.009 111.798 106.484L107.971 113.302L96.2834 106.758L99.564 100.914C103.135 94.5579 101.444 86.3563 95.6341 81.9138C93.3103 92.1316 84.2031 99.7523 73.2847 99.7523C62.3663 99.7523 53.2078 92.0974 50.9182 81.8454C45.0233 86.2709 43.3146 94.5238 46.9028 100.914L50.1834 106.758L38.4961 113.319L35.2155 107.475C28.6713 95.8223 31.4051 80.974 41.4521 72.311C31.2684 73.6096 23.3744 82.2897 23.3744 92.8493V106.091H9.9784V92.8493C9.9784 74.9424 23.8016 60.2649 41.3496 58.8638C43.4342 62.4178 46.0826 65.5959 49.1753 68.2615C49.8759 68.8595 50.8327 68.8937 51.5675 68.4323C51.7383 68.3298 51.9092 68.1931 52.0288 68.0223C52.0459 67.9881 52.0801 67.9881 52.0972 67.9539C52.7294 67.1167 52.6439 65.9206 51.8409 65.2371C44.1518 58.6587 39.4872 48.663 40.3244 37.5908C41.5205 21.9394 53.8913 9.0731 69.4914 7.33025C89.3804 5.10898 106.296 20.6921 106.296 40.1538C106.296 50.1666 101.785 59.1372 94.7285 65.203C93.9254 65.8864 93.8229 67.0825 94.4551 67.9197C94.4893 67.9539 94.5406 67.971 94.5576 68.0223C94.6602 68.159 94.7969 68.2615 94.9335 68.3469C95.6683 68.8424 96.6764 68.8253 97.394 68.2102C100.487 65.5447 103.118 62.3836 105.203 58.8467L105.22 58.8809ZM29.3206 38.1205C30.3629 38.1205 33.4897 38.2572 36.2578 39.5729C36.2749 38.2059 36.3603 36.8219 36.5312 35.4208C36.9754 31.8326 37.9835 28.4153 39.4017 25.22C36.702 23.836 33.2847 23.5455 31.0121 23.5455C29.5598 23.5455 28.5858 23.6481 28.5858 23.6481L28.9447 38.1205C28.9447 38.1205 29.0813 38.1205 29.3035 38.1205H29.3206ZM30.9609 55.754C34.5149 55.754 37.9835 56.6767 41.0591 58.4366C38.4107 53.7719 36.7362 48.4921 36.3261 42.8364C34.532 42.4947 33.0113 42.3409 31.3026 42.3409C31.2514 42.3409 31.183 42.3409 31.1147 42.3409C30.4825 42.3409 29.799 42.3751 29.0813 42.4093C23.0839 42.7339 17.2573 44.6476 12.268 47.9283L6.66357 51.6019L14.0279 62.7937L19.6324 59.1201C22.6567 57.138 26.1766 55.9761 29.8161 55.7882C30.192 55.7711 30.585 55.754 30.9609 55.754ZM117.249 38.1205C117.488 38.1205 117.608 38.1205 117.608 38.1205L117.966 23.6481C117.966 23.6481 116.993 23.5455 115.54 23.5455C113.268 23.5455 109.816 23.836 107.133 25.22C109.081 29.6284 110.209 34.481 110.277 39.5729C113.063 38.2401 116.189 38.1034 117.249 38.1034V38.1205ZM132.507 62.7937L139.872 51.6019L134.267 47.9283C129.261 44.6476 123.451 42.7339 117.454 42.4093C116.873 42.3751 116.326 42.358 115.779 42.358C115.643 42.358 115.523 42.358 115.403 42.358C114.908 42.358 114.429 42.3922 113.951 42.4264C113.814 42.4264 113.661 42.4264 113.507 42.4435C112.943 42.4947 112.379 42.5631 111.815 42.6485C111.764 42.6485 111.713 42.6485 111.661 42.6656C111.183 42.7339 110.687 42.8364 110.192 42.939C109.782 48.5263 108.107 53.789 105.476 58.4195C108.569 56.6596 112.02 55.7369 115.574 55.7369C115.95 55.7369 116.343 55.7369 116.719 55.7711C120.359 55.9761 123.878 57.1209 126.903 59.103L132.507 62.7766V62.7937Z" fill="#242424"/>
|
||||
<path d="M88.2016 76.3265C88.2016 79.368 85.7411 81.8284 82.6997 81.8284C79.6582 81.8284 77.1978 79.368 77.1978 76.3265C77.1978 75.2671 77.5053 74.2761 78.035 73.4389C80.5638 73.0117 83.0243 72.3795 85.3994 71.5764C87.0568 72.5162 88.1845 74.2932 88.1845 76.3265H88.2016Z" fill="#AC6DFF"/>
|
||||
<path d="M58.2998 76.3265C58.2998 79.368 60.7603 81.8284 63.8017 81.8284C66.8432 81.8284 69.3037 79.368 69.3037 76.3265C69.3037 75.2671 68.9961 74.2761 68.4664 73.4389C65.9376 73.0117 63.4771 72.3795 61.102 71.5764C59.4446 72.5162 58.3169 74.2932 58.3169 76.3265H58.2998Z" fill="#AC6DFF"/>
|
||||
<path d="M63.7678 54.0453C58.8981 51.3798 55.0706 47.3132 52.7298 42.2897L52.576 41.9479C52.0463 40.8031 52.0463 39.4875 52.576 38.3597L52.8323 37.813C55.1048 33.0287 58.7443 29.1158 63.3235 26.5015L63.6994 26.2965L67.8857 33.6096L67.5098 33.8147C64.8442 35.3525 62.6571 37.5225 61.1193 40.1709C62.7084 42.9048 64.998 45.1261 67.8173 46.6639L63.7678 54.0624V54.0453Z" fill="#AC6DFF"/>
|
||||
<path d="M82.6823 54.0112L78.4961 46.698L78.872 46.493C81.5375 44.9552 83.7246 42.7852 85.2624 40.1367C83.6734 37.4029 81.3838 35.1816 78.5644 33.6438L82.614 26.2452C87.4837 28.9108 91.3111 32.9774 93.652 38.0009L93.8229 38.3597C94.3526 39.5045 94.3526 40.8202 93.8229 41.9479L93.5666 42.4776C91.294 47.2448 87.6717 51.1577 83.0753 53.789L82.6994 53.9941L82.6823 54.0112Z" fill="#AC6DFF"/>
|
||||
</g>
|
||||
<defs>
|
||||
<clipPath id="clip0_118_212">
|
||||
<rect width="146.536" height="120" fill="white"/>
|
||||
</clipPath>
|
||||
</defs>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 6.4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 54 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 53 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 67 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 60 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 48 KiB |
+226
@@ -0,0 +1,226 @@
|
||||
## 🚀 **正在寻找更快、更简单的规模化抓取方式（只需5行代码）？** 查看我们在 [**ScrapeGraphAI.com**](https://scrapegraphai.com/?utm_source=github&utm_medium=readme&utm_campaign=oss_cta&utm_content=top_banner) 的增强版本！🚀
|
||||
|
||||
---
|
||||
|
||||
# 🕷️ ScrapeGraphAI: 只需抓取一次
|
||||
|
||||
[English](../README.md) | [中文](chinese.md) | [日本語](japanese.md)
|
||||
| [한국어](korean.md)
|
||||
| [Русский](russian.md) | [Türkçe](turkish.md)
|
||||
| [Deutsch](german.md)
|
||||
| [Español](spanish.md)
|
||||
| [français](french.md)
|
||||
| [Português](portuguese.md)
|
||||
| [Italiano](italian.md)
|
||||
|
||||
[](https://pepy.tech/projects/scrapegraphai)
|
||||
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
[](https://discord.gg/gkxQDAjfeX)
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/15078" target="_blank"><img src="https://trendshift.io/api/badge/repositories/15078" alt="ScrapeGraphAI%2FScrapegraph-ai | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
<p align="center">
|
||||
|
||||
[ScrapeGraphAI](https://scrapegraphai.com) 是一个*网络爬虫* Python 库，使用大型语言模型和直接图逻辑为网站和本地文档（XML，HTML，JSON，Markdown 等）创建爬取管道。
|
||||
|
||||
只需告诉库您想提取哪些信息，它将为您完成！
|
||||
|
||||
|
||||
|
||||
## 🚀 集成
|
||||
|
||||
ScrapeGraphAI 提供与流行框架和工具的无缝集成，以增强您的抓取能力。无论您使用 Python 还是 Node.js 构建，使用 LLM 框架，还是使用无代码平台，我们都为您提供全面的集成选项。
|
||||
|
||||
<p align="center">
|
||||
<img src="https://raw.githubusercontent.com/ScrapeGraphAI/Scrapegraph-ai/main/docs/assets/sgai-hero.png" alt="ScrapeGraphAI Hero" style="width: 100%;">
|
||||
</p>
|
||||
您可以在以下[链接](https://scrapegraphai.com)找到更多信息
|
||||
|
||||
**集成**：
|
||||
- **API**: [文档](https://docs.scrapegraphai.com/introduction)
|
||||
- **SDKs**: [Python](https://docs.scrapegraphai.com/sdks/python), [Node](https://docs.scrapegraphai.com/sdks/javascript)
|
||||
- **LLM 框架**: [Langchain](https://docs.scrapegraphai.com/integrations/langchain), [Llama Index](https://docs.scrapegraphai.com/integrations/llamaindex), [Crew.ai](https://docs.scrapegraphai.com/integrations/crewai), [Agno](https://docs.scrapegraphai.com/integrations/agno), [CamelAI](https://github.com/camel-ai/camel)
|
||||
- **低代码框架**: [Pipedream](https://pipedream.com/apps/scrapegraphai), [Bubble](https://bubble.io/plugin/scrapegraphai-1745408893195x213542371433906180), [Zapier](https://zapier.com/apps/scrapegraphai/integrations), [n8n](http://localhost:5001/dashboard), [Dify](https://dify.ai), [Toolhouse](https://app.toolhouse.ai/mcp-servers/scrapegraph_smartscraper)
|
||||
- **MCP 服务器**: [链接](https://smithery.ai/server/@ScrapeGraphAI/scrapegraph-mcp)
|
||||
|
||||
## 🚀 快速安装
|
||||
|
||||
Scrapegraph-ai 的参考页面可以在 PyPI 的官方网站上找到: [pypi](https://pypi.org/project/scrapegraphai/)。
|
||||
|
||||
```bash
|
||||
pip install scrapegraphai
|
||||
|
||||
# 重要（用于获取网站内容）
|
||||
playwright install
|
||||
```
|
||||
|
||||
**注意**: 建议在虚拟环境中安装该库，以避免与其他库发生冲突 🐱
|
||||
|
||||
|
||||
## 💻 用法
|
||||
有多种标准抓取管道可用于从网站（或本地文件）提取信息。
|
||||
|
||||
最常见的是 `SmartScraperGraph`，它在给定用户提示和源 URL 的情况下从单个页面提取信息。
|
||||
|
||||
|
||||
```python
|
||||
from scrapegraphai.graphs import SmartScraperGraph
|
||||
|
||||
# 定义抓取管道的配置
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"model": "ollama/llama3.2",
|
||||
"model_tokens": 8192,
|
||||
"format": "json",
|
||||
},
|
||||
"verbose": True,
|
||||
"headless": False,
|
||||
}
|
||||
|
||||
# 创建 SmartScraperGraph 实例
|
||||
smart_scraper_graph = SmartScraperGraph(
|
||||
prompt="从网页中提取有用信息，包括公司描述、创始人和社交媒体链接",
|
||||
source="https://scrapegraphai.com/",
|
||||
config=graph_config
|
||||
)
|
||||
|
||||
# 运行管道
|
||||
result = smart_scraper_graph.run()
|
||||
|
||||
import json
|
||||
print(json.dumps(result, indent=4))
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
> 对于 OpenAI 和其他模型，您只需要更改 llm 配置！
|
||||
> ```python
|
||||
>graph_config = {
|
||||
> "llm": {
|
||||
> "api_key": "YOUR_OPENAI_API_KEY",
|
||||
> "model": "openai/gpt-4o-mini",
|
||||
> },
|
||||
> "verbose": True,
|
||||
> "headless": False,
|
||||
>}
|
||||
>```
|
||||
|
||||
|
||||
输出将是一个类似以下的字典：
|
||||
|
||||
```python
|
||||
{
|
||||
"description": "ScrapeGraphAI transforms websites into clean, organized data for AI agents and data analytics. It offers an AI-powered API for effortless and cost-effective data extraction.",
|
||||
"founders": [
|
||||
{
|
||||
"name": "",
|
||||
"role": "Founder & Technical Lead",
|
||||
"linkedin": "https://www.linkedin.com/in/perinim/"
|
||||
},
|
||||
{
|
||||
"name": "Marco Vinciguerra",
|
||||
"role": "Founder & Software Engineer",
|
||||
"linkedin": "https://www.linkedin.com/in/marco-vinciguerra-7ba365242/"
|
||||
},
|
||||
{
|
||||
"name": "Lorenzo Padoan",
|
||||
"role": "Founder & Product Engineer",
|
||||
"linkedin": "https://www.linkedin.com/in/lorenzo-padoan-4521a2154/"
|
||||
}
|
||||
],
|
||||
"social_media_links": {
|
||||
"linkedin": "https://www.linkedin.com/company/101881123",
|
||||
"twitter": "https://x.com/scrapegraphai",
|
||||
"github": "https://github.com/ScrapeGraphAI/Scrapegraph-ai"
|
||||
}
|
||||
}
|
||||
```
|
||||
还有其他管道可用于从多个页面提取信息、生成 Python 脚本，甚至生成音频文件。
|
||||
|
||||
| 管道名称 | 描述 |
|
||||
|-------------------------|------------------------------------------------------------------------------------------------------------------|
|
||||
| SmartScraperGraph | 单页抓取器，只需要用户提示和输入源。 |
|
||||
| SearchGraph | 多页抓取器，从搜索引擎的前 n 个搜索结果中提取信息。 |
|
||||
| SpeechGraph | 单页抓取器，从网站提取信息并生成音频文件。 |
|
||||
| ScriptCreatorGraph | 单页抓取器，从网站提取信息并生成 Python 脚本。 |
|
||||
| SmartScraperMultiGraph | 多页抓取器，在给定单个提示和源列表的情况下从多个页面提取信息。 |
|
||||
| ScriptCreatorMultiGraph | 多页抓取器，生成用于从多个页面和源提取信息的 Python 脚本。 |
|
||||
|
||||
对于这些图中的每一个，都有多版本。它允许并行调用 LLM。
|
||||
|
||||
可以通过 API 使用不同的 LLM，例如 **OpenAI**、**Groq**、**Azure** 和 **Gemini**，或使用 **Ollama** 的本地模型。
|
||||
|
||||
如果您想使用本地模型，请记住安装 [Ollama](https://ollama.com/) 并使用 **ollama pull** 命令下载模型。
|
||||
|
||||
|
||||
## 📖 文档
|
||||
|
||||
[](https://colab.research.google.com/drive/1sEZBonBMGP44CtO6GQTwAlL0BGJXjtfd?usp=sharing)
|
||||
|
||||
ScrapeGraphAI 的文档可以在[这里](https://docs.scrapegraphai.com/introduction)找到。
|
||||
|
||||
## 🤝 贡献
|
||||
|
||||
欢迎贡献并加入我们的 Discord 服务器与我们讨论改进和提出建议！
|
||||
|
||||
请参阅[贡献指南](https://github.com/ScrapeGraphAI/Scrapegraph-ai/blob/main/CONTRIBUTING.md)。
|
||||
|
||||
[](https://discord.gg/uJN7TYcpNa)
|
||||
[](https://www.linkedin.com/company/scrapegraphai/)
|
||||
[](https://twitter.com/scrapegraphai)
|
||||
|
||||
## 🔗 ScrapeGraph API & SDKs
|
||||
如果您正在寻找快速解决方案来将 ScrapeGraph 集成到您的系统中，请查看我们的强大 API [这里！](https://dashboard.scrapegraphai.com/login)
|
||||
|
||||
[](https://dashboard.scrapegraphai.com/login)
|
||||
|
||||
我们提供 Python 和 Node.js 的 SDK，使您可以轻松集成到您的项目中。请在下面查看：
|
||||
|
||||
| SDK | 语言 | GitHub 链接 |
|
||||
|-----------|----------|-----------------------------------------------------------------------------|
|
||||
| Python SDK | Python | [scrapegraph-py](https://docs.scrapegraphai.com/sdks/python) |
|
||||
| Node.js SDK | Node.js | [scrapegraph-js](https://docs.scrapegraphai.com/sdks/javascript) |
|
||||
|
||||
官方 API 文档可以在[这里](https://docs.scrapegraphai.com/introduction)找到。
|
||||
|
||||
## 🔥 基准测试
|
||||
|
||||
根据 Firecrawl 基准测试 [Firecrawl benchmark](https://scrapegraphai.com/compare/firecrawl)，ScrapeGraph 是市场上最好的抓取工具！
|
||||
|
||||
|
||||
|
||||
## 📈 遥测
|
||||
我们收集匿名使用指标以增强我们包的质量和用户体验。这些数据帮助我们确定改进的优先级并确保兼容性。如果您希望退出，请设置环境变量 SCRAPEGRAPHAI_TELEMETRY_ENABLED=false。有关更多信息，请参阅[这里](https://docs.scrapegraphai.com/introduction)的文档。
|
||||
|
||||
## ❤️ 贡献者
|
||||
[](https://github.com/ScrapeGraphAI/Scrapegraph-ai/graphs/contributors)
|
||||
|
||||
## 🎓 引用
|
||||
如果您将我们的库用于研究目的，请使用以下参考文献引用我们：
|
||||
```text
|
||||
@misc{scrapegraph-ai,
|
||||
author = {Lorenzo Padoan, Marco Vinciguerra},
|
||||
title = {Scrapegraph-ai},
|
||||
year = {2024},
|
||||
url = {https://github.com/ScrapeGraphAI/Scrapegraph-ai},
|
||||
note = {一个利用大型语言模型进行爬取的 Python 库}
|
||||
}
|
||||
```
|
||||
## 作者
|
||||
|
||||
| | 联系信息 |
|
||||
|--------------------|----------------------|
|
||||
| Marco Vinciguerra | [](https://www.linkedin.com/in/marco-vinciguerra-7ba365242/) |
|
||||
| Lorenzo Padoan | [](https://www.linkedin.com/in/lorenzo-padoan-4521a2154/) |
|
||||
|
||||
## 📜 许可证
|
||||
|
||||
ScrapeGraphAI 采用 MIT 许可证。更多信息请查看 [LICENSE](https://github.com/ScrapeGraphAI/Scrapegraph-ai/blob/main/LICENSE) 文件。
|
||||
|
||||
## 鸣谢
|
||||
|
||||
- 我们要感谢所有项目贡献者和开源社区的支持。
|
||||
- ScrapeGraphAI 仅用于数据探索和研究目的。我们不对任何滥用该库的行为负责。
|
||||
|
||||
Made with ❤️ by [ScrapeGraph AI](https://scrapegraphai.com)
|
||||
|
||||
[Scarf tracking](https://static.scarf.sh/a.png?x-pxid=102d4b8c-cd6a-4b9e-9a16-d6d141b9212d)
|
||||
+228
@@ -0,0 +1,228 @@
|
||||
## 🚀 **Vous cherchez un moyen encore plus rapide et plus simple de scraper à grande échelle (seulement 5 lignes de code) ?** Découvrez notre version améliorée sur [**ScrapeGraphAI.com**](https://scrapegraphai.com/?utm_source=github&utm_medium=readme&utm_campaign=oss_cta&utm_content=top_banner) ! 🚀
|
||||
|
||||
---
|
||||
|
||||
# 🕷️ ScrapeGraphAI: Vous ne scrapez qu'une fois
|
||||
|
||||
<p align="center">
|
||||
<a href="https://scrapegraphai.com">
|
||||
<img src="https://raw.githubusercontent.com/ScrapeGraphAI/Scrapegraph-ai/main/media/banner.png" alt="ScrapeGraphAI" style="width: 100%;">
|
||||
</a>
|
||||
</p>
|
||||
|
||||
[English](../README.md) | [中文](chinese.md) | [日本語](japanese.md)
|
||||
| [한국어](korean.md)
|
||||
| [Русский](russian.md) | [Türkçe](turkish.md)
|
||||
| [Deutsch](german.md)
|
||||
| [Español](spanish.md)
|
||||
| [français](french.md)
|
||||
| [Português](portuguese.md)
|
||||
| [Italiano](italian.md)
|
||||
|
||||
[](https://pepy.tech/projects/scrapegraphai)
|
||||
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
[](https://discord.gg/gkxQDAjfeX)
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/15078" target="_blank"><img src="https://trendshift.io/api/badge/repositories/15078" alt="ScrapeGraphAI%2FScrapegraph-ai | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
<p align="center">
|
||||
|
||||
[ScrapeGraphAI](https://scrapegraphai.com) est une bibliothèque Python de *web scraping* qui utilise les LLM et la logique de graphes directs pour créer des pipelines de scraping pour les sites web et les documents locaux (XML, HTML, JSON, Markdown, etc.).
|
||||
|
||||
Dites simplement quelles informations vous souhaitez extraire et la bibliothèque le fera pour vous !
|
||||
|
||||
<p align="center">
|
||||
<img src="https://raw.githubusercontent.com/ScrapeGraphAI/Scrapegraph-ai/main/docs/assets/sgai-hero.png" alt="ScrapeGraphAI Hero" style="width: 100%;">
|
||||
</p>
|
||||
|
||||
## 🚀 Intégrations
|
||||
|
||||
ScrapeGraphAI offre une intégration transparente avec les frameworks et outils populaires pour améliorer vos capacités de scraping. Que vous développiez avec Python ou Node.js, utilisiez des frameworks LLM ou travailliez avec des plateformes no-code, nous avons tout prévu avec nos options d'intégration complètes.
|
||||
|
||||
Vous pouvez trouver plus d'informations sur le [lien](https://scrapegraphai.com) suivant.
|
||||
|
||||
**Intégrations**:
|
||||
- **API**: [Documentation](https://docs.scrapegraphai.com/introduction)
|
||||
- **SDKs**: [Python](https://docs.scrapegraphai.com/sdks/python), [Node](https://docs.scrapegraphai.com/sdks/javascript)
|
||||
- **Frameworks LLM**: [Langchain](https://docs.scrapegraphai.com/integrations/langchain), [Llama Index](https://docs.scrapegraphai.com/integrations/llamaindex), [Crew.ai](https://docs.scrapegraphai.com/integrations/crewai), [Agno](https://docs.scrapegraphai.com/integrations/agno), [CamelAI](https://github.com/camel-ai/camel)
|
||||
- **Frameworks Low-code**: [Pipedream](https://pipedream.com/apps/scrapegraphai), [Bubble](https://bubble.io/plugin/scrapegraphai-1745408893195x213542371433906180), [Zapier](https://zapier.com/apps/scrapegraphai/integrations), [n8n](http://localhost:5001/dashboard), [Dify](https://dify.ai), [Toolhouse](https://app.toolhouse.ai/mcp-servers/scrapegraph_smartscraper)
|
||||
- **Serveur MCP**: [Lien](https://smithery.ai/server/@ScrapeGraphAI/scrapegraph-mcp)
|
||||
|
||||
## 🚀 Installation rapide
|
||||
|
||||
La page de référence pour Scrapegraph-ai est disponible sur la page officielle de PyPI : [pypi](https://pypi.org/project/scrapegraphai/).
|
||||
|
||||
```bash
|
||||
pip install scrapegraphai
|
||||
|
||||
# IMPORTANT (pour récupérer le contenu des sites web)
|
||||
playwright install
|
||||
```
|
||||
|
||||
**Remarque** : il est recommandé d'installer la bibliothèque dans un environnement virtuel pour éviter les conflits avec d'autres bibliothèques 🐱
|
||||
|
||||
## 💻 Utilisation
|
||||
Il existe plusieurs pipelines de scraping standards qui peuvent être utilisés pour extraire des informations d'un site web (ou d'un fichier local).
|
||||
|
||||
Le plus courant est le `SmartScraperGraph`, qui extrait les informations d'une seule page à partir d'une invite utilisateur et d'une URL source.
|
||||
|
||||
```python
|
||||
from scrapegraphai.graphs import SmartScraperGraph
|
||||
|
||||
# Définir la configuration pour le pipeline de scraping
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"model": "ollama/llama3.2",
|
||||
"model_tokens": 8192,
|
||||
"format": "json",
|
||||
},
|
||||
"verbose": True,
|
||||
"headless": False,
|
||||
}
|
||||
|
||||
# Créer l'instance SmartScraperGraph
|
||||
smart_scraper_graph = SmartScraperGraph(
|
||||
prompt="Extract useful information from the webpage, including a description of what the company does, founders and social media links",
|
||||
source="https://scrapegraphai.com/",
|
||||
config=graph_config
|
||||
)
|
||||
|
||||
# Exécuter le pipeline
|
||||
result = smart_scraper_graph.run()
|
||||
|
||||
import json
|
||||
print(json.dumps(result, indent=4))
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
> Pour OpenAI et d'autres modèles, il vous suffit de modifier la configuration LLM !
|
||||
> ```python
|
||||
>graph_config = {
|
||||
> "llm": {
|
||||
> "api_key": "YOUR_OPENAI_API_KEY",
|
||||
> "model": "openai/gpt-4o-mini",
|
||||
> },
|
||||
> "verbose": True,
|
||||
> "headless": False,
|
||||
>}
|
||||
>```
|
||||
|
||||
La sortie sera un dictionnaire comme suit :
|
||||
|
||||
```python
|
||||
{
|
||||
"description": "ScrapeGraphAI transforms websites into clean, organized data for AI agents and data analytics. It offers an AI-powered API for effortless and cost-effective data extraction.",
|
||||
"founders": [
|
||||
{
|
||||
"name": "",
|
||||
"role": "Founder & Technical Lead",
|
||||
"linkedin": "https://www.linkedin.com/in/perinim/"
|
||||
},
|
||||
{
|
||||
"name": "Marco Vinciguerra",
|
||||
"role": "Founder & Software Engineer",
|
||||
"linkedin": "https://www.linkedin.com/in/marco-vinciguerra-7ba365242/"
|
||||
},
|
||||
{
|
||||
"name": "Lorenzo Padoan",
|
||||
"role": "Founder & Product Engineer",
|
||||
"linkedin": "https://www.linkedin.com/in/lorenzo-padoan-4521a2154/"
|
||||
}
|
||||
],
|
||||
"social_media_links": {
|
||||
"linkedin": "https://www.linkedin.com/company/101881123",
|
||||
"twitter": "https://x.com/scrapegraphai",
|
||||
"github": "https://github.com/ScrapeGraphAI/Scrapegraph-ai"
|
||||
}
|
||||
}
|
||||
```
|
||||
Il existe d'autres pipelines qui peuvent être utilisés pour extraire des informations de plusieurs pages, générer des scripts Python, ou même générer des fichiers audio.
|
||||
|
||||
| Nom du pipeline | Description |
|
||||
|-------------------------|------------------------------------------------------------------------------------------------------------------|
|
||||
| SmartScraperGraph | Scraper d'une seule page qui n'a besoin que d'une invite utilisateur et d'une source d'entrée. |
|
||||
| SearchGraph | Scraper multi-pages qui extrait des informations des n premiers résultats d'un moteur de recherche. |
|
||||
| SpeechGraph | Scraper d'une seule page qui extrait des informations d'un site web et génère un fichier audio. |
|
||||
| ScriptCreatorGraph | Scraper d'une seule page qui extrait des informations d'un site web et génère un script Python. |
|
||||
| SmartScraperMultiGraph | Scraper multi-pages qui extrait des informations de plusieurs pages à partir d'une seule invite et d'une liste de sources. |
|
||||
| ScriptCreatorMultiGraph | Scraper multi-pages qui génère un script Python pour extraire des informations de plusieurs pages et sources. |
|
||||
|
||||
Pour chacun de ces graphes, il existe la version multi. Elle permet de faire des appels au LLM en parallèle.
|
||||
|
||||
Il est possible d'utiliser différents LLM via des API, telles qu'**OpenAI**, **Groq**, **Azure**, **Gemini**, **MiniMax** et d'autres, ou des modèles locaux en utilisant **Ollama**.
|
||||
|
||||
N'oubliez pas d'installer [Ollama](https://ollama.com/) et de télécharger les modèles en utilisant la commande **ollama pull**, si vous souhaitez utiliser des modèles locaux.
|
||||
|
||||
## 📖 Documentation
|
||||
|
||||
[](https://colab.research.google.com/drive/1sEZBonBMGP44CtO6GQTwAlL0BGJXjtfd?usp=sharing)
|
||||
|
||||
La documentation pour ScrapeGraphAI se trouve [ici](https://docs.scrapegraphai.com/introduction).
|
||||
|
||||
## 🤝 Contribuer
|
||||
|
||||
N'hésitez pas à contribuer et à rejoindre notre serveur Discord pour discuter avec nous des améliorations et nous donner des suggestions !
|
||||
|
||||
Veuillez consulter les [directives de contribution](https://github.com/ScrapeGraphAI/Scrapegraph-ai/blob/main/CONTRIBUTING.md).
|
||||
|
||||
[](https://discord.gg/uJN7TYcpNa)
|
||||
[](https://www.linkedin.com/company/scrapegraphai/)
|
||||
[](https://twitter.com/scrapegraphai)
|
||||
|
||||
## 🔗 API et SDK ScrapeGraph
|
||||
Si vous cherchez une solution rapide pour intégrer ScrapeGraph dans votre système, consultez notre puissante API [ici !](https://dashboard.scrapegraphai.com/login)
|
||||
|
||||
[](https://dashboard.scrapegraphai.com/login)
|
||||
|
||||
Nous proposons des SDK en Python et Node.js, ce qui facilite leur intégration dans vos projets. Découvrez-les ci-dessous :
|
||||
|
||||
| SDK | Langage | Lien GitHub |
|
||||
|-----------|----------|-----------------------------------------------------------------------------|
|
||||
| Python SDK | Python | [scrapegraph-py](https://docs.scrapegraphai.com/sdks/python) |
|
||||
| Node.js SDK | Node.js | [scrapegraph-js](https://docs.scrapegraphai.com/sdks/javascript) |
|
||||
|
||||
La documentation officielle de l'API est disponible [ici](https://docs.scrapegraphai.com/introduction).
|
||||
|
||||
## 🔥 Benchmark
|
||||
|
||||
Selon le benchmark Firecrawl [Firecrawl benchmark](https://scrapegraphai.com/compare/firecrawl), ScrapeGraph est le meilleur scraper sur le marché !
|
||||
|
||||
|
||||
|
||||
## 📈 Télémétrie
|
||||
Nous recueillons des mesures d'utilisation anonymes pour améliorer la qualité de notre package et l'expérience utilisateur. Les données nous aident à hiérarchiser les améliorations et à assurer la compatibilité. Si vous souhaitez vous désinscrire, définissez la variable d'environnement SCRAPEGRAPHAI_TELEMETRY_ENABLED=false. Pour plus d'informations, veuillez consulter la documentation [ici](https://docs.scrapegraphai.com/introduction).
|
||||
|
||||
## ❤️ Contributeurs
|
||||
[](https://github.com/ScrapeGraphAI/Scrapegraph-ai/graphs/contributors)
|
||||
|
||||
## 🎓 Citations
|
||||
Si vous avez utilisé notre bibliothèque à des fins de recherche, veuillez nous citer avec la référence suivante :
|
||||
```text
|
||||
@misc{scrapegraph-ai,
|
||||
author = {Lorenzo Padoan, Marco Vinciguerra},
|
||||
title = {Scrapegraph-ai},
|
||||
year = {2024},
|
||||
url = {https://github.com/ScrapeGraphAI/Scrapegraph-ai},
|
||||
note = {A Python library for scraping leveraging large language models}
|
||||
}
|
||||
```
|
||||
## Auteurs
|
||||
|
||||
| | Info Contact |
|
||||
|--------------------|----------------------|
|
||||
| Marco Vinciguerra | [](https://www.linkedin.com/in/marco-vinciguerra-7ba365242/) |
|
||||
| Lorenzo Padoan | [](https://www.linkedin.com/in/lorenzo-padoan-4521a2154/) |
|
||||
|
||||
## 📜 Licence
|
||||
|
||||
ScrapeGraphAI est sous licence MIT. Consultez le fichier [LICENSE](https://github.com/ScrapeGraphAI/Scrapegraph-ai/blob/main/LICENSE) pour plus d'informations.
|
||||
|
||||
## Remerciements
|
||||
|
||||
- Nous tenons à remercier tous les contributeurs au projet et la communauté open-source pour leur soutien.
|
||||
- ScrapeGraphAI est destiné uniquement à l'exploration de données et à la recherche. Nous ne sommes pas responsables de toute mauvaise utilisation de la bibliothèque.
|
||||
|
||||
Fait avec ❤️ par [ScrapeGraph AI](https://scrapegraphai.com)
|
||||
|
||||
[Scarf tracking](https://static.scarf.sh/a.png?x-pxid=102d4b8c-cd6a-4b9e-9a16-d6d141b9212d)
|
||||
+228
@@ -0,0 +1,228 @@
|
||||
## 🚀 **Suchen Sie nach einem noch schnelleren und einfacheren Weg, um im großen Maßstab zu scrapen (nur 5 Codezeilen)?** Schauen Sie sich unsere erweiterte Version auf [**ScrapeGraphAI.com**](https://scrapegraphai.com/?utm_source=github&utm_medium=readme&utm_campaign=oss_cta&utm_content=top_banner) an! 🚀
|
||||
|
||||
---
|
||||
|
||||
# 🕷️ ScrapeGraphAI: You Only Scrape Once
|
||||
|
||||
<p align="center">
|
||||
<a href="https://scrapegraphai.com">
|
||||
<img src="https://raw.githubusercontent.com/ScrapeGraphAI/Scrapegraph-ai/main/media/banner.png" alt="ScrapeGraphAI" style="width: 100%;">
|
||||
</a>
|
||||
</p>
|
||||
|
||||
[English](../README.md) | [中文](chinese.md) | [日本語](japanese.md)
|
||||
| [한국어](korean.md)
|
||||
| [Русский](russian.md) | [Türkçe](turkish.md)
|
||||
| [Deutsch](german.md)
|
||||
| [Español](spanish.md)
|
||||
| [français](french.md)
|
||||
| [Português](portuguese.md)
|
||||
| [Italiano](italian.md)
|
||||
|
||||
[](https://pepy.tech/projects/scrapegraphai)
|
||||
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
[](https://discord.gg/gkxQDAjfeX)
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/15078" target="_blank"><img src="https://trendshift.io/api/badge/repositories/15078" alt="ScrapeGraphAI%2FScrapegraph-ai | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
<p align="center">
|
||||
|
||||
[ScrapeGraphAI](https://scrapegraphai.com) ist eine Python-Bibliothek für *Web-Scraping*, die LLM- und direkte Graphenlogik verwendet, um Scraping-Pipelines für Websites und lokale Dokumente (XML, HTML, JSON, Markdown usw.) zu erstellen.
|
||||
|
||||
Sagen Sie einfach, welche Informationen Sie extrahieren möchten, und die Bibliothek erledigt das für Sie!
|
||||
|
||||
<p align="center">
|
||||
<img src="https://raw.githubusercontent.com/ScrapeGraphAI/Scrapegraph-ai/main/docs/assets/sgai-hero.png" alt="ScrapeGraphAI Hero" style="width: 100%;">
|
||||
</p>
|
||||
|
||||
## 🚀 Integrationen
|
||||
|
||||
ScrapeGraphAI bietet eine nahtlose Integration mit gängigen Frameworks und Tools, um Ihre Scraping-Fähigkeiten zu erweitern. Egal, ob Sie mit Python oder Node.js entwickeln, LLM-Frameworks verwenden oder mit No-Code-Plattformen arbeiten, wir decken alles mit unseren umfassenden Integrationsmöglichkeiten ab.
|
||||
|
||||
Weitere Informationen finden Sie unter folgendem [Link](https://scrapegraphai.com)
|
||||
|
||||
**Integrationen**:
|
||||
- **API**: [Dokumentation](https://docs.scrapegraphai.com/introduction)
|
||||
- **SDKs**: [Python](https://docs.scrapegraphai.com/sdks/python), [Node](https://docs.scrapegraphai.com/sdks/javascript)
|
||||
- **LLM-Frameworks**: [Langchain](https://docs.scrapegraphai.com/integrations/langchain), [Llama Index](https://docs.scrapegraphai.com/integrations/llamaindex), [Crew.ai](https://docs.scrapegraphai.com/integrations/crewai), [Agno](https://docs.scrapegraphai.com/integrations/agno), [CamelAI](https://github.com/camel-ai/camel)
|
||||
- **Low-Code-Frameworks**: [Pipedream](https://pipedream.com/apps/scrapegraphai), [Bubble](https://bubble.io/plugin/scrapegraphai-1745408893195x213542371433906180), [Zapier](https://zapier.com/apps/scrapegraphai/integrations), [n8n](http://localhost:5001/dashboard), [Dify](https://dify.ai), [Toolhouse](https://app.toolhouse.ai/mcp-servers/scrapegraph_smartscraper)
|
||||
- **MCP-Server**: [Link](https://smithery.ai/server/@ScrapeGraphAI/scrapegraph-mcp)
|
||||
|
||||
## 🚀 Schnelle Installation
|
||||
|
||||
Die Referenzseite für Scrapegraph-ai finden Sie auf der offiziellen PyPI-Seite: [pypi](https://pypi.org/project/scrapegraphai/).
|
||||
|
||||
```bash
|
||||
pip install scrapegraphai
|
||||
|
||||
# WICHTIG (für das Abrufen von Website-Inhalten)
|
||||
playwright install
|
||||
```
|
||||
|
||||
**Hinweis**: Es wird empfohlen, die Bibliothek in einer virtuellen Umgebung zu installieren, um Konflikte mit anderen Bibliotheken zu vermeiden 🐱
|
||||
|
||||
## 💻 Verwendung
|
||||
Es gibt mehrere Standard-Scraping-Pipelines, mit denen Informationen aus einer Website (oder lokalen Datei) extrahiert werden können.
|
||||
|
||||
Die häufigste ist der `SmartScraperGraph`, der anhand einer Benutzereingabe und einer Quell-URL Informationen aus einer einzelnen Seite extrahiert.
|
||||
|
||||
```python
|
||||
from scrapegraphai.graphs import SmartScraperGraph
|
||||
|
||||
# Definieren Sie die Konfiguration für die Scraping-Pipeline
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"model": "ollama/llama3.2",
|
||||
"model_tokens": 8192,
|
||||
"format": "json",
|
||||
},
|
||||
"verbose": True,
|
||||
"headless": False,
|
||||
}
|
||||
|
||||
# Erstellen Sie die SmartScraperGraph-Instanz
|
||||
smart_scraper_graph = SmartScraperGraph(
|
||||
prompt="Extract useful information from the webpage, including a description of what the company does, founders and social media links",
|
||||
source="https://scrapegraphai.com/",
|
||||
config=graph_config
|
||||
)
|
||||
|
||||
# Führen Sie die Pipeline aus
|
||||
result = smart_scraper_graph.run()
|
||||
|
||||
import json
|
||||
print(json.dumps(result, indent=4))
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
> Für OpenAI und andere Modelle müssen Sie lediglich die LLM-Konfiguration ändern!
|
||||
> ```python
|
||||
>graph_config = {
|
||||
> "llm": {
|
||||
> "api_key": "YOUR_OPENAI_API_KEY",
|
||||
> "model": "openai/gpt-4o-mini",
|
||||
> },
|
||||
> "verbose": True,
|
||||
> "headless": False,
|
||||
>}
|
||||
>```
|
||||
|
||||
Die Ausgabe ist ein Wörterbuch wie folgt:
|
||||
|
||||
```python
|
||||
{
|
||||
"description": "ScrapeGraphAI transforms websites into clean, organized data for AI agents and data analytics. It offers an AI-powered API for effortless and cost-effective data extraction.",
|
||||
"founders": [
|
||||
{
|
||||
"name": "",
|
||||
"role": "Founder & Technical Lead",
|
||||
"linkedin": "https://www.linkedin.com/in/perinim/"
|
||||
},
|
||||
{
|
||||
"name": "Marco Vinciguerra",
|
||||
"role": "Founder & Software Engineer",
|
||||
"linkedin": "https://www.linkedin.com/in/marco-vinciguerra-7ba365242/"
|
||||
},
|
||||
{
|
||||
"name": "Lorenzo Padoan",
|
||||
"role": "Founder & Product Engineer",
|
||||
"linkedin": "https://www.linkedin.com/in/lorenzo-padoan-4521a2154/"
|
||||
}
|
||||
],
|
||||
"social_media_links": {
|
||||
"linkedin": "https://www.linkedin.com/company/101881123",
|
||||
"twitter": "https://x.com/scrapegraphai",
|
||||
"github": "https://github.com/ScrapeGraphAI/Scrapegraph-ai"
|
||||
}
|
||||
}
|
||||
```
|
||||
Es gibt weitere Pipelines, mit denen Informationen aus mehreren Seiten extrahiert, Python-Skripte generiert oder sogar Audiodateien generiert werden können.
|
||||
|
||||
| Pipeline Name | Beschreibung |
|
||||
|-------------------------|-------------------------------------------------------------------------------------------------------------------|
|
||||
| SmartScraperGraph | Single-Page-Scraper, der nur eine Benutzereingabe und eine Eingabequelle benötigt. |
|
||||
| SearchGraph | Multi-Page-Scraper, der Informationen aus den oberen n Suchergebnissen einer Suchmaschine extrahiert. |
|
||||
| SpeechGraph | Single-Page-Scraper, der Informationen aus einer Website extrahiert und eine Audiodatei generiert. |
|
||||
| ScriptCreatorGraph | Single-Page-Scraper, der Informationen aus einer Website extrahiert und ein Python-Skript generiert. |
|
||||
| SmartScraperMultiGraph | Multi-Page-Scraper, der Informationen aus mehreren Seiten mit einer Eingabe und einer Liste von Quellen extrahiert.|
|
||||
| ScriptCreatorMultiGraph | Multi-Page-Scraper, der ein Python-Skript zum Extrahieren von Informationen aus mehreren Seiten und Quellen generiert.|
|
||||
|
||||
Für jeden dieser Graphen gibt es die Multi-Version. Sie ermöglicht parallele Aufrufe des LLM.
|
||||
|
||||
Es ist möglich, verschiedene LLMs über APIs wie **OpenAI**, **Groq**, **Azure**, **Gemini**, **MiniMax** und andere zu verwenden, oder lokale Modelle über **Ollama**.
|
||||
|
||||
Denken Sie daran, [Ollama](https://ollama.com/) installiert zu haben und die Modelle mit dem Befehl **ollama pull** herunterzuladen, wenn Sie lokale Modelle verwenden möchten.
|
||||
|
||||
## 📖 Dokumentation
|
||||
|
||||
[](https://colab.research.google.com/drive/1sEZBonBMGP44CtO6GQTwAlL0BGJXjtfd?usp=sharing)
|
||||
|
||||
Die Dokumentation zu ScrapeGraphAI finden Sie [hier](https://docs.scrapegraphai.com/introduction).
|
||||
|
||||
## 🤝 Mitwirken
|
||||
|
||||
Fühlen Sie sich frei, beizutragen und treten Sie unserem Discord-Server bei, um mit uns über Verbesserungen zu diskutieren und uns Vorschläge zu machen!
|
||||
|
||||
Bitte lesen Sie die [Richtlinien für Mitwirkende](https://github.com/ScrapeGraphAI/Scrapegraph-ai/blob/main/CONTRIBUTING.md).
|
||||
|
||||
[](https://discord.gg/uJN7TYcpNa)
|
||||
[](https://www.linkedin.com/company/scrapegraphai/)
|
||||
[](https://twitter.com/scrapegraphai)
|
||||
|
||||
## 🔗 ScrapeGraph API & SDKs
|
||||
Wenn Sie nach einer schnellen Lösung zur Integration von ScrapeGraph in Ihr System suchen, sehen Sie sich unsere leistungsstarke API [hier!](https://dashboard.scrapegraphai.com/login) an.
|
||||
|
||||
[](https://dashboard.scrapegraphai.com/login)
|
||||
|
||||
Wir bieten SDKs für Python und Node.js an, die eine einfache Integration in Ihre Projekte ermöglichen. Sehen Sie sie sich unten an:
|
||||
|
||||
| SDK | Sprache | GitHub Link |
|
||||
|-----------|----------|-----------------------------------------------------------------------------|
|
||||
| Python SDK | Python | [scrapegraph-py](https://docs.scrapegraphai.com/sdks/python) |
|
||||
| Node.js SDK | Node.js | [scrapegraph-js](https://docs.scrapegraphai.com/sdks/javascript) |
|
||||
|
||||
Die offizielle API-Dokumentation finden Sie [hier](https://docs.scrapegraphai.com/introduction).
|
||||
|
||||
## 🔥 Benchmark
|
||||
|
||||
Laut dem Benchmark von Firecrawl [Firecrawl benchmark](https://scrapegraphai.com/compare/firecrawl) ist ScrapeGraph der beste Fetcher auf dem Markt!
|
||||
|
||||
|
||||
|
||||
## 📈 Telemetrie
|
||||
Wir erfassen anonyme Nutzungsmetriken, um die Qualität unseres Pakets und das Benutzererlebnis zu verbessern. Die Daten helfen uns, Verbesserungen zu priorisieren und die Kompatibilität sicherzustellen. Wenn Sie dies nicht möchten, setzen Sie die Umgebungsvariable SCRAPEGRAPHAI_TELEMETRY_ENABLED=false. Weitere Informationen finden Sie in der Dokumentation [hier](https://docs.scrapegraphai.com/introduction).
|
||||
|
||||
## ❤️ Mitwirkende
|
||||
[](https://github.com/ScrapeGraphAI/Scrapegraph-ai/graphs/contributors)
|
||||
|
||||
## 🎓 Zitate
|
||||
Wenn Sie unsere Bibliothek für Forschungszwecke verwendet haben, zitieren Sie uns bitte mit folgendem Hinweis:
|
||||
```text
|
||||
@misc{scrapegraph-ai,
|
||||
author = {Lorenzo Padoan, Marco Vinciguerra},
|
||||
title = {Scrapegraph-ai},
|
||||
year = {2024},
|
||||
url = {https://github.com/ScrapeGraphAI/Scrapegraph-ai},
|
||||
note = {A Python library for scraping leveraging large language models}
|
||||
}
|
||||
```
|
||||
## Autoren
|
||||
|
||||
| | Kontakt Info |
|
||||
|--------------------|----------------------|
|
||||
| Marco Vinciguerra | [](https://www.linkedin.com/in/marco-vinciguerra-7ba365242/) |
|
||||
| Lorenzo Padoan | [](https://www.linkedin.com/in/lorenzo-padoan-4521a2154/) |
|
||||
|
||||
## 📜 Lizenz
|
||||
|
||||
ScrapeGraphAI ist unter der MIT-Lizenz lizenziert. Weitere Informationen finden Sie in der Datei [LICENSE](https://github.com/ScrapeGraphAI/Scrapegraph-ai/blob/main/LICENSE).
|
||||
|
||||
## Danksagungen
|
||||
|
||||
- Wir möchten allen Mitwirkenden am Projekt und der Open-Source-Community für ihre Unterstützung danken.
|
||||
- ScrapeGraphAI ist nur für Datenexploration und Forschungszwecke vorgesehen. Wir sind nicht verantwortlich für einen Missbrauch der Bibliothek.
|
||||
|
||||
Gemacht mit ❤️ von [ScrapeGraph AI](https://scrapegraphai.com)
|
||||
|
||||
[Scarf tracking](https://static.scarf.sh/a.png?x-pxid=102d4b8c-cd6a-4b9e-9a16-d6d141b9212d)
|
||||
+236
@@ -0,0 +1,236 @@
|
||||
## 🚀 **Cerchi un modo ancora più veloce e semplice per fare scraping su larga scala (con sole 5 righe di codice)?** Scopri la nostra versione potenziata su [**ScrapeGraphAI.com**](https://scrapegraphai.com/?utm_source=github&utm_medium=readme&utm_campaign=oss_cta&utm_content=top_banner)! 🚀
|
||||
|
||||
---
|
||||
|
||||
# 🕷️ ScrapeGraphAI: You Only Scrape Once
|
||||
|
||||
<p align="center">
|
||||
<a href="https://scrapegraphai.com">
|
||||
<img src="https://raw.githubusercontent.com/ScrapeGraphAI/Scrapegraph-ai/main/media/banner.png" alt="ScrapeGraphAI" style="width: 100%;">
|
||||
</a>
|
||||
</p>
|
||||
|
||||
[English](../README.md) | [中文](chinese.md) | [日本語](japanese.md)
|
||||
| [한국어](korean.md)
|
||||
| [Русский](russian.md) | [Türkçe](turkish.md)
|
||||
| [Deutsch](german.md)
|
||||
| [Español](spanish.md)
|
||||
| [français](french.md)
|
||||
| [Português](portuguese.md)
|
||||
| [Italiano](italian.md)
|
||||
|
||||
[](https://pepy.tech/projects/scrapegraphai)
|
||||
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
[](https://discord.gg/gkxQDAjfeX)
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/15078" target="_blank"><img src="https://trendshift.io/api/badge/repositories/15078" alt="ScrapeGraphAI%2FScrapegraph-ai | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
<p align="center">
|
||||
|
||||
[ScrapeGraphAI](https://scrapegraphai.com) è una libreria Python per il *web scraping* che utilizza LLM e logica basata sui grafi per creare pipeline di scraping per siti web e documenti locali (XML, HTML, JSON, Markdown, ecc.).
|
||||
|
||||
Indica semplicemente quali informazioni vuoi estrarre e la libreria lo farà per te!
|
||||
<p align="center">
|
||||
<img src="https://raw.githubusercontent.com/ScrapeGraphAI/Scrapegraph-ai/main/docs/assets/sgai-hero.png" alt="ScrapeGraphAI Hero" style="width: 100%;">
|
||||
</p>
|
||||
|
||||
|
||||
## 🚀 Integrazioni
|
||||
|
||||
ScrapeGraphAI offre integrazioni con i framework e gli strumenti più diffusi per potenziare le tue capacità di scraping. Che tu stia sviluppando in Python o Node.js, usando framework LLM o piattaforme no-code, offriamo un'ampia gamma di opzioni di integrazione.
|
||||
|
||||
Puoi trovare ulteriori informazioni al seguente [link](https://scrapegraphai.com)
|
||||
|
||||
**Integrazioni**:
|
||||
- **API**: [Documentazione](https://docs.scrapegraphai.com/introduction)
|
||||
- **SDK**: [Python](https://docs.scrapegraphai.com/sdks/python), [Node](https://docs.scrapegraphai.com/sdks/javascript)
|
||||
- **Framework LLM**: [Langchain](https://docs.scrapegraphai.com/integrations/langchain), [Llama Index](https://docs.scrapegraphai.com/integrations/llamaindex), [Crew.ai](https://docs.scrapegraphai.com/integrations/crewai), [Agno](https://docs.scrapegraphai.com/integrations/agno), [CamelAI](https://github.com/camel-ai/camel)
|
||||
- **Framework Low-code**: [Pipedream](https://pipedream.com/apps/scrapegraphai), [Bubble](https://bubble.io/plugin/scrapegraphai-1745408893195x213542371433906180), [Zapier](https://zapier.com/apps/scrapegraphai/integrations), [n8n](http://localhost:5001/dashboard), [Dify](https://dify.ai), [Toolhouse](https://app.toolhouse.ai/mcp-servers/scrapegraph_smartscraper)
|
||||
- **Server MCP**: [Link](https://smithery.ai/server/@ScrapeGraphAI/scrapegraph-mcp)
|
||||
|
||||
## 🚀 Installazione rapida
|
||||
|
||||
La pagina di riferimento per scrapegraph-ai è disponibile sulla pagina ufficiale di PyPI: [pypi](https://pypi.org/project/scrapegraphai/).
|
||||
|
||||
```bash
|
||||
pip install scrapegraphai
|
||||
|
||||
# IMPORTANTE (per il recupero del contenuto dei siti web)
|
||||
playwright install
|
||||
```
|
||||
|
||||
**Nota**: si consiglia di installare la libreria in un ambiente virtuale per evitare conflitti con altre librerie 🐱
|
||||
|
||||
## 💻 Utilizzo
|
||||
|
||||
Esistono diverse pipeline di scraping predefinite che possono essere utilizzate per estrarre informazioni da un sito web (o da un file locale).
|
||||
|
||||
La più comune è `SmartScraperGraph`, che estrae informazioni da una singola pagina dato un prompt dell'utente e un URL sorgente.
|
||||
|
||||
```python
|
||||
from scrapegraphai.graphs import SmartScraperGraph
|
||||
|
||||
# Definisci la configurazione per la pipeline di scraping
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"model": "ollama/llama3.2",
|
||||
"model_tokens": 8192,
|
||||
"format": "json",
|
||||
},
|
||||
"verbose": True,
|
||||
"headless": False,
|
||||
}
|
||||
|
||||
# Crea l'istanza di SmartScraperGraph
|
||||
smart_scraper_graph = SmartScraperGraph(
|
||||
prompt="Estrai informazioni utili dalla pagina web, inclusa una descrizione di cosa fa l'azienda, i fondatori e i link ai social media",
|
||||
source="https://scrapegraphai.com/",
|
||||
config=graph_config
|
||||
)
|
||||
|
||||
# Esegui la pipeline
|
||||
result = smart_scraper_graph.run()
|
||||
|
||||
import json
|
||||
print(json.dumps(result, indent=4))
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
> Per OpenAI e altri modelli è sufficiente modificare la configurazione llm!
|
||||
> ```python
|
||||
> graph_config = {
|
||||
> "llm": {
|
||||
> "api_key": "LA_TUA_OPENAI_API_KEY",
|
||||
> "model": "openai/gpt-4o-mini",
|
||||
> },
|
||||
> "verbose": True,
|
||||
> "headless": False,
|
||||
> }
|
||||
> ```
|
||||
|
||||
L'output sarà un dizionario simile al seguente:
|
||||
|
||||
```python
|
||||
{
|
||||
"description": "ScrapeGraphAI transforms websites into clean, organized data for AI agents and data analytics. It offers an AI-powered API for effortless and cost-effective data extraction.",
|
||||
"founders": [
|
||||
{
|
||||
"name": "",
|
||||
"role": "Founder & Technical Lead",
|
||||
"linkedin": "https://www.linkedin.com/in/perinim/"
|
||||
},
|
||||
{
|
||||
"name": "Marco Vinciguerra",
|
||||
"role": "Founder & Software Engineer",
|
||||
"linkedin": "https://www.linkedin.com/in/marco-vinciguerra-7ba365242/"
|
||||
},
|
||||
{
|
||||
"name": "Lorenzo Padoan",
|
||||
"role": "Founder & Product Engineer",
|
||||
"linkedin": "https://www.linkedin.com/in/lorenzo-padoan-4521a2154/"
|
||||
}
|
||||
],
|
||||
"social_media_links": {
|
||||
"linkedin": "https://www.linkedin.com/company/101881123",
|
||||
"twitter": "https://x.com/scrapegraphai",
|
||||
"github": "https://github.com/ScrapeGraphAI/Scrapegraph-ai"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Esistono altre pipeline che possono essere utilizzate per estrarre informazioni da più pagine, generare script Python o persino generare file audio.
|
||||
|
||||
| Nome Pipeline | Descrizione |
|
||||
|-------------------------|------------------------------------------------------------------------------------------------------------------|
|
||||
| SmartScraperGraph | Scraper di singole pagine che richiede solo un prompt utente e una sorgente. |
|
||||
| SearchGraph | Scraper multi-pagina che estrae informazioni dai primi n risultati di un motore di ricerca. |
|
||||
| SpeechGraph | Scraper di singole pagine che estrae informazioni da un sito web e genera un file audio. |
|
||||
| ScriptCreatorGraph | Scraper di singole pagine che estrae informazioni da un sito web e genera uno script Python. |
|
||||
| SmartScraperMultiGraph | Scraper multi-pagina che estrae informazioni da più pagine dato un singolo prompt e una lista di sorgenti. |
|
||||
| ScriptCreatorMultiGraph | Scraper multi-pagina che genera uno script Python per estrarre informazioni da più pagine e sorgenti. |
|
||||
|
||||
Per ciascuno di questi grafi esiste una versione multi, che consente di effettuare chiamate all'LLM in parallelo.
|
||||
|
||||
È possibile utilizzare diversi LLM tramite API, come **OpenAI**, **Groq**, **Azure**, **Gemini**, **MiniMax** e altri, oppure modelli locali tramite **Ollama**.
|
||||
|
||||
Ricordati di avere [Ollama](https://ollama.com/) installato e di scaricare i modelli con il comando **ollama pull**, se desideri utilizzare modelli locali.
|
||||
|
||||
## 📖 Documentazione
|
||||
|
||||
[](https://colab.research.google.com/drive/1sEZBonBMGP44CtO6GQTwAlL0BGJXjtfd?usp=sharing)
|
||||
|
||||
La documentazione di ScrapeGraphAI è disponibile [qui](https://scrapegraph-ai.readthedocs.io/en/latest/).
|
||||
Consulta anche il Docusaurus [qui](https://docs-oss.scrapegraphai.com/).
|
||||
|
||||
## 🤝 Vuoi contribuire?
|
||||
|
||||
Sentiti libero di contribuire e unisciti al nostro server Discord per discutere con noi su cosa migliorare e darci suggerimenti!
|
||||
|
||||
Consulta le [linee guida per i contributi](https://github.com/ScrapeGraphAI/Scrapegraph-ai/blob/main/CONTRIBUTING.md).
|
||||
|
||||
[](https://discord.gg/uJN7TYcpNa)
|
||||
[](https://www.linkedin.com/company/scrapegraphai/)
|
||||
[](https://twitter.com/scrapegraphai)
|
||||
|
||||
## 🔗 API e SDK di ScrapeGraph
|
||||
|
||||
Se stai cercando una soluzione rapida per integrare ScrapeGraph nel tuo sistema, scopri la nostra potente API [qui!](https://dashboard.scrapegraphai.com/login)
|
||||
|
||||
[](https://dashboard.scrapegraphai.com/login)
|
||||
|
||||
Offriamo gli SDK sia in Python che in Node.js, per una facile integrazione nei tuoi progetti. Scoprili di seguito:
|
||||
|
||||
| SDK | Linguaggio | Link GitHub |
|
||||
|------------|------------|-----------------------------------------------------------------------------|
|
||||
| Python SDK | Python | [scrapegraph-py](https://docs.scrapegraphai.com/sdks/python) |
|
||||
| Node.js SDK | Node.js | [scrapegraph-js](https://docs.scrapegraphai.com/sdks/javascript) |
|
||||
|
||||
La documentazione ufficiale dell'API è disponibile [qui](https://docs.scrapegraphai.com/).
|
||||
|
||||
## 🔥 Benchmark
|
||||
|
||||
Secondo il benchmark di Firecrawl [Firecrawl benchmark](https://scrapegraphai.com/compare/firecrawl), ScrapeGraph è il miglior fetcher sul mercato!
|
||||
|
||||
|
||||
|
||||
## 📈 Telemetria
|
||||
|
||||
Raccogliamo metriche di utilizzo anonimizzate per migliorare la qualità e la user experience del nostro pacchetto. I dati ci aiutano a stabilire le priorità e a garantire la compatibilità. Se desideri disattivare la telemetria, imposta la variabile d'ambiente `SCRAPEGRAPHAI_TELEMETRY_ENABLED=false`. Per ulteriori informazioni, consulta la documentazione [qui](https://scrapegraph-ai.readthedocs.io/en/latest/scrapers/telemetry.html).
|
||||
|
||||
## ❤️ Collaboratori
|
||||
|
||||
[](https://github.com/ScrapeGraphAI/Scrapegraph-ai/graphs/contributors)
|
||||
|
||||
## 🎓 Citazioni
|
||||
|
||||
Se hai utilizzato la nostra libreria per scopi di ricerca, citaci con il seguente riferimento:
|
||||
|
||||
```text
|
||||
@misc{scrapegraph-ai,
|
||||
author = {Lorenzo Padoan, Marco Vinciguerra},
|
||||
title = {Scrapegraph-ai},
|
||||
year = {2024},
|
||||
url = {https://github.com/ScrapeGraphAI/Scrapegraph-ai},
|
||||
note = {A Python library for scraping leveraging large language models}
|
||||
}
|
||||
```
|
||||
|
||||
## Autori
|
||||
|
||||
| | Contatti |
|
||||
|--------------------|----------------------|
|
||||
| Marco Vinciguerra | [](https://www.linkedin.com/in/marco-vinciguerra-7ba365242/) |
|
||||
| Lorenzo Padoan | [](https://www.linkedin.com/in/lorenzo-padoan-4521a2154/) |
|
||||
|
||||
## 📜 Licenza
|
||||
|
||||
ScrapeGraphAI è rilasciato sotto la Licenza MIT. Consulta il file [LICENSE](https://github.com/ScrapeGraphAI/Scrapegraph-ai/blob/main/LICENSE) per ulteriori informazioni.
|
||||
|
||||
## Ringraziamenti
|
||||
|
||||
- Ringraziamo tutti i collaboratori del progetto e la comunità open-source per il loro supporto.
|
||||
- ScrapeGraphAI è destinato esclusivamente a scopi di esplorazione dei dati e ricerca. Non siamo responsabili per eventuali usi impropri della libreria.
|
||||
|
||||
Fatto con il ❤️ da [ScrapeGraph AI](https://scrapegraphai.com)
|
||||
|
||||
[Scarf tracking](https://static.scarf.sh/a.png?x-pxid=102d4b8c-cd6a-4b9e-9a16-d6d141b9212d)
|
||||
@@ -0,0 +1,226 @@
|
||||
## 🚀 **さらに高速でシンプルな大規模スクレイピング方法（わずか5行のコード）をお探しですか？** [**ScrapeGraphAI.com**](https://scrapegraphai.com/?utm_source=github&utm_medium=readme&utm_campaign=oss_cta&utm_content=top_banner) の拡張版をご覧ください！🚀
|
||||
|
||||
---
|
||||
|
||||
# 🕷️ ScrapeGraphAI: 一度のクロールで完結
|
||||
|
||||
[English](../README.md) | [中文](chinese.md) | [日本語](japanese.md)
|
||||
| [한국어](korean.md)
|
||||
| [Русский](russian.md) | [Türkçe](turkish.md)
|
||||
| [Deutsch](german.md)
|
||||
| [Español](spanish.md)
|
||||
| [français](french.md)
|
||||
| [Português](portuguese.md)
|
||||
| [Italiano](italian.md)
|
||||
|
||||
[](https://pepy.tech/projects/scrapegraphai)
|
||||
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
[](https://discord.gg/gkxQDAjfeX)
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/15078" target="_blank"><img src="https://trendshift.io/api/badge/repositories/15078" alt="ScrapeGraphAI%2FScrapegraph-ai | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
<p align="center">
|
||||
|
||||
ScrapeGraphAIは、大規模言語モデルと直接グラフロジックを使用して、ウェブサイトやローカルドキュメント（XML、HTML、JSON、Markdownなど）のクローリングパイプラインを作成するPythonライブラリです。
|
||||
|
||||
クロールしたい情報をライブラリに伝えるだけで、残りはすべてライブラリが行います！
|
||||
|
||||
|
||||
|
||||
## 🚀 統合
|
||||
|
||||
ScrapeGraphAIは、人気のあるフレームワークやツールとのシームレスな統合を提供し、スクレイピング機能を強化します。PythonまたはNode.jsで構築する場合でも、LLMフレームワークを使用する場合でも、ノーコードプラットフォームで作業する場合でも、包括的な統合オプションを提供しています。
|
||||
|
||||
<p align="center">
|
||||
<img src="https://raw.githubusercontent.com/ScrapeGraphAI/Scrapegraph-ai/main/docs/assets/sgai-hero.png" alt="ScrapeGraphAI Hero" style="width: 100%;">
|
||||
</p>
|
||||
詳細情報は以下の[リンク](https://scrapegraphai.com)で確認できます
|
||||
|
||||
**統合**：
|
||||
- **API**: [ドキュメント](https://docs.scrapegraphai.com/introduction)
|
||||
- **SDKs**: [Python](https://docs.scrapegraphai.com/sdks/python), [Node](https://docs.scrapegraphai.com/sdks/javascript)
|
||||
- **LLMフレームワーク**: [Langchain](https://docs.scrapegraphai.com/integrations/langchain), [Llama Index](https://docs.scrapegraphai.com/integrations/llamaindex), [Crew.ai](https://docs.scrapegraphai.com/integrations/crewai), [Agno](https://docs.scrapegraphai.com/integrations/agno), [CamelAI](https://github.com/camel-ai/camel)
|
||||
- **ローコードフレームワーク**: [Pipedream](https://pipedream.com/apps/scrapegraphai), [Bubble](https://bubble.io/plugin/scrapegraphai-1745408893195x213542371433906180), [Zapier](https://zapier.com/apps/scrapegraphai/integrations), [n8n](http://localhost:5001/dashboard), [Dify](https://dify.ai), [Toolhouse](https://app.toolhouse.ai/mcp-servers/scrapegraph_smartscraper)
|
||||
- **MCPサーバー**: [リンク](https://smithery.ai/server/@ScrapeGraphAI/scrapegraph-mcp)
|
||||
|
||||
## 🚀 クイックインストール
|
||||
|
||||
Scrapegraph-aiの参照ページはPyPIの公式サイトで見ることができます: [pypi](https://pypi.org/project/scrapegraphai/)。
|
||||
|
||||
```bash
|
||||
pip install scrapegraphai
|
||||
|
||||
# 重要（ウェブサイトコンテンツの取得用）
|
||||
playwright install
|
||||
```
|
||||
|
||||
**注意**: 他のライブラリとの競合を避けるため、このライブラリは仮想環境でのインストールを推奨します 🐱
|
||||
|
||||
|
||||
## 💻 使い方
|
||||
ウェブサイト（またはローカルファイル）から情報を抽出するために使用できる複数の標準スクレイピングパイプラインがあります。
|
||||
|
||||
最も一般的なのは `SmartScraperGraph` で、ユーザープロンプトとソースURLが与えられた場合に単一ページから情報を抽出します。
|
||||
|
||||
|
||||
```python
|
||||
from scrapegraphai.graphs import SmartScraperGraph
|
||||
|
||||
# スクレイピングパイプラインの設定を定義
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"model": "ollama/llama3.2",
|
||||
"model_tokens": 8192,
|
||||
"format": "json",
|
||||
},
|
||||
"verbose": True,
|
||||
"headless": False,
|
||||
}
|
||||
|
||||
# SmartScraperGraphインスタンスを作成
|
||||
smart_scraper_graph = SmartScraperGraph(
|
||||
prompt="ウェブページから有用な情報を抽出してください。会社の説明、創設者、ソーシャルメディアリンクを含めてください",
|
||||
source="https://scrapegraphai.com/",
|
||||
config=graph_config
|
||||
)
|
||||
|
||||
# パイプラインを実行
|
||||
result = smart_scraper_graph.run()
|
||||
|
||||
import json
|
||||
print(json.dumps(result, indent=4))
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
> OpenAIやその他のモデルの場合は、llm設定を変更するだけです！
|
||||
> ```python
|
||||
>graph_config = {
|
||||
> "llm": {
|
||||
> "api_key": "YOUR_OPENAI_API_KEY",
|
||||
> "model": "openai/gpt-4o-mini",
|
||||
> },
|
||||
> "verbose": True,
|
||||
> "headless": False,
|
||||
>}
|
||||
>```
|
||||
|
||||
|
||||
出力は次のような辞書になります：
|
||||
|
||||
```python
|
||||
{
|
||||
"description": "ScrapeGraphAI transforms websites into clean, organized data for AI agents and data analytics. It offers an AI-powered API for effortless and cost-effective data extraction.",
|
||||
"founders": [
|
||||
{
|
||||
"name": "",
|
||||
"role": "Founder & Technical Lead",
|
||||
"linkedin": "https://www.linkedin.com/in/perinim/"
|
||||
},
|
||||
{
|
||||
"name": "Marco Vinciguerra",
|
||||
"role": "Founder & Software Engineer",
|
||||
"linkedin": "https://www.linkedin.com/in/marco-vinciguerra-7ba365242/"
|
||||
},
|
||||
{
|
||||
"name": "Lorenzo Padoan",
|
||||
"role": "Founder & Product Engineer",
|
||||
"linkedin": "https://www.linkedin.com/in/lorenzo-padoan-4521a2154/"
|
||||
}
|
||||
],
|
||||
"social_media_links": {
|
||||
"linkedin": "https://www.linkedin.com/company/101881123",
|
||||
"twitter": "https://x.com/scrapegraphai",
|
||||
"github": "https://github.com/ScrapeGraphAI/Scrapegraph-ai"
|
||||
}
|
||||
}
|
||||
```
|
||||
複数のページから情報を抽出したり、Pythonスクリプトを生成したり、さらにはオーディオファイルを生成したりするために使用できる他のパイプラインもあります。
|
||||
|
||||
| パイプライン名 | 説明 |
|
||||
|-------------------------|------------------------------------------------------------------------------------------------------------------|
|
||||
| SmartScraperGraph | ユーザープロンプトと入力ソースのみが必要な単一ページスクレイパー。 |
|
||||
| SearchGraph | 検索エンジンの上位n個の検索結果から情報を抽出する複数ページスクレイパー。 |
|
||||
| SpeechGraph | ウェブサイトから情報を抽出し、オーディオファイルを生成する単一ページスクレイパー。 |
|
||||
| ScriptCreatorGraph | ウェブサイトから情報を抽出し、Pythonスクリプトを生成する単一ページスクレイパー。 |
|
||||
| SmartScraperMultiGraph | 単一のプロンプトとソースのリストが与えられた場合に複数のページから情報を抽出する複数ページスクレイパー。 |
|
||||
| ScriptCreatorMultiGraph | 複数のページとソースから情報を抽出するためのPythonスクリプトを生成する複数ページスクレイパー。 |
|
||||
|
||||
これらのグラフのそれぞれには、マルチバージョンがあります。これにより、LLMの呼び出しを並列で行うことができます。
|
||||
|
||||
**OpenAI**、**Groq**、**Azure**、**Gemini**などのAPIを介して、または**Ollama**を使用してローカルモデルを使用して、異なるLLMを使用することができます。
|
||||
|
||||
ローカルモデルを使用する場合は、[Ollama](https://ollama.com/)がインストールされていること、および**ollama pull**コマンドを使用してモデルをダウンロードしていることを確認してください。
|
||||
|
||||
|
||||
## 📖 ドキュメント
|
||||
|
||||
[](https://colab.research.google.com/drive/1sEZBonBMGP44CtO6GQTwAlL0BGJXjtfd?usp=sharing)
|
||||
|
||||
ScrapeGraphAIのドキュメントは[こちら](https://docs.scrapegraphai.com/introduction)で見ることができます。
|
||||
|
||||
## 🤝 貢献
|
||||
|
||||
貢献を歓迎し、Discordサーバーで改善や提案について話し合います！
|
||||
|
||||
[貢献ガイド](https://github.com/ScrapeGraphAI/Scrapegraph-ai/blob/main/CONTRIBUTING.md)をご覧ください。
|
||||
|
||||
[](https://discord.gg/uJN7TYcpNa)
|
||||
[](https://www.linkedin.com/company/scrapegraphai/)
|
||||
[](https://twitter.com/scrapegraphai)
|
||||
|
||||
## 🔗 ScrapeGraph API & SDKs
|
||||
システムにScrapeGraphを統合するための迅速なソリューションをお探しの場合は、強力なAPIを[こちら！](https://dashboard.scrapegraphai.com/login)でご確認ください。
|
||||
|
||||
[](https://dashboard.scrapegraphai.com/login)
|
||||
|
||||
PythonとNode.jsの両方でSDKを提供しており、プロジェクトに簡単に統合できます。以下をご覧ください：
|
||||
|
||||
| SDK | 言語 | GitHubリンク |
|
||||
|-----------|----------|-----------------------------------------------------------------------------|
|
||||
| Python SDK | Python | [scrapegraph-py](https://docs.scrapegraphai.com/sdks/python) |
|
||||
| Node.js SDK | Node.js | [scrapegraph-js](https://docs.scrapegraphai.com/sdks/javascript) |
|
||||
|
||||
公式APIドキュメントは[こちら](https://docs.scrapegraphai.com/introduction)で見ることができます。
|
||||
|
||||
## 🔥 ベンチマーク
|
||||
|
||||
Firecrawlベンチマーク [Firecrawl benchmark](https://scrapegraphai.com/compare/firecrawl)によると、ScrapeGraphは市場で最高のフェッチャーです！
|
||||
|
||||
|
||||
|
||||
## 📈 テレメトリ
|
||||
パッケージの品質とユーザーエクスペリエンスを向上させるために、匿名の使用メトリクスを収集しています。このデータは、改善の優先順位付けと互換性の確保に役立ちます。オプトアウトする場合は、環境変数SCRAPEGRAPHAI_TELEMETRY_ENABLED=falseを設定してください。詳細については、[こちら](https://docs.scrapegraphai.com/introduction)のドキュメントを参照してください。
|
||||
|
||||
## ❤️ 貢献者
|
||||
[](https://github.com/ScrapeGraphAI/Scrapegraph-ai/graphs/contributors)
|
||||
|
||||
## 🎓 引用
|
||||
研究目的で当社のライブラリを使用する場合は、以下の参考文献を引用してください：
|
||||
```text
|
||||
@misc{scrapegraph-ai,
|
||||
author = {Lorenzo Padoan, Marco Vinciguerra},
|
||||
title = {Scrapegraph-ai},
|
||||
year = {2024},
|
||||
url = {https://github.com/ScrapeGraphAI/Scrapegraph-ai},
|
||||
note = {大規模言語モデルを活用したスクレイピング用のPythonライブラリ}
|
||||
}
|
||||
```
|
||||
## 作者
|
||||
|
||||
| | 連絡先 |
|
||||
|--------------------|----------------------|
|
||||
| Marco Vinciguerra | [](https://www.linkedin.com/in/marco-vinciguerra-7ba365242/) |
|
||||
| Lorenzo Padoan | [](https://www.linkedin.com/in/lorenzo-padoan-4521a2154/) |
|
||||
|
||||
## 📜 ライセンス
|
||||
|
||||
ScrapeGraphAIはMITライセンスの下で提供されています。詳細は[LICENSE](https://github.com/ScrapeGraphAI/Scrapegraph-ai/blob/main/LICENSE)ファイルをご覧ください。
|
||||
|
||||
## 謝辞
|
||||
|
||||
- プロジェクトの貢献者とオープンソースコミュニティのサポートに感謝します。
|
||||
- ScrapeGraphAIはデータ探索と研究目的のみに使用されます。このライブラリの不正使用については一切責任を負いません。
|
||||
|
||||
Made with ❤️ by [ScrapeGraph AI](https://scrapegraphai.com)
|
||||
|
||||
[Scarf tracking](https://static.scarf.sh/a.png?x-pxid=102d4b8c-cd6a-4b9e-9a16-d6d141b9212d)
|
||||
+227
@@ -0,0 +1,227 @@
|
||||
## 🚀 **더 빠르고 간단한 대규모 스크래핑 방법(단 5줄의 코드)을 찾고 계신가요?** [**ScrapeGraphAI.com**](https://scrapegraphai.com/?utm_source=github&utm_medium=readme&utm_campaign=oss_cta&utm_content=top_banner)의 향상된 버전을 확인해보세요! 🚀
|
||||
|
||||
---
|
||||
|
||||
# 🕷️ ScrapeGraphAI: 한 방에 끝내는 웹스크래핑
|
||||
|
||||
[English](../README.md) | [中文](chinese.md) | [日本語](japanese.md)
|
||||
| [한국어](korean.md)
|
||||
| [Русский](russian.md) | [Türkçe](turkish.md)
|
||||
| [Deutsch](german.md)
|
||||
| [Español](spanish.md)
|
||||
| [français](french.md)
|
||||
| [Português](portuguese.md)
|
||||
| [Italiano](italian.md)
|
||||
|
||||
[](https://pepy.tech/projects/scrapegraphai)
|
||||
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
[](https://discord.gg/gkxQDAjfeX)
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/15078" target="_blank"><img src="https://trendshift.io/api/badge/repositories/15078" alt="ScrapeGraphAI%2FScrapegraph-ai | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
<p align="center">
|
||||
|
||||
[ScrapeGraphAI](https://scrapegraphai.com)는 웹 사이트와 로컬 문서(XML, HTML, JSON, Markdown 등)에 대한 스크래핑 파이프라인을 만들기 위해 LLM 및 직접 그래프 로직을 사용하는 파이썬 웹스크래핑 라이브러리입니다.
|
||||
|
||||
추출하려는 정보를 말하기만 하면 라이브러리가 알아서 처리해 줍니다!
|
||||
|
||||
|
||||
|
||||
## 🚀 통합
|
||||
|
||||
ScrapeGraphAI는 인기 있는 프레임워크 및 도구와의 원활한 통합을 제공하여 스크래핑 능력을 향상시킵니다. 파이썬이든 Node.js로 개발하든, LLM 프레임워크를 사용하든, 노코드 플랫폼이든 저희의 포괄적인 통합 옵션을 제공합니다.
|
||||
|
||||
<p align="center">
|
||||
<img src="https://raw.githubusercontent.com/ScrapeGraphAI/Scrapegraph-ai/main/docs/assets/sgai-hero.png" alt="ScrapeGraphAI Hero" style="width: 100%;">
|
||||
</p>
|
||||
더 많은 정보는 다음 [링크](https://scrapegraphai.com)에서 확인할 수 있습니다
|
||||
|
||||
**통합**:
|
||||
- **API**: [문서](https://docs.scrapegraphai.com/introduction)
|
||||
- **SDKs**: [Python](https://docs.scrapegraphai.com/sdks/python), [Node](https://docs.scrapegraphai.com/sdks/javascript)
|
||||
- **LLM 프레임워크**: [Langchain](https://docs.scrapegraphai.com/integrations/langchain), [Llama Index](https://docs.scrapegraphai.com/integrations/llamaindex), [Crew.ai](https://docs.scrapegraphai.com/integrations/crewai), [Agno](https://docs.scrapegraphai.com/integrations/agno), [CamelAI](https://github.com/camel-ai/camel)
|
||||
- **로우코드 프레임워크**: [Pipedream](https://pipedream.com/apps/scrapegraphai), [Bubble](https://bubble.io/plugin/scrapegraphai-1745408893195x213542371433906180), [Zapier](https://zapier.com/apps/scrapegraphai/integrations), [n8n](http://localhost:5001/dashboard), [Dify](https://dify.ai), [Toolhouse](https://app.toolhouse.ai/mcp-servers/scrapegraph_smartscraper)
|
||||
- **MCP 서버**: [링크](https://smithery.ai/server/@ScrapeGraphAI/scrapegraph-mcp)
|
||||
|
||||
## 🚀 빠른 설치
|
||||
|
||||
Scrapegraph-ai에 대한 참조 페이지는 PyPI의 공식 페이지에서 확인할 수 있습니다: [pypi](https://pypi.org/project/scrapegraphai/).
|
||||
|
||||
```bash
|
||||
pip install scrapegraphai
|
||||
|
||||
# 중요 (웹사이트 콘텐츠 가져오기용)
|
||||
playwright install
|
||||
```
|
||||
|
||||
**참고**: 다른 라이브러리와의 충돌을 피하기 위해 라이브러리를 가상 환경에 설치하는 것이 좋습니다 🐱
|
||||
|
||||
|
||||
## 💻 사용법
|
||||
웹사이트(또는 로컬 파일)에서 정보를 추출하기 위해 사용할 수 있는 여러 표준 스크래핑 파이프라인이 있습니다.
|
||||
|
||||
가장 일반적인 것은 `SmartScraperGraph`로, 사용자 프롬프트와 소스 URL이 주어진 단일 페이지에서 정보를 추출합니다.
|
||||
|
||||
|
||||
```python
|
||||
from scrapegraphai.graphs import SmartScraperGraph
|
||||
|
||||
# 스크래핑 파이프라인에 대한 구성 정의
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"model": "ollama/llama3.2",
|
||||
"model_tokens": 8192,
|
||||
"format": "json",
|
||||
},
|
||||
"verbose": True,
|
||||
"headless": False,
|
||||
}
|
||||
|
||||
# SmartScraperGraph 인스턴스 생성
|
||||
smart_scraper_graph = SmartScraperGraph(
|
||||
prompt="웹페이지에서 유용한 정보를 추출하세요. 회사가 하는 일에 대한 설명, 창립자 및 소셜 미디어 링크를 포함하세요",
|
||||
source="https://scrapegraphai.com/",
|
||||
config=graph_config
|
||||
)
|
||||
|
||||
# 파이프라인 실행
|
||||
result = smart_scraper_graph.run()
|
||||
|
||||
import json
|
||||
print(json.dumps(result, indent=4))
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
> OpenAI나 다른 모델들은 LLM 설정만 바꾸면 됩니다!
|
||||
> ```python
|
||||
>graph_config = {
|
||||
> "llm": {
|
||||
> "api_key": "YOUR_OPENAI_API_KEY",
|
||||
> "model": "openai/gpt-4o-mini",
|
||||
> },
|
||||
> "verbose": True,
|
||||
> "headless": False,
|
||||
>}
|
||||
>```
|
||||
|
||||
|
||||
출력은 다음과 같은 dictionary 형태가 될 것입니다:
|
||||
|
||||
```python
|
||||
{
|
||||
"description": "ScrapeGraphAI transforms websites into clean, organized data for AI agents and data analytics. It offers an AI-powered API for effortless and cost-effective data extraction.",
|
||||
"founders": [
|
||||
{
|
||||
"name": "",
|
||||
"role": "Founder & Technical Lead",
|
||||
"linkedin": "https://www.linkedin.com/in/perinim/"
|
||||
},
|
||||
{
|
||||
"name": "Marco Vinciguerra",
|
||||
"role": "Founder & Software Engineer",
|
||||
"linkedin": "https://www.linkedin.com/in/marco-vinciguerra-7ba365242/"
|
||||
},
|
||||
{
|
||||
"name": "Lorenzo Padoan",
|
||||
"role": "Founder & Product Engineer",
|
||||
"linkedin": "https://www.linkedin.com/in/lorenzo-padoan-4521a2154/"
|
||||
}
|
||||
],
|
||||
"social_media_links": {
|
||||
"linkedin": "https://www.linkedin.com/company/101881123",
|
||||
"twitter": "https://x.com/scrapegraphai",
|
||||
"github": "https://github.com/ScrapeGraphAI/Scrapegraph-ai"
|
||||
}
|
||||
}
|
||||
```
|
||||
여러 페이지에서 정보를 추출하거나, Python 스크립트를 생성하거나, 심지어 오디오 파일을 생성하는 데 사용할 수 있는 다른 파이프라인도 있습니다.
|
||||
|
||||
| 파이프라인 이름 | 설명 |
|
||||
|-------------------------|------------------------------------------------------------------------------------------------------------------|
|
||||
| SmartScraperGraph | 사용자 프롬프트와 입력 소스만 있으면 되는 단일 페이지 스크래퍼입니다. |
|
||||
| SearchGraph | 검색 엔진의 상위 n개 검색 결과에서 정보를 추출하는 다중 페이지 스크래퍼입니다. |
|
||||
| SpeechGraph | 웹사이트에서 정보를 추출하고 오디오 파일을 생성하는 단일 페이지 스크래퍼입니다. |
|
||||
| ScriptCreatorGraph | 웹사이트에서 정보를 추출하고 파이썬 스크립트를 생성하는 단일 페이지 스크래퍼입니다. |
|
||||
| SmartScraperMultiGraph | 단일 프롬프트와 출처 목록이 주어지면 여러 페이지에서 정보를 추출하는 다중 페이지 스크래퍼입니다. |
|
||||
| ScriptCreatorMultiGraph | 여러 페이지와 소스에서 정보를 추출하기 위한 파이썬 스크립트를 생성하는 다중 페이지 스크래퍼입니다. |
|
||||
|
||||
각 그래프에는 다중 버전이 있습니다. 이를 통해 LLM을 병렬로 호출할 수 있습니다.
|
||||
|
||||
OpenAI, Groq, Azure, Gemini와 같은 API를 통해 다양한 LLM을 사용할 수 있으며, Ollama를 이용한 로컬 모델도 가능합니다.
|
||||
|
||||
로컬 모델을 사용하려면 [Ollama](https://ollama.com/)를 설치하고 ollama pull 명령을 사용하여 모델을 다운로드해야 합니다.
|
||||
|
||||
|
||||
## 📖 문서
|
||||
|
||||
[](https://colab.research.google.com/drive/1sEZBonBMGP44CtO6GQTwAlL0BGJXjtfd?usp=sharing)
|
||||
|
||||
ScrapeGraphAI 관련 문서는 [여기](https://docs.scrapegraphai.com/introduction)에서 확인하실 수 있습니다.
|
||||
|
||||
## 🤝 기여
|
||||
|
||||
자유롭게 기여하고 Discord 서버에 참여하여 개선 사항을 논의하고 제안해 주세요!
|
||||
|
||||
[기여 가이드라인](https://github.com/ScrapeGraphAI/Scrapegraph-ai/blob/main/CONTRIBUTING.md)을 참고하세요.
|
||||
|
||||
[](https://discord.gg/uJN7TYcpNa)
|
||||
[](https://www.linkedin.com/company/scrapegraphai/)
|
||||
[](https://twitter.com/scrapegraphai)
|
||||
|
||||
## 🔗 ScrapeGraph API & SDKs
|
||||
시스템에 ScrapeGraph를 통합하기 위한 빠른 솔루션을 찾고 있다면, [여기!](https://dashboard.scrapegraphai.com/login)에서 강력한 API를 확인해 보세요.
|
||||
|
||||
[](https://dashboard.scrapegraphai.com/login)
|
||||
|
||||
Python과 Node.js SDK를 제공하여 프로젝트에 쉽게 통합할 수 있습니다. 아래에서 확인해 보세요.
|
||||
|
||||
| SDK | 언어 | GitHub 링크 |
|
||||
|-----------|----------|-----------------------------------------------------------------------------|
|
||||
| Python SDK | Python | [scrapegraph-py](https://docs.scrapegraphai.com/sdks/python) |
|
||||
| Node.js SDK | Node.js | [scrapegraph-js](https://docs.scrapegraphai.com/sdks/javascript) |
|
||||
|
||||
공식 API 문서는 [여기](https://docs.scrapegraphai.com/introduction)에서 확인할 수 있습니다.
|
||||
|
||||
## 🔥 벤치마크
|
||||
|
||||
Firecrawl 벤치마크 [Firecrawl benchmark](https://scrapegraphai.com/compare/firecrawl)에 따르면, ScrapeGraph는 시장에서 최고의 페처입니다!
|
||||
|
||||
|
||||
|
||||
## 📈 텔레메트리
|
||||
저희는 패키지의 품질과 사용자 경험을 향상시키기 위해 익명의 사용 지표를 수집합니다. 이 데이터는 개선 사항의 우선순위를 정하고 호환성을 보장하는 데 도움이 됩니다. 옵트아웃하려면 환경 변수 SCRAPEGRAPHAI_TELEMETRY_ENABLED=false를 설정하세요. 자세한 내용은 [여기](https://docs.scrapegraphai.com/introduction)에서 설명서를 참조하세요.
|
||||
|
||||
## ❤️ 기여자들
|
||||
[](https://github.com/ScrapeGraphAI/Scrapegraph-ai/graphs/contributors)
|
||||
|
||||
## 🎓 인용
|
||||
우리의 라이브러리를 연구 목적으로 사용한 경우 다음과 같이 인용해 주세요:
|
||||
```text
|
||||
@misc{scrapegraph-ai,
|
||||
author = {Lorenzo Padoan, Marco Vinciguerra},
|
||||
title = {Scrapegraph-ai},
|
||||
year = {2024},
|
||||
url = {https://github.com/ScrapeGraphAI/Scrapegraph-ai},
|
||||
note = {대규모 언어 모델을 활용한 스크래핑용 Python 라이브러리}
|
||||
}
|
||||
```
|
||||
|
||||
## 저자들
|
||||
|
||||
| | 연락처 |
|
||||
|--------------------|---------------|
|
||||
| Marco Vinciguerra | [](https://www.linkedin.com/in/marco-vinciguerra-7ba365242/) |
|
||||
| Lorenzo Padoan | [](https://www.linkedin.com/in/lorenzo-padoan-4521a2154/) |
|
||||
|
||||
## 📜 라이선스
|
||||
|
||||
ScrapeGraphAI는 MIT License로 배포되었습니다. 자세한 내용은 [LICENSE](https://github.com/ScrapeGraphAI/Scrapegraph-ai/blob/main/LICENSE) 파일을 참조하세요.
|
||||
|
||||
## 감사의 말
|
||||
|
||||
- 프로젝트에 기여한 모든 분들과 오픈 소스 커뮤니티에 감사드립니다.
|
||||
- ScrapeGraphAI는 데이터 탐색 및 연구 목적으로만 사용되어야 합니다. 우리는 라이브러리의 오용에 대해 책임을 지지 않습니다.
|
||||
|
||||
Made with ❤️ by [ScrapeGraph AI](https://scrapegraphai.com)
|
||||
|
||||
[Scarf tracking](https://static.scarf.sh/a.png?x-pxid=102d4b8c-cd6a-4b9e-9a16-d6d141b9212d)
|
||||
@@ -0,0 +1,227 @@
|
||||
## 🚀 **Procurando uma forma ainda mais rápida e simples de fazer scraping em escala (apenas 5 linhas de código)?** Confira nossa versão aprimorada em [**ScrapeGraphAI.com**](https://scrapegraphai.com/?utm_source=github&utm_medium=readme&utm_campaign=oss_cta&utm_content=top_banner)! 🚀
|
||||
|
||||
---
|
||||
|
||||
# 🕷️ ScrapeGraphAI: Você Só Faz Scraping Uma Vez
|
||||
|
||||
[English](../README.md) | [中文](chinese.md) | [日本語](japanese.md)
|
||||
| [한국어](korean.md)
|
||||
| [Русский](russian.md) | [Türkçe](turkish.md)
|
||||
| [Deutsch](german.md)
|
||||
| [Español](spanish.md)
|
||||
| [français](french.md)
|
||||
| [Português](portuguese.md)
|
||||
| [Italiano](italian.md)
|
||||
|
||||
[](https://pepy.tech/projects/scrapegraphai)
|
||||
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
[](https://discord.gg/gkxQDAjfeX)
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/15078" target="_blank"><img src="https://trendshift.io/api/badge/repositories/15078" alt="ScrapeGraphAI%2FScrapegraph-ai | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
<p align="center">
|
||||
|
||||
[ScrapeGraphAI](https://scrapegraphai.com) é uma biblioteca Python de *web scraping* que usa LLM e lógica de grafo direto para criar pipelines de scraping para sites e documentos locais (XML, HTML, JSON, Markdown, etc.).
|
||||
|
||||
Basta dizer qual informação você quer extrair e a biblioteca fará isso por você!
|
||||
|
||||
|
||||
|
||||
## 🚀 Integrações
|
||||
|
||||
O ScrapeGraphAI oferece integração perfeita com frameworks e ferramentas populares para aprimorar suas capacidades de scraping. Seja você construindo com Python ou Node.js, usando frameworks LLM ou trabalhando com plataformas no-code, temos você coberto com nossas opções abrangentes de integração.
|
||||
|
||||
<p align="center">
|
||||
<img src="https://raw.githubusercontent.com/ScrapeGraphAI/Scrapegraph-ai/main/docs/assets/sgai-hero.png" alt="ScrapeGraphAI Hero" style="width: 100%;">
|
||||
</p>
|
||||
Você pode encontrar mais informações no seguinte [link](https://scrapegraphai.com)
|
||||
|
||||
**Integrações**:
|
||||
- **API**: [Documentação](https://docs.scrapegraphai.com/introduction)
|
||||
- **SDKs**: [Python](https://docs.scrapegraphai.com/sdks/python), [Node](https://docs.scrapegraphai.com/sdks/javascript)
|
||||
- **Frameworks LLM**: [Langchain](https://docs.scrapegraphai.com/integrations/langchain), [Llama Index](https://docs.scrapegraphai.com/integrations/llamaindex), [Crew.ai](https://docs.scrapegraphai.com/integrations/crewai), [Agno](https://docs.scrapegraphai.com/integrations/agno), [CamelAI](https://github.com/camel-ai/camel)
|
||||
- **Frameworks Low-code**: [Pipedream](https://pipedream.com/apps/scrapegraphai), [Bubble](https://bubble.io/plugin/scrapegraphai-1745408893195x213542371433906180), [Zapier](https://zapier.com/apps/scrapegraphai/integrations), [n8n](http://localhost:5001/dashboard), [Dify](https://dify.ai), [Toolhouse](https://app.toolhouse.ai/mcp-servers/scrapegraph_smartscraper)
|
||||
- **Servidor MCP**: [Link](https://smithery.ai/server/@ScrapeGraphAI/scrapegraph-mcp)
|
||||
|
||||
## 🚀 Instalação Rápida
|
||||
|
||||
A página de referência para Scrapegraph-ai está disponível na página oficial do PyPI: [pypi](https://pypi.org/project/scrapegraphai/).
|
||||
|
||||
```bash
|
||||
pip install scrapegraphai
|
||||
|
||||
# IMPORTANTE (para buscar conteúdo de sites)
|
||||
playwright install
|
||||
```
|
||||
|
||||
**Nota**: é recomendado instalar a biblioteca em um ambiente virtual para evitar conflitos com outras bibliotecas 🐱
|
||||
|
||||
|
||||
## 💻 Uso
|
||||
Existem múltiplos pipelines de scraping padrão que podem ser usados para extrair informações de um site (ou arquivo local).
|
||||
|
||||
O mais comum é o `SmartScraperGraph`, que extrai informações de uma única página dado um prompt do usuário e uma URL de origem.
|
||||
|
||||
|
||||
```python
|
||||
from scrapegraphai.graphs import SmartScraperGraph
|
||||
|
||||
# Defina a configuração para o pipeline de scraping
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"model": "ollama/llama3.2",
|
||||
"model_tokens": 8192,
|
||||
"format": "json",
|
||||
},
|
||||
"verbose": True,
|
||||
"headless": False,
|
||||
}
|
||||
|
||||
# Crie a instância SmartScraperGraph
|
||||
smart_scraper_graph = SmartScraperGraph(
|
||||
prompt="Extraia informações úteis da página web, incluindo uma descrição do que a empresa faz, fundadores e links de redes sociais",
|
||||
source="https://scrapegraphai.com/",
|
||||
config=graph_config
|
||||
)
|
||||
|
||||
# Execute o pipeline
|
||||
result = smart_scraper_graph.run()
|
||||
|
||||
import json
|
||||
print(json.dumps(result, indent=4))
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
> Para OpenAI e outros modelos, você só precisa mudar a configuração do llm!
|
||||
> ```python
|
||||
>graph_config = {
|
||||
> "llm": {
|
||||
> "api_key": "YOUR_OPENAI_API_KEY",
|
||||
> "model": "openai/gpt-4o-mini",
|
||||
> },
|
||||
> "verbose": True,
|
||||
> "headless": False,
|
||||
>}
|
||||
>```
|
||||
|
||||
|
||||
A saída será um dicionário como o seguinte:
|
||||
|
||||
```python
|
||||
{
|
||||
"description": "ScrapeGraphAI transforms websites into clean, organized data for AI agents and data analytics. It offers an AI-powered API for effortless and cost-effective data extraction.",
|
||||
"founders": [
|
||||
{
|
||||
"name": "",
|
||||
"role": "Founder & Technical Lead",
|
||||
"linkedin": "https://www.linkedin.com/in/perinim/"
|
||||
},
|
||||
{
|
||||
"name": "Marco Vinciguerra",
|
||||
"role": "Founder & Software Engineer",
|
||||
"linkedin": "https://www.linkedin.com/in/marco-vinciguerra-7ba365242/"
|
||||
},
|
||||
{
|
||||
"name": "Lorenzo Padoan",
|
||||
"role": "Founder & Product Engineer",
|
||||
"linkedin": "https://www.linkedin.com/in/lorenzo-padoan-4521a2154/"
|
||||
}
|
||||
],
|
||||
"social_media_links": {
|
||||
"linkedin": "https://www.linkedin.com/company/101881123",
|
||||
"twitter": "https://x.com/scrapegraphai",
|
||||
"github": "https://github.com/ScrapeGraphAI/Scrapegraph-ai"
|
||||
}
|
||||
}
|
||||
```
|
||||
Existem outros pipelines que podem ser usados para extrair informações de múltiplas páginas, gerar scripts Python ou até mesmo gerar arquivos de áudio.
|
||||
|
||||
| Nome do Pipeline | Descrição |
|
||||
|-------------------------|------------------------------------------------------------------------------------------------------------------|
|
||||
| SmartScraperGraph | Scraper de página única que só precisa de um prompt do usuário e uma fonte de entrada. |
|
||||
| SearchGraph | Scraper de múltiplas páginas que extrai informações dos n principais resultados de pesquisa de um mecanismo de busca. |
|
||||
| SpeechGraph | Scraper de página única que extrai informações de um site e gera um arquivo de áudio. |
|
||||
| ScriptCreatorGraph | Scraper de página única que extrai informações de um site e gera um script Python. |
|
||||
| SmartScraperMultiGraph | Scraper de múltiplas páginas que extrai informações de múltiplas páginas dado um único prompt e uma lista de fontes. |
|
||||
| ScriptCreatorMultiGraph | Scraper de múltiplas páginas que gera um script Python para extrair informações de múltiplas páginas e fontes. |
|
||||
|
||||
Para cada um desses grafos existe a versão multi. Isso permite fazer chamadas do LLM em paralelo.
|
||||
|
||||
É possível usar diferentes LLMs através de APIs, como **OpenAI**, **Groq**, **Azure** e **Gemini**, ou modelos locais usando **Ollama**.
|
||||
|
||||
Lembre-se de ter o [Ollama](https://ollama.com/) instalado e baixar os modelos usando o comando **ollama pull**, se você quiser usar modelos locais.
|
||||
|
||||
|
||||
## 📖 Documentação
|
||||
|
||||
[](https://colab.research.google.com/drive/1sEZBonBMGP44CtO6GQTwAlL0BGJXjtfd?usp=sharing)
|
||||
|
||||
A documentação do ScrapeGraphAI pode ser encontrada [aqui](https://docs.scrapegraphai.com/introduction).
|
||||
|
||||
## 🤝 Contribuindo
|
||||
|
||||
Sinta-se à vontade para contribuir e junte-se ao nosso servidor Discord para discutir melhorias e nos dar sugestões!
|
||||
|
||||
Por favor, veja as [diretrizes de contribuição](https://github.com/ScrapeGraphAI/Scrapegraph-ai/blob/main/CONTRIBUTING.md).
|
||||
|
||||
[](https://discord.gg/uJN7TYcpNa)
|
||||
[](https://www.linkedin.com/company/scrapegraphai/)
|
||||
[](https://twitter.com/scrapegraphai)
|
||||
|
||||
## 🔗 ScrapeGraph API & SDKs
|
||||
Se você está procurando uma solução rápida para integrar o ScrapeGraph em seu sistema, confira nossa poderosa API [aqui!](https://dashboard.scrapegraphai.com/login)
|
||||
|
||||
[](https://dashboard.scrapegraphai.com/login)
|
||||
|
||||
Oferecemos SDKs em Python e Node.js, facilitando a integração em seus projetos. Confira abaixo:
|
||||
|
||||
| SDK | Linguagem | Link do GitHub |
|
||||
|-----------|----------|-----------------------------------------------------------------------------|
|
||||
| Python SDK | Python | [scrapegraph-py](https://docs.scrapegraphai.com/sdks/python) |
|
||||
| Node.js SDK | Node.js | [scrapegraph-js](https://docs.scrapegraphai.com/sdks/javascript) |
|
||||
|
||||
A Documentação Oficial da API pode ser encontrada [aqui](https://docs.scrapegraphai.com/introduction).
|
||||
|
||||
## 🔥 Benchmark
|
||||
|
||||
De acordo com o benchmark do Firecrawl [Firecrawl benchmark](https://scrapegraphai.com/compare/firecrawl), o ScrapeGraph é o melhor fetcher do mercado!
|
||||
|
||||
|
||||
|
||||
## 📈 Telemetria
|
||||
Coletamos métricas de uso anônimas para melhorar a qualidade e a experiência do usuário do nosso pacote. Os dados nos ajudam a priorizar melhorias e garantir compatibilidade. Se você deseja optar por não participar, defina a variável de ambiente SCRAPEGRAPHAI_TELEMETRY_ENABLED=false. Para mais informações, consulte a documentação [aqui](https://docs.scrapegraphai.com/introduction).
|
||||
|
||||
## ❤️ Contribuidores
|
||||
[](https://github.com/ScrapeGraphAI/Scrapegraph-ai/graphs/contributors)
|
||||
|
||||
## 🎓 Citações
|
||||
Se você usou nossa biblioteca para fins de pesquisa, por favor, cite-nos com a seguinte referência:
|
||||
```text
|
||||
@misc{scrapegraph-ai,
|
||||
author = {Lorenzo Padoan, Marco Vinciguerra},
|
||||
title = {Scrapegraph-ai},
|
||||
year = {2024},
|
||||
url = {https://github.com/ScrapeGraphAI/Scrapegraph-ai},
|
||||
note = {Uma biblioteca Python para scraping aproveitando grandes modelos de linguagem}
|
||||
}
|
||||
```
|
||||
|
||||
## Autores
|
||||
|
||||
| | Informações de Contato |
|
||||
|--------------------|----------------------|
|
||||
| Marco Vinciguerra | [](https://www.linkedin.com/in/marco-vinciguerra-7ba365242/) |
|
||||
| Lorenzo Padoan | [](https://www.linkedin.com/in/lorenzo-padoan-4521a2154/) |
|
||||
|
||||
## 📜 Licença
|
||||
|
||||
O ScrapeGraphAI está licenciado sob a Licença MIT. Veja o arquivo [LICENSE](https://github.com/ScrapeGraphAI/Scrapegraph-ai/blob/main/LICENSE) para mais informações.
|
||||
|
||||
## Agradecimentos
|
||||
|
||||
- Gostaríamos de agradecer a todos os contribuidores do projeto e à comunidade de código aberto pelo seu apoio.
|
||||
- O ScrapeGraphAI destina-se apenas a fins de exploração de dados e pesquisa. Não nos responsabilizamos por qualquer uso indevido da biblioteca.
|
||||
|
||||
Made with ❤️ by [ScrapeGraph AI](https://scrapegraphai.com)
|
||||
|
||||
[Scarf tracking](https://static.scarf.sh/a.png?x-pxid=102d4b8c-cd6a-4b9e-9a16-d6d141b9212d)
|
||||
+230
@@ -0,0 +1,230 @@
|
||||
## 🚀 **Ищете еще более быстрый и простой способ масштабного скрейпинга (всего 5 строк кода)?** Ознакомьтесь с нашей улучшенной версией на [**ScrapeGraphAI.com**](https://scrapegraphai.com/?utm_source=github&utm_medium=readme&utm_campaign=oss_cta&utm_content=top_banner)! 🚀
|
||||
|
||||
---
|
||||
|
||||
# 🕷️ ScrapeGraphAI: Вы скрейпите только один раз
|
||||
|
||||
[English](../README.md) | [中文](chinese.md) | [日本語](japanese.md)
|
||||
| [한국어](korean.md)
|
||||
| [Русский](russian.md) | [Türkçe](turkish.md)
|
||||
| [Deutsch](german.md)
|
||||
| [Español](spanish.md)
|
||||
| [français](french.md)
|
||||
| [Português](portuguese.md)
|
||||
| [Italiano](italian.md)
|
||||
|
||||
[](https://pepy.tech/projects/scrapegraphai)
|
||||
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
[](https://discord.gg/gkxQDAjfeX)
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/15078" target="_blank"><img src="https://trendshift.io/api/badge/repositories/15078" alt="ScrapeGraphAI%2FScrapegraph-ai | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
<p align="center">
|
||||
|
||||
ScrapeGraphAI - это библиотека для веб-скрейпинга на Python, которая использует LLM и прямую графовую логику для создания скрейпинговых пайплайнов для веб-сайтов и локальных документов (XML, HTML, JSON, Markdown и т.д.).
|
||||
|
||||
Просто укажите, какую информацию вы хотите извлечь, и библиотека сделает это за вас!
|
||||
|
||||
|
||||
|
||||
## 🚀 Интеграции
|
||||
|
||||
ScrapeGraphAI предлагает бесшовную интеграцию с популярными фреймворками и инструментами для улучшения ваших возможностей скрейпинга. Независимо от того, создаете ли вы приложения на Python или Node.js, используете ли LLM-фреймворки или работаете с платформами без кода, мы предоставляем комплексные варианты интеграции.
|
||||
|
||||
<p align="center">
|
||||
<img src="https://raw.githubusercontent.com/ScrapeGraphAI/Scrapegraph-ai/main/docs/assets/sgai-hero.png" alt="ScrapeGraphAI Hero" style="width: 100%;">
|
||||
</p>
|
||||
Вы можете найти больше информации по следующей [ссылке](https://scrapegraphai.com)
|
||||
|
||||
**Интеграции**:
|
||||
- **API**: [Документация](https://docs.scrapegraphai.com/introduction)
|
||||
- **SDKs**: [Python](https://docs.scrapegraphai.com/sdks/python), [Node](https://docs.scrapegraphai.com/sdks/javascript)
|
||||
- **LLM Фреймворки**: [Langchain](https://docs.scrapegraphai.com/integrations/langchain), [Llama Index](https://docs.scrapegraphai.com/integrations/llamaindex), [Crew.ai](https://docs.scrapegraphai.com/integrations/crewai), [Agno](https://docs.scrapegraphai.com/integrations/agno), [CamelAI](https://github.com/camel-ai/camel)
|
||||
- **Low-code Фреймворки**: [Pipedream](https://pipedream.com/apps/scrapegraphai), [Bubble](https://bubble.io/plugin/scrapegraphai-1745408893195x213542371433906180), [Zapier](https://zapier.com/apps/scrapegraphai/integrations), [n8n](http://localhost:5001/dashboard), [Dify](https://dify.ai), [Toolhouse](https://app.toolhouse.ai/mcp-servers/scrapegraph_smartscraper)
|
||||
- **MCP сервер**: [Ссылка](https://smithery.ai/server/@ScrapeGraphAI/scrapegraph-mcp)
|
||||
|
||||
## 🚀 Быстрая установка
|
||||
|
||||
Референсная страница для Scrapegraph-ai доступна на официальной странице PyPI: [pypi](https://pypi.org/project/scrapegraphai/).
|
||||
|
||||
```bash
|
||||
pip install scrapegraphai
|
||||
|
||||
# ВАЖНО (для получения содержимого веб-сайтов)
|
||||
playwright install
|
||||
```
|
||||
|
||||
**Примечание**: рекомендуется устанавливать библиотеку в виртуальную среду, чтобы избежать конфликтов с другими библиотеками 🐱
|
||||
|
||||
|
||||
## 💻 Использование
|
||||
Существует несколько стандартных скрейпинговых пайплайнов, которые можно использовать для извлечения информации с веб-сайта (или локального файла).
|
||||
|
||||
Наиболее распространенным является `SmartScraperGraph`, который извлекает информацию с одной страницы при наличии пользовательского запроса и исходного URL.
|
||||
|
||||
|
||||
```python
|
||||
from scrapegraphai.graphs import SmartScraperGraph
|
||||
|
||||
# Определите конфигурацию для скрейпингового пайплайна
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"model": "ollama/llama3.2",
|
||||
"model_tokens": 8192,
|
||||
"format": "json",
|
||||
},
|
||||
"verbose": True,
|
||||
"headless": False,
|
||||
}
|
||||
|
||||
# Создайте экземпляр SmartScraperGraph
|
||||
smart_scraper_graph = SmartScraperGraph(
|
||||
prompt="Извлеките полезную информацию с веб-страницы, включая описание деятельности компании, основателей и ссылки на социальные сети",
|
||||
source="https://scrapegraphai.com/",
|
||||
config=graph_config
|
||||
)
|
||||
|
||||
# Запустите пайплайн
|
||||
result = smart_scraper_graph.run()
|
||||
|
||||
import json
|
||||
print(json.dumps(result, indent=4))
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
> Для OpenAI и других моделей вам просто нужно изменить конфигурацию llm!
|
||||
> ```python
|
||||
>graph_config = {
|
||||
> "llm": {
|
||||
> "api_key": "YOUR_OPENAI_API_KEY",
|
||||
> "model": "openai/gpt-4o-mini",
|
||||
> },
|
||||
> "verbose": True,
|
||||
> "headless": False,
|
||||
>}
|
||||
>```
|
||||
|
||||
|
||||
Выходные данные будут представлять собой словарь, например:
|
||||
|
||||
```python
|
||||
{
|
||||
"description": "ScrapeGraphAI transforms websites into clean, organized data for AI agents and data analytics. It offers an AI-powered API for effortless and cost-effective data extraction.",
|
||||
"founders": [
|
||||
{
|
||||
"name": "",
|
||||
"role": "Founder & Technical Lead",
|
||||
"linkedin": "https://www.linkedin.com/in/perinim/"
|
||||
},
|
||||
{
|
||||
"name": "Marco Vinciguerra",
|
||||
"role": "Founder & Software Engineer",
|
||||
"linkedin": "https://www.linkedin.com/in/marco-vinciguerra-7ba365242/"
|
||||
},
|
||||
{
|
||||
"name": "Lorenzo Padoan",
|
||||
"role": "Founder & Product Engineer",
|
||||
"linkedin": "https://www.linkedin.com/in/lorenzo-padoan-4521a2154/"
|
||||
}
|
||||
],
|
||||
"social_media_links": {
|
||||
"linkedin": "https://www.linkedin.com/company/101881123",
|
||||
"twitter": "https://x.com/scrapegraphai",
|
||||
"github": "https://github.com/ScrapeGraphAI/Scrapegraph-ai"
|
||||
}
|
||||
}
|
||||
```
|
||||
Существуют другие пайплайны, которые можно использовать для извлечения информации с нескольких страниц, генерации Python-скриптов или даже генерации аудиофайлов.
|
||||
|
||||
| Название пайплайна | Описание |
|
||||
|-------------------------|------------------------------------------------------------------------------------------------------------------|
|
||||
| SmartScraperGraph | Скрейпер одной страницы, которому требуется только пользовательский запрос и источник ввода. |
|
||||
| SearchGraph | Многопользовательский скрейпер, который извлекает информацию из топ n результатов поиска поисковой системы. |
|
||||
| SpeechGraph | Скрейпер одной страницы, который извлекает информацию с веб-сайта и генерирует аудиофайл. |
|
||||
| ScriptCreatorGraph | Скрейпер одной страницы, который извлекает информацию с веб-сайта и генерирует Python-скрипт. |
|
||||
| SmartScraperMultiGraph | Многопользовательский скрейпер, который извлекает информацию с нескольких страниц при наличии одного запроса и списка источников. |
|
||||
| ScriptCreatorMultiGraph | Многопользовательский скрейпер, который генерирует Python-скрипт для извлечения информации с нескольких страниц и источников. |
|
||||
|
||||
Для каждого из этих графов существует мульти-версия. Это позволяет выполнять вызовы LLM параллельно.
|
||||
|
||||
Можно использовать различные LLM через API, такие как **OpenAI**, **Groq**, **Azure** и **Gemini**, или локальные модели, используя **Ollama**.
|
||||
|
||||
Не забудьте установить [Ollama](https://ollama.com/) и загрузить модели, используя команду **ollama pull**, если вы хотите использовать локальные модели.
|
||||
|
||||
|
||||
## 📖 Документация
|
||||
|
||||
[](https://colab.research.google.com/drive/1sEZBonBMGP44CtO6GQTwAlL0BGJXjtfd?usp=sharing)
|
||||
|
||||
Документация для ScrapeGraphAI доступна [здесь](https://docs.scrapegraphai.com/introduction).
|
||||
|
||||
## 🤝 Участие
|
||||
|
||||
Не стесняйтесь вносить свой вклад и присоединяйтесь к нашему серверу Discord, чтобы обсудить с нами улучшения и дать нам предложения!
|
||||
|
||||
Пожалуйста, ознакомьтесь с [руководством по участию](https://github.com/ScrapeGraphAI/Scrapegraph-ai/blob/main/CONTRIBUTING.md).
|
||||
|
||||
[](https://discord.gg/uJN7TYcpNa)
|
||||
[](https://www.linkedin.com/company/scrapegraphai/)
|
||||
[](https://twitter.com/scrapegraphai)
|
||||
|
||||
## 🔗 ScrapeGraph API & SDKs
|
||||
Если вы ищете быстрое решение для интеграции ScrapeGraph в вашу систему, ознакомьтесь с нашим мощным API [здесь!](https://dashboard.scrapegraphai.com/login)
|
||||
|
||||
[](https://dashboard.scrapegraphai.com/login)
|
||||
|
||||
Мы предлагаем SDK для Python и Node.js, что упрощает интеграцию в ваши проекты. Ознакомьтесь с ними ниже:
|
||||
|
||||
| SDK | Язык | GitHub Ссылка |
|
||||
|-----------|----------|-----------------------------------------------------------------------------|
|
||||
| Python SDK | Python | [scrapegraph-py](https://docs.scrapegraphai.com/sdks/python) |
|
||||
| Node.js SDK | Node.js | [scrapegraph-js](https://docs.scrapegraphai.com/sdks/javascript) |
|
||||
|
||||
Официальная документация API доступна [здесь](https://docs.scrapegraphai.com/introduction).
|
||||
|
||||
## 🔥 Бенчмарк
|
||||
|
||||
Согласно бенчмарку Firecrawl [Firecrawl benchmark](https://scrapegraphai.com/compare/firecrawl), ScrapeGraph является лучшим фетчером на рынке!
|
||||
|
||||
|
||||
|
||||
## 📈 Телеметрия
|
||||
Мы собираем анонимные метрики использования для повышения качества нашего пакета и пользовательского опыта. Данные помогают нам определять приоритеты улучшений и обеспечивать совместимость. Если вы хотите отказаться, установите переменную окружения SCRAPEGRAPHAI_TELEMETRY_ENABLED=false. Для получения дополнительной информации обратитесь к документации [здесь](https://docs.scrapegraphai.com/introduction).
|
||||
|
||||
## ❤️ Разработчики программного обеспечения
|
||||
|
||||
[](https://github.com/ScrapeGraphAI/Scrapegraph-ai/graphs/contributors)
|
||||
|
||||
## 🎓 Цитаты
|
||||
|
||||
Если вы использовали нашу библиотеку для научных исследований, пожалуйста, укажите нас в следующем виде:
|
||||
|
||||
```text
|
||||
@misc{scrapegraph-ai,
|
||||
author = {Lorenzo Padoan, Marco Vinciguerra},
|
||||
title = {Scrapegraph-ai},
|
||||
year = {2024},
|
||||
url = {https://github.com/ScrapeGraphAI/Scrapegraph-ai},
|
||||
note = {Библиотека на Python для скрейпинга с использованием больших языковых моделей}
|
||||
}
|
||||
```
|
||||
|
||||
## Авторы
|
||||
|
||||
| | Контактная информация |
|
||||
|--------------------|----------------------|
|
||||
| Marco Vinciguerra | [](https://www.linkedin.com/in/marco-vinciguerra-7ba365242/) |
|
||||
| Lorenzo Padoan | [](https://www.linkedin.com/in/lorenzo-padoan-4521a2154/) |
|
||||
|
||||
## 📜 Лицензия
|
||||
|
||||
ScrapeGraphAI лицензирован под MIT License. Подробнее см. в файле [LICENSE](https://github.com/ScrapeGraphAI/Scrapegraph-ai/blob/main/LICENSE).
|
||||
|
||||
## Благодарности
|
||||
|
||||
- Мы хотели бы поблагодарить всех участников проекта и сообщество с открытым исходным кодом за их поддержку.
|
||||
- ScrapeGraphAI предназначен только для исследования данных и научных целей. Мы не несем ответственности за неправильное использование библиотеки.
|
||||
|
||||
Made with ❤️ by [ScrapeGraph AI](https://scrapegraphai.com)
|
||||
|
||||
[Scarf tracking](https://static.scarf.sh/a.png?x-pxid=102d4b8c-cd6a-4b9e-9a16-d6d141b9212d)
|
||||
+239
@@ -0,0 +1,239 @@
|
||||
## 🚀 **¿Buscas una forma aún más rápida y sencilla de hacer scraping a escala (con solo 5 líneas de código)?** ¡Checa nuestra versión mejorada en [**ScrapeGraphAI.com**](https://scrapegraphai.com/?utm_source=github&utm_medium=readme&utm_campaign=oss_cta&utm_content=top_banner)! 🚀
|
||||
|
||||
---
|
||||
|
||||
# 🕷️ ScrapeGraphAI: You Only Scrape Once
|
||||
|
||||
<p align="center">
|
||||
<a href="https://scrapegraphai.com">
|
||||
<img src="https://raw.githubusercontent.com/ScrapeGraphAI/Scrapegraph-ai/main/media/banner.png" alt="ScrapeGraphAI" style="width: 100%;">
|
||||
</a>
|
||||
</p>
|
||||
|
||||
[English](../README.md) | [中文](chinese.md) | [日本語](japanese.md)
|
||||
| [한국어](korean.md)
|
||||
| [Русский](russian.md) | [Türkçe](turkish.md)
|
||||
| [Deutsch](german.md)
|
||||
| [Español](spanish.md)
|
||||
| [français](french.md)
|
||||
| [Português](portuguese.md)
|
||||
| [Italiano](italian.md)
|
||||
|
||||
[](https://pepy.tech/projects/scrapegraphai)
|
||||
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
[](https://discord.gg/gkxQDAjfeX)
|
||||
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/15078" target="_blank"><img src="https://trendshift.io/api/badge/repositories/15078" alt="ScrapeGraphAI%2FScrapegraph-ai | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
<p align="center">
|
||||
|
||||
[ScrapeGraphAI](https://scrapegraphai.com) es una librería de Python para *web scraping* que usa LLMs y lógica de grafos para crear pipelines de extracción de datos en sitios web y documentos locales (XML, HTML, JSON, Markdown, etc.).
|
||||
|
||||
¡Solo dile qué información quieres extraer y la librería lo hace por ti!
|
||||
|
||||
<p align="center">
|
||||
<img src="https://raw.githubusercontent.com/ScrapeGraphAI/Scrapegraph-ai/main/docs/assets/sgai-hero.png" alt="ScrapeGraphAI Hero" style="width: 100%;">
|
||||
</p>
|
||||
|
||||
## 🚀 Integraciones
|
||||
|
||||
ScrapeGraphAI se integra de forma nativa con los frameworks y herramientas más populares para potenciar tus capacidades de scraping. Ya sea que estés desarrollando en Python o Node.js, usando frameworks de LLMs o plataformas no-code, tenemos todo cubierto.
|
||||
|
||||
Puedes encontrar más información en el siguiente [link](https://scrapegraphai.com)
|
||||
|
||||
**Integraciones**:
|
||||
- **API**: [Documentación](https://docs.scrapegraphai.com/introduction)
|
||||
- **SDKs**: [Python](https://docs.scrapegraphai.com/sdks/python), [Node](https://docs.scrapegraphai.com/sdks/javascript)
|
||||
- **Frameworks de LLMs**: [Langchain](https://docs.scrapegraphai.com/integrations/langchain), [Llama Index](https://docs.scrapegraphai.com/integrations/llamaindex), [Crew.ai](https://docs.scrapegraphai.com/integrations/crewai), [Agno](https://docs.scrapegraphai.com/integrations/agno), [CamelAI](https://github.com/camel-ai/camel)
|
||||
- **Frameworks low-code**: [Pipedream](https://pipedream.com/apps/scrapegraphai), [Bubble](https://bubble.io/plugin/scrapegraphai-1745408893195x213542371433906180), [Zapier](https://zapier.com/apps/scrapegraphai/integrations), [n8n](http://localhost:5001/dashboard), [Dify](https://dify.ai), [Toolhouse](https://app.toolhouse.ai/mcp-servers/scrapegraph_smartscraper)
|
||||
- **Servidor MCP**: [Link](https://smithery.ai/server/@ScrapeGraphAI/scrapegraph-mcp)
|
||||
|
||||
## 🚀 Instalación rápida
|
||||
|
||||
La página de referencia de Scrapegraph-ai está disponible en PyPI: [pypi](https://pypi.org/project/scrapegraphai/).
|
||||
|
||||
```bash
|
||||
pip install scrapegraphai
|
||||
|
||||
# IMPORTANTE (para hacer fetch del contenido de los sitios web)
|
||||
playwright install
|
||||
```
|
||||
|
||||
**Nota**: se recomienda instalar la librería en un entorno virtual para evitar conflictos con otras dependencias 🐱
|
||||
|
||||
|
||||
## 💻 Uso
|
||||
|
||||
Hay múltiples pipelines de scraping estándar para extraer información de un sitio web o archivo local.
|
||||
|
||||
El más común es `SmartScraperGraph`, que extrae información de una sola página dado un prompt y una URL de origen.
|
||||
|
||||
```python
|
||||
from scrapegraphai.graphs import SmartScraperGraph
|
||||
|
||||
# Define the configuration for the scraping pipeline
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"model": "ollama/llama3.2",
|
||||
"model_tokens": 8192,
|
||||
"format": "json",
|
||||
},
|
||||
"verbose": True,
|
||||
"headless": False,
|
||||
}
|
||||
|
||||
# Create the SmartScraperGraph instance
|
||||
smart_scraper_graph = SmartScraperGraph(
|
||||
prompt="Extract useful information from the webpage, including a description of what the company does, founders and social media links",
|
||||
source="https://scrapegraphai.com/",
|
||||
config=graph_config
|
||||
)
|
||||
|
||||
# Run the pipeline
|
||||
result = smart_scraper_graph.run()
|
||||
|
||||
import json
|
||||
print(json.dumps(result, indent=4))
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
> Para OpenAI y otros modelos solo necesitas cambiar el config del LLM:
|
||||
> ```python
|
||||
> graph_config = {
|
||||
> "llm": {
|
||||
> "api_key": "YOUR_OPENAI_API_KEY",
|
||||
> "model": "openai/gpt-4o-mini",
|
||||
> },
|
||||
> "verbose": True,
|
||||
> "headless": False,
|
||||
> }
|
||||
> ```
|
||||
|
||||
|
||||
El output será un diccionario como el siguiente:
|
||||
|
||||
```python
|
||||
{
|
||||
"description": "ScrapeGraphAI transforms websites into clean, organized data for AI agents and data analytics. It offers an AI-powered API for effortless and cost-effective data extraction.",
|
||||
"founders": [
|
||||
{
|
||||
"name": "",
|
||||
"role": "Founder & Technical Lead",
|
||||
"linkedin": "https://www.linkedin.com/in/perinim/"
|
||||
},
|
||||
{
|
||||
"name": "Marco Vinciguerra",
|
||||
"role": "Founder & Software Engineer",
|
||||
"linkedin": "https://www.linkedin.com/in/marco-vinciguerra-7ba365242/"
|
||||
},
|
||||
{
|
||||
"name": "Lorenzo Padoan",
|
||||
"role": "Founder & Product Engineer",
|
||||
"linkedin": "https://www.linkedin.com/in/lorenzo-padoan-4521a2154/"
|
||||
}
|
||||
],
|
||||
"social_media_links": {
|
||||
"linkedin": "https://www.linkedin.com/company/101881123",
|
||||
"twitter": "https://x.com/scrapegraphai",
|
||||
"github": "https://github.com/ScrapeGraphAI/Scrapegraph-ai"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Existen otros pipelines para extraer información de múltiples páginas, generar scripts de Python o incluso generar archivos de audio.
|
||||
|
||||
| Nombre del Pipeline | Descripción |
|
||||
|-------------------------|-----------------------------------------------------------------------------------------------------------------------|
|
||||
| SmartScraperGraph | Scraper de una sola página que solo requiere un prompt y una fuente de entrada. |
|
||||
| SearchGraph | Scraper multi-página que extrae información de los primeros n resultados de un motor de búsqueda. |
|
||||
| SpeechGraph | Scraper de una sola página que extrae información de un sitio web y genera un archivo de audio. |
|
||||
| ScriptCreatorGraph | Scraper de una sola página que extrae información de un sitio web y genera un script de Python. |
|
||||
| SmartScraperMultiGraph | Scraper multi-página que extrae información de múltiples páginas con un solo prompt y una lista de fuentes. |
|
||||
| ScriptCreatorMultiGraph | Scraper multi-página que genera un script de Python para extraer información de múltiples páginas y fuentes. |
|
||||
|
||||
Cada uno de estos grafos tiene su versión multi, que permite hacer llamadas al LLM en paralelo.
|
||||
|
||||
Es posible usar diferentes LLMs mediante APIs como **OpenAI**, **Groq**, **Azure**, **Gemini**, **MiniMax** y más, o modelos locales usando **Ollama**.
|
||||
|
||||
Recuerda tener [Ollama](https://ollama.com/) instalado y descargar los modelos con el comando **ollama pull** si quieres usar modelos locales.
|
||||
|
||||
|
||||
## 📖 Documentación
|
||||
|
||||
[](https://colab.research.google.com/drive/1sEZBonBMGP44CtO6GQTwAlL0BGJXjtfd?usp=sharing)
|
||||
|
||||
La documentación de ScrapeGraphAI está disponible [aquí](https://docs.scrapegraphai.com/introduction).
|
||||
|
||||
## 🤝 Contribuciones
|
||||
|
||||
¡Siéntete libre de contribuir y únete a nuestro servidor de Discord para discutir mejoras y compartir sugerencias!
|
||||
|
||||
Por favor revisa las [guías de contribución](https://github.com/ScrapeGraphAI/Scrapegraph-ai/blob/main/CONTRIBUTING.md).
|
||||
|
||||
[](https://discord.gg/uJN7TYcpNa)
|
||||
[](https://www.linkedin.com/company/scrapegraphai/)
|
||||
[](https://twitter.com/scrapegraphai)
|
||||
|
||||
## 🔗 API y SDKs de ScrapeGraph
|
||||
|
||||
Si buscas una solución rápida para integrar ScrapeGraph en tu sistema, checa nuestra API [aquí](https://dashboard.scrapegraphai.com/login).
|
||||
|
||||
[](https://dashboard.scrapegraphai.com/login)
|
||||
|
||||
Ofrecemos SDKs en Python y Node.js para facilitar la integración en tus proyectos:
|
||||
|
||||
| SDK | Lenguaje | GitHub Link |
|
||||
|-------------|----------|----------------------------------------------------------------------------------------------|
|
||||
| Python SDK | Python | [scrapegraph-py](https://docs.scrapegraphai.com/sdks/python) |
|
||||
| Node.js SDK | Node.js | [scrapegraph-js](https://docs.scrapegraphai.com/sdks/javascript) |
|
||||
|
||||
La documentación oficial de la API está disponible [aquí](https://docs.scrapegraphai.com/introduction).
|
||||
|
||||
## 🔥 Benchmark
|
||||
|
||||
¡Según el benchmark de Firecrawl [Firecrawl benchmark](https://scrapegraphai.com/compare/firecrawl), ScrapeGraph es el mejor fetcher del mercado!
|
||||
|
||||
|
||||
|
||||
## 📈 Telemetría
|
||||
|
||||
Recopilamos métricas de uso anónimas para mejorar la calidad del paquete y la experiencia del usuario. Los datos nos ayudan a priorizar mejoras y garantizar compatibilidad. Si deseas hacer opt-out, configura la variable de entorno `SCRAPEGRAPHAI_TELEMETRY_ENABLED=false`. Para más información consulta la documentación [aquí](https://docs.scrapegraphai.com/introduction).
|
||||
|
||||
## ❤️ Contributors
|
||||
|
||||
[](https://github.com/ScrapeGraphAI/Scrapegraph-ai/graphs/contributors)
|
||||
|
||||
## 🎓 Citas
|
||||
|
||||
Si usaste nuestra librería para investigación, por favor cítanos con la siguiente referencia:
|
||||
|
||||
```text
|
||||
@misc{scrapegraph-ai,
|
||||
author = {Lorenzo Padoan, Marco Vinciguerra},
|
||||
title = {Scrapegraph-ai},
|
||||
year = {2024},
|
||||
url = {https://github.com/ScrapeGraphAI/Scrapegraph-ai},
|
||||
note = {A Python library for scraping leveraging large language models}
|
||||
}
|
||||
```
|
||||
|
||||
## Autores
|
||||
|
||||
| | Contacto |
|
||||
|--------------------|----------------------|
|
||||
| Marco Vinciguerra | [](https://www.linkedin.com/in/marco-vinciguerra-7ba365242/) |
|
||||
| Lorenzo Padoan | [](https://www.linkedin.com/in/lorenzo-padoan-4521a2154/) |
|
||||
|
||||
## 📜 Licencia
|
||||
|
||||
ScrapeGraphAI está licenciado bajo la Licencia MIT. Consulta el archivo [LICENSE](https://github.com/ScrapeGraphAI/Scrapegraph-ai/blob/main/LICENSE) para más información.
|
||||
|
||||
## Agradecimientos
|
||||
|
||||
- Queremos agradecer a todos los contributors del proyecto y a la comunidad open-source por su apoyo.
|
||||
- ScrapeGraphAI está pensado únicamente para exploración de datos e investigación. No nos hacemos responsables del mal uso de la librería.
|
||||
|
||||
Hecho con ❤️ por [ScrapeGraph AI](https://scrapegraphai.com)
|
||||
|
||||
[Scarf tracking](https://static.scarf.sh/a.png?x-pxid=102d4b8c-cd6a-4b9e-9a16-d6d141b9212d)
|
||||
@@ -0,0 +1,292 @@
|
||||
# FetchNode Timeout Configuration
|
||||
|
||||
## Overview
|
||||
|
||||
The `FetchNode` in ScrapeGraphAI supports configurable timeouts for all blocking operations to prevent indefinite hangs when fetching web content or parsing files. This feature allows you to control execution time limits for:
|
||||
|
||||
- HTTP requests (when using `use_soup=True`)
|
||||
- PDF file parsing
|
||||
- ChromiumLoader operations
|
||||
|
||||
## Configuration
|
||||
|
||||
### Default Behavior
|
||||
|
||||
By default, `FetchNode` uses a **30-second timeout** for all blocking operations when a `node_config` is provided:
|
||||
|
||||
```python
|
||||
from scrapegraphai.nodes import FetchNode
|
||||
|
||||
# Default 30-second timeout
|
||||
node = FetchNode(
|
||||
input="url",
|
||||
output=["doc"],
|
||||
node_config={}
|
||||
)
|
||||
```
|
||||
|
||||
### Custom Timeout
|
||||
|
||||
You can specify a custom timeout value (in seconds) via the `timeout` parameter:
|
||||
|
||||
```python
|
||||
# Custom 10-second timeout
|
||||
node = FetchNode(
|
||||
input="url",
|
||||
output=["doc"],
|
||||
node_config={"timeout": 10}
|
||||
)
|
||||
```
|
||||
|
||||
### Disabling Timeout
|
||||
|
||||
To disable timeout and allow operations to run indefinitely, set `timeout` to `None`:
|
||||
|
||||
```python
|
||||
# No timeout - operations will wait indefinitely
|
||||
node = FetchNode(
|
||||
input="url",
|
||||
output=["doc"],
|
||||
node_config={"timeout": None}
|
||||
)
|
||||
```
|
||||
|
||||
### No Configuration
|
||||
|
||||
If you don't provide any `node_config`, the timeout defaults to `None` (no timeout):
|
||||
|
||||
```python
|
||||
# No timeout (backward compatible)
|
||||
node = FetchNode(
|
||||
input="url",
|
||||
output=["doc"],
|
||||
node_config=None
|
||||
)
|
||||
```
|
||||
|
||||
## Use Cases
|
||||
|
||||
### HTTP Requests
|
||||
|
||||
When `use_soup=True`, the timeout applies to `requests.get()` calls:
|
||||
|
||||
```python
|
||||
node = FetchNode(
|
||||
input="url",
|
||||
output=["doc"],
|
||||
node_config={
|
||||
"use_soup": True,
|
||||
"timeout": 15 # HTTP request will timeout after 15 seconds
|
||||
}
|
||||
)
|
||||
|
||||
state = {"url": "https://example.com"}
|
||||
result = node.execute(state)
|
||||
```
|
||||
|
||||
If the timeout is `None`, no timeout parameter is passed to `requests.get()`:
|
||||
|
||||
```python
|
||||
node = FetchNode(
|
||||
input="url",
|
||||
output=["doc"],
|
||||
node_config={
|
||||
"use_soup": True,
|
||||
"timeout": None # No timeout for HTTP requests
|
||||
}
|
||||
)
|
||||
```
|
||||
|
||||
### PDF Parsing
|
||||
|
||||
The timeout applies to PDF file parsing operations using `PyPDFLoader`:
|
||||
|
||||
```python
|
||||
node = FetchNode(
|
||||
input="pdf",
|
||||
output=["doc"],
|
||||
node_config={
|
||||
"timeout": 60 # PDF parsing will timeout after 60 seconds
|
||||
}
|
||||
)
|
||||
|
||||
state = {"pdf": "/path/to/large_document.pdf"}
|
||||
try:
|
||||
result = node.execute(state)
|
||||
except TimeoutError as e:
|
||||
print(f"PDF parsing took too long: {e}")
|
||||
```
|
||||
|
||||
If parsing exceeds the timeout, a `TimeoutError` is raised with a descriptive message:
|
||||
|
||||
```
|
||||
TimeoutError: PDF parsing exceeded timeout of 60 seconds
|
||||
```
|
||||
|
||||
### ChromiumLoader
|
||||
|
||||
The timeout is automatically propagated to `ChromiumLoader` via `loader_kwargs`:
|
||||
|
||||
```python
|
||||
node = FetchNode(
|
||||
input="url",
|
||||
output=["doc"],
|
||||
node_config={
|
||||
"timeout": 30, # ChromiumLoader will use 30-second timeout
|
||||
"headless": True
|
||||
}
|
||||
)
|
||||
|
||||
state = {"url": "https://example.com"}
|
||||
result = node.execute(state)
|
||||
```
|
||||
|
||||
If you need different timeout behavior for ChromiumLoader specifically, you can override it in `loader_kwargs`:
|
||||
|
||||
```python
|
||||
node = FetchNode(
|
||||
input="url",
|
||||
output=["doc"],
|
||||
node_config={
|
||||
"timeout": 30, # General timeout for other operations
|
||||
"loader_kwargs": {
|
||||
"timeout": 60 # ChromiumLoader gets 60-second timeout
|
||||
}
|
||||
}
|
||||
)
|
||||
```
|
||||
|
||||
## Graph Examples
|
||||
|
||||
### SmartScraperGraph
|
||||
|
||||
```python
|
||||
from scrapegraphai.graphs import SmartScraperGraph
|
||||
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"model": "gpt-3.5-turbo",
|
||||
"api_key": "your-api-key"
|
||||
},
|
||||
"timeout": 20 # 20-second timeout for fetch operations
|
||||
}
|
||||
|
||||
smart_scraper = SmartScraperGraph(
|
||||
prompt="Extract all article titles",
|
||||
source="https://news.example.com",
|
||||
config=graph_config
|
||||
)
|
||||
|
||||
result = smart_scraper.run()
|
||||
```
|
||||
|
||||
### Custom Graph with FetchNode
|
||||
|
||||
```python
|
||||
from scrapegraphai.nodes import FetchNode
|
||||
from langgraph.graph import StateGraph
|
||||
|
||||
# Create a custom graph with timeout
|
||||
fetch_node = FetchNode(
|
||||
input="url",
|
||||
output=["doc"],
|
||||
node_config={
|
||||
"timeout": 15,
|
||||
"headless": True
|
||||
}
|
||||
)
|
||||
|
||||
# Add to graph...
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
1. **Choose appropriate timeouts**: Consider the expected response time of your target websites
|
||||
- Fast APIs: 5-10 seconds
|
||||
- Regular websites: 15-30 seconds
|
||||
- Large PDFs or slow sites: 60+ seconds
|
||||
|
||||
2. **Handle TimeoutError**: Always wrap your code in try-except when using timeouts:
|
||||
|
||||
```python
|
||||
try:
|
||||
result = node.execute(state)
|
||||
except TimeoutError as e:
|
||||
logger.error(f"Operation timed out: {e}")
|
||||
# Handle timeout gracefully
|
||||
```
|
||||
|
||||
3. **Use different timeouts for different operations**: Set higher timeouts for PDF parsing and lower for HTTP requests:
|
||||
|
||||
```python
|
||||
# For PDFs
|
||||
pdf_node = FetchNode("pdf", ["doc"], {"timeout": 120})
|
||||
|
||||
# For web pages
|
||||
web_node = FetchNode("url", ["doc"], {"timeout": 15})
|
||||
```
|
||||
|
||||
4. **Monitor timeout occurrences**: Log timeout errors to identify problematic sources:
|
||||
|
||||
```python
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
try:
|
||||
result = node.execute(state)
|
||||
except TimeoutError as e:
|
||||
logger.warning(f"Timeout for {state.get('url', 'unknown')}: {e}")
|
||||
```
|
||||
|
||||
## Implementation Details
|
||||
|
||||
The timeout feature is implemented using:
|
||||
|
||||
- **HTTP requests**: `requests.get(url, timeout=X)` parameter
|
||||
- **PDF parsing**: `concurrent.futures.ThreadPoolExecutor` with `future.result(timeout=X)`
|
||||
- **ChromiumLoader**: Propagated via `loader_kwargs` dictionary
|
||||
|
||||
When `timeout=None`, no timeout constraints are applied, allowing operations to run until completion.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Timeout is too short
|
||||
|
||||
If you're seeing frequent timeout errors, increase the timeout value:
|
||||
|
||||
```python
|
||||
node_config = {"timeout": 60} # Increase from 30 to 60 seconds
|
||||
```
|
||||
|
||||
### Need different timeouts for different operations
|
||||
|
||||
Use separate FetchNode instances with different configurations:
|
||||
|
||||
```python
|
||||
fast_fetcher = FetchNode("url", ["doc"], {"timeout": 10})
|
||||
slow_fetcher = FetchNode("pdf", ["doc"], {"timeout": 120})
|
||||
```
|
||||
|
||||
### ChromiumLoader timeout not working
|
||||
|
||||
Ensure you're not overriding the timeout in `loader_kwargs`:
|
||||
|
||||
```python
|
||||
# ❌ Wrong - explicit loader_kwargs timeout overrides node timeout
|
||||
node_config = {
|
||||
"timeout": 30,
|
||||
"loader_kwargs": {"timeout": 10} # This takes precedence
|
||||
}
|
||||
|
||||
# ✅ Correct - let node timeout propagate
|
||||
node_config = {
|
||||
"timeout": 30 # ChromiumLoader will use 30 seconds
|
||||
}
|
||||
```
|
||||
|
||||
## See Also
|
||||
|
||||
- [FetchNode API Documentation](../api/nodes/fetch_node.md)
|
||||
- [Graph Configuration](./graph_configuration.md)
|
||||
- [Error Handling](./error_handling.md)
|
||||
+231
@@ -0,0 +1,231 @@
|
||||
## 🚀 **Daha hızlı ve daha basit bir ölçekli kazıma yöntemi (sadece 5 satır kod) mi arıyorsunuz?** [**ScrapeGraphAI.com**](https://scrapegraphai.com/?utm_source=github&utm_medium=readme&utm_campaign=oss_cta&utm_content=top_banner)'daki geliştirilmiş sürümümüze göz atın! 🚀
|
||||
|
||||
---
|
||||
|
||||
# 🕷️ ScrapeGraphAI: Yalnızca Bir Kez Kazıyın
|
||||
|
||||
[English](../README.md) | [中文](chinese.md) | [日本語](japanese.md)
|
||||
| [한국어](korean.md)
|
||||
| [Русский](russian.md) | [Türkçe](turkish.md)
|
||||
| [Deutsch](german.md)
|
||||
| [Español](spanish.md)
|
||||
| [français](french.md)
|
||||
| [Português](portuguese.md)
|
||||
| [Italiano](italian.md)
|
||||
|
||||
[](https://pepy.tech/projects/scrapegraphai)
|
||||
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
[](https://discord.gg/gkxQDAjfeX)
|
||||
<p align="center">
|
||||
<a href="https://trendshift.io/repositories/15078" target="_blank"><img src="https://trendshift.io/api/badge/repositories/15078" alt="ScrapeGraphAI%2FScrapegraph-ai | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
<p align="center">
|
||||
|
||||
ScrapeGraphAI, LLM ve grafik mantığını kullanarak web siteleri ve yerel belgeler (XML, HTML, JSON, Markdown vb.) için kazıma süreçleri oluşturan bir _web kazıma_ Python kütüphanesidir.
|
||||
|
||||
Sadece hangi bilgiyi çıkarmak istediğinizi söyleyin, kütüphane sizin için yapar!
|
||||
|
||||
|
||||
|
||||
## 🚀 Entegrasyonlar
|
||||
|
||||
ScrapeGraphAI, kazıma yeteneklerinizi geliştirmek için popüler çerçeveler ve araçlarla sorunsuz entegrasyon sunar. Python veya Node.js ile geliştirme yapıyor olsanız da, LLM çerçeveleri kullanıyor olsanız da, no-code platformlarda çalışıyor olsanız da, kapsamlı entegrasyon seçeneklerimizle yanınızdayız.
|
||||
|
||||
<p align="center">
|
||||
<img src="https://raw.githubusercontent.com/ScrapeGraphAI/Scrapegraph-ai/main/docs/assets/sgai-hero.png" alt="ScrapeGraphAI Hero" style="width: 100%;">
|
||||
</p>
|
||||
Daha fazla bilgiyi aşağıdaki [bağlantıda](https://scrapegraphai.com) bulabilirsiniz
|
||||
|
||||
**Entegrasyonlar**:
|
||||
- **API**: [Dokümantasyon](https://docs.scrapegraphai.com/introduction)
|
||||
- **SDKs**: [Python](https://docs.scrapegraphai.com/sdks/python), [Node](https://docs.scrapegraphai.com/sdks/javascript)
|
||||
- **LLM Çerçeveleri**: [Langchain](https://docs.scrapegraphai.com/integrations/langchain), [Llama Index](https://docs.scrapegraphai.com/integrations/llamaindex), [Crew.ai](https://docs.scrapegraphai.com/integrations/crewai), [Agno](https://docs.scrapegraphai.com/integrations/agno), [CamelAI](https://github.com/camel-ai/camel)
|
||||
- **Low-code Çerçeveleri**: [Pipedream](https://pipedream.com/apps/scrapegraphai), [Bubble](https://bubble.io/plugin/scrapegraphai-1745408893195x213542371433906180), [Zapier](https://zapier.com/apps/scrapegraphai/integrations), [n8n](http://localhost:5001/dashboard), [Dify](https://dify.ai), [Toolhouse](https://app.toolhouse.ai/mcp-servers/scrapegraph_smartscraper)
|
||||
- **MCP sunucusu**: [Bağlantı](https://smithery.ai/server/@ScrapeGraphAI/scrapegraph-mcp)
|
||||
|
||||
## 🚀 Hızlı Kurulum
|
||||
|
||||
Scrapegraph-ai için referans sayfası PyPI'nin resmi sayfasında mevcuttur: [pypi](https://pypi.org/project/scrapegraphai/).
|
||||
|
||||
```bash
|
||||
pip install scrapegraphai
|
||||
|
||||
# ÖNEMLİ (web sitesi içeriğini almak için)
|
||||
playwright install
|
||||
```
|
||||
|
||||
**Not**: Diğer kütüphanelerle çakışmaları önlemek için kütüphaneyi sanal bir ortamda kurmanız önerilir 🐱
|
||||
|
||||
|
||||
## 💻 Kullanım
|
||||
|
||||
Web sitesinden (veya yerel dosyadan) bilgi çıkarmak için kullanılabilecek birden fazla standart kazıma süreci vardır.
|
||||
|
||||
En yaygın olanı `SmartScraperGraph`'tır; bu, bir kullanıcı isteği ve kaynak URL'si verildiğinde tek bir sayfadan bilgi çıkarır.
|
||||
|
||||
|
||||
```python
|
||||
from scrapegraphai.graphs import SmartScraperGraph
|
||||
|
||||
# Kazıma süreci için yapılandırmayı tanımlayın
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"model": "ollama/llama3.2",
|
||||
"model_tokens": 8192,
|
||||
"format": "json",
|
||||
},
|
||||
"verbose": True,
|
||||
"headless": False,
|
||||
}
|
||||
|
||||
# SmartScraperGraph örneğini oluşturun
|
||||
smart_scraper_graph = SmartScraperGraph(
|
||||
prompt="Web sayfasından yararlı bilgileri çıkarın, şirketin ne yaptığına dair bir açıklama, kurucular ve sosyal medya bağlantılarını dahil edin",
|
||||
source="https://scrapegraphai.com/",
|
||||
config=graph_config
|
||||
)
|
||||
|
||||
# Süreci çalıştırın
|
||||
result = smart_scraper_graph.run()
|
||||
|
||||
import json
|
||||
print(json.dumps(result, indent=4))
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
> OpenAI ve diğer modeller için sadece llm yapılandırmasını değiştirmeniz yeterlidir!
|
||||
> ```python
|
||||
>graph_config = {
|
||||
> "llm": {
|
||||
> "api_key": "YOUR_OPENAI_API_KEY",
|
||||
> "model": "openai/gpt-4o-mini",
|
||||
> },
|
||||
> "verbose": True,
|
||||
> "headless": False,
|
||||
>}
|
||||
>```
|
||||
|
||||
|
||||
Çıktı aşağıdaki gibi bir sözlük olacaktır:
|
||||
|
||||
```python
|
||||
{
|
||||
"description": "ScrapeGraphAI transforms websites into clean, organized data for AI agents and data analytics. It offers an AI-powered API for effortless and cost-effective data extraction.",
|
||||
"founders": [
|
||||
{
|
||||
"name": "",
|
||||
"role": "Founder & Technical Lead",
|
||||
"linkedin": "https://www.linkedin.com/in/perinim/"
|
||||
},
|
||||
{
|
||||
"name": "Marco Vinciguerra",
|
||||
"role": "Founder & Software Engineer",
|
||||
"linkedin": "https://www.linkedin.com/in/marco-vinciguerra-7ba365242/"
|
||||
},
|
||||
{
|
||||
"name": "Lorenzo Padoan",
|
||||
"role": "Founder & Product Engineer",
|
||||
"linkedin": "https://www.linkedin.com/in/lorenzo-padoan-4521a2154/"
|
||||
}
|
||||
],
|
||||
"social_media_links": {
|
||||
"linkedin": "https://www.linkedin.com/company/101881123",
|
||||
"twitter": "https://x.com/scrapegraphai",
|
||||
"github": "https://github.com/ScrapeGraphAI/Scrapegraph-ai"
|
||||
}
|
||||
}
|
||||
```
|
||||
Birden fazla sayfadan bilgi çıkarmak, Python scriptleri oluşturmak veya hatta ses dosyaları oluşturmak için kullanılabilecek diğer süreçler de vardır.
|
||||
|
||||
| Süreç Adı | Açıklama |
|
||||
| ----------------------- | -------------------------------------------------------------------------------------------------------- |
|
||||
| SmartScraperGraph | Sadece bir kullanıcı isteği ve bir kaynak girişi gerektiren tek sayfalık kazıyıcı. |
|
||||
| SearchGraph | Bir arama motorunun en iyi n arama sonucundan bilgi çıkaran çok sayfalı kazıyıcı. |
|
||||
| SpeechGraph | Bir web sitesinden bilgi çıkaran ve bir ses dosyası oluşturan tek sayfalık kazıyıcı. |
|
||||
| ScriptCreatorGraph | Bir web sitesinden bilgi çıkaran ve bir Python scripti oluşturan tek sayfalık kazıyıcı. |
|
||||
| SmartScraperMultiGraph | Tek bir bilgi istemi ve kaynak listesi verilen birden çok sayfadan bilgi ayıklayan çok sayfalı kazıyıcı. |
|
||||
| ScriptCreatorMultiGraph | Birden fazla sayfa veya kaynaktan bilgi çıkarmak için bir Python scripti oluşturan çok sayfalı kazıyıcı. |
|
||||
|
||||
Bu süreçlerin her biri için çoklu versiyon vardır. Bu, LLM çağrılarını paralel olarak yapmanızı sağlar.
|
||||
|
||||
**OpenAI**, **Groq**, **Azure** ve **Gemini** gibi API'ler aracılığıyla farklı LLM'leri kullanmak veya **Ollama** kullanarak yerel modelleri kullanmak mümkündür.
|
||||
|
||||
Yerel modelleri kullanmak istiyorsanız, [Ollama](https://ollama.com/) kurulu olduğundan ve **ollama pull** komutunu kullanarak modelleri indirdiğinizden emin olun.
|
||||
|
||||
|
||||
## 📖 Dokümantasyon
|
||||
|
||||
[](https://colab.research.google.com/drive/1sEZBonBMGP44CtO6GQTwAlL0BGJXjtfd?usp=sharing)
|
||||
|
||||
ScrapeGraphAI dokümantasyonuna [buradan](https://docs.scrapegraphai.com/introduction) ulaşabilirsiniz.
|
||||
|
||||
## 🤝 Katkıda Bulunun
|
||||
|
||||
Projeye katkıda bulunmaktan çekinmeyin ve geliştirmeleri tartışmak ve bize önerilerde bulunmak için Discord sunucumuza katılın!
|
||||
|
||||
Lütfen [katkıda bulunma yönergelerine](https://github.com/ScrapeGraphAI/Scrapegraph-ai/blob/main/CONTRIBUTING.md) bakın.
|
||||
|
||||
[](https://discord.gg/uJN7TYcpNa)
|
||||
[](https://www.linkedin.com/company/scrapegraphai/)
|
||||
[](https://twitter.com/scrapegraphai)
|
||||
|
||||
## 🔗 ScrapeGraph API & SDKs
|
||||
Sisteminize ScrapeGraph'u entegre etmek için hızlı bir çözüm arıyorsanız, güçlü API'mizi [burada!](https://dashboard.scrapegraphai.com/login) kontrol edin
|
||||
|
||||
[](https://dashboard.scrapegraphai.com/login)
|
||||
|
||||
Python ve Node.js için SDK'lar sunuyoruz, böylece projelerinize kolayca entegre edebilirsiniz. Aşağıda kontrol edin:
|
||||
|
||||
| SDK | Dil | GitHub Bağlantısı |
|
||||
|-----------|----------|-----------------------------------------------------------------------------|
|
||||
| Python SDK | Python | [scrapegraph-py](https://docs.scrapegraphai.com/sdks/python) |
|
||||
| Node.js SDK | Node.js | [scrapegraph-js](https://docs.scrapegraphai.com/sdks/javascript) |
|
||||
|
||||
Resmi API Dokümantasyonu [burada](https://docs.scrapegraphai.com/introduction) bulunabilir.
|
||||
|
||||
## 🔥 Kıyaslama
|
||||
|
||||
Firecrawl kıyaslamasına göre [Firecrawl benchmark](https://scrapegraphai.com/compare/firecrawl), ScrapeGraph piyasadaki en iyi getirici!
|
||||
|
||||
|
||||
|
||||
## 📈 Telemetri
|
||||
Paketimizin kalitesini ve kullanıcı deneyimini geliştirmek amacıyla anonim kullanım metrikleri topluyoruz. Bu veriler, iyileştirmelere öncelik vermemize ve uyumluluğu sağlamamıza yardımcı olur. İsterseniz, SCRAPEGRAPHAI_TELEMETRY_ENABLED=false ortam değişkenini ayarlayarak devre dışı bırakabilirsiniz. Daha fazla bilgi için lütfen [buraya](https://docs.scrapegraphai.com/introduction) bakın.
|
||||
|
||||
## ❤️ Katkıda Bulunanlar
|
||||
|
||||
[](https://github.com/ScrapeGraphAI/Scrapegraph-ai/graphs/contributors)
|
||||
|
||||
## 🎓 Atıflar
|
||||
|
||||
Kütüphanemizi araştırma amaçlı kullandıysanız, lütfen bizi aşağıdaki referansla alıntılayın:
|
||||
|
||||
```text
|
||||
@misc{scrapegraph-ai,
|
||||
author = {Lorenzo Padoan, Marco Vinciguerra},
|
||||
title = {Scrapegraph-ai},
|
||||
year = {2024},
|
||||
url = {https://github.com/ScrapeGraphAI/Scrapegraph-ai},
|
||||
note = {Büyük dil modellerinden yararlanan kazıma için bir Python kütüphanesi}
|
||||
}
|
||||
```
|
||||
|
||||
## Yazarlar
|
||||
|
||||
| | İletişim Bilgileri |
|
||||
| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| Marco Vinciguerra | [](https://www.linkedin.com/in/marco-vinciguerra-7ba365242/) |
|
||||
| Lorenzo Padoan | [](https://www.linkedin.com/in/lorenzo-padoan-4521a2154/) |
|
||||
|
||||
## 📜 Lisans
|
||||
|
||||
ScrapeGraphAI, MIT Lisansı altında lisanslanmıştır. Daha fazla bilgi için [LİSANS](https://github.com/ScrapeGraphAI/Scrapegraph-ai/blob/main/LICENSE) dosyasına bakın.
|
||||
|
||||
## Teşekkürler
|
||||
|
||||
- Projeye katkıda bulunan tüm katılımcılara ve açık kaynak topluluğuna destekleri için teşekkür ederiz.
|
||||
- ScrapeGraphAI, yalnızca veri arama ve araştırma amacıyla kullanılmak üzere tasarlanmıştır. Kütüphanenin kötüye kullanılmasından sorumlu değiliz.
|
||||
|
||||
Made with ❤️ by [ScrapeGraph AI](https://scrapegraphai.com)
|
||||
|
||||
[Scarf tracking](https://static.scarf.sh/a.png?x-pxid=102d4b8c-cd6a-4b9e-9a16-d6d141b9212d)
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,14 @@
|
||||
# OpenAI API Configuration
|
||||
OPENAI_API_KEY=your-openai-api-key-here
|
||||
|
||||
# Optional Configurations
|
||||
MAX_TOKENS=4000
|
||||
MODEL_NAME=gpt-4-1106-preview
|
||||
TEMPERATURE=0.7
|
||||
|
||||
# Code Generator Settings
|
||||
DEFAULT_LANGUAGE=python
|
||||
GENERATE_TESTS=true
|
||||
ADD_DOCUMENTATION=true
|
||||
CODE_STYLE=pep8
|
||||
TYPE_CHECKING=true
|
||||
@@ -0,0 +1,30 @@
|
||||
# Code Generator Graph Example
|
||||
|
||||
This example demonstrates how to use Scrapegraph-ai to generate code based on specifications and requirements.
|
||||
|
||||
## Features
|
||||
|
||||
- Code generation from specifications
|
||||
- Multiple programming languages support
|
||||
- Code documentation
|
||||
- Best practices implementation
|
||||
|
||||
## Setup
|
||||
|
||||
1. Install required dependencies
|
||||
2. Copy `.env.example` to `.env`
|
||||
3. Configure your API keys in the `.env` file
|
||||
|
||||
## Usage
|
||||
|
||||
```python
|
||||
from scrapegraphai.graphs import CodeGeneratorGraph
|
||||
|
||||
graph = CodeGeneratorGraph()
|
||||
code = graph.generate("code specification")
|
||||
```
|
||||
|
||||
## Environment Variables
|
||||
|
||||
Required environment variables:
|
||||
- `OPENAI_API_KEY`: Your OpenAI API key
|
||||
@@ -0,0 +1,65 @@
|
||||
"""
|
||||
Basic example of scraping pipeline using Code Generator with schema
|
||||
"""
|
||||
|
||||
from typing import List
|
||||
|
||||
from dotenv import load_dotenv
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from scrapegraphai.graphs import CodeGeneratorGraph
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# ************************************************
|
||||
# Define the output schema for the graph
|
||||
# ************************************************
|
||||
|
||||
|
||||
class Project(BaseModel):
|
||||
title: str = Field(description="The title of the project")
|
||||
description: str = Field(description="The description of the project")
|
||||
|
||||
|
||||
class Projects(BaseModel):
|
||||
projects: List[Project]
|
||||
|
||||
|
||||
# ************************************************
|
||||
# Define the configuration for the graph
|
||||
# ************************************************
|
||||
|
||||
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"model": "ollama/llama3",
|
||||
"temperature": 0,
|
||||
"format": "json",
|
||||
"base_url": "http://localhost:11434",
|
||||
},
|
||||
"verbose": True,
|
||||
"headless": False,
|
||||
"reduction": 2,
|
||||
"max_iterations": {
|
||||
"overall": 10,
|
||||
"syntax": 3,
|
||||
"execution": 3,
|
||||
"validation": 3,
|
||||
"semantic": 3,
|
||||
},
|
||||
"output_file_name": "extracted_data.py",
|
||||
}
|
||||
|
||||
# ************************************************
|
||||
# Create the SmartScraperGraph instance and run it
|
||||
# ************************************************
|
||||
|
||||
code_generator_graph = CodeGeneratorGraph(
|
||||
prompt="List me all the projects with their description",
|
||||
source="https://perinim.github.io/projects/",
|
||||
schema=Projects,
|
||||
config=graph_config,
|
||||
)
|
||||
|
||||
result = code_generator_graph.run()
|
||||
print(result)
|
||||
@@ -0,0 +1,65 @@
|
||||
"""
|
||||
Basic example of scraping pipeline using Code Generator with schema
|
||||
"""
|
||||
|
||||
import os
|
||||
from typing import List
|
||||
|
||||
from dotenv import load_dotenv
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from scrapegraphai.graphs import CodeGeneratorGraph
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# ************************************************
|
||||
# Define the output schema for the graph
|
||||
# ************************************************
|
||||
|
||||
|
||||
class Project(BaseModel):
|
||||
title: str = Field(description="The title of the project")
|
||||
description: str = Field(description="The description of the project")
|
||||
|
||||
|
||||
class Projects(BaseModel):
|
||||
projects: List[Project]
|
||||
|
||||
|
||||
# ************************************************
|
||||
# Define the configuration for the graph
|
||||
# ************************************************
|
||||
|
||||
openai_key = os.getenv("OPENAI_APIKEY")
|
||||
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"api_key": openai_key,
|
||||
"model": "openai/gpt-4o-mini",
|
||||
},
|
||||
"verbose": True,
|
||||
"headless": False,
|
||||
"reduction": 2,
|
||||
"max_iterations": {
|
||||
"overall": 10,
|
||||
"syntax": 3,
|
||||
"execution": 3,
|
||||
"validation": 3,
|
||||
"semantic": 3,
|
||||
},
|
||||
"output_file_name": "extracted_data.py",
|
||||
}
|
||||
|
||||
# ************************************************
|
||||
# Create the SmartScraperGraph instance and run it
|
||||
# ************************************************
|
||||
|
||||
code_generator_graph = CodeGeneratorGraph(
|
||||
prompt="List me all the projects with their description",
|
||||
source="https://perinim.github.io/projects/",
|
||||
schema=Projects,
|
||||
config=graph_config,
|
||||
)
|
||||
|
||||
result = code_generator_graph.run()
|
||||
print(result)
|
||||
@@ -0,0 +1,11 @@
|
||||
# OpenAI API Configuration
|
||||
OPENAI_API_KEY=your-openai-api-key-here
|
||||
|
||||
# Optional Configurations
|
||||
MAX_TOKENS=4000
|
||||
MODEL_NAME=gpt-4-1106-preview
|
||||
TEMPERATURE=0.7
|
||||
|
||||
# CSV Scraper Settings
|
||||
CSV_DELIMITER=,
|
||||
MAX_ROWS=1000
|
||||
@@ -0,0 +1,30 @@
|
||||
# CSV Scraper Graph Example
|
||||
|
||||
This example demonstrates how to use Scrapegraph-ai to extract data from web sources and save it in CSV format.
|
||||
|
||||
## Features
|
||||
|
||||
- Table data extraction
|
||||
- CSV formatting
|
||||
- Data cleaning
|
||||
- Structured output
|
||||
|
||||
## Setup
|
||||
|
||||
1. Install required dependencies
|
||||
2. Copy `.env.example` to `.env`
|
||||
3. Configure your API keys in the `.env` file
|
||||
|
||||
## Usage
|
||||
|
||||
```python
|
||||
from scrapegraphai.graphs import CsvScraperGraph
|
||||
|
||||
graph = CsvScraperGraph()
|
||||
csv_data = graph.scrape("https://example.com/table")
|
||||
```
|
||||
|
||||
## Environment Variables
|
||||
|
||||
Required environment variables:
|
||||
- `OPENAI_API_KEY`: Your OpenAI API key
|
||||
@@ -0,0 +1,59 @@
|
||||
"""
|
||||
Basic example of scraping pipeline using CSVScraperMultiGraph from CSV documents
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from scrapegraphai.graphs import CSVScraperMultiGraph
|
||||
from scrapegraphai.utils import prettify_exec_info
|
||||
|
||||
# ************************************************
|
||||
# Read the CSV file
|
||||
# ************************************************
|
||||
|
||||
FILE_NAME = "inputs/username.csv"
|
||||
curr_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
file_path = os.path.join(curr_dir, FILE_NAME)
|
||||
|
||||
with open(file_path, "r") as file:
|
||||
text = file.read()
|
||||
|
||||
# ************************************************
|
||||
# Define the configuration for the graph
|
||||
# ************************************************
|
||||
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"model": "ollama/llama3",
|
||||
"temperature": 0,
|
||||
"format": "json", # Ollama needs the format to be specified explicitly
|
||||
# "model_tokens": 2000, # set context length arbitrarily
|
||||
"base_url": "http://localhost:11434",
|
||||
},
|
||||
"embeddings": {
|
||||
"model": "ollama/nomic-embed-text",
|
||||
"temperature": 0,
|
||||
"base_url": "http://localhost:11434",
|
||||
},
|
||||
"verbose": True,
|
||||
}
|
||||
|
||||
# ************************************************
|
||||
# Create the CSVScraperMultiGraph instance and run it
|
||||
# ************************************************
|
||||
|
||||
csv_scraper_graph = CSVScraperMultiGraph(
|
||||
prompt="List me all the last names",
|
||||
source=[str(text), str(text)],
|
||||
config=graph_config,
|
||||
)
|
||||
|
||||
result = csv_scraper_graph.run()
|
||||
print(result)
|
||||
|
||||
# ************************************************
|
||||
# Get graph execution info
|
||||
# ************************************************
|
||||
|
||||
graph_exec_info = csv_scraper_graph.get_execution_info()
|
||||
print(prettify_exec_info(graph_exec_info))
|
||||
@@ -0,0 +1,59 @@
|
||||
"""
|
||||
Basic example of scraping pipeline using CSVScraperGraph from CSV documents
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from scrapegraphai.graphs import CSVScraperGraph
|
||||
from scrapegraphai.utils import prettify_exec_info
|
||||
|
||||
# ************************************************
|
||||
# Read the CSV file
|
||||
# ************************************************
|
||||
|
||||
FILE_NAME = "inputs/username.csv"
|
||||
curr_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
file_path = os.path.join(curr_dir, FILE_NAME)
|
||||
|
||||
with open(file_path, "r") as file:
|
||||
text = file.read()
|
||||
|
||||
# ************************************************
|
||||
# Define the configuration for the graph
|
||||
# ************************************************
|
||||
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"model": "ollama/llama3",
|
||||
"temperature": 0,
|
||||
"format": "json", # Ollama needs the format to be specified explicitly
|
||||
# "model_tokens": 2000, # set context length arbitrarily
|
||||
"base_url": "http://localhost:11434",
|
||||
},
|
||||
"embeddings": {
|
||||
"model": "ollama/nomic-embed-text",
|
||||
"temperature": 0,
|
||||
"base_url": "http://localhost:11434",
|
||||
},
|
||||
"verbose": True,
|
||||
}
|
||||
|
||||
# ************************************************
|
||||
# Create the CSVScraperGraph instance and run it
|
||||
# ************************************************
|
||||
|
||||
csv_scraper_graph = CSVScraperGraph(
|
||||
prompt="List me all the last names",
|
||||
source=str(text), # Pass the content of the file, not the file object
|
||||
config=graph_config,
|
||||
)
|
||||
|
||||
result = csv_scraper_graph.run()
|
||||
print(result)
|
||||
|
||||
# ************************************************
|
||||
# Get graph execution info
|
||||
# ************************************************
|
||||
|
||||
graph_exec_info = csv_scraper_graph.get_execution_info()
|
||||
print(prettify_exec_info(graph_exec_info))
|
||||
@@ -0,0 +1,6 @@
|
||||
Username; Identifier;First name;Last name
|
||||
booker12;9012;Rachel;Booker
|
||||
grey07;2070;Laura;Grey
|
||||
johnson81;4081;Craig;Johnson
|
||||
jenkins46;9346;Mary;Jenkins
|
||||
smith79;5079;Jamie;Smith
|
||||
|
@@ -0,0 +1,54 @@
|
||||
"""
|
||||
Basic example of scraping pipeline using CSVScraperMultiGraph from CSV documents
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from scrapegraphai.graphs import CSVScraperMultiGraph
|
||||
from scrapegraphai.utils import prettify_exec_info
|
||||
|
||||
load_dotenv()
|
||||
# ************************************************
|
||||
# Read the CSV file
|
||||
# ************************************************
|
||||
|
||||
FILE_NAME = "inputs/username.csv"
|
||||
curr_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
file_path = os.path.join(curr_dir, FILE_NAME)
|
||||
|
||||
with open(file_path, "r") as file:
|
||||
text = file.read()
|
||||
|
||||
# ************************************************
|
||||
# Define the configuration for the graph
|
||||
# ************************************************
|
||||
openai_key = os.getenv("OPENAI_APIKEY")
|
||||
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"api_key": openai_key,
|
||||
"model": "openai/gpt-4o",
|
||||
},
|
||||
}
|
||||
|
||||
# ************************************************
|
||||
# Create the CSVScraperMultiGraph instance and run it
|
||||
# ************************************************
|
||||
|
||||
csv_scraper_graph = CSVScraperMultiGraph(
|
||||
prompt="List me all the last names",
|
||||
source=[str(text), str(text)],
|
||||
config=graph_config,
|
||||
)
|
||||
|
||||
result = csv_scraper_graph.run()
|
||||
print(result)
|
||||
|
||||
# ************************************************
|
||||
# Get graph execution info
|
||||
# ************************************************
|
||||
|
||||
graph_exec_info = csv_scraper_graph.get_execution_info()
|
||||
print(prettify_exec_info(graph_exec_info))
|
||||
@@ -0,0 +1,56 @@
|
||||
"""
|
||||
Basic example of scraping pipeline using CSVScraperGraph from CSV documents
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from scrapegraphai.graphs import CSVScraperGraph
|
||||
from scrapegraphai.utils import prettify_exec_info
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# ************************************************
|
||||
# Read the CSV file
|
||||
# ************************************************
|
||||
|
||||
FILE_NAME = "inputs/username.csv"
|
||||
curr_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
file_path = os.path.join(curr_dir, FILE_NAME)
|
||||
|
||||
with open(file_path, "r") as file:
|
||||
text = file.read()
|
||||
|
||||
# ************************************************
|
||||
# Define the configuration for the graph
|
||||
# ************************************************
|
||||
|
||||
openai_key = os.getenv("OPENAI_APIKEY")
|
||||
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"api_key": openai_key,
|
||||
"model": "openai/gpt-4o",
|
||||
},
|
||||
}
|
||||
|
||||
# ************************************************
|
||||
# Create the CSVScraperGraph instance and run it
|
||||
# ************************************************
|
||||
|
||||
csv_scraper_graph = CSVScraperGraph(
|
||||
prompt="List me all the last names",
|
||||
source=str(text), # Pass the content of the file, not the file object
|
||||
config=graph_config,
|
||||
)
|
||||
|
||||
result = csv_scraper_graph.run()
|
||||
print(result)
|
||||
|
||||
# ************************************************
|
||||
# Get graph execution info
|
||||
# ************************************************
|
||||
|
||||
graph_exec_info = csv_scraper_graph.get_execution_info()
|
||||
print(prettify_exec_info(graph_exec_info))
|
||||
@@ -0,0 +1,6 @@
|
||||
Username; Identifier;First name;Last name
|
||||
booker12;9012;Rachel;Booker
|
||||
grey07;2070;Laura;Grey
|
||||
johnson81;4081;Craig;Johnson
|
||||
jenkins46;9346;Mary;Jenkins
|
||||
smith79;5079;Jamie;Smith
|
||||
|
@@ -0,0 +1,13 @@
|
||||
# OpenAI API Configuration
|
||||
OPENAI_API_KEY=your-openai-api-key-here
|
||||
|
||||
# Optional Configurations
|
||||
MAX_TOKENS=4000
|
||||
MODEL_NAME=gpt-4-1106-preview
|
||||
TEMPERATURE=0.7
|
||||
|
||||
# Custom Graph Settings
|
||||
CUSTOM_NODE_TIMEOUT=30
|
||||
MAX_NODES=10
|
||||
DEBUG_MODE=false
|
||||
LOG_LEVEL=info
|
||||
@@ -0,0 +1,31 @@
|
||||
# Custom Graph Example
|
||||
|
||||
This example demonstrates how to create and implement custom graphs using Scrapegraph-ai.
|
||||
|
||||
## Features
|
||||
|
||||
- Custom node creation
|
||||
- Graph customization
|
||||
- Pipeline configuration
|
||||
- Custom data processing
|
||||
|
||||
## Setup
|
||||
|
||||
1. Install required dependencies
|
||||
2. Copy `.env.example` to `.env`
|
||||
3. Configure your API keys in the `.env` file
|
||||
|
||||
## Usage
|
||||
|
||||
```python
|
||||
from scrapegraphai.graphs import CustomGraph
|
||||
|
||||
graph = CustomGraph()
|
||||
graph.add_node("custom_node", CustomNode())
|
||||
results = graph.process()
|
||||
```
|
||||
|
||||
## Environment Variables
|
||||
|
||||
Required environment variables:
|
||||
- `OPENAI_API_KEY`: Your OpenAI API key
|
||||
@@ -0,0 +1,103 @@
|
||||
"""
|
||||
Example of custom graph using existing nodes
|
||||
"""
|
||||
|
||||
from langchain_openai import ChatOpenAI, OpenAIEmbeddings
|
||||
|
||||
from scrapegraphai.graphs import BaseGraph
|
||||
from scrapegraphai.nodes import (
|
||||
FetchNode,
|
||||
GenerateAnswerNode,
|
||||
ParseNode,
|
||||
RobotsNode,
|
||||
)
|
||||
|
||||
# ************************************************
|
||||
# Define the configuration for the graph
|
||||
# ************************************************
|
||||
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"model": "ollama/mistral",
|
||||
"temperature": 0,
|
||||
"format": "json", # Ollama needs the format to be specified explicitly
|
||||
# "model_tokens": 2000, # set context length arbitrarily
|
||||
"base_url": "http://localhost:11434",
|
||||
},
|
||||
"verbose": True,
|
||||
}
|
||||
|
||||
# ************************************************
|
||||
# Define the graph nodes
|
||||
# ************************************************
|
||||
|
||||
llm_model = ChatOpenAI(graph_config["llm"])
|
||||
embedder = OpenAIEmbeddings(api_key=llm_model.openai_api_key)
|
||||
|
||||
# define the nodes for the graph
|
||||
robot_node = RobotsNode(
|
||||
input="url",
|
||||
output=["is_scrapable"],
|
||||
node_config={
|
||||
"llm_model": llm_model,
|
||||
"force_scraping": True,
|
||||
"verbose": True,
|
||||
},
|
||||
)
|
||||
|
||||
fetch_node = FetchNode(
|
||||
input="url | local_dir",
|
||||
output=["doc"],
|
||||
node_config={
|
||||
"verbose": True,
|
||||
"headless": True,
|
||||
},
|
||||
)
|
||||
parse_node = ParseNode(
|
||||
input="doc",
|
||||
output=["parsed_doc"],
|
||||
node_config={
|
||||
"chunk_size": 4096,
|
||||
"verbose": True,
|
||||
},
|
||||
)
|
||||
|
||||
generate_answer_node = GenerateAnswerNode(
|
||||
input="user_prompt & (relevant_chunks | parsed_doc | doc)",
|
||||
output=["answer"],
|
||||
node_config={
|
||||
"llm_model": llm_model,
|
||||
"verbose": True,
|
||||
},
|
||||
)
|
||||
|
||||
# ************************************************
|
||||
# Create the graph by defining the connections
|
||||
# ************************************************
|
||||
|
||||
graph = BaseGraph(
|
||||
nodes=[
|
||||
robot_node,
|
||||
fetch_node,
|
||||
parse_node,
|
||||
generate_answer_node,
|
||||
],
|
||||
edges=[
|
||||
(robot_node, fetch_node),
|
||||
(fetch_node, parse_node),
|
||||
(parse_node, generate_answer_node),
|
||||
],
|
||||
entry_point=robot_node,
|
||||
)
|
||||
|
||||
# ************************************************
|
||||
# Execute the graph
|
||||
# ************************************************
|
||||
|
||||
result, execution_info = graph.execute(
|
||||
{"user_prompt": "Describe the content", "url": "https://example.com/"}
|
||||
)
|
||||
|
||||
# get the answer from the result
|
||||
result = result.get("answer", "No answer found.")
|
||||
print(result)
|
||||
@@ -0,0 +1,116 @@
|
||||
"""
|
||||
Example of custom graph using existing nodes
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from dotenv import load_dotenv
|
||||
from langchain_openai import ChatOpenAI, OpenAIEmbeddings
|
||||
|
||||
from scrapegraphai.graphs import BaseGraph
|
||||
from scrapegraphai.nodes import (
|
||||
FetchNode,
|
||||
GenerateAnswerNode,
|
||||
ParseNode,
|
||||
RAGNode,
|
||||
RobotsNode,
|
||||
)
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# ************************************************
|
||||
# Define the configuration for the graph
|
||||
# ************************************************
|
||||
|
||||
openai_key = os.getenv("OPENAI_APIKEY")
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"api_key": openai_key,
|
||||
"model": "gpt-4o",
|
||||
},
|
||||
}
|
||||
|
||||
# ************************************************
|
||||
# Define the graph nodes
|
||||
# ************************************************
|
||||
|
||||
llm_model = ChatOpenAI(graph_config["llm"])
|
||||
embedder = OpenAIEmbeddings(api_key=llm_model.openai_api_key)
|
||||
|
||||
# define the nodes for the graph
|
||||
robot_node = RobotsNode(
|
||||
input="url",
|
||||
output=["is_scrapable"],
|
||||
node_config={
|
||||
"llm_model": llm_model,
|
||||
"force_scraping": True,
|
||||
"verbose": True,
|
||||
},
|
||||
)
|
||||
|
||||
fetch_node = FetchNode(
|
||||
input="url | local_dir",
|
||||
output=["doc"],
|
||||
node_config={
|
||||
"verbose": True,
|
||||
"headless": True,
|
||||
},
|
||||
)
|
||||
parse_node = ParseNode(
|
||||
input="doc",
|
||||
output=["parsed_doc"],
|
||||
node_config={
|
||||
"chunk_size": 4096,
|
||||
"verbose": True,
|
||||
},
|
||||
)
|
||||
rag_node = RAGNode(
|
||||
input="user_prompt & (parsed_doc | doc)",
|
||||
output=["relevant_chunks"],
|
||||
node_config={
|
||||
"llm_model": llm_model,
|
||||
"embedder_model": embedder,
|
||||
"verbose": True,
|
||||
},
|
||||
)
|
||||
generate_answer_node = GenerateAnswerNode(
|
||||
input="user_prompt & (relevant_chunks | parsed_doc | doc)",
|
||||
output=["answer"],
|
||||
node_config={
|
||||
"llm_model": llm_model,
|
||||
"verbose": True,
|
||||
},
|
||||
)
|
||||
|
||||
# ************************************************
|
||||
# Create the graph by defining the connections
|
||||
# ************************************************
|
||||
|
||||
graph = BaseGraph(
|
||||
nodes=[
|
||||
robot_node,
|
||||
fetch_node,
|
||||
parse_node,
|
||||
rag_node,
|
||||
generate_answer_node,
|
||||
],
|
||||
edges=[
|
||||
(robot_node, fetch_node),
|
||||
(fetch_node, parse_node),
|
||||
(parse_node, rag_node),
|
||||
(rag_node, generate_answer_node),
|
||||
],
|
||||
entry_point=robot_node,
|
||||
)
|
||||
|
||||
# ************************************************
|
||||
# Execute the graph
|
||||
# ************************************************
|
||||
|
||||
result, execution_info = graph.execute(
|
||||
{"user_prompt": "Describe the content", "url": "https://example.com/"}
|
||||
)
|
||||
|
||||
# get the answer from the result
|
||||
result = result.get("answer", "No answer found.")
|
||||
print(result)
|
||||
@@ -0,0 +1,14 @@
|
||||
# OpenAI API Configuration
|
||||
OPENAI_API_KEY=your-openai-api-key-here
|
||||
|
||||
# Optional Configurations
|
||||
MAX_TOKENS=4000
|
||||
MODEL_NAME=gpt-4-1106-preview
|
||||
TEMPERATURE=0.7
|
||||
|
||||
# Depth Search Settings
|
||||
MAX_DEPTH=5
|
||||
CRAWL_DELAY=1
|
||||
RESPECT_ROBOTS_TXT=true
|
||||
MAX_PAGES_PER_DOMAIN=100
|
||||
USER_AGENT=Mozilla/5.0
|
||||
@@ -0,0 +1,30 @@
|
||||
# Depth Search Graph Example
|
||||
|
||||
This example demonstrates how to use Scrapegraph-ai for deep web crawling and content exploration.
|
||||
|
||||
## Features
|
||||
|
||||
- Deep web crawling
|
||||
- Content discovery
|
||||
- Link analysis
|
||||
- Recursive search
|
||||
|
||||
## Setup
|
||||
|
||||
1. Install required dependencies
|
||||
2. Copy `.env.example` to `.env`
|
||||
3. Configure your API keys in the `.env` file
|
||||
|
||||
## Usage
|
||||
|
||||
```python
|
||||
from scrapegraphai.graphs import DepthSearchGraph
|
||||
|
||||
graph = DepthSearchGraph()
|
||||
results = graph.search("https://example.com", depth=3)
|
||||
```
|
||||
|
||||
## Environment Variables
|
||||
|
||||
Required environment variables:
|
||||
- `OPENAI_API_KEY`: Your OpenAI API key
|
||||
@@ -0,0 +1,35 @@
|
||||
"""
|
||||
depth_search_graph_opeani example
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from scrapegraphai.graphs import DepthSearchGraph
|
||||
|
||||
load_dotenv()
|
||||
|
||||
openai_key = os.getenv("OPENAI_APIKEY")
|
||||
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"model": "ollama/llama3.1",
|
||||
"temperature": 0,
|
||||
"format": "json", # Ollama needs the format to be specified explicitly
|
||||
# "base_url": "http://localhost:11434", # set ollama URL arbitrarily
|
||||
},
|
||||
"verbose": True,
|
||||
"headless": False,
|
||||
"depth": 2,
|
||||
"only_inside_links": False,
|
||||
}
|
||||
|
||||
search_graph = DepthSearchGraph(
|
||||
prompt="List me all the projects with their description",
|
||||
source="https://perinim.github.io",
|
||||
config=graph_config,
|
||||
)
|
||||
|
||||
result = search_graph.run()
|
||||
print(result)
|
||||
@@ -0,0 +1,33 @@
|
||||
"""
|
||||
depth_search_graph_opeani example
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from scrapegraphai.graphs import DepthSearchGraph
|
||||
|
||||
load_dotenv()
|
||||
|
||||
openai_key = os.getenv("OPENAI_API_KEY")
|
||||
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"api_key": openai_key,
|
||||
"model": "openai/gpt-4o-mini",
|
||||
},
|
||||
"verbose": True,
|
||||
"headless": False,
|
||||
"depth": 2,
|
||||
"only_inside_links": False,
|
||||
}
|
||||
|
||||
search_graph = DepthSearchGraph(
|
||||
prompt="List me all the projects with their description",
|
||||
source="https://perinim.github.io",
|
||||
config=graph_config,
|
||||
)
|
||||
|
||||
result = search_graph.run()
|
||||
print(result)
|
||||
@@ -0,0 +1,13 @@
|
||||
# OpenAI API Configuration
|
||||
OPENAI_API_KEY=your-openai-api-key-here
|
||||
|
||||
# Optional Configurations
|
||||
MAX_TOKENS=4000
|
||||
MODEL_NAME=gpt-4-1106-preview
|
||||
TEMPERATURE=0.7
|
||||
|
||||
# Document Scraper Settings
|
||||
OCR_ENABLED=true
|
||||
EXTRACT_METADATA=true
|
||||
MAX_FILE_SIZE=10485760 # 10MB
|
||||
SUPPORTED_FORMATS=pdf,doc,docx,txt
|
||||
@@ -0,0 +1,30 @@
|
||||
# Document Scraper Graph Example
|
||||
|
||||
This example demonstrates how to use Scrapegraph-ai to extract data from various document formats (PDF, DOC, DOCX, etc.).
|
||||
|
||||
## Features
|
||||
|
||||
- Multi-format document support
|
||||
- Text extraction
|
||||
- Document parsing
|
||||
- Metadata extraction
|
||||
|
||||
## Setup
|
||||
|
||||
1. Install required dependencies
|
||||
2. Copy `.env.example` to `.env`
|
||||
3. Configure your API keys in the `.env` file
|
||||
|
||||
## Usage
|
||||
|
||||
```python
|
||||
from scrapegraphai.graphs import DocumentScraperGraph
|
||||
|
||||
graph = DocumentScraperGraph()
|
||||
content = graph.scrape("document.pdf")
|
||||
```
|
||||
|
||||
## Environment Variables
|
||||
|
||||
Required environment variables:
|
||||
- `OPENAI_API_KEY`: Your OpenAI API key
|
||||
@@ -0,0 +1,45 @@
|
||||
"""
|
||||
document_scraper example
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from scrapegraphai.graphs import DocumentScraperGraph
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# ************************************************
|
||||
# Define the configuration for the graph
|
||||
# ************************************************
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"model": "ollama/llama3",
|
||||
"temperature": 0,
|
||||
"format": "json", # Ollama needs the format to be specified explicitly
|
||||
"model_tokens": 4000,
|
||||
},
|
||||
"verbose": True,
|
||||
"headless": False,
|
||||
}
|
||||
|
||||
source = """
|
||||
The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian
|
||||
circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature.
|
||||
Divided into three major sections—Inferno, Purgatorio, and Paradiso—the narrative traces the journey of Dante
|
||||
from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God.
|
||||
Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood
|
||||
through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided
|
||||
by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love,
|
||||
the Beatrice of his earlier poetry, through the celestial spheres of Paradise.
|
||||
"""
|
||||
|
||||
pdf_scraper_graph = DocumentScraperGraph(
|
||||
prompt="Summarize the text and find the main topics",
|
||||
source=source,
|
||||
config=graph_config,
|
||||
)
|
||||
result = pdf_scraper_graph.run()
|
||||
|
||||
print(json.dumps(result, indent=4))
|
||||
@@ -0,0 +1,105 @@
|
||||
<body class="fixed-top-nav " style="padding-top: 57px;">
|
||||
<header>
|
||||
<nav id="navbar" class="navbar navbar-light navbar-expand-sm fixed-top">
|
||||
<div class="container">
|
||||
<a class="navbar-brand title font-weight-lighter" href="/"><span class="font-weight-bold">Marco </span>Perini</a> <button class="navbar-toggler collapsed ml-auto" type="button" data-toggle="collapse" data-target="#navbarNav" aria-controls="navbarNav" aria-expanded="false" aria-label="Toggle navigation"> <span class="sr-only">Toggle navigation</span> <span class="icon-bar top-bar"></span> <span class="icon-bar middle-bar"></span> <span class="icon-bar bottom-bar"></span> </button>
|
||||
<div class="collapse navbar-collapse text-right" id="navbarNav">
|
||||
<ul class="navbar-nav ml-auto flex-nowrap">
|
||||
<li class="nav-item "> <a class="nav-link" href="/">About</a> </li>
|
||||
<li class="nav-item dropdown active">
|
||||
<a class="nav-link dropdown-toggle" href="#" id="navbarDropdown" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">Projects<span class="sr-only">(current)</span></a>
|
||||
<div class="dropdown-menu dropdown-menu-right" aria-labelledby="navbarDropdown">
|
||||
<a class="dropdown-item" href="/projects/">Projects</a>
|
||||
<div class="dropdown-divider"></div>
|
||||
<a class="dropdown-item" href="/competitions/">Competitions</a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="nav-item "> <a class="nav-link" href="/cv/">CV</a> </li>
|
||||
<li class="toggle-container"> <button id="light-toggle" title="Change theme"> <i class="fa-solid fa-moon"></i> <i class="fa-solid fa-sun"></i> </button> </li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</nav>
|
||||
<progress id="progress" value="0" max="284" style="top: 57px;">
|
||||
<div class="progress-container"> <span class="progress-bar"></span> </div>
|
||||
</progress>
|
||||
</header>
|
||||
<div class="container mt-5">
|
||||
<div class="post">
|
||||
<header class="post-header">
|
||||
<h1 class="post-title">Projects</h1>
|
||||
<p class="post-description"></p>
|
||||
</header>
|
||||
<article>
|
||||
<div class="projects">
|
||||
<div class="grid" style="position: relative; height: 861.992px;">
|
||||
<div class="grid-sizer"></div>
|
||||
<div class="grid-item" style="position: absolute; left: 0px; top: 0px;">
|
||||
<a href="/projects/rotary-pendulum-rl/">
|
||||
<div class="card hoverable">
|
||||
<figure>
|
||||
<picture> <img src="/assets/img/rotary_pybullet.jpg" width="auto" height="auto" alt="project thumbnail" onerror="this.onerror=null; $('.responsive-img-srcset').remove();"> </picture>
|
||||
</figure>
|
||||
<div class="card-body">
|
||||
<h4 class="card-title">Rotary Pendulum RL</h4>
|
||||
<p class="card-text">Open Source project aimed at controlling a real life rotary pendulum using RL algorithms</p>
|
||||
<div class="row ml-1 mr-1 p-0"> </div>
|
||||
</div>
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
<div class="grid-sizer"></div>
|
||||
<div class="grid-item" style="position: absolute; left: 260px; top: 0px;">
|
||||
<a href="https://github.com/PeriniM/DQN-SwingUp" rel="external nofollow noopener" target="_blank">
|
||||
<div class="card hoverable">
|
||||
<figure>
|
||||
<picture> <img src="/assets/img/value-policy-heatmaps.jpg" width="auto" height="auto" alt="project thumbnail" onerror="this.onerror=null; $('.responsive-img-srcset').remove();"> </picture>
|
||||
</figure>
|
||||
<div class="card-body">
|
||||
<h4 class="card-title">DQN Implementation from scratch</h4>
|
||||
<p class="card-text">Developed a Deep Q-Network algorithm to train a simple and double pendulum</p>
|
||||
<div class="row ml-1 mr-1 p-0"> </div>
|
||||
</div>
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
<div class="grid-sizer"></div>
|
||||
<div class="grid-item" style="position: absolute; left: 0px; top: 447.414px;">
|
||||
<a href="https://github.com/PeriniM/Multi-Agents-HAED" rel="external nofollow noopener" target="_blank">
|
||||
<div class="card hoverable">
|
||||
<figure>
|
||||
<picture> <img src="/assets/img/multi_agents_haed.gif" width="auto" height="auto" alt="project thumbnail" onerror="this.onerror=null; $('.responsive-img-srcset').remove();"> </picture>
|
||||
</figure>
|
||||
<div class="card-body">
|
||||
<h4 class="card-title">Multi Agents HAED</h4>
|
||||
<p class="card-text">University project which focuses on simulating a multi-agent system to perform environment mapping. Agents, equipped with sensors, explore and record their surroundings, considering uncertainties in their readings.</p>
|
||||
<div class="row ml-1 mr-1 p-0"> </div>
|
||||
</div>
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
<div class="grid-sizer"></div>
|
||||
<div class="grid-item" style="position: absolute; left: 260px; top: 370.172px;">
|
||||
<a href="/projects/wireless-esc-drone/">
|
||||
<div class="card hoverable">
|
||||
<figure>
|
||||
<picture> <img src="/assets/img/wireless_esc.gif" width="auto" height="auto" alt="project thumbnail" onerror="this.onerror=null; $('.responsive-img-srcset').remove();"> </picture>
|
||||
</figure>
|
||||
<div class="card-body">
|
||||
<h4 class="card-title">Wireless ESC for Modular Drones</h4>
|
||||
<p class="card-text">Modular drone architecture proposal and proof of concept. The project received maximum grade.</p>
|
||||
<div class="row ml-1 mr-1 p-0"> </div>
|
||||
</div>
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</article>
|
||||
</div>
|
||||
</div>
|
||||
<footer class="fixed-bottom">
|
||||
<div class="container mt-0"> © Copyright 2023 . Powered by <a href="https://jekyllrb.com/" target="_blank" rel="external nofollow noopener">Jekyll</a> with <a href="https://github.com/alshedivat/al-folio" rel="external nofollow noopener" target="_blank">al-folio</a> theme. Hosted by <a href="https://pages.github.com/" target="_blank" rel="external nofollow noopener">GitHub Pages</a>. </div>
|
||||
</footer>
|
||||
<div class="hiddendiv common"></div>
|
||||
</body>
|
||||
@@ -0,0 +1,42 @@
|
||||
"""
|
||||
document_scraper example
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from scrapegraphai.graphs import DocumentScraperGraph
|
||||
|
||||
load_dotenv()
|
||||
|
||||
|
||||
openai_key = os.getenv("OPENAI_APIKEY")
|
||||
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"api_key": openai_key,
|
||||
"model": "openai/gpt-4o",
|
||||
}
|
||||
}
|
||||
|
||||
source = """
|
||||
The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian
|
||||
circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature.
|
||||
Divided into three major sections—Inferno, Purgatorio, and Paradiso—the narrative traces the journey of Dante
|
||||
from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God.
|
||||
Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood
|
||||
through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided
|
||||
by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love,
|
||||
the Beatrice of his earlier poetry, through the celestial spheres of Paradise.
|
||||
"""
|
||||
|
||||
pdf_scraper_graph = DocumentScraperGraph(
|
||||
prompt="Summarize the text and find the main topics",
|
||||
source=source,
|
||||
config=graph_config,
|
||||
)
|
||||
result = pdf_scraper_graph.run()
|
||||
|
||||
print(json.dumps(result, indent=4))
|
||||
@@ -0,0 +1,35 @@
|
||||
Toggle navigation
|
||||
|
||||
* About
|
||||
* Projects(current)
|
||||
|
||||
Projects
|
||||
|
||||
Competitions
|
||||
|
||||
* CV
|
||||
* ____
|
||||
|
||||
# Projects
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
© Copyright 2023 . Powered by Jekyll with
|
||||
al-folio theme. Hosted by [GitHub
|
||||
Pages](https://pages.github.com/).
|
||||
@@ -0,0 +1,105 @@
|
||||
<body class="fixed-top-nav " style="padding-top: 57px;">
|
||||
<header>
|
||||
<nav id="navbar" class="navbar navbar-light navbar-expand-sm fixed-top">
|
||||
<div class="container">
|
||||
<a class="navbar-brand title font-weight-lighter" href="/"><span class="font-weight-bold">Marco </span>Perini</a> <button class="navbar-toggler collapsed ml-auto" type="button" data-toggle="collapse" data-target="#navbarNav" aria-controls="navbarNav" aria-expanded="false" aria-label="Toggle navigation"> <span class="sr-only">Toggle navigation</span> <span class="icon-bar top-bar"></span> <span class="icon-bar middle-bar"></span> <span class="icon-bar bottom-bar"></span> </button>
|
||||
<div class="collapse navbar-collapse text-right" id="navbarNav">
|
||||
<ul class="navbar-nav ml-auto flex-nowrap">
|
||||
<li class="nav-item "> <a class="nav-link" href="/">About</a> </li>
|
||||
<li class="nav-item dropdown active">
|
||||
<a class="nav-link dropdown-toggle" href="#" id="navbarDropdown" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">Projects<span class="sr-only">(current)</span></a>
|
||||
<div class="dropdown-menu dropdown-menu-right" aria-labelledby="navbarDropdown">
|
||||
<a class="dropdown-item" href="/projects/">Projects</a>
|
||||
<div class="dropdown-divider"></div>
|
||||
<a class="dropdown-item" href="/competitions/">Competitions</a>
|
||||
</div>
|
||||
</li>
|
||||
<li class="nav-item "> <a class="nav-link" href="/cv/">CV</a> </li>
|
||||
<li class="toggle-container"> <button id="light-toggle" title="Change theme"> <i class="fa-solid fa-moon"></i> <i class="fa-solid fa-sun"></i> </button> </li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</nav>
|
||||
<progress id="progress" value="0" max="284" style="top: 57px;">
|
||||
<div class="progress-container"> <span class="progress-bar"></span> </div>
|
||||
</progress>
|
||||
</header>
|
||||
<div class="container mt-5">
|
||||
<div class="post">
|
||||
<header class="post-header">
|
||||
<h1 class="post-title">Projects</h1>
|
||||
<p class="post-description"></p>
|
||||
</header>
|
||||
<article>
|
||||
<div class="projects">
|
||||
<div class="grid" style="position: relative; height: 861.992px;">
|
||||
<div class="grid-sizer"></div>
|
||||
<div class="grid-item" style="position: absolute; left: 0px; top: 0px;">
|
||||
<a href="/projects/rotary-pendulum-rl/">
|
||||
<div class="card hoverable">
|
||||
<figure>
|
||||
<picture> <img src="/assets/img/rotary_pybullet.jpg" width="auto" height="auto" alt="project thumbnail" onerror="this.onerror=null; $('.responsive-img-srcset').remove();"> </picture>
|
||||
</figure>
|
||||
<div class="card-body">
|
||||
<h4 class="card-title">Rotary Pendulum RL</h4>
|
||||
<p class="card-text">Open Source project aimed at controlling a real life rotary pendulum using RL algorithms</p>
|
||||
<div class="row ml-1 mr-1 p-0"> </div>
|
||||
</div>
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
<div class="grid-sizer"></div>
|
||||
<div class="grid-item" style="position: absolute; left: 260px; top: 0px;">
|
||||
<a href="https://github.com/PeriniM/DQN-SwingUp" rel="external nofollow noopener" target="_blank">
|
||||
<div class="card hoverable">
|
||||
<figure>
|
||||
<picture> <img src="/assets/img/value-policy-heatmaps.jpg" width="auto" height="auto" alt="project thumbnail" onerror="this.onerror=null; $('.responsive-img-srcset').remove();"> </picture>
|
||||
</figure>
|
||||
<div class="card-body">
|
||||
<h4 class="card-title">DQN Implementation from scratch</h4>
|
||||
<p class="card-text">Developed a Deep Q-Network algorithm to train a simple and double pendulum</p>
|
||||
<div class="row ml-1 mr-1 p-0"> </div>
|
||||
</div>
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
<div class="grid-sizer"></div>
|
||||
<div class="grid-item" style="position: absolute; left: 0px; top: 447.414px;">
|
||||
<a href="https://github.com/PeriniM/Multi-Agents-HAED" rel="external nofollow noopener" target="_blank">
|
||||
<div class="card hoverable">
|
||||
<figure>
|
||||
<picture> <img src="/assets/img/multi_agents_haed.gif" width="auto" height="auto" alt="project thumbnail" onerror="this.onerror=null; $('.responsive-img-srcset').remove();"> </picture>
|
||||
</figure>
|
||||
<div class="card-body">
|
||||
<h4 class="card-title">Multi Agents HAED</h4>
|
||||
<p class="card-text">University project which focuses on simulating a multi-agent system to perform environment mapping. Agents, equipped with sensors, explore and record their surroundings, considering uncertainties in their readings.</p>
|
||||
<div class="row ml-1 mr-1 p-0"> </div>
|
||||
</div>
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
<div class="grid-sizer"></div>
|
||||
<div class="grid-item" style="position: absolute; left: 260px; top: 370.172px;">
|
||||
<a href="/projects/wireless-esc-drone/">
|
||||
<div class="card hoverable">
|
||||
<figure>
|
||||
<picture> <img src="/assets/img/wireless_esc.gif" width="auto" height="auto" alt="project thumbnail" onerror="this.onerror=null; $('.responsive-img-srcset').remove();"> </picture>
|
||||
</figure>
|
||||
<div class="card-body">
|
||||
<h4 class="card-title">Wireless ESC for Modular Drones</h4>
|
||||
<p class="card-text">Modular drone architecture proposal and proof of concept. The project received maximum grade.</p>
|
||||
<div class="row ml-1 mr-1 p-0"> </div>
|
||||
</div>
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</article>
|
||||
</div>
|
||||
</div>
|
||||
<footer class="fixed-bottom">
|
||||
<div class="container mt-0"> © Copyright 2023 . Powered by <a href="https://jekyllrb.com/" target="_blank" rel="external nofollow noopener">Jekyll</a> with <a href="https://github.com/alshedivat/al-folio" rel="external nofollow noopener" target="_blank">al-folio</a> theme. Hosted by <a href="https://pages.github.com/" target="_blank" rel="external nofollow noopener">GitHub Pages</a>. </div>
|
||||
</footer>
|
||||
<div class="hiddendiv common"></div>
|
||||
</body>
|
||||
@@ -0,0 +1,4 @@
|
||||
OPENAI_API_KEY="YOUR_OPENAI_API_KEY"
|
||||
BROWSER_BASE_PROJECT_ID="YOUR_BROWSER_BASE_PROJECT_ID"
|
||||
BROWSER_BASE_API_KEY="YOUR_BROWSERBASE_API_KEY"
|
||||
SCRAPE_DO_API_KEY="YOUR_SCRAPE_DO_API_KEY"
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 174 KiB |
@@ -0,0 +1,94 @@
|
||||
"""
|
||||
Example leveraging a state file containing session cookies which
|
||||
might be leveraged to authenticate to a website and scrape protected
|
||||
content.
|
||||
"""
|
||||
|
||||
import os
|
||||
import random
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# import playwright so we can use it to create the state file
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
from scrapegraphai.graphs import OmniScraperGraph
|
||||
from scrapegraphai.utils import prettify_exec_info
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# ************************************************
|
||||
# Leveraging Playwright external to the invocation of the graph to
|
||||
# login and create the state file
|
||||
# ************************************************
|
||||
|
||||
|
||||
# note this is just an example and probably won't actually work on
|
||||
# LinkedIn, the implementation of the login is highly dependent on the website
|
||||
async def do_login():
|
||||
async with async_playwright() as playwright:
|
||||
browser = await playwright.chromium.launch(
|
||||
timeout=30000,
|
||||
headless=False,
|
||||
slow_mo=random.uniform(500, 1500),
|
||||
)
|
||||
page = await browser.new_page()
|
||||
|
||||
# very basic implementation of a login, in reality it may be trickier
|
||||
await page.goto("https://www.linkedin.com/login")
|
||||
await page.get_by_label("Email or phone").fill("some_bloke@some_domain.com")
|
||||
await page.get_by_label("Password").fill("test1234")
|
||||
await page.get_by_role("button", name="Sign in").click()
|
||||
await page.wait_for_timeout(3000)
|
||||
|
||||
# assuming a successful login, we save the cookies to a file
|
||||
await page.context.storage_state(path="./state.json")
|
||||
|
||||
|
||||
async def main():
|
||||
await do_login()
|
||||
|
||||
# ************************************************
|
||||
# Define the configuration for the graph
|
||||
# ************************************************
|
||||
|
||||
openai_api_key = os.getenv("OPENAI_APIKEY")
|
||||
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"api_key": openai_api_key,
|
||||
"model": "openai/gpt-4o",
|
||||
},
|
||||
"max_images": 10,
|
||||
"headless": False,
|
||||
# provide the path to the state file
|
||||
"storage_state": "./state.json",
|
||||
}
|
||||
|
||||
# ************************************************
|
||||
# Create the OmniScraperGraph instance and run it
|
||||
# ************************************************
|
||||
|
||||
omni_scraper_graph = OmniScraperGraph(
|
||||
prompt="List me all the projects with their description.",
|
||||
source="https://www.linkedin.com/feed/",
|
||||
config=graph_config,
|
||||
)
|
||||
|
||||
# the storage_state is used to load the cookies from the state file
|
||||
# so we are authenticated and able to scrape protected content
|
||||
result = omni_scraper_graph.run()
|
||||
print(result)
|
||||
|
||||
# ************************************************
|
||||
# Get graph execution info
|
||||
# ************************************************
|
||||
|
||||
graph_exec_info = omni_scraper_graph.get_execution_info()
|
||||
print(prettify_exec_info(graph_exec_info))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import asyncio
|
||||
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,51 @@
|
||||
"""
|
||||
Basic example of scraping pipeline using SmartScraper
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from scrapegraphai.graphs import SmartScraperGraph
|
||||
from scrapegraphai.utils import prettify_exec_info
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# ************************************************
|
||||
# Define the configuration for the graph
|
||||
# ************************************************
|
||||
|
||||
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"api_key": os.getenv("OPENAI_API_KEY"),
|
||||
"model": "openai/gpt-4o",
|
||||
},
|
||||
"browser_base": {
|
||||
"api_key": os.getenv("BROWSER_BASE_API_KEY"),
|
||||
"project_id": os.getenv("BROWSER_BASE_PROJECT_ID"),
|
||||
},
|
||||
"verbose": True,
|
||||
"headless": False,
|
||||
}
|
||||
|
||||
# ************************************************
|
||||
# Create the SmartScraperGraph instance and run it
|
||||
# ************************************************
|
||||
|
||||
smart_scraper_graph = SmartScraperGraph(
|
||||
prompt="List me what does the company do, the name and a contact email.",
|
||||
source="https://scrapegraphai.com/",
|
||||
config=graph_config,
|
||||
)
|
||||
|
||||
result = smart_scraper_graph.run()
|
||||
print(json.dumps(result, indent=4))
|
||||
|
||||
# ************************************************
|
||||
# Get graph execution info
|
||||
# ************************************************
|
||||
|
||||
graph_exec_info = smart_scraper_graph.get_execution_info()
|
||||
print(prettify_exec_info(graph_exec_info))
|
||||
@@ -0,0 +1,148 @@
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
|
||||
from aiohttp import ClientError
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from scrapegraphai.docloaders.chromium import ( # Import your ChromiumLoader class
|
||||
ChromiumLoader,
|
||||
)
|
||||
from scrapegraphai.graphs import SmartScraperGraph
|
||||
|
||||
# Load environment variables for API keys
|
||||
load_dotenv()
|
||||
|
||||
|
||||
# ************************************************
|
||||
# Define function to analyze content with ScrapegraphAI
|
||||
# ************************************************
|
||||
async def analyze_content_with_scrapegraph(content: str):
|
||||
"""
|
||||
Analyze scraped content using ScrapegraphAI.
|
||||
|
||||
Args:
|
||||
content (str): The scraped HTML or text content.
|
||||
|
||||
Returns:
|
||||
dict: The result from ScrapegraphAI analysis.
|
||||
"""
|
||||
try:
|
||||
# Initialize ScrapegraphAI SmartScraperGraph
|
||||
smart_scraper = SmartScraperGraph(
|
||||
prompt="Summarize the main content of this webpage and extract any contact information.",
|
||||
source=content, # Pass the content directly
|
||||
config={
|
||||
"llm": {
|
||||
"api_key": os.getenv("OPENAI_API_KEY"),
|
||||
"model": "openai/gpt-4o",
|
||||
},
|
||||
"verbose": True,
|
||||
},
|
||||
)
|
||||
result = smart_scraper.run()
|
||||
return result
|
||||
except Exception as e:
|
||||
print(f"❌ ScrapegraphAI analysis failed: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
|
||||
# ************************************************
|
||||
# Test scraper and ScrapegraphAI pipeline
|
||||
# ************************************************
|
||||
async def test_scraper_with_analysis(scraper: ChromiumLoader, urls: list):
|
||||
"""
|
||||
Test scraper for the given backend and URLs, then analyze content with ScrapegraphAI.
|
||||
|
||||
Args:
|
||||
scraper (ChromiumLoader): The ChromiumLoader instance.
|
||||
urls (list): A list of URLs to scrape.
|
||||
"""
|
||||
for url in urls:
|
||||
try:
|
||||
print(f"\n🔎 Scraping: {url} using {scraper.backend}...")
|
||||
result = await scraper.scrape(url)
|
||||
|
||||
if "Error" in result or not result.strip():
|
||||
print(f"❌ Failed to scrape {url}: {result}")
|
||||
else:
|
||||
print(
|
||||
f"✅ Successfully scraped {url}. Content (first 200 chars): {result[:200]}"
|
||||
)
|
||||
|
||||
# Pass scraped content to ScrapegraphAI for analysis
|
||||
print("🤖 Analyzing content with ScrapegraphAI...")
|
||||
analysis_result = await analyze_content_with_scrapegraph(result)
|
||||
print("📝 Analysis Result:")
|
||||
print(json.dumps(analysis_result, indent=4))
|
||||
|
||||
except ClientError as ce:
|
||||
print(f"❌ Network error while scraping {url}: {ce}")
|
||||
except Exception as e:
|
||||
print(f"❌ Unexpected error while scraping {url}: {e}")
|
||||
|
||||
|
||||
# ************************************************
|
||||
# Main Execution
|
||||
# ************************************************
|
||||
async def main():
|
||||
urls_to_scrape = [
|
||||
"https://example.com",
|
||||
"https://www.python.org",
|
||||
"https://invalid-url.test",
|
||||
]
|
||||
|
||||
# Test with Playwright backend
|
||||
print("\n--- Testing Playwright Backend ---")
|
||||
try:
|
||||
scraper_playwright_chromium = ChromiumLoader(
|
||||
urls=urls_to_scrape,
|
||||
backend="playwright",
|
||||
headless=True,
|
||||
browser_name="chromium",
|
||||
)
|
||||
await test_scraper_with_analysis(scraper_playwright_chromium, urls_to_scrape)
|
||||
|
||||
scraper_playwright_firefox = ChromiumLoader(
|
||||
urls=urls_to_scrape,
|
||||
backend="playwright",
|
||||
headless=True,
|
||||
browser_name="firefox",
|
||||
)
|
||||
await test_scraper_with_analysis(scraper_playwright_firefox, urls_to_scrape)
|
||||
except ImportError as ie:
|
||||
print(f"❌ Playwright ImportError: {ie}")
|
||||
except Exception as e:
|
||||
print(f"❌ Error initializing Playwright ChromiumLoader: {e}")
|
||||
|
||||
# Test with Selenium backend
|
||||
print("\n--- Testing Selenium Backend ---")
|
||||
try:
|
||||
scraper_selenium_chromium = ChromiumLoader(
|
||||
urls=urls_to_scrape,
|
||||
backend="selenium",
|
||||
headless=True,
|
||||
browser_name="chromium",
|
||||
)
|
||||
await test_scraper_with_analysis(scraper_selenium_chromium, urls_to_scrape)
|
||||
|
||||
scraper_selenium_firefox = ChromiumLoader(
|
||||
urls=urls_to_scrape,
|
||||
backend="selenium",
|
||||
headless=True,
|
||||
browser_name="firefox",
|
||||
)
|
||||
await test_scraper_with_analysis(scraper_selenium_firefox, urls_to_scrape)
|
||||
except ImportError as ie:
|
||||
print(f"❌ Selenium ImportError: {ie}")
|
||||
except Exception as e:
|
||||
print(f"❌ Error initializing Selenium ChromiumLoader: {e}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
asyncio.run(main())
|
||||
except KeyboardInterrupt:
|
||||
print("❌ Program interrupted by user.")
|
||||
except Exception as e:
|
||||
print(f"❌ Program crashed: {e}")
|
||||
@@ -0,0 +1,40 @@
|
||||
"""
|
||||
Basic example of scraping pipeline using SmartScraperMultiConcatGraph with Groq
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from scrapegraphai.graphs import SmartScraperGraph
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# ************************************************
|
||||
# Define the configuration for the graph
|
||||
# ************************************************
|
||||
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"api_key": os.getenv("GROQ_APIKEY"),
|
||||
"model": "groq/gemma-7b-it",
|
||||
},
|
||||
"verbose": True,
|
||||
"headless": True,
|
||||
"reattempt": True, # Setting this to True will allow the graph to reattempt the scraping process
|
||||
}
|
||||
|
||||
# *******************************************************
|
||||
# Create the SmartScraperMultiCondGraph instance and run it
|
||||
# *******************************************************
|
||||
|
||||
multiple_search_graph = SmartScraperGraph(
|
||||
prompt="Who is ?",
|
||||
source="https://perinim.github.io/",
|
||||
schema=None,
|
||||
config=graph_config,
|
||||
)
|
||||
|
||||
result = multiple_search_graph.run()
|
||||
print(json.dumps(result, indent=4))
|
||||
@@ -0,0 +1,39 @@
|
||||
"""
|
||||
Basic example of scraping pipeline using SmartScraperMultiConcatGraph with Groq
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from scrapegraphai.graphs import SmartScraperMultiGraph
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# ************************************************
|
||||
# Define the configuration for the graph
|
||||
# ************************************************
|
||||
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"api_key": os.getenv("OPENAI_API_KEY"),
|
||||
"model": "openai/gpt-4o",
|
||||
},
|
||||
"verbose": True,
|
||||
"headless": False,
|
||||
}
|
||||
|
||||
# *******************************************************
|
||||
# Create the SmartScraperMultiCondGraph instance and run it
|
||||
# *******************************************************
|
||||
|
||||
multiple_search_graph = SmartScraperMultiGraph(
|
||||
prompt="Who is Marco Perini?",
|
||||
source=["https://perinim.github.io/", "https://perinim.github.io/cv/"],
|
||||
schema=None,
|
||||
config=graph_config,
|
||||
)
|
||||
|
||||
result = multiple_search_graph.run()
|
||||
print(json.dumps(result, indent=4))
|
||||
@@ -0,0 +1,53 @@
|
||||
"""
|
||||
Basic example of scraping pipeline using SmartScraper
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from scrapegraphai.graphs import SmartScraperGraph
|
||||
from scrapegraphai.utils import prettify_exec_info
|
||||
|
||||
load_dotenv()
|
||||
|
||||
|
||||
# ************************************************
|
||||
# Define the configuration for the graph
|
||||
# ************************************************
|
||||
|
||||
openai_key = os.getenv("OPENAI_APIKEY")
|
||||
|
||||
prompt = "Some more info"
|
||||
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"api_key": openai_key,
|
||||
"model": "openai/gpt-3.5-turbo",
|
||||
},
|
||||
"additional_info": prompt,
|
||||
"verbose": True,
|
||||
"headless": False,
|
||||
}
|
||||
|
||||
# ************************************************
|
||||
# Create the SmartScraperGraph instance and run it
|
||||
# ************************************************
|
||||
|
||||
smart_scraper_graph = SmartScraperGraph(
|
||||
prompt="List me all the projects with their description",
|
||||
# also accepts a string with the already downloaded HTML code
|
||||
source="https://perinim.github.io/projects/",
|
||||
config=graph_config,
|
||||
)
|
||||
|
||||
result = smart_scraper_graph.run()
|
||||
print(json.dumps(result, indent=4))
|
||||
|
||||
# ************************************************
|
||||
# Get graph execution info
|
||||
# ************************************************
|
||||
|
||||
graph_exec_info = smart_scraper_graph.get_execution_info()
|
||||
print(prettify_exec_info(graph_exec_info))
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"llm": {
|
||||
"model": "ollama/llama3",
|
||||
"temperature": 0,
|
||||
"format": "json",
|
||||
# "base_url": "http://localhost:11434",
|
||||
},
|
||||
"embeddings": {
|
||||
"model": "ollama/nomic-embed-text",
|
||||
"temperature": 0,
|
||||
# "base_url": "http://localhost:11434",
|
||||
},
|
||||
"verbose": true,
|
||||
"headless": false
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
"""
|
||||
Basic example of scraping pipeline using SmartScraper
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from scrapegraphai.graphs import SmartScraperGraph
|
||||
from scrapegraphai.utils import prettify_exec_info
|
||||
|
||||
load_dotenv()
|
||||
|
||||
|
||||
# ************************************************
|
||||
# Define the configuration for the graph
|
||||
# ************************************************
|
||||
|
||||
openai_key = os.getenv("OPENAI_APIKEY")
|
||||
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"model": "ollama/llama3",
|
||||
"temperature": 0,
|
||||
# "format": "json", # Ollama needs the format to be specified explicitly
|
||||
# "base_url": "http://localhost:11434", # set ollama URL arbitrarily
|
||||
},
|
||||
"embeddings": {
|
||||
"model": "ollama/nomic-embed-text",
|
||||
"temperature": 0,
|
||||
# "base_url": "http://localhost:11434", # set ollama URL arbitrarily
|
||||
},
|
||||
"force": True,
|
||||
"caching": True,
|
||||
}
|
||||
|
||||
# ************************************************
|
||||
# Create the SmartScraperGraph instance and run it
|
||||
# ************************************************
|
||||
|
||||
smart_scraper_graph = SmartScraperGraph(
|
||||
prompt="List me all the projects with their description.",
|
||||
# also accepts a string with the already downloaded HTML code
|
||||
source="https://perinim.github.io/projects/",
|
||||
config=graph_config,
|
||||
)
|
||||
|
||||
result = smart_scraper_graph.run()
|
||||
print(result)
|
||||
|
||||
# ************************************************
|
||||
# Get graph execution info
|
||||
# ************************************************
|
||||
|
||||
graph_exec_info = smart_scraper_graph.get_execution_info()
|
||||
print(prettify_exec_info(graph_exec_info))
|
||||
@@ -0,0 +1,51 @@
|
||||
"""
|
||||
Basic example of scraping pipeline using SmartScraper
|
||||
By default smart scraper converts in md format the
|
||||
code. If you want to just use the original code, you have
|
||||
to specify in the confi
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from scrapegraphai.graphs import SmartScraperGraph
|
||||
from scrapegraphai.utils import prettify_exec_info
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# ************************************************
|
||||
# Define the configuration for the graph
|
||||
# ************************************************
|
||||
|
||||
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"api_key": os.getenv("OPENAI_API_KEY"),
|
||||
"model": "openai/gpt-4o",
|
||||
},
|
||||
"html_mode": True,
|
||||
"verbose": True,
|
||||
"headless": False,
|
||||
}
|
||||
|
||||
# ************************************************
|
||||
# Create the SmartScraperGraph instance and run it
|
||||
# ************************************************
|
||||
|
||||
smart_scraper_graph = SmartScraperGraph(
|
||||
prompt="List me what does the company do, the name and a contact email.",
|
||||
source="https://scrapegraphai.com/",
|
||||
config=graph_config,
|
||||
)
|
||||
|
||||
result = smart_scraper_graph.run()
|
||||
print(json.dumps(result, indent=4))
|
||||
|
||||
# ************************************************
|
||||
# Get graph execution info
|
||||
# ************************************************
|
||||
|
||||
graph_exec_info = smart_scraper_graph.get_execution_info()
|
||||
print(prettify_exec_info(graph_exec_info))
|
||||
@@ -0,0 +1,34 @@
|
||||
"""
|
||||
Basic example of scraping pipeline using SmartScraper
|
||||
"""
|
||||
|
||||
import yaml
|
||||
|
||||
from scrapegraphai.graphs import SmartScraperGraph
|
||||
from scrapegraphai.utils import prettify_exec_info
|
||||
|
||||
# ************************************************
|
||||
# Define the configuration for the graph
|
||||
# ************************************************
|
||||
with open("example.yml", "r") as file:
|
||||
graph_config = yaml.safe_load(file)
|
||||
|
||||
# ************************************************
|
||||
# Create the SmartScraperGraph instance and run it
|
||||
# ************************************************
|
||||
|
||||
smart_scraper_graph = SmartScraperGraph(
|
||||
prompt="List me all the titles",
|
||||
source="https://sport.sky.it/nba?gr=www",
|
||||
config=graph_config,
|
||||
)
|
||||
|
||||
result = smart_scraper_graph.run()
|
||||
print(result)
|
||||
|
||||
# ************************************************
|
||||
# Get graph execution info
|
||||
# ************************************************
|
||||
|
||||
graph_exec_info = smart_scraper_graph.get_execution_info()
|
||||
print(prettify_exec_info(graph_exec_info))
|
||||
@@ -0,0 +1,43 @@
|
||||
"""
|
||||
This example shows how to do not process the html code in the fetch phase
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
from scrapegraphai.graphs import SmartScraperGraph
|
||||
from scrapegraphai.utils import prettify_exec_info
|
||||
|
||||
# ************************************************
|
||||
# Define the configuration for the graph
|
||||
# ************************************************
|
||||
|
||||
|
||||
graph_config = {
|
||||
"llm": {
|
||||
"api_key": "s",
|
||||
"model": "openai/gpt-3.5-turbo",
|
||||
},
|
||||
"cut": False,
|
||||
"verbose": True,
|
||||
"headless": False,
|
||||
}
|
||||
|
||||
# ************************************************
|
||||
# Create the SmartScraperGraph instance and run it
|
||||
# ************************************************
|
||||
|
||||
smart_scraper_graph = SmartScraperGraph(
|
||||
prompt="Extract me the python code inside the page",
|
||||
source="https://www.exploit-db.com/exploits/51447",
|
||||
config=graph_config,
|
||||
)
|
||||
|
||||
result = smart_scraper_graph.run()
|
||||
print(json.dumps(result, indent=4))
|
||||
|
||||
# ************************************************
|
||||
# Get graph execution info
|
||||
# ************************************************
|
||||
|
||||
graph_exec_info = smart_scraper_graph.get_execution_info()
|
||||
print(prettify_exec_info(graph_exec_info))
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user