wehub-resource-sync
440 lines
16 KiB
JavaScript
440 lines
16 KiB
JavaScript
/* eslint-disable global-require */
|
|
const path = require('path');
|
|
const { externalLinkProcessor } = require('./tools/utils/externalLink');
|
|
|
|
const packages = [
|
|
'core',
|
|
'browser-pool',
|
|
'basic-crawler',
|
|
'browser-crawler',
|
|
'http-crawler',
|
|
'cheerio-crawler',
|
|
'puppeteer-crawler',
|
|
'playwright-crawler',
|
|
'jsdom-crawler',
|
|
'linkedom-crawler',
|
|
'stagehand-crawler',
|
|
'memory-storage',
|
|
'utils',
|
|
'types',
|
|
'impit-client',
|
|
];
|
|
const packagesOrder = [
|
|
'@crawlee/core',
|
|
'@crawlee/cheerio',
|
|
'@crawlee/playwright',
|
|
'@crawlee/puppeteer',
|
|
'@crawlee/jsdom',
|
|
'@crawlee/linkedom',
|
|
'@crawlee/stagehand',
|
|
'@crawlee/basic',
|
|
'@crawlee/http',
|
|
'@crawlee/browser',
|
|
'@crawlee/memory-storage',
|
|
'@crawlee/browser-pool',
|
|
'@crawlee/utils',
|
|
'@crawlee/types',
|
|
'@crawlee/impit-client',
|
|
];
|
|
|
|
/** @type {Partial<import('@docusaurus/types').DocusaurusConfig>} */
|
|
module.exports = {
|
|
title: 'Crawlee for JavaScript · Build reliable crawlers. Fast.',
|
|
tagline: 'Build reliable crawlers. Fast.',
|
|
url: 'https://crawlee.dev',
|
|
baseUrl: '/',
|
|
trailingSlash: false,
|
|
organizationName: 'apify',
|
|
projectName: 'crawlee',
|
|
scripts: ['/js/custom.js'],
|
|
favicon: 'img/favicon.ico',
|
|
customFields: {
|
|
markdownOptions: {
|
|
html: true,
|
|
},
|
|
gaGtag: true,
|
|
repoUrl: 'https://github.com/apify/crawlee',
|
|
},
|
|
onBrokenLinks: 'throw',
|
|
markdown: {
|
|
mermaid: true,
|
|
hooks: {
|
|
onBrokenMarkdownLinks: 'throw',
|
|
},
|
|
},
|
|
themes: [
|
|
'@docusaurus/theme-mermaid',
|
|
],
|
|
future: {
|
|
faster: {
|
|
// ssgWorkerThreads: true,
|
|
swcJsLoader: true,
|
|
swcJsMinimizer: true,
|
|
swcHtmlMinimizer: true,
|
|
lightningCssMinimizer: true,
|
|
rspackBundler: true,
|
|
mdxCrossCompilerCache: true,
|
|
rspackPersistentCache: true,
|
|
},
|
|
v4: {
|
|
removeLegacyPostBuildHeadAttribute: true,
|
|
useCssCascadeLayers: false,
|
|
},
|
|
},
|
|
presets: /** @type {import('@docusaurus/types').PresetConfig[]} */ ([
|
|
[
|
|
'@docusaurus/preset-classic',
|
|
/** @type {import('@docusaurus/preset-classic').Options} */
|
|
({
|
|
docs: {
|
|
showLastUpdateAuthor: true,
|
|
showLastUpdateTime: true,
|
|
path: '../docs',
|
|
routeBasePath: 'js/docs',
|
|
sidebarPath: './sidebars.js',
|
|
rehypePlugins: [externalLinkProcessor],
|
|
disableVersioning: !!process.env.CRAWLEE_DOCS_FAST,
|
|
editUrl: (doc) => {
|
|
return `https://github.com/apify/crawlee/edit/master/website/${doc.versionDocsDirPath}/${doc.docPath}`;
|
|
},
|
|
},
|
|
blog: {
|
|
blogTitle: 'Crawlee Blog - learn how to build better scrapers',
|
|
// eslint-disable-next-line max-len
|
|
blogDescription: 'Guides and tutorials on using Crawlee, the most reliable open-source web scraping and browser automation library for JavaScript and Node.js developers.',
|
|
blogSidebarTitle: 'All posts',
|
|
blogSidebarCount: 'ALL',
|
|
},
|
|
theme: {
|
|
customCss: '/src/css/custom.css',
|
|
},
|
|
}),
|
|
],
|
|
]),
|
|
headTags: [
|
|
// Intercom messenger
|
|
{
|
|
tagName: 'script',
|
|
innerHTML: `window.intercomSettings={api_base:"https://api-iam.intercom.io",app_id:"kod1r788"};`,
|
|
attributes: {},
|
|
},
|
|
// Intercom messenger
|
|
{
|
|
tagName: 'script',
|
|
innerHTML: `(function(){var w=window;var ic=w.Intercom;if(typeof ic==="function"){ic('reattach_activator');ic('update',w.intercomSettings);}else{var d=document;var i=function(){i.c(arguments);};i.q=[];i.c=function(args){i.q.push(args);};w.Intercom=i;var l=function(){var s=d.createElement('script');s.type='text/javascript';s.async=true;s.src='https://widget.intercom.io/widget/kod1r788';var x=d.getElementsByTagName('script')[0];x.parentNode.insertBefore(s,x);};if(document.readyState==='complete'){l();}else if(w.attachEvent){w.attachEvent('onload',l);}else{w.addEventListener('load',l,false);}}})()`,
|
|
attributes: {},
|
|
},
|
|
],
|
|
plugins: [
|
|
[
|
|
'@apify/docusaurus-plugin-typedoc-api',
|
|
{
|
|
projectRoot: `${__dirname}/..`,
|
|
changelogs: true,
|
|
readmes: true,
|
|
routeBasePath: 'js/api',
|
|
sortPackages: (a, b) => {
|
|
return packagesOrder.indexOf(a.packageName) - packagesOrder.indexOf(b.packageName);
|
|
},
|
|
packages: packages.map((name) => ({ path: `packages/${name}` })),
|
|
typedocOptions: {
|
|
excludeExternals: false,
|
|
},
|
|
},
|
|
],
|
|
[
|
|
'@docusaurus/plugin-client-redirects',
|
|
{
|
|
redirects: [
|
|
{
|
|
from: '/js/docs',
|
|
to: '/js/docs/quick-start',
|
|
},
|
|
{
|
|
from: '/js/docs/next',
|
|
to: '/js/docs/next/quick-start',
|
|
},
|
|
{
|
|
from: '/js/docs/guides/environment-variables',
|
|
to: '/js/docs/guides/configuration',
|
|
},
|
|
{
|
|
from: '/js/docs/guides/getting-started',
|
|
to: '/js/docs/introduction',
|
|
},
|
|
{
|
|
from: '/js/docs/guides/apify-platform',
|
|
to: '/js/docs/deployment/apify-platform',
|
|
},
|
|
{
|
|
from: '/js/docs/experiments/experiments-system-infomation-v2',
|
|
to: '/js/docs/experiments/experiments-system-information-v2',
|
|
},
|
|
{
|
|
from: '/js/docs/next/experiments/experiments-system-infomation-v2',
|
|
to: '/js/docs/next/experiments/experiments-system-information-v2',
|
|
},
|
|
{
|
|
from: '/js/docs/3.13/experiments/experiments-system-infomation-v2',
|
|
to: '/js/docs/3.13/experiments/experiments-system-information-v2',
|
|
},
|
|
{
|
|
from: '/js/docs/3.14/experiments/experiments-system-infomation-v2',
|
|
to: '/js/docs/3.14/experiments/experiments-system-information-v2',
|
|
},
|
|
{
|
|
from: '/js/docs/3.15/experiments/experiments-system-infomation-v2',
|
|
to: '/js/docs/3.15/experiments/experiments-system-information-v2',
|
|
},
|
|
{
|
|
from: '/js/docs/3.16/experiments/experiments-system-infomation-v2',
|
|
to: '/js/docs/experiments/experiments-system-information-v2',
|
|
},
|
|
],
|
|
// createRedirects(existingPath) {
|
|
// if (!existingPath.endsWith('/')) {
|
|
// return `${existingPath}/`;
|
|
// }
|
|
//
|
|
// return undefined; // Return a falsy value: no redirect created
|
|
// },
|
|
},
|
|
],
|
|
[
|
|
'docusaurus-gtm-plugin',
|
|
{
|
|
id: 'GTM-5P7MCS7',
|
|
},
|
|
],
|
|
[
|
|
'@signalwire/docusaurus-plugin-llms-txt',
|
|
{
|
|
enableDescriptions: false,
|
|
optionalLinks: [
|
|
{
|
|
title: 'Crawlee for Python llms.txt',
|
|
url: 'https://crawlee.dev/python/llms.txt',
|
|
},
|
|
{
|
|
title: 'Crawlee for Python llms-full.txt',
|
|
url: 'https://crawlee.dev/python/llms-full.txt',
|
|
}
|
|
],
|
|
content: {
|
|
excludeRoutes: ['/js/api/3.*/**', '/js/api/3.*', '/js/api/next/**', '/js/api/next'],
|
|
includeVersionedDocs: false,
|
|
enableLlmsFullTxt: true,
|
|
includeBlog: true,
|
|
includePages: true,
|
|
relativePaths: false,
|
|
},
|
|
}
|
|
],
|
|
async function runnableCodeBlock() {
|
|
return {
|
|
name: 'runnable-code-block',
|
|
configureWebpack() {
|
|
return {
|
|
resolveLoader: {
|
|
alias: {
|
|
'roa-loader': require.resolve(`${__dirname}/roa-loader/`),
|
|
},
|
|
},
|
|
};
|
|
},
|
|
};
|
|
},
|
|
// skipping svgo for animated crawlee logo
|
|
async function doNotUseSVGO() {
|
|
return {
|
|
name: 'docusaurus-svgo',
|
|
configureWebpack(config) {
|
|
// find the svg rule
|
|
const svgRule = config.module.rules.find((r) => typeof r === 'object' && r.test.toString() === '/\\.svg$/i');
|
|
|
|
// find the svgr loader
|
|
const svgrLoader = svgRule?.oneOf?.[0];
|
|
|
|
// make copy of svgr loader and disable svgo
|
|
const svgrLoaderCopy = JSON.parse(JSON.stringify(svgrLoader));
|
|
|
|
// include only animated logo
|
|
svgrLoaderCopy.include = /animated-crawlee-logo/;
|
|
|
|
// turn off svgo
|
|
svgrLoaderCopy.use[0].options.svgo = false;
|
|
|
|
// insert the copy after the original svgr loader
|
|
svgRule.oneOf.splice(1, 0, svgrLoaderCopy);
|
|
|
|
// exclude animated logo from the first svgr loader (with svgo enabled)
|
|
svgrLoader.exclude = /animated-crawlee-logo/;
|
|
|
|
return {
|
|
mergeStrategy: {
|
|
'module.rules': 'replace',
|
|
},
|
|
module: {
|
|
rules: config.module.rules,
|
|
},
|
|
};
|
|
},
|
|
};
|
|
},
|
|
[
|
|
path.resolve(__dirname, 'src/plugins/docusaurus-plugin-segment'),
|
|
{
|
|
writeKey: process.env.SEGMENT_TOKEN,
|
|
allowedInDev: false,
|
|
},
|
|
],
|
|
],
|
|
themeConfig:
|
|
/** @type {import('@docusaurus/preset-classic').ThemeConfig} */ ({
|
|
docs: {
|
|
versionPersistence: 'localStorage',
|
|
sidebar: {
|
|
hideable: true,
|
|
},
|
|
},
|
|
navbar: {
|
|
hideOnScroll: true,
|
|
title: 'Crawlee',
|
|
logo: {
|
|
src: 'img/crawlee-light.svg',
|
|
srcDark: 'img/crawlee-dark.svg',
|
|
},
|
|
items: [
|
|
{
|
|
type: 'doc',
|
|
docId: 'quick-start/quick-start',
|
|
label: 'Docs',
|
|
position: 'left',
|
|
},
|
|
{
|
|
type: 'doc',
|
|
docId: '/examples',
|
|
label: 'Examples',
|
|
position: 'left',
|
|
},
|
|
{
|
|
type: 'custom-api',
|
|
to: 'core',
|
|
label: 'API',
|
|
position: 'left',
|
|
activeBaseRegex: 'api/(?!.*/changelog)',
|
|
},
|
|
{
|
|
type: 'custom-api',
|
|
to: 'core/changelog',
|
|
label: 'Changelog',
|
|
position: 'left',
|
|
className: 'changelog',
|
|
activeBaseRegex: 'changelog',
|
|
},
|
|
{
|
|
to: 'blog',
|
|
label: 'Blog',
|
|
position: 'left',
|
|
},
|
|
],
|
|
},
|
|
colorMode: {
|
|
defaultMode: 'light',
|
|
disableSwitch: false,
|
|
respectPrefersColorScheme: true,
|
|
},
|
|
prism: {
|
|
defaultLanguage: 'typescript',
|
|
theme: require('prism-react-renderer').themes.github,
|
|
darkTheme: require('prism-react-renderer').themes.dracula,
|
|
additionalLanguages: ['docker', 'log', 'bash', 'diff', 'json'],
|
|
},
|
|
metadata: [
|
|
// eslint-disable-next-line max-len
|
|
{ name: 'description', content: `Crawlee helps you build and maintain your crawlers. It's open source, but built by developers who scrape millions of pages every day for a living.` },
|
|
// eslint-disable-next-line max-len
|
|
{ name: 'og:description', content: `Crawlee helps you build and maintain your crawlers. It's open source, but built by developers who scrape millions of pages every day for a living.` },
|
|
],
|
|
image: 'img/crawlee-js-og.png',
|
|
footer: {
|
|
links: [
|
|
{
|
|
title: 'Docs',
|
|
items: [
|
|
{
|
|
label: 'Guides',
|
|
to: 'js/docs/guides',
|
|
},
|
|
{
|
|
label: 'Examples',
|
|
to: 'js/docs/examples',
|
|
},
|
|
{
|
|
label: 'API reference',
|
|
to: 'js/api/core',
|
|
},
|
|
{
|
|
label: 'Changelog',
|
|
to: 'js/api/core/changelog',
|
|
},
|
|
],
|
|
},
|
|
{
|
|
title: 'Product',
|
|
items: [
|
|
{
|
|
label: 'Discord',
|
|
href: 'https://discord.com/invite/jyEM2PRvMU',
|
|
},
|
|
{
|
|
label: 'Stack Overflow',
|
|
href: 'https://stackoverflow.com/questions/tagged/crawlee',
|
|
},
|
|
{
|
|
label: 'Twitter',
|
|
href: 'https://twitter.com/apify',
|
|
},
|
|
{
|
|
label: 'YouTube',
|
|
href: 'https://www.youtube.com/apify',
|
|
},
|
|
|
|
],
|
|
},
|
|
{
|
|
title: 'More',
|
|
items: [
|
|
{
|
|
label: 'Apify platform',
|
|
href: 'https://apify.com',
|
|
},
|
|
{
|
|
label: 'Docusaurus',
|
|
href: 'https://docusaurus.io',
|
|
},
|
|
{
|
|
label: 'GitHub',
|
|
href: 'https://github.com/apify/crawlee',
|
|
},
|
|
],
|
|
},
|
|
],
|
|
},
|
|
algolia: {
|
|
appId: '5JC94MPMLY',
|
|
apiKey: '267679200b833c2ca1255ab276731869', // search only (public) API key
|
|
indexName: 'crawlee',
|
|
placeholder: 'Search documentation',
|
|
algoliaOptions: {
|
|
facetFilters: ['version:VERSION'],
|
|
},
|
|
translations: {
|
|
button: {
|
|
buttonText: 'Search documentation...',
|
|
},
|
|
},
|
|
},
|
|
}),
|
|
};
|