import { cli, Strategy } from '@jackwener/opencli/registry'; import { AuthRequiredError, EmptyResultError } from '@jackwener/opencli/errors'; import { compactRepeatedText } from './shared.js'; function normalizeWhitespace(value) { return String(value ?? '').replace(/\s+/g, ' ').trim(); } function parseMetric(value) { const raw = normalizeWhitespace(value).toLowerCase(); if (!raw) return 0; const compact = raw.replace(/,/g, ''); const match = compact.match(/(\d+(?:\.\d+)?)(k|m)?/i); if (!match) return 0; const base = Number(match[1]); const suffix = (match[2] || '').toLowerCase(); if (suffix === 'k') return Math.round(base * 1000); if (suffix === 'm') return Math.round(base * 1000000); return Math.round(base); } function buildPostId(post) { const url = normalizeWhitespace(post.url); if (url) return url; const author = compactRepeatedText(post.author); const text = normalizeWhitespace(post.text); const postedAt = normalizeTimestamp(post.posted_at); return `${author}::${postedAt}::${text.slice(0, 120)}`; } function mergeTimelinePosts(existing, batch) { const seen = new Set(existing.map(post => post.id)); const merged = [...existing]; for (const rawPost of batch) { const post = { id: buildPostId(rawPost), author: compactRepeatedText(rawPost.author), author_url: normalizeWhitespace(rawPost.author_url), headline: compactRepeatedText(rawPost.headline), text: normalizeWhitespace(rawPost.text), posted_at: normalizeTimestamp(rawPost.posted_at), reactions: Number(rawPost.reactions) || 0, comments: Number(rawPost.comments) || 0, url: normalizeWhitespace(rawPost.url), }; if (!post.author || !post.text) continue; if (seen.has(post.id)) continue; seen.add(post.id); merged.push(post); } return merged; } function normalizeTimestamp(value) { const text = normalizeWhitespace(value).replace(/[•·.]$/g, '').trim(); const match = text.match(/\b(\d+\s*(?:s|m|h|d|w|mo|yr|min))\b/i); return match ? match[1].replace(/\s+/g, '') : text; } async function extractVisiblePosts(page) { return page.evaluate(`(function () { function normalize(value) { return String(value || '').replace(/\\s+/g, ' ').trim(); } function textOf(root, selector) { var el = root.querySelector(selector); return el ? el.textContent : ''; } function hrefOf(root, selector) { var el = root.querySelector(selector); return el && el.href ? el.href : ''; } function attrOf(root, selector, attr) { var el = root.querySelector(selector); return el ? el.getAttribute(attr) : ''; } function cleanTimestamp(value) { return normalize(String(value || '').replace(/[•.]/g, ' ')); } function parseMetric(value) { var raw = normalize(value).toLowerCase(); var match; var base; var suffix; if (!raw) return 0; match = raw.replace(/,/g, '').match(/(\\d+(?:\\.\\d+)?)(k|m)?/i); if (!match) return 0; base = Number(match[1]); suffix = (match[2] || '').toLowerCase(); if (suffix === 'k') return Math.round(base * 1000); if (suffix === 'm') return Math.round(base * 1000000); return Math.round(base); } function splitBlocks(text) { var lines = String(text || '').split('\\n'); var blocks = []; var current = []; var i; var line; for (i = 0; i < lines.length; i += 1) { line = normalize(lines[i]); if (!line) { if (current.length) { blocks.push(normalize(current.join(' '))); current = []; } continue; } current.push(line); } if (current.length) blocks.push(normalize(current.join(' '))); return blocks; } function looksLikeTimestamp(value) { var lower = String(value || '').toLowerCase(); return /^\\d+\\s*(s|m|h|d|w|mo|yr|min)(\\s*[•.])?$/i.test(lower); } function looksLikeBadge(value) { var lower = String(value || '').toLowerCase(); return String(value || '').indexOf('•') === 0 || lower === '1st' || lower === '2nd' || lower === '3rd' || lower === 'degree connection'; } function looksLikeAction(value) { return /^(follow|send message|connect|visit my website|view my newsletter|subscribe)$/i.test((value || '').toLowerCase()); } function looksLikeCta(value) { return /^(book an appointment|view my services|visit my website|view my newsletter|subscribe|learn more|contact us)$/i.test((value || '').toLowerCase()); } function looksLikeEngagement(value) { return /(reactions?|comments?|reposts?)/i.test(String(value || '')); } function looksLikeFooterAction(value) { return /^(like|comment|repost|send|reply|load more comments)$/i.test((value || '').toLowerCase()); } function findActivityUrn(root) { var elements = [root].concat(Array.from(root.querySelectorAll('*'))); var i; var j; var attrs; var value; var match; for (i = 0; i < elements.length; i += 1) { attrs = Array.from(elements[i].attributes || []); for (j = 0; j < attrs.length; j += 1) { value = String(attrs[j].value || ''); match = value.match(/urn:li:activity:\\d+/); if (match) return match[0]; } } return ''; } function parseReactionCount(root, blocks) { var direct = textOf(root, '.social-details-social-counts__reactions-count'); var rootText = String(root.innerText || ''); var i; var value; value = rootText.match(/and\\s+(\\d[\\d,]*)\\s+others\\s+reacted/i); if (value) return parseMetric(value[1]) + 1; value = rootText.match(/and\\s+(\\d[\\d,]*)\\s+others(?!\\s+comments?)(?!\\s+reposts?)/i); if (value) return parseMetric(value[1]) + 1; value = rootText.match(/(\\d[\\d,]*)\\s+reactions?/i); if (value) return parseMetric(value[0]); if (direct) return parseMetric(direct); for (i = 0; i < blocks.length; i += 1) { value = blocks[i]; if (/and\\s+\\d[\\d,]*\\s+others(?!\\s+comments?)(?!\\s+reposts?)/i.test(value)) { return parseMetric(value) + 1; } if (/reactions?/i.test(value)) return parseMetric(value); if (/and\\s+\\d+[\\d,]*\\s+others\\s+reacted/i.test(value)) return parseMetric(value) + 1; } return 0; } function parseCommentCount(blocks) { var i; var text = blocks.join(' '); var match = text.match(/(\\d[\\d,]*)\\s+comments?/i); if (match) return parseMetric(match[0]); for (i = 0; i < blocks.length; i += 1) { if (/comments?/i.test(blocks[i])) return parseMetric(blocks[i]); } return 0; } function selectProfileLink(root, author) { var links = Array.from(root.querySelectorAll('a[href*="/in/"], a[href*="/company/"]')); var normalizedAuthor = normalize(author).toLowerCase(); var i; var label; for (i = 0; i < links.length; i += 1) { label = normalize(links[i].textContent || links[i].getAttribute('aria-label')).toLowerCase(); if (!links[i].href) continue; if (normalizedAuthor && label.indexOf(normalizedAuthor) >= 0) return links[i]; } return links[0] || null; } function selectProfileUrl(root, author) { var link = selectProfileLink(root, author); return link && link.href ? link.href : ''; } function parseActorLinkMeta(root, author) { var link = selectProfileLink(root, author); var text = normalize(link ? link.textContent : ''); var normalizedAuthor = normalize(author); var match; var rest; var headline = ''; var postedAt = ''; if (!text || !normalizedAuthor) return { headline: '', postedAt: '' }; if (text.indexOf(normalizedAuthor) === 0) { rest = normalize(text.slice(normalizedAuthor.length)); } else { rest = text; } rest = normalize(rest.replace(/^[•·]\\s*(1st|2nd|3rd\\+?|3rd|degree connection)/i, '')); match = rest.match(/(\\d+\\s*(?:s|m|h|d|w|mo|yr|min))\\s*[•·]?$/i); if (match) { postedAt = cleanTimestamp(match[1]); headline = normalize(rest.slice(0, rest.length - match[0].length)); } else { headline = rest; } headline = normalize(headline.replace(/^(book an appointment|view my services|visit my website|view my newsletter)\\s*/i, '')); return { headline: headline, postedAt: postedAt }; } function stripBodyTail(value) { return normalize(String(value || '') .replace(/\\s+\\d[\\d,]*\\s+reactions?[\\s\\S]*$/i, '') .replace(/\\s+\\d[\\d,]*\\s+comments?[\\s\\S]*$/i, '') .replace(/\\s+[A-Z][^\\n]+\\s+and\\s+\\d[\\d,]*\\s+others\\s+reacted[\\s\\S]*$/i, '') .replace(/\\s+Like\\s+Comment\\s+Repost\\s+Send[\\s\\S]*$/i, '') .replace(/\\s+Reaction button state:[\\s\\S]*$/i, '') .replace(/^\\d+\\s*(?:s|m|h|d|w|mo|yr|min)\\s*[•.]?\\s*Follow\\s+/i, '') ); } function parseActorMeta(root) { var actorLink = root.querySelector('a[href*="/in/"], a[href*="/company/"]'); var actorText = normalize(actorLink ? actorLink.textContent : ''); var author = ''; var headline = ''; var postedAt = ''; var match; if (actorText) { match = actorText.match(/^(.+?)\\s+[•·]\\s+(1st|2nd|3rd\\+?|3rd|degree connection)(.*)$/i); if (match) { author = normalize(match[1]); actorText = normalize(match[3]); } } match = actorText.match(/(.+?)\\s+(\\d+\\s*(?:s|m|h|d|w|mo|yr|min))\\s*[•·]?$/i); if (match) { headline = normalize(match[1]); postedAt = cleanTimestamp(match[2]); } else if (actorText) { headline = actorText; } return { author: author, headline: headline, postedAt: postedAt, authorUrl: actorLink && actorLink.href ? actorLink.href : '', }; } function extractFromListItem(root) { var blocks = splitBlocks(root.innerText || ''); var filtered = []; var i; var value; var author = ''; var authorUrl = ''; var headline = ''; var postedAt = ''; var text = ''; var bodyStart = -1; var permalink; var url; var reactions; var comments; var endIndex = -1; var urn; if (blocks.length < 5) return null; if (blocks[0] !== 'Feed post') return null; for (i = 1; i < blocks.length; i += 1) { value = blocks[i]; if (!value) continue; if (/commented on this|reposted this|liked this|suggested/i.test(value)) continue; filtered.push(value); } if (filtered.length < 4) return null; for (i = 0; i < filtered.length; i += 1) { value = filtered[i]; if (!author && !looksLikeBadge(value) && !looksLikeAction(value) && !looksLikeTimestamp(value)) { author = value; continue; } if (author && !headline && !looksLikeBadge(value) && !looksLikeAction(value) && !looksLikeTimestamp(value) && !looksLikeCta(value)) { headline = value; continue; } if (!postedAt && looksLikeTimestamp(value)) { postedAt = cleanTimestamp(value); continue; } } if (!author) return null; authorUrl = selectProfileUrl(root, author); if (!headline || !postedAt) { var actorMeta = parseActorLinkMeta(root, author); if (!headline && actorMeta.headline) headline = actorMeta.headline; if (!postedAt && actorMeta.postedAt) postedAt = actorMeta.postedAt; } for (i = 0; i < filtered.length; i += 1) { value = filtered[i]; if (looksLikeAction(value)) { bodyStart = i + 1; break; } } if (bodyStart < 0 && postedAt) { bodyStart = filtered.indexOf(postedAt) + 1; } if (bodyStart < 0) bodyStart = Math.min(filtered.length, headline ? 2 : 1); for (i = bodyStart; i < filtered.length; i += 1) { value = filtered[i]; if (looksLikeEngagement(value) || looksLikeFooterAction(value)) { endIndex = i; break; } } if (endIndex < 0) endIndex = filtered.length; text = stripBodyTail(filtered.slice(bodyStart, endIndex).join('\\n\\n')); if (!text) return null; permalink = root.querySelector('a[href*="/feed/update/"], a[href*="/posts/"], a[href*="/pulse/"]'); url = permalink ? permalink.href : ''; urn = findActivityUrn(root); if (!url && urn) url = 'https://www.linkedin.com/feed/update/' + urn + '/'; reactions = parseReactionCount(root, filtered); comments = parseCommentCount(filtered); return { id: url || (author + '::' + postedAt + '::' + text.slice(0, 120)), author: author, author_url: authorUrl, headline: headline, text: text, posted_at: postedAt, reactions: reactions, comments: comments, url: url, }; } function commentMetric(root) { var links = Array.from(root.querySelectorAll('button, a')); var i; var label; for (i = 0; i < links.length; i += 1) { label = normalize(links[i].textContent || links[i].getAttribute('aria-label')); if (/comment/i.test(label)) return parseMetric(label); } return 0; } var currentUrl = window.location.href; var path = String(window.location.pathname || ''); var loginRequired = path.indexOf('/login') >= 0 || path.indexOf('/checkpoint/') >= 0 || Boolean(document.querySelector('input[name="session_key"], form.login__form')); var moreButtons = Array.from(document.querySelectorAll('button, a[role="button"]')) .filter(function (el) { return /see more|more/i.test(normalize(el.textContent)) || /see more|more/i.test(normalize(el.getAttribute('aria-label'))); }) .slice(0, 8); var cards = Array.from(document.querySelectorAll('article, [role="article"], .feed-shared-update-v2, .occludable-update, [role="listitem"]')); var seen = new Set(); var posts = []; var i; var card; var root; var author; var headline; var text; var postedAt; var permalink; var url; var reactions; var comments; for (i = 0; i < moreButtons.length; i += 1) { try { moreButtons[i].click(); } catch (err) {} } for (i = 0; i < cards.length; i += 1) { card = cards[i]; root = card.closest('article, [role="article"], .feed-shared-update-v2, .occludable-update, [role="listitem"]') || card; if (!root || seen.has(root)) continue; seen.add(root); if (String(root.getAttribute('role') || '') === 'listitem') { var extracted = extractFromListItem(root); if (extracted) posts.push(extracted); continue; } author = normalize( textOf(root, '.update-components-actor__title span[dir="ltr"]') || textOf(root, '.update-components-actor__title') || textOf(root, '[data-control-name="actor"] span[dir="ltr"]') || textOf(root, '[data-control-name="actor"]') ); headline = normalize( textOf(root, '.update-components-actor__description') || textOf(root, '.update-components-actor__sub-description') ); text = normalize( textOf(root, '.update-components-text span[dir="ltr"]') || textOf(root, '.update-components-text') || textOf(root, '.feed-shared-inline-show-more-text span[dir="ltr"]') || textOf(root, '.feed-shared-inline-show-more-text') || textOf(root, '[data-test-id="main-feed-activity-card"] .break-words') ); postedAt = normalize( textOf(root, '.update-components-actor__sub-description a') || textOf(root, '.update-components-actor__sub-description span[aria-hidden="true"]') || textOf(root, 'time') ); permalink = root.querySelector('a[href*="/feed/update/"], a[href*="/posts/"], a[href*="/pulse/"]'); url = permalink ? permalink.href : ''; if (url && url.indexOf('/') === 0) url = new URL(url, currentUrl).toString(); reactions = parseMetric( textOf(root, '.social-details-social-counts__reactions-count') || attrOf(root, '[aria-label*="reaction"]', 'aria-label') || attrOf(root, '[aria-label*="like"]', 'aria-label') ); comments = commentMetric(root); if (!author || !text) continue; posts.push({ id: url || (author + '::' + postedAt + '::' + text.slice(0, 120)), author: author, author_url: hrefOf(root, 'a[href*="/in/"], a[href*="/company/"]'), headline: headline, text: text, posted_at: postedAt, reactions: reactions, comments: comments, url: url, }); } return { loginRequired: loginRequired, posts: posts }; })()`); } cli({ site: 'linkedin', name: 'timeline', access: 'read', description: 'Read LinkedIn home timeline posts', domain: 'www.linkedin.com', strategy: Strategy.COOKIE, browser: true, args: [ { name: 'limit', type: 'int', default: 20, help: 'Number of posts to return (max 100)' }, ], columns: ['rank', 'author', 'author_url', 'headline', 'text', 'posted_at', 'reactions', 'comments', 'url'], func: async (page, kwargs) => { const limit = Math.max(1, Math.min(kwargs.limit ?? 20, 100)); await page.goto('https://www.linkedin.com/feed/'); await page.wait(4); let posts = []; let sawLoginWall = false; for (let i = 0; i < 6 && posts.length < limit; i++) { const batch = await extractVisiblePosts(page); if (batch?.loginRequired) sawLoginWall = true; posts = mergeTimelinePosts(posts, Array.isArray(batch?.posts) ? batch.posts : []); if (posts.length >= limit) break; await page.autoScroll({ times: 1, delayMs: 1200 }); await page.wait(1); } if (sawLoginWall && posts.length === 0) { throw new AuthRequiredError('linkedin.com', 'LinkedIn timeline requires an active signed-in browser session'); } if (posts.length === 0) { throw new EmptyResultError('linkedin timeline', 'Make sure your LinkedIn home feed is visible in the browser.'); } return posts.slice(0, limit).map((post, index) => ({ rank: index + 1, ...post, })); }, }); export const __test__ = { parseMetric, buildPostId, mergeTimelinePosts, normalizeTimestamp, };