wehub-resource-sync
9b395f5cc3
Build Chrome Extension / build (push) Waiting to run
Trigger Website Rebuild (Docs Updated) / dispatch (push) Waiting to run
E2E Headed Chrome / e2e-headed (macos-15) (push) Has been cancelled
E2E Headed Chrome / e2e-headed (ubuntu-latest) (push) Has been cancelled
E2E Headed Chrome / e2e-headed (windows-latest) (push) Has been cancelled
CI / build (macos-latest) (push) Has been cancelled
CI / build (ubuntu-latest) (push) Has been cancelled
CI / build (windows-latest) (push) Has been cancelled
CI / unit-test (push) Has been cancelled
CI / bun-test (push) Has been cancelled
CI / adapter-test (push) Has been cancelled
CI / smoke-test (macos-latest) (push) Has been cancelled
CI / smoke-test (ubuntu-latest) (push) Has been cancelled
Security Audit / audit (push) Has been cancelled
512 lines
19 KiB
JavaScript
512 lines
19 KiB
JavaScript
import { cli, Strategy } from '@jackwener/opencli/registry';
|
|
import { AuthRequiredError, EmptyResultError } from '@jackwener/opencli/errors';
|
|
import { compactRepeatedText } from './shared.js';
|
|
function normalizeWhitespace(value) {
|
|
return String(value ?? '').replace(/\s+/g, ' ').trim();
|
|
}
|
|
function parseMetric(value) {
|
|
const raw = normalizeWhitespace(value).toLowerCase();
|
|
if (!raw)
|
|
return 0;
|
|
const compact = raw.replace(/,/g, '');
|
|
const match = compact.match(/(\d+(?:\.\d+)?)(k|m)?/i);
|
|
if (!match)
|
|
return 0;
|
|
const base = Number(match[1]);
|
|
const suffix = (match[2] || '').toLowerCase();
|
|
if (suffix === 'k')
|
|
return Math.round(base * 1000);
|
|
if (suffix === 'm')
|
|
return Math.round(base * 1000000);
|
|
return Math.round(base);
|
|
}
|
|
function buildPostId(post) {
|
|
const url = normalizeWhitespace(post.url);
|
|
if (url)
|
|
return url;
|
|
const author = compactRepeatedText(post.author);
|
|
const text = normalizeWhitespace(post.text);
|
|
const postedAt = normalizeTimestamp(post.posted_at);
|
|
return `${author}::${postedAt}::${text.slice(0, 120)}`;
|
|
}
|
|
function mergeTimelinePosts(existing, batch) {
|
|
const seen = new Set(existing.map(post => post.id));
|
|
const merged = [...existing];
|
|
for (const rawPost of batch) {
|
|
const post = {
|
|
id: buildPostId(rawPost),
|
|
author: compactRepeatedText(rawPost.author),
|
|
author_url: normalizeWhitespace(rawPost.author_url),
|
|
headline: compactRepeatedText(rawPost.headline),
|
|
text: normalizeWhitespace(rawPost.text),
|
|
posted_at: normalizeTimestamp(rawPost.posted_at),
|
|
reactions: Number(rawPost.reactions) || 0,
|
|
comments: Number(rawPost.comments) || 0,
|
|
url: normalizeWhitespace(rawPost.url),
|
|
};
|
|
if (!post.author || !post.text)
|
|
continue;
|
|
if (seen.has(post.id))
|
|
continue;
|
|
seen.add(post.id);
|
|
merged.push(post);
|
|
}
|
|
return merged;
|
|
}
|
|
function normalizeTimestamp(value) {
|
|
const text = normalizeWhitespace(value).replace(/[•·.]$/g, '').trim();
|
|
const match = text.match(/\b(\d+\s*(?:s|m|h|d|w|mo|yr|min))\b/i);
|
|
return match ? match[1].replace(/\s+/g, '') : text;
|
|
}
|
|
async function extractVisiblePosts(page) {
|
|
return page.evaluate(`(function () {
|
|
function normalize(value) {
|
|
return String(value || '').replace(/\\s+/g, ' ').trim();
|
|
}
|
|
function textOf(root, selector) {
|
|
var el = root.querySelector(selector);
|
|
return el ? el.textContent : '';
|
|
}
|
|
function hrefOf(root, selector) {
|
|
var el = root.querySelector(selector);
|
|
return el && el.href ? el.href : '';
|
|
}
|
|
function attrOf(root, selector, attr) {
|
|
var el = root.querySelector(selector);
|
|
return el ? el.getAttribute(attr) : '';
|
|
}
|
|
function cleanTimestamp(value) {
|
|
return normalize(String(value || '').replace(/[•.]/g, ' '));
|
|
}
|
|
function parseMetric(value) {
|
|
var raw = normalize(value).toLowerCase();
|
|
var match;
|
|
var base;
|
|
var suffix;
|
|
if (!raw) return 0;
|
|
match = raw.replace(/,/g, '').match(/(\\d+(?:\\.\\d+)?)(k|m)?/i);
|
|
if (!match) return 0;
|
|
base = Number(match[1]);
|
|
suffix = (match[2] || '').toLowerCase();
|
|
if (suffix === 'k') return Math.round(base * 1000);
|
|
if (suffix === 'm') return Math.round(base * 1000000);
|
|
return Math.round(base);
|
|
}
|
|
function splitBlocks(text) {
|
|
var lines = String(text || '').split('\\n');
|
|
var blocks = [];
|
|
var current = [];
|
|
var i;
|
|
var line;
|
|
for (i = 0; i < lines.length; i += 1) {
|
|
line = normalize(lines[i]);
|
|
if (!line) {
|
|
if (current.length) {
|
|
blocks.push(normalize(current.join(' ')));
|
|
current = [];
|
|
}
|
|
continue;
|
|
}
|
|
current.push(line);
|
|
}
|
|
if (current.length) blocks.push(normalize(current.join(' ')));
|
|
return blocks;
|
|
}
|
|
function looksLikeTimestamp(value) {
|
|
var lower = String(value || '').toLowerCase();
|
|
return /^\\d+\\s*(s|m|h|d|w|mo|yr|min)(\\s*[•.])?$/i.test(lower);
|
|
}
|
|
function looksLikeBadge(value) {
|
|
var lower = String(value || '').toLowerCase();
|
|
return String(value || '').indexOf('•') === 0
|
|
|| lower === '1st'
|
|
|| lower === '2nd'
|
|
|| lower === '3rd'
|
|
|| lower === 'degree connection';
|
|
}
|
|
function looksLikeAction(value) {
|
|
return /^(follow|send message|connect|visit my website|view my newsletter|subscribe)$/i.test((value || '').toLowerCase());
|
|
}
|
|
function looksLikeCta(value) {
|
|
return /^(book an appointment|view my services|visit my website|view my newsletter|subscribe|learn more|contact us)$/i.test((value || '').toLowerCase());
|
|
}
|
|
function looksLikeEngagement(value) {
|
|
return /(reactions?|comments?|reposts?)/i.test(String(value || ''));
|
|
}
|
|
function looksLikeFooterAction(value) {
|
|
return /^(like|comment|repost|send|reply|load more comments)$/i.test((value || '').toLowerCase());
|
|
}
|
|
function findActivityUrn(root) {
|
|
var elements = [root].concat(Array.from(root.querySelectorAll('*')));
|
|
var i;
|
|
var j;
|
|
var attrs;
|
|
var value;
|
|
var match;
|
|
for (i = 0; i < elements.length; i += 1) {
|
|
attrs = Array.from(elements[i].attributes || []);
|
|
for (j = 0; j < attrs.length; j += 1) {
|
|
value = String(attrs[j].value || '');
|
|
match = value.match(/urn:li:activity:\\d+/);
|
|
if (match) return match[0];
|
|
}
|
|
}
|
|
return '';
|
|
}
|
|
function parseReactionCount(root, blocks) {
|
|
var direct = textOf(root, '.social-details-social-counts__reactions-count');
|
|
var rootText = String(root.innerText || '');
|
|
var i;
|
|
var value;
|
|
value = rootText.match(/and\\s+(\\d[\\d,]*)\\s+others\\s+reacted/i);
|
|
if (value) return parseMetric(value[1]) + 1;
|
|
value = rootText.match(/and\\s+(\\d[\\d,]*)\\s+others(?!\\s+comments?)(?!\\s+reposts?)/i);
|
|
if (value) return parseMetric(value[1]) + 1;
|
|
value = rootText.match(/(\\d[\\d,]*)\\s+reactions?/i);
|
|
if (value) return parseMetric(value[0]);
|
|
if (direct) return parseMetric(direct);
|
|
for (i = 0; i < blocks.length; i += 1) {
|
|
value = blocks[i];
|
|
if (/and\\s+\\d[\\d,]*\\s+others(?!\\s+comments?)(?!\\s+reposts?)/i.test(value)) {
|
|
return parseMetric(value) + 1;
|
|
}
|
|
if (/reactions?/i.test(value)) return parseMetric(value);
|
|
if (/and\\s+\\d+[\\d,]*\\s+others\\s+reacted/i.test(value)) return parseMetric(value) + 1;
|
|
}
|
|
return 0;
|
|
}
|
|
function parseCommentCount(blocks) {
|
|
var i;
|
|
var text = blocks.join(' ');
|
|
var match = text.match(/(\\d[\\d,]*)\\s+comments?/i);
|
|
if (match) return parseMetric(match[0]);
|
|
for (i = 0; i < blocks.length; i += 1) {
|
|
if (/comments?/i.test(blocks[i])) return parseMetric(blocks[i]);
|
|
}
|
|
return 0;
|
|
}
|
|
function selectProfileLink(root, author) {
|
|
var links = Array.from(root.querySelectorAll('a[href*="/in/"], a[href*="/company/"]'));
|
|
var normalizedAuthor = normalize(author).toLowerCase();
|
|
var i;
|
|
var label;
|
|
for (i = 0; i < links.length; i += 1) {
|
|
label = normalize(links[i].textContent || links[i].getAttribute('aria-label')).toLowerCase();
|
|
if (!links[i].href) continue;
|
|
if (normalizedAuthor && label.indexOf(normalizedAuthor) >= 0) return links[i];
|
|
}
|
|
return links[0] || null;
|
|
}
|
|
function selectProfileUrl(root, author) {
|
|
var link = selectProfileLink(root, author);
|
|
return link && link.href ? link.href : '';
|
|
}
|
|
function parseActorLinkMeta(root, author) {
|
|
var link = selectProfileLink(root, author);
|
|
var text = normalize(link ? link.textContent : '');
|
|
var normalizedAuthor = normalize(author);
|
|
var match;
|
|
var rest;
|
|
var headline = '';
|
|
var postedAt = '';
|
|
if (!text || !normalizedAuthor) return { headline: '', postedAt: '' };
|
|
if (text.indexOf(normalizedAuthor) === 0) {
|
|
rest = normalize(text.slice(normalizedAuthor.length));
|
|
} else {
|
|
rest = text;
|
|
}
|
|
rest = normalize(rest.replace(/^[•·]\\s*(1st|2nd|3rd\\+?|3rd|degree connection)/i, ''));
|
|
match = rest.match(/(\\d+\\s*(?:s|m|h|d|w|mo|yr|min))\\s*[•·]?$/i);
|
|
if (match) {
|
|
postedAt = cleanTimestamp(match[1]);
|
|
headline = normalize(rest.slice(0, rest.length - match[0].length));
|
|
} else {
|
|
headline = rest;
|
|
}
|
|
headline = normalize(headline.replace(/^(book an appointment|view my services|visit my website|view my newsletter)\\s*/i, ''));
|
|
return { headline: headline, postedAt: postedAt };
|
|
}
|
|
function stripBodyTail(value) {
|
|
return normalize(String(value || '')
|
|
.replace(/\\s+\\d[\\d,]*\\s+reactions?[\\s\\S]*$/i, '')
|
|
.replace(/\\s+\\d[\\d,]*\\s+comments?[\\s\\S]*$/i, '')
|
|
.replace(/\\s+[A-Z][^\\n]+\\s+and\\s+\\d[\\d,]*\\s+others\\s+reacted[\\s\\S]*$/i, '')
|
|
.replace(/\\s+Like\\s+Comment\\s+Repost\\s+Send[\\s\\S]*$/i, '')
|
|
.replace(/\\s+Reaction button state:[\\s\\S]*$/i, '')
|
|
.replace(/^\\d+\\s*(?:s|m|h|d|w|mo|yr|min)\\s*[•.]?\\s*Follow\\s+/i, '')
|
|
);
|
|
}
|
|
function parseActorMeta(root) {
|
|
var actorLink = root.querySelector('a[href*="/in/"], a[href*="/company/"]');
|
|
var actorText = normalize(actorLink ? actorLink.textContent : '');
|
|
var author = '';
|
|
var headline = '';
|
|
var postedAt = '';
|
|
var match;
|
|
if (actorText) {
|
|
match = actorText.match(/^(.+?)\\s+[•·]\\s+(1st|2nd|3rd\\+?|3rd|degree connection)(.*)$/i);
|
|
if (match) {
|
|
author = normalize(match[1]);
|
|
actorText = normalize(match[3]);
|
|
}
|
|
}
|
|
match = actorText.match(/(.+?)\\s+(\\d+\\s*(?:s|m|h|d|w|mo|yr|min))\\s*[•·]?$/i);
|
|
if (match) {
|
|
headline = normalize(match[1]);
|
|
postedAt = cleanTimestamp(match[2]);
|
|
} else if (actorText) {
|
|
headline = actorText;
|
|
}
|
|
return {
|
|
author: author,
|
|
headline: headline,
|
|
postedAt: postedAt,
|
|
authorUrl: actorLink && actorLink.href ? actorLink.href : '',
|
|
};
|
|
}
|
|
function extractFromListItem(root) {
|
|
var blocks = splitBlocks(root.innerText || '');
|
|
var filtered = [];
|
|
var i;
|
|
var value;
|
|
var author = '';
|
|
var authorUrl = '';
|
|
var headline = '';
|
|
var postedAt = '';
|
|
var text = '';
|
|
var bodyStart = -1;
|
|
var permalink;
|
|
var url;
|
|
var reactions;
|
|
var comments;
|
|
var endIndex = -1;
|
|
var urn;
|
|
|
|
if (blocks.length < 5) return null;
|
|
if (blocks[0] !== 'Feed post') return null;
|
|
|
|
for (i = 1; i < blocks.length; i += 1) {
|
|
value = blocks[i];
|
|
if (!value) continue;
|
|
if (/commented on this|reposted this|liked this|suggested/i.test(value)) continue;
|
|
filtered.push(value);
|
|
}
|
|
if (filtered.length < 4) return null;
|
|
|
|
for (i = 0; i < filtered.length; i += 1) {
|
|
value = filtered[i];
|
|
if (!author && !looksLikeBadge(value) && !looksLikeAction(value) && !looksLikeTimestamp(value)) {
|
|
author = value;
|
|
continue;
|
|
}
|
|
if (author && !headline && !looksLikeBadge(value) && !looksLikeAction(value) && !looksLikeTimestamp(value) && !looksLikeCta(value)) {
|
|
headline = value;
|
|
continue;
|
|
}
|
|
if (!postedAt && looksLikeTimestamp(value)) {
|
|
postedAt = cleanTimestamp(value);
|
|
continue;
|
|
}
|
|
}
|
|
|
|
if (!author) return null;
|
|
authorUrl = selectProfileUrl(root, author);
|
|
if (!headline || !postedAt) {
|
|
var actorMeta = parseActorLinkMeta(root, author);
|
|
if (!headline && actorMeta.headline) headline = actorMeta.headline;
|
|
if (!postedAt && actorMeta.postedAt) postedAt = actorMeta.postedAt;
|
|
}
|
|
|
|
for (i = 0; i < filtered.length; i += 1) {
|
|
value = filtered[i];
|
|
if (looksLikeAction(value)) {
|
|
bodyStart = i + 1;
|
|
break;
|
|
}
|
|
}
|
|
if (bodyStart < 0 && postedAt) {
|
|
bodyStart = filtered.indexOf(postedAt) + 1;
|
|
}
|
|
if (bodyStart < 0) bodyStart = Math.min(filtered.length, headline ? 2 : 1);
|
|
|
|
for (i = bodyStart; i < filtered.length; i += 1) {
|
|
value = filtered[i];
|
|
if (looksLikeEngagement(value) || looksLikeFooterAction(value)) {
|
|
endIndex = i;
|
|
break;
|
|
}
|
|
}
|
|
if (endIndex < 0) endIndex = filtered.length;
|
|
|
|
text = stripBodyTail(filtered.slice(bodyStart, endIndex).join('\\n\\n'));
|
|
if (!text) return null;
|
|
|
|
permalink = root.querySelector('a[href*="/feed/update/"], a[href*="/posts/"], a[href*="/pulse/"]');
|
|
url = permalink ? permalink.href : '';
|
|
urn = findActivityUrn(root);
|
|
if (!url && urn) url = 'https://www.linkedin.com/feed/update/' + urn + '/';
|
|
reactions = parseReactionCount(root, filtered);
|
|
comments = parseCommentCount(filtered);
|
|
|
|
return {
|
|
id: url || (author + '::' + postedAt + '::' + text.slice(0, 120)),
|
|
author: author,
|
|
author_url: authorUrl,
|
|
headline: headline,
|
|
text: text,
|
|
posted_at: postedAt,
|
|
reactions: reactions,
|
|
comments: comments,
|
|
url: url,
|
|
};
|
|
}
|
|
function commentMetric(root) {
|
|
var links = Array.from(root.querySelectorAll('button, a'));
|
|
var i;
|
|
var label;
|
|
for (i = 0; i < links.length; i += 1) {
|
|
label = normalize(links[i].textContent || links[i].getAttribute('aria-label'));
|
|
if (/comment/i.test(label)) return parseMetric(label);
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
var currentUrl = window.location.href;
|
|
var path = String(window.location.pathname || '');
|
|
var loginRequired = path.indexOf('/login') >= 0
|
|
|| path.indexOf('/checkpoint/') >= 0
|
|
|| Boolean(document.querySelector('input[name="session_key"], form.login__form'));
|
|
var moreButtons = Array.from(document.querySelectorAll('button, a[role="button"]'))
|
|
.filter(function (el) {
|
|
return /see more|more/i.test(normalize(el.textContent))
|
|
|| /see more|more/i.test(normalize(el.getAttribute('aria-label')));
|
|
})
|
|
.slice(0, 8);
|
|
var cards = Array.from(document.querySelectorAll('article, [role="article"], .feed-shared-update-v2, .occludable-update, [role="listitem"]'));
|
|
var seen = new Set();
|
|
var posts = [];
|
|
var i;
|
|
var card;
|
|
var root;
|
|
var author;
|
|
var headline;
|
|
var text;
|
|
var postedAt;
|
|
var permalink;
|
|
var url;
|
|
var reactions;
|
|
var comments;
|
|
|
|
for (i = 0; i < moreButtons.length; i += 1) {
|
|
try { moreButtons[i].click(); } catch (err) {}
|
|
}
|
|
|
|
for (i = 0; i < cards.length; i += 1) {
|
|
card = cards[i];
|
|
root = card.closest('article, [role="article"], .feed-shared-update-v2, .occludable-update, [role="listitem"]') || card;
|
|
if (!root || seen.has(root)) continue;
|
|
seen.add(root);
|
|
|
|
if (String(root.getAttribute('role') || '') === 'listitem') {
|
|
var extracted = extractFromListItem(root);
|
|
if (extracted) posts.push(extracted);
|
|
continue;
|
|
}
|
|
|
|
author = normalize(
|
|
textOf(root, '.update-components-actor__title span[dir="ltr"]')
|
|
|| textOf(root, '.update-components-actor__title')
|
|
|| textOf(root, '[data-control-name="actor"] span[dir="ltr"]')
|
|
|| textOf(root, '[data-control-name="actor"]')
|
|
);
|
|
headline = normalize(
|
|
textOf(root, '.update-components-actor__description')
|
|
|| textOf(root, '.update-components-actor__sub-description')
|
|
);
|
|
text = normalize(
|
|
textOf(root, '.update-components-text span[dir="ltr"]')
|
|
|| textOf(root, '.update-components-text')
|
|
|| textOf(root, '.feed-shared-inline-show-more-text span[dir="ltr"]')
|
|
|| textOf(root, '.feed-shared-inline-show-more-text')
|
|
|| textOf(root, '[data-test-id="main-feed-activity-card"] .break-words')
|
|
);
|
|
postedAt = normalize(
|
|
textOf(root, '.update-components-actor__sub-description a')
|
|
|| textOf(root, '.update-components-actor__sub-description span[aria-hidden="true"]')
|
|
|| textOf(root, 'time')
|
|
);
|
|
permalink = root.querySelector('a[href*="/feed/update/"], a[href*="/posts/"], a[href*="/pulse/"]');
|
|
url = permalink ? permalink.href : '';
|
|
if (url && url.indexOf('/') === 0) url = new URL(url, currentUrl).toString();
|
|
reactions = parseMetric(
|
|
textOf(root, '.social-details-social-counts__reactions-count')
|
|
|| attrOf(root, '[aria-label*="reaction"]', 'aria-label')
|
|
|| attrOf(root, '[aria-label*="like"]', 'aria-label')
|
|
);
|
|
comments = commentMetric(root);
|
|
|
|
if (!author || !text) continue;
|
|
|
|
posts.push({
|
|
id: url || (author + '::' + postedAt + '::' + text.slice(0, 120)),
|
|
author: author,
|
|
author_url: hrefOf(root, 'a[href*="/in/"], a[href*="/company/"]'),
|
|
headline: headline,
|
|
text: text,
|
|
posted_at: postedAt,
|
|
reactions: reactions,
|
|
comments: comments,
|
|
url: url,
|
|
});
|
|
}
|
|
|
|
return { loginRequired: loginRequired, posts: posts };
|
|
})()`);
|
|
}
|
|
cli({
|
|
site: 'linkedin',
|
|
name: 'timeline',
|
|
access: 'read',
|
|
description: 'Read LinkedIn home timeline posts',
|
|
domain: 'www.linkedin.com',
|
|
strategy: Strategy.COOKIE,
|
|
browser: true,
|
|
args: [
|
|
{ name: 'limit', type: 'int', default: 20, help: 'Number of posts to return (max 100)' },
|
|
],
|
|
columns: ['rank', 'author', 'author_url', 'headline', 'text', 'posted_at', 'reactions', 'comments', 'url'],
|
|
func: async (page, kwargs) => {
|
|
const limit = Math.max(1, Math.min(kwargs.limit ?? 20, 100));
|
|
await page.goto('https://www.linkedin.com/feed/');
|
|
await page.wait(4);
|
|
let posts = [];
|
|
let sawLoginWall = false;
|
|
for (let i = 0; i < 6 && posts.length < limit; i++) {
|
|
const batch = await extractVisiblePosts(page);
|
|
if (batch?.loginRequired)
|
|
sawLoginWall = true;
|
|
posts = mergeTimelinePosts(posts, Array.isArray(batch?.posts) ? batch.posts : []);
|
|
if (posts.length >= limit)
|
|
break;
|
|
await page.autoScroll({ times: 1, delayMs: 1200 });
|
|
await page.wait(1);
|
|
}
|
|
if (sawLoginWall && posts.length === 0) {
|
|
throw new AuthRequiredError('linkedin.com', 'LinkedIn timeline requires an active signed-in browser session');
|
|
}
|
|
if (posts.length === 0) {
|
|
throw new EmptyResultError('linkedin timeline', 'Make sure your LinkedIn home feed is visible in the browser.');
|
|
}
|
|
return posts.slice(0, limit).map((post, index) => ({
|
|
rank: index + 1,
|
|
...post,
|
|
}));
|
|
},
|
|
});
|
|
export const __test__ = {
|
|
parseMetric,
|
|
buildPostId,
|
|
mergeTimelinePosts,
|
|
normalizeTimestamp,
|
|
};
|