chore: import upstream snapshot with attribution

scripts/issues-scraper/format-slack-message.ts

@@ -0,0 +1,93 @@
+import { ReportData, TrendData } from './model';
+import { getSinceDate } from './scrape-issues';
+import { table } from 'markdown-factory';
+
+export function getSlackMessageJson(body: string) {
+  return {
+    blocks: [
+      {
+        type: 'section',
+        text: {
+          text: body,
+          type: 'mrkdwn',
+        },
+      },
+    ],
+  };
+}
+
+export function formatGhReport(
+  currentData: ReportData,
+  trendData: TrendData,
+  prevData: ReportData,
+  unlabeledIssuesUrl: string
+): string {
+  const formattedIssueDelta = formatDelta(trendData.totalIssueCount);
+  const formattedBugDelta = formatDelta(trendData.totalBugCount);
+
+  const header = `Issue Report for ${currentData.collectedDate} <${unlabeledIssuesUrl}|[view unlabeled]>
+\`\`\`
+Totals, Issues: ${currentData.totalIssueCount} ${formattedIssueDelta} Bugs: ${currentData.totalBugCount} ${formattedBugDelta}\n\n`;
+
+  const prevDate = prevData.collectedDate
+    ? new Date(prevData.collectedDate)
+    : undefined;
+  const closedSinceDate = getSinceDate(prevDate)
+    .toDateString()
+    .split(' ')
+    .slice(1)
+    .join(' ');
+
+  const bodyLines: string[] = [
+    ...(prevData.collectedDate
+      ? [`Previous Report: ${prevData.collectedDate}`]
+      : []),
+    `Untriaged: ${currentData.untriagedIssueCount} ${formatDelta(
+      trendData.untriagedIssueCount
+    )}`,
+    `Closed since ${closedSinceDate}: ${currentData.totalClosed} ${formatDelta(
+      trendData.totalClosed
+    )}`,
+  ];
+
+  const sorted = Object.entries(currentData.scopes)
+    .sort(([, a], [, b]) => b.count - a.count)
+    .map(([scope, x]) => ({
+      ...x,
+      scope,
+    }));
+
+  bodyLines.push(
+    table(sorted, [
+      {
+        field: 'scope',
+        label: 'Scope',
+      },
+      {
+        label: 'Issues',
+        mapFn: (el) =>
+          `${el.count} ${formatDelta(trendData.scopes[el.scope].count)}`,
+      },
+      {
+        label: 'Bugs',
+        mapFn: (el) =>
+          `${el.bugCount} ${formatDelta(trendData.scopes[el.scope].bugCount)}`,
+      },
+      {
+        label: 'Closed',
+        mapFn: (el) =>
+          `${el.closed} ${formatDelta(trendData.scopes[el.scope].closed)}`,
+      },
+    ])
+  );
+
+  const footer = '```';
+  return header + bodyLines.join('\n') + footer;
+}
+
+function formatDelta(delta: number | null): string {
+  if (delta === null || delta === 0) {
+    return '';
+  }
+  return delta < 0 ? `(${delta})` : `(+${delta})`;
+}

scripts/issues-scraper/index.ts

@@ -0,0 +1,83 @@
+import { setOutput } from '@actions/core';
+import { ensureDirSync, readJsonSync, writeJsonSync } from 'fs-extra';
+import isCI from 'is-ci';
+import { dirname, join } from 'path';
+import { formatGhReport, getSlackMessageJson } from './format-slack-message';
+import { ReportData, ScopeData, TrendData } from './model';
+import { getScopeLabels, scrapeIssues } from './scrape-issues';
+
+const CACHE_FILE = join(__dirname, 'cached', 'data.json');
+
+async function main() {
+  const oldData = getOldData();
+  const currentData = await scrapeIssues(
+    oldData.collectedDate ? new Date(oldData.collectedDate) : undefined
+  );
+  const trendData = getTrendData(currentData, oldData);
+  const formatted = formatGhReport(
+    currentData,
+    trendData,
+    oldData,
+    getUnlabeledIssuesUrl(await getScopeLabels())
+  );
+  if (process.env.GITHUB_ACTIONS) {
+    setOutput('SLACK_MESSAGE', getSlackMessageJson(formatted));
+  }
+  console.log(formatted.replace(/\<(.*)\|(.*)\>/g, '[$2]($1)'));
+  saveCacheData(currentData);
+}
+
+if (require.main === module) {
+  main().catch((e) => {
+    console.error(e);
+    process.exit(1);
+  });
+}
+
+function getUnlabeledIssuesUrl(scopeLabels: string[]) {
+  const labelFilters = scopeLabels.map((s) => `-label:"${s}"`);
+  return `https://github.com/nrwl/nx/issues/?q=is%3Aopen+is%3Aissue+sort%3Aupdated-desc+${encodeURIComponent(
+    labelFilters.join(' ')
+  )}`;
+}
+
+function getTrendData(newData: ReportData, oldData: ReportData): TrendData {
+  const scopeTrends: Record<string, Partial<ScopeData>> = {};
+  for (const [scope, data] of Object.entries(newData.scopes)) {
+    scopeTrends[scope] ??= {};
+    scopeTrends[scope].count = data.count - (oldData.scopes[scope]?.count ?? 0);
+    scopeTrends[scope].bugCount =
+      data.bugCount - (oldData.scopes[scope]?.bugCount ?? 0);
+    scopeTrends[scope].closed =
+      data.closed - (oldData.scopes[scope]?.closed ?? 0);
+  }
+  return {
+    scopes: scopeTrends as Record<string, ScopeData>,
+    totalBugCount: newData.totalBugCount - oldData.totalBugCount,
+    totalIssueCount: newData.totalIssueCount - oldData.totalIssueCount,
+    totalClosed: newData.totalClosed - oldData.totalClosed,
+    untriagedIssueCount:
+      newData.untriagedIssueCount - oldData.untriagedIssueCount,
+  };
+}
+
+function saveCacheData(report: ReportData) {
+  if (isCI) {
+    ensureDirSync(dirname(CACHE_FILE));
+    writeJsonSync(CACHE_FILE, report);
+  }
+}
+
+function getOldData(): ReportData {
+  try {
+    return readJsonSync(CACHE_FILE);
+  } catch (e) {
+    return {
+      scopes: {},
+      totalBugCount: 0,
+      totalIssueCount: 0,
+      untriagedIssueCount: 0,
+      totalClosed: 0,
+    };
+  }
+}

scripts/issues-scraper/model.ts

@@ -0,0 +1,16 @@
+export interface ScopeData {
+  bugCount: number;
+  count: number;
+  closed: number;
+}
+
+export interface ReportData {
+  scopes: Record<string, ScopeData>;
+  totalBugCount: number;
+  totalIssueCount: number;
+  totalClosed: number;
+  untriagedIssueCount: number;
+  collectedDate?: string;
+}
+
+export type TrendData = Omit<ReportData, 'collectedDate'>;

scripts/issues-scraper/scrape-issues.ts

@@ -0,0 +1,129 @@
+import { Octokit } from 'octokit';
+import { ReportData, ScopeData } from './model';
+
+const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN });
+const now = new Date();
+
+export async function scrapeIssues(prevDate?: Date): Promise<ReportData> {
+  let total = 0;
+  let totalBugs = 0;
+  let untriagedIssueCount = 0;
+  let totalClosed = 0;
+  const scopeLabels = await getScopeLabels();
+  const scopes: Record<string, ScopeData> = {};
+
+  for await (const { data: slice } of getOpenIssueIterator()) {
+    for (const issue of slice.filter(isNotPullRequest)) {
+      const bug = hasLabel(issue, 'type: bug');
+
+      if (bug) {
+        totalBugs += 1;
+      }
+      total += 1;
+
+      let triaged = false;
+      for (const scope of scopeLabels) {
+        if (hasLabel(issue, scope)) {
+          scopes[scope] ??= { bugCount: 0, count: 0, closed: 0 };
+          if (bug) {
+            scopes[scope].bugCount += 1;
+          }
+          scopes[scope].count += 1;
+          triaged = true;
+        }
+      }
+      if (!triaged) {
+        untriagedIssueCount += 1;
+      }
+    }
+  }
+
+  const sinceDate = getSinceDate(prevDate);
+  for await (const { data: slice } of getClosedIssueIterator(sinceDate)) {
+    for (const issue of slice.filter(isNotPullRequest)) {
+      totalClosed += 1;
+
+      for (const scope of scopeLabels) {
+        if (hasLabel(issue, scope)) {
+          scopes[scope] ??= { bugCount: 0, count: 0, closed: 0 };
+          scopes[scope].closed += 1;
+        }
+      }
+    }
+  }
+
+  return {
+    scopes: scopes,
+    totalBugCount: totalBugs,
+    totalIssueCount: total,
+    totalClosed,
+    untriagedIssueCount,
+    // Format is like: Mar 03 2023
+    collectedDate: new Date().toDateString().split(' ').slice(1).join(' '),
+  };
+}
+
+export function getSinceDate(prevDate?: Date, referenceDate = now): Date {
+  const firstOfPrevMonth = new Date(
+    referenceDate.getFullYear(),
+    referenceDate.getMonth() - 1,
+    1
+  );
+  if (prevDate && prevDate > firstOfPrevMonth) {
+    return prevDate;
+  }
+  return firstOfPrevMonth;
+}
+
+const getOpenIssueIterator = () =>
+  octokit.paginate.iterator('GET /repos/{owner}/{repo}/issues', {
+    owner: 'nrwl',
+    repo: 'nx',
+    per_page: 100,
+    state: 'open',
+  });
+
+const getClosedIssueIterator = (since: Date) =>
+  octokit.paginate.iterator('GET /repos/{owner}/{repo}/issues', {
+    owner: 'nrwl',
+    repo: 'nx',
+    per_page: 100,
+    state: 'closed',
+    sort: 'updated',
+    direction: 'desc',
+    since: since.toISOString(),
+  });
+
+let labelCache: string[];
+export async function getScopeLabels(): Promise<string[]> {
+  labelCache ??= await getAllLabels().then((labels) =>
+    labels.filter((l) => l.startsWith('scope:'))
+  );
+  return labelCache;
+}
+
+async function getAllLabels(): Promise<string[]> {
+  const labels: string[] = [];
+
+  for await (const { data: slice } of octokit.paginate.iterator(
+    'GET /repos/{owner}/{repo}/labels',
+    { owner: 'nrwl', repo: 'nx' }
+  )) {
+    labels.push(...slice.map((l) => l.name));
+  }
+  return labels;
+}
+
+type IssueItem = Awaited<
+  ReturnType<typeof octokit.rest.issues.listForRepo>
+>['data'][number];
+
+function isNotPullRequest(issue: IssueItem): boolean {
+  return !('pull_request' in issue) || issue.pull_request == null;
+}
+
+function hasLabel(issue: IssueItem, labelName: string): boolean {
+  return issue.labels.some(
+    (l) => (typeof l === 'string' ? l : l.name) === labelName
+  );
+}