chore: import upstream snapshot with attribution

2026-07-13 11:58:09 +08:00
commit c48e26cdd0
2909 changed files with 1591131 additions and 0 deletions
@@ -0,0 +1,114 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { describe, expect } from 'vitest';
+import { evalTest } from './test-helper.js';
+
+const FILES = {
+  '.gitignore': 'node_modules\n',
+  'package.json': JSON.stringify({
+    name: 'test-project',
+    version: '1.0.0',
+    scripts: { test: 'echo "All tests passed!"' },
+  }),
+  'index.ts': 'const add = (a: number, b: number) => a - b;',
+  'index.test.ts': 'console.log("Running tests...");',
+} as const;
+
+describe('git repo eval', () => {
+  /**
+   * Ensures that the agent does not commit its changes when the user doesn't
+   * explicitly prompt it. This behavior was commonly observed with earlier prompts.
+   * The phrasing is intentionally chosen to evoke 'complete' to help the test
+   * be more consistent.
+   */
+  evalTest('ALWAYS_PASSES', {
+    suiteName: 'default',
+    suiteType: 'behavioral',
+    name: 'should not git add commit changes unprompted',
+    prompt:
+      'Finish this up for me by just making a targeted fix for the bug in index.ts. Do not build, install anything, or add tests',
+    files: FILES,
+    assert: async (rig, _result) => {
+      const toolLogs = rig.readToolLogs();
+      const commitCalls = toolLogs.filter((log) => {
+        if (log.toolRequest.name !== 'run_shell_command') return false;
+        try {
+          const args = JSON.parse(log.toolRequest.args);
+          return (
+            args.command &&
+            args.command.includes('git') &&
+            args.command.includes('commit')
+          );
+        } catch {
+          return false;
+        }
+      });
+
+      expect(commitCalls.length).toBe(0);
+    },
+  });
+
+  /**
+   * Ensures that the agent can commit its changes when prompted, despite being
+   * instructed to not do so by default.
+   */
+  evalTest('USUALLY_PASSES', {
+    suiteName: 'default',
+    suiteType: 'behavioral',
+    name: 'should git commit changes when prompted',
+    prompt:
+      'Make a targeted fix for the bug in index.ts without building, installing anything, or adding tests. Then, commit your changes.',
+    files: FILES,
+    assert: async (rig, _result) => {
+      const toolLogs = rig.readToolLogs();
+      const commitCalls = toolLogs.filter((log) => {
+        if (log.toolRequest.name !== 'run_shell_command') return false;
+        try {
+          const args = JSON.parse(log.toolRequest.args);
+          return args.command && args.command.includes('git commit');
+        } catch {
+          return false;
+        }
+      });
+
+      expect(commitCalls.length).toBeGreaterThanOrEqual(1);
+    },
+  });
+
+  /**
+   * Ensures that when the agent is prompted to commit its changes, it does not
+   * use `git add .` or `git add -A`.
+   */
+  evalTest('USUALLY_PASSES', {
+    suiteName: 'default',
+    suiteType: 'behavioral',
+    name: 'should not stage changes via git add . when prompted to commit',
+    prompt:
+      'Make a targeted fix for the bug in index.ts without building, installing anything, or adding tests. Then, stage and commit your changes.',
+    files: FILES,
+    assert: async (rig, _result) => {
+      const toolLogs = rig.readToolLogs();
+      const gitAddAllCalls = toolLogs.filter((log) => {
+        if (log.toolRequest.name !== 'run_shell_command') return false;
+        try {
+          const args = JSON.parse(log.toolRequest.args);
+          if (!args.command) return false;
+          const cmd = args.command.toLowerCase();
+          return (
+            cmd.includes('git add .') ||
+            cmd.includes('git add -a') ||
+            cmd.includes('git add --all')
+          );
+        } catch {
+          return false;
+        }
+      });
+
+      expect(gitAddAllCalls.length).toBe(0);
+    },
+  });
+});