wehub-resource-sync
db620d33df
dotnet-build-and-test / dotnet-test-functions (push) Has been cancelled
dotnet-build-and-test / paths-filter (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Debug, windows-latest, net9.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net8.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-foundry-hosted-it (push) Has been cancelled
dotnet-build-and-test / dotnet-build-and-test-check (push) Has been cancelled
dotnet-build-and-test / Integration Test Report (push) Has been cancelled
CodeQL / Analyze (csharp) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
440 lines
17 KiB
C#
440 lines
17 KiB
C#
// Copyright (c) Microsoft. All rights reserved.
|
|
|
|
using System.ComponentModel;
|
|
using System.Net;
|
|
using System.Net.Sockets;
|
|
using System.Text.Json;
|
|
using System.Text.RegularExpressions;
|
|
using Microsoft.Extensions.AI;
|
|
|
|
namespace SampleApp;
|
|
|
|
/// <summary>
|
|
/// An AI function that downloads HTML pages and converts them to markdown.
|
|
/// Access is controlled by <see cref="WebBrowsingToolOptions"/> — by default, no hosts are accessible.
|
|
/// </summary>
|
|
internal sealed partial class WebBrowsingTool : AIFunction
|
|
{
|
|
private static readonly HttpClient s_httpClient = new();
|
|
private readonly AIFunction _inner;
|
|
private readonly WebBrowsingToolOptions _options;
|
|
|
|
/// <summary>
|
|
/// Initializes a new instance of the <see cref="WebBrowsingTool"/> class.
|
|
/// </summary>
|
|
/// <param name="options">Options controlling which URLs are permitted. By default, no hosts are accessible.</param>
|
|
public WebBrowsingTool(WebBrowsingToolOptions options)
|
|
{
|
|
this._options = options ?? throw new ArgumentNullException(nameof(options));
|
|
this._inner = AIFunctionFactory.Create(this.DownloadUriAsync);
|
|
}
|
|
|
|
/// <inheritdoc/>
|
|
public override string Name => this._inner.Name;
|
|
|
|
/// <inheritdoc/>
|
|
public override string Description => this._inner.Description;
|
|
|
|
/// <inheritdoc/>
|
|
public override JsonElement JsonSchema => this._inner.JsonSchema;
|
|
|
|
/// <inheritdoc/>
|
|
protected override ValueTask<object?> InvokeCoreAsync(
|
|
AIFunctionArguments arguments,
|
|
CancellationToken cancellationToken) =>
|
|
this._inner.InvokeAsync(arguments, cancellationToken);
|
|
|
|
[Description("Fetch the html from the given url as markdown")]
|
|
private async Task<string> DownloadUriAsync(
|
|
[Description("The URL to download")] string uri,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
if (!Uri.TryCreate(uri, UriKind.Absolute, out Uri? parsedUri))
|
|
{
|
|
return $"Error: '{uri}' is not a valid URL.";
|
|
}
|
|
|
|
if (parsedUri.Scheme is not "http" and not "https")
|
|
{
|
|
return $"Error: Only HTTP and HTTPS URLs are supported. Got: '{parsedUri.Scheme}'.";
|
|
}
|
|
|
|
// Check access policy.
|
|
string? accessError = await this.CheckAccessAsync(parsedUri, cancellationToken);
|
|
if (accessError is not null)
|
|
{
|
|
return accessError;
|
|
}
|
|
|
|
try
|
|
{
|
|
string html = await s_httpClient.GetStringAsync(parsedUri, cancellationToken);
|
|
return HtmlToMarkdownConverter.Convert(html);
|
|
}
|
|
catch (HttpRequestException ex)
|
|
{
|
|
return $"Error downloading {uri}: {ex.Message}";
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Checks whether the given URI is permitted by the configured access policy.
|
|
/// Returns null if allowed, or an error message string if blocked.
|
|
/// </summary>
|
|
private async Task<string?> CheckAccessAsync(Uri uri, CancellationToken cancellationToken)
|
|
{
|
|
string host = uri.Host;
|
|
|
|
// 1. Check AllowedHosts.
|
|
if (this._options.AllowedHosts is { Count: > 0 } allowedHosts)
|
|
{
|
|
foreach (string pattern in allowedHosts)
|
|
{
|
|
if (HostMatchesPattern(host, pattern))
|
|
{
|
|
return null; // Allowed by explicit host list.
|
|
}
|
|
}
|
|
}
|
|
|
|
// 2. Short-circuit when the policy is guaranteed to block.
|
|
if (!this._options.AllowPublicNetworks &&
|
|
!this._options.AllowPrivateNetworks &&
|
|
!this._options.AllowAllHosts)
|
|
{
|
|
return $"Error: Access to '{host}' is blocked by the current access policy. Configure WebBrowsingToolOptions to allow access.";
|
|
}
|
|
|
|
// 3. Resolve DNS to determine if the host is public or private.
|
|
IPAddress[] addresses;
|
|
try
|
|
{
|
|
addresses = await Dns.GetHostAddressesAsync(host, cancellationToken);
|
|
}
|
|
catch (SocketException)
|
|
{
|
|
return $"Error: Could not resolve host '{host}'.";
|
|
}
|
|
|
|
if (addresses.Length == 0)
|
|
{
|
|
return $"Error: Could not resolve host '{host}'.";
|
|
}
|
|
|
|
bool isPrivate = Array.Exists(addresses, IsPrivateAddress);
|
|
|
|
// 4. If public and AllowPublicNetworks is true → allow.
|
|
if (!isPrivate && this._options.AllowPublicNetworks)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
// 5. If private and AllowPrivateNetworks is true → allow.
|
|
if (isPrivate && this._options.AllowPrivateNetworks)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
// 6. If AllowAllHosts is true → allow.
|
|
if (this._options.AllowAllHosts)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
// 7. Block.
|
|
string networkType = isPrivate ? "private/internal network" : "public network";
|
|
return $"Error: Access to '{host}' is blocked. The host resolves to a {networkType} address and the current access policy does not permit this. " +
|
|
"Configure WebBrowsingToolOptions to allow access.";
|
|
}
|
|
|
|
/// <summary>
|
|
/// Checks whether a host matches a pattern. Supports exact match and wildcard prefix (e.g., "*.example.com").
|
|
/// </summary>
|
|
private static bool HostMatchesPattern(string host, string pattern)
|
|
{
|
|
if (string.Equals(host, pattern, StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
return true;
|
|
}
|
|
|
|
// Wildcard prefix: "*.example.com" matches "sub.example.com" and "a.b.example.com".
|
|
if (pattern.StartsWith("*.", StringComparison.Ordinal))
|
|
{
|
|
string suffix = pattern[1..]; // ".example.com"
|
|
return host.EndsWith(suffix, StringComparison.OrdinalIgnoreCase);
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Determines whether an IP address is private, loopback, or link-local.
|
|
/// </summary>
|
|
private static bool IsPrivateAddress(IPAddress address)
|
|
{
|
|
if (address.IsIPv4MappedToIPv6)
|
|
{
|
|
address = address.MapToIPv4();
|
|
}
|
|
|
|
if (IPAddress.IsLoopback(address))
|
|
{
|
|
return true;
|
|
}
|
|
|
|
if (address.AddressFamily == AddressFamily.InterNetwork)
|
|
{
|
|
byte[] bytes = address.GetAddressBytes();
|
|
return bytes[0] switch
|
|
{
|
|
10 => true, // 10.0.0.0/8
|
|
172 => bytes[1] >= 16 && bytes[1] <= 31, // 172.16.0.0/12
|
|
192 => bytes[1] == 168, // 192.168.0.0/16
|
|
169 => bytes[1] == 254, // 169.254.0.0/16 (link-local + metadata)
|
|
_ => false
|
|
};
|
|
}
|
|
|
|
if (address.AddressFamily == AddressFamily.InterNetworkV6)
|
|
{
|
|
// fe80::/10 (link-local) or fc00::/7 (unique local).
|
|
byte[] bytes = address.GetAddressBytes();
|
|
if (bytes[0] == 0xfe && (bytes[1] & 0xc0) == 0x80)
|
|
{
|
|
return true; // Link-local
|
|
}
|
|
|
|
if ((bytes[0] & 0xfe) == 0xfc)
|
|
{
|
|
return true; // Unique local
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
/// <summary>
|
|
/// A simple HTML to Markdown converter using regex-based transformations.
|
|
/// Handles the most common HTML elements without requiring external dependencies.
|
|
/// </summary>
|
|
private static partial class HtmlToMarkdownConverter
|
|
{
|
|
public static string Convert(string html)
|
|
{
|
|
// Extract body content if present, otherwise use the full HTML.
|
|
var bodyMatch = BodyRegex().Match(html);
|
|
string content = bodyMatch.Success ? bodyMatch.Groups[1].Value : html;
|
|
|
|
// Remove script, style, and head blocks.
|
|
content = ScriptRegex().Replace(content, string.Empty);
|
|
content = StyleRegex().Replace(content, string.Empty);
|
|
content = HeadRegex().Replace(content, string.Empty);
|
|
content = CommentRegex().Replace(content, string.Empty);
|
|
|
|
// Convert block elements before inline elements.
|
|
content = ConvertHeadings(content);
|
|
content = ConvertCodeBlocks(content);
|
|
content = ConvertBlockquotes(content);
|
|
content = ConvertLists(content);
|
|
content = ConvertHorizontalRules(content);
|
|
|
|
// Convert inline elements.
|
|
content = ConvertLinks(content);
|
|
content = ConvertImages(content);
|
|
content = ConvertBold(content);
|
|
content = ConvertItalic(content);
|
|
content = ConvertInlineCode(content);
|
|
|
|
// Convert structural elements.
|
|
content = ConvertParagraphs(content);
|
|
content = ConvertLineBreaks(content);
|
|
|
|
// Strip remaining HTML tags.
|
|
content = StripTagsRegex().Replace(content, string.Empty);
|
|
|
|
// Decode HTML entities.
|
|
content = WebUtility.HtmlDecode(content);
|
|
|
|
// Clean up excessive whitespace.
|
|
content = ExcessiveNewlinesRegex().Replace(content, "\n\n");
|
|
|
|
return content.Trim();
|
|
}
|
|
|
|
private static string ConvertHeadings(string html)
|
|
{
|
|
html = H1Regex().Replace(html, m => $"\n# {StripInnerTags(m.Groups[1].Value).Trim()}\n");
|
|
html = H2Regex().Replace(html, m => $"\n## {StripInnerTags(m.Groups[1].Value).Trim()}\n");
|
|
html = H3Regex().Replace(html, m => $"\n### {StripInnerTags(m.Groups[1].Value).Trim()}\n");
|
|
html = H4Regex().Replace(html, m => $"\n#### {StripInnerTags(m.Groups[1].Value).Trim()}\n");
|
|
html = H5Regex().Replace(html, m => $"\n##### {StripInnerTags(m.Groups[1].Value).Trim()}\n");
|
|
html = H6Regex().Replace(html, m => $"\n###### {StripInnerTags(m.Groups[1].Value).Trim()}\n");
|
|
return html;
|
|
}
|
|
|
|
private static string ConvertLinks(string html) =>
|
|
LinkRegex().Replace(html, m =>
|
|
{
|
|
string href = m.Groups[1].Value;
|
|
string text = StripInnerTags(m.Groups[2].Value).Trim();
|
|
|
|
// Skip javascript and data links.
|
|
if (href.StartsWith("javascript:", StringComparison.OrdinalIgnoreCase) ||
|
|
href.StartsWith("data:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
return text;
|
|
}
|
|
|
|
return string.IsNullOrWhiteSpace(text) ? string.Empty : $"[{text}]({href})";
|
|
});
|
|
|
|
private static string ConvertImages(string html) =>
|
|
ImageRegex().Replace(html, m =>
|
|
{
|
|
string src = m.Groups[1].Value;
|
|
string alt = m.Groups[2].Value;
|
|
|
|
// Truncate data URIs.
|
|
if (src.StartsWith("data:", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
src = src.Split(',')[0] + "...";
|
|
}
|
|
|
|
return $"";
|
|
});
|
|
|
|
private static string ConvertBold(string html) =>
|
|
BoldRegex().Replace(html, m => $"**{m.Groups[2].Value}**");
|
|
|
|
private static string ConvertItalic(string html) =>
|
|
ItalicRegex().Replace(html, m => $"*{m.Groups[2].Value}*");
|
|
|
|
private static string ConvertInlineCode(string html) =>
|
|
InlineCodeRegex().Replace(html, m => $"`{m.Groups[1].Value}`");
|
|
|
|
private static string ConvertCodeBlocks(string html) =>
|
|
CodeBlockRegex().Replace(html, m => $"\n```\n{StripInnerTags(m.Groups[1].Value).Trim()}\n```\n");
|
|
|
|
private static string ConvertBlockquotes(string html) =>
|
|
BlockquoteRegex().Replace(html, m =>
|
|
{
|
|
string inner = StripInnerTags(m.Groups[1].Value).Trim();
|
|
// Prefix each line with "> ".
|
|
string quoted = string.Join("\n", inner.Split('\n').Select(line => $"> {line.Trim()}"));
|
|
return $"\n{quoted}\n";
|
|
});
|
|
|
|
private static string ConvertLists(string html)
|
|
{
|
|
// Unordered lists.
|
|
html = UlRegex().Replace(html, m =>
|
|
{
|
|
string items = LiRegex().Replace(m.Groups[1].Value, li => $"- {StripInnerTags(li.Groups[1].Value).Trim()}\n");
|
|
return $"\n{items}";
|
|
});
|
|
|
|
// Ordered lists.
|
|
html = OlRegex().Replace(html, m =>
|
|
{
|
|
int index = 1;
|
|
string items = LiRegex().Replace(m.Groups[1].Value, li => $"{index++}. {StripInnerTags(li.Groups[1].Value).Trim()}\n");
|
|
return $"\n{items}";
|
|
});
|
|
|
|
return html;
|
|
}
|
|
|
|
private static string ConvertHorizontalRules(string html) =>
|
|
HrRegex().Replace(html, "\n---\n");
|
|
|
|
private static string ConvertParagraphs(string html) =>
|
|
ParagraphRegex().Replace(html, m => $"\n\n{m.Groups[1].Value}\n\n");
|
|
|
|
private static string ConvertLineBreaks(string html) =>
|
|
BrRegex().Replace(html, "\n");
|
|
|
|
private static string StripInnerTags(string html) =>
|
|
StripTagsRegex().Replace(html, string.Empty);
|
|
|
|
// Source-generated regex patterns for performance and AOT compatibility.
|
|
|
|
[GeneratedRegex(@"<body[^>]*>(.*?)</body>", RegexOptions.Singleline | RegexOptions.IgnoreCase)]
|
|
private static partial Regex BodyRegex();
|
|
|
|
[GeneratedRegex(@"<script[^>]*>.*?</script>", RegexOptions.Singleline | RegexOptions.IgnoreCase)]
|
|
private static partial Regex ScriptRegex();
|
|
|
|
[GeneratedRegex(@"<style[^>]*>.*?</style>", RegexOptions.Singleline | RegexOptions.IgnoreCase)]
|
|
private static partial Regex StyleRegex();
|
|
|
|
[GeneratedRegex(@"<head[^>]*>.*?</head>", RegexOptions.Singleline | RegexOptions.IgnoreCase)]
|
|
private static partial Regex HeadRegex();
|
|
|
|
[GeneratedRegex(@"<!--.*?-->", RegexOptions.Singleline)]
|
|
private static partial Regex CommentRegex();
|
|
|
|
[GeneratedRegex(@"<h1[^>]*>(.*?)</h1>", RegexOptions.Singleline | RegexOptions.IgnoreCase)]
|
|
private static partial Regex H1Regex();
|
|
|
|
[GeneratedRegex(@"<h2[^>]*>(.*?)</h2>", RegexOptions.Singleline | RegexOptions.IgnoreCase)]
|
|
private static partial Regex H2Regex();
|
|
|
|
[GeneratedRegex(@"<h3[^>]*>(.*?)</h3>", RegexOptions.Singleline | RegexOptions.IgnoreCase)]
|
|
private static partial Regex H3Regex();
|
|
|
|
[GeneratedRegex(@"<h4[^>]*>(.*?)</h4>", RegexOptions.Singleline | RegexOptions.IgnoreCase)]
|
|
private static partial Regex H4Regex();
|
|
|
|
[GeneratedRegex(@"<h5[^>]*>(.*?)</h5>", RegexOptions.Singleline | RegexOptions.IgnoreCase)]
|
|
private static partial Regex H5Regex();
|
|
|
|
[GeneratedRegex(@"<h6[^>]*>(.*?)</h6>", RegexOptions.Singleline | RegexOptions.IgnoreCase)]
|
|
private static partial Regex H6Regex();
|
|
|
|
[GeneratedRegex(@"<a\s[^>]*href=[""']([^""']*)[""'][^>]*>(.*?)</a>", RegexOptions.Singleline | RegexOptions.IgnoreCase)]
|
|
private static partial Regex LinkRegex();
|
|
|
|
[GeneratedRegex(@"<img\s[^>]*src=[""']([^""']*)[""'][^>]*?(?:alt=[""']([^""']*)[""'])?[^>]*/?>", RegexOptions.Singleline | RegexOptions.IgnoreCase)]
|
|
private static partial Regex ImageRegex();
|
|
|
|
[GeneratedRegex(@"<(strong|b)\b[^>]*>(.*?)</\1>", RegexOptions.Singleline | RegexOptions.IgnoreCase)]
|
|
private static partial Regex BoldRegex();
|
|
|
|
[GeneratedRegex(@"<(em|i)\b[^>]*>(.*?)</\1>", RegexOptions.Singleline | RegexOptions.IgnoreCase)]
|
|
private static partial Regex ItalicRegex();
|
|
|
|
[GeneratedRegex(@"<code[^>]*>(.*?)</code>", RegexOptions.Singleline | RegexOptions.IgnoreCase)]
|
|
private static partial Regex InlineCodeRegex();
|
|
|
|
[GeneratedRegex(@"<pre[^>]*>(.*?)</pre>", RegexOptions.Singleline | RegexOptions.IgnoreCase)]
|
|
private static partial Regex CodeBlockRegex();
|
|
|
|
[GeneratedRegex(@"<blockquote[^>]*>(.*?)</blockquote>", RegexOptions.Singleline | RegexOptions.IgnoreCase)]
|
|
private static partial Regex BlockquoteRegex();
|
|
|
|
[GeneratedRegex(@"<ul[^>]*>(.*?)</ul>", RegexOptions.Singleline | RegexOptions.IgnoreCase)]
|
|
private static partial Regex UlRegex();
|
|
|
|
[GeneratedRegex(@"<ol[^>]*>(.*?)</ol>", RegexOptions.Singleline | RegexOptions.IgnoreCase)]
|
|
private static partial Regex OlRegex();
|
|
|
|
[GeneratedRegex(@"<li[^>]*>(.*?)</li>", RegexOptions.Singleline | RegexOptions.IgnoreCase)]
|
|
private static partial Regex LiRegex();
|
|
|
|
[GeneratedRegex(@"<hr\s*/?>", RegexOptions.IgnoreCase)]
|
|
private static partial Regex HrRegex();
|
|
|
|
[GeneratedRegex(@"<p[^>]*>(.*?)</p>", RegexOptions.Singleline | RegexOptions.IgnoreCase)]
|
|
private static partial Regex ParagraphRegex();
|
|
|
|
[GeneratedRegex(@"<br\s*/?>", RegexOptions.IgnoreCase)]
|
|
private static partial Regex BrRegex();
|
|
|
|
[GeneratedRegex(@"<[^>]+>")]
|
|
private static partial Regex StripTagsRegex();
|
|
|
|
[GeneratedRegex(@"\n{3,}")]
|
|
private static partial Regex ExcessiveNewlinesRegex();
|
|
}
|
|
}
|