// Copyright (c) Microsoft. All rights reserved. using System.ComponentModel; using System.Net; using System.Net.Sockets; using System.Text.Json; using System.Text.RegularExpressions; using Microsoft.Extensions.AI; namespace SampleApp; /// /// An AI function that downloads HTML pages and converts them to markdown. /// Access is controlled by — by default, no hosts are accessible. /// internal sealed partial class WebBrowsingTool : AIFunction { private static readonly HttpClient s_httpClient = new(); private readonly AIFunction _inner; private readonly WebBrowsingToolOptions _options; /// /// Initializes a new instance of the class. /// /// Options controlling which URLs are permitted. By default, no hosts are accessible. public WebBrowsingTool(WebBrowsingToolOptions options) { this._options = options ?? throw new ArgumentNullException(nameof(options)); this._inner = AIFunctionFactory.Create(this.DownloadUriAsync); } /// public override string Name => this._inner.Name; /// public override string Description => this._inner.Description; /// public override JsonElement JsonSchema => this._inner.JsonSchema; /// protected override ValueTask InvokeCoreAsync( AIFunctionArguments arguments, CancellationToken cancellationToken) => this._inner.InvokeAsync(arguments, cancellationToken); [Description("Fetch the html from the given url as markdown")] private async Task DownloadUriAsync( [Description("The URL to download")] string uri, CancellationToken cancellationToken = default) { if (!Uri.TryCreate(uri, UriKind.Absolute, out Uri? parsedUri)) { return $"Error: '{uri}' is not a valid URL."; } if (parsedUri.Scheme is not "http" and not "https") { return $"Error: Only HTTP and HTTPS URLs are supported. Got: '{parsedUri.Scheme}'."; } // Check access policy. string? accessError = await this.CheckAccessAsync(parsedUri, cancellationToken); if (accessError is not null) { return accessError; } try { string html = await s_httpClient.GetStringAsync(parsedUri, cancellationToken); return HtmlToMarkdownConverter.Convert(html); } catch (HttpRequestException ex) { return $"Error downloading {uri}: {ex.Message}"; } } /// /// Checks whether the given URI is permitted by the configured access policy. /// Returns null if allowed, or an error message string if blocked. /// private async Task CheckAccessAsync(Uri uri, CancellationToken cancellationToken) { string host = uri.Host; // 1. Check AllowedHosts. if (this._options.AllowedHosts is { Count: > 0 } allowedHosts) { foreach (string pattern in allowedHosts) { if (HostMatchesPattern(host, pattern)) { return null; // Allowed by explicit host list. } } } // 2. Short-circuit when the policy is guaranteed to block. if (!this._options.AllowPublicNetworks && !this._options.AllowPrivateNetworks && !this._options.AllowAllHosts) { return $"Error: Access to '{host}' is blocked by the current access policy. Configure WebBrowsingToolOptions to allow access."; } // 3. Resolve DNS to determine if the host is public or private. IPAddress[] addresses; try { addresses = await Dns.GetHostAddressesAsync(host, cancellationToken); } catch (SocketException) { return $"Error: Could not resolve host '{host}'."; } if (addresses.Length == 0) { return $"Error: Could not resolve host '{host}'."; } bool isPrivate = Array.Exists(addresses, IsPrivateAddress); // 4. If public and AllowPublicNetworks is true → allow. if (!isPrivate && this._options.AllowPublicNetworks) { return null; } // 5. If private and AllowPrivateNetworks is true → allow. if (isPrivate && this._options.AllowPrivateNetworks) { return null; } // 6. If AllowAllHosts is true → allow. if (this._options.AllowAllHosts) { return null; } // 7. Block. string networkType = isPrivate ? "private/internal network" : "public network"; return $"Error: Access to '{host}' is blocked. The host resolves to a {networkType} address and the current access policy does not permit this. " + "Configure WebBrowsingToolOptions to allow access."; } /// /// Checks whether a host matches a pattern. Supports exact match and wildcard prefix (e.g., "*.example.com"). /// private static bool HostMatchesPattern(string host, string pattern) { if (string.Equals(host, pattern, StringComparison.OrdinalIgnoreCase)) { return true; } // Wildcard prefix: "*.example.com" matches "sub.example.com" and "a.b.example.com". if (pattern.StartsWith("*.", StringComparison.Ordinal)) { string suffix = pattern[1..]; // ".example.com" return host.EndsWith(suffix, StringComparison.OrdinalIgnoreCase); } return false; } /// /// Determines whether an IP address is private, loopback, or link-local. /// private static bool IsPrivateAddress(IPAddress address) { if (address.IsIPv4MappedToIPv6) { address = address.MapToIPv4(); } if (IPAddress.IsLoopback(address)) { return true; } if (address.AddressFamily == AddressFamily.InterNetwork) { byte[] bytes = address.GetAddressBytes(); return bytes[0] switch { 10 => true, // 10.0.0.0/8 172 => bytes[1] >= 16 && bytes[1] <= 31, // 172.16.0.0/12 192 => bytes[1] == 168, // 192.168.0.0/16 169 => bytes[1] == 254, // 169.254.0.0/16 (link-local + metadata) _ => false }; } if (address.AddressFamily == AddressFamily.InterNetworkV6) { // fe80::/10 (link-local) or fc00::/7 (unique local). byte[] bytes = address.GetAddressBytes(); if (bytes[0] == 0xfe && (bytes[1] & 0xc0) == 0x80) { return true; // Link-local } if ((bytes[0] & 0xfe) == 0xfc) { return true; // Unique local } } return false; } /// /// A simple HTML to Markdown converter using regex-based transformations. /// Handles the most common HTML elements without requiring external dependencies. /// private static partial class HtmlToMarkdownConverter { public static string Convert(string html) { // Extract body content if present, otherwise use the full HTML. var bodyMatch = BodyRegex().Match(html); string content = bodyMatch.Success ? bodyMatch.Groups[1].Value : html; // Remove script, style, and head blocks. content = ScriptRegex().Replace(content, string.Empty); content = StyleRegex().Replace(content, string.Empty); content = HeadRegex().Replace(content, string.Empty); content = CommentRegex().Replace(content, string.Empty); // Convert block elements before inline elements. content = ConvertHeadings(content); content = ConvertCodeBlocks(content); content = ConvertBlockquotes(content); content = ConvertLists(content); content = ConvertHorizontalRules(content); // Convert inline elements. content = ConvertLinks(content); content = ConvertImages(content); content = ConvertBold(content); content = ConvertItalic(content); content = ConvertInlineCode(content); // Convert structural elements. content = ConvertParagraphs(content); content = ConvertLineBreaks(content); // Strip remaining HTML tags. content = StripTagsRegex().Replace(content, string.Empty); // Decode HTML entities. content = WebUtility.HtmlDecode(content); // Clean up excessive whitespace. content = ExcessiveNewlinesRegex().Replace(content, "\n\n"); return content.Trim(); } private static string ConvertHeadings(string html) { html = H1Regex().Replace(html, m => $"\n# {StripInnerTags(m.Groups[1].Value).Trim()}\n"); html = H2Regex().Replace(html, m => $"\n## {StripInnerTags(m.Groups[1].Value).Trim()}\n"); html = H3Regex().Replace(html, m => $"\n### {StripInnerTags(m.Groups[1].Value).Trim()}\n"); html = H4Regex().Replace(html, m => $"\n#### {StripInnerTags(m.Groups[1].Value).Trim()}\n"); html = H5Regex().Replace(html, m => $"\n##### {StripInnerTags(m.Groups[1].Value).Trim()}\n"); html = H6Regex().Replace(html, m => $"\n###### {StripInnerTags(m.Groups[1].Value).Trim()}\n"); return html; } private static string ConvertLinks(string html) => LinkRegex().Replace(html, m => { string href = m.Groups[1].Value; string text = StripInnerTags(m.Groups[2].Value).Trim(); // Skip javascript and data links. if (href.StartsWith("javascript:", StringComparison.OrdinalIgnoreCase) || href.StartsWith("data:", StringComparison.OrdinalIgnoreCase)) { return text; } return string.IsNullOrWhiteSpace(text) ? string.Empty : $"[{text}]({href})"; }); private static string ConvertImages(string html) => ImageRegex().Replace(html, m => { string src = m.Groups[1].Value; string alt = m.Groups[2].Value; // Truncate data URIs. if (src.StartsWith("data:", StringComparison.OrdinalIgnoreCase)) { src = src.Split(',')[0] + "..."; } return $"![{alt}]({src})"; }); private static string ConvertBold(string html) => BoldRegex().Replace(html, m => $"**{m.Groups[2].Value}**"); private static string ConvertItalic(string html) => ItalicRegex().Replace(html, m => $"*{m.Groups[2].Value}*"); private static string ConvertInlineCode(string html) => InlineCodeRegex().Replace(html, m => $"`{m.Groups[1].Value}`"); private static string ConvertCodeBlocks(string html) => CodeBlockRegex().Replace(html, m => $"\n```\n{StripInnerTags(m.Groups[1].Value).Trim()}\n```\n"); private static string ConvertBlockquotes(string html) => BlockquoteRegex().Replace(html, m => { string inner = StripInnerTags(m.Groups[1].Value).Trim(); // Prefix each line with "> ". string quoted = string.Join("\n", inner.Split('\n').Select(line => $"> {line.Trim()}")); return $"\n{quoted}\n"; }); private static string ConvertLists(string html) { // Unordered lists. html = UlRegex().Replace(html, m => { string items = LiRegex().Replace(m.Groups[1].Value, li => $"- {StripInnerTags(li.Groups[1].Value).Trim()}\n"); return $"\n{items}"; }); // Ordered lists. html = OlRegex().Replace(html, m => { int index = 1; string items = LiRegex().Replace(m.Groups[1].Value, li => $"{index++}. {StripInnerTags(li.Groups[1].Value).Trim()}\n"); return $"\n{items}"; }); return html; } private static string ConvertHorizontalRules(string html) => HrRegex().Replace(html, "\n---\n"); private static string ConvertParagraphs(string html) => ParagraphRegex().Replace(html, m => $"\n\n{m.Groups[1].Value}\n\n"); private static string ConvertLineBreaks(string html) => BrRegex().Replace(html, "\n"); private static string StripInnerTags(string html) => StripTagsRegex().Replace(html, string.Empty); // Source-generated regex patterns for performance and AOT compatibility. [GeneratedRegex(@"]*>(.*?)", RegexOptions.Singleline | RegexOptions.IgnoreCase)] private static partial Regex BodyRegex(); [GeneratedRegex(@"]*>.*?", RegexOptions.Singleline | RegexOptions.IgnoreCase)] private static partial Regex ScriptRegex(); [GeneratedRegex(@"]*>.*?", RegexOptions.Singleline | RegexOptions.IgnoreCase)] private static partial Regex StyleRegex(); [GeneratedRegex(@"]*>.*?", RegexOptions.Singleline | RegexOptions.IgnoreCase)] private static partial Regex HeadRegex(); [GeneratedRegex(@"", RegexOptions.Singleline)] private static partial Regex CommentRegex(); [GeneratedRegex(@"]*>(.*?)", RegexOptions.Singleline | RegexOptions.IgnoreCase)] private static partial Regex H1Regex(); [GeneratedRegex(@"]*>(.*?)", RegexOptions.Singleline | RegexOptions.IgnoreCase)] private static partial Regex H2Regex(); [GeneratedRegex(@"]*>(.*?)", RegexOptions.Singleline | RegexOptions.IgnoreCase)] private static partial Regex H3Regex(); [GeneratedRegex(@"]*>(.*?)", RegexOptions.Singleline | RegexOptions.IgnoreCase)] private static partial Regex H4Regex(); [GeneratedRegex(@"]*>(.*?)", RegexOptions.Singleline | RegexOptions.IgnoreCase)] private static partial Regex H5Regex(); [GeneratedRegex(@"]*>(.*?)", RegexOptions.Singleline | RegexOptions.IgnoreCase)] private static partial Regex H6Regex(); [GeneratedRegex(@"]*href=[""']([^""']*)[""'][^>]*>(.*?)", RegexOptions.Singleline | RegexOptions.IgnoreCase)] private static partial Regex LinkRegex(); [GeneratedRegex(@"]*src=[""']([^""']*)[""'][^>]*?(?:alt=[""']([^""']*)[""'])?[^>]*/?>", RegexOptions.Singleline | RegexOptions.IgnoreCase)] private static partial Regex ImageRegex(); [GeneratedRegex(@"<(strong|b)\b[^>]*>(.*?)", RegexOptions.Singleline | RegexOptions.IgnoreCase)] private static partial Regex BoldRegex(); [GeneratedRegex(@"<(em|i)\b[^>]*>(.*?)", RegexOptions.Singleline | RegexOptions.IgnoreCase)] private static partial Regex ItalicRegex(); [GeneratedRegex(@"]*>(.*?)", RegexOptions.Singleline | RegexOptions.IgnoreCase)] private static partial Regex InlineCodeRegex(); [GeneratedRegex(@"]*>(.*?)", RegexOptions.Singleline | RegexOptions.IgnoreCase)] private static partial Regex CodeBlockRegex(); [GeneratedRegex(@"]*>(.*?)", RegexOptions.Singleline | RegexOptions.IgnoreCase)] private static partial Regex BlockquoteRegex(); [GeneratedRegex(@"]*>(.*?)", RegexOptions.Singleline | RegexOptions.IgnoreCase)] private static partial Regex UlRegex(); [GeneratedRegex(@"]*>(.*?)", RegexOptions.Singleline | RegexOptions.IgnoreCase)] private static partial Regex OlRegex(); [GeneratedRegex(@"]*>(.*?)", RegexOptions.Singleline | RegexOptions.IgnoreCase)] private static partial Regex LiRegex(); [GeneratedRegex(@"", RegexOptions.IgnoreCase)] private static partial Regex HrRegex(); [GeneratedRegex(@"]*>(.*?)

", RegexOptions.Singleline | RegexOptions.IgnoreCase)] private static partial Regex ParagraphRegex(); [GeneratedRegex(@"", RegexOptions.IgnoreCase)] private static partial Regex BrRegex(); [GeneratedRegex(@"<[^>]+>")] private static partial Regex StripTagsRegex(); [GeneratedRegex(@"\n{3,}")] private static partial Regex ExcessiveNewlinesRegex(); } }