chore: import upstream snapshot with attribution

2026-07-13 12:32:25 +08:00
commit e014feafe1
2285 changed files with 1131979 additions and 0 deletions
@@ -0,0 +1,107 @@
+// Copyright 2023 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/dolthub/doltgresql/testing/generation/utils"
+
+	"github.com/PuerkitoBio/goquery"
+	"golang.org/x/net/html"
+)
+
+const (
+	postgresBaseLink = `https://www.postgresql.org/docs/15/`
+	commandPageLink  = `sql-commands.html`
+)
+
+type Link struct {
+	CommandName string
+	Link        string
+}
+
+// DownloadAllSynopses downloads and writes all synopses from the internet. It is assumed that this will be run from
+// within an IDE, and will download to the `commands_docs/synopses` folder. Uses `postgresBaseLink` to determine which
+// version of the command page will be downloaded, and `commandPageLink` specifies the exact page that the commands are
+// contained on. This function is only necessary if we're upgrading the Postgres version.
+func DownloadAllSynopses() {
+	commandDocument, err := FetchDocument(postgresBaseLink + commandPageLink)
+	if err != nil {
+		panic(err)
+	}
+	var allLinks []Link
+	_ = commandDocument.Find(".toc a").Each(func(i int, e *goquery.Selection) {
+		href, exists := e.Attr("href")
+		if exists {
+			allLinks = append(allLinks, Link{
+				CommandName: e.Text(),
+				Link:        postgresBaseLink + strings.Trim(href, `\/`),
+			})
+		}
+	})
+	for _, link := range allLinks {
+		linkDocument, err := FetchDocument(link.Link)
+		if err != nil {
+			panic(err)
+		}
+		synopsis := linkDocument.Find(".synopsis").First()
+		if strings.Contains(synopsis.Text(), "$") {
+			panic(fmt.Errorf("Synopsis has a $, which is unexpected:\n\n%s\n\n%s", link, synopsis.Text()))
+		}
+		synopsis.Find(".replaceable").Each(func(i int, e *goquery.Selection) {
+			e.SetText(fmt.Sprintf("$%s$", e.Text()))
+		})
+		func() {
+			fileLocation, err := GetCommandDocsFolder()
+			if err != nil {
+				fmt.Println(err.Error())
+				return
+			}
+			fileName := strings.ToLower(strings.ReplaceAll(link.CommandName, " ", "_"))
+			data := []byte(synopsis.Text())
+			if err = fileLocation.WriteFileToDirectory("synopses/%s.txt", fileName+".txt", data, 0644); err != nil {
+				fmt.Println(err.Error())
+			}
+			fmt.Printf("Downloaded: %s\n", link.Link)
+		}()
+	}
+}
+
+// FetchDocument fetches the document from the given link. The link should be a full HTTP/HTTPS URL.
+func FetchDocument(link string) (*goquery.Document, error) {
+	res, err := http.Get(link)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+	node, err := html.Parse(res.Body)
+	if err != nil {
+		return nil, err
+	}
+	return goquery.NewDocumentFromNode(node), nil
+}
+
+// GetCommandDocsFolder returns the directory of this particular Go file: download.go. This is useful to locate relative
+// directories, such as the synopses folder. It is assumed that this will always be called from within an IDE.
+func GetCommandDocsFolder() (utils.RootFolderLocation, error) {
+	root, err := utils.GetRootFolder()
+	if err != nil {
+		return utils.RootFolderLocation{}, err
+	}
+	return root.MoveRoot("testing/generation/command_docs"), nil
+}