Add sys.http.html2text

Author: ibuildthecloud

examples/search.gpt

@@ -0,0 +1,16 @@
+tools: search
+
+Search the web for vacation spots in Florida, for each
+vacation spot give the name and description.
+
+---
+name: search
+description: Searches the internet for content
+args: query: The query to search for
+tools: sys.http.text?
+
+First download the content of "https://html.duckduckgo.com/html/?q=${query}".
+Look for the first 5 search results. Download each search result and look for content
+that would best answer the query ${query}.
+
+With all that information try your best to provide an answer or useful context to ${query}.

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/adrg/xdg v0.4.0
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/hexops/autogold/v2 v2.1.0
+	github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056
 	github.com/sashabaranov/go-openai v1.18.3
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.8.0
@@ -34,16 +35,19 @@ require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
 	github.com/nightlyone/lockfile v1.0.0 // indirect
+	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rogpeppe/go-internal v1.11.0 // indirect
 	github.com/samber/lo v1.38.1 // indirect
 	github.com/samber/slog-logrus v1.0.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
 	golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect
 	golang.org/x/mod v0.14.0 // indirect
+	golang.org/x/net v0.19.0 // indirect
 	golang.org/x/sys v0.16.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/tools v0.16.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/klog/v2 v2.110.1 // indirect

go.sum

@@ -44,6 +44,8 @@ github.com/hexops/valast v1.4.3 h1:oBoGERMJh6UZdRc6cduE1CTPK+VAdXA59Y1HFgu3sm0=
 github.com/hexops/valast v1.4.3/go.mod h1:Iqx2kLj3Jn47wuXpj3wX40xn6F93QNFBHuiKBerkTGA=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056 h1:iCHtR9CQyktQ5+f3dMVZfwD2KWJUgm7M0gdL9NGr8KA=
+github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
@@ -60,8 +62,12 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
 github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/nightlyone/lockfile v1.0.0 h1:RHep2cFKK4PonZJDdEl4GmkabuhbsRMgk/k3uAmxBiA=
 github.com/nightlyone/lockfile v1.0.0/go.mod h1:rywoIealpdNse2r832aiD9jRk8ErCatROs6LzC841CI=
+github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
+github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=
 github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
 github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg=
@@ -86,6 +92,8 @@ github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
 github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo=
+github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=

pkg/builtin/builtin.go

@@ -14,6 +14,7 @@ import (
 	"strings"
 
 	"github.com/acorn-io/gptscript/pkg/types"
+	"github.com/jaytaylor/html2text"
 )
 
 var Tools = map[string]types.Tool{
@@ -36,6 +37,12 @@ var Tools = map[string]types.Tool{
 			"url", "The URL to download"),
 		BuiltinFunc: SysHTTPGet,
 	},
+	"sys.http.html2text": {
+		Description: "Download the contents of a http or https URL returning the content as rendered text converted from HTML",
+		Arguments: types.ObjectSchema(
+			"url", "The URL to download"),
+		BuiltinFunc: SysHTTPHtml2Text,
+	},
 	"sys.abort": {
 		Description: "Aborts execution",
 		Arguments: types.ObjectSchema(
@@ -91,10 +98,21 @@ func ListTools() (result []types.Tool) {
 }
 
 func Builtin(name string) (types.Tool, bool) {
+	name, dontFail := strings.CutSuffix(name, "?")
 	t, ok := Tools[name]
 	t.Name = name
 	t.ID = name
 	t.Instructions = "#!" + name
+	if ok && dontFail {
+		orig := t.BuiltinFunc
+		t.BuiltinFunc = func(ctx context.Context, env []string, input string) (string, error) {
+			s, err := orig(ctx, env, input)
+			if err != nil {
+				return fmt.Sprintf("ERROR: %s", err.Error()), nil
+			}
+			return s, err
+		}
+	}
 	return t, ok
 }
 
@@ -218,6 +236,16 @@ func SysHTTPGet(ctx context.Context, env []string, input string) (string, error)
 	return string(data), nil
 }
 
+func SysHTTPHtml2Text(ctx context.Context, env []string, input string) (string, error) {
+	content, err := SysHTTPGet(ctx, env, input)
+	if err != nil {
+		return "", err
+	}
+	return html2text.FromString(content, html2text.Options{
+		PrettyTables: true,
+	})
+}
+
 func SysHTTPPost(ctx context.Context, env []string, input string) (string, error) {
 	var params struct {
 		URL         string `json:"url,omitempty"`

pkg/types/tool.go

@@ -69,7 +69,7 @@ func (t Tool) String() string {
 		sort.Strings(keys)
 		for _, key := range keys {
 			prop := t.Arguments.Properties[key]
-			_, _ = fmt.Fprintf(buf, "Arg: %s: %s\n", key, prop.Description)
+			_, _ = fmt.Fprintf(buf, "Args: %s: %s\n", key, prop.Description)
 		}
 	}
 	if t.Instructions != "" && t.BuiltinFunc == nil {