Merge pull request #461 from tylerslaton/parse-remote

feat: add support for remote file parsing

Author: tylerslaton

pkg/cli/parse.go

@@ -2,10 +2,12 @@ package cli
 
 import (
 	"encoding/json"
+	"fmt"
 	"os"
 	"strings"
 
 	"github.com/gptscript-ai/gptscript/pkg/input"
+	"github.com/gptscript-ai/gptscript/pkg/loader"
 	"github.com/gptscript-ai/gptscript/pkg/parser"
 	"github.com/spf13/cobra"
 )
@@ -26,12 +28,28 @@ func locationName(l string) string {
 }
 
 func (e *Parse) Run(_ *cobra.Command, args []string) error {
-	input, err := input.FromFile(args[0])
+	var (
+		content string
+		err     error
+	)
+
+	// Attempt to read the file first, if that fails, try to load the URL. Finally,
+	// return an error if both fail.
+	content, err = input.FromFile(args[0])
 	if err != nil {
-		return err
+		log.Debugf("failed to read file %s (due to %v) attempting to load the URL...", args[0], err)
+		content, err = loader.ContentFromURL(args[0])
+		if err != nil {
+			return err
+		}
+		// If the content is empty and there was no error, this is not a remote file. Return a generic
+		// error indicating that the file could not be loaded.
+		if content == "" {
+			return fmt.Errorf("failed to load %v", args[0])
+		}
 	}
 
-	docs, err := parser.Parse(strings.NewReader(input), parser.Options{
+	docs, err := parser.Parse(strings.NewReader(content), parser.Options{
 		Location: locationName(args[0]),
 	})
 	if err != nil {

pkg/loader/url.go

@@ -137,3 +137,21 @@ func loadURL(ctx context.Context, cache *cache.Client, base *source, name string
 
 	return result, true, nil
 }
+
+func ContentFromURL(url string) (string, error) {
+	cache, err := cache.New()
+	if err != nil {
+		return "", fmt.Errorf("failed to create cache: %w", err)
+	}
+
+	source, ok, err := loadURL(context.Background(), cache, &source{}, url)
+	if err != nil {
+		return "", fmt.Errorf("failed to load %s: %w", url, err)
+	}
+
+	if !ok {
+		return "", nil
+	}
+
+	return string(source.Content), nil
+}