Add RemoteS3ConnectionProvider plugin implementations

Added generic implementations for the RemoteS3ConnectionProvider interface: config, file and http.

Added a README for the file and http ones as those are a bit more complex.

Added more tests to dynamic RemoteS3Facade creation

Author: mosiac1

trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/credentials/CredentialsProvider.java

@@ -15,7 +15,6 @@
 
 import java.util.Optional;
 
-// TODO: Add back file-based provider
 public interface CredentialsProvider
 {
     CredentialsProvider NOOP = (_, _) -> Optional.empty();

trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/remote/RemoteS3ConnectionProvider.java

@@ -24,5 +24,5 @@ public interface RemoteS3ConnectionProvider
 {
     RemoteS3ConnectionProvider NOOP = (_, _, _) -> Optional.empty();
 
-    Optional<RemoteS3Connection> remoteConnection(SigningMetadata signingMetadata, Optional<Identity> identity, ParsedS3Request request);
+    Optional<? extends RemoteS3Connection> remoteConnection(SigningMetadata signingMetadata, Optional<Identity> identity, ParsedS3Request request);
 }

trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/TrinoAwsProxyServerModule.java

@@ -34,6 +34,9 @@
 import io.trino.aws.proxy.server.credentials.http.HttpCredentialsModule;
 import io.trino.aws.proxy.server.remote.DefaultRemoteS3Module;
 import io.trino.aws.proxy.server.remote.RemoteS3ConnectionController;
+import io.trino.aws.proxy.server.remote.provider.config.ConfigRemoteS3ConnectionProviderModule;
+import io.trino.aws.proxy.server.remote.provider.file.FileBasedRemoteS3ConnectionModule;
+import io.trino.aws.proxy.server.remote.provider.http.HttpRemoteS3ConnectionProviderModule;
 import io.trino.aws.proxy.server.rest.LimitStreamController;
 import io.trino.aws.proxy.server.rest.ResourceSecurityDynamicFeature;
 import io.trino.aws.proxy.server.rest.RestModule;
@@ -134,6 +137,9 @@ protected void setup(Binder binder)
         install(new FileBasedCredentialsModule());
         install(new OpaS3SecurityModule());
         install(new HttpCredentialsModule());
+        install(new FileBasedRemoteS3ConnectionModule());
+        install(new ConfigRemoteS3ConnectionProviderModule());
+        install(new HttpRemoteS3ConnectionProviderModule());
 
         configBinder(binder).bindConfig(RemoteS3Config.class);
         // RemoteS3 provided implementation

trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/DefaultRemoteS3Config.java

@@ -13,6 +13,7 @@
  */
 package io.trino.aws.proxy.server.remote;
 
+import com.fasterxml.jackson.annotation.JsonSetter;
 import io.airlift.configuration.Config;
 import jakarta.validation.constraints.Max;
 import jakarta.validation.constraints.Min;
@@ -29,34 +30,39 @@ public class DefaultRemoteS3Config
     private String hostnameTemplate = "${bucket}.s3.${region}.${domain}";
 
     @Config("remoteS3.https")
+    @JsonSetter("remoteS3.https")
     public DefaultRemoteS3Config setHttps(boolean https)
     {
         this.https = https;
         return this;
     }
 
     @Config("remoteS3.domain")
+    @JsonSetter("remoteS3.domain")
     public DefaultRemoteS3Config setDomain(String s3Domain)
     {
         this.domain = s3Domain;
         return this;
     }
 
     @Config("remoteS3.port")
+    @JsonSetter("remoteS3.port")
     public DefaultRemoteS3Config setPort(Integer port)
     {
         this.port = Optional.ofNullable(port);
         return this;
     }
 
     @Config("remoteS3.virtual-host-style")
+    @JsonSetter("remoteS3.virtual-host-style")
     public DefaultRemoteS3Config setVirtualHostStyle(boolean virtualHostStyle)
     {
         this.virtualHostStyle = virtualHostStyle;
         return this;
     }
 
     @Config("remoteS3.hostname.template")
+    @JsonSetter("remoteS3.hostname.template")
     public DefaultRemoteS3Config setHostnameTemplate(String hostnameTemplate)
     {
         this.hostnameTemplate = hostnameTemplate;

trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/SerializableRemoteS3Connection.java

@@ -0,0 +1,73 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.aws.proxy.server.remote.provider;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.inject.ConfigurationException;
+import io.airlift.configuration.ConfigurationFactory;
+import io.trino.aws.proxy.server.remote.DefaultRemoteS3Config;
+import io.trino.aws.proxy.server.remote.PathStyleRemoteS3Facade;
+import io.trino.aws.proxy.server.remote.VirtualHostStyleRemoteS3Facade;
+import io.trino.aws.proxy.spi.credentials.Credential;
+import io.trino.aws.proxy.spi.remote.RemoteS3Connection;
+import io.trino.aws.proxy.spi.remote.RemoteS3Facade;
+import io.trino.aws.proxy.spi.remote.RemoteSessionRole;
+
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+import static com.google.common.collect.Sets.difference;
+import static java.lang.String.format;
+import static java.util.Objects.requireNonNull;
+
+public record SerializableRemoteS3Connection(
+        Credential remoteCredential,
+        Optional<RemoteSessionRole> remoteSessionRole,
+        Optional<RemoteS3Facade> remoteS3Facade)
+        implements RemoteS3Connection
+{
+    public SerializableRemoteS3Connection
+    {
+        requireNonNull(remoteCredential, "remoteCredential is null");
+        requireNonNull(remoteSessionRole, "remoteSessionRole is null");
+        requireNonNull(remoteS3Facade, "remoteS3Facade is null");
+    }
+
+    @JsonCreator
+    public static SerializableRemoteS3Connection fromConfig(
+            @JsonProperty("remoteCredential") Credential remoteCredential,
+            @JsonProperty("remoteSessionRole") Optional<RemoteSessionRole> remoteSessionRole,
+            @JsonProperty("remoteS3FacadeConfiguration") Optional<Map<String, String>> remoteS3FacadeConfiguration)
+    {
+        Optional<RemoteS3Facade> facade = remoteS3FacadeConfiguration.map(config -> {
+            ConfigurationFactory configurationFactory = new ConfigurationFactory(config);
+            DefaultRemoteS3Config parsedConfig;
+            try {
+                parsedConfig = configurationFactory.build(DefaultRemoteS3Config.class);
+            }
+            catch (ConfigurationException e) {
+                throw new IllegalArgumentException("Failed create RemoteS3Facade from RemoteS3FacadeConfiguration", e);
+            }
+            Set<String> unusedProperties = difference(configurationFactory.getProperties().keySet(), configurationFactory.getUsedProperties());
+            if (!unusedProperties.isEmpty()) {
+                throw new IllegalArgumentException(format("Failed to create RemoteS3Facade from RemoteS3FacadeConfiguration. Unused properties when instantiating " +
+                        "DefaultRemoteS3Config: %s", unusedProperties));
+            }
+            return parsedConfig.getVirtualHostStyle() ? new VirtualHostStyleRemoteS3Facade(parsedConfig) : new PathStyleRemoteS3Facade(parsedConfig);
+        });
+        return new SerializableRemoteS3Connection(remoteCredential, remoteSessionRole, facade);
+    }
+}

trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/config/ConfigRemoteS3ConnectionProviderConfig.java

@@ -0,0 +1,54 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.aws.proxy.server.remote.provider.config;
+
+import io.airlift.configuration.Config;
+import io.airlift.configuration.ConfigSecuritySensitive;
+import jakarta.validation.constraints.NotEmpty;
+import jakarta.validation.constraints.NotNull;
+
+public class ConfigRemoteS3ConnectionProviderConfig
+{
+    private String accessKey;
+    private String secretKey;
+
+    @NotNull
+    @NotEmpty
+    public String getAccessKey()
+    {
+        return accessKey;
+    }
+
+    @Config("remote-s3-connection-provider.access-key")
+    public ConfigRemoteS3ConnectionProviderConfig setAccessKey(String accessKey)
+    {
+        this.accessKey = accessKey;
+        return this;
+    }
+
+    @NotNull
+    @NotEmpty
+    public String getSecretKey()
+    {
+        return secretKey;
+    }
+
+    @ConfigSecuritySensitive
+    @Config("remote-s3-connection-provider.secret-key")
+    public ConfigRemoteS3ConnectionProviderConfig setSecretKey(String secretKey)
+    {
+        this.secretKey = secretKey;
+        return this;
+    }
+}

trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/config/ConfigRemoteS3ConnectionProviderModule.java

@@ -0,0 +1,46 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.aws.proxy.server.remote.provider.config;
+
+import com.google.inject.Binder;
+import io.airlift.configuration.AbstractConfigurationAwareModule;
+import io.trino.aws.proxy.spi.credentials.Credential;
+import io.trino.aws.proxy.spi.plugin.config.RemoteS3ConnectionProviderConfig;
+import io.trino.aws.proxy.spi.remote.RemoteS3Connection.StaticRemoteS3Connection;
+import io.trino.aws.proxy.spi.remote.RemoteS3ConnectionProvider;
+
+import java.util.Optional;
+
+import static com.google.inject.multibindings.OptionalBinder.newOptionalBinder;
+import static io.airlift.configuration.ConditionalModule.conditionalModule;
+
+public class ConfigRemoteS3ConnectionProviderModule
+        extends AbstractConfigurationAwareModule
+{
+    public static final String CONFIG_REMOTE_S3_CONNECTION_PROVIDER = "config";
+
+    @Override
+    protected void setup(Binder binder)
+    {
+        install(conditionalModule(
+                RemoteS3ConnectionProviderConfig.class,
+                config -> config.getPluginIdentifier().map(CONFIG_REMOTE_S3_CONNECTION_PROVIDER::equals).orElse(false),
+                innerBinder -> {
+                    ConfigRemoteS3ConnectionProviderConfig config = buildConfigObject(ConfigRemoteS3ConnectionProviderConfig.class);
+                    newOptionalBinder(innerBinder, RemoteS3ConnectionProvider.class)
+                            .setBinding()
+                            .toInstance((_, _, _) -> Optional.of(new StaticRemoteS3Connection(new Credential(config.getAccessKey(), config.getSecretKey()))));
+                }));
+    }
+}

trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/file/FileBasedRemoteS3ConnectionModule.java

@@ -0,0 +1,42 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.aws.proxy.server.remote.provider.file;
+
+import com.google.inject.Binder;
+import io.airlift.configuration.AbstractConfigurationAwareModule;
+import io.trino.aws.proxy.server.remote.provider.SerializableRemoteS3Connection;
+
+import static io.airlift.configuration.ConfigBinder.configBinder;
+import static io.airlift.json.JsonCodecBinder.jsonCodecBinder;
+import static io.trino.aws.proxy.spi.plugin.TrinoAwsProxyServerBinding.remoteS3ConnectionProviderModule;
+
+public class FileBasedRemoteS3ConnectionModule
+        extends AbstractConfigurationAwareModule
+{
+    // set as config value for "remote-s3-connection-provider.type"
+    public static final String FILE_BASED_REMOTE_S3_CONNECTION_PROVIDER = "file";
+
+    @Override
+    protected void setup(Binder binder)
+    {
+        install(remoteS3ConnectionProviderModule(
+                FILE_BASED_REMOTE_S3_CONNECTION_PROVIDER,
+                FileBasedRemoteS3ConnectionProvider.class,
+                innerBinder -> {
+                    configBinder(innerBinder).bindConfig(FileBasedRemoteS3ConnectionProviderConfig.class);
+                    innerBinder.bind(FileBasedRemoteS3ConnectionProvider.class);
+                    jsonCodecBinder(innerBinder).bindMapJsonCodec(String.class, SerializableRemoteS3Connection.class);
+                }));
+    }
+}

trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/file/FileBasedRemoteS3ConnectionProvider.java

@@ -0,0 +1,77 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.aws.proxy.server.remote.provider.file;
+
+import com.google.common.io.Files;
+import com.google.inject.Inject;
+import io.airlift.json.JsonCodec;
+import io.trino.aws.proxy.server.remote.provider.SerializableRemoteS3Connection;
+import io.trino.aws.proxy.spi.credentials.Identity;
+import io.trino.aws.proxy.spi.remote.RemoteS3Connection;
+import io.trino.aws.proxy.spi.remote.RemoteS3ConnectionProvider;
+import io.trino.aws.proxy.spi.rest.ParsedS3Request;
+import io.trino.aws.proxy.spi.signing.SigningMetadata;
+
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * <p>File-based RemoteS3ConnectionProvider that reads a JSON file containing a mapping from emulated access key to
+ * RemoteS3Connection.</p>
+ * <pre>{@code
+ * {
+ *   "emulated-access-key-1": {
+ *     "remoteCredential": {
+ *       "accessKey": "remote-access-key",
+ *       "secretKey": "remote-secret-key"
+ *     },
+ *     "remoteSessionRole": {
+ *       "region": "us-east-1",
+ *       "roleArn": "arn:aws:iam::123456789012:role/role-name",
+ *       "externalId": "external-id",
+ *       "stsEndpoint": "https://sts.us-east-1.amazonaws.com"
+ *     },
+ *     "remoteS3FacadeConfiguration": {
+ *       "remoteS3.https": true,
+ *       "remoteS3.domain": "s3.amazonaws.com",
+ *       "remoteS3.port": 443,
+ *       "remoteS3.virtual-host-style": false,
+ *       "remoteS3.hostname.template": "${domain}"
+ *     }
+ *   }
+ * }
+ * }</pre>
+ */
+public class FileBasedRemoteS3ConnectionProvider
+        implements RemoteS3ConnectionProvider
+{
+    private final Map<String, SerializableRemoteS3Connection> remoteS3Connections;
+
+    @Inject
+    public FileBasedRemoteS3ConnectionProvider(FileBasedRemoteS3ConnectionProviderConfig config, JsonCodec<Map<String, SerializableRemoteS3Connection>> jsonCodec)
+    {
+        try {
+            this.remoteS3Connections = jsonCodec.fromJson(Files.toByteArray(config.getConnectionsFile()));
+        }
+        catch (Exception e) {
+            throw new RuntimeException("Failed to read remote S3 connections file", e);
+        }
+    }
+
+    @Override
+    public Optional<RemoteS3Connection> remoteConnection(SigningMetadata signingMetadata, Optional<Identity> identity, ParsedS3Request request)
+    {
+        return Optional.ofNullable(remoteS3Connections.get(signingMetadata.credential().accessKey()));
+    }
+}