Skip to content

Commit 74bef8b

Browse files
ivica-kheitorlessa
ivica-k
and
heitorlessa
authored
feat(parser): support for S3 Event Notifications via EventBridge (#1982)
Co-authored-by: heitorlessa <[email protected]>
1 parent eab9463 commit 74bef8b

11 files changed

+289
-22
lines changed

‎aws_lambda_powertools/utilities/parser/models/__init__.py

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,13 @@
4343
KinesisFirehoseRecordMetadata,
4444
)
4545
from .lambda_function_url import LambdaFunctionUrlModel
46-
from .s3 import S3Model, S3RecordModel
46+
from .s3 import (
47+
S3EventNotificationEventBridgeDetailModel,
48+
S3EventNotificationEventBridgeModel,
49+
S3EventNotificationObjectModel,
50+
S3Model,
51+
S3RecordModel,
52+
)
4753
from .s3_object_event import (
4854
S3ObjectConfiguration,
4955
S3ObjectContext,
@@ -105,6 +111,9 @@
105111
"S3ObjectUserRequest",
106112
"S3ObjectConfiguration",
107113
"S3ObjectContext",
114+
"S3EventNotificationObjectModel",
115+
"S3EventNotificationEventBridgeModel",
116+
"S3EventNotificationEventBridgeDetailModel",
108117
"SesModel",
109118
"SesRecordModel",
110119
"SesMessage",

‎aws_lambda_powertools/utilities/parser/models/event_bridge.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
from datetime import datetime
2-
from typing import Any, Dict, List, Optional, Type, Union
2+
from typing import List, Optional
33

44
from pydantic import BaseModel, Field
55

6+
from aws_lambda_powertools.utilities.parser.types import RawDictOrModel
7+
68

79
class EventBridgeModel(BaseModel):
810
version: str
@@ -13,5 +15,5 @@ class EventBridgeModel(BaseModel):
1315
region: str
1416
resources: List[str]
1517
detail_type: str = Field(None, alias="detail-type")
16-
detail: Union[Dict[str, Any], Type[BaseModel]]
18+
detail: RawDictOrModel
1719
replay_name: Optional[str] = Field(None, alias="replay-name")

‎aws_lambda_powertools/utilities/parser/models/s3.py

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88

99
from aws_lambda_powertools.utilities.parser.types import Literal
1010

11+
from .event_bridge import EventBridgeModel
12+
1113

1214
class S3EventRecordGlacierRestoreEventData(BaseModel):
1315
lifecycleRestorationExpiryTime: datetime
@@ -56,6 +58,37 @@ class S3Message(BaseModel):
5658
object: S3Object # noqa: A003,VNE003
5759

5860

61+
class S3EventNotificationObjectModel(BaseModel):
62+
key: str
63+
size: Optional[NonNegativeFloat]
64+
etag: str
65+
version_id: str = Field(None, alias="version-id")
66+
sequencer: Optional[str]
67+
68+
69+
class S3EventNotificationEventBridgeBucketModel(BaseModel):
70+
name: str
71+
72+
73+
class S3EventNotificationEventBridgeDetailModel(BaseModel):
74+
version: str
75+
bucket: S3EventNotificationEventBridgeBucketModel
76+
object: S3EventNotificationObjectModel # noqa: A003,VNE003
77+
request_id: str = Field(None, alias="request-id")
78+
requester: str
79+
source_ip_address: str = Field(None, alias="source-ip-address")
80+
reason: Optional[str]
81+
deletion_type: Optional[str] = Field(None, alias="deletion-type")
82+
restore_expiry_time: Optional[str] = Field(None, alias="restore-expiry-time")
83+
source_storage_class: Optional[str] = Field(None, alias="source-storage-class")
84+
destination_storage_class: Optional[str] = Field(None, alias="destination-storage-class")
85+
destination_access_tier: Optional[str] = Field(None, alias="destination-access-tier")
86+
87+
88+
class S3EventNotificationEventBridgeModel(EventBridgeModel):
89+
detail: S3EventNotificationEventBridgeDetailModel
90+
91+
5992
class S3RecordModel(BaseModel):
6093
eventVersion: str
6194
eventSource: Literal["aws:s3"]

‎aws_lambda_powertools/utilities/parser/types.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
"""Generics and other shared types used across parser"""
22

33
import sys
4-
from typing import TypeVar
4+
from typing import Any, Dict, Type, TypeVar, Union
55

66
from pydantic import BaseModel
77

@@ -14,3 +14,5 @@
1414
Model = TypeVar("Model", bound=BaseModel)
1515
EnvelopeModel = TypeVar("EnvelopeModel")
1616
EventParserReturnType = TypeVar("EventParserReturnType")
17+
AnyInheritedModel = Union[Type[BaseModel], BaseModel]
18+
RawDictOrModel = Union[Dict[str, Any], AnyInheritedModel]

‎docs/utilities/parser.md

Lines changed: 19 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -156,24 +156,25 @@ def my_function():
156156

157157
Parser comes with the following built-in models:
158158

159-
| Model name | Description |
160-
| ------------------------------- | ------------------------------------------------------------------ |
161-
| **DynamoDBStreamModel** | Lambda Event Source payload for Amazon DynamoDB Streams |
162-
| **EventBridgeModel** | Lambda Event Source payload for Amazon EventBridge |
163-
| **SqsModel** | Lambda Event Source payload for Amazon SQS |
164-
| **AlbModel** | Lambda Event Source payload for Amazon Application Load Balancer |
165-
| **CloudwatchLogsModel** | Lambda Event Source payload for Amazon CloudWatch Logs |
166-
| **S3Model** | Lambda Event Source payload for Amazon S3 |
167-
| **S3ObjectLambdaEvent** | Lambda Event Source payload for Amazon S3 Object Lambda |
168-
| **KinesisDataStreamModel** | Lambda Event Source payload for Amazon Kinesis Data Streams |
169-
| **KinesisFirehoseModel** | Lambda Event Source payload for Amazon Kinesis Firehose |
170-
| **SesModel** | Lambda Event Source payload for Amazon Simple Email Service |
171-
| **SnsModel** | Lambda Event Source payload for Amazon Simple Notification Service |
172-
| **APIGatewayProxyEventModel** | Lambda Event Source payload for Amazon API Gateway |
173-
| **APIGatewayProxyEventV2Model** | Lambda Event Source payload for Amazon API Gateway v2 payload |
174-
| **LambdaFunctionUrlModel** | Lambda Event Source payload for Lambda Function URL payload |
175-
| **KafkaSelfManagedEventModel** | Lambda Event Source payload for self managed Kafka payload |
176-
| **KafkaMskEventModel** | Lambda Event Source payload for AWS MSK payload |
159+
| Model name | Description |
160+
| --------------------------------------- | ---------------------------------------------------------------------------- |
161+
| **DynamoDBStreamModel** | Lambda Event Source payload for Amazon DynamoDB Streams |
162+
| **EventBridgeModel** | Lambda Event Source payload for Amazon EventBridge |
163+
| **SqsModel** | Lambda Event Source payload for Amazon SQS |
164+
| **AlbModel** | Lambda Event Source payload for Amazon Application Load Balancer |
165+
| **CloudwatchLogsModel** | Lambda Event Source payload for Amazon CloudWatch Logs |
166+
| **S3Model** | Lambda Event Source payload for Amazon S3 |
167+
| **S3ObjectLambdaEvent** | Lambda Event Source payload for Amazon S3 Object Lambda |
168+
| **S3EventNotificationEventBridgeModel** | Lambda Event Source payload for Amazon S3 Event Notification to EventBridge. |
169+
| **KinesisDataStreamModel** | Lambda Event Source payload for Amazon Kinesis Data Streams |
170+
| **KinesisFirehoseModel** | Lambda Event Source payload for Amazon Kinesis Firehose |
171+
| **SesModel** | Lambda Event Source payload for Amazon Simple Email Service |
172+
| **SnsModel** | Lambda Event Source payload for Amazon Simple Notification Service |
173+
| **APIGatewayProxyEventModel** | Lambda Event Source payload for Amazon API Gateway |
174+
| **APIGatewayProxyEventV2Model** | Lambda Event Source payload for Amazon API Gateway v2 payload |
175+
| **LambdaFunctionUrlModel** | Lambda Event Source payload for Lambda Function URL payload |
176+
| **KafkaSelfManagedEventModel** | Lambda Event Source payload for self managed Kafka payload |
177+
| **KafkaMskEventModel** | Lambda Event Source payload for AWS MSK payload |
177178

178179
#### Extending built-in models
179180

‎tests/events/s3EventBridgeNotificationObjectCreatedEvent.json

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
{
2+
"version": "0",
3+
"id": "f5f1e65c-dc3a-93ca-6c1e-b1647eac7963",
4+
"detail-type": "Object Created",
5+
"source": "aws.s3",
6+
"account": "123456789012",
7+
"time": "2023-03-08T17:50:14Z",
8+
"region": "eu-west-1",
9+
"resources": [
10+
"arn:aws:s3:::example-bucket"
11+
],
12+
"detail": {
13+
"version": "0",
14+
"bucket": {
15+
"name": "example-bucket"
16+
},
17+
"object": {
18+
"key": "IMG_m7fzo3.jpg",
19+
"size": 184662,
20+
"etag": "4e68adba0abe2dc8653dc3354e14c01d",
21+
"sequencer": "006408CAD69598B05E"
22+
},
23+
"request-id": "57H08PA84AB1JZW0",
24+
"requester": "123456789012",
25+
"source-ip-address": "34.252.34.74",
26+
"reason": "PutObject"
27+
}
28+
}

‎tests/events/s3EventBridgeNotificationObjectDeletedEvent.json

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
{
2+
"version": "0",
3+
"id": "2ee9cc15-d022-99ea-1fb8-1b1bac4850f9",
4+
"detail-type": "Object Deleted",
5+
"source": "aws.s3",
6+
"account": "111122223333",
7+
"time": "2021-11-12T00:00:00Z",
8+
"region": "ca-central-1",
9+
"resources": [
10+
"arn:aws:s3:::example-bucket"
11+
],
12+
"detail": {
13+
"version": "0",
14+
"bucket": {
15+
"name": "example-bucket"
16+
},
17+
"object": {
18+
"key": "IMG_m7fzo3.jpg",
19+
"size": 184662,
20+
"etag": "4e68adba0abe2dc8653dc3354e14c01d",
21+
"sequencer": "006408CAD69598B05E"
22+
},
23+
"request-id": "0BH729840619AG5K",
24+
"requester": "123456789012",
25+
"source-ip-address": "34.252.34.74",
26+
"reason": "DeleteObject",
27+
"deletion-type": "Delete Marker Created"
28+
}
29+
}

‎tests/events/s3EventBridgeNotificationObjectExpiredEvent.json

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
{
2+
"version": "0",
3+
"id": "ad1de317-e409-eba2-9552-30113f8d88e3",
4+
"detail-type": "Object Deleted",
5+
"source": "aws.s3",
6+
"account": "111122223333",
7+
"time": "2021-11-12T00:00:00Z",
8+
"region": "ca-central-1",
9+
"resources": [
10+
"arn:aws:s3:::example-bucket"
11+
],
12+
"detail": {
13+
"version": "0",
14+
"bucket": {
15+
"name": "example-bucket"
16+
},
17+
"object": {
18+
"key": "IMG_m7fzo3.jpg",
19+
"size": 184662,
20+
"etag": "4e68adba0abe2dc8653dc3354e14c01d",
21+
"sequencer": "006408CAD69598B05E"
22+
},
23+
"request-id": "20EB74C14654DC47",
24+
"requester": "s3.amazonaws.com",
25+
"reason": "Lifecycle Expiration",
26+
"deletion-type": "Delete Marker Created"
27+
}
28+
}

‎tests/events/s3EventBridgeNotificationObjectRestoreCompletedEvent.json

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
{
2+
"version": "0",
3+
"id": "6924de0d-13e2-6bbf-c0c1-b903b753565e",
4+
"detail-type": "Object Restore Completed",
5+
"source": "aws.s3",
6+
"account": "111122223333",
7+
"time": "2021-11-12T00:00:00Z",
8+
"region": "ca-central-1",
9+
"resources": [
10+
"arn:aws:s3:::example-bucket"
11+
],
12+
"detail": {
13+
"version": "0",
14+
"bucket": {
15+
"name": "example-bucket"
16+
},
17+
"object": {
18+
"key": "IMG_m7fzo3.jpg",
19+
"size": 184662,
20+
"etag": "4e68adba0abe2dc8653dc3354e14c01d",
21+
"sequencer": "006408CAD69598B05E"
22+
},
23+
"request-id": "189F19CB7FB1B6A4",
24+
"requester": "s3.amazonaws.com",
25+
"restore-expiry-time": "2021-11-13T00:00:00Z",
26+
"source-storage-class": "GLACIER"
27+
}
28+
}

‎tests/unit/parser/__init__.py

Whitespace-only changes.

‎tests/unit/parser/test_s3.py

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
from datetime import datetime
2+
3+
from aws_lambda_powertools.utilities.parser.models import (
4+
S3EventNotificationEventBridgeModel,
5+
)
6+
from tests.functional.utils import load_event
7+
8+
9+
def test_s3_eventbridge_notification_object_created_event():
10+
raw_event = load_event("s3EventBridgeNotificationObjectCreatedEvent.json")
11+
model = S3EventNotificationEventBridgeModel(**raw_event)
12+
13+
assert model.version == raw_event["version"]
14+
assert model.id == raw_event["id"]
15+
assert model.detail_type == raw_event["detail-type"]
16+
assert model.source == raw_event["source"]
17+
assert model.account == raw_event["account"]
18+
assert model.time == datetime.fromisoformat(raw_event["time"].replace("Z", "+00:00"))
19+
assert model.region == raw_event["region"]
20+
assert model.resources == raw_event["resources"]
21+
22+
assert model.detail.version == raw_event["detail"]["version"]
23+
assert model.detail.bucket.name == raw_event["detail"]["bucket"]["name"]
24+
assert model.detail.object.key == raw_event["detail"]["object"]["key"]
25+
assert model.detail.object.size == raw_event["detail"]["object"]["size"]
26+
assert model.detail.object.etag == raw_event["detail"]["object"]["etag"]
27+
assert model.detail.object.sequencer == raw_event["detail"]["object"]["sequencer"]
28+
assert model.detail.request_id == raw_event["detail"]["request-id"]
29+
assert model.detail.requester == raw_event["detail"]["requester"]
30+
assert model.detail.source_ip_address == raw_event["detail"]["source-ip-address"]
31+
assert model.detail.reason == raw_event["detail"]["reason"]
32+
33+
34+
def test_s3_eventbridge_notification_object_deleted_event():
35+
raw_event = load_event("s3EventBridgeNotificationObjectDeletedEvent.json")
36+
model = S3EventNotificationEventBridgeModel(**raw_event)
37+
38+
assert model.version == raw_event["version"]
39+
assert model.id == raw_event["id"]
40+
assert model.detail_type == raw_event["detail-type"]
41+
assert model.source == raw_event["source"]
42+
assert model.account == raw_event["account"]
43+
assert model.time == datetime.fromisoformat(raw_event["time"].replace("Z", "+00:00"))
44+
assert model.region == raw_event["region"]
45+
assert model.resources == raw_event["resources"]
46+
47+
assert model.detail.version == raw_event["detail"]["version"]
48+
assert model.detail.bucket.name == raw_event["detail"]["bucket"]["name"]
49+
assert model.detail.object.key == raw_event["detail"]["object"]["key"]
50+
assert model.detail.object.size == raw_event["detail"]["object"]["size"]
51+
assert model.detail.object.etag == raw_event["detail"]["object"]["etag"]
52+
assert model.detail.object.sequencer == raw_event["detail"]["object"]["sequencer"]
53+
assert model.detail.request_id == raw_event["detail"]["request-id"]
54+
assert model.detail.requester == raw_event["detail"]["requester"]
55+
assert model.detail.source_ip_address == raw_event["detail"]["source-ip-address"]
56+
assert model.detail.reason == raw_event["detail"]["reason"]
57+
assert model.detail.deletion_type == raw_event["detail"]["deletion-type"]
58+
59+
60+
def test_s3_eventbridge_notification_object_expired_event():
61+
raw_event = load_event("s3EventBridgeNotificationObjectExpiredEvent.json")
62+
model = S3EventNotificationEventBridgeModel(**raw_event)
63+
64+
assert model.version == raw_event["version"]
65+
assert model.id == raw_event["id"]
66+
assert model.detail_type == raw_event["detail-type"]
67+
assert model.source == raw_event["source"]
68+
assert model.account == raw_event["account"]
69+
assert model.time == datetime.fromisoformat(raw_event["time"].replace("Z", "+00:00"))
70+
assert model.region == raw_event["region"]
71+
assert model.resources == raw_event["resources"]
72+
73+
assert model.detail.version == raw_event["detail"]["version"]
74+
assert model.detail.bucket.name == raw_event["detail"]["bucket"]["name"]
75+
assert model.detail.object.key == raw_event["detail"]["object"]["key"]
76+
assert model.detail.object.size == raw_event["detail"]["object"]["size"]
77+
assert model.detail.object.etag == raw_event["detail"]["object"]["etag"]
78+
assert model.detail.object.sequencer == raw_event["detail"]["object"]["sequencer"]
79+
assert model.detail.request_id == raw_event["detail"]["request-id"]
80+
assert model.detail.requester == raw_event["detail"]["requester"]
81+
assert model.detail.reason == raw_event["detail"]["reason"]
82+
assert model.detail.deletion_type == raw_event["detail"]["deletion-type"]
83+
84+
85+
def test_s3_eventbridge_notification_object_restore_completed_event():
86+
raw_event = load_event("s3EventBridgeNotificationObjectRestoreCompletedEvent.json")
87+
model = S3EventNotificationEventBridgeModel(**raw_event)
88+
89+
assert model.version == raw_event["version"]
90+
assert model.id == raw_event["id"]
91+
assert model.detail_type == raw_event["detail-type"]
92+
assert model.source == raw_event["source"]
93+
assert model.account == raw_event["account"]
94+
assert model.time == datetime.fromisoformat(raw_event["time"].replace("Z", "+00:00"))
95+
assert model.region == raw_event["region"]
96+
assert model.resources == raw_event["resources"]
97+
98+
assert model.detail.version == raw_event["detail"]["version"]
99+
assert model.detail.bucket.name == raw_event["detail"]["bucket"]["name"]
100+
assert model.detail.object.key == raw_event["detail"]["object"]["key"]
101+
assert model.detail.object.size == raw_event["detail"]["object"]["size"]
102+
assert model.detail.object.etag == raw_event["detail"]["object"]["etag"]
103+
assert model.detail.object.sequencer == raw_event["detail"]["object"]["sequencer"]
104+
assert model.detail.request_id == raw_event["detail"]["request-id"]
105+
assert model.detail.requester == raw_event["detail"]["requester"]
106+
assert model.detail.restore_expiry_time == raw_event["detail"]["restore-expiry-time"]
107+
assert model.detail.source_storage_class == raw_event["detail"]["source-storage-class"]

0 commit comments

Comments
 (0)