Merge pull request #421 from Turbo87/db-dumps

crates-io: Add support for `db-dump.zip` files

Author: jdno

terragrunt/modules/crates-io/compute-static/src/main.rs

@@ -116,7 +116,9 @@ fn handle_request(config: &Config, mut request: Request) -> Result<Response, Err
 
     // Database dump is too big to cache on Fastly
     if request.get_url_str().ends_with("db-dump.tar.gz") {
-        redirect_db_dump_to_cloudfront(config)
+        redirect_to_cloudfront(config, "db-dump.tar.gz")
+    } else if request.get_url_str().ends_with("db-dump.zip") {
+        redirect_to_cloudfront(config, "db-dump.zip")
     } else {
         send_request_to_s3(config, &request)
     }
@@ -192,8 +194,8 @@ fn rewrite_download_urls(request: &mut Request) {
 ///
 /// As of early 2023, certain files are too large to be served through Fastly. One of those is the
 /// database dump, which gets redirected to CloudFront.
-fn redirect_db_dump_to_cloudfront(config: &Config) -> Result<Response, Error> {
-    let url = format!("https://{}/db-dump.tar.gz", config.cloudfront_url);
+fn redirect_to_cloudfront(config: &Config, path: &str) -> Result<Response, Error> {
+    let url = format!("https://{}/{path}", config.cloudfront_url);
     Ok(Response::temporary_redirect(url))
 }
 

terragrunt/modules/crates-io/s3-static.tf

@@ -13,17 +13,21 @@ resource "aws_s3_bucket" "static" {
     max_age_seconds = 3000
   }
 
-  // Keep only the live db-dump.tar.gz and the previous day's version, removing
+  // Keep only the live db-dumps and the previous day's versions, removing
   // all the other ones. This is needed because we don't want this file to be
   // versioned, while all the other ones in the bucket should be versioned.
-  lifecycle_rule {
-    id      = "purge-db-dump"
-    enabled = true
-    prefix  = "db-dump.tar.gz"
+  dynamic "lifecycle_rule" {
+    for_each = toset(["db-dump.tar.gz", "db-dump.zip"])
+
+    content {
+      id      = "purge-${replace(lifecycle_rule.key, ".", "-")}"
+      enabled = true
+      prefix  = lifecycle_rule.key
 
-    abort_incomplete_multipart_upload_days = 1
-    noncurrent_version_expiration {
-      days = 1
+      abort_incomplete_multipart_upload_days = 1
+      noncurrent_version_expiration {
+        days = 1
+      }
     }
   }