Cannot use aws_sdk_s3::Client::head_bucket() to get bucket region without already knowing bucket region

[jwodder]

Describe the bug

Consider the following program:

use anyhow::Context;
use aws_sdk_s3::Client;
use clap::Parser;

#[derive(Clone, Debug, Eq, Parser, PartialEq)]
struct Arguments {
    bucket: String,
}

#[tokio::main]
async fn main() -> anyhow::Result<()> {
    let args = Arguments::parse();
    let config = aws_config::from_env()
        .no_credentials()
        .load()
        .await;
    let client = Client::new(&config);
    let r = client
        .head_bucket()
        .bucket(args.bucket)
        .send()
        .await
        .context("failed to determine bucket region")?;
    let region = r
        .bucket_region
        .ok_or(anyhow::anyhow!("no bucket region available"))?;
    println!("{region}");
    Ok(())
}

Dependencies:

[dependencies]
anyhow = "1.0.79"
aws-config = { version = "1.1.3", features = ["behavior-version-latest"] }
aws-sdk-s3 = "1.13.0"
clap = { version = "4.4.18", default-features = false, features = ["derive", "error-context", "help", "std", "suggestions", "usage", "wrap_help"] }
tokio = { version = "1.35.1", features = ["macros", "rt", "rt-multi-thread"] }

When I run this program, passing the name of an S3 general-purpose bucket (which happens to be in us-east-2) as the argument, it fails with:

Error: failed to determine bucket region

Caused by:
    0: dispatch failure
    1: other
    2: A region must be set when sending requests to S3.

If I call region() on the ConfigLoader (after the call to no_credentials()) but use a region that the bucket argument is not in, the program fails with:

Error: failed to determine bucket region

Caused by:
    0: service error
    1: unhandled error
    2: Error { s3_extended_request_id: "TpF4GdonwjKkO77qxaK3b9nlPPjLbdP1xIcvJGny5gBiSNU6NOeJFmFVmBDnKrUvyqvfsDMuTfM=", aws_request_id: "F2DQ48S1FB31W533" }

The only way I can get the program to run successfully is to call region() with the name of the region the bucket is in, yet the whole point of the code is to determine what that region is.

In contrast, if I just do curl -fIsSL https://BUCKET_NAME.amazonaws.com on the command line, the bucket's region is given in the x-amz-bucket-region header, so it clearly shouldn't be hard for the SDK to get this information without the user having to set a region.

Expected Behavior

The program should have successfully retrieved & output the region of the given S3 bucket without having to specify a region in the code.

Current Behavior

See above.

Reproduction Steps

See above.

Possible Solution

No response

Additional Information/Context

No response

Version

├── aws-config v1.1.3
│   ├── aws-credential-types v1.1.3
│   │   ├── aws-smithy-async v1.1.4
│   │   ├── aws-smithy-runtime-api v1.1.4
│   │   │   ├── aws-smithy-async v1.1.4 (*)
│   │   │   ├── aws-smithy-types v1.1.4
│   │   ├── aws-smithy-types v1.1.4 (*)
│   ├── aws-http v0.60.3
│   │   ├── aws-smithy-runtime-api v1.1.4 (*)
│   │   ├── aws-smithy-types v1.1.4 (*)
│   │   ├── aws-types v1.1.3
│   │   │   ├── aws-credential-types v1.1.3 (*)
│   │   │   ├── aws-smithy-async v1.1.4 (*)
│   │   │   ├── aws-smithy-runtime-api v1.1.4 (*)
│   │   │   ├── aws-smithy-types v1.1.4 (*)
│   ├── aws-runtime v1.1.3
│   │   ├── aws-credential-types v1.1.3 (*)
│   │   ├── aws-http v0.60.3 (*)
│   │   ├── aws-sigv4 v1.1.3
│   │   │   ├── aws-credential-types v1.1.3 (*)
│   │   │   ├── aws-smithy-eventstream v0.60.4
│   │   │   │   ├── aws-smithy-types v1.1.4 (*)
│   │   │   ├── aws-smithy-http v0.60.4
│   │   │   │   ├── aws-smithy-eventstream v0.60.4 (*)
│   │   │   │   ├── aws-smithy-runtime-api v1.1.4 (*)
│   │   │   │   ├── aws-smithy-types v1.1.4 (*)
│   │   │   ├── aws-smithy-runtime-api v1.1.4 (*)
│   │   │   ├── aws-smithy-types v1.1.4 (*)
│   │   ├── aws-smithy-async v1.1.4 (*)
│   │   ├── aws-smithy-eventstream v0.60.4 (*)
│   │   ├── aws-smithy-http v0.60.4 (*)
│   │   ├── aws-smithy-runtime-api v1.1.4 (*)
│   │   ├── aws-smithy-types v1.1.4 (*)
│   │   ├── aws-types v1.1.3 (*)
│   ├── aws-sdk-sso v1.11.0
│   │   ├── aws-credential-types v1.1.3 (*)
│   │   ├── aws-http v0.60.3 (*)
│   │   ├── aws-runtime v1.1.3 (*)
│   │   ├── aws-smithy-async v1.1.4 (*)
│   │   ├── aws-smithy-http v0.60.4 (*)
│   │   ├── aws-smithy-json v0.60.4
│   │   │   └── aws-smithy-types v1.1.4 (*)
│   │   ├── aws-smithy-runtime v1.1.4
│   │   │   ├── aws-smithy-async v1.1.4 (*)
│   │   │   ├── aws-smithy-http v0.60.4 (*)
│   │   │   ├── aws-smithy-runtime-api v1.1.4 (*)
│   │   │   ├── aws-smithy-types v1.1.4 (*)
│   │   ├── aws-smithy-runtime-api v1.1.4 (*)
│   │   ├── aws-smithy-types v1.1.4 (*)
│   │   ├── aws-types v1.1.3 (*)
│   ├── aws-sdk-ssooidc v1.11.0
│   │   ├── aws-credential-types v1.1.3 (*)
│   │   ├── aws-http v0.60.3 (*)
│   │   ├── aws-runtime v1.1.3 (*)
│   │   ├── aws-smithy-async v1.1.4 (*)
│   │   ├── aws-smithy-http v0.60.4 (*)
│   │   ├── aws-smithy-json v0.60.4 (*)
│   │   ├── aws-smithy-runtime v1.1.4 (*)
│   │   ├── aws-smithy-runtime-api v1.1.4 (*)
│   │   ├── aws-smithy-types v1.1.4 (*)
│   │   ├── aws-types v1.1.3 (*)
│   ├── aws-sdk-sts v1.11.0
│   │   ├── aws-credential-types v1.1.3 (*)
│   │   ├── aws-http v0.60.3 (*)
│   │   ├── aws-runtime v1.1.3 (*)
│   │   ├── aws-smithy-async v1.1.4 (*)
│   │   ├── aws-smithy-http v0.60.4 (*)
│   │   ├── aws-smithy-json v0.60.4 (*)
│   │   ├── aws-smithy-query v0.60.4
│   │   │   ├── aws-smithy-types v1.1.4 (*)
│   │   ├── aws-smithy-runtime v1.1.4 (*)
│   │   ├── aws-smithy-runtime-api v1.1.4 (*)
│   │   ├── aws-smithy-types v1.1.4 (*)
│   │   ├── aws-smithy-xml v0.60.4
│   │   ├── aws-types v1.1.3 (*)
│   ├── aws-smithy-async v1.1.4 (*)
│   ├── aws-smithy-http v0.60.4 (*)
│   ├── aws-smithy-json v0.60.4 (*)
│   ├── aws-smithy-runtime v1.1.4 (*)
│   ├── aws-smithy-runtime-api v1.1.4 (*)
│   ├── aws-smithy-types v1.1.4 (*)
│   ├── aws-types v1.1.3 (*)
├── aws-sdk-s3 v1.13.0
│   ├── aws-credential-types v1.1.3 (*)
│   ├── aws-http v0.60.3 (*)
│   ├── aws-runtime v1.1.3 (*)
│   ├── aws-sigv4 v1.1.3 (*)
│   ├── aws-smithy-async v1.1.4 (*)
│   ├── aws-smithy-checksums v0.60.4
│   │   ├── aws-smithy-http v0.60.4 (*)
│   │   ├── aws-smithy-types v1.1.4 (*)
│   ├── aws-smithy-eventstream v0.60.4 (*)
│   ├── aws-smithy-http v0.60.4 (*)
│   ├── aws-smithy-json v0.60.4 (*)
│   ├── aws-smithy-runtime v1.1.4 (*)
│   ├── aws-smithy-runtime-api v1.1.4 (*)
│   ├── aws-smithy-types v1.1.4 (*)
│   ├── aws-smithy-xml v0.60.4 (*)
│   ├── aws-types v1.1.3 (*)

Environment details (OS name and version, etc.)

macOS Sonoma 14.2.1, Intel, rustc 1.75

Logs

No response

[rcoh]

ah yeah, that's annoying. One option could be to grab the raw response from the error where you set the region and extract the header manually. Since that header isn't in the model currently, it doesn't get surfaced into the error message unfortunately.

In parallel, I'll work on our side to see if they can model that field. Then the error can contain it directly, at least.

[Velfi]

This is still easiest to accomplish outside of the SDK. Here's an example that uses reqwest to get the region:

use reqwest::{Client, Url};

#[tokio::main]
async fn main() {
    const BUCKET_NAME: &str = "zhessler-test-bucket";
    let client = Client::new();

    let url = Url::parse(format!("https://{}.s3.amazonaws.com", BUCKET_NAME).as_str()).unwrap();
    let res = client.head(url).send().await.unwrap();
    let bucket_region = res.headers().get("x-amz-bucket-region").unwrap().to_str().unwrap();
    println!("Bucket region: {}", bucket_region);
}

Which would print Bucket region: us-east-1 when run.