opendal
opendal copied to clipboard
apache
Implement `Write Returns Metadata` for all services
#5562 has implemented all the fundamental elements for Write Returns Metadata. Now, we need to extend this feature for every service. This issue will be used to track the progress.
The modification is straightforward -- we just need to extract metadata from all write responses.
for example:
diff --git a/core/src/services/azblob/backend.rs b/core/src/services/azblob/backend.rs
index eafe32ddb..331d9b954 100644
--- a/core/src/services/azblob/backend.rs
+++ b/core/src/services/azblob/backend.rs
@@ -32,6 +32,7 @@ use sha2::Digest;
use sha2::Sha256;
use super::core::constants::X_MS_META_PREFIX;
+use super::core::constants::X_MS_VERSION_ID;
use super::core::AzblobCore;
use super::delete::AzblobDeleter;
use super::error::parse_error;
@@ -567,6 +568,9 @@ impl Access for AzblobBackend {
StatusCode::OK => {
let headers = resp.headers();
let mut meta = parse_into_metadata(path, headers)?;
+ if let Some(version_id) = parse_header_to_str(headers, X_MS_VERSION_ID)? {
+ meta.set_version(version_id);
+ }
let user_meta = parse_prefixed_headers(headers, X_MS_META_PREFIX);
if !user_meta.is_empty() {
diff --git a/core/src/services/azblob/core.rs b/core/src/services/azblob/core.rs
index 87fd4da11..7f664e64b 100644
--- a/core/src/services/azblob/core.rs
+++ b/core/src/services/azblob/core.rs
@@ -47,6 +47,7 @@ use crate::raw::*;
use crate::*;
pub mod constants {
+ // Indicates the Blob Storage version that was used to execute the request
pub const X_MS_VERSION: &str = "x-ms-version";
pub const X_MS_BLOB_TYPE: &str = "x-ms-blob-type";
@@ -55,6 +56,9 @@ pub mod constants {
pub const X_MS_BLOB_CONDITION_APPENDPOS: &str = "x-ms-blob-condition-appendpos";
pub const X_MS_META_PREFIX: &str = "x-ms-meta-";
+ // indicates the version of the blob, and it can be used in subsequent requests to access the blob.
+ pub const X_MS_VERSION_ID: &str = "x-ms-version-id";
+
// Server-side encryption with customer-provided headers
pub const X_MS_ENCRYPTION_KEY: &str = "x-ms-encryption-key";
pub const X_MS_ENCRYPTION_KEY_SHA256: &str = "x-ms-encryption-key-sha256";
diff --git a/core/src/services/azblob/writer.rs b/core/src/services/azblob/writer.rs
index 17cf8b676..d67b43e5a 100644
--- a/core/src/services/azblob/writer.rs
+++ b/core/src/services/azblob/writer.rs
@@ -20,6 +20,7 @@ use std::sync::Arc;
use http::StatusCode;
use uuid::Uuid;
+use super::core::constants::X_MS_VERSION_ID;
use super::core::AzblobCore;
use super::error::parse_error;
use crate::raw::*;
@@ -40,6 +41,26 @@ impl AzblobWriter {
pub fn new(core: Arc<AzblobCore>, op: OpWrite, path: String) -> Self {
AzblobWriter { core, op, path }
}
+
+ // skip extracting `content-md5` here, as it pertains to the content of the request rather than
+ // the content of the block itself for the `append` and `complete put block list` operations.
+ fn parse_metadata(headers: &http::HeaderMap) -> Result<Metadata> {
+ let mut metadata = Metadata::default();
+
+ if let Some(last_modified) = parse_last_modified(headers)? {
+ metadata.set_last_modified(last_modified);
+ }
+ let etag = parse_etag(headers)?;
+ if let Some(etag) = etag {
+ metadata.set_etag(etag);
+ }
+ let version_id = parse_header_to_str(headers, X_MS_VERSION_ID)?;
+ if let Some(version_id) = version_id {
+ metadata.set_version(version_id);
+ }
+
+ Ok(metadata)
+ }
}
impl oio::AppendWrite for AzblobWriter {
@@ -97,9 +118,10 @@ impl oio::AppendWrite for AzblobWriter {
let resp = self.core.send(req).await?;
+ let meta = AzblobWriter::parse_metadata(resp.headers())?;
let status = resp.status();
match status {
- StatusCode::CREATED => Ok(Metadata::default()),
+ StatusCode::CREATED => Ok(meta),
_ => Err(parse_error(resp)),
}
}
@@ -116,8 +138,13 @@ impl oio::BlockWrite for AzblobWriter {
let status = resp.status();
+ let mut meta = AzblobWriter::parse_metadata(resp.headers())?;
+ let md5 = parse_content_md5(resp.headers())?;
+ if let Some(md5) = md5 {
+ meta.set_content_md5(md5);
+ }
match status {
- StatusCode::CREATED | StatusCode::OK => Ok(Metadata::default()),
+ StatusCode::CREATED | StatusCode::OK => Ok(meta),
_ => Err(parse_error(resp)),
}
}
@@ -141,9 +168,10 @@ impl oio::BlockWrite for AzblobWriter {
.azblob_complete_put_block_list(&self.path, block_ids, &self.op)
.await?;
+ let meta = AzblobWriter::parse_metadata(resp.headers())?;
let status = resp.status();
match status {
- StatusCode::CREATED | StatusCode::OK => Ok(Metadata::default()),
+ StatusCode::CREATED | StatusCode::OK => Ok(meta),
_ => Err(parse_error(resp)),
}
}
The whole change could be seen at: #5958
Note: we only need to extract metadata that represents the entire object, not just a portion of it or the request. For example: the content_length in HTTP always indicates the size of response, so we should't include it. Please review the documentations carefully and include only relevant metadata.
Services
- [x] s3 #5562
- [ ] aliyun_drive
- [ ] alluxio
- [ ] atomicserver
- [x] azblob #5958
- [x] azdls #6368
- [ ] azfile
- [ ] b2
- [ ] comfs
- [x] cos #5777
- [ ] dbfs
- [x] dropbox #6673
- [ ] foundationdb
- [x] fs #5562
- [ ] ftp
- [x] gcs #5933
- [x] gdrive #6683
- [ ] ghac
- [ ] github
- [ ] gridfs
- [ ] hdfs
- [ ] hdfs_native
- [ ] http
- [ ] icloud
- [ ] ipmfs
- [ ] koofr
- [x] lakefs #6770
- [x] monoiofs #5562
- [x] obs #5912
- [x] onedrive #5784
- [x] oss #5688
- [ ] seafile
- [ ] sftp
- [x] swift #6669
- [x] webdav #6660
- [ ] webhdfs
- [ ] yandex_disk
Now that the changes from #5908 are in place, I'll update this issue later.
#5933 isn't a suitable example. I'll implement this feature for azblob as a demonstration instead.
Hi, is this issue still open for PRs? I'd like to work on the FTP feature.
Feel free to go, but I don't know what metadata that FTP can return.
