feat: store raw upload chunks
Add the chunk upload and progress APIs from PLAN.md. PUT
/api/uploads/{id}/chunks/{index} now accepts raw octet-stream bodies, validates
unknown upload ids, out-of-range chunk indexes, and exact chunk lengths, then
writes through a temporary .part.tmp path before renaming the completed chunk
into place. Re-uploading an already-complete chunk is idempotent when the
existing file length matches the expected length.
GET /api/uploads/{id} now reports server-authoritative progress by scanning the
chunk directory and only counting chunk files whose lengths match metadata. The
router also raises Axum's request body limit to 64 MiB so the planned 16 MiB
chunks can reach the handler, matching the nginx deployment guidance.
Document the chunk storage responsibility and extend the reusable test checklist
with the new progress and validation coverage.
Test Plan:
- just check
Refs: PLAN.md milestones 3 and 4
This commit is contained in:
@@ -0,0 +1,187 @@
|
||||
use std::{
|
||||
net::{Ipv4Addr, SocketAddr},
|
||||
path::Path,
|
||||
};
|
||||
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{Method, Request, StatusCode, header},
|
||||
};
|
||||
use http_body_util::BodyExt;
|
||||
use serde_json::json;
|
||||
use tempfile::TempDir;
|
||||
use tower::ServiceExt;
|
||||
use upl::{
|
||||
app::{AppConfig, build_router},
|
||||
model::{CHUNK_SIZE, CreateUploadResponse, UploadProgressResponse},
|
||||
};
|
||||
|
||||
#[tokio::test]
|
||||
async fn stores_chunks_and_reports_progress() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let temp_dir = TempDir::new()?;
|
||||
let app = test_app(temp_dir.path());
|
||||
let upload = create_upload(&app, temp_dir.path(), CHUNK_SIZE + 3).await?;
|
||||
|
||||
let final_chunk = vec![b'z'; 3];
|
||||
let response = app
|
||||
.clone()
|
||||
.oneshot(chunk_request(&upload.upload_id, 1, final_chunk)?)
|
||||
.await?;
|
||||
assert_eq!(response.status(), StatusCode::NO_CONTENT);
|
||||
|
||||
let progress = get_progress(&app, &upload.upload_id).await?;
|
||||
assert_eq!(progress.completed_chunks, vec![1]);
|
||||
|
||||
let first_chunk = vec![b'a'; usize::try_from(CHUNK_SIZE)?];
|
||||
let response = app
|
||||
.clone()
|
||||
.oneshot(chunk_request(&upload.upload_id, 0, first_chunk)?)
|
||||
.await?;
|
||||
assert_eq!(response.status(), StatusCode::NO_CONTENT);
|
||||
|
||||
let progress = get_progress(&app, &upload.upload_id).await?;
|
||||
assert_eq!(progress.completed_chunks, vec![0, 1]);
|
||||
|
||||
let chunk_path = temp_dir
|
||||
.path()
|
||||
.join("staging")
|
||||
.join(&upload.upload_id)
|
||||
.join("chunks")
|
||||
.join("000000.part");
|
||||
assert_eq!(tokio::fs::metadata(chunk_path).await?.len(), CHUNK_SIZE);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejects_wrong_size_non_final_chunk() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let temp_dir = TempDir::new()?;
|
||||
let app = test_app(temp_dir.path());
|
||||
let upload = create_upload(&app, temp_dir.path(), CHUNK_SIZE + 1).await?;
|
||||
|
||||
let response = app
|
||||
.oneshot(chunk_request(&upload.upload_id, 0, b"too short".to_vec())?)
|
||||
.await?;
|
||||
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accepts_duplicate_chunk_when_existing_length_matches()
|
||||
-> Result<(), Box<dyn std::error::Error>> {
|
||||
let temp_dir = TempDir::new()?;
|
||||
let app = test_app(temp_dir.path());
|
||||
let upload = create_upload(&app, temp_dir.path(), 4).await?;
|
||||
|
||||
let first = app
|
||||
.clone()
|
||||
.oneshot(chunk_request(&upload.upload_id, 0, b"data".to_vec())?)
|
||||
.await?;
|
||||
let second = app
|
||||
.oneshot(chunk_request(&upload.upload_id, 0, b"data".to_vec())?)
|
||||
.await?;
|
||||
|
||||
assert_eq!(first.status(), StatusCode::NO_CONTENT);
|
||||
assert_eq!(second.status(), StatusCode::NO_CONTENT);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejects_unknown_upload_id() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let temp_dir = TempDir::new()?;
|
||||
let app = test_app(temp_dir.path());
|
||||
|
||||
let response = app
|
||||
.oneshot(chunk_request("missing", 0, b"data".to_vec())?)
|
||||
.await?;
|
||||
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn create_upload(
|
||||
app: &axum::Router,
|
||||
data_dir: &Path,
|
||||
size: u64,
|
||||
) -> Result<CreateUploadResponse, Box<dyn std::error::Error>> {
|
||||
let response = app
|
||||
.clone()
|
||||
.oneshot(json_request(
|
||||
Method::POST,
|
||||
"/api/uploads",
|
||||
&json!({
|
||||
"name": "chunked.bin",
|
||||
"size": size,
|
||||
"last_modified": 1_760_000_000_000_i64
|
||||
}),
|
||||
)?)
|
||||
.await?;
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert!(data_dir.join("staging").is_dir());
|
||||
|
||||
decode_body(response).await
|
||||
}
|
||||
|
||||
async fn get_progress(
|
||||
app: &axum::Router,
|
||||
upload_id: &str,
|
||||
) -> Result<UploadProgressResponse, Box<dyn std::error::Error>> {
|
||||
let response = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method(Method::GET)
|
||||
.uri(format!("/api/uploads/{upload_id}"))
|
||||
.body(Body::empty())?,
|
||||
)
|
||||
.await?;
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
|
||||
decode_body(response).await
|
||||
}
|
||||
|
||||
fn test_app(data_dir: &Path) -> axum::Router {
|
||||
build_router(&AppConfig::new(
|
||||
SocketAddr::from((Ipv4Addr::LOCALHOST, 0)),
|
||||
concat!(env!("CARGO_MANIFEST_DIR"), "/static"),
|
||||
data_dir,
|
||||
))
|
||||
}
|
||||
|
||||
fn json_request(
|
||||
method: Method,
|
||||
uri: &str,
|
||||
body: &serde_json::Value,
|
||||
) -> Result<Request<Body>, Box<dyn std::error::Error>> {
|
||||
Ok(Request::builder()
|
||||
.method(method)
|
||||
.uri(uri)
|
||||
.header(header::CONTENT_TYPE, "application/json")
|
||||
.body(Body::from(serde_json::to_vec(body)?))?)
|
||||
}
|
||||
|
||||
fn chunk_request(
|
||||
upload_id: &str,
|
||||
index: u64,
|
||||
body: Vec<u8>,
|
||||
) -> Result<Request<Body>, Box<dyn std::error::Error>> {
|
||||
Ok(Request::builder()
|
||||
.method(Method::PUT)
|
||||
.uri(format!("/api/uploads/{upload_id}/chunks/{index}"))
|
||||
.header(header::CONTENT_TYPE, "application/octet-stream")
|
||||
.body(Body::from(body))?)
|
||||
}
|
||||
|
||||
async fn decode_body<T>(response: axum::response::Response) -> Result<T, Box<dyn std::error::Error>>
|
||||
where
|
||||
T: serde::de::DeserializeOwned,
|
||||
{
|
||||
let body = response.into_body().collect().await?.to_bytes();
|
||||
Ok(serde_json::from_slice(&body)?)
|
||||
}
|
||||
Reference in New Issue
Block a user