feat: write chunks directly to temp upload files
Completed uploads used to copy every staged chunk into a second file before renaming the result into data/complete. That doubled write volume and required peak disk space for both the chunk set and the final file. Write each chunk directly into one private temp upload file at its final offset instead. After a chunk write succeeds, record a tiny durable completion marker for progress and resume scans. Completion now verifies the temp file length and all markers, then renames the temp file into the completed upload directory. Add UPL_TEMP_DIR and --temp-dir so operators can choose where upload metadata, markers, and temp files live. The default remains data/staging, and docs call out that the temp directory must be on the same filesystem as data/complete for atomic promotion. The nginx example now aliases only the completed upload directory, and the smoke test verifies that final-file alias. This keeps the existing length-based validation model; it does not add per-chunk hashing. Test Plan: - just check - just nginx-smoke - cargo clippy && cargo clippy --benches && cargo clippy --tests - cargo +nightly fmt --all - cargo clippy && cargo clippy --benches && cargo clippy --tests Refs: none
This commit is contained in:
+11
-9
@@ -42,13 +42,16 @@ async fn stores_chunks_and_reports_progress() -> Result<(), Box<dyn std::error::
|
||||
let progress = get_progress(&app, &upload.upload_id).await?;
|
||||
assert_eq!(progress.completed_chunks, vec![0, 1]);
|
||||
|
||||
let chunk_path = temp_dir
|
||||
.path()
|
||||
.join("staging")
|
||||
.join(&upload.upload_id)
|
||||
.join("chunks")
|
||||
.join("000000.part");
|
||||
assert_eq!(tokio::fs::metadata(chunk_path).await?.len(), CHUNK_SIZE);
|
||||
let upload_dir = temp_dir.path().join("staging").join(&upload.upload_id);
|
||||
assert_eq!(
|
||||
tokio::fs::metadata(upload_dir.join(".upload.tmp"))
|
||||
.await?
|
||||
.len(),
|
||||
CHUNK_SIZE + 3
|
||||
);
|
||||
assert!(upload_dir.join("completed").join("000000.done").is_file());
|
||||
assert!(upload_dir.join("completed").join("000001.done").is_file());
|
||||
assert!(!upload_dir.join("chunks").exists());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -84,8 +87,7 @@ async fn rejects_out_of_range_chunk_index() -> Result<(), Box<dyn std::error::Er
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accepts_duplicate_chunk_when_existing_length_matches()
|
||||
-> Result<(), Box<dyn std::error::Error>> {
|
||||
async fn accepts_duplicate_completed_chunk() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let temp_dir = TempDir::new()?;
|
||||
let app = test_app(temp_dir.path());
|
||||
let upload = create_upload(&app, temp_dir.path(), 4).await?;
|
||||
|
||||
+4
-8
@@ -98,18 +98,14 @@ async fn rejects_incomplete_upload() -> Result<(), Box<dyn std::error::Error>> {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejects_corrupt_chunk_file() -> Result<(), Box<dyn std::error::Error>> {
|
||||
async fn rejects_tampered_temp_upload_file() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let temp_dir = TempDir::new()?;
|
||||
let app = test_app(temp_dir.path());
|
||||
let upload = create_upload(&app, "corrupt.bin", 4).await?;
|
||||
|
||||
let chunk_path = temp_dir
|
||||
.path()
|
||||
.join("staging")
|
||||
.join(&upload.upload_id)
|
||||
.join("chunks")
|
||||
.join("000000.part");
|
||||
tokio::fs::write(chunk_path, b"bad").await?;
|
||||
let upload_dir = temp_dir.path().join("staging").join(&upload.upload_id);
|
||||
tokio::fs::write(upload_dir.join(".upload.tmp"), b"bad").await?;
|
||||
tokio::fs::write(upload_dir.join("completed").join("000000.done"), b"").await?;
|
||||
|
||||
let response = app
|
||||
.oneshot(empty_request(
|
||||
|
||||
@@ -43,7 +43,8 @@ async fn creates_upload_metadata_on_disk() -> Result<(), Box<dyn std::error::Err
|
||||
|
||||
let upload_dir = temp_dir.path().join("staging").join(&response.upload_id);
|
||||
let meta_path = upload_dir.join("meta.json");
|
||||
assert!(upload_dir.join("chunks").is_dir());
|
||||
assert!(upload_dir.join(".upload.tmp").is_file());
|
||||
assert!(upload_dir.join("completed").is_dir());
|
||||
assert!(temp_dir.path().join("complete").is_dir());
|
||||
|
||||
let meta: UploadMeta = serde_json::from_slice(&tokio::fs::read(meta_path).await?)?;
|
||||
|
||||
Reference in New Issue
Block a user