diff --git a/PLAN.md b/PLAN.md index bc61e68..6fdc7b2 100644 --- a/PLAN.md +++ b/PLAN.md @@ -26,9 +26,9 @@ The program should stay simple: The browser owns file selection and chunk scheduling. -- Let the user pick one file. +- Let the user pick one or more files. - Slice it into fixed-size chunks with `Blob.slice()`. -- Upload a few chunks concurrently. +- Upload a few files concurrently, with a separate chunk pool per file. - Retry failed chunks with exponential backoff. - Persist pending upload state in IndexedDB. - Use the File System Access API when available so the same local file can be @@ -194,11 +194,12 @@ The server should not delete staging data until assembly succeeds. ### First Upload -1. User selects a file. -2. Browser calls `POST /api/uploads`. -3. Browser stores the returned `upload_id` and file handle in IndexedDB. -4. Browser uploads missing chunks with a small concurrency pool. -5. Browser calls `/complete` when all chunks are uploaded. +1. User selects one or more files. +2. Browser creates one selected upload row per file. +3. Browser calls `POST /api/uploads` once for each file being started. +4. Browser stores each returned `upload_id` and file handle in IndexedDB. +5. Browser uploads missing chunks with bounded file and chunk concurrency pools. +6. Browser calls `/complete` for each file when all of its chunks are uploaded. ### After Interruption @@ -239,7 +240,8 @@ Start with these defaults: ```text chunk size: 16 MiB -concurrency: 3 +file concurrency: 3 +chunk concurrency per file: 3 max retries per chunk: 5 ``` diff --git a/README.md b/README.md index 9b1c015..ad9f54b 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ upl Browser UI static/index.html upload tool markup static/styles.css responsive tool styling - static/app.js upload scheduler, retries, and browser resume state + static/app.js multi-file scheduler, retries, and browser resume state Deployment deploy/nginx/ nginx reverse proxy example scripts/ reusable local smoke tests @@ -62,6 +62,13 @@ just run `just check` also syntax-checks the static browser JavaScript with `node`. +## Browser Uploads + +The browser UI accepts multiple selected files. `Start all` runs up to three +file uploads at the same time, and each file still uploads up to three chunks +concurrently. Every selected file keeps its own upload id, progress markers, +abort controller, retry state, and saved IndexedDB resume record. + ## nginx Run `upl` on localhost and put nginx in front of it for TLS and access control: diff --git a/TESTS.md b/TESTS.md index 34d66a3..7fd54ff 100644 --- a/TESTS.md +++ b/TESTS.md @@ -21,6 +21,8 @@ Keep this file as the reusable verification checklist while implementing - `GET /api/uploads/:id` reports completed chunks from disk markers. - `POST /api/uploads/:id/complete` renames the verified temp upload file and removes staging data. + - Parallel upload requests for separate files complete without crossing + bytes between temp upload files. - `POST /api/uploads/:id/complete` rejects incomplete uploads. - `POST /api/uploads/:id/complete` rejects tampered temp upload files. - `static/app.js` passes `node --check`. @@ -38,6 +40,8 @@ deployment retests. - Upload a small file in one pass. - Upload a file larger than one chunk. +- Select multiple files and confirm several upload rows advance at the same + time. - Kill the browser tab mid-upload and resume. - Restart the Rust server mid-upload and resume. - Interrupt the network and resume. diff --git a/static/app.js b/static/app.js index 744d6f3..49d6fd2 100644 --- a/static/app.js +++ b/static/app.js @@ -1,37 +1,33 @@ const DB_NAME = "upl"; const DB_VERSION = 1; const STORE_NAME = "uploads"; -const CONCURRENCY = 3; +const CHUNK_CONCURRENCY = 3; +const FILE_CONCURRENCY = 3; const MAX_RETRIES = 5; const BASE_RETRY_DELAY_MS = 500; const COMPLETE_EXISTS_MESSAGE = "complete file already exists"; const fileInput = document.querySelector("#file-input"); const pickButton = document.querySelector("#pick-button"); -const fileSummary = document.querySelector("#file-summary"); -const fileName = document.querySelector("#file-name"); -const fileSize = document.querySelector("#file-size"); +const uploadSection = document.querySelector("#upload-section"); +const uploadList = document.querySelector("#upload-list"); const startButton = document.querySelector("#start-button"); const pauseButton = document.querySelector("#pause-button"); const resumeButton = document.querySelector("#resume-button"); const eventLog = document.querySelector("#event-log"); -const progressBar = document.querySelector("#progress-bar"); -const progressMeta = document.querySelector("#progress-meta"); const pendingSection = document.querySelector("#pending-section"); const pendingList = document.querySelector("#pending-list"); const state = { - abortController: null, - completedChunks: new Set(), - file: null, - fileHandle: null, pendingRecords: [], - record: null, resumeAfterReselect: null, - running: false, + schedulerAbortController: null, + schedulerRunning: false, + uploadItems: [], }; const dbReady = "indexedDB" in window ? openDatabase() : Promise.resolve(null); +let nextUploadItemId = 1; let saveChain = Promise.resolve(); function formatBytes(bytes) { @@ -101,7 +97,9 @@ async function withStore(mode, callback) { async function loadRecords() { const records = (await withStore("readonly", (store) => store.getAll())) ?? []; - records.sort((left, right) => right.updated_at.localeCompare(left.updated_at)); + records.sort((left, right) => + (right.updated_at ?? "").localeCompare(left.updated_at ?? ""), + ); state.pendingRecords = records; renderPendingRecords(); } @@ -109,7 +107,7 @@ async function loadRecords() { async function saveRecord(record) { const nextSave = saveChain.catch(() => null).then(() => writeRecord(record)); saveChain = nextSave; - await nextSave; + return nextSave; } async function writeRecord(record) { @@ -125,23 +123,31 @@ async function writeRecord(record) { await withStore("readwrite", (store) => store.put(storedRecord)); log("Saved resume state without a reusable file handle."); } - state.record = storedRecord; + await loadRecords(); + return storedRecord; } async function deleteRecord(uploadId) { await withStore("readwrite", (store) => store.delete(uploadId)); - if (state.record?.upload_id === uploadId) { - state.record = null; - } await loadRecords(); } function renderPendingRecords() { pendingList.replaceChildren(); - pendingSection.hidden = state.pendingRecords.length === 0; - for (const record of state.pendingRecords) { + const activeUploadIds = new Set( + state.uploadItems + .map((item) => item.record?.upload_id) + .filter((uploadId) => Boolean(uploadId)), + ); + const visibleRecords = state.pendingRecords.filter( + (record) => !activeUploadIds.has(record.upload_id), + ); + + pendingSection.hidden = visibleRecords.length === 0; + + for (const record of visibleRecords) { const item = document.createElement("li"); item.className = "pending-item"; @@ -158,6 +164,7 @@ function renderPendingRecords() { resume.type = "button"; resume.className = "secondary"; resume.textContent = isReadyToFinish(record) ? "Finish" : "Resume"; + resume.disabled = state.schedulerRunning || !hasAvailableFileSlot(); resume.addEventListener("click", () => { void resumePendingRecord(record); }); @@ -166,6 +173,7 @@ function renderPendingRecords() { remove.type = "button"; remove.className = "danger"; remove.textContent = "Clear"; + remove.disabled = state.schedulerRunning; remove.addEventListener("click", () => { void deleteRecord(record.upload_id); }); @@ -176,31 +184,111 @@ function renderPendingRecords() { } } -function renderFile(file) { - if (!file) { - fileSummary.hidden = true; - fileName.textContent = ""; - fileSize.textContent = ""; - return; - } +function renderUploadItems() { + uploadList.replaceChildren(); + uploadSection.hidden = state.uploadItems.length === 0; - fileName.textContent = file.name; - fileSize.textContent = formatBytes(file.size); - fileSummary.hidden = false; + for (const item of state.uploadItems) { + const row = document.createElement("li"); + row.className = "upload-item"; + + const header = document.createElement("div"); + header.className = "upload-item-header"; + + const meta = document.createElement("div"); + meta.className = "upload-meta"; + + const title = document.createElement("strong"); + title.textContent = item.file.name; + + const detail = document.createElement("span"); + detail.textContent = uploadItemDetail(item); + + meta.append(title, detail); + + const actions = document.createElement("div"); + actions.className = "upload-item-actions"; + + const start = document.createElement("button"); + start.type = "button"; + start.textContent = uploadActionLabel(item); + start.disabled = + !canRunItem(item) || state.schedulerRunning || !hasAvailableFileSlot(); + start.addEventListener("click", () => { + void runUploadItem(item); + }); + + const pause = document.createElement("button"); + pause.type = "button"; + pause.className = "secondary"; + pause.textContent = "Pause"; + pause.disabled = !item.running; + pause.addEventListener("click", () => { + item.abortController?.abort(); + }); + + const remove = document.createElement("button"); + remove.type = "button"; + remove.className = "secondary"; + remove.textContent = "Remove"; + remove.disabled = item.running || item.queued; + remove.addEventListener("click", () => { + removeUploadItem(item); + }); + + actions.append(start, pause, remove); + header.append(meta, actions); + + const progress = document.createElement("div"); + progress.className = "upload-progress"; + + const progressWrap = document.createElement("div"); + progressWrap.className = "progress-wrap"; + progressWrap.setAttribute("aria-label", `${item.file.name} upload progress`); + + const progressBar = document.createElement("div"); + progressBar.className = "progress-bar"; + progressBar.style.width = `${progressPercentage( + item.completedCount, + item.totalChunks, + )}%`; + + const progressMeta = document.createElement("div"); + progressMeta.className = "progress-meta"; + progressMeta.textContent = `${item.completedCount} of ${item.totalChunks} chunks`; + + progressWrap.append(progressBar); + progress.append(progressWrap, progressMeta); + row.append(header, progress); + uploadList.append(row); + } } function renderButtons() { - startButton.disabled = !state.file || state.running || Boolean(state.record); - pauseButton.disabled = !state.running; - resumeButton.disabled = !state.file || state.running || !state.record; - resumeButton.textContent = - state.record && isReadyToFinish(state.record) ? "Finish" : "Resume"; + const hasRunnable = state.uploadItems.some((item) => canRunItem(item)); + const hasRunnableResume = state.uploadItems.some( + (item) => item.record && canRunItem(item), + ); + const hasRunningOrQueued = state.uploadItems.some((item) => item.running || item.queued); + const hasFileSlot = hasAvailableFileSlot(); + + startButton.disabled = !hasRunnable || state.schedulerRunning || !hasFileSlot; + pauseButton.disabled = !hasRunningOrQueued; + resumeButton.disabled = !hasRunnableResume || state.schedulerRunning || !hasFileSlot; } -function updateProgress(completedCount, totalChunks) { - const percentage = totalChunks === 0 ? 0 : (completedCount / totalChunks) * 100; - progressBar.style.width = `${percentage}%`; - progressMeta.textContent = `${completedCount} of ${totalChunks} chunks`; +function renderAll() { + renderUploadItems(); + renderPendingRecords(); + renderButtons(); +} + +function progressPercentage(completedCount, totalChunks) { + if (totalChunks <= 0) { + return 0; + } + + return Math.min(100, Math.max(0, (completedCount / totalChunks) * 100)); } function sameFile(record, file) { @@ -211,10 +299,34 @@ function sameFile(record, file) { ); } +function sameUploadItemFile(item, file) { + return ( + item.file.name === file.name && + item.file.size === file.size && + item.file.lastModified === file.lastModified + ); +} + function findPendingRecord(file) { return state.pendingRecords.find((record) => sameFile(record, file)) ?? null; } +function findUploadItem(file, record = null) { + return ( + state.uploadItems.find((item) => { + if (item.finished || item.terminal) { + return false; + } + + if (record?.upload_id && item.record?.upload_id === record.upload_id) { + return true; + } + + return sameUploadItemFile(item, file); + }) ?? null + ); +} + function completedChunkCount(record) { return Math.min(record.completed_chunks ?? 0, record.total_chunks ?? 0); } @@ -224,6 +336,13 @@ function isReadyToFinish(record) { return totalChunks === 0 || completedChunkCount(record) >= totalChunks; } +function isUploadItemReadyToFinish(item) { + return ( + Boolean(item.record) && + (item.totalChunks === 0 || item.completedCount >= item.totalChunks) + ); +} + function savedUploadDetail(record) { const totalChunks = record.total_chunks ?? 0; const completedChunks = completedChunkCount(record); @@ -239,50 +358,149 @@ function savedUploadDetail(record) { return `${formatBytes(record.size)} - ${completedChunks} of ${totalChunks} chunks uploaded`; } -function clearSelection({ resetProgress = true } = {}) { - state.file = null; - state.fileHandle = null; - state.record = null; - state.completedChunks = new Set(); - state.resumeAfterReselect = null; - fileInput.value = ""; - renderFile(null); - if (resetProgress) { - updateProgress(0, 0); - } - renderButtons(); +function uploadItemDetail(item) { + return `${formatBytes(item.file.size)} - ${item.statusText}`; } -async function selectFile(file, fileHandle = null, record = null) { - if (record && !sameFile(record, file)) { - log("Selected file does not match the pending upload."); +function uploadActionLabel(item) { + if (item.finished) { + return "Done"; + } + + if (!item.record) { + return "Start"; + } + + return isUploadItemReadyToFinish(item) ? "Finish" : "Resume"; +} + +function initialUploadStatus(record) { + if (!record) { + return "Ready to create an upload record."; + } + + if (isReadyToFinish(record)) { + return "Ready to finish saved upload."; + } + + return "Ready to resume upload."; +} + +function canRunItem(item) { + return ( + Boolean(item.file) && + !item.running && + !item.queued && + !item.finished && + !item.terminal + ); +} + +function runningFileCount() { + return state.uploadItems.filter((item) => item.running).length; +} + +function hasAvailableFileSlot() { + return runningFileCount() < FILE_CONCURRENCY; +} + +function setItemProgress(item, completedCount, totalChunks) { + item.totalChunks = Math.max(0, totalChunks); + item.completedCount = + item.totalChunks === 0 + ? 0 + : Math.min(Math.max(0, completedCount), item.totalChunks); +} + +async function selectFiles(files, fileHandles = []) { + const selectedFiles = Array.from(files); + if (selectedFiles.length === 0) { + log("Choose files to begin."); + renderButtons(); return; } - state.file = file; - state.fileHandle = fileHandle; - state.record = record ?? findPendingRecord(file); - state.completedChunks = new Set(); + const resumeRecord = state.resumeAfterReselect; + let matchedResumeRecord = false; + let addedCount = 0; - renderFile(file); - updateProgress(state.record?.completed_chunks ?? 0, state.record?.total_chunks ?? 0); - renderButtons(); + for (const [index, file] of selectedFiles.entries()) { + let record = findPendingRecord(file); + if (resumeRecord && sameFile(resumeRecord, file)) { + record = resumeRecord; + matchedResumeRecord = true; + } - if (!state.record) { - log("Ready to create an upload record."); - } else if (isReadyToFinish(state.record)) { - log("Ready to finish saved upload."); - } else { - log("Ready to resume upload."); + const previousCount = state.uploadItems.length; + addUploadItem(file, fileHandles[index] ?? null, record); + if (state.uploadItems.length > previousCount) { + addedCount += 1; + } } + + if (resumeRecord && !matchedResumeRecord) { + log("Selected files did not include the pending upload file."); + } + + state.resumeAfterReselect = null; + fileInput.value = ""; + renderAll(); + + if (addedCount === 1) { + log("Ready to upload 1 file."); + } else if (addedCount > 1) { + log(`Ready to upload ${addedCount} files.`); + } +} + +function addUploadItem(file, fileHandle = null, record = null) { + if (record && !sameFile(record, file)) { + log("Selected file does not match the pending upload."); + return null; + } + + const existingItem = findUploadItem(file, record); + if (existingItem) { + log(`${file.name} is already selected.`); + return existingItem; + } + + const item = { + abortController: null, + completedChunks: new Set(), + completedCount: record ? completedChunkCount(record) : 0, + file, + fileHandle, + finished: false, + id: nextUploadItemId, + queued: false, + record, + running: false, + statusText: initialUploadStatus(record), + terminal: false, + totalChunks: record?.total_chunks ?? 0, + }; + + nextUploadItemId += 1; + state.uploadItems.push(item); + return item; +} + +function removeUploadItem(item) { + if (item.running || item.queued) { + return; + } + + state.uploadItems = state.uploadItems.filter((candidate) => candidate.id !== item.id); + renderAll(); } async function pickFile() { if ("showOpenFilePicker" in window) { try { - const [handle] = await window.showOpenFilePicker({ multiple: false }); - const file = await handle.getFile(); - await selectFile(file, handle); + const handles = await window.showOpenFilePicker({ multiple: true }); + const files = await Promise.all(handles.map((handle) => handle.getFile())); + await selectFiles(files, handles); return; } catch (error) { if (isAbortError(error)) { @@ -296,18 +514,7 @@ async function pickFile() { } fileInput.addEventListener("change", () => { - const [file] = fileInput.files; - - if (!file) { - renderFile(null); - renderButtons(); - log("Choose a file to begin."); - return; - } - - const record = state.resumeAfterReselect ?? findPendingRecord(file); - state.resumeAfterReselect = null; - void selectFile(file, null, record); + void selectFiles(fileInput.files); }); pickButton.addEventListener("click", () => { @@ -315,21 +522,19 @@ pickButton.addEventListener("click", () => { }); startButton.addEventListener("click", () => { - void runUpload(); + void runUploadItems(state.uploadItems); }); pauseButton.addEventListener("click", () => { - if (state.abortController) { - state.abortController.abort(); - } + pauseUploads(); }); resumeButton.addEventListener("click", () => { - void runUpload(); + void runUploadItems(state.uploadItems.filter((item) => item.record)); }); async function resumePendingRecord(record) { - if (state.running) { + if (state.schedulerRunning) { return; } @@ -337,8 +542,11 @@ async function resumePendingRecord(record) { const granted = await requestFileHandlePermission(record.file_handle); if (granted) { const file = await record.file_handle.getFile(); - await selectFile(file, record.file_handle, record); - await runUpload(); + const item = addUploadItem(file, record.file_handle, record); + renderAll(); + if (item) { + await runUploadItem(item); + } return; } } @@ -365,127 +573,224 @@ async function requestFileHandlePermission(handle) { return (await handle.requestPermission(options)) === "granted"; } -async function runUpload() { - if (!state.file || state.running) { +async function runUploadItems(items) { + if (state.schedulerRunning) { + return; + } + + const runnableItems = items.filter((item) => canRunItem(item)); + const availableSlots = FILE_CONCURRENCY - runningFileCount(); + if (runnableItems.length === 0 || availableSlots <= 0) { return; } const controller = new AbortController(); - state.abortController = controller; - state.running = true; - renderButtons(); + state.schedulerAbortController = controller; + state.schedulerRunning = true; + + for (const item of runnableItems) { + item.queued = true; + item.statusText = "Queued."; + } + renderAll(); try { - if (!state.record) { - await createUploadRecord(); + await runPool( + runnableItems, + async (item) => { + throwIfAborted(controller.signal); + if (!item.queued) { + return; + } + await runUploadItem(item); + }, + availableSlots, + controller.signal, + ); + } catch (error) { + if (!isAbortError(error)) { + log(`Upload scheduler failed: ${error.message}`); + } + } finally { + for (const item of runnableItems) { + if (item.queued) { + item.queued = false; + item.statusText = "Paused."; + } } - const progress = await fetchJson(`/api/uploads/${state.record.upload_id}`, { + if (state.schedulerAbortController === controller) { + state.schedulerAbortController = null; + } + state.schedulerRunning = false; + renderAll(); + } +} + +function pauseUploads() { + state.schedulerAbortController?.abort(); + + for (const item of state.uploadItems) { + if (item.running) { + item.abortController?.abort(); + } else if (item.queued) { + item.queued = false; + item.statusText = "Paused."; + } + } + + renderAll(); +} + +async function runUploadItem(item) { + if ( + item.running || + item.finished || + item.terminal || + (!item.queued && !hasAvailableFileSlot()) || + (!item.queued && !canRunItem(item)) + ) { + return; + } + + const controller = new AbortController(); + item.abortController = controller; + item.queued = false; + item.running = true; + item.statusText = item.record + ? "Checking saved progress." + : "Creating upload record."; + renderAll(); + + try { + if (!item.record) { + await createUploadRecord(item, controller.signal); + } + + const progress = await fetchJson(`/api/uploads/${item.record.upload_id}`, { signal: controller.signal, }); - state.completedChunks = new Set(progress.completed_chunks); - await saveRecord({ - ...state.record, - completed_chunks: state.completedChunks.size, + item.completedChunks = new Set(progress.completed_chunks); + setItemProgress(item, item.completedChunks.size, progress.total_chunks); + item.record = await saveRecord({ + ...item.record, + completed_chunks: item.completedCount, chunk_size: progress.chunk_size, total_chunks: progress.total_chunks, }); - updateProgress(state.completedChunks.size, progress.total_chunks); + renderAll(); - const missingChunks = buildMissingChunkList(progress.total_chunks, state.completedChunks); - log( + const missingChunks = buildMissingChunkList(progress.total_chunks, item.completedChunks); + item.statusText = missingChunks.length === 0 ? "All chunks already uploaded." - : `Uploading ${missingChunks.length} missing chunks.`, - ); + : `Uploading ${missingChunks.length} missing chunks.`; + renderAll(); - await runPool(missingChunks, (index) => - uploadChunkWithRetry(index, progress.chunk_size, progress.total_chunks, controller.signal), + await runPool( + missingChunks, + (index) => + uploadChunkWithRetry( + item, + index, + progress.chunk_size, + progress.total_chunks, + controller.signal, + ), + CHUNK_CONCURRENCY, + controller.signal, ); if (controller.signal.aborted) { return; } - log("Completing upload."); - const uploadId = state.record.upload_id; + item.statusText = "Completing upload."; + renderAll(); + const uploadId = item.record.upload_id; const complete = await fetchJson(`/api/uploads/${uploadId}/complete`, { method: "POST", signal: controller.signal, }); - updateProgress(progress.total_chunks, progress.total_chunks); - log(`Complete: ${complete.file_path}`); + setItemProgress(item, progress.total_chunks, progress.total_chunks); + item.finished = true; + item.statusText = `Complete: ${complete.file_path}`; await deleteRecord(uploadId); - clearSelection({ resetProgress: false }); } catch (error) { if (controller.signal.aborted || isAbortError(error)) { - log("Upload paused."); - } else if (await handleTerminalUploadError(error)) { + item.statusText = "Paused."; + } else if (await handleTerminalUploadError(item, error)) { controller.abort(); } else { controller.abort(); - log(`Upload failed: ${error.message}`); + item.statusText = `Upload failed: ${error.message}`; } } finally { - if (state.abortController === controller) { - state.abortController = null; + if (item.abortController === controller) { + item.abortController = null; } - state.running = false; - renderButtons(); + item.running = false; + renderAll(); } } -async function handleTerminalUploadError(error) { - if (!state.record || typeof error.status !== "number") { +async function handleTerminalUploadError(item, error) { + if (!item.record || typeof error.status !== "number") { return false; } - const uploadId = state.record.upload_id; + const uploadId = item.record.upload_id; if (error.status === 404) { await deleteRecord(uploadId); - clearSelection(); - log("Saved upload progress no longer exists on the server."); + item.record = null; + item.completedChunks = new Set(); + setItemProgress(item, 0, 0); + item.statusText = "Saved upload progress no longer exists. Start again."; + log(`${item.file.name}: saved upload progress no longer exists on the server.`); return true; } if (error.status === 409 && error.message === COMPLETE_EXISTS_MESSAGE) { await deleteRecord(uploadId); - clearSelection(); - log("The completed file already exists on the server. Cleared saved upload progress."); + item.terminal = true; + item.statusText = "Complete file already exists on the server."; + log(`${item.file.name}: complete file already exists on the server.`); return true; } return false; } -async function createUploadRecord() { +async function createUploadRecord(item, signal) { const response = await fetchJson("/api/uploads", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ - name: state.file.name, - size: state.file.size, - last_modified: state.file.lastModified, + name: item.file.name, + size: item.file.size, + last_modified: item.file.lastModified, }), + signal, }); const record = { upload_id: response.upload_id, - name: state.file.name, - size: state.file.size, - last_modified: state.file.lastModified, + name: item.file.name, + size: item.file.size, + last_modified: item.file.lastModified, chunk_size: response.chunk_size, total_chunks: response.total_chunks, completed_chunks: 0, - file_handle: state.fileHandle, + file_handle: item.fileHandle, updated_at: new Date().toISOString(), }; - await saveRecord(record); - updateProgress(0, response.total_chunks); - log("Upload record created."); + item.record = await saveRecord(record); + setItemProgress(item, 0, response.total_chunks); + item.statusText = "Upload record created."; + renderAll(); } function buildMissingChunkList(totalChunks, completedChunks) { @@ -498,18 +803,19 @@ function buildMissingChunkList(totalChunks, completedChunks) { return missing; } -async function uploadChunkWithRetry(index, chunkSize, totalChunks, signal) { +async function uploadChunkWithRetry(item, index, chunkSize, totalChunks, signal) { for (let attempt = 0; attempt <= MAX_RETRIES; attempt += 1) { throwIfAborted(signal); try { - await uploadChunk(index, chunkSize, signal); - state.completedChunks.add(index); - updateProgress(state.completedChunks.size, totalChunks); - await saveRecord({ - ...state.record, - completed_chunks: state.completedChunks.size, + await uploadChunk(item, index, chunkSize, signal); + item.completedChunks.add(index); + setItemProgress(item, item.completedChunks.size, totalChunks); + item.record = await saveRecord({ + ...item.record, + completed_chunks: item.completedCount, }); + renderAll(); return; } catch (error) { if (isAbortError(error) || attempt === MAX_RETRIES) { @@ -517,34 +823,42 @@ async function uploadChunkWithRetry(index, chunkSize, totalChunks, signal) { } const delayMs = BASE_RETRY_DELAY_MS * 2 ** attempt; - log(`Retrying chunk ${index} after ${delayMs} ms.`); + item.statusText = `Retrying chunk ${index} after ${delayMs} ms.`; + renderAll(); await delay(delayMs, signal); } } } -async function uploadChunk(index, chunkSize, signal) { +async function uploadChunk(item, index, chunkSize, signal) { const start = index * chunkSize; - const end = Math.min(state.file.size, start + chunkSize); - const body = state.file.slice(start, end); - const response = await fetch(`/api/uploads/${state.record.upload_id}/chunks/${index}`, { - method: "PUT", - headers: { "Content-Type": "application/octet-stream" }, - body, - signal, - }); + const end = Math.min(item.file.size, start + chunkSize); + const body = item.file.slice(start, end); + const response = await fetch( + `/api/uploads/${item.record.upload_id}/chunks/${index}`, + { + method: "PUT", + headers: { "Content-Type": "application/octet-stream" }, + body, + signal, + }, + ); if (!response.ok) { - throw new Error(await responseError(response)); + throw new ApiRequestError(await responseError(response), response.status); } } -async function runPool(items, worker) { +async function runPool(items, worker, concurrency, signal = null) { let nextIndex = 0; const workers = Array.from( - { length: Math.min(CONCURRENCY, items.length) }, + { length: Math.min(concurrency, items.length) }, async () => { while (nextIndex < items.length) { + if (signal) { + throwIfAborted(signal); + } + const item = items[nextIndex]; nextIndex += 1; await worker(item); @@ -612,7 +926,7 @@ function isAbortError(error) { async function initialize() { await loadRecords(); - renderButtons(); + renderAll(); } void initialize(); diff --git a/static/index.html b/static/index.html index 309f6ed..4ee8a70 100644 --- a/static/index.html +++ b/static/index.html @@ -18,33 +18,28 @@
- - + +
- - -
-
-
-
0 of 0 chunks
-
- - - + + +
+ +
    -
  1. Choose a file to begin.
    2. +
  2. Choose files to begin.
diff --git a/static/styles.css b/static/styles.css index 308646f..8abe800 100644 --- a/static/styles.css +++ b/static/styles.css @@ -88,16 +88,48 @@ h1 { clip: rect(0, 0, 0, 0); } -.file-summary { +.upload-section { display: grid; - gap: 4px; - padding: 14px; + gap: 12px; +} + +.upload-list { + display: grid; + gap: 12px; + margin: 0; + padding: 0; + list-style: none; +} + +.upload-item { + display: grid; + gap: 12px; + padding: 12px; border: 1px solid var(--line); border-radius: 8px; } -.file-summary span { +.upload-item-header { + display: grid; + grid-template-columns: minmax(0, 1fr) auto; + align-items: start; + gap: 12px; +} + +.upload-meta { + display: grid; + gap: 4px; + min-width: 0; +} + +.upload-meta span { color: var(--muted); + font-size: 0.875rem; +} + +.upload-progress { + display: grid; + gap: 6px; } .progress-wrap { @@ -115,7 +147,6 @@ h1 { } .progress-meta { - margin-top: -12px; color: var(--muted); font-size: 0.875rem; } @@ -126,6 +157,13 @@ h1 { gap: 10px; } +.upload-item-actions { + display: flex; + flex-wrap: wrap; + justify-content: flex-end; + gap: 8px; +} + button { min-width: 96px; min-height: 40px; @@ -184,7 +222,7 @@ h2 { } .pending-item strong, -.file-summary strong { +.upload-meta strong { overflow-wrap: anywhere; } @@ -232,4 +270,12 @@ h2 { .pending-item { grid-template-columns: 1fr; } + + .upload-item-header { + grid-template-columns: 1fr; + } + + .upload-item-actions { + justify-content: flex-start; + } } diff --git a/tests/completion.rs b/tests/completion.rs index 88e80b4..e6c96ed 100644 --- a/tests/completion.rs +++ b/tests/completion.rs @@ -66,6 +66,80 @@ async fn assembles_completed_upload() -> Result<(), Box> Ok(()) } +#[tokio::test] +async fn parallel_uploads_keep_bytes_separate() -> Result<(), Box> { + let temp_dir = TempDir::new()?; + let app = test_app(temp_dir.path()); + let chunk_size = usize::try_from(CHUNK_SIZE)?; + let left_upload = create_upload(&app, "left.bin", CHUNK_SIZE + 4).await?; + let right_upload = create_upload(&app, "right.bin", CHUNK_SIZE + 5).await?; + + let mut expected_left = vec![b'l'; chunk_size]; + expected_left.extend_from_slice(b"eft!"); + let mut expected_right = vec![b'r'; chunk_size]; + expected_right.extend_from_slice(b"ight!"); + + let left_first = chunk_request( + &left_upload.upload_id, + 0, + expected_left[..chunk_size].to_vec(), + )?; + let left_final = chunk_request( + &left_upload.upload_id, + 1, + expected_left[chunk_size..].to_vec(), + )?; + let right_first = chunk_request( + &right_upload.upload_id, + 0, + expected_right[..chunk_size].to_vec(), + )?; + let right_final = chunk_request( + &right_upload.upload_id, + 1, + expected_right[chunk_size..].to_vec(), + )?; + + let (left_first, right_first, left_final, right_final) = tokio::join!( + app.clone().oneshot(left_first), + app.clone().oneshot(right_first), + app.clone().oneshot(left_final), + app.clone().oneshot(right_final), + ); + + assert_eq!(left_first?.status(), StatusCode::NO_CONTENT); + assert_eq!(right_first?.status(), StatusCode::NO_CONTENT); + assert_eq!(left_final?.status(), StatusCode::NO_CONTENT); + assert_eq!(right_final?.status(), StatusCode::NO_CONTENT); + + let left_complete = empty_request( + Method::POST, + &format!("/api/uploads/{}/complete", left_upload.upload_id), + )?; + let right_complete = empty_request( + Method::POST, + &format!("/api/uploads/{}/complete", right_upload.upload_id), + )?; + + let (left_complete, right_complete) = tokio::join!( + app.clone().oneshot(left_complete), + app.clone().oneshot(right_complete), + ); + + assert_eq!(left_complete?.status(), StatusCode::OK); + assert_eq!(right_complete?.status(), StatusCode::OK); + assert_eq!( + tokio::fs::read(temp_dir.path().join("complete").join("left.bin")).await?, + expected_left + ); + assert_eq!( + tokio::fs::read(temp_dir.path().join("complete").join("right.bin")).await?, + expected_right + ); + + Ok(()) +} + #[tokio::test] async fn rejects_incomplete_upload() -> Result<(), Box> { let temp_dir = TempDir::new()?; diff --git a/tests/static_server.rs b/tests/static_server.rs index 11dfd84..3e1445a 100644 --- a/tests/static_server.rs +++ b/tests/static_server.rs @@ -22,7 +22,8 @@ async fn serves_index_page() -> Result<(), Box> { let body = String::from_utf8(body.to_vec())?; assert!(body.contains("upl")); - assert!(body.contains("Choose file")); + assert!(body.contains("Choose files")); + assert!(body.contains("Selected uploads")); assert!(body.contains("Saved upload progress")); assert!(!body.contains("Server online"));