From 35cd0657bd19e59d79afd63a2a0ea88865605a11 Mon Sep 17 00:00:00 2001 From: ddidderr Date: Sat, 30 May 2026 17:08:02 +0200 Subject: [PATCH] feat: add browser resumable upload client Replace the placeholder browser script with the PLAN.md upload flow. The static UI now creates upload records, slices the selected file into fixed-size chunks, uploads missing chunks with a concurrency pool of three workers, retries failed chunks with exponential backoff, pauses via AbortController, and completes the upload once the server has accepted every chunk. Persist pending upload records in IndexedDB and render them in the page so a reload can resume from server-authoritative progress. When the File System Access API is available, the app stores a file handle and asks for read permission during resume; when it is unavailable or permission is denied, the same pending record resumes after the user reselects the matching file. Browser state is helpful but not trusted: every resume starts by querying the server for completed chunks. Add a JavaScript syntax check to the justfile, update the static-page test and documentation, and extend TESTS.md with the manual resume scenarios that still need real-browser repetition. Test Plan: - just check - UPL_BIND=127.0.0.1:39123 UPL_DATA_DIR=$(mktemp -d) cargo run - curl -fsS http://127.0.0.1:39123/healthz - curl -fsS http://127.0.0.1:39123/ | rg "Choose file|Pending uploads|app.js" - firefox --headless --screenshot /tmp/upl-page.png http://127.0.0.1:39123/ Refs: PLAN.md milestones 5, 6, and 7 --- README.md | 4 +- TESTS.md | 7 + justfile | 4 + static/app.js | 502 ++++++++++++++++++++++++++++++++++++++++- static/index.html | 12 +- static/styles.css | 83 ++++++- tests/static_server.rs | 3 +- 7 files changed, 596 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 70d242f..3c8ce82 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ upl Browser UI static/index.html upload tool markup static/styles.css responsive tool styling - static/app.js file-selection behavior + static/app.js upload scheduler, retries, and browser resume state Validation tests/ integration tests for server behavior TESTS.md reusable manual and automated test checklist @@ -48,3 +48,5 @@ Use the `justfile` for routine tasks: just check just run ``` + +`just check` also syntax-checks the static browser JavaScript with `node`. diff --git a/TESTS.md b/TESTS.md index 693921b..43f4756 100644 --- a/TESTS.md +++ b/TESTS.md @@ -18,6 +18,7 @@ Keep this file as the reusable verification checklist while implementing - `GET /api/uploads/:id` reports completed chunks from disk. - `POST /api/uploads/:id/complete` assembles verified chunks. - `POST /api/uploads/:id/complete` rejects incomplete uploads. + - `static/app.js` passes `node --check`. ## Manual @@ -29,6 +30,12 @@ features land. - Kill the browser tab mid-upload and resume. - Restart the Rust server mid-upload and resume. - Interrupt the network and resume. +- Pause from the browser controls and resume. +- Reload the page and resume from the pending upload list. +- In a browser with the File System Access API, resume without reselecting the + file after granting read permission. +- In a browser without the File System Access API, resume after reselecting the + same file. - Retry a duplicate chunk and confirm it is accepted idempotently. - Attempt an invalid chunk index and confirm it is rejected. - Attempt a wrong-size non-final chunk and confirm it is rejected. diff --git a/justfile b/justfile index 06c82a2..d1d140c 100644 --- a/justfile +++ b/justfile @@ -4,12 +4,16 @@ fmt: test: cargo test --all-targets +static-check: + node --check static/app.js + clippy: cargo clippy --all-targets check: just fmt just test + just static-check just clippy run: diff --git a/static/app.js b/static/app.js index f82b3fc..48887e9 100644 --- a/static/app.js +++ b/static/app.js @@ -1,9 +1,37 @@ +const DB_NAME = "upl"; +const DB_VERSION = 1; +const STORE_NAME = "uploads"; +const CONCURRENCY = 3; +const MAX_RETRIES = 5; +const BASE_RETRY_DELAY_MS = 500; + const fileInput = document.querySelector("#file-input"); +const pickButton = document.querySelector("#pick-button"); const fileSummary = document.querySelector("#file-summary"); const fileName = document.querySelector("#file-name"); const fileSize = document.querySelector("#file-size"); const startButton = document.querySelector("#start-button"); +const pauseButton = document.querySelector("#pause-button"); +const resumeButton = document.querySelector("#resume-button"); const eventLog = document.querySelector("#event-log"); +const progressBar = document.querySelector("#progress-bar"); +const progressMeta = document.querySelector("#progress-meta"); +const pendingSection = document.querySelector("#pending-section"); +const pendingList = document.querySelector("#pending-list"); + +const state = { + abortController: null, + completedChunks: new Set(), + file: null, + fileHandle: null, + pendingRecords: [], + record: null, + resumeAfterReselect: null, + running: false, +}; + +const dbReady = "indexedDB" in window ? openDatabase() : Promise.resolve(null); +let saveChain = Promise.resolve(); function formatBytes(bytes) { const formatter = new Intl.NumberFormat(undefined, { @@ -15,7 +43,7 @@ function formatBytes(bytes) { for (const candidate of units) { unit = candidate; - if (value < 1024 || candidate === units.at(-1)) { + if (value < 1024 || candidate === units[units.length - 1]) { break; } value /= 1024; @@ -25,25 +53,481 @@ function formatBytes(bytes) { } function log(message) { - eventLog.replaceChildren(); const item = document.createElement("li"); item.textContent = message; - eventLog.append(item); + eventLog.prepend(item); + + while (eventLog.children.length > 8) { + eventLog.lastElementChild.remove(); + } } -fileInput.addEventListener("change", () => { - const [file] = fileInput.files; +function openDatabase() { + return new Promise((resolve, reject) => { + const request = indexedDB.open(DB_NAME, DB_VERSION); + request.addEventListener("upgradeneeded", () => { + const db = request.result; + if (!db.objectStoreNames.contains(STORE_NAME)) { + db.createObjectStore(STORE_NAME, { keyPath: "upload_id" }); + } + }); + + request.addEventListener("success", () => resolve(request.result)); + request.addEventListener("error", () => reject(request.error)); + }).catch((error) => { + log(`IndexedDB unavailable: ${error.message}`); + return null; + }); +} + +function requestToPromise(request) { + return new Promise((resolve, reject) => { + request.addEventListener("success", () => resolve(request.result)); + request.addEventListener("error", () => reject(request.error)); + }); +} + +async function withStore(mode, callback) { + const db = await dbReady; + if (!db) { + return null; + } + + const transaction = db.transaction(STORE_NAME, mode); + return requestToPromise(callback(transaction.objectStore(STORE_NAME))); +} + +async function loadRecords() { + const records = (await withStore("readonly", (store) => store.getAll())) ?? []; + records.sort((left, right) => right.updated_at.localeCompare(left.updated_at)); + state.pendingRecords = records; + renderPendingRecords(); +} + +async function saveRecord(record) { + const nextSave = saveChain.catch(() => null).then(() => writeRecord(record)); + saveChain = nextSave; + await nextSave; +} + +async function writeRecord(record) { + const storedRecord = { ...record, updated_at: new Date().toISOString() }; + try { + await withStore("readwrite", (store) => store.put(storedRecord)); + } catch (error) { + if (!storedRecord.file_handle) { + throw error; + } + + delete storedRecord.file_handle; + await withStore("readwrite", (store) => store.put(storedRecord)); + log("Saved resume state without a reusable file handle."); + } + state.record = storedRecord; + await loadRecords(); +} + +async function deleteRecord(uploadId) { + await withStore("readwrite", (store) => store.delete(uploadId)); + if (state.record?.upload_id === uploadId) { + state.record = null; + } + await loadRecords(); +} + +function renderPendingRecords() { + pendingList.replaceChildren(); + pendingSection.hidden = state.pendingRecords.length === 0; + + for (const record of state.pendingRecords) { + const item = document.createElement("li"); + item.className = "pending-item"; + + const meta = document.createElement("div"); + meta.className = "pending-meta"; + + const title = document.createElement("strong"); + title.textContent = record.name; + + const detail = document.createElement("span"); + detail.textContent = `${formatBytes(record.size)} - ${record.completed_chunks ?? 0} of ${record.total_chunks} chunks`; + + const resume = document.createElement("button"); + resume.type = "button"; + resume.className = "secondary"; + resume.textContent = "Resume"; + resume.addEventListener("click", () => { + void resumePendingRecord(record); + }); + + const remove = document.createElement("button"); + remove.type = "button"; + remove.className = "danger"; + remove.textContent = "Remove"; + remove.addEventListener("click", () => { + void deleteRecord(record.upload_id); + }); + + meta.append(title, detail); + item.append(meta, resume, remove); + pendingList.append(item); + } +} + +function renderFile(file) { if (!file) { fileSummary.hidden = true; - startButton.disabled = true; - log("Choose a file to begin."); + fileName.textContent = ""; + fileSize.textContent = ""; return; } fileName.textContent = file.name; fileSize.textContent = formatBytes(file.size); fileSummary.hidden = false; - startButton.disabled = false; - log("Ready to create an upload record."); +} + +function renderButtons() { + startButton.disabled = !state.file || state.running || Boolean(state.record); + pauseButton.disabled = !state.running; + resumeButton.disabled = !state.file || state.running || !state.record; +} + +function updateProgress(completedCount, totalChunks) { + const percentage = totalChunks === 0 ? 100 : (completedCount / totalChunks) * 100; + progressBar.style.width = `${percentage}%`; + progressMeta.textContent = `${completedCount} of ${totalChunks} chunks`; +} + +function sameFile(record, file) { + return ( + record.name === file.name && + record.size === file.size && + record.last_modified === file.lastModified + ); +} + +function findPendingRecord(file) { + return state.pendingRecords.find((record) => sameFile(record, file)) ?? null; +} + +async function selectFile(file, fileHandle = null, record = null) { + if (record && !sameFile(record, file)) { + log("Selected file does not match the pending upload."); + return; + } + + state.file = file; + state.fileHandle = fileHandle; + state.record = record ?? findPendingRecord(file); + state.completedChunks = new Set(); + + renderFile(file); + updateProgress(state.record?.completed_chunks ?? 0, state.record?.total_chunks ?? 0); + renderButtons(); + + log(state.record ? "Ready to resume upload." : "Ready to create an upload record."); +} + +async function pickFile() { + if ("showOpenFilePicker" in window) { + try { + const [handle] = await window.showOpenFilePicker({ multiple: false }); + const file = await handle.getFile(); + await selectFile(file, handle); + return; + } catch (error) { + if (isAbortError(error)) { + return; + } + log(`File picker failed: ${error.message}`); + } + } + + fileInput.click(); +} + +fileInput.addEventListener("change", () => { + const [file] = fileInput.files; + + if (!file) { + renderFile(null); + renderButtons(); + log("Choose a file to begin."); + return; + } + + const record = state.resumeAfterReselect ?? findPendingRecord(file); + state.resumeAfterReselect = null; + void selectFile(file, null, record); }); + +pickButton.addEventListener("click", () => { + void pickFile(); +}); + +startButton.addEventListener("click", () => { + void runUpload(); +}); + +pauseButton.addEventListener("click", () => { + if (state.abortController) { + state.abortController.abort(); + } +}); + +resumeButton.addEventListener("click", () => { + void runUpload(); +}); + +async function resumePendingRecord(record) { + if (state.running) { + return; + } + + if (record.file_handle) { + const granted = await requestFileHandlePermission(record.file_handle); + if (granted) { + const file = await record.file_handle.getFile(); + await selectFile(file, record.file_handle, record); + await runUpload(); + return; + } + } + + state.resumeAfterReselect = record; + log("Select the same file to resume."); + fileInput.click(); +} + +async function requestFileHandlePermission(handle) { + if (!("queryPermission" in handle) || !("requestPermission" in handle)) { + return false; + } + + const options = { mode: "read" }; + if ((await handle.queryPermission(options)) === "granted") { + return true; + } + + return (await handle.requestPermission(options)) === "granted"; +} + +async function runUpload() { + if (!state.file || state.running) { + return; + } + + const controller = new AbortController(); + state.abortController = controller; + state.running = true; + renderButtons(); + + try { + if (!state.record) { + await createUploadRecord(); + } + + const progress = await fetchJson(`/api/uploads/${state.record.upload_id}`, { + signal: controller.signal, + }); + state.completedChunks = new Set(progress.completed_chunks); + await saveRecord({ + ...state.record, + completed_chunks: state.completedChunks.size, + chunk_size: progress.chunk_size, + total_chunks: progress.total_chunks, + }); + updateProgress(state.completedChunks.size, progress.total_chunks); + + const missingChunks = buildMissingChunkList(progress.total_chunks, state.completedChunks); + log( + missingChunks.length === 0 + ? "All chunks already uploaded." + : `Uploading ${missingChunks.length} missing chunks.`, + ); + + await runPool(missingChunks, (index) => + uploadChunkWithRetry(index, progress.chunk_size, progress.total_chunks, controller.signal), + ); + + if (controller.signal.aborted) { + return; + } + + log("Completing upload."); + const complete = await fetchJson(`/api/uploads/${state.record.upload_id}/complete`, { + method: "POST", + signal: controller.signal, + }); + + updateProgress(progress.total_chunks, progress.total_chunks); + log(`Complete: ${complete.file_path}`); + await deleteRecord(state.record.upload_id); + state.completedChunks = new Set(); + state.record = null; + } catch (error) { + if (controller.signal.aborted || isAbortError(error)) { + log("Upload paused."); + } else { + controller.abort(); + log(`Upload failed: ${error.message}`); + } + } finally { + if (state.abortController === controller) { + state.abortController = null; + } + state.running = false; + renderButtons(); + } +} + +async function createUploadRecord() { + const response = await fetchJson("/api/uploads", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + name: state.file.name, + size: state.file.size, + last_modified: state.file.lastModified, + }), + }); + + const record = { + upload_id: response.upload_id, + name: state.file.name, + size: state.file.size, + last_modified: state.file.lastModified, + chunk_size: response.chunk_size, + total_chunks: response.total_chunks, + completed_chunks: 0, + file_handle: state.fileHandle, + updated_at: new Date().toISOString(), + }; + + await saveRecord(record); + updateProgress(0, response.total_chunks); + log("Upload record created."); +} + +function buildMissingChunkList(totalChunks, completedChunks) { + const missing = []; + for (let index = 0; index < totalChunks; index += 1) { + if (!completedChunks.has(index)) { + missing.push(index); + } + } + return missing; +} + +async function uploadChunkWithRetry(index, chunkSize, totalChunks, signal) { + for (let attempt = 0; attempt <= MAX_RETRIES; attempt += 1) { + throwIfAborted(signal); + + try { + await uploadChunk(index, chunkSize, signal); + state.completedChunks.add(index); + updateProgress(state.completedChunks.size, totalChunks); + await saveRecord({ + ...state.record, + completed_chunks: state.completedChunks.size, + }); + return; + } catch (error) { + if (isAbortError(error) || attempt === MAX_RETRIES) { + throw error; + } + + const delayMs = BASE_RETRY_DELAY_MS * 2 ** attempt; + log(`Retrying chunk ${index} after ${delayMs} ms.`); + await delay(delayMs, signal); + } + } +} + +async function uploadChunk(index, chunkSize, signal) { + const start = index * chunkSize; + const end = Math.min(state.file.size, start + chunkSize); + const body = state.file.slice(start, end); + const response = await fetch(`/api/uploads/${state.record.upload_id}/chunks/${index}`, { + method: "PUT", + headers: { "Content-Type": "application/octet-stream" }, + body, + signal, + }); + + if (!response.ok) { + throw new Error(await responseError(response)); + } +} + +async function runPool(items, worker) { + let nextIndex = 0; + const workers = Array.from( + { length: Math.min(CONCURRENCY, items.length) }, + async () => { + while (nextIndex < items.length) { + const item = items[nextIndex]; + nextIndex += 1; + await worker(item); + } + }, + ); + + await Promise.all(workers); +} + +async function fetchJson(url, options = {}) { + const response = await fetch(url, options); + if (!response.ok) { + throw new Error(await responseError(response)); + } + return response.json(); +} + +async function responseError(response) { + const text = await response.text(); + if (!text) { + return `${response.status} ${response.statusText}`; + } + + try { + return JSON.parse(text).error ?? text; + } catch { + return text; + } +} + +function delay(ms, signal) { + return new Promise((resolve, reject) => { + throwIfAborted(signal); + + const id = window.setTimeout(resolve, ms); + signal.addEventListener( + "abort", + () => { + window.clearTimeout(id); + reject(new DOMException("Aborted", "AbortError")); + }, + { once: true }, + ); + }); +} + +function throwIfAborted(signal) { + if (signal.aborted) { + throw new DOMException("Aborted", "AbortError"); + } +} + +function isAbortError(error) { + return error?.name === "AbortError"; +} + +async function initialize() { + await loadRecords(); + renderButtons(); +} + +void initialize(); diff --git a/static/index.html b/static/index.html index 0df875b..c315eca 100644 --- a/static/index.html +++ b/static/index.html @@ -18,10 +18,10 @@ Server online -