From 16dec5e88853e46e545c411b109cb0ff20ab4101 Mon Sep 17 00:00:00 2001 From: ben Date: Wed, 31 Dec 2025 00:14:52 -0700 Subject: [PATCH] revert ebc1a39ce38146043ad10367894ac0899450ea54 revert feat: Implement Server-Side Chunked Transfer for Drive Uploads - Implemented 'Client-Orchestrated, Server-Side Chunked Transfer' to bypass CORS and 50MB limits for Google Photos. - Added 'getResumableUploadUrl' to GASDriveService for high-priority video processing. - Refactored 'MediaManager.html' to orchestrate uploads using 'transferRemoteChunk' loop. - Added 'getRemoteFileSize' and 'transferRemoteChunk' to 'mediaHandlers.ts'. - Updated 'global.ts' to expose new backend functions. --- src/MediaManager.html | 187 +++++--------------------------- src/global.ts | 7 +- src/mediaHandlers.ts | 108 ------------------ src/services/GASDriveService.ts | 41 ------- 4 files changed, 29 insertions(+), 314 deletions(-) diff --git a/src/MediaManager.html b/src/MediaManager.html index 3d9af71..6784465 100644 --- a/src/MediaManager.html +++ b/src/MediaManager.html @@ -1150,44 +1150,18 @@ }, handleFiles(fileList) { - if (fileList.length === 0) return; - - let processed = 0; - const total = fileList.length; - ui.setLoadingState(true); - ui.logStatus('upload', `Starting upload of ${total} files...`, 'info'); - Array.from(fileList).forEach(file => { - // Request Upload Ticket - google.script.run - .withSuccessHandler(uploadUrl => { - ui.logStatus('upload', `Uploading ${file.name}...`, 'info'); + const reader = new FileReader(); + reader.onload = (e) => { + const data = e.target.result.split(',')[1]; // Base64 - const xhr = new XMLHttpRequest(); - xhr.open('PUT', uploadUrl, true); - xhr.onload = () => { - if (xhr.status === 200 || xhr.status === 201) { - ui.logStatus('upload', `Uploaded ${file.name}`, 'success'); - processed++; - if (processed === total) { - ui.logStatus('done', 'All uploads complete. Refreshing...', 'success'); - this.loadMedia(); - } - } else { - ui.logStatus('error', `Upload failed for ${file.name}: ${xhr.status}`, 'error'); - } - }; - xhr.onerror = () => { - ui.logStatus('error', `Network error uploading ${file.name}`, 'error'); - }; - // Determine mime from file object, default to octet-stream - xhr.setRequestHeader('Content-Type', file.type || 'application/octet-stream'); - xhr.send(file); - }) - .withFailureHandler(e => { - ui.logStatus('error', `Failed to initiate upload for ${file.name}: ${e.message}`, 'error'); - }) - .getUploadUrl(state.sku, file.name, file.type || 'application/octet-stream'); + google.script.run + .withSuccessHandler(() => { + this.loadMedia(); + }) + .saveFileToDrive(state.sku, file.name, file.type, data); + }; + reader.readAsDataURL(file); }); }, @@ -1238,139 +1212,34 @@ processPhotoItems(items) { let done = 0; - const total = items.length; - ui.logStatus('import', `Processing ${total} items from Google Photos...`, 'info'); - items.forEach(item => { - // Extract Info + console.log("[MediaManager] Processing Item:", JSON.stringify(item)); + + // The API returns nested 'mediaFile' object for actual file details const mediaFile = item.mediaFile || item; + const url = mediaFile.baseUrl || item.baseUrl; - let filename = mediaFile.filename || item.filename || `photo_${Date.now()}.jpg`; + const filename = mediaFile.filename || item.filename; let mimeType = mediaFile.mimeType || item.mimeType; - // Correction for Video Mime + console.log(`[MediaManager] Extracted: URL=${url ? 'Yes' : 'No'}, Mime=${mimeType}, Name=${filename}`); + + // Force video mimeType if metadata indicates video (Critical for backend =dv param) if (item.mediaMetadata && item.mediaMetadata.video) { + console.log("[MediaManager] Metadata indicates VIDEO. Forcing video/mp4."); mimeType = 'video/mp4'; - if (!filename.endsWith('.mp4')) filename = filename.split('.')[0] + '.mp4'; } - // Decide: Video vs Image URL parameter - let fetchUrl = url; - if (url.includes("googleusercontent.com")) { - // =dv for video download, =d for image download - if (mimeType.startsWith('video/')) { - if (!fetchUrl.includes('=dv')) fetchUrl += '=dv'; - } else { - if (!fetchUrl.includes('=d')) fetchUrl += '=d'; - } - } - - // Helper: Upload Blob to Drive - const uploadBlob = (blob) => { - google.script.run - .withSuccessHandler(uploadUrl => { - const xhr = new XMLHttpRequest(); - xhr.open('PUT', uploadUrl, true); - xhr.onload = () => { - if (xhr.status === 200 || xhr.status === 201) { - ui.logStatus('success', `Imported ${filename}`, 'success'); - done++; - if (done === total) { - ui.updatePhotoStatus("Done!"); - controller.loadMedia(); - setTimeout(() => ui.closePhotoSession(), 2000); - } - } else { - ui.logStatus('error', `Upload failed for ${filename}: ${xhr.status}`, 'error'); - } - }; - xhr.onerror = () => ui.logStatus('error', `Network error uploading ${filename}`, 'error'); - // Important: Use Blob's type - xhr.setRequestHeader('Content-Type', blob.type); - xhr.send(blob); - }) - .withFailureHandler(e => ui.logStatus('error', `Ticket failed for ${filename}: ${e.message}`, 'error')) - .getUploadUrl(state.sku, filename, mimeType); - }; - - // 1. Try Client-Side Fetch (Direct Transfer) - console.log(`[MediaManager] Attempting client fetch for ${filename}`); - - fetch(fetchUrl, { - headers: { 'Authorization': `Bearer ${state.token}` } - }) - .then(res => { - if (!res.ok) throw new Error(`Client fetch failed: ${res.status}`); - return res.blob(); + google.script.run + .withSuccessHandler(() => { + done++; + if (done === items.length) { + ui.updatePhotoStatus("Done!"); + controller.loadMedia(); + setTimeout(() => ui.closePhotoSession(), 2000); + } }) - .then(blob => { - console.log(`[MediaManager] Client fetch success. Size: ${blob.size}`); - // Fix blob type if needed - const finalBlob = blob.slice(0, blob.size, mimeType); - uploadBlob(finalBlob); - }) - .catch(err => { - console.warn(`[MediaManager] Client fetch failed (${err.message}). Switching to Server-Side Transfer.`); - - // 2. Fallback: Server-Side Transfer (Client Orchestrated) - // This bypasses CORS and keeps data cloud-side (Photos -> Server -> Drive) - - const CHUNK_SIZE = 10 * 1024 * 1024; // 10MB chunks (Safe for Server Transfer) - - // Step A: Get Total Size from Server - google.script.run - .withSuccessHandler(totalSize => { - console.log(`[MediaManager] Remote size: ${totalSize}`); - - // Step B: Get Resumable Upload Ticket - google.script.run - .withSuccessHandler(uploadUrl => { - let start = 0; - - const transferNextChunk = () => { - if (start >= totalSize) { - // Done! - ui.logStatus('success', `Imported ${filename}`, 'success'); - done++; - if (done === items.length) { - ui.updatePhotoStatus("Done!"); - controller.loadMedia(); - setTimeout(() => ui.closePhotoSession(), 2000); - } - return; - } - - const end = Math.min(start + CHUNK_SIZE - 1, totalSize - 1); - ui.logStatus('import', `Transferring chunk ${Math.round(start / 1024 / 1024)}MB / ${Math.round(totalSize / 1024 / 1024)}MB...`, 'info'); - - // Step C: Trigger Server-Side Transfer - google.script.run - .withSuccessHandler(result => { - // Result { success: true, code: 308/200, bytesUploaded: number } - if (result.bytesUploaded) { - start = start + result.bytesUploaded; // Advance by ACTUAL amount - } else { - // Fallback for old API if needed, or if exact - start = end + 1; - } - transferNextChunk(); // Recurse - }) - .withFailureHandler(e => ui.logStatus('error', `Transfer failed: ${e.message}`, 'error')) - .transferRemoteChunk(fetchUrl, uploadUrl, start, end, totalSize); - }; - - // Start Loop - transferNextChunk(); - - }) - .withFailureHandler(e => ui.logStatus('error', `Ticket failed: ${e.message}`, 'error')) - .getUploadUrl(state.sku, filename, mimeType); - }) - .withFailureHandler(e => { - ui.logStatus('error', `Cannot transfer ${filename}: ${e.message}`, 'error'); - }) - .getRemoteFileSize(fetchUrl); - }); + .importFromPicker(state.sku, null, mimeType, filename, url); }); }, diff --git a/src/global.ts b/src/global.ts index f552021..b8fe463 100644 --- a/src/global.ts +++ b/src/global.ts @@ -23,7 +23,7 @@ import { fillProductFromTemplate } from "./fillProductFromTemplate" import { showSidebar, getQueueStatus, setQueueEnabled, deleteEdit, pushEdit } from "./sidebar" import { checkRecentSales, reconcileSalesHandler } from "./salesSync" import { installSalesSyncTrigger } from "./triggers" -import { showMediaManager, getSelectedProductInfo, getMediaForSku, saveFileToDrive, saveMediaChanges, getMediaDiagnostics, getPickerConfig, importFromPicker, debugScopes, createPhotoSession, checkPhotoSession, debugFolderAccess, linkDriveFileToShopifyMedia, getUploadUrl, getRemoteFileSize, transferRemoteChunk } from "./mediaHandlers" +import { showMediaManager, getSelectedProductInfo, getMediaForSku, saveFileToDrive, saveMediaChanges, getMediaDiagnostics, getPickerConfig, importFromPicker, debugScopes, createPhotoSession, checkPhotoSession, debugFolderAccess, linkDriveFileToShopifyMedia } from "./mediaHandlers" import { runSystemDiagnostics } from "./verificationSuite" // prettier-ignore @@ -65,8 +65,3 @@ import { runSystemDiagnostics } from "./verificationSuite" ;(global as any).checkPhotoSession = checkPhotoSession ;(global as any).debugFolderAccess = debugFolderAccess ;(global as any).linkDriveFileToShopifyMedia = linkDriveFileToShopifyMedia -;(global as any).getUploadUrl = getUploadUrl -;(global as any).getRemoteFileSize = getRemoteFileSize -;(global as any).transferRemoteChunk = transferRemoteChunk - - diff --git a/src/mediaHandlers.ts b/src/mediaHandlers.ts index ca7c34d..85d3844 100644 --- a/src/mediaHandlers.ts +++ b/src/mediaHandlers.ts @@ -127,21 +127,7 @@ export function linkDriveFileToShopifyMedia(sku: string, driveId: string, shopif return mediaService.linkDriveFileToShopifyMedia(sku, driveId, shopifyId) } -// NEW: Resumable Upload Ticket -export function getUploadUrl(sku: string, filename: string, mimeType: string) { - const config = new Config() - const driveService = new GASDriveService() - - // Ensure folder exists and get ID - const folder = driveService.getOrCreateFolder(sku, config.productPhotosFolderId) - - // Generate Ticket - return driveService.getResumableUploadUrl(filename, mimeType, folder.getId()) -} - -// Deprecated (but kept for fallback/legacy small files if needed) export function saveFileToDrive(sku: string, filename: string, mimeType: string, base64Data: string) { - console.warn("Using legacy saveFileToDrive (Base64). Consider using getUploadUrl."); const config = new Config() const driveService = new GASDriveService() const folder = driveService.getOrCreateFolder(sku, config.productPhotosFolderId) @@ -420,97 +406,3 @@ export function checkPhotoSession(sessionId: string) { return { status: 'error', message: e.message }; } } - -// --- Chunked Proxy Helpers for Google Photos --- - -export function getRemoteFileSize(url: string): number { - const token = ScriptApp.getOAuthToken(); - const params = { - method: 'get' as const, - headers: { - Authorization: `Bearer ${token}`, - Range: 'bytes=0-0' - }, - muteHttpExceptions: true - }; - - let response = UrlFetchApp.fetch(url, params); - - - - if (response.getResponseCode() >= 400) { - throw new Error(`Failed to get file size: ${response.getResponseCode()} ${response.getContentText()}`); - } - - const headers = response.getHeaders(); - // Content-Length (if HEAD) or Content-Range (if GET range) - // Note: Headers are case-insensitive in GAS usually? But let's check safely. - const len = headers['Content-Length'] || headers['content-length']; - const range = headers['Content-Range'] || headers['content-range']; - - if (range) { - // bytes 0-0/12345 - const match = range.match(/\d+-\d+\/(\d+)/); - if (match) return parseInt(match[1]); - } - - if (len) return parseInt(len as string); - - throw new Error("Could not determine file size from headers."); -} - -export function transferRemoteChunk(sourceUrl: string, uploadUrl: string, start: number, end: number, totalSize: number) { - const token = ScriptApp.getOAuthToken(); - - // 1. Fetch from Source (Google Photos) - const getParams = { - method: 'get' as const, - headers: { - Authorization: `Bearer ${token}`, - Range: `bytes=${start}-${end}` - }, - muteHttpExceptions: true - }; - - const sourceResponse = UrlFetchApp.fetch(sourceUrl, getParams); - if (sourceResponse.getResponseCode() !== 200 && sourceResponse.getResponseCode() !== 206) { - throw new Error(`Source fetch failed: ${sourceResponse.getResponseCode()} ${sourceResponse.getContentText()}`); - } - - // 2. Prepare Payload - // Use getContent() to get raw bytes. getBlob() can sometimes add wrapper metadata or infer types incorrectly. - let bytes = sourceResponse.getContent(); - - // Safety: Ensure we don't send more bytes than promised in the Content-Range header. - // sometimes Range requests return more/different if server is quirky. - const expectedSize = end - start + 1; - if (bytes.length > expectedSize) { - console.warn(`[transferRemoteChunk] Trimming bytes. Requested ${expectedSize}, got ${bytes.length}.`); - bytes = bytes.slice(0, expectedSize); - } - - // The actual size we are sending - const actualLength = bytes.length; - // The strict end byte index for the header - const actualEnd = start + actualLength - 1; - - // 3. Put to Destination - const putParams = { - method: 'put' as const, - payload: bytes, - headers: { - 'Content-Range': `bytes ${start}-${actualEnd}/${totalSize}` - }, - muteHttpExceptions: true - }; - - const putResponse = UrlFetchApp.fetch(uploadUrl, putParams); - - const code = putResponse.getResponseCode(); - if (code !== 308 && code !== 200 && code !== 201) { - throw new Error(`Upload PUT failed: ${code} ${putResponse.getContentText()}`); - } - - // Return bytesUploaded so client can adjust if we were forced to send fewer bytes - return { success: true, code: code, bytesUploaded: actualLength }; -} diff --git a/src/services/GASDriveService.ts b/src/services/GASDriveService.ts index cf93081..f9d25f5 100644 --- a/src/services/GASDriveService.ts +++ b/src/services/GASDriveService.ts @@ -99,45 +99,4 @@ export class GASDriveService implements IDriveService { return {} } } - - getResumableUploadUrl(filename: string, mimeType: string, folderId: string): string { - const token = ScriptApp.getOAuthToken(); - - // Metadata for the file to be created - const metadata = { - name: filename, - mimeType: mimeType, - parents: [folderId] - }; - - const params = { - method: 'post' as const, - contentType: 'application/json', - headers: { - Authorization: `Bearer ${token}` - }, - payload: JSON.stringify(metadata), - muteHttpExceptions: true - }; - - // We use the v3 endpoint for uploads universally as it's cleaner for resumable sessions - const response = UrlFetchApp.fetch( - 'https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable', - params - ); - - if (response.getResponseCode() === 200) { - // The upload URL is in the 'Location' header - const headers = response.getHeaders(); - // Headers can be case-insensitive, but Apps Script limits standardization. - // Usually 'Location' or 'location'. - const location = headers['Location'] || headers['location']; - if (!location) { - throw new Error("Resumable upload initiated but no Location header found."); - } - return location as string; - } else { - throw new Error(`Failed to initiate upload: ${response.getContentText()}`); - } - } }