diff --git a/src/edlStore.js b/src/edlStore.js index 7ac6ddd3..6835a28b 100644 --- a/src/edlStore.js +++ b/src/edlStore.js @@ -5,39 +5,39 @@ import { parseSrt, formatSrt, parseCuesheet, parseXmeml, parseFcpXml, parseCsv, import { askForYouTubeInput, showOpenDialog } from './dialogs'; import { getOutPath } from './util'; -const fs = window.require('fs-extra'); +const { readFile, writeFile } = window.require('fs/promises'); const cueParser = window.require('cue-parser'); const { basename } = window.require('path'); const { dialog } = window.require('@electron/remote'); export async function loadCsvSeconds(path) { - return parseCsv(await fs.readFile(path, 'utf-8'), parseCsvTime); + return parseCsv(await readFile(path, 'utf-8'), parseCsvTime); } export async function loadCsvFrames(path, fps) { if (!fps) throw new Error('The loaded file has an unknown framerate'); - return parseCsv(await fs.readFile(path, 'utf-8'), getFrameValParser(fps)); + return parseCsv(await readFile(path, 'utf-8'), getFrameValParser(fps)); } export async function loadXmeml(path) { - return parseXmeml(await fs.readFile(path, 'utf-8')); + return parseXmeml(await readFile(path, 'utf-8')); } export async function loadFcpXml(path) { - return parseFcpXml(await fs.readFile(path, 'utf-8')); + return parseFcpXml(await readFile(path, 'utf-8')); } export async function loadDvAnalyzerSummaryTxt(path) { - return parseDvAnalyzerSummaryTxt(await fs.readFile(path, 'utf-8')); + return parseDvAnalyzerSummaryTxt(await readFile(path, 'utf-8')); } export async function loadPbf(path) { - return parsePbf(await fs.readFile(path)); + return parsePbf(await readFile(path)); } export async function loadMplayerEdl(path) { - return parseMplayerEdl(await fs.readFile(path, 'utf-8')); + return parseMplayerEdl(await readFile(path, 'utf-8')); } export async function loadCue(path) { @@ -45,41 +45,40 @@ export async function loadCue(path) { } export async function loadSrt(path) { - return parseSrt(await fs.readFile(path, 'utf-8')); + return parseSrt(await readFile(path, 'utf-8')); } export async function saveCsv(path, cutSegments) { - await fs.writeFile(path, await formatCsvSeconds(cutSegments)); + await writeFile(path, await formatCsvSeconds(cutSegments)); } export async function saveCsvHuman(path, cutSegments) { - await fs.writeFile(path, await formatCsvHuman(cutSegments)); + await writeFile(path, await formatCsvHuman(cutSegments)); } export async function saveCsvFrames({ path, cutSegments, getFrameCount }) { - await fs.writeFile(path, await formatCsvFrames({ cutSegments, getFrameCount })); + await writeFile(path, await formatCsvFrames({ cutSegments, getFrameCount })); } export async function saveTsv(path, cutSegments) { - await fs.writeFile(path, await formatTsv(cutSegments)); + await writeFile(path, await formatTsv(cutSegments)); } export async function saveSrt(path, cutSegments) { - await fs.writeFile(path, await formatSrt(cutSegments)); + await writeFile(path, await formatSrt(cutSegments)); } - export async function saveLlcProject({ savePath, filePath, cutSegments }) { const projectData = { version: 1, mediaFileName: basename(filePath), cutSegments: cutSegments.map(({ start, end, name, tags }) => ({ start, end, name, tags })), }; - await fs.writeFile(savePath, JSON5.stringify(projectData, null, 2)); + await writeFile(savePath, JSON5.stringify(projectData, null, 2)); } export async function loadLlcProject(path) { - return JSON5.parse(await fs.readFile(path)); + return JSON5.parse(await readFile(path)); } diff --git a/src/hooks/useFfmpegOperations.js b/src/hooks/useFfmpegOperations.js index 51495231..27bf1762 100644 --- a/src/hooks/useFfmpegOperations.js +++ b/src/hooks/useFfmpegOperations.js @@ -3,7 +3,7 @@ import flatMap from 'lodash/flatMap'; import sum from 'lodash/sum'; import pMap from 'p-map'; -import { getSuffixedOutPath, transferTimestamps, getOutFileExtension, getOutDir, deleteDispositionValue, getHtml5ifiedPath } from '../util'; +import { getSuffixedOutPath, transferTimestamps, getOutFileExtension, getOutDir, deleteDispositionValue, getHtml5ifiedPath, unlinkWithRetry } from '../util'; import { isCuttingStart, isCuttingEnd, runFfmpegWithProgress, getFfCommandLine, getDuration, createChaptersFromSegments, readFileMeta, cutEncodeSmartPart, getExperimentalArgs, html5ify as ffmpegHtml5ify, getVideoTimescaleArgs, logStdoutStderr, runFfmpegConcat } from '../ffmpeg'; import { getMapStreamsArgs, getStreamIdsToCopy } from '../util/streams'; import { getSmartCutParams } from '../smartcut'; @@ -11,7 +11,7 @@ import { isDurationValid } from '../segments'; const { join, resolve, dirname } = window.require('path'); const { pathExists } = window.require('fs-extra'); -const { writeFile, unlink, mkdir } = window.require('fs/promises'); +const { writeFile, mkdir } = window.require('fs/promises'); async function writeChaptersFfmetadata(outDir, chapters) { if (!chapters || chapters.length === 0) return undefined; @@ -52,9 +52,9 @@ function getMatroskaFlags() { const getChaptersInputArgs = (ffmetadataPath) => (ffmetadataPath ? ['-f', 'ffmetadata', '-i', ffmetadataPath] : []); -const tryDeleteFiles = async (paths) => pMap(paths, (path) => { - unlink(path).catch((err) => console.error('Failed to delete', path, err)); -}, { concurrency: 5 }); +async function tryDeleteFiles(paths) { + return pMap(paths, (path) => unlinkWithRetry(path).catch((err) => console.error('Failed to delete', path, err)), { concurrency: 5 }); +} function useFfmpegOperations({ filePath, treatInputFileModifiedTimeAsStart, treatOutputFileModifiedTimeAsStart, needSmartCut, enableOverwriteOutput, outputPlaybackRate }) { const shouldSkipExistingFile = useCallback(async (path) => { diff --git a/src/hooks/useFrameCapture.js b/src/hooks/useFrameCapture.js index 2ed81288..6dde5d89 100644 --- a/src/hooks/useFrameCapture.js +++ b/src/hooks/useFrameCapture.js @@ -1,14 +1,13 @@ import dataUriToBuffer from 'data-uri-to-buffer'; import pMap from 'p-map'; -import { getSuffixedOutPath, getOutDir, transferTimestamps, getSuffixedFileName, getOutPath, escapeRegExp } from '../util'; +import { getSuffixedOutPath, getOutDir, transferTimestamps, getSuffixedFileName, getOutPath, escapeRegExp, fsOperationWithRetry } from '../util'; import { getNumDigits } from '../segments'; import { captureFrame as ffmpegCaptureFrame, captureFrames as ffmpegCaptureFrames } from '../ffmpeg'; -const fs = window.require('fs-extra'); const mime = window.require('mime-types'); -const { rename, readdir } = window.require('fs/promises'); +const { rename, readdir, writeFile } = window.require('fs/promises'); function getFrameFromVideo(video, format, quality) { @@ -63,7 +62,7 @@ export default ({ formatTimecode, treatOutputFileModifiedTimeAsStart }) => { const duration = formatTimecode({ seconds: fromTime + (frameNum / fps), fileNameFriendly: true }); const renameFromPath = getOutPath({ customOutDir, filePath, fileName }); const renameToPath = getOutPath({ customOutDir, filePath, fileName: getSuffixedFileName(filePath, getSuffix(duration, captureFormat)) }); - await rename(renameFromPath, renameToPath); + await fsOperationWithRetry(async () => rename(renameFromPath, renameToPath)); return renameToPath; }, { concurrency: 1 }); @@ -87,7 +86,7 @@ export default ({ formatTimecode, treatOutputFileModifiedTimeAsStart }) => { const time = formatTimecode({ seconds: currentTime, fileNameFriendly: true }); const outPath = getSuffixedOutPath({ customOutDir, filePath, nameSuffix: `${time}.${ext}` }); - await fs.writeFile(outPath, buf); + await writeFile(outPath, buf); await transferTimestamps({ inPath: filePath, outPath, cutFrom: currentTime, treatOutputFileModifiedTimeAsStart }); return outPath; diff --git a/src/util.js b/src/util.js index 076643d9..d45f04fd 100644 --- a/src/util.js +++ b/src/util.js @@ -11,13 +11,11 @@ import { ffmpegExtractWindow } from './util/constants'; const { dirname, parse: parsePath, join, extname, isAbsolute, resolve, basename } = window.require('path'); const fsExtra = window.require('fs-extra'); -const { stat, readdir } = window.require('fs/promises'); +const { stat, readdir, utimes, unlink } = window.require('fs/promises'); const os = window.require('os'); const { ipcRenderer } = window.require('electron'); const remote = window.require('@electron/remote'); -const { unlink } = fsExtra; - const trashFile = async (path) => ipcRenderer.invoke('tryTrashItem', path); @@ -95,6 +93,30 @@ export async function dirExists(dirPath) { return (await pathExists(dirPath)) && (await fsExtra.lstat(dirPath)).isDirectory(); } +// const testFailFsOperation = isDev; +const testFailFsOperation = false; + +// Retry because sometimes write operations fail on windows due to the file being locked for various reasons (often anti-virus) #272 #1797 #1704 +export async function fsOperationWithRetry(operation, { signal, retries = 10, minTimeout = 100, maxTimeout = 2000, ...opts }) { + return pRetry(async () => { + if (testFailFsOperation && Math.random() > 0.3) throw Object.assign(new Error('test delete failure'), { code: 'EPERM' }); + await operation(); + }, { + retries, + signal, + minTimeout, + maxTimeout, + // mimic fs.rm `maxRetries` https://nodejs.org/api/fs.html#fspromisesrmpath-options + shouldRetry: (err) => err instanceof Error && 'code' in err && ['EBUSY', 'EMFILE', 'ENFILE', 'EPERM'].includes(err.code), + ...opts, + }); +} + +// example error: index-18074aaf.js:166 Failed to delete C:\Users\USERNAME\Desktop\RC\New folder\2023-12-27 21-45-22 (GMT p5)-merged-1703933052361-00.01.04.915-00.01.07.424-seg1.mp4 Error: EPERM: operation not permitted, unlink 'C:\Users\USERNAME\Desktop\RC\New folder\2023-12-27 21-45-22 (GMT p5)-merged-1703933052361-00.01.04.915-00.01.07.424-seg1.mp4' +export const unlinkWithRetry = async (path, options) => fsOperationWithRetry(async () => unlink(path), { ...options, onFailedAttempt: (error) => console.warn('Retrying delete', path, error.attemptNumber) }); +// example error: index-18074aaf.js:160 Error: EPERM: operation not permitted, utime 'C:\Users\USERNAME\Desktop\RC\New folder\2023-12-27 21-45-22 (GMT p5)-merged-1703933052361-cut-merged-1703933070237.mp4' +export const utimesWithRetry = async (path, atime, mtime, options) => fsOperationWithRetry(async () => utimes(path, atime, mtime), { ...options, onFailedAttempt: (error) => console.warn('Retrying utimes', path, error.attemptNumber) }); + export async function transferTimestamps({ inPath, outPath, cutFrom = 0, cutTo = 0, duration = 0, treatInputFileModifiedTimeAsStart = true, treatOutputFileModifiedTimeAsStart }) { if (treatOutputFileModifiedTimeAsStart == null) return; // null means disabled; @@ -115,7 +137,7 @@ export async function transferTimestamps({ inPath, outPath, cutFrom = 0, cutTo = try { const { atime, mtime } = await stat(inPath); - await fsExtra.utimes(outPath, calculateTime((atime.getTime() / 1000)), calculateTime((mtime.getTime() / 1000))); + await utimesWithRetry(outPath, calculateTime((atime.getTime() / 1000)), calculateTime((mtime.getTime() / 1000))); } catch (err) { console.error('Failed to set output file modified time', err); } @@ -239,13 +261,10 @@ export function getHtml5ifiedPath(cod, fp, type) { export async function deleteFiles({ paths, deleteIfTrashFails, signal }) { const failedToTrashFiles = []; - // const testFail = isDev; - const testFail = false; - // eslint-disable-next-line no-restricted-syntax for (const path of paths) { try { - if (testFail) throw new Error('test trash failure'); + if (testFailFsOperation) throw new Error('test trash failure'); // eslint-disable-next-line no-await-in-loop await trashFile(path); signal.throwIfAborted(); @@ -267,19 +286,7 @@ export async function deleteFiles({ paths, deleteIfTrashFails, signal }) { if (!value) return; } - // Retry because sometimes it fails on windows #272 #1797 - await pMap(failedToTrashFiles, async (path) => { - await pRetry(async () => { - if (testFail) throw new Error('test delete failure'); - await unlink(path); - }, { - retries: 3, - signal, - onFailedAttempt: async () => { - console.warn('Retrying delete', path); - }, - }); - }, { concurrency: 1 }); + await pMap(failedToTrashFiles, async (path) => unlinkWithRetry(path, { signal }), { concurrency: 5 }); } export const deleteDispositionValue = 'llc_disposition_remove';