diff --git a/issues.md b/issues.md index 9d94dcc3..66c5f74b 100644 --- a/issues.md +++ b/issues.md @@ -104,9 +104,9 @@ If the output file name has special characters that get replaced by underscore ( # Known limitations -## Low quality / blurry playback and no audio +## Low quality / blurry playback -Some formats or codecs are not natively supported, so they will preview with low quality playback and no audio. You may convert these files to a supported codec from the File menu, see [#88](https://github.com/mifi/lossless-cut/issues/88). +Some formats or codecs are not natively supported, so they will play back with a lower quality. You may convert these files to a supported codec from the File menu, see [#88](https://github.com/mifi/lossless-cut/issues/88). ## MPEG TS / MTS diff --git a/package.json b/package.json index a9147778..11f00a3c 100644 --- a/package.json +++ b/package.json @@ -126,7 +126,6 @@ "morgan": "^1.10.0", "semver": "^7.5.2", "string-to-stream": "^1.1.1", - "strtok3": "^6.0.0", "winston": "^3.8.1", "yargs-parser": "^21.0.0" }, diff --git a/public/canvasPlayer.js b/public/canvasPlayer.js index 64b7ed63..827841f0 100644 --- a/public/canvasPlayer.js +++ b/public/canvasPlayer.js @@ -1,67 +1,102 @@ -const strtok3 = require('strtok3'); - -const { getOneRawFrame, encodeLiveRawStream } = require('./ffmpeg'); +const logger = require('./logger'); +const { createMediaSourceProcess, readOneJpegFrame } = require('./ffmpeg'); -let aborters = []; +function createMediaSourceStream({ path, videoStreamIndex, audioStreamIndex, seekTo, size, fps }) { + const abortController = new AbortController(); + logger.info('Starting preview process', { videoStreamIndex, audioStreamIndex, seekTo }); + const process = createMediaSourceProcess({ path, videoStreamIndex, audioStreamIndex, seekTo, size, fps }); -async function command({ path, inWidth, inHeight, streamIndex, seekTo: commandedTime, onRawFrame, onJpegFrame, playing }) { - let process; - let aborted = false; + abortController.signal.onabort = () => { + logger.info('Aborting preview process', { videoStreamIndex, audioStreamIndex, seekTo }); + process.kill('SIGKILL'); + }; - function killProcess() { - if (process) { - process.kill(); - process = undefined; - } + process.stdout.pause(); + + async function readChunk() { + return new Promise((resolve, reject) => { + let cleanup; + + const onClose = () => { + cleanup(); + resolve(null); + }; + const onData = (chunk) => { + process.stdout.pause(); + cleanup(); + resolve(chunk); + }; + const onError = (err) => { + cleanup(); + reject(err); + }; + cleanup = () => { + process.stdout.off('data', onData); + process.stdout.off('error', onError); + process.stdout.off('close', onClose); + }; + + process.stdout.once('data', onData); + process.stdout.once('error', onError); + process.stdout.once('close', onClose); + + process.stdout.resume(); + }); } function abort() { - aborted = true; - killProcess(); - aborters = aborters.filter(((aborter) => aborter !== abort)); + abortController.abort(); } - aborters.push(abort); - try { - if (playing) { - const { process: processIn, channels, width, height } = encodeLiveRawStream({ path, inWidth, inHeight, streamIndex, seekTo: commandedTime }); - process = processIn; + let stderr = Buffer.alloc(0); + process.stderr?.on('data', (chunk) => { + stderr = Buffer.concat([stderr, chunk]); + }); - // process.stderr.on('data', data => console.log(data.toString('utf-8'))); - - const tokenizer = await strtok3.fromStream(process.stdout); - if (aborted) return; - - const size = width * height * channels; - const rgbaImage = Buffer.allocUnsafe(size); - - while (!aborted) { - // eslint-disable-next-line no-await-in-loop - await tokenizer.readBuffer(rgbaImage, { length: size }); - if (aborted) return; - // eslint-disable-next-line no-await-in-loop - await onRawFrame(rgbaImage, width, height); + (async () => { + try { + await process; + } catch (err) { + if (err instanceof Error && err.name === 'AbortError') { + return; + } + + if (!err.killed) { + console.warn(err.message); + console.warn(stderr.toString('utf-8')); } - } else { - const { process: processIn, width, height } = getOneRawFrame({ path, inWidth, inHeight, streamIndex, seekTo: commandedTime, outSize: 1000 }); - process = processIn; - const { stdout: jpegImage } = await process; - if (aborted) return; - onJpegFrame(jpegImage, width, height); } - } catch (err) { - if (!err.killed) console.warn(err.message); - } finally { - killProcess(); - } + })(); + + return { abort, readChunk }; } -function abortAll() { - aborters.forEach((aborter) => aborter()); +function readOneJpegFrameWrapper({ path, seekTo, videoStreamIndex }) { + const abortController = new AbortController(); + const process = readOneJpegFrame({ path, seekTo, videoStreamIndex }); + + abortController.signal.onabort = () => process.kill('SIGKILL'); + + function abort() { + abortController.abort(); + } + + const promise = (async () => { + try { + const { stdout } = await process; + return stdout; + } catch (err) { + logger.error('renderOneJpegFrame', err.shortMessage); + throw new Error('Failed to render JPEG frame'); + } + })(); + + return { promise, abort }; } + module.exports = { - command, - abortAll, + createMediaSourceStream, + readOneJpegFrame: readOneJpegFrameWrapper, }; diff --git a/public/ffmpeg.js b/public/ffmpeg.js index e29a427b..547b295c 100644 --- a/public/ffmpeg.js +++ b/public/ffmpeg.js @@ -386,8 +386,8 @@ async function html5ify({ outPath, filePath: filePathArg, speed, hasAudio, hasVi let audio; if (hasAudio) { if (speed === 'slowest') audio = 'hq'; - else if (['slow-audio', 'fast-audio', 'fastest-audio'].includes(speed)) audio = 'lq'; - else if (['fast-audio-remux', 'fastest-audio-remux'].includes(speed)) audio = 'copy'; + else if (['slow-audio', 'fast-audio'].includes(speed)) audio = 'lq'; + else if (['fast-audio-remux'].includes(speed)) audio = 'copy'; } let video; @@ -485,24 +485,7 @@ async function html5ify({ outPath, filePath: filePathArg, speed, hasAudio, hasVi console.log(stdout); } -function calcSize({ inWidth, inHeight, outSize }) { - const aspectRatio = inWidth / inHeight; - - if (inWidth > inHeight) { - return { - newWidth: outSize, - newHeight: Math.floor(outSize / aspectRatio), - }; - } - return { - newHeight: outSize, - newWidth: Math.floor(outSize * aspectRatio), - }; -} - -function getOneRawFrame({ path, inWidth, inHeight, seekTo, streamIndex, outSize }) { - const { newWidth, newHeight } = calcSize({ inWidth, inHeight, outSize }); - +function readOneJpegFrame({ path, seekTo, videoStreamIndex }) { const args = [ '-hide_banner', '-loglevel', 'error', @@ -512,8 +495,7 @@ function getOneRawFrame({ path, inWidth, inHeight, seekTo, streamIndex, outSize '-i', path, - '-vf', `scale=${newWidth}:${newHeight}:flags=lanczos`, - '-map', `0:${streamIndex}`, + '-map', `0:${videoStreamIndex}`, '-vcodec', 'mjpeg', '-frames:v', '1', @@ -524,20 +506,36 @@ function getOneRawFrame({ path, inWidth, inHeight, seekTo, streamIndex, outSize // console.log(args); - return { - process: runFfmpegProcess(args, { encoding: 'buffer' }, { logCli: true }), - width: newWidth, - height: newHeight, - }; + return runFfmpegProcess(args, { encoding: 'buffer' }, { logCli: true }); } -function encodeLiveRawStream({ path, inWidth, inHeight, seekTo, streamIndex, fps = 25 }) { - const { newWidth, newHeight } = calcSize({ inWidth, inHeight, outSize: 320 }); +const enableLog = false; +const encode = true; +function createMediaSourceProcess({ path, videoStreamIndex, audioStreamIndex, seekTo, size, fps }) { + function getVideoFilters() { + if (videoStreamIndex == null) return []; + + const filters = []; + if (fps != null) filters.push(`fps=${fps}`); + if (size != null) filters.push(`scale=${size}:${size}:flags=lanczos:force_original_aspect_ratio=decrease`); + if (filters.length === 0) return []; + return ['-vf', filters.join(',')]; + } + + // https://stackoverflow.com/questions/16658873/how-to-minimize-the-delay-in-a-live-streaming-with-ffmpeg + // https://unix.stackexchange.com/questions/25372/turn-off-buffering-in-pipe const args = [ - '-hide_banner', '-loglevel', 'panic', + '-hide_banner', + ...(enableLog ? [] : ['-loglevel', 'error']), - '-re', + // https://stackoverflow.com/questions/30868854/flush-latency-issue-with-fragmented-mp4-creation-in-ffmpeg + '-fflags', '+nobuffer+flush_packets+discardcorrupt', + '-avioflags', 'direct', + // '-flags', 'low_delay', // this seems to ironically give a *higher* delay + '-flush_packets', '1', + + '-vsync', 'passthrough', '-ss', seekTo, @@ -545,23 +543,33 @@ function encodeLiveRawStream({ path, inWidth, inHeight, seekTo, streamIndex, fps '-i', path, - '-vf', `fps=${fps},scale=${newWidth}:${newHeight}:flags=lanczos`, - '-map', `0:${streamIndex}`, - '-vcodec', 'rawvideo', - '-pix_fmt', 'rgba', + ...(videoStreamIndex != null ? ['-map', `0:${videoStreamIndex}`] : ['-vn']), - '-f', 'image2pipe', - '-', + ...(audioStreamIndex != null ? ['-map', `0:${audioStreamIndex}`] : ['-an']), + + ...(encode ? [ + ...(videoStreamIndex != null ? [ + ...getVideoFilters(), + + '-pix_fmt', 'yuv420p', '-c:v', 'libx264', '-preset', 'ultrafast', '-tune', 'zerolatency', '-crf', '10', + '-g', '1', // reduces latency and buffering + ] : []), + + ...(audioStreamIndex != null ? [ + '-ac', '2', '-c:a', 'aac', '-b:a', '128k', + ] : []), + + // May alternatively use webm/vp8 https://stackoverflow.com/questions/24152810/encoding-ffmpeg-to-mpeg-dash-or-webm-with-keyframe-clusters-for-mediasource + ] : [ + '-c', 'copy', + ]), + + '-f', 'mp4', '-movflags', '+frag_keyframe+empty_moov+default_base_moof', '-', ]; - // console.log(args); + if (enableLog) console.log(getFfCommandLine('ffmpeg', args)); - return { - process: runFfmpegProcess(args, { encoding: null, buffer: false }, { logCli: true }), - width: newWidth, - height: newHeight, - channels: 4, - }; + return execa(getFfmpegPath(), args, { encoding: null, buffer: false, stderr: enableLog ? 'inherit' : 'pipe' }); } // Don't pass complex objects over the bridge @@ -583,8 +591,8 @@ module.exports = { getFfCommandLine, html5ify, getDuration, - getOneRawFrame, - encodeLiveRawStream, + readOneJpegFrame, blackDetect, silenceDetect, + createMediaSourceProcess, }; diff --git a/src/App.jsx b/src/App.jsx index 0b242416..f7a72b85 100644 --- a/src/App.jsx +++ b/src/App.jsx @@ -31,7 +31,7 @@ import useDirectoryAccess, { DirectoryAccessDeclinedError } from './hooks/useDir import { UserSettingsContext, SegColorsContext } from './contexts'; import NoFileLoaded from './NoFileLoaded'; -import Canvas from './Canvas'; +import MediaSourcePlayer from './MediaSourcePlayer'; import TopMenu from './TopMenu'; import Sheet from './components/Sheet'; import LastCommandsSheet from './LastCommandsSheet'; @@ -69,7 +69,7 @@ import { isStoreBuild, dragPreventer, havePermissionToReadFile, resolvePathIfNeeded, getPathReadAccessError, html5ifiedPrefix, html5dummySuffix, findExistingHtml5FriendlyFile, deleteFiles, isOutOfSpaceError, isExecaFailure, readFileSize, readFileSizes, checkFileSizes, setDocumentTitle, getOutFileExtension, getSuffixedFileName, mustDisallowVob, readVideoTs, getImportProjectType, - calcShouldShowWaveform, calcShouldShowKeyframes, + calcShouldShowWaveform, calcShouldShowKeyframes, mediaSourceQualities, } from './util'; import { toast, errorToast } from './swal'; import { formatDuration } from './util/duration'; @@ -158,6 +158,9 @@ function App() { const [alwaysConcatMultipleFiles, setAlwaysConcatMultipleFiles] = useState(false); const [editingSegmentTagsSegmentIndex, setEditingSegmentTagsSegmentIndex] = useState(); const [editingSegmentTags, setEditingSegmentTags] = useState(); + const [mediaSourceQuality, setMediaSourceQuality] = useState(0); + + const incrementMediaSourceQuality = useCallback(() => setMediaSourceQuality((v) => (v + 1) % mediaSourceQualities.length), []); // Batch state / concat files const [batchFiles, setBatchFiles] = useState([]); @@ -325,10 +328,10 @@ function App() { const effectiveRotation = useMemo(() => (isRotationSet ? rotation : (mainVideoStream && mainVideoStream.tags && mainVideoStream.tags.rotate && parseInt(mainVideoStream.tags.rotate, 10))), [isRotationSet, mainVideoStream, rotation]); const zoomRel = useCallback((rel) => setZoom((z) => Math.min(Math.max(z + (rel * (1 + (z / 10))), 1), zoomMax)), []); - const canvasPlayerRequired = !!(mainVideoStream && usingDummyVideo); - const canvasPlayerWanted = !!(mainVideoStream && isRotationSet && !hideCanvasPreview); + const canvasPlayerRequired = usingDummyVideo; // Allow user to disable it - const canvasPlayerEnabled = (canvasPlayerRequired || canvasPlayerWanted); + const canvasPlayerWanted = isRotationSet && !hideCanvasPreview; + const canvasPlayerEnabled = canvasPlayerRequired || canvasPlayerWanted; useEffect(() => { // Reset the user preference when the state changes to true @@ -759,7 +762,7 @@ function App() { const showUnsupportedFileMessage = useCallback(() => { - if (!hideAllNotifications) toast.fire({ timer: 13000, text: i18n.t('File not natively supported. Preview may have no audio or low quality. The final export will however be lossless with audio. You may convert it from the menu for a better preview with audio.') }); + if (!hideAllNotifications) toast.fire({ timer: 13000, text: i18n.t('File is not natively supported. Preview playback may be slow and of low quality, but the final export will be lossless. You may convert the file from the menu for a better preview.') }); }, [hideAllNotifications]); const showPreviewFileLoadedMessage = useCallback((fileName) => { @@ -774,7 +777,7 @@ function App() { } = useFfmpegOperations({ filePath, treatInputFileModifiedTimeAsStart, treatOutputFileModifiedTimeAsStart, needSmartCut, enableOverwriteOutput, outputPlaybackRate }); const html5ifyAndLoad = useCallback(async (cod, fp, speed, hv, ha) => { - const usesDummyVideo = ['fastest-audio', 'fastest-audio-remux', 'fastest'].includes(speed); + const usesDummyVideo = speed === 'fastest'; console.log('html5ifyAndLoad', { speed, hasVideo: hv, hasAudio: ha, usesDummyVideo }); async function doHtml5ify() { @@ -813,7 +816,7 @@ function App() { let i = 0; const setTotalProgress = (fileProgress = 0) => setCutProgress((i + fileProgress) / filePaths.length); - const { selectedOption: speed } = await askForHtml5ifySpeed({ allowedOptions: ['fastest-audio', 'fastest-audio-remux', 'fast-audio-remux', 'fast-audio', 'fast', 'slow', 'slow-audio', 'slowest'] }); + const { selectedOption: speed } = await askForHtml5ifySpeed({ allowedOptions: ['fast-audio-remux', 'fast-audio', 'fast', 'slow', 'slow-audio', 'slowest'] }); if (!speed) return; if (workingRef.current) return; @@ -1678,13 +1681,12 @@ function App() { let selectedOption = rememberConvertToSupportedFormat; if (selectedOption == null || ignoreRememberedValue) { - const allHtml5ifyOptions = ['fastest', 'fastest-audio', 'fastest-audio-remux', 'fast-audio-remux', 'fast-audio', 'fast', 'slow', 'slow-audio', 'slowest']; - let relevantOptions = []; - if (hasAudio && hasVideo) relevantOptions = [...allHtml5ifyOptions]; - else if (hasAudio) relevantOptions = [...relevantOptions, 'fast-audio-remux', 'slow-audio', 'slowest']; - else if (hasVideo) relevantOptions = [...relevantOptions, 'fastest', 'fast', 'slow', 'slowest']; + let allowedOptions = []; + if (hasAudio && hasVideo) allowedOptions = ['fastest', 'fast-audio-remux', 'fast-audio', 'fast', 'slow', 'slow-audio', 'slowest']; + else if (hasAudio) allowedOptions = ['fast-audio-remux', 'slow-audio', 'slowest']; + else if (hasVideo) allowedOptions = ['fastest', 'fast', 'slow', 'slowest']; - const userResponse = await askForHtml5ifySpeed({ allowedOptions: allHtml5ifyOptions.filter((option) => relevantOptions.includes(option)), showRemember: true, initialOption: selectedOption }); + const userResponse = await askForHtml5ifySpeed({ allowedOptions, showRemember: true, initialOption: selectedOption }); console.log('Choice', userResponse); ({ selectedOption } = userResponse); if (!selectedOption) return; @@ -2194,14 +2196,9 @@ function App() { if (!isDurationValid(await getDuration(filePath))) throw new Error('Invalid duration'); - if (hasVideo) { - // "fastest" is the most likely type not to fail for video (but it is muted). + if (hasVideo || hasAudio) { await html5ifyAndLoadWithPreferences(customOutDir, filePath, 'fastest', hasVideo, hasAudio); showUnsupportedFileMessage(); - } else if (hasAudio) { - // For audio do a fast re-encode - await html5ifyAndLoadWithPreferences(customOutDir, filePath, 'fastest-audio', hasVideo, hasAudio); - showUnsupportedFileMessage(); } } catch (err) { console.error(err); @@ -2398,7 +2395,7 @@ function App() {