Implement a canvas for better playback of more formats #88

package.json

@@ -59,7 +59,6 @@
     "moment": "^2.18.1",
     "mousetrap": "^1.6.1",
     "p-map": "^3.0.0",
-    "p-queue": "^6.2.0",
     "patch-package": "^6.2.1",
     "pretty-ms": "^6.0.0",
     "react": "^16.12.0",
@@ -96,6 +95,7 @@
     "read-chunk": "^2.0.0",
     "semver": "^7.1.3",
     "string-to-stream": "^1.1.1",
+    "strtok3": "^6.0.0",
     "trash": "^6.1.1"
   },
   "eslintConfig": {

src/App.jsx

@@ -6,7 +6,6 @@ import Lottie from 'react-lottie';
 import { SideSheet, Button, Position, SegmentedControl, Select } from 'evergreen-ui';
 import { useStateWithHistory } from 'react-use/lib/useStateWithHistory';
 import useDebounce from 'react-use/lib/useDebounce';
-import PQueue from 'p-queue';
 import filePathToUrl from 'file-url';
 import Mousetrap from 'mousetrap';
 import uuid from 'uuid';
@@ -22,6 +21,7 @@ import flatMap from 'lodash/flatMap';
 import isEqual from 'lodash/isEqual';
 
 
+import Canvas from './Canvas';
 import TopMenu from './TopMenu';
 import HelpSheet from './HelpSheet';
 import SettingsSheet from './SettingsSheet';
@@ -39,7 +39,7 @@ import allOutFormats from './outFormats';
 import { captureFrameFromTag, captureFrameFfmpeg } from './capture-frame';
 import {
   defaultProcessedCodecTypes, getStreamFps, isCuttingStart, isCuttingEnd,
-  getDefaultOutFormat, getFormatData, renderFrame, mergeFiles as ffmpegMergeFiles, renderThumbnails as ffmpegRenderThumbnails,
+  getDefaultOutFormat, getFormatData, mergeFiles as ffmpegMergeFiles, renderThumbnails as ffmpegRenderThumbnails,
   readFrames, renderWaveformPng, html5ifyDummy, cutMultiple, extractStreams, autoMergeSegments, getAllStreams,
   findNearestKeyFrameTime, html5ify as ffmpegHtml5ify, isStreamThumbnail,
 } from './ffmpeg';
@@ -121,12 +121,8 @@ const zoomMax = 2 ** 14;
 const videoStyle = { width: '100%', height: '100%', objectFit: 'contain' };
 
 
-const queue = new PQueue({ concurrency: 1 });
-
-
 const App = memo(() => {
   // Per project state
-  const [framePath, setFramePath] = useState();
   const [waveform, setWaveform] = useState();
   const [html5FriendlyPath, setHtml5FriendlyPath] = useState();
   const [working, setWorking] = useState(false);
@@ -140,7 +136,6 @@ const App = memo(() => {
   const [rotation, setRotation] = useState(360);
   const [cutProgress, setCutProgress] = useState();
   const [startTimeOffset, setStartTimeOffset] = useState(0);
-  const [rotationPreviewRequested, setRotationPreviewRequested] = useState(false);
   const [filePath, setFilePath] = useState('');
   const [externalStreamFiles, setExternalStreamFiles] = useState([]);
   const [detectedFps, setDetectedFps] = useState();
@@ -235,7 +230,6 @@ const App = memo(() => {
     firstUpdateRef.current = false;
   }, []);
 
-
   // Global state
   const [helpVisible, setHelpVisible] = useState(false);
   const [settingsVisible, setSettingsVisible] = useState(false);
@@ -326,8 +320,12 @@ const App = memo(() => {
     if (dummyVideoPath) unlink(dummyVideoPath).catch(console.error);
   }, [dummyVideoPath]);
 
+  // 360 means we don't modify rotation
+  const isRotationSet = rotation !== 360;
+  const effectiveRotation = isRotationSet ? rotation : (mainVideoStream && mainVideoStream.tags && mainVideoStream.tags.rotate && parseInt(mainVideoStream.tags.rotate, 10));
+
   const zoomRel = useCallback((rel) => setZoom(z => Math.min(Math.max(z + rel, 1), zoomMax)), []);
-  const frameRenderEnabled = !!(rotationPreviewRequested || dummyVideoPath);
+  const canvasPlayerEnabled = !!(mainVideoStream && (isRotationSet || dummyVideoPath));
 
   const comfortZoom = duration ? Math.max(duration / 100, 1) : undefined;
   const toggleComfortZoom = useCallback(() => {
@@ -565,38 +563,6 @@ const App = memo(() => {
     save();
   }, [debouncedCutSegments, edlFilePath, autoSaveProjectFile]);
 
-  // 360 means we don't modify rotation
-  const isRotationSet = rotation !== 360;
-  const effectiveRotation = isRotationSet ? rotation : undefined;
-
-  useEffect(() => {
-    async function throttledRender() {
-      if (queue.size < 2) {
-        queue.add(async () => {
-          if (!frameRenderEnabled) return;
-
-          if (playerTime == null || !filePath) return;
-
-          try {
-            const framePathNew = await renderFrame(playerTime, filePath, effectiveRotation);
-            setFramePath(framePathNew);
-          } catch (err) {
-            console.error(err);
-          }
-        });
-      }
-
-      await queue.onIdle();
-    }
-
-    throttledRender();
-  }, [
-    filePath, playerTime, frameRenderEnabled, effectiveRotation,
-  ]);
-
-  // Cleanup old
-  useEffect(() => () => URL.revokeObjectURL(framePath), [framePath]);
-
   function onPlayingChange(val) {
     setPlaying(val);
     if (!val) {
@@ -614,13 +580,11 @@ const App = memo(() => {
   const onTimeUpdate = useCallback((e) => {
     const { currentTime } = e.target;
     if (playerTime === currentTime) return;
-    setRotationPreviewRequested(false); // Reset this
     setPlayerTime(currentTime);
   }, [playerTime]);
 
   const increaseRotation = useCallback(() => {
     setRotation((r) => (r + 90) % 450);
-    setRotationPreviewRequested(true);
   }, []);
 
   const assureOutDirAccess = useCallback(async (outFilePath) => {
@@ -812,7 +776,6 @@ const App = memo(() => {
     video.playbackRate = 1;
 
     setFileNameTitle();
-    setFramePath();
     setHtml5FriendlyPath();
     setDummyVideoPath();
     setWorking(false);
@@ -830,7 +793,6 @@ const App = memo(() => {
     setRotation(360);
     setCutProgress();
     setStartTimeOffset(0);
-    setRotationPreviewRequested(false);
     setFilePath(''); // Setting video src="" prevents memory leak in chromium
     setExternalStreamFiles([]);
     setDetectedFps();
@@ -1015,7 +977,7 @@ const App = memo(() => {
         outFormat: fileFormat,
         isCustomFormatSelected,
         videoDuration: duration,
-        rotation: effectiveRotation,
+        rotation: isRotationSet ? effectiveRotation : undefined,
         copyFileStreams,
         keyframeCut,
         segments: outSegments,
@@ -1062,7 +1024,7 @@ const App = memo(() => {
       setWorking(false);
     }
   }, [
-    effectiveRotation, outSegments, handleCutFailed,
+    effectiveRotation, outSegments, handleCutFailed, isRotationSet,
     working, duration, filePath, keyframeCut,
     autoMerge, customOutDir, fileFormat, haveInvalidSegs, copyFileStreams, numStreamsToCopy,
     exportExtraStreams, nonCopiedExtraStreams, outputDir, shortestFlag, isCustomFormatSelected,
@@ -1075,7 +1037,7 @@ const App = memo(() => {
       const currentTime = currentTimeRef.current;
       const video = videoRef.current;
       const outPath = mustCaptureFfmpeg
-        ? await captureFrameFfmpeg({ customOutDir, videoPath: filePath, currentTime, captureFormat, duration: video.duration })
+        ? await captureFrameFfmpeg({ customOutDir, videoPath: filePath, currentTime, captureFormat, duration })
         : await captureFrameFromTag({ customOutDir, filePath, video, currentTime, captureFormat });
 
       openDirToast({ dirPath: outputDir, text: `${i18n.t('Screenshot captured to:')} ${outPath}` });
@@ -1083,7 +1045,7 @@ const App = memo(() => {
       console.error(err);
       errorToast(i18n.t('Failed to capture frame'));
     }
-  }, [filePath, captureFormat, customOutDir, html5FriendlyPath, dummyVideoPath, outputDir]);
+  }, [filePath, captureFormat, customOutDir, html5FriendlyPath, dummyVideoPath, outputDir, duration]);
 
   const changePlaybackRate = useCallback((dir) => {
     const video = videoRef.current;
@@ -1825,24 +1787,15 @@ const App = memo(() => {
           onError={onVideoError}
         />
 
-        {framePath && frameRenderEnabled && (
-          <img
-            draggable={false}
-            style={{
-              width: '100%', height: '100%', objectFit: 'contain', left: 0, right: 0, top: 0, bottom: 0, position: 'absolute', background: 'black',
-            }}
-            src={framePath}
-            alt=""
-          />
-        )}
+        {canvasPlayerEnabled && <Canvas rotate={effectiveRotation} filePath={filePath} width={mainVideoStream.width} height={mainVideoStream.height} playerTime={playerTime} commandedTime={commandedTime} playing={playing} />}
       </div>
 
-      {rotationPreviewRequested && (
+      {isRotationSet && (
         <div style={{
           position: 'absolute', top: topBarHeight, marginTop: '1em', marginRight: '1em', right: sideBarWidth, color: 'white',
         }}
         >
-          {t('Lossless rotation preview')}
+          {t('Rotation preview')}
         </div>
       )}
 

src/Canvas.jsx

@@ -0,0 +1,33 @@
+
+import React, { memo, useEffect, useRef, useMemo } from 'react';
+
+import CanvasPlayer from './CanvasPlayer';
+
+const Canvas = memo(({ rotate, filePath, width, height, playerTime, commandedTime, playing }) => {
+  const canvasRef = useRef();
+
+  const canvasPlayer = useMemo(() => CanvasPlayer({ path: filePath, width, height }),
+    [filePath, width, height]);
+
+  useEffect(() => {
+    canvasPlayer.setCanvas(canvasRef.current);
+
+    return () => {
+      canvasPlayer.setCanvas();
+      if (canvasPlayer) canvasPlayer.dispose();
+    };
+  }, [canvasPlayer]);
+
+  useEffect(() => {
+    if (playing) canvasPlayer.play(commandedTime);
+    else canvasPlayer.pause(playerTime);
+  }, [canvasPlayer, commandedTime, playerTime, playing]);
+
+  return (
+    <div style={{ width: '100%', height: '100%', left: 0, right: 0, top: 0, bottom: 0, position: 'absolute', overflow: 'hidden', background: 'black' }}>
+      <canvas ref={canvasRef} style={{ display: 'block', width: '100%', height: '100%', objectFit: 'contain', transform: `rotate(${rotate}deg)` }} />
+    </div>
+  );
+});
+
+export default Canvas;

src/CanvasPlayer.js

@@ -0,0 +1,105 @@
+import { encodeLiveRawStream, getOneRawFrame } from './ffmpeg';
+
+// TODO keep everything in electron land?
+const strtok3 = window.require('strtok3');
+
+export default ({ path, width: inWidth, height: inHeight }) => {
+  let canvas;
+
+  let terminated;
+  let cancel;
+  let commandedTime;
+  let playing;
+
+  function drawOnCanvas(rgbaImage, width, height) {
+    if (!canvas || rgbaImage.length === 0) return;
+
+    canvas.width = width;
+    canvas.height = height;
+
+    const ctx = canvas.getContext('2d');
+    // https://developer.mozilla.org/en-US/docs/Web/API/ImageData/ImageData
+    // https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/putImageData
+    ctx.putImageData(new ImageData(Uint8ClampedArray.from(rgbaImage), width, height), 0, 0);
+  }
+
+  async function run() {
+    let process;
+    let cancelled;
+
+    cancel = () => {
+      cancelled = true;
+      if (process) process.cancel();
+      cancel = undefined;
+    };
+
+    if (playing) {
+      try {
+        const { process: processIn, channels, width, height } = encodeLiveRawStream({ path, inWidth, inHeight, seekTo: commandedTime });
+        process = processIn;
+
+        // process.stderr.on('data', data => console.log(data.toString('utf-8')));
+
+        const tokenizer = await strtok3.fromStream(process.stdout);
+
+        const size = width * height * channels;
+        const buf = Buffer.allocUnsafe(size);
+
+        while (!cancelled) {
+          // eslint-disable-next-line no-await-in-loop
+          await tokenizer.readBuffer(buf, { length: size });
+          if (!cancelled) drawOnCanvas(buf, width, height);
+        }
+      } catch (err) {
+        if (!err.isCanceled) console.warn(err.message);
+      }
+    } else {
+      try {
+        const { process: processIn, width, height } = getOneRawFrame({ path, inWidth, inHeight, seekTo: commandedTime });
+        process = processIn;
+        const { stdout: rgbaImage } = await process;
+
+        if (!cancelled) drawOnCanvas(rgbaImage, width, height);
+      } catch (err) {
+        if (!err.isCanceled) console.warn(err.message);
+      }
+    }
+  }
+
+  function command() {
+    if (cancel) cancel();
+    run();
+  }
+
+  function pause(seekTo) {
+    if (terminated) return;
+    if (!playing && commandedTime === seekTo) return;
+    playing = false;
+    commandedTime = seekTo;
+    command();
+  }
+
+  function play(playFrom) {
+    if (terminated) return;
+    if (playing && commandedTime === playFrom) return;
+    playing = true;
+    commandedTime = playFrom;
+    command();
+  }
+
+  function setCanvas(c) {
+    canvas = c;
+  }
+
+  function dispose() {
+    terminated = true;
+    if (cancel) cancel();
+  }
+
+  return {
+    play,
+    pause,
+    setCanvas,
+    dispose,
+  };
+};

src/ffmpeg.js

@@ -217,7 +217,7 @@ async function cut({
     ...cutToArgs,
   ];
 
-  const rotationArgs = rotation !== undefined ? ['-metadata:s:v:0', `rotate=${rotation}`] : [];
+  const rotationArgs = rotation !== undefined ? ['-metadata:s:v:0', `rotate=${360 - rotation}`] : [];
 
   const ffmpegArgs = [
     '-hide_banner',
@@ -310,6 +310,7 @@ export async function cutMultiple({
   return outFiles;
 }
 
+// TODO merge with getFormatData
 export async function getDuration(filePath) {
   // https://superuser.com/questions/650291/how-to-get-video-duration-in-seconds
   const { stdout } = await runFfprobe(['-i', filePath, '-show_entries', 'format=duration', '-print_format', 'json']);
@@ -648,35 +649,6 @@ export async function renderWaveformPng({ filePath, aroundTime, window, color })
   }
 }
 
-export async function renderFrame(timestamp, filePath, rotation) {
-  const transpose = {
-    90: 'transpose=2',
-    180: 'transpose=1,transpose=1',
-    270: 'transpose=1',
-  };
-  const args = [
-    '-ss', timestamp,
-    ...(rotation !== undefined ? ['-noautorotate'] : []),
-    '-i', filePath,
-    // ...(rotation !== undefined ? ['-metadata:s:v:0', 'rotate=0'] : []), // Reset the rotation metadata first
-    ...(rotation !== undefined && rotation > 0 ? ['-vf', `${transpose[rotation]}`] : []),
-    '-f', 'image2',
-    '-vframes', '1',
-    '-q:v', '10',
-    '-',
-    // '-y', outPath,
-  ];
-
-  // console.time('ffmpeg');
-  const ffmpegPath = getFfmpegPath();
-  // console.timeEnd('ffmpeg');
-  // console.log('ffmpeg', args);
-  const { stdout } = await execa(ffmpegPath, args, { encoding: null });
-
-  const blob = new Blob([stdout], { type: 'image/jpeg' });
-  return URL.createObjectURL(blob);
-}
-
 export async function extractWaveform({ filePath, outPath }) {
   const numSegs = 10;
   const duration = 60 * 60;
@@ -739,3 +711,66 @@ export function getStreamFps(stream) {
   }
   return undefined;
 }
+
+function createRawFfmpeg({ fps = 25, path, inWidth, inHeight, seekTo, oneFrameOnly, execaOpts }) {
+  // const fps = 25; // TODO
+
+  const aspectRatio = inWidth / inHeight;
+
+  let newWidth;
+  let newHeight;
+  if (inWidth > inHeight) {
+    newWidth = 320;
+    newHeight = Math.floor(newWidth / aspectRatio);
+  } else {
+    newHeight = 320;
+    newWidth = Math.floor(newHeight * aspectRatio);
+  }
+
+  const args = [
+    '-hide_banner', '-loglevel', 'panic',
+
+    '-re',
+
+    '-ss', seekTo,
+
+    '-noautorotate',
+
+    '-i', path,
+
+    '-vf', `fps=${fps},scale=${newWidth}:${newHeight}:flags=lanczos`,
+    '-map', 'v:0',
+    '-vcodec', 'rawvideo',
+    '-pix_fmt', 'rgba',
+
+    ...(oneFrameOnly ? ['-frames:v', '1'] : []),
+
+    '-f', 'image2pipe',
+    '-',
+  ];
+
+  // console.log(args);
+
+  return {
+    process: execa(getFfmpegPath(), args, execaOpts),
+    width: newWidth,
+    height: newHeight,
+    channels: 4,
+  };
+}
+
+export function getOneRawFrame({ path, inWidth, inHeight, seekTo }) {
+  const { process, width, height, channels } = createRawFfmpeg({ path, inWidth, inHeight, seekTo, oneFrameOnly: true, execaOpts: { encoding: null } });
+  return { process, width, height, channels };
+}
+
+export function encodeLiveRawStream({ path, inWidth, inHeight, seekTo }) {
+  const { process, width, height, channels } = createRawFfmpeg({ path, inWidth, inHeight, seekTo, execaOpts: { encoding: null, buffer: false } });
+
+  return {
+    process,
+    width,
+    height,
+    channels,
+  };
+}

yarn.lock

@@ -1544,6 +1544,11 @@
   dependencies:
     defer-to-connect "^1.0.1"
 
+"@tokenizer/token@^0.1.1":
+  version "0.1.1"
+  resolved "https://registry.yarnpkg.com/@tokenizer/token/-/token-0.1.1.tgz#f0d92c12f87079ddfd1b29f614758b9696bc29e3"
+  integrity sha512-XO6INPbZCxdprl+9qa/AAbFFOMzzwqYxpjPgLICrMD6C2FCw6qfJOPcBk6JqqPLSaZ/Qx87qn4rpPmPMwaAK6w==
+
 "@types/babel__core@^7.1.0":
   version "7.1.6"
   resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.1.6.tgz#16ff42a5ae203c9af1c6e190ed1f30f83207b610"
@@ -8973,14 +8978,6 @@ p-map@^3.0.0:
   dependencies:
     aggregate-error "^3.0.0"
 
-p-queue@^6.2.0:
-  version "6.2.0"
-  resolved "https://registry.yarnpkg.com/p-queue/-/p-queue-6.2.0.tgz#c8122b9514d2bbe5f16d8a47f17dc2f9a8ac7235"
-  integrity sha512-B2LXNONcyn/G6uz2UBFsGjmSa0e/br3jznlzhEyCXg56c7VhEpiT2pZxGOfv32Q3FSyugAdys9KGpsv3kV+Sbg==
-  dependencies:
-    eventemitter3 "^4.0.0"
-    p-timeout "^3.1.0"
-
 p-reduce@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/p-reduce/-/p-reduce-1.0.0.tgz#18c2b0dd936a4690a529f8231f58a0fdb6a47dfa"
@@ -8993,13 +8990,6 @@ p-retry@^3.0.1:
   dependencies:
     retry "^0.12.0"
 
-p-timeout@^3.1.0:
-  version "3.2.0"
-  resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-3.2.0.tgz#c7e17abc971d2a7962ef83626b35d635acf23dfe"
-  integrity sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==
-  dependencies:
-    p-finally "^1.0.0"
-
 p-try@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/p-try/-/p-try-1.0.0.tgz#cbc79cdbaf8fd4228e13f621f2b1a237c1b207b3"
@@ -9243,6 +9233,11 @@ pbkdf2@^3.0.3:
     safe-buffer "^5.0.1"
     sha.js "^2.4.8"
 
+peek-readable@^3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/peek-readable/-/peek-readable-3.1.0.tgz#250b08b7de09db8573d7fd8ea475215bbff14348"
+  integrity sha512-KGuODSTV6hcgdZvDrIDBUkN0utcAVj1LL7FfGbM0viKTtCHmtZcuEJ+lGqsp0fTFkGqesdtemV2yUSMeyy3ddA==
+
 pend@~1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50"
@@ -12026,6 +12021,16 @@ strong-data-uri@^1.0.5:
   dependencies:
     truncate "^2.0.1"
 
+strtok3@^6.0.0:
+  version "6.0.0"
+  resolved "https://registry.yarnpkg.com/strtok3/-/strtok3-6.0.0.tgz#d6b900863daeacfe6c1724c6e7bb36d7a58e83c8"
+  integrity sha512-ZXlmE22LZnIBvEU3n/kZGdh770fYFie65u5+2hLK9s74DoFtpkQIdBZVeYEzlolpGa+52G5IkzjUWn+iXynOEQ==
+  dependencies:
+    "@tokenizer/token" "^0.1.1"
+    "@types/debug" "^4.1.5"
+    debug "^4.1.1"
+    peek-readable "^3.1.0"
+
 style-loader@0.23.1:
   version "0.23.1"
   resolved "https://registry.yarnpkg.com/style-loader/-/style-loader-0.23.1.tgz#cb9154606f3e771ab6c4ab637026a1049174d925"