Implement audio waveform #6

also try to improve zooming

.gitattributes

@@ -1 +0,0 @@
-src/font-awesome*/** linguist-vendored

package.json

@@ -61,7 +61,7 @@
     "electron-is-dev": "^0.1.2",
     "electron-store": "^5.1.0",
     "evergreen-ui": "^4.23.0",
-    "execa": "^0.5.0",
+    "execa": "^4.0.0",
     "ffmpeg-static": "^4.0.1",
     "ffprobe-static": "^3.0.0",
     "file-type": "^12.4.0",

src/LeftMenu.jsx

@@ -7,7 +7,7 @@ const { withBlur } = require('./util');
 
 
 const LeftMenu = memo(({ zoom, setZoom, invertCutSegments, setInvertCutSegments }) => (
-  <div className="no-user-select" style={{ position: 'absolute', left: 0, bottom: 0, padding: '.3em', display: 'flex', alignItems: 'center' }}>
+  <div className="no-user-select" style={{ padding: '.3em', display: 'flex', alignItems: 'center' }}>
     <div style={{ marginLeft: 5 }}>
       <motion.div
         animate={{ rotateX: invertCutSegments ? 0 : 180, width: 26, height: 26 }}

src/RightMenu.jsx

@@ -15,7 +15,7 @@ const RightMenu = memo(({
   const CutIcon = areWeCutting ? FiScissors : FaFileExport;
 
   return (
-    <div className="no-user-select" style={{ position: 'absolute', right: 0, bottom: 0, padding: '.3em', display: 'flex', alignItems: 'center' }}>
+    <div className="no-user-select" style={{ padding: '.3em', display: 'flex', alignItems: 'center' }}>
       <div>
         <span style={{ width: 40, textAlign: 'right', display: 'inline-block' }}>{isRotationSet && rotationStr}</span>
         <MdRotate90DegreesCcw

src/SegmentList.jsx

@@ -65,7 +65,7 @@ const SegmentList = memo(({
           <FaAngleRight
             title="Close sidebar"
             size={18}
-            style={{ verticalAlign: 'middle' }}
+            style={{ verticalAlign: 'middle', color: 'white' }}
             role="button"
             onClick={toggleSideBar}
           />

src/Timeline.jsx

@@ -1,4 +1,4 @@
-import React, { memo, useRef, useMemo, useCallback, useEffect } from 'react';
+import React, { memo, useRef, useMemo, useCallback, useEffect, useState } from 'react';
 import { motion } from 'framer-motion';
 import Hammer from 'react-hammerjs';
 
@@ -12,10 +12,33 @@ import { getSegColors } from './util';
 
 const hammerOptions = { recognizers: {} };
 
+const Waveform = memo(({ calculateTimelinePos, durationSafe, waveform, zoom, timelineHeight }) => {
+  const imgRef = useRef();
+  const [style, setStyle] = useState({ display: 'none' });
+
+  const leftPos = calculateTimelinePos(waveform.from);
+
+  const toTruncated = Math.min(waveform.to, durationSafe);
+
+  // Prevents flash
+  function onLoad() {
+    setStyle({
+      position: 'absolute', height: '100%', left: leftPos, width: `${((toTruncated - waveform.from) / durationSafe) * 100}%`,
+    });
+  }
+
+  return (
+    <div style={{ height: timelineHeight, width: `${zoom * 100}%`, position: 'relative' }}>
+      <img ref={imgRef} src={waveform.url} draggable={false} style={style} alt="" onLoad={onLoad} />
+    </div>
+  );
+});
+
 const Timeline = memo(({
   durationSafe, getCurrentTime, startTimeOffset, playerTime, commandedTime,
   zoom, neighbouringFrames, seekAbs, seekRel, duration, apparentCutSegments, zoomRel,
   setCurrentSegIndex, currentSegIndexSafe, invertCutSegments, inverseCutSegments, mainVideoStream, formatTimecode,
+  waveform, shouldShowWaveform, shouldShowKeyframes, timelineHeight, timelineExpanded,
 }) => {
   const timelineScrollerRef = useRef();
   const timelineScrollerSkipEventRef = useRef();
@@ -23,6 +46,7 @@ const Timeline = memo(({
 
   const offsetCurrentTime = (getCurrentTime() || 0) + startTimeOffset;
 
+
   const calculateTimelinePos = useCallback((time) => (time !== undefined && time < durationSafe ? `${(time / durationSafe) * 100}%` : undefined), [durationSafe]);
 
   const currentTimePos = useMemo(() => calculateTimelinePos(playerTime), [calculateTimelinePos, playerTime]);
@@ -33,13 +57,13 @@ const Timeline = memo(({
   const currentTimeWidth = 1;
   // Prevent it from overflowing (and causing scroll) when end of timeline
 
-  const shouldShowKeyframes = neighbouringFrames.length >= 2 && (neighbouringFrames[neighbouringFrames.length - 1].time - neighbouringFrames[0].time) / durationSafe > (0.1 / zoom);
+  // Keep cursor in view while scrolling
-
   useEffect(() => {
     timelineScrollerSkipEventRef.current = true;
     if (zoom > 1) {
-      timelineScrollerRef.current.scrollLeft = (getCurrentTime() / durationSafe)
+      const zoomedTargetWidth = timelineScrollerRef.current.offsetWidth * (zoom - 1);
-        * (timelineWrapperRef.current.offsetWidth - timelineScrollerRef.current.offsetWidth);
+
+      timelineScrollerRef.current.scrollLeft = (getCurrentTime() / durationSafe) * zoomedTargetWidth;
     }
   }, [zoom, durationSafe, getCurrentTime]);
 
@@ -49,9 +73,9 @@ const Timeline = memo(({
       return;
     }
     if (!zoomed) return;
-    seekAbs((((e.target.scrollLeft + (timelineScrollerRef.current.offsetWidth / 2))
+    seekAbs((((e.target.scrollLeft + (timelineScrollerRef.current.offsetWidth * 0.5))
-      / timelineWrapperRef.current.offsetWidth) * duration));
+      / (timelineScrollerRef.current.offsetWidth * zoom)) * duration));
-  }, [duration, seekAbs, zoomed]);
+  }, [duration, seekAbs, zoomed, zoom]);
 
   const handleTap = useCallback((e) => {
     const target = timelineWrapperRef.current;
@@ -81,8 +105,18 @@ const Timeline = memo(({
           onScroll={onTimelineScroll}
           ref={timelineScrollerRef}
         >
+          {timelineExpanded && shouldShowWaveform && waveform && (
+            <Waveform
+              calculateTimelinePos={calculateTimelinePos}
+              durationSafe={durationSafe}
+              waveform={waveform}
+              zoom={zoom}
+              timelineHeight={timelineHeight}
+            />
+          )}
+
           <div
-            style={{ height: 36, width: `${zoom * 100}%`, position: 'relative', backgroundColor: timelineBackground }}
+            style={{ height: timelineHeight, width: `${zoom * 100}%`, position: 'relative', backgroundColor: timelineBackground }}
             ref={timelineWrapperRef}
           >
             {currentTimePos !== undefined && <motion.div transition={{ type: 'spring', damping: 70, stiffness: 800 }} animate={{ left: currentTimePos }} style={{ position: 'absolute', bottom: 0, top: 0, zIndex: 3, backgroundColor: 'black', width: currentTimeWidth, pointerEvents: 'none' }} />}
@@ -129,7 +163,11 @@ const Timeline = memo(({
           </div>
         </div>
 
-        <div style={{ position: 'absolute', left: 0, right: 0, top: 0, bottom: 0, display: 'flex', alignItems: 'center', justifyContent: 'center', pointerEvents: 'none' }}>
+        {timelineExpanded && !shouldShowWaveform && (
+          <div style={{ position: 'absolute', display: 'flex', alignItems: 'center', justifyContent: 'center', height: timelineHeight, bottom: timelineHeight, left: 0, right: 0, color: 'rgba(255,255,255,0.6)' }}>Zoom in more to view waveform</div>
+        )}
+
+        <div style={{ position: 'absolute', height: timelineHeight, left: 0, right: 0, bottom: 0, display: 'flex', alignItems: 'center', justifyContent: 'center', pointerEvents: 'none' }}>
           <div style={{ background: 'rgba(0,0,0,0.4)', borderRadius: 3, padding: '2px 4px', color: 'rgba(255, 255, 255, 0.8)' }}>
             {formatTimecode(offsetCurrentTime)}
           </div>

src/TimelineControls.jsx

@@ -1,5 +1,6 @@
 import React, { memo } from 'react';
 import { FaHandPointLeft, FaHandPointRight, FaStepBackward, FaStepForward, FaCaretLeft, FaCaretRight, FaPause, FaPlay } from 'react-icons/fa';
+import { GiSoundWaves } from 'react-icons/gi';
 // import useTraceUpdate from 'use-trace-update';
 
 import { getSegColors, parseDuration, formatDuration } from './util';
@@ -9,7 +10,7 @@ const TimelineControls = memo(({
   seekAbs, currentSegIndexSafe, cutSegments, currentCutSeg, setCutStart, setCutEnd,
   setCurrentSegIndex, cutStartTimeManual, setCutStartTimeManual, cutEndTimeManual, setCutEndTimeManual,
   duration, jumpCutEnd, jumpCutStart, startTimeOffset, setCutTime, currentApparentCutSeg,
-  playing, shortStep, playCommand,
+  playing, shortStep, playCommand, setTimelineExpanded, hasAudio,
 }) => {
   const {
     segActiveBgColor: currentSegActiveBgColor,
@@ -108,8 +109,24 @@ const TimelineControls = memo(({
 
   const PlayPause = playing ? FaPause : FaPlay;
 
+  const leftRightWidth = 50;
+
   return (
-    <div style={{ display: 'flex', justifyContent: 'center', alignItems: 'center' }}>
+    <div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}>
+      <div style={{ display: 'flex', alignItems: 'center', flexBasis: leftRightWidth }}>
+        {hasAudio && (
+          <GiSoundWaves
+            size={24}
+            style={{ padding: '0 5px' }}
+            role="button"
+            title="Expand timeline"
+            onClick={() => setTimelineExpanded(v => !v)}
+          />
+        )}
+      </div>
+
+      <div style={{ flexGrow: 1 }} />
+
       <FaStepBackward
         size={16}
         title="Jump to start of video"
@@ -157,6 +174,10 @@ const TimelineControls = memo(({
         role="button"
         onClick={() => seekAbs(duration)}
       />
+
+      <div style={{ flexGrow: 1 }} />
+
+      <div style={{ flexBasis: leftRightWidth }} />
     </div>
   );
 });

src/colors.js

@@ -1,4 +1,5 @@
 export const saveColor = 'hsl(158, 100%, 43%)';
 export const primaryColor = 'hsl(194, 78%, 47%)';
+export const waveformColor = '#ffffff'; // Must be hex because used by ffmpeg
 export const controlsBackground = '#6b6b6b';
 export const timelineBackground = '#444';

src/ffmpeg.js

@@ -97,8 +97,19 @@ function getExtensionForFormat(format) {
   return ext || format;
 }
 
-async function readFrames({ filePath, aroundTime, window = 30, stream }) {
+function getIntervalAroundTime(time, window) {
-  const intervalsArgs = aroundTime != null ? ['-read_intervals', `${Math.max(aroundTime - window, 0)}%${aroundTime + window}`] : [];
+  return {
+    from: Math.max(time - window / 2, 0),
+    to: time + window / 2,
+  };
+}
+
+async function readFrames({ filePath, aroundTime, window, stream }) {
+  let intervalsArgs = [];
+  if (aroundTime != null) {
+    const { from, to } = getIntervalAroundTime(aroundTime, window);
+    intervalsArgs = ['-read_intervals', `${from}%${to}`];
+  }
   const { stdout } = await runFfprobe(['-v', 'error', ...intervalsArgs, '-show_packets', '-select_streams', stream, '-show_entries', 'packet=pts_time,flags', '-of', 'json', filePath]);
   return sortBy(JSON.parse(stdout).packets.map(p => ({ keyframe: p.flags[0] === 'K', time: parseFloat(p.pts_time, 10) })), 'time');
 }
@@ -475,6 +486,57 @@ async function extractStreams({ filePath, customOutDir, streams }) {
   console.log(stdout);
 }
 
+async function renderWaveformPng({ filePath, aroundTime, window, color }) {
+  const { from, to } = getIntervalAroundTime(aroundTime, window);
+
+  const args1 = [
+    '-i', filePath,
+    '-ss', from,
+    '-t', to - from,
+    '-c', 'copy',
+    '-vn',
+    '-map', 'a:0',
+    '-f', 'matroska', // mpegts doesn't support vorbis etc
+    '-',
+  ];
+
+  const args2 = [
+    '-i', '-',
+    '-filter_complex', `aformat=channel_layouts=mono,showwavespic=s=640x120:scale=sqrt:colors=${color}`,
+    '-frames:v', '1',
+    '-vcodec', 'png',
+    '-f', 'image2',
+    '-',
+  ];
+
+  console.log(getFfCommandLine('ffmpeg1', args1));
+  console.log(getFfCommandLine('ffmpeg2', args2));
+
+  let ps1;
+  let ps2;
+  try {
+    const ffmpegPath = getFfmpegPath();
+    ps1 = execa(ffmpegPath, args1, { encoding: null, buffer: false });
+    ps2 = execa(ffmpegPath, args2, { encoding: null });
+    ps1.stdout.pipe(ps2.stdin);
+
+    const { stdout } = await ps2;
+
+    const blob = new Blob([stdout], { type: 'image/png' });
+
+    return {
+      url: URL.createObjectURL(blob),
+      from,
+      aroundTime,
+      to,
+    };
+  } catch (err) {
+    if (ps1) ps1.kill();
+    if (ps2) ps2.kill();
+    throw err;
+  }
+}
+
 async function renderFrame(timestamp, filePath, rotation) {
   const transpose = {
     90: 'transpose=2',
@@ -501,8 +563,7 @@ async function renderFrame(timestamp, filePath, rotation) {
   const { stdout } = await execa(ffmpegPath, args, { encoding: null });
 
   const blob = new Blob([stdout], { type: 'image/jpeg' });
-  const url = URL.createObjectURL(blob);
+  return URL.createObjectURL(blob);
-  return url;
 }
 
 // https://www.ffmpeg.org/doxygen/3.2/libavutil_2utils_8c_source.html#l00079
@@ -540,4 +601,5 @@ module.exports = {
   isCuttingEnd,
   readFrames,
   getNextPrevKeyframe,
+  renderWaveformPng,
 };

src/renderer.jsx

@@ -24,7 +24,7 @@ import Timeline from './Timeline';
 import RightMenu from './RightMenu';
 import TimelineControls from './TimelineControls';
 import { loadMifiLink } from './mifi';
-import { primaryColor, controlsBackground } from './colors';
+import { primaryColor, controlsBackground, waveformColor } from './colors';
 
 import loadingLottie from './7077-magic-flow.json';
 
@@ -96,19 +96,28 @@ function doesPlayerSupportFile(streams) {
   // return true;
 }
 
+const ffmpegExtractWindow = 60;
+const calcShouldShowWaveform = (duration, zoom) => (duration != null && duration / zoom < ffmpegExtractWindow * 8);
+const calcShouldShowKeyframes = (duration, zoom) => (duration != null && duration / zoom < ffmpegExtractWindow * 8);
+
+
 const commonFormats = ['mov', 'mp4', 'matroska', 'mp3', 'ipod'];
 
 
-const topBarHeight = '2rem';
+// TODO flex
-const bottomBarHeight = '6rem';
+const topBarHeight = 32;
+const timelineHeight = 36;
 const zoomMax = 2 ** 14;
 
+const videoStyle = { width: '100%', height: '100%', objectFit: 'contain' };
+
 
 const queue = new PQueue({ concurrency: 1 });
 
 const App = memo(() => {
   // Per project state
   const [framePath, setFramePath] = useState();
+  const [waveform, setWaveform] = useState();
   const [html5FriendlyPath, setHtml5FriendlyPath] = useState();
   const [working, setWorking] = useState(false);
   const [dummyVideoPath, setDummyVideoPath] = useState(false);
@@ -127,14 +136,17 @@ const App = memo(() => {
   const [detectedFps, setDetectedFps] = useState();
   const [mainStreams, setMainStreams] = useState([]);
   const [mainVideoStream, setMainVideoStream] = useState();
+  const [mainAudioStream, setMainAudioStream] = useState();
   const [copyStreamIdsByFile, setCopyStreamIdsByFile] = useState({});
   const [streamsSelectorShown, setStreamsSelectorShown] = useState(false);
   const [zoom, setZoom] = useState(1);
   const [commandedTime, setCommandedTime] = useState(0);
-  const [debouncedCommandedTime, setDebouncedCommandedTime] = useState(0);
   const [ffmpegCommandLog, setFfmpegCommandLog] = useState([]);
   const [neighbouringFrames, setNeighbouringFrames] = useState([]);
   const [shortestFlag, setShortestFlag] = useState(false);
+  const [debouncedWaveformData, setDebouncedWaveformData] = useState();
+  const [debouncedReadKeyframesData, setDebouncedReadKeyframesData] = useState();
+  const [timelineExpanded, setTimelineExpanded] = useState(false);
 
   const [showSideBar, setShowSideBar] = useState(true);
 
@@ -150,14 +162,19 @@ const App = memo(() => {
     createInitialCutSegments(),
   );
 
-  const [, cancelCommandedTimeDebounce] = useDebounce(() => {
-    setDebouncedCommandedTime(commandedTime);
-  }, 300, [commandedTime]);
-
   const [, cancelCutSegmentsDebounce] = useDebounce(() => {
     setDebouncedCutSegments(cutSegments);
   }, 500, [cutSegments]);
 
+  const durationSafe = duration || 1;
+
+  const [, cancelWaveformDataDebounce] = useDebounce(() => {
+    setDebouncedWaveformData({ filePath, commandedTime, duration, zoom, timelineExpanded, mainAudioStream });
+  }, 500, [filePath, commandedTime, duration, zoom, timelineExpanded, mainAudioStream]);
+
+  const [, cancelReadKeyframeDataDebounce] = useDebounce(() => {
+    setDebouncedReadKeyframesData({ filePath, commandedTime, duration, zoom, mainVideoStream });
+  }, 500, [filePath, commandedTime, duration, zoom, mainVideoStream]);
 
   // Preferences
   const [captureFormat, setCaptureFormat] = useState(configStore.get('captureFormat'));
@@ -188,8 +205,55 @@ const App = memo(() => {
   const videoRef = useRef();
   const lastSavedCutSegmentsRef = useRef();
   const readingKeyframesPromise = useRef();
+  const creatingWaveformPromise = useRef();
   const currentTimeRef = useRef();
 
+  const resetState = useCallback(() => {
+    const video = videoRef.current;
+    setCommandedTime(0);
+    video.currentTime = 0;
+    video.playbackRate = 1;
+
+    setFileNameTitle();
+    setFramePath();
+    setHtml5FriendlyPath();
+    setDummyVideoPath();
+    setWorking(false);
+    setPlaying(false);
+    setDuration();
+    cutSegmentsHistory.go(0);
+    cancelCutSegmentsDebounce(); // TODO auto save when loading new file/closing file
+    setDebouncedCutSegments(createInitialCutSegments());
+    setCutSegments(createInitialCutSegments()); // TODO this will cause two history items
+    setCutStartTimeManual();
+    setCutEndTimeManual();
+    setFileFormat();
+    setFileFormatData();
+    setDetectedFileFormat();
+    setRotation(360);
+    setCutProgress();
+    setStartTimeOffset(0);
+    setRotationPreviewRequested(false);
+    setFilePath(''); // Setting video src="" prevents memory leak in chromium
+    setExternalStreamFiles([]);
+    setDetectedFps();
+    setMainStreams([]);
+    setMainVideoStream();
+    setMainAudioStream();
+    setCopyStreamIdsByFile({});
+    setStreamsSelectorShown(false);
+    setZoom(1);
+    setShortestFlag(false);
+
+    setWaveform();
+    cancelWaveformDataDebounce();
+    setDebouncedWaveformData();
+
+    setNeighbouringFrames([]);
+    cancelReadKeyframeDataDebounce();
+    setDebouncedReadKeyframesData();
+  }, [cutSegmentsHistory, cancelCutSegmentsDebounce, setCutSegments, cancelWaveformDataDebounce, cancelReadKeyframeDataDebounce]);
+
   function appendFfmpegCommandLog(command) {
     setFfmpegCommandLog(old => [...old, { command, time: new Date() }]);
   }
@@ -242,45 +306,6 @@ const App = memo(() => {
     seekRel((1 / (detectedFps || 60)) * dir);
   }, [seekRel, detectedFps]);
 
-  const resetState = useCallback(() => {
-    const video = videoRef.current;
-    cancelCommandedTimeDebounce();
-    setDebouncedCommandedTime(0);
-    setCommandedTime(0);
-    video.currentTime = 0;
-    video.playbackRate = 1;
-
-    setFileNameTitle();
-    setFramePath();
-    setHtml5FriendlyPath();
-    setDummyVideoPath();
-    setWorking(false);
-    setPlaying(false);
-    setDuration();
-    cutSegmentsHistory.go(0);
-    cancelCutSegmentsDebounce(); // TODO auto save when loading new file/closing file
-    setDebouncedCutSegments(createInitialCutSegments());
-    setCutSegments(createInitialCutSegments()); // TODO this will cause two history items
-    setCutStartTimeManual();
-    setCutEndTimeManual();
-    setFileFormat();
-    setFileFormatData();
-    setDetectedFileFormat();
-    setRotation(360);
-    setCutProgress();
-    setStartTimeOffset(0);
-    setRotationPreviewRequested(false);
-    setFilePath(''); // Setting video src="" prevents memory leak in chromium
-    setExternalStreamFiles([]);
-    setDetectedFps();
-    setMainStreams([]);
-    setCopyStreamIdsByFile({});
-    setStreamsSelectorShown(false);
-    setZoom(1);
-    setNeighbouringFrames([]);
-    setShortestFlag(false);
-  }, [cutSegmentsHistory, setCutSegments, cancelCommandedTimeDebounce, cancelCutSegmentsDebounce]);
-
   useEffect(() => () => {
     if (dummyVideoPath) unlink(dummyVideoPath).catch(console.error);
   }, [dummyVideoPath]);
@@ -545,7 +570,7 @@ const App = memo(() => {
     filePath, playerTime, frameRenderEnabled, effectiveRotation,
   ]);
 
-  // Cleanup old frames
+  // Cleanup old
   useEffect(() => () => URL.revokeObjectURL(framePath), [framePath]);
 
   function onPlayingChange(val) {
@@ -640,7 +665,49 @@ const App = memo(() => {
     setCutSegments(cutSegmentsNew);
   }, [currentSegIndexSafe, cutSegments, setCutSegments]);
 
-  const durationSafe = duration || 1;
+  const shouldShowKeyframes = calcShouldShowKeyframes(duration, zoom);
+
+  useEffect(() => {
+    async function run() {
+      const d = debouncedReadKeyframesData;
+      if (!d || !d.filePath || !d.mainVideoStream || d.commandedTime == null || !calcShouldShowKeyframes(d.duration, d.zoom) || readingKeyframesPromise.current) return;
+
+      try {
+        const promise = ffmpeg.readFrames({ filePath: d.filePath, aroundTime: d.commandedTime, stream: d.mainVideoStream.index, window: ffmpegExtractWindow });
+        readingKeyframesPromise.current = promise;
+        const newFrames = await promise;
+        // console.log(newFrames);
+        setNeighbouringFrames(newFrames);
+      } catch (err) {
+        console.error('Failed to read keyframes', err);
+      } finally {
+        readingKeyframesPromise.current = undefined;
+      }
+    }
+    run();
+  }, [debouncedReadKeyframesData]);
+
+  useEffect(() => {
+    async function run() {
+      const d = debouncedWaveformData;
+      if (!d || !d.filePath || !d.mainAudioStream || d.commandedTime == null || !calcShouldShowWaveform(d.duration, d.zoom) || !d.timelineExpanded || creatingWaveformPromise.current) return;
+      try {
+        const promise = ffmpeg.renderWaveformPng({ filePath: d.filePath, aroundTime: d.commandedTime, window: ffmpegExtractWindow, color: waveformColor });
+        creatingWaveformPromise.current = promise;
+        const wf = await promise;
+        setWaveform(wf);
+      } catch (err) {
+        console.error('Failed to render waveform', err);
+      } finally {
+        creatingWaveformPromise.current = undefined;
+      }
+    }
+
+    run();
+  }, [debouncedWaveformData]);
+
+  // Cleanup old
+  useEffect(() => () => waveform && URL.revokeObjectURL(waveform.url), [waveform]);
 
   function showUnsupportedFileMessage() {
     toast.fire({ timer: 10000, icon: 'warning', title: 'This video is not natively supported', text: 'This means that there is no audio in the preview and it has low quality. The final export operation will however be lossless and contains audio!' });
@@ -877,7 +944,9 @@ const App = memo(() => {
       ])));
 
       const videoStream = streams.find(stream => stream.codec_type === 'video');
+      const audioStream = streams.find(stream => stream.codec_type === 'audio');
       setMainVideoStream(videoStream);
+      setMainAudioStream(audioStream);
       if (videoStream) {
         const streamFps = getStreamFps(videoStream);
         if (streamFps != null) setDetectedFps(streamFps);
@@ -1261,24 +1330,6 @@ const App = memo(() => {
   // eslint-disable-next-line react-hooks/exhaustive-deps
   }, []);
 
-  useEffect(() => {
-    async function run() {
-      if (!filePath || debouncedCommandedTime == null || !mainVideoStream || readingKeyframesPromise.current) return;
-      try {
-        const promise = ffmpeg.readFrames({ filePath, aroundTime: debouncedCommandedTime, stream: mainVideoStream.index });
-        readingKeyframesPromise.current = promise;
-        const newFrames = await promise;
-        // console.log(newFrames);
-        setNeighbouringFrames(newFrames);
-      } catch (err) {
-        console.error('Failed to read keyframes', err);
-      } finally {
-        readingKeyframesPromise.current = undefined;
-      }
-    }
-    run();
-  }, [filePath, debouncedCommandedTime, mainVideoStream]);
-
   const VolumeIcon = muted || dummyVideoPath ? FaVolumeMute : FaVolumeUp;
 
   useEffect(() => {
@@ -1295,6 +1346,10 @@ const App = memo(() => {
 
   const sideBarWidth = showSideBar ? 200 : 0;
 
+  const hasAudio = !!mainAudioStream;
+  const shouldShowWaveform = calcShouldShowWaveform(duration, zoom);
+  const bottomBarHeight = 96 + (hasAudio && timelineExpanded ? timelineHeight : 0);
+
   return (
     <div>
       <div className="no-user-select" style={{ background: controlsBackground, height: topBarHeight, display: 'flex', alignItems: 'center', padding: '0 5px', justifyContent: 'space-between', flexWrap: 'wrap' }}>
@@ -1395,7 +1450,7 @@ const App = memo(() => {
         <video
           muted={muted}
           ref={videoRef}
-          style={{ width: '100%', height: '100%', objectFit: 'contain' }}
+          style={videoStyle}
           src={fileUri}
           onPlay={onSartPlaying}
           onPause={onStopPlaying}
@@ -1479,8 +1534,16 @@ const App = memo(() => {
         </Fragment>
       )}
 
-      <div className="no-user-select" style={{ height: bottomBarHeight, background: controlsBackground, position: 'absolute', left: 0, right: 0, bottom: 0, textAlign: 'center' }}>
+      <motion.div
+        className="no-user-select"
+        style={{ background: controlsBackground, position: 'absolute', left: 0, right: 0, bottom: 0, display: 'flex', flexDirection: 'column', justifyContent: 'flex-end' }}
+        animate={{ height: bottomBarHeight }}
+      >
         <Timeline
+          shouldShowKeyframes={shouldShowKeyframes}
+          shouldShowWaveform={shouldShowWaveform}
+          timelineExpanded={timelineExpanded}
+          waveform={waveform}
           getCurrentTime={getCurrentTime}
           startTimeOffset={startTimeOffset}
           playerTime={playerTime}
@@ -1499,6 +1562,7 @@ const App = memo(() => {
           inverseCutSegments={inverseCutSegments}
           mainVideoStream={mainVideoStream}
           formatTimecode={formatTimecode}
+          timelineHeight={timelineHeight}
         />
 
         <TimelineControls
@@ -1522,27 +1586,31 @@ const App = memo(() => {
           playing={playing}
           shortStep={shortStep}
           playCommand={playCommand}
+          setTimelineExpanded={setTimelineExpanded}
+          hasAudio={hasAudio}
         />
-      </div>
 
-      <LeftMenu
+        <div style={{ display: 'flex', justifyContent: 'space-between' }}>
-        zoom={zoom}
+          <LeftMenu
-        setZoom={setZoom}
+            zoom={zoom}
-        invertCutSegments={invertCutSegments}
+            setZoom={setZoom}
-        setInvertCutSegments={setInvertCutSegments}
+            invertCutSegments={invertCutSegments}
-      />
+            setInvertCutSegments={setInvertCutSegments}
+          />
 
-      <RightMenu
+          <RightMenu
-        isRotationSet={isRotationSet}
+            isRotationSet={isRotationSet}
-        rotation={rotation}
+            rotation={rotation}
-        areWeCutting={areWeCutting}
+            areWeCutting={areWeCutting}
-        increaseRotation={increaseRotation}
+            increaseRotation={increaseRotation}
-        deleteSource={deleteSource}
+            deleteSource={deleteSource}
-        renderCaptureFormatButton={renderCaptureFormatButton}
+            renderCaptureFormatButton={renderCaptureFormatButton}
-        capture={capture}
+            capture={capture}
-        cutClick={cutClick}
+            cutClick={cutClick}
-        multipleCutSegments={cutSegments.length > 1}
+            multipleCutSegments={cutSegments.length > 1}
-      />
+          />
+        </div>
+      </motion.div>
 
       <HelpSheet
         visible={!!helpVisible}

yarn.lock

@@ -1795,14 +1795,6 @@ cross-spawn-async@^2.1.1:
     lru-cache "^4.0.0"
     which "^1.2.8"
 
-cross-spawn@^4.0.0:
-  version "4.0.2"
-  resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-4.0.2.tgz#7b9247621c23adfdd3856004a823cbe397424d41"
-  integrity sha1-e5JHYhwjrf3ThWAEqCPL45dCTUE=
-  dependencies:
-    lru-cache "^4.0.1"
-    which "^1.2.9"
-
 cross-spawn@^6.0.5:
   version "6.0.5"
   resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4"
@@ -1814,6 +1806,15 @@ cross-spawn@^6.0.5:
     shebang-command "^1.2.0"
     which "^1.2.9"
 
+cross-spawn@^7.0.0:
+  version "7.0.1"
+  resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.1.tgz#0ab56286e0f7c24e153d04cc2aa027e43a9a5d14"
+  integrity sha512-u7v4o84SwFpD32Z8IIcPZ6z1/ie24O6RU3RbtL5Y316l3KuHVPx9ItBgWQ6VlfAFnRnTtMUrsQ9MUUTuEZjogg==
+  dependencies:
+    path-key "^3.1.0"
+    shebang-command "^2.0.0"
+    which "^2.0.1"
+
 cross-unzip@0.0.2:
   version "0.0.2"
   resolved "https://registry.yarnpkg.com/cross-unzip/-/cross-unzip-0.0.2.tgz#5183bc47a09559befcf98cc4657964999359372f"
@@ -2550,18 +2551,20 @@ execa@^0.2.2:
     path-key "^1.0.0"
     strip-eof "^1.0.0"
 
-execa@^0.5.0:
+execa@^4.0.0:
-  version "0.5.1"
+  version "4.0.0"
-  resolved "https://registry.yarnpkg.com/execa/-/execa-0.5.1.tgz#de3fb85cb8d6e91c85bcbceb164581785cb57b36"
+  resolved "https://registry.yarnpkg.com/execa/-/execa-4.0.0.tgz#7f37d6ec17f09e6b8fc53288611695b6d12b9daf"
-  integrity sha1-3j+4XLjW6RyFvLzrFkWBeFy1ezY=
+  integrity sha512-JbDUxwV3BoT5ZVXQrSVbAiaXhXUkIwvbhPIwZ0N13kX+5yCzOhUNdocxB/UQRuYOHRYYwAxKYwJYc0T4D12pDA==
   dependencies:
-    cross-spawn "^4.0.0"
+    cross-spawn "^7.0.0"
-    get-stream "^2.2.0"
+    get-stream "^5.0.0"
-    is-stream "^1.1.0"
+    human-signals "^1.1.1"
-    npm-run-path "^2.0.0"
+    is-stream "^2.0.0"
-    p-finally "^1.0.0"
+    merge-stream "^2.0.0"
-    signal-exit "^3.0.0"
+    npm-run-path "^4.0.0"
-    strip-eof "^1.0.0"
+    onetime "^5.1.0"
+    signal-exit "^3.0.2"
+    strip-final-newline "^2.0.0"
 
 expand-brackets@^0.1.4:
   version "0.1.5"
@@ -2935,14 +2938,6 @@ get-caller-file@^2.0.1:
   resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e"
   integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==
 
-get-stream@^2.2.0:
-  version "2.3.1"
-  resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-2.3.1.tgz#5f38f93f346009666ee0150a054167f91bdd95de"
-  integrity sha1-Xzj5PzRgCWZu4BUKBUFn+Rvdld4=
-  dependencies:
-    object-assign "^4.0.1"
-    pinkie-promise "^2.0.0"
-
 get-stream@^4.1.0:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-4.1.0.tgz#c1b255575f3dc21d59bfc79cd3d2b46b1c3a54b5"
@@ -2950,7 +2945,7 @@ get-stream@^4.1.0:
   dependencies:
     pump "^3.0.0"
 
-get-stream@^5.1.0:
+get-stream@^5.0.0, get-stream@^5.1.0:
   version "5.1.0"
   resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-5.1.0.tgz#01203cdc92597f9b909067c3e656cc1f4d3c4dc9"
   integrity sha512-EXr1FOzrzTfGeL0gQdeFEvOMm2mzMOglyiOXSTpPC+iAjAKftbr3jpCMWynogwYnM+eSj9sHGc6wjIcDvYiygw==
@@ -3296,6 +3291,11 @@ http-signature@~1.2.0:
     jsprim "^1.2.2"
     sshpk "^1.7.0"
 
+human-signals@^1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-1.1.1.tgz#c5b1cd14f50aeae09ab6c59fe63ba3395fe4dfa3"
+  integrity sha512-SEQu7vl8KjNL2eoGBLF3+wAjpsNfA9XMlXAYj/3EdaNfAlxKthD1xjEQfGOUhllCGGJVNY34bRr6lPINhNjyZw==
+
 hyphenate-style-name@^1.0.2:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/hyphenate-style-name/-/hyphenate-style-name-1.0.3.tgz#097bb7fa0b8f1a9cf0bd5c734cf95899981a9b48"
@@ -3628,11 +3628,16 @@ is-regex@^1.0.5:
   dependencies:
     has "^1.0.3"
 
-is-stream@^1.0.1, is-stream@^1.1.0:
+is-stream@^1.0.1:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44"
   integrity sha1-EtSj3U5o4Lec6428hBc66A2RykQ=
 
+is-stream@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.0.tgz#bde9c32680d6fae04129d6ac9d921ce7815f78e3"
+  integrity sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==
+
 is-string@^1.0.5:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/is-string/-/is-string-1.0.5.tgz#40493ed198ef3ff477b8c7f92f644ec82a5cd3a6"
@@ -4002,7 +4007,7 @@ lowercase-keys@^2.0.0:
   resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-2.0.0.tgz#2603e78b7b4b0006cbca2fbcc8a3202558ac9479"
   integrity sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==
 
-lru-cache@^4.0.0, lru-cache@^4.0.1:
+lru-cache@^4.0.0:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-4.1.1.tgz#622e32e82488b49279114a4f9ecf45e7cd6bba55"
   integrity sha512-q4spe4KTfsAS1SUHLO0wz8Qiyf1+vMIAgpRYioFYDMNqKfHQbg+AVDH3i4fvpl71/P1L0dBl+fQi+P37UYf0ew==
@@ -4044,6 +4049,11 @@ mdn-data@2.0.6:
   resolved "https://registry.yarnpkg.com/mdn-data/-/mdn-data-2.0.6.tgz#852dc60fcaa5daa2e8cf6c9189c440ed3e042978"
   integrity sha512-rQvjv71olwNHgiTbfPZFkJtjNMciWgswYeciZhtvWLO8bmX3TnhyA62I6sTWOyZssWHJJjY6/KiWwqQsWWsqOA==
 
+merge-stream@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/merge-stream/-/merge-stream-2.0.0.tgz#52823629a14dd00c9770fb6ad47dc6310f2c1f60"
+  integrity sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==
+
 micromatch@^2.1.5:
   version "2.3.11"
   resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-2.3.11.tgz#86677c97d1720b363431d04d0d15293bd38c1565"
@@ -4307,12 +4317,12 @@ npm-run-path@^1.0.0:
   dependencies:
     path-key "^1.0.0"
 
-npm-run-path@^2.0.0:
+npm-run-path@^4.0.0:
-  version "2.0.2"
+  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/npm-run-path/-/npm-run-path-2.0.2.tgz#35a9232dfa35d7067b4cb2ddf2357b1871536c5f"
+  resolved "https://registry.yarnpkg.com/npm-run-path/-/npm-run-path-4.0.1.tgz#b7ecd1e5ed53da8e37a55e1c2269e0b97ed748ea"
-  integrity sha1-NakjLfo11wZ7TLLd8jV7GHFTbF8=
+  integrity sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==
   dependencies:
-    path-key "^2.0.0"
+    path-key "^3.0.0"
 
 npmlog@^4.0.2:
   version "4.1.0"
@@ -4620,11 +4630,16 @@ path-key@^1.0.0:
   resolved "https://registry.yarnpkg.com/path-key/-/path-key-1.0.0.tgz#5d53d578019646c0d68800db4e146e6bdc2ac7af"
   integrity sha1-XVPVeAGWRsDWiADbThRua9wqx68=
 
-path-key@^2.0.0, path-key@^2.0.1:
+path-key@^2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/path-key/-/path-key-2.0.1.tgz#411cadb574c5a140d3a4b1910d40d80cc9f40b40"
   integrity sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=
 
+path-key@^3.0.0, path-key@^3.1.0:
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375"
+  integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==
+
 path-parse@^1.0.6:
   version "1.0.6"
   resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.6.tgz#d62dbb5679405d72c4737ec58600e9ddcf06d24c"
@@ -5523,11 +5538,23 @@ shebang-command@^1.2.0:
   dependencies:
     shebang-regex "^1.0.0"
 
+shebang-command@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-2.0.0.tgz#ccd0af4f8835fbdc265b82461aaf0c36663f34ea"
+  integrity sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==
+  dependencies:
+    shebang-regex "^3.0.0"
+
 shebang-regex@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-1.0.0.tgz#da42f49740c0b42db2ca9728571cb190c98efea3"
   integrity sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=
 
+shebang-regex@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172"
+  integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==
+
 side-channel@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.2.tgz#df5d1abadb4e4bf4af1cd8852bf132d2f7876947"
@@ -5826,6 +5853,11 @@ strip-eof@^1.0.0:
   resolved "https://registry.yarnpkg.com/strip-eof/-/strip-eof-1.0.0.tgz#bb43ff5598a6eb05d89b59fcd129c983313606bf"
   integrity sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=
 
+strip-final-newline@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/strip-final-newline/-/strip-final-newline-2.0.0.tgz#89b852fb2fcbe936f6f4b3187afb0a12c1ab58ad"
+  integrity sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==
+
 strip-json-comments@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-3.0.1.tgz#85713975a91fb87bf1b305cca77395e40d2a64a7"
@@ -6292,6 +6324,13 @@ which@^1.2.11, which@^1.2.8, which@^1.2.9:
   dependencies:
     isexe "^2.0.0"
 
+which@^2.0.1:
+  version "2.0.2"
+  resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1"
+  integrity sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==
+  dependencies:
+    isexe "^2.0.0"
+
 wide-align@^1.1.0:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/wide-align/-/wide-align-1.1.2.tgz#571e0f1b0604636ebc0dfc21b0339bbe31341710"