Added command 'create_video' to run FFmpeg

This requires FFmpeg to be installed (not just for the command, but also
for running the accompanying unit test). Because of this external
dependency, the worker has to declare task_type='video-encoding' in its
configuration file before it gets such tasks. This is not enabled by
default.

CHANGELOG.md

@@ -27,6 +27,8 @@ changed functionality, fixed bugs).
 - Log lines produced by subprocesses are now prefixed with 'PID=nnn'.
 - Moved from pip-installing requirements.txt to Pipenv.
 - Upgraded Python from 3.5 to 3.7
+- Added a new command `create_video` which uses FFmpeg to create a video after rendering an image
+  sequence. It's up to Flamenco Server to include (or not) this command in a render job.
 
 
 ## Version 2.1.0 (2018-01-04)

README.md

@@ -6,15 +6,14 @@ Author: Sybren A. Stüvel <sybren@blender.studio>
 
 ## Installation
 
-Before you begin, make sure you have Flamenco Manager up and running.
-
-There are two ways to install Flamenco Worker:
-
-- If you have a distributable zip file (see [Packaging for distribution](#packaging-for-distribution))
-  unzip it, `cd` into it, then run `./flamenco-worker` (or `flamenco-worker.exe` on Windows).
-
-- If you have a copy of the source files, run `pipenv install` then run `flamenco-worker`. This
-  requires Python 3.7 or newer.
+- Make sure you have Flamenco Manager up and running.
+- Install [FFmpeg](https://ffmpeg.org/) and make sure the `ffmpeg` binary is on `$PATH`.
+- Install Flamenco Worker in one of two ways:
+    - If you have a distributable zip file (see
+      [Packaging for distribution](#packaging-for-distribution)) unzip it, `cd` into it,
+      then run `./flamenco-worker` (or `flamenco-worker.exe` on Windows).
+    - If you have a copy of the source files, run `pipenv install` then run `flamenco-worker`.
+      This requires Python 3.7 or newer.
 
 
 ## Upgrading

flamenco-worker.cfg

@@ -3,6 +3,7 @@
 # The URL of the Flamenco Manager. Leave empty for auto-discovery via UPnP/SSDP.
 manager_url =
 
+# Add the 'video-encoding' task type if you have ffmpeg installed.
 task_types = sleep blender-render file-management exr-merge debug
 task_update_queue_db = flamenco-worker.db
 

flamenco_worker/commands.py

@@ -11,6 +11,7 @@ import shlex
 import shutil
 import subprocess
 import tempfile
+
 import time
 import typing
 from pathlib import Path
@@ -183,7 +184,8 @@ class AbstractCommand(metaclass=abc.ABCMeta):
         return None
 
     def _setting(self, settings: Settings, key: str, is_required: bool,
-                 valtype: InstanceOfType = str) \
+                 valtype: InstanceOfType = str,
+                 default: typing.Any = None) \
             -> typing.Tuple[typing.Any, typing.Optional[str]]:
         """Parses a setting, returns either (value, None) or (None, errormsg)"""
 
@@ -192,10 +194,12 @@ class AbstractCommand(metaclass=abc.ABCMeta):
         except KeyError:
             if is_required:
                 return None, 'Missing "%s"' % key
-            return None, None
+            settings.setdefault(key, default)
+            return default, None
 
         if value is None and not is_required:
-            return None, None
+            settings.setdefault(key, default)
+            return default, None
 
         if not isinstance(value, valtype):
             return None, '"%s" must be a %s, not a %s' % (key, valtype, type(value))
@@ -884,3 +888,65 @@ class MergeProgressiveRendersCommand(AbstractSubprocessCommand):
 
         await self.worker.register_log('Moving %s to %s', src, dst)
         shutil.move(str(src), str(dst))
+
+
+@command_executor('create_video')
+class CreateVideoCommand(AbstractSubprocessCommand):
+    """Create a video from individual frames.
+
+    Requires FFmpeg to be installed and available with the 'ffmpeg' command.
+    """
+
+    codec_video = 'h264'
+
+    # Select some settings that are useful for scrubbing through the video.
+    constant_rate_factor = 17  # perceptually lossless
+    keyframe_interval = 1  # GOP size
+    max_b_frames: typing.Optional[int] = 0
+
+    def validate(self, settings: Settings) -> typing.Optional[str]:
+        # Check that FFmpeg can be found.
+        ffmpeg, err = self._setting(settings, 'ffmpeg', is_required=False, default='ffmpeg')
+        if err:
+            return err
+        executable_path: typing.Optional[str] = shutil.which(ffmpeg)
+        if not executable_path:
+            return f'FFmpeg command {ffmpeg!r} not found on $PATH'
+        self._log.debug('Found FFmpeg command at %r', executable_path)
+
+        # Check that we know our input and output image files.
+        input_files, err = self._setting(settings, 'input_files', is_required=True)
+        if err:
+            return err
+        self._log.debug('Input files: %s', input_files)
+        output_file, err = self._setting(settings, 'output_file', is_required=True)
+        if err:
+            return err
+        self._log.debug('Output file: %s', output_file)
+
+        fps, err = self._setting(settings, 'fps', is_required=True, valtype=(int, float))
+        if err:
+            return err
+        self._log.debug('Frame rate: %r fps', fps)
+        return None
+
+    async def execute(self, settings: Settings) -> None:
+        cmd = self._build_ffmpeg_command(settings)
+        await self.subprocess(cmd)
+
+    def _build_ffmpeg_command(self, settings) -> typing.List[str]:
+        cmd = [
+            settings['ffmpeg'],
+            '-pattern_type', 'glob',
+            '-i', settings['input_files'],
+            '-c:v', self.codec_video,
+            '-crf', str(self.constant_rate_factor),
+            '-g', str(self.keyframe_interval),
+            '-r', str(settings['fps']),
+        ]
+        if self.max_b_frames is not None:
+            cmd.extend(['-bf', str(self.max_b_frames)])
+        cmd += [
+            settings['output_file']
+        ]
+        return cmd

flamenco_worker/config.py

@@ -16,7 +16,8 @@ CONFIG_SECTION = 'flamenco-worker'
 DEFAULT_CONFIG = {
     'flamenco-worker': collections.OrderedDict([
         ('manager_url', ''),
-        ('task_types', 'unknown sleep blender-render'),
+        # The 'video-encoding' tasks require ffmpeg to be installed, so it's not enabled by default.
+        ('task_types', 'sleep blender-render file-management exr-merge'),
         ('task_update_queue_db', 'flamenco-worker.db'),
         ('subprocess_pid_file', 'flamenco-worker-subprocess.pid'),
         ('may_i_run_interval_seconds', '5'),

tests/test_commands_create_video.py

+import logging
+import shutil
+import typing
+from pathlib import Path
+import shlex
+import subprocess
+import sys
+import tempfile
+
+from tests.test_runner import AbstractCommandTest
+
+log = logging.getLogger(__name__)
+
+
+class BlenderRenderTest(AbstractCommandTest):
+    settings: typing.Dict[str, typing.Any] = {
+        'ffmpeg': sys.executable,
+        'input_files': '/tmp/*.png',
+        'output_file': '/tmp/merged.mkv',
+        'fps': 24,
+    }
+
+    def setUp(self):
+        super().setUp()
+
+        from flamenco_worker.commands import CreateVideoCommand
+
+        self.cmd = CreateVideoCommand(
+            worker=self.fworker,
+            task_id='12345',
+            command_idx=0,
+        )
+
+    def test_validate(self):
+        self.assertIn('not found on $PATH', self.cmd.validate({'ffmpeg': '/does/not/exist'}))
+        self.assertIsNone(self.cmd.validate(self.settings))
+
+    def test_validate_without_ffmpeg(self):
+        settings = self.settings.copy()
+        del settings['ffmpeg']
+
+        self.assertIsNone(self.cmd.validate(settings))
+        self.assertEqual('ffmpeg', settings['ffmpeg'],
+                         'The default setting should be stored in the dict after validation')
+
+    def test_build_ffmpeg_cmd(self):
+        self.assertEqual([
+            sys.executable,
+            '-pattern_type', 'glob',
+            '-i', '/tmp/*.png',
+            '-c:v', 'h264',
+            '-crf', '17',
+            '-g', '1',
+            '-r', '24',
+            '-bf', '0',
+            '/tmp/merged.mkv',
+        ], self.cmd._build_ffmpeg_command(self.settings))
+
+    def test_run_ffmpeg(self):
+        with tempfile.TemporaryDirectory() as tempdir:
+            outfile = Path(tempdir) / 'merged.mkv'
+            frame_dir = Path(__file__).with_name('test_frames')
+            settings: typing.Dict[str, typing.Any] = {
+                **self.settings,
+                'ffmpeg': 'ffmpeg',  # use the real FFmpeg for this test.
+                'input_files': f'{frame_dir}/*.png',
+                'output_file': str(outfile),
+            }
+
+            self.loop.run_until_complete(self.cmd.run(settings))
+            self.assertTrue(outfile.exists())
+
+            ffprobe_cmd = [shutil.which('ffprobe'), '-v', 'error',
+                           '-show_entries', 'format=duration',
+                           '-of', 'default=noprint_wrappers=1:nokey=1',
+                           str(outfile)]
+            log.debug('Running %s', ' '.join(shlex.quote(arg) for arg in ffprobe_cmd))
+            probe_out = subprocess.check_output(ffprobe_cmd)
+            probed_duration = float(probe_out)
+            expect_duration = len(list(frame_dir.glob('*.png'))) / settings['fps']
+            self.assertAlmostEqual(expect_duration, probed_duration, places=3)