merge

.gitignore

@@ -4,9 +4,10 @@
 
 .cache/
 .coverage
+.mypy_cache/
 
 /flamenco_worker.egg-info/
 /flamenco-worker.db
 /build/
 /dist/
-/flamenco-worker*.log
+/flamenco-worker*.log*

CHANGELOG.md

@@ -6,6 +6,7 @@ changed functionality, fixed bugs).
 ## Version 2.2 (in development)
 
 - Always log the version of Flamenco Worker.
+- Requires Flamenco Manager 2.2 or newer.
 - Include missing merge-exr.blend, required for progressive rendering, in the distribution bundle.
 - Include `exr-merge` task type in default configuration, which is required for progressive
   rendering.
@@ -15,6 +16,28 @@ changed functionality, fixed bugs).
   amounts of logs before pushing to Flamenco Manager).
 - Fixed a memory leak in the ask update queue.
 - Added a new `log_a_lot` command and task type `debug` to aid in debugging.
+- Fixed bug where task updates would be sent in an infinite loop when the Manager didn't
+  know the task, blocking all other task updates.
+- Added a `pre_task_check` section to the configuration file, which can contain `write.N` and
+  `read.N` keys (where `N` can be anything to make the keys unique). Every value is a path to be
+  checked for writability or readability. Note that write checks are lossy, and bytes are appended
+  to any existing file used to check writability. When such a check fails, the Worker will go to
+  status `error` and sleep for 10 minutes before trying again.
+- Subprocess commands now write the spawned process PID in a text file, and refuse to run if there
+  already is such a file with an alive PID.
+- Log lines produced by subprocesses are now prefixed with 'PID=nnn'.
+- Moved from pip-installing requirements.txt to Pipenv.
+- Upgraded Python from 3.5 to 3.7
+- Added a new command `create_video` which uses FFmpeg to create a video after rendering an image
+  sequence. It's up to Flamenco Server to include (or not) this command in a render job.
+- Explicitly return tasks to the Manager queue when stopping them (that is, when going asleep or
+  shutting down). Requires Flamenco Manager 2.2 or newer.
+- Added support for commands used in the blender-video-chunks job type:
+    - blender_render_audio
+    - concat_videos
+    - create_video
+    - move_with_counter
+    - mux_audio
 
 
 ## Version 2.1.0 (2018-01-04)

Pipfile

+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+attrs = "*"
+requests = "*"
+psutil = "*"
+
+[dev-packages]
+pytest = "<4"  # pytest-cov uses deprecated function (removed in pytest 4) when using --no-cov
+pytest-cov = "*"
+wheel = "*"
+pyinstaller = "*"
+ipython = "*"
+pathlib2 = {version = "*", markers="python_version < '3.6'"}
+mypy = "*"
+
+[requires]
+python_version = "3.7"

Pipfile.lock

+{
+    "_meta": {
+        "hash": {
+            "sha256": "b0736d28b87f6c3f3541d2bab2740817ad8f51db7f8e92ce7bd6068ab2ac32c8"
+        },
+        "pipfile-spec": 6,
+        "requires": {
+            "python_version": "3.7"
+        },
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "attrs": {
+            "hashes": [
+                "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69",
+                "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb"
+            ],
+            "index": "pypi",
+            "version": "==18.2.0"
+        },
+        "certifi": {
+            "hashes": [
+                "sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c",
+                "sha256:6d58c986d22b038c8c0df30d639f23a3e6d172a05c3583e766f4c0b785c0986a"
+            ],
+            "version": "==2018.10.15"
+        },
+        "chardet": {
+            "hashes": [
+                "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
+                "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
+            ],
+            "version": "==3.0.4"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e",
+                "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16"
+            ],
+            "version": "==2.7"
+        },
+        "psutil": {
+            "hashes": [
+                "sha256:1c19957883e0b93d081d41687089ad630e370e26dc49fd9df6951d6c891c4736",
+                "sha256:1c71b9716790e202a00ab0931a6d1e25db1aa1198bcacaea2f5329f75d257fff",
+                "sha256:3b7a4daf4223dae171a67a89314ac5ca0738e94064a78d99cfd751c55d05f315",
+                "sha256:3e19be3441134445347af3767fa7770137d472a484070840eee6653b94ac5576",
+                "sha256:6e265c8f3da00b015d24b842bfeb111f856b13d24f2c57036582568dc650d6c3",
+                "sha256:809c9cef0402e3e48b5a1dddc390a8a6ff58b15362ea5714494073fa46c3d293",
+                "sha256:b4d1b735bf5b120813f4c89db8ac22d89162c558cbd7fdd298866125fe906219",
+                "sha256:bbffac64cfd01c6bcf90eb1bedc6c80501c4dae8aef4ad6d6dd49f8f05f6fc5a",
+                "sha256:bfcea4f189177b2d2ce4a34b03c4ac32c5b4c22e21f5b093d9d315e6e253cd81"
+            ],
+            "index": "pypi",
+            "version": "==5.4.8"
+        },
+        "requests": {
+            "hashes": [
+                "sha256:65b3a120e4329e33c9889db89c80976c5272f56ea92d3e74da8a463992e3ff54",
+                "sha256:ea881206e59f41dbd0bd445437d792e43906703fff75ca8ff43ccdb11f33f263"
+            ],
+            "index": "pypi",
+            "version": "==2.20.1"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39",
+                "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22"
+            ],
+            "version": "==1.24.1"
+        }
+    },
+    "develop": {
+        "altgraph": {
+            "hashes": [
+                "sha256:d6814989f242b2b43025cba7161fc1b8fb487a62cd49c49245d6fd01c18ac997",
+                "sha256:ddf5320017147ba7b810198e0b6619bd7b5563aa034da388cea8546b877f9b0c"
+            ],
+            "version": "==0.16.1"
+        },
+        "atomicwrites": {
+            "hashes": [
+                "sha256:0312ad34fcad8fac3704d441f7b317e50af620823353ec657a53e981f92920c0",
+                "sha256:ec9ae8adaae229e4f8446952d204a3e4b5fdd2d099f9be3aaf556120135fb3ee"
+            ],
+            "version": "==1.2.1"
+        },
+        "attrs": {
+            "hashes": [
+                "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69",
+                "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb"
+            ],
+            "index": "pypi",
+            "version": "==18.2.0"
+        },
+        "backcall": {
+            "hashes": [
+                "sha256:38ecd85be2c1e78f77fd91700c76e14667dc21e2713b63876c0eb901196e01e4",
+                "sha256:bbbf4b1e5cd2bdb08f915895b51081c041bac22394fdfcfdfbe9f14b77c08bf2"
+            ],
+            "version": "==0.1.0"
+        },
+        "coverage": {
+            "hashes": [
+                "sha256:09e47c529ff77bf042ecfe858fb55c3e3eb97aac2c87f0349ab5a7efd6b3939f",
+                "sha256:0a1f9b0eb3aa15c990c328535655847b3420231af299386cfe5efc98f9c250fe",
+                "sha256:0cc941b37b8c2ececfed341444a456912e740ecf515d560de58b9a76562d966d",
+                "sha256:10e8af18d1315de936d67775d3a814cc81d0747a1a0312d84e27ae5610e313b0",
+                "sha256:1b4276550b86caa60606bd3572b52769860a81a70754a54acc8ba789ce74d607",
+                "sha256:1e8a2627c48266c7b813975335cfdea58c706fe36f607c97d9392e61502dc79d",
+                "sha256:2b224052bfd801beb7478b03e8a66f3f25ea56ea488922e98903914ac9ac930b",
+                "sha256:447c450a093766744ab53bf1e7063ec82866f27bcb4f4c907da25ad293bba7e3",
+                "sha256:46101fc20c6f6568561cdd15a54018bb42980954b79aa46da8ae6f008066a30e",
+                "sha256:4710dc676bb4b779c4361b54eb308bc84d64a2fa3d78e5f7228921eccce5d815",
+                "sha256:510986f9a280cd05189b42eee2b69fecdf5bf9651d4cd315ea21d24a964a3c36",
+                "sha256:5535dda5739257effef56e49a1c51c71f1d37a6e5607bb25a5eee507c59580d1",
+                "sha256:5a7524042014642b39b1fcae85fb37556c200e64ec90824ae9ecf7b667ccfc14",
+                "sha256:5f55028169ef85e1fa8e4b8b1b91c0b3b0fa3297c4fb22990d46ff01d22c2d6c",
+                "sha256:6694d5573e7790a0e8d3d177d7a416ca5f5c150742ee703f3c18df76260de794",
+                "sha256:6831e1ac20ac52634da606b658b0b2712d26984999c9d93f0c6e59fe62ca741b",
+                "sha256:77f0d9fa5e10d03aa4528436e33423bfa3718b86c646615f04616294c935f840",
+                "sha256:828ad813c7cdc2e71dcf141912c685bfe4b548c0e6d9540db6418b807c345ddd",
+                "sha256:85a06c61598b14b015d4df233d249cd5abfa61084ef5b9f64a48e997fd829a82",
+                "sha256:8cb4febad0f0b26c6f62e1628f2053954ad2c555d67660f28dfb1b0496711952",
+                "sha256:a5c58664b23b248b16b96253880b2868fb34358911400a7ba39d7f6399935389",
+                "sha256:aaa0f296e503cda4bc07566f592cd7a28779d433f3a23c48082af425d6d5a78f",
+                "sha256:ab235d9fe64833f12d1334d29b558aacedfbca2356dfb9691f2d0d38a8a7bfb4",
+                "sha256:b3b0c8f660fae65eac74fbf003f3103769b90012ae7a460863010539bb7a80da",
+                "sha256:bab8e6d510d2ea0f1d14f12642e3f35cefa47a9b2e4c7cea1852b52bc9c49647",
+                "sha256:c45297bbdbc8bb79b02cf41417d63352b70bcb76f1bbb1ee7d47b3e89e42f95d",
+                "sha256:d19bca47c8a01b92640c614a9147b081a1974f69168ecd494687c827109e8f42",
+                "sha256:d64b4340a0c488a9e79b66ec9f9d77d02b99b772c8b8afd46c1294c1d39ca478",
+                "sha256:da969da069a82bbb5300b59161d8d7c8d423bc4ccd3b410a9b4d8932aeefc14b",
+                "sha256:ed02c7539705696ecb7dc9d476d861f3904a8d2b7e894bd418994920935d36bb",
+                "sha256:ee5b8abc35b549012e03a7b1e86c09491457dba6c94112a2482b18589cc2bdb9"
+            ],
+            "version": "==4.5.2"
+        },
+        "decorator": {
+            "hashes": [
+                "sha256:2c51dff8ef3c447388fe5e4453d24a2bf128d3a4c32af3fabef1f01c6851ab82",
+                "sha256:c39efa13fbdeb4506c476c9b3babf6a718da943dab7811c206005a4a956c080c"
+            ],
+            "version": "==4.3.0"
+        },
+        "future": {
+            "hashes": [
+                "sha256:67045236dcfd6816dc439556d009594abf643e5eb48992e36beac09c2ca659b8"
+            ],
+            "version": "==0.17.1"
+        },
+        "ipython": {
+            "hashes": [
+                "sha256:a5781d6934a3341a1f9acb4ea5acdc7ea0a0855e689dbe755d070ca51e995435",
+                "sha256:b10a7ddd03657c761fc503495bc36471c8158e3fc948573fb9fe82a7029d8efd"
+            ],
+            "index": "pypi",
+            "version": "==7.1.1"
+        },
+        "ipython-genutils": {
+            "hashes": [
+                "sha256:72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8",
+                "sha256:eb2e116e75ecef9d4d228fdc66af54269afa26ab4463042e33785b887c628ba8"
+            ],
+            "version": "==0.2.0"
+        },
+        "jedi": {
+            "hashes": [
+                "sha256:0191c447165f798e6a730285f2eee783fff81b0d3df261945ecb80983b5c3ca7",
+                "sha256:b7493f73a2febe0dc33d51c99b474547f7f6c0b2c8fb2b21f453eef204c12148"
+            ],
+            "version": "==0.13.1"
+        },
+        "macholib": {
+            "hashes": [
+                "sha256:ac02d29898cf66f27510d8f39e9112ae00590adb4a48ec57b25028d6962b1ae1",
+                "sha256:c4180ffc6f909bf8db6cd81cff4b6f601d575568f4d5dee148c830e9851eb9db"
+            ],
+            "version": "==1.11"
+        },
+        "more-itertools": {
+            "hashes": [
+                "sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092",
+                "sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e",
+                "sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d"
+            ],
+            "version": "==4.3.0"
+        },
+        "mypy": {
+            "hashes": [
+                "sha256:8e071ec32cc226e948a34bbb3d196eb0fd96f3ac69b6843a5aff9bd4efa14455",
+                "sha256:fb90c804b84cfd8133d3ddfbd630252694d11ccc1eb0166a1b2efb5da37ecab2"
+            ],
+            "index": "pypi",
+            "version": "==0.641"
+        },
+        "mypy-extensions": {
+            "hashes": [
+                "sha256:37e0e956f41369209a3d5f34580150bcacfabaa57b33a15c0b25f4b5725e0812",
+                "sha256:b16cabe759f55e3409a7d231ebd2841378fb0c27a5d1994719e340e4f429ac3e"
+            ],
+            "version": "==0.4.1"
+        },
+        "parso": {
+            "hashes": [
+                "sha256:35704a43a3c113cce4de228ddb39aab374b8004f4f2407d070b6a2ca784ce8a2",
+                "sha256:895c63e93b94ac1e1690f5fdd40b65f07c8171e3e53cbd7793b5b96c0e0a7f24"
+            ],
+            "version": "==0.3.1"
+        },
+        "pathlib2": {
+            "hashes": [
+                "sha256:8eb170f8d0d61825e09a95b38be068299ddeda82f35e96c3301a8a5e7604cb83",
+                "sha256:d1aa2a11ba7b8f7b21ab852b1fb5afb277e1bb99d5dfc663380b5015c0d80c5a"
+            ],
+            "index": "pypi",
+            "markers": "python_version < '3.6'",
+            "version": "==2.3.2"
+        },
+        "pefile": {
+            "hashes": [
+                "sha256:4c5b7e2de0c8cb6c504592167acf83115cbbde01fe4a507c16a1422850e86cd6"
+            ],
+            "version": "==2018.8.8"
+        },
+        "pexpect": {
+            "hashes": [
+                "sha256:2a8e88259839571d1251d278476f3eec5db26deb73a70be5ed5dc5435e418aba",
+                "sha256:3fbd41d4caf27fa4a377bfd16fef87271099463e6fa73e92a52f92dfee5d425b"
+            ],
+            "markers": "sys_platform != 'win32'",
+            "version": "==4.6.0"
+        },
+        "pickleshare": {
+            "hashes": [
+                "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca",
+                "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"
+            ],
+            "version": "==0.7.5"
+        },
+        "pluggy": {
+            "hashes": [
+                "sha256:447ba94990e8014ee25ec853339faf7b0fc8050cdc3289d4d71f7f410fb90095",
+                "sha256:bde19360a8ec4dfd8a20dcb811780a30998101f078fc7ded6162f0076f50508f"
+            ],
+            "version": "==0.8.0"
+        },
+        "prompt-toolkit": {
+            "hashes": [
+                "sha256:c1d6aff5252ab2ef391c2fe498ed8c088066f66bc64a8d5c095bbf795d9fec34",
+                "sha256:d4c47f79b635a0e70b84fdb97ebd9a274203706b1ee5ed44c10da62755cf3ec9",
+                "sha256:fd17048d8335c1e6d5ee403c3569953ba3eb8555d710bfc548faf0712666ea39"
+            ],
+            "version": "==2.0.7"
+        },
+        "ptyprocess": {
+            "hashes": [
+                "sha256:923f299cc5ad920c68f2bc0bc98b75b9f838b93b599941a6b63ddbc2476394c0",
+                "sha256:d7cc528d76e76342423ca640335bd3633420dc1366f258cb31d05e865ef5ca1f"
+            ],
+            "version": "==0.6.0"
+        },
+        "py": {
+            "hashes": [
+                "sha256:bf92637198836372b520efcba9e020c330123be8ce527e535d185ed4b6f45694",
+                "sha256:e76826342cefe3c3d5f7e8ee4316b80d1dd8a300781612ddbc765c17ba25a6c6"
+            ],
+            "version": "==1.7.0"
+        },
+        "pygments": {
+            "hashes": [
+                "sha256:78f3f434bcc5d6ee09020f92ba487f95ba50f1e3ef83ae96b9d5ffa1bab25c5d",
+                "sha256:dbae1046def0efb574852fab9e90209b23f556367b5a320c0bcb871c77c3e8cc"
+            ],
+            "version": "==2.2.0"
+        },
+        "pyinstaller": {
+            "hashes": [
+                "sha256:a5a6e04a66abfcf8761e89a2ebad937919c6be33a7b8963e1a961b55cb35986b"
+            ],
+            "index": "pypi",
+            "version": "==3.4"
+        },
+        "pytest": {
+            "hashes": [
+                "sha256:3f193df1cfe1d1609d4c583838bea3d532b18d6160fd3f55c9447fdca30848ec",
+                "sha256:e246cf173c01169b9617fc07264b7b1316e78d7a650055235d6d897bc80d9660"
+            ],
+            "index": "pypi",
+            "version": "==3.10.1"
+        },
+        "pytest-cov": {
+            "hashes": [
+                "sha256:513c425e931a0344944f84ea47f3956be0e416d95acbd897a44970c8d926d5d7",
+                "sha256:e360f048b7dae3f2f2a9a4d067b2dd6b6a015d384d1577c994a43f3f7cbad762"
+            ],
+            "index": "pypi",
+            "version": "==2.6.0"
+        },
+        "six": {
+            "hashes": [
+                "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9",
+                "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb"
+            ],
+            "version": "==1.11.0"
+        },
+        "traitlets": {
+            "hashes": [
+                "sha256:9c4bd2d267b7153df9152698efb1050a5d84982d3384a37b2c1f7723ba3e7835",
+                "sha256:c6cb5e6f57c5a9bdaa40fa71ce7b4af30298fbab9ece9815b5d995ab6217c7d9"
+            ],
+            "version": "==4.3.2"
+        },
+        "typed-ast": {
+            "hashes": [
+                "sha256:0948004fa228ae071054f5208840a1e88747a357ec1101c17217bfe99b299d58",
+                "sha256:10703d3cec8dcd9eef5a630a04056bbc898abc19bac5691612acba7d1325b66d",
+                "sha256:1f6c4bd0bdc0f14246fd41262df7dfc018d65bb05f6e16390b7ea26ca454a291",
+                "sha256:25d8feefe27eb0303b73545416b13d108c6067b846b543738a25ff304824ed9a",
+                "sha256:29464a177d56e4e055b5f7b629935af7f49c196be47528cc94e0a7bf83fbc2b9",
+                "sha256:2e214b72168ea0275efd6c884b114ab42e316de3ffa125b267e732ed2abda892",
+                "sha256:3e0d5e48e3a23e9a4d1a9f698e32a542a4a288c871d33ed8df1b092a40f3a0f9",
+                "sha256:519425deca5c2b2bdac49f77b2c5625781abbaf9a809d727d3a5596b30bb4ded",
+                "sha256:57fe287f0cdd9ceaf69e7b71a2e94a24b5d268b35df251a88fef5cc241bf73aa",
+                "sha256:668d0cec391d9aed1c6a388b0d5b97cd22e6073eaa5fbaa6d2946603b4871efe",
+                "sha256:68ba70684990f59497680ff90d18e756a47bf4863c604098f10de9716b2c0bdd",
+                "sha256:6de012d2b166fe7a4cdf505eee3aaa12192f7ba365beeefaca4ec10e31241a85",
+                "sha256:79b91ebe5a28d349b6d0d323023350133e927b4de5b651a8aa2db69c761420c6",
+                "sha256:8550177fa5d4c1f09b5e5f524411c44633c80ec69b24e0e98906dd761941ca46",
+                "sha256:898f818399cafcdb93cbbe15fc83a33d05f18e29fb498ddc09b0214cdfc7cd51",
+                "sha256:94b091dc0f19291adcb279a108f5d38de2430411068b219f41b343c03b28fb1f",
+                "sha256:a26863198902cda15ab4503991e8cf1ca874219e0118cbf07c126bce7c4db129",
+                "sha256:a8034021801bc0440f2e027c354b4eafd95891b573e12ff0418dec385c76785c",
+                "sha256:bc978ac17468fe868ee589c795d06777f75496b1ed576d308002c8a5756fb9ea",
+                "sha256:c05b41bc1deade9f90ddc5d988fe506208019ebba9f2578c622516fd201f5863",
+                "sha256:c9b060bd1e5a26ab6e8267fd46fc9e02b54eb15fffb16d112d4c7b1c12987559",
+                "sha256:edb04bdd45bfd76c8292c4d9654568efaedf76fe78eb246dde69bdb13b2dad87",
+                "sha256:f19f2a4f547505fe9072e15f6f4ae714af51b5a681a97f187971f50c283193b6"
+            ],
+            "version": "==1.1.0"
+        },
+        "wcwidth": {
+            "hashes": [
+                "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e",
+                "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c"
+            ],
+            "version": "==0.1.7"
+        },
+        "wheel": {
+            "hashes": [
+                "sha256:196c9842d79262bb66fcf59faa4bd0deb27da911dbc7c6cdca931080eb1f0783",
+                "sha256:c93e2d711f5f9841e17f53b0e6c0ff85593f3b416b6eec7a9452041a59a42688"
+            ],
+            "index": "pypi",
+            "version": "==0.32.2"
+        }
+    }
+}

README.md

@@ -6,15 +6,14 @@ Author: Sybren A. Stüvel <sybren@blender.studio>
 
 ## Installation
 
-Before you begin, make sure you have Flamenco Manager up and running.
-
-There are two ways to install Flamenco Worker:
-
-- If you have a distributable zip file (see [Packaging for distribution](#packaging-for-distribution))
-  unzip it, `cd` into it, then run `./flamenco-worker` (or `flamenco-worker.exe` on Windows).
-
-- If you have a copy of the source files, run `pip3 install -e .` then run `flamenco-worker`. This
-  requires Python 3.5.2 or newer.
+- Make sure you have Flamenco Manager up and running.
+- Install [FFmpeg](https://ffmpeg.org/) and make sure the `ffmpeg` binary is on `$PATH`.
+- Install Flamenco Worker in one of two ways:
+    - If you have a distributable zip file (see
+      [Packaging for distribution](#packaging-for-distribution)) unzip it, `cd` into it,
+      then run `./flamenco-worker` (or `flamenco-worker.exe` on Windows).
+    - If you have a copy of the source files, run `pipenv install` then run `flamenco-worker`.
+      This requires Python 3.7 or newer.
 
 
 ## Upgrading

deploy_via_biflamanager.sh

+#!/usr/bin/env bash
+
+ssh -o ClearAllForwardings=yes biflamanager -T <<EOT
+set -e
+cd \$HOME/flamenco-worker
+
+git reset --hard
+git pull
+pipenv install --dev --deploy
+pipenv run ./mkdistfile.py
+
+last_file=\$(ls -rt dist/flamenco-worker* | tail -n 1)
+dirname=\$(basename \$last_file | sed s/-linux.*//)
+tar_path=\$(pwd)/\$last_file
+
+echo
+echo "--------------------------------------------------------------"
+echo "Deploying \$last_file"
+echo "--------------------------------------------------------------"
+
+cd /shared/bin/flamenco-worker
+tar zxvf \$tar_path
+rm -f flamenco-worker
+ln -s \$dirname/flamenco-worker .
+
+echo
+echo "--------------------------------------------------------------"
+echo "Done! Now restart workers to pick up the changes."
+echo "--------------------------------------------------------------"
+EOT

flamenco-worker.cfg

@@ -3,17 +3,22 @@
 # The URL of the Flamenco Manager. Leave empty for auto-discovery via UPnP/SSDP.
 manager_url = http://LOCALHOST:5983
 
-task_types = sleep blender-render file-management
+# Add the 'video-encoding' task type if you have ffmpeg installed.
+task_types = sleep blender-render file-management exr-merge debug
 task_update_queue_db = flamenco-worker.db
 
 may_i_run_interval_seconds = 5
-push_log_max_interval_seconds = 20
-push_log_max_entries = 200
-push_act_max_interval_seconds = 10
+push_log_max_interval_seconds = 30
+push_log_max_entries = 2000
+push_act_max_interval_seconds = 15
 worker_storage_dir = /home/milanjaros/work/temp/public/flamenco/render/in
 worker_output_dir = /home/milanjaros/work/temp/public/flamenco/render/out
 worker_blender_cmd = ./run_icc_mpi.sh
 
+[pre_task_check]
+write.0 = /home/milanjaros/work/temp/public/flamenco/render/_flamenco
+write.1 = /home/milanjaros/work/temp/public/flamenco/render/spring/frames
+
 [loggers]
 keys = root,flamenco_worker
 

flamenco_worker/__init__.py

-__version__ = '2.2-dev1'
+__version__ = '2.2-dev10'

flamenco_worker/cli.py

@@ -6,6 +6,7 @@ import logging
 import logging.config
 import os
 import pathlib
+import typing
 
 import requests
 
@@ -100,7 +101,11 @@ def main():
         shutdown_future=shutdown_future,
     )
     trunner = runner.TaskRunner(
-        shutdown_future=shutdown_future)
+        shutdown_future=shutdown_future,
+        subprocess_pid_file=confparser.value('subprocess_pid_file'),
+    )
+
+    pretask_check_params = parse_pretask_check_config(confparser, log)
 
     fworker = worker.FlamencoWorker(
         manager=fmanager,
@@ -120,6 +125,7 @@ def main():
         worker_blender_cmd=confparser.value('worker_blender_cmd'),
         initial_state='testing' if args.test else 'awake',
         run_single_task=args.single,
+        pretask_check_params=pretask_check_params,
     )
 
     mir = may_i_run.MayIRun(
@@ -171,6 +177,9 @@ def main():
             loop.run_until_complete(asyncio.wait_for(mir_work_task, 5))
         except requests.exceptions.ConnectionError:
             log.warning("Unable to connect to HTTP server, but that's fine as we're shutting down.")
+        except asyncio.TimeoutError:
+            log.debug("Timeout waiting for may-I-run task, "
+                      "but that's fine as we're shutting down.")
 
         fworker.shutdown()
 
@@ -203,6 +212,31 @@ def main():
     log.warning('Flamenco Worker is shut down')
 
 
+def parse_pretask_check_config(confparser, log):
+    """Parse the [pre_task_check] config section.
+
+    :rtype: flamenco.worker.PreTaskCheckParams
+    """
+    from . import worker
+
+    check_read = []
+    check_write = []
+    for name, value in confparser.items(section='pre_task_check'):
+        if name.startswith('write'):
+            check_write.append(pathlib.Path(value))
+        elif name.startswith('read'):
+            check_read.append(pathlib.Path(value))
+        else:
+            log.fatal('Config section "pre_task_check" should only have keys starting with '
+                      '"read" or "write"; found %r', value)
+            raise SystemExit(47)
+    pretask_check_params = worker.PreTaskCheckParams(
+        pre_task_check_write=tuple(check_write),
+        pre_task_check_read=tuple(check_read),
+    )
+    return pretask_check_params
+
+
 def asyncio_report_tasks(signum=0, stackframe=None):
     """Runs the garbage collector, then reports all AsyncIO tasks on the log.
 

flamenco_worker/config.py

@@ -5,6 +5,7 @@ import configparser
 import datetime
 import pathlib
 import logging
+import typing
 
 from . import worker
 
@@ -15,9 +16,10 @@ CONFIG_SECTION = 'flamenco-worker'
 DEFAULT_CONFIG = {
     'flamenco-worker': collections.OrderedDict([
         ('manager_url', ''),
-        ('task_types', 'unknown sleep blender-render'),
+        # The 'video-encoding' tasks require ffmpeg to be installed, so it's not enabled by default.
+        ('task_types', 'sleep blender-render file-management exr-merge'),
         ('task_update_queue_db', 'flamenco-worker.db'),
-        ('task_only_one', 'False'),
+        ('subprocess_pid_file', 'flamenco-worker-subprocess.pid'),
         ('may_i_run_interval_seconds', '5'),
         ('worker_id', ''),
         ('worker_secret', ''),
@@ -26,8 +28,9 @@ DEFAULT_CONFIG = {
         ('push_log_max_interval_seconds', str(worker.PUSH_LOG_MAX_INTERVAL.total_seconds())),
         ('push_log_max_entries', str(worker.PUSH_LOG_MAX_ENTRIES)),
         ('push_act_max_interval_seconds', str(worker.PUSH_ACT_MAX_INTERVAL.total_seconds())),
-    ])
-}
+    ]),
+    'pre_task_check': collections.OrderedDict([]),
+}  # type: typing.Mapping[str, typing.Mapping[str, typing.Any]]
 
 # Will be assigned to the config key 'task_types' when started with --test CLI arg.
 TESTING_TASK_TYPES = 'test-blender-render'
@@ -52,8 +55,8 @@ class ConfigParser(configparser.ConfigParser):
         secs = self.value(key, float)
         return datetime.timedelta(seconds=secs)
 
-    def erase(self, key: str) -> bool:
-        return self.set(CONFIG_SECTION, key, '')
+    def erase(self, key: str) -> None:
+        self.set(CONFIG_SECTION, key, '')
 
 
 def merge_with_home_config(new_conf: dict):
@@ -92,13 +95,13 @@ def load_config(config_file: pathlib.Path = None,
     if config_file:
         log.info('Loading configuration from %s', config_file)
         if not config_file.exists():
-            log.fatal('Config file %s does not exist', config_file)
-            raise SystemExit()
+            log.error('Config file %s does not exist', config_file)
+            raise SystemExit(47)
         loaded = confparser.read(str(config_file), encoding='utf8')
     else:
         if not GLOBAL_CONFIG_FILE.exists():
-            log.fatal('Config file %s does not exist', GLOBAL_CONFIG_FILE)
-            raise SystemExit()
+            log.error('Config file %s does not exist', GLOBAL_CONFIG_FILE)
+            raise SystemExit(47)
 
         config_files = [GLOBAL_CONFIG_FILE, HOME_CONFIG_FILE]
         filenames = [str(f.absolute()) for f in config_files]

flamenco_worker/may_i_run.py

@@ -38,11 +38,11 @@ class MayIRun:
             return
 
         if await self.may_i_run(task_id):
-            self._log.debug('Current task may run')
+            self._log.debug('Current task %s may run', task_id)
             return
 
         self._log.warning('We have to stop task %s', task_id)
-        await self.worker.stop_current_task()
+        await self.worker.stop_current_task(task_id)
 
     async def may_i_run(self, task_id: str) -> bool:
         """Asks the Manager whether we are still allowed to run the given task."""

flamenco_worker/runner.py

@@ -17,6 +17,7 @@ class TaskRunner:
     """Runs tasks, sending updates back to the worker."""
 
     shutdown_future = attr.ib(validator=attr.validators.instance_of(asyncio.Future))
+    subprocess_pid_file = attr.ib(validator=attr.validators.instance_of(str))
     last_command_idx = attr.ib(default=0, init=False)
 
     _log = attrs_extra.log('%s.TaskRunner' % __name__)

flamenco_worker/ssdp_discover.py

@@ -30,9 +30,16 @@ class Response(HTTPResponse):
         self.fp = BytesIO(payload)
         self.debuglevel = 0
         self.strict = 0
-        self.headers = self.msg = None
+
+        # This is also done in the HTTPResponse __init__ function, but
+        # MyPy still doesn't like it.
+        self.headers = self.msg = None  # type: ignore
+
         self._method = None
-        self.begin()
+
+        # This function is available on the superclass, but still
+        # MyPy doesn't think it is.
+        self.begin()  # type: ignore
 
 
 def interface_addresses():

flamenco_worker/tz.py

@@ -22,7 +22,9 @@ class tzutc(datetime.tzinfo):
 
         return True
 
-    __hash__ = None
+    # Assigning a different type than object.__hash__ (None resp. Callable)
+    # is not allowed by MyPy. Here it's intentional, though.
+    __hash__ = None  # type: ignore
 
     def __ne__(self, other):
         return not (self == other)

flamenco_worker/upstream.py

 import attr
 import concurrent.futures
-import functools
+import json as json_module  # to prevent shadowing of 'json' parameter
 import requests
 
 from . import attrs_extra
@@ -9,6 +9,13 @@ HTTP_RETRY_COUNT = 5
 HTTP_TIMEOUT = 3  # in seconds
 
 
+def elide(string: str, length: int) -> str:
+    """Cut the string to be no longer than 'length' characters."""
+    if len(string) <= length:
+        return string
+    return f'{string[:length-3]}...'
+
+
 @attr.s
 class FlamencoManager:
     manager_url = attr.ib(validator=attr.validators.instance_of(str))
@@ -78,7 +85,8 @@ class FlamencoManager:
             if json is None:
                 self._log.debug('%s %s', method, abs_url)
             else:
-                self._log.debug('%s %s with JSON: %s', method, abs_url, json)
+                for_log = elide(json_module.dumps(json), 80)
+                self._log.debug('%s %s with JSON: %s', method, abs_url, for_log)
 
         if headers is None:
             headers = {}

flamenco_worker/upstream_update_queue.py

@@ -98,6 +98,7 @@ class TaskUpdateQueue:
 
         if self._db is None:
             self._connect_db()
+            assert self._db is not None
 
         result = self._db.execute('''
             SELECT rowid, url, payload
@@ -114,6 +115,7 @@ class TaskUpdateQueue:
         """Return the number of items queued."""
         if self._db is None:
             self._connect_db()
+            assert self._db is not None
 
         result = self._db.execute('SELECT count(*) FROM fworker_queue')
         count = next(result)[0]
@@ -132,7 +134,7 @@ class TaskUpdateQueue:
         Returns True iff the queue was empty, even before flushing.
         """
 
-        with (await self._queue_lock):
+        async with self._queue_lock:
             queue_is_empty = True
             queue_size_before = self.queue_size()
             handled = 0
@@ -142,15 +144,16 @@ class TaskUpdateQueue:
                 queue_size = self.queue_size()
                 self._log.info('Pushing task update to Manager, queue size is %d', queue_size)
                 resp = await self.manager.post(url, json=payload, loop=loop)
-                if resp.status_code == 409:
-                    # The task was assigned to another worker, so we're not allowed to
-                    # push updates for it. We have to un-queue this update, as it will
-                    # never be accepted.
+                if resp.status_code in {404, 409}:
+                    # 404: Task doesn't exist (any more).
+                    # 409: The task was assigned to another worker, so we're not allowed to
+                    #      push updates for it. We have to un-queue this update, as it will
+                    #      never be accepted.
                     self._log.warning('discarding update, Manager says %s', resp.text)
                     # TODO(sybren): delete all queued updates to the same URL?
                 else:
                     resp.raise_for_status()
-                    self._log.debug('Master accepted pushed update.')
+                    self._log.debug('Manager accepted pushed update.')
                 self._unqueue(rowid)
 
                 handled += 1

publish-online.sh

 #!/bin/bash -e
 
-FLAMENCO_VERSION="2.2-dev1"
+FLAMENCO_VERSION="2.2-dev10"
 
 cd dist
 

requirements-dev.txt

--r requirements-test.txt
-ipython
-pyinstaller
-wheel