Skip to content
Snippets Groups Projects
Commit 6d92fba7 authored by Sybren A. Stüvel's avatar Sybren A. Stüvel
Browse files

Server: progressive rendering is looking good

parent f9ccc242
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
packages/flamenco/flamenco/job_compilers/blender_render_progressive.py
+ 41
− 38
View file @ 6d92fba7
...@@ -19,11 +19,6 @@ class BlenderRenderProgressive(BlenderRender): ...@@ -19,11 +19,6 @@ class BlenderRenderProgressive(BlenderRender):
REQUIRED_SETTINGS = ('blender_cmd', 'filepath', 'render_output', 'frames', 'chunk_size', REQUIRED_SETTINGS = ('blender_cmd', 'filepath', 'render_output', 'frames', 'chunk_size',
'format', 'cycles_sample_count', 'cycles_num_chunks') 'format', 'cycles_sample_count', 'cycles_num_chunks')
def __init__(self):
import re
self.hash_re = re.compile('#+')
def compile(self, job): def compile(self, job):
import math import math
from pathlib2 import Path from pathlib2 import Path
...@@ -42,23 +37,23 @@ class BlenderRenderProgressive(BlenderRender): ...@@ -42,23 +37,23 @@ class BlenderRenderProgressive(BlenderRender):
sample_count_per_chunk = int(math.ceil(float(cycles_sample_count) / self.cycles_num_chunks)) sample_count_per_chunk = int(math.ceil(float(cycles_sample_count) / self.cycles_num_chunks))
next_merge_task_deps = [] next_merge_task_deps = []
prev_samples_from = prev_samples_to = 0 prev_samples_to = 0
for cycles_chunk_idx in range(int(job_settings['cycles_num_chunks'])): for cycles_chunk_idx in range(int(job_settings['cycles_num_chunks'])):
# Compute the Cycles sample range for this chunk, in base-0. # Compute the Cycles sample range for this chunk (chunk_idx in base-0), in base-1.
cycles_samples_from = cycles_chunk_idx * sample_count_per_chunk cycles_samples_from = cycles_chunk_idx * sample_count_per_chunk + 1
cycles_samples_to = min((cycles_chunk_idx + 1) * sample_count_per_chunk, cycles_samples_to = min((cycles_chunk_idx + 1) * sample_count_per_chunk,
cycles_sample_count - 1) cycles_sample_count)
# Create render tasks for each frame chunk. Only this function uses the base-0 # Create render tasks for each frame chunk. Only this function uses the base-0
# chunk/sample numbers, so we also convert to the base-1 numbers that Blender # chunk/sample numbers, so we also convert to the base-1 numbers that Blender
# uses. # uses.
render_task_ids = self._make_progressive_render_tasks( render_task_ids = self._make_progressive_render_tasks(
job, job,
'render-smpl%i-%i-frm%%s' % (cycles_samples_from + 1, cycles_samples_to + 1), 'render-smpl%i-%i-frm%%s' % (cycles_samples_from, cycles_samples_to),
move_existing_task_id, move_existing_task_id,
cycles_chunk_idx + 1, cycles_chunk_idx + 1,
cycles_samples_from + 1, cycles_samples_from,
cycles_samples_to + 1, cycles_samples_to,
task_priority=-cycles_chunk_idx * 10, task_priority=-cycles_chunk_idx * 10,
) )
task_count += len(render_task_ids) task_count += len(render_task_ids)
...@@ -70,7 +65,7 @@ class BlenderRenderProgressive(BlenderRender): ...@@ -70,7 +65,7 @@ class BlenderRenderProgressive(BlenderRender):
else: else:
merge_task_ids = self._make_merge_tasks( merge_task_ids = self._make_merge_tasks(
job, job,
'merge-to-smpl%i-frm%%s' % (cycles_samples_to + 1), 'merge-to-smpl%i-frm%%s' % cycles_samples_to,
cycles_chunk_idx + 1, cycles_chunk_idx + 1,
next_merge_task_deps, next_merge_task_deps,
render_task_ids, render_task_ids,
...@@ -81,7 +76,6 @@ class BlenderRenderProgressive(BlenderRender): ...@@ -81,7 +76,6 @@ class BlenderRenderProgressive(BlenderRender):
) )
task_count += len(merge_task_ids) task_count += len(merge_task_ids)
next_merge_task_deps = merge_task_ids next_merge_task_deps = merge_task_ids
prev_samples_from = cycles_samples_from
prev_samples_to = cycles_samples_to prev_samples_to = cycles_samples_to
self._log.info('Created %i tasks for job %s', task_count, job['_id']) self._log.info('Created %i tasks for job %s', task_count, job['_id'])
...@@ -97,6 +91,13 @@ class BlenderRenderProgressive(BlenderRender): ...@@ -97,6 +91,13 @@ class BlenderRenderProgressive(BlenderRender):
raise exceptions.JobSettingError( raise exceptions.JobSettingError(
u'Job %s must use format="EXR", not %r' % (job[u'_id'], render_format)) u'Job %s must use format="EXR", not %r' % (job[u'_id'], render_format))
# This is quite a limitation, but makes our code to predict the
# filename that Blender will use a lot simpler.
render_output = job['settings']['render_output']
if u'######' not in render_output or u'#######' in render_output:
raise exceptions.JobSettingError(
u'Setting "render_output" must contain exactly 6 "#" marks.')
def _make_progressive_render_tasks(self, def _make_progressive_render_tasks(self,
job, name_fmt, parents, job, name_fmt, parents,
cycles_chunk_idx, cycles_chunk_idx,
...@@ -106,6 +107,7 @@ class BlenderRenderProgressive(BlenderRender): ...@@ -106,6 +107,7 @@ class BlenderRenderProgressive(BlenderRender):
:param parents: either a list of parents, one for each task, or a :param parents: either a list of parents, one for each task, or a
single parent used for all tasks. single parent used for all tasks.
:param cycles_chunk_idx: base-1 sample chunk index
:returns: created task IDs, one render task per frame chunk. :returns: created task IDs, one render task per frame chunk.
:rtype: list :rtype: list
...@@ -147,8 +149,7 @@ class BlenderRenderProgressive(BlenderRender): ...@@ -147,8 +149,7 @@ class BlenderRenderProgressive(BlenderRender):
parent_task_id = parents parent_task_id = parents
if not isinstance(parent_task_id, ObjectId): if not isinstance(parent_task_id, ObjectId):
raise TypeError('parents should be list of ObjectIds or ObjectId, not %s', raise TypeError('parents should be list of ObjectIds or ObjectId, not %s' % parents)
parents)
task_id = self.task_manager.api_create_task( task_id = self.task_manager.api_create_task(
job, task_cmds, name, parents=[parent_task_id], job, task_cmds, name, parents=[parent_task_id],
...@@ -158,14 +159,14 @@ class BlenderRenderProgressive(BlenderRender): ...@@ -158,14 +159,14 @@ class BlenderRenderProgressive(BlenderRender):
return task_ids return task_ids
def _render_output(self, cycles_samples_from, cycles_samples_to): def _render_output(self, cycles_samples_from, cycles_samples_to):
"""Intermediate render output path""" """Intermediate render output path, with ###### placeholder for the frame nr"""
render_fname = u'render-smpl%i-%i-frm-######' % (cycles_samples_from, cycles_samples_to) render_fname = u'render-smpl-%i-%i-frm-######' % (cycles_samples_from, cycles_samples_to)
render_output = self.intermediate_path / render_fname render_output = self.intermediate_path / render_fname
return render_output return render_output
def _merge_output(self, cycles_samples_to): def _merge_output(self, cycles_samples_to):
"""Intermediate merge output path""" """Intermediate merge output path, with ###### placeholder for the frame nr"""
merge_fname = u'merge-smpl%i-frm-######' % cycles_samples_to merge_fname = u'merge-smpl-%i-frm-######' % cycles_samples_to
merge_output = self.intermediate_path / merge_fname merge_output = self.intermediate_path / merge_fname
return merge_output return merge_output
...@@ -176,7 +177,14 @@ class BlenderRenderProgressive(BlenderRender): ...@@ -176,7 +177,14 @@ class BlenderRenderProgressive(BlenderRender):
cycles_samples_from2, cycles_samples_from2,
cycles_samples_to2, cycles_samples_to2,
task_priority): task_priority):
"""Creates merge tasks for each chunk, consisting of merges for each frame.""" """Creates merge tasks for each chunk, consisting of merges for each frame.
:param cycles_chunk_idx: base-1 sample chunk index
"""
# Merging cannot happen unless we have at least two chunks
assert cycles_chunk_idx >= 2
from flamenco.utils import iter_frame_range, frame_range_merge from flamenco.utils import iter_frame_range, frame_range_merge
...@@ -186,7 +194,7 @@ class BlenderRenderProgressive(BlenderRender): ...@@ -186,7 +194,7 @@ class BlenderRenderProgressive(BlenderRender):
cycles_num_chunks = int(job_settings['cycles_num_chunks']) cycles_num_chunks = int(job_settings['cycles_num_chunks'])
weight1 = cycles_samples_from2 weight1 = cycles_samples_to1
weight2 = cycles_samples_to2 - cycles_samples_from2 + 1 weight2 = cycles_samples_to2 - cycles_samples_from2 + 1
# Replace Blender formatting with Python formatting in render output path # Replace Blender formatting with Python formatting in render output path
...@@ -196,32 +204,27 @@ class BlenderRenderProgressive(BlenderRender): ...@@ -196,32 +204,27 @@ class BlenderRenderProgressive(BlenderRender):
input1 = self._render_output(1, cycles_samples_to1) input1 = self._render_output(1, cycles_samples_to1)
else: else:
input1 = self._merge_output(cycles_samples_to1) input1 = self._merge_output(cycles_samples_to1)
input1 = unicode(input1).replace(u'######', u'%06i')
input2 = self._render_output(cycles_samples_from2, cycles_samples_to2) input2 = self._render_output(cycles_samples_from2, cycles_samples_to2)
input2 = unicode(input2).replace(u'######', u'%06i')
if cycles_chunk_idx == cycles_num_chunks: if cycles_chunk_idx == cycles_num_chunks:
# At the last merge, we merge to the actual render output, not to intermediary. # At the last merge, we merge to the actual render output, not to intermediary.
output = output = job['settings']['render_output']
else: else:
output = self._merge_output(cycles_samples_to2) output = self._merge_output(cycles_samples_to2)
output = unicode(output).replace(u'######', u'%06i')
frame_chunk_iter = iter_frame_range(job_settings['frames'], job_settings['chunk_size']) frame_chunk_iter = iter_frame_range(job_settings['frames'], job_settings['chunk_size'])
for chunk_idx, chunk_frames in enumerate(frame_chunk_iter): for chunk_idx, chunk_frames in enumerate(frame_chunk_iter):
# Create a merge command for every frame in the chunk. # Create a merge command for every frame in the chunk.
task_cmds = [ task_cmds = []
commands.MergeProgressiveRenders( for framenr in chunk_frames:
input1=input1 % framenr, task_cmds.append(
input2=input2 % framenr, commands.MergeProgressiveRenders(
output=output % framenr, input1=unicode(input1).replace(u'######', u'%06i') % framenr,
weight1=weight1, input2=unicode(input2).replace(u'######', u'%06i') % framenr,
weight2=weight2, output=unicode(output).replace(u'######', u'%06i') % framenr,
) weight1=weight1,
for framenr in chunk_frames weight2=weight2,
] ))
name = name_fmt % frame_range_merge(chunk_frames) name = name_fmt % frame_range_merge(chunk_frames)
......
packages/flamenco/tests/test_job_compilers.py
+ 331
− 0
View file @ 6d92fba7
...@@ -60,3 +60,334 @@ class CommandTest(unittest.TestCase): ...@@ -60,3 +60,334 @@ class CommandTest(unittest.TestCase):
'message': u'Preparing to sleep', 'message': u'Preparing to sleep',
} }
}, cmd.to_dict()) }, cmd.to_dict())
class BlenderRenderProgressiveTest(unittest.TestCase):
def test_nonexr_job(self):
from flamenco.job_compilers import blender_render_progressive
from flamenco.exceptions import JobSettingError
job_doc = {
u'_id': ObjectId(24 * 'f'),
u'settings': {
u'frames': u'1-6',
u'chunk_size': 2,
u'render_output': u'/render/out/frames-######',
u'format': u'JPEG',
u'filepath': u'/agent327/scenes/someshot/somefile.blend',
u'blender_cmd': u'/path/to/blender --enable-new-depsgraph',
u'cycles_sample_count': 30,
u'cycles_num_chunks': 3,
}
}
task_manager = mock.Mock()
compiler = blender_render_progressive.BlenderRenderProgressive(task_manager=task_manager)
self.assertRaises(JobSettingError, compiler.compile, job_doc)
def test_small_job(self):
from flamenco.job_compilers import blender_render_progressive, commands
job_doc = {
u'_id': ObjectId(24 * 'f'),
u'settings': {
u'frames': u'1-6',
u'chunk_size': 2,
u'render_output': u'/render/out/frames-######',
u'format': u'EXR',
u'filepath': u'/agent327/scenes/someshot/somefile.blend',
u'blender_cmd': u'/path/to/blender --enable-new-depsgraph',
u'cycles_sample_count': 30,
u'cycles_num_chunks': 3,
}
}
task_manager = mock.Mock()
# We expect:
# - 1 move-out-of-way task
# - 3 frame chunks of 2 frames each
# - 3 progressive renders per frame
# - 2 merge tasks per frame chunk
# so that's a 9 render tasks and 6 merge tasks, giving 16 tasks in total.
task_ids = [ObjectId() for _ in range(16)]
task_manager.api_create_task.side_effect = task_ids
compiler = blender_render_progressive.BlenderRenderProgressive(task_manager=task_manager)
compiler.compile(job_doc)
task_manager.api_create_task.assert_has_calls([
mock.call(job_doc,
[commands.MoveOutOfWay(src=u'/render/out')],
u'move-existing-frames'),
# First Cycles chunk
mock.call(
job_doc,
[commands.BlenderRenderProgressive(
blender_cmd=u'/path/to/blender --enable-new-depsgraph',
filepath=u'/agent327/scenes/someshot/somefile.blend',
format=u'EXR',
render_output=u'/render/out/_intermediate/render-smpl-1-10-frm-######',
frames=u'1,2',
cycles_num_chunks=3,
cycles_chunk=1,
cycles_samples_from=1,
cycles_samples_to=10)],
u'render-smpl1-10-frm1,2',
parents=[task_ids[0]],
priority=0),
mock.call(
job_doc,
[commands.BlenderRenderProgressive(
blender_cmd=u'/path/to/blender --enable-new-depsgraph',
filepath=u'/agent327/scenes/someshot/somefile.blend',
format=u'EXR',
render_output=u'/render/out/_intermediate/render-smpl-1-10-frm-######',
frames=u'3,4',
cycles_num_chunks=3,
cycles_chunk=1,
cycles_samples_from=1,
cycles_samples_to=10)],
u'render-smpl1-10-frm3,4',
parents=[task_ids[0]],
priority=0),
mock.call(
job_doc,
[commands.BlenderRenderProgressive(
blender_cmd=u'/path/to/blender --enable-new-depsgraph',
filepath=u'/agent327/scenes/someshot/somefile.blend',
format=u'EXR',
render_output=u'/render/out/_intermediate/render-smpl-1-10-frm-######',
frames=u'5,6',
cycles_num_chunks=3,
cycles_chunk=1,
cycles_samples_from=1,
cycles_samples_to=10)],
u'render-smpl1-10-frm5,6',
parents=[task_ids[0]],
priority=0),
# Second Cycles chunk
mock.call(
job_doc,
[commands.BlenderRenderProgressive(
blender_cmd=u'/path/to/blender --enable-new-depsgraph',
filepath=u'/agent327/scenes/someshot/somefile.blend',
format=u'EXR',
render_output=u'/render/out/_intermediate/render-smpl-11-20-frm-######',
frames=u'1,2',
cycles_num_chunks=3,
cycles_chunk=2,
cycles_samples_from=11,
cycles_samples_to=20)],
u'render-smpl11-20-frm1,2',
parents=[task_ids[0]],
priority=-10),
mock.call(
job_doc,
[commands.BlenderRenderProgressive(
blender_cmd=u'/path/to/blender --enable-new-depsgraph',
filepath=u'/agent327/scenes/someshot/somefile.blend',
format=u'EXR',
render_output=u'/render/out/_intermediate/render-smpl-11-20-frm-######',
frames=u'3,4',
cycles_num_chunks=3,
cycles_chunk=2,
cycles_samples_from=11,
cycles_samples_to=20)],
u'render-smpl11-20-frm3,4',
parents=[task_ids[0]],
priority=-10),
mock.call(
job_doc,
[commands.BlenderRenderProgressive(
blender_cmd=u'/path/to/blender --enable-new-depsgraph',
filepath=u'/agent327/scenes/someshot/somefile.blend',
format=u'EXR',
render_output=u'/render/out/_intermediate/render-smpl-11-20-frm-######',
frames=u'5,6',
cycles_num_chunks=3,
cycles_chunk=2,
cycles_samples_from=11,
cycles_samples_to=20)],
u'render-smpl11-20-frm5,6',
parents=[task_ids[0]],
priority=-10),
# First merge pass
mock.call(
job_doc,
[
commands.MergeProgressiveRenders(
input1=u'/render/out/_intermediate/render-smpl-1-10-frm-000001',
input2=u'/render/out/_intermediate/render-smpl-11-20-frm-000001',
output=u'/render/out/_intermediate/merge-smpl-20-frm-000001',
weight1=10,
weight2=10,
),
commands.MergeProgressiveRenders(
input1=u'/render/out/_intermediate/render-smpl-1-10-frm-000002',
input2=u'/render/out/_intermediate/render-smpl-11-20-frm-000002',
output=u'/render/out/_intermediate/merge-smpl-20-frm-000002',
weight1=10,
weight2=10,
),
],
u'merge-to-smpl20-frm1,2',
parents=[task_ids[1], task_ids[4]],
priority=-11),
mock.call(
job_doc,
[
commands.MergeProgressiveRenders(
input1=u'/render/out/_intermediate/render-smpl-1-10-frm-000003',
input2=u'/render/out/_intermediate/render-smpl-11-20-frm-000003',
output=u'/render/out/_intermediate/merge-smpl-20-frm-000003',
weight1=10,
weight2=10,
),
commands.MergeProgressiveRenders(
input1=u'/render/out/_intermediate/render-smpl-1-10-frm-000004',
input2=u'/render/out/_intermediate/render-smpl-11-20-frm-000004',
output=u'/render/out/_intermediate/merge-smpl-20-frm-000004',
weight1=10,
weight2=10,
),
],
u'merge-to-smpl20-frm3,4',
parents=[task_ids[2], task_ids[5]],
priority=-11),
mock.call(
job_doc,
[
commands.MergeProgressiveRenders(
input1=u'/render/out/_intermediate/render-smpl-1-10-frm-000005',
input2=u'/render/out/_intermediate/render-smpl-11-20-frm-000005',
output=u'/render/out/_intermediate/merge-smpl-20-frm-000005',
weight1=10,
weight2=10,
),
commands.MergeProgressiveRenders(
input1=u'/render/out/_intermediate/render-smpl-1-10-frm-000006',
input2=u'/render/out/_intermediate/render-smpl-11-20-frm-000006',
output=u'/render/out/_intermediate/merge-smpl-20-frm-000006',
weight1=10,
weight2=10,
),
],
u'merge-to-smpl20-frm5,6',
parents=[task_ids[3], task_ids[6]],
priority=-11),
# Third Cycles chunk
mock.call(
job_doc,
[commands.BlenderRenderProgressive(
blender_cmd=u'/path/to/blender --enable-new-depsgraph',
filepath=u'/agent327/scenes/someshot/somefile.blend',
format=u'EXR',
render_output=u'/render/out/_intermediate/render-smpl-21-30-frm-######',
frames=u'1,2',
cycles_num_chunks=3,
cycles_chunk=3,
cycles_samples_from=21,
cycles_samples_to=30)],
u'render-smpl21-30-frm1,2',
parents=[task_ids[0]],
priority=-20),
mock.call(
job_doc,
[commands.BlenderRenderProgressive(
blender_cmd=u'/path/to/blender --enable-new-depsgraph',
filepath=u'/agent327/scenes/someshot/somefile.blend',
format=u'EXR',
render_output=u'/render/out/_intermediate/render-smpl-21-30-frm-######',
frames=u'3,4',
cycles_num_chunks=3,
cycles_chunk=3,
cycles_samples_from=21,
cycles_samples_to=30)],
u'render-smpl21-30-frm3,4',
parents=[task_ids[0]],
priority=-20),
mock.call(
job_doc,
[commands.BlenderRenderProgressive(
blender_cmd=u'/path/to/blender --enable-new-depsgraph',
filepath=u'/agent327/scenes/someshot/somefile.blend',
format=u'EXR',
render_output=u'/render/out/_intermediate/render-smpl-21-30-frm-######',
frames=u'5,6',
cycles_num_chunks=3,
cycles_chunk=3,
cycles_samples_from=21,
cycles_samples_to=30)],
u'render-smpl21-30-frm5,6',
parents=[task_ids[0]],
priority=-20),
# Final merge pass
mock.call(
job_doc,
[
commands.MergeProgressiveRenders(
input1=u'/render/out/_intermediate/merge-smpl-20-frm-000001',
input2=u'/render/out/_intermediate/render-smpl-21-30-frm-000001',
output=u'/render/out/frames-000001',
weight1=20,
weight2=10,
),
commands.MergeProgressiveRenders(
input1=u'/render/out/_intermediate/merge-smpl-20-frm-000002',
input2=u'/render/out/_intermediate/render-smpl-21-30-frm-000002',
output=u'/render/out/frames-000002',
weight1=20,
weight2=10,
),
],
u'merge-to-smpl30-frm1,2',
parents=[task_ids[7], task_ids[10]],
priority=-21),
mock.call(
job_doc,
[
commands.MergeProgressiveRenders(
input1=u'/render/out/_intermediate/merge-smpl-20-frm-000003',
input2=u'/render/out/_intermediate/render-smpl-21-30-frm-000003',
output=u'/render/out/frames-000003',
weight1=20,
weight2=10,
),
commands.MergeProgressiveRenders(
input1=u'/render/out/_intermediate/merge-smpl-20-frm-000004',
input2=u'/render/out/_intermediate/render-smpl-21-30-frm-000004',
output=u'/render/out/frames-000004',
weight1=20,
weight2=10,
),
],
u'merge-to-smpl30-frm3,4',
parents=[task_ids[8], task_ids[11]],
priority=-21),
mock.call(
job_doc,
[
commands.MergeProgressiveRenders(
input1=u'/render/out/_intermediate/merge-smpl-20-frm-000005',
input2=u'/render/out/_intermediate/render-smpl-21-30-frm-000005',
output=u'/render/out/frames-000005',
weight1=20,
weight2=10,
),
commands.MergeProgressiveRenders(
input1=u'/render/out/_intermediate/merge-smpl-20-frm-000006',
input2=u'/render/out/_intermediate/render-smpl-21-30-frm-000006',
output=u'/render/out/frames-000006',
weight1=20,
weight2=10,
),
],
u'merge-to-smpl30-frm5,6',
parents=[task_ids[9], task_ids[12]],
priority=-21),
])
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment