Commit 91ec4459 authored by Milan Jaros's avatar Milan Jaros
Browse files

add new version v3.2

parent 7d4ad138
......@@ -14,11 +14,11 @@
bl_info = {
"name" : "bheappe",
"author" : "Milan Jaros, Petr Strakos",
"description" : "Rendering-as-a-service",
"blender" : (2, 80, 0),
"version" : (0, 0, 2),
"description" : "Rendering-as-a-Service",
"blender" : (2, 93, 0),
"version" : (3, 2, 0),
"location" : "Addon Preferences panel",
"wiki_url" : "http://blender.it4i.cz/",
"wiki_url" : "https://blender.it4i.cz/",
"category" : "System",
}
......@@ -31,40 +31,17 @@ if 'wheels' in locals():
wheels = importlib.reload(wheels)
wheels.load_wheels()
#pillar = importlib.reload(pillar)
else:
from . import wheels
wheels.load_wheels()
#from . import pillar
log = logging.getLogger(__name__)
# def _monkey_patch_requests():
# """Monkey-patch old versions of Requests.
# This is required for the Mac version of Blender 2.77a.
# """
# import requests
# if requests.__build__ >= 0x020601:
# return
# log.info('Monkey-patching requests version %s', requests.__version__)
# from requests.packages.urllib3.response import HTTPResponse
# HTTPResponse.chunked = False
# HTTPResponse.chunk_left = None
def register():
"""Late-loads and registers the Blender-dependent submodules."""
import sys
#_monkey_patch_requests()
#sys.path.append('raas_lib')
# Support reloading
if '%s.raas_pref' % __name__ in sys.modules:
......@@ -114,8 +91,10 @@ def unregister():
from . import raas_pref
from . import raas_render
async_loop.unregister()
raas_pref.unregister()
raas_render.unregister()
try:
async_loop.unregister()
raas_pref.unregister()
raas_render.unregister()
except RuntimeError:
pass
......@@ -16,6 +16,8 @@
#
# ##### END GPL LICENSE BLOCK #####
# (c) Blender Foundation
"""Manages the asyncio loop."""
import asyncio
......@@ -24,7 +26,6 @@ import concurrent.futures
import logging
import gc
import typing
#import aiohttp
import bpy
......@@ -39,11 +40,13 @@ def setup_asyncio_executor():
import sys
if sys.platform == 'win32':
if sys.platform == "win32":
asyncio.get_event_loop().close()
# On Windows, the default event loop is SelectorEventLoop, which does
# not support subprocesses. ProactorEventLoop should be used instead.
# Source: https://docs.python.org/3/library/asyncio-subprocess.html
#
# NOTE: this is actually the default even loop in Python 3.9+.
loop = asyncio.ProactorEventLoop()
asyncio.set_event_loop(loop)
else:
......@@ -53,18 +56,10 @@ def setup_asyncio_executor():
loop.set_default_executor(executor)
# loop.set_debug(True)
#from . import raas_server
#import certifi
#import ssl
#import requests
# No more than this many Heappe calls should be made simultaneously
#raas_server.raas_semaphore = asyncio.Semaphore(1, loop=loop)
#raas_server.raas_client = aiohttp.ClientSession(loop=loop, connector=aiohttp.TCPConnector(verify_ssl=False))
#sslcontext = ssl.create_default_context(cafile=certifi.where())
#sslcontext = ssl.create_default_context(purpose=ssl.Purpose.CLIENT_AUTH)
#raas_server.raas_client = requests.session() #aiohttp.ClientSession(loop=loop, connector=aiohttp.TCPConnector(ssl_context=sslcontext))
# from . import pillar
# No more than this many Pillar calls should be made simultaneously
# pillar.pillar_semaphore = asyncio.Semaphore(3, loop=loop)
def kick_async_loop(*args) -> bool:
"""Performs a single iteration of the asyncio event loop.
......@@ -79,17 +74,23 @@ def kick_async_loop(*args) -> bool:
stop_after_this_kick = False
if loop.is_closed():
log.warning('loop closed, stopping immediately.')
log.warning("loop closed, stopping immediately.")
return True
all_tasks = asyncio.Task.all_tasks()
# Passing an explicit loop is required. Without it, the function uses
# asyncio.get_running_loop(), which raises a RuntimeError as the current
# loop isn't running.
all_tasks = asyncio.all_tasks(loop=loop)
if not len(all_tasks):
log.debug('no more scheduled tasks, stopping after this kick.')
log.debug("no more scheduled tasks, stopping after this kick.")
stop_after_this_kick = True
elif all(task.done() for task in all_tasks):
log.debug('all %i tasks are done, fetching results and stopping after this kick.',
len(all_tasks))
log.debug(
"all %i tasks are done, fetching results and stopping after this kick.",
len(all_tasks),
)
stop_after_this_kick = True
# Clean up circular references between tasks.
......@@ -102,12 +103,12 @@ def kick_async_loop(*args) -> bool:
# noinspection PyBroadException
try:
res = task.result()
log.debug(' task #%i: result=%r', task_idx, res)
log.debug(" task #%i: result=%r", task_idx, res)
except asyncio.CancelledError:
# No problem, we want to stop anyway.
log.debug(' task #%i: cancelled', task_idx)
log.debug(" task #%i: cancelled", task_idx)
except Exception:
print('{}: resulted in exception'.format(task))
print("{}: resulted in exception".format(task))
traceback.print_exc()
# for ref in gc.get_referrers(task):
......@@ -120,26 +121,26 @@ def kick_async_loop(*args) -> bool:
def ensure_async_loop():
log.debug('Starting asyncio loop')
log.debug("Starting asyncio loop")
result = bpy.ops.asyncio.loop()
log.debug('Result of starting modal operator is %r', result)
log.debug("Result of starting modal operator is %r", result)
def erase_async_loop():
global _loop_kicking_operator_running
log.debug('Erasing async loop')
log.debug("Erasing async loop")
loop = asyncio.get_event_loop()
loop.stop()
class AsyncLoopModalOperator(bpy.types.Operator):
bl_idname = 'asyncio.loop'
bl_label = 'Runs the asyncio main loop'
bl_idname = "asyncio.loop"
bl_label = "Runs the asyncio main loop"
timer = None
log = logging.getLogger(__name__ + '.AsyncLoopModalOperator')
log = logging.getLogger(__name__ + ".AsyncLoopModalOperator")
def __del__(self):
global _loop_kicking_operator_running
......@@ -150,16 +151,16 @@ class AsyncLoopModalOperator(bpy.types.Operator):
_loop_kicking_operator_running = False
def execute(self, context):
context.window_manager.raas_status = 'COMMUNICATING'
context.window_manager.raas_status_txt = ''
context.window_manager.raas_status = "COMMUNICATING"
context.window_manager.raas_status_txt = ""
return self.invoke(context, None)
def invoke(self, context, event):
global _loop_kicking_operator_running
if _loop_kicking_operator_running:
self.log.debug('Another loop-kicking operator is already running.')
return {'PASS_THROUGH'}
self.log.debug("Another loop-kicking operator is already running.")
return {"PASS_THROUGH"}
context.window_manager.modal_handler_add(self)
_loop_kicking_operator_running = True
......@@ -167,7 +168,7 @@ class AsyncLoopModalOperator(bpy.types.Operator):
wm = context.window_manager
self.timer = wm.event_timer_add(0.00001, window=context.window)
return {'RUNNING_MODAL'}
return {"RUNNING_MODAL"}
def modal(self, context, event):
global _loop_kicking_operator_running
......@@ -176,10 +177,10 @@ class AsyncLoopModalOperator(bpy.types.Operator):
# erase_async_loop(). This is a signal that we really should stop
# running.
if not _loop_kicking_operator_running:
return {'FINISHED'}
return {"FINISHED"}
if event.type != 'TIMER':
return {'PASS_THROUGH'}
if event.type != "TIMER":
return {"PASS_THROUGH"}
# self.log.debug('KICKING LOOP')
stop_after_this_kick = kick_async_loop()
......@@ -187,34 +188,38 @@ class AsyncLoopModalOperator(bpy.types.Operator):
context.window_manager.event_timer_remove(self.timer)
_loop_kicking_operator_running = False
self.log.debug('Stopped asyncio loop kicking')
return {'FINISHED'}
self.log.debug("Stopped asyncio loop kicking")
return {"FINISHED"}
return {'RUNNING_MODAL'}
return {"RUNNING_MODAL"}
def quit(self):
"""Signals the state machine to stop this operator from running."""
bpy.context.window_manager.raas_status = 'DONE'
bpy.context.window_manager.raas_status = "DONE"
super.quit()
# noinspection PyAttributeOutsideInit
class AsyncModalOperatorMixin:
async_task = None # asyncio task for fetching thumbnails
signalling_future = None # asyncio future for signalling that we want to cancel everything.
log = logging.getLogger('%s.AsyncModalOperatorMixin' % __name__)
signalling_future = (
None # asyncio future for signalling that we want to cancel everything.
)
log = logging.getLogger("%s.AsyncModalOperatorMixin" % __name__)
_state = 'INITIALIZING'
stop_upon_exception = True
_state = "INITIALIZING"
stop_upon_exception = False
def invoke(self, context, event):
context.window_manager.modal_handler_add(self)
self.timer = context.window_manager.event_timer_add(1 / 15, window=context.window)
self.timer = context.window_manager.event_timer_add(
1 / 15, window=context.window
)
self.log.info('Starting')
self.log.info("Starting")
self._new_async_task(self.async_execute(context))
return {'RUNNING_MODAL'}
return {"RUNNING_MODAL"}
async def async_execute(self, context):
"""Entry point of the asynchronous operator.
......@@ -225,63 +230,67 @@ class AsyncModalOperatorMixin:
def quit(self):
"""Signals the state machine to stop this operator from running."""
if bpy.context.window_manager.raas_status != 'ERROR':
bpy.context.window_manager.raas_status = 'DONE'
if bpy.context.window_manager.raas_status != "ERROR":
bpy.context.window_manager.raas_status = "DONE"
self._state = 'QUIT'
self._state = "QUIT"
def execute(self, context):
context.window_manager.raas_status = 'COMMUNICATING'
context.window_manager.raas_status_txt = ''
context.window_manager.raas_status = "COMMUNICATING"
context.window_manager.raas_status_txt = ""
return self.invoke(context, None)
def modal(self, context, event):
task = self.async_task
if self._state != 'EXCEPTION' and task and task.done() and not task.cancelled():
if self._state != "EXCEPTION" and task and task.done() and not task.cancelled():
ex = task.exception()
if ex is not None:
self._state = 'EXCEPTION'
self.log.error('Exception while running task: %s', ex)
self._state = "EXCEPTION"
self.log.error("Exception while running task: %s", ex)
context.window_manager.raas_status = 'ERROR'
context.window_manager.raas_status_txt = 'Exception while running task: %s' % ex
context.window_manager.raas_status = "ERROR"
context.window_manager.raas_status_txt = "Exception while running task: %s" % ex
if self.stop_upon_exception:
self.quit()
self._finish(context)
return {'FINISHED'}
return {"FINISHED"}
return {'RUNNING_MODAL'}
return {"RUNNING_MODAL"}
if self._state == 'QUIT':
if self._state == "QUIT":
self._finish(context)
return {'FINISHED'}
return {"FINISHED"}
return {'PASS_THROUGH'}
return {"PASS_THROUGH"}
def _finish(self, context):
self._stop_async_task()
context.window_manager.event_timer_remove(self.timer)
def _new_async_task(self, async_task: typing.Coroutine, future: asyncio.Future = None):
def _new_async_task(
self, async_task: typing.Coroutine, future: asyncio.Future = None
):
"""Stops the currently running async task, and starts another one."""
self.log.debug('Setting up a new task %r, so any existing task must be stopped', async_task)
self.log.debug(
"Setting up a new task %r, so any existing task must be stopped", async_task
)
self._stop_async_task()
# Download the previews asynchronously.
self.signalling_future = future or asyncio.Future()
self.async_task = asyncio.ensure_future(async_task)
self.log.debug('Created new task %r', self.async_task)
self.log.debug("Created new task %r", self.async_task)
# Start the async manager so everything happens.
ensure_async_loop()
def _stop_async_task(self):
self.log.debug('Stopping async task')
self.log.debug("Stopping async task")
if self.async_task is None:
self.log.debug('No async task, trivially stopped')
self.log.debug("No async task, trivially stopped")
return
# Signal that we want to stop.
......@@ -297,14 +306,14 @@ class AsyncModalOperatorMixin:
try:
loop.run_until_complete(self.async_task)
except asyncio.CancelledError:
self.log.info('Asynchronous task was cancelled')
self.log.info("Asynchronous task was cancelled")
return
# noinspection PyBroadException
try:
self.async_task.result() # This re-raises any exception of the task.
except asyncio.CancelledError:
self.log.info('Asynchronous task was cancelled')
self.log.info("Asynchronous task was cancelled")
except Exception:
self.log.exception("Exception from asynchronous task")
......
......@@ -11,6 +11,8 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# (c) Blender Foundation
"""BAT🦇 packing interface for Raas."""
import asyncio
......@@ -49,7 +51,6 @@ class BatProgress(progress.Callback):
self.loop = asyncio.get_event_loop()
def _set_attr(self, attr: str, value):
async def do_it():
setattr(bpy.context.window_manager, attr, value)
......@@ -83,21 +84,21 @@ class BatProgress(progress.Callback):
def trace_blendfile(self, filename: pathlib.Path) -> None:
"""Called for every blendfile opened when tracing dependencies."""
self._txt('Inspecting %s' % filename.name)
self._txt("Inspecting %s" % filename.name)
def trace_asset(self, filename: pathlib.Path) -> None:
if filename.stem == '.blend':
if filename.stem == ".blend":
return
self._txt('Found asset %s' % filename.name)
self._txt("Found asset %s" % filename.name)
def rewrite_blendfile(self, orig_filename: pathlib.Path) -> None:
self._txt('Rewriting %s' % orig_filename.name)
self._txt("Rewriting %s" % orig_filename.name)
def transfer_file(self, src: pathlib.Path, dst: pathlib.Path) -> None:
self._txt('Transferring %s' % src.name)
self._txt("Transferring %s" % src.name)
def transfer_file_skipped(self, src: pathlib.Path, dst: pathlib.Path) -> None:
self._txt('Skipped %s' % src.name)
self._txt("Skipped %s" % src.name)
def transfer_progress(self, total_bytes: int, transferred_bytes: int) -> None:
self._progress(round(100 * transferred_bytes / total_bytes))
......@@ -125,15 +126,6 @@ class ShamanPacker(shaman.ShamanPacker):
def _get_auth_token(self) -> str:
"""get a token from Raas Server"""
#from ..blender import PILLAR_SERVER_URL
#from ..pillar import blender_id_subclient, uncached_session, SUBCLIENT_ID
#url = urllib.parse.urljoin(PILLAR_SERVER_URL,
# 'raas/jwt/generate-token/%s' % self.manager_id)
#auth_token = blender_id_subclient()['token']
#resp = uncached_session.get(url, auth=(auth_token, SUBCLIENT_ID))
#resp.raise_for_status()
return "resp.text"
......@@ -145,28 +137,33 @@ async def copy(context,
*,
relative_only: bool,
packer_class=pack.Packer,
**packer_args) \
-> typing.Tuple[pathlib.Path, typing.Set[pathlib.Path]]:
**packer_args
) -> typing.Tuple[pathlib.Path, typing.Set[pathlib.Path]]:
"""Use BAT🦇 to copy the given file and dependencies to the target location.
:raises: FileTransferError if a file couldn't be transferred.
:returns: the path of the packed blend file, and a set of missing sources.
"""
global _running_packer
#from concurrent.futures import ThreadPoolExecutor
loop = asyncio.get_event_loop()
wm = bpy.context.window_manager
packer = packer_class(base_blendfile, project, target,
compress=True, relative_only=relative_only, **packer_args)
packer = packer_class(
base_blendfile,
project,
target,
compress=True,
relative_only=relative_only,
**packer_args
)
with packer:
with _packer_lock:
if exclusion_filter:
# There was a mistake in an older version of the property tooltip,
# showing semicolon-separated instead of space-separated. We now
# just handle both.
filter_parts = re.split('[ ;]+', exclusion_filter.strip(' ;'))
filter_parts = re.split("[ ;]+", exclusion_filter.strip(" ;"))
packer.exclude(*filter_parts)
packer.progress_cb = BatProgress()
......@@ -175,18 +172,12 @@ async def copy(context,
log.debug('awaiting strategise')
wm.raas_status = 'INVESTIGATING'
#executor = ThreadPoolExecutor(max_workers=1)
#futures = []
#futures.append(await loop.run_in_executor(executor, packer.strategise))
await loop.run_in_executor(None, packer.strategise)
log.debug('awaiting execute')
wm.raas_status = 'TRANSFERRING'
#futures.append(await loop.run_in_executor(executor, packer.execute))
await loop.run_in_executor(None, packer.execute)
#await asyncio.wait(futures)
log.debug('done')
wm.raas_status = 'DONE'
......@@ -204,7 +195,7 @@ def abort() -> None:
with _packer_lock:
if _running_packer is None:
log.debug('No running packer, ignoring call to bat_abort()')
log.debug("No running packer, ignoring call to bat_abort()")
return
log.info('Aborting running packer')
log.info("Aborting running packer")
_running_packer.abort()
This diff is collapsed.
# ##### BEGIN GPL LICENSE BLOCK #####
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# ##### END GPL LICENSE BLOCK #####
"""RaaS config."""
import bpy
Cluster_items = [
("BARBORA", "Barbora", ""),
]
JobQueueBarbora_items = [
("ORIGCPU", "Blender2.93 CPU", ""),
("ORIGCUDA", "Blender2.93 CUDA", ""),
]
from . import raas_jobs
# async def CreateJobSimple(context, token, job_nodes, ClusterNodeTypeId, CommandTemplateId, FileTransferMethodId, ClusterId)
async def CreateJob(context, token):
blender_job_info_new = context.scene.raas_blender_job_info_new
if blender_job_info_new.cluster_type == 'BARBORA':
if blender_job_info_new.job_queue_barbora == 'ORIGCPU':
await raas_jobs.CreateJobSimple(context, token, 36, 14, 10, 3, 3)
if blender_job_info_new.job_queue_barbora == 'ORIGCUDA':
await raas_jobs.CreateJobSimple(context, token, 24, 16, 7, 3, 3)
# ##### BEGIN GPL LICENSE BLOCK #####
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# ##### END GPL LICENSE BLOCK #####
"""RaaS config."""
import bpy
Cluster_items = [
("SALOMON", "Salomon", ""),
("BARBORA", "Barbora", ""),
]
JobQueueSalomon_items = [
("ORIGCPU", "Blender2.93 CPU", ""),
]
JobQueueBarbora_items = [
("ORIGCPU", "Blender2.93 CPU", ""),
("ORIGCUDA", "Blender2.93 CUDA", ""),
]
JobQueueKarolina_items = [
]
from . import raas_jobs
# async def CreateJobSimple(context, token, job_nodes, ClusterNodeTypeId, CommandTemplateId, FileTransferMethodId, ClusterId)
async def CreateJob(context, token):