Commit 13697712 authored by Milan Jaros's avatar Milan Jaros
Browse files

add paramiko

parent 096de5d1
......@@ -49,6 +49,127 @@ def factor(factor: float) -> dict:
return {'percentage': factor}
return {'factor': factor}
##################################################
from collections import namedtuple
import subprocess, sys, importlib
Dependency = namedtuple("Dependency", ["module", "package", "name"])
python_dependencies = (Dependency(module="paramiko", package="paramiko", name=None),
Dependency(module="scp", package="scp", name=None),
)
internal_dependencies = []
def import_module(module_name, global_name=None, reload=True):
"""
Import a module.
:param module_name: Module to import.
:param global_name: (Optional) Name under which the module is imported. If None the module_name will be used.
This allows to import under a different name with the same effect as e.g. "import numpy as np" where "np" is
the global_name under which the module can be accessed.
:raises: ImportError and ModuleNotFoundError
"""
if global_name is None:
global_name = module_name
if global_name in globals():
importlib.reload(globals()[global_name])
else:
# Attempt to import the module and assign it to globals dictionary. This allow to access the module under
# the given name, just like the regular import would.
globals()[global_name] = importlib.import_module(module_name)
def install_pip():
"""
Installs pip if not already present. Please note that ensurepip.bootstrap() also calls pip, which adds the
environment variable PIP_REQ_TRACKER. After ensurepip.bootstrap() finishes execution, the directory doesn't exist
anymore. However, when subprocess is used to call pip, in order to install a package, the environment variables
still contain PIP_REQ_TRACKER with the now nonexistent path. This is a problem since pip checks if PIP_REQ_TRACKER
is set and if it is, attempts to use it as temp directory. This would result in an error because the
directory can't be found. Therefore, PIP_REQ_TRACKER needs to be removed from environment variables.
:return:
"""
try:
# Check if pip is already installed
subprocess.run([sys.executable, "-m", "pip", "--version"], check=True)
except subprocess.CalledProcessError:
import ensurepip
ensurepip.bootstrap()
os.environ.pop("PIP_REQ_TRACKER", None)
def install_and_import_module(module_name, package_name=None, global_name=None):
"""
Installs the package through pip and attempts to import the installed module.
:param module_name: Module to import.
:param package_name: (Optional) Name of the package that needs to be installed. If None it is assumed to be equal
to the module_name.
:param global_name: (Optional) Name under which the module is imported. If None the module_name will be used.
This allows to import under a different name with the same effect as e.g. "import numpy as np" where "np" is
the global_name under which the module can be accessed.
:raises: subprocess.CalledProcessError and ImportError
"""
if package_name is None:
package_name = module_name
if global_name is None:
global_name = module_name
# Blender disables the loading of user site-packages by default. However, pip will still check them to determine
# if a dependency is already installed. This can cause problems if the packages is installed in the user
# site-packages and pip deems the requirement satisfied, but Blender cannot import the package from the user
# site-packages. Hence, the environment variable PYTHONNOUSERSITE is set to disallow pip from checking the user
# site-packages. If the package is not already installed for Blender's Python interpreter, it will then try to.
# The paths used by pip can be checked with `subprocess.run([bpy.app.binary_path_python, "-m", "site"], check=True)`
# Create a copy of the environment variables and modify them for the subprocess call
environ_copy = dict(os.environ)
environ_copy["PYTHONNOUSERSITE"] = "1"
subprocess.run([sys.executable, "-m", "pip", "install", package_name], check=True, env=environ_copy)
# The installation succeeded, attempt to import the module again
import_module(module_name, global_name)
class RAAS_OT_install_dependencies(Operator):
bl_idname = 'raas.install_dependencies'
bl_label = 'Install dependencies'
bl_description = ("Downloads and installs the required python packages for this add-on. "
"Internet connection is required. Blender may have to be started with "
"elevated permissions in order to install the package")
@classmethod
def poll(self, context):
# Deactivate when dependencies have been installed
dependencies_installed = preferences().dependencies_installed
return not dependencies_installed
def execute(self, context):
try:
install_pip()
for dependency in python_dependencies:
install_and_import_module(module_name=dependency.module,
package_name=dependency.package,
global_name=dependency.name)
#enable_internal_addons()
#install_external_addons()
except (subprocess.CalledProcessError, ImportError) as err:
self.report({"ERROR"}, str(err))
return {"CANCELLED"}
preferences().dependencies_installed = True
# Register the panels, operators, etc. since dependencies are installed
#from . import sim_scene
#sim_scene.register()
return {"FINISHED"}
##################################################
class RaasPreferences(AddonPreferences):
bl_idname = ADDON_NAME
......@@ -87,6 +208,15 @@ class RaasPreferences(AddonPreferences):
default=tempfile.gettempdir()
)
dependencies_installed: BoolProperty(
default=False
)
raas_use_paramiko: BoolProperty(
name='Use Paramiko',
default=False
)
def reset_messages(self):
self.ok_message = ''
self.error_message = ''
......@@ -131,6 +261,23 @@ class RaasPreferences(AddonPreferences):
props = path_box.operator('raas.explore_file_path', text='', icon='DISK_DRIVE')
props.path = self.raas_job_storage_path
box = layout.box()
par_split = box.split(**factor(0.25), align=True)
par_split.label(text='Use Paramiko:')
user_box = par_split.row(align=True)
user_box.prop(self, 'raas_use_paramiko', text='')
if self.raas_use_paramiko == True:
box.label(text='Dependencies:')
dependencies_installed = preferences().dependencies_installed
if not dependencies_installed:
box.label(text='Dependencies not installed', icon='ERROR')
box.operator(RAAS_OT_install_dependencies.bl_idname, icon="CONSOLE")
def ctx_preferences():
"""Returns bpy.context.preferences in a 2.79-compatible way."""
......@@ -171,7 +318,20 @@ def register():
"""register."""
bpy.utils.register_class(RaasPreferences)
bpy.utils.register_class(RaasAuthValidate)
bpy.utils.register_class(RaasAuthValidate)
bpy.utils.register_class(RAAS_OT_install_dependencies)
try:
for dependency in python_dependencies:
import_module(module_name=dependency.module, global_name=dependency.name)
#check_internal_addons()
#check_external_addons()
preferences().dependencies_installed = True
except ModuleNotFoundError:
preferences().dependencies_installed = False
return
......@@ -181,5 +341,6 @@ def unregister():
bpy.utils.unregister_class(RaasAuthValidate)
bpy.utils.unregister_class(RaasPreferences)
bpy.utils.unregister_class(RAAS_OT_install_dependencies)
return
......@@ -300,7 +300,22 @@ class RAAS_UL_SubmittedJobInfoExt(bpy.types.UIList):
layout.label(text=('%d' % item.Id))
#layout.label(text=item.Name)
layout.label(text=item.Project)
layout.label(text=item.State)
layout.label(text=item.State)
def filter_items(self, context, data, propname):
"""Filter and order items in the list."""
filtered = []
ordered = []
items = getattr(data, propname)
helpers = bpy.types.UI_UL_list
filtered = helpers.filter_items_by_name(self.filter_name,
self.bitflag_filter_item,
items, "Name", reverse=False)
return filtered, ordered
......@@ -400,6 +415,65 @@ async def _scp(key_file, source, destination):
else:
raise Exception("scp command failed: %s -> %s" % (source, destination))
async def _paramiko_put(privateKey, serverHostname, username, source, destination):
""" Execute an paramiko command """
import paramiko
from io import StringIO
from base64 import b64decode
from scp import SCPClient
ssh = None
scp = None
try:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
key = paramiko.RSAKey.from_private_key(StringIO(privateKey))
ssh.connect(serverHostname, username=username, pkey=key)
#scp = SCPClient(ssh.get_transport(), progress = scp_progress, socket_timeout=60.0)
scp = SCPClient(ssh.get_transport())
scp.put(source, recursive=True, remote_path=destination)
scp.close()
except Exception as e:
if scp is not None:
scp.close()
if ssh is not None:
ssh.close()
raise Exception("paramiko command failed: %s: %s" % (e.__class__, e))
async def _paramiko_get(privateKey, serverHostname, username, source, destination):
""" Execute an paramiko command """
import paramiko
from io import StringIO
from base64 import b64decode
from scp import SCPClient
ssh = None
scp = None
try:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
key = paramiko.RSAKey.from_private_key(StringIO(privateKey))
ssh.connect(serverHostname, username=username, pkey=key)
scp = SCPClient(ssh.get_transport())
scp.get(source, local_path=destination, recursive=True)
scp.close()
except Exception as e:
if scp is not None:
scp.close()
if ssh is not None:
ssh.close()
raise Exception("paramiko command failed: %s: %s" % (e.__class__, e))
async def start_transfer_files(context, job_id: int, token: str) -> None:
"""Start Transfer files."""
......@@ -445,19 +519,35 @@ async def transfer_files(context, fileTransfer, job_local_dir: str, job_remote_d
sharedBasepath = fileTransfer[raas_server.local_to_server_map["SharedBasepath"]]
credentials = fileTransfer[raas_server.local_to_server_map["Credentials"]]
username = credentials[raas_server.local_to_server_map["UserName"]]
pkey = credentials[raas_server.local_to_server_map["PrivateKey"]]
key_file = str(get_ssh_key_file())
prefs = raas_pref.preferences()
if to_cluster == True:
source = job_local_dir
destination = '%s@%s:%s/%s' % (username, serverHostname, str(sharedBasepath), job_remote_dir)
print('copy from %s to server' % (job_local_dir))
if prefs.raas_use_paramiko == True:
if to_cluster == True:
source = job_local_dir
destination = '%s/%s' % (str(sharedBasepath), job_remote_dir)
print('copy from %s to server' % (job_local_dir))
await _paramiko_put(pkey, serverHostname, username, source, destination)
else:
destination = job_local_dir
source = '%s/%s' % (str(sharedBasepath), job_remote_dir)
print('copy from server to: %s' % (job_local_dir))
await _paramiko_get(pkey, serverHostname, username, source, destination)
else:
destination = job_local_dir
source = '%s@%s:%s/%s' % (username, serverHostname, str(sharedBasepath), job_remote_dir)
print('copy from server to: %s' % (job_local_dir))
key_file = str(get_ssh_key_file())
if to_cluster == True:
source = job_local_dir
destination = '%s@%s:%s/%s' % (username, serverHostname, str(sharedBasepath), job_remote_dir)
print('copy from %s to server' % (job_local_dir))
else:
destination = job_local_dir
source = '%s@%s:%s/%s' % (username, serverHostname, str(sharedBasepath), job_remote_dir)
print('copy from server to: %s' % (job_local_dir))
await _scp(key_file, source, destination)
await _scp(key_file, source, destination)
async def transfer_files_to_cluster(context, fileTransfer, job_local_dir: str, job_remote_dir: str, job_id: int, token: str) -> None:
"""Transfer files."""
......@@ -869,7 +959,21 @@ class RAAS_PT_MessageOfTheDay(RaasButtonsPanel, Panel):
box.operator(RAAS_OT_MessageOfTheDay.bl_idname,
text='Message of the day',
icon='WORLD')
icon='WORLD')
# class RAAS_PT_Report(RaasButtonsPanel, Panel):
# bl_label = "Report"
# bl_parent_id = "RAAS_PT_simplify"
# COMPAT_ENGINES = {'CYCLES'}
# def draw(self, context):
# layout = self.layout
# box = layout.box()
# box.prop(context.scene, "raas_total_core_hours_usage")
# box.operator(RAAS_OT_GetUserGroupResourceUsageReport.bl_idname,
# text='Core Hours Usage')
class RAAS_PT_NewJob(RaasButtonsPanel, Panel):
bl_label = "New Job"
......@@ -1003,7 +1107,7 @@ class RAAS_PT_NewJob(RaasButtonsPanel, Panel):
box.operator(RAAS_OT_prepare_files.bl_idname,
text='Submit Job',
icon='RENDER_ANIMATION')
icon='RENDER_ANIMATION')
##########################################################################
......@@ -1036,6 +1140,36 @@ class RAAS_OT_GetCurrentInfoForJob(
self.quit()
##########################################################################
async def GetUserGroupResourceUsageReport(context, token):
data = {
"GroupId": 1,
"StartTime": "2000-01-01T12:00:00.000Z",
"EndTime": "2100-01-01T12:00:00.000Z",
"SessionCode" : token
}
resp_json = await raas_server.post("JobReporting/GetUserGroupResourceUsageReport", data)
pass
class RAAS_OT_GetUserGroupResourceUsageReport(
async_loop.AsyncModalOperatorMixin,
AuthenticatedRaasOperatorMixin,
Operator):
"""returns a resource usage for user group"""
bl_idname = 'raas.get_user_group_resource_usage_report'
bl_label = 'GetUserGroupResourceUsageReport'
async def async_execute(self, context):
if not await self.authenticate(context):
return
await GetUserGroupResourceUsageReport(context, self.token)
self.quit()
##########################################################################
async def ListJobsForCurrentUser(context, token):
data = {
"SessionCode" : token
......@@ -1045,7 +1179,7 @@ async def ListJobsForCurrentUser(context, token):
#context.scene.raas_list_jobs_index = -1
context.scene.raas_list_jobs.clear()
for key in resp_json:
for key in reversed(resp_json):
item = context.scene.raas_list_jobs.add()
raas_server.fill_items(item, key)
......@@ -1318,6 +1452,7 @@ def register():
scene.raas_list_jobs_index = bpy.props.IntProperty(default=-1, options={'SKIP_SAVE'})
scene.raas_blender_job_info_new = bpy.props.PointerProperty(type=RAAS_PG_BlenderJobInfo, options={'SKIP_SAVE'})
scene.raas_submitted_job_info_ext_new = bpy.props.PointerProperty(type=RAAS_PG_SubmittedJobInfoExt, options={'SKIP_SAVE'})
scene.raas_total_core_hours_usage = bpy.props.IntProperty(default=0)
#################################
bpy.types.WindowManager.raas_status = EnumProperty(
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment