Skip to content

Commit

Permalink
hotfix(compute_for): protect PythonMonkey from Python
Browse files Browse the repository at this point in the history
This is temporary... Better patch coming later...

Basically, PythonMonkey can't support many types... such as
multi dimensional np arrays

SO I hide it from JS by nesting it a level deeper which dcp-client
never touches... Until job exec...
  • Loading branch information
wiwichips committed Sep 3, 2024
1 parent 7e9f09b commit b0aaaf1
Show file tree
Hide file tree
Showing 3 changed files with 54 additions and 14 deletions.
43 changes: 41 additions & 2 deletions dcp/api/compute_for.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,54 @@
import pythonmonkey as pm
import dill
from .. import dry
from .. import js
from types import FunctionType

def compute_for_maker(Job):
def compute_for(*args, **kwargs):
args = list(args)
for i in range(len(args)):
arg = args[i]

for i, arg in enumerate(args):
if isinstance(arg, FunctionType):
args[i] = dill.source.getsource(arg)


# Hide values from PythonMonkey which aren't supported
# TODO: This is bad for a number of reasons:
####################################################
# - Not comprehensively guarding every call to JS
# - Only applies to compute_for and not compute_do
# - Bizarre that it only applies to some types.....
# - Only transforming values that are one level deep

job_input_idx = None
job_args_idx = None

# compute.for(start, end, step, work, args)
if len(args) == 5:
job_args_idx = 4

# compute.for(iterableObject, work, args)
elif len(args) <= 3:
job_input_idx = 0

if len(args) == 3:
job_args_idx = 2

# clean up job input for PythonMonkey
if job_input_idx != None:
for i, val in enumerate(args[job_input_idx]):
if js.utils.throws_in_pm(val):
args[job_input_idx][i] = { '__pythonmonkey_guard': val }

# clean up job args for PythonMonkey
if job_args_idx != None:
for i, val in enumerate(args[job_args_idx]):
if js.utils.throws_in_pm(val):
args[job_args_idx][i] = { '__pythonmonkey_guard': val }

####################################################

compute_for_js = pm.eval("globalThis.dcp.compute.for")
job_js = dry.aio.blockify(compute_for_js)(*args, **kwargs)
return Job(job_js)
Expand Down
13 changes: 13 additions & 0 deletions dcp/api/job.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,10 +61,13 @@ def _get_raw_work_function(self):
return unindented_work_function

def _before_exec(self, *args, **kwargs):

# Any other worktime, do not apply serializers, env, jobfs
if not self.js_ref.worktime == "pyodide":
return

# pyodide worktime / bifrost 2 flavoured setup below

work_function = self._get_raw_work_function()

meta_arguments = [
Expand All @@ -79,6 +82,11 @@ def _before_exec(self, *args, **kwargs):
super_range_object = pm.eval("globalThis.dcp['range-object'].SuperRangeObject")
if isinstance(self.js_ref.jobInputData, list) or utils.instanceof(self.js_ref.jobInputData, pm.globalThis.Array):
for input_slice in self.js_ref.jobInputData:
# TODO - find better solution
# un-hide values from PythonMonkey which aren't supported
if '__pythonmonkey_guard' in input_slice:
input_slice = input_slice['__pythonmonkey_guard']

serialized_slice = serialize(input_slice, self.serializers)
serialized_input_data.append(serialized_slice)
elif isinstance(self.js_ref.jobInputData, Iterator) and not utils.instanceof(self.js_ref.jobInputData, super_range_object):
Expand All @@ -87,6 +95,11 @@ def _before_exec(self, *args, **kwargs):
serialized_input_data = self.js_ref.jobInputData

for argument in self.js_ref.jobArguments:
# TODO - find better solution
# un-hide values from PythonMonkey which aren't supported
if '__pythonmonkey_guard' in argument:
argument = argument['__pythonmonkey_guard']

serialized_argument = serialize(argument, self.serializers)
serialized_arguments.append(serialized_argument)

Expand Down
12 changes: 0 additions & 12 deletions dcp/dry/package.json

This file was deleted.

0 comments on commit b0aaaf1

Please sign in to comment.