dependencies

- nbconvert dependency <7.0.0 is too new for some distrubtions
- kombu relies on deprecated importlib metadata,
celery/kombu#1601
- pass along qualifier to apphub when set
- drop nosetests
- update selenium dependency

update livetests
- adopt to jupyter nb naming
- adopt to splinter api
- update dependencies
- enable config update from env only if enabled
- enable base auth env to work for omegaml-tutorial livetest
- drop 3.10 livesting for now (python-chromedriver image does not
provide)

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.15.2-rc1
+current_version = 0.15.2-rc4
 commit = True
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)([-](?P<release>(dev|rc))+(?P<build>\d+))?

.circleci/config.yml

@@ -42,10 +42,12 @@ jobs:
               echo "Not a PR build."
             fi
       - run:
-          name: Install Miniforge
+          name: setup env
           command: |
             curl -L -O --silent --show-error "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh"
             bash Miniforge3-Linux-x86_64.sh -b
+            apt update && apt install -y docker-compose
+            pip3 install tox tox-run-before tox-conda
       - run:
           name: Run unit tests
           command: |
@@ -74,8 +76,7 @@ jobs:
               scripts/runtests.sh --rmi --specs scripts/docker/test_images_minimal.ini
             else
               # if this is not a release build, do a quick build for speed
-              PYVER=3.9
-              dotest $PYVER
+              dotest 3.9
             fi
             docker-compose down --remove-orphans
           shell: /bin/bash -l -eo pipefail
@@ -112,7 +113,7 @@ jobs:
             docker exec -i $(docker ps -qf name=mongo) mongo < scripts/mongoinit.js
             # build python specific wheel to allow dependency matching, e.g. tensorflow 2.5 requires py3.9
             # TODO consider using https://cibuildwheel.readthedocs.io
-            for PYVER in 3.7 3.8 3.9; do
+            for PYVER in 3.8 3.9; do
               echo "Building distribution for $PYVER"
               # setup pyenv and build
               conda create -q -y -n omenv$PYVER python=$PYVER

Makefile

@@ -9,15 +9,15 @@ install:
 	# in some images pip is outdated, some packages are system-level installed
 	# https://stackoverflow.com/questions/49911550/how-to-upgrade-disutils-package-pyyaml
 	pip install --ignore-installed -U pip
-	pip install pytest -U
+	pip install -U pytest tox tox-conda tox-run-before
 	[ -f .gitlinks ] && (pip install gil && gil clone && pip install -r requirements.dev) || echo "no .gitlinks, using packaeges from pypi only"
 	pip install ${PIPOPTS} --progress-bar off -e ".[${EXTRAS}]" "${PIPREQ}"
 	(which R && scripts/setup-r.sh) || echo "R is not installed"
 
 test:
 	# add -x to fail on first error
 	# PATH is required for tensorflow images
-	unset DJANGO_SETTINGS_MODULE && PATH=${HOME}/.local/bin:${PATH} pytest -v -s --tb=native ${TESTS}
+	unset DJANGO_SETTINGS_MODULE; PATH=${HOME}/.local/bin:${PATH} OMEGA_TEST_MODE=1 pytest -v -s --tb=native ${TESTS}
 
 freeze:
 	echo "Writing pip requirements to pip-requirements.lst"
@@ -91,6 +91,7 @@ old:
 	cd dist/staging/tensorflow-gpu && docker build -t omegaml/omegaml-tf:${VERSION} .
 	docker push omegaml/omegaml-tf:${VERSION}
 
+# TODO consider https://github.com/semantic-release/semantic-release
 bumppatch:
 	bumpversion patch
 

docs/source/nb/omegaml-tutorial.ipynb

@@ -180,7 +180,7 @@
    "outputs": [],
    "source": [
     "# filter data in the database -- notice the nice syntax\n",
-    "%time om.datasets.get('google', Close__gte=900, Close__lte=920)"
+    "%time om.datasets.get('google', Close__gte=120, Close__lte=140)"
    ]
   },
   {
@@ -208,7 +208,7 @@
     "# filter & aggregate data locally (let's make it large)\n",
     "def getdata():\n",
     "  data = om.datasets.get('google-large')\n",
-    "  return data[(data.Close >= 900) & (data.Close <= 920)].mean() \n",
+    "  return data[(data.Close >= 100) & (data.Close <= 140)].mean() \n",
     "\n",
     "%time getdata()"
    ]
@@ -220,7 +220,7 @@
    "outputs": [],
    "source": [
     "# filter and aggregate by database - 2-3x faster\n",
-    "%time om.datasets.getl('google-large', Close__gte=900, Close__lte=920).mean().iloc[0]"
+    "%time om.datasets.getl('google-large', Close__gte=100, Close__lte=140).mean().iloc[0]"
    ]
   },
   {
@@ -395,7 +395,7 @@
     "import omegaml as om \n",
     "\n",
     "# -- setup authentication and API URL\n",
-    "auth = AuthenticationEnv.secure().get_restapi_auth(om=om)\n",
+    "auth = AuthenticationEnv.active().get_restapi_auth(om=om)\n",
     "url = getattr(om.defaults, 'OMEGA_RESTAPI_URL', 'http://localhost:5000')\n",
     "modelname = 'iris-model'\n",
     "dataset = 'iris'\n",

omegaml/VERSION

@@ -1 +1 @@
-0.15.2-rc1
+0.15.2-rc4

omegaml/backends/rsystem/rscripts.py

@@ -7,11 +7,12 @@
 from subprocess import run
 
 from omegaml.backends.basedata import BaseDataBackend
+from omegaml.backends.package.packager import RunnablePackageMixin
 from omegaml.runtimes.rsystem import rhelper
 from omegaml.util import tryOr
 
 
-class RPackageData(BaseDataBackend):
+class RPackageData(RunnablePackageMixin, BaseDataBackend):
     """
     Backend to support R packages
 

omegaml/client/auth.py

@@ -101,6 +101,15 @@ def get_omega_from_apikey(cls, auth=None):
         om = setup()
         return om
 
+    @classmethod
+    def get_restapi_auth(cls, defaults=None, om=None,
+                         userid=None, apikey=None, qualifier=None):
+        return None
+
+    @classmethod
+    def get_runtime_auth(cls, defaults=None, om=None):
+        return None
+
     @classmethod
     def active(cls):
         # load the currently active auth env

omegaml/client/userconf.py

@@ -125,7 +125,7 @@ def _get_omega_from_apikey(userid, apikey, api_url=None, requested_userid=None,
     _base_config.update_from_env(defaults)
     _base_config.load_framework_support(defaults)
     _base_config.load_user_extensions(defaults)
-    # update config to reflect request
+    # update config to reflect request (never update from env or received config)
     defaults.OMEGA_RESTAPI_URL = api_url
     defaults.OMEGA_USERID = userid
     defaults.OMEGA_APIKEY = apikey

omegaml/defaults.py

@@ -14,8 +14,8 @@
 truefalse = lambda v: (v if isinstance(v, bool) else
                        any(str(v).lower().startswith(c) for c in ('y', 't', '1')))
 is_cli_run = os.path.basename(sys.argv[0]) == 'om'
-is_test_run = truefalse(os.environ.get('OMEGA_TEST_MODE') or
-                        any('test' in basename(v) for v in sys.argv[:3]))
+is_test_run = truefalse(os.environ.get('OMEGA_TEST_MODE'))
+is_test_run |= any('test' in basename(v) for v in sys.argv[:3]) and 'omegaml-ce' in str(Path().cwd())
 
 #: configuration file, by default will be searched in current directory, user config or site config
 OMEGA_CONFIG_FILE = os.environ.get('OMEGA_CONFIG_FILE') or 'config.yml'
@@ -188,6 +188,8 @@
     'maxsize': 1,  # cache at most one session
     'ttl': 3600,  # keep it for 1 hour
 }
+#: allow overrides from local env upon retrieving config from hub (disable in workers)
+OMEGA_ALLOW_ENV_CONFIG = truefalse(os.environ.get('OMEGA_ALLOW_ENV_CONFIG', '1'))
 
 
 # =========================================
@@ -216,6 +218,9 @@ def update_from_config(vars=globals(), config_file=OMEGA_CONFIG_FILE):
 
 def update_from_env(vars=globals()):
     # simple override from env vars
+    # -- only allow if enabled
+    if not truefalse(vars.get('OMEGA_ALLOW_ENV_CONFIG', True)):
+        return vars
     # -- top-level OMEGA_*
     for k in [k for k in os.environ.keys() if k.startswith('OMEGA')]:
         nv = os.environ.get(k, None) or vars.get(k)

omegaml/tests/features/util.py

@@ -86,7 +86,7 @@ def login_hub(self):
 
     def login_nb(self):
         br = self.browser
-        assert br.is_element_present_by_id('ipython-main-app', wait_time=10)
+        assert br.is_element_present_by_id('jupyter-main-app', wait_time=10)
         br.find_by_id('password_input').fill(self.password)
         br.find_by_id('login_submit').click()
         br.visit(jburl(br.url, '', nbstyle='tree'))
@@ -124,7 +124,7 @@ def open_notebook(self, name, retry=5):
             br.reload()
             found = br.is_text_present(name, wait_time=60)
             retry = 0 if found else retry
-        item = br.find_link_by_partial_text(name)
+        item = br.links.find_by_partial_text(name)
         item.click()
         sleep(2)
         self.last_notebook
@@ -133,9 +133,9 @@ def open_notebook(self, name, retry=5):
     def restart(self, wait=False):
         br = self.browser
         assert br.is_element_present_by_text('Cell', wait_time=60)
-        br.find_link_by_text('Kernel', )[0].click()
+        br.links.find_by_text('Kernel', )[0].click()
         sleep(1)
-        br.find_link_by_text('Restart')[0].click()
+        br.links.find_by_text('Restart')[0].click()
         if wait:
             busy = True
             while busy:
@@ -145,9 +145,9 @@ def restart(self, wait=False):
     def run_all_cells(self, wait=False):
         br = self.browser
         assert br.is_element_present_by_text('Cell', wait_time=30)
-        br.find_link_by_text('Cell', )[0].click()
+        br.links.find_by_text('Cell', )[0].click()
         sleep(1)
-        br.find_link_by_text('Run All')[0].click()
+        br.links.find_by_text('Run All')[0].click()
         if wait:
             busy = True
             while busy:
@@ -157,7 +157,7 @@ def run_all_cells(self, wait=False):
     def open_folder(self, folder=None):
         br = self.browser
         folder = quote(folder.encode('utf-8'))
-        item = br.find_link_by_href('/tree/{folder}'.format(**locals()))[0]
+        item = br.links.find_by_href('/tree/{folder}'.format(**locals()))[0]
         item.click()
         return self
 

scripts/docker/livetest/Dockerfile

@@ -1,3 +1,4 @@
+# TODO consider using selenium/standalone-chrome + install python
 FROM joyzoursky/python-chromedriver:3.9-selenium
 # chrome debug port https://developers.google.com/web/updates/2017/04/headless-chrome#frontend
 EXPOSE 9222/tcp

scripts/docker/test_images_minimal.ini

@@ -15,8 +15,8 @@ image=jupyter/scipy-notebook:python-3.9.7
 tests=omegaml/tests/core
 extras=all
 
-[jscnb388]
-image=jupyter/scipy-notebook:python-3.8.8
+[jscnb3813]
+image=jupyter/scipy-notebook:python-3.8.13
 tests=omegaml/tests/core
 extras=all
 
@@ -31,8 +31,3 @@ image=tensorflow/tensorflow:2.7.1-jupyter
 tests=omegaml/tests/tensorflow
 dockeropts="-e HOME=/tf"
 
-[tf153]
-image=tensorflow/tensorflow:1.15.3-py3-jupyter
-tests=omegaml/tests/tensorflow
-pipreq="keras<2"
-dockeropts="-e HOME=/tf"

setup.py

@@ -17,7 +17,7 @@
 streaming_deps = ['minibatch[all]>=0.5.0']
 jupyter_deps = ['jupyterlab', 'jupyterhub==2.2.1']
 mlflow_deps = ['mlflow~=1.21']
-dev_deps = ['nose', 'twine', 'flake8', 'mock', 'behave', 'splinter', 'ipdb', 'bumpversion']
+dev_deps = ['pytest', 'twine', 'flake8', 'mock', 'behave', 'splinter[selenium3]', 'ipdb', 'bumpversion']
 
 # -- tensorflow specifics
 #    see https://www.tensorflow.org/install/source
@@ -80,11 +80,12 @@
     ],
     install_requires=[
         'celery>4.4,<5.0',
+        'importlib-metadata<5.0', # due to https://github.com/celery/kombu/pull/1601
         'joblib>=0.9.4',
         'jupyter-client>=4.1.1',
         'mongoengine~=0.24.1',
         'pymongo~=4.0.2',  # mongoengine 0.24.1 compatibility
-        'pandas>1.1,<1.4',  # 1.1 fails on indexes, 1.4 fails some libraries, e.g. yfinance
+        'pandas>1.1',
         'numpy>=1.16.4',
         'scipy>=0.17.0',
         'scikit-learn>=0.21',
@@ -94,7 +95,7 @@
         'markupsafe==2.0.1', # due to flask/markupsafe, https://github.com/pallets/markupsafe/issues/284
         'croniter>=0.3.30',
         'nbformat>=4.0.1',
-        'nbconvert>=7.0.0',
+        'nbconvert>=6.4.0',
         'dill>=0.3.2',
         'callable-pip>=1.0.0',
         'appdirs>=1.4.3',

tox.ini

@@ -0,0 +1,18 @@
+[tox]
+envlist = py38,py39,py310
+
+[testenv]
+deps =
+    -e .[all,dev,mlflow,tensorflow]
+run_before =
+    docker-compose -f docker-compose-dev.yml up -d
+    sleep 10
+    scripts/initlocal.sh
+commands =
+    pytest
+allowlist_externals =
+    docker-compose
+    bash
+setenv =
+    OMEGA_TEST_MODE = 1
+