Skip to content
This repository has been archived by the owner on Aug 25, 2024. It is now read-only.

Commit

Permalink
df: Real DataFlows and HTTP MultiComm
Browse files Browse the repository at this point in the history
Fixes: #213 
Fixes: #211  
Fixes: #209 
Fixes: #204 
Fixes: #199 
Fixes: #193 
Fixes: #186 
Fixes: #90 

Signed-off-by: John Andersen <[email protected]>
  • Loading branch information
pdxjohnny authored Oct 26, 2019
1 parent 013877a commit 9587a42
Show file tree
Hide file tree
Showing 138 changed files with 8,241 additions and 1,643 deletions.
7 changes: 7 additions & 0 deletions .ci/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ function run_plugin() {
./scripts/docs.sh
# Try running create for real
cd $(mktemp -d)
# TODO Bash array
# Create model
dffml service dev create model travis-test-model
cd travis-test-model
Expand All @@ -44,6 +45,12 @@ function run_plugin() {
python setup.py install
python setup.py test
cd ..
# Create config
dffml service dev create config travis-test-config
cd travis-test-config
python setup.py install
python setup.py test
cd ..
fi
}

Expand Down
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -20,4 +20,5 @@ htmlcov/
html/
pages/
examples/shouldi/response.json
docs/plugins/service/http
docs/plugins/service/http
*pip-wheel-metadata/
3 changes: 2 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,9 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [Unreleased]
## [0.3.0] - 2019-10-26
### Added
- Real DataFlows, see operations tutorial and usage examples
- Async helper concurrently nocancel optional keyword argument which, if set is
a set of tasks not to cancel when the concurrently execution loop completes.
- FileSourceTest has a `test_label` method which checks that a FileSource knows
Expand Down
13 changes: 13 additions & 0 deletions config/yaml/.coveragerc
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
[run]
source =
dffml_config_yaml
tests
branch = True

[report]
exclude_lines =
no cov
no qa
noqa
pragma: no cover
if __name__ == .__main__.:
20 changes: 20 additions & 0 deletions config/yaml/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
*.log
*.pyc
.cache/
.coverage
.idea/
.vscode/
*.egg-info/
build/
dist/
docs/build/
venv/
wheelhouse/
*.egss
.mypy_cache/
*.swp
.venv/
.eggs/
*.modeldir
*.db
htmlcov/
21 changes: 21 additions & 0 deletions config/yaml/LICENSE
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
Copyright (c) 2019 Intel Corporation

MIT License

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
3 changes: 3 additions & 0 deletions config/yaml/MANIFEST.in
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
include README.md
include LICENSE
include setup_common.py
10 changes: 10 additions & 0 deletions config/yaml/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# DFFML Config For YAML Format

## About

Allows for reading and writing of ``yaml`` files.

## License

model_name Models are distributed under the terms of the
[MIT License](LICENSE).
Empty file.
31 changes: 31 additions & 0 deletions config/yaml/dffml_config_yaml/config.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
"""
Description of what this config does
"""
import yaml
from typing import Dict, Any

from dffml.util.entrypoint import entry_point
from dffml.util.cli.arg import Arg
from dffml.base import BaseConfig
from dffml.config.config import BaseConfigLoaderContext, BaseConfigLoader


class YamlConfigLoaderContext(BaseConfigLoaderContext):
async def loadb(self, resource: bytes) -> Dict:
return yaml.safe_load(resource.decode())

async def dumpb(self, resource: Dict) -> bytes:
return yaml.dump(resource, default_flow_style=False).encode()


@entry_point("yaml")
class YamlConfigLoader(BaseConfigLoader):
CONTEXT = YamlConfigLoaderContext

@classmethod
def args(cls, args, *above) -> Dict[str, Arg]:
return args

@classmethod
def config(cls, config, *above) -> BaseConfig:
return BaseConfig()
1 change: 1 addition & 0 deletions config/yaml/dffml_config_yaml/version.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
VERSION = "0.0.3"
20 changes: 20 additions & 0 deletions config/yaml/pyproject.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
[tool.black]
line-length = 79
target-version = ['py37']

exclude = '''
(
/(
\.eggs # exclude a few common directories in the
| \.git # root of the project
| \.hg
| \.mypy_cache
| \.tox
| \.venv
| _build
| buck-out
| build
| dist
)
)
'''
17 changes: 17 additions & 0 deletions config/yaml/setup.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
import os
import importlib.util
from setuptools import setup

# Boilerplate to load commonalities
spec = importlib.util.spec_from_file_location(
"setup_common", os.path.join(os.path.dirname(__file__), "setup_common.py")
)
common = importlib.util.module_from_spec(spec)
spec.loader.exec_module(common)

common.KWARGS["install_requires"] += ["PyYAML>=5.1.2"]
common.KWARGS["entry_points"] = {
"dffml.config": [f"yaml = {common.IMPORT_NAME}.config:YamlConfigLoader"]
}

setup(**common.KWARGS)
79 changes: 79 additions & 0 deletions config/yaml/setup_common.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
import os
import sys
import ast
from io import open
from pathlib import Path
from setuptools import find_packages

ORG = "intel"
NAME = "dffml-config-yaml"
DESCRIPTION = "DFFML config dffml-config-yaml"
AUTHOR_NAME = "John Andersen"
AUTHOR_EMAIL = "[email protected]"
# Install dffml if it is not installed in development mode
INSTALL_REQUIRES = [] + (
["dffml>=0.3.0"]
if not any(
list(
map(
os.path.isfile,
list(
map(
lambda syspath: os.path.join(
syspath, "dffml.egg-link"
),
sys.path,
)
),
)
)
)
else []
)

IMPORT_NAME = (
NAME
if "replace_package_name".upper() != NAME
else "replace_import_package_name".upper()
).replace("-", "_")

SELF_PATH = Path(sys.argv[0]).parent.resolve()
if not (SELF_PATH / Path(IMPORT_NAME, "version.py")).is_file():
SELF_PATH = os.path.dirname(os.path.realpath(__file__))

VERSION = ast.literal_eval(
Path(SELF_PATH, IMPORT_NAME, "version.py")
.read_text()
.split("=")[-1]
.strip()
)

README = Path(SELF_PATH, "README.md").read_text()

KWARGS = dict(
name=NAME,
version=VERSION,
description=DESCRIPTION,
long_description=README,
long_description_content_type="text/markdown",
author=AUTHOR_NAME,
author_email=AUTHOR_EMAIL,
maintainer=AUTHOR_NAME,
maintainer_email=AUTHOR_EMAIL,
url="https://github.com/intel/dffml/blob/master/config/yaml/README.md",
license="MIT",
keywords=["dffml"],
classifiers=[
"Development Status :: 3 - Alpha",
"Intended Audience :: Developers",
"License :: OSI Approved :: MIT License",
"Natural Language :: English",
"Operating System :: OS Independent",
"Programming Language :: Python :: 3 :: Only",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: Implementation :: CPython",
"Programming Language :: Python :: Implementation :: PyPy",
],
install_requires=INSTALL_REQUIRES,
packages=find_packages(),
)
Empty file added config/yaml/tests/__init__.py
Empty file.
12 changes: 12 additions & 0 deletions config/yaml/tests/test_config.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
from dffml.util.asynctestcase import AsyncTestCase

from dffml_config_yaml.config import YamlConfigLoader


class TestConfig(AsyncTestCase):
async def test_dumpb_loadb(self):
async with YamlConfigLoader.withconfig({}) as configloader:
async with configloader() as ctx:
original = {"Test": ["dict"]}
reloaded = await ctx.loadb(await ctx.dumpb(original))
self.assertEqual(original, reloaded)
6 changes: 6 additions & 0 deletions dffml/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,12 @@ class BaseConfig(object):
as their config.
"""

def __repr__(self):
return "BaseConfig()"

def __str__(self):
return repr(self)


class ConfigurableParsingNamespace(object):
def __init__(self):
Expand Down
1 change: 1 addition & 0 deletions dffml/cli/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
from .cli import *
Loading

0 comments on commit 9587a42

Please sign in to comment.