##############################################################################
# Institute for the Design of Advanced Energy Systems Process Systems
# Engineering Framework (IDAES PSE Framework) Copyright (c) 2018, by the
# software owners: The Regents of the University of California, through
# Lawrence Berkeley National Laboratory, National Technology & Engineering
# Solutions of Sandia, LLC, Carnegie Mellon University, West Virginia
# University Research Corporation, et al. All rights reserved.
#
# Please see the files COPYRIGHT.txt and LICENSE.txt for full copyright and
# license information, respectively. Both files are also available online
# at the URL "https://github.com/IDAES/idaes".
##############################################################################
"""
Resource representaitons.
"""
# stdlib
from collections import namedtuple
from copy import copy
from datetime import datetime
import getpass
import json
import os
import re
from tempfile import NamedTemporaryFile
import uuid
# third-party
import pendulum
import six
from traitlets import HasTraits, TraitType, default, TraitError
from traitlets import List, Dict, Instance, Enum, Unicode, Integer
# local
from . import propdata, tabular
from .util import get_logger, datetime_timestamp
_log = get_logger('resource')
[docs]class ResourceTypes(object):
"""Standard resource type names.
Use these as opaque constants to indicate standard resource
types. For example, when creating a Resource::
rsrc = Resource(type=ResourceTypes.property_data, ...)
"""
#: Experiment
experiment = 'experiment'
xp = experiment
#: Tabular data
tabular_data = 'tabular_data'
#: Property data resource, e.g. the contents are
#: created via classes in the :mod:`idaes.dmf.propdata` module.
property_data = 'propertydb'
#: Flowsheet resource.
fs = 'flowsheet'
#: Jupyter Notebook
nb = 'notebook'
jupyter_nb = jupyter = nb # aliases
#: Python code
python = 'python'
#: Surrogate model
surrmod = 'surrogate_model'
#: Data (e.g. result data)
data = 'data'
[docs]class DateTime(TraitType):
"""A trait type for a datetime.
Input can be a string, float, or tuple. Specifically:
- string, ISO8601: YYYY[-MM-DD[Thh:mm:ss[.uuuuuu]]]
- float: seconds since Unix epoch (1/1/1970)
- tuple: format accepted by datetime.datetime()
No matter the input, validation will transform it into
a floating point number, since this is the easiest form
to store and search.
"""
default_value = 0
info_text = 'a datetime'
[docs] def validate(self, obj, value):
dt = None
usec = 0
if isinstance(value, datetime):
dt = value
elif isinstance(value, tuple):
try:
dt = datetime(*value)
except TypeError:
self.error(obj, value)
elif isinstance(value, six.string_types):
if '.' in value:
value, usec_str = value.rsplit('.', 1)
try:
usec = int(usec_str) / 1e6
except ValueError:
self.error(obj, value)
try:
dt = datetime.strptime(value, '%Y')
except ValueError:
try:
dt = datetime.strptime(value, '%Y-%m-%d')
except ValueError:
try:
dt = datetime.strptime(value, '%Y-%m-%dT%H:%M:%S')
except ValueError:
self.error(obj, value)
elif isinstance(value, float) or isinstance(value, int):
try:
dt = datetime.fromtimestamp(value)
except ValueError:
self.error(obj, value)
if dt is None:
self.error(obj, value)
return datetime_timestamp(dt) + usec # just a float
[docs]class SemanticVersion(TraitType):
"""Semantic version.
Three numeric identifiers, separated by a dot.
Trailing non-numeric characters allowed.
Inputs, string or tuple, may have less than three numeric
identifiers, but internally the value will be padded with
zeros to always be of length four.
A leading dash or underscore in the trailing non-numeric characters
is removed.
Some examples:
- 1 => valid => (1, 0, 0, '')
- rc3 => invalid: no number
- 1.1 => valid => (1, 1, 0, '')
- 1a => valid => (1, 0, 0, 'a')
- 1.a.1 => invalid: non-numeric can only go at end
- 1.12.1 => valid => (1, 12, 1, '')
- 1.12.13-1 => valid => (1, 12, 13, '1')
- 1.12.13.x => invalid: too many parts
"""
default_value = (0, 0, 0, '')
info_text = 'semantic version major, minor, patch, & modifier'
[docs] def validate(self, obj, value):
ver = ()
if isinstance(value, list):
ver = tuple(value)
elif isinstance(value, str):
ver = value.split('.', 2)
elif isinstance(value, tuple):
ver = value
elif isinstance(value, int):
ver = (value, 0, 0)
else:
self.error(obj, value)
if len(ver) < 1:
self.error(obj, value)
verlist = []
# leading version numbers
for i in range(len(ver) - 1):
try:
verlist.append(int(ver[i]))
except ValueError:
self.error(obj, value)
# last version number
s = ver[-1]
extra = ''
if isinstance(s, int):
verlist.append(s if len(verlist) < 3 else str(s))
elif isinstance(s, six.string_types):
if s:
m = re.match('([0-9]+)?(.*)', s)
if m.group(1) is not None:
verlist.append(int(m.group(1)))
extra = m.group(2)
else: # last version must be int or str
self.error(obj, value)
# must have at least one numbered version
if len(verlist) == 0:
self.error(obj, value)
# pad with zeros, and add non-numeric ID
while len(verlist) < 3:
verlist.append(0)
if extra and extra[0] == '.':
# cannot start extra version with '.'
self.error(obj, value)
if extra and extra[0] in ('-', '_'):
extra = extra[1:]
verlist.append(extra)
return tuple(verlist)
[docs] @classmethod
def pretty(cls, values):
s = '{}.{}.{}'.format(*values[:3])
if values[3]:
s += '-{}'.format(values[3])
return s
[docs]class Identifier(TraitType):
"""Unique identifier.
Will set it itself automatically to a 32-byte
unique hex string. Can only be set to strings
"""
default_value = None
info_text = 'Unique identifier'
# regular expression for hex string len=32
expr = re.compile('[0-9a-f]{32}')
[docs] def validate(self, obj, value):
m = None
if value is None:
value = uuid.uuid4().hex
try:
m = self.expr.match(value)
except TypeError:
self.error(obj, value)
if m is None:
self.error(obj, value)
return value
[docs]class TraitContainer(HasTraits):
"""Base class for Resource, that knows how to
serialize and parse its traits.
"""
[docs] def as_dict(self):
d = {}
for name in self.trait_names():
obj = getattr(self, name)
d[name] = self._value(obj)
return d
@classmethod
def _value(cls, obj):
if isinstance(obj, TraitContainer):
return obj.as_dict()
elif isinstance(obj, list):
return [cls._value(o) for o in obj]
elif isinstance(obj, dict):
return {k: cls._value(v) for k, v in six.iteritems(obj)}
else:
return obj
[docs] @classmethod
def from_dict(cls, d):
obj = cls()
traits = cls.class_traits()
for key, value in six.iteritems(d):
try:
trait = traits[key]
except KeyError:
# XXX: be more tolerant here? just print warning?
raise KeyError('Cannot deserialize: "{}" has no attribute '
'"{}"'.format(cls.__name__, key))
if isinstance(trait, List):
if not isinstance(value, list):
raise ValueError(
'Cannot deserialize list of values "{}" from "{}"'
.format(key, value))
c = cls._get_item_class(trait)
if isinstance(c, TraitContainer):
# use contained item's class
setattr(obj, key, [c.from_dict(v) for v in value])
else:
# let the trait's class convert values
setattr(obj, key, value)
elif isinstance(trait, Dict):
if not isinstance(value, dict):
raise ValueError(
'Cannot deserialize dict of values "{}" from "{}"'
.format(key, value))
c = cls._get_item_class(trait)
if c is None: # raw dict
setattr(obj, key, value)
else:
setattr(obj, key, {k: cls._value(v)
for k, v in six.iteritems(value)})
elif isinstance(trait, Instance):
c = trait.klass()
if isinstance(c, TraitContainer):
setattr(obj, key, c.from_dict(value))
else:
setattr(obj, key, value)
else:
setattr(obj, key, value)
return obj
@staticmethod
def _get_item_class(trait):
c = trait._trait
if isinstance(c, Instance):
c = c.klass()
elif isinstance(c, dict) or isinstance(c, list):
c = None
return c
[docs]class FilePath(TraitContainer):
"""Path to a file, plus optional description and metadata.
So that the DMF does not break when data files are moved
or copied, the default is to copy the datafile into the
DMF workspace. This behavior can be controlled by the
`copy` and `tempfile` keywords to the constructor.
For example, if you have a big file you do NOT want to
copy when you create the resource::
FilePath(path='/my/big.file', desc='100GB file', copy=False)
On the other hand, if you have a file that you want
the DMF to manage entirely::
FilePath(path='/some/file.txt', desc='a file', tempfile=True)
"""
CSV_MIMETYPE = 'text/csv'
#: Path to file
path = Unicode()
#: Unique subdir
subdir = Unicode()
#: Description of the file's contents
desc = Unicode('')
#: MIME type
mimetype = Unicode('text')
#: Metadata to associate with the file
metadata = Dict(default_value={})
def __init__(self, tempfile=False, copy=True, **kwargs):
"""Constructor.
Args:
tempfile (bool): if True, when copying the file, remove original.
"""
super(FilePath, self).__init__(**kwargs)
if tempfile and not copy:
raise ValueError('Options tempfile=True and copy=False conflict')
self._tmp = tempfile
self._copy = copy
self._root = ''
@property
def is_tmp(self):
return self._tmp
@property
def do_copy(self):
return self._copy
@property
def fullpath(self):
return self.path if self._abspath() else\
os.path.join(self._root, self.subdir, self.path)
def _abspath(self):
r = os.path.isabs(self.path)
return r
@property
def root(self):
return None if self._abspath() else self._root
@root.setter
def root(self, value):
self._root = value
[docs] def open(self, mode='r'):
return open(self.fullpath, mode=mode)
[docs] def read(self, *args):
return self.open().read(*args)
[docs]class Version(TraitContainer):
"""Version of something (code, usually).
"""
#: Name given to version
name = Unicode('', help='Name given to version, if any')
#: When this version was created. Default "empty", which is
#: encoded as the start of Unix epoch (1970/01/01).
created = DateTime(help='When this version was created')
#: Revision, e.g. 1.0.0rc3
revision = SemanticVersion(help='Revision, e.g. 1.0.0rc3')
def __eq__(self, other):
if isinstance(other, Version):
return self.revision == other.revision
return super(Version, self).__eq__(other)
@default('created')
def _default_created(self):
return pendulum.now()
def __str__(self):
s = '{v} ({d})'.format(v=SemanticVersion.pretty(self.revision),
d=DateTime.isoformat(self.created))
if self.name:
s = self.name + ' ' + s
return s
__repr__ = __str__
[docs]class Source(TraitContainer):
"""A work from which the resource is derived.
"""
#: Digital object identifier
doi = Unicode('', help='DOI if any')
#: ISBN
isbn = Unicode('', help='ISBN if any')
#: The work, either print or electronic, from which the
#: resource was derived
source = Unicode(help='The work, either print or electronic, from which '
'the resource is delivered (if applicable)')
#: The primary language of the intellectual content of the resource
language = Unicode('English', help='The language of the intellectual'
'content of the resource')
#: Date associated with resource
date = DateTime()
[docs]class Code(TraitContainer):
"""Some source code, such as a Python module or C file.
This can also refer to packages or entire Git repositories.
"""
#: Type of code resource, must be one of: 'method', 'function',
#: 'module', 'class', 'file', 'package', 'repository', or 'notebook'.
type = Enum(('method', 'function', 'module', 'class', 'file', 'package',
'repository', 'notebook'),
help='Type of code object that this represents')
#: Description of the code
desc = Unicode(help='Description of the code')
#: Name of the code object, e.g. Python module name
name = Unicode(help='Name of the code object')
#: Programming language, e.g. "Python" (the default).
language = Unicode(default_value='Python', help='Programming language')
#: Version of the release, default is '0.0.0'
release = Instance(Version, kw=dict(revision='0.0.0'))
#: Git or other unique hash
idhash = Unicode('', help='Git or other unique hash for the file')
#: Flie path or URL location for the code
location = Unicode(help='URL for repo or path where this code object '
'is stored')
#: Provide attribute access to an RDF subject, predicate, object triple
Triple = namedtuple('Triple', 'subject predicate object')
#: Constants for RelationType predicates
R_DERIVED = 'derived' # derivedFrom
R_CONTAINS = 'contains'
R_USES = 'uses'
R_VERSION = 'version'
[docs]class RelationType(TraitType):
"""Traitlets type for RDF-style triples relating resources to each other.
"""
Predicates = {R_DERIVED, R_CONTAINS, R_USES, R_VERSION}
info_text = 'triple of (subject-id, predicate, object-id), all strings, '\
'with a predicate in {{{}}}'\
.format(', '.join(list(Predicates)))
[docs] def validate(self, obj, value):
# split strings up, allowing "subject-id predicate object-id"
# as a valid initialization value
if isinstance(value, str):
value = value.split()
# accept triples or split-strings which are triples too
if isinstance(value, tuple) or isinstance(value, list):
n = len(value)
if n != 3:
_log.error('Bad relation: length {:d} != 3'.format(n))
return self.error(obj, value)
# now all values should be strings
if not all([isinstance(v, six.string_types) for v in value]):
_log.error('Bad relation: non-string value')
return self.error(obj, value)
if value[0] == value[2]:
_log.error('Bad relation: subject = object')
return self.error(obj, value)
if value[1] not in self.Predicates:
_log.error('Bad relation: unknown predicate "{}"'
.format(value[1]))
return self.error(obj, value)
else:
return self.error(obj, value)
return Triple(*value)
class _VList(list): # pragma: nocover
"""Act like a list, but validate all added elements against
a given TraitType.
This is a ValidatingList helper class, and should not be used alone.
"""
def __init__(self, obj, trait_type, values):
self._tt, self._obj = trait_type, obj
super(_VList, self).__init__()
if values:
self.extend(values) # will be validated
def _validate(self, value):
try:
return self._tt.validate(self._obj, value)
except TraitError as err:
_log.error('Setting value in List: {}'.format(err))
raise
def append(self, value):
value = self._validate(value)
super(_VList, self).append(value)
def insert(self, index, value):
value = self._validate(value)
super(_VList, self).insert(index, value)
def extend(self, values):
new_values = []
for v in values:
new_values.append(self._validate(v))
super(_VList, self).extend(new_values)
[docs]class ValidatingList(List):
"""Validate values in a list as belonging to a given TraitType.
This can be used in place of the Traitlets.List class.
"""
def __init__(self, *args, **kwargs):
self._trait_type = kwargs.get('trait', args[0])
super(ValidatingList, self).__init__(*args, **kwargs)
[docs] def validate_elements(self, obj, value=None):
"""This is called when the initial value is set.
"""
return _VList(obj, self._trait_type, value or [])
[docs]class Resource(TraitContainer):
"""A dynamically typed resource.
Resources have metadata and (same for all resoures)
a type-specific "data" section (unique to that type of resource).
"""
ID_FIELD = 'id_' # special field name
TYPE_FIELD = 'type' # special field name
#: Integer identifier for this Resource. You should not
#: set this yourself. The value will be automatically
#: overwritten with the database's value when the resource is added
#: to the DMF (with the `.add()` method).
id_ = Integer(0, help='Internal Identifier')
#: Universal identifier for this resource
uuid = Identifier(help='External Identifier')
#: Type of this Resource. See :class:`ResourceTypes` for standard
#: values for this attribute.
type = Unicode(default_value='resource', help='Resource type')
#: Human-readable name for the resource (optional)
name = Unicode(default_value='', help='Short resource name')
#: Description of the resource
desc = Unicode(default_value='', help='Resource description')
#: Date and time when the resource was created. This defaults
#: to the time when the object was created. Value is a :class:`DateTime`.
created = DateTime(help='Date and time created')
#: Date and time the resource was last modified. This defaults
#: to the time when the object was created. Value is a :class:`DateTime`.
modified = DateTime(help='Date and time last modified')
#: Version of the resource. Value is a :class:`SemanticVersion`.
version = Instance(Version, kw=dict(revision='0.0.0'))
#: Creator of the resource. Value is a :class:`Contact`.
creator = Instance(Contact, help='Creator of resource')
#: List of other people involved. Each value is a :class:`Contact`.
collaborators = List(Instance(Contact), help='Other people involved')
#: Sources from which resource is derived, i.e. its provenance.
#: Each value is a :class:`Source`.
sources = List(Instance(Source), help='Provenance of resource')
#: List of code objects (including repositories and packages) associated
#: with the resource. Each value is a :class:`Code`.
codes = List(Instance(Code), help='Associated code objects')
#: List of data files associated with the resource.
#: Each value is a :class:`FilePath`.
datafiles = List(Instance(FilePath), help='Associated data objects')
#: Datafiles subdirectory (single directory name)
datafiles_dir = Unicode()
#: List of aliases for the resource
aliases = List(Unicode(), help='User-assigned name(s)')
#: List of tags for the resource
tags = List(Unicode(), help='User-assigned keywords')
relations = ValidatingList(RelationType(),
help='Relations to other resources')
data = Dict(help='Type-specific data')
def __init__(self, *args, **kwargs):
self._now = pendulum.utcnow()
self._preprocess_kw(kwargs)
# Pass args up to superclass
super(Resource, self).__init__(*args, **kwargs)
def _preprocess_kw(self, kwargs):
# If a non-Version object is given for Version, attempt to set
# that as the revision and pass the full Version object up
ver_fld = 'version'
if ver_fld in kwargs and not isinstance(kwargs[ver_fld], Version):
kwargs[ver_fld] = Version(revision=kwargs[ver_fld])
@default('created')
def _default_created(self):
return self._now
@default('modified')
def _default_modified(self):
return self._now
@default('creator')
def _default_creator(self):
name = getpass.getuser()
return Contact(name=name)
@default('datafiles_dir')
def _default_datafiles_dir(self):
id_ = str(uuid.uuid4())
return id_
[docs] def help(self, name):
"""Return descriptive 'help' for the given attribute.
Args:
name (str): Name of attribute
Returns:
str: Help string, or error starting with "Error: "
"""
try:
result = self.trait_metadata(name, 'help')
except AttributeError:
result = 'Error: No help available for "{}"'.format(name)
except TraitError:
result = 'Error: No attribute named "{}"'.format(name)
return result
[docs] @staticmethod
def create_relation(subj, pred, obj):
"""Create a relationship between two Resource instances.
Args:
subj (Resource): Subject
pred (str): Predicate
obj (Resource): Object
Returns:
None
Raises:
TypeError: if subject & object are not Resource instances.
"""
if not isinstance(subj, Resource) or not isinstance(obj, Resource):
raise TypeError('subject and object must both be of type Resource,'
' got subj={} obj={}'
.format(type(subj), type(obj)))
subj.relations.append((subj.uuid, pred, obj.uuid))
obj.relations.append((subj.uuid, pred, obj.uuid))
[docs] def copy(self, **kwargs):
"""Get a copy of this Resource.
As a convenience, optionally set some attributes in the copy.
Args:
kwargs: Attributes to set in new instance after copying.
Returns:
Resource: A deep copy.
The copy will have an empty (zero)
`identifier` and a new unique value for `uuid`.
The relations are not copied.
"""
newr = self._copy()
for k, v in six.iteritems(kwargs):
setattr(newr, k, v)
return newr
def _copy(self, *args, **kwargs):
# extremely manual deep copy
r = self.__class__(*args)
r.id_ = 0 # don't copy the ID
r.uuid = None # this forces creation of new unique ID
r.type = str(self.type)
r.name = str(self.name)
r.desc = str(self.desc)
r.created = self.created
r.modified = self.modified
r.version = Version(name=self.version.name,
revision=self.version.revision,
creted=self.version.created)
r.creator = Contact(name=self.creator.name, email=self.creator.email)
r.collaborators = [Contact(name=c.name, email=c.email)
for c in self.collaborators]
r.sources = [Source(doi=s.doi, isbn=s.isbn, source=s.source,
language=s.language, date=s.date)
for s in self.sources]
r.codes = [Code(type=c.type, desc=c.desc, name=c.name,
language=c.language, release=c.release,
idhash=c.idhash,
location=c.location) for c in self.codes]
r.datafiles = [FilePath(path=f.path, subdir=f.subdir, desc=f.desc,
mimetype=f.mimetype, metadata=copy(f.metadata))
for f in self.datafiles]
r.datafiles_dir = str(self.datafiles_dir)
r.tags = [str(t) for t in self.tags]
r.relations = [] # do not copy these
r.data = copy(self.data)
# Overwrite with any provided keyword args
self._preprocess_kw(kwargs)
for k, v in six.iteritems(kwargs):
setattr(r, k, v)
return r
@property
def table(self):
"""For tabular data resources, this property builds and returns
a Table object.
Returns:
`tabular.Table`: A representation of metadata and data
in this resource.
Raises:
TypeError: if this resource is not of the correct type.
"""
# type: () -> tabular.Table
self._must_be_type(ResourceTypes.tabular_data)
return self._build_table(tabular.Table)
@property
def property_table(self):
"""For property data resources, this property builds and returns
a PropertyTable object.
Returns:
`propdata.PropertyTable`: A representation of metadata and data
in this resource.
Raises:
TypeError: if this resource is not of the correct type.
"""
# type: () -> propdata.PropertyTable
self._must_be_type(ResourceTypes.property_data)
return self._build_table(propdata.PropertyTable)
def _build_table(self, table_class):
# (a) embedded in .data attribute
if 'data' in self.data:
data = self.data['data']
meta = self.data.get('meta', None)
table = table_class(data=data, metadata=meta)
# (b) saved to JSON in a file
elif len(self.datafiles) > 0:
for datafile in self.datafiles:
f = datafile.fullpath
_log.debug('Load table from file "{}"'.format(f))
try:
table = table_class.load(f)
break
except (ValueError, json.JSONDecodeError) as err:
if f.endswith('.json'):
_log.warn('Unable to parse file "{}" as property '
'data: {}'.format(f, err))
else:
df_str = ', '.join([df.path for df in self.datafiles])
raise ValueError('Table data not found in datafiles: '
'{}'.format(df_str))
# (c) Error
else:
raise ValueError('Table data not found in resource')
return table
def _must_be_type(self, t):
if self.type != t:
msg = 'Resource type "{}" is not ' \
'required type "{}"'.format(self.type, t)
raise TypeError(msg)
def _repr_html_(self):
"""HTML representation.
"""
items = self._get_items(html=True)
keystyle = '<span style="font-weight: 800;">'
hdr = '<h2>{}</h2>'.format(self.desc or 'Resource')
result = hdr
result += '<ul style="list-style-type: none; padding: 0;">'
for k, v in items:
if isinstance(v, str):
result += '<li>{}{}</span>: {}</li>'.format(keystyle, k, v)
else:
result += '<li>{}{}</span>:'.format(keystyle, k)
result += '<ul style="list-style-type: circle; ' \
'margin: 0 0 0 10px;">'
for v2 in v:
result += '<li>{}</li>'.format(v2)
result += '</ul></li>'
result += '</ul>'
return result
def _repr_text_(self, **kw):
"""Text representation.
"""
items = self._get_items(**kw)
lines = [self.desc or 'Resource']
for k, v in items:
if isinstance(v, str):
lines.append('{}: {}'.format(k, v))
else:
lines.append('{}:'.format(k))
for v2 in v:
lines.append(' - {}'.format(v2))
return '\n'.join(lines)
__str__ = _repr_text_
def _get_items(self, html=False, include_empty=False, include_data=False):
"""Get attribute tree.
Tree is a simple depth=2 tree.
Returns:
list: Items, list of (str, str|list of (str, str))
"""
def datestr(t):
return pendulum.from_timestamp(t).to_datetime_string()
def verstr(v):
return '.'.join([str(x) for x in v[:3]]) + v[3]
if html:
def ctstr(c):
return '{} <{}>'.format(c.name, c.email)
else:
def ctstr(c):
return '{} <{}>'.format(c.name, c.email)
items = [('Created', datestr(self.created)),
('Modified', datestr(self.modified)),
('Version', verstr(self.version.revision)),
('Creator', ctstr(self.creator))]
if self.collaborators:
collabstr = ', '.join([ctstr(c) for c in self.collaborators])
items.append(('Collaborators', collabstr))
if self.sources or include_empty:
subitems = []
for src in self.sources:
hsrc = src.source
if src.doi:
if html:
hsrc += ' <a href="https://doi.org/{}" ' \
'target=_blank>{}</a>'.format(src.doi, src.doi)
else:
hsrc += ' DOI:{}'.format(src.doi)
subitems.append(hsrc)
items.append(('Sources', subitems))
if self.codes or include_empty:
subitems = []
for code in self.codes:
if code.location:
if html:
code_name = '<a href="{}" target=_blank>{}</a>'.format(
code.location, code.name)
else:
code_name = ' {}, URL:{}'.format(
code.name, code.location)
else:
code_name = code.name
if code.language:
code_type = '{}/{}'.format(code.type, code.language)
else:
code_type = code.type
hcode = '({}) {}: {}'.format(code_type, code_name, code.desc)
subitems.append(hcode)
items.append(('Codes', subitems))
if self.datafiles or include_empty:
subitems = []
for dfile in self.datafiles:
hdfile = '{}: {}'.format(dfile.path, dfile.desc)
subitems.append(hdfile)
items.append(('Data files', subitems))
if self.aliases or include_empty:
sub_str = ', '.join(self.aliases)
items.append(('Aliases', sub_str))
if self.tags or include_empty:
sub_str = ', '.join(self.tags)
items.append(('Tags', sub_str))
if self.relations or include_empty:
subitems = []
for rel in self.relations:
sub_str = '{} === {} ===>> {}'.format(rel.subject,
rel.predicate,
rel.object)
subitems.append(sub_str)
items.append(('Relations', subitems))
if include_data:
items.append(('Data', self.data))
return items
[docs]class TabularDataResource(Resource):
"""Tabular data resource & factory.
"""
def __init__(self, table=None, **kwargs):
super(TabularDataResource, self).__init__(**kwargs)
self.type = ResourceTypes.tabular_data
# If a table is supplied, put its data in a temporary file
if table is not None:
self.data = {}
self._set_metadata(table)
path = self._dump_table(table)
self.datafiles.append(FilePath(tempfile=True, path=path))
@staticmethod
def _dump_table(table):
tfile = NamedTemporaryFile(mode='w', suffix='.json', delete=False)
table.dump(tfile.file)
return tfile.name
def _set_metadata(self, meta):
datatypes = set()
for m in meta.metadata:
work = '{a}, "{t}". {i}, {d}'.format(
a=m.author, t=m.title, i=m.info, d=m.date)
self.sources.append(Source(source=work, date=m.date))
self.data['meta'] = m.as_dict()
if m.datatype:
datatypes.add(m.datatype)
for dtype in datatypes:
self.tags.append(dtype)
[docs]class PropertyDataResource(TabularDataResource):
"""Property data resource & factory.
"""
def __init__(self, property_table=None, **kwargs):
super(PropertyDataResource, self).__init__(table=property_table,
**kwargs)
self.type = ResourceTypes.property_data
[docs]class FlowsheetResource(Resource):
"""Flowsheet resource & factory.
"""
[docs] @classmethod
def from_flowsheet(cls, obj, **kw):
r = FlowsheetResource()
cls.init_resource(r, kw)
r.type = ResourceTypes.fs
code = Code(type='class')
if hasattr(obj, '_orig_module'):
code.name = '{}.{}'.format(obj._orig_module, obj._orig_name)
else:
code.name = obj.__class__
try:
ver = obj.__version__
code.release = Version(revision=ver)
except AttributeError:
_log.warn('No __version__ found for code object: {}'.format(obj))
r.codes = [code]
return r
[docs]def get_resource_structure():
r = Resource()
items = r._get_items(include_empty=True, include_data=True)
lines = ['digraph hierarchy {',
' rankdir=LR;'
' node [color=Black,fontname=Courier,shape=box];',
' edge [color=Blue, style=solid];']
for k, v in items:
k = k.replace(' ', '').lower()
lines.append(' Resource -> {};'.format(k))
if isinstance(v, list):
lines.append(' {} -> {}_values;'.format(k, k))
lines.append('}')
return '\n'.join(lines)