Merge pull request #303 from mpsonntag/idcheck

Add validation for unique IDs within a document

Author: jgrewe

docs/data_model.md

@@ -13,7 +13,7 @@ interoperability, and being customizable. The model defines four
 entities (Property, Section, Value, RootSection) whose relations and
 elements are shown in the figure below.
 
-![odml_logo](./images/erModel.png "odml data model")
+![odml_logo](images/erModel.png "odml data model")
 
 Property and Section are the core entities. A Section contains
 Properties and can further have subsection thus building a tree-like

docs/images/erModel.png

@@ -50,6 +50,18 @@ pip install odml
 - We have assembled a set of
  [tutorials](http://github.com/G-Node/python-odml/blob/master/doc/tutorial.rst "Python Tutorial").
 
+### Python convenience scripts
+
+The Python installation features two convenience commandline scripts.
+
+- `odmlconversion`: Converts odML files of previous file versions into the current one.
+- `odmltordf`: Converts odML files to the supported RDF version of odML.
+
+Both scripts provide detailed usage descriptions by adding the help flag to the command.
+
+    odmlconversion -h
+    odmltordf -h
+
 ## Support
 
 If you experience problems using *odml* feel free to join our IRC channel

docs/index.md

@@ -3,37 +3,40 @@
 Generic odML validation framework
 """
 
+LABEL_ERROR = 'error'
+LABEL_WARNING = 'warning'
+
 
 class ValidationError(object):
     """
     Represents an error found in the validation process
 
-    The error is bound to an odML-object (*obj*) or a list of those
-    and contains a message and a type which may be one of:
-    'error', 'warning', 'info'
+    The error is bound to an odML-object (*obj*) or a list of
+    those and contains a message and a rank which may be one of:
+    'error', 'warning'.
     """
 
-    def __init__(self, obj, msg, type='error'):
+    def __init__(self, obj, msg, rank=LABEL_ERROR):
         self.obj = obj
         self.msg = msg
-        self.type = type
+        self.rank = rank
 
     @property
     def is_warning(self):
-        return self.type == 'warning'
+        return self.rank == LABEL_WARNING
 
     @property
     def is_error(self):
-        return self.type == 'error'
+        return self.rank == LABEL_ERROR
 
     @property
     def path(self):
         return self.obj.get_path()
 
     def __repr__(self):
-        return "<ValidationError(%s):%s \"%s\">" % (self.type,
-                                                    self.obj,
-                                                    self.msg)
+        return "<ValidationError(%s):%s '%s'>" % (self.rank,
+                                                  self.obj,
+                                                  self.msg)
 
 
 class Validation(object):
@@ -65,7 +68,7 @@ def __init__(self, doc):
         self.doc = doc  # may also be a section
         self.errors = []
         self.validate(doc)
-        # TODO isn't there a 'walk' method for these things?
+
         for sec in doc.itersections(recursive=True):
             self.validate(sec)
             for prop in sec.properties:
@@ -98,34 +101,115 @@ def __getitem__(self, obj):
 def section_type_must_be_defined(sec):
     """test that no section has an undefined type"""
     if sec.type is None or sec.type == '' or sec.type == 'undefined':
-        yield ValidationError(sec, 'Section type undefined', 'warning')
+        yield ValidationError(sec, 'Section type undefined', LABEL_WARNING)
+
 
 Validation.register_handler('section', section_type_must_be_defined)
 
 
-def section_repository_should_be_present(sec):
+def section_repository_present(sec):
     """
     1. warn, if a section has no repository or
     2. the section type is not present in the repository
     """
     repo = sec.get_repository()
     if repo is None:
-        yield ValidationError(sec, 'A section should have an associated '
-                                   'repository', 'warning')
+        yield ValidationError(sec,
+                              'A section should have an associated repository',
+                              LABEL_WARNING)
         return
 
     try:
         tsec = sec.get_terminology_equivalent()
-    except Exception as e:
-        yield ValidationError(sec, 'Could not load terminology: %s' % e,
-                                   'warning')
+    except Exception as exc:
+        yield ValidationError(sec,
+                              'Could not load terminology: %s' % exc,
+                              LABEL_WARNING)
         return
 
     if tsec is None:
-        yield ValidationError(sec, "Section type '%s' not found in terminology" % sec.type,
-                                   'warning')
+        yield ValidationError(sec,
+                              "Section type '%s' not found in terminology" % sec.type,
+                              LABEL_WARNING)
+
+
+Validation.register_handler('section', section_repository_present)
+
+
+def document_unique_ids(doc):
+    """
+    Traverse an odML Document and check whether all
+    assigned ids are unique within the document.
+
+    Yields all duplicate odML object id entries
+    that are encountered.
+
+    :param doc: odML document
+    """
+    id_map = {doc.id: "Document '%s'" % doc.get_path()}
+    for i in section_unique_ids(doc, id_map):
+        yield i
+
+
+def section_unique_ids(parent, id_map=None):
+    """
+    Traverse a parent (odML Document or Section)
+    and check whether all assigned ids are unique.
+
+    A "id":"odML object / path" dictionary of additional
+    'to-be-excluded' ids may be handed in via the
+    *id_map* attribute.
+
+    Yields all duplicate odML object id entries
+    that are encountered.
 
-Validation.register_handler('section', section_repository_should_be_present)
+    :param parent: odML Document or Section
+    :param id_map: "id":"odML object / path" dictionary
+    """
+    if not id_map:
+        id_map = {}
+
+    for sec in parent.sections:
+        for i in property_unique_ids(sec, id_map):
+            yield i
+
+        if sec.id in id_map:
+            yield ValidationError(sec, "Duplicate id in Section '%s' and '%s'" %
+                                  (sec.get_path(), id_map[sec.id]))
+        else:
+            id_map[sec.id] = "Section '%s'" % sec.get_path()
+
+        for i in section_unique_ids(sec, id_map):
+            yield i
+
+
+def property_unique_ids(section, id_map=None):
+    """
+    Check whether all ids assigned to the odML
+    Properties of an odML Section are unique.
+
+    A "id":"odML object / path" dictionary of additional
+    'to-be-excluded' ids may be handed in via the
+    *id_map* attribute.
+
+    Yields all duplicate odML object id entries
+    that are encountered.
+
+    :param section: odML Section
+    :param id_map: "id":"odML object / path" dictionary
+    """
+    if not id_map:
+        id_map = {}
+
+    for prop in section.properties:
+        if prop.id in id_map:
+            yield ValidationError(prop, "Duplicate id in Property '%s' and '%s'" %
+                                  (prop.get_path(), id_map[prop.id]))
+        else:
+            id_map[prop.id] = "Property '%s'" % prop.get_path()
+
+
+Validation.register_handler('odML', document_unique_ids)
 
 
 def object_unique_names(obj, children, attr=lambda x: x.name,
@@ -143,13 +227,13 @@ def object_unique_names(obj, children, attr=lambda x: x.name,
     if len(names) == len(children(obj)):
         return  # quick exit
     names = set()
-    for s in children(obj):
-        if attr(s) in names:
-            yield ValidationError(s, msg, 'error')
-        names.add(attr(s))
+    for i in children(obj):
+        if attr(i) in names:
+            yield ValidationError(i, msg, LABEL_ERROR)
+        names.add(attr(i))
 
 
-def section_unique_name_type_combination(obj):
+def section_unique_name_type(obj):
     for i in object_unique_names(
             obj,
             attr=lambda x: (x.name, x.type),
@@ -162,8 +246,9 @@ def property_unique_names(obj):
     for i in object_unique_names(obj, lambda x: x.properties):
         yield i
 
-Validation.register_handler('odML',    section_unique_name_type_combination)
-Validation.register_handler('section', section_unique_name_type_combination)
+
+Validation.register_handler('odML', section_unique_name_type)
+Validation.register_handler('section', section_unique_name_type)
 Validation.register_handler('section', property_unique_names)
 
 
@@ -179,11 +264,12 @@ def property_terminology_check(prop):
     if tsec is None:
         return
     try:
-        tprop = tsec.properties[prop.name]
+        tsec.properties[prop.name]
     except KeyError:
-        tprop = None
-        yield ValidationError(prop, "Property '%s' not found in terminology" %
-                              prop.name, 'warning')
+        yield ValidationError(prop,
+                              "Property '%s' not found in terminology" % prop.name,
+                              LABEL_WARNING)
+
 
 Validation.register_handler('property', property_terminology_check)
 
@@ -200,12 +286,14 @@ def property_dependency_check(prop):
     try:
         dep_obj = prop.parent[dep]
     except KeyError:
-        yield ValidationError(prop, "Property refers to a non-existent "
-                                    "dependency object", 'warning')
+        yield ValidationError(prop,
+                              "Property refers to a non-existent dependency object",
+                              LABEL_WARNING)
         return
 
-    if prop.dependency_value not in dep_obj.value[0]:  # FIXME
+    if prop.dependency_value not in dep_obj.value[0]:
         yield ValidationError(prop, "Dependency-value is not equal to value of"
-                              " the property's dependency", 'warning')
+                              " the property's dependency", LABEL_WARNING)
+
 
 Validation.register_handler('property', property_dependency_check)