Merge pull request #386 from irushchyshyn/feature/build-requires-ui

Load build dependencies into the database

Author: irushchyshyn

portingdb/cli.py

@@ -170,12 +170,12 @@ def report(ctx):
     query = db.query(tables.Package)
     query = query.order_by(func.lower(tables.Package.name))
     query = query.options(eagerload(tables.Package.by_collection))
-    query = query.options(subqueryload(tables.Package.requirements))
+    query = query.options(subqueryload(tables.Package.run_time_requirements))
     for package in query:
         print_collection_info(package, collections)
         print(' ' + package.name, end=' ')
         reqs = []
-        for req in package.requirements:
+        for req in package.run_time_requirements:
             for cp in req.by_collection.values():
                 if cp.status not in ('released', 'legacy-leaf', 'py3-only'):
                     reqs.append(req.name)
@@ -262,7 +262,7 @@ def can_ignore(pkg):
             e = '✔'
         else:
             seen.add(pkg)
-            reqs = [p for p in pkg.requirements if p is not pkg]
+            reqs = [p for p in pkg.run_time_requirements if p is not pkg]
             if skip:
                 reqs = [r for r in reqs
                         if not (can_ignore(r) or r.name in exclude)]

portingdb/htmlreport.py

@@ -210,6 +210,15 @@ def package(pkg):
 
     dependents = list(queries.dependents(db, package))
 
+    query = queries.build_dependencies(db, package)
+    query = query.options(eagerload('status_obj'))
+    query = query.options(subqueryload('collection_packages'))
+    query = query.options(subqueryload('collection_packages.links'))
+    query = query.options(eagerload('collection_packages.status_obj'))
+    build_dependencies = list(query)
+
+    build_dependents = list(queries.build_dependents(db, package))
+
     return render_template(
         'package.html',
         breadcrumbs=(
@@ -221,8 +230,11 @@ def package(pkg):
         dependencies=dependencies,
         dependents=dependents,
         deptree=[(package, gen_deptree(dependencies))],
-        len_dependencies=len(dependencies),
         dependencies_status_counts=get_status_counts(dependencies),
+        build_dependencies=build_dependencies,
+        build_dependents=build_dependents,
+        build_deptree=[(package, gen_deptree(build_dependencies, run_time=False, build_time=True))],
+        build_dependencies_status_counts=get_status_counts(build_dependencies),
     )
 
 
@@ -258,21 +270,24 @@ def group(grp):
         grp=group,
         packages=packages,
         len_packages=len(packages),
-        deptree=list(gen_deptree(seed_groups)),
+        deptree=list(gen_deptree(seed_groups, run_time=True, build_time=True)),
         status_counts=get_status_counts(packages),
     )
 
 
-def gen_deptree(base, *, seen=None):
+def gen_deptree(base, *, seen=None, run_time=True, build_time=False):
     seen = seen or set()
     base = tuple(base)
     for pkg in base:
         if pkg in seen or pkg.status in {'idle'} | DONE_STATUSES:
             yield pkg, []
         else:
-            reqs = sorted(pkg.requirements,
-                          key=lambda p: (-p.status_obj.weight, p.name))
-            yield pkg, gen_deptree(reqs, seen=seen | {pkg})
+            reqs = sorted(
+                set((pkg.run_time_requirements if run_time else []) +
+                    (pkg.build_time_requirements if build_time else [])),
+                key=lambda p: (-p.status_obj.weight, p.name))
+            yield pkg, gen_deptree(reqs, seen=seen | {pkg},
+                                   run_time=run_time, build_time=build_time)
         seen.add(pkg)
 
 
@@ -347,7 +362,7 @@ def graph_json(grp=None, pkg=None):
         if grp:
             query = query.join(tables.GroupPackage)
             query = query.filter(tables.GroupPackage.group_ident == grp)
-        query = query.options(joinedload(tables.Package.requirers))
+        query = query.options(joinedload(tables.Package.run_time_requirers))
         packages = list(query)
     else:
         query = db.query(tables.Package)
@@ -360,20 +375,21 @@ def graph_json(grp=None, pkg=None):
             package = todo.pop()
             if package not in requirements:
                 requirements.add(package)
-                todo.update(p for p in package.requirements
+                todo.update(p for p in package.run_time_requirements
                             if p.status not in DONE_STATUSES)
         todo = {root_package}
         requirers = set()
         while todo:
             package = todo.pop()
             if package not in requirers:
                 requirers.add(package)
-                todo.update(p for p in package.requirers
+                todo.update(p for p in package.run_time_requirers
                             if p.status not in DONE_STATUSES)
         packages = list(requirements | requirers | {root_package})
 
     package_names = {p.name for p in packages}
     query = db.query(tables.Dependency)
+    query = query.filter(tables.Dependency.run_time)
     linked_pairs = {(d.requirer_name, d.requirement_name)
                     for d in query
                     if d.requirer_name in package_names

portingdb/load.py

@@ -230,18 +230,33 @@ def load_from_directories(db, directories):
         bulk_load(db, values, tables.Package.__table__, id_column="name")
 
         # Dependencies
-        values = [{'requirer_name': a, 'requirement_name': b, 'unversioned': False}
-                  for a, v in package_infos.items() for b in v.get('deps', ())
-                  if a != b]
+        ambiguous_deps = {}
         for requirement, info in package_infos.items():
             for requirer in info.get('unversioned_requirers', ()):
-                row = {'requirer_name': requirer,
-                       'requirement_name': requirement,
-                       'unversioned': False}
-                if row in values:
-                    values.remove(row)
-                row['unversioned'] = True
-                values.append(row)
+                ambiguous_deps[(requirer, requirement)] = True
+
+        values = []
+        for package, info in package_infos.items():
+            for dep in set(info.get('deps', ()) + info.get('build_deps', ())):
+                if package != dep:
+                    values.append(
+                        {'requirer_name': package,
+                         'requirement_name': dep,
+                         'run_time': dep in info.get('deps', ()),
+                         'build_time': dep in info.get('build_deps', ()),
+                         'unversioned': ambiguous_deps.pop((package, dep), False)}
+                    )
+
+        # Add the rest of ambiguous deps, which are not in portingdb (non-Python).
+        for requirer, requirement in ambiguous_deps:
+            values.append(
+                {'requirer_name': requirer,
+                 'requirement_name': requirement,
+                 # Ambiguous deps are not categorized into build
+                 # and run time. And do not need to be.
+                 'run_time': False, 'build_time': False,
+                 'unversioned': True}
+            )
 
         bulk_load(db, values, tables.Dependency.__table__,
                   key_columns=['requirer_name', 'requirement_name'])

portingdb/queries.py

@@ -38,9 +38,8 @@ def order_by_weight(db, query):
     return query
 
 
-def _deps(db, package, forward=True):
+def _deps(db, package, forward=True, run_time=True, build_time=False):
     query = packages(db)
-    package_table = aliased(tables.Package)
 
     if forward:
         col_a = tables.Dependency.requirement_name
@@ -51,6 +50,10 @@ def _deps(db, package, forward=True):
     query = query.outerjoin(tables.Dependency, col_a == tables.Package.name)
     query = query.filter(col_b == package.name)
 
+    query = query.filter(
+        or_(tables.Dependency.run_time == run_time,
+            tables.Dependency.build_time == build_time))
+
     query = query.join(tables.Status, tables.Status.ident == tables.Package.status)
     query = query.order_by(-tables.Status.weight)
     query = order_by_name(db, query)
@@ -66,6 +69,14 @@ def dependents(db, package):
     return _deps(db, package, False)
 
 
+def build_dependencies(db, package):
+    return _deps(db, package, True, False, True)
+
+
+def build_dependents(db, package):
+    return _deps(db, package, False, False, True)
+
+
 def order_by_name(db, query):
     return query.order_by(func.lower(tables.Package.name))
 
@@ -115,6 +126,7 @@ def update_status_summaries(db):
     subquery = subquery.join(tables.Dependency, tables.Dependency.requirement_name == dep.name)
     subquery = subquery.filter(tables.Dependency.requirer_name == tables.Package.name)
     subquery = subquery.filter(tables.Dependency.requirer_name != tables.Dependency.requirement_name)
+    subquery = subquery.filter(tables.Dependency.run_time)
 
     update = main_update
     update = update.where(subquery.exists())

portingdb/tables.py

@@ -1,5 +1,5 @@
 from sqlalchemy import Column, ForeignKey, MetaData, extract, desc
-from sqlalchemy import UniqueConstraint
+from sqlalchemy import UniqueConstraint, and_
 from sqlalchemy.types import Boolean, Integer, Unicode, UnicodeText, Date, DateTime
 from sqlalchemy.types import Enum
 from sqlalchemy.orm import backref, relationship
@@ -154,12 +154,12 @@ def __repr__(self):
 
     @property
     def pending_requirements(self):
-        return [r for r in self.requirements
+        return [r for r in self.run_time_requirements
                 if r.status not in ('released', 'dropped', 'py3-only', 'legacy-leaf')]
 
     @property
     def pending_requirers(self):
-        return [r for r in self.requirers
+        return [r for r in self.run_time_requirers
                 if r.status not in ('released', 'dropped', 'py3-only', 'legacy-leaf')]
 
     @property
@@ -285,6 +285,8 @@ class Dependency(TableBase):
     unversioned = Column(
         Boolean(), default=False,
         doc=u"True if the requirement name should be changed to a versioned one")
+    build_time = Column(Boolean())
+    run_time = Column(Boolean())
 
     requirer = relationship(
         'Package', backref=backref('requirement_dependencies'),
@@ -487,6 +489,20 @@ class HistoryNamingEntry(TableBase):
     secondaryjoin=Package.name == Dependency.requirement_name,
     backref="requirers")
 
+Package.run_time_requirements = relationship(
+    Package,
+    secondary=Dependency.__table__,
+    primaryjoin=and_(Package.name == Dependency.requirer_name, Dependency.run_time),
+    secondaryjoin=Package.name == Dependency.requirement_name,
+    backref="run_time_requirers")
+
+Package.build_time_requirements = relationship(
+    Package,
+    secondary=Dependency.__table__,
+    primaryjoin=and_(Package.name == Dependency.requirer_name, Dependency.build_time),
+    secondaryjoin=Package.name == Dependency.requirement_name,
+    backref="build_time_requirers")
+
 Package.groups = relationship(
     Group,
     secondary=GroupPackage.__table__,

portingdb/templates/package.html

@@ -122,6 +122,7 @@ <h3>
                 {% endif %}
             {% endfor %}
             <h3>Dependent packages</h3>
+            <h4>Run time</h4>
             {% if dependents %}
                 <ul class="simple-pkg-list">
                     {% for pkg in dependents %}
@@ -131,8 +132,21 @@ <h3>Dependent packages</h3>
             {% else %}
                 None.
             {% endif %}
+            <h4>Build time</h4>
+            {% if build_dependents %}
+                <ul class="simple-pkg-list">
+                    {% for pkg in build_dependents %}
+                        <li>{{ pkglink(pkg) }}</li>
+                    {% endfor %}
+                </ul>
+            {% else %}
+                None.
+            {% endif %}
             <h3>Dependency Tree</h3>
+            <h4>Run time</h4>
             {{ print_deptree(deptree) }}
+            <h4>Build time</h4>
+            {{ print_deptree(build_deptree) }}
             {% if pkg['loc_total'] %}
                 <h3>Project Size</h3>
                 {{ loc_chart(pkg) }}
@@ -181,6 +195,7 @@ <h3>Groups</h3>
         <div class="col-md-1"></div>
         <div class="col-md-8">
             <h2>Dependencies</h2>
+            <h4>Run time</h4>
             {{ progress_summary_legend(dependencies_status_counts.items()) }}
             {% if dependencies %}
                 <table class="table table-striped table-condensed table-hovered">
@@ -189,6 +204,15 @@ <h2>Dependencies</h2>
             {% else %}
                 None.
             {% endif %}
+            <h4>Build time</h4>
+            {{ progress_summary_legend(build_dependencies_status_counts.items()) }}
+            {% if build_dependencies %}
+                <table class="table table-striped table-condensed table-hovered">
+                    {{ pkglist_table_content(build_dependencies, collections, show_nonblocking=1) }}
+                </table>
+            {% else %}
+                None.
+            {% endif %}
         </div>
     </div>
 {% endblock bodycontent %}