Improvements to tackle wrong expression outputs

mathics/builtin/evaluation.py

@@ -406,13 +406,31 @@ class OutputSizeLimit(Predefined):
         To set no limit on output size, use $OutputSizeLimit = Infinity.
     </dl>
 
-    >> $OutputSizeLimit = 100;
-    >> Table[i, {i, 1, 100}]
-     = {1, 2, 3, 4, 5, <<90>>, 96, 97, 98, 99, 100}
-    >> $OutputSizeLimit = 10;
+    >> $OutputSizeLimit = 50;
+
     >> Table[i, {i, 1, 100}]
-     = {1, <<98>>, 100}
-    >> $OutputSizeLimit = Infinity;
+     : Parts of this output were omitted (see <<71>>). To generate the whole output, please set $OutputSizeLimit = Infinity.
+     = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, <<71>>, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100}
+
+    #> Take[Range[1000], 1001]
+     : Cannot take positions 1 through 1001 in {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, <<976>>, 989, 990, 991, 992, 993, 994, 995, 996, 997, 998, 999, 1000}.
+     : Parts of this output were omitted (see <<976>>). To generate the whole output, please set $OutputSizeLimit = Infinity.
+     = Take[{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, <<976>>, 989, 990, 991, 992, 993, 994, 995, 996, 997, 998, 999, 1000}, 1001]
+
+    #> {}
+     = {}
+
+    #> $OutputSizeLimit = 100;
+
+    #> Table[Graphics[Table[Circle[],{10}]], {5}]
+     = {-Graphics-, -Graphics-, -Graphics-, -Graphics-, -Graphics-}
+
+    #> Quiet[ImageAvailable = SameQ[Head[Image[{{0, 1}, {1, 0}}] // ToBoxes], ImageBox]];
+    #> If[ImageAvailable, Table[Image[{{1, 0}, {0, 1}}], {5}], {"-Image-", "-Image-", "-Image-", "-Image-", "-Image-"}]
+     = {-Image-, -Image-, -Image-, -Image-, -Image-}
+
+    #> $OutputSizeLimit = Infinity;
+
     """
 
     name = '$OutputSizeLimit'

mathics/builtin/inout.py

@@ -1140,6 +1140,12 @@ def apply_makeboxes(self, s, args, f, evaluation):
         '''MakeBoxes[StringForm[s_String, args___],
             f:StandardForm|TraditionalForm|OutputForm]'''
 
+        # StringForm does not call evaluation.make_boxes
+        # since we use it for messages and we never want
+        # to omit parts of the message. args are subject
+        # to MakeBoxes (see below) and thus can get parts
+        # omitted.
+
         s = s.value
         args = args.get_sequence()
         result = []
@@ -1640,6 +1646,8 @@ class General(Builtin):
         'invalidargs': "Invalid arguments.",
 
         'notboxes': "`1` is not a valid box structure.",
+        'omit': "Parts of this output were omitted (see `1`). " +
+                  "To generate the whole output, please set $OutputSizeLimit = Infinity.",
 
         'pyimport': "`1`[] is not available. Your Python installation misses the \"`2`\" module.",
     }

mathics/core/evaluation.py

@@ -14,7 +14,7 @@
 import itertools
 
 from mathics import settings
-from mathics.core.expression import ensure_context, KeyComparable, make_boxes_strategy
+from mathics.core.expression import ensure_context, KeyComparable, make_boxes_strategy, Omissions
 
 FORMATS = ['StandardForm', 'FullForm', 'TraditionalForm',
            'OutputForm', 'InputForm',
@@ -178,7 +178,7 @@ def __init__(self, definitions=None,
 
         self.quiet_all = False
         self.format = format
-        self.boxes_strategy = make_boxes_strategy(None, self)
+        self.boxes_strategy = make_boxes_strategy(None, None, self)
         self.catch_interrupt = catch_interrupt
 
     def parse(self, query):
@@ -317,7 +317,7 @@ def get_stored_result(self, result):
     def stop(self):
         self.stopped = True
 
-    def format_output(self, expr, format=None):
+    def format_output(self, expr, format=None, warn_about_omitted=True):
         if format is None:
             format = self.format
 
@@ -329,18 +329,24 @@ def format_output(self, expr, format=None):
         old_boxes_strategy = self.boxes_strategy
         try:
             capacity = self.definitions.get_config_value('System`$OutputSizeLimit')
-            self.boxes_strategy = make_boxes_strategy(capacity, self)
+            omissions = Omissions()
+            self.boxes_strategy = make_boxes_strategy(capacity, omissions, self)
 
             options = {}
 
+            # for xml/MathMLForm and tex/TexForm, output size limits are applied in the output
+            # form's apply methods (e.g. see MathMLForm.apply) and then passed through
+            # result.boxes_to_text which, in these two cases, must not apply any additional
+            # clipping (it would clip already clipped string material).
+
+            # for text/OutputForm, on the other hand, the call to result.boxes_to_text is the
+            # only place there is to apply output size limits, which has us resort to setting
+            # options['output_size_limit']. NOTE: disabled right now, causes problems with
+            # long message outputs (see test case Table[i, {i, 1, 100}] under OutputSizeLimit).
+
             if format == 'text':
                 result = expr.format(self, 'System`OutputForm')
-                # for MathMLForm and TexForm, output size limits are applied in the form's apply
-                # methods (e.g. see MathMLForm.apply) and then passed through result.boxes_to_text
-                # which must, in these cases, not apply additional clipping, as this would clip
-                # already clipped string material. for OutputForm, on the other hand, the call to
-                # result.boxes_to_text is the only place we have to apply output size limits.
-                options['output_size_limit'] = capacity
+                # options['output_size_limit'] = capacity
             elif format == 'xml':
                 result = Expression(
                     'StandardForm', expr).format(self, 'System`MathMLForm')
@@ -356,6 +362,9 @@ def format_output(self, expr, format=None):
                 self.message('General', 'notboxes',
                              Expression('FullForm', result).evaluate(self))
                 boxes = None
+
+            if warn_about_omitted:
+                omissions.warn(self)
         finally:
             self.boxes_strategy = old_boxes_strategy
 
@@ -415,7 +424,7 @@ def message(self, symbol, tag, *args):
             text = String("Message %s::%s not found." % (symbol_shortname, tag))
 
         text = self.format_output(Expression(
-            'StringForm', text, *(from_python(arg) for arg in args)), 'text')
+            'StringForm', text, *(from_python(arg) for arg in args)), 'text', warn_about_omitted=False)
 
         self.out.append(Message(symbol_shortname, tag, text))
         self.output.out(self.out[-1])

mathics/core/expression.py

@@ -62,6 +62,15 @@ def system_symbols_dict(d):
     return {ensure_context(k): v for k, v in six.iteritems(d)}
 
 
+_layout_boxes = system_symbols(
+    'RowBox',
+    'SuperscriptBox',
+    'SubscriptBox',
+    'SubsuperscriptBox',
+    'FractionBox',
+    'SqrtBox')
+
+
 class BoxError(Exception):
     def __init__(self, box, form):
         super(BoxError, self).__init__(
@@ -311,6 +320,12 @@ def format(self, evaluation, form):
             'MakeBoxes', expr, Symbol(form)).evaluate(evaluation)
         return result
 
+    def output_cost(self):
+        # the cost of outputting this item, usually the number of
+        # characters without any formatting elements or whitespace;
+        # e.g. "a, b", would be counted as 3.
+        return 1  # fallback implementation: count as one character
+
     def is_free(self, form, evaluation):
         from mathics.core.pattern import StopGenerator
 
@@ -1155,6 +1170,27 @@ def block(tex, only_subsup=False):
         else:
             raise BoxError(self, 'tex')
 
+    def output_cost(self):
+        name = self.get_head_name()
+
+        if name in ('System`ImageBox', 'System`GraphicsBox', 'System`Graphics3DBox'):
+            return 1  # count as expensive as one character
+
+        leaves = self.leaves
+        cost_of_leaves = sum(leaf.output_cost() for leaf in leaves)
+
+        if name in _layout_boxes:
+            return cost_of_leaves
+        else:
+            separator_cost = 1  # i.e. ","
+            n_separators = max(len(leaves) - 1, 0)
+            total_cost = 2 + cost_of_leaves + separator_cost * n_separators  # {a, b, c}, [a, b, c]
+
+            if name != 'System`List':
+                total_cost += self.head.output_cost() + separator_cost * n_separators
+
+            return total_cost
+
     def default_format(self, evaluation, form):
         return '%s[%s]' % (self.head.default_format(evaluation, form),
                            ', '.join([leaf.default_format(evaluation, form)
@@ -1435,6 +1471,9 @@ def __str__(self):
     def do_copy(self):
         return Symbol(self.name)
 
+    def output_cost(self):
+        return len(self.name)
+
     def boxes_to_text(self, **options):
         return str(self.name)
 
@@ -1583,6 +1622,7 @@ def _NumberFormat(man, base, exp, options):
     'NumberMultiplier': '\u00d7',
 }
 
+
 class Integer(Number):
     def __new__(cls, value):
         n = int(value)
@@ -1596,6 +1636,9 @@ def boxes_to_text(self, **options):
     def boxes_to_xml(self, **options):
         return self.make_boxes('MathMLForm').boxes_to_xml(**options)
 
+    def output_cost(self):
+        return len(str(self.value))
+
     def boxes_to_tex(self, **options):
         return str(self.value)
 
@@ -1665,6 +1708,10 @@ def __new__(cls, numerator, denominator=None):
         self.value = sympy.Rational(numerator, denominator)
         return self
 
+    def output_cost(self):
+        numer, denom = self.value.as_numer_denom()
+        return len(str(numer)) + len(str(denom))
+
     def atom_to_boxes(self, f, evaluation):
         return self.format(evaluation, f.get_name())
 
@@ -1771,6 +1818,9 @@ def __new__(cls, value, p=None):
         else:
             return PrecisionReal.__new__(PrecisionReal, value)
 
+    def output_cost(self):
+        return len(self.boxes_to_text())
+
     def boxes_to_text(self, **options):
         return self.make_boxes('System`OutputForm').boxes_to_text(**options)
 
@@ -1970,6 +2020,9 @@ def atom_to_boxes(self, f, evaluation):
     def __str__(self):
         return str(self.to_sympy())
 
+    def output_cost(self):
+        return self.real.output_cost() + self.imag.output_cost() + 2  # "+", "I"
+
     def to_sympy(self, **kwargs):
         return self.real.to_sympy() + sympy.I * self.imag.to_sympy()
 
@@ -2134,6 +2187,9 @@ def __new__(cls, value):
     def __str__(self):
         return '"%s"' % self.value
 
+    def output_cost(self):
+        return len(self.value)
+
     def boxes_to_text(self, show_string_characters=False, output_size_limit=None, **options):
         value = self.value
         if (not show_string_characters and      # nopep8
@@ -2289,7 +2345,8 @@ def boxes_to_text(self, **options):
 
     def boxes_to_xml(self, **options):
         new_options = dict((k, v) for k, v in options.items() if k != 'output_size_limit')
-        return super(Omitted, self).boxes_to_xml(**new_options)
+        s = super(Omitted, self).boxes_to_xml(**new_options)
+        return "<mtext mathcolor='#4040a0'>%s</mtext>" % s
 
     def boxes_to_tex(self, **options):
         new_options = dict((k, v) for k, v in options.items() if k != 'output_size_limit')
@@ -2342,9 +2399,25 @@ def make(self, items, form, segment=None):
         raise NotImplementedError()
 
 
+class Omissions:
+    def __init__(self):
+        self._omissions = []
+
+    def add(self, count):
+        n = len(self._omissions)
+        if n < 3:
+            self._omissions.append('<<%d>>' % count)
+        if n == 3:
+            self._omissions.append('...')
+
+    def warn(self, evaluation):
+        if self._omissions:
+            evaluation.message('General', 'omit', ', '.join(self._omissions))
+
+
 class _UnlimitedMakeBoxesStrategy(_MakeBoxesStrategy):
     def __init__(self):
-        pass
+        self.omissions_occured = False
 
     def capacity(self):
         return None
@@ -2365,11 +2438,12 @@ def __init__(self, capacity, side, both_sides, depth):
 
 
 class _LimitedMakeBoxesStrategy(_MakeBoxesStrategy):
-    def __init__(self, capacity, evaluation):
+    def __init__(self, capacity, omissions, evaluation):
         self._capacity = capacity
         self._evaluation = evaluation
         self._state = _LimitedMakeBoxesState(self._capacity, 1, True, 1)
         self._unlimited = _UnlimitedMakeBoxesStrategy()
+        self._omissions = omissions
 
     def capacity(self):
         return self._capacity
@@ -2378,8 +2452,8 @@ def make(self, items, form, segment=None):
         state = self._state
         capacity = state.capacity
 
-        if capacity is None or len(items) < 1:
-            return self._unlimited(items, form, segment)
+        if capacity is None or len(items) <= 2:
+            return self._unlimited.make(items, form, segment)
 
         left_leaves = []
         right_leaves = []
@@ -2447,6 +2521,8 @@ def make(self, items, form, segment=None):
                     break
 
         ellipsis_size = len(items) - (len(left_leaves) + len(right_leaves))
+        if ellipsis_size > 0 and self._omissions:
+            self._omissions.add(ellipsis_size)
         ellipsis = [Omitted('<<%d>>' % ellipsis_size)] if ellipsis_size > 0 else []
 
         if segment is not None:
@@ -2469,16 +2545,16 @@ def _evaluate(self, item, form, **kwargs):
             # the simple solution; the problem is that it's redundant, as for {{{a}, b}, c}, we'd
             # call boxes_to_xml first on {a}, then on {{a}, b}, then on {{{a}, b}, c}. a good fix
             # is not simple though, so let's keep it this way for now.
-            cost = len(box.boxes_to_xml(evaluation=self._evaluation))  # evaluate len as XML
+            cost = box.output_cost()
 
             return box, cost
         finally:
             self._state = old_state
 
 
-def make_boxes_strategy(capacity, evaluation):
+def make_boxes_strategy(capacity, omissions, evaluation):
     if capacity is None:
         return _UnlimitedMakeBoxesStrategy()
     else:
-        return _LimitedMakeBoxesStrategy(capacity, evaluation)
+        return _LimitedMakeBoxesStrategy(capacity, omissions, evaluation)