[MOD] file:read, fn:unparsed-text: options unified

Author: ChristianGruen

basex-core/src/main/java/org/basex/core/CommonOptions.java

@@ -42,4 +42,6 @@ public interface CommonOptions {
   StringOption ENCODING = new StringOption("encoding");
   /** Normalize newlines. */
   BooleanOption NORMALIZE_NEWLINES = new BooleanOption("normalize-newlines");
+  /** Fallback. */
+  BooleanOption FALLBACK = new BooleanOption("fallback", false);
 }

basex-core/src/main/java/org/basex/gui/view/editor/EditorView.java

@@ -649,7 +649,7 @@ private EditorArea execEditor() {
   private byte[] read(final IOFile file) throws IOException {
     try {
       // try to open as validated UTF-8 document
-      return new NewlineInput(file).validate(true).content();
+      return new NewlineInput(file).fallback(false).content();
     } catch(final InputException ex) {
       // error...
       Util.debug(ex);

basex-core/src/main/java/org/basex/io/in/NewlineInput.java

@@ -57,8 +57,8 @@ public NewlineInput(final IO in, final String encoding) throws IOException {
   }
 
   @Override
-  public NewlineInput validate(final boolean flag) {
-    super.validate(flag);
+  public NewlineInput fallback(final boolean flag) {
+    super.fallback(flag);
     return this;
   }
 

basex-core/src/main/java/org/basex/io/in/TextDecoder.java

@@ -21,8 +21,8 @@
 abstract class TextDecoder {
   /** Encoding. */
   final String encoding;
-  /** Indicates if input must be valid. */
-  boolean validate;
+  /** Replace invalid input with Unicode replacement character. */
+  boolean fallback = true;
 
   /**
    * Constructor.
@@ -91,17 +91,16 @@ final int readUTF16(final TextInput ti, final boolean be) throws IOException {
    * @throws IOException I/O exception
    */
   final int invalid(final boolean incomplete, final byte... bytes) throws IOException {
-    if(validate) {
-      final TokenBuilder tb = new TokenBuilder();
-      final IntUnaryOperator toHex = c -> c + (c > 9 ? '7' : '0');
-      for(final int b : bytes) {
-        if(!tb.isEmpty()) tb.add(", ");
-        tb.add(toHex.applyAsInt(b >> 4 & 0x0F)).add(toHex.applyAsInt(b & 0x0F));
-      }
-      if(incomplete) tb.add(", ??");
-      throw new DecodingException("Invalid " + encoding + " character encoding: " + tb);
+    if(fallback) return Token.REPLACEMENT;
+
+    final TokenBuilder tb = new TokenBuilder();
+    final IntUnaryOperator toHex = c -> c + (c > 9 ? '7' : '0');
+    for(final int b : bytes) {
+      if(!tb.isEmpty()) tb.add(", ");
+      tb.add(toHex.applyAsInt(b >> 4 & 0x0F)).add(toHex.applyAsInt(b & 0x0F));
     }
-    return Token.REPLACEMENT;
+    if(incomplete) tb.add(", ??");
+    throw new DecodingException("Invalid " + encoding + " character encoding: " + tb);
   }
 
   /** UTF8 Decoder. */

basex-core/src/main/java/org/basex/io/in/TextInput.java

@@ -19,8 +19,8 @@
 public class TextInput extends BufferInput {
   /** Decoder. */
   private TextDecoder decoder;
-  /** Indicates if the input is to be checked for valid XML 1.0.5 characters. */
-  private boolean validate;
+  /** Replace invalid input with Unicode replacement character. */
+  private boolean fallback = true;
 
   /**
    * Constructor.
@@ -105,9 +105,9 @@ private void guess(final String encoding) throws IOException {
    * @param flag flag to be set
    * @return self reference
    */
-  public TextInput validate(final boolean flag) {
-    validate = flag;
-    decoder.validate = flag;
+  public TextInput fallback(final boolean flag) {
+    fallback = flag;
+    decoder.fallback = flag;
     return this;
   }
 
@@ -122,7 +122,7 @@ public TextInput encoding(final String encoding) throws IOException {
       final String enc = normEncoding(encoding, false);
       decoder = TextDecoder.get(enc != UTF16 ? enc :
         decoder.encoding.equals(UTF16LE) ? UTF16LE : UTF16BE);
-      decoder.validate = validate;
+      decoder.fallback = fallback;
     }
     return this;
   }
@@ -144,8 +144,8 @@ public String encoding() {
   public int read() throws IOException {
     final int cp = decoder.read(this);
     if(cp != -1 && !XMLToken.valid(cp)) {
-      if(validate) throw new InputException(cp);
-      return Token.REPLACEMENT;
+      if(fallback) return Token.REPLACEMENT;
+      throw new InputException(cp);
     }
     return cp;
   }

basex-core/src/main/java/org/basex/query/func/archive/ArchiveFn.java

@@ -37,7 +37,7 @@ final byte[] encode(final byte[] value, final String encoding, final boolean enf
     if(encoding == Strings.UTF8 && !enforce) return value;
     try {
       final boolean validate = qc.context.options.get(MainOptions.CHECKSTRINGS);
-      return ConvertFn.toString(new ArrayInput(value), encoding, validate);
+      return ConvertFn.toString(new ArrayInput(value), encoding, !validate);
     } catch(final IOException ex) {
       throw ARCHIVE_ENCODE2_X.get(info, ex);
     }

basex-core/src/main/java/org/basex/query/func/bin/BinDecodeString.java

@@ -36,7 +36,7 @@ public Item item(final QueryContext qc, final InputInfo ii) throws QueryExceptio
     }
 
     try {
-      return Str.get(ConvertFn.toString(new ArrayInput(bytes), encoding, true));
+      return Str.get(ConvertFn.toString(new ArrayInput(bytes), encoding, false));
     } catch(final IOException ex) {
       throw BIN_CE_X.get(info, ex);
     }

basex-core/src/main/java/org/basex/query/func/convert/ConvertBinaryToString.java

@@ -23,7 +23,7 @@ public Item item(final QueryContext qc, final InputInfo ii) throws QueryExceptio
     final boolean fallback = toBooleanOrFalse(arg(2), qc);
 
     try(BufferInput bi = value.input(info)) {
-      return Str.get(toString(bi, encoding, !fallback));
+      return Str.get(toString(bi, encoding, fallback));
     } catch(final IOException ex) {
       throw CONVERT_STRING_X.get(info, ex);
     }

basex-core/src/main/java/org/basex/query/func/convert/ConvertFn.java

@@ -56,14 +56,14 @@ public static byte[] toBinary(final byte[] in, final String encoding)
    * Converts the specified input to a string in the specified encoding.
    * @param is input stream
    * @param encoding encoding (can be {@code null})
-   * @param validate validate string
+   * @param fallback return Unicode replacement character to invalid characters
    * @return resulting value
    * @throws IOException I/O exception
    */
-  public static byte[] toString(final InputStream is, final String encoding, final boolean validate)
+  public static byte[] toString(final InputStream is, final String encoding, final boolean fallback)
       throws IOException {
     try(TextInput ti = new TextInput(is, encoding)) {
-      return ti.validate(validate).content();
+      return ti.fallback(fallback).content();
     }
   }
 }

basex-core/src/main/java/org/basex/query/func/fetch/FetchText.java

@@ -20,6 +20,6 @@ public Item item(final QueryContext qc, final InputInfo ii) throws QueryExceptio
     final String encoding = toEncodingOrNull(arg(1), FETCH_ENCODING_X, qc);
     final boolean fallback = toBooleanOrFalse(arg(2), qc);
 
-    return new StrLazy(source, encoding, FETCH_OPEN_X, !fallback);
+    return new StrLazy(source, encoding, FETCH_OPEN_X, fallback);
   }
 }