RESTCOMM-1856

Motivation:
-Fix inbound and outbound smpp encoding
Modifications:
-Add default inbound and outbound configs for Smpp object
-Decode and encode with default encoding when DCS==DEFAULT
-Decode content string in Downloader with encoding header or default RVD encoding (UTF-8)
Results:
-Inbound and outbound smpp connection can be configured with specific character encodings
-Any inbound and outbound default and UCS2 messages will be decoded and encoded correctly

Author: abdulazizali77

restcomm/restcomm.application/src/main/webapp/WEB-INF/conf/restcomm.xml

@@ -670,6 +670,11 @@
 					If no response received for 3 consecutive requests, connection will be killed
 					and attempted to connect again -->
 				<enquirelinkdelay>30000</enquirelinkdelay>
+				<!-- when no values are set, default value from code is UTF-8-->
+				<!-- 
+				<incomingencoding>UTF-8</incomingencoding>
+				<outgoingencoding>GSM7</outgoingencoding>
+				 -->
 			</connection>
 		</connections>
 	</smpp>

restcomm/restcomm.http/src/main/java/org/restcomm/connect/http/client/Downloader.java

@@ -22,6 +22,8 @@
 import akka.actor.ActorRef;
 import akka.event.Logging;
 import akka.event.LoggingAdapter;
+
+import org.apache.commons.io.IOUtils;
 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpHeaders;
@@ -39,7 +41,6 @@
 import org.restcomm.connect.commons.common.http.CustomHttpClientBuilder;
 import org.restcomm.connect.commons.configuration.RestcommConfiguration;
 import org.restcomm.connect.commons.faulttolerance.RestcommUntypedActor;
-import org.restcomm.connect.commons.util.StringUtils;
 import org.xml.sax.InputSource;
 
 import javax.xml.parsers.DocumentBuilder;
@@ -86,6 +87,7 @@ public HttpResponseDescriptor fetch (final HttpRequestDescriptor descriptor) thr
         try {
             do {
                 request = request(temp);
+                //FIXME:should we externalize RVD encoding default?
                 request.setHeader("http.protocol.content-charset", "UTF-8");
                 if (descriptor.getTimeout() > 0){
                     HttpContext httpContext = new BasicHttpContext();
@@ -161,6 +163,7 @@ private HttpResponseDescriptor validateXML (final HttpResponseDescriptor descrip
                 // parse an XML document into a DOM tree
                 String xml = descriptor.getContentAsString().trim().replaceAll("&([^;]+(?!(?:\\w|;)))", "&$1");
                 DocumentBuilder parser = DocumentBuilderFactory.newInstance().newDocumentBuilder();
+                //FIXME:should we externalize RVD encoding default?
                 parser.parse(new InputSource(new ByteArrayInputStream(xml.getBytes("utf-8"))));
                 return descriptor;
             } catch (final Exception e) {
@@ -218,6 +221,7 @@ public HttpUriRequest request (final HttpRequestDescriptor descriptor) throws Il
         } else if ("POST".equalsIgnoreCase(method)) {
             final List<NameValuePair> parameters = descriptor.getParameters();
             final HttpPost post = new HttpPost(uri);
+            //FIXME:should we externalize RVD encoding default?
             post.setEntity(new UrlEncodedFormEntity(parameters, "UTF-8"));
             return post;
         } else {
@@ -237,14 +241,17 @@ private HttpResponseDescriptor response (final HttpRequest request, final HttpRe
             InputStream stream = entity.getContent();
             try {
                 final Header contentEncoding = entity.getContentEncoding();
-                if (contentEncoding != null) {
-                    builder.setContentEncoding(contentEncoding.getValue());
+                //FIXME:should we externalize RVD encoding default?
+                String encodingValue = "UTF-8";
+                if (contentEncoding != null && !contentEncoding.getValue().isEmpty()) {
+                    encodingValue = contentEncoding.getValue();
+                    builder.setContentEncoding(encodingValue);
                 }
                 final Header contentType = entity.getContentType();
                 if (contentType != null) {
                     builder.setContentType(contentType.getValue());
                 }
-                builder.setContent(StringUtils.toString(stream));
+                builder.setContent(IOUtils.toString(stream, encodingValue));
                 builder.setContentLength(entity.getContentLength());
                 builder.setIsChunked(entity.isChunked());
             } finally {

restcomm/restcomm.http/src/main/java/org/restcomm/connect/http/client/HttpRequestDescriptor.java

@@ -56,6 +56,7 @@ public HttpRequestDescriptor(final URI uri, final String method,
         }
         final String query = uri.getQuery();
         if (query != null) {
+            //FIXME:should we externalize RVD encoding default?
             final List<NameValuePair> other = URLEncodedUtils.parse(uri, "UTF-8");
             parameters.addAll(other);
         }
@@ -104,6 +105,7 @@ public List<NameValuePair> getParameters() {
     }
 
     public String getParametersAsString() {
+        //FIXME:should we externalize RVD encoding default?
         return URLEncodedUtils.format(parameters, "UTF-8");
     }
 

restcomm/restcomm.sms/src/main/java/org/restcomm/connect/sms/smpp/Smpp.java

@@ -1,5 +1,7 @@
 package org.restcomm.connect.sms.smpp;
 
+import com.cloudhopper.commons.charset.Charset;
+import com.cloudhopper.commons.charset.CharsetUtil;
 import com.cloudhopper.smpp.SmppBindType;
 import com.cloudhopper.smpp.impl.DefaultSmppSession;
 import com.cloudhopper.smpp.type.Address;
@@ -11,6 +13,7 @@
  */
 public class Smpp {
 
+    private static final String DEFAULT_SMPP_ENCODING = "UTF-8";
     private String name;
     private String systemId;
     private String peerIp;
@@ -37,6 +40,9 @@ public class Smpp {
 
     private long enquireLinkDelay;
 
+    private Charset inboundCharacterEncoding;
+    private Charset outboundCharacterEncoding;
+
     // not used as of today, but later we can allow users to stop each SMPP
     private boolean started = true;
 
@@ -45,7 +51,7 @@ public class Smpp {
     public Smpp(String name, String systemId, String peerIp, int peerPort, SmppBindType smppBindType, String password,
             String systemType, byte interfaceVersion, Address address, long connectTimeout, int windowSize,
             long windowWaitTimeout, long requestExpiryTimeout, long windowMonitorInterval, boolean countersEnabled,
-            boolean logBytes, long enquireLinkDelay) {
+            boolean logBytes, long enquireLinkDelay, String incomingCharacterEncoding, String outgoingCharacterEncoding) {
         super();
         this.name = name;
         this.systemId = systemId;
@@ -64,6 +70,16 @@ public Smpp(String name, String systemId, String peerIp, int peerPort, SmppBindT
         this.countersEnabled = countersEnabled;
         this.logBytes = logBytes;
         this.enquireLinkDelay = enquireLinkDelay;
+        try {
+            this.inboundCharacterEncoding = CharsetUtil.map(incomingCharacterEncoding);
+        } catch (Exception e) {
+            this.inboundCharacterEncoding = CharsetUtil.map(DEFAULT_SMPP_ENCODING);
+        }
+        try {
+            this.outboundCharacterEncoding = CharsetUtil.map(outgoingCharacterEncoding);
+        } catch (Exception e) {
+            this.outboundCharacterEncoding = CharsetUtil.map(DEFAULT_SMPP_ENCODING);
+        }
     }
 
     public String getName() {
@@ -202,6 +218,13 @@ public void setEnquireLinkDelay(long enquireLinkDelay) {
         this.enquireLinkDelay = enquireLinkDelay;
     }
 
+    public Charset getInboundDefaultEncoding() {
+        return inboundCharacterEncoding;
+    }
+    public Charset getOutboundDefaultEncoding() {
+        return outboundCharacterEncoding;
+    }
+
     public boolean isStarted() {
         return started;
     }
@@ -228,7 +251,10 @@ public String toString() {
                 + ", interfaceVersion=" + interfaceVersion + ", address=" + address + ", connectTimeout=" + connectTimeout
                 + ", windowSize=" + windowSize + ", windowWaitTimeout=" + windowWaitTimeout + ", requestExpiryTimeout="
                 + requestExpiryTimeout + ", windowMonitorInterval=" + windowMonitorInterval + ", countersEnabled="
-                + countersEnabled + ", logBytes=" + logBytes + ", enquireLinkDelay=" + enquireLinkDelay + "]";
+                + countersEnabled + ", logBytes=" + logBytes + ", enquireLinkDelay=" + enquireLinkDelay
+                + ", inboundCharacterEncoding=" + inboundCharacterEncoding.toString()
+                + ", outboundCharacterEncoding=" + outboundCharacterEncoding.toString()
+                + "]";
     }
 
     @Override
@@ -255,5 +281,4 @@ public boolean equals(Object obj) {
             return false;
         return true;
     }
-
 }

restcomm/restcomm.sms/src/main/java/org/restcomm/connect/sms/smpp/SmppClientOpsThread.java

@@ -25,6 +25,7 @@
 import com.cloudhopper.commons.charset.CharsetUtil;
 import com.cloudhopper.commons.charset.Charset;
 import com.cloudhopper.smpp.PduAsyncResponse;
+import com.cloudhopper.smpp.SmppConstants;
 import com.cloudhopper.smpp.SmppSession;
 import com.cloudhopper.smpp.SmppSessionConfiguration;
 import com.cloudhopper.smpp.SmppSessionHandler;
@@ -59,6 +60,10 @@ public class SmppClientOpsThread implements Runnable {
     private Object waitObject = new Object();
     private final DefaultSmppClient clientBootstrap;
     private static SmppSession getSmppSession;
+    //FIXME: like getSmppSession, this is bad design
+    //this assumes singular SMPP connection all the time,so it works
+    private static Charset outboundEncoding;
+    private static Charset inboundEncoding;
     protected volatile boolean started = true;
     private static int sipPort;
 
@@ -294,6 +299,8 @@ protected class ClientSmppSessionHandler implements SmppSessionHandler {
         public ClientSmppSessionHandler(Smpp esme) {
             super();
             this.esme = esme;
+            inboundEncoding = esme.getInboundDefaultEncoding();
+            outboundEncoding = esme.getOutboundDefaultEncoding();
         }
 
         @Override
@@ -352,18 +359,20 @@ public PduResponse firePduRequestReceived(PduRequest pduRequest) {
 
                 DeliverSm deliverSm = (DeliverSm) pduRequest;
                 try {
-                    String decodedPduMessage = CharsetUtil.CHARSET_MODIFIED_UTF8.decode(deliverSm.getShortMessage());
+                    byte dcs = deliverSm.getDataCoding();
                     String destSmppAddress = deliverSm.getDestAddress().getAddress();
                     String sourceSmppAddress = deliverSm.getSourceAddress().getAddress();
-                    Charset charset;
-                    if (DataCoding.DATA_CODING_UCS2 == deliverSm.getDataCoding()) {
-                        charset = CharsetUtil.CHARSET_UCS_2;
-                    } else {
-                        charset = CharsetUtil.CHARSET_GSM;
+
+                    //this default esme encoding only applies to DCS==0
+                    Charset encoding = esme.getInboundDefaultEncoding();
+                    if(dcs == SmppConstants.DATA_CODING_UCS2) {
+                        encoding = CharsetUtil.CHARSET_UCS_2;
                     }
+
+                    String decodedPduMessage = CharsetUtil.decode(deliverSm.getShortMessage(), encoding);
                     //send received SMPP PDU message to restcomm
                     try {
-                        sendSmppMessageToRestcomm(decodedPduMessage, destSmppAddress, sourceSmppAddress, charset);
+                        sendSmppMessageToRestcomm(decodedPduMessage, destSmppAddress, sourceSmppAddress, encoding);
                     } catch (IOException | ServletException e) {
                         logger.error("Exception while trying to dispatch incoming SMPP message to Restcomm: " + e);
                     }
@@ -429,4 +438,12 @@ public void sendSmppMessageToRestcomm(String smppMessage, String smppTo, String
         SmppInboundMessageEntity smppInboundMessage = new SmppInboundMessageEntity(to, from, inboundMessage, charset);
         smppMessageHandler.tell(smppInboundMessage, null);
     }
+
+    public static Charset getInboundDefaultEncoding() {
+        return inboundEncoding;
+    }
+
+    public static Charset getOutboundDefaultEncoding() {
+        return outboundEncoding;
+    }
 }

restcomm/restcomm.sms/src/main/java/org/restcomm/connect/sms/smpp/SmppMessageHandler.java

@@ -66,6 +66,7 @@
 import org.restcomm.smpp.parameter.TlvSet;
 
 import com.cloudhopper.commons.charset.CharsetUtil;
+import com.cloudhopper.smpp.SmppConstants;
 import com.cloudhopper.smpp.pdu.SubmitSm;
 import com.cloudhopper.smpp.tlv.Tlv;
 import com.cloudhopper.smpp.type.Address;
@@ -357,11 +358,11 @@ public void outbound(SmppOutboundMessageEntity request) throws SmppInvalidArgume
         submit0.setSourceAddress(new Address((byte) smppTonNpiValue, (byte) smppTonNpiValue, request.getSmppFrom()));
         submit0.setDestAddress(new Address((byte) smppTonNpiValue, (byte) smppTonNpiValue, request.getSmppTo()));
         if (CharsetUtil.CHARSET_UCS_2 == request.getSmppEncoding()) {
-            submit0.setDataCoding(DataCoding.DATA_CODING_UCS2);
+            submit0.setDataCoding(SmppConstants.DATA_CODING_UCS2);
             textBytes = CharsetUtil.encode(request.getSmppContent(), CharsetUtil.CHARSET_UCS_2);
         } else {
-            submit0.setDataCoding(DataCoding.DATA_CODING_GSM7);
-            textBytes = CharsetUtil.encode(request.getSmppContent(), request.getSmppEncoding());
+            submit0.setDataCoding(SmppConstants.DATA_CODING_DEFAULT);
+            textBytes = CharsetUtil.encode(request.getSmppContent(), SmppClientOpsThread.getOutboundDefaultEncoding());
         }
 
         //TODO reverted from https://telestax.atlassian.net/browse/RESTCOMM-1595 as it caused SMS loop at SMSC

restcomm/restcomm.sms/src/main/java/org/restcomm/connect/sms/smpp/SmppService.java

@@ -159,10 +159,12 @@ private void initializeSmppConnections() {
             boolean countersEnabled = smppConfiguration.getBoolean("connections.connection(" + count + ").countersenabled");
 
             long enquireLinkDelay = smppConfiguration.getLong("connections.connection(" + count + ").enquirelinkdelay");
+            String inboundCharacterEncoding = smppConfiguration.getString("connections.connection(" + count + ").inboundencoding");
+            String outboundCharacterEncoding = smppConfiguration.getString("connections.connection(" + count + ").outboundencoding");
 
             Smpp smpp = new Smpp(name, systemId, peerIp, peerPort, bindtype, password, systemType, interfaceVersion, address,
                     connectTimeout, windowSize, windowWaitTimeout, requestExpiryTimeout, windowMonitorInterval,
-                    countersEnabled, logBytes, enquireLinkDelay);
+                    countersEnabled, logBytes, enquireLinkDelay, inboundCharacterEncoding, outboundCharacterEncoding);
 
             this.smppList.add(smpp);