Merge pull request #2863 from RestComm/RESTCOMM-1856

jaimecasero · web-flow · commit 6efef1f52cbc · 2018-04-10T00:56:40.000+02:00
RESTCOM-1856: Allow specific inbound and outbound SMPP encoding
diff --git a/restcomm/configuration/config-scripts/as7-config-scripts/restcomm/autoconfig.d/config-restcomm.sh b/restcomm/configuration/config-scripts/as7-config-scripts/restcomm/autoconfig.d/config-restcomm.sh
@@ -492,6 +492,8 @@ configSMPPAccount() {
 	peerPort="$6"
 	sourceMap="$7"
 	destinationMap="$8"
+	inboundEncoding="$9"
+	outboundEncoding="${10}"
 
 
 	sed -i "s|<smpp class=\"org.restcomm.connect.sms.smpp.SmppService\" activateSmppConnection =\".*\">|<smpp class=\"org.restcomm.connect.sms.smpp.SmppService\" activateSmppConnection =\"$activate\">|g" $FILE
@@ -528,6 +530,13 @@ configSMPPAccount() {
 		}" $FILE
 
         sed -i "s|<connection activateAddressMapping=\"false\" sourceAddressMap=\"\" destinationAddressMap=\"\" tonNpiValue=\"1\">|<connection activateAddressMapping=\"false\" sourceAddressMap=\"${sourceMap}\" destinationAddressMap=\"${destinationMap}\" tonNpiValue=\"1\">|" $FILE
+
+        if [ ! -z "${inboundEncoding}" ]; then
+            xmlstarlet ed -L -P -u  "/restcomm/smpp/connections/connection/inboundencoding" -v $inboundEncoding $FILE
+        fi
+        if [ ! -z "${outboundEncoding}" ]; then
+            xmlstarlet ed -L -P -u  "/restcomm/smpp/connections/connection/outboundencoding" -v $outboundEncoding $FILE
+        fi
 		echo 'Configured SMPP Account Details'
 
 	else
@@ -560,6 +569,10 @@ configSMPPAccount() {
 		}" $FILE
 
         sed -i "s|<connection activateAddressMapping=\"false\" sourceAddressMap=\"\" destinationAddressMap=\"\" tonNpiValue=\"1\">|<connection activateAddressMapping=\"false\" sourceAddressMap=\"\" destinationAddressMap=\"\" tonNpiValue=\"1\">|" $FILE
+
+        xmlstarlet ed -L -P -u  "/restcomm/smpp/connections/connection/inboundencoding" -v "" $FILE
+        xmlstarlet ed -L -P -u  "/restcomm/smpp/connections/connection/outboundencoding" -v "" $FILE
+
 		echo 'Configured SMPP Account Details'
 	fi
 }
@@ -880,7 +893,7 @@ configSpeechRecognizer "$ISPEECH_KEY"
 configSpeechSynthesizers
 configTelestaxProxy "$ACTIVE_PROXY" "$TP_LOGIN" "$TP_PASSWORD" "$INSTANCE_ID" "$PROXY_IP" "$SITE_ID"
 configMediaServerManager "$BIND_ADDRESS" "$MS_ADDRESS" "$MEDIASERVER_EXTERNAL_ADDRESS"
-configSMPPAccount "$SMPP_ACTIVATE" "$SMPP_SYSTEM_ID" "$SMPP_PASSWORD" "$SMPP_SYSTEM_TYPE" "$SMPP_PEER_IP" "$SMPP_PEER_PORT" "$SMPP_SOURCE_MAP" "$SMPP_DEST_MAP"
+configSMPPAccount "$SMPP_ACTIVATE" "$SMPP_SYSTEM_ID" "$SMPP_PASSWORD" "$SMPP_SYSTEM_TYPE" "$SMPP_PEER_IP" "$SMPP_PEER_PORT" "$SMPP_SOURCE_MAP" "$SMPP_DEST_MAP" "$SMPP_INBOUND_ENCODING" "$SMPP_OUTBOUND_ENCODING"
 configRestCommURIs
 updateRecordingsPath
 configHypertextPort
diff --git a/restcomm/configuration/config-scripts/as7-config-scripts/restcomm/restcomm.conf b/restcomm/configuration/config-scripts/as7-config-scripts/restcomm/restcomm.conf
@@ -55,6 +55,8 @@ SMPP_PEER_IP=''	        #use IP or DNS name of peer SMPP server
 SMPP_PEER_PORT=''
 SMPP_SOURCE_MAP=''
 SMPP_DEST_MAP=''
+SMPP_INBOUND_ENCODING=''
+SMPP_OUTBOUND_ENCODING=''
 
 # DID Provision provider variable declarations
 PROVISION_PROVIDER=''               # values: VI (VoipInnovation), BW (Bandwidth), NX (Nexmo), VB (Voxbone)
diff --git a/restcomm/restcomm.application/src/main/webapp/WEB-INF/conf/restcomm.xml b/restcomm/restcomm.application/src/main/webapp/WEB-INF/conf/restcomm.xml
@@ -672,6 +672,10 @@
 					If no response received for 3 consecutive requests, connection will be killed
 					and attempted to connect again -->
 				<enquirelinkdelay>30000</enquirelinkdelay>
+				<!-- when no values are set, default value from code is MODIFIED-UTF8 inbound, GSM7 for outbound-->
+				<inboundencoding/>
+				<outboundencoding/>
+
 			</connection>
 		</connections>
 	</smpp>
diff --git a/restcomm/restcomm.docs/sources-asciidoc/src/main/asciidoc/configuration/Restcomm- Automatic Configuration Options.adoc b/restcomm/restcomm.docs/sources-asciidoc/src/main/asciidoc/configuration/Restcomm- Automatic Configuration Options.adoc
@@ -75,6 +75,8 @@ SMPP_PEER_IP=''	#use IP or DNS name of peer SMPP server
 SMPP_PEER_PORT=''
 SMPP_SOURCE_MAP=''
 SMPP_DEST_MAP=''
+SMPP_INBOUND_ENCODING=''
+SMPP_OUTBOUND_ENCODING=''
 ----
 
 * Section *#Restcomm DID provider integration*.
diff --git a/restcomm/restcomm.http/src/main/java/org/restcomm/connect/http/client/Downloader.java b/restcomm/restcomm.http/src/main/java/org/restcomm/connect/http/client/Downloader.java
@@ -22,6 +22,8 @@
 import akka.actor.ActorRef;
 import akka.event.Logging;
 import akka.event.LoggingAdapter;
+
+import org.apache.commons.io.IOUtils;
 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpHeaders;
@@ -39,7 +41,6 @@
 import org.restcomm.connect.commons.common.http.CustomHttpClientBuilder;
 import org.restcomm.connect.commons.configuration.RestcommConfiguration;
 import org.restcomm.connect.commons.faulttolerance.RestcommUntypedActor;
-import org.restcomm.connect.commons.util.StringUtils;
 import org.xml.sax.InputSource;
 
 import javax.xml.parsers.DocumentBuilder;
@@ -86,6 +87,7 @@ public HttpResponseDescriptor fetch (final HttpRequestDescriptor descriptor) thr
         try {
             do {
                 request = request(temp);
+                //FIXME:should we externalize RVD encoding default?
                 request.setHeader("http.protocol.content-charset", "UTF-8");
                 if (descriptor.getTimeout() > 0){
                     HttpContext httpContext = new BasicHttpContext();
@@ -161,6 +163,7 @@ private HttpResponseDescriptor validateXML (final HttpResponseDescriptor descrip
                 // parse an XML document into a DOM tree
                 String xml = descriptor.getContentAsString().trim().replaceAll("&([^;]+(?!(?:\\w|;)))", "&amp;$1");
                 DocumentBuilder parser = DocumentBuilderFactory.newInstance().newDocumentBuilder();
+                //FIXME:should we externalize RVD encoding default?
                 parser.parse(new InputSource(new ByteArrayInputStream(xml.getBytes("utf-8"))));
                 return descriptor;
             } catch (final Exception e) {
@@ -218,6 +221,7 @@ public HttpUriRequest request (final HttpRequestDescriptor descriptor) throws Il
         } else if ("POST".equalsIgnoreCase(method)) {
             final List<NameValuePair> parameters = descriptor.getParameters();
             final HttpPost post = new HttpPost(uri);
+            //FIXME:should we externalize RVD encoding default?
             post.setEntity(new UrlEncodedFormEntity(parameters, "UTF-8"));
             return post;
         } else {
@@ -237,14 +241,17 @@ private HttpResponseDescriptor response (final HttpRequest request, final HttpRe
             InputStream stream = entity.getContent();
             try {
                 final Header contentEncoding = entity.getContentEncoding();
-                if (contentEncoding != null) {
-                    builder.setContentEncoding(contentEncoding.getValue());
+                //FIXME:should we externalize RVD encoding default?
+                String encodingValue = "UTF-8";
+                if (contentEncoding != null && !contentEncoding.getValue().isEmpty()) {
+                    encodingValue = contentEncoding.getValue();
+                    builder.setContentEncoding(encodingValue);
                 }
                 final Header contentType = entity.getContentType();
                 if (contentType != null) {
                     builder.setContentType(contentType.getValue());
                 }
-                builder.setContent(StringUtils.toString(stream));
+                builder.setContent(IOUtils.toString(stream, encodingValue));
                 builder.setContentLength(entity.getContentLength());
                 builder.setIsChunked(entity.isChunked());
             } finally {
diff --git a/restcomm/restcomm.http/src/main/java/org/restcomm/connect/http/client/HttpRequestDescriptor.java b/restcomm/restcomm.http/src/main/java/org/restcomm/connect/http/client/HttpRequestDescriptor.java
@@ -56,6 +56,7 @@ public HttpRequestDescriptor(final URI uri, final String method,
         }
         final String query = uri.getQuery();
         if (query != null) {
+            //FIXME:should we externalize RVD encoding default?
             final List<NameValuePair> other = URLEncodedUtils.parse(uri, "UTF-8");
             parameters.addAll(other);
         }
@@ -104,6 +105,7 @@ public List<NameValuePair> getParameters() {
     }
 
     public String getParametersAsString() {
+        //FIXME:should we externalize RVD encoding default?
         return URLEncodedUtils.format(parameters, "UTF-8");
     }
 
diff --git a/restcomm/restcomm.sms/src/main/java/org/restcomm/connect/sms/smpp/Smpp.java b/restcomm/restcomm.sms/src/main/java/org/restcomm/connect/sms/smpp/Smpp.java
@@ -1,5 +1,9 @@
 package org.restcomm.connect.sms.smpp;
 
+import org.apache.log4j.Logger;
+
+import com.cloudhopper.commons.charset.Charset;
+import com.cloudhopper.commons.charset.CharsetUtil;
 import com.cloudhopper.smpp.SmppBindType;
 import com.cloudhopper.smpp.impl.DefaultSmppSession;
 import com.cloudhopper.smpp.type.Address;
@@ -11,6 +15,10 @@
  */
 public class Smpp {
 
+    private static final Logger logger = Logger.getLogger(Smpp.class);
+
+    private static final String DEFAULT_SMPP_INBOUND_ENCODING = "MODIFIED-UTF8";
+    private static final String DEFAULT_SMPP_OUTBOUND_ENCODING = "GSM7";
     private String name;
     private String systemId;
     private String peerIp;
@@ -37,6 +45,9 @@ public class Smpp {
 
     private long enquireLinkDelay;
 
+    private Charset inboundCharacterEncoding;
+    private Charset outboundCharacterEncoding;
+
     // not used as of today, but later we can allow users to stop each SMPP
     private boolean started = true;
 
@@ -45,7 +56,7 @@ public class Smpp {
     public Smpp(String name, String systemId, String peerIp, int peerPort, SmppBindType smppBindType, String password,
             String systemType, byte interfaceVersion, Address address, long connectTimeout, int windowSize,
             long windowWaitTimeout, long requestExpiryTimeout, long windowMonitorInterval, boolean countersEnabled,
-            boolean logBytes, long enquireLinkDelay) {
+            boolean logBytes, long enquireLinkDelay, String inboundCharacterEncoding, String outboundCharacterEncoding) {
         super();
         this.name = name;
         this.systemId = systemId;
@@ -64,6 +75,22 @@ public Smpp(String name, String systemId, String peerIp, int peerPort, SmppBindT
         this.countersEnabled = countersEnabled;
         this.logBytes = logBytes;
         this.enquireLinkDelay = enquireLinkDelay;
+        try {
+            this.inboundCharacterEncoding = CharsetUtil.map(inboundCharacterEncoding);
+        } catch (Exception e) {
+            this.inboundCharacterEncoding = CharsetUtil.map(DEFAULT_SMPP_INBOUND_ENCODING);
+            if(logger.isInfoEnabled()) {
+                logger.info("Charset " + inboundCharacterEncoding + " does not exist. Inbound encoding is set to default " + DEFAULT_SMPP_INBOUND_ENCODING + "\n" + e.getMessage());
+            }
+        }
+        try {
+            this.outboundCharacterEncoding = CharsetUtil.map(outboundCharacterEncoding);
+        } catch (Exception e) {
+            this.outboundCharacterEncoding = CharsetUtil.map(DEFAULT_SMPP_OUTBOUND_ENCODING);
+            if(logger.isInfoEnabled()) {
+                logger.info("Charset " + outboundCharacterEncoding+ " does not exist. Outbound encoding is set to default " + DEFAULT_SMPP_OUTBOUND_ENCODING + "\n" + e.getMessage());
+            }
+        }
     }
 
     public String getName() {
@@ -202,6 +229,13 @@ public void setEnquireLinkDelay(long enquireLinkDelay) {
         this.enquireLinkDelay = enquireLinkDelay;
     }
 
+    public Charset getInboundDefaultEncoding() {
+        return inboundCharacterEncoding;
+    }
+    public Charset getOutboundDefaultEncoding() {
+        return outboundCharacterEncoding;
+    }
+
     public boolean isStarted() {
         return started;
     }
@@ -228,7 +262,10 @@ public String toString() {
                 + ", interfaceVersion=" + interfaceVersion + ", address=" + address + ", connectTimeout=" + connectTimeout
                 + ", windowSize=" + windowSize + ", windowWaitTimeout=" + windowWaitTimeout + ", requestExpiryTimeout="
                 + requestExpiryTimeout + ", windowMonitorInterval=" + windowMonitorInterval + ", countersEnabled="
-                + countersEnabled + ", logBytes=" + logBytes + ", enquireLinkDelay=" + enquireLinkDelay + "]";
+                + countersEnabled + ", logBytes=" + logBytes + ", enquireLinkDelay=" + enquireLinkDelay
+                + ", inboundCharacterEncoding=" + inboundCharacterEncoding.toString()
+                + ", outboundCharacterEncoding=" + outboundCharacterEncoding.toString()
+                + "]";
     }
 
     @Override
@@ -255,5 +292,4 @@ public boolean equals(Object obj) {
             return false;
         return true;
     }
-
 }
diff --git a/restcomm/restcomm.sms/src/main/java/org/restcomm/connect/sms/smpp/SmppClientOpsThread.java b/restcomm/restcomm.sms/src/main/java/org/restcomm/connect/sms/smpp/SmppClientOpsThread.java
@@ -25,6 +25,7 @@
 import com.cloudhopper.commons.charset.CharsetUtil;
 import com.cloudhopper.commons.charset.Charset;
 import com.cloudhopper.smpp.PduAsyncResponse;
+import com.cloudhopper.smpp.SmppConstants;
 import com.cloudhopper.smpp.SmppSession;
 import com.cloudhopper.smpp.SmppSessionConfiguration;
 import com.cloudhopper.smpp.SmppSessionHandler;
@@ -59,6 +60,10 @@ public class SmppClientOpsThread implements Runnable {
     private Object waitObject = new Object();
     private final DefaultSmppClient clientBootstrap;
     private static SmppSession getSmppSession;
+    //FIXME: like getSmppSession, this is bad design
+    //this assumes singular SMPP connection all the time,so it works
+    private static Charset outboundEncoding;
+    private static Charset inboundEncoding;
     protected volatile boolean started = true;
     private static int sipPort;
 
@@ -294,6 +299,8 @@ protected class ClientSmppSessionHandler implements SmppSessionHandler {
         public ClientSmppSessionHandler(Smpp esme) {
             super();
             this.esme = esme;
+            inboundEncoding = esme.getInboundDefaultEncoding();
+            outboundEncoding = esme.getOutboundDefaultEncoding();
         }
 
         @Override
@@ -352,18 +359,20 @@ public PduResponse firePduRequestReceived(PduRequest pduRequest) {
 
                 DeliverSm deliverSm = (DeliverSm) pduRequest;
                 try {
-                    String decodedPduMessage = CharsetUtil.CHARSET_MODIFIED_UTF8.decode(deliverSm.getShortMessage());
+                    byte dcs = deliverSm.getDataCoding();
                     String destSmppAddress = deliverSm.getDestAddress().getAddress();
                     String sourceSmppAddress = deliverSm.getSourceAddress().getAddress();
-                    Charset charset;
-                    if (DataCoding.DATA_CODING_UCS2 == deliverSm.getDataCoding()) {
-                        charset = CharsetUtil.CHARSET_UCS_2;
-                    } else {
-                        charset = CharsetUtil.CHARSET_GSM;
+
+                    //this default esme encoding only applies to DCS==0
+                    Charset encoding = esme.getInboundDefaultEncoding();
+                    if(dcs == SmppConstants.DATA_CODING_UCS2) {
+                        encoding = CharsetUtil.CHARSET_UCS_2;
                     }
+
+                    String decodedPduMessage = CharsetUtil.decode(deliverSm.getShortMessage(), encoding);
                     //send received SMPP PDU message to restcomm
                     try {
-                        sendSmppMessageToRestcomm(decodedPduMessage, destSmppAddress, sourceSmppAddress, charset);
+                        sendSmppMessageToRestcomm(decodedPduMessage, destSmppAddress, sourceSmppAddress, encoding);
                     } catch (IOException | ServletException e) {
                         logger.error("Exception while trying to dispatch incoming SMPP message to Restcomm: " + e);
                     }
@@ -429,4 +438,12 @@ public void sendSmppMessageToRestcomm(String smppMessage, String smppTo, String
         SmppInboundMessageEntity smppInboundMessage = new SmppInboundMessageEntity(to, from, inboundMessage, charset);
         smppMessageHandler.tell(smppInboundMessage, null);
     }
+
+    public static Charset getInboundDefaultEncoding() {
+        return inboundEncoding;
+    }
+
+    public static Charset getOutboundDefaultEncoding() {
+        return outboundEncoding;
+    }
 }
diff --git a/restcomm/restcomm.sms/src/main/java/org/restcomm/connect/sms/smpp/SmppMessageHandler.java b/restcomm/restcomm.sms/src/main/java/org/restcomm/connect/sms/smpp/SmppMessageHandler.java
@@ -66,6 +66,7 @@
 import org.restcomm.smpp.parameter.TlvSet;
 
 import com.cloudhopper.commons.charset.CharsetUtil;
+import com.cloudhopper.smpp.SmppConstants;
 import com.cloudhopper.smpp.pdu.SubmitSm;
 import com.cloudhopper.smpp.tlv.Tlv;
 import com.cloudhopper.smpp.type.Address;
@@ -357,11 +358,11 @@ public void outbound(SmppOutboundMessageEntity request) throws SmppInvalidArgume
         submit0.setSourceAddress(new Address((byte) smppTonNpiValue, (byte) smppTonNpiValue, request.getSmppFrom()));
         submit0.setDestAddress(new Address((byte) smppTonNpiValue, (byte) smppTonNpiValue, request.getSmppTo()));
         if (CharsetUtil.CHARSET_UCS_2 == request.getSmppEncoding()) {
-            submit0.setDataCoding(DataCoding.DATA_CODING_UCS2);
+            submit0.setDataCoding(SmppConstants.DATA_CODING_UCS2);
             textBytes = CharsetUtil.encode(request.getSmppContent(), CharsetUtil.CHARSET_UCS_2);
         } else {
-            submit0.setDataCoding(DataCoding.DATA_CODING_GSM7);
-            textBytes = CharsetUtil.encode(request.getSmppContent(), request.getSmppEncoding());
+            submit0.setDataCoding(SmppConstants.DATA_CODING_DEFAULT);
+            textBytes = CharsetUtil.encode(request.getSmppContent(), SmppClientOpsThread.getOutboundDefaultEncoding());
         }
 
         //TODO reverted from https://telestax.atlassian.net/browse/RESTCOMM-1595 as it caused SMS loop at SMSC
diff --git a/restcomm/restcomm.sms/src/main/java/org/restcomm/connect/sms/smpp/SmppService.java b/restcomm/restcomm.sms/src/main/java/org/restcomm/connect/sms/smpp/SmppService.java
@@ -159,10 +159,12 @@ private void initializeSmppConnections() {
             boolean countersEnabled = smppConfiguration.getBoolean("connections.connection(" + count + ").countersenabled");
 
             long enquireLinkDelay = smppConfiguration.getLong("connections.connection(" + count + ").enquirelinkdelay");
+            String inboundCharacterEncoding = smppConfiguration.getString("connections.connection(" + count + ").inboundencoding");
+            String outboundCharacterEncoding = smppConfiguration.getString("connections.connection(" + count + ").outboundencoding");
 
             Smpp smpp = new Smpp(name, systemId, peerIp, peerPort, bindtype, password, systemType, interfaceVersion, address,
                     connectTimeout, windowSize, windowWaitTimeout, requestExpiryTimeout, windowMonitorInterval,
-                    countersEnabled, logBytes, enquireLinkDelay);
+                    countersEnabled, logBytes, enquireLinkDelay, inboundCharacterEncoding, outboundCharacterEncoding);
 
             this.smppList.add(smpp);
 
diff --git a/restcomm/restcomm.testsuite/src/main/java/org/restcomm/connect/testsuite/smpp/MockSmppServer.java b/restcomm/restcomm.testsuite/src/main/java/org/restcomm/connect/testsuite/smpp/MockSmppServer.java
diff --git a/restcomm/restcomm.testsuite/src/test/java/org/restcomm/connect/testsuite/smpp/SmppTest.java b/restcomm/restcomm.testsuite/src/test/java/org/restcomm/connect/testsuite/smpp/SmppTest.java
diff --git a/restcomm/restcomm.testsuite/src/test/resources/restcomm-smpp.xml b/restcomm/restcomm.testsuite/src/test/resources/restcomm-smpp.xml

Original file line number	Diff line number	Diff line change
`@@ -56,6 +56,7 @@ public HttpRequestDescriptor(final URI uri, final String method,`
`56`	`56`	`}`
`57`	`57`	`final String query = uri.getQuery();`
`58`	`58`	`if (query != null) {`
	`59`	`+ //FIXME:should we externalize RVD encoding default?`
`59`	`60`	`final List<NameValuePair> other = URLEncodedUtils.parse(uri, "UTF-8");`
`60`	`61`	`parameters.addAll(other);`
`61`	`62`	`}`
`@@ -104,6 +105,7 @@ public List<NameValuePair> getParameters() {`
`104`	`105`	`}`
`105`	`106`
`106`	`107`	`public String getParametersAsString() {`
	`108`	`+ //FIXME:should we externalize RVD encoding default?`
`107`	`109`	`return URLEncodedUtils.format(parameters, "UTF-8");`
`108`	`110`	`}`
`109`	`111`