Merge branch 'master' of https://github.com/RestComm/RestComm-Connect into FS-56_change_rvd_context

Author: Orestis Tsakiridis

Jenkinsfile

@@ -9,9 +9,9 @@ def buildRC() {
         try {
             if (env.BRANCH_NAME == 'master') {
                 //do sonar just in master
-                sh "mvn -f restcomm/pom.xml -pl \\!restcomm.testsuite -Dmaven.test.redirectTestOutputToFile=true -Dsonar.host.url=https://sonarqube.com -Dsonar.login=dd43f79a4bd32b1f2c484362e8a4de676a8388c4 -Dsonar.organization=jaimecasero-github -Dsonar.branch=master install sonar:sonar"
+                sh "mvn -f restcomm/pom.xml -pl \\!restcomm.testsuite -Dmaven.test.redirectTestOutputToFile=true -Dsonar.host.url=https://sonarqube.com -Dsonar.login=dd43f79a4bd32b1f2c484362e8a4de676a8388c4 -Dsonar.organization=jaimecasero-github -Dsonar.branch=master clean install sonar:sonar"
             } else {
-                sh "mvn -f restcomm/pom.xml -pl \\!restcomm.testsuite -Dmaven.test.redirectTestOutputToFile=true install"
+                sh "mvn -f restcomm/pom.xml -pl \\!restcomm.testsuite -Dmaven.test.redirectTestOutputToFile=true clean install"
             }
         } catch(err) {
             publishRCResults()

restcomm/configuration/config-scripts/as7-config-scripts/restcomm/advanced.conf

@@ -149,6 +149,8 @@ HTTP_RESPONSE_TIMEOUT=6000
 
 # If set to true RestComm will NOT use cache for *.wav files playback.If set to false RestComm will use cache for *.wav files playback.
 CACHE_NO_WAV=false
+#Path in FS to where the cache will be hosted.
+CACHE_PATH=${restcomm:home}/cache
 
 #MSS Configuration
 TLS_CLIENT_AUTH_TYPE="Disabled" #Possible values Enabled, Want, Disabled,DisabledAll

restcomm/configuration/config-scripts/as7-config-scripts/restcomm/autoconfig.d/config-restcomm.sh

@@ -607,11 +607,14 @@ configRestCommURIs() {
 		REMOTE_ADDRESS="${SCHEME}://${PUBLIC_IP}:${PORT}"
 
 		sed -i "s|<prompts-uri>.*</prompts-uri>|<prompts-uri>$REMOTE_ADDRESS/restcomm/audio<\/prompts-uri>|" $FILE
-		sed -i "s|<cache-uri>.*/cache-uri>|<cache-uri>$REMOTE_ADDRESS/restcomm/cache</cache-uri>|" $FILE
+		sed -i "s|<cache-uri>.*</cache-uri>|<cache-uri>$REMOTE_ADDRESS/restcomm/cache</cache-uri>|" $FILE
+		sed -i "s|<cache-path>.*</cache-path>|<cache-path>$CACHE_PATH</cache-path>|" $FILE
 		sed -i "s|<error-dictionary-uri>.*</error-dictionary-uri>|<error-dictionary-uri>$REMOTE_ADDRESS/restcomm/errors</error-dictionary-uri>|" $FILE
 
+                #Equivalent changes in case XML tags are empty after empty values
 		sed -i "s|<prompts-uri/>|<prompts-uri>$REMOTE_ADDRESS/restcomm/audio<\/prompts-uri>|" $FILE
 		sed -i "s|<cache-uri/>|<cache-uri>$REMOTE_ADDRESS/restcomm/cache</cache-uri>|" $FILE
+                sed -i "s|<cache-path/>|<cache-path>$CACHE_PATH</cache-path>|" $FILE
 		sed -i "s|<error-dictionary-uri/>|<error-dictionary-uri>$REMOTE_ADDRESS/restcomm/errors</error-dictionary-uri>|" $FILE
 
 		echo "Updated prompts-uri cache-uri error-dictionary-uri External MSaddress for "

restcomm/restcomm.application/src/main/java/org/restcomm/connect/application/Bootstrapper.java

@@ -30,6 +30,7 @@
 import org.mobicents.servlet.sip.SipConnector;
 import org.restcomm.connect.application.config.ConfigurationStringLookup;
 import org.restcomm.connect.commons.Version;
+import org.restcomm.connect.commons.amazonS3.S3AccessTool;
 import org.restcomm.connect.commons.common.http.CustomHttpClientBuilder;
 import org.restcomm.connect.commons.configuration.RestcommConfiguration;
 import org.restcomm.connect.commons.dao.Sid;
@@ -232,10 +233,10 @@ private String home(final ServletContext context) {
         }
     }
 
-    private DaoManager storage(final Configuration configuration, Configuration daoManagerConfiguration, final ClassLoader loader, final ExecutionContext ec) throws ObjectInstantiationException {
+    private DaoManager storage(final Configuration configuration, Configuration daoManagerConfiguration, S3AccessTool s3AccessTool, final ClassLoader loader, final ExecutionContext ec) throws ObjectInstantiationException {
         final String classpath = daoManagerConfiguration.getString("dao-manager[@class]");
         final DaoManager daoManager = (DaoManager) new ObjectFactory(loader).getObjectInstance(classpath);
-        daoManager.configure(configuration, daoManagerConfiguration, ec);
+        daoManager.configure(configuration, daoManagerConfiguration, s3AccessTool, ec);
         daoManager.start();
         return daoManager;
     }
@@ -335,6 +336,27 @@ private void generateDefaultProfile(final DaoManager storage, final String profi
         }
     }
 
+    private S3AccessTool prepareS3AccessTool(Configuration configuration) {
+        Configuration amazonS3Configuration = configuration.subset("amazon-s3");
+        if (!amazonS3Configuration.isEmpty()) { // Do not fail with NPE is amazonS3Configuration is not present for older install
+            boolean amazonS3Enabled = amazonS3Configuration.getBoolean("enabled");
+            if (amazonS3Enabled) {
+                final String accessKey = amazonS3Configuration.getString("access-key");
+                final String securityKey = amazonS3Configuration.getString("security-key");
+                final String bucketName = amazonS3Configuration.getString("bucket-name");
+                final String folder = amazonS3Configuration.getString("folder");
+                final boolean reducedRedundancy = amazonS3Configuration.getBoolean("reduced-redundancy");
+                final int minutesToRetainPublicUrl = amazonS3Configuration.getInt("minutes-to-retain-public-url", 10);
+                final boolean removeOriginalFile = amazonS3Configuration.getBoolean("remove-original-file");
+                final String bucketRegion = amazonS3Configuration.getString("bucket-region");
+                final boolean testing = amazonS3Configuration.getBoolean("testing", false);
+                final String testingUrl = amazonS3Configuration.getString("testing-url", null);
+                return new S3AccessTool(accessKey, securityKey, bucketName, folder, reducedRedundancy, minutesToRetainPublicUrl, removeOriginalFile, bucketRegion, testing, testingUrl);
+            }
+        }
+        return null;
+    }
+
     @Override
     public void servletInitialized(SipServletContextEvent event) {
         if (event.getSipServlet().getClass().equals(Bootstrapper.class)) {
@@ -387,10 +409,14 @@ public void servletInitialized(SipServletContextEvent event) {
             // Share the actor system with other servlets.
             context.setAttribute(ActorSystem.class.getName(), system);
             ec = system.dispatchers().lookup("restcomm-blocking-dispatcher");
+
+            S3AccessTool s3AccessTool = prepareS3AccessTool(xml);
+            context.setAttribute(S3AccessTool.class.getName(), s3AccessTool);
+
             // Create the storage system.
             DaoManager storage = null;
             try {
-                storage = storage(xml, daoManagerConf, loader, ec);
+                storage = storage(xml, daoManagerConf, s3AccessTool, loader, ec);
             } catch (final ObjectInstantiationException exception) {
                 logger.error("ObjectInstantiationException during initialization: ", exception);
             }

restcomm/restcomm.application/src/main/java/org/restcomm/connect/application/RestcommConnectServiceProvider.java

@@ -22,12 +22,15 @@
 
 import javax.servlet.ServletContext;
 
+import org.restcomm.connect.commons.amazonS3.S3AccessTool;
 import org.restcomm.connect.core.service.api.ClientPasswordHashingService;
 import org.restcomm.connect.core.service.api.NumberSelectorService;
 import org.restcomm.connect.core.service.api.ProfileService;
+import org.restcomm.connect.core.service.api.RecordingService;
 import org.restcomm.connect.core.service.client.ClientPasswordHashingServiceImpl;
 import org.restcomm.connect.core.service.number.NumberSelectorServiceImpl;
 import org.restcomm.connect.core.service.profile.ProfileServiceImpl;
+import org.restcomm.connect.core.service.recording.RecordingsServiceImpl;
 import org.restcomm.connect.dao.DaoManager;
 
 /**
@@ -41,6 +44,7 @@ public class RestcommConnectServiceProvider {
     private NumberSelectorService numberSelector;
     private ProfileService profileService;
     private ClientPasswordHashingService clientPasswordHashingService;
+    private RecordingService recordingService;
 
     public static RestcommConnectServiceProvider getInstance() {
         if (instance == null) {
@@ -61,6 +65,11 @@ public void startServices(ServletContext ctx) {
         ctx.setAttribute(ProfileService.class.getName(), profileService);
         this.clientPasswordHashingService = new ClientPasswordHashingServiceImpl(daoManager);
         ctx.setAttribute(ClientPasswordHashingService.class.getName(), clientPasswordHashingService);
+
+        S3AccessTool s3AccessTool = (S3AccessTool) ctx.getAttribute(S3AccessTool.class.getName());
+
+        this.recordingService = new RecordingsServiceImpl(daoManager.getRecordingsDao(), s3AccessTool);
+        ctx.setAttribute(RecordingService.class.getName(), recordingService);
     }
 
     /**
@@ -77,6 +86,14 @@ public ProfileService provideProfileService() {
         return profileService;
     }
 
+    /**
+     * @return
+     */
     public ClientPasswordHashingService clientPasswordHashingService() { return clientPasswordHashingService; }
 
+    /**
+     * @return
+     */
+    public RecordingService recordingService() { return recordingService; }
+
 }

restcomm/restcomm.application/src/main/webapp/WEB-INF/data/hsql/restcomm.script

@@ -13,7 +13,7 @@ CREATE MEMORY TABLE "restcomm_conference_detail_records" ( "sid" VARCHAR(34) NOT
 CREATE MEMORY TABLE "restcomm_clients"("sid" VARCHAR(34) NOT NULL PRIMARY KEY,"date_created" DATETIME NOT NULL,"date_updated" DATETIME NOT NULL,"account_sid" VARCHAR(34) NOT NULL,"api_version" VARCHAR(10) NOT NULL,"friendly_name" VARCHAR(64) NOT NULL,"login" VARCHAR(64) NOT NULL,"password" VARCHAR(64) NOT NULL,"status" INTEGER NOT NULL,"voice_url" LONGVARCHAR,"voice_method" VARCHAR(4),"voice_fallback_url" LONGVARCHAR,"voice_fallback_method" VARCHAR(4),"voice_application_sid" VARCHAR(34),"uri" LONGVARCHAR NOT NULL, "push_client_identity" VARCHAR(34), "password_algorithm" VARCHAR(34))
 CREATE MEMORY TABLE "restcomm_registrations"("sid" VARCHAR(34) NOT NULL PRIMARY KEY,"date_created" DATETIME NOT NULL,"date_updated" DATETIME NOT NULL,"date_expires" DATETIME NOT NULL,"address_of_record" LONGVARCHAR NOT NULL,"display_name" VARCHAR(255),"user_name" VARCHAR(64) NOT NULL,"user_agent" LONGVARCHAR,"ttl" INTEGER NOT NULL,"location" LONGVARCHAR NOT NULL, "webrtc" BOOLEAN DEFAULT FALSE, "instanceid" VARCHAR(255), "isLBPresent" BOOLEAN DEFAULT FALSE, "organization_sid" VARCHAR(34))
 CREATE MEMORY TABLE "restcomm_short_codes"("sid" VARCHAR(34) NOT NULL PRIMARY KEY,"date_created" DATETIME NOT NULL,"date_updated" DATETIME NOT NULL,"friendly_name" VARCHAR(64) NOT NULL,"account_sid" VARCHAR(34) NOT NULL,"short_code" INTEGER NOT NULL,"api_version" VARCHAR(10) NOT NULL,"sms_url" LONGVARCHAR,"sms_method" VARCHAR(4),"sms_fallback_url" LONGVARCHAR,"sms_fallback_method" VARCHAR(4),"uri" LONGVARCHAR NOT NULL)
-CREATE MEMORY TABLE "restcomm_sms_messages"("sid" VARCHAR(34) NOT NULL PRIMARY KEY,"date_created" DATETIME NOT NULL,"date_updated" DATETIME NOT NULL,"date_sent" DATETIME,"account_sid" VARCHAR(34) NOT NULL,"sender" VARCHAR(255) NOT NULL,"recipient" VARCHAR(64) NOT NULL,"body" VARCHAR(999) NOT NULL,"status" VARCHAR(20) NOT NULL,"direction" VARCHAR(14) NOT NULL,"price" VARCHAR(8) NOT NULL,"api_version" VARCHAR(10) NOT NULL,"uri" LONGVARCHAR NOT NULL, "smpp_message_id" VARCHAR(1000))
+CREATE MEMORY TABLE "restcomm_sms_messages"("sid" VARCHAR(34) NOT NULL PRIMARY KEY,"date_created" DATETIME NOT NULL,"date_updated" DATETIME NOT NULL,"date_sent" DATETIME,"account_sid" VARCHAR(34) NOT NULL,"sender" VARCHAR(255) NOT NULL,"recipient" VARCHAR(64) NOT NULL,"body" VARCHAR(999) NOT NULL,"status" VARCHAR(20) NOT NULL,"direction" VARCHAR(14) NOT NULL,"price" VARCHAR(8) NOT NULL,"api_version" VARCHAR(10) NOT NULL,"uri" LONGVARCHAR NOT NULL, "smpp_message_id" VARCHAR(1000), "error_code" INT, "status_callback" LONGVARCHAR, "status_callback_method" LONGVARCHAR)
 CREATE MEMORY TABLE "restcomm_recordings"("sid" VARCHAR(34) NOT NULL PRIMARY KEY,"date_created" DATETIME NOT NULL,"date_updated" DATETIME NOT NULL,"account_sid" VARCHAR(34) NOT NULL,"call_sid" VARCHAR(1000) NOT NULL,"duration" DOUBLE NOT NULL,"api_version" VARCHAR(10) NOT NULL,"uri" LONGVARCHAR NOT NULL, "file_uri" LONGVARCHAR, "s3_uri" LONGVARCHAR)
 CREATE MEMORY TABLE "restcomm_transcriptions"("sid" VARCHAR(34) NOT NULL PRIMARY KEY,"date_created" DATETIME NOT NULL,"date_updated" DATETIME NOT NULL,"account_sid" VARCHAR(34) NOT NULL,"status" VARCHAR(11) NOT NULL,"recording_sid" VARCHAR(34) NOT NULL,"duration" DOUBLE NOT NULL,"transcription_text" LONGVARCHAR,"price" VARCHAR(8) NOT NULL,"uri" LONGVARCHAR NOT NULL)
 CREATE MEMORY TABLE "restcomm_notifications"("sid" VARCHAR(34) NOT NULL PRIMARY KEY,"date_created" DATETIME NOT NULL,"date_updated" DATETIME NOT NULL,"account_sid" VARCHAR(34) NOT NULL,"call_sid" VARCHAR(1000),"api_version" VARCHAR(10) NOT NULL,"log" TINYINT NOT NULL,"error_code" SMALLINT NOT NULL,"more_info" LONGVARCHAR NOT NULL,"message_text" LONGVARCHAR NOT NULL,"message_date" DATETIME NOT NULL,"request_url" LONGVARCHAR NOT NULL,"request_method" VARCHAR(4) NOT NULL,"request_variables" LONGVARCHAR NOT NULL,"response_headers" LONGVARCHAR,"response_body" LONGVARCHAR,"uri" LONGVARCHAR NOT NULL)

restcomm/restcomm.application/src/main/webapp/WEB-INF/scripts/mariadb/init.sql

@@ -249,7 +249,10 @@ direction VARCHAR(14) NOT NULL,
 price VARCHAR(8) NOT NULL,
 api_version VARCHAR(10) NOT NULL,
 uri MEDIUMTEXT NOT NULL,
-smpp_message_id MEDIUMTEXT
+smpp_message_id MEDIUMTEXT,
+status_callback MEDIUMTEXT,
+status_callback_method MEDIUMTEXT,
+error_code BIGINT
 );
 
 CREATE TABLE restcomm_recordings (
@@ -466,6 +469,9 @@ CREATE INDEX idx_cdr_conference_sid ON restcomm_call_detail_records (conference_
 /* Create index on restcomm_call_detail_records on conference_sid column */
 CREATE INDEX idx_cdr_conference_status ON restcomm_conference_detail_records (status);
 
+/* Create index on restcomm_sms_messages on smpp_message_id column */
+CREATE INDEX idx_restcomm_sms_messages_smpp_message_id ON restcomm_sms_messages (smpp_message_id);
+
 DELIMITER //
 DROP PROCEDURE IF EXISTS addConferenceDetailRecord;
 CREATE PROCEDURE addConferenceDetailRecord(	IN in_sid VARCHAR(34),

restcomm/restcomm.application/src/main/webapp/WEB-INF/scripts/mariadb/sql/sms-messages.xml

@@ -6,8 +6,9 @@
 <mapper namespace="org.mobicents.servlet.sip.restcomm.dao.SmsMessagesDao">
   <insert id="addSmsMessage" parameterType="map">
     INSERT INTO restcomm_sms_messages (sid, date_created, date_updated, date_sent, account_sid, sender, recipient, body, status, direction,
-		price, api_version, uri) VALUES (#{sid}, #{date_created}, #{date_updated}, #{date_sent}, #{account_sid}, #{sender}, #{recipient}, #{body},
-    #{status}, #{direction}, #{price}, #{api_version}, #{uri});
+		price, api_version, uri, status_callback, status_callback_method) VALUES (#{sid}, #{date_created}, #{date_updated}, #{date_sent}, #{account_sid}, #{sender}, #{recipient}, #{body},
+    #{status}, #{direction}, #{price}, #{api_version}, #{uri}, #{status_callback)},
+      #{status_callback_method)});
   </insert>
 
   <select id="getSmsMessage" parameterType="string" resultType="hashmap">
@@ -31,7 +32,7 @@
   </delete>
 
   <update id="updateSmsMessage" parameterType="map">
-    UPDATE restcomm_sms_messages SET date_sent=#{date_sent}, status=#{status}, price=#{price}, smpp_message_id=#{smpp_message_id}, date_updated=NOW() WHERE sid=#{sid};
+    UPDATE restcomm_sms_messages SET date_sent=#{date_sent}, status=#{status}, price=#{price}, smpp_message_id=#{smpp_message_id}, error_code=#{error_code}, date_updated=NOW() WHERE sid=#{sid};
   </update>
 
   <select id="getSmsMessagesPerAccountLastPerMinute" parameterType="map" resultType="int">
@@ -117,9 +118,14 @@
                 AND date_created &gt;= #{startTime}
             </if>
             <if test="endTime != null">
-                    AND date_created &lt;= DATE_ADD(#{endTime},INTERVAL 1 DAY)
+                AND date_created &lt;= DATE_ADD(#{endTime},INTERVAL 1 DAY)
             </if>
             order by date_created
             LIMIT #{limit} OFFSET #{offset}
     </select>
+
+    <select id="findBySmppMessageId" parameterType="map" resultType="hashmap">
+        SELECT * FROM restcomm_sms_messages AS restcomm_sms_messages
+        WHERE smpp_message_id = #{smppMessageId}
+    </select>
 </mapper>

restcomm/restcomm.application/src/main/webapp/WEB-INF/sql/sms-messages.xml

@@ -6,8 +6,9 @@
 <mapper namespace="org.mobicents.servlet.sip.restcomm.dao.SmsMessagesDao">
   <insert id="addSmsMessage" parameterType="map">
     INSERT INTO "restcomm_sms_messages" ("sid", "date_created", "date_updated", "date_sent", "account_sid", "sender", "recipient", "body", "status", "direction", "price",
-    "api_version", "uri") VALUES (#{sid}, #{date_created}, #{date_updated}, #{date_sent}, #{account_sid}, #{sender}, #{recipient}, #{body},
-    #{status}, #{direction}, #{price}, #{api_version}, #{uri});
+    "api_version", "uri", "status_callback", "status_callback_method") VALUES (#{sid}, #{date_created}, #{date_updated}, #{date_sent}, #{account_sid}, #{sender}, #{recipient}, #{body},
+    #{status}, #{direction}, #{price}, #{api_version}, #{uri}, #{status_callback},
+      #{status_callback_method});
   </insert>
 
   <select id="getSmsMessage" parameterType="string" resultType="hashmap">
@@ -31,7 +32,7 @@
   </delete>
 
   <update id="updateSmsMessage" parameterType="map">
-    UPDATE "restcomm_sms_messages" SET "date_sent"=#{date_sent}, "status"=#{status}, "price"=#{price}, "smpp_message_id"=#{smpp_message_id}, "date_updated"=NOW() WHERE "sid"=#{sid};
+    UPDATE "restcomm_sms_messages" SET "date_sent"=#{date_sent}, "status"=#{status}, "price"=#{price}, "smpp_message_id"=#{smpp_message_id}, "error_code"=#{error_code}, "date_updated"=NOW() WHERE "sid"=#{sid};
   </update>
 
   <select id="getSmsMessagesPerAccountLastPerMinute" parameterType="map" resultType="int">
@@ -117,10 +118,15 @@
                 AND "date_created" &gt;= #{startTime}
             </if>
             <if test="endTime != null">
-                    AND "date_created" &lt;= DATE_ADD(#{endTime},INTERVAL 1 DAY)
+                AND "date_created" &lt;= DATE_ADD(#{endTime},INTERVAL 1 DAY)
             </if>
             order by "date_created"
             LIMIT #{limit} OFFSET #{offset}
     </select>
+
+    <select id="findBySmppMessageId" parameterType="map" resultType="hashmap">
+        SELECT * FROM "restcomm_sms_messages" AS "restcomm_sms_messages"
+        WHERE "smpp_message_id" = #{smppMessageId}
+    </select>
 
 </mapper>

restcomm/restcomm.application/src/main/webapp/WEB-INF/web.xml

@@ -27,6 +27,11 @@
             <param-value>true</param-value>
     </init-param> 
   </servlet>
+  
+  <servlet>
+    <servlet-name>RestcommFileCache</servlet-name>
+    <servlet-class>org.restcomm.connect.http.filters.FileCacheServlet</servlet-class>
+  </servlet>  
 
   <context-param>
     <param-name>resteasy.scan</param-name>
@@ -47,6 +52,12 @@
     <servlet-name>Jersey</servlet-name>
     <url-pattern>/2012-04-24/*</url-pattern>
   </servlet-mapping>
+  
+  <servlet-mapping>
+    <servlet-name>RestcommFileCache</servlet-name>
+    <url-pattern>/cache/*</url-pattern>
+  </servlet-mapping>
+  
 
   <welcome-file-list>
     <welcome-file>index.html</welcome-file>