Merge pull request #7 from yiiyama/production

Express partition

Author: dr-stringfellow

etc/db/dynamoregister.sql

@@ -2,7 +2,7 @@
 --
 -- Host: localhost    Database: dynamoregister
 -- ------------------------------------------------------
--- Server version	5.1.73
+-- Server version	5.1.73-log
 
 /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
 /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
@@ -55,10 +55,11 @@ DROP TABLE IF EXISTS `deletion_queue`;
 /*!40101 SET @saved_cs_client     = @@character_set_client */;
 /*!40101 SET character_set_client = utf8 */;
 CREATE TABLE `deletion_queue` (
+  `reqid` int(10) unsigned NOT NULL DEFAULT '0',
   `file` varchar(512) COLLATE latin1_general_cs NOT NULL,
-  `target` varchar(64) COLLATE latin1_general_cs NOT NULL,
-  `created` datetime NOT NULL,
-  UNIQUE KEY `files` (`file`,`target`)
+  `site` varchar(32) COLLATE latin1_general_cs NOT NULL,
+  `status` enum('new','done','failed') COLLATE latin1_general_cs NOT NULL,
+  UNIQUE KEY `file` (`file`,`site`)
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs;
 /*!40101 SET character_set_client = @saved_cs_client */;
 
@@ -103,6 +104,57 @@ CREATE TABLE `domains` (
 ) ENGINE=MyISAM AUTO_INCREMENT=1 DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs;
 /*!40101 SET character_set_client = @saved_cs_client */;
 
+--
+-- Table structure for table `invalidations`
+--
+
+DROP TABLE IF EXISTS `invalidations`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `invalidations` (
+  `item` varchar(512) CHARACTER SET latin1 COLLATE latin1_general_cs NOT NULL,
+  `user_id` int(10) unsigned NOT NULL DEFAULT '0',
+  `timestamp` datetime NOT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Table structure for table `requests`
+--
+
+DROP TABLE IF EXISTS `requests`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `requests` (
+  `item` varchar(512) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+  `datatype` enum('dataset','block') CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+  `site` varchar(32) NOT NULL,
+  `reqtype` enum('copy','delete') NOT NULL,
+  `created` datetime NOT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Table structure for table `requests_unified`
+--
+
+DROP TABLE IF EXISTS `requests_unified`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `requests_unified` (
+  `reqid` int(10) unsigned NOT NULL AUTO_INCREMENT,
+  `item` varchar(512) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+  `datatype` enum('dataset','block') CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+  `site` varchar(32) NOT NULL,
+  `reqtype` enum('copy','delete') NOT NULL,
+  `rank` int(10) unsigned DEFAULT '0',
+  `status` enum('new','queued') NOT NULL,
+  `created` datetime NOT NULL,
+  `updated` datetime DEFAULT NULL,
+  PRIMARY KEY (`reqid`)
+) ENGINE=MyISAM AUTO_INCREMENT=1 DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
 --
 -- Table structure for table `services`
 --
@@ -162,4 +214,4 @@ CREATE TABLE `users` (
 /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
 /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
 
--- Dump completed on 2017-06-26 15:52:45
+-- Dump completed on 2017-10-07 11:12:50

etc/policies.tag

@@ -1 +1 @@
-v49
+v50

lib/common/configuration.py

@@ -184,7 +184,7 @@
         ('IB RelVal', lambda r: r.group is not None and r.group.name == 'IB RelVal'),
         ('Tape', lambda r: r.site.storage_type == Site.TYPE_MSS),
         ('Unsubscribed', lambda r: r.group is None),
-        ('Express', lambda r: r.group is not None and r.group.name == 'express' and re.match('/.*Express.*/.+/.+$', r.block.dataset.name)),
+        ('Express', lambda r: r.group is not None and r.group.name == 'express' and re.match('/(?:.*Express.*/.+|.+/.*Express.*)/.+$', r.block.dataset.name)),
         ('Physics', ['AnalysisOps', 'DataOps']),
     ],
     # list of conditions for a PRODUCTION state dataset to become IGNORED (will still be reset to PRODUCTION if a new block replica is found)

web/cgi-bin/registry/invalidation.php

@@ -0,0 +1,62 @@
+<?php
+
+date_default_timezone_set('UTC');
+
+$command = substr($_SERVER['PATH_INFO'], 1); # dynamo.mit.edu/registry/invalidate/command -> /command
+
+if ($command == "") {
+  # show webpage
+  exit(0);
+}
+
+if ($command == 'help') {
+#  echo file_get_contents(__DIR__ . '/html/invalidate_help.html');
+  exit(0);
+}
+
+include_once(__DIR__ . '/../dynamo/common/db_conf.php');
+include_once(__DIR__ . '/common.php');
+
+/* if ($_SERVER['SSL_CLIENT_VERIFY'] != 'SUCCESS') */
+/*   send_response(401, 'AuthFailed', 'SSL authentication failed.'); */
+
+if ($command == 'invalidate' || $command == 'clear') {
+  if (!isset($_REQUEST['item']))
+    send_response(400, 'BadRequest', '\'item\' field is required', NULL, 'json');
+
+  $item = $_REQUEST['item'];
+
+  $db = new mysqli($db_conf['host'], $db_conf['user'], $db_conf['password'], 'dynamoregister');
+
+  if ($command == 'invalidate') {
+    $stmt = $db->prepare('INSERT INTO `invalidations` (`item`, `timestamp`) VALUES (?, NOW())');
+    $message = 'Item added';   
+  }
+  else {
+    $stmt = $db->prepare('DELETE FROM `invalidations` WHERE `item` = ?');
+    $message = 'Item removed';
+  }
+
+  $return_data = array();
+
+  if (is_array($item)) {
+    foreach ($item as $t) {
+      $stmt->bind_param('s', $t);
+      $stmt->execute();
+      $return_data[] = array('item' => $t);
+    }
+  }
+  else {
+    $stmt->bind_param('s', $item);
+    $stmt->execute();
+    $return_data[] = array('item' => $item);
+  }
+  $stmt->close();
+
+  send_response(200, 'OK', $message, $return_data, 'json');
+}
+else {
+  send_response(400, 'BadRequest', 'Invalid command (possible values: help, invalidate)');
+}
+
+?>