Merge pull request #30 from gkandemi/master

artiedaj · artiedaj · commit 45aefc2d6368 · 2014-04-14T10:49:41.000-05:00
Generic File Monitoring Script
diff --git a/Pledges_View_SSB/pledge/pledges.py b/Pledges_View_SSB/pledge/pledges.py
@@ -54,7 +54,8 @@ def fetch_all_pledges(url,api):
 
 def matchPledges(pledgeList):
   pledges = {}
-  sitesList =  ['T1_TW_ASGC','T1_FR_CCIN2P3','T1_CH_CERN','T1_IT_CNAF','T1_US_FNAL','T1_US_FNAL_Disk','T1_RU_JINR','T1_RU_JINR_Disk','T1_DE_KIT','T1_ES_PIC','T1_UK_RAL','T1_UK_RAL_Disk','T2_IT_Bari','T2_CN_Beijing','T2_K_SGrid_Bristol','T2_K_London_Brunel','T2_FR_CCIN2P3','T2_CH_CERN','T2_CH_CERN_AI','T2_CH_CERN_HLT','T2_CH_CERN_T0','T2_ES_CIEMAT','T2_CH_CSCS','T2_TH_CUNSTDA','T2_S_Caltech','T2_DE_DESY','T2_EE_Estonia','T2_S_Florida','T2_FR_GRIF_IRFU','T2_FR_GRIF_LLR','T2_BR_UERJ','T2_FI_HIP','T2_AT_Vienna','T2_HU_Budapest','T2_UK_London_IC','T2_ES_IFCA','T2_RU_IHEP','T2_BE_IIHE','T2_RU_INR', 'T2_FR_IPHC','T2_RU_ITEP','T2_GR_Ioannina','T2_RU_JINR','T2_UA_KIPT','T2_KR_KNU','T2_IT_Legnaro','T2_BE_UCL','T2_TR_METU','T2_US_MIT','T2_PT_NCG_Lisbon','T2_PK_NCP','T2_US_Nebraska','T2_RU_PNPI','T2_IT_Pisa','T2_US_Purde', 'T2_RU_RRC_KI','T2_DE_RWTH','T2_IT_Rome','T2_UK_SGrid_RALPP','T2_RU_SINP','T2_BR_SPRACE','T2_IN_TIFR','T2_TW_Taiwan','T2_US_UCSD','T2_MY_UPM_BIRUNI', 'T2_US_Vanderbilt','T2_PL_Warsaw','T2_US_Wisconsin']
+  #sitesList =  ['T1_TW_ASGC','T1_FR_CCIN2P3','T1_CH_CERN','T1_IT_CNAF','T1_US_FNAL','T1_US_FNAL_Disk','T1_RU_JINR','T1_RU_JINR_Disk','T1_DE_KIT','T1_ES_PIC','T1_UK_RAL','T1_UK_RAL_Disk','T2_IT_Bari','T2_CN_Beijing','T2_K_SGrid_Bristol','T2_K_London_Brunel','T2_FR_CCIN2P3','T2_CH_CERN','T2_CH_CERN_AI','T2_CH_CERN_HLT','T2_CH_CERN_T0','T2_ES_CIEMAT','T2_CH_CSCS','T2_TH_CUNSTDA','T2_S_Caltech','T2_DE_DESY','T2_EE_Estonia','T2_S_Florida','T2_FR_GRIF_IRFU','T2_FR_GRIF_LLR','T2_BR_UERJ','T2_FI_HIP','T2_AT_Vienna','T2_HU_Budapest','T2_UK_London_IC','T2_ES_IFCA','T2_RU_IHEP','T2_BE_IIHE','T2_RU_INR', 'T2_FR_IPHC','T2_RU_ITEP','T2_GR_Ioannina','T2_RU_JINR','T2_UA_KIPT','T2_KR_KNU','T2_IT_Legnaro','T2_BE_UCL','T2_TR_METU','T2_US_MIT','T2_PT_NCG_Lisbon','T2_PK_NCP','T2_US_Nebraska','T2_RU_PNPI','T2_IT_Pisa','T2_US_Purde', 'T2_RU_RRC_KI','T2_DE_RWTH','T2_IT_Rome','T2_UK_SGrid_RALPP','T2_RU_SINP','T2_BR_SPRACE','T2_IN_TIFR','T2_TW_Taiwan','T2_US_UCSD','T2_MY_UPM_BIRUNI', 'T2_US_Vanderbilt','T2_PL_Warsaw','T2_US_Wisconsin']
+  sitesList =  ['T1_TW_ASGC','T1_FR_CCIN2P3','T1_CH_CERN','T1_IT_CNAF','T1_US_FNAL','T1_US_FNAL_Disk','T1_RU_JINR','T1_RU_JINR_Disk','T1_DE_KIT','T1_ES_PIC','T1_UK_RAL','T1_UK_RAL_Disk','T2_IT_Bari','T2_CN_Beijing','T2_UK_SGrid_Bristol','T2_UK_London_Brunel','T2_FR_CCIN2P3','T2_CH_CERN','T2_CH_CERN_AI','T2_CH_CERN_HLT','T2_CH_CERN_T0','T2_ES_CIEMAT','T2_CH_CSCS','T2_TH_CUNSTDA','T2_US_Caltech','T2_DE_DESY','T2_EE_Estonia','T2_US_Florida','T2_FR_GRIF_IRFU','T2_FR_GRIF_LLR','T2_BR_UERJ','T2_FI_HIP','T2_AT_Vienna','T2_HU_Budapest','T2_UK_London_IC','T2_ES_IFCA','T2_RU_IHEP','T2_BE_IIHE','T2_RU_INR', 'T2_FR_IPHC','T2_RU_ITEP','T2_GR_Ioannina','T2_RU_JINR','T2_UA_KIPT','T2_KR_KNU','T2_IT_Legnaro','T2_BE_UCL','T2_TR_METU','T2_US_MIT','T2_PT_NCG_Lisbon','T2_PK_NCP','T2_US_Nebraska','T2_RU_PNPI','T2_IT_Pisa','T2_US_Purdue', 'T2_RU_RRC_KI','T2_DE_RWTH','T2_IT_Rome','T2_UK_SGrid_RALPP','T2_RU_SINP','T2_BR_SPRACE','T2_IN_TIFR','T2_TW_Taiwan','T2_US_UCSD','T2_MY_UPM_BIRUNI', 'T2_US_Vanderbilt','T2_PL_Warsaw','T2_US_Wisconsin']
   #reportRoot = GetXMLFromURL( "https://cmsweb.cern.ch/sitedb/reports/showXMLReport?reportid=naming_convention.ini")
   #result   = reportRoot.find('result')
   #matchList = {'T3_US_PuertoRico': 'UPRM', 'T2_FI_HIP': 'Helsinki Institute of Physics', 'T2_UK_SGrid_RALPP': 'Rutherford PPD', 'T2_FR_GRIF_LLR': 'GRIF_LLR', 'T2_UK_London_IC': 'IC', 'T3_UK_London_QMUL': 'QMUL', 'T3_TW_NTU_HEP': 'NTU_HEP', 'T3_US_Omaha': 'Firefly', 'T2_KR_KNU': 'KNU', 'T2_RU_SINP': 'SINP', 'T3_US_UMD': 'UMD', 'T2_CH_CERN_AI': 'CERN Tier-2 AI', 'T1_TW_ASGC': 'ASGC', 'T3_US_Colorado': 'Colorado', 'T3_US_UB': 'SUNY_BUFFALO', 'T1_UK_RAL_Disk': 'RALDISK', 'T3_IT_Napoli': 'INFN-NAPOLI-CMS', 'T3_NZ_UOA': 'NZ-UOA', 'T2_TH_CUNSTDA': 'CUNSTDA', 'T3_US_Kansas': 'Kansas', 'T3_US_ParrotTest': 'T3 US ParrotTest', 'T3_GR_IASA': 'IASA', 'T3_US_Parrot': 'T3 US Parrot', 'T2_IT_Bari': 'Bari', 'T2_US_UCSD': 'UCSD', 'T2_RU_IHEP': 'IHEP', 'T3_US_Vanderbilt_EC2': 'Vanderbilt_EC2', 'T1_RU_JINR': 'JINR-T1', 'T2_RU_RRC_KI': 'RRC_KI', 'T2_CH_CERN': 'CERN Tier-2', 'T3_BY_NCPHEP': 'BY-NCPHEP', 'T2_US_Vanderbilt': 'Vanderbilt', 'T3_GR_Demokritos': 'Demokritos', 'T3_US_UTENN': 'UTenn', 'T3_US_UCR': 'UC Riverside', 'T3_TW_NCU': 'NCU', 'T2_CH_CSCS': 'CSCS', 'T2_UA_KIPT': 'KIPT', 'T2_PK_NCP': 'NCP-LCG2', 'T2_RU_PNPI': 'PNPI', 'T2_IN_TIFR': 'TIFR', 'T3_UK_London_UCL': 'University College London', 'T3_US_Brown': 'Brown-CMS', 'T3_US_UCD': 'UCD', 'T3_CO_Uniandes': 'UNIANDES', 'T3_KR_KNU': 'KR_KNU', 'T2_FR_IPHC': 'IPHC', 'T3_US_OSU': 'OSU', 'T3_US_TAMU': 'TAMU', 'T1_US_FNAL': 'FNAL', 'T3_IT_Trieste': 'Trieste', 'T2_IT_Rome': 'Rome', 'T2_UK_London_Brunel': 'Brunel', 'T3_IN_PUHEP': 'PUHEP', 'T3_IT_Firenze': 'Firenze', 'T1_US_FNAL_Disk': 'FNALDISK', 'T2_EE_Estonia': 'Estonia', 'T3_UK_ScotGrid_ECDF': 'ECDF', 'T2_CN_Beijing': 'Beijing', 'T2_US_Florida': 'Florida', 'T3_US_Princeton_ICSE': 'Princeton ICSE ', 'T3_IT_MIB': 'INFN-MIB', 'T3_US_FNALXEN': 'FNALXEN', 'T3_US_Rutgers': 'Rutgers', 'T1_DE_KIT': 'KIT', 'T3_IR_IPM': 'IPM', 'T2_US_Wisconsin': 'Wisconsin', 'T2_HU_Budapest': 'Hungary', 'T2_DE_RWTH': 'RWTH', 'T3_IT_Perugia': 'Perugia', 'T3_UK_SGrid_Oxford': 'Oxford', 'T3_US_NU': 'Northwestern', 'T2_BR_UERJ': 'HEPGRID_UERJ', 'T3_MX_Cinvestav': 'cinvestav', 'T3_US_FNALLPC': 'FNALLPC', 'T3_US_UIowa': 'UIowa', 'T3_RU_FIAN': 'FIAN', 'T3_US_Cornell': 'Cornell', 'T2_ES_IFCA': 'IFCA', 'T3_US_UVA': 'UVA', 'T3_ES_Oviedo': 'Oviedo', 'T3_US_NotreDame': 'NWICG_NDCMS', 'T2_DE_DESY': 'DESY', 'T1_UK_RAL': 'RAL', 'T2_US_Caltech': 'Caltech', 'T3_FR_IPNL': 'IN2P3-IPNL', 'T2_TW_Taiwan': 'Taiwan', 'T3_US_NEU': 'Northeastern', 'T3_UK_London_RHUL': 'RHUL', 'T0_CH_CERN': 'CERN Tier-0', 'T1_RU_JINR_Disk': 'JINR-T1DISK', 'T3_CN_PKU': 'CN_PKU', 'T3_US_Baylor': 'Baylor University Tier3', 'T2_US_Nebraska': 'Nebraska', 'T2_ES_CIEMAT': 'CIEMAT', 'T3_US_Princeton': 'Princeton', 'T3_UK_ScotGrid_GLA': 'UKI-SCOTGRID-GLASGOW', 'T2_CH_CERN_T0': 'CERN Tier-2 T0', 'T3_US_TTU': 'TTU', 'T3_US_FSU': 'T3_US_FSU', 'T3_KR_UOS': 'UOS', 'T2_BR_SPRACE': 'SPRACE', 'T1_IT_CNAF': 'CNAF', 'T3_US_Minnesota': 'Minnesota', 'T2_TR_METU': 'METU', 'T2_AT_Vienna': 'Hephy-Vienna', 'T2_US_Purdue': 'Purdue', 'T3_US_Rice': 'Rice', 'T3_HR_IRB': 'IRB', 'T2_BE_UCL': 'Louvain', 'T3_US_FIT': 'FLTECH', 'T2_UK_SGrid_Bristol': 'Bristol', 'T2_PT_NCG_Lisbon': 'NCG-INGRID-PT', 'T1_ES_PIC': 'PIC', 'T3_US_JHU': 'JHU', 'T2_IT_Legnaro': 'Legnaro', 'T2_RU_INR': 'INR', 'T3_US_FIU': 'T3_US_FIU', 'T3_EU_Parrot': 'T3 EU Parrot', 'T2_RU_JINR': 'JINR', 'T2_IT_Pisa': 'Pisa', 'T2_GR_Ioannina': 'Ioannina', 'T3_US_MIT': 'T3 US MIT', 'T2_CH_CERN_HLT': 'CERN Tier-2 HLT', 'T2_MY_UPM_BIRUNI': 'UPM Biruni', 'T1_FR_CCIN2P3': 'CC-IN2P3', 'T2_FR_GRIF_IRFU': 'GRIF_IRFU', 'T3_US_UMiss': 'UMissHEP', 'T2_FR_CCIN2P3': 'CC-IN2P3 AF', 'T2_PL_Warsaw': 'Warsaw', 'T3_AS_Parrot': 'T3 AS Parrot', 'T2_US_MIT': 'MIT', 'T2_BE_IIHE': 'IIHE', 'T2_RU_ITEP': 'ITEP', 'T1_CH_CERN': 'CERN', 'T3_CH_PSI': 'PSI', 'T3_IT_Bologna': 'Bologna-T3'}
diff --git a/Site_Config/generic_file_monitoring/run_site_local_config.sh b/Site_Config/generic_file_monitoring/run_site_local_config.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+# written by Gökhan Kandemir => gokhan.kandemir@cern.ch
+# usercert and userkey files must be in folder .globus
+# this script calculates the number of T1s and T2s counts and writes results to console and file.
+clear
+echo "To Prevent ask typing password constantly, Registering your KEY and CERT to Grid Environment"
+source /afs/cern.ch/project/gd/LCG-share/new_3.2/etc/profile.d/grid_env.sh
+voms-proxy-init -voms cms
+# Email if things are running slowly
+
+if [ -f scriptRunning.run ];
+then
+   echo "run_site_local_config.sh is already running. Will send an email to the admin."
+   # script to send simple email
+   # email subject
+   SUBJECT="[TFC] load site-local-config"
+   # Email To ?
+   EMAIL="gokhan.kandemir@cern.ch"
+   # Email text/message
+   if [ -f emailmessage.txt ];
+   then
+      rm emailmessage.txt
+   fi
+   touch emailmessage.txt
+   EMAILMESSAGE="/tmp/emailmessage.txt"
+   echo "run_site_local_config.sh  is running to slowly."
+   # send an email using /bin/mail
+   /bin/mail -s "$SUBJECT" "$EMAIL" < $EMAILMESSAGE
+
+else
+     echo "bash run_site_local_config.sh succesfully"
+     touch scriptRunning.run
+fi
+
+
+#Run the script
+txt="gfm"
+echo "python site_local_config.py > $txt.txt and $txt.json"
+python site_local_config.py $txt &> site_local_config.log
+
+problem="$?"
+echo "problem: $problem"
+echo "The files were created succesfully."
+
+cp $txt".txt" /afs/cern.ch/user/c/cmst1/www/SST
+cp $txt".json"  /afs/cern.ch/user/c/cmst1/www/SST
+
+rm scriptRunning.run
+
diff --git a/Site_Config/generic_file_monitoring/site_local_config.py b/Site_Config/generic_file_monitoring/site_local_config.py
@@ -0,0 +1,81 @@
+#!/usr/bin/python
+import os, sys
+import time
+import urllib, httplib, urllib2
+import string
+from xml.dom.minidom import parse, parseString
+#_____________________________________________________________________________
+
+siteList = {}
+T1Count = 0
+T2Count = 0
+#________________________________________________________________________________________
+# functions fetchs all siteName and gets all xml file.
+def fetch_all_site():
+  sitesList =  ['T1_TW_ASGC','T1_FR_CCIN2P3','T1_CH_CERN','T1_IT_CNAF','T1_US_FNAL','T1_US_FNAL_Disk','T1_RU_JINR','T1_RU_JINR_Disk','T1_DE_KIT','T1_ES_PIC','T1_UK_RAL','T1_UK_RAL_Disk','T2_IT_Bari','T2_CN_Beijing','T2_UK_SGrid_Bristol','T2_UK_London_Brunel','T2_FR_CCIN2P3','T2_CH_CERN','T2_CH_CERN_AI','T2_CH_CERN_HLT','T2_CH_CERN_T0','T2_ES_CIEMAT','T2_CH_CSCS','T2_TH_CUNSTDA','T2_US_Caltech','T2_DE_DESY','T2_EE_Estonia','T2_US_Florida','T2_FR_GRIF_IRFU','T2_FR_GRIF_LLR','T2_BR_UERJ','T2_FI_HIP','T2_AT_Vienna','T2_HU_Budapest','T2_UK_London_IC','T2_ES_IFCA','T2_RU_IHEP','T2_BE_IIHE','T2_RU_INR', 'T2_FR_IPHC','T2_RU_ITEP','T2_GR_Ioannina','T2_RU_JINR','T2_UA_KIPT','T2_KR_KNU','T2_IT_Legnaro','T2_BE_UCL','T2_TR_METU','T2_US_MIT','T2_PT_NCG_Lisbon','T2_PK_NCP','T2_US_Nebraska','T2_RU_PNPI','T2_IT_Pisa','T2_US_Purdue', 'T2_RU_RRC_KI','T2_DE_RWTH','T2_IT_Rome','T2_UK_SGrid_RALPP','T2_RU_SINP','T2_BR_SPRACE','T2_IN_TIFR','T2_TW_Taiwan','T2_US_UCSD','T2_MY_UPM_BIRUNI', 'T2_US_Vanderbilt','T2_PL_Warsaw','T2_US_Wisconsin']
+  # To get all site-local-config information.
+  for siteName in sitesList:
+    xml = getXmlfromURL('cmsweb.cern.ch','/gitweb/?p=siteconf/.git;a=blob_plain;f=' + siteName + '/JobConfig/site-local-config.xml;hb=HEAD')
+    match(xml, siteName)
+#__________________________________________________________________________________________
+# function calculates the number of T1s and T2s counts and writes results to console and file.
+def match(xml, siteName):
+  global T1Count, T2Count, file, jsonCode
+  findTxt = "404 - Cannot find file"
+  saveTime = time.strftime('%Y-%m-%d %H:%M:%S')
+  url = "cmsweb.cern.ch/gitweb/?p=siteconf/.git;a=blob_plain;f=" + siteName + "/JobConfig/site-local-config.xml;hb=HEAD"
+  if (findTxt in xml) == False:
+    dom  = parseString(xml)
+    tag  = dom.getElementsByTagName('site-local-config')[0].getElementsByTagName('site')[0].getElementsByTagName('statistics-destination')
+    if len(tag):
+      print saveTime + "\t" + siteName + "\t" + "yes" + "\t" + "green" + "\t" + url
+      file.write(saveTime + "\t" + siteName + "\t" + "yes" + "\t" + "green" + "\t" + url + "\n")
+      jsonCode = jsonCode + "{" + '"siteName":"' + siteName + '", "yes_no":' + '"yes"' + ',' + '"color":"green' + '",' + '"url":"' + url + '"},'    
+      if (siteName[0:2] == "T1") : T1Count = T1Count + 1
+      if (siteName[0:2] == "T2") : T2Count = T2Count + 1
+    else:
+      print saveTime + "\t" + siteName + "\t" + "no" + "\t" + "red" + "\t" + url
+      file.write(saveTime + "\t" + siteName + "\t" + "no" + "\t" + "red" + "\t" + url + "\n")
+      jsonCode = jsonCode + "{" + '"siteName":"' + siteName + '", "yes_no":' + '"no"' + ',' + '"color":"red' + '",' + '"url":"' + url + '"},' 
+  else:
+    print saveTime + "\t" + siteName + "\t" + "no" + "\t" + "red" + "\t" + url
+    file.write(saveTime + "\t" + siteName + "\t" + "no" + "\t" + "red" + "\t" + url + "\n")
+    jsonCode = jsonCode + "{" + '"siteName":"' + siteName + '", "yes_no":' + '"no"' + ',' + '"color":"red' + '",' + '"url":"' + url + '"},'
+#_______________________________________________________________________________
+# function reads  your certificate and gets site-local-config.xml from URL
+def getXmlfromURL(url,api):
+  headers = {"Accept": "application/xml"}
+  if 'X509_USER_PROXY' in os.environ:
+      #print 'X509_USER_PROXY found'
+      conn = httplib.HTTPSConnection(url, cert_file = os.getenv('X509_USER_PROXY'), key_file = os.getenv('X509_USER_PROXY'))
+  elif 'X509_USER_CERT' in os.environ and 'X509_USER_KEY' in os.environ:
+      #print 'X509_USER_CERT and X509_USER_KEY found'
+      conn = httplib.HTTPSConnection(url, cert_file = os.getenv('X509_USER_CERT'), key_file = os.getenv('X509_USER_KEY'))
+  else:
+      #print 'You need a valid proxy or cert/key files'
+      sys.exit()
+  r1=conn.request("GET",api, None, headers)
+  r2=conn.getresponse()
+  xml = r2.read()
+  return xml
+#_____________________________________________________________________________
+if __name__ == '__main__':
+  outputfile_txt=sys.argv[1]
+  print 'starting to fetch all site and calculate the number of T1s and T2s counts.'
+  file = open(outputfile_txt + ".txt", "w")
+  jsonFile = open(outputfile_txt + ".json", "w")
+  jsonCode = '{ "site_local_config":['
+  jsonCodeEnd = ']}'
+  fetch_all_site()
+  saveTime = time.strftime('%Y-%m-%d %H:%M:%S')
+  url = "https://cmsweb.cern.ch/gitweb/?p=siteconf/.git;a=tree"
+  print saveTime + "\t" + "T1s" + "\t" + str(T1Count) + "\t" + "green" + "\t" + url
+  print saveTime + "\t" + "T2s" + "\t" + str(T2Count) + "\t" + "green" + "\t" + url
+  file.write(saveTime + "\t" + "T1s" + "\t" + str(T1Count) + "\t" + "green" + "\t" + url + "\n")
+  file.write(saveTime + "\t" + "T2s" + "\t" + str(T2Count) + "\t" + "green" + "\t" + url + "\n")
+  file.close()
+  jsonCode = jsonCode + "{" + '"T1s":' + str(T1Count)  + ',' + '"color":"green' + '",' + '"url":"' + url + '"},'
+  jsonCode = jsonCode + "{" + '"T2s":' + str(T2Count)  + ',' + '"color":"green' + '",' + '"url":"' + url + '"},'
+  jsonCode = (jsonCode + jsonCodeEnd).replace("},]}", "}]}")
+  jsonFile.write(jsonCode)
+  jsonFile.close()
