Merge pull request #33 from gkandemi/master

artiedaj · artiedaj · commit 4ae861cb8553 · 2014-05-12T09:36:33.000-05:00
siteDB v2__
diff --git a/Pledges_View_SSB/pledge/pledges.py b/Pledges_View_SSB/pledge/pledges.py
@@ -1,7 +1,7 @@
 #!/usr/bin/python
 import os, sys
 import simplejson
-import json
+import simplejson as json
 import time
 import urllib, httplib, urllib2
 import string
@@ -30,14 +30,16 @@ def fetch_all_pledges(url,api):
   pledges= {}
   pledgesSites = {}
   count = 0
+  pledgeCurTime = int(time.strftime("%Y"))
   for i in jsn['result']:
     #_____________________________
+    print i
     pledgeDate     = i[2]
     pledgeTime     = i[1]
     pledgeSiteName = i[0]
     #pledgeCpuValue = i[3]
     #_____________________________
-    if pledgeDate == 2014:
+    if pledgeDate == pledgeCurTime:
       current = time.time()
       diff = current - pledgeTime
       if pledges.has_key(pledgeSiteName):
@@ -53,13 +55,27 @@ def fetch_all_pledges(url,api):
 # function matchs pledges values gets from siteDB with SiteName
 
 def matchPledges(pledgeList):
+  #_______________________fetch all siteName : FederationName because Pledges is defined by federation name not siteName._____________________
+  headers = {"Accept": "application/json"}
+  url = "cmsweb.cern.ch"
+  api = "/sitedb/data/prod/federations-sites"
+  sitesList=[]
+  matchList = {}
+  if 'X509_USER_PROXY' in os.environ:
+    print 'X509_USER_PROXY found'
+    conn = httplib.HTTPSConnection(url, cert_file = os.getenv('X509_USER_PROXY'), key_file = os.getenv('X509_USER_PROXY'))
+  r1=conn.request("GET",api, None, headers)
+  r2=conn.getresponse()
+  inputjson=r2.read()
+  jn = simplejson.loads(inputjson)
+  conn.close()
+
+  for siteName in jn['result']:
+      if siteName[3][0:2] != 'T3':
+         matchList[siteName[3]] = siteName[2]
+         sitesList.append(siteName[3])
+  #___________________________________________________________________________________________________________________________
   pledges = {}
-  #sitesList =  ['T1_TW_ASGC','T1_FR_CCIN2P3','T1_CH_CERN','T1_IT_CNAF','T1_US_FNAL','T1_US_FNAL_Disk','T1_RU_JINR','T1_RU_JINR_Disk','T1_DE_KIT','T1_ES_PIC','T1_UK_RAL','T1_UK_RAL_Disk','T2_IT_Bari','T2_CN_Beijing','T2_K_SGrid_Bristol','T2_K_London_Brunel','T2_FR_CCIN2P3','T2_CH_CERN','T2_CH_CERN_AI','T2_CH_CERN_HLT','T2_CH_CERN_T0','T2_ES_CIEMAT','T2_CH_CSCS','T2_TH_CUNSTDA','T2_S_Caltech','T2_DE_DESY','T2_EE_Estonia','T2_S_Florida','T2_FR_GRIF_IRFU','T2_FR_GRIF_LLR','T2_BR_UERJ','T2_FI_HIP','T2_AT_Vienna','T2_HU_Budapest','T2_UK_London_IC','T2_ES_IFCA','T2_RU_IHEP','T2_BE_IIHE','T2_RU_INR', 'T2_FR_IPHC','T2_RU_ITEP','T2_GR_Ioannina','T2_RU_JINR','T2_UA_KIPT','T2_KR_KNU','T2_IT_Legnaro','T2_BE_UCL','T2_TR_METU','T2_US_MIT','T2_PT_NCG_Lisbon','T2_PK_NCP','T2_US_Nebraska','T2_RU_PNPI','T2_IT_Pisa','T2_US_Purde', 'T2_RU_RRC_KI','T2_DE_RWTH','T2_IT_Rome','T2_UK_SGrid_RALPP','T2_RU_SINP','T2_BR_SPRACE','T2_IN_TIFR','T2_TW_Taiwan','T2_US_UCSD','T2_MY_UPM_BIRUNI', 'T2_US_Vanderbilt','T2_PL_Warsaw','T2_US_Wisconsin']
-  sitesList =  ['T1_TW_ASGC','T1_FR_CCIN2P3','T1_CH_CERN','T1_IT_CNAF','T1_US_FNAL','T1_US_FNAL_Disk','T1_RU_JINR','T1_RU_JINR_Disk','T1_DE_KIT','T1_ES_PIC','T1_UK_RAL','T1_UK_RAL_Disk','T2_IT_Bari','T2_CN_Beijing','T2_UK_SGrid_Bristol','T2_UK_London_Brunel','T2_FR_CCIN2P3','T2_CH_CERN','T2_CH_CERN_AI','T2_CH_CERN_HLT','T2_CH_CERN_T0','T2_ES_CIEMAT','T2_CH_CSCS','T2_TH_CUNSTDA','T2_US_Caltech','T2_DE_DESY','T2_EE_Estonia','T2_US_Florida','T2_FR_GRIF_IRFU','T2_FR_GRIF_LLR','T2_BR_UERJ','T2_FI_HIP','T2_AT_Vienna','T2_HU_Budapest','T2_UK_London_IC','T2_ES_IFCA','T2_RU_IHEP','T2_BE_IIHE','T2_RU_INR', 'T2_FR_IPHC','T2_RU_ITEP','T2_GR_Ioannina','T2_RU_JINR','T2_UA_KIPT','T2_KR_KNU','T2_IT_Legnaro','T2_BE_UCL','T2_TR_METU','T2_US_MIT','T2_PT_NCG_Lisbon','T2_PK_NCP','T2_US_Nebraska','T2_RU_PNPI','T2_IT_Pisa','T2_US_Purdue', 'T2_RU_RRC_KI','T2_DE_RWTH','T2_IT_Rome','T2_UK_SGrid_RALPP','T2_RU_SINP','T2_BR_SPRACE','T2_IN_TIFR','T2_TW_Taiwan','T2_US_UCSD','T2_MY_UPM_BIRUNI', 'T2_US_Vanderbilt','T2_PL_Warsaw','T2_US_Wisconsin']
-  #reportRoot = GetXMLFromURL( "https://cmsweb.cern.ch/sitedb/reports/showXMLReport?reportid=naming_convention.ini")
-  #result   = reportRoot.find('result')
-  #matchList = {'T3_US_PuertoRico': 'UPRM', 'T2_FI_HIP': 'Helsinki Institute of Physics', 'T2_UK_SGrid_RALPP': 'Rutherford PPD', 'T2_FR_GRIF_LLR': 'GRIF_LLR', 'T2_UK_London_IC': 'IC', 'T3_UK_London_QMUL': 'QMUL', 'T3_TW_NTU_HEP': 'NTU_HEP', 'T3_US_Omaha': 'Firefly', 'T2_KR_KNU': 'KNU', 'T2_RU_SINP': 'SINP', 'T3_US_UMD': 'UMD', 'T2_CH_CERN_AI': 'CERN Tier-2 AI', 'T1_TW_ASGC': 'ASGC', 'T3_US_Colorado': 'Colorado', 'T3_US_UB': 'SUNY_BUFFALO', 'T1_UK_RAL_Disk': 'RALDISK', 'T3_IT_Napoli': 'INFN-NAPOLI-CMS', 'T3_NZ_UOA': 'NZ-UOA', 'T2_TH_CUNSTDA': 'CUNSTDA', 'T3_US_Kansas': 'Kansas', 'T3_US_ParrotTest': 'T3 US ParrotTest', 'T3_GR_IASA': 'IASA', 'T3_US_Parrot': 'T3 US Parrot', 'T2_IT_Bari': 'Bari', 'T2_US_UCSD': 'UCSD', 'T2_RU_IHEP': 'IHEP', 'T3_US_Vanderbilt_EC2': 'Vanderbilt_EC2', 'T1_RU_JINR': 'JINR-T1', 'T2_RU_RRC_KI': 'RRC_KI', 'T2_CH_CERN': 'CERN Tier-2', 'T3_BY_NCPHEP': 'BY-NCPHEP', 'T2_US_Vanderbilt': 'Vanderbilt', 'T3_GR_Demokritos': 'Demokritos', 'T3_US_UTENN': 'UTenn', 'T3_US_UCR': 'UC Riverside', 'T3_TW_NCU': 'NCU', 'T2_CH_CSCS': 'CSCS', 'T2_UA_KIPT': 'KIPT', 'T2_PK_NCP': 'NCP-LCG2', 'T2_RU_PNPI': 'PNPI', 'T2_IN_TIFR': 'TIFR', 'T3_UK_London_UCL': 'University College London', 'T3_US_Brown': 'Brown-CMS', 'T3_US_UCD': 'UCD', 'T3_CO_Uniandes': 'UNIANDES', 'T3_KR_KNU': 'KR_KNU', 'T2_FR_IPHC': 'IPHC', 'T3_US_OSU': 'OSU', 'T3_US_TAMU': 'TAMU', 'T1_US_FNAL': 'FNAL', 'T3_IT_Trieste': 'Trieste', 'T2_IT_Rome': 'Rome', 'T2_UK_London_Brunel': 'Brunel', 'T3_IN_PUHEP': 'PUHEP', 'T3_IT_Firenze': 'Firenze', 'T1_US_FNAL_Disk': 'FNALDISK', 'T2_EE_Estonia': 'Estonia', 'T3_UK_ScotGrid_ECDF': 'ECDF', 'T2_CN_Beijing': 'Beijing', 'T2_US_Florida': 'Florida', 'T3_US_Princeton_ICSE': 'Princeton ICSE ', 'T3_IT_MIB': 'INFN-MIB', 'T3_US_FNALXEN': 'FNALXEN', 'T3_US_Rutgers': 'Rutgers', 'T1_DE_KIT': 'KIT', 'T3_IR_IPM': 'IPM', 'T2_US_Wisconsin': 'Wisconsin', 'T2_HU_Budapest': 'Hungary', 'T2_DE_RWTH': 'RWTH', 'T3_IT_Perugia': 'Perugia', 'T3_UK_SGrid_Oxford': 'Oxford', 'T3_US_NU': 'Northwestern', 'T2_BR_UERJ': 'HEPGRID_UERJ', 'T3_MX_Cinvestav': 'cinvestav', 'T3_US_FNALLPC': 'FNALLPC', 'T3_US_UIowa': 'UIowa', 'T3_RU_FIAN': 'FIAN', 'T3_US_Cornell': 'Cornell', 'T2_ES_IFCA': 'IFCA', 'T3_US_UVA': 'UVA', 'T3_ES_Oviedo': 'Oviedo', 'T3_US_NotreDame': 'NWICG_NDCMS', 'T2_DE_DESY': 'DESY', 'T1_UK_RAL': 'RAL', 'T2_US_Caltech': 'Caltech', 'T3_FR_IPNL': 'IN2P3-IPNL', 'T2_TW_Taiwan': 'Taiwan', 'T3_US_NEU': 'Northeastern', 'T3_UK_London_RHUL': 'RHUL', 'T0_CH_CERN': 'CERN Tier-0', 'T1_RU_JINR_Disk': 'JINR-T1DISK', 'T3_CN_PKU': 'CN_PKU', 'T3_US_Baylor': 'Baylor University Tier3', 'T2_US_Nebraska': 'Nebraska', 'T2_ES_CIEMAT': 'CIEMAT', 'T3_US_Princeton': 'Princeton', 'T3_UK_ScotGrid_GLA': 'UKI-SCOTGRID-GLASGOW', 'T2_CH_CERN_T0': 'CERN Tier-2 T0', 'T3_US_TTU': 'TTU', 'T3_US_FSU': 'T3_US_FSU', 'T3_KR_UOS': 'UOS', 'T2_BR_SPRACE': 'SPRACE', 'T1_IT_CNAF': 'CNAF', 'T3_US_Minnesota': 'Minnesota', 'T2_TR_METU': 'METU', 'T2_AT_Vienna': 'Hephy-Vienna', 'T2_US_Purdue': 'Purdue', 'T3_US_Rice': 'Rice', 'T3_HR_IRB': 'IRB', 'T2_BE_UCL': 'Louvain', 'T3_US_FIT': 'FLTECH', 'T2_UK_SGrid_Bristol': 'Bristol', 'T2_PT_NCG_Lisbon': 'NCG-INGRID-PT', 'T1_ES_PIC': 'PIC', 'T3_US_JHU': 'JHU', 'T2_IT_Legnaro': 'Legnaro', 'T2_RU_INR': 'INR', 'T3_US_FIU': 'T3_US_FIU', 'T3_EU_Parrot': 'T3 EU Parrot', 'T2_RU_JINR': 'JINR', 'T2_IT_Pisa': 'Pisa', 'T2_GR_Ioannina': 'Ioannina', 'T3_US_MIT': 'T3 US MIT', 'T2_CH_CERN_HLT': 'CERN Tier-2 HLT', 'T2_MY_UPM_BIRUNI': 'UPM Biruni', 'T1_FR_CCIN2P3': 'CC-IN2P3', 'T2_FR_GRIF_IRFU': 'GRIF_IRFU', 'T3_US_UMiss': 'UMissHEP', 'T2_FR_CCIN2P3': 'CC-IN2P3 AF', 'T2_PL_Warsaw': 'Warsaw', 'T3_AS_Parrot': 'T3 AS Parrot', 'T2_US_MIT': 'MIT', 'T2_BE_IIHE': 'IIHE', 'T2_RU_ITEP': 'ITEP', 'T1_CH_CERN': 'CERN', 'T3_CH_PSI': 'PSI', 'T3_IT_Bologna': 'Bologna-T3'}
-  matchList = {'T2_FI_HIP': 'Helsinki Institute of Physics', 'T2_UK_SGrid_RALPP': 'Rutherford PPD', 'T2_FR_GRIF_LLR': 'GRIF_LLR', 'T2_UK_London_IC': 'IC', 'T3_US_Omaha': 'Firefly', 'T2_KR_KNU': 'KNU', 'T2_RU_SINP': 'SINP', 'T2_CH_CERN_AI': 'CERN Tier-2 AI', 'T1_TW_ASGC': 'ASGC', 'T3_US_Colorado': 'Colorado', 'T1_UK_RAL_Disk': 'RALDISK', 'T2_TH_CUNSTDA': 'CUNSTDA', 'T2_IT_Bari': 'Bari', 'T2_US_UCSD': 'UCSD', 'T2_RU_IHEP': 'IHEP', 'T1_RU_JINR': 'JINR-T1', 'T2_RU_RRC_KI': 'RRC_KI', 'T2_CH_CERN': 'CERN Tier-2', 'T2_US_Vanderbilt': 'Vanderbilt', 'T2_CH_CSCS': 'CSCS', 'T2_UA_KIPT': 'KIPT', 'T2_PK_NCP': 'NCP-LCG2', 'T2_RU_PNPI': 'PNPI', 'T2_IN_TIFR': 'TIFR', 'T2_FR_IPHC': 'IPHC', 'T1_US_FNAL': 'FNAL', 'T2_IT_Rome': 'Rome', 'T2_UK_London_Brunel': 'Brunel', 'T1_US_FNAL_Disk': 'FNALDISK', 'T2_EE_Estonia': 'Estonia', 'T2_CN_Beijing': 'Beijing', 'T2_US_Florida': 'Florida', 'T1_DE_KIT': 'KIT', 'T2_US_Wisconsin': 'Wisconsin', 'T2_HU_Budapest': 'Hungary', 'T2_DE_RWTH': 'RWTH', 'T2_BR_UERJ': 'HEPGRID_UERJ', 'T2_ES_IFCA': 'IFCA', 'T2_DE_DESY': 'DESY', 'T1_UK_RAL': 'RAL', 'T2_US_Caltech': 'Caltech', 'T2_TW_Taiwan': 'Taiwan', 'T0_CH_CERN': 'CERN Tier-0', 'T1_RU_JINR_Disk': 'JINR-T1DISK', 'T2_US_Nebraska': 'Nebraska', 'T2_ES_CIEMAT': 'CIEMAT', 'T2_CH_CERN_T0': 'CERN Tier-2 T0', 'T2_BR_SPRACE': 'SPRACE', 'T1_IT_CNAF': 'CNAF', 'T2_TR_METU': 'METU', 'T2_AT_Vienna': 'Hephy-Vienna', 'T2_US_Purdue': 'Purdue', 'T2_BE_UCL': 'Louvain', 'T2_UK_SGrid_Bristol': 'Bristol', 'T2_PT_NCG_Lisbon': 'NCG-INGRID-PT', 'T1_ES_PIC': 'PIC', 'T2_IT_Legnaro': 'Legnaro', 'T2_RU_INR': 'INR', 'T2_RU_JINR': 'JINR', 'T2_IT_Pisa': 'Pisa', 'T2_GR_Ioannina': 'Ioannina', 'T2_CH_CERN_HLT': 'CERN Tier-2 HLT', 'T2_MY_UPM_BIRUNI': 'UPM Biruni', 'T1_FR_CCIN2P3': 'CC-IN2P3', 'T2_FR_GRIF_IRFU': 'GRIF_IRFU', 'T2_FR_CCIN2P3': 'CC-IN2P3 AF', 'T2_PL_Warsaw': 'Warsaw', 'T2_US_MIT': 'MIT', 'T2_BE_IIHE': 'IIHE', 'T2_RU_ITEP': 'ITEP', 'T1_CH_CERN': 'CERN'}
 #_____________________________________________________________________________
   for site in matchList:
     if pledgeList.has_key(matchList[site]):
@@ -69,15 +85,14 @@ def matchPledges(pledgeList):
       	pledges[site] = "n/a"
   return pledges
 #____________________function creates JSON TXT file________________
-def savetoFile(pledges, year,outputfile_txt):
+def savetoFile(pledges, outputfile_txt):
   saveTime = time.strftime('%Y-%m-%d %H:%M:%S')
   url = "https://cmsweb.cern.ch/sitedb/prod/pledges "
   #_______________JSON__________________________________________________
   filename = outputfile_txt + ".json"
   fileOp = open(filename, "w")
   fileOp.write(unicode(simplejson.dumps(pledges, ensure_ascii=False)))
   fileOp.close()
-
   #_______________the List_____________________________________________
   filename = outputfile_txt + ".txt"
   fileOp = open(filename, "w")
@@ -86,13 +101,11 @@ def savetoFile(pledges, year,outputfile_txt):
       if (pledges[tmpPledges]      > 0)      : color = 'green'
       if (str(pledges[tmpPledges]) == 'n/a') : color = 'white'
       fileOp.write(saveTime + "\t" + tmpPledges + "\t" + str(pledges[tmpPledges]) + "\t" + color + "\t" +  url + "\n" )
-
   fileOp.close()
 
 if __name__ == '__main__':
   outputfile_txt=sys.argv[1]
-  year = sys.argv[2]
   print 'starting to fetch all pledges from siteDB'
   allPledgeList = fetch_all_pledges('cmsweb.cern.ch','/sitedb/data/prod/resource-pledges')
   pledges       = matchPledges(allPledgeList)
-  savetoFile(pledges, year, outputfile_txt)
+  savetoFile(pledges, outputfile_txt)
diff --git a/Pledges_View_SSB/pledge/run_pledges.sh b/Pledges_View_SSB/pledge/run_pledges.sh
@@ -2,15 +2,11 @@
 # written by Gökhan Kandemir => gokhan.kandemir@cern.ch
 # outputfile [year]_pledges.txt
 # outputfile [year]_pledges.json
-# outputfile [year]_pledges.html
-# usercert and userkey files must be in folder "data"
 # this script fetchs all pledges values and matchs with siteName from siteDB.
 clear
-echo "exporting KEY and CERT"
-
 #fixing access
-export X509_USER_CERT=./data/usercert.pem
-export X509_USER_KEY=./data/userkey.pem
+source /afs/cern.ch/project/gd/LCG-share/new_3.2/etc/profile.d/grid_env.sh
+voms-proxy-init -voms cms
 
 # Email if things are running slowly
 
@@ -40,10 +36,9 @@ fi
 
 
 #Run the script
-year="2014"
-txt="pledge"  #postfix in code itself
+txt="pledges"  #postfix in code itself
 echo "python pledges.py $txt"
-python pledges.py $txt $year &> pledges.log
+python pledges.py $txt &> pledges.log
 
 problem="$?"
 echo "problem: $problem"
@@ -53,3 +48,4 @@ cp $txt".txt"  /afs/cern.ch/user/c/cmst1/www/SST
 cp $txt".json"  /afs/cern.ch/user/c/cmst1/www/SST
 
 rm scriptRunning.run
+
diff --git a/SR_View_SSB/WRDays/run_WaitingRoom_Sites.sh b/SR_View_SSB/WRDays/run_WaitingRoom_Sites.sh
@@ -1,24 +1,15 @@
 #!/bin/bash
-# Sten Luyckx
 # Script in acrontab t1
 # 5,20,35,50 * * * * lxplus ssh vocms202 /afs/cern.ch/user/c/cmst1/scratch0/Waitingroom_Dashboard/Waitingroom_SummedMetric/run_WaitingRoom_Sites.sh &> /dev/null
 # Script for Dashboard metric 154, 155, 156
 # outputfile WaitingRoom_1MonthSum.txt
 # outputfile WaitingRoom_2MonthSum.txt
 # outputfile WaitingRoom_3MonthSum.txt
-# outputdir /afs/cern.ch/user/c/cmst1/www/WFMon/
-# usercert and userkey files must be in folder "data"
 # this script read all of data from http://dashb-ssb.cern.ch/dashboard/ according to column, dateFrom, dateTo, sites and it calculates How many days Sites are in WaitingRoom as last 1 month, last 2 months, last 3 months. 
 clear
-cd /afs/cern.ch/user/c/cmst1/scratch0/Waitingroom_Dashboard/Waitingroom_SummedMetric/
-
-echo "exporting KEY and CERT"
-
 #fixing access
-export X509_USER_CERT=/data/certs/servicecert.pem
-export X509_USER_KEY=/data/certs/servicekey.pem
-
-
+source /afs/cern.ch/project/gd/LCG-share/new_3.2/etc/profile.d/grid_env.sh
+voms-proxy-init -voms cms
 # Email if things are running slowly
 
 if [ -f scriptRunning.run ];
diff --git a/SR_View_SSB/WRDays/waitingRoom_SummedMetrics.py b/SR_View_SSB/WRDays/waitingRoom_SummedMetrics.py
@@ -39,10 +39,6 @@ def extractJson(col, startDate, endDate):
 
 # function needed to fetch a list of all sites from siteDB
 def fetch_all_sites(url,api):
-  # examples in:
-  # https://github.com/gutsche/old-scripts/blob/master/SiteDB/extract_site_executive_mail_addresses.py
-  # https://github.com/dballesteros7/dev-scripts/blob/master/src/reqmgr/input-blocks.py
-  # https://twiki.cern.ch/twiki/bin/view/CMSPublic/CompOpsWorkflowOperationsWMAgentScripts#Resubmit
   headers = {"Accept": "application/json"}
   if 'X509_USER_PROXY' in os.environ:
       print 'X509_USER_PROXY found'
@@ -64,6 +60,10 @@ def fetch_all_sites(url,api):
   conn.close()
 
   site_T2= []
+  for siteName in jn['result']:
+      if siteName[3][0:2] == 'T2':
+          site_T2.append(siteName[3])
+
   for i in jn['result']:
     if i[jn['desc']['columns'].index('type')]=='cms':
       sitedbname=i[jn['desc']['columns'].index('alias')]
@@ -142,7 +142,8 @@ def main_function(outputfile_txt, submonths,allSites):
 if __name__ == '__main__':
   outputfile_txt=sys.argv[1]
   print 'starting to fetch all sites from siteDB'
-  allSitesList = fetch_all_sites('cmsweb.cern.ch','/sitedb/data/prod/site-names')
+  #allSitesList = fetch_all_sites('cmsweb.cern.ch','/sitedb/data/prod/site-names')
+  allSitesList = fetch_all_sites('cmsweb.cern.ch','/sitedb/data/prod/federations-sites')
   main_function(outputfile_txt+'1MonthSum.txt',-1,allSitesList)
   print '__________________________________________________'
   print '__________________________________________________'
diff --git a/Site_Config/generic_file_monitoring/run_site_local_config.sh b/Site_Config/generic_file_monitoring/run_site_local_config.sh
@@ -1,6 +1,5 @@
 #!/bin/bash
 # written by Gökhan Kandemir => gokhan.kandemir@cern.ch
-# usercert and userkey files must be in folder .globus
 # this script calculates the number of T1s and T2s counts and writes results to console and file.
 clear
 echo "To Prevent ask typing password constantly, Registering your KEY and CERT to Grid Environment"
diff --git a/Site_Config/generic_file_monitoring/site_local_config.py b/Site_Config/generic_file_monitoring/site_local_config.py
@@ -3,16 +3,31 @@
 import time
 import urllib, httplib, urllib2
 import string
+import simplejson
 from xml.dom.minidom import parse, parseString
+from xml import xpath
 #_____________________________________________________________________________
 
 siteList = {}
 T1Count = 0
 T2Count = 0
 #________________________________________________________________________________________
 # functions fetchs all siteName and gets all xml file.
-def fetch_all_site():
-  sitesList =  ['T1_TW_ASGC','T1_FR_CCIN2P3','T1_CH_CERN','T1_IT_CNAF','T1_US_FNAL','T1_US_FNAL_Disk','T1_RU_JINR','T1_RU_JINR_Disk','T1_DE_KIT','T1_ES_PIC','T1_UK_RAL','T1_UK_RAL_Disk','T2_IT_Bari','T2_CN_Beijing','T2_UK_SGrid_Bristol','T2_UK_London_Brunel','T2_FR_CCIN2P3','T2_CH_CERN','T2_CH_CERN_AI','T2_CH_CERN_HLT','T2_CH_CERN_T0','T2_ES_CIEMAT','T2_CH_CSCS','T2_TH_CUNSTDA','T2_US_Caltech','T2_DE_DESY','T2_EE_Estonia','T2_US_Florida','T2_FR_GRIF_IRFU','T2_FR_GRIF_LLR','T2_BR_UERJ','T2_FI_HIP','T2_AT_Vienna','T2_HU_Budapest','T2_UK_London_IC','T2_ES_IFCA','T2_RU_IHEP','T2_BE_IIHE','T2_RU_INR', 'T2_FR_IPHC','T2_RU_ITEP','T2_GR_Ioannina','T2_RU_JINR','T2_UA_KIPT','T2_KR_KNU','T2_IT_Legnaro','T2_BE_UCL','T2_TR_METU','T2_US_MIT','T2_PT_NCG_Lisbon','T2_PK_NCP','T2_US_Nebraska','T2_RU_PNPI','T2_IT_Pisa','T2_US_Purdue', 'T2_RU_RRC_KI','T2_DE_RWTH','T2_IT_Rome','T2_UK_SGrid_RALPP','T2_RU_SINP','T2_BR_SPRACE','T2_IN_TIFR','T2_TW_Taiwan','T2_US_UCSD','T2_MY_UPM_BIRUNI', 'T2_US_Vanderbilt','T2_PL_Warsaw','T2_US_Wisconsin']
+def fetch_all_sites(url, api):
+  #________________fetch all siteName from siteDB v2________________________________
+  headers = {"Accept": "application/json"}
+  if 'X509_USER_PROXY' in os.environ:
+    conn = httplib.HTTPSConnection(url, cert_file = os.getenv('X509_USER_PROXY'), key_file = os.getenv('X509_USER_PROXY'))
+  r1=conn.request("GET",api, None, headers)
+  r2=conn.getresponse()
+  inputjson=r2.read()
+  jn = simplejson.loads(inputjson)
+  conn.close()
+  sitesList=[]
+  for site in jn['result']:
+    if site[3][0:2] != 'T3':
+       sitesList.append(site[3])    
+  #______________________________________________________________________________________
   # To get all site-local-config information.
   for siteName in sitesList:
     xml = getXmlfromURL('cmsweb.cern.ch','/gitweb/?p=siteconf/.git;a=blob_plain;f=' + siteName + '/JobConfig/site-local-config.xml;hb=HEAD')
@@ -66,7 +81,7 @@ def getXmlfromURL(url,api):
   jsonFile = open(outputfile_txt + ".json", "w")
   jsonCode = '{ "site_local_config":['
   jsonCodeEnd = ']}'
-  fetch_all_site()
+  fetch_all_sites('cmsweb.cern.ch','/sitedb/data/prod/federations-sites')
   saveTime = time.strftime('%Y-%m-%d %H:%M:%S')
   url = "https://cmsweb.cern.ch/gitweb/?p=siteconf/.git;a=tree"
   print saveTime + "\t" + "T1s" + "\t" + str(T1Count) + "\t" + "green" + "\t" + url
