Revert 5b2d0dd..46018fd

This rolls back to commit 5b2d0dd.

Author: John P. Artieda

GGUS/ggus.py

@@ -12,23 +12,33 @@
 #extract nonwaitingroommsites from ActiveSites script output
 url2 = "https://cmsdoc.cern.ch/cms/LCG/SiteComm/T2WaitingList/WasCommissionedT2ForSiteMonitor.txt"
 
-def extractJson():
-  url = "http://dashb-ssb.cern.ch/dashboard/request.py/getplotdata?columnid=153&time=24&dateFrom=&dateTo=&site=T2_AT_Vienna&sites=all&clouds=undefined&batch=1"
-  print "Getting the url %s" % url
-  request = urllib2.Request(url, headers = {"Accept":"application/json"})
-  response = urllib2.urlopen(request)
-  data = response.read()
-  rows = simplejson.loads(data)
-  return rows
-
-# function needed to fetch a list of all sites from metric
-def fetch_all_sites(jsn):
-  site_T2 = []
-  for row in jsn['csvdata']:
-    if row['VOName'][0:2] == 'T2':
-      site_T2.append(row['VOName'])
-  return site_T2
-
+# function needed to fetch a list of all sites from siteDB
+def fetch_all_sites(url,api):
+  # examples in:
+  # https://github.com/gutsche/old-scripts/blob/master/SiteDB/extract_site_executive_mail_addresses.py
+  # https://github.com/dballesteros7/dev-scripts/blob/master/src/reqmgr/input-blocks.py
+  # https://twiki.cern.ch/twiki/bin/view/CMSPublic/CompOpsWorkflowOperationsWMAgentScripts#Resubmit
+  headers = {"Accept": "application/json"}
+  if 'X509_USER_PROXY' in os.environ:
+      print 'X509_USER_PROXY found'
+      conn = httplib.HTTPSConnection(url, cert_file = os.getenv('X509_USER_PROXY'), key_file = os.getenv('X509_USER_PROXY'))
+  elif 'X509_USER_CERT' in os.environ and 'X509_USER_KEY' in os.environ:
+      print 'X509_USER_CERT and X509_USER_KEY found'
+      conn = httplib.HTTPSConnection(url, cert_file = os.getenv('X509_USER_CERT'), key_file = os.getenv('X509_USER_KEY'))
+  else:
+      print 'You need a valid proxy or cert/key files'
+      sys.exit()
+  print 'conn found in if else structure'
+  r1=conn.request("GET",api, None, headers)
+  print 'r1 passed'
+  r2=conn.getresponse()
+  print 'r2 passed'
+  inputjson=r2.read()
+  print '-------------------------------------------------------------'
+  #print inputjson
+  result = simplejson.loads(inputjson)
+  conn.close()
+  return result
 
 def getNonWaitingRoomSites(url):
     tmp_sites = []
@@ -41,6 +51,8 @@ def getNonWaitingRoomSites(url):
       if not row.strip().startswith("#"):
         split_row = row.split('\t')
         if len(split_row)>1:
+          #print split_row
+          #print split_row[1] 
           tmp_sites.append(split_row[1]) 
     return tmp_sites
 
@@ -53,8 +65,32 @@ def main_function(outputfile_txt):
   print '------------------------------------------'
 
   # all sites
-  print 'starting to fetch all sites from metric'
-  site_T2= fetch_all_sites(extractJson())
+  print 'starting to fetch all sites from siteDB'
+  jn = fetch_all_sites('cmsweb.cern.ch','/sitedb/data/prod/site-names') 
+  #json form:
+  #{"desc": {"columns": ["type", "site_name", "alias"]}, 
+  # "result": [
+  #             ["cms", "ASGC", "T1_TW_ASGC"],
+  #             ["cms", "BY-NCPHEP", "T3_BY_NCPHEP"]
+  #            ...
+  
+  # match the information
+  #print jn['result']
+  #site_T1= []
+  site_T2= []
+  #site_T3= []
+  #print 'printing i'
+  for i in jn['result']:
+    if i[jn['desc']['columns'].index('type')]=='cms':
+      sitedbname=i[jn['desc']['columns'].index('alias')]
+      #print sitedbname
+      #if 'T1' in sitedbname: 
+      #  site_T1.append(sitedbname)
+      if 'T2' in sitedbname:
+        site_T2.append(sitedbname)
+      #if 'T3' in sitedbname: 
+      #   site_T3.append(sitedbname)
+  #print site_T2
   print '--------------------------------------------------------'
   print 'Sites in waiting room:'
   waitingRoom_sites = [ site for site in site_T2 if not site in nonWaitingRoom_Sites]
@@ -90,4 +126,4 @@ def main_function(outputfile_txt):
 
 if __name__ == '__main__':
   outputfile_txt=sys.argv[1]
-  main_function(outputfile_txt)
+  main_function(outputfile_txt)

GGUS/readme.rm

@@ -1,15 +1,24 @@
 #!/bin/bash
+# Sten Luyckx
 # Script in acrontab t1
 # 5,20,35,50 * * * * lxplus ssh vocms202 /afs/cern.ch/user/c/cmst1/scratch0/Waitingroom_Dashboard/Waitingroom_SummedMetric/run_WaitingRoom_Sites.sh &> /dev/null
 # Script for Dashboard metric 154, 155, 156
 # outputfile WaitingRoom_1MonthSum.txt
 # outputfile WaitingRoom_2MonthSum.txt
 # outputfile WaitingRoom_3MonthSum.txt
+# outputdir /afs/cern.ch/user/c/cmst1/www/WFMon/
+# usercert and userkey files must be in folder "data"
 # this script read all of data from http://dashb-ssb.cern.ch/dashboard/ according to column, dateFrom, dateTo, sites and it calculates How many days Sites are in WaitingRoom as last 1 month, last 2 months, last 3 months. 
 clear
+cd /afs/cern.ch/user/c/cmst1/scratch0/Waitingroom_Dashboard/Waitingroom_SummedMetric/
+
+echo "exporting KEY and CERT"
+
 #fixing access
-#source /afs/cern.ch/project/gd/LCG-share/new_3.2/etc/profile.d/grid_env.sh
-#voms-proxy-init -voms cms
+export X509_USER_CERT=/data/certs/servicecert.pem
+export X509_USER_KEY=/data/certs/servicekey.pem
+
+
 # Email if things are running slowly
 
 if [ -f scriptRunning.run ];
@@ -49,4 +58,4 @@ echo "problem: $problem"
 
 cp $txt*.txt /afs/cern.ch/user/c/cmst1/www/WFMon/
 echo "WaitingRoom_XMonthSum.txt files copied to: /afs/cern.ch/user/c/cmst1/www/WFMon/ "
-rm scriptRunning.run
+rm scriptRunning.run

GGUS/run_ggus.sh

@@ -37,24 +37,54 @@ def extractJson(col, startDate, endDate):
 
 #_______________________________________________________________________
 
-# function needed to fetch a list of all sites from metric 153
-def fetch_all_sites(jsn):
-  site_T2 = []
-  for row in jsn['csvdata']:
-    if row['VOName'][0:2] == 'T2':
-      site_T2.append(row['VOName'])
+# function needed to fetch a list of all sites from siteDB
+def fetch_all_sites(url,api):
+  # examples in:
+  # https://github.com/gutsche/old-scripts/blob/master/SiteDB/extract_site_executive_mail_addresses.py
+  # https://github.com/dballesteros7/dev-scripts/blob/master/src/reqmgr/input-blocks.py
+  # https://twiki.cern.ch/twiki/bin/view/CMSPublic/CompOpsWorkflowOperationsWMAgentScripts#Resubmit
+  headers = {"Accept": "application/json"}
+  if 'X509_USER_PROXY' in os.environ:
+      print 'X509_USER_PROXY found'
+      conn = httplib.HTTPSConnection(url, cert_file = os.getenv('X509_USER_PROXY'), key_file = os.getenv('X509_USER_PROXY'))
+  elif 'X509_USER_CERT' in os.environ and 'X509_USER_KEY' in os.environ:
+      print 'X509_USER_CERT and X509_USER_KEY found'
+      conn = httplib.HTTPSConnection(url, cert_file = os.getenv('X509_USER_CERT'), key_file = os.getenv('X509_USER_KEY'))
+  else:
+      print 'You need a valid proxy or cert/key files'
+      sys.exit()
+  print 'conn found in if else structure'
+  r1=conn.request("GET",api, None, headers)
+  print 'r1 passed'
+  r2=conn.getresponse()
+  print 'r2 passed'
+  inputjson=r2.read()
+  print '-------------------------------------------------------------'
+  jn = simplejson.loads(inputjson)
+  conn.close()
+
+  site_T2= []
+  for i in jn['result']:
+    if i[jn['desc']['columns'].index('type')]=='cms':
+      sitedbname=i[jn['desc']['columns'].index('alias')]
+      if 'T2' in sitedbname:
+        site_T2.append(sitedbname)
   return site_T2
 #_______________________________________________________________________
 
-def main_function(outputfile_txt, submonths):
+def main_function(outputfile_txt, submonths,allSites):
   print outputfile_txt
   columnnumber='153'
   enddate=date.today() # query date
   startdate=addmonths(enddate,submonths) # query date
   start=startdate.strftime("%Y-%m-%d") # as string field to query
   end=enddate.strftime("%Y-%m-%d") # as string field to query
   jsn=extractJson(columnnumber, start, end) # results from dashboard extracts into JSON file
-  allSites = fetch_all_sites(jsn) 
+  filename = 'tmpjson' + start + '.json'
+  f=open(filename,'w')
+  f.write(unicode(simplejson.dumps(jsn, ensure_ascii=False)))
+  f.close()
+ 
   # sum @ days 
   days_per_site={}
   for site in allSites: # Read all sites from allsites variable
@@ -93,12 +123,12 @@ def main_function(outputfile_txt, submonths):
     	print '------------------------------'
   print days_per_site
 
+
   # write to file
-  #f1=open('./'+outputfile_txt, 'w+')
-  f1=open('/afs/cern.ch/user/c/cmst1/scratch0/Waitingroom_Dashboard/Waitingroom_SummedMetric/'+outputfile_txt, 'w')
+  f1=open('./'+outputfile_txt, 'w+')
   f1.write('# This txt goes into SSB and shows the number of days a site has been in the Waiting Room for X months --> See filename)\n')
   f1.write('# Readme:\n# https://raw.githubusercontent.com/CMSCompOps/MonitoringScripts/master/SR_View_SSB/WRDays/Readme.txt\n')
-  now_write = time.strftime('%Y-%m-%d %H:%M:%S')
+  now_write=(datetime.utcnow()).strftime("%Y-%m-%d %H:%M:%S")
   print "Local current time :", now_write
   link="https://dashb-ssb.cern.ch/dashboard/request.py/siteviewhistorywithstatistics?columnid=153#time=2184&start_date=&end_date=&use_downtimes=false&merge_colors=false&sites=all"
   for key, number in days_per_site.iteritems():
@@ -112,12 +142,12 @@ def main_function(outputfile_txt, submonths):
 if __name__ == '__main__':
   outputfile_txt=sys.argv[1]
   print 'starting to fetch all sites from siteDB'
-  main_function(outputfile_txt+'1MonthSum.txt',-1)
+  allSitesList = fetch_all_sites('cmsweb.cern.ch','/sitedb/data/prod/site-names')
+  main_function(outputfile_txt+'1MonthSum.txt',-1,allSitesList)
   print '__________________________________________________'
   print '__________________________________________________'
-  main_function(outputfile_txt+'2MonthSum.txt',-2)
+  main_function(outputfile_txt+'2MonthSum.txt',-2,allSitesList)
   print '__________________________________________________'
   print '__________________________________________________'
-  main_function(outputfile_txt+'3MonthSum.txt',-3)
-
+  main_function(outputfile_txt+'3MonthSum.txt',-3,allSitesList)