Skip to content

Commit 2da24e9

Browse files
author
artiedaj
committed
Merge pull request #37 from gkandemi/master
WRControl, WRDays, GGUS, gLExec
2 parents 49b519e + 8c0dced commit 2da24e9

20 files changed

Lines changed: 35440 additions & 113 deletions

‎GGUS/ggus.py‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
import os,sys,time
2+
from xml.dom.minidom import parse, parseString
3+
import simplejson
4+
import simplejson as json
5+
import httplib
6+
import operator
7+
8+
#_______________________________________________________________
9+
def getJSONfromSiteDB():
10+
headers = {"Accept": "application/json"}
11+
url = "cmsweb.cern.ch"
12+
api = "/sitedb/data/prod/site-names"
13+
if 'X509_USER_PROXY' in os.environ:
14+
print 'X509_USER_PROXY found'
15+
conn = httplib.HTTPSConnection(url, cert_file = os.getenv('X509_USER_PROXY'), key_file = os.getenv('X509_USER_PROXY'))
16+
elif 'X509_USER_CERT' in os.environ and 'X509_USER_KEY' in os.environ:
17+
print 'X509_USER_CERT and X509_USER_KEY found'
18+
conn = httplib.HTTPSConnection(url, cert_file = os.getenv('X509_USER_CERT'), key_file = os.getenv('X509_USER_KEY'))
19+
elif os.path.isfile('/data/certs/servicecert.pem') and os.path.isfile('/data/certs/servicekey.pem'):
20+
conn = httplib.HTTPSConnection(url, cert_file = '/data/certs/servicecert.pem', key_file = '/data/certs/servicekey.pem')
21+
else:
22+
print 'You need a valid proxy or cert/key files'
23+
sys.exit()
24+
r1=conn.request("GET",api, None, headers)
25+
r2=conn.getresponse()
26+
inputjson=r2.read()
27+
jn = simplejson.loads(inputjson)
28+
conn.close()
29+
return jn
30+
31+
#______________________________________________________________
32+
def convertLCGtoCMS(jn, getlcgName, mode):
33+
lcgNames = {}
34+
for row in jn['result']:
35+
if row[0] == 'lcg':
36+
lcgNames[row[2]] = row[1]
37+
siteNames = {}
38+
for row in jn['result']:
39+
if row[0] == 'cms':
40+
siteNames[row[1]] = row[2]
41+
lcgtoCms = {}
42+
43+
for lcgName in lcgNames:
44+
if siteNames.has_key(lcgNames[lcgName]):
45+
lcgtoCms[lcgName] = siteNames[lcgNames[lcgName]]
46+
47+
if mode != 'cms':
48+
if not lcgtoCms.has_key(getlcgName):
49+
return 'cms'
50+
else:
51+
return lcgtoCms[getlcgName]
52+
else:
53+
return lcgtoCms
54+
55+
#______________________________________________________________
56+
def ReadXML(fileName):
57+
fileHandle = open(fileName)
58+
data = fileHandle.read()
59+
if "“" in data:
60+
data = data.replace("“" , " ")
61+
if "”" in data:
62+
data = data.replace("”" , " ")
63+
fileHandle.close()
64+
return data
65+
66+
#______________________________________________________________
67+
def getTicketNumbers(dom, lcgName):
68+
tag = dom.getElementsByTagName('tickets')[0].childNodes
69+
count = len(tag)
70+
ticketlist = []
71+
for i in range(count):
72+
try:
73+
siteName = dom.getElementsByTagName('tickets')[0].getElementsByTagName('ticket')[i].getElementsByTagName('affected_site')[0].firstChild.nodeValue
74+
except AttributeError:
75+
continue
76+
if siteName == lcgName:
77+
ticketNumber = dom.getElementsByTagName('tickets')[0].getElementsByTagName('ticket')[i].getElementsByTagName('request_id')[0].firstChild.nodeValue
78+
url = "https://ggus.eu/?mode=ticket_info&ticket_id=" + ticketNumber
79+
ticketURL = "[" + "[" + url + "]" + "[" + ticketNumber + "]" + "]"
80+
ticketlist.append(ticketURL)
81+
return ticketlist
82+
#______________________________________________________________
83+
84+
if __name__ == '__main__':
85+
json = getJSONfromSiteDB()
86+
text = ReadXML('/afs/cern.ch/user/c/cmst1/scratch0/ggus/tickets.xml')
87+
dom = parseString(text)
88+
tag = dom.getElementsByTagName('tickets')[0].childNodes
89+
count = len(tag)
90+
sites = {}
91+
#________________________get ticket information and calculate number of tickets______________________________
92+
for i in range(count):
93+
try:
94+
siteName = dom.getElementsByTagName('tickets')[0].getElementsByTagName('ticket')[i].getElementsByTagName('affected_site')[0].firstChild.nodeValue
95+
except AttributeError:
96+
continue
97+
if not sites.has_key(siteName): sites[siteName] = 0
98+
sites[siteName] += 1
99+
#____________________________________________________________________________________________________________
100+
saveTime = time.strftime('%Y-%m-%d %H:%M:%S')
101+
fileOp = open("/afs/cern.ch/user/c/cmst1/scratch0/ggus/ggusticketmetrics.txt" , "w")
102+
fileMeet = open("/afs/cern.ch/user/c/cmst1/scratch0/ggus/ggusticketmeeting.txt" , "w")
103+
fileMeet.write("| *Order* | *SiteName* | *Ticket Count* | *Tickets* |" + "\n")
104+
lcgNames = convertLCGtoCMS(json, '', 'cms')
105+
order = 0
106+
for lcgName in lcgNames:
107+
color = "green"
108+
if sites.has_key(lcgName):
109+
cmsSiteName = convertLCGtoCMS(json, lcgName, 'lcg')
110+
ticketCount = sites[lcgName]
111+
order = order + 1
112+
#________________________To write Tickets List for CompOps Meeting______________________________
113+
fileMeet.write(" | " + str(order) + " | " + cmsSiteName + " | " + str(ticketCount) + " | " + ' , '.join(getTicketNumbers(dom, lcgName)) + " | " + "\n")
114+
#_______________________________________________________________________________________________
115+
else:
116+
cmsSiteName = lcgNames[lcgName]
117+
ticketCount = 0
118+
url = "http://para.ms/GGUS/" + lcgName
119+
if ticketCount > 0 : color = "red"
120+
i = i + 1
121+
#________________________To write sites for metricg______________________________
122+
fileOp.write(saveTime + "\t" + cmsSiteName + "\t" + str(ticketCount) + "\t" + color + "\t" + url + "\n" )
123+
#________________________________________________________________________________
124+
fileMeet.close()
125+
fileOp.close()

‎GGUS/readme.rm‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# Author : Gökhan Kandemir
2+
# E-mail : gokhan.kandemir@cern.ch
3+
# Script Responsible : Site Support Team
4+
# Script is located : /afs/cern.ch/user/c/cmst1/scratch0/ggus
5+
# Outputs :
6+
# /afs/cern.ch/user/c/cmst1/www/SST/ggusticketmeeting.txt (https://cmst1.web.cern.ch/CMST1/SST/ggusticketmeeting.txt)
7+
# /afs/cern.ch/user/c/cmst1/www/SST/ggusticketmetrics.txt (https://cmst1.web.cern.ch/CMST1/SST/ggusticketmetrics.txt)
8+
# The script is being run by an acronjob
9+
# The acronojb is in the acrontab of the user: cmst1
10+
# 5,20,35,50 * * * * lxplus ssh vocms202 /afs/cern.ch/user/c/cmst1/scratch0/ggus/run_ggus.sh &> /dev/null
11+
# Description : The script is used for getting ggus ticket information for each site. Also it calcultes how many tickets the site has and it creates 2 files. One of them is for metric 198 [1], the other one is for twiki table.
12+
13+
[1] https://dashb-ssb.cern.ch/dashboard/request.py/siteviewhistory?columnid=198&view=Site%20Readiness
14+
[2] https://cmst1.web.cern.ch/CMST1/SST/ggusticketmeeting.txt

‎GGUS/run_ggus.sh‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/bin/bash
2+
# Script in cmst1 acrontab
3+
# 5,20,35,50 * * * * lxplus ssh vocms202 /afs/cern.ch/user/c/cmst1/scratch0/ggus/run_ggus.sh &> /dev/null
4+
# Script for Dashboard metric
5+
# outputfile ggusticketmetrics.txt
6+
# outputfile ticketmeeting.txt
7+
# outputfile tickets.xml
8+
# Script obtains ggus ticket information and creates 2 files. One of them is ggusticketmetrics.txt for metric in dashboard, the other one is ticketmeeting for CompOps Meeting
9+
curl -kvv --cert /data/certs/servicecert.pem --key /data/certs/servicekey.pem "https://ggus.eu/index.php?mode=ticket_search&show_columns_check%5B%5D=TICKET_TYPE&show_columns_check%5B%5D=AFFECTED_VO&show_columns_check%5B%5D=AFFECTED_SITE&show_columns_check%5B%5D=PRIORITY&show_columns_check%5B%5D=RESPONSIBLE_UNIT&show_columns_check%5B%5D=STATUS&show_columns_check%5B%5D=DATE_OF_CHANGE&show_columns_check%5B%5D=SHORT_DESCRIPTION&ticket_id=&supportunit=&su_hierarchy=0&vo=cms&cms_su=&user=&keyword=&involvedsupporter=&assignedto=&affectedsite=&specattrib=none&status=open&priority=&typeofproblem=&ticket_category=all&mouarea=&date_type=creation+date&tf_radio=1&timeframe=any&from_date=&to_date=&untouched_date=&orderticketsby=REQUEST_ID&orderhow=desc&search_submit=GO%21&writeFormat=XML" -o /afs/cern.ch/user/c/cmst1/scratch0/ggus/tickets.xml
10+
python /afs/cern.ch/user/c/cmst1/scratch0/ggus/ggus.py &> /afs/cern.ch/user/c/cmst1/scratch0/ggus/ticket.log
11+
cp /afs/cern.ch/user/c/cmst1/scratch0/ggus/ggusticketmetrics.txt /afs/cern.ch/user/c/cmst1/www/SST/
12+
cp /afs/cern.ch/user/c/cmst1/scratch0/ggus/ggusticketmeeting.txt /afs/cern.ch/user/c/cmst1/www/SST/

‎SR_View_SSB/WRControl/WaitingRoom_Sites.py‎

Lines changed: 20 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -12,33 +12,23 @@
1212
#extract nonwaitingroommsites from ActiveSites script output
1313
url2 = "https://cmsdoc.cern.ch/cms/LCG/SiteComm/T2WaitingList/WasCommissionedT2ForSiteMonitor.txt"
1414

15-
# function needed to fetch a list of all sites from siteDB
16-
def fetch_all_sites(url,api):
17-
# examples in:
18-
# https://github.com/gutsche/old-scripts/blob/master/SiteDB/extract_site_executive_mail_addresses.py
19-
# https://github.com/dballesteros7/dev-scripts/blob/master/src/reqmgr/input-blocks.py
20-
# https://twiki.cern.ch/twiki/bin/view/CMSPublic/CompOpsWorkflowOperationsWMAgentScripts#Resubmit
21-
headers = {"Accept": "application/json"}
22-
if 'X509_USER_PROXY' in os.environ:
23-
print 'X509_USER_PROXY found'
24-
conn = httplib.HTTPSConnection(url, cert_file = os.getenv('X509_USER_PROXY'), key_file = os.getenv('X509_USER_PROXY'))
25-
elif 'X509_USER_CERT' in os.environ and 'X509_USER_KEY' in os.environ:
26-
print 'X509_USER_CERT and X509_USER_KEY found'
27-
conn = httplib.HTTPSConnection(url, cert_file = os.getenv('X509_USER_CERT'), key_file = os.getenv('X509_USER_KEY'))
28-
else:
29-
print 'You need a valid proxy or cert/key files'
30-
sys.exit()
31-
print 'conn found in if else structure'
32-
r1=conn.request("GET",api, None, headers)
33-
print 'r1 passed'
34-
r2=conn.getresponse()
35-
print 'r2 passed'
36-
inputjson=r2.read()
37-
print '-------------------------------------------------------------'
38-
#print inputjson
39-
result = simplejson.loads(inputjson)
40-
conn.close()
41-
return result
15+
def extractJson():
16+
url = "http://dashb-ssb.cern.ch/dashboard/request.py/getplotdata?columnid=153&time=24&dateFrom=&dateTo=&site=T2_AT_Vienna&sites=all&clouds=undefined&batch=1"
17+
print "Getting the url %s" % url
18+
request = urllib2.Request(url, headers = {"Accept":"application/json"})
19+
response = urllib2.urlopen(request)
20+
data = response.read()
21+
rows = simplejson.loads(data)
22+
return rows
23+
24+
# function needed to fetch a list of all sites from metric
25+
def fetch_all_sites(jsn):
26+
site_T2 = []
27+
for row in jsn['csvdata']:
28+
if row['VOName'][0:2] == 'T2':
29+
site_T2.append(row['VOName'])
30+
return site_T2
31+
4232

4333
def getNonWaitingRoomSites(url):
4434
tmp_sites = []
@@ -51,8 +41,6 @@ def getNonWaitingRoomSites(url):
5141
if not row.strip().startswith("#"):
5242
split_row = row.split('\t')
5343
if len(split_row)>1:
54-
#print split_row
55-
#print split_row[1]
5644
tmp_sites.append(split_row[1])
5745
return tmp_sites
5846

@@ -65,32 +53,8 @@ def main_function(outputfile_txt):
6553
print '------------------------------------------'
6654

6755
# all sites
68-
print 'starting to fetch all sites from siteDB'
69-
jn = fetch_all_sites('cmsweb.cern.ch','/sitedb/data/prod/site-names')
70-
#json form:
71-
#{"desc": {"columns": ["type", "site_name", "alias"]},
72-
# "result": [
73-
# ["cms", "ASGC", "T1_TW_ASGC"],
74-
# ["cms", "BY-NCPHEP", "T3_BY_NCPHEP"]
75-
# ...
76-
77-
# match the information
78-
#print jn['result']
79-
#site_T1= []
80-
site_T2= []
81-
#site_T3= []
82-
#print 'printing i'
83-
for i in jn['result']:
84-
if i[jn['desc']['columns'].index('type')]=='cms':
85-
sitedbname=i[jn['desc']['columns'].index('alias')]
86-
#print sitedbname
87-
#if 'T1' in sitedbname:
88-
# site_T1.append(sitedbname)
89-
if 'T2' in sitedbname:
90-
site_T2.append(sitedbname)
91-
#if 'T3' in sitedbname:
92-
# site_T3.append(sitedbname)
93-
#print site_T2
56+
print 'starting to fetch all sites from metric'
57+
site_T2= fetch_all_sites(extractJson())
9458
print '--------------------------------------------------------'
9559
print 'Sites in waiting room:'
9660
waitingRoom_sites = [ site for site in site_T2 if not site in nonWaitingRoom_Sites]
@@ -126,4 +90,4 @@ def main_function(outputfile_txt):
12690

12791
if __name__ == '__main__':
12892
outputfile_txt=sys.argv[1]
129-
main_function(outputfile_txt)
93+
main_function(outputfile_txt)

‎SR_View_SSB/WRDays/run_WaitingRoom_Sites.sh‎

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,15 @@
11
#!/bin/bash
2-
# Sten Luyckx
32
# Script in acrontab t1
43
# 5,20,35,50 * * * * lxplus ssh vocms202 /afs/cern.ch/user/c/cmst1/scratch0/Waitingroom_Dashboard/Waitingroom_SummedMetric/run_WaitingRoom_Sites.sh &> /dev/null
54
# Script for Dashboard metric 154, 155, 156
65
# outputfile WaitingRoom_1MonthSum.txt
76
# outputfile WaitingRoom_2MonthSum.txt
87
# outputfile WaitingRoom_3MonthSum.txt
9-
# outputdir /afs/cern.ch/user/c/cmst1/www/WFMon/
10-
# usercert and userkey files must be in folder "data"
118
# this script read all of data from http://dashb-ssb.cern.ch/dashboard/ according to column, dateFrom, dateTo, sites and it calculates How many days Sites are in WaitingRoom as last 1 month, last 2 months, last 3 months.
129
clear
13-
cd /afs/cern.ch/user/c/cmst1/scratch0/Waitingroom_Dashboard/Waitingroom_SummedMetric/
14-
15-
echo "exporting KEY and CERT"
16-
1710
#fixing access
18-
export X509_USER_CERT=/data/certs/servicecert.pem
19-
export X509_USER_KEY=/data/certs/servicekey.pem
20-
21-
11+
#source /afs/cern.ch/project/gd/LCG-share/new_3.2/etc/profile.d/grid_env.sh
12+
#voms-proxy-init -voms cms
2213
# Email if things are running slowly
2314

2415
if [ -f scriptRunning.run ];
@@ -58,4 +49,4 @@ echo "problem: $problem"
5849

5950
cp $txt*.txt /afs/cern.ch/user/c/cmst1/www/WFMon/
6051
echo "WaitingRoom_XMonthSum.txt files copied to: /afs/cern.ch/user/c/cmst1/www/WFMon/ "
61-
rm scriptRunning.run
52+
rm scriptRunning.run

‎SR_View_SSB/WRDays/waitingRoom_SummedMetrics.py‎

Lines changed: 15 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -37,54 +37,24 @@ def extractJson(col, startDate, endDate):
3737

3838
#_______________________________________________________________________
3939

40-
# function needed to fetch a list of all sites from siteDB
41-
def fetch_all_sites(url,api):
42-
# examples in:
43-
# https://github.com/gutsche/old-scripts/blob/master/SiteDB/extract_site_executive_mail_addresses.py
44-
# https://github.com/dballesteros7/dev-scripts/blob/master/src/reqmgr/input-blocks.py
45-
# https://twiki.cern.ch/twiki/bin/view/CMSPublic/CompOpsWorkflowOperationsWMAgentScripts#Resubmit
46-
headers = {"Accept": "application/json"}
47-
if 'X509_USER_PROXY' in os.environ:
48-
print 'X509_USER_PROXY found'
49-
conn = httplib.HTTPSConnection(url, cert_file = os.getenv('X509_USER_PROXY'), key_file = os.getenv('X509_USER_PROXY'))
50-
elif 'X509_USER_CERT' in os.environ and 'X509_USER_KEY' in os.environ:
51-
print 'X509_USER_CERT and X509_USER_KEY found'
52-
conn = httplib.HTTPSConnection(url, cert_file = os.getenv('X509_USER_CERT'), key_file = os.getenv('X509_USER_KEY'))
53-
else:
54-
print 'You need a valid proxy or cert/key files'
55-
sys.exit()
56-
print 'conn found in if else structure'
57-
r1=conn.request("GET",api, None, headers)
58-
print 'r1 passed'
59-
r2=conn.getresponse()
60-
print 'r2 passed'
61-
inputjson=r2.read()
62-
print '-------------------------------------------------------------'
63-
jn = simplejson.loads(inputjson)
64-
conn.close()
65-
66-
site_T2= []
67-
for i in jn['result']:
68-
if i[jn['desc']['columns'].index('type')]=='cms':
69-
sitedbname=i[jn['desc']['columns'].index('alias')]
70-
if 'T2' in sitedbname:
71-
site_T2.append(sitedbname)
40+
# function needed to fetch a list of all sites from metric 153
41+
def fetch_all_sites(jsn):
42+
site_T2 = []
43+
for row in jsn['csvdata']:
44+
if row['VOName'][0:2] == 'T2':
45+
site_T2.append(row['VOName'])
7246
return site_T2
7347
#_______________________________________________________________________
7448

75-
def main_function(outputfile_txt, submonths,allSites):
49+
def main_function(outputfile_txt, submonths):
7650
print outputfile_txt
7751
columnnumber='153'
7852
enddate=date.today() # query date
7953
startdate=addmonths(enddate,submonths) # query date
8054
start=startdate.strftime("%Y-%m-%d") # as string field to query
8155
end=enddate.strftime("%Y-%m-%d") # as string field to query
8256
jsn=extractJson(columnnumber, start, end) # results from dashboard extracts into JSON file
83-
filename = 'tmpjson' + start + '.json'
84-
f=open(filename,'w')
85-
f.write(unicode(simplejson.dumps(jsn, ensure_ascii=False)))
86-
f.close()
87-
57+
allSites = fetch_all_sites(jsn)
8858
# sum @ days
8959
days_per_site={}
9060
for site in allSites: # Read all sites from allsites variable
@@ -123,12 +93,12 @@ def main_function(outputfile_txt, submonths,allSites):
12393
print '------------------------------'
12494
print days_per_site
12595

126-
12796
# write to file
128-
f1=open('./'+outputfile_txt, 'w+')
97+
#f1=open('./'+outputfile_txt, 'w+')
98+
f1=open('/afs/cern.ch/user/c/cmst1/scratch0/Waitingroom_Dashboard/Waitingroom_SummedMetric/'+outputfile_txt, 'w')
12999
f1.write('# This txt goes into SSB and shows the number of days a site has been in the Waiting Room for X months --> See filename)\n')
130100
f1.write('# Readme:\n# https://raw.githubusercontent.com/CMSCompOps/MonitoringScripts/master/SR_View_SSB/WRDays/Readme.txt\n')
131-
now_write=(datetime.utcnow()).strftime("%Y-%m-%d %H:%M:%S")
101+
now_write = time.strftime('%Y-%m-%d %H:%M:%S')
132102
print "Local current time :", now_write
133103
link="https://dashb-ssb.cern.ch/dashboard/request.py/siteviewhistorywithstatistics?columnid=153#time=2184&start_date=&end_date=&use_downtimes=false&merge_colors=false&sites=all"
134104
for key, number in days_per_site.iteritems():
@@ -142,12 +112,12 @@ def main_function(outputfile_txt, submonths,allSites):
142112
if __name__ == '__main__':
143113
outputfile_txt=sys.argv[1]
144114
print 'starting to fetch all sites from siteDB'
145-
allSitesList = fetch_all_sites('cmsweb.cern.ch','/sitedb/data/prod/site-names')
146-
main_function(outputfile_txt+'1MonthSum.txt',-1,allSitesList)
115+
main_function(outputfile_txt+'1MonthSum.txt',-1)
147116
print '__________________________________________________'
148117
print '__________________________________________________'
149-
main_function(outputfile_txt+'2MonthSum.txt',-2,allSitesList)
118+
main_function(outputfile_txt+'2MonthSum.txt',-2)
150119
print '__________________________________________________'
151120
print '__________________________________________________'
152-
main_function(outputfile_txt+'3MonthSum.txt',-3,allSitesList)
121+
main_function(outputfile_txt+'3MonthSum.txt',-3)
122+
153123

0 commit comments

Comments
 (0)