SourceFEWSAdapter/pre.py at master · eWaterTest/SourceFEWSAdapter · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
import clr
import sys
import traceback
import System
import xml.dom.minidom

sys.path.Add("C:\\src\\SourceFEWSAdapter\\SourceFEWSAdapter.Core\\bin\\Debug")
sys.path.Add("C:\\src\\trunk\\Solutions\\Output")

clr.AddReferenceToFile("SourceFEWSAdapter.Core.dll")
clr.AddReferenceToFile("TIME.dll")
import TIME
io = TIME.Management.NonInteractiveIO
TimeSeries = TIME.DataTypes.TimeSeries
TimeStep = TIME.DataTypes.TimeStep

import SourceFEWSAdapter.Core
proxy = SourceFEWSAdapter.Core.FEWSPIProxy
timeStepUnits = SourceFEWSAdapter.Core.FEWSPI.timeStepUnitEnumStringType
var = SourceFEWSAdapter.Core.Diagnostics

def timeStepFromPI(piData):
  if piData.unit == timeStepUnits.second:
    if piData.multiplier == '3600':
      return(TimeStep.Hourly)
    if piData.multiplier == '86400':
      return(TimeStep.Daily)
  None

def buildTimeSeries(pi_timeseries):
  h = pi_timeseries.header
  tsStart = proxy.DateTimeFromPI(h.startDate)
  tsEnd = proxy.DateTimeFromPI(h.endDate)
  tsStep = timeStepFromPI(h.timeStep)
  timeSeries = TimeSeries(tsStart,tsEnd,tsStep)

  for e in pi_timeseries.event:
    dt = proxy.Merge(e.date,e.time)
    timeSeries.setTime(dt,float(e.value))

  timeSeries.name = h.parameterId + "_" + h.locationId
  return(timeSeries)

def fileForParameter(parameter):
	marker = 'ExpectedFile_'
	for property in run_data.properties.Items:
		if property.key.rfind(marker) == 0:
			if property.key.replace(marker,"") == parameter:
				return property.value
	return "defaultFile.csv"


# def writeLogFile(path):
   # doc = xml.dom.minidom.Document()
   # diag = doc.createElement("Diag")
   # diag.attributes["xmlns:xsi"]="http://www.w3.org/2001/XMLSchema-instance"
   # diag.attributes["xmlns"]="http://www.wldelft.nl/fews/PI"
   # diag.attributes["xsi:schemaLocation"]="http://www.wldelft.nl/fews/PI http://fews.wldelft.nl/schemas/version1.0/pi-schemas/pi_run.xsd"
   # diag.attributes["version"]="1.5"
   # doc.childNodes.append(diag)

   # line = doc.createElement("line")
   # line.attributes["level"] = "3"
   # line.attributes["description"] = "Success"
  ##text = doc.createTextNode("Preprocesser successful")
  ##line.childNodes.append(text)

   # diag.childNodes.append(line)
   # doc.writexml(open(path,'a'),addindent="  ")

#def writeLogFile(path):
#  diagnostics = Diagnostics(path)
#  diagnostics.Log(3,"Using the C# proxy")
#  diagnostics.Save()


#print sys.argv
run_file = None
run_file = sys.argv[1]
if run_file is None:
  pi_path = "C:\\FEWS\\Ovens\\OvensFEWS\\Modules\\source\\ovens\\input\\"
  run_file = pi_path+"RunParameters.xml"

run_data = proxy.ReadRunFile(run_file)
#sys.stdout = open(run_data.outputDiagnosticFile,'a')
diagnostics = Diagnostics(run_data.outputDiagnosticFile)

runStart = proxy.DateTimeFromPI(run_data.startDateTime)
runEnd = proxy.DateTimeFromPI(run_data.endDateTime)

timeSeries=[]
for fn in run_data.inputTimeSeriesFile:
  print(fn)
  c = proxy.ReadTimeSeries(fn)
  for ts in c.series: timeSeries.append(ts)

#ts0 = timeSeries[4]
#timeStepFromPI(ts0.header.timeStep)
#tts0 = buildTimeSeries(ts0)

outputSets={}
convertedTimeSeries=[]

timeSeries.sort(lambda x,y: (x.header.locationId>y.header.locationId))
for ts in timeSeries:
	destFile = fileForParameter(ts.header.parameterId)
	if not outputSets.Contains(destFile):
		outputSets[destFile] = TIME.DataTypes.IO.CsvFileIo.CSVFileIO()

	converted = buildTimeSeries(ts)
	convertedTimeSeries.append(converted)
	outputSets[destFile].use(converted.name,converted)

[ts.name for ts in convertedTimeSeries]
nullCounts = [ts.countEqual(-9999) for ts in convertedTimeSeries]
nullCounts
# vals = [convertedTimeSeries[0][i] for i in range(0,convertedTimeSeries[0].count()-1)]
# vals

System.IO.Directory.CreateDirectory(run_data.workDir)

# Save to individual files
#for ts in convertedTimeSeries:
#  fn = run_data.workDir + "\\" + ts.name + ".csv"
#  diagnostics.Log(3,"Writing time series to " + fn)
#  io.Save(fn,ts)


for outputFile in outputSets:
	outputSets[outputFile].save(run_data.workDir + "\\" + outputFile)


#csv = TIME.DataTypes.IO.CsvFileIo.CSVFileIO()
#for ts in convertedTimeSeries:
#	csv.use(ts.name,ts)
#
#csv.save(run_data.workDir + "\\allcatchments.csv")

#writeLogFile(run_data.outputDiagnosticFile)
diagnostics.Log(3,"Pre Adapter - All done")
diagnostics.Save()