import time from collections import OrderedDict Logger = system.util.getLogger("reports.MBR.coolingdata") def test1_0(): res={} endDate= system.date.midnight(system.date.now()) startDate = system.date.addDays(endDate, -2) tempDS, humDS = reports.MBR.coolingdata.getRawDataMeasurement(startDate, endDate, "DH1", customArgs= {"noInterpolation":False}) # res["tempDS"]= tempDS # res["humDS"] = humDS # group the sensors by aisle then perform an interval average tempDS = reports.MBR.common.util.averageIntervalData(groupColumnLocation(tempDS, "Temp"), 60) humDS = reports.MBR.common.util.averageIntervalData(groupColumnLocation(humDS, "Humidity"), 60) tempSummaryDS = reports.MBR.coolingdata.generateSummaryColumnDS(tempDS, "Temp") humiditySummaryDS = reports.MBR.coolingdata.generateSummaryColumnDS(humDS, "Humidity") summaryDS = combineTempHumidity(tempSummaryDS, humiditySummaryDS) del tempSummaryDS del humiditySummaryDS measurementDS = reports.MBR.common.util.mergeHistorianData(combineTempHumidity(tempDS, humDS),summaryDS) res["measurementDS"]=measurementDS res["summaryDS"]=summaryDS return res def genSummCols(ds, snsrType, aggRow): if aggRow: return reports.MBR.coolingdata.generateSummaryColumnDS(ds, snsrType) else: return reports.MBR.coolingdata.generateRangeSummaryCol(ds, snsrType) def doReport(startDate, endDate, locations, intervalTime = 60, aggRow= True): """ Args: aggRow boolean, true: get the min/max/avg on row, false: get the min max over the range, and avg per row """ res={} customArgs= {"noInterpolation":False} tempDS, humDS = None, None if locations is None: tempDS, humDS = reports.MBR.coolingdata.getRawDataMeasurement(startDate, endDate, None,customArgs) else: for loc in locations: ds1, ds2 = getRawDataMeasurement(startDate, endDate, loc, customArgs) if tempDS is None and humDS is None: tempDS = ds1 humDS = ds2 else: tempDS= reports.MBR.common.util.mergeHistorianData(tempDS, ds1) humDS= reports.MBR.common.util.mergeHistorianData(humDS, ds1) # res["tempDS"]= tempDS # res["humDS"] = humDS # group the sensors by aisle then perform an interval average tempDS = reports.MBR.common.util.averageIntervalData(groupColumnLocation(tempDS, "Temp"), intervalTime) humDS = reports.MBR.common.util.averageIntervalData(groupColumnLocation(humDS, "Humidity"), intervalTime) tempSummaryDS = genSummCols(tempDS, "Temp", aggRow) humiditySummaryDS = genSummCols(humDS, "Humidity", aggRow) summaryDS = combineTempHumidity(tempSummaryDS, humiditySummaryDS) del tempSummaryDS del humiditySummaryDS measurementDS = addStaticLimits(reports.MBR.common.util.mergeHistorianData(combineTempHumidity(tempDS, humDS),summaryDS)) res["measurementDS"]=measurementDS res["summaryDS"]=summaryDS return res def doReport2(startDate, endDate, locations, intervalTime = 60, aggRow= False): """ Args: aggRow boolean, true: get the min/max/avg on row, false: get the min max over the range, and avg per row """ tSt = time.time() res={} customArgs= {"noInterpolation":False} superTempDS, superHumDS, tempDS, humDS = None, None, None, None if locations is None: locations = ["DH1","DH2","DH3","DH4","DH5"] summaryRows = [] for loc in locations: ds1, ds2 = getRawDataMeasurement(startDate, endDate, loc, customArgs) tempDS = ds1 humDS = ds2 if superTempDS is None and superHumDS is None: superTempDS = ds1 superHumDS = ds2 else: superTempDS= reports.MBR.common.util.mergeHistorianData(superTempDS, ds1) superHumDS= reports.MBR.common.util.mergeHistorianData(superHumDS, ds2) tempDS = reports.MBR.common.util.averageIntervalData(groupColumnLocation(tempDS, "Temp"), intervalTime) humDS = reports.MBR.common.util.averageIntervalData(groupColumnLocation(humDS, "Humidity"), intervalTime) tempSummaryDS = genSummCols(tempDS, "Temp", aggRow) humiditySummaryDS = genSummCols(humDS, "Humidity", aggRow) summaryDS = combineTempHumidity(tempSummaryDS, humiditySummaryDS) del tempSummaryDS del humiditySummaryDS summaryRows.append([loc.replace("DH", "Datahall "), system.dataset.toDataSet(summaryDS) ]) print "elapsed 1: ", time.time() - tSt # group the sensors by aisle then perform an interval average print "elapsed 2: ", time.time()- tSt superTempDS = reports.MBR.common.util.averageIntervalData(groupColumnLocation(superTempDS, "Temp"), intervalTime) superHumDS = reports.MBR.common.util.averageIntervalData(groupColumnLocation(superHumDS, "Humidity"), intervalTime) tempSummaryDS = genSummCols(superTempDS, "Temp", aggRow) humiditySummaryDS = genSummCols(superHumDS, "Humidity", aggRow) summaryDS = combineTempHumidity(tempSummaryDS, humiditySummaryDS) measurementDS = addStaticLimits(reports.MBR.common.util.mergeHistorianData(combineTempHumidity(superTempDS, superHumDS),summaryDS)) print "elapsed 3: ", time.time()- tSt res["summaryDS"]=summaryDS res["measurementDS"]=measurementDS res["locSummaryDS"] = system.dataset.toDataSet(["Location", "data"], summaryRows) return res # res["summaryDS"]=system.dataset.toDataSet([],summaryDS) def getRawDataMeasurement(startDate, endDate, location, customArgs= {}): """ Get the history 1-minute interval for the relevant location sensors Args: startDate: datetime endDate: datetime location: str DH1, DH2.. customArgs: optional overrides for the history retrieval Returns: tuple (temp Dataset, humidity dataset) """ logger = Logger.createSubLogger("getRawDataMeasurement") stTime= time.time() tempTags = reports.MBR.common.coolingTags.generateTags(location, "temp") humidityTags = reports.MBR.common.coolingTags.generateTags(location, "humidity") tempDS = reports.MBR.common.util.getHistory(tempTags, startDate, endDate, customArgs) humidityDS = reports.MBR.common.util.getHistory(humidityTags, startDate, endDate, customArgs) logger.debug("complete duration: %s"%(time.time()-stTime)) return tempDS, humidityDS def groupColumnLocation(histDS, sensorType): """ Take the full range of tag columns and group them by location and average the value ex. [Ignition_Common_IO_Gtwy]DH1/SATB1_DH1_PNL24_C1_HT01/Val [Ignition_Common_IO_Gtwy]DH1/SATB1_DH1_PNL24_C1_HT02/Val [Ignition_Common_IO_Gtwy]DH1/SATB1_DH1_PNL24_C1_HT03/Val becomes DH1_C1 Args: histDS: pyds of history sensorType: str Temp/Humidity Returns: py dataset """ def groupColNames(colNames): colMap = OrderedDict() for name in colNames: cNameParts = name.split("/")[1].split("_") gName = "%s_%s %s"%(cNameParts[1], cNameParts[3], sensorType) colMap.setdefault(gName, []) colMap[gName].append(name) return colMap oColNames = system.dataset.getColumnHeaders(histDS)[1:] colMap = groupColNames(oColNames) newCols = colMap.keys() # now let's average across the new names allRows = [] for row in histDS: oneRow = [row["t_stamp"]] for grp in newCols: groupVals = [row[c] for c in colMap[grp]] try: oneRow.append(sum(groupVals)/len(groupVals)) except: oneRow.append(-1.0) allRows.append(oneRow) return system.dataset.toPyDataSet(system.dataset.toDataSet(["t_stamp"]+newCols , allRows)) def combineTempHumidity(tempDS, humidityDS): """ Build the custom column sort order C1 Humidity, C1 Temp, C2 Humidity, etc.. Args: tempDS: pyDataset humidityDS: humidityDS Returns: pydataset """ allRows = [] header = ["t_stamp"] tempHeaders = system.dataset.getColumnHeaders(tempDS) humHeaders = system.dataset.getColumnHeaders(humidityDS) for rCnt, row in enumerate(humidityDS): newRow = [row["t_stamp"]] for cCnt in range(1, len(row)): if rCnt == 0: header.extend([humHeaders[cCnt], tempHeaders[cCnt]]) newRow.extend([row[cCnt], tempDS[rCnt][cCnt]]) allRows.append(newRow) # print header, len(header), len(allRows[0]) return system.dataset.toPyDataSet(system.dataset.toDataSet(header, allRows)) def generateSummaryColumnDS(ds1, sensorType): """ Receive the grouped dataset and calculate the Min/Max/Avg/Limit for each row Args: ds1: pydataset grouped by sensors sensorType: str (Temp or Humidity) Returns: pydataset with columns t_stamp, Max , Avg, Min """ header = ["t_stamp", "Max %s"%(sensorType), "Avg %s"%(sensorType), "Min %s"%(sensorType)] allRows = [] for row in ds1: rowVals= [c for c in row[1:]] allRows.append([row["t_stamp"], max(rowVals), sum(rowVals)/len(rowVals), min(rowVals)]) return system.dataset.toPyDataSet(system.dataset.toDataSet(header, allRows)) def generateRangeSummaryCol(ds1, sensorType): """ Receive the grouped dataset and calculate the Min/Max/Avg/Limit for the entire range Args: ds1: pydataset grouped by sensors sensorType: str (Temp or Humidity) Returns: pydataset with columns t_stamp, Max , Avg, Min """ header = ["t_stamp", "Max %s"%(sensorType), "Avg %s"%(sensorType), "Min %s"%(sensorType)] allRows = [] vals = [] for row in ds1: vals.extend([v for v in row[1:]]) rowVals= [c for c in row[1:]] allRows.append([row["t_stamp"], None, sum(rowVals)/len(rowVals), None]) valMin = min(vals) valMax = max(vals) for sRow in allRows: sRow[1] = valMax sRow[3] = valMin return system.dataset.toPyDataSet(system.dataset.toDataSet(header, allRows)) def addStaticLimits(ds): limits= [] limits.append( [reports.MBR.common.static.humidityHighLimit for i in range(len(ds))]) limits.append( [reports.MBR.common.static.humidityLowLimit for i in range(len(ds))]) limits.append( [reports.MBR.common.static.tempHighLimit for i in range(len(ds))]) limits.append( [reports.MBR.common.static.tempLowLimit for i in range(len(ds))]) for i,cName in enumerate(["Hum High Lim", "Hum Low Lim", "Temp High Lim", "Temp Low Lim"]): ds = system.dataset.addColumn(ds, limits[i], cName, float) return system.dataset.toPyDataSet(ds)