// continuously monitors a service by polling
continuously monitors a service by polling
// continuously monitors a service by polling
func MonitorService(serviceName string, mutex *sync.Mutex, anomalyfilename *string, jobmap *map[string]JobInfoM, currentYear *int, currentMonth *time.Month, currentDay *int) {
	for {
		healthresponse := CheckJobStatus((*jobmap)[serviceName].JobID, serviceName)
		status := healthresponse.Status
		if status == "Healthy" {
			ForemastQuery(jobmap, serviceName, (*jobmap)[serviceName].MetricMap)
		} else if status == "Unhealthy" {
			dashboardUrl := CreateDashboardURL(serviceName, jobmap, healthresponse)
			log.Printf("[%s] dashboardUrl: %s", serviceName, dashboardUrl)
			s := time.Now().Format("2006-01-02T15:04:05Z07:00") + "\t" + serviceName + "\t" + (*jobmap)[serviceName].JobID + "\t" + healthresponse.Reason + "\t" + dashboardUrl + "\n" 
			mutex.Lock()
			cury, curm, curd := time.Now().Date()
			if cury != *currentYear || curm != *currentMonth || curd != *currentDay {
				*currentYear = cury
				*currentMonth = curm
				*currentDay = curd
				*anomalyfilename = os.Getenv("VOLUME_PATH") + "/anomaly_" + strconv.Itoa(*currentYear) + "-" + (*currentMonth).String() + "-" + strconv.Itoa(*currentDay) + ".tsv"
			}
			anomalyPath, _ := filepath.Abs(*anomalyfilename)
			anomalyfile, err := os.OpenFile(anomalyPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0777)
			if err != nil {
				log.Printf("[%s] error opening anomaly file %s\n%s\n", serviceName, err, err.Error())
			}
			_, err = anomalyfile.WriteString(html.UnescapeString(s))
			if err != nil {
				log.Printf("[%s] error writing to anomaly file %s\n%s\n", serviceName, err, err.Error())
			}
			anomalyfile.Close()
			mutex.Unlock()
			ForemastQuery(jobmap, serviceName, (*jobmap)[serviceName].MetricMap)
		} else if status == "Abort" || status == "Warning" {
			ForemastQuery(jobmap, serviceName, (*jobmap)[serviceName].MetricMap)
		} else {
			time.Sleep(time.Second * 10)
		}
	}
}
func MonitorService ( serviceName string , mutex * sync . Mutex , anomalyfilename * string , jobmap * map [ string ] JobInfoM , currentYear * int , currentMonth * time . Month , currentDay * int ) { for { healthresponse := CheckJobStatus ( ( * jobmap ) [ serviceName ] . JobID , serviceName ) status := healthresponse . Status if status == " " { ForemastQuery ( jobmap , serviceName , ( * jobmap ) [ serviceName ] . MetricMap ) } else if status == " " { dashboardUrl := CreateDashboardURL ( serviceName , jobmap , healthresponse ) log . Printf ( " " , serviceName , dashboardUrl ) s := time . Now ( ) . Format ( " " ) + " \t " + serviceName + " \t " + ( * jobmap ) [ serviceName ] . JobID + " \t " + healthresponse . Reason + " \t " + dashboardUrl + " \n " mutex . Lock ( ) cury , curm , curd := time . Now ( ) . Date ( ) if cury != * currentYear || curm != * currentMonth || curd != * currentDay { * currentYear = cury * currentMonth = curm * currentDay = curd * anomalyfilename = os . Getenv ( " " ) + " " + strconv . Itoa ( * currentYear ) + " " + ( * currentMonth ) . String ( ) + " " + strconv . Itoa ( * currentDay ) + " " } anomalyPath , _ := filepath . Abs ( * anomalyfilename ) anomalyfile , err := os . OpenFile ( anomalyPath , os . O_CREATE | os . O_APPEND | os . O_WRONLY , 0777 ) if err != nil { log . Printf ( " \n \n " , serviceName , err , err . Error ( ) ) } _ , err = anomalyfile . WriteString ( html . UnescapeString ( s ) ) if err != nil { log . Printf ( " \n \n " , serviceName , err , err . Error ( ) ) } anomalyfile . Close ( ) mutex . Unlock ( ) ForemastQuery ( jobmap , serviceName , ( * jobmap ) [ serviceName ] . MetricMap ) } else if status == " " || status == " " { ForemastQuery ( jobmap , serviceName , ( * jobmap ) [ serviceName ] . MetricMap ) } else { time . Sleep ( time . Second * 10 ) } } }
continuously monitors a service by polling
continuously monitors a service by polling
// check if the job completed // we're done with this job, run the next query //we're done with this job, write the anomaly to file and run next query //timestamp + servicename + jobid // create daily anoamly files, so ensure that nothing's changed since the last update // open and update file // give up and run another query // not done, check again later