implement Box.runChecks()
This commit is contained in:
parent
295c26237b
commit
778b2ae90f
9 changed files with 258 additions and 121 deletions
23
cmd/args.go
23
cmd/args.go
|
@ -1,10 +1,11 @@
|
||||||
package cmd
|
package cmd
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"../core"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/spf13/cobra"
|
|
||||||
"regexp"
|
"regexp"
|
||||||
|
|
||||||
|
"../core"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
)
|
)
|
||||||
|
|
||||||
func isValidBoxId(boxId string) bool {
|
func isValidBoxId(boxId string) bool {
|
||||||
|
@ -27,19 +28,15 @@ func BoxIdValidator(cmd *cobra.Command, args []string) error {
|
||||||
|
|
||||||
// TODO: actually to be read from arg / file
|
// TODO: actually to be read from arg / file
|
||||||
var defaultConf = &core.NotifyConfig{
|
var defaultConf = &core.NotifyConfig{
|
||||||
// Transports: struct {
|
|
||||||
// Slack: SlackConfig{
|
|
||||||
// Channel: "asdf"
|
|
||||||
// Token: "qwer"
|
|
||||||
// }
|
|
||||||
// },
|
|
||||||
Events: []core.NotifyEvent{
|
Events: []core.NotifyEvent{
|
||||||
core.NotifyEvent{
|
core.NotifyEvent{
|
||||||
Type: "measurementAge",
|
Type: "measurement_age",
|
||||||
Target: "593bcd656ccf3b0011791f5d",
|
Target: "all",
|
||||||
Threshold: "5h",
|
Threshold: "15m",
|
||||||
|
},
|
||||||
|
core.NotifyEvent{
|
||||||
|
Type: "measurement_suspicious",
|
||||||
|
Target: "all",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
// func parseNotifyConfig(conf string) NotifyConfig, error {}
|
|
||||||
|
|
10
cmd/check.go
10
cmd/check.go
|
@ -1,8 +1,6 @@
|
||||||
package cmd
|
package cmd
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"../core"
|
|
||||||
"fmt"
|
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -23,13 +21,11 @@ var checkBoxCmd = &cobra.Command{
|
||||||
Long: "specify box IDs to check them for events",
|
Long: "specify box IDs to check them for events",
|
||||||
Args: BoxIdValidator,
|
Args: BoxIdValidator,
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
notifications, err := core.CheckNotifications(args, defaultConf)
|
cmd.SilenceUsage = true
|
||||||
|
_, err := CheckBoxes(args, defaultConf)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("error checking for notifications:", err)
|
return err
|
||||||
}
|
}
|
||||||
fmt.Println(notifications)
|
|
||||||
|
|
||||||
// logNotifications(notifications)
|
|
||||||
if shouldNotify {
|
if shouldNotify {
|
||||||
// TODO
|
// TODO
|
||||||
}
|
}
|
||||||
|
|
65
cmd/jobs.go
Normal file
65
cmd/jobs.go
Normal file
|
@ -0,0 +1,65 @@
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"../core"
|
||||||
|
log "github.com/sirupsen/logrus"
|
||||||
|
)
|
||||||
|
|
||||||
|
func CheckBoxes(boxIds []string, defaultConf *core.NotifyConfig) ([]core.CheckResult, error) {
|
||||||
|
log.Debug("Checking notifications for ", len(boxIds), " box(es)")
|
||||||
|
|
||||||
|
// TODO: return a map of Box: []Notification instead?
|
||||||
|
results := []core.CheckResult{}
|
||||||
|
for _, boxId := range boxIds {
|
||||||
|
r, err := checkBox(boxId, defaultConf)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if r != nil {
|
||||||
|
results = append(results, r...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return results, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkBox(boxId string, defaultConf *core.NotifyConfig) ([]core.CheckResult, error) {
|
||||||
|
boxLogger := log.WithFields(log.Fields{"boxId": boxId})
|
||||||
|
boxLogger.Info("checking box for due notifications")
|
||||||
|
|
||||||
|
// get box data
|
||||||
|
box, err := core.Osem.GetBox(boxId)
|
||||||
|
if err != nil {
|
||||||
|
boxLogger.Error(err)
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// if box has no notify config, we use the defaultConf
|
||||||
|
if box.NotifyConf == nil {
|
||||||
|
box.NotifyConf = defaultConf
|
||||||
|
}
|
||||||
|
|
||||||
|
// run checks
|
||||||
|
results, err2 := box.RunChecks()
|
||||||
|
if err2 != nil {
|
||||||
|
boxLogger.Error("could not run checks on box: ", err2)
|
||||||
|
return results, err2
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, r := range results {
|
||||||
|
resultLog := boxLogger.WithFields(log.Fields{
|
||||||
|
"status": r.Status,
|
||||||
|
"event": r.Event,
|
||||||
|
"value": r.Value,
|
||||||
|
"target": r.Target,
|
||||||
|
})
|
||||||
|
if r.Status == core.CheckOk {
|
||||||
|
resultLog.Debug(r)
|
||||||
|
} else {
|
||||||
|
resultLog.Warn(r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return results, nil
|
||||||
|
}
|
33
cmd/root.go
33
cmd/root.go
|
@ -1,26 +1,53 @@
|
||||||
package cmd
|
package cmd
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"os"
|
||||||
|
|
||||||
|
log "github.com/sirupsen/logrus"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
)
|
)
|
||||||
|
|
||||||
var rootCmd = &cobra.Command{
|
var rootCmd = &cobra.Command{
|
||||||
Use: "osem_notify",
|
Use: "osem_notify",
|
||||||
Long: "Run healthchecks and send notifications for boxes on opensensemap.org",
|
Long: "Run healthchecks and send notifications for boxes on opensensemap.org",
|
||||||
|
PersistentPreRun: func(cmd *cobra.Command, args []string) {
|
||||||
|
log.SetOutput(os.Stdout)
|
||||||
|
switch logLevel {
|
||||||
|
case "debug":
|
||||||
|
log.SetLevel(log.DebugLevel)
|
||||||
|
case "info":
|
||||||
|
log.SetLevel(log.InfoLevel)
|
||||||
|
case "warn":
|
||||||
|
log.SetLevel(log.WarnLevel)
|
||||||
|
case "error":
|
||||||
|
log.SetLevel(log.ErrorLevel)
|
||||||
|
}
|
||||||
|
switch logFormat {
|
||||||
|
case "json":
|
||||||
|
log.SetFormatter(&log.JSONFormatter{})
|
||||||
|
}
|
||||||
|
},
|
||||||
Run: func(cmd *cobra.Command, args []string) {
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
cmd.Help()
|
cmd.Help()
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
var shouldNotify bool
|
var (
|
||||||
var defaultConfig string
|
shouldNotify bool
|
||||||
|
defaultConfig string
|
||||||
|
logLevel string
|
||||||
|
logFormat string
|
||||||
|
)
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
|
rootCmd.PersistentFlags().StringVarP(&logLevel, "log-level", "", "info", "log level, can be one of debug, info, warn, error")
|
||||||
|
rootCmd.PersistentFlags().StringVarP(&logFormat, "log-format", "", "plain", "log format, can be plain or json")
|
||||||
rootCmd.PersistentFlags().BoolVarP(&shouldNotify, "notify", "n", false, "if set, will send out notifications.\nOtherwise results are printed to stdout only")
|
rootCmd.PersistentFlags().BoolVarP(&shouldNotify, "notify", "n", false, "if set, will send out notifications.\nOtherwise results are printed to stdout only")
|
||||||
rootCmd.PersistentFlags().StringVarP(&defaultConfig, "confdefault", "c", "", "default JSON config to use for event checking")
|
rootCmd.PersistentFlags().StringVarP(&defaultConfig, "conf-default", "c", "", "default JSON config to use for event checking")
|
||||||
}
|
}
|
||||||
|
|
||||||
func Execute() {
|
func Execute() {
|
||||||
if err := rootCmd.Execute(); err != nil {
|
if err := rootCmd.Execute(); err != nil {
|
||||||
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
27
cmd/watch.go
27
cmd/watch.go
|
@ -1,7 +1,6 @@
|
||||||
package cmd
|
package cmd
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
|
@ -34,14 +33,18 @@ var watchBoxesCmd = &cobra.Command{
|
||||||
Long: "specify box IDs to watch them for events",
|
Long: "specify box IDs to watch them for events",
|
||||||
Args: BoxIdValidator,
|
Args: BoxIdValidator,
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
cmd.SilenceUsage = true
|
||||||
exec := func() error {
|
exec := func() error {
|
||||||
notifications, err := core.CheckNotifications(args, defaultConf)
|
results, err := CheckBoxes(args, defaultConf)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("error checking for notifications: ", err)
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
results, err = filterFromCache(results)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
fmt.Println(notifications)
|
|
||||||
|
|
||||||
// logNotifications(notifications)
|
|
||||||
if shouldNotify {
|
if shouldNotify {
|
||||||
// TODO
|
// TODO
|
||||||
}
|
}
|
||||||
|
@ -59,7 +62,17 @@ var watchBoxesCmd = &cobra.Command{
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func filterFromCache(results []core.CheckResult) ([]core.CheckResult, error) {
|
||||||
|
// get results from cache. they are indexed by ______
|
||||||
|
|
||||||
|
// filter, so that only changed result.Status remain
|
||||||
|
|
||||||
|
// extract additional results with Status ERR from cache with time.Since(lastNotifyDate) > thresh
|
||||||
|
|
||||||
|
// upate cache set lastNotifyDate to Now()
|
||||||
|
|
||||||
|
return results, nil
|
||||||
|
}
|
||||||
|
|
137
core/Box.go
137
core/Box.go
|
@ -1,5 +1,44 @@
|
||||||
package core
|
package core
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strconv"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
CheckOk = "OK"
|
||||||
|
CheckErr = "ERROR"
|
||||||
|
eventMeasurementAge = "measurement_age" // errors if age of last measurement is higher than a duration
|
||||||
|
eventMeasurementValMin = "measurement_min" // errors if value of last measurement is lower than threshold
|
||||||
|
eventMeasurementValMax = "measurement_max" // errors if value of last measurement is higher than threshold
|
||||||
|
eventMeasurementValSuspicious = "measurement_suspicious" // checks value of last measurement against a blacklist of values
|
||||||
|
eventTargetAll = "all" // if event.Target is this value, all sensors will be checked
|
||||||
|
)
|
||||||
|
|
||||||
|
type SuspiciousValue struct {
|
||||||
|
sensor string
|
||||||
|
val float64
|
||||||
|
}
|
||||||
|
|
||||||
|
var suspiciousVals = map[SuspiciousValue]bool{
|
||||||
|
SuspiciousValue{sensor: "BMP280", val: 0.0}: true,
|
||||||
|
SuspiciousValue{sensor: "HDC1008", val: 0.0}: true,
|
||||||
|
SuspiciousValue{sensor: "HDC1008", val: -40}: true,
|
||||||
|
SuspiciousValue{sensor: "SDS 011", val: 0.0}: true,
|
||||||
|
}
|
||||||
|
|
||||||
|
type CheckResult struct {
|
||||||
|
Status string
|
||||||
|
Event string
|
||||||
|
Target string
|
||||||
|
Value string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r CheckResult) String() string {
|
||||||
|
return fmt.Sprintf("check %s on sensor %s: %s with value %s\n", r.Event, r.Target, r.Status, r.Value)
|
||||||
|
}
|
||||||
|
|
||||||
type NotifyEvent struct {
|
type NotifyEvent struct {
|
||||||
Type string `json:"type"`
|
Type string `json:"type"`
|
||||||
Target string `json:"target"`
|
Target string `json:"target"`
|
||||||
|
@ -7,7 +46,6 @@ type NotifyEvent struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
type NotifyConfig struct {
|
type NotifyConfig struct {
|
||||||
// Transports interface{} `json:"transports"`
|
|
||||||
Events []NotifyEvent `json:"events"`
|
Events []NotifyEvent `json:"events"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -15,25 +53,94 @@ type Box struct {
|
||||||
Id string `json:"_id"`
|
Id string `json:"_id"`
|
||||||
Sensors []struct {
|
Sensors []struct {
|
||||||
Id string `json:"_id"`
|
Id string `json:"_id"`
|
||||||
|
Type string `json:"sensorType"`
|
||||||
LastMeasurement *struct {
|
LastMeasurement *struct {
|
||||||
Value string `json:"value"`
|
Value string `json:"value"`
|
||||||
Date string `json:"createdAt"`
|
Date time.Time `json:"createdAt"`
|
||||||
} `json:"lastMeasurement"`
|
} `json:"lastMeasurement"`
|
||||||
} `json:"sensors"`
|
} `json:"sensors"`
|
||||||
NotifyConf *NotifyConfig `json:"notify"`
|
NotifyConf *NotifyConfig `json:"notify"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (box Box) runChecks() ([]Notification, error) {
|
func (box Box) RunChecks() ([]CheckResult, error) {
|
||||||
|
var results = []CheckResult{}
|
||||||
|
|
||||||
|
for _, event := range box.NotifyConf.Events {
|
||||||
|
target := event.Target
|
||||||
|
|
||||||
|
for _, s := range box.Sensors {
|
||||||
|
// if a sensor never measured anything, thats ok. checks would fail anyway
|
||||||
|
if s.LastMeasurement == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if target == eventTargetAll || target == s.Id {
|
||||||
|
|
||||||
|
switch event.Type {
|
||||||
|
case eventMeasurementAge:
|
||||||
|
// check if age of lastMeasurement is within threshold
|
||||||
|
status := CheckOk
|
||||||
|
thresh, err := time.ParseDuration(event.Threshold)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if time.Since(s.LastMeasurement.Date) > thresh {
|
||||||
|
status = CheckErr
|
||||||
|
}
|
||||||
|
|
||||||
|
results = append(results, CheckResult{
|
||||||
|
Event: event.Type,
|
||||||
|
Target: s.Id,
|
||||||
|
Value: s.LastMeasurement.Date.String(),
|
||||||
|
Status: status,
|
||||||
|
})
|
||||||
|
|
||||||
|
case eventMeasurementValMin, eventMeasurementValMax:
|
||||||
|
status := CheckOk
|
||||||
|
thresh, err := strconv.ParseFloat(event.Threshold, 64)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
val, err2 := strconv.ParseFloat(s.LastMeasurement.Value, 64)
|
||||||
|
if err2 != nil {
|
||||||
|
return nil, err2
|
||||||
|
}
|
||||||
|
if event.Type == eventMeasurementValMax && val > thresh ||
|
||||||
|
event.Type == eventMeasurementValMin && val < thresh {
|
||||||
|
status = CheckErr
|
||||||
|
}
|
||||||
|
|
||||||
|
results = append(results, CheckResult{
|
||||||
|
Event: event.Type,
|
||||||
|
Target: s.Id,
|
||||||
|
Value: s.LastMeasurement.Value,
|
||||||
|
Status: status,
|
||||||
|
})
|
||||||
|
|
||||||
|
case eventMeasurementValSuspicious:
|
||||||
|
status := CheckOk
|
||||||
|
|
||||||
|
val, err := strconv.ParseFloat(s.LastMeasurement.Value, 64)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if suspiciousVals[SuspiciousValue{
|
||||||
|
sensor: s.Type,
|
||||||
|
val: val,
|
||||||
|
}] {
|
||||||
|
status = CheckErr
|
||||||
|
}
|
||||||
|
|
||||||
|
results = append(results, CheckResult{
|
||||||
|
Event: event.Type,
|
||||||
|
Target: s.Id,
|
||||||
|
Value: s.LastMeasurement.Value,
|
||||||
|
Status: status,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
// must return ALL events to enable Notifier to clear previous notifications
|
// must return ALL events to enable Notifier to clear previous notifications
|
||||||
return nil, nil
|
return results, nil
|
||||||
}
|
|
||||||
|
|
||||||
func (box Box) getNotifier() (AbstractNotifier, error) {
|
|
||||||
// validate box.NotifyConf.transport
|
|
||||||
|
|
||||||
// try to get notifier state from persistence
|
|
||||||
|
|
||||||
// return
|
|
||||||
var notifier AbstractNotifier
|
|
||||||
return notifier, nil
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,8 +2,9 @@ package core
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"github.com/dghubble/sling"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
|
"github.com/dghubble/sling"
|
||||||
)
|
)
|
||||||
|
|
||||||
type OsemError struct {
|
type OsemError struct {
|
||||||
|
@ -31,4 +32,4 @@ func (client *OsemClient) GetBox(boxId string) (Box, error) {
|
||||||
return box, nil
|
return box, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var osem = NewOsemClient(&http.Client{}) // default client
|
var Osem = NewOsemClient(&http.Client{}) // default client
|
||||||
|
|
74
core/jobs.go
74
core/jobs.go
|
@ -1,74 +0,0 @@
|
||||||
package core
|
|
||||||
|
|
||||||
import (
|
|
||||||
log "github.com/sirupsen/logrus"
|
|
||||||
"os"
|
|
||||||
)
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
log.SetLevel(log.DebugLevel)
|
|
||||||
log.SetOutput(os.Stdout)
|
|
||||||
// log.SetFormatter(&log.JSONFormatter{})
|
|
||||||
}
|
|
||||||
|
|
||||||
func CheckNotifications(boxIds []string, defaultConf *NotifyConfig) ([]Notification, []error) {
|
|
||||||
log.Info("Checking notifications for ", len(boxIds), " box(es)")
|
|
||||||
|
|
||||||
// TODO: return a map of Box: []Notification instead?
|
|
||||||
notifications := []Notification{}
|
|
||||||
errors := []error{}
|
|
||||||
for _, boxId := range boxIds {
|
|
||||||
n, err := checkBox(boxId, defaultConf)
|
|
||||||
if notifications != nil {
|
|
||||||
notifications = append(notifications, n...)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
errors = append(errors, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(errors) == 0 {
|
|
||||||
errors = nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return notifications, errors
|
|
||||||
}
|
|
||||||
|
|
||||||
func checkBox(boxId string, defaultConf *NotifyConfig) ([]Notification, error) {
|
|
||||||
boxLogger := log.WithFields(log.Fields{"boxId": boxId})
|
|
||||||
boxLogger.Debug("checking box for due notifications")
|
|
||||||
|
|
||||||
// get box data
|
|
||||||
box, err := osem.GetBox(boxId)
|
|
||||||
if err != nil {
|
|
||||||
boxLogger.Error(err)
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// if box has no notify config, we use the defaultConf
|
|
||||||
if box.NotifyConf == nil {
|
|
||||||
box.NotifyConf = defaultConf
|
|
||||||
}
|
|
||||||
boxLogger.Debug(box.NotifyConf)
|
|
||||||
|
|
||||||
// run checks
|
|
||||||
notifications, err2 := box.runChecks()
|
|
||||||
if err2 != nil {
|
|
||||||
boxLogger.Error("could not run checks on box: ", err)
|
|
||||||
return notifications, err2
|
|
||||||
}
|
|
||||||
if notifications == nil {
|
|
||||||
boxLogger.Debug("all is fine")
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// store notifications for later submit
|
|
||||||
// notifier, err3 := box.getNotifier()
|
|
||||||
// if err3 != nil {
|
|
||||||
// boxLogger.Error("could not get notifier for box: ", err)
|
|
||||||
// return notifications, err3
|
|
||||||
// }
|
|
||||||
// notifier.AddNotifications(notifications)
|
|
||||||
|
|
||||||
return notifications, nil
|
|
||||||
}
|
|
5
run.sh
Executable file
5
run.sh
Executable file
|
@ -0,0 +1,5 @@
|
||||||
|
go fmt ./ ./cmd/ ./core/ && \
|
||||||
|
go build ./ && \
|
||||||
|
./osem_notify check boxes \
|
||||||
|
593bcd656ccf3b0011791f5a 5b26181b1fef04001b69093c 59b31b8dd67eb50011165a04 562bdcf3b3de1fe005e03d2a $@ \
|
||||||
|
--log-level debug
|
Loading…
Add table
Reference in a new issue