diff --git a/3rd/load-watcher/.github/workflows/ci.yml b/3rd/load-watcher/.github/workflows/ci.yml new file mode 100644 index 00000000..15b27373 --- /dev/null +++ b/3rd/load-watcher/.github/workflows/ci.yml @@ -0,0 +1,25 @@ +name: CI + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.22 + + - name: Build + run: go build -o load-watcher main.go + + - name: Test + run: go test ./... diff --git a/3rd/load-watcher/.gitignore b/3rd/load-watcher/.gitignore new file mode 100644 index 00000000..9f11b755 --- /dev/null +++ b/3rd/load-watcher/.gitignore @@ -0,0 +1 @@ +.idea/ diff --git a/3rd/load-watcher/.grenrc b/3rd/load-watcher/.grenrc new file mode 100644 index 00000000..ab468ca3 --- /dev/null +++ b/3rd/load-watcher/.grenrc @@ -0,0 +1,10 @@ +{ + "dataSource": "milestones", + "groupBy": { + "Features:": ["enhancement", "documentation"], + "Bug Fixes:": ["bug"] + }, + "ignoreLabels": ["help wanted", "question", "good first issue"], + "milestoneMatch": "{{tag_name}}", + "changelogFilename": "CHANGELOG.md" +} diff --git a/3rd/load-watcher/CHANGELOG.md b/3rd/load-watcher/CHANGELOG.md new file mode 100644 index 00000000..c6dc6787 --- /dev/null +++ b/3rd/load-watcher/CHANGELOG.md @@ -0,0 +1,23 @@ +# Changelog + +## v0.1.1 (26/02/2021) + +#### Bug Fixes: + +- [**bug**] Cannot use the LibraryClient with the watcher [#18](https://github.com/paypal/load-watcher/issues/18) + +--- + +## v0.1.0 (24/02/2021) + +#### Enhancements: + +- [**enhancement**] Maintaining consistent release version [#17](https://github.com/paypal/load-watcher/issues/17) +- [**enhancement**] Move metricsprovider to internal [#15](https://github.com/paypal/load-watcher/issues/15) +- [**enhancement**] Need lib to run `load-watcher` client [#12](https://github.com/paypal/load-watcher/issues/12) +- [**enhancement**] Prometheus Client is Missing [#10](https://github.com/paypal/load-watcher/issues/10) +- [**documentation**][**enhancement**] Dockerfile and k8s deployment tutorial are needed [#5](https://github.com/paypal/load-watcher/issues/5) + +#### Bug Fixes: + +- [**bug**] Is there some errors? [#7](https://github.com/paypal/load-watcher/issues/7) diff --git a/3rd/load-watcher/CODEOWNERS b/3rd/load-watcher/CODEOWNERS new file mode 100644 index 00000000..bf88a0b5 --- /dev/null +++ b/3rd/load-watcher/CODEOWNERS @@ -0,0 +1 @@ +* @lenhattan86 @wangchen615 diff --git a/3rd/load-watcher/Dockerfile b/3rd/load-watcher/Dockerfile new file mode 100644 index 00000000..00bfc54a --- /dev/null +++ b/3rd/load-watcher/Dockerfile @@ -0,0 +1,10 @@ +FROM golang:1.23 +WORKDIR /go/src/github.com/paypal/load-watcher +COPY . . +RUN make build + +FROM alpine:3.12 + +COPY --from=0 /go/src/github.com/paypal/load-watcher/bin/load-watcher /bin/load-watcher + +CMD ["/bin/load-watcher"] diff --git a/3rd/load-watcher/LICENSE b/3rd/load-watcher/LICENSE new file mode 100644 index 00000000..9b3082d3 --- /dev/null +++ b/3rd/load-watcher/LICENSE @@ -0,0 +1,15 @@ +/* +Copyright 2020 PayPal + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ diff --git a/3rd/load-watcher/Makefile b/3rd/load-watcher/Makefile new file mode 100644 index 00000000..b4b06af7 --- /dev/null +++ b/3rd/load-watcher/Makefile @@ -0,0 +1,28 @@ +# Copyright 2021 PayPal +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +COMMONENVVAR=GOOS=$(shell uname -s | tr A-Z a-z) GOARCH=$(subst x86_64,amd64,$(patsubst i%86,386,$(shell uname -m))) +BUILDENVVAR=CGO_ENABLED=0 + +.PHONY: all +all: build + chmod +x bin/load-watcher + +.PHONY: build +build: + $(COMMONENVVAR) $(BUILDENVVAR) go build -o bin/load-watcher main.go + +.PHONY: clean +clean: + rm -rf ./bin diff --git a/3rd/load-watcher/README.md b/3rd/load-watcher/README.md new file mode 100644 index 00000000..30d15a47 --- /dev/null +++ b/3rd/load-watcher/README.md @@ -0,0 +1,54 @@ +# Load Watcher [![Go Reference](https://pkg.go.dev/badge/github.com/paypal/load-watcher.svg)](https://pkg.go.dev/github.com/paypal/load-watcher) ![CI Build Status](https://github.com/paypal/load-watcher/actions/workflows/ci.yml/badge.svg) [![Automated Release Notes by gren](https://img.shields.io/badge/%F0%9F%A4%96-release%20notes-00B2EE.svg)](https://github-tools.github.io/github-release-notes/) + +The load watcher is responsible for the cluster-wide aggregation of resource usage metrics like CPU, memory, network, and IO stats over time windows from a metrics provider like SignalFx, Prometheus, Kubernetes Metrics Server etc. developed for [Trimaran: Real Load Aware Scheduling](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/kep/61-Trimaran-real-load-aware-scheduling/README.md) in Kubernetes. +It stores the metrics in its local cache, which can be queried from scheduler plugins. + +The following metrics provider clients are currently supported: + +1) SignalFx +2) Kubernetes Metrics Server +3) Prometheus + +These clients fetch CPU usage currently, support for other resources will be added later as needed. + +# Tutorial + +This tutorial will guide you to build load watcher Docker image, which can be deployed to work with Trimaran scheduler plugins. + +The default `main.go` is configured to watch Kubernetes Metrics Server. +You can change this to any available metrics provider in `pkg/metricsprovider`. +To build a client for new metrics provider, you will need to implement `FetcherClient` interface. + +From the root folder, run the following commands to build docker image of load watcher, tag it and push to your docker repository: + +``` +docker build -t load-watcher: . +docker tag load-watcher: : +docker push +``` + +Note that load watcher runs on default port 2020. Once deployed, you can use the following API to read watcher metrics: + +``` +GET /watcher +``` + +This will return metrics for all nodes. A query parameter to filter by host can be added with `host`. + +## Metrics Provider Configuration +- By default Kubernetes Metrics Server client is configured. Set `KUBE_CONFIG` env var to your kubernetes client configuration file path if running out of cluster. + +- To use the Prometheus client, please configure environment variables `METRICS_PROVIDER_NAME`, `METRICS_PROVIDER_ADDRESS` and `METRICS_PROVIDER_TOKEN` to `Prometheus`, Prometheus address and auth token. Please do not set `METRICS_PROVIDER_TOKEN` if no authentication + is needed to access the Prometheus APIs. Default value of address set is `http://prometheus-k8s:9090` for Prometheus client. + +- To use the SignalFx client, please configure environment variables `METRICS_PROVIDER_NAME`, `METRICS_PROVIDER_ADDRESS` and `METRICS_PROVIDER_TOKEN` to `SignalFx`, SignalFx address and auth token respectively. Default value of address set is `https://api.signalfx.com` for SignalFx client. + +## Deploy `load-watcher` as a service +To deploy `load-watcher` as a monitoring service in your Kubernetes cluster, you should replace the values in the `[]` with your own cluster monitoring stack and then you can run the following. +```bash +> kubectl create -f manifests/load-watcher-deployment.yaml +``` + +## Using `load-watcher` client +- `load-watcher-client.go` shows an example to use `load-watcher` packages as libraries in a client mode. When `load-watcher` is running as a +service exposing an endpoint in a cluster, a client, such as Trimaran plugins, can use its libraries to create a client getting the latest metrics. diff --git a/3rd/load-watcher/go.mod b/3rd/load-watcher/go.mod new file mode 100644 index 00000000..a7006398 --- /dev/null +++ b/3rd/load-watcher/go.mod @@ -0,0 +1,64 @@ +module github.com/paypal/load-watcher + +go 1.22.0 + +require ( + github.com/francoispqt/gojay v1.2.13 + github.com/prometheus/client_golang v1.19.1 + github.com/prometheus/common v0.55.0 + github.com/sirupsen/logrus v1.6.0 + github.com/stretchr/testify v1.9.0 + k8s.io/apimachinery v0.31.2 + k8s.io/client-go v0.31.2 + k8s.io/klog/v2 v2.130.1 + k8s.io/metrics v0.31.2 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.4 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/imdario/mergo v0.3.6 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/jpillora/backoff v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/x448/float16 v0.8.4 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/oauth2 v0.21.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/term v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect + golang.org/x/time v0.3.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/api v0.31.2 // indirect + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect + k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/3rd/load-watcher/go.sum b/3rd/load-watcher/go.sum new file mode 100644 index 00000000..52117f40 --- /dev/null +++ b/3rd/load-watcher/go.sum @@ -0,0 +1,328 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo= +dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= +dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= +dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= +dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= +git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= +github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= +github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= +github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af h1:kmjWCqn2qkEml422C2Rrd27c3VGxi6a/6HNq8QmHRKM= +github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= +github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= +github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= +github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= +github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= +github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= +github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= +github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= +github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= +github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= +github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= +github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw= +github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI= +github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU= +github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag= +github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg= +github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw= +github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y= +github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= +github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= +github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ= +github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I= +github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0= +github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ= +github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk= +github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= +github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= +github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= +github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= +github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= +go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= +golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= +golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= +google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +k8s.io/api v0.31.2 h1:3wLBbL5Uom/8Zy98GRPXpJ254nEFpl+hwndmk9RwmL0= +k8s.io/api v0.31.2/go.mod h1:bWmGvrGPssSK1ljmLzd3pwCQ9MgoTsRCuK35u6SygUk= +k8s.io/apimachinery v0.31.2 h1:i4vUt2hPK56W6mlT7Ry+AO8eEsyxMD1U44NR22CLTYw= +k8s.io/apimachinery v0.31.2/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/client-go v0.31.2 h1:Y2F4dxU5d3AQj+ybwSMqQnpZH9F30//1ObxOKlTI9yc= +k8s.io/client-go v0.31.2/go.mod h1:NPa74jSVR/+eez2dFsEIHNa+3o09vtNaWwWwb1qSxSs= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/metrics v0.31.2 h1:sQhujR9m3HN/Nu/0fTfTscjnswQl0qkQAodEdGBS0N4= +k8s.io/metrics v0.31.2/go.mod h1:QqqyReApEWO1UEgXOSXiHCQod6yTxYctbAAQBWZkboU= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= +sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= +sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= diff --git a/3rd/load-watcher/main.go b/3rd/load-watcher/main.go new file mode 100644 index 00000000..af3548d4 --- /dev/null +++ b/3rd/load-watcher/main.go @@ -0,0 +1,51 @@ +/* +Copyright 2020 PayPal + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/paypal/load-watcher/pkg/watcher" + "github.com/paypal/load-watcher/pkg/watcher/api" + log "github.com/sirupsen/logrus" + "os" +) + +func init() { + log.SetReportCaller(true) + logLevel, evnLogLevelSet := os.LookupEnv("LOG_LEVEL") + parsedLogLevel, err := log.ParseLevel(logLevel) + if evnLogLevelSet && err != nil { + log.Infof("unable to parse log level set; defaulting to: %v", log.GetLevel()) + } + if err == nil { + log.SetLevel(parsedLogLevel) + } +} + +func main() { + client, err := api.NewLibraryClient(watcher.EnvMetricProviderOpts) + if err != nil { + log.Fatalf("unable to create client: %v", err) + } + metrics, err := client.GetLatestWatcherMetrics() + if err != nil { + log.Errorf("unable to get watcher metrics: %v", err) + } + log.Debugf("received metrics: %v", metrics) + + // Keep the watcher server up + select {} +} diff --git a/3rd/load-watcher/manifests/load-watcher-deployment.yaml b/3rd/load-watcher/manifests/load-watcher-deployment.yaml new file mode 100644 index 00000000..493a98af --- /dev/null +++ b/3rd/load-watcher/manifests/load-watcher-deployment.yaml @@ -0,0 +1,51 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: loadwatcher +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: load-watcher-deployment + namespace: loadwatcher + labels: + app: load-watcher +spec: + replicas: 1 + selector: + matchLabels: + app: load-watcher + template: + metadata: + labels: + app: load-watcher + spec: + containers: + - name: load-watcher + image: [load-watcher image] + env: + - name: METRICS_PROVIDER_NAME + value: [Prometheus/SignalFx] + - name: METRICS_PROVIDER_ADDRESS + value: [metrics_provider_endpoint] + - name: METRICS_PROVIDER_TOKEN + value: [token] + ports: + - containerPort: 2020 +--- +apiVersion: v1 +kind: Service +metadata: + namespace: loadwatcher + name: load-watcher + labels: + app: load-watcher +spec: + type: ClusterIP + ports: + - name: http + port: 2020 + targetPort: 2020 + protocol: TCP + selector: + app: load-watcher diff --git a/3rd/load-watcher/pkg/watcher/api/api.go b/3rd/load-watcher/pkg/watcher/api/api.go new file mode 100644 index 00000000..966bf001 --- /dev/null +++ b/3rd/load-watcher/pkg/watcher/api/api.go @@ -0,0 +1,25 @@ +/* +Copyright 2021 PayPal + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import "github.com/paypal/load-watcher/pkg/watcher" + +// Watcher Client API +type Client interface { + // Returns latest metrics present in load Watcher cache + GetLatestWatcherMetrics() (*watcher.WatcherMetrics, error) +} diff --git a/3rd/load-watcher/pkg/watcher/api/client.go b/3rd/load-watcher/pkg/watcher/api/client.go new file mode 100644 index 00000000..0c5e47f5 --- /dev/null +++ b/3rd/load-watcher/pkg/watcher/api/client.go @@ -0,0 +1,113 @@ +/* +Copyright 2021 PayPal + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "fmt" + "net/http" + "time" + + "github.com/francoispqt/gojay" + "github.com/paypal/load-watcher/pkg/watcher" + "github.com/paypal/load-watcher/pkg/watcher/internal/metricsprovider" + + "k8s.io/klog/v2" +) + +const ( + httpClientTimeoutSeconds = 55 * time.Second +) + +// Client for Watcher APIs as a library +type libraryClient struct { + fetcherClient watcher.MetricsProviderClient + watcher *watcher.Watcher +} + +// Client for Watcher APIs as a service +type serviceClient struct { + httpClient http.Client + watcherAddress string +} + +// Creates a new watcher client when using watcher as a library +func NewLibraryClient(opts watcher.MetricsProviderOpts) (Client, error) { + var err error + client := libraryClient{} + switch opts.Name { + case watcher.PromClientName: + client.fetcherClient, err = metricsprovider.NewPromClient(opts) + case watcher.SignalFxClientName: + client.fetcherClient, err = metricsprovider.NewSignalFxClient(opts) + default: + client.fetcherClient, err = metricsprovider.NewMetricsServerClient() + } + if err != nil { + return client, err + } + client.watcher = watcher.NewWatcher(client.fetcherClient) + client.watcher.StartWatching() + return client, nil +} + +// Creates a new watcher client when using watcher as a service +func NewServiceClient(watcherAddress string) (Client, error) { + return serviceClient{ + httpClient: http.Client{ + Timeout: httpClientTimeoutSeconds, + }, + watcherAddress: watcherAddress, + }, nil +} + +func (c libraryClient) GetLatestWatcherMetrics() (*watcher.WatcherMetrics, error) { + return c.watcher.GetLatestWatcherMetrics(watcher.FifteenMinutes) +} + +func (c serviceClient) GetLatestWatcherMetrics() (*watcher.WatcherMetrics, error) { + req, err := http.NewRequest(http.MethodGet, c.watcherAddress+watcher.BaseUrl, nil) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + + //TODO(aqadeer): Add a couple of retries for transient errors + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + klog.V(6).Infof("received status code %v from watcher", resp.StatusCode) + if resp.StatusCode == http.StatusOK { + data := watcher.Data{NodeMetricsMap: make(map[string]watcher.NodeMetrics)} + metrics := watcher.WatcherMetrics{Data: data} + dec := gojay.BorrowDecoder(resp.Body) + defer dec.Release() + err = dec.Decode(&metrics) + if err != nil { + klog.Errorf("unable to decode watcher metrics: %v", err) + return nil, err + } else { + return &metrics, nil + } + } else { + err = fmt.Errorf("received status code %v from watcher", resp.StatusCode) + klog.Error(err) + return nil, err + } + return nil, nil +} diff --git a/3rd/load-watcher/pkg/watcher/internal/metricsprovider/k8s.go b/3rd/load-watcher/pkg/watcher/internal/metricsprovider/k8s.go new file mode 100644 index 00000000..59ab0ee4 --- /dev/null +++ b/3rd/load-watcher/pkg/watcher/internal/metricsprovider/k8s.go @@ -0,0 +1,166 @@ +/* +Copyright 2020 PayPal + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metricsprovider + +import ( + "context" + "fmt" + "net/http" + "os" + + "github.com/paypal/load-watcher/pkg/watcher" + log "github.com/sirupsen/logrus" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + metricsv "k8s.io/metrics/pkg/client/clientset/versioned" +) + +var ( + kubeConfigPresent = false + kubeConfigPath string +) + +const ( + // env variable that provides path to kube config file, if deploying from outside K8s cluster + kubeConfig = "KUBE_CONFIG" +) + +func init() { + var ok bool + kubeConfigPath, ok = os.LookupEnv(kubeConfig) + if ok { + kubeConfigPresent = true + } +} + +// This is a client for K8s provided Metric Server +type metricsServerClient struct { + // This client fetches node metrics from metric server + metricsClientSet *metricsv.Clientset + // This client fetches node capacity + coreClientSet *kubernetes.Clientset +} + +func NewMetricsServerClient() (watcher.MetricsProviderClient, error) { + var config *rest.Config + var err error + kubeConfig := "" + if kubeConfigPresent { + kubeConfig = kubeConfigPath + } + config, err = clientcmd.BuildConfigFromFlags("", kubeConfig) + if err != nil { + return nil, err + } + + metricsClientSet, err := metricsv.NewForConfig(config) + clientSet, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, err + } + return metricsServerClient{ + metricsClientSet: metricsClientSet, + coreClientSet: clientSet}, nil +} + +func (m metricsServerClient) Name() string { + return watcher.K8sClientName +} + +func (m metricsServerClient) FetchHostMetrics(host string, window *watcher.Window) ([]watcher.Metric, error) { + var metrics = []watcher.Metric{} + + nodeMetrics, err := m.metricsClientSet.MetricsV1beta1().NodeMetricses().Get(context.TODO(), host, metav1.GetOptions{}) + if err != nil { + return metrics, err + } + var cpuFetchedMetric watcher.Metric + var memFetchedMetric watcher.Metric + node, err := m.coreClientSet.CoreV1().Nodes().Get(context.Background(), host, metav1.GetOptions{}) + if err != nil { + return metrics, err + } + + // Added CPU latest metric + cpuFetchedMetric.Value = float64(100*nodeMetrics.Usage.Cpu().MilliValue()) / float64(node.Status.Capacity.Cpu().MilliValue()) + cpuFetchedMetric.Type = watcher.CPU + cpuFetchedMetric.Operator = watcher.Latest + metrics = append(metrics, cpuFetchedMetric) + + // Added Memory latest metric + memFetchedMetric.Value = float64(100*nodeMetrics.Usage.Memory().Value()) / float64(node.Status.Capacity.Memory().Value()) + memFetchedMetric.Type = watcher.Memory + memFetchedMetric.Operator = watcher.Latest + metrics = append(metrics, memFetchedMetric) + return metrics, nil +} + +func (m metricsServerClient) FetchAllHostsMetrics(window *watcher.Window) (map[string][]watcher.Metric, error) { + metrics := make(map[string][]watcher.Metric) + + nodeMetricsList, err := m.metricsClientSet.MetricsV1beta1().NodeMetricses().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return metrics, err + } + nodeList, err := m.coreClientSet.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) + if err != nil { + return metrics, err + } + + cpuNodeCapacityMap := make(map[string]int64) + memNodeCPUCapacityMap := make(map[string]int64) + for _, host := range nodeList.Items { + cpuNodeCapacityMap[host.Name] = host.Status.Capacity.Cpu().MilliValue() + memNodeCPUCapacityMap[host.Name] = host.Status.Capacity.Memory().Value() + } + for _, host := range nodeMetricsList.Items { + var cpuFetchedMetric watcher.Metric + cpuFetchedMetric.Type = watcher.CPU + cpuFetchedMetric.Operator = watcher.Latest + if _, ok := cpuNodeCapacityMap[host.Name]; !ok { + log.Errorf("unable to find host %v in node list caching cpu capacity", host.Name) + continue + } + + cpuFetchedMetric.Value = float64(100*host.Usage.Cpu().MilliValue()) / float64(cpuNodeCapacityMap[host.Name]) + metrics[host.Name] = append(metrics[host.Name], cpuFetchedMetric) + + var memFetchedMetric watcher.Metric + memFetchedMetric.Type = watcher.Memory + memFetchedMetric.Operator = watcher.Latest + if _, ok := memNodeCPUCapacityMap[host.Name]; !ok { + log.Errorf("unable to find host %v in node list caching memory capacity", host.Name) + continue + } + memFetchedMetric.Value = float64(100*host.Usage.Memory().Value()) / float64(memNodeCPUCapacityMap[host.Name]) + metrics[host.Name] = append(metrics[host.Name], memFetchedMetric) + } + + return metrics, nil +} + +func (m metricsServerClient) Health() (int, error) { + var status int + m.metricsClientSet.RESTClient().Verb("HEAD").Do(context.Background()).StatusCode(&status) + if status != http.StatusOK { + return -1, fmt.Errorf("received response status code: %v", status) + } + return 0, nil +} diff --git a/3rd/load-watcher/pkg/watcher/internal/metricsprovider/prometheus.go b/3rd/load-watcher/pkg/watcher/internal/metricsprovider/prometheus.go new file mode 100644 index 00000000..5cd57277 --- /dev/null +++ b/3rd/load-watcher/pkg/watcher/internal/metricsprovider/prometheus.go @@ -0,0 +1,319 @@ +/* +Copyright 2020 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metricsprovider + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "io/ioutil" + "net" + "net/http" + "net/url" + "os" + "time" + + "k8s.io/client-go/transport" + + "github.com/paypal/load-watcher/pkg/watcher" + "github.com/prometheus/client_golang/api" + v1 "github.com/prometheus/client_golang/api/prometheus/v1" + "github.com/prometheus/common/config" + "github.com/prometheus/common/model" + log "github.com/sirupsen/logrus" + + _ "k8s.io/client-go/plugin/pkg/client/auth/oidc" +) + +const ( + EnableOpenShiftAuth = "ENABLE_OPENSHIFT_AUTH" + K8sPodCAFilePath = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + DefaultPromAddress = "http://prometheus-k8s:9090" + promStd = "stddev_over_time" + promAvg = "avg_over_time" + promCpuMetric = "instance:node_cpu:ratio" + promMemMetric = "instance:node_memory_utilisation:ratio" + promTransBandMetric = "instance:node_network_transmit_bytes:rate:sum" + promTransBandDropMetric = "instance:node_network_transmit_drop_excluding_lo:rate5m" + promRecBandMetric = "instance:node_network_receive_bytes:rate:sum" + promRecBandDropMetric = "instance:node_network_receive_drop_excluding_lo:rate5m" + promDiskIOMetric = "instance_device:node_disk_io_time_seconds:rate5m" + promScaphHostPower = "scaph_host_power_microwatts" + promScaphHostJoules = "scaph_host_energy_microjoules" + promKeplerHostCoreJoules = "kepler_node_core_joules_total" + promKeplerHostUncoreJoules = "kepler_node_uncore_joules_total" + promKeplerHostDRAMJoules = "kepler_node_dram_joules_total" + promKeplerHostPackageJoules = "kepler_node_package_joules_total" + promKeplerHostOtherJoules = "kepler_node_other_joules_total" + promKeplerHostGPUJoules = "kepler_node_gpu_joules_total" + promKeplerHostPlatformJoules = "kepler_node_platform_joules_total" + promKeplerHostEnergyStat = "kepler_node_energy_stat" + allHosts = "all" + hostMetricKey = "node" +) + +type promClient struct { + client api.Client + promAddress string +} + +func loadCAFile(filepath string) (*x509.CertPool, error) { + caCert, err := ioutil.ReadFile(filepath) + if err != nil { + return nil, err + } + + caCertPool := x509.NewCertPool() + if ok := caCertPool.AppendCertsFromPEM(caCert); !ok { + return nil, fmt.Errorf("failed to append CA certificate to the pool") + } + + return caCertPool, nil +} + +func NewPromClient(opts watcher.MetricsProviderOpts) (watcher.MetricsProviderClient, error) { + if opts.Name != watcher.PromClientName { + return nil, fmt.Errorf("metric provider name should be %v, found %v", watcher.PromClientName, opts.Name) + } + + var client api.Client + var err error + var promToken, promAddress = "", DefaultPromAddress + if opts.AuthToken != "" { + promToken = opts.AuthToken + } + if opts.Address != "" { + promAddress = opts.Address + } + + // Ignore TLS verify errors if InsecureSkipVerify is set + roundTripper := api.DefaultRoundTripper + + // Check if EnableOpenShiftAuth is set. + _, enableOpenShiftAuth := os.LookupEnv(EnableOpenShiftAuth) + if enableOpenShiftAuth { + // Retrieve Pod CA cert + caCertPool, err := loadCAFile(K8sPodCAFilePath) + if err != nil { + return nil, fmt.Errorf("Error loading CA file: %v", err) + } + + // Get Prometheus Host + u, _ := url.Parse(opts.Address) + roundTripper = transport.NewBearerAuthRoundTripper( + opts.AuthToken, + &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + TLSHandshakeTimeout: 10 * time.Second, + TLSClientConfig: &tls.Config{ + RootCAs: caCertPool, + ServerName: u.Host, + }, + }, + ) + } else if opts.InsecureSkipVerify { + roundTripper = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + TLSHandshakeTimeout: 10 * time.Second, + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + } + + if promToken != "" { + client, err = api.NewClient(api.Config{ + Address: promAddress, + RoundTripper: config.NewAuthorizationCredentialsRoundTripper("Bearer", config.NewInlineSecret(opts.AuthToken), roundTripper), + }) + } else { + client, err = api.NewClient(api.Config{ + Address: promAddress, + }) + } + + if err != nil { + log.Errorf("error creating prometheus client: %v", err) + return nil, err + } + + return promClient{ + client: client, + promAddress: promAddress, + }, err +} + +func (s promClient) Name() string { + return watcher.PromClientName +} + +func (s promClient) FetchHostMetrics(host string, window *watcher.Window) ([]watcher.Metric, error) { + var metricList []watcher.Metric + var anyerr error + + for _, method := range []string{promAvg, promStd} { + for _, metric := range []string{promCpuMetric, promMemMetric, promTransBandMetric, promTransBandDropMetric, promRecBandMetric, promRecBandDropMetric, + promDiskIOMetric, promScaphHostPower, promScaphHostJoules, promKeplerHostCoreJoules, promKeplerHostUncoreJoules, promKeplerHostDRAMJoules, + promKeplerHostPackageJoules, promKeplerHostOtherJoules, promKeplerHostGPUJoules, promKeplerHostPlatformJoules, promKeplerHostEnergyStat} { + promQuery := s.buildPromQuery(host, metric, method, window.Duration) + promResults, err := s.getPromResults(promQuery) + + if err != nil { + log.Errorf("error querying Prometheus for query %v: %v\n", promQuery, err) + anyerr = err + continue + } + + curMetricMap := s.promResults2MetricMap(promResults, metric, method, window.Duration) + metricList = append(metricList, curMetricMap[host]...) + } + } + + return metricList, anyerr +} + +// FetchAllHostsMetrics Fetch all host metrics with different operators (avg_over_time, stddev_over_time) and different resource types (CPU, Memory) +func (s promClient) FetchAllHostsMetrics(window *watcher.Window) (map[string][]watcher.Metric, error) { + hostMetrics := make(map[string][]watcher.Metric) + var anyerr error + + for _, method := range []string{promAvg, promStd} { + for _, metric := range []string{promCpuMetric, promMemMetric, promTransBandMetric, promTransBandDropMetric, promRecBandMetric, promRecBandDropMetric, + promDiskIOMetric, promScaphHostPower, promScaphHostJoules, promKeplerHostCoreJoules, promKeplerHostUncoreJoules, promKeplerHostDRAMJoules, + promKeplerHostPackageJoules, promKeplerHostOtherJoules, promKeplerHostGPUJoules, promKeplerHostPlatformJoules, promKeplerHostEnergyStat} { + promQuery := s.buildPromQuery(allHosts, metric, method, window.Duration) + promResults, err := s.getPromResults(promQuery) + + if err != nil { + log.Errorf("error querying Prometheus for query %v: %v\n", promQuery, err) + anyerr = err + continue + } + + curMetricMap := s.promResults2MetricMap(promResults, metric, method, window.Duration) + + for k, v := range curMetricMap { + // skip empty keys + if k == "" { + continue + } + hostMetrics[k] = append(hostMetrics[k], v...) + } + } + } + + return hostMetrics, anyerr +} + +func (s promClient) Health() (int, error) { + req, err := http.NewRequest("HEAD", s.promAddress, nil) + if err != nil { + return -1, err + } + resp, _, err := s.client.Do(context.Background(), req) + if err != nil { + return -1, err + } + if resp.StatusCode != http.StatusOK { + return -1, fmt.Errorf("received response status code: %v", resp.StatusCode) + } + return 0, nil +} + +func (s promClient) buildPromQuery(host string, metric string, method string, rollup string) string { + var promQuery string + + if host == allHosts { + promQuery = fmt.Sprintf("%s(%s[%s])", method, metric, rollup) + } else { + promQuery = fmt.Sprintf("%s(%s{%s=\"%s\"}[%s])", method, metric, hostMetricKey, host, rollup) + } + + return promQuery +} + +func (s promClient) getPromResults(promQuery string) (model.Value, error) { + v1api := v1.NewAPI(s.client) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + results, warnings, err := v1api.Query(ctx, promQuery, time.Now()) + if err != nil { + return nil, err + } + if len(warnings) > 0 { + log.Warnf("Warnings: %v\n", warnings) + } + log.Debugf("result:\n%v\n", results) + + return results, nil +} + +func (s promClient) promResults2MetricMap(promresults model.Value, metric string, method string, rollup string) map[string][]watcher.Metric { + var metricType string + var operator string + + curMetrics := make(map[string][]watcher.Metric) + + switch metric { + case promCpuMetric: // CPU metrics + metricType = watcher.CPU + case promMemMetric: // Memory metrics + metricType = watcher.Memory + case promDiskIOMetric: // Storage metrics + metricType = watcher.Storage + case promScaphHostPower, promScaphHostJoules, // Energy-related metrics + promKeplerHostCoreJoules, promKeplerHostUncoreJoules, + promKeplerHostDRAMJoules, promKeplerHostPackageJoules, + promKeplerHostOtherJoules, promKeplerHostGPUJoules, + promKeplerHostPlatformJoules, promKeplerHostEnergyStat: + metricType = watcher.Energy + case promTransBandMetric, promTransBandDropMetric, // Bandwidth-related metrics + promRecBandMetric, promRecBandDropMetric: + metricType = watcher.Bandwidth + default: + metricType = watcher.Unknown + } + + if method == promAvg { + operator = watcher.Average + } else if method == promStd { + operator = watcher.Std + } else { + operator = watcher.UnknownOperator + } + + switch promresults.(type) { + case model.Vector: + for _, result := range promresults.(model.Vector) { + curMetric := watcher.Metric{Name: metric, Type: metricType, Operator: operator, Rollup: rollup, Value: float64(result.Value * 100)} + curHost := string(result.Metric[hostMetricKey]) + curMetrics[curHost] = append(curMetrics[curHost], curMetric) + } + default: + log.Errorf("error: The Prometheus results should not be type: %v.\n", promresults.Type()) + } + + return curMetrics +} diff --git a/3rd/load-watcher/pkg/watcher/internal/metricsprovider/signalfx.go b/3rd/load-watcher/pkg/watcher/internal/metricsprovider/signalfx.go new file mode 100644 index 00000000..c67a680b --- /dev/null +++ b/3rd/load-watcher/pkg/watcher/internal/metricsprovider/signalfx.go @@ -0,0 +1,467 @@ +/* +Copyright 2020 PayPal + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metricsprovider + +import ( + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "time" + + "github.com/paypal/load-watcher/pkg/watcher" + log "github.com/sirupsen/logrus" +) + +const ( + // SignalFX Request Params + DefaultSignalFxAddress = "https://api.signalfx.com" + signalFxMetricsAPI = "/v1/timeserieswindow" + signalFxMetdataAPI = "/v2/metrictimeseries" + signalFxHostFilter = "host:" + signalFxClusterFilter = "cluster:" + signalFxHostNameSuffixKey = "SIGNALFX_HOST_NAME_SUFFIX" + signalFxClusterName = "SIGNALFX_CLUSTER_NAME" + // SignalFX Query Params + oneMinuteResolutionMs = 60000 + cpuUtilizationMetric = `sf_metric:"cpu.utilization"` + memoryUtilizationMetric = `sf_metric:"memory.utilization"` + AND = "AND" + resultSetLimit = "10000" + + // Miscellaneous + httpClientTimeout = 55 * time.Second +) + +type signalFxClient struct { + client http.Client + authToken string + signalFxAddress string + hostNameSuffix string + clusterName string +} + +func NewSignalFxClient(opts watcher.MetricsProviderOpts) (watcher.MetricsProviderClient, error) { + if opts.Name != watcher.SignalFxClientName { + return nil, fmt.Errorf("metric provider name should be %v, found %v", watcher.SignalFxClientName, opts.Name) + } + tlsConfig := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: opts.InsecureSkipVerify}, // TODO(aqadeer): Figure out a secure way to let users add SSL certs + } + hostNameSuffix, _ := os.LookupEnv(signalFxHostNameSuffixKey) + clusterName, _ := os.LookupEnv(signalFxClusterName) + var signalFxAddress, signalFxAuthToken = DefaultSignalFxAddress, "" + if opts.Address != "" { + signalFxAddress = opts.Address + } + if opts.AuthToken != "" { + signalFxAuthToken = opts.AuthToken + } + if signalFxAuthToken == "" { + log.Fatalf("No auth token found to connect with SignalFx server") + } + return signalFxClient{client: http.Client{ + Timeout: httpClientTimeout, + Transport: tlsConfig}, + authToken: signalFxAuthToken, + signalFxAddress: signalFxAddress, + hostNameSuffix: hostNameSuffix, + clusterName: clusterName}, nil +} + +func (s signalFxClient) Name() string { + return watcher.SignalFxClientName +} + +func (s signalFxClient) FetchHostMetrics(host string, window *watcher.Window) ([]watcher.Metric, error) { + log.Debugf("fetching metrics for host %v", host) + var metrics []watcher.Metric + hostFilter := signalFxHostFilter + host + s.hostNameSuffix + clusterFilter := signalFxClusterFilter + s.clusterName + for _, metric := range []string{cpuUtilizationMetric, memoryUtilizationMetric} { + uri, err := s.buildMetricURL(hostFilter, clusterFilter, metric, window) + if err != nil { + return metrics, fmt.Errorf("received error when building metric URL: %v", err) + } + req, _ := http.NewRequest(http.MethodGet, uri.String(), nil) + req.Header.Set("X-SF-Token", s.authToken) + req.Header.Set("Content-Type", "application/json") + + resp, err := s.client.Do(req) + if err != nil { + return metrics, fmt.Errorf("received error in metric API call: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return metrics, fmt.Errorf("received status code: %v", resp.StatusCode) + } + var res interface{} + err = json.NewDecoder(resp.Body).Decode(&res) + if err != nil { + return metrics, fmt.Errorf("received error in decoding resp: %v", err) + } + + var fetchedMetric watcher.Metric + addMetadata(&fetchedMetric, metric) + fetchedMetric.Value, err = decodeMetricsPayload(res) + if err != nil { + return metrics, err + } + metrics = append(metrics, fetchedMetric) + } + return metrics, nil +} + +func (s signalFxClient) FetchAllHostsMetrics(window *watcher.Window) (map[string][]watcher.Metric, error) { + hostFilter := signalFxHostFilter + "*" + s.hostNameSuffix + clusterFilter := signalFxClusterFilter + s.clusterName + metrics := make(map[string][]watcher.Metric) + for _, metric := range []string{cpuUtilizationMetric, memoryUtilizationMetric} { + uri, err := s.buildMetricURL(hostFilter, clusterFilter, metric, window) + if err != nil { + return metrics, fmt.Errorf("received error when building metric URL: %v", err) + } + req := s.requestWithAuthToken(uri.String()) + metricResp, err := s.client.Do(req) + if err != nil { + return metrics, fmt.Errorf("received error in metric API call: %v", err) + } + defer metricResp.Body.Close() + if metricResp.StatusCode != http.StatusOK { + return metrics, fmt.Errorf("received status code for metric resp: %v", metricResp.StatusCode) + } + var metricPayload interface{} + err = json.NewDecoder(metricResp.Body).Decode(&metricPayload) + if err != nil { + return metrics, fmt.Errorf("received error in decoding resp: %v", err) + } + + uri, err = s.buildMetadataURL(hostFilter, clusterFilter, metric) + if err != nil { + return metrics, fmt.Errorf("received error when building metadata URL: %v", err) + } + req = s.requestWithAuthToken(uri.String()) + metadataResp, err := s.client.Do(req) + if err != nil { + return metrics, fmt.Errorf("received error in metadata API call: %v", err) + } + defer metadataResp.Body.Close() + if metadataResp.StatusCode != http.StatusOK { + return metrics, fmt.Errorf("received status code for metadata resp: %v", metadataResp.StatusCode) + } + var metadataPayload interface{} + err = json.NewDecoder(metadataResp.Body).Decode(&metadataPayload) + if err != nil { + return metrics, fmt.Errorf("received error in decoding metadata payload: %v", err) + } + mappedMetrics, err := getMetricsFromPayloads(metricPayload, metadataPayload) + if err != nil { + return metrics, fmt.Errorf("received error in getting metrics from payload: %v", err) + } + for k, v := range mappedMetrics { + addMetadata(&v, metric) + metrics[k] = append(metrics[k], v) + } + } + return metrics, nil +} + +func (s signalFxClient) Health() (int, error) { + return Ping(s.client, s.signalFxAddress) +} + +func (s signalFxClient) requestWithAuthToken(uri string) *http.Request { + req, _ := http.NewRequest(http.MethodGet, uri, nil) + req.Header.Set("X-SF-Token", s.authToken) + req.Header.Set("Content-Type", "application/json") + return req +} + +// Simple ping utility to a given URL +// Returns -1 if unhealthy, 0 if healthy along with error if any +func Ping(client http.Client, url string) (int, error) { + req, err := http.NewRequest("HEAD", url, nil) + if err != nil { + return -1, err + } + resp, err := client.Do(req) + if err != nil { + return -1, err + } + if resp.StatusCode != http.StatusOK { + return -1, fmt.Errorf("received response code: %v", resp.StatusCode) + } + return 0, nil +} + +func addMetadata(metric *watcher.Metric, metricType string) { + metric.Operator = watcher.Average + if metricType == cpuUtilizationMetric { + metric.Name = cpuUtilizationMetric + metric.Type = watcher.CPU + } else { + metric.Name = memoryUtilizationMetric + metric.Type = watcher.Memory + } +} + +func (s signalFxClient) buildMetricURL(hostFilter string, clusterFilter string, metric string, window *watcher.Window) (uri *url.URL, err error) { + uri, err = url.Parse(s.signalFxAddress + signalFxMetricsAPI) + if err != nil { + return nil, err + } + q := uri.Query() + + builder := strings.Builder{} + builder.WriteString(hostFilter) + builder.WriteString(fmt.Sprintf(" %v ", AND)) + builder.WriteString(clusterFilter) + builder.WriteString(fmt.Sprintf(" %v ", AND)) + builder.WriteString(metric) + q.Set("query", builder.String()) + q.Set("startMs", strconv.FormatInt(window.Start*1000, 10)) + q.Set("endMs", strconv.FormatInt(window.End*1000, 10)) + q.Set("resolution", strconv.Itoa(oneMinuteResolutionMs)) + uri.RawQuery = q.Encode() + return +} + +func (s signalFxClient) buildMetadataURL(host string, clusterFilter string, metric string) (uri *url.URL, err error) { + uri, err = url.Parse(s.signalFxAddress + signalFxMetdataAPI) + if err != nil { + return nil, err + } + q := uri.Query() + + builder := strings.Builder{} + builder.WriteString(host) + builder.WriteString(fmt.Sprintf(" %v ", AND)) + builder.WriteString(clusterFilter) + builder.WriteString(fmt.Sprintf(" %v ", AND)) + builder.WriteString(metric) + q.Set("query", builder.String()) + q.Set("limit", resultSetLimit) + uri.RawQuery = q.Encode() + return +} + +/** +Sample payload: +{ + "data": { + "Ehql_bxBgAc": [ + [ + 1600213380000, + 84.64246793530153 + ] + ] + }, + "errors": [] +} +*/ +func decodeMetricsPayload(payload interface{}) (float64, error) { + var data interface{} + data = payload.(map[string]interface{})["data"] + if data == nil { + return -1, errors.New("unexpected payload: missing data field") + } + keyMap, ok := data.(map[string]interface{}) + if !ok { + return -1, errors.New("unable to deserialise data field") + } + + var values []interface{} + if len(keyMap) == 0 { + return -1, errors.New("no values found") + } + for _, v := range keyMap { + values, ok = v.([]interface{}) + if !ok { + return -1, errors.New("unable to deserialise values") + } + break + } + if len(values) == 0 { + return -1, errors.New("no metric value array could be decoded") + } + + var timestampUtilisation []interface{} + // Choose the latest window out of multiple values returned + timestampUtilisation, ok = values[len(values)-1].([]interface{}) + if !ok { + return -1, errors.New("unable to deserialise metric values") + } + return timestampUtilisation[1].(float64), nil +} + +/** +Sample metricData payload: +{ + "data": { + "Ehql_bxBgAc": [ + [ + 1600213380000, + 84.64246793530153 + ] + ], + "EuXgJm7BkAA": [ + [ + 1614634260000, + 5.450946379084264 + ] + ], + .... + .... + }, + "errors": [] +} + +https://dev.splunk.com/observability/reference/api/metrics_metadata/latest#endpoint-retrieve-metric-timeseries-metadata +Sample metaData payload: +{ + "count": 5, + "partialCount": false, + "results": [ + { + "active": true, + "created": 1614534848000, + "creator": null, + "dimensions": { + "host": "test.dev.com", + "sf_metric": null + }, + "id": "EvVH6P7BgAA", + "lastUpdated": 0, + "lastUpdatedBy": null, + "metric": "cpu.utilization" + }, + .... + .... + ] +} +*/ +func getMetricsFromPayloads(metricData interface{}, metadata interface{}) (map[string]watcher.Metric, error) { + keyHostMap := make(map[string]string) + hostMetricMap := make(map[string]watcher.Metric) + if _, ok := metadata.(map[string]interface{}); !ok { + return hostMetricMap, fmt.Errorf("type conversion failed, found %T", metadata) + } + results := metadata.(map[string]interface{})["results"] + if results == nil { + return hostMetricMap, errors.New("unexpected payload: missing results field") + } + + for _, v := range results.([]interface{}) { + _, ok := v.(map[string]interface{}) + if !ok { + log.Errorf("type conversion failed, found %T", v) + continue + } + id := v.(map[string]interface{})["id"] + if id == nil { + log.Errorf("id not found in %v", v) + continue + } + _, ok = id.(string) + if !ok { + log.Errorf("id not expected type string, found %T", id) + continue + } + dimensions := v.(map[string]interface{})["dimensions"] + if dimensions == nil { + log.Errorf("no dimensions found in %v", v) + continue + } + _, ok = dimensions.(map[string]interface{}) + if !ok { + log.Errorf("type conversion failed, found %T", dimensions) + continue + } + host := dimensions.(map[string]interface{})["host"] + if host == nil { + log.Errorf("no host found in %v", dimensions) + continue + } + if _, ok := host.(string); !ok { + log.Errorf("host not expected type string, found %T", host) + } + + keyHostMap[id.(string)] = extractHostName(host.(string)) + } + + var data interface{} + data = metricData.(map[string]interface{})["data"] + if data == nil { + return hostMetricMap, errors.New("unexpected payload: missing data field") + } + keyMetricMap, ok := data.(map[string]interface{}) + if !ok { + return hostMetricMap, errors.New("unable to deserialise data field") + } + for key, metric := range keyMetricMap { + if _, ok := keyHostMap[key]; !ok { + log.Errorf("no metadata found for key %v", key) + continue + } + values, ok := metric.([]interface{}) + if !ok { + log.Errorf("unable to deserialise values for key %v", key) + continue + } + if len(values) == 0 { + log.Errorf("no metric value array could be decoded for key %v", key) + continue + } + // Find the average across returned values per 1 minute resolution + var sum float64 + var count float64 + for _, value := range values { + var timestampUtilisation []interface{} + timestampUtilisation, ok = value.([]interface{}) + if !ok || len(timestampUtilisation) < 2 { + log.Errorf("unable to deserialise metric values for key %v", key) + continue + } + if _, ok := timestampUtilisation[1].(float64); !ok { + log.Errorf("unable to typecast value to float64: %v of type %T", timestampUtilisation, timestampUtilisation) + } + sum += timestampUtilisation[1].(float64) + count += 1 + } + + fetchedMetric := watcher.Metric{Value: sum / count} + hostMetricMap[keyHostMap[key]] = fetchedMetric + } + + return hostMetricMap, nil +} + +// This function checks and extracts node name from its FQDN if present +// It assumes that node names themselves don't contain "." +// Example: alpha.dev.k8s.com is returned as alpha +func extractHostName(fqdn string) string { + index := strings.Index(fqdn, ".") + if index != -1 { + return fqdn[:index] + } + return fqdn +} diff --git a/3rd/load-watcher/pkg/watcher/internal/metricsprovider/signalfx_test.go b/3rd/load-watcher/pkg/watcher/internal/metricsprovider/signalfx_test.go new file mode 100644 index 00000000..f49715d9 --- /dev/null +++ b/3rd/load-watcher/pkg/watcher/internal/metricsprovider/signalfx_test.go @@ -0,0 +1,99 @@ +package metricsprovider + +import ( + "github.com/paypal/load-watcher/pkg/watcher" + "github.com/stretchr/testify/assert" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestNewSignalFxClient(t *testing.T) { + opts := watcher.MetricsProviderOpts{ + Name: watcher.SignalFxClientName, + Address: "", + AuthToken: "Test", + } + _, err := NewSignalFxClient(opts) + assert.Nil(t, err) + + opts.Name = "invalid" + _, err = NewSignalFxClient(opts) + assert.NotNil(t, err) +} + +func TestFetchAllHostMetrics(t *testing.T) { + metricData := `{ + "data": { + "Ehql_bxBgAc": [ + [ + 1600213380000, + 84.64246793530153 + ] + ], + "EuXgJm7BkAA": [ + [ + 1614634260000, + 5.450946379084264 + ] + ] + }, + "errors": [] +}` + metaData := `{ + "count":2, + "partialCount":false, + "results":[ + { + "active":true, + "created":1614534848000, + "creator":null, + "dimensions":{ + "host":"test1.dev.com", + "sf_metric":null + }, + "id":"Ehql_bxBgAc", + "lastUpdated":0, + "lastUpdatedBy":null, + "metric":"cpu.utilization" + }, + { + "active":true, + "created":1614534848000, + "creator":null, + "dimensions":{ + "host":"test2.dev.com", + "sf_metric":null + }, + "id":"EuXgJm7BkAA", + "lastUpdated":0, + "lastUpdatedBy":null, + "metric":"cpu.utilization" + } + ] +}` + server := httptest.NewServer(http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { + if strings.Contains(req.URL.Path, signalFxMetdataAPI) { + resp.Write([]byte(metaData)) + } else { + resp.Write([]byte(metricData)) + } + })) + opts := watcher.MetricsProviderOpts{ + Name: watcher.SignalFxClientName, + Address: server.URL, + AuthToken: "PWNED", + } + + client, err := NewSignalFxClient(opts) + assert.Nil(t, err) + + metrics, err := client.FetchAllHostsMetrics(watcher.CurrentFifteenMinuteWindow()) + assert.Nil(t, err) + assert.NotNil(t, metrics) + assert.NotNil(t, metrics["test1"]) + assert.NotNil(t, metrics["test2"]) + + defer server.Close() +} diff --git a/3rd/load-watcher/pkg/watcher/metricsprovider.go b/3rd/load-watcher/pkg/watcher/metricsprovider.go new file mode 100644 index 00000000..26368605 --- /dev/null +++ b/3rd/load-watcher/pkg/watcher/metricsprovider.go @@ -0,0 +1,74 @@ +/* +Copyright 2020 PayPal + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package watcher + +import ( + "os" + "strings" +) + +const ( + K8sClientName = "KubernetesMetricsServer" + PromClientName = "Prometheus" + SignalFxClientName = "SignalFx" + + MetricsProviderNameKey = "METRICS_PROVIDER_NAME" + MetricsProviderAddressKey = "METRICS_PROVIDER_ADDRESS" + MetricsProviderTokenKey = "METRICS_PROVIDER_TOKEN" + InsecureSkipVerify = "INSECURE_SKIP_VERIFY" +) + +var ( + EnvMetricProviderOpts MetricsProviderOpts +) + +func init() { + var ok bool + EnvMetricProviderOpts.Name, ok = os.LookupEnv(MetricsProviderNameKey) + if !ok { + EnvMetricProviderOpts.Name = K8sClientName + } + EnvMetricProviderOpts.Address, ok = os.LookupEnv(MetricsProviderAddressKey) + EnvMetricProviderOpts.AuthToken, ok = os.LookupEnv(MetricsProviderTokenKey) + insecureVerify, _ := os.LookupEnv(InsecureSkipVerify) + if strings.ToLower(insecureVerify) == "true" { + EnvMetricProviderOpts.InsecureSkipVerify = true + } else { + EnvMetricProviderOpts.InsecureSkipVerify = false + } +} + +// Interface to be implemented by any metrics provider client to interact with Watcher +type MetricsProviderClient interface { + // Return the client name + Name() string + // Fetch metrics for given host + FetchHostMetrics(host string, window *Window) ([]Metric, error) + // Fetch metrics for all hosts + FetchAllHostsMetrics(window *Window) (map[string][]Metric, error) + // Get metric provider server health status + // Returns 0 if healthy, -1 if unhealthy along with error if any + Health() (int, error) +} + +// Generic metrics provider options +type MetricsProviderOpts struct { + Name string + Address string + AuthToken string + InsecureSkipVerify bool +} diff --git a/3rd/load-watcher/pkg/watcher/schema/watcher-example.json b/3rd/load-watcher/pkg/watcher/schema/watcher-example.json new file mode 100644 index 00000000..435f9524 --- /dev/null +++ b/3rd/load-watcher/pkg/watcher/schema/watcher-example.json @@ -0,0 +1,53 @@ +{ + "timestamp": 1556987522, + "window": { + "duration": "15m", + "start": 1556984522, + "end": 1556985422 + }, + "source": "InfluxDB", + "data": { + "node-1": { + "metrics": [ + { + "name": "host.cpu.utilisation", + "type": "cpu", + "rollup": "AVG", + "value": 20 + }, + { + "name": "host.memory.utilisation", + "type": "memory", + "rollup": "STD", + "value": 5 + } + ], + "tags": {}, + "metadata": { + "dataCenter": "data-center-1", + "pool": "critical-apps" + } + }, + "node-2": { + "metrics": [ + { + "name": "host.cpu.utilisation", + "type": "cpu", + "rollup": "AVG", + "value": 20 + }, + { + "name": "host.memory.utilisation", + "type": "memory", + "rollup": "STD", + "value": 5 + } + ] + }, + "metadata": { + "dataCenter": "data-center-2", + "pool": "light-apps" + }, + "tags": {} + } +} \ No newline at end of file diff --git a/3rd/load-watcher/pkg/watcher/schema/watcher-schema.json b/3rd/load-watcher/pkg/watcher/schema/watcher-schema.json new file mode 100644 index 00000000..e3f55a1b --- /dev/null +++ b/3rd/load-watcher/pkg/watcher/schema/watcher-schema.json @@ -0,0 +1,125 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "timestamp": { + "type": "integer" + }, + "window": { + "type": "object", + "properties": { + "duration": { + "type": "string" + }, + "start": { + "type": "integer" + }, + "end": { + "type": "integer" + } + }, + "required": [ + "duration", + "start", + "end" + ] + }, + "source": { + "type": "string" + }, + "data": { + "type": "object", + "patternProperties": { + "^[0-9]+$": { + "type": "object", + "properties": { + "metrics": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "type": { + "type": "string" + }, + "operator": { + "type": "string" + }, + "rollup": { + "type": "string" + }, + "value": { + "type": "integer", + "description": "percentage value" + } + }, + "required": [ + "name", + "type", + "operator", + "rollup", + "value" + ] + }, + { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "type": { + "type": "string" + }, + "operator": { + "type": "string" + }, + "rollup": { + "type": "string" + }, + "value": { + "type": "integer", + "description": "percentage value" + } + }, + "required": [ + "name", + "type", + "operator", + "rollup", + "value" + ] + } + ] + }, + "tags": { + "type": "object" + }, + "metadata": { + "type": "object", + "properties": { + "dataCenter": { + "type": "string" + }, + "pool": { + "type": "string" + } + } + } + }, + "required": [ + "metrics" + ] + } + } + } + }, + "required": [ + "timestamp", + "window", + "source", + "data" + ] +} \ No newline at end of file diff --git a/3rd/load-watcher/pkg/watcher/testserver.go b/3rd/load-watcher/pkg/watcher/testserver.go new file mode 100644 index 00000000..4910f6e8 --- /dev/null +++ b/3rd/load-watcher/pkg/watcher/testserver.go @@ -0,0 +1,121 @@ +/* +Copyright 2020 PayPal + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package watcher + +var FifteenMinutesMetricsMap = map[string][]Metric{ + FirstNode: { + { + Name: "test-cpu", + Type: CPU, + Value: 26, + }, + }, + SecondNode: { + { + Name: "test-cpu", + Type: CPU, + Value: 60, + }, + }, +} + +var TenMinutesMetricsMap = map[string][]Metric{ + FirstNode: { + { + Name: "test-cpu", + Type: CPU, + Value: 22, + }, + }, + SecondNode: { + { + Name: "test-cpu", + Type: CPU, + Value: 65, + }, + }, +} + +var FiveMinutesMetricsMap = map[string][]Metric{ + FirstNode: { + { + Name: "test-cpu", + Type: CPU, + Value: 21, + }, + }, + SecondNode: { + { + Name: "test-cpu", + Type: CPU, + Value: 50, + }, + }, +} + +var _ MetricsProviderClient = &testServerClient{} + +const ( + FirstNode = "worker-1" + SecondNode = "worker-2" + TestServerClientName = "TestServerClient" +) + +type testServerClient struct { +} + +func (t testServerClient) Name() string { + return TestServerClientName +} + +func NewTestMetricsServerClient() MetricsProviderClient { + return testServerClient{} +} + +func (t testServerClient) FetchHostMetrics(host string, window *Window) ([]Metric, error) { + if _, ok := FifteenMinutesMetricsMap[host]; !ok { + return nil, nil + } + if _, ok := TenMinutesMetricsMap[host]; !ok { + return nil, nil + } + if _, ok := FiveMinutesMetricsMap[host]; !ok { + return nil, nil + } + + if window.Duration == TenMinutes { + return TenMinutesMetricsMap[host], nil + } else if window.Duration == FiveMinutes { + return FiveMinutesMetricsMap[host], nil + } + + return FifteenMinutesMetricsMap[host], nil +} + +func (t testServerClient) FetchAllHostsMetrics(window *Window) (map[string][]Metric, error) { + if window.Duration == TenMinutes { + return TenMinutesMetricsMap, nil + } else if window.Duration == FiveMinutes { + return FiveMinutesMetricsMap, nil + } + + return FifteenMinutesMetricsMap, nil +} + +func (t testServerClient) Health() (int, error) { + return 0, nil +} diff --git a/3rd/load-watcher/pkg/watcher/watcher.go b/3rd/load-watcher/pkg/watcher/watcher.go new file mode 100644 index 00000000..7d6576c5 --- /dev/null +++ b/3rd/load-watcher/pkg/watcher/watcher.go @@ -0,0 +1,350 @@ +/* +Copyright 2020 PayPal + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Package Watcher is responsible for watching latest metrics from metrics provider via a fetcher client. +It exposes an HTTP REST endpoint to get these metrics, in addition to application API via clients +This also uses a fast json parser +*/ +package watcher + +import ( + "context" + "errors" + "net/http" + "os" + "os/signal" + "sync" + "syscall" + "time" + + "github.com/francoispqt/gojay" + log "github.com/sirupsen/logrus" +) + +const ( + BaseUrl = "/watcher" + HealthCheckUrl = "/watcher/health" + FifteenMinutes = "15m" + TenMinutes = "10m" + FiveMinutes = "5m" + CPU = "CPU" + Memory = "Memory" + Bandwidth = "Bandwidth" + Storage = "Storage" + Energy = "Energy" + Unknown = "Unknown" + Average = "AVG" + Std = "STD" + Latest = "Latest" + UnknownOperator = "Unknown" +) + +type Watcher struct { + mutex sync.RWMutex // For thread safe access to cache + fifteenMinute []WatcherMetrics + tenMinute []WatcherMetrics + fiveMinute []WatcherMetrics + cacheSize int + client MetricsProviderClient + isStarted bool // Indicates if the Watcher is started by calling StartWatching() + shutdown chan os.Signal +} + +type Window struct { + Duration string `json:"duration"` + Start int64 `json:"start"` + End int64 `json:"end"` +} + +type Metric struct { + Name string `json:"name"` // Name of metric at the provider + Type string `json:"type"` // CPU or Memory + Operator string `json:"operator"` // STD or AVE or SUM, etc. + Rollup string `json:"rollup,omitempty"` // Rollup used for metric calculation + Value float64 `json:"value"` // Value is expected to be in % +} + +type NodeMetricsMap map[string]NodeMetrics + +type Data struct { + NodeMetricsMap NodeMetricsMap +} + +type WatcherMetrics struct { + Timestamp int64 `json:"timestamp"` + Window Window `json:"window"` + Source string `json:"source"` + Data Data `json:"data"` +} + +type Tags struct { +} + +type Metadata struct { + DataCenter string `json:"dataCenter,omitempty"` +} + +type NodeMetrics struct { + Metrics []Metric `json:"metrics,omitempty"` + Tags Tags `json:"tags,omitempty"` + Metadata Metadata `json:"metadata,omitempty"` +} + +// NewWatcher Returns a new initialised Watcher +func NewWatcher(client MetricsProviderClient) *Watcher { + sizePerWindow := 5 + return &Watcher{ + mutex: sync.RWMutex{}, + fifteenMinute: make([]WatcherMetrics, 0, sizePerWindow), + tenMinute: make([]WatcherMetrics, 0, sizePerWindow), + fiveMinute: make([]WatcherMetrics, 0, sizePerWindow), + cacheSize: sizePerWindow, + client: client, + shutdown: make(chan os.Signal, 1), + } +} + +// StartWatching This function needs to be called to begin actual watching +func (w *Watcher) StartWatching() { + w.mutex.RLock() + if w.isStarted { + w.mutex.RUnlock() + return + } + w.mutex.RUnlock() + + fetchOnce := func(duration string) { + curWindow, metric := w.getCurrentWindow(duration) + hostMetrics, err := w.client.FetchAllHostsMetrics(curWindow) + + if err != nil { + log.Errorf("received error while fetching metrics: %v", err) + return + } + log.Debugf("fetched metrics for window: %v", curWindow) + + // TODOļ¼š add tags, etc. + watcherMetrics := metricMapToWatcherMetrics(hostMetrics, w.client.Name(), *curWindow) + w.appendWatcherMetrics(metric, &watcherMetrics) + } + + windowWatcher := func(duration string) { + for { + fetchOnce(duration) + // This is assuming fetching of metrics won't exceed more than 1 minute. If it happens we need to throttle rate of fetches + time.Sleep(time.Minute) + } + } + + durations := [3]string{FifteenMinutes, TenMinutes, FiveMinutes} + for _, duration := range durations { + // Populate cache initially before returning + fetchOnce(duration) + go windowWatcher(duration) + } + + http.HandleFunc(BaseUrl, w.handler) + http.HandleFunc(HealthCheckUrl, w.healthCheckHandler) + server := &http.Server{ + Addr: ":2020", + Handler: http.DefaultServeMux, + } + + go func() { + log.Warn(server.ListenAndServe()) + }() + + signal.Notify(w.shutdown, os.Interrupt, syscall.SIGTERM) + + go func() { + <-w.shutdown + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + if err := server.Shutdown(ctx); err != nil { + log.Errorf("Unable to shutdown server: %v", err) + } + }() + + w.mutex.Lock() + w.isStarted = true + w.mutex.Unlock() + log.Info("Started watching metrics") +} + +// GetLatestWatcherMetrics It starts from 15 minute window, and falls back to 10 min, 5 min windows subsequently +// if metrics are not present. StartWatching() should be called before calling this. +func (w *Watcher) GetLatestWatcherMetrics(duration string) (*WatcherMetrics, error) { + w.mutex.RLock() + defer w.mutex.RUnlock() + if !w.isStarted { + return nil, errors.New("need to call StartWatching() first") + } + + switch { + case duration == FifteenMinutes && len(w.fifteenMinute) > 0: + return w.deepCopyWatcherMetrics(&w.fifteenMinute[len(w.fifteenMinute)-1]), nil + case (duration == FifteenMinutes || duration == TenMinutes) && len(w.tenMinute) > 0: + return w.deepCopyWatcherMetrics(&w.tenMinute[len(w.tenMinute)-1]), nil + case (duration == TenMinutes || duration == FiveMinutes) && len(w.fiveMinute) > 0: + return w.deepCopyWatcherMetrics(&w.fiveMinute[len(w.fiveMinute)-1]), nil + default: + return nil, errors.New("unable to get any latest metrics") + } +} + +func (w *Watcher) getCurrentWindow(duration string) (*Window, *[]WatcherMetrics) { + var curWindow *Window + var watcherMetrics *[]WatcherMetrics + switch duration { + case FifteenMinutes: + curWindow = CurrentFifteenMinuteWindow() + watcherMetrics = &w.fifteenMinute + case TenMinutes: + curWindow = CurrentTenMinuteWindow() + watcherMetrics = &w.tenMinute + case FiveMinutes: + curWindow = CurrentFiveMinuteWindow() + watcherMetrics = &w.fiveMinute + default: + log.Error("received unexpected window duration, defaulting to 15m") + curWindow = CurrentFifteenMinuteWindow() + } + return curWindow, watcherMetrics +} + +func (w *Watcher) appendWatcherMetrics(recentMetrics *[]WatcherMetrics, metric *WatcherMetrics) { + w.mutex.Lock() + defer w.mutex.Unlock() + if len(*recentMetrics) == w.cacheSize { + *recentMetrics = (*recentMetrics)[1:] + } + *recentMetrics = append(*recentMetrics, *metric) +} + +func (w *Watcher) deepCopyWatcherMetrics(src *WatcherMetrics) *WatcherMetrics { + nodeMetricsMap := make(map[string]NodeMetrics) + for host, fetchedMetric := range src.Data.NodeMetricsMap { + nodeMetric := NodeMetrics{ + Metrics: make([]Metric, len(fetchedMetric.Metrics)), + Tags: fetchedMetric.Tags, + } + copy(nodeMetric.Metrics, fetchedMetric.Metrics) + nodeMetric.Metadata = fetchedMetric.Metadata + nodeMetricsMap[host] = nodeMetric + } + return &WatcherMetrics{ + Timestamp: src.Timestamp, + Window: src.Window, + Source: src.Source, + Data: Data{ + NodeMetricsMap: nodeMetricsMap, + }, + } +} + +// HTTP Handler for BaseUrl endpoint +func (w *Watcher) handler(resp http.ResponseWriter, r *http.Request) { + resp.Header().Set("Content-Type", "application/json") + + metrics, err := w.GetLatestWatcherMetrics(FifteenMinutes) + if metrics == nil { + if err != nil { + resp.WriteHeader(http.StatusInternalServerError) + log.Error(err) + return + } + resp.WriteHeader(http.StatusNotFound) + return + } + + host := r.URL.Query().Get("host") + var bytes []byte + if host != "" { + if _, ok := metrics.Data.NodeMetricsMap[host]; ok { + hostMetricsData := make(map[string]NodeMetrics) + hostMetricsData[host] = metrics.Data.NodeMetricsMap[host] + hostMetrics := WatcherMetrics{Timestamp: metrics.Timestamp, + Window: metrics.Window, + Source: metrics.Source, + Data: Data{NodeMetricsMap: hostMetricsData}, + } + bytes, err = gojay.MarshalJSONObject(&hostMetrics) + } else { + resp.WriteHeader(http.StatusNotFound) + return + } + } else { + bytes, err = gojay.MarshalJSONObject(metrics) + } + + if err != nil { + log.Error(err) + resp.WriteHeader(http.StatusInternalServerError) + return + } + + _, err = resp.Write(bytes) + if err != nil { + log.Error(err) + resp.WriteHeader(http.StatusInternalServerError) + } +} + +// Simple server status handler +func (w *Watcher) healthCheckHandler(resp http.ResponseWriter, r *http.Request) { + if status, err := w.client.Health(); status != 0 { + log.Warnf("health check failed with: %v", err) + resp.WriteHeader(http.StatusServiceUnavailable) + return + } + resp.WriteHeader(http.StatusOK) +} + +// Utility functions + +func metricMapToWatcherMetrics(metricMap map[string][]Metric, clientName string, window Window) WatcherMetrics { + metricsMap := make(map[string]NodeMetrics) + for host, metricList := range metricMap { + nodeMetric := NodeMetrics{ + Metrics: make([]Metric, len(metricList)), + } + copy(nodeMetric.Metrics, metricList) + metricsMap[host] = nodeMetric + } + + watcherMetrics := WatcherMetrics{Timestamp: time.Now().Unix(), + Data: Data{NodeMetricsMap: metricsMap}, + Source: clientName, + Window: window, + } + return watcherMetrics +} + +func CurrentFifteenMinuteWindow() *Window { + curTime := time.Now().Unix() + return &Window{FifteenMinutes, curTime - 15*60, curTime} +} + +func CurrentTenMinuteWindow() *Window { + curTime := time.Now().Unix() + return &Window{TenMinutes, curTime - 10*60, curTime} +} + +func CurrentFiveMinuteWindow() *Window { + curTime := time.Now().Unix() + return &Window{FiveMinutes, curTime - 5*60, curTime} +} diff --git a/3rd/load-watcher/pkg/watcher/watcher_json.go b/3rd/load-watcher/pkg/watcher/watcher_json.go new file mode 100644 index 00000000..057e5555 --- /dev/null +++ b/3rd/load-watcher/pkg/watcher/watcher_json.go @@ -0,0 +1,293 @@ +/* +Copyright 2020 PayPal + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Generated from gojay generator tool, with some bug fixes + +package watcher + +import ( + "github.com/francoispqt/gojay" +) + +type Metrices []Metric + +func (s *Metrices) UnmarshalJSONArray(dec *gojay.Decoder) error { + var value = Metric{} + if err := dec.Object(&value); err != nil { + return err + } + *s = append(*s, value) + return nil +} + +func (s Metrices) MarshalJSONArray(enc *gojay.Encoder) { + for i := range s { + enc.Object(&s[i]) + } +} + +func (s Metrices) IsNil() bool { + return len(s) == 0 +} + +// MarshalJSONObject implements MarshalerJSONObject +func (d *Data) MarshalJSONObject(enc *gojay.Encoder) { + enc.ObjectKey("NodeMetricsMap", &d.NodeMetricsMap) +} + +// IsNil checks if instance is nil +func (d *Data) IsNil() bool { + return d == nil +} + +// UnmarshalJSONObject implements gojay's UnmarshalerJSONObject +func (d *Data) UnmarshalJSONObject(dec *gojay.Decoder, k string) error { + + switch k { + case "NodeMetricsMap": + err := dec.Object(&d.NodeMetricsMap) + return err + } + return nil +} + +// NKeys returns the number of keys to unmarshal +func (d *Data) NKeys() int { return 1 } + +// MarshalJSONObject implements MarshalerJSONObject +func (m *Metadata) MarshalJSONObject(enc *gojay.Encoder) { + enc.StringKey("dataCenter", m.DataCenter) +} + +// IsNil checks if instance is nil +func (m *Metadata) IsNil() bool { + return m == nil +} + +// UnmarshalJSONObject implements gojay's UnmarshalerJSONObject +func (m *Metadata) UnmarshalJSONObject(dec *gojay.Decoder, k string) error { + + switch k { + case "dataCenter": + return dec.String(&m.DataCenter) + + } + return nil +} + +// NKeys returns the number of keys to unmarshal +func (m *Metadata) NKeys() int { return 1 } + +// MarshalJSONObject implements MarshalerJSONObject +func (m *Metric) MarshalJSONObject(enc *gojay.Encoder) { + enc.StringKey("name", m.Name) + enc.StringKey("type", m.Type) + enc.StringKey("operator", m.Operator) + enc.StringKey("rollup", m.Rollup) + enc.Float64Key("value", m.Value) +} + +// IsNil checks if instance is nil +func (m *Metric) IsNil() bool { + return m == nil +} + +// UnmarshalJSONObject implements gojay's UnmarshalerJSONObject +func (m *Metric) UnmarshalJSONObject(dec *gojay.Decoder, k string) error { + + switch k { + case "name": + return dec.String(&m.Name) + + case "type": + return dec.String(&m.Type) + + case "operator": + return dec.String(&m.Operator) + + case "rollup": + return dec.String(&m.Rollup) + + case "value": + return dec.Float64(&m.Value) + + } + return nil +} + +// NKeys returns the number of keys to unmarshal +func (m *Metric) NKeys() int { return 5 } + +// MarshalJSONObject implements MarshalerJSONObject +func (m *NodeMetrics) MarshalJSONObject(enc *gojay.Encoder) { + var metricsSlice = Metrices(m.Metrics) + enc.ArrayKey("metrics", metricsSlice) + enc.ObjectKey("tags", &m.Tags) + enc.ObjectKey("metadata", &m.Metadata) +} + +// IsNil checks if instance is nil +func (m *NodeMetrics) IsNil() bool { + return m == nil +} + +// UnmarshalJSONObject implements gojay's UnmarshalerJSONObject +func (m *NodeMetrics) UnmarshalJSONObject(dec *gojay.Decoder, k string) error { + + switch k { + case "metrics": + var aSlice = Metrices{} + err := dec.Array(&aSlice) + if err == nil && len(aSlice) > 0 { + m.Metrics = []Metric(aSlice) + } + return err + + case "tags": + err := dec.Object(&m.Tags) + + return err + + case "metadata": + err := dec.Object(&m.Metadata) + + return err + + } + return nil +} + +// NKeys returns the number of keys to unmarshal +func (m *NodeMetrics) NKeys() int { return 3 } + +// MarshalJSONObject implements MarshalerJSONObject +func (m *NodeMetricsMap) MarshalJSONObject(enc *gojay.Encoder) { + for k, v := range *m { + enc.ObjectKey(k, &v) + } +} + +// IsNil checks if instance is nil +func (m *NodeMetricsMap) IsNil() bool { + return m == nil +} + +// UnmarshalJSONObject implements gojay's UnmarshalerJSONObject +func (m *NodeMetricsMap) UnmarshalJSONObject(dec *gojay.Decoder, k string) error { + var value NodeMetrics + if err := dec.Object(&value); err != nil { + return err + } + (*m)[k] = value + return nil +} + +// NKeys returns the number of keys to unmarshal +func (m *NodeMetricsMap) NKeys() int { return 0 } + +// MarshalJSONObject implements MarshalerJSONObject +func (t *Tags) MarshalJSONObject(enc *gojay.Encoder) { + +} + +// IsNil checks if instance is nil +func (t *Tags) IsNil() bool { + return t == nil +} + +// UnmarshalJSONObject implements gojay's UnmarshalerJSONObject +func (t *Tags) UnmarshalJSONObject(dec *gojay.Decoder, k string) error { + + switch k { + + } + return nil +} + +// NKeys returns the number of keys to unmarshal +func (t *Tags) NKeys() int { return 0 } + +// MarshalJSONObject implements MarshalerJSONObject +func (m *WatcherMetrics) MarshalJSONObject(enc *gojay.Encoder) { + enc.Int64Key("timestamp", m.Timestamp) + enc.ObjectKey("window", &m.Window) + enc.StringKey("source", m.Source) + enc.ObjectKey("data", &m.Data) +} + +// IsNil checks if instance is nil +func (m *WatcherMetrics) IsNil() bool { + return m == nil +} + +// UnmarshalJSONObject implements gojay's UnmarshalerJSONObject +func (m *WatcherMetrics) UnmarshalJSONObject(dec *gojay.Decoder, k string) error { + + switch k { + case "timestamp": + return dec.Int64(&m.Timestamp) + + case "window": + err := dec.Object(&m.Window) + + return err + + case "source": + return dec.String(&m.Source) + + case "data": + err := dec.Object(&m.Data) + + return err + + } + return nil +} + +// NKeys returns the number of keys to unmarshal +func (m *WatcherMetrics) NKeys() int { return 4 } + +// MarshalJSONObject implements MarshalerJSONObject +func (w *Window) MarshalJSONObject(enc *gojay.Encoder) { + enc.StringKey("duration", w.Duration) + enc.Int64Key("start", w.Start) + enc.Int64Key("end", w.End) +} + +// IsNil checks if instance is nil +func (w *Window) IsNil() bool { + return w == nil +} + +// UnmarshalJSONObject implements gojay's UnmarshalerJSONObject +func (w *Window) UnmarshalJSONObject(dec *gojay.Decoder, k string) error { + + switch k { + case "duration": + return dec.String(&w.Duration) + + case "start": + return dec.Int64(&w.Start) + + case "end": + return dec.Int64(&w.End) + + } + return nil +} + +// NKeys returns the number of keys to unmarshal +func (w *Window) NKeys() int { return 3 } diff --git a/3rd/load-watcher/pkg/watcher/watcher_test.go b/3rd/load-watcher/pkg/watcher/watcher_test.go new file mode 100644 index 00000000..1fdd6a28 --- /dev/null +++ b/3rd/load-watcher/pkg/watcher/watcher_test.go @@ -0,0 +1,141 @@ +/* +Copyright 2020 PayPal + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package watcher + +import ( + "net/http" + "net/http/httptest" + "net/url" + "os" + "testing" + + "github.com/francoispqt/gojay" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var w *Watcher + +func TestGetLatestWatcherMetrics(t *testing.T) { + var metrics *WatcherMetrics + metrics, err := w.GetLatestWatcherMetrics(FifteenMinutes) + require.Nil(t, err) + assert.Equal(t, FifteenMinutesMetricsMap[FirstNode], metrics.Data.NodeMetricsMap[FirstNode].Metrics) + assert.Equal(t, FifteenMinutesMetricsMap[SecondNode], metrics.Data.NodeMetricsMap[SecondNode].Metrics) + + metrics, err = w.GetLatestWatcherMetrics(TenMinutes) + require.Nil(t, err) + assert.Equal(t, TenMinutesMetricsMap[FirstNode], metrics.Data.NodeMetricsMap[FirstNode].Metrics) + assert.Equal(t, TenMinutesMetricsMap[SecondNode], metrics.Data.NodeMetricsMap[SecondNode].Metrics) + + metrics, err = w.GetLatestWatcherMetrics(FiveMinutes) + require.Nil(t, err) + assert.Equal(t, FiveMinutesMetricsMap[FirstNode], metrics.Data.NodeMetricsMap[FirstNode].Metrics) + assert.Equal(t, FiveMinutesMetricsMap[SecondNode], metrics.Data.NodeMetricsMap[SecondNode].Metrics) +} + +func TestWatcherAPIAllHosts(t *testing.T) { + req, err := http.NewRequest("GET", BaseUrl, nil) + require.Nil(t, err) + + rr := httptest.NewRecorder() + handler := http.HandlerFunc(w.handler) + + handler.ServeHTTP(rr, req) + require.Equal(t, http.StatusOK, rr.Code) + + expectedMetrics, err := w.GetLatestWatcherMetrics(FifteenMinutes) + require.Nil(t, err) + data := Data{NodeMetricsMap: make(map[string]NodeMetrics)} + var watcherMetrics = &WatcherMetrics{Data: data} + err = gojay.UnmarshalJSONObject(rr.Body.Bytes(), watcherMetrics) + require.Nil(t, err) + assert.Equal(t, expectedMetrics, watcherMetrics) +} + +func TestWatcherAPISingleHost(t *testing.T) { + uri, _ := url.Parse(BaseUrl) + q := uri.Query() + q.Set("host", FirstNode) + uri.RawQuery = q.Encode() + req, err := http.NewRequest("GET", uri.String(), nil) + require.Nil(t, err) + + rr := httptest.NewRecorder() + handler := http.HandlerFunc(w.handler) + + handler.ServeHTTP(rr, req) + require.Equal(t, http.StatusOK, rr.Code) + + expectedMetrics, err := w.GetLatestWatcherMetrics(FifteenMinutes) + require.Nil(t, err) + data := Data{NodeMetricsMap: make(map[string]NodeMetrics)} + var watcherMetrics = &WatcherMetrics{Data: data} + err = gojay.UnmarshalJSONObject(rr.Body.Bytes(), watcherMetrics) + require.Nil(t, err) + assert.Equal(t, expectedMetrics.Data.NodeMetricsMap[FirstNode], watcherMetrics.Data.NodeMetricsMap[FirstNode]) + assert.Equal(t, expectedMetrics.Source, watcherMetrics.Source) +} + +func TestWatcherMetricsNotFound(t *testing.T) { + uri, _ := url.Parse(BaseUrl) + q := uri.Query() + q.Set("host", "deadbeef") + uri.RawQuery = q.Encode() + req, err := http.NewRequest("GET", uri.String(), nil) + require.Nil(t, err) + + rr := httptest.NewRecorder() + handler := http.HandlerFunc(w.handler) + + handler.ServeHTTP(rr, req) + assert.Equal(t, http.StatusNotFound, rr.Code) +} + +func TestWatcherInternalServerError(t *testing.T) { + client := NewTestMetricsServerClient() + unstartedWatcher := NewWatcher(client) + + req, err := http.NewRequest("GET", BaseUrl, nil) + require.Nil(t, err) + + rr := httptest.NewRecorder() + handler := http.HandlerFunc(unstartedWatcher.handler) + + handler.ServeHTTP(rr, req) + assert.Equal(t, http.StatusInternalServerError, rr.Code) +} + +func TestWatcherHealthCheck(t *testing.T) { + req, err := http.NewRequest("GET", HealthCheckUrl, nil) + require.Nil(t, err) + + rr := httptest.NewRecorder() + handler := http.HandlerFunc(w.handler) + + handler.ServeHTTP(rr, req) + require.Equal(t, http.StatusOK, rr.Code) +} + +func TestMain(m *testing.M) { + client := NewTestMetricsServerClient() + w = NewWatcher(client) + w.StartWatching() + + ret := m.Run() + os.Exit(ret) +}