From e6d653eeace81be869a9b907a1b6ca38a81da021 Mon Sep 17 00:00:00 2001 From: Adam Pietrzycki Date: Fri, 15 Feb 2019 13:52:50 +0000 Subject: [PATCH 01/21] Bump go version and alpine version --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5dd9417e..055e3f59 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.9-alpine as build +FROM golang:1.10.8-alpine as build LABEL maintainer "Infinity Works" RUN apk --no-cache add ca-certificates \ @@ -11,7 +11,7 @@ RUN go get \ && go test ./... \ && go build -o /bin/main -FROM alpine:3.6 +FROM alpine:3.9 RUN apk --no-cache add ca-certificates \ && addgroup exporter \ From e3b8d415d71ff41fc4f3c264aa05220ff5006a22 Mon Sep 17 00:00:00 2001 From: Adam Pietrzycki Date: Fri, 15 Feb 2019 13:53:35 +0000 Subject: [PATCH 02/21] Markdown linting --- METRICS.md | 2 +- README.md | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/METRICS.md b/METRICS.md index 78a404c3..629df874 100644 --- a/METRICS.md +++ b/METRICS.md @@ -2,7 +2,7 @@ Below are an example of the metrics as exposed by this exporter. -``` +```plaintext # HELP github_rate_limit Number of API queries allowed in a 60 minute window # TYPE github_rate_limit gauge github_rate_limit 5000 diff --git a/README.md b/README.md index bab6f0d3..94a0bb3d 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ Exposes basic metrics for your repositories from the GitHub API, to a Prometheus This exporter is setup to take input from environment variables: ### Required + * `ORGS` If supplied, the exporter will enumerate all repositories for that organization. Expected in the format "org1, org2". * `REPOS` If supplied, The repos you wish to monitor, expected in the format "user/repo1, user/repo2". Can be across different Github users/orgs. * `USERS` If supplied, the exporter will enumerate all repositories for that users. Expected in @@ -15,6 +16,7 @@ the format "user1, user2". At least one of those 3 options should be provided. ### Optional + * `GITHUB_TOKEN` If supplied, enables the user to supply a github authentication token that allows the API to be queried more often. Optional, but recommended. * `GITHUB_TOKEN_FILE` If supplied _instead of_ `GITHUB_TOKEN`, enables the user to supply a path to a file containing a github authentication token that allows the API to be queried more often. Optional, but recommended. * `API_URL` Github API URL, shouldn't need to change this. Defaults to `https://api.github.com` @@ -26,19 +28,21 @@ At least one of those 3 options should be provided. ## Install and deploy Run manually from Docker Hub: -``` + +```sh docker run -d --restart=always -p 9171:9171 -e REPOS="infinityworks/ranch-eye, infinityworks/prom-conf" infinityworks/github-exporter ``` Build a docker image: -``` + +```sh docker build -t . docker run -d --restart=always -p 9171:9171 -e REPOS="infinityworks/ranch-eye, infinityworks/prom-conf" ``` ## Docker compose -``` +```yaml github-exporter: tty: true stdin_open: true @@ -50,7 +54,6 @@ github-exporter: environment: - REPOS= - GITHUB_TOKEN= - ``` ## Metrics @@ -59,4 +62,5 @@ Metrics will be made available on port 9171 by default An example of these metrics can be found in the `METRICS.md` markdown file in the root of this repository ## Metadata -[![](https://images.microbadger.com/badges/image/infinityworks/github-exporter.svg)](http://microbadger.com/images/infinityworks/github-exporter "Get your own image badge on microbadger.com") [![](https://images.microbadger.com/badges/version/infinityworks/github-exporter.svg)](http://microbadger.com/images/infinityworks/github-exporter "Get your own version badge on microbadger.com") + +[![](https://images.microbadger.com/badges/image/infinityworks/github-exporter.svg)](http://microbadger.com/images/infinityworks/github-exporter "Get your own image badge on microbadger.com")[![](https://images.microbadger.com/badges/version/infinityworks/github-exporter.svg)](http://microbadger.com/images/infinityworks/github-exporter "Get your own version badge on microbadger.com") From d5ceff31503a82994a9c9a24bef4a3b59ca043bb Mon Sep 17 00:00:00 2001 From: Adam Pietrzycki Date: Fri, 15 Feb 2019 13:54:02 +0000 Subject: [PATCH 03/21] Build docker image instead of using generated one --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index a007ef26..1d4895d0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,13 +2,13 @@ version: "2" services: github-exporter: + build: . tty: true stdin_open: true expose: - 9171 ports: - 9171:9171 - image: infinityworks/github-exporter:latest environment: - REPOS= - GITHUB_TOKEN= From d001a86dbc92ba9c2f27ff892985cdeaa7b8aaf2 Mon Sep 17 00:00:00 2001 From: Adam Pietrzycki Date: Fri, 15 Feb 2019 13:54:10 +0000 Subject: [PATCH 04/21] Spelling --- config/config.go | 2 +- exporter/http.go | 2 +- exporter/metrics.go | 2 +- exporter/prometheus.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/config.go b/config/config.go index 7ca8b0ec..87734878 100644 --- a/config/config.go +++ b/config/config.go @@ -12,7 +12,7 @@ import ( cfg "github.com/infinityworks/go-common/config" ) -// Config struct holds all of the runtime confgiguration for the application +// Config struct holds all of the runtime configuration for the application type Config struct { *cfg.BaseConfig APIURL string diff --git a/exporter/http.go b/exporter/http.go index 59a39e6f..aa0fadc5 100644 --- a/exporter/http.go +++ b/exporter/http.go @@ -66,7 +66,7 @@ func getResponse(url string, token string, ch chan<- *Response) error { // Triggers if a user specifies an invalid or not visible repository if resp.StatusCode == 404 { - return fmt.Errorf("Error: Received 404 status from Github API, ensure the repsository URL is correct. If it's a privare repository, also check the oauth token is correct") + return fmt.Errorf("Error: Received 404 status from Github API, ensure the repository URL is correct. If it's a private repository, also check the oauth token is correct") } ch <- &Response{url, resp, body, err} diff --git a/exporter/metrics.go b/exporter/metrics.go index 558180ea..5c4ea958 100644 --- a/exporter/metrics.go +++ b/exporter/metrics.go @@ -3,7 +3,7 @@ package exporter import "github.com/prometheus/client_golang/prometheus" import "strconv" -// AddMetrics - Add's all of the metrics to a map of strings, returns the map. +// AddMetrics - Adds all of the metrics to a map of strings, returns the map. func AddMetrics() map[string]*prometheus.Desc { APIMetrics := make(map[string]*prometheus.Desc) diff --git a/exporter/prometheus.go b/exporter/prometheus.go index 90715dcb..0b2f0331 100644 --- a/exporter/prometheus.go +++ b/exporter/prometheus.go @@ -15,7 +15,7 @@ func (e *Exporter) Describe(ch chan<- *prometheus.Desc) { } // Collect function, called on by Prometheus Client library -// This function is called when a scrape is peformed on the /metrics page +// This function is called when a scrape is performed on the /metrics page func (e *Exporter) Collect(ch chan<- prometheus.Metric) { // Scrape the Data from Github From 864f7d68bdacbe3e444703c3aef06865cca3344d Mon Sep 17 00:00:00 2001 From: Adam Pietrzycki Date: Fri, 15 Feb 2019 13:59:35 +0000 Subject: [PATCH 05/21] More markdown linting --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 94a0bb3d..85f5d71b 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,6 @@ At least one of those 3 options should be provided. * `METRICS_PATH` the metrics URL path you wish to use, defaults to `/metrics` * `LOG_LEVEL` The level of logging the exporter will run with, defaults to `debug` - ## Install and deploy Run manually from Docker Hub: @@ -63,4 +62,5 @@ An example of these metrics can be found in the `METRICS.md` markdown file in th ## Metadata -[![](https://images.microbadger.com/badges/image/infinityworks/github-exporter.svg)](http://microbadger.com/images/infinityworks/github-exporter "Get your own image badge on microbadger.com")[![](https://images.microbadger.com/badges/version/infinityworks/github-exporter.svg)](http://microbadger.com/images/infinityworks/github-exporter "Get your own version badge on microbadger.com") +[![Get your own image badge on microbadger.com](https://images.microbadger.com/badges/image/infinityworks/github-exporter.svg)](http://microbadger.com/images/infinityworks/github-exporter) +[![Get your own version badge on microbadger.com](https://images.microbadger.com/badges/version/infinityworks/github-exporter.svg)](http://microbadger.com/images/infinityworks/github-exporter) From 21171ffd8297416b312fb57c1f9326e1325c4ced Mon Sep 17 00:00:00 2001 From: Adam Pietrzycki Date: Sat, 16 Feb 2019 15:07:00 +0000 Subject: [PATCH 06/21] Go mod Signed-off-by: Adam Pietrzycki --- go.mod | 8 ++++++++ go.sum | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 go.mod create mode 100644 go.sum diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..b6489050 --- /dev/null +++ b/go.mod @@ -0,0 +1,8 @@ +module github.com/infinityworks/github-exporter + +require ( + github.com/fatih/structs v1.1.0 + github.com/infinityworks/go-common v0.0.0-20170820165359-7f20a140fd37 + github.com/prometheus/client_golang v0.9.2 + github.com/sirupsen/logrus v1.3.0 +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..740500ff --- /dev/null +++ b/go.sum @@ -0,0 +1,36 @@ +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/infinityworks/go-common v0.0.0-20170820165359-7f20a140fd37 h1:Lm6kyC3JBiJQvJrus66He0E4viqDc/m5BdiFNSkIFfU= +github.com/infinityworks/go-common v0.0.0-20170820165359-7f20a140fd37/go.mod h1:+OaHNKQvQ9oOCr+DgkF95PkiDx20fLHpzMp8SmRPQTg= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= +github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME= +github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= From 422617cb2b368db7d399dd42f86b46881ed66884 Mon Sep 17 00:00:00 2001 From: Adam Pietrzycki Date: Sat, 16 Feb 2019 15:07:26 +0000 Subject: [PATCH 07/21] Bump version + fix label Signed-off-by: Adam Pietrzycki --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 055e3f59..ec60b3c7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ -FROM golang:1.10.8-alpine as build -LABEL maintainer "Infinity Works" +FROM golang:1.11.5-alpine as build +LABEL maintainer="Infinity Works" RUN apk --no-cache add ca-certificates \ && apk --no-cache add --virtual build-deps git From 4674217e71f6a9660e395f9410d762119ea012a5 Mon Sep 17 00:00:00 2001 From: Adam Pietrzycki Date: Sat, 16 Feb 2019 15:12:44 +0000 Subject: [PATCH 08/21] Added gitignore Signed-off-by: Adam Pietrzycki --- .gitignore | 236 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 236 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..10ca28b0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,236 @@ +# Created by https://www.gitignore.io/api/go,macos,linux,windows,jetbrains,jetbrains+all,visualstudiocode +# Edit at https://www.gitignore.io/?templates=go,macos,linux,windows,jetbrains,jetbrains+all,visualstudiocode + +### Go ### +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +### Go Patch ### +/vendor/ +/Godeps/ + +### JetBrains ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/modules.xml +# .idea/*.iml +# .idea/modules + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### JetBrains Patch ### +# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 + +# *.iml +# modules.xml +# .idea/misc.xml +# *.ipr + +# Sonarlint plugin +.idea/sonarlint + +### JetBrains+all ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff + +# Generated files + +# Sensitive or high-churn files + +# Gradle + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/modules.xml +# .idea/*.iml +# .idea/modules + +# CMake + +# Mongo Explorer plugin + +# File-based project format + +# IntelliJ + +# mpeltonen/sbt-idea plugin + +# JIRA plugin + +# Cursive Clojure plugin + +# Crashlytics plugin (for Android Studio and IntelliJ) + +# Editor-based Rest Client + +# Android studio 3.1+ serialized cache file + +### JetBrains+all Patch ### +# Ignores the whole .idea folder and all .iml files +# See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360 + +.idea/ + +# Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 + +*.iml +modules.xml +.idea/misc.xml +*.ipr + +# Sonarlint plugin + .idea/sonarlint + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### VisualStudioCode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history + +### Windows ### +# Windows thumbnail cache files +Thumbs.db +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# End of https://www.gitignore.io/api/go,macos,linux,windows,jetbrains,jetbrains+all,visualstudiocode \ No newline at end of file From d63883f88d0a5c4e0f3dcdc74a82aed143a7ad27 Mon Sep 17 00:00:00 2001 From: Adam Pietrzycki Date: Sat, 16 Feb 2019 15:58:54 +0000 Subject: [PATCH 09/21] Lint + spelling Signed-off-by: Adam Pietrzycki --- config/config.go | 4 ++-- exporter/gather.go | 4 ++-- exporter/http.go | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/config/config.go b/config/config.go index 87734878..033374ad 100644 --- a/config/config.go +++ b/config/config.go @@ -67,7 +67,7 @@ func getScrapeURLs(apiURL, repos, orgs, users string) ([]string, error) { // User input validation, check that either repositories or organisations have been passed in if len(repos) == 0 && len(orgs) == 0 && len(users) == 0 { - return urls, fmt.Errorf("No targets specified") + return urls, fmt.Errorf("no targets specified") } // Append repositories to the array @@ -79,7 +79,7 @@ func getScrapeURLs(apiURL, repos, orgs, users string) ([]string, error) { } } - // Append github orginisations to the array + // Append github organisations to the array if orgs != "" { o := strings.Split(orgs, ", ") for _, x := range o { diff --git a/exporter/gather.go b/exporter/gather.go index 14b6b7fb..abb09f45 100644 --- a/exporter/gather.go +++ b/exporter/gather.go @@ -52,7 +52,7 @@ func (e *Exporter) gatherData() ([]*Datum, *RateLimits, error) { // Especially useful when operating without oauth and the subsequent lower cap. func getRates(baseURL string, token string) (*RateLimits, error) { - rateEndPoint := ("/rate_limit") + rateEndPoint := "/rate_limit" url := fmt.Sprintf("%s%s", baseURL, rateEndPoint) resp, err := getHTTPResponse(url, token) @@ -63,7 +63,7 @@ func getRates(baseURL string, token string) (*RateLimits, error) { // Triggers if rate-limiting isn't enabled on private Github Enterprise installations if resp.StatusCode == 404 { - return &RateLimits{}, fmt.Errorf("Rate Limiting not enabled in GitHub API") + return &RateLimits{}, fmt.Errorf("rate Limiting not enabled in GitHub API") } limit, err := strconv.ParseFloat(resp.Header.Get("X-RateLimit-Limit"), 64) diff --git a/exporter/http.go b/exporter/http.go index aa0fadc5..4cbe211a 100644 --- a/exporter/http.go +++ b/exporter/http.go @@ -52,7 +52,7 @@ func getResponse(url string, token string, ch chan<- *Response) error { resp, err := getHTTPResponse(url, token) // do this earlier if err != nil { - return fmt.Errorf("Error converting body to byte array: %v", err) + return fmt.Errorf("error converting body to byte array: %v", err) } // Read the body to a byte array so it can be used elsewhere @@ -61,12 +61,12 @@ func getResponse(url string, token string, ch chan<- *Response) error { defer resp.Body.Close() if err != nil { - return fmt.Errorf("Error converting body to byte array: %v", err) + return fmt.Errorf("error converting body to byte array: %v", err) } // Triggers if a user specifies an invalid or not visible repository if resp.StatusCode == 404 { - return fmt.Errorf("Error: Received 404 status from Github API, ensure the repository URL is correct. If it's a private repository, also check the oauth token is correct") + return fmt.Errorf("error: Received 404 status from Github API, ensure the repository URL is correct. If it's a private repository, also check the oauth token is correct") } ch <- &Response{url, resp, body, err} From e63f30f52033b94e507b058a4ea9bd366d51470d Mon Sep 17 00:00:00 2001 From: Adam Pietrzycki Date: Sat, 16 Feb 2019 21:09:57 +0000 Subject: [PATCH 10/21] PLAYGROUND - Delete me Mac battery nearly dead, moving to PC Signed-off-by: Adam Pietrzycki --- config/exporter.go | 8 ++++++ config/github.go | 7 +++++ config/query.go | 9 ++++++ exporter/github.go | 25 +++++++++++++++++ exporter/graphql.txt | 29 +++++++++++++++++++ go.mod | 15 ++++++++++ go.sum | 66 ++++++++++++++++++++++++++++++++++++++++++++ main.go | 18 ++++++++++-- 8 files changed, 174 insertions(+), 3 deletions(-) create mode 100644 config/exporter.go create mode 100644 config/github.go create mode 100644 config/query.go create mode 100644 exporter/github.go create mode 100644 exporter/graphql.txt diff --git a/config/exporter.go b/config/exporter.go new file mode 100644 index 00000000..76d6bc4f --- /dev/null +++ b/config/exporter.go @@ -0,0 +1,8 @@ +package config + +type ExporterConfig struct { + MetricsPath string + ListenPort string + LogLevel string + ApplicationName string +} diff --git a/config/github.go b/config/github.go new file mode 100644 index 00000000..eaf26be4 --- /dev/null +++ b/config/github.go @@ -0,0 +1,7 @@ +package config + +type RateLimitConfig struct { + Limit int + Remaining int + Reset string +} diff --git a/config/query.go b/config/query.go new file mode 100644 index 00000000..3160aa7c --- /dev/null +++ b/config/query.go @@ -0,0 +1,9 @@ +package config + +type QueryConfiguration struct { + Stars bool + OpenIssues bool + Watchers bool + Forks bool + Size bool +} diff --git a/exporter/github.go b/exporter/github.go new file mode 100644 index 00000000..77c7afe8 --- /dev/null +++ b/exporter/github.go @@ -0,0 +1,25 @@ +package exporter + +import ( + "context" + "fmt" + "github.com/prometheus/common/log" + "github.com/shurcooL/githubv4" +) + +var query struct { + Viewer struct { + Login githubv4.String + CreatedAt githubv4.DateTime + } +} + +// Query GitHub API +func Query(client *githubv4.Client) { + err := client.Query(context.Background(), &query, nil) + if err != nil { + log.Fatal(err) + } + fmt.Println(" Login:", query.Viewer.Login) + fmt.Println("CreatedAt:", query.Viewer.CreatedAt) +} diff --git a/exporter/graphql.txt b/exporter/graphql.txt new file mode 100644 index 00000000..e9ecddfe --- /dev/null +++ b/exporter/graphql.txt @@ -0,0 +1,29 @@ +query ($owner: String!, $repo: String!, $stars: Boolean!, $issues: Boolean!, $watchers: Boolean!, $forks: Boolean!) { + repository(owner: $owner, name: $repo) { + stargazers @include(if: $stars) { + totalCount + } + issues @include(if: $issues) { + totalCount + } + watchers @include(if: $watchers) { + totalCount + } + forkCount @include(if: $forks) + } + rateLimit { + limit + cost + remaining + resetAt + } +} + +{ + "owner": "infinityworks", + "repo": "github-exporter", + "stars": true, + "issues": true, + "watchers": true, + "forks": true +} \ No newline at end of file diff --git a/go.mod b/go.mod index b6489050..634efb65 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,23 @@ module github.com/infinityworks/github-exporter require ( + github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc // indirect + github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf // indirect + github.com/coreos/etcd v3.3.12+incompatible // indirect github.com/fatih/structs v1.1.0 github.com/infinityworks/go-common v0.0.0-20170820165359-7f20a140fd37 github.com/prometheus/client_golang v0.9.2 + github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 + github.com/shurcooL/githubv4 v0.0.0-20190119021625-d9689b595017 + github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f // indirect github.com/sirupsen/logrus v1.3.0 + github.com/spf13/afero v1.2.1 // indirect + github.com/spf13/viper v1.3.1 + github.com/stretchr/testify v1.3.0 // indirect + github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43 // indirect + golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67 // indirect + golang.org/x/oauth2 v0.0.0-20190212230446-3e8b2be13635 + golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a // indirect + gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect + gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect ) diff --git a/go.sum b/go.sum index 740500ff..24986ee7 100644 --- a/go.sum +++ b/go.sum @@ -1,17 +1,38 @@ +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.12+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/infinityworks/go-common v0.0.0-20170820165359-7f20a140fd37 h1:Lm6kyC3JBiJQvJrus66He0E4viqDc/m5BdiFNSkIFfU= github.com/infinityworks/go-common v0.0.0-20170820165359-7f20a140fd37/go.mod h1:+OaHNKQvQ9oOCr+DgkF95PkiDx20fLHpzMp8SmRPQTg= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= @@ -22,15 +43,60 @@ github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jO github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/shurcooL/githubv4 v0.0.0-20190119021625-d9689b595017 h1:7ykwAbXU6ZgqeMACeXEbN2ZQFPsI7fbtOEJLQPA+B/4= +github.com/shurcooL/githubv4 v0.0.0-20190119021625-d9689b595017/go.mod h1:hAF0iLZy4td2EX+/8Tw+4nodhlMrwN3HupfaXj3zkGo= +github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f h1:tygelZueB1EtXkPI6mQ4o9DQ0+FKW41hTbunoXZCTqk= +github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f/go.mod h1:AuYgA5Kyo4c7HfUmvRGs/6rGlMMV/6B1bVnB9JxJEEg= github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.1 h1:qgMbHoJbPbw579P+1zVY+6n4nIFuIchaIjzZ/I/Yq8M= +github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.1 h1:5+8j8FTpnFV4nEImW/ofkzEt8VoOiLXxdYIDsB73T38= +github.com/spf13/viper v1.3.1/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/ugorji/go v1.1.2/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67 h1:ng3VDlRp5/DHpSWl02R4rM9I+8M2rhmsuLwAMmkLQWE= +golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc h1:a3CU5tJYVj92DY2LaA1kUkrsqD5/3mLDhx2NcNqyW+0= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e h1:bRhVy7zSSasaqNksaRZiA5EEI+Ei4I1nO5Jh72wfHlg= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/oauth2 v0.0.0-20190212230446-3e8b2be13635 h1:dOJmQysgY8iOBECuNp0vlKHWEtfiTnyjisEizRV3/4o= +golang.org/x/oauth2 v0.0.0-20190212230446-3e8b2be13635/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/main.go b/main.go index 7b9d0952..d827412a 100644 --- a/main.go +++ b/main.go @@ -1,14 +1,18 @@ package main import ( + "context" "net/http" + "os" "github.com/fatih/structs" conf "github.com/infinityworks/github-exporter/config" "github.com/infinityworks/github-exporter/exporter" "github.com/infinityworks/go-common/logger" "github.com/prometheus/client_golang/prometheus" + "github.com/shurcooL/githubv4" "github.com/sirupsen/logrus" + "golang.org/x/oauth2" ) var ( @@ -27,17 +31,25 @@ func main() { log.WithFields(structs.Map(applicationCfg)).Info("Starting Exporter") - exporter := exporter.Exporter{ + conf := exporter.Exporter{ APIMetrics: mets, Config: applicationCfg, } + src := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: os.Getenv("GITHUB_TOKEN")}, + ) + httpClient := oauth2.NewClient(context.Background(), src) + + client := githubv4.NewClient(httpClient) + exporter.Query(client) + // Register Metrics from each of the endpoints // This invokes the Collect method through the prometheus client libraries. - prometheus.MustRegister(&exporter) + prometheus.MustRegister(&conf) // Setup HTTP handler - http.Handle(applicationCfg.MetricsPath(), prometheus.Handler()) + // http.Handle(applicationCfg.MetricsPath(), prometheus.Handler()) http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(` Github Exporter From 489cd6e193a8c145504086cba00716d14fcd9300 Mon Sep 17 00:00:00 2001 From: Adam Pietrzycki Date: Sun, 17 Feb 2019 01:39:00 +0000 Subject: [PATCH 11/21] Use promhttp --- main.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index d827412a..c94706bc 100644 --- a/main.go +++ b/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "github.com/prometheus/client_golang/prometheus/promhttp" "net/http" "os" @@ -49,7 +50,8 @@ func main() { prometheus.MustRegister(&conf) // Setup HTTP handler - // http.Handle(applicationCfg.MetricsPath(), prometheus.Handler()) + http.Handle(applicationCfg.MetricsPath(), promhttp.Handler()) + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(` Github Exporter From fb154dc4fa0b54fb004ffbb8c0ea7ccaac1b6fa2 Mon Sep 17 00:00:00 2001 From: Adam Pietrzycki Date: Sun, 17 Feb 2019 02:14:04 +0000 Subject: [PATCH 12/21] Testing v4 setup --- exporter/github.go | 29 ++++++++++++++++++++++------- go.mod | 5 ----- go.sum | 37 +------------------------------------ 3 files changed, 23 insertions(+), 48 deletions(-) diff --git a/exporter/github.go b/exporter/github.go index 77c7afe8..5063786f 100644 --- a/exporter/github.go +++ b/exporter/github.go @@ -8,18 +8,33 @@ import ( ) var query struct { - Viewer struct { - Login githubv4.String - CreatedAt githubv4.DateTime - } + Repository struct { + Stargazers struct { + TotalCount githubv4.Int + } + Issues struct { + TotalCount githubv4.Int + } + Watchers struct { + TotalCount githubv4.Int + } + ForkCount githubv4.Int + } `graphql:"repository(owner: $owner, name: $name)"` } // Query GitHub API func Query(client *githubv4.Client) { - err := client.Query(context.Background(), &query, nil) + variables := map[string]interface{}{ + "owner": githubv4.String("infinityworks"), + "name": githubv4.String("github-exporter"), + } + + err := client.Query(context.Background(), &query, variables) if err != nil { log.Fatal(err) } - fmt.Println(" Login:", query.Viewer.Login) - fmt.Println("CreatedAt:", query.Viewer.CreatedAt) + fmt.Println("Stars - ", query.Repository.Stargazers.TotalCount) + fmt.Println("Issues - ", query.Repository.Issues.TotalCount) + fmt.Println("Watchers - ", query.Repository.Watchers.TotalCount) + fmt.Println("Forks - ", query.Repository.ForkCount) } diff --git a/go.mod b/go.mod index 634efb65..82ed6e8f 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/infinityworks/github-exporter require ( github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc // indirect github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf // indirect - github.com/coreos/etcd v3.3.12+incompatible // indirect github.com/fatih/structs v1.1.0 github.com/infinityworks/go-common v0.0.0-20170820165359-7f20a140fd37 github.com/prometheus/client_golang v0.9.2 @@ -11,13 +10,9 @@ require ( github.com/shurcooL/githubv4 v0.0.0-20190119021625-d9689b595017 github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f // indirect github.com/sirupsen/logrus v1.3.0 - github.com/spf13/afero v1.2.1 // indirect - github.com/spf13/viper v1.3.1 github.com/stretchr/testify v1.3.0 // indirect - github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43 // indirect golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67 // indirect golang.org/x/oauth2 v0.0.0-20190212230446-3e8b2be13635 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect - gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect ) diff --git a/go.sum b/go.sum index 24986ee7..c4f5c6b9 100644 --- a/go.sum +++ b/go.sum @@ -3,36 +3,21 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5Vpd github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.12+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/infinityworks/go-common v0.0.0-20170820165359-7f20a140fd37 h1:Lm6kyC3JBiJQvJrus66He0E4viqDc/m5BdiFNSkIFfU= github.com/infinityworks/go-common v0.0.0-20170820165359-7f20a140fd37/go.mod h1:+OaHNKQvQ9oOCr+DgkF95PkiDx20fLHpzMp8SmRPQTg= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= @@ -49,29 +34,14 @@ github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f h1:tygelZueB1EtXk github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f/go.mod h1:AuYgA5Kyo4c7HfUmvRGs/6rGlMMV/6B1bVnB9JxJEEg= github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.1 h1:qgMbHoJbPbw579P+1zVY+6n4nIFuIchaIjzZ/I/Yq8M= -github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/viper v1.3.1 h1:5+8j8FTpnFV4nEImW/ofkzEt8VoOiLXxdYIDsB73T38= -github.com/spf13/viper v1.3.1/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/ugorji/go v1.1.2/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67 h1:ng3VDlRp5/DHpSWl02R4rM9I+8M2rhmsuLwAMmkLQWE= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -87,7 +57,6 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FY golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= @@ -96,7 +65,3 @@ google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO50 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From 02227c4473f3b998f3419def95c09244dcaad099 Mon Sep 17 00:00:00 2001 From: Adam Pietrzycki Date: Wed, 27 Feb 2019 21:43:40 +0000 Subject: [PATCH 13/21] Use go 1.12 Signed-off-by: Adam Pietrzycki --- Dockerfile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index ec60b3c7..c0ca4572 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,10 @@ -FROM golang:1.11.5-alpine as build +FROM golang:1.12-alpine as build LABEL maintainer="Infinity Works" -RUN apk --no-cache add ca-certificates \ - && apk --no-cache add --virtual build-deps git +RUN apk --no-cache add ca-certificates build-base git -COPY ./ /go/src/github.com/infinityworks/github-exporter -WORKDIR /go/src/github.com/infinityworks/github-exporter +COPY ./ /github-exporter +WORKDIR /github-exporter RUN go get \ && go test ./... \ From f43c9551c2ebee91cf44d1d36daa0d5d37b38716 Mon Sep 17 00:00:00 2001 From: Adam Pietrzycki Date: Wed, 27 Feb 2019 22:17:44 +0000 Subject: [PATCH 14/21] Testing viper for config Signed-off-by: Adam Pietrzycki --- exporter/metrics.go | 26 +++++++++++++++++++------- go.mod | 2 ++ go.sum | 34 ++++++++++++++++++++++++++++++++++ main.go | 9 +++++++++ 4 files changed, 64 insertions(+), 7 deletions(-) diff --git a/exporter/metrics.go b/exporter/metrics.go index 5c4ea958..2e812a94 100644 --- a/exporter/metrics.go +++ b/exporter/metrics.go @@ -1,6 +1,9 @@ package exporter -import "github.com/prometheus/client_golang/prometheus" +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/spf13/viper" +) import "strconv" // AddMetrics - Adds all of the metrics to a map of strings, returns the map. @@ -57,12 +60,21 @@ func (e *Exporter) processMetrics(data []*Datum, rates *RateLimits, ch chan<- pr // APIMetrics - range through the data slice for _, x := range data { - ch <- prometheus.MustNewConstMetric(e.APIMetrics["Stars"], prometheus.GaugeValue, x.Stars, x.Name, x.Owner.Login, strconv.FormatBool(x.Private), strconv.FormatBool(x.Fork), strconv.FormatBool(x.Archived), x.License.Key, x.Language) - ch <- prometheus.MustNewConstMetric(e.APIMetrics["Forks"], prometheus.GaugeValue, x.Forks, x.Name, x.Owner.Login, strconv.FormatBool(x.Private), strconv.FormatBool(x.Fork), strconv.FormatBool(x.Archived), x.License.Key, x.Language) - ch <- prometheus.MustNewConstMetric(e.APIMetrics["OpenIssues"], prometheus.GaugeValue, x.OpenIssues, x.Name, x.Owner.Login, strconv.FormatBool(x.Private), strconv.FormatBool(x.Fork), strconv.FormatBool(x.Archived), x.License.Key, x.Language) - ch <- prometheus.MustNewConstMetric(e.APIMetrics["Watchers"], prometheus.GaugeValue, x.Watchers, x.Name, x.Owner.Login, strconv.FormatBool(x.Private), strconv.FormatBool(x.Fork), strconv.FormatBool(x.Archived), x.License.Key, x.Language) - ch <- prometheus.MustNewConstMetric(e.APIMetrics["Size"], prometheus.GaugeValue, x.Size, x.Name, x.Owner.Login, strconv.FormatBool(x.Private), strconv.FormatBool(x.Fork), strconv.FormatBool(x.Archived), x.License.Key, x.Language) - + if viper.GetBool("stars") { + ch <- prometheus.MustNewConstMetric(e.APIMetrics["Stars"], prometheus.GaugeValue, x.Stars, x.Name, x.Owner.Login, strconv.FormatBool(x.Private), strconv.FormatBool(x.Fork), strconv.FormatBool(x.Archived), x.License.Key, x.Language) + } + if viper.GetBool("forks") { + ch <- prometheus.MustNewConstMetric(e.APIMetrics["Forks"], prometheus.GaugeValue, x.Forks, x.Name, x.Owner.Login, strconv.FormatBool(x.Private), strconv.FormatBool(x.Fork), strconv.FormatBool(x.Archived), x.License.Key, x.Language) + } + if viper.GetBool("issues") { + ch <- prometheus.MustNewConstMetric(e.APIMetrics["OpenIssues"], prometheus.GaugeValue, x.OpenIssues, x.Name, x.Owner.Login, strconv.FormatBool(x.Private), strconv.FormatBool(x.Fork), strconv.FormatBool(x.Archived), x.License.Key, x.Language) + } + if viper.GetBool("watchers") { + ch <- prometheus.MustNewConstMetric(e.APIMetrics["Watchers"], prometheus.GaugeValue, x.Watchers, x.Name, x.Owner.Login, strconv.FormatBool(x.Private), strconv.FormatBool(x.Fork), strconv.FormatBool(x.Archived), x.License.Key, x.Language) + } + if viper.GetBool("size") { + ch <- prometheus.MustNewConstMetric(e.APIMetrics["Size"], prometheus.GaugeValue, x.Size, x.Name, x.Owner.Login, strconv.FormatBool(x.Private), strconv.FormatBool(x.Fork), strconv.FormatBool(x.Archived), x.License.Key, x.Language) + } } // Set Rate limit stats diff --git a/go.mod b/go.mod index 82ed6e8f..6b81c6c8 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,7 @@ module github.com/infinityworks/github-exporter require ( + github.com/BurntSushi/toml v0.3.1 // indirect github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc // indirect github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf // indirect github.com/fatih/structs v1.1.0 @@ -10,6 +11,7 @@ require ( github.com/shurcooL/githubv4 v0.0.0-20190119021625-d9689b595017 github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f // indirect github.com/sirupsen/logrus v1.3.0 + github.com/spf13/viper v1.3.1 github.com/stretchr/testify v1.3.0 // indirect golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67 // indirect golang.org/x/oauth2 v0.0.0-20190212230446-3e8b2be13635 diff --git a/go.sum b/go.sum index c4f5c6b9..9533381b 100644 --- a/go.sum +++ b/go.sum @@ -1,23 +1,39 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/infinityworks/go-common v0.0.0-20170820165359-7f20a140fd37 h1:Lm6kyC3JBiJQvJrus66He0E4viqDc/m5BdiFNSkIFfU= github.com/infinityworks/go-common v0.0.0-20170820165359-7f20a140fd37/go.mod h1:+OaHNKQvQ9oOCr+DgkF95PkiDx20fLHpzMp8SmRPQTg= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= @@ -34,14 +50,27 @@ github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f h1:tygelZueB1EtXk github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f/go.mod h1:AuYgA5Kyo4c7HfUmvRGs/6rGlMMV/6B1bVnB9JxJEEg= github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.1 h1:5+8j8FTpnFV4nEImW/ofkzEt8VoOiLXxdYIDsB73T38= +github.com/spf13/viper v1.3.1/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67 h1:ng3VDlRp5/DHpSWl02R4rM9I+8M2rhmsuLwAMmkLQWE= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -57,6 +86,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FY golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= @@ -65,3 +95,7 @@ google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO50 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/main.go b/main.go index c94706bc..5a19a3fb 100644 --- a/main.go +++ b/main.go @@ -14,6 +14,8 @@ import ( "github.com/shurcooL/githubv4" "github.com/sirupsen/logrus" "golang.org/x/oauth2" + + "github.com/spf13/viper" ) var ( @@ -26,6 +28,13 @@ func init() { applicationCfg = conf.Init() mets = exporter.AddMetrics() log = logger.Start(&applicationCfg) + + viper.SetDefault("stars", false) + viper.SetDefault("forks", false) + viper.SetDefault("issues", false) + viper.SetDefault("watchers", false) + viper.SetDefault("size", false) + } func main() { From 6bf94b3efe09258a55c153355c1650f815fb1823 Mon Sep 17 00:00:00 2001 From: Adam Pietrzycki Date: Thu, 28 Feb 2019 00:01:25 +0000 Subject: [PATCH 15/21] Playing around with config Signed-off-by: Adam Pietrzycki --- config/config.go | 38 ++++++++++++++++++++++++++++++-------- config/github.go | 6 ++---- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/config/config.go b/config/config.go index 033374ad..a3236c29 100644 --- a/config/config.go +++ b/config/config.go @@ -2,20 +2,19 @@ package config import ( "fmt" + "github.com/spf13/viper" "io/ioutil" "strings" log "github.com/sirupsen/logrus" "os" - - cfg "github.com/infinityworks/go-common/config" ) // Config struct holds all of the runtime configuration for the application type Config struct { - *cfg.BaseConfig - APIURL string + *ExporterConfig + *APIConfig Repositories string Organisations string Users string @@ -25,26 +24,49 @@ type Config struct { TargetURLs []string } +func init() { + viper.SetDefault("METRICS_PATH", "/metrics") + viper.SetDefault("LISTEN_PORT", "8080") + viper.SetDefault("LOG_LEVEL", "debug") + viper.SetDefault("APP_NAME", "app") + viper.SetDefault("API_URL", "https://api.github.com") + + viper.BindEnv("METRICS_PATH") + viper.BindEnv("LISTEN_PORT") + viper.BindEnv("LOG_LEVEL") + viper.BindEnv("APP_NAME") +} + // Init populates the Config struct based on environmental runtime configuration func Init() Config { - ac := cfg.Init() - url := cfg.GetEnv("API_URL", "https://api.github.com") + ec := ExporterConfig{ + MetricsPath: viper.GetString("METRICS_PATH"), + ListenPort: viper.GetString("LISTEN_PORT"), + LogLevel: viper.GetString("LOG_LEVEL"), + ApplicationName: viper.GetString("APP_NAME"), + } + + ac := APIConfig{ + APIURL: viper.GetString("API_URL"), + } + repos := os.Getenv("REPOS") orgs := os.Getenv("ORGS") users := os.Getenv("USERS") tokenEnv := os.Getenv("GITHUB_TOKEN") tokenFile := os.Getenv("GITHUB_TOKEN_FILE") + token, err := getAuth(tokenEnv, tokenFile) - scraped, err := getScrapeURLs(url, repos, orgs, users) + scraped, err := getScrapeURLs(ac.APIURL, repos, orgs, users) if err != nil { log.Errorf("Error initialising Configuration, Error: %v", err) } appConfig := Config{ + &ec, &ac, - url, repos, orgs, users, diff --git a/config/github.go b/config/github.go index eaf26be4..a1d18f76 100644 --- a/config/github.go +++ b/config/github.go @@ -1,7 +1,5 @@ package config -type RateLimitConfig struct { - Limit int - Remaining int - Reset string +type APIConfig struct { + APIURL string } From b401d849abcfc5e9f3af5256914262a30e35eee0 Mon Sep 17 00:00:00 2001 From: Adam Pietrzycki Date: Thu, 28 Feb 2019 00:01:45 +0000 Subject: [PATCH 16/21] Move log out of go-common Signed-off-by: Adam Pietrzycki --- config/log.go | 23 ++++++++++++++++++++ go.mod | 2 +- main.go | 58 ++++++++++++++++++++++----------------------------- 3 files changed, 49 insertions(+), 34 deletions(-) create mode 100644 config/log.go diff --git a/config/log.go b/config/log.go new file mode 100644 index 00000000..75da7746 --- /dev/null +++ b/config/log.go @@ -0,0 +1,23 @@ +package config + +import ( + log "github.com/sirupsen/logrus" + "github.com/spf13/viper" +) + +func LogLevel() log.Level { + switch viper.GetString("LOG_LEVEL") { + case "debug": + return log.DebugLevel + case "info": + return log.InfoLevel + case "warn": + return log.WarnLevel + case "fatal": + return log.FatalLevel + case "panic": + return log.PanicLevel + default: + return log.DebugLevel + } +} diff --git a/go.mod b/go.mod index 6b81c6c8..8f1226ed 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/spf13/viper v1.3.1 github.com/stretchr/testify v1.3.0 // indirect golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67 // indirect - golang.org/x/oauth2 v0.0.0-20190212230446-3e8b2be13635 + golang.org/x/oauth2 v0.0.0-20190212230446-3e8b2be13635 // indirect golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect ) diff --git a/main.go b/main.go index 5a19a3fb..88834782 100644 --- a/main.go +++ b/main.go @@ -1,39 +1,29 @@ package main import ( - "context" - "github.com/prometheus/client_golang/prometheus/promhttp" "net/http" - "os" "github.com/fatih/structs" conf "github.com/infinityworks/github-exporter/config" "github.com/infinityworks/github-exporter/exporter" - "github.com/infinityworks/go-common/logger" "github.com/prometheus/client_golang/prometheus" - "github.com/shurcooL/githubv4" - "github.com/sirupsen/logrus" - "golang.org/x/oauth2" - + "github.com/prometheus/client_golang/prometheus/promhttp" + log "github.com/sirupsen/logrus" "github.com/spf13/viper" ) var ( - log *logrus.Logger applicationCfg conf.Config mets map[string]*prometheus.Desc ) func init() { + log.SetFormatter(&log.JSONFormatter{}) + viper.AutomaticEnv() + log.SetLevel(conf.LogLevel()) applicationCfg = conf.Init() mets = exporter.AddMetrics() - log = logger.Start(&applicationCfg) - - viper.SetDefault("stars", false) - viper.SetDefault("forks", false) - viper.SetDefault("issues", false) - viper.SetDefault("watchers", false) - viper.SetDefault("size", false) + //log = logger.Start(&applicationCfg) } @@ -46,31 +36,33 @@ func main() { Config: applicationCfg, } - src := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: os.Getenv("GITHUB_TOKEN")}, - ) - httpClient := oauth2.NewClient(context.Background(), src) + //--- + //src := oauth2.StaticTokenSource( + // &oauth2.Token{AccessToken: os.Getenv("GITHUB_TOKEN")}, + //) + //httpClient := oauth2.NewClient(context.Background(), src) - client := githubv4.NewClient(httpClient) - exporter.Query(client) + //client := githubv4.NewClient(httpClient) + //exporter.Query(client) + //--- // Register Metrics from each of the endpoints // This invokes the Collect method through the prometheus client libraries. prometheus.MustRegister(&conf) // Setup HTTP handler - http.Handle(applicationCfg.MetricsPath(), promhttp.Handler()) + http.Handle(applicationCfg.MetricsPath, promhttp.Handler()) http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(` - Github Exporter - -

GitHub Prometheus Metrics Exporter

-

For more information, visit GitHub

-

Metrics

- - - `)) + w.Write([]byte( + ` + Github Exporter + +

GitHub Prometheus Metrics Exporter

+

For more information, visit GitHub

+

Metrics

+ + `)) }) - log.Fatal(http.ListenAndServe(":"+applicationCfg.ListenPort(), nil)) + log.Fatal(http.ListenAndServe(":"+applicationCfg.ListenPort, nil)) } From 4c44609ed42bc2ad7250cc2476684e064a87d4a6 Mon Sep 17 00:00:00 2001 From: Adam Pietrzycki Date: Thu, 28 Feb 2019 00:04:53 +0000 Subject: [PATCH 17/21] fmt Signed-off-by: Adam Pietrzycki --- config/config.go | 2 +- config/github.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.go b/config/config.go index a3236c29..f31c0f92 100644 --- a/config/config.go +++ b/config/config.go @@ -50,7 +50,7 @@ func Init() Config { ac := APIConfig{ APIURL: viper.GetString("API_URL"), } - + repos := os.Getenv("REPOS") orgs := os.Getenv("ORGS") users := os.Getenv("USERS") diff --git a/config/github.go b/config/github.go index a1d18f76..910eda41 100644 --- a/config/github.go +++ b/config/github.go @@ -1,5 +1,5 @@ package config type APIConfig struct { - APIURL string + APIURL string } From aad98b6a71ca2d5a0dc0bc8ffdee451c996b773c Mon Sep 17 00:00:00 2001 From: Adam Pietrzycki Date: Thu, 28 Feb 2019 00:05:00 +0000 Subject: [PATCH 18/21] Add changelog file Signed-off-by: Adam Pietrzycki --- CHANGELOG.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..e69de29b From d06dc13824eeef6603377720ce9920b2377f97a1 Mon Sep 17 00:00:00 2001 From: Adam Pietrzycki Date: Thu, 28 Feb 2019 00:16:45 +0000 Subject: [PATCH 19/21] Clean up Signed-off-by: Adam Pietrzycki --- main.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/main.go b/main.go index 88834782..47e87007 100644 --- a/main.go +++ b/main.go @@ -23,15 +23,13 @@ func init() { log.SetLevel(conf.LogLevel()) applicationCfg = conf.Init() mets = exporter.AddMetrics() - //log = logger.Start(&applicationCfg) - } func main() { log.WithFields(structs.Map(applicationCfg)).Info("Starting Exporter") - conf := exporter.Exporter{ + exporter := exporter.Exporter{ APIMetrics: mets, Config: applicationCfg, } @@ -48,7 +46,7 @@ func main() { // Register Metrics from each of the endpoints // This invokes the Collect method through the prometheus client libraries. - prometheus.MustRegister(&conf) + prometheus.MustRegister(&exporter) // Setup HTTP handler http.Handle(applicationCfg.MetricsPath, promhttp.Handler()) From 08a0c828442814ff122af67dc2012eca46d7a831 Mon Sep 17 00:00:00 2001 From: Adam Pietrzycki Date: Thu, 28 Feb 2019 22:08:28 +0000 Subject: [PATCH 20/21] Clean up configs and use viper autoenv Signed-off-by: Adam Pietrzycki --- config/config.go | 28 +++++++--------------------- config/exporter.go | 9 +++++++++ config/github.go | 6 ++++++ config/query.go | 10 ++++++++++ main.go | 5 +++-- 5 files changed, 35 insertions(+), 23 deletions(-) diff --git a/config/config.go b/config/config.go index f31c0f92..a1dca76c 100644 --- a/config/config.go +++ b/config/config.go @@ -2,13 +2,12 @@ package config import ( "fmt" - "github.com/spf13/viper" "io/ioutil" "strings" - log "github.com/sirupsen/logrus" + "github.com/spf13/viper" - "os" + log "github.com/sirupsen/logrus" ) // Config struct holds all of the runtime configuration for the application @@ -24,19 +23,6 @@ type Config struct { TargetURLs []string } -func init() { - viper.SetDefault("METRICS_PATH", "/metrics") - viper.SetDefault("LISTEN_PORT", "8080") - viper.SetDefault("LOG_LEVEL", "debug") - viper.SetDefault("APP_NAME", "app") - viper.SetDefault("API_URL", "https://api.github.com") - - viper.BindEnv("METRICS_PATH") - viper.BindEnv("LISTEN_PORT") - viper.BindEnv("LOG_LEVEL") - viper.BindEnv("APP_NAME") -} - // Init populates the Config struct based on environmental runtime configuration func Init() Config { @@ -51,11 +37,11 @@ func Init() Config { APIURL: viper.GetString("API_URL"), } - repos := os.Getenv("REPOS") - orgs := os.Getenv("ORGS") - users := os.Getenv("USERS") - tokenEnv := os.Getenv("GITHUB_TOKEN") - tokenFile := os.Getenv("GITHUB_TOKEN_FILE") + repos := viper.GetString("REPOS") + orgs := viper.GetString("ORGS") + users := viper.GetString("USERS") + tokenEnv := viper.GetString("GITHUB_TOKEN") + tokenFile := viper.GetString("GITHUB_TOKEN_FILE") token, err := getAuth(tokenEnv, tokenFile) scraped, err := getScrapeURLs(ac.APIURL, repos, orgs, users) diff --git a/config/exporter.go b/config/exporter.go index 76d6bc4f..5b6dc611 100644 --- a/config/exporter.go +++ b/config/exporter.go @@ -1,8 +1,17 @@ package config +import "github.com/spf13/viper" + type ExporterConfig struct { MetricsPath string ListenPort string LogLevel string ApplicationName string } + +func init() { + viper.SetDefault("METRICS_PATH", "/metrics") + viper.SetDefault("LISTEN_PORT", "8080") + viper.SetDefault("LOG_LEVEL", "debug") + viper.SetDefault("APP_NAME", "app") +} diff --git a/config/github.go b/config/github.go index 910eda41..24252bc6 100644 --- a/config/github.go +++ b/config/github.go @@ -1,5 +1,11 @@ package config +import "github.com/spf13/viper" + type APIConfig struct { APIURL string } + +func init() { + viper.SetDefault("API_URL", "https://api.github.com") +} diff --git a/config/query.go b/config/query.go index 3160aa7c..8880f820 100644 --- a/config/query.go +++ b/config/query.go @@ -1,5 +1,7 @@ package config +import "github.com/spf13/viper" + type QueryConfiguration struct { Stars bool OpenIssues bool @@ -7,3 +9,11 @@ type QueryConfiguration struct { Forks bool Size bool } + +func init() { + viper.SetDefault("STARS", true) + viper.SetDefault("ISSUES", true) + viper.SetDefault("WATCHERS", true) + viper.SetDefault("FORKS", true) + viper.SetDefault("SIZE", true) +} diff --git a/main.go b/main.go index 47e87007..d785d2f0 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + "github.com/spf13/viper" "net/http" "github.com/fatih/structs" @@ -9,7 +10,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" log "github.com/sirupsen/logrus" - "github.com/spf13/viper" ) var ( @@ -18,9 +18,10 @@ var ( ) func init() { - log.SetFormatter(&log.JSONFormatter{}) viper.AutomaticEnv() + log.SetFormatter(&log.JSONFormatter{}) log.SetLevel(conf.LogLevel()) + applicationCfg = conf.Init() mets = exporter.AddMetrics() } From a488de3c107ac47605f6f5c848a1d38c77a3f6f9 Mon Sep 17 00:00:00 2001 From: Adam Pietrzycki Date: Thu, 28 Feb 2019 23:36:36 +0000 Subject: [PATCH 21/21] Make query smarter but not fetching all the data Signed-off-by: Adam Pietrzycki --- exporter/github.go | 71 +++++++++++++++++++++++++++++++++++----------- go.mod | 2 +- main.go | 44 ++++++++++++++-------------- 3 files changed, 78 insertions(+), 39 deletions(-) diff --git a/exporter/github.go b/exporter/github.go index 5063786f..5c409175 100644 --- a/exporter/github.go +++ b/exporter/github.go @@ -3,23 +3,47 @@ package exporter import ( "context" "fmt" - "github.com/prometheus/common/log" "github.com/shurcooL/githubv4" + "github.com/spf13/viper" + "log" ) +type starGazers struct { + Stargazers struct { + TotalCount githubv4.Int + } +} + +type issues struct { + Issues struct { + TotalCount githubv4.Int + } +} + +type watchers struct { + Watchers struct { + TotalCount githubv4.Int + } +} + +type forkCount struct { + ForkCount githubv4.Int +} + var query struct { Repository struct { - Stargazers struct { - TotalCount githubv4.Int - } - Issues struct { - TotalCount githubv4.Int - } - Watchers struct { - TotalCount githubv4.Int - } - ForkCount githubv4.Int + starGazers `graphql:"...@include(if:$stars)"` + issues `graphql:"...@include(if:$issues)"` + watchers `graphql:"...@include(if:$watchers)"` + forkCount `graphql:"...@include(if:$forks)"` } `graphql:"repository(owner: $owner, name: $name)"` + + RateLimit struct { + Cost githubv4.Int + Limit githubv4.Int + Remaining githubv4.Int + ResetAt githubv4.DateTime + } } // Query GitHub API @@ -27,14 +51,29 @@ func Query(client *githubv4.Client) { variables := map[string]interface{}{ "owner": githubv4.String("infinityworks"), "name": githubv4.String("github-exporter"), + + "stars": githubv4.Boolean(viper.GetBool("STARS")), + "issues": githubv4.Boolean(viper.GetBool("ISSUES")), + "watchers": githubv4.Boolean(viper.GetBool("WATCHERS")), + "forks": githubv4.Boolean(viper.GetBool("FORKS")), } err := client.Query(context.Background(), &query, variables) if err != nil { - log.Fatal(err) + log.Println(err) } - fmt.Println("Stars - ", query.Repository.Stargazers.TotalCount) - fmt.Println("Issues - ", query.Repository.Issues.TotalCount) - fmt.Println("Watchers - ", query.Repository.Watchers.TotalCount) - fmt.Println("Forks - ", query.Repository.ForkCount) + + fmt.Println() + fmt.Println("Stars - ", query.Repository.Stargazers.TotalCount) + fmt.Println("Issues - ", query.Repository.Issues.TotalCount) + fmt.Println("Watchers - ", query.Repository.Watchers.TotalCount) + fmt.Println("Forks - ", query.Repository.ForkCount) + fmt.Println("Ratelimit - ", + query.RateLimit.Cost, + query.RateLimit.Limit, + query.RateLimit.Remaining, + query.RateLimit.ResetAt) + + fmt.Println(query) + } diff --git a/go.mod b/go.mod index 8f1226ed..6b81c6c8 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/spf13/viper v1.3.1 github.com/stretchr/testify v1.3.0 // indirect golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67 // indirect - golang.org/x/oauth2 v0.0.0-20190212230446-3e8b2be13635 // indirect + golang.org/x/oauth2 v0.0.0-20190212230446-3e8b2be13635 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect ) diff --git a/main.go b/main.go index d785d2f0..6f80bce5 100644 --- a/main.go +++ b/main.go @@ -1,20 +1,22 @@ package main +import "C" import ( - "github.com/spf13/viper" - "net/http" - + "context" "github.com/fatih/structs" conf "github.com/infinityworks/github-exporter/config" "github.com/infinityworks/github-exporter/exporter" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/shurcooL/githubv4" log "github.com/sirupsen/logrus" + "github.com/spf13/viper" + "golang.org/x/oauth2" + "net/http" ) var ( - applicationCfg conf.Config - mets map[string]*prometheus.Desc + c conf.Config + mets map[string]*prometheus.Desc ) func init() { @@ -22,35 +24,33 @@ func init() { log.SetFormatter(&log.JSONFormatter{}) log.SetLevel(conf.LogLevel()) - applicationCfg = conf.Init() + c = conf.Init() mets = exporter.AddMetrics() } func main() { - log.WithFields(structs.Map(applicationCfg)).Info("Starting Exporter") + log.WithFields(structs.Map(c)).Info("Starting Exporter") - exporter := exporter.Exporter{ + applicationCfg := exporter.Exporter{ APIMetrics: mets, - Config: applicationCfg, + Config: c, } - //--- - //src := oauth2.StaticTokenSource( - // &oauth2.Token{AccessToken: os.Getenv("GITHUB_TOKEN")}, - //) - //httpClient := oauth2.NewClient(context.Background(), src) + src := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: viper.GetString("GITHUB_TOKEN")}, + ) + httpClient := oauth2.NewClient(context.Background(), src) - //client := githubv4.NewClient(httpClient) - //exporter.Query(client) - //--- + client := githubv4.NewClient(httpClient) + exporter.Query(client) // Register Metrics from each of the endpoints // This invokes the Collect method through the prometheus client libraries. - prometheus.MustRegister(&exporter) + prometheus.MustRegister(&applicationCfg) // Setup HTTP handler - http.Handle(applicationCfg.MetricsPath, promhttp.Handler()) + //http.Handle(c.MetricsPath, promhttp.Handler()) http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte( @@ -59,9 +59,9 @@ func main() {

GitHub Prometheus Metrics Exporter

For more information, visit GitHub

-

Metrics

+

Metrics

`)) }) - log.Fatal(http.ListenAndServe(":"+applicationCfg.ListenPort, nil)) + log.Fatal(http.ListenAndServe(":"+c.ListenPort, nil)) }