Commit 7872c9b0 authored by Kevin Di Lallo's avatar Kevin Di Lallo
Browse files

added session metrics in auth-svc

parent 710d41e9

charts/meep-auth-svc/values.yaml

+1 −1
Original line number Diff line number Diff line
@@ -77,7 +77,7 @@ prometheus: 
  monitor:

    enabled: true

    port: 9000

    interval: 5s

    interval: 10s

    additionalLabels: {}

    namespace: ""

    relabelings: []

charts/nginx-ingress/values.yaml

+1 −0
Original line number Diff line number Diff line
@@ -114,6 +114,7 @@ controller: 
  ##   default-ssl-certificate: "<namespace>/<secret_name>"

  extraArgs:

    default-ssl-certificate: "default/meep-ingress"

    metrics-per-host: "false"



  ## Additional environment variables to set

  extraEnvs: []

go-apps/meep-auth-svc/server/auth-svc.go

+97 −16
Original line number Diff line number Diff line
@@ -142,10 +142,42 @@ var influxDBAddr string = "http://meep-influxdb.default.svc.cluster.local:8086" 
// Auth Service

var authSvc *AuthSvc



var authRequests = promauto.NewCounter(prometheus.CounterOpts{

	Name: "auth_svc_auth_req_total",

	Help: "The total number of auuthentication requests",

// Metrics

var (

	metricAuthRequests = promauto.NewCounterVec(prometheus.CounterOpts{

		Name: "auth_svc_http_request_total",

		Help: "The total number of http requests authenticated",

	}, []string{"svc", "method", "path", "resp"})

	metricSessionLogin = promauto.NewCounterVec(prometheus.CounterOpts{

		Name: "auth_svc_session_login_total",

		Help: "The total number of session login attempts",

	}, []string{"type"})

	metricSessionLogout = promauto.NewCounter(prometheus.CounterOpts{

		Name: "auth_svc_session_logout_total",

		Help: "The total number of session logout attempts",

	})

	metricSessionSuccess = promauto.NewCounter(prometheus.CounterOpts{

		Name: "auth_svc_session_success_total",

		Help: "The total number of successful sessions",

	})

	metricSessionFail = promauto.NewCounterVec(prometheus.CounterOpts{

		Name: "auth_svc_session_fail_total",

		Help: "The total number of failed session login attempts",

	}, []string{"type"})

	metricSessionTimeout = promauto.NewCounter(prometheus.CounterOpts{

		Name: "auth_svc_session_timeout_total",

		Help: "The total number of timed out sessions",

	})

	metricSessionActive = promauto.NewGauge(prometheus.GaugeOpts{

		Name: "auth_svc_session_active",

		Help: "The number of active sessions",

	})

	metricSessionDuration = promauto.NewHistogram(prometheus.HistogramOpts{

		Name:    "auth_svc_session_duration",

		Help:    "A histogram of session durations",

		Buckets: prometheus.LinearBuckets(0, 20, 6),

	})

)



func Init() (err error) {
@@ -450,8 +482,14 @@ func sessionTimeoutCb(session *sm.Session) { 
	metric.Sandbox = session.Sandbox

	_ = authSvc.metricStore.SetSessionMetric(ms.SesMetTypeTimeout, metric)



	metricSessionTimeout.Inc()



	// Destroy session sandbox

	_, _ = authSvc.pfmCtrlClient.SandboxControlApi.DeleteSandbox(context.TODO(), session.Sandbox)

	_, err := authSvc.pfmCtrlClient.SandboxControlApi.DeleteSandbox(context.TODO(), session.Sandbox)

	if err == nil {

		metricSessionActive.Dec()

		metricSessionDuration.Observe(time.Since(session.StartTime).Minutes())

	}

}



// Generate a random state string
@@ -517,19 +555,19 @@ func getErrUrl(err string) string { 


func asAuthenticate(w http.ResponseWriter, r *http.Request) {



	authRequests.Inc()



	// Get service & sandbox name from request query parameters

	query := r.URL.Query()

	svcName := query.Get("svc")

	// sboxName := query.Get("sbox")

	var sboxName string

	var path string



	// Get original request URL & method

	originalUrl := r.Header.Get("X-Original-URL")

	originalMethod := r.Header.Get("X-Original-Method")

	if originalUrl == "" || originalMethod == "" {

		http.Error(w, "Unauthorized", http.StatusUnauthorized)

		metricAuthRequests.WithLabelValues(svcName, originalMethod, path, strconv.Itoa(http.StatusUnauthorized)).Inc()

		return

	}
@@ -539,6 +577,7 @@ func asAuthenticate(w http.ResponseWriter, r *http.Request) { 
	r.URL, err = url.ParseRequestURI(originalUrl)

	if err != nil {

		http.Error(w, "Unauthorized", http.StatusUnauthorized)

		metricAuthRequests.WithLabelValues(svcName, originalMethod, path, strconv.Itoa(http.StatusUnauthorized)).Inc()

		return

	}
@@ -550,6 +589,9 @@ func asAuthenticate(w http.ResponseWriter, r *http.Request) { 
		sboxName = match.Vars["sbox"]

		log.Debug("routeName: ", routeName, " sboxName: ", sboxName)



		// Get path template

		path, _ = match.Route.GetPathTemplate()



		// Check service-specific routes

		if svcName != "" {

			if svcPermissions, found := authSvc.cache.Services[svcName]; found {
@@ -569,6 +611,7 @@ func asAuthenticate(w http.ResponseWriter, r *http.Request) { 
	// Verify permission

	if permission == nil {

		http.Error(w, "Unauthorized", http.StatusUnauthorized)

		metricAuthRequests.WithLabelValues(svcName, originalMethod, path, strconv.Itoa(http.StatusUnauthorized)).Inc()

		return

	}
@@ -578,12 +621,14 @@ func asAuthenticate(w http.ResponseWriter, r *http.Request) { 
		// break

	case sm.ModeBlock:

		http.Error(w, "Unauthorized", http.StatusUnauthorized)

		metricAuthRequests.WithLabelValues(svcName, originalMethod, path, strconv.Itoa(http.StatusUnauthorized)).Inc()

		return

	case sm.ModeVerify:

		// Retrieve user session, if any

		session, err := authSvc.sessionMgr.GetSessionStore().Get(r)

		if err != nil || session == nil {

			http.Error(w, "Unauthorized", http.StatusUnauthorized)

			metricAuthRequests.WithLabelValues(svcName, originalMethod, path, strconv.Itoa(http.StatusUnauthorized)).Inc()

			return

		}
@@ -591,28 +636,33 @@ func asAuthenticate(w http.ResponseWriter, r *http.Request) { 
		role := session.Role

		if role == "" {

			http.Error(w, "Unauthorized", http.StatusUnauthorized)

			metricAuthRequests.WithLabelValues(svcName, originalMethod, path, strconv.Itoa(http.StatusUnauthorized)).Inc()

			return

		}

		access := permission.Roles[role]

		if access != sm.AccessGranted {

			http.Error(w, "Unauthorized", http.StatusUnauthorized)

			metricAuthRequests.WithLabelValues(svcName, originalMethod, path, strconv.Itoa(http.StatusUnauthorized)).Inc()

			return

		}



		// For non-admin users, verify session sandbox matches service sandbox, if any

		if session.Role != sm.RoleAdmin && sboxName != "" && sboxName != session.Sandbox {

			http.Error(w, "Unauthorized", http.StatusUnauthorized)

			metricAuthRequests.WithLabelValues(svcName, originalMethod, path, strconv.Itoa(http.StatusUnauthorized)).Inc()

			return

		}



	default:

		http.Error(w, "Unauthorized", http.StatusUnauthorized)

		metricAuthRequests.WithLabelValues(svcName, originalMethod, path, strconv.Itoa(http.StatusUnauthorized)).Inc()

		return

	}



	// Allow request

	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	w.WriteHeader(http.StatusOK)

	metricAuthRequests.WithLabelValues(svcName, originalMethod, path, strconv.Itoa(http.StatusOK)).Inc()

}



func asAuthorize(w http.ResponseWriter, r *http.Request) {
@@ -631,6 +681,7 @@ func asAuthorize(w http.ResponseWriter, r *http.Request) { 
		metric.Description = err.Error()

		_ = authSvc.metricStore.SetSessionMetric(ms.SesMetTypeError, metric)

		http.Redirect(w, r, getErrUrl(err.Error()), http.StatusFound)

		metricSessionFail.WithLabelValues("OAuth").Inc()

		return

	}
@@ -643,6 +694,7 @@ func asAuthorize(w http.ResponseWriter, r *http.Request) { 
		metric.Description = err.Error()

		_ = authSvc.metricStore.SetSessionMetric(ms.SesMetTypeError, metric)

		http.Redirect(w, r, getErrUrl(err.Error()), http.StatusFound)

		metricSessionFail.WithLabelValues("Internal").Inc()

		return

	}

	metric.Provider = provider
@@ -657,6 +709,7 @@ func asAuthorize(w http.ResponseWriter, r *http.Request) { 
		metric.Description = err.Error()

		_ = authSvc.metricStore.SetSessionMetric(ms.SesMetTypeError, metric)

		http.Redirect(w, r, getErrUrl(err.Error()), http.StatusFound)

		metricSessionFail.WithLabelValues("Internal").Inc()

		return

	}
@@ -667,6 +720,7 @@ func asAuthorize(w http.ResponseWriter, r *http.Request) { 
		metric.Description = err.Error()

		_ = authSvc.metricStore.SetSessionMetric(ms.SesMetTypeError, metric)

		http.Redirect(w, r, getErrUrl(err.Error()), http.StatusFound)

		metricSessionFail.WithLabelValues("OAuth").Inc()

		return

	}
@@ -681,6 +735,7 @@ func asAuthorize(w http.ResponseWriter, r *http.Request) { 
			metric.Description = err.Error()

			_ = authSvc.metricStore.SetSessionMetric(ms.SesMetTypeError, metric)

			http.Redirect(w, r, getErrUrl(err.Error()), http.StatusFound)

			metricSessionFail.WithLabelValues("OAuth").Inc()

			return

		}

		user, _, err := client.Users.Get(context.Background(), "")
@@ -689,6 +744,7 @@ func asAuthorize(w http.ResponseWriter, r *http.Request) { 
			metric.Description = err.Error()

			_ = authSvc.metricStore.SetSessionMetric(ms.SesMetTypeError, metric)

			http.Redirect(w, r, getErrUrl("Failed to retrieve GitHub user ID"), http.StatusFound)

			metricSessionFail.WithLabelValues("OAuth").Inc()

			return

		}

		userId = *user.Login
@@ -701,6 +757,7 @@ func asAuthorize(w http.ResponseWriter, r *http.Request) { 
			metric.Description = err.Error()

			_ = authSvc.metricStore.SetSessionMetric(ms.SesMetTypeError, metric)

			http.Redirect(w, r, getErrUrl(err.Error()), http.StatusFound)

			metricSessionFail.WithLabelValues("OAuth").Inc()

			return

		}
@@ -712,6 +769,7 @@ func asAuthorize(w http.ResponseWriter, r *http.Request) { 
				metric.Description = err.Error()

				_ = authSvc.metricStore.SetSessionMetric(ms.SesMetTypeError, metric)

				http.Redirect(w, r, getErrUrl("Failed to set GitLab API base url"), http.StatusFound)

				metricSessionFail.WithLabelValues("OAuth").Inc()

				return

			}

		}
@@ -722,6 +780,7 @@ func asAuthorize(w http.ResponseWriter, r *http.Request) { 
			metric.Description = err.Error()

			_ = authSvc.metricStore.SetSessionMetric(ms.SesMetTypeError, metric)

			http.Redirect(w, r, getErrUrl("Failed to retrieve GitLab user ID"), http.StatusFound)

			metricSessionFail.WithLabelValues("OAuth").Inc()

			return

		}

		userId = user.Username
@@ -730,12 +789,13 @@ func asAuthorize(w http.ResponseWriter, r *http.Request) { 
	metric.User = userId



	// Start user session

	sandboxName, err, errCode := startSession(provider, userId, w, r)

	sandboxName, isNew, err, errCode := startSession(provider, userId, w, r)

	if err != nil {

		log.Error(err.Error())

		metric.Description = err.Error()

		_ = authSvc.metricStore.SetSessionMetric(ms.SesMetTypeError, metric)

		http.Redirect(w, r, getErrUrl(err.Error()), errCode)

		metricSessionFail.WithLabelValues("Session").Inc()

		return

	}
@@ -744,11 +804,16 @@ func asAuthorize(w http.ResponseWriter, r *http.Request) { 


	// Redirect user to sandbox

	http.Redirect(w, r, authSvc.uri+"?sbox="+sandboxName+"&user="+userId, http.StatusFound)

	metricSessionSuccess.Inc()

	if isNew {

		metricSessionActive.Inc()

	}

}



func asLogin(w http.ResponseWriter, r *http.Request) {

	log.Info("----- OAUTH LOGIN -----")

	var metric ms.SessionMetric

	metricSessionLogin.WithLabelValues("OAuth").Inc()



	// Retrieve query parameters

	query := r.URL.Query()
@@ -763,6 +828,7 @@ func asLogin(w http.ResponseWriter, r *http.Request) { 
		metric.Description = err.Error()

		_ = authSvc.metricStore.SetSessionMetric(ms.SesMetTypeError, metric)

		http.Redirect(w, r, getErrUrl(err.Error()), http.StatusFound)

		metricSessionFail.WithLabelValues("Internal").Inc()

		return

	}
@@ -773,6 +839,7 @@ func asLogin(w http.ResponseWriter, r *http.Request) { 
		metric.Description = err.Error()

		_ = authSvc.metricStore.SetSessionMetric(ms.SesMetTypeError, metric)

		http.Redirect(w, r, getErrUrl(err.Error()), http.StatusFound)

		metricSessionFail.WithLabelValues("Internal").Inc()

		return

	}
@@ -797,6 +864,7 @@ func asLogin(w http.ResponseWriter, r *http.Request) { 
func asLoginUser(w http.ResponseWriter, r *http.Request) {

	log.Info("----- LOGIN -----")

	var metric ms.SessionMetric

	metricSessionLogin.WithLabelValues("Basic").Inc()



	// Get form data

	username := r.FormValue("username")
@@ -819,7 +887,7 @@ func asLoginUser(w http.ResponseWriter, r *http.Request) { 
	}



	// Start user session

	sandboxName, err, errCode := startSession(OAUTH_PROVIDER_LOCAL, username, w, r)

	sandboxName, isNew, err, errCode := startSession(OAUTH_PROVIDER_LOCAL, username, w, r)

	if err != nil {

		log.Error(err.Error())

		metric.Description = err.Error()
@@ -830,6 +898,9 @@ func asLoginUser(w http.ResponseWriter, r *http.Request) { 


	metric.Sandbox = sandboxName

	_ = authSvc.metricStore.SetSessionMetric(ms.SesMetTypeLogin, metric)

	if isNew {

		metricSessionActive.Inc()

	}



	// Prepare response

	var sandbox dataModel.Sandbox
@@ -850,7 +921,7 @@ func asLoginUser(w http.ResponseWriter, r *http.Request) { 
}



// Retrieve existing user session or create a new one

func startSession(provider string, username string, w http.ResponseWriter, r *http.Request) (sandboxName string, err error, code int) {

func startSession(provider string, username string, w http.ResponseWriter, r *http.Request) (sandboxName string, isNew bool, err error, code int) {



	// Get existing session by user name, if any

	sessionStore := authSvc.sessionMgr.GetSessionStore()
@@ -860,7 +931,7 @@ func startSession(provider string, username string, w http.ResponseWriter, r *ht 
		count := sessionStore.GetCount()

		if count >= authSvc.maxSessions {

			err = errors.New("Maximum session count exceeded")

			return "", err, http.StatusServiceUnavailable

			return "", isNew, err, http.StatusServiceUnavailable

		}



		// Get requested sandbox name & role from user profile, if any
@@ -876,13 +947,13 @@ func startSession(provider string, username string, w http.ResponseWriter, r *ht 
		if sandboxName == "" {

			sandbox, _, err := authSvc.pfmCtrlClient.SandboxControlApi.CreateSandbox(context.TODO(), sandboxConfig)

			if err != nil {

				return "", err, http.StatusInternalServerError

				return "", isNew, err, http.StatusInternalServerError

			}

			sandboxName = sandbox.Name

		} else {

			_, err := authSvc.pfmCtrlClient.SandboxControlApi.CreateSandboxWithName(context.TODO(), sandboxName, sandboxConfig)

			if err != nil {

				return "", err, http.StatusInternalServerError

				return "", isNew, err, http.StatusInternalServerError

			}

		}
@@ -893,6 +964,7 @@ func startSession(provider string, username string, w http.ResponseWriter, r *ht 
		session.Provider = provider

		session.Sandbox = sandboxName

		session.Role = role

		isNew = true

	} else {

		sandboxName = session.Sandbox

	}
@@ -905,14 +977,16 @@ func startSession(provider string, username string, w http.ResponseWriter, r *ht 
		if session.ID == "" {

			_, _ = authSvc.pfmCtrlClient.SandboxControlApi.DeleteSandbox(context.TODO(), sandboxName)

		}

		return "", err, code

		return "", isNew, err, code

	}

	return sandboxName, nil, http.StatusOK

	return sandboxName, isNew, nil, http.StatusOK

}



func asLogout(w http.ResponseWriter, r *http.Request) {

	log.Info("----- LOGOUT -----")

	var metric ms.SessionMetric

	sandboxDeleted := false

	metricSessionLogout.Inc()



	// Get existing session

	sessionStore := authSvc.sessionMgr.GetSessionStore()
@@ -922,7 +996,10 @@ func asLogout(w http.ResponseWriter, r *http.Request) { 
		metric.User = session.Username

		metric.Sandbox = session.Sandbox

		// Delete sandbox

		_, _ = authSvc.pfmCtrlClient.SandboxControlApi.DeleteSandbox(context.TODO(), session.Sandbox)

		_, err = authSvc.pfmCtrlClient.SandboxControlApi.DeleteSandbox(context.TODO(), session.Sandbox)

		if err == nil {

			sandboxDeleted = true

		}

	}



	// Delete session
@@ -934,6 +1011,10 @@ func asLogout(w http.ResponseWriter, r *http.Request) { 
	}



	_ = authSvc.metricStore.SetSessionMetric(ms.SesMetTypeLogout, metric)

	if sandboxDeleted {

		metricSessionActive.Dec()

		metricSessionDuration.Observe(time.Since(session.StartTime).Minutes())

	}



	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	w.WriteHeader(http.StatusOK)

go-apps/meep-mon-engine/go.sum

+1 −0
Original line number Diff line number Diff line
@@ -83,6 +83,7 @@ k8s.io/apimachinery v0.0.0-20181127025237-2b1284ed4c93 h1:tT6oQBi0qwLbbZSfDkdIsb 
k8s.io/apimachinery v0.0.0-20181127025237-2b1284ed4c93/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0=

k8s.io/client-go v10.0.0+incompatible h1:F1IqCqw7oMBzDkqlcBymRq1450wD0eNqLE9jzUrIi34=

k8s.io/client-go v10.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s=

k8s.io/client-go v11.0.0+incompatible h1:LBbX2+lOwY9flffWlJM7f1Ct8V2SRNiMRDFeiwnJo9o=

k8s.io/klog v0.0.0-20181108234604-8139d8cb77af h1:s6rm8OxBbyDNSRkpyAd5OL4icUdBICVw9+mFADa+t5E=

k8s.io/klog v0.0.0-20181108234604-8139d8cb77af/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=

sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=

go-packages/meep-sessions/session-store.go

+12 −0
Original line number Diff line number Diff line
@@ -44,6 +44,7 @@ const ( 
	ValSandbox   = "sbox"

	ValRole      = "role"

	ValTimestamp = "timestamp"

	ValStartTime = "starttime"

)



const (
@@ -59,6 +60,7 @@ type Session struct { 
	Sandbox   string

	Role      string

	Timestamp time.Time

	StartTime time.Time

}



type SessionStore struct {
@@ -137,6 +139,7 @@ func (ss *SessionStore) Get(r *http.Request) (s *Session, err error) { 
	s.Sandbox = session[ValSandbox]

	s.Role = session[ValRole]

	s.Timestamp, _ = time.Parse(time.RFC3339, session[ValTimestamp])

	s.StartTime, _ = time.Parse(time.RFC3339, session[ValStartTime])

	return s, nil

}
@@ -173,6 +176,7 @@ func getSessionEntryHandler(key string, fields map[string]string, userData inter 
	s.Sandbox = fields[ValSandbox]

	s.Role = fields[ValRole]

	s.Timestamp, _ = time.Parse(time.RFC3339, fields[ValTimestamp])

	s.StartTime, _ = time.Parse(time.RFC3339, fields[ValStartTime])

	*sessionList = append(*sessionList, s)

	return nil

}
@@ -209,6 +213,7 @@ func getUserEntryHandler(key string, fields map[string]string, userData interfac 
		s.Sandbox = fields[ValSandbox]

		s.Role = fields[ValRole]

		s.Timestamp, _ = time.Parse(time.RFC3339, fields[ValTimestamp])

		s.StartTime, _ = time.Parse(time.RFC3339, fields[ValStartTime])

	}

	return nil

}
@@ -226,6 +231,12 @@ func (ss *SessionStore) Set(s *Session, w http.ResponseWriter, r *http.Request) 
		}

	}



	// Set session start time on initial request

	sessionStartTime := s.StartTime

	if sessionStartTime.IsZero() {

		sessionStartTime = time.Now()

	}



	// Update existing session or create new one if not found

	sessionId := s.ID

	if s.ID == "" {
@@ -238,6 +249,7 @@ func (ss *SessionStore) Set(s *Session, w http.ResponseWriter, r *http.Request) 
	fields[ValSandbox] = s.Sandbox

	fields[ValRole] = s.Role

	fields[ValTimestamp] = time.Now().Format(time.RFC3339)

	fields[ValStartTime] = sessionStartTime.Format(time.RFC3339)

	err = ss.rc.SetEntry(ss.baseKey+sessionId, fields)

	if err != nil {

		return err, http.StatusInternalServerError