mirror of
https://github.com/optim-enterprises-bv/vault.git
synced 2025-10-30 18:17:55 +00:00
VAULT-22642: Include secret syncs in activity log responses (#24710)
* refactor and include secret sync * add secret sync tests * changelog * include secret syncs in clients * pr comments * add godocs
This commit is contained in:
3
changelog/24710.txt
Normal file
3
changelog/24710.txt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
```release-note:improvement
|
||||||
|
core/activity: Include secret_syncs in activity log responses
|
||||||
|
```
|
||||||
@@ -32,6 +32,11 @@ type CountsRecord struct {
|
|||||||
SecretSyncs int `json:"secret_syncs"`
|
SecretSyncs int `json:"secret_syncs"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// HasCounts returns true when any of the record's fields have a non-zero value
|
||||||
|
func (c *CountsRecord) HasCounts() bool {
|
||||||
|
return c.EntityClients+c.NonEntityClients+c.SecretSyncs != 0
|
||||||
|
}
|
||||||
|
|
||||||
type NewClientRecord struct {
|
type NewClientRecord struct {
|
||||||
Counts *CountsRecord `json:"counts"`
|
Counts *CountsRecord `json:"counts"`
|
||||||
Namespaces []*MonthlyNamespaceRecord `json:"namespaces"`
|
Namespaces []*MonthlyNamespaceRecord `json:"namespaces"`
|
||||||
|
|||||||
@@ -1578,6 +1578,20 @@ type ResponseCounts struct {
|
|||||||
NonEntityTokens int `json:"non_entity_tokens" mapstructure:"non_entity_tokens"`
|
NonEntityTokens int `json:"non_entity_tokens" mapstructure:"non_entity_tokens"`
|
||||||
NonEntityClients int `json:"non_entity_clients" mapstructure:"non_entity_clients"`
|
NonEntityClients int `json:"non_entity_clients" mapstructure:"non_entity_clients"`
|
||||||
Clients int `json:"clients"`
|
Clients int `json:"clients"`
|
||||||
|
SecretSyncs int `json:"secret_syncs" mapstructure:"secret_syncs"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add adds the new record's counts to the existing record
|
||||||
|
func (r *ResponseCounts) Add(newRecord *ResponseCounts) {
|
||||||
|
if newRecord == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
r.EntityClients += newRecord.EntityClients
|
||||||
|
r.Clients += newRecord.Clients
|
||||||
|
r.DistinctEntities += newRecord.DistinctEntities
|
||||||
|
r.NonEntityClients += newRecord.NonEntityClients
|
||||||
|
r.NonEntityTokens += newRecord.NonEntityTokens
|
||||||
|
r.SecretSyncs += newRecord.SecretSyncs
|
||||||
}
|
}
|
||||||
|
|
||||||
type ResponseNamespace struct {
|
type ResponseNamespace struct {
|
||||||
@@ -1587,6 +1601,30 @@ type ResponseNamespace struct {
|
|||||||
Mounts []*ResponseMount `json:"mounts"`
|
Mounts []*ResponseMount `json:"mounts"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Add adds the namespace counts to the existing record, then either adds the
|
||||||
|
// mount counts to the existing mount (if it exists) or appends the mount to the
|
||||||
|
// list of mounts
|
||||||
|
func (r *ResponseNamespace) Add(newRecord *ResponseNamespace) {
|
||||||
|
// Create a map of the existing mounts, so we don't duplicate them
|
||||||
|
mountMap := make(map[string]*ResponseCounts)
|
||||||
|
for _, erm := range r.Mounts {
|
||||||
|
mountMap[erm.MountPath] = erm.Counts
|
||||||
|
}
|
||||||
|
|
||||||
|
r.Counts.Add(&newRecord.Counts)
|
||||||
|
|
||||||
|
// Check the current month mounts against the existing mounts and if there are matches, update counts
|
||||||
|
// accordingly. If there is no match, append the new mount to the existing mounts, so it will be counted
|
||||||
|
// later.
|
||||||
|
for _, newRecordMount := range newRecord.Mounts {
|
||||||
|
if existingRecordMountCounts, ok := mountMap[newRecordMount.MountPath]; ok {
|
||||||
|
existingRecordMountCounts.Add(&newRecord.Counts)
|
||||||
|
} else {
|
||||||
|
r.Mounts = append(r.Mounts, newRecordMount)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type ResponseMonth struct {
|
type ResponseMonth struct {
|
||||||
Timestamp string `json:"timestamp"`
|
Timestamp string `json:"timestamp"`
|
||||||
Counts *ResponseCounts `json:"counts"`
|
Counts *ResponseCounts `json:"counts"`
|
||||||
@@ -1681,7 +1719,7 @@ func (a *ActivityLog) handleQuery(ctx context.Context, startTime, endTime time.T
|
|||||||
|
|
||||||
// Calculate the namespace response breakdowns and totals for entities and tokens from the initial
|
// Calculate the namespace response breakdowns and totals for entities and tokens from the initial
|
||||||
// namespace data.
|
// namespace data.
|
||||||
totalEntities, totalTokens, byNamespaceResponse, err := a.calculateByNamespaceResponseForQuery(ctx, pq.Namespaces)
|
totalCounts, byNamespaceResponse, err := a.calculateByNamespaceResponseForQuery(ctx, pq.Namespaces)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -1690,9 +1728,8 @@ func (a *ActivityLog) handleQuery(ctx context.Context, startTime, endTime time.T
|
|||||||
// breakdown for the current month as well.
|
// breakdown for the current month as well.
|
||||||
var partialByMonth map[int64]*processMonth
|
var partialByMonth map[int64]*processMonth
|
||||||
var partialByNamespace map[string]*processByNamespace
|
var partialByNamespace map[string]*processByNamespace
|
||||||
var totalEntitiesCurrent int
|
|
||||||
var totalTokensCurrent int
|
|
||||||
var byNamespaceResponseCurrent []*ResponseNamespace
|
var byNamespaceResponseCurrent []*ResponseNamespace
|
||||||
|
var totalCurrentCounts *ResponseCounts
|
||||||
if computePartial {
|
if computePartial {
|
||||||
// Traverse through current month's activitylog data and group clients
|
// Traverse through current month's activitylog data and group clients
|
||||||
// into months and namespaces
|
// into months and namespaces
|
||||||
@@ -1705,9 +1742,9 @@ func (a *ActivityLog) handleQuery(ctx context.Context, startTime, endTime time.T
|
|||||||
// endpoints.
|
// endpoints.
|
||||||
byNamespaceComputation := a.transformALNamespaceBreakdowns(partialByNamespace)
|
byNamespaceComputation := a.transformALNamespaceBreakdowns(partialByNamespace)
|
||||||
|
|
||||||
// Calculate the namespace response breakdowns and totals for entities and tokens from the initial
|
// Calculate the namespace response breakdowns and totals for entities
|
||||||
// namespace data.
|
// and tokens from current month namespace data.
|
||||||
totalEntitiesCurrent, totalTokensCurrent, byNamespaceResponseCurrent, err = a.calculateByNamespaceResponseForQuery(ctx, byNamespaceComputation)
|
totalCurrentCounts, byNamespaceResponseCurrent, err = a.calculateByNamespaceResponseForQuery(ctx, byNamespaceComputation)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -1723,34 +1760,7 @@ func (a *ActivityLog) handleQuery(ctx context.Context, startTime, endTime time.T
|
|||||||
// month counts, and append or update as necessary. We also want to account for mounts and their counts.
|
// month counts, and append or update as necessary. We also want to account for mounts and their counts.
|
||||||
for _, nrc := range byNamespaceResponseCurrent {
|
for _, nrc := range byNamespaceResponseCurrent {
|
||||||
if ndx, ok := nsrMap[nrc.NamespaceID]; ok {
|
if ndx, ok := nsrMap[nrc.NamespaceID]; ok {
|
||||||
existingRecord := byNamespaceResponse[ndx]
|
byNamespaceResponse[ndx].Add(nrc)
|
||||||
|
|
||||||
// Create a map of the existing mounts, so we don't duplicate them
|
|
||||||
mountMap := make(map[string]*ResponseCounts)
|
|
||||||
for _, erm := range existingRecord.Mounts {
|
|
||||||
mountMap[erm.MountPath] = erm.Counts
|
|
||||||
}
|
|
||||||
|
|
||||||
existingRecord.Counts.EntityClients += nrc.Counts.EntityClients
|
|
||||||
existingRecord.Counts.Clients += nrc.Counts.Clients
|
|
||||||
existingRecord.Counts.DistinctEntities += nrc.Counts.DistinctEntities
|
|
||||||
existingRecord.Counts.NonEntityClients += nrc.Counts.NonEntityClients
|
|
||||||
existingRecord.Counts.NonEntityTokens += nrc.Counts.NonEntityTokens
|
|
||||||
|
|
||||||
// Check the current month mounts against the existing mounts and if there are matches, update counts
|
|
||||||
// accordingly. If there is no match, append the new mount to the existing mounts, so it will be counted
|
|
||||||
// later.
|
|
||||||
for _, nrcMount := range nrc.Mounts {
|
|
||||||
if existingRecordMountCounts, ook := mountMap[nrcMount.MountPath]; ook {
|
|
||||||
existingRecordMountCounts.EntityClients += nrcMount.Counts.EntityClients
|
|
||||||
existingRecordMountCounts.Clients += nrcMount.Counts.Clients
|
|
||||||
existingRecordMountCounts.DistinctEntities += nrcMount.Counts.DistinctEntities
|
|
||||||
existingRecordMountCounts.NonEntityClients += nrcMount.Counts.NonEntityClients
|
|
||||||
existingRecordMountCounts.NonEntityTokens += nrcMount.Counts.NonEntityTokens
|
|
||||||
} else {
|
|
||||||
existingRecord.Mounts = append(existingRecord.Mounts, nrcMount)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
byNamespaceResponse = append(byNamespaceResponse, nrc)
|
byNamespaceResponse = append(byNamespaceResponse, nrc)
|
||||||
}
|
}
|
||||||
@@ -1761,10 +1771,10 @@ func (a *ActivityLog) handleQuery(ctx context.Context, startTime, endTime time.T
|
|||||||
a.sortALResponseNamespaces(byNamespaceResponse)
|
a.sortALResponseNamespaces(byNamespaceResponse)
|
||||||
|
|
||||||
if limitNamespaces > 0 {
|
if limitNamespaces > 0 {
|
||||||
totalEntities, totalTokens, byNamespaceResponse = a.limitNamespacesInALResponse(byNamespaceResponse, limitNamespaces)
|
totalCounts, byNamespaceResponse = a.limitNamespacesInALResponse(byNamespaceResponse, limitNamespaces)
|
||||||
}
|
}
|
||||||
|
|
||||||
distinctEntitiesResponse := totalEntities
|
distinctEntitiesResponse := totalCounts.EntityClients
|
||||||
if computePartial {
|
if computePartial {
|
||||||
currentMonth, err := a.computeCurrentMonthForBillingPeriod(ctx, partialByMonth, startTime, endTime)
|
currentMonth, err := a.computeCurrentMonthForBillingPeriod(ctx, partialByMonth, startTime, endTime)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1808,13 +1818,9 @@ func (a *ActivityLog) handleQuery(ctx context.Context, startTime, endTime time.T
|
|||||||
}
|
}
|
||||||
|
|
||||||
responseData["by_namespace"] = byNamespaceResponse
|
responseData["by_namespace"] = byNamespaceResponse
|
||||||
responseData["total"] = &ResponseCounts{
|
totalCounts.Add(totalCurrentCounts)
|
||||||
DistinctEntities: distinctEntitiesResponse,
|
totalCounts.DistinctEntities = distinctEntitiesResponse
|
||||||
EntityClients: totalEntities + totalEntitiesCurrent,
|
responseData["total"] = totalCounts
|
||||||
NonEntityTokens: totalTokens + totalTokensCurrent,
|
|
||||||
NonEntityClients: totalTokens + totalTokensCurrent,
|
|
||||||
Clients: totalEntities + totalEntitiesCurrent + totalTokens + totalTokensCurrent,
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create and populate the month response structs based on the monthly breakdown.
|
// Create and populate the month response structs based on the monthly breakdown.
|
||||||
months, err := a.prepareMonthsResponseForQuery(ctx, pq.Months)
|
months, err := a.prepareMonthsResponseForQuery(ctx, pq.Months)
|
||||||
@@ -2610,32 +2616,25 @@ func (a *ActivityLog) transformMonthBreakdowns(byMonth map[int64]*processMonth)
|
|||||||
return monthly
|
return monthly
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *ActivityLog) calculateByNamespaceResponseForQuery(ctx context.Context, byNamespace []*activity.NamespaceRecord) (int, int, []*ResponseNamespace, error) {
|
func (a *ActivityLog) calculateByNamespaceResponseForQuery(ctx context.Context, byNamespace []*activity.NamespaceRecord) (*ResponseCounts, []*ResponseNamespace, error) {
|
||||||
queryNS, err := namespace.FromContext(ctx)
|
queryNS, err := namespace.FromContext(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, 0, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
byNamespaceResponse := make([]*ResponseNamespace, 0)
|
byNamespaceResponse := make([]*ResponseNamespace, 0)
|
||||||
totalEntities := 0
|
totalCounts := &ResponseCounts{}
|
||||||
totalTokens := 0
|
|
||||||
|
|
||||||
for _, nsRecord := range byNamespace {
|
for _, nsRecord := range byNamespace {
|
||||||
ns, err := NamespaceByID(ctx, nsRecord.NamespaceID, a.core)
|
ns, err := NamespaceByID(ctx, nsRecord.NamespaceID, a.core)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, 0, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
if a.includeInResponse(queryNS, ns) {
|
if a.includeInResponse(queryNS, ns) {
|
||||||
mountResponse := make([]*ResponseMount, 0, len(nsRecord.Mounts))
|
mountResponse := make([]*ResponseMount, 0, len(nsRecord.Mounts))
|
||||||
for _, mountRecord := range nsRecord.Mounts {
|
for _, mountRecord := range nsRecord.Mounts {
|
||||||
mountResponse = append(mountResponse, &ResponseMount{
|
mountResponse = append(mountResponse, &ResponseMount{
|
||||||
MountPath: mountRecord.MountPath,
|
MountPath: mountRecord.MountPath,
|
||||||
Counts: &ResponseCounts{
|
Counts: a.countsRecordToCountsResponse(mountRecord.Counts, true),
|
||||||
DistinctEntities: int(mountRecord.Counts.EntityClients),
|
|
||||||
EntityClients: int(mountRecord.Counts.EntityClients),
|
|
||||||
NonEntityClients: int(mountRecord.Counts.NonEntityClients),
|
|
||||||
NonEntityTokens: int(mountRecord.Counts.NonEntityClients),
|
|
||||||
Clients: int(mountRecord.Counts.EntityClients + mountRecord.Counts.NonEntityClients),
|
|
||||||
},
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
// Sort the mounts in descending order of usage
|
// Sort the mounts in descending order of usage
|
||||||
@@ -2649,55 +2648,41 @@ func (a *ActivityLog) calculateByNamespaceResponseForQuery(ctx context.Context,
|
|||||||
} else {
|
} else {
|
||||||
displayPath = ns.Path
|
displayPath = ns.Path
|
||||||
}
|
}
|
||||||
|
nsCounts := a.namespaceRecordToCountsResponse(nsRecord)
|
||||||
byNamespaceResponse = append(byNamespaceResponse, &ResponseNamespace{
|
byNamespaceResponse = append(byNamespaceResponse, &ResponseNamespace{
|
||||||
NamespaceID: nsRecord.NamespaceID,
|
NamespaceID: nsRecord.NamespaceID,
|
||||||
NamespacePath: displayPath,
|
NamespacePath: displayPath,
|
||||||
Counts: ResponseCounts{
|
Counts: *nsCounts,
|
||||||
DistinctEntities: int(nsRecord.Entities),
|
Mounts: mountResponse,
|
||||||
EntityClients: int(nsRecord.Entities),
|
|
||||||
NonEntityTokens: int(nsRecord.NonEntityTokens),
|
|
||||||
NonEntityClients: int(nsRecord.NonEntityTokens),
|
|
||||||
Clients: int(nsRecord.Entities + nsRecord.NonEntityTokens),
|
|
||||||
},
|
|
||||||
Mounts: mountResponse,
|
|
||||||
})
|
})
|
||||||
totalEntities += int(nsRecord.Entities)
|
totalCounts.Add(nsCounts)
|
||||||
totalTokens += int(nsRecord.NonEntityTokens)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return totalEntities, totalTokens, byNamespaceResponse, nil
|
return totalCounts, byNamespaceResponse, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *ActivityLog) prepareMonthsResponseForQuery(ctx context.Context, byMonth []*activity.MonthRecord) ([]*ResponseMonth, error) {
|
func (a *ActivityLog) prepareMonthsResponseForQuery(ctx context.Context, byMonth []*activity.MonthRecord) ([]*ResponseMonth, error) {
|
||||||
months := make([]*ResponseMonth, 0, len(byMonth))
|
months := make([]*ResponseMonth, 0, len(byMonth))
|
||||||
for _, monthsRecord := range byMonth {
|
for _, monthsRecord := range byMonth {
|
||||||
newClientsResponse := &ResponseNewClients{}
|
newClientsResponse := &ResponseNewClients{}
|
||||||
if int(monthsRecord.NewClients.Counts.EntityClients+monthsRecord.NewClients.Counts.NonEntityClients) != 0 {
|
if monthsRecord.NewClients.Counts.HasCounts() {
|
||||||
newClientsNSResponse, err := a.prepareNamespaceResponse(ctx, monthsRecord.NewClients.Namespaces)
|
newClientsNSResponse, err := a.prepareNamespaceResponse(ctx, monthsRecord.NewClients.Namespaces)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
newClientsResponse.Counts = &ResponseCounts{
|
newClientsResponse.Counts = a.countsRecordToCountsResponse(monthsRecord.NewClients.Counts, false)
|
||||||
EntityClients: int(monthsRecord.NewClients.Counts.EntityClients),
|
|
||||||
NonEntityClients: int(monthsRecord.NewClients.Counts.NonEntityClients),
|
|
||||||
Clients: int(monthsRecord.NewClients.Counts.EntityClients + monthsRecord.NewClients.Counts.NonEntityClients),
|
|
||||||
}
|
|
||||||
newClientsResponse.Namespaces = newClientsNSResponse
|
newClientsResponse.Namespaces = newClientsNSResponse
|
||||||
}
|
}
|
||||||
|
|
||||||
monthResponse := &ResponseMonth{
|
monthResponse := &ResponseMonth{
|
||||||
Timestamp: time.Unix(monthsRecord.Timestamp, 0).UTC().Format(time.RFC3339),
|
Timestamp: time.Unix(monthsRecord.Timestamp, 0).UTC().Format(time.RFC3339),
|
||||||
}
|
}
|
||||||
if int(monthsRecord.Counts.EntityClients+monthsRecord.Counts.NonEntityClients) != 0 {
|
if monthsRecord.Counts.HasCounts() {
|
||||||
nsResponse, err := a.prepareNamespaceResponse(ctx, monthsRecord.Namespaces)
|
nsResponse, err := a.prepareNamespaceResponse(ctx, monthsRecord.Namespaces)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
monthResponse.Counts = &ResponseCounts{
|
monthResponse.Counts = a.countsRecordToCountsResponse(monthsRecord.Counts, false)
|
||||||
EntityClients: int(monthsRecord.Counts.EntityClients),
|
|
||||||
NonEntityClients: int(monthsRecord.Counts.NonEntityClients),
|
|
||||||
Clients: int(monthsRecord.Counts.EntityClients + monthsRecord.Counts.NonEntityClients),
|
|
||||||
}
|
|
||||||
monthResponse.Namespaces = nsResponse
|
monthResponse.Namespaces = nsResponse
|
||||||
monthResponse.NewClients = newClientsResponse
|
monthResponse.NewClients = newClientsResponse
|
||||||
months = append(months, monthResponse)
|
months = append(months, monthResponse)
|
||||||
@@ -2715,7 +2700,7 @@ func (a *ActivityLog) prepareNamespaceResponse(ctx context.Context, nsRecords []
|
|||||||
}
|
}
|
||||||
nsResponse := make([]*ResponseNamespace, 0, len(nsRecords))
|
nsResponse := make([]*ResponseNamespace, 0, len(nsRecords))
|
||||||
for _, nsRecord := range nsRecords {
|
for _, nsRecord := range nsRecords {
|
||||||
if int(nsRecord.Counts.EntityClients) == 0 && int(nsRecord.Counts.NonEntityClients) == 0 {
|
if !nsRecord.Counts.HasCounts() {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2726,17 +2711,13 @@ func (a *ActivityLog) prepareNamespaceResponse(ctx context.Context, nsRecords []
|
|||||||
if a.includeInResponse(queryNS, ns) {
|
if a.includeInResponse(queryNS, ns) {
|
||||||
mountResponse := make([]*ResponseMount, 0, len(nsRecord.Mounts))
|
mountResponse := make([]*ResponseMount, 0, len(nsRecord.Mounts))
|
||||||
for _, mountRecord := range nsRecord.Mounts {
|
for _, mountRecord := range nsRecord.Mounts {
|
||||||
if int(mountRecord.Counts.EntityClients) == 0 && int(mountRecord.Counts.NonEntityClients) == 0 {
|
if !mountRecord.Counts.HasCounts() {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
mountResponse = append(mountResponse, &ResponseMount{
|
mountResponse = append(mountResponse, &ResponseMount{
|
||||||
MountPath: mountRecord.MountPath,
|
MountPath: mountRecord.MountPath,
|
||||||
Counts: &ResponseCounts{
|
Counts: a.countsRecordToCountsResponse(mountRecord.Counts, false),
|
||||||
EntityClients: int(mountRecord.Counts.EntityClients),
|
|
||||||
NonEntityClients: int(mountRecord.Counts.NonEntityClients),
|
|
||||||
Clients: int(mountRecord.Counts.EntityClients + mountRecord.Counts.NonEntityClients),
|
|
||||||
},
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2749,12 +2730,8 @@ func (a *ActivityLog) prepareNamespaceResponse(ctx context.Context, nsRecords []
|
|||||||
nsResponse = append(nsResponse, &ResponseNamespace{
|
nsResponse = append(nsResponse, &ResponseNamespace{
|
||||||
NamespaceID: nsRecord.NamespaceID,
|
NamespaceID: nsRecord.NamespaceID,
|
||||||
NamespacePath: displayPath,
|
NamespacePath: displayPath,
|
||||||
Counts: ResponseCounts{
|
Counts: *a.countsRecordToCountsResponse(nsRecord.Counts, false),
|
||||||
EntityClients: int(nsRecord.Counts.EntityClients),
|
Mounts: mountResponse,
|
||||||
NonEntityClients: int(nsRecord.Counts.NonEntityClients),
|
|
||||||
Clients: int(nsRecord.Counts.EntityClients + nsRecord.Counts.NonEntityClients),
|
|
||||||
},
|
|
||||||
Mounts: mountResponse,
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2783,7 +2760,7 @@ func (a *ActivityLog) partialMonthClientCount(ctx context.Context) (map[string]i
|
|||||||
|
|
||||||
// Calculate the namespace response breakdowns and totals for entities and tokens from the initial
|
// Calculate the namespace response breakdowns and totals for entities and tokens from the initial
|
||||||
// namespace data.
|
// namespace data.
|
||||||
totalEntities, totalTokens, byNamespaceResponse, err := a.calculateByNamespaceResponseForQuery(ctx, byNamespaceComputation)
|
totalCounts, byNamespaceResponse, err := a.calculateByNamespaceResponseForQuery(ctx, byNamespaceComputation)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -2794,11 +2771,12 @@ func (a *ActivityLog) partialMonthClientCount(ctx context.Context) (map[string]i
|
|||||||
// Now populate the response based on breakdowns.
|
// Now populate the response based on breakdowns.
|
||||||
responseData := make(map[string]interface{})
|
responseData := make(map[string]interface{})
|
||||||
responseData["by_namespace"] = byNamespaceResponse
|
responseData["by_namespace"] = byNamespaceResponse
|
||||||
responseData["distinct_entities"] = totalEntities
|
responseData["distinct_entities"] = totalCounts.EntityClients
|
||||||
responseData["entity_clients"] = totalEntities
|
responseData["entity_clients"] = totalCounts.EntityClients
|
||||||
responseData["non_entity_tokens"] = totalTokens
|
responseData["non_entity_tokens"] = totalCounts.NonEntityClients
|
||||||
responseData["non_entity_clients"] = totalTokens
|
responseData["non_entity_clients"] = totalCounts.NonEntityClients
|
||||||
responseData["clients"] = totalEntities + totalTokens
|
responseData["clients"] = totalCounts.Clients
|
||||||
|
responseData["secret_syncs"] = totalCounts.SecretSyncs
|
||||||
|
|
||||||
// The partialMonthClientCount should not have more than one month worth of data.
|
// The partialMonthClientCount should not have more than one month worth of data.
|
||||||
// If it does, something has gone wrong and we should warn that the activity log data
|
// If it does, something has gone wrong and we should warn that the activity log data
|
||||||
|
|||||||
@@ -197,19 +197,17 @@ func (a *ActivityLog) transformALNamespaceBreakdowns(nsData map[string]*processB
|
|||||||
|
|
||||||
// limitNamespacesInALResponse will truncate the number of namespaces shown in the activity
|
// limitNamespacesInALResponse will truncate the number of namespaces shown in the activity
|
||||||
// endpoints to the number specified in limitNamespaces (the API filtering parameter)
|
// endpoints to the number specified in limitNamespaces (the API filtering parameter)
|
||||||
func (a *ActivityLog) limitNamespacesInALResponse(byNamespaceResponse []*ResponseNamespace, limitNamespaces int) (int, int, []*ResponseNamespace) {
|
func (a *ActivityLog) limitNamespacesInALResponse(byNamespaceResponse []*ResponseNamespace, limitNamespaces int) (*ResponseCounts, []*ResponseNamespace) {
|
||||||
if limitNamespaces > len(byNamespaceResponse) {
|
if limitNamespaces > len(byNamespaceResponse) {
|
||||||
limitNamespaces = len(byNamespaceResponse)
|
limitNamespaces = len(byNamespaceResponse)
|
||||||
}
|
}
|
||||||
byNamespaceResponse = byNamespaceResponse[:limitNamespaces]
|
byNamespaceResponse = byNamespaceResponse[:limitNamespaces]
|
||||||
// recalculate total entities and tokens
|
// recalculate total entities and tokens
|
||||||
totalEntities := 0
|
totalCounts := &ResponseCounts{}
|
||||||
totalTokens := 0
|
|
||||||
for _, namespaceData := range byNamespaceResponse {
|
for _, namespaceData := range byNamespaceResponse {
|
||||||
totalEntities += namespaceData.Counts.DistinctEntities
|
totalCounts.Add(&namespaceData.Counts)
|
||||||
totalTokens += namespaceData.Counts.NonEntityTokens
|
|
||||||
}
|
}
|
||||||
return totalEntities, totalTokens, byNamespaceResponse
|
return totalCounts, byNamespaceResponse
|
||||||
}
|
}
|
||||||
|
|
||||||
// transformActivityLogMounts is a helper used to reformat data for transformMonthlyNamespaceBreakdowns.
|
// transformActivityLogMounts is a helper used to reformat data for transformMonthlyNamespaceBreakdowns.
|
||||||
@@ -383,3 +381,35 @@ func (e *segmentReader) ReadEntity(ctx context.Context) (*activity.EntityActivit
|
|||||||
}
|
}
|
||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// namespaceRecordToCountsResponse converts the record to the ResponseCounts
|
||||||
|
// type. The function sums entity, non-entity, and secret sync counts to get the
|
||||||
|
// total client count. If includeDeprecated is true, the deprecated fields
|
||||||
|
// NonEntityTokens and DistinctEntities are populated
|
||||||
|
func (a *ActivityLog) countsRecordToCountsResponse(record *activity.CountsRecord, includeDeprecated bool) *ResponseCounts {
|
||||||
|
response := &ResponseCounts{
|
||||||
|
EntityClients: record.EntityClients,
|
||||||
|
NonEntityClients: record.NonEntityClients,
|
||||||
|
Clients: record.EntityClients + record.NonEntityClients + record.SecretSyncs,
|
||||||
|
SecretSyncs: record.SecretSyncs,
|
||||||
|
}
|
||||||
|
if includeDeprecated {
|
||||||
|
response.NonEntityTokens = response.NonEntityClients
|
||||||
|
response.DistinctEntities = response.EntityClients
|
||||||
|
}
|
||||||
|
return response
|
||||||
|
}
|
||||||
|
|
||||||
|
// namespaceRecordToCountsResponse converts the namespace counts to the
|
||||||
|
// ResponseCounts type. The function sums entity, non-entity, and secret sync
|
||||||
|
// counts to get the total client count.
|
||||||
|
func (a *ActivityLog) namespaceRecordToCountsResponse(record *activity.NamespaceRecord) *ResponseCounts {
|
||||||
|
return &ResponseCounts{
|
||||||
|
DistinctEntities: int(record.Entities),
|
||||||
|
EntityClients: int(record.Entities),
|
||||||
|
NonEntityTokens: int(record.NonEntityTokens),
|
||||||
|
NonEntityClients: int(record.NonEntityTokens),
|
||||||
|
Clients: int(record.Entities + record.NonEntityTokens + record.SecretSyncs),
|
||||||
|
SecretSyncs: int(record.SecretSyncs),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ package activity_testonly
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -237,3 +238,147 @@ func getMonthsData(t *testing.T, resp *api.Secret) []vault.ResponseMonth {
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
return monthsResponse
|
return monthsResponse
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func getNamespaceData(t *testing.T, resp *api.Secret) []vault.ResponseNamespace {
|
||||||
|
t.Helper()
|
||||||
|
nsRaw, ok := resp.Data["by_namespace"]
|
||||||
|
require.True(t, ok)
|
||||||
|
nsResponse := make([]vault.ResponseNamespace, 0)
|
||||||
|
err := mapstructure.Decode(nsRaw, &nsResponse)
|
||||||
|
require.NoError(t, err)
|
||||||
|
return nsResponse
|
||||||
|
}
|
||||||
|
|
||||||
|
func getTotals(t *testing.T, resp *api.Secret) vault.ResponseCounts {
|
||||||
|
t.Helper()
|
||||||
|
totalRaw, ok := resp.Data["total"]
|
||||||
|
require.True(t, ok)
|
||||||
|
total := vault.ResponseCounts{}
|
||||||
|
err := mapstructure.Decode(totalRaw, &total)
|
||||||
|
require.NoError(t, err)
|
||||||
|
return total
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_ActivityLog_SecretSyncResponse creates 10 secret sync clients and
|
||||||
|
// verifies that the activity log query response returns 10 secret sync clients
|
||||||
|
// at every level of the response hierarchy
|
||||||
|
func Test_ActivityLog_SecretSyncResponse(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
cluster := minimal.NewTestSoloCluster(t, nil)
|
||||||
|
client := cluster.Cores[0].Client
|
||||||
|
_, err := client.Logical().Write("sys/internal/counters/config", map[string]interface{}{
|
||||||
|
"enabled": "enable",
|
||||||
|
})
|
||||||
|
_, err = clientcountutil.NewActivityLogData(client).
|
||||||
|
NewCurrentMonthData().
|
||||||
|
NewClientsSeen(10, clientcountutil.WithClientType("secret-sync")).
|
||||||
|
Write(context.Background(), generation.WriteOptions_WRITE_ENTITIES)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
now := time.Now().UTC()
|
||||||
|
resp, err := client.Logical().ReadWithData("sys/internal/counters/activity", map[string][]string{
|
||||||
|
"end_time": {timeutil.EndOfMonth(now).Format(time.RFC3339)},
|
||||||
|
"start_time": {timeutil.StartOfMonth(now).Format(time.RFC3339)},
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
total := getTotals(t, resp)
|
||||||
|
require.Equal(t, 10, total.SecretSyncs)
|
||||||
|
require.Equal(t, 10, total.Clients)
|
||||||
|
|
||||||
|
byNamespace := getNamespaceData(t, resp)
|
||||||
|
require.Equal(t, 10, byNamespace[0].Counts.SecretSyncs)
|
||||||
|
require.Equal(t, 10, byNamespace[0].Mounts[0].Counts.SecretSyncs)
|
||||||
|
require.Equal(t, 10, byNamespace[0].Counts.Clients)
|
||||||
|
require.Equal(t, 10, byNamespace[0].Mounts[0].Counts.Clients)
|
||||||
|
|
||||||
|
byMonth := getMonthsData(t, resp)
|
||||||
|
require.Equal(t, 10, byMonth[0].NewClients.Counts.SecretSyncs)
|
||||||
|
require.Equal(t, 10, byMonth[0].Counts.SecretSyncs)
|
||||||
|
require.Equal(t, 10, byMonth[0].Namespaces[0].Counts.SecretSyncs)
|
||||||
|
require.Equal(t, 10, byMonth[0].Namespaces[0].Mounts[0].Counts.SecretSyncs)
|
||||||
|
require.Equal(t, 10, byMonth[0].NewClients.Counts.Clients)
|
||||||
|
require.Equal(t, 10, byMonth[0].Counts.Clients)
|
||||||
|
require.Equal(t, 10, byMonth[0].Namespaces[0].Counts.Clients)
|
||||||
|
require.Equal(t, 10, byMonth[0].Namespaces[0].Mounts[0].Counts.Clients)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_ActivityLogCurrentMonth_SecretSyncResponse creates 10 secret sync
|
||||||
|
// clients and verifies that the activity log partial month response returns
|
||||||
|
// 10 secret sync clients at every level of the response hierarchy
|
||||||
|
func Test_ActivityLogCurrentMonth_SecretSyncResponse(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
cluster := minimal.NewTestSoloCluster(t, nil)
|
||||||
|
client := cluster.Cores[0].Client
|
||||||
|
_, err := client.Logical().Write("sys/internal/counters/config", map[string]interface{}{
|
||||||
|
"enabled": "enable",
|
||||||
|
})
|
||||||
|
_, err = clientcountutil.NewActivityLogData(client).
|
||||||
|
NewCurrentMonthData().
|
||||||
|
NewClientsSeen(10, clientcountutil.WithClientType("secret-sync")).
|
||||||
|
Write(context.Background(), generation.WriteOptions_WRITE_ENTITIES)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
resp, err := client.Logical().Read("sys/internal/counters/activity/monthly")
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
secretSyncs, ok := resp.Data["secret_syncs"]
|
||||||
|
require.True(t, ok)
|
||||||
|
require.Equal(t, json.Number("10"), secretSyncs)
|
||||||
|
clients, ok := resp.Data["clients"]
|
||||||
|
require.True(t, ok)
|
||||||
|
require.Equal(t, json.Number("10"), clients)
|
||||||
|
|
||||||
|
byNamespace := getNamespaceData(t, resp)
|
||||||
|
require.Equal(t, 10, byNamespace[0].Counts.SecretSyncs)
|
||||||
|
require.Equal(t, 10, byNamespace[0].Mounts[0].Counts.SecretSyncs)
|
||||||
|
require.Equal(t, 10, byNamespace[0].Counts.Clients)
|
||||||
|
require.Equal(t, 10, byNamespace[0].Mounts[0].Counts.Clients)
|
||||||
|
|
||||||
|
byMonth := getMonthsData(t, resp)
|
||||||
|
require.Equal(t, 10, byMonth[0].NewClients.Counts.SecretSyncs)
|
||||||
|
require.Equal(t, 10, byMonth[0].Counts.SecretSyncs)
|
||||||
|
require.Equal(t, 10, byMonth[0].Namespaces[0].Counts.SecretSyncs)
|
||||||
|
require.Equal(t, 10, byMonth[0].Namespaces[0].Mounts[0].Counts.SecretSyncs)
|
||||||
|
require.Equal(t, 10, byMonth[0].NewClients.Counts.Clients)
|
||||||
|
require.Equal(t, 10, byMonth[0].Counts.Clients)
|
||||||
|
require.Equal(t, 10, byMonth[0].Namespaces[0].Counts.Clients)
|
||||||
|
require.Equal(t, 10, byMonth[0].Namespaces[0].Mounts[0].Counts.Clients)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test_SecretSync_Deduplication verifies that secret sync clients are
|
||||||
|
// deduplicated across months. The test creates 10 secret sync clients and
|
||||||
|
// repeats those clients in later months, then also registers 3 and then 2 new
|
||||||
|
// secret sync clients. The test verifies that the total number of secret sync
|
||||||
|
// clients is 15 (10 + 2 + 3), ensuring that the duplicates are not included
|
||||||
|
func Test_SecretSync_Deduplication(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
cluster := minimal.NewTestSoloCluster(t, nil)
|
||||||
|
client := cluster.Cores[0].Client
|
||||||
|
_, err := client.Logical().Write("sys/internal/counters/config", map[string]interface{}{
|
||||||
|
"enabled": "enable",
|
||||||
|
})
|
||||||
|
_, err = clientcountutil.NewActivityLogData(client).
|
||||||
|
NewPreviousMonthData(3).
|
||||||
|
NewClientsSeen(10, clientcountutil.WithClientType("secret-sync")).
|
||||||
|
NewPreviousMonthData(2).
|
||||||
|
RepeatedClientsSeen(4, clientcountutil.WithClientType("secret-sync")).
|
||||||
|
NewClientsSeen(3, clientcountutil.WithClientType("secret-sync")).
|
||||||
|
NewPreviousMonthData(1).
|
||||||
|
RepeatedClientsSeen(5, clientcountutil.WithClientType("secret-sync")).
|
||||||
|
NewClientsSeen(2, clientcountutil.WithClientType("secret-sync")).
|
||||||
|
Write(context.Background(), generation.WriteOptions_WRITE_PRECOMPUTED_QUERIES)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
now := time.Now().UTC()
|
||||||
|
resp, err := client.Logical().ReadWithData("sys/internal/counters/activity", map[string][]string{
|
||||||
|
"end_time": {timeutil.StartOfMonth(now).Format(time.RFC3339)},
|
||||||
|
"start_time": {timeutil.StartOfMonth(timeutil.MonthsPreviousTo(4, now)).Format(time.RFC3339)},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
total := getTotals(t, resp)
|
||||||
|
require.Equal(t, 15, total.SecretSyncs)
|
||||||
|
require.Equal(t, 15, total.Clients)
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user