scottk/vault
1
0
Fork 0
mirror of https://github.com/optim-enterprises-bv/vault.git synced 2025-11-02 03:27:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

backports for ActivityLog and Reporting 1.13.x (#21140)

Browse Source 
* backport of commit 9f7f8d5bfa

* backport of commit e3c59773e9

* backport of commit b4fab6ac2a

* backport of commit 54904e4cd6

* backport of commit 4b6ec4079d

* backport of commit 05ba6bbddd

* backport of commit 002a59a370

* backport of commit 77f83d9fe8

* backport of commit 730d0e2821

* backport of commit 35e2c1665f

* backport of commit 810d504e4f

* backport of commit 5b23dd506f

* backport of commit 018ea84997

* backport of commit 541f18eeb7

* backport of commit b4e2751a09

* backport of commit dc5dd71c72

* backport of commit 5002489d27

---------

Co-authored-by: miagilepner <mia.epner@hashicorp.com>
Co-authored-by: Nick Cabatoff <ncabatoff@hashicorp.com>
This commit is contained in:
Mike Palmiotto
2023-06-14 17:07:26 -04:00
committed by GitHub
parent 43bdbde214
commit fa4153dc1f
31 changed files with 2203 additions and 416 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
changelog/19625.txt Normal file
View File

@@ -0,0 +1,4 @@
```release-note:feature
core (enterprise): Add background worker for automatic reporting of billing
information.
```

3
changelog/19891.txt Normal file
View File

@@ -0,0 +1,3 @@
```release-note:improvement
core (enterprise): add configuration for license reporting
```

3
changelog/20073.txt Normal file
View File

@@ -0,0 +1,3 @@
```release-note:improvement
core/activity: refactor the activity log's generation of precomputed queries
```

3
changelog/20078.txt Normal file
View File

@@ -0,0 +1,3 @@
```release-note:improvement
core/activity: error when attempting to update retention configuration below the minimum
```

4
changelog/20086.txt Normal file
View File

@@ -0,0 +1,4 @@
```release-note:improvement
api: `/sys/internal/counters/config` endpoint now contains read-only
`reporting_enabled` and `billing_start_timestamp` fields.
```

4
changelog/20150.txt Normal file
View File

@@ -0,0 +1,4 @@
```release-note:improvement
api: `/sys/internal/counters/config` endpoint now contains read-only
`minimum_retention_months`.
```

6
changelog/20680.txt Normal file
View File

@@ -0,0 +1,6 @@
```release-note:improvement
core (enterprise): support reloading configuration for automated reporting via SIGHUP
```
```release-note:improvement
core (enterprise): license updates trigger a reload of reporting and the activity log
```

4
changelog/20694.txt Normal file
View File

@@ -0,0 +1,4 @@
```release-note:improvement
api: GET ... /sys/internal/counters/activity?current_billing_period=true now
results in a response which contains the full billing period
```

3
command/server.go
View File

@@ -1631,6 +1631,9 @@ func (c *ServerCommand) Run(args []string) int {
c.UI.Error(err.Error())
}
if err := core.ReloadCensus(); err != nil {
c.UI.Error(err.Error())
}
select {
case c.licenseReloadedCh <- err:
default:

1
command/server/config_test_helpers.go
View File

@@ -1097,6 +1097,7 @@ func testParseSeals(t *testing.T) {
},
},
}
addExpectedDefaultEntConfig(expected)
config.Prune()
require.Equal(t, config, expected)
}

1
command/server/config_test_helpers_util.go
View File

@@ -3,4 +3,5 @@
package server
func addExpectedEntConfig(c *Config, sentinelModules []string) {}
func addExpectedDefaultEntConfig(c *Config) {}
func addExpectedEntSanitizedConfig(c map[string]interface{}, sentinelModules []string) {}

25
helper/metricsutil/gauge_process.go
View File

@@ -8,24 +8,9 @@ import (
"github.com/armon/go-metrics"
log "github.com/hashicorp/go-hclog"
"github.com/hashicorp/vault/helper/timeutil"
)
// This interface allows unit tests to substitute in a simulated clock.
type clock interface {
Now() time.Time
NewTicker(time.Duration) *time.Ticker
}
type defaultClock struct{}
func (_ defaultClock) Now() time.Time {
return time.Now()
}
func (_ defaultClock) NewTicker(d time.Duration) *time.Ticker {
return time.NewTicker(d)
}
// GaugeLabelValues is one gauge in a set sharing a single key, that
// are measured in a batch.
type GaugeLabelValues struct {
@@ -73,7 +58,7 @@ type GaugeCollectionProcess struct {
maxGaugeCardinality int
// time source
clock clock
clock timeutil.Clock
}
// NewGaugeCollectionProcess creates a new collection process for the callback
@@ -98,7 +83,7 @@ func NewGaugeCollectionProcess(
gaugeInterval,
maxGaugeCardinality,
logger,
defaultClock{},
timeutil.DefaultClock{},
)
}
@@ -121,7 +106,7 @@ func (m *ClusterMetricSink) NewGaugeCollectionProcess(
m.GaugeInterval,
m.MaxGaugeCardinality,
logger,
defaultClock{},
timeutil.DefaultClock{},
)
}
@@ -134,7 +119,7 @@ func newGaugeCollectionProcessWithClock(
gaugeInterval time.Duration,
maxGaugeCardinality int,
logger log.Logger,
clock clock,
clock timeutil.Clock,
) (*GaugeCollectionProcess, error) {
process := &GaugeCollectionProcess{
stop: make(chan struct{}, 1),

6
helper/metricsutil/gauge_process_test.go
View File

@@ -12,6 +12,7 @@ import (
"github.com/armon/go-metrics"
log "github.com/hashicorp/go-hclog"
"github.com/hashicorp/vault/helper/timeutil"
)
// SimulatedTime maintains a virtual clock so the test isn't
@@ -21,9 +22,10 @@ import (
type SimulatedTime struct {
now time.Time
tickerBarrier chan *SimulatedTicker
timeutil.DefaultClock
}
var _ clock = &SimulatedTime{}
var _ timeutil.Clock = &SimulatedTime{}
type SimulatedTicker struct {
ticker *time.Ticker
@@ -118,7 +120,7 @@ func TestGauge_Creation(t *testing.T) {
t.Fatalf("Error creating collection process: %v", err)
}
if _, ok := p.clock.(defaultClock); !ok {
if _, ok := p.clock.(timeutil.DefaultClock); !ok {
t.Error("Default clock not installed.")
}

23
helper/timeutil/timeutil.go
View File

@@ -139,3 +139,26 @@ func SkipAtEndOfMonth(t *testing.T) {
t.Skip("too close to end of month")
}
}
// This interface allows unit tests to substitute in a simulated Clock.
type Clock interface {
Now() time.Time
NewTicker(time.Duration) *time.Ticker
NewTimer(time.Duration) *time.Timer
}
type DefaultClock struct{}
var _ Clock = (*DefaultClock)(nil)
func (_ DefaultClock) Now() time.Time {
return time.Now()
}
func (_ DefaultClock) NewTicker(d time.Duration) *time.Ticker {
return time.NewTicker(d)
}
func (_ DefaultClock) NewTimer(d time.Duration) *time.Timer {
return time.NewTimer(d)
}

72
vault/activity/generation/generate_data.pb.go
View File

@@ -439,12 +439,11 @@ type Client struct {
Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
Count int32 `protobuf:"varint,2,opt,name=count,proto3" json:"count,omitempty"`
TimesSeen int32 `protobuf:"varint,3,opt,name=times_seen,json=timesSeen,proto3" json:"times_seen,omitempty"`
Repeated bool `protobuf:"varint,4,opt,name=repeated,proto3" json:"repeated,omitempty"`
RepeatedFromMonth int32 `protobuf:"varint,5,opt,name=repeated_from_month,json=repeatedFromMonth,proto3" json:"repeated_from_month,omitempty"`
Namespace string `protobuf:"bytes,6,opt,name=namespace,proto3" json:"namespace,omitempty"`
Mount string `protobuf:"bytes,7,opt,name=mount,proto3" json:"mount,omitempty"`
NonEntity bool `protobuf:"varint,8,opt,name=non_entity,json=nonEntity,proto3" json:"non_entity,omitempty"`
Repeated bool `protobuf:"varint,3,opt,name=repeated,proto3" json:"repeated,omitempty"`
RepeatedFromMonth int32 `protobuf:"varint,4,opt,name=repeated_from_month,json=repeatedFromMonth,proto3" json:"repeated_from_month,omitempty"`
Namespace string `protobuf:"bytes,5,opt,name=namespace,proto3" json:"namespace,omitempty"`
Mount string `protobuf:"bytes,6,opt,name=mount,proto3" json:"mount,omitempty"`
NonEntity bool `protobuf:"varint,7,opt,name=non_entity,json=nonEntity,proto3" json:"non_entity,omitempty"`
}
func (x *Client) Reset() {
@@ -493,13 +492,6 @@ func (x *Client) GetCount() int32 {
return 0
}
func (x *Client) GetTimesSeen() int32 {
if x != nil {
return x.TimesSeen
}
return 0
}
func (x *Client) GetRepeated() bool {
if x != nil {
return x.Repeated
@@ -584,36 +576,34 @@ var file_vault_activity_generation_generate_data_proto_rawDesc = []byte{
0x74, 0x73, 0x12, 0x2c, 0x0a, 0x07, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20,
0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e,
0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, 0x07, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73,
0x22, 0xec, 0x01, 0x0a, 0x06, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69,
0x22, 0xcd, 0x01, 0x0a, 0x06, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69,
0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x63,
0x6f, 0x75, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e,
0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x5f, 0x73, 0x65, 0x65, 0x6e, 0x18,
0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x53, 0x65, 0x65, 0x6e,
0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x65, 0x64, 0x18, 0x04, 0x20, 0x01,
0x28, 0x08, 0x52, 0x08, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x65, 0x64, 0x12, 0x2e, 0x0a, 0x13,
0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x66, 0x72, 0x6f, 0x6d, 0x5f, 0x6d, 0x6f,
0x6e, 0x74, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x11, 0x72, 0x65, 0x70, 0x65, 0x61,
0x74, 0x65, 0x64, 0x46, 0x72, 0x6f, 0x6d, 0x4d, 0x6f, 0x6e, 0x74, 0x68, 0x12, 0x1c, 0x0a, 0x09,
0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52,
0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x6f,
0x75, 0x6e, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6d, 0x6f, 0x75, 0x6e, 0x74,
0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x6f, 0x6e, 0x5f, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x18, 0x08,
0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x6e, 0x6f, 0x6e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2a,
0xa0, 0x01, 0x0a, 0x0c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73,
0x12, 0x11, 0x0a, 0x0d, 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57,
0x4e, 0x10, 0x00, 0x12, 0x1d, 0x0a, 0x19, 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x50, 0x52, 0x45,
0x43, 0x4f, 0x4d, 0x50, 0x55, 0x54, 0x45, 0x44, 0x5f, 0x51, 0x55, 0x45, 0x52, 0x49, 0x45, 0x53,
0x10, 0x01, 0x12, 0x1a, 0x0a, 0x16, 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x44, 0x49, 0x53, 0x54,
0x49, 0x4e, 0x43, 0x54, 0x5f, 0x43, 0x4c, 0x49, 0x45, 0x4e, 0x54, 0x53, 0x10, 0x02, 0x12, 0x12,
0x0a, 0x0e, 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x4e, 0x54, 0x49, 0x54, 0x49, 0x45, 0x53,
0x10, 0x03, 0x12, 0x17, 0x0a, 0x13, 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x44, 0x49, 0x52, 0x45,
0x43, 0x54, 0x5f, 0x54, 0x4f, 0x4b, 0x45, 0x4e, 0x53, 0x10, 0x04, 0x12, 0x15, 0x0a, 0x11, 0x57,
0x52, 0x49, 0x54, 0x45, 0x5f, 0x49, 0x4e, 0x54, 0x45, 0x4e, 0x54, 0x5f, 0x4c, 0x4f, 0x47, 0x53,
0x10, 0x05, 0x42, 0x36, 0x5a, 0x34, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d,
0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x76, 0x61, 0x75, 0x6c, 0x74,
0x2f, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2f, 0x61, 0x63, 0x74, 0x69, 0x76, 0x69, 0x74, 0x79, 0x2f,
0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x33,
0x74, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20,
0x01, 0x28, 0x08, 0x52, 0x08, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x65, 0x64, 0x12, 0x2e, 0x0a,
0x13, 0x72, 0x65, 0x70, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x66, 0x72, 0x6f, 0x6d, 0x5f, 0x6d,
0x6f, 0x6e, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x11, 0x72, 0x65, 0x70, 0x65,
0x61, 0x74, 0x65, 0x64, 0x46, 0x72, 0x6f, 0x6d, 0x4d, 0x6f, 0x6e, 0x74, 0x68, 0x12, 0x1c, 0x0a,
0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09,
0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6d,
0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6d, 0x6f, 0x75, 0x6e,
0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x6f, 0x6e, 0x5f, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x18,
0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x6e, 0x6f, 0x6e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79,
0x2a, 0xa0, 0x01, 0x0a, 0x0c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e,
0x73, 0x12, 0x11, 0x0a, 0x0d, 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f,
0x57, 0x4e, 0x10, 0x00, 0x12, 0x1d, 0x0a, 0x19, 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x50, 0x52,
0x45, 0x43, 0x4f, 0x4d, 0x50, 0x55, 0x54, 0x45, 0x44, 0x5f, 0x51, 0x55, 0x45, 0x52, 0x49, 0x45,
0x53, 0x10, 0x01, 0x12, 0x1a, 0x0a, 0x16, 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x44, 0x49, 0x53,
0x54, 0x49, 0x4e, 0x43, 0x54, 0x5f, 0x43, 0x4c, 0x49, 0x45, 0x4e, 0x54, 0x53, 0x10, 0x02, 0x12,
0x12, 0x0a, 0x0e, 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x4e, 0x54, 0x49, 0x54, 0x49, 0x45,
0x53, 0x10, 0x03, 0x12, 0x17, 0x0a, 0x13, 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x44, 0x49, 0x52,
0x45, 0x43, 0x54, 0x5f, 0x54, 0x4f, 0x4b, 0x45, 0x4e, 0x53, 0x10, 0x04, 0x12, 0x15, 0x0a, 0x11,
0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x49, 0x4e, 0x54, 0x45, 0x4e, 0x54, 0x5f, 0x4c, 0x4f, 0x47,
0x53, 0x10, 0x05, 0x42, 0x36, 0x5a, 0x34, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x76, 0x61, 0x75, 0x6c,
0x74, 0x2f, 0x76, 0x61, 0x75, 0x6c, 0x74, 0x2f, 0x61, 0x63, 0x74, 0x69, 0x76, 0x69, 0x74, 0x79,
0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x62, 0x06, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x33,
}
var (

11
vault/activity/generation/generate_data.proto
View File

@@ -48,10 +48,9 @@ message Clients {
message Client {
string id = 1;
int32 count = 2;
int32 times_seen = 3;
bool repeated = 4;
int32 repeated_from_month = 5;
string namespace = 6;
string mount = 7;
bool non_entity = 8;
bool repeated = 3;
int32 repeated_from_month = 4;
string namespace = 5;
string mount = 6;
bool non_entity = 7;
}

767
vault/activity_log.go
View File

File diff suppressed because it is too large Load Diff

636
vault/activity_log_test.go
View File

@@ -32,6 +32,7 @@ import (
"github.com/mitchellh/mapstructure"
)
// TestActivityLog_Creation calls AddEntityToFragment and verifies that it appears correctly in a.fragment.
func TestActivityLog_Creation(t *testing.T) {
core, _, _ := TestCoreUnsealed(t)
@@ -102,6 +103,8 @@ func TestActivityLog_Creation(t *testing.T) {
}
}
// TestActivityLog_Creation_WrappingTokens calls HandleTokenUsage for two wrapping tokens, and verifies that this
// doesn't create a fragment.
func TestActivityLog_Creation_WrappingTokens(t *testing.T) {
core, _, _ := TestCoreUnsealed(t)
@@ -170,6 +173,8 @@ func checkExpectedEntitiesInMap(t *testing.T, a *ActivityLog, entityIDs []string
}
}
// TestActivityLog_UniqueEntities calls AddEntityToFragment 4 times with 2 different clients, then verifies that there
// are only 2 clients in the fragment and that they have the earlier timestamps.
func TestActivityLog_UniqueEntities(t *testing.T) {
core, _, _ := TestCoreUnsealed(t)
a := core.activityLog
@@ -274,6 +279,9 @@ func expectedEntityIDs(t *testing.T, out *activity.EntityActivityLog, ids []stri
}
}
// TestActivityLog_SaveTokensToStorage calls AddTokenToFragment with duplicate namespaces and then saves the segment to
// storage. The test then reads and unmarshals the segment, and verifies that the results have the correct counts by
// namespace.
func TestActivityLog_SaveTokensToStorage(t *testing.T) {
core, _, _ := TestCoreUnsealed(t)
ctx := context.Background()
@@ -426,6 +434,8 @@ func TestActivityLog_SaveTokensToStorageDoesNotUpdateTokenCount(t *testing.T) {
}
}
// TestActivityLog_SaveEntitiesToStorage calls AddEntityToFragment with clients with different namespaces and then
// writes the segment to storage. Read back from storage, and verify that client IDs exist in storage.
func TestActivityLog_SaveEntitiesToStorage(t *testing.T) {
core, _, _ := TestCoreUnsealed(t)
ctx := context.Background()
@@ -477,7 +487,8 @@ func TestActivityLog_SaveEntitiesToStorage(t *testing.T) {
expectedEntityIDs(t, out, ids)
}
// Test to check store hyperloglog and fetch hyperloglog from storage
// TestActivityLog_StoreAndReadHyperloglog inserts into a hyperloglog, stores it and then reads it back. The test
// verifies the estimate count is correct.
func TestActivityLog_StoreAndReadHyperloglog(t *testing.T) {
core, _, _ := TestCoreUnsealed(t)
ctx := context.Background()
@@ -505,12 +516,16 @@ func TestActivityLog_StoreAndReadHyperloglog(t *testing.T) {
}
}
// TestModifyResponseMonthsNilAppend calls modifyResponseMonths for a range of 5 months ago to now. It verifies that the
// 5 months in the range are correct.
func TestModifyResponseMonthsNilAppend(t *testing.T) {
core, _, _ := TestCoreUnsealed(t)
a := core.activityLog
end := time.Now().UTC()
start := timeutil.StartOfMonth(end).AddDate(0, -5, 0)
responseMonthTimestamp := timeutil.StartOfMonth(end).AddDate(0, -3, 0).Format(time.RFC3339)
responseMonths := []*ResponseMonth{{Timestamp: responseMonthTimestamp}}
months := modifyResponseMonths(responseMonths, start, end)
months := a.modifyResponseMonths(responseMonths, start, end)
if len(months) != 5 {
t.Fatal("wrong number of months padded")
}
@@ -535,6 +550,9 @@ func TestModifyResponseMonthsNilAppend(t *testing.T) {
}
}
// TestActivityLog_ReceivedFragment calls receivedFragment with a fragment and verifies it gets added to
// standbyFragmentsReceived. Send the same fragment again and then verify that it doesn't change the entity map but does
// get added to standbyFragmentsReceived.
func TestActivityLog_ReceivedFragment(t *testing.T) {
core, _, _ := TestCoreUnsealed(t)
a := core.activityLog
@@ -586,6 +604,8 @@ func TestActivityLog_ReceivedFragment(t *testing.T) {
}
}
// TestActivityLog_availableLogsEmptyDirectory verifies that availableLogs returns an empty slice when the log directory
// is empty.
func TestActivityLog_availableLogsEmptyDirectory(t *testing.T) {
// verify that directory is empty, and nothing goes wrong
core, _, _ := TestCoreUnsealed(t)
@@ -599,6 +619,8 @@ func TestActivityLog_availableLogsEmptyDirectory(t *testing.T) {
}
}
// TestActivityLog_availableLogs writes to the direct token paths and entity paths and verifies that the correct start
// times are returned.
func TestActivityLog_availableLogs(t *testing.T) {
// set up a few files in storage
core, _, _ := TestCoreUnsealed(t)
@@ -626,22 +648,24 @@ func TestActivityLog_availableLogs(t *testing.T) {
}
}
// TestActivityLog_MultipleFragmentsAndSegments adds 4000 clients to a fragment
// and saves it and reads it. The test then adds 4000 more clients and calls
// receivedFragment with 200 more entities. The current segment is saved to
// storage and read back. The test verifies that there are 5000 clients in the
// first segment index, then the rest in the second index.
func TestActivityLog_MultipleFragmentsAndSegments(t *testing.T) {
core, _, _ := TestCoreUnsealed(t)
core, _, _ := TestCoreUnsealedWithConfig(t, &CoreConfig{
ActivityLogConfig: ActivityLogCoreConfig{
DisableFragmentWorker: true,
DisableTimers: true,
},
})
a := core.activityLog
// enabled check is now inside AddClientToFragment
a.SetEnable(true)
a.SetStartTimestamp(time.Now().Unix()) // set a nonzero segment
// Stop timers for test purposes
close(a.doneCh)
defer func() {
a.l.Lock()
a.doneCh = make(chan struct{}, 1)
a.l.Unlock()
}()
startTimestamp := a.GetStartTimestamp()
path0 := fmt.Sprintf("sys/counters/activity/log/entity/%d/0", startTimestamp)
path1 := fmt.Sprintf("sys/counters/activity/log/entity/%d/1", startTimestamp)
@@ -794,6 +818,7 @@ func TestActivityLog_MultipleFragmentsAndSegments(t *testing.T) {
}
}
// TestActivityLog_API_ConfigCRUD performs various CRUD operations on internal/counters/config.
func TestActivityLog_API_ConfigCRUD(t *testing.T) {
core, b, _ := testCoreSystemBackend(t)
view := core.systemBarrierView
@@ -807,10 +832,13 @@ func TestActivityLog_API_ConfigCRUD(t *testing.T) {
t.Fatalf("err: %v", err)
}
defaults := map[string]interface{}{
"default_report_months": 12,
"retention_months": 24,
"enabled": activityLogEnabledDefaultValue,
"queries_available": false,
"default_report_months": 12,
"retention_months": 24,
"enabled": activityLogEnabledDefaultValue,
"queries_available": false,
"reporting_enabled": core.CensusLicensingEnabled(),
"billing_start_timestamp": core.BillingStart(),
"minimum_retention_months": core.activityLog.configOverrides.MinimumRetentionMonths,
}
if diff := deep.Equal(resp.Data, defaults); len(diff) > 0 {
@@ -888,10 +916,13 @@ func TestActivityLog_API_ConfigCRUD(t *testing.T) {
t.Fatalf("err: %v", err)
}
expected := map[string]interface{}{
"default_report_months": 1,
"retention_months": 2,
"enabled": "enable",
"queries_available": false,
"default_report_months": 1,
"retention_months": 2,
"enabled": "enable",
"queries_available": false,
"reporting_enabled": core.CensusLicensingEnabled(),
"billing_start_timestamp": core.BillingStart(),
"minimum_retention_months": core.activityLog.configOverrides.MinimumRetentionMonths,
}
if diff := deep.Equal(resp.Data, expected); len(diff) > 0 {
@@ -924,10 +955,13 @@ func TestActivityLog_API_ConfigCRUD(t *testing.T) {
}
defaults := map[string]interface{}{
"default_report_months": 12,
"retention_months": 24,
"enabled": activityLogEnabledDefaultValue,
"queries_available": false,
"default_report_months": 12,
"retention_months": 24,
"enabled": activityLogEnabledDefaultValue,
"queries_available": false,
"reporting_enabled": core.CensusLicensingEnabled(),
"billing_start_timestamp": core.BillingStart(),
"minimum_retention_months": core.activityLog.configOverrides.MinimumRetentionMonths,
}
if diff := deep.Equal(resp.Data, defaults); len(diff) > 0 {
@@ -936,6 +970,7 @@ func TestActivityLog_API_ConfigCRUD(t *testing.T) {
}
}
// TestActivityLog_parseSegmentNumberFromPath verifies that the segment number is extracted correctly from a path.
func TestActivityLog_parseSegmentNumberFromPath(t *testing.T) {
testCases := []struct {
input string
@@ -985,6 +1020,7 @@ func TestActivityLog_parseSegmentNumberFromPath(t *testing.T) {
}
}
// TestActivityLog_getLastEntitySegmentNumber verifies that the last segment number is correctly returned.
func TestActivityLog_getLastEntitySegmentNumber(t *testing.T) {
core, _, _ := TestCoreUnsealed(t)
a := core.activityLog
@@ -1040,6 +1076,8 @@ func TestActivityLog_getLastEntitySegmentNumber(t *testing.T) {
}
}
// TestActivityLog_tokenCountExists writes to the direct tokens segment path and verifies that segment count exists
// returns true for the segments at these paths.
func TestActivityLog_tokenCountExists(t *testing.T) {
core, _, _ := TestCoreUnsealed(t)
a := core.activityLog
@@ -1164,6 +1202,8 @@ func (a *ActivityLog) resetEntitiesInMemory(t *testing.T) {
a.partialMonthClientTracker = make(map[string]*activity.EntityRecord)
}
// TestActivityLog_loadCurrentClientSegment writes entity segments and calls loadCurrentClientSegment, then verifies
// that the correct values are returned when querying the current segment.
func TestActivityLog_loadCurrentClientSegment(t *testing.T) {
core, _, _ := TestCoreUnsealed(t)
a := core.activityLog
@@ -1280,6 +1320,8 @@ func TestActivityLog_loadCurrentClientSegment(t *testing.T) {
}
}
// TestActivityLog_loadPriorEntitySegment writes entities to two months and calls loadPriorEntitySegment for each month,
// verifying that the active clients are correct.
func TestActivityLog_loadPriorEntitySegment(t *testing.T) {
core, _, _ := TestCoreUnsealed(t)
a := core.activityLog
@@ -1424,6 +1466,9 @@ func TestActivityLog_loadTokenCount(t *testing.T) {
}
}
// TestActivityLog_StopAndRestart disables the activity log, waits for deletes to complete, and then enables the
// activity log. The activity log is then stopped and started again, to simulate a seal and unseal. The test then
// verifies that there's no error adding an entity, direct token, and when writing a segment to storage.
func TestActivityLog_StopAndRestart(t *testing.T) {
core, b, _ := testCoreSystemBackend(t)
sysView := core.systemBarrierView
@@ -1555,6 +1600,8 @@ func setupActivityRecordsInStorage(t *testing.T, base time.Time, includeEntities
return a, entityRecords, tokenRecords
}
// TestActivityLog_refreshFromStoredLog writes records for 3 months ago and this month, then calls refreshFromStoredLog.
// The test verifies that current entities and current tokens are correct.
func TestActivityLog_refreshFromStoredLog(t *testing.T) {
a, expectedClientRecords, expectedTokenCounts := setupActivityRecordsInStorage(t, time.Now().UTC(), true, true)
a.SetEnable(true)
@@ -1592,6 +1639,9 @@ func TestActivityLog_refreshFromStoredLog(t *testing.T) {
}
}
// TestActivityLog_refreshFromStoredLogWithBackgroundLoadingCancelled writes data from 3 months ago to this month. The
// test closes a.doneCh and calls refreshFromStoredLog, which will not do any processing because the doneCh is closed.
// The test verifies that the current data is not loaded.
func TestActivityLog_refreshFromStoredLogWithBackgroundLoadingCancelled(t *testing.T) {
a, expectedClientRecords, expectedTokenCounts := setupActivityRecordsInStorage(t, time.Now().UTC(), true, true)
a.SetEnable(true)
@@ -1633,6 +1683,8 @@ func TestActivityLog_refreshFromStoredLogWithBackgroundLoadingCancelled(t *testi
}
}
// TestActivityLog_refreshFromStoredLogContextCancelled writes data from 3 months ago to this month and calls
// refreshFromStoredLog with a canceled context, verifying that the function errors because of the canceled context.
func TestActivityLog_refreshFromStoredLogContextCancelled(t *testing.T) {
a, _, _ := setupActivityRecordsInStorage(t, time.Now().UTC(), true, true)
@@ -1646,6 +1698,8 @@ func TestActivityLog_refreshFromStoredLogContextCancelled(t *testing.T) {
}
}
// TestActivityLog_refreshFromStoredLogNoTokens writes only entities from 3 months ago to today, then calls
// refreshFromStoredLog. It verifies that there are no tokens loaded.
func TestActivityLog_refreshFromStoredLogNoTokens(t *testing.T) {
a, expectedClientRecords, _ := setupActivityRecordsInStorage(t, time.Now().UTC(), true, false)
a.SetEnable(true)
@@ -1681,6 +1735,8 @@ func TestActivityLog_refreshFromStoredLogNoTokens(t *testing.T) {
}
}
// TestActivityLog_refreshFromStoredLogNoEntities writes only direct tokens from 3 months ago to today, and runs
// refreshFromStoredLog. It verifies that there are no entities or clients loaded.
func TestActivityLog_refreshFromStoredLogNoEntities(t *testing.T) {
a, _, expectedTokenCounts := setupActivityRecordsInStorage(t, time.Now().UTC(), false, true)
a.SetEnable(true)
@@ -1708,6 +1764,8 @@ func TestActivityLog_refreshFromStoredLogNoEntities(t *testing.T) {
}
}
// TestActivityLog_refreshFromStoredLogNoData writes nothing and calls refreshFromStoredLog, and verifies that the
// current segment counts are zero.
func TestActivityLog_refreshFromStoredLogNoData(t *testing.T) {
now := time.Now().UTC()
a, _, _ := setupActivityRecordsInStorage(t, now, false, false)
@@ -1723,6 +1781,8 @@ func TestActivityLog_refreshFromStoredLogNoData(t *testing.T) {
a.ExpectCurrentSegmentRefreshed(t, now.Unix(), false)
}
// TestActivityLog_refreshFromStoredLogTwoMonthsPrevious creates segment data from 5 months ago to 2 months ago and
// calls refreshFromStoredLog, then verifies that the current segment counts are zero.
func TestActivityLog_refreshFromStoredLogTwoMonthsPrevious(t *testing.T) {
// test what happens when the most recent data is from month M-2 (or earlier - same effect)
now := time.Now().UTC()
@@ -1740,6 +1800,8 @@ func TestActivityLog_refreshFromStoredLogTwoMonthsPrevious(t *testing.T) {
a.ExpectCurrentSegmentRefreshed(t, now.Unix(), false)
}
// TestActivityLog_refreshFromStoredLogPreviousMonth creates segment data from 4 months ago to 1 month ago, then calls
// refreshFromStoredLog, then verifies that these clients are included in the current segment.
func TestActivityLog_refreshFromStoredLogPreviousMonth(t *testing.T) {
// test what happens when most recent data is from month M-1
// we expect to load the data from the previous month so that the activeFragmentWorker
@@ -1782,6 +1844,8 @@ func TestActivityLog_refreshFromStoredLogPreviousMonth(t *testing.T) {
}
}
// TestActivityLog_Export writes overlapping client for 5 months with various mounts and namespaces. It performs an
// export for various month ranges in the range, and verifies that the outputs are correct.
func TestActivityLog_Export(t *testing.T) {
timeutil.SkipAtEndOfMonth(t)
@@ -1973,6 +2037,8 @@ func (f *fakeResponseWriter) WriteHeader(statusCode int) {
panic("unimplmeneted")
}
// TestActivityLog_IncludeNamespace verifies that includeInResponse returns true for namespaces that are children of
// their parents.
func TestActivityLog_IncludeNamespace(t *testing.T) {
root := namespace.RootNamespace
a := &ActivityLog{}
@@ -2020,6 +2086,8 @@ func TestActivityLog_IncludeNamespace(t *testing.T) {
}
}
// TestActivityLog_DeleteWorker writes segments for entities and direct tokens for 2 different timestamps, then runs the
// deleteLogWorker for one of the timestamps. The test verifies that the correct segment is deleted, and the other remains.
func TestActivityLog_DeleteWorker(t *testing.T) {
core, _, _ := TestCoreUnsealed(t)
a := core.activityLog
@@ -2075,6 +2143,9 @@ func checkAPIWarnings(t *testing.T, originalEnabled, newEnabled bool, resp *logi
}
}
// TestActivityLog_EnableDisable writes a segment, adds an entity to the in-memory fragment, then disables the activity
// log. The test verifies that the segment doesn't exist. The activity log is enabled, then verified that an empty
// segment is written and new clients can be added and written to segments.
func TestActivityLog_EnableDisable(t *testing.T) {
timeutil.SkipAtEndOfMonth(t)
@@ -2712,6 +2783,9 @@ func TestActivityLog_SaveAfterDisable(t *testing.T) {
expectMissingSegment(t, core, path)
}
// TestActivityLog_Precompute creates segments over a range of 11 months, with overlapping clients and namespaces.
// Create intent logs and run precomputedQueryWorker for various month ranges. Verify that the precomputed queries have
// the correct counts, including per namespace.
func TestActivityLog_Precompute(t *testing.T) {
timeutil.SkipAtEndOfMonth(t)
@@ -3599,6 +3673,8 @@ func (b *BlockingInmemStorage) Delete(ctx context.Context, key string) error {
return errors.New("fake implementation")
}
// TestActivityLog_PrecomputeCancel stops the activity log before running the precomputedQueryWorker, and verifies that
// the context used to query storage has been canceled.
func TestActivityLog_PrecomputeCancel(t *testing.T) {
core, _, _ := TestCoreUnsealed(t)
a := core.activityLog
@@ -3627,6 +3703,8 @@ func TestActivityLog_PrecomputeCancel(t *testing.T) {
}
}
// TestActivityLog_NextMonthStart sets the activity log start timestamp, then verifies that StartOfNextMonth returns the
// correct value.
func TestActivityLog_NextMonthStart(t *testing.T) {
timeutil.SkipAtEndOfMonth(t)
@@ -3679,6 +3757,8 @@ func waitForRetentionWorkerToFinish(t *testing.T, a *ActivityLog) {
}
}
// TestActivityLog_Deletion writes entity, direct tokens, and queries for dates ranging over 20 months. Then the test
// calls the retentionWorker with decreasing retention values, and verifies that the correct paths are being deleted.
func TestActivityLog_Deletion(t *testing.T) {
timeutil.SkipAtEndOfMonth(t)
@@ -3794,6 +3874,8 @@ func TestActivityLog_Deletion(t *testing.T) {
checkPresent(21)
}
// TestActivityLog_partialMonthClientCount writes segment data for the curren month and runs refreshFromStoredLog and
// then partialMonthClientCount. The test verifies that the values returned by partialMonthClientCount are correct.
func TestActivityLog_partialMonthClientCount(t *testing.T) {
timeutil.SkipAtEndOfMonth(t)
@@ -3863,6 +3945,8 @@ func TestActivityLog_partialMonthClientCount(t *testing.T) {
}
}
// TestActivityLog_partialMonthClientCountUsingHandleQuery writes segments for the current month and calls
// refreshFromStoredLog, then handleQuery. The test verifies that the results from handleQuery are correct.
func TestActivityLog_partialMonthClientCountUsingHandleQuery(t *testing.T) {
timeutil.SkipAtEndOfMonth(t)
@@ -3990,7 +4074,7 @@ func TestActivityLog_partialMonthClientCountUsingHandleQuery(t *testing.T) {
}
// TestActivityLog_handleQuery_normalizedMountPaths ensures that the mount paths returned by the activity log always have a trailing slash and client accounting is done correctly when there's no trailing slash.
// Two clients that have the same mount path, but one has a trailing slash, should be considered part of the same mount path
// Two clients that have the same mount path, but one has a trailing slash, should be considered part of the same mount path.
func TestActivityLog_handleQuery_normalizedMountPaths(t *testing.T) {
timeutil.SkipAtEndOfMonth(t)
@@ -4156,3 +4240,507 @@ func TestActivityLog_partialMonthClientCountWithMultipleMountPaths(t *testing.T)
}
}
}
// TestActivityLog_processNewClients_delete ensures that the correct clients are deleted from a processNewClients struct
func TestActivityLog_processNewClients_delete(t *testing.T) {
mount := "mount"
namespace := "namespace"
clientID := "client-id"
run := func(t *testing.T, isNonEntity bool) {
t.Helper()
record := &activity.EntityRecord{
MountAccessor: mount,
NamespaceID: namespace,
ClientID: clientID,
NonEntity: isNonEntity,
}
newClients := newProcessNewClients()
newClients.add(record)
require.True(t, newClients.Counts.contains(record))
require.True(t, newClients.Namespaces[namespace].Counts.contains(record))
require.True(t, newClients.Namespaces[namespace].Mounts[mount].Counts.contains(record))
newClients.delete(record)
byNS := newClients.Namespaces
counts := newClients.Counts
require.NotContains(t, counts.NonEntities, clientID)
require.NotContains(t, counts.Entities, clientID)
require.NotContains(t, counts.NonEntities, clientID)
require.NotContains(t, counts.Entities, clientID)
require.NotContains(t, byNS[namespace].Mounts[mount].Counts.NonEntities, clientID)
require.NotContains(t, byNS[namespace].Counts.NonEntities, clientID)
require.NotContains(t, byNS[namespace].Mounts[mount].Counts.Entities, clientID)
require.NotContains(t, byNS[namespace].Counts.Entities, clientID)
}
t.Run("entity", func(t *testing.T) {
run(t, false)
})
t.Run("non-entity", func(t *testing.T) {
run(t, true)
})
}
// TestActivityLog_processClientRecord calls processClientRecord for an entity and a non-entity record and verifies that
// the record is present in the namespace and month maps
func TestActivityLog_processClientRecord(t *testing.T) {
startTime := time.Now()
mount := "mount"
namespace := "namespace"
clientID := "client-id"
run := func(t *testing.T, isNonEntity bool) {
t.Helper()
record := &activity.EntityRecord{
MountAccessor: mount,
NamespaceID: namespace,
ClientID: clientID,
NonEntity: isNonEntity,
}
byNS := make(summaryByNamespace)
byMonth := make(summaryByMonth)
processClientRecord(record, byNS, byMonth, startTime)
require.Contains(t, byNS, namespace)
require.Contains(t, byNS[namespace].Mounts, mount)
monthIndex := timeutil.StartOfMonth(startTime).UTC().Unix()
require.Contains(t, byMonth, monthIndex)
require.Equal(t, byMonth[monthIndex].Namespaces, byNS)
require.Equal(t, byMonth[monthIndex].NewClients.Namespaces, byNS)
if isNonEntity {
require.Contains(t, byMonth[monthIndex].Counts.NonEntities, clientID)
require.NotContains(t, byMonth[monthIndex].Counts.Entities, clientID)
require.Contains(t, byMonth[monthIndex].NewClients.Counts.NonEntities, clientID)
require.NotContains(t, byMonth[monthIndex].NewClients.Counts.Entities, clientID)
require.Contains(t, byNS[namespace].Mounts[mount].Counts.NonEntities, clientID)
require.Contains(t, byNS[namespace].Counts.NonEntities, clientID)
require.NotContains(t, byNS[namespace].Mounts[mount].Counts.Entities, clientID)
require.NotContains(t, byNS[namespace].Counts.Entities, clientID)
} else {
require.Contains(t, byMonth[monthIndex].Counts.Entities, clientID)
require.NotContains(t, byMonth[monthIndex].Counts.NonEntities, clientID)
require.Contains(t, byMonth[monthIndex].NewClients.Counts.Entities, clientID)
require.NotContains(t, byMonth[monthIndex].NewClients.Counts.NonEntities, clientID)
require.Contains(t, byNS[namespace].Mounts[mount].Counts.Entities, clientID)
require.Contains(t, byNS[namespace].Counts.Entities, clientID)
require.NotContains(t, byNS[namespace].Mounts[mount].Counts.NonEntities, clientID)
require.NotContains(t, byNS[namespace].Counts.NonEntities, clientID)
}
}
t.Run("non entity", func(t *testing.T) {
run(t, true)
})
t.Run("entity", func(t *testing.T) {
run(t, false)
})
}
func verifyByNamespaceContains(t *testing.T, s summaryByNamespace, clients ...*activity.EntityRecord) {
t.Helper()
for _, c := range clients {
require.Contains(t, s, c.NamespaceID)
counts := s[c.NamespaceID].Counts
require.True(t, counts.contains(c))
mounts := s[c.NamespaceID].Mounts
require.Contains(t, mounts, c.MountAccessor)
require.True(t, mounts[c.MountAccessor].Counts.contains(c))
}
}
func (s summaryByMonth) firstSeen(t *testing.T, client *activity.EntityRecord) time.Time {
t.Helper()
var seen int64
for month, data := range s {
present := data.NewClients.Counts.contains(client)
if present {
if seen != 0 {
require.Fail(t, "client seen more than once", client.ClientID, s)
}
seen = month
}
}
return time.Unix(seen, 0).UTC()
}
// TestActivityLog_handleEntitySegment verifies that the by namespace and by month summaries are correctly filled in a
// variety of scenarios
func TestActivityLog_handleEntitySegment(t *testing.T) {
finalTime := timeutil.StartOfMonth(time.Date(2022, 12, 1, 0, 0, 0, 0, time.UTC))
addMonths := func(i int) time.Time {
return timeutil.StartOfMonth(finalTime.AddDate(0, i, 0))
}
currentSegmentClients := make([]*activity.EntityRecord, 0, 3)
for i := 0; i < 3; i++ {
currentSegmentClients = append(currentSegmentClients, &activity.EntityRecord{
ClientID: fmt.Sprintf("id-%d", i),
NamespaceID: fmt.Sprintf("ns-%d", i),
MountAccessor: fmt.Sprintf("mnt-%d", i),
NonEntity: i == 0,
})
}
a := &ActivityLog{}
t.Run("older segment empty", func(t *testing.T) {
hll := hyperloglog.New()
byNS := make(summaryByNamespace)
byMonth := make(summaryByMonth)
segmentTime := addMonths(-3)
// our 3 clients were seen 3 months ago, with no other clients having been seen
err := a.handleEntitySegment(&activity.EntityActivityLog{Clients: currentSegmentClients}, segmentTime, hll, pqOptions{
byNamespace: byNS,
byMonth: byMonth,
endTime: timeutil.EndOfMonth(segmentTime),
activePeriodStart: addMonths(-12),
activePeriodEnd: addMonths(12),
})
require.NoError(t, err)
require.Len(t, byNS, 3)
verifyByNamespaceContains(t, byNS, currentSegmentClients...)
require.Len(t, byMonth, 1)
// they should all be registered as having first been seen 3 months ago
require.Equal(t, byMonth.firstSeen(t, currentSegmentClients[0]), segmentTime)
require.Equal(t, byMonth.firstSeen(t, currentSegmentClients[1]), segmentTime)
require.Equal(t, byMonth.firstSeen(t, currentSegmentClients[2]), segmentTime)
// and all 3 should be in the hyperloglog
require.Equal(t, hll.Estimate(), uint64(3))
})
t.Run("older segment clients seen earlier", func(t *testing.T) {
hll := hyperloglog.New()
byNS := make(summaryByNamespace)
byNS.add(currentSegmentClients[0])
byNS.add(currentSegmentClients[1])
byMonth := make(summaryByMonth)
segmentTime := addMonths(-3)
seenBefore2Months := addMonths(-2)
seenBefore1Month := addMonths(-1)
// client 0 was seen 2 months ago
byMonth.add(currentSegmentClients[0], seenBefore2Months)
// client 1 was seen 1 month ago
byMonth.add(currentSegmentClients[1], seenBefore1Month)
// handle clients 0, 1, and 2 as having been seen 3 months ago
err := a.handleEntitySegment(&activity.EntityActivityLog{Clients: currentSegmentClients}, segmentTime, hll, pqOptions{
byNamespace: byNS,
byMonth: byMonth,
endTime: timeutil.EndOfMonth(segmentTime),
activePeriodStart: addMonths(-12),
activePeriodEnd: addMonths(12),
})
require.NoError(t, err)
require.Len(t, byNS, 3)
verifyByNamespaceContains(t, byNS, currentSegmentClients...)
// we expect that they will only be registered as new 3 months ago, because that's when they were first seen
require.Equal(t, byMonth.firstSeen(t, currentSegmentClients[0]), segmentTime)
require.Equal(t, byMonth.firstSeen(t, currentSegmentClients[1]), segmentTime)
require.Equal(t, byMonth.firstSeen(t, currentSegmentClients[2]), segmentTime)
require.Equal(t, hll.Estimate(), uint64(3))
})
t.Run("disjoint set of clients", func(t *testing.T) {
hll := hyperloglog.New()
byNS := make(summaryByNamespace)
byNS.add(currentSegmentClients[0])
byNS.add(currentSegmentClients[1])
byMonth := make(summaryByMonth)
segmentTime := addMonths(-3)
seenBefore2Months := addMonths(-2)
seenBefore1Month := addMonths(-1)
// client 0 was seen 2 months ago
byMonth.add(currentSegmentClients[0], seenBefore2Months)
// client 1 was seen 1 month ago
byMonth.add(currentSegmentClients[1], seenBefore1Month)
// handle client 2 as having been seen 3 months ago
err := a.handleEntitySegment(&activity.EntityActivityLog{Clients: currentSegmentClients[2:]}, segmentTime, hll, pqOptions{
byNamespace: byNS,
byMonth: byMonth,
endTime: timeutil.EndOfMonth(segmentTime),
activePeriodStart: addMonths(-12),
activePeriodEnd: addMonths(12),
})
require.NoError(t, err)
require.Len(t, byNS, 3)
verifyByNamespaceContains(t, byNS, currentSegmentClients...)
// client 2 should be added to the map, and the other clients should stay where they were
require.Equal(t, byMonth.firstSeen(t, currentSegmentClients[0]), seenBefore2Months)
require.Equal(t, byMonth.firstSeen(t, currentSegmentClients[1]), seenBefore1Month)
require.Equal(t, byMonth.firstSeen(t, currentSegmentClients[2]), segmentTime)
// the hyperloglog will have 1 element, because there was only 1 client in the segment
require.Equal(t, hll.Estimate(), uint64(1))
})
t.Run("new clients same namespaces", func(t *testing.T) {
hll := hyperloglog.New()
byNS := make(summaryByNamespace)
byNS.add(currentSegmentClients[0])
byNS.add(currentSegmentClients[1])
byNS.add(currentSegmentClients[2])
byMonth := make(summaryByMonth)
segmentTime := addMonths(-3)
seenBefore2Months := addMonths(-2)
seenBefore1Month := addMonths(-1)
// client 0 and 2 were seen 2 months ago
byMonth.add(currentSegmentClients[0], seenBefore2Months)
byMonth.add(currentSegmentClients[2], seenBefore2Months)
// client 1 was seen 1 month ago
byMonth.add(currentSegmentClients[1], seenBefore1Month)
// create 3 additional clients
// these have ns-1, ns-2, ns-3 and mnt-1, mnt-2, mnt-3
moreSegmentClients := make([]*activity.EntityRecord, 0, 3)
for i := 0; i < 3; i++ {
moreSegmentClients = append(moreSegmentClients, &activity.EntityRecord{
ClientID: fmt.Sprintf("id-%d", i+3),
NamespaceID: fmt.Sprintf("ns-%d", i),
MountAccessor: fmt.Sprintf("ns-%d", i),
NonEntity: i == 1,
})
}
// 3 new clients have been seen 3 months ago
err := a.handleEntitySegment(&activity.EntityActivityLog{Clients: moreSegmentClients}, segmentTime, hll, pqOptions{
byNamespace: byNS,
byMonth: byMonth,
endTime: timeutil.EndOfMonth(segmentTime),
activePeriodStart: addMonths(-12),
activePeriodEnd: addMonths(12),
})
require.NoError(t, err)
// there are only 3 namespaces, since both currentSegmentClients and moreSegmentClients use the same namespaces
require.Len(t, byNS, 3)
verifyByNamespaceContains(t, byNS, currentSegmentClients...)
verifyByNamespaceContains(t, byNS, moreSegmentClients...)
// The segment clients that have already been seen have their same first seen dates
require.Equal(t, byMonth.firstSeen(t, currentSegmentClients[0]), seenBefore2Months)
require.Equal(t, byMonth.firstSeen(t, currentSegmentClients[1]), seenBefore1Month)
require.Equal(t, byMonth.firstSeen(t, currentSegmentClients[2]), seenBefore2Months)
// and the new clients should be first seen at segmentTime
require.Equal(t, byMonth.firstSeen(t, moreSegmentClients[0]), segmentTime)
require.Equal(t, byMonth.firstSeen(t, moreSegmentClients[1]), segmentTime)
require.Equal(t, byMonth.firstSeen(t, moreSegmentClients[2]), segmentTime)
// the hyperloglog will have 3 elements, because there were the 3 new elements in moreSegmentClients seen
require.Equal(t, hll.Estimate(), uint64(3))
})
}
// TestActivityLog_breakdownTokenSegment verifies that tokens are correctly added to a map that tracks counts per namespace
func TestActivityLog_breakdownTokenSegment(t *testing.T) {
toAdd := map[string]uint64{
"a": 1,
"b": 2,
"c": 3,
}
a := &ActivityLog{}
testCases := []struct {
name string
existingNamespaceCounts map[string]uint64
wantCounts map[string]uint64
}{
{
name: "empty",
wantCounts: toAdd,
},
{
name: "some overlap",
existingNamespaceCounts: map[string]uint64{
"a": 2,
"z": 1,
},
wantCounts: map[string]uint64{
"a": 3,
"b": 2,
"c": 3,
"z": 1,
},
},
{
name: "disjoint sets",
existingNamespaceCounts: map[string]uint64{
"z": 5,
"y": 3,
"x": 2,
},
wantCounts: map[string]uint64{
"a": 1,
"b": 2,
"c": 3,
"z": 5,
"y": 3,
"x": 2,
},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
byNamespace := make(map[string]*processByNamespace)
for k, v := range tc.existingNamespaceCounts {
byNamespace[k] = newByNamespace()
byNamespace[k].Counts.Tokens = v
}
a.breakdownTokenSegment(&activity.TokenCount{CountByNamespaceID: toAdd}, byNamespace)
got := make(map[string]uint64)
for k, v := range byNamespace {
got[k] = v.Counts.Tokens
}
require.Equal(t, tc.wantCounts, got)
})
}
}
// TestActivityLog_writePrecomputedQuery calls writePrecomputedQuery for a segment with 1 non entity and 1 entity client,
// which have different namespaces and mounts. The precomputed query is then retrieved from storage and we verify that
// the data structure is filled correctly
func TestActivityLog_writePrecomputedQuery(t *testing.T) {
core, _, _ := TestCoreUnsealed(t)
a := core.activityLog
a.SetEnable(true)
byMonth := make(summaryByMonth)
byNS := make(summaryByNamespace)
clientEntity := &activity.EntityRecord{
ClientID: "id-1",
NamespaceID: "ns-1",
MountAccessor: "mnt-1",
}
clientNonEntity := &activity.EntityRecord{
ClientID: "id-2",
NamespaceID: "ns-2",
MountAccessor: "mnt-2",
NonEntity: true,
}
now := time.Now()
// add the 2 clients to the namespace and month summaries
processClientRecord(clientEntity, byNS, byMonth, now)
processClientRecord(clientNonEntity, byNS, byMonth, now)
endTime := timeutil.EndOfMonth(now)
opts := pqOptions{
byNamespace: byNS,
byMonth: byMonth,
endTime: endTime,
}
err := a.writePrecomputedQuery(context.Background(), now, opts)
require.NoError(t, err)
// read the query back from storage
val, err := a.queryStore.Get(context.Background(), now, endTime)
require.NoError(t, err)
require.Equal(t, now.UTC().Unix(), val.StartTime.UTC().Unix())
require.Equal(t, endTime.UTC().Unix(), val.EndTime.UTC().Unix())
// ns-1 and ns-2 should both be present in the results
require.Len(t, val.Namespaces, 2)
require.Len(t, val.Months, 1)
resultByNS := make(map[string]*activity.NamespaceRecord)
for _, ns := range val.Namespaces {
resultByNS[ns.NamespaceID] = ns
}
ns1 := resultByNS["ns-1"]
ns2 := resultByNS["ns-2"]
require.Equal(t, ns1.Entities, uint64(1))
require.Equal(t, ns1.NonEntityTokens, uint64(0))
require.Equal(t, ns2.Entities, uint64(0))
require.Equal(t, ns2.NonEntityTokens, uint64(1))
require.Len(t, ns1.Mounts, 1)
require.Len(t, ns2.Mounts, 1)
// ns-1 needs to have mnt-1
require.Contains(t, ns1.Mounts[0].MountPath, "mnt-1")
// ns-2 needs to have mnt-2
require.Contains(t, ns2.Mounts[0].MountPath, "mnt-2")
require.Equal(t, 1, ns1.Mounts[0].Counts.EntityClients)
require.Equal(t, 0, ns1.Mounts[0].Counts.NonEntityClients)
require.Equal(t, 0, ns2.Mounts[0].Counts.EntityClients)
require.Equal(t, 1, ns2.Mounts[0].Counts.NonEntityClients)
monthRecord := val.Months[0]
// there should only be one month present, since the clients were added with the same timestamp
require.Equal(t, monthRecord.Timestamp, timeutil.StartOfMonth(now).UTC().Unix())
require.Equal(t, 1, monthRecord.Counts.NonEntityClients)
require.Equal(t, 1, monthRecord.Counts.EntityClients)
require.Len(t, monthRecord.Namespaces, 2)
require.Len(t, monthRecord.NewClients.Namespaces, 2)
require.Equal(t, 1, monthRecord.NewClients.Counts.EntityClients)
require.Equal(t, 1, monthRecord.NewClients.Counts.NonEntityClients)
}
type mockTimeNowClock struct {
timeutil.DefaultClock
start time.Time
created time.Time
}
func newMockTimeNowClock(startAt time.Time) timeutil.Clock {
return &mockTimeNowClock{start: startAt, created: time.Now()}
}
// NewTimer returns a timer with a channel that will return the correct time,
// relative to the starting time. This is used when testing the
// activeFragmentWorker, as that function uses the returned value from timer.C
// to perform additional functionality
func (m mockTimeNowClock) NewTimer(d time.Duration) *time.Timer {
timerStarted := m.Now()
t := time.NewTimer(d)
readCh := t.C
writeCh := make(chan time.Time, 1)
go func() {
<-readCh
writeCh <- timerStarted.Add(d)
}()
t.C = writeCh
return t
}
func (m mockTimeNowClock) Now() time.Time {
return m.start.Add(time.Since(m.created))
}
// TestActivityLog_HandleEndOfMonth runs the activity log with a mock clock.
// The current time is set to be 3 seconds before the end of a month. The test
// verifies that the precomputedQueryWorker runs and writes precomputed queries
// with the proper start and end times when the end of the month is triggered
func TestActivityLog_HandleEndOfMonth(t *testing.T) {
// 3 seconds until a new month
now := time.Date(2021, 1, 31, 23, 59, 57, 0, time.UTC)
core, _, _ := TestCoreUnsealedWithConfig(t, &CoreConfig{ActivityLogConfig: ActivityLogCoreConfig{Clock: newMockTimeNowClock(now)}})
done := make(chan struct{})
go func() {
defer close(done)
<-core.activityLog.precomputedQueryWritten
}()
core.activityLog.SetEnable(true)
core.activityLog.SetStartTimestamp(now.Unix())
core.activityLog.AddClientToFragment("id", "ns", now.Unix(), false, "mount")
// wait for the end of month to be triggered
select {
case <-done:
case <-time.After(10 * time.Second):
t.Fatal("timeout waiting for precomputed query")
}
// verify that a precomputed query was written
exists, err := core.activityLog.queryStore.QueriesAvailable(context.Background())
require.NoError(t, err)
require.True(t, exists)
// verify that the timestamp is correct
pq, err := core.activityLog.queryStore.Get(context.Background(), now, now.Add(24*time.Hour))
require.NoError(t, err)
require.Equal(t, now, pq.StartTime)
require.Equal(t, timeutil.EndOfMonth(now), pq.EndTime)
}

6
vault/activity_log_testing_util.go
View File

@@ -5,10 +5,8 @@ import (
"fmt"
"math/rand"
"testing"
"time"
"github.com/hashicorp/vault/helper/constants"
"github.com/hashicorp/vault/sdk/logical"
"github.com/hashicorp/vault/vault/activity"
)
@@ -29,7 +27,7 @@ func (c *Core) InjectActivityLogDataThisMonth(t *testing.T) map[string]*activity
ClientID: fmt.Sprintf("testclientid-%d", i),
NamespaceID: "root",
MountAccessor: fmt.Sprintf("testmountaccessor-%d", i),
Timestamp: time.Now().Unix(),
Timestamp: c.activityLog.clock.Now().Unix(),
NonEntity: i%2 == 0,
}
c.activityLog.partialMonthClientTracker[er.ClientID] = er
@@ -42,7 +40,7 @@ func (c *Core) InjectActivityLogDataThisMonth(t *testing.T) map[string]*activity
ClientID: fmt.Sprintf("ns-%d-testclientid-%d", j, i),
NamespaceID: fmt.Sprintf("ns-%d", j),
MountAccessor: fmt.Sprintf("ns-%d-testmountaccessor-%d", j, i),
Timestamp: time.Now().Unix(),
Timestamp: c.activityLog.clock.Now().Unix(),
NonEntity: i%2 == 0,
}
c.activityLog.partialMonthClientTracker[er.ClientID] = er

8
vault/activity_log_util.go
View File

@@ -2,9 +2,15 @@
package vault
import "context"
import (
"context"
"time"
)
// sendCurrentFragment is a no-op on OSS
func (a *ActivityLog) sendCurrentFragment(ctx context.Context) error {
return nil
}
// CensusReport is a no-op on OSS
func (a *ActivityLog) CensusReport(context.Context, CensusReporter, time.Time) {}

2
vault/activity_log_util_common.go
View File

@@ -72,7 +72,7 @@ func (a *ActivityLog) StoreHyperlogLog(ctx context.Context, startTime time.Time,
}
func (a *ActivityLog) computeCurrentMonthForBillingPeriodInternal(ctx context.Context, byMonth map[int64]*processMonth, hllGetFunc HLLGetter, startTime time.Time, endTime time.Time) (*activity.MonthRecord, error) {
if timeutil.IsCurrentMonth(startTime, time.Now().UTC()) {
if timeutil.IsCurrentMonth(startTime, a.clock.Now().UTC()) {
monthlyComputation := a.transformMonthBreakdowns(byMonth)
if len(monthlyComputation) > 1 {
a.logger.Warn("monthly in-memory activitylog computation returned multiple months of data", "months returned", len(byMonth))

5
vault/activity_log_util_common_test.go
View File

@@ -15,6 +15,11 @@ import (
"google.golang.org/protobuf/proto"
)
// Test_ActivityLog_ComputeCurrentMonthForBillingPeriodInternal creates 3 months of hyperloglogs and fills them with
// overlapping clients. The test calls computeCurrentMonthForBillingPeriodInternal with the current month map having
// some overlap with the previous months. The test then verifies that the results have the correct number of entity and
// non-entity clients. The test also calls computeCurrentMonthForBillingPeriodInternal with an empty current month map,
// and verifies that the results are all 0.
func Test_ActivityLog_ComputeCurrentMonthForBillingPeriodInternal(t *testing.T) {
// populate the first month with clients 1-10
monthOneHLL := hyperloglog.New()

16
vault/census.go Normal file
View File

@@ -0,0 +1,16 @@
//go:build !enterprise
package vault
import "time"
// CensusAgent is a stub for OSS
type CensusReporter interface{}
// setupCensusAgent is a stub for OSS.
func (c *Core) setupCensusAgent() error { return nil }
func (c *Core) BillingStart() time.Time { return time.Time{} }
func (c *Core) CensusLicensingEnabled() bool { return false }
func (c *Core) CensusAgent() CensusReporter { return nil }
func (c *Core) ReloadCensus() error { return nil }
func (c *Core) teardownCensusAgent() error { return nil }

18
vault/core.go
View File

@@ -416,6 +416,8 @@ type Core struct {
// activityLog is used to track active client count
activityLog *ActivityLog
// activityLogLock protects the activityLog and activityLogConfig
activityLogLock sync.RWMutex
// metricsCh is used to stop the metrics streaming
metricsCh chan struct{}
@@ -630,8 +632,12 @@ type Core struct {
clusterHeartbeatInterval time.Duration
// activityLogConfig contains override values for the activity log
// it is protected by activityLogLock
activityLogConfig ActivityLogCoreConfig
censusConfig atomic.Value
// activeTime is set on active nodes indicating the time at which this node
// became active.
activeTime time.Time
@@ -795,6 +801,9 @@ type CoreConfig struct {
LicensePath string
LicensingConfig *LicensingConfig
// Configured Census Agent
CensusAgent CensusReporter
DisablePerformanceStandby bool
DisableIndexing bool
DisableKeyEncodingChecks bool
@@ -2333,6 +2342,11 @@ func (s standardUnsealStrategy) unseal(ctx context.Context, logger log.Logger, c
if err := c.setupAuditedHeadersConfig(ctx); err != nil {
return err
}
if err := c.setupCensusAgent(); err != nil {
c.logger.Error("skipping reporting for nil agent", "error", err)
}
// not waiting on wg to avoid changing existing behavior
var wg sync.WaitGroup
if err := c.setupActivityLog(ctx, &wg); err != nil {
@@ -2533,6 +2547,10 @@ func (c *Core) preSeal() error {
result = multierror.Append(result, fmt.Errorf("error stopping expiration: %w", err))
}
c.stopActivityLog()
// Clean up the censusAgent on seal
if err := c.teardownCensusAgent(); err != nil {
result = multierror.Append(result, fmt.Errorf("error tearing down reporting agent: %w", err))
}
if err := c.teardownCredentials(context.Background()); err != nil {
result = multierror.Append(result, fmt.Errorf("error tearing down credentials: %w", err))

50
vault/logical_system_activity.go
View File

@@ -19,6 +19,10 @@ func (b *SystemBackend) activityQueryPath() *framework.Path {
return &framework.Path{
Pattern: "internal/counters/activity$",
Fields: map[string]*framework.FieldSchema{
"current_billing_period": {
Type: framework.TypeBool,
Description: "Query utilization for configured billing period",
},
"start_time": {
Type: framework.TypeTime,
Description: "Start of query interval",
@@ -167,7 +171,9 @@ func parseStartEndTimes(a *ActivityLog, d *framework.FieldData) (time.Time, time
// This endpoint is not used by the UI. The UI's "export" feature is entirely client-side.
func (b *SystemBackend) handleClientExport(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
b.Core.activityLogLock.RLock()
a := b.Core.activityLog
b.Core.activityLogLock.RUnlock()
if a == nil {
return logical.ErrorResponse("no activity log present"), nil
}
@@ -198,14 +204,23 @@ func (b *SystemBackend) handleClientExport(ctx context.Context, req *logical.Req
}
func (b *SystemBackend) handleClientMetricQuery(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
var startTime, endTime time.Time
b.Core.activityLogLock.RLock()
a := b.Core.activityLog
b.Core.activityLogLock.RUnlock()
if a == nil {
return logical.ErrorResponse("no activity log present"), nil
}
startTime, endTime, err := parseStartEndTimes(a, d)
if err != nil {
return logical.ErrorResponse(err.Error()), nil
if d.Get("current_billing_period").(bool) {
startTime = b.Core.BillingStart()
endTime = time.Now().UTC()
} else {
var err error
startTime, endTime, err = parseStartEndTimes(a, d)
if err != nil {
return logical.ErrorResponse(err.Error()), nil
}
}
var limitNamespaces int
@@ -228,7 +243,9 @@ func (b *SystemBackend) handleClientMetricQuery(ctx context.Context, req *logica
}
func (b *SystemBackend) handleMonthlyActivityCount(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
b.Core.activityLogLock.RLock()
a := b.Core.activityLog
b.Core.activityLogLock.RUnlock()
if a == nil {
return logical.ErrorResponse("no activity log present"), nil
}
@@ -247,7 +264,9 @@ func (b *SystemBackend) handleMonthlyActivityCount(ctx context.Context, req *log
}
func (b *SystemBackend) handleActivityConfigRead(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
b.Core.activityLogLock.RLock()
a := b.Core.activityLog
b.Core.activityLogLock.RUnlock()
if a == nil {
return logical.ErrorResponse("no activity log present"), nil
}
@@ -268,16 +287,21 @@ func (b *SystemBackend) handleActivityConfigRead(ctx context.Context, req *logic
return &logical.Response{
Data: map[string]interface{}{
"default_report_months": config.DefaultReportMonths,
"retention_months": config.RetentionMonths,
"enabled": config.Enabled,
"queries_available": qa,
"default_report_months": config.DefaultReportMonths,
"retention_months": config.RetentionMonths,
"enabled": config.Enabled,
"queries_available": qa,
"reporting_enabled": b.Core.CensusLicensingEnabled(),
"billing_start_timestamp": b.Core.BillingStart(),
"minimum_retention_months": a.configOverrides.MinimumRetentionMonths,
},
}, nil
}
func (b *SystemBackend) handleActivityConfigUpdate(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
b.Core.activityLogLock.RLock()
a := b.Core.activityLog
b.Core.activityLogLock.RUnlock()
if a == nil {
return logical.ErrorResponse("no activity log present"), nil
}
@@ -326,6 +350,11 @@ func (b *SystemBackend) handleActivityConfigUpdate(ctx context.Context, req *log
if config.Enabled == "enable" && enabledStr == "disable" ||
!activityLogEnabledDefault && config.Enabled == "enable" && enabledStr == "default" ||
activityLogEnabledDefault && config.Enabled == "default" && enabledStr == "disable" {
// if census is enabled, the activity log cannot be disabled
if a.core.CensusLicensingEnabled() {
return logical.ErrorResponse("cannot disable the activity log while Reporting is enabled"), logical.ErrInvalidRequest
}
warnings = append(warnings, "the current monthly segment will be deleted because the activity log was disabled")
}
@@ -338,6 +367,9 @@ func (b *SystemBackend) handleActivityConfigUpdate(ctx context.Context, req *log
}
}
a.core.activityLogLock.RLock()
minimumRetentionMonths := a.configOverrides.MinimumRetentionMonths
a.core.activityLogLock.RUnlock()
enabled := config.Enabled == "enable"
if !enabled && config.Enabled == "default" {
enabled = activityLogEnabledDefault
@@ -347,6 +379,10 @@ func (b *SystemBackend) handleActivityConfigUpdate(ctx context.Context, req *log
return logical.ErrorResponse("retention_months cannot be 0 while enabled"), logical.ErrInvalidRequest
}
if a.core.CensusLicensingEnabled() && config.RetentionMonths < minimumRetentionMonths {
return logical.ErrorResponse("retention_months must be at least %d while Reporting is enabled", minimumRetentionMonths), logical.ErrInvalidRequest
}
// Store the config
entry, err := logical.StorageEntryJSON(path.Join(activitySubPath, activityConfigKey), config)
if err != nil {

390
vault/logical_system_activity_write_testonly.go
View File

@@ -7,9 +7,17 @@ package vault
import (
"context"
"fmt"
"io"
"sync"
"time"
"github.com/hashicorp/go-uuid"
"github.com/hashicorp/vault/helper/namespace"
"github.com/hashicorp/vault/helper/timeutil"
"github.com/hashicorp/vault/sdk/framework"
"github.com/hashicorp/vault/sdk/logical"
"github.com/hashicorp/vault/vault/activity"
"github.com/hashicorp/vault/vault/activity/generation"
"google.golang.org/protobuf/encoding/protojson"
)
@@ -49,5 +57,385 @@ func (b *SystemBackend) handleActivityWriteData(ctx context.Context, request *lo
if len(input.Data) == 0 {
return logical.ErrorResponse("Missing required \"data\" values"), logical.ErrInvalidRequest
}
return nil, nil
numMonths := 0
for _, month := range input.Data {
if int(month.GetMonthsAgo()) > numMonths {
numMonths = int(month.GetMonthsAgo())
}
}
generated := newMultipleMonthsActivityClients(numMonths + 1)
for _, month := range input.Data {
err := generated.processMonth(ctx, b.Core, month)
if err != nil {
return logical.ErrorResponse("failed to process data for month %d", month.GetMonthsAgo()), err
}
}
opts := make(map[generation.WriteOptions]struct{}, len(input.Write))
for _, opt := range input.Write {
opts[opt] = struct{}{}
}
paths, err := generated.write(ctx, opts, b.Core.activityLog)
if err != nil {
return logical.ErrorResponse("failed to write data"), err
}
return &logical.Response{
Data: map[string]interface{}{
"paths": paths,
},
}, nil
}
// singleMonthActivityClients holds a single month's client IDs, in the order they were seen
type singleMonthActivityClients struct {
// clients are indexed by ID
clients []*activity.EntityRecord
// predefinedSegments map from the segment number to the client's index in
// the clients slice
predefinedSegments map[int][]int
// generationParameters holds the generation request
generationParameters *generation.Data
}
// multipleMonthsActivityClients holds multiple month's data
type multipleMonthsActivityClients struct {
// months are in order, with month 0 being the current month and index 1 being 1 month ago
months []*singleMonthActivityClients
}
func (s *singleMonthActivityClients) addEntityRecord(record *activity.EntityRecord, segmentIndex *int) {
s.clients = append(s.clients, record)
if segmentIndex != nil {
index := len(s.clients) - 1
s.predefinedSegments[*segmentIndex] = append(s.predefinedSegments[*segmentIndex], index)
}
}
// populateSegments converts a month of clients into a segmented map. The map's
// keys are the segment index, and the value are the clients that were seen in
// that index. If the value is an empty slice, then it's an empty index. If the
// value is nil, then it's a skipped index
func (s *singleMonthActivityClients) populateSegments() (map[int][]*activity.EntityRecord, error) {
segments := make(map[int][]*activity.EntityRecord)
ignoreIndexes := make(map[int]struct{})
skipIndexes := s.generationParameters.SkipSegmentIndexes
emptyIndexes := s.generationParameters.EmptySegmentIndexes
for _, i := range skipIndexes {
segments[int(i)] = nil
ignoreIndexes[int(i)] = struct{}{}
}
for _, i := range emptyIndexes {
segments[int(i)] = make([]*activity.EntityRecord, 0, 0)
ignoreIndexes[int(i)] = struct{}{}
}
// if we have predefined segments, then we can construct the map using those
if len(s.predefinedSegments) > 0 {
for segment, clientIndexes := range s.predefinedSegments {
clientsInSegment := make([]*activity.EntityRecord, 0, len(clientIndexes))
for _, idx := range clientIndexes {
clientsInSegment = append(clientsInSegment, s.clients[idx])
}
segments[segment] = clientsInSegment
}
return segments, nil
}
totalSegmentCount := 1
if s.generationParameters.GetNumSegments() > 0 {
totalSegmentCount = int(s.generationParameters.GetNumSegments())
}
numNonUsable := len(skipIndexes) + len(emptyIndexes)
usableSegmentCount := totalSegmentCount - numNonUsable
if usableSegmentCount <= 0 {
return nil, fmt.Errorf("num segments %d is too low, it must be greater than %d (%d skipped indexes + %d empty indexes)", totalSegmentCount, numNonUsable, len(skipIndexes), len(emptyIndexes))
}
// determine how many clients should be in each segment
segmentSizes := len(s.clients) / usableSegmentCount
if len(s.clients)%usableSegmentCount != 0 {
segmentSizes++
}
clientIndex := 0
for i := 0; i < totalSegmentCount; i++ {
if clientIndex >= len(s.clients) {
break
}
if _, ok := ignoreIndexes[i]; ok {
continue
}
for len(segments[i]) < segmentSizes && clientIndex < len(s.clients) {
segments[i] = append(segments[i], s.clients[clientIndex])
clientIndex++
}
}
return segments, nil
}
// addNewClients generates clients according to the given parameters, and adds them to the month
// the client will always have the mountAccessor as its mount accessor
func (s *singleMonthActivityClients) addNewClients(c *generation.Client, mountAccessor string, segmentIndex *int) error {
count := 1
if c.Count > 1 {
count = int(c.Count)
}
for i := 0; i < count; i++ {
record := &activity.EntityRecord{
ClientID: c.Id,
NamespaceID: c.Namespace,
NonEntity: c.NonEntity,
MountAccessor: mountAccessor,
}
if record.ClientID == "" {
var err error
record.ClientID, err = uuid.GenerateUUID()
if err != nil {
return err
}
}
s.addEntityRecord(record, segmentIndex)
}
return nil
}
// processMonth populates a month of client data
func (m *multipleMonthsActivityClients) processMonth(ctx context.Context, core *Core, month *generation.Data) error {
// default to using the root namespace and the first mount on the root namespace
mounts, err := core.ListMounts()
if err != nil {
return err
}
defaultMountAccessorRootNS := ""
for _, mount := range mounts {
if mount.NamespaceID == namespace.RootNamespaceID {
defaultMountAccessorRootNS = mount.Accessor
break
}
}
m.months[month.GetMonthsAgo()].generationParameters = month
add := func(c []*generation.Client, segmentIndex *int) error {
for _, clients := range c {
if clients.Namespace == "" {
clients.Namespace = namespace.RootNamespaceID
}
// verify that the namespace exists
ns, err := core.NamespaceByID(ctx, clients.Namespace)
if err != nil {
return err
}
// verify that the mount exists
if clients.Mount != "" {
nctx := namespace.ContextWithNamespace(ctx, ns)
mountEntry := core.router.MatchingMountEntry(nctx, clients.Mount)
if mountEntry == nil {
return fmt.Errorf("unable to find matching mount in namespace %s", clients.Namespace)
}
}
mountAccessor := defaultMountAccessorRootNS
if clients.Namespace != namespace.RootNamespaceID && clients.Mount == "" {
// if we're not using the root namespace, find a mount on the namespace that we are using
found := false
for _, mount := range mounts {
if mount.NamespaceID == clients.Namespace {
mountAccessor = mount.Accessor
found = true
break
}
}
if !found {
return fmt.Errorf("unable to find matching mount in namespace %s", clients.Namespace)
}
}
err = m.addClientToMonth(month.GetMonthsAgo(), clients, mountAccessor, segmentIndex)
if err != nil {
return err
}
}
return nil
}
if month.GetAll() != nil {
return add(month.GetAll().GetClients(), nil)
}
predefinedSegments := month.GetSegments()
for i, segment := range predefinedSegments.GetSegments() {
index := i
if segment.SegmentIndex != nil {
index = int(*segment.SegmentIndex)
}
err = add(segment.GetClients().GetClients(), &index)
if err != nil {
return err
}
}
return nil
}
func (m *multipleMonthsActivityClients) addClientToMonth(monthsAgo int32, c *generation.Client, mountAccessor string, segmentIndex *int) error {
if c.Repeated || c.RepeatedFromMonth > 0 {
return m.addRepeatedClients(monthsAgo, c, mountAccessor, segmentIndex)
}
return m.months[monthsAgo].addNewClients(c, mountAccessor, segmentIndex)
}
func (m *multipleMonthsActivityClients) addRepeatedClients(monthsAgo int32, c *generation.Client, mountAccessor string, segmentIndex *int) error {
addingTo := m.months[monthsAgo]
repeatedFromMonth := monthsAgo + 1
if c.RepeatedFromMonth > 0 {
repeatedFromMonth = c.RepeatedFromMonth
}
repeatedFrom := m.months[repeatedFromMonth]
numClients := 1
if c.Count > 0 {
numClients = int(c.Count)
}
for _, client := range repeatedFrom.clients {
if c.NonEntity == client.NonEntity && mountAccessor == client.MountAccessor && c.Namespace == client.NamespaceID {
addingTo.addEntityRecord(client, segmentIndex)
numClients--
if numClients == 0 {
break
}
}
}
if numClients > 0 {
return fmt.Errorf("missing repeated %d clients matching given parameters", numClients)
}
return nil
}
func (m *multipleMonthsActivityClients) write(ctx context.Context, opts map[generation.WriteOptions]struct{}, activityLog *ActivityLog) ([]string, error) {
now := timeutil.StartOfMonth(time.Now().UTC())
paths := []string{}
_, writePQ := opts[generation.WriteOptions_WRITE_PRECOMPUTED_QUERIES]
_, writeDistinctClients := opts[generation.WriteOptions_WRITE_DISTINCT_CLIENTS]
pqOpts := pqOptions{}
if writePQ || writeDistinctClients {
pqOpts.byNamespace = make(map[string]*processByNamespace)
pqOpts.byMonth = make(map[int64]*processMonth)
pqOpts.activePeriodEnd = m.latestTimestamp(now)
pqOpts.endTime = timeutil.EndOfMonth(pqOpts.activePeriodEnd)
pqOpts.activePeriodStart = m.earliestTimestamp(now)
}
for i, month := range m.months {
if month.generationParameters == nil {
continue
}
var timestamp time.Time
if i > 0 {
timestamp = timeutil.StartOfMonth(timeutil.MonthsPreviousTo(i, now))
} else {
timestamp = now
}
segments, err := month.populateSegments()
if err != nil {
return nil, err
}
for segmentIndex, segment := range segments {
if _, ok := opts[generation.WriteOptions_WRITE_ENTITIES]; ok {
if segment == nil {
// skip the index
continue
}
entityPath, err := activityLog.saveSegmentEntitiesInternal(ctx, segmentInfo{
startTimestamp: timestamp.Unix(),
currentClients: &activity.EntityActivityLog{Clients: segment},
clientSequenceNumber: uint64(segmentIndex),
tokenCount: &activity.TokenCount{},
}, true)
if err != nil {
return nil, err
}
paths = append(paths, entityPath)
}
}
if writePQ || writeDistinctClients {
reader := newProtoSegmentReader(segments)
err = activityLog.segmentToPrecomputedQuery(ctx, timestamp, reader, pqOpts)
if err != nil {
return nil, err
}
}
}
wg := sync.WaitGroup{}
err := activityLog.refreshFromStoredLog(ctx, &wg, now)
if err != nil {
return nil, err
}
return paths, nil
}
func (m *multipleMonthsActivityClients) latestTimestamp(now time.Time) time.Time {
for i, month := range m.months {
if month.generationParameters != nil {
return timeutil.StartOfMonth(timeutil.MonthsPreviousTo(i, now))
}
}
return time.Time{}
}
func (m *multipleMonthsActivityClients) earliestTimestamp(now time.Time) time.Time {
for i := len(m.months) - 1; i >= 0; i-- {
month := m.months[i]
if month.generationParameters != nil {
return timeutil.StartOfMonth(timeutil.MonthsPreviousTo(i, now))
}
}
return time.Time{}
}
func newMultipleMonthsActivityClients(numberOfMonths int) *multipleMonthsActivityClients {
m := &multipleMonthsActivityClients{
months: make([]*singleMonthActivityClients, numberOfMonths),
}
for i := 0; i < numberOfMonths; i++ {
m.months[i] = &singleMonthActivityClients{
predefinedSegments: make(map[int][]int),
}
}
return m
}
func newProtoSegmentReader(segments map[int][]*activity.EntityRecord) SegmentReader {
allRecords := make([][]*activity.EntityRecord, 0, len(segments))
for _, records := range segments {
if segments == nil {
continue
}
allRecords = append(allRecords, records)
}
return &sliceSegmentReader{
records: allRecords,
}
}
type sliceSegmentReader struct {
records [][]*activity.EntityRecord
i int
}
func (p *sliceSegmentReader) ReadToken(ctx context.Context) (*activity.TokenCount, error) {
return nil, io.EOF
}
func (p *sliceSegmentReader) ReadEntity(ctx context.Context) (*activity.EntityActivityLog, error) {
if p.i == len(p.records) {
return nil, io.EOF
}
record := p.records[p.i]
p.i++
return &activity.EntityActivityLog{Clients: record}, nil
}

523
vault/logical_system_activity_write_testonly_test.go
View File

@@ -6,11 +6,19 @@
package vault
import (
"context"
"sort"
"testing"
"time"
"github.com/hashicorp/vault/helper/namespace"
"github.com/hashicorp/vault/helper/timeutil"
"github.com/hashicorp/vault/sdk/logical"
"github.com/hashicorp/vault/vault/activity"
"github.com/hashicorp/vault/vault/activity/generation"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/proto"
)
// TestSystemBackend_handleActivityWriteData calls the activity log write endpoint and confirms that the inputs are
@@ -21,6 +29,7 @@ func TestSystemBackend_handleActivityWriteData(t *testing.T) {
operation logical.Operation
input map[string]interface{}
wantError error
wantPaths int
}{
{
name: "read fails",
@@ -67,6 +76,12 @@ func TestSystemBackend_handleActivityWriteData(t *testing.T) {
operation: logical.CreateOperation,
input: map[string]interface{}{"input": `{"write":["WRITE_PRECOMPUTED_QUERIES"],"data":[{"current_month":true,"all":{"clients":[{"count":5}]}}]}`},
},
{
name: "entities with multiple segments",
operation: logical.CreateOperation,
input: map[string]interface{}{"input": `{"write":["WRITE_ENTITIES"],"data":[{"current_month":true,"num_segments":3,"all":{"clients":[{"count":5}]}}]}`},
wantPaths: 3,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
@@ -78,7 +93,515 @@ func TestSystemBackend_handleActivityWriteData(t *testing.T) {
require.Equal(t, tc.wantError, err, resp.Error())
} else {
require.NoError(t, err)
paths := resp.Data["paths"].([]string)
require.Len(t, paths, tc.wantPaths)
}
})
}
}
// Test_singleMonthActivityClients_addNewClients verifies that new clients are
// created correctly, adhering to the requested parameters. The clients should
// use the inputted mount and a generated ID if one is not supplied. The new
// client should be added to the month's `clients` slice and segment map, if
// a segment index is supplied
func Test_singleMonthActivityClients_addNewClients(t *testing.T) {
segmentIndex := 0
tests := []struct {
name string
mount string
clients *generation.Client
wantNamespace string
wantMount string
wantID string
segmentIndex *int
}{
{
name: "default mount is used",
mount: "default_mount",
wantMount: "default_mount",
clients: &generation.Client{},
},
{
name: "record namespace is used, default mount is used",
mount: "default_mount",
wantNamespace: "ns",
wantMount: "default_mount",
clients: &generation.Client{
Namespace: "ns",
Mount: "mount",
},
},
{
name: "predefined ID is used",
clients: &generation.Client{
Id: "client_id",
},
wantID: "client_id",
},
{
name: "non zero count",
clients: &generation.Client{
Count: 5,
},
},
{
name: "non entity client",
clients: &generation.Client{
NonEntity: true,
},
},
{
name: "added to segment",
clients: &generation.Client{},
segmentIndex: &segmentIndex,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
m := &singleMonthActivityClients{
predefinedSegments: make(map[int][]int),
}
err := m.addNewClients(tt.clients, tt.mount, tt.segmentIndex)
require.NoError(t, err)
numNew := tt.clients.Count
if numNew == 0 {
numNew = 1
}
require.Len(t, m.clients, int(numNew))
for i, rec := range m.clients {
require.NotNil(t, rec)
require.Equal(t, tt.wantNamespace, rec.NamespaceID)
require.Equal(t, tt.wantMount, rec.MountAccessor)
require.Equal(t, tt.clients.NonEntity, rec.NonEntity)
if tt.wantID != "" {
require.Equal(t, tt.wantID, rec.ClientID)
} else {
require.NotEqual(t, "", rec.ClientID)
}
if tt.segmentIndex != nil {
require.Contains(t, m.predefinedSegments[*tt.segmentIndex], i)
}
}
})
}
}
// Test_multipleMonthsActivityClients_processMonth verifies that a month of data
// is added correctly. The test checks that default values are handled correctly
// for mounts and namespaces.
func Test_multipleMonthsActivityClients_processMonth(t *testing.T) {
core, _, _ := TestCoreUnsealed(t)
tests := []struct {
name string
clients *generation.Data
wantError bool
numMonths int
}{
{
name: "specified namespace and mount exist",
clients: &generation.Data{
Clients: &generation.Data_All{All: &generation.Clients{Clients: []*generation.Client{{
Namespace: namespace.RootNamespaceID,
Mount: "identity/",
}}}},
},
numMonths: 1,
},
{
name: "specified namespace exists, mount empty",
clients: &generation.Data{
Clients: &generation.Data_All{All: &generation.Clients{Clients: []*generation.Client{{
Namespace: namespace.RootNamespaceID,
}}}},
},
numMonths: 1,
},
{
name: "empty namespace and mount",
clients: &generation.Data{
Clients: &generation.Data_All{All: &generation.Clients{Clients: []*generation.Client{{}}}},
},
numMonths: 1,
},
{
name: "namespace doesn't exist",
clients: &generation.Data{
Clients: &generation.Data_All{All: &generation.Clients{Clients: []*generation.Client{{
Namespace: "abcd",
}}}},
},
wantError: true,
numMonths: 1,
},
{
name: "namespace exists, mount doesn't exist",
clients: &generation.Data{
Clients: &generation.Data_All{All: &generation.Clients{Clients: []*generation.Client{{
Namespace: namespace.RootNamespaceID,
Mount: "mount",
}}}},
},
wantError: true,
numMonths: 1,
},
{
name: "older month",
clients: &generation.Data{
Month: &generation.Data_MonthsAgo{MonthsAgo: 4},
Clients: &generation.Data_All{All: &generation.Clients{Clients: []*generation.Client{{}}}},
},
numMonths: 5,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
m := newMultipleMonthsActivityClients(tt.numMonths)
err := m.processMonth(context.Background(), core, tt.clients)
if tt.wantError {
require.Error(t, err)
} else {
require.NoError(t, err)
require.Len(t, m.months[tt.clients.GetMonthsAgo()].clients, len(tt.clients.GetAll().Clients))
for _, month := range m.months {
for _, c := range month.clients {
require.NotEmpty(t, c.NamespaceID)
require.NotEmpty(t, c.MountAccessor)
}
}
}
})
}
}
// Test_multipleMonthsActivityClients_processMonth_segmented verifies that segments
// are filled correctly when a month is processed with segmented data. The clients
// should be in the clients array, and should also be in the predefinedSegments map
// at the correct segment index
func Test_multipleMonthsActivityClients_processMonth_segmented(t *testing.T) {
index7 := int32(7)
data := &generation.Data{
Clients: &generation.Data_Segments{
Segments: &generation.Segments{
Segments: []*generation.Segment{
{
Clients: &generation.Clients{Clients: []*generation.Client{
{},
}},
},
{
Clients: &generation.Clients{Clients: []*generation.Client{{}}},
},
{
SegmentIndex: &index7,
Clients: &generation.Clients{Clients: []*generation.Client{{}}},
},
},
},
},
}
m := newMultipleMonthsActivityClients(1)
core, _, _ := TestCoreUnsealed(t)
require.NoError(t, m.processMonth(context.Background(), core, data))
require.Len(t, m.months[0].predefinedSegments, 3)
require.Len(t, m.months[0].clients, 3)
// segment indexes are correct
require.Contains(t, m.months[0].predefinedSegments, 0)
require.Contains(t, m.months[0].predefinedSegments, 1)
require.Contains(t, m.months[0].predefinedSegments, 7)
// the data in each segment is correct
require.Contains(t, m.months[0].predefinedSegments[0], 0)
require.Contains(t, m.months[0].predefinedSegments[1], 1)
require.Contains(t, m.months[0].predefinedSegments[7], 2)
}
// Test_multipleMonthsActivityClients_addRepeatedClients adds repeated clients
// from 1 month ago and 2 months ago, and verifies that the correct clients are
// added based on namespace, mount, and non-entity attributes
func Test_multipleMonthsActivityClients_addRepeatedClients(t *testing.T) {
m := newMultipleMonthsActivityClients(3)
defaultMount := "default"
require.NoError(t, m.addClientToMonth(2, &generation.Client{Count: 2}, "identity", nil))
require.NoError(t, m.addClientToMonth(2, &generation.Client{Count: 2, Namespace: "other_ns"}, defaultMount, nil))
require.NoError(t, m.addClientToMonth(1, &generation.Client{Count: 2}, defaultMount, nil))
require.NoError(t, m.addClientToMonth(1, &generation.Client{Count: 2, NonEntity: true}, defaultMount, nil))
month2Clients := m.months[2].clients
month1Clients := m.months[1].clients
thisMonth := m.months[0]
// this will match the first client in month 1
require.NoError(t, m.addRepeatedClients(0, &generation.Client{Count: 1, Repeated: true}, defaultMount, nil))
require.Contains(t, month1Clients, thisMonth.clients[0])
// this will match the 3rd client in month 1
require.NoError(t, m.addRepeatedClients(0, &generation.Client{Count: 1, Repeated: true, NonEntity: true}, defaultMount, nil))
require.Equal(t, month1Clients[2], thisMonth.clients[1])
// this will match the first two clients in month 1
require.NoError(t, m.addRepeatedClients(0, &generation.Client{Count: 2, Repeated: true}, defaultMount, nil))
require.Equal(t, month1Clients[0:2], thisMonth.clients[2:4])
// this will match the first client in month 2
require.NoError(t, m.addRepeatedClients(0, &generation.Client{Count: 1, RepeatedFromMonth: 2}, "identity", nil))
require.Equal(t, month2Clients[0], thisMonth.clients[4])
// this will match the 3rd client in month 2
require.NoError(t, m.addRepeatedClients(0, &generation.Client{Count: 1, RepeatedFromMonth: 2, Namespace: "other_ns"}, defaultMount, nil))
require.Equal(t, month2Clients[2], thisMonth.clients[5])
require.Error(t, m.addRepeatedClients(0, &generation.Client{Count: 1, RepeatedFromMonth: 2, Namespace: "other_ns"}, "other_mount", nil))
}
// Test_singleMonthActivityClients_populateSegments calls populateSegments for a
// collection of 5 clients, segmented in various ways. The test ensures that the
// resulting map has the correct clients for each segment index
func Test_singleMonthActivityClients_populateSegments(t *testing.T) {
clients := []*activity.EntityRecord{
{ClientID: "a"},
{ClientID: "b"},
{ClientID: "c"},
{ClientID: "d"},
{ClientID: "e"},
}
cases := []struct {
name string
segments map[int][]int
numSegments int
emptyIndexes []int32
skipIndexes []int32
wantSegments map[int][]*activity.EntityRecord
}{
{
name: "segmented",
segments: map[int][]int{
0: {0, 1},
1: {2, 3},
2: {4},
},
wantSegments: map[int][]*activity.EntityRecord{
0: {{ClientID: "a"}, {ClientID: "b"}},
1: {{ClientID: "c"}, {ClientID: "d"}},
2: {{ClientID: "e"}},
},
},
{
name: "segmented with skip and empty",
segments: map[int][]int{
0: {0, 1},
2: {0, 1},
},
emptyIndexes: []int32{1, 4},
skipIndexes: []int32{3},
wantSegments: map[int][]*activity.EntityRecord{
0: {{ClientID: "a"}, {ClientID: "b"}},
1: {},
2: {{ClientID: "a"}, {ClientID: "b"}},
3: nil,
4: {},
},
},
{
name: "all clients",
numSegments: 0,
wantSegments: map[int][]*activity.EntityRecord{
0: {{ClientID: "a"}, {ClientID: "b"}, {ClientID: "c"}, {ClientID: "d"}, {ClientID: "e"}},
},
},
{
name: "all clients split",
numSegments: 2,
wantSegments: map[int][]*activity.EntityRecord{
0: {{ClientID: "a"}, {ClientID: "b"}, {ClientID: "c"}},
1: {{ClientID: "d"}, {ClientID: "e"}},
},
},
{
name: "all clients with skip and empty",
numSegments: 5,
skipIndexes: []int32{0, 3},
emptyIndexes: []int32{2},
wantSegments: map[int][]*activity.EntityRecord{
0: nil,
1: {{ClientID: "a"}, {ClientID: "b"}, {ClientID: "c"}},
2: {},
3: nil,
4: {{ClientID: "d"}, {ClientID: "e"}},
},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
s := singleMonthActivityClients{predefinedSegments: tc.segments, clients: clients, generationParameters: &generation.Data{EmptySegmentIndexes: tc.emptyIndexes, SkipSegmentIndexes: tc.skipIndexes, NumSegments: int32(tc.numSegments)}}
gotSegments, err := s.populateSegments()
require.NoError(t, err)
require.Equal(t, tc.wantSegments, gotSegments)
})
}
}
// Test_handleActivityWriteData writes 4 months of data splitting some months
// across segments and using empty segments and skipped segments. Entities and
// precomputed queries are written. written and then storage is queried. The
// test verifies that the correct timestamps are present in the activity log and
// that the correct segment numbers for each month contain the correct number of
// clients
func Test_handleActivityWriteData(t *testing.T) {
index5 := int32(5)
index4 := int32(4)
data := []*generation.Data{
{
// segments: 0:[x,y], 1:[z]
Month: &generation.Data_MonthsAgo{MonthsAgo: 3},
Clients: &generation.Data_All{All: &generation.Clients{Clients: []*generation.Client{{Count: 3}}}},
NumSegments: 2,
},
{
// segments: 1:[a,b,c], 2:[d,e]
Month: &generation.Data_MonthsAgo{MonthsAgo: 2},
Clients: &generation.Data_All{All: &generation.Clients{Clients: []*generation.Client{{Count: 5}}}},
NumSegments: 3,
SkipSegmentIndexes: []int32{0},
},
{
// segments: 5:[f,g]
Month: &generation.Data_MonthsAgo{MonthsAgo: 1},
Clients: &generation.Data_Segments{
Segments: &generation.Segments{Segments: []*generation.Segment{{
SegmentIndex: &index5,
Clients: &generation.Clients{Clients: []*generation.Client{{Count: 2}}},
}}},
},
},
{
// segments: 1:[], 2:[], 4:[n], 5:[o]
Month: &generation.Data_CurrentMonth{},
EmptySegmentIndexes: []int32{1, 2},
Clients: &generation.Data_Segments{
Segments: &generation.Segments{Segments: []*generation.Segment{
{
SegmentIndex: &index5,
Clients: &generation.Clients{Clients: []*generation.Client{{Count: 1}}},
},
{
SegmentIndex: &index4,
Clients: &generation.Clients{Clients: []*generation.Client{{Count: 1}}},
},
}},
},
},
}
t.Run("write entitites", func(t *testing.T) {
core, _, _ := TestCoreUnsealed(t)
marshaled, err := protojson.Marshal(&generation.ActivityLogMockInput{
Data: data,
Write: []generation.WriteOptions{generation.WriteOptions_WRITE_ENTITIES},
})
require.NoError(t, err)
req := logical.TestRequest(t, logical.CreateOperation, "internal/counters/activity/write")
req.Data = map[string]interface{}{"input": string(marshaled)}
resp, err := core.systemBackend.HandleRequest(namespace.RootContext(nil), req)
require.NoError(t, err)
paths := resp.Data["paths"].([]string)
require.Len(t, paths, 9)
times, err := core.activityLog.availableLogs(context.Background())
require.NoError(t, err)
require.Len(t, times, 4)
sortPaths := func(monthPaths []string) {
sort.Slice(monthPaths, func(i, j int) bool {
iVal, _ := parseSegmentNumberFromPath(monthPaths[i])
jVal, _ := parseSegmentNumberFromPath(monthPaths[j])
return iVal < jVal
})
}
month0Paths := paths[0:4]
month1Paths := paths[4:5]
month2Paths := paths[5:7]
month3Paths := paths[7:9]
sortPaths(month0Paths)
sortPaths(month1Paths)
sortPaths(month2Paths)
sortPaths(month3Paths)
entities := func(paths []string) map[int][]*activity.EntityRecord {
segments := make(map[int][]*activity.EntityRecord)
for _, path := range paths {
segmentNum, _ := parseSegmentNumberFromPath(path)
entry, err := core.activityLog.view.Get(context.Background(), path)
require.NoError(t, err)
if entry == nil {
segments[segmentNum] = []*activity.EntityRecord{}
continue
}
activities := &activity.EntityActivityLog{}
err = proto.Unmarshal(entry.Value, activities)
require.NoError(t, err)
segments[segmentNum] = activities.Clients
}
return segments
}
month0Entities := entities(month0Paths)
require.Len(t, month0Entities, 4)
require.Contains(t, month0Entities, 1)
require.Contains(t, month0Entities, 2)
require.Contains(t, month0Entities, 4)
require.Contains(t, month0Entities, 5)
require.Len(t, month0Entities[1], 0)
require.Len(t, month0Entities[2], 0)
require.Len(t, month0Entities[4], 1)
require.Len(t, month0Entities[5], 1)
month1Entities := entities(month1Paths)
require.Len(t, month1Entities, 1)
require.Contains(t, month1Entities, 5)
require.Len(t, month1Entities[5], 2)
month2Entities := entities(month2Paths)
require.Len(t, month2Entities, 2)
require.Contains(t, month2Entities, 1)
require.Contains(t, month2Entities, 2)
require.Len(t, month2Entities[1], 3)
require.Len(t, month2Entities[2], 2)
month3Entities := entities(month3Paths)
require.Len(t, month3Entities, 2)
require.Contains(t, month3Entities, 0)
require.Contains(t, month3Entities, 1)
require.Len(t, month3Entities[0], 2)
require.Len(t, month3Entities[1], 1)
})
t.Run("write precomputed queries", func(t *testing.T) {
core, _, _ := TestCoreUnsealed(t)
marshaled, err := protojson.Marshal(&generation.ActivityLogMockInput{
Data: data,
Write: []generation.WriteOptions{generation.WriteOptions_WRITE_PRECOMPUTED_QUERIES},
})
require.NoError(t, err)
req := logical.TestRequest(t, logical.CreateOperation, "internal/counters/activity/write")
req.Data = map[string]interface{}{"input": string(marshaled)}
_, err = core.systemBackend.HandleRequest(namespace.RootContext(nil), req)
require.NoError(t, err)
queries, err := core.activityLog.queryStore.QueriesAvailable(context.Background())
require.NoError(t, err)
require.True(t, queries)
now := time.Now().UTC()
start := timeutil.StartOfMonth(timeutil.MonthsPreviousTo(3, now))
end := timeutil.EndOfMonth(now)
pq, err := core.activityLog.queryStore.Get(context.Background(), start, end)
require.NoError(t, err)
require.NotNil(t, pq)
require.Equal(t, end, pq.EndTime)
require.Equal(t, start, pq.StartTime)
require.Len(t, pq.Namespaces, 1)
require.Equal(t, uint64(12), pq.Namespaces[0].Entities)
require.Len(t, pq.Months, 4)
})
}

6
vault/logical_system_test.go
View File

@@ -30,6 +30,7 @@ import (
"github.com/hashicorp/vault/sdk/helper/compressutil"
"github.com/hashicorp/vault/sdk/helper/consts"
"github.com/hashicorp/vault/sdk/helper/jsonutil"
"github.com/hashicorp/vault/sdk/helper/logging"
"github.com/hashicorp/vault/sdk/helper/pluginutil"
"github.com/hashicorp/vault/sdk/helper/testhelpers/schema"
"github.com/hashicorp/vault/sdk/logical"
@@ -5096,7 +5097,10 @@ func TestSystemBackend_LoggersByName(t *testing.T) {
t.Run(fmt.Sprintf("loggers-by-name-%s", tc.logger), func(t *testing.T) {
t.Parallel()
core, b, _ := testCoreSystemBackend(t)
core, _, _ := TestCoreUnsealedWithConfig(t, &CoreConfig{
Logger: logging.NewVaultLogger(hclog.Trace),
})
b := core.systemBackend
// Test core overrides logging level outside of config,
// an initial delete will ensure that we an initial read

7
vault/request_handling.go
View File

@@ -431,9 +431,12 @@ func (c *Core) CheckToken(ctx context.Context, req *logical.Request, unauth bool
auth.PolicyResults.GrantingPolicies = append(auth.PolicyResults.GrantingPolicies, authResults.SentinelResults.GrantingPolicies...)
}
c.activityLogLock.RLock()
activityLog := c.activityLog
c.activityLogLock.RUnlock()
// If it is an authenticated ( i.e with vault token ) request, increment client count
if !unauth && c.activityLog != nil {
c.activityLog.HandleTokenUsage(ctx, te, clientID, isTWE)
if !unauth && activityLog != nil {
activityLog.HandleTokenUsage(ctx, te, clientID, isTWE)
}
return auth, te, nil
}

4
vault/testing.go
View File

@@ -187,7 +187,7 @@ func TestCoreWithSealAndUI(t testing.T, opts *CoreConfig) *Core {
}
func TestCoreWithSealAndUINoCleanup(t testing.T, opts *CoreConfig) *Core {
logger := logging.NewVaultLogger(log.Trace)
logger := logging.NewVaultLogger(log.Trace).Named(t.Name())
physicalBackend, err := physInmem.NewInmem(nil, logger)
if err != nil {
t.Fatal(err)
@@ -214,6 +214,7 @@ func TestCoreWithSealAndUINoCleanup(t testing.T, opts *CoreConfig) *Core {
conf.PluginDirectory = opts.PluginDirectory
conf.DetectDeadlocks = opts.DetectDeadlocks
conf.Experiments = []string{experiments.VaultExperimentEventsAlpha1}
conf.CensusAgent = opts.CensusAgent
if opts.Logger != nil {
conf.Logger = opts.Logger
@@ -235,6 +236,7 @@ func TestCoreWithSealAndUINoCleanup(t testing.T, opts *CoreConfig) *Core {
}
conf.ActivityLogConfig = opts.ActivityLogConfig
testApplyEntBaseConfig(conf, opts)
c, err := NewCore(conf)
if err != nil {

8
website/content/api-docs/system/internal-counters.mdx
View File

@@ -346,6 +346,10 @@ This endpoint was added in Vault 1.6.
- `limit_namespaces` `(int, optional)` - Controls the total number of by_namespace data returned. This can
be used to return the client counts for the specified number of namespaces having highest activity.
If no `limit_namespaces` parameter is specified, client counts for all namespaces in specified usage period is returned.
- `current_billing_period` `(bool, optional)` - Uses the builtin billing start
timestamp as `start_time` and the current time as the `end_time`, returning a
response with the current billing period information without having to
explicitly provide a start and end time.
### Sample Request
@@ -924,7 +928,9 @@ $ curl \
"default_report_months": 12,
"enabled": "default-enabled",
"queries_available": true,
"retention_months": 24
"retention_months": 24,
"reporting_enabled": false,
"billing_start_timestamp": "2022-03-01T00:00:00Z",
},
"warnings": null
}