diff --git a/cmd/influx/cli/cli.go b/cmd/influx/cli/cli.go index ebaa9d68b2b..3ccbc76fb82 100644 --- a/cmd/influx/cli/cli.go +++ b/cmd/influx/cli/cli.go @@ -16,6 +16,7 @@ import ( "path/filepath" "reflect" "runtime" + "slices" "sort" "strconv" "strings" @@ -880,6 +881,9 @@ func headersEqual(prev, current models.Row) bool { if prev.Name != current.Name { return false } + if !slices.Equal(prev.GroupingKeys, current.GroupingKeys) { + return false + } return tagsEqual(prev.Tags, current.Tags) && columnsEqual(prev.Columns, current.Columns) } @@ -890,9 +894,10 @@ func (c *CommandLine) writeCSV(response *client.Response, w io.Writer) { suppressHeaders := len(result.Series) > 0 && headersEqual(previousHeaders, result.Series[0]) if !suppressHeaders && len(result.Series) > 0 { previousHeaders = models.Row{ - Name: result.Series[0].Name, - Tags: result.Series[0].Tags, - Columns: result.Series[0].Columns, + Name: result.Series[0].Name, + Tags: result.Series[0].Tags, + GroupingKeys: result.Series[0].GroupingKeys, + Columns: result.Series[0].Columns, } } @@ -920,9 +925,10 @@ func (c *CommandLine) writeColumns(response *client.Response, w io.Writer) { suppressHeaders := len(result.Series) > 0 && headersEqual(previousHeaders, result.Series[0]) if !suppressHeaders && len(result.Series) > 0 { previousHeaders = models.Row{ - Name: result.Series[0].Name, - Tags: result.Series[0].Tags, - Columns: result.Series[0].Columns, + Name: result.Series[0].Name, + Tags: result.Series[0].Tags, + GroupingKeys: result.Series[0].GroupingKeys, + Columns: result.Series[0].Columns, } } @@ -984,6 +990,9 @@ func (c *CommandLine) formatResults(result client.Result, separator string, supp t := fmt.Sprintf("tags: %s", (strings.Join(tags, ", "))) rows = append(rows, t) } + if len(row.GroupingKeys) > 0 { + rows = append(rows, fmt.Sprintf("group: %s", strings.Join(row.GroupingKeys, ", "))) + } } if !suppressHeaders { diff --git a/coordinator/statement_executor.go b/coordinator/statement_executor.go index 66bbe588c62..8e06b746e57 100644 --- a/coordinator/statement_executor.go +++ b/coordinator/statement_executor.go @@ -618,7 +618,7 @@ func (e *StatementExecutor) executeSelectStatement(ctx *query.ExecutionContext, Messages: messages, Series: []*models.Row{{ Name: "result", - Columns: []string{"time", "written"}, + Columns: []string{models.TimeString, "written"}, Values: [][]interface{}{{time.Unix(0, 0).UTC(), writeN}}, }}, }) @@ -1394,12 +1394,34 @@ var errNoDatabaseInTarget = errors.New("no database in target") // convertRowToPoints will convert a query result Row into Points that can be written back in. func convertRowToPoints(measurementName string, row *models.Row, strictErrorHandling bool) ([]models.Point, error) { - // figure out which parts of the result are the time and which are the fields + // figure out which parts of the result are the time, the GROUP BY date_part + // grouping dimensions, and which are the regular fields. + // + // A GROUP BY date_part dimension (e.g. "year") is injected as an output column + // and recorded in row.GroupingKeys, but it identifies the series rather than + // carrying a value: every group shares the same representative bucket timestamp + // and the same base tag set (single window, Interval=0). Writing such a column + // as a field would collapse every group onto an identical series+timestamp + // (last-write-wins, silent data loss), so promote these columns to tags instead + // — mirroring how GROUP BY INTO writes its grouping tag. + var groupingCols map[string]struct{} + if len(row.GroupingKeys) > 0 { + groupingCols = make(map[string]struct{}, len(row.GroupingKeys)) + for _, k := range row.GroupingKeys { + groupingCols[k] = struct{}{} + } + } + timeIndex := -1 fieldIndexes := make(map[string]int) + tagIndexes := make(map[string]int, len(row.GroupingKeys)) for i, c := range row.Columns { - if c == "time" { + if c == models.TimeString { timeIndex = i + } else if _, ok := groupingCols[c]; ok { + // A GROUP BY date_part grouping dimension is written as a tag + // rather than a field. + tagIndexes[c] = i } else { fieldIndexes[c] = i } @@ -1423,7 +1445,7 @@ func convertRowToPoints(measurementName string, row *models.Row, strictErrorHand } } - p, err := models.NewPoint(measurementName, models.NewTags(row.Tags), vals, v[timeIndex].(time.Time)) + p, err := models.NewPoint(measurementName, groupingTags(row.Tags, tagIndexes, v), vals, v[timeIndex].(time.Time)) if err != nil { if !strictErrorHandling { // Drop points that can't be stored @@ -1438,6 +1460,48 @@ func convertRowToPoints(measurementName string, row *models.Row, strictErrorHand return points, nil } +// groupingTags builds the tag set for a single INTO point. It starts from the +// row's base tags (shared by every group) and adds the per-group date_part +// dimension values from tagIndexes so each group becomes a distinct series. When +// there are no grouping dimensions this is equivalent to models.NewTags(base). +func groupingTags(base map[string]string, tagIndexes map[string]int, v []interface{}) models.Tags { + if len(tagIndexes) == 0 { + return models.NewTags(base) + } + + merged := make(map[string]string, len(base)+len(tagIndexes)) + for k, val := range base { + merged[k] = val + } + for tagName, tagIndex := range tagIndexes { + if s, ok := tagValueString(v[tagIndex]); ok { + merged[tagName] = s + } + } + return models.NewTags(merged) +} + +// tagValueString renders a date_part dimension value as a tag value. date_part +// dimensions are emitted as int64 (influxql.Integer); other types are rendered +// defensively. A nil value yields ok=false so the tag is omitted rather than +// written as an empty (and therefore dropped) tag. +func tagValueString(val interface{}) (string, bool) { + switch x := val.(type) { + case nil: + return "", false + case int64: + return strconv.FormatInt(x, 10), true + case int: + return strconv.FormatInt(int64(x), 10), true + case int32: + return strconv.FormatInt(int64(x), 10), true + case string: + return x, x != "" + default: + return fmt.Sprintf("%v", x), true + } +} + // NormalizeStatement adds a default database and policy to the measurements in statement. // Parameter defaultRetentionPolicy can be "". func (e *StatementExecutor) NormalizeStatement(stmt influxql.Statement, defaultDatabase, defaultRetentionPolicy string) (err error) { diff --git a/coordinator/statement_executor_test.go b/coordinator/statement_executor_test.go index ecc13b21c5f..a1b30f22f06 100644 --- a/coordinator/statement_executor_test.go +++ b/coordinator/statement_executor_test.go @@ -710,9 +710,9 @@ func TestStatementExecutor_ExecuteShowMeasurementsStatement_Partial(t *testing.T // Source labels match influxql.QuoteIdent: simple names emit unquoted, // joined by '.'. Update these alongside formatMeasurementSource if the // labeling format changes. - quotedDB0 = DefaultDatabase - quotedDB0RP0 = DefaultDatabase + "." + DefaultRetentionPolicy - quotedDB1RP0 = db1Name + "." + DefaultRetentionPolicy + quotedDB0 = DefaultDatabase + quotedDB0RP0 = DefaultDatabase + "." + DefaultRetentionPolicy + quotedDB1RP0 = db1Name + "." + DefaultRetentionPolicy ) wildcardDBs := func() []meta.DatabaseInfo { diff --git a/models/rows.go b/models/rows.go index c087a4882d0..6a14979623a 100644 --- a/models/rows.go +++ b/models/rows.go @@ -1,21 +1,24 @@ package models import ( + "slices" "sort" ) // Row represents a single row returned from the execution of a statement. type Row struct { - Name string `json:"name,omitempty"` - Tags map[string]string `json:"tags,omitempty"` - Columns []string `json:"columns,omitempty"` - Values [][]interface{} `json:"values,omitempty"` - Partial bool `json:"partial,omitempty"` + Name string `json:"name,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + GroupingKeys []string `json:"grouping_keys,omitempty"` + Columns []string `json:"columns,omitempty"` + Values [][]interface{} `json:"values,omitempty"` + Partial bool `json:"partial,omitempty"` } // SameSeries returns true if r contains values for the same series as o. +// GroupingKeys are emitted sorted, so they can be compared element-wise. func (r *Row) SameSeries(o *Row) bool { - return r.tagsHash() == o.tagsHash() && r.Name == o.Name + return r.tagsHash() == o.tagsHash() && r.Name == o.Name && slices.Equal(r.GroupingKeys, o.GroupingKeys) } // tagsHash returns a hash of tag key/value pairs. diff --git a/models/rows_test.go b/models/rows_test.go new file mode 100644 index 00000000000..cee3ae16663 --- /dev/null +++ b/models/rows_test.go @@ -0,0 +1,64 @@ +package models_test + +import ( + "testing" + + "github.com/influxdata/influxdb/models" + "github.com/stretchr/testify/require" +) + +func TestRow_SameSeries(t *testing.T) { + for _, tt := range []struct { + name string + a, b models.Row + want bool + }{ + { + name: "same name and tags", + a: models.Row{Name: "cpu", Tags: map[string]string{"host": "a"}}, + b: models.Row{Name: "cpu", Tags: map[string]string{"host": "a"}}, + want: true, + }, + { + name: "different name", + a: models.Row{Name: "cpu"}, + b: models.Row{Name: "mem"}, + want: false, + }, + { + name: "different tags", + a: models.Row{Name: "cpu", Tags: map[string]string{"host": "a"}}, + b: models.Row{Name: "cpu", Tags: map[string]string{"host": "b"}}, + want: false, + }, + { + name: "same grouping keys", + a: models.Row{Name: "cpu", GroupingKeys: []string{"month", "year"}}, + b: models.Row{Name: "cpu", GroupingKeys: []string{"month", "year"}}, + want: true, + }, + { + name: "different grouping keys", + a: models.Row{Name: "cpu", GroupingKeys: []string{"year"}}, + b: models.Row{Name: "cpu", GroupingKeys: []string{"month"}}, + want: false, + }, + { + name: "grouping keys only on one side", + a: models.Row{Name: "cpu", GroupingKeys: []string{"year"}}, + b: models.Row{Name: "cpu"}, + want: false, + }, + { + name: "grouping keys prefix of the other", + a: models.Row{Name: "cpu", GroupingKeys: []string{"year"}}, + b: models.Row{Name: "cpu", GroupingKeys: []string{"month", "year"}}, + want: false, + }, + } { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.want, tt.a.SameSeries(&tt.b)) + require.Equal(t, tt.want, tt.b.SameSeries(&tt.a), "SameSeries must be symmetric") + }) + } +} diff --git a/models/time.go b/models/time.go index 297892c6da3..95860f1bc26 100644 --- a/models/time.go +++ b/models/time.go @@ -39,6 +39,12 @@ var ( // ErrTimeOutOfRange gets returned when time is out of the representable range using int64 nanoseconds since the epoch. ErrTimeOutOfRange = fmt.Errorf("time outside range %d - %d", MinNanoTime, MaxNanoTime) + + // Static objects to prevent small allocs. + TimeBytes = []byte("time") + + // TimeString is a variable representing the string "time" + TimeString = string(TimeBytes) ) // SafeCalcTime safely calculates the time given. Will return error if the time is outside the diff --git a/query/compile.go b/query/compile.go index 30f9ae31375..db53021f6b1 100644 --- a/query/compile.go +++ b/query/compile.go @@ -10,6 +10,20 @@ import ( "github.com/influxdata/influxql" ) +// Sentinel errors returned during compilation. Named values so tests can +// reference them (via export_test.go) instead of duplicating the strings. +var ( + errOnlyTimeAndDatePartDimensions = errors.New("only time() and date_part() calls allowed in dimensions") + errTimeDimensionArgCount = errors.New("time dimension expected 1 or 2 arguments") + errTimeDimensionDurationArg = errors.New("time dimension must have duration argument") + errMultipleTimeDimensions = errors.New("multiple time dimensions not allowed") + errTimeOffsetFunctionMustBeNow = errors.New("time dimension offset function must be now()") + errTimeOffsetNowNoArgs = errors.New("time dimension offset now() function requires no arguments") + errInvalidTimeOffset = errors.New("time dimension offset must be duration or now()") + errAtLeastOneNonTimeField = errors.New("at least 1 non-time field must be queried") + errMixedMultipleSelectors = errors.New("mixing multiple selector functions with tags or fields is not supported") +) + // CompileOptions are the customization options for the compiler. type CompileOptions struct { Now time.Time @@ -98,7 +112,7 @@ func newCompiler(opt CompileOptions) *compiledStatement { } return &compiledStatement{ OnlySelectors: true, - TimeFieldName: "time", + TimeFieldName: models.TimeString, Options: opt, } } @@ -149,7 +163,7 @@ func (c *compiledStatement) preprocess(stmt *influxql.SelectStatement) error { return err } // Verify that the condition is actually ok to use. - if err := c.validateCondition(cond); err != nil { + if err := c.validateCondition(cond, stmt.Sources); err != nil { return err } c.Condition = cond @@ -188,6 +202,9 @@ func (c *compiledStatement) compile(stmt *influxql.SelectStatement) error { if err := c.validateFields(); err != nil { return err } + if err := c.validateDatePartSelectFields(stmt); err != nil { + return err + } // Look through the sources and compile each of the subqueries (if they exist). // We do this after compiling the outside because subqueries may require @@ -205,7 +222,10 @@ func (c *compiledStatement) compile(stmt *influxql.SelectStatement) error { } func (c *compiledStatement) compileFields(stmt *influxql.SelectStatement) error { - valuer := MathValuer{} + valuer := influxql.MultiValuer( + MathValuer{}, + DatePartValuer{}, + ) c.Fields = make([]*compiledField, 0, len(stmt.Fields)) for _, f := range stmt.Fields { @@ -214,7 +234,7 @@ func (c *compiledStatement) compileFields(stmt *influxql.SelectStatement) error // Such as SELECT time, max(value) FROM cpu will be SELECT max(value) FROM cpu // and SELECT time AS timestamp, max(value) FROM cpu will return "timestamp" // as the column name for the time. - if ref, ok := f.Expr.(*influxql.VarRef); ok && ref.Val == "time" { + if ref, ok := f.Expr.(*influxql.VarRef); ok && ref.Val == models.TimeString { if f.Alias != "" { c.TimeFieldName = f.Alias } @@ -222,7 +242,7 @@ func (c *compiledStatement) compileFields(stmt *influxql.SelectStatement) error } // Append this field to the list of processed fields and compile it. - f.Expr = influxql.Reduce(f.Expr, &valuer) + f.Expr = influxql.Reduce(f.Expr, valuer) field := &compiledField{ global: c, Field: f, @@ -391,6 +411,8 @@ func (c *compiledField) compileFunction(expr *influxql.Call) error { // Validate the function call and mark down some meta properties // related to the function for query validation. switch expr.Name { + case DatePartString: + return ValidateDatePart(expr.Args) case "max", "min", "first", "last": // top/bottom are not included here since they are not typical functions. case "count", "sum", "mean", "median", "mode", "stddev", "spread", "sum_hll": @@ -926,56 +948,25 @@ func (c *compiledStatement) compileDimensions(stmt *influxql.SelectStatement) er switch expr := expr.(type) { case *influxql.VarRef: - if strings.EqualFold(expr.Val, "time") { + if strings.EqualFold(expr.Val, models.TimeString) { return errors.New("time() is a function and expects at least one argument") } case *influxql.Call: - // Ensure the call is time() and it has one or two duration arguments. - // If we already have a duration - if expr.Name != "time" { - return errors.New("only time() calls allowed in dimensions") - } else if got := len(expr.Args); got < 1 || got > 2 { - return errors.New("time dimension expected 1 or 2 arguments") - } else if lit, ok := expr.Args[0].(*influxql.DurationLiteral); !ok { - return errors.New("time dimension must have duration argument") - } else if c.Interval.Duration != 0 { - return errors.New("multiple time dimensions not allowed") - } else { - c.Interval.Duration = lit.Val - if len(expr.Args) == 2 { - switch lit := expr.Args[1].(type) { - case *influxql.DurationLiteral: - c.Interval.Offset = lit.Val % c.Interval.Duration - case *influxql.TimeLiteral: - c.Interval.Offset = lit.Val.Sub(lit.Val.Truncate(c.Interval.Duration)) - case *influxql.Call: - if lit.Name != "now" { - return errors.New("time dimension offset function must be now()") - } else if len(lit.Args) != 0 { - return errors.New("time dimension offset now() function requires no arguments") - } - now := c.Options.Now - c.Interval.Offset = now.Sub(now.Truncate(c.Interval.Duration)) - - // Use the evaluated offset to replace the argument. Ideally, we would - // use the interval assigned above, but the query engine hasn't been changed - // to use the compiler information yet. - expr.Args[1] = &influxql.DurationLiteral{Val: c.Interval.Offset} - case *influxql.StringLiteral: - // If literal looks like a date time then parse it as a time literal. - if lit.IsTimeLiteral() { - t, err := lit.ToTimeLiteral(stmt.Location) - if err != nil { - return err - } - c.Interval.Offset = t.Val.Sub(t.Val.Truncate(c.Interval.Duration)) - } else { - return errors.New("time dimension offset must be duration or now()") - } - default: - return errors.New("time dimension offset must be duration or now()") - } + switch expr.Name { + case models.TimeString: + err := c.compileTimeDimension(expr, stmt) + if err != nil { + return err + } + case DatePartString: + if err := ValidateDatePart(expr.Args); err != nil { + return err } + // GROUP BY date_part over a subquery source is resolved at the + // subquery boundary by datePartMap (see query/subquery.go), which + // computes the dimension value from each row's timestamp. + default: + return errOnlyTimeAndDatePartDimensions } case *influxql.Wildcard: case *influxql.RegexLiteral: @@ -989,17 +980,96 @@ func (c *compiledStatement) compileDimensions(stmt *influxql.SelectStatement) er return nil } +func (c *compiledStatement) compileTimeDimension(expr *influxql.Call, stmt *influxql.SelectStatement) error { + // Ensure the call is time() and it has one or two duration arguments. + // If we already have a duration + if expr.Name != models.TimeString { + return errOnlyTimeAndDatePartDimensions + } else if got := len(expr.Args); got < 1 || got > 2 { + return errTimeDimensionArgCount + } else if lit, ok := expr.Args[0].(*influxql.DurationLiteral); !ok { + return errTimeDimensionDurationArg + } else if c.Interval.Duration != 0 { + return errMultipleTimeDimensions + } else { + c.Interval.Duration = lit.Val + if len(expr.Args) == 2 { + switch lit := expr.Args[1].(type) { + case *influxql.DurationLiteral: + c.Interval.Offset = lit.Val % c.Interval.Duration + case *influxql.TimeLiteral: + c.Interval.Offset = lit.Val.Sub(lit.Val.Truncate(c.Interval.Duration)) + case *influxql.Call: + if lit.Name != "now" { + return errTimeOffsetFunctionMustBeNow + } else if len(lit.Args) != 0 { + return errTimeOffsetNowNoArgs + } + now := c.Options.Now + c.Interval.Offset = now.Sub(now.Truncate(c.Interval.Duration)) + + // Use the evaluated offset to replace the argument. Ideally, we would + // use the interval assigned above, but the query engine hasn't been changed + // to use the compiler information yet. + expr.Args[1] = &influxql.DurationLiteral{Val: c.Interval.Offset} + case *influxql.StringLiteral: + // If literal looks like a date time then parse it as a time literal. + if lit.IsTimeLiteral() { + t, err := lit.ToTimeLiteral(stmt.Location) + if err != nil { + return err + } + c.Interval.Offset = t.Val.Sub(t.Val.Truncate(c.Interval.Duration)) + } else { + return errInvalidTimeOffset + } + default: + return errInvalidTimeOffset + } + } + } + return nil +} + // validateFields validates that the fields are mutually compatible with each other. // This runs at the end of compilation but before linking. func (c *compiledStatement) validateFields() error { // Validate that at least one field has been selected. if len(c.Fields) == 0 { - return errors.New("at least 1 non-time field must be queried") + return errAtLeastOneNonTimeField + } + // date_part('part', time) derives its value purely from the row timestamp and + // references no stored field. A SELECT whose only fields are such date_part + // expressions has nothing to anchor the scan on (the storage engine needs a + // real field cursor to emit points), so it would silently return no data, like + // SELECT time. Reject it with a clear error instead. Queries that also select a + // bare field (HasAuxiliaryFields) or an aggregate/selector call other than + // date_part carry an anchor and are unaffected. + // + // This is a schema-blind early check: HasAuxiliaryFields is true for any bare + // VarRef including a tag, which is not a real anchor. The tag-only case can only + // be detected once field types are known, so it is caught later by the + // authoritative validateDatePartAnchor in Prepare. Keep both in sync. + datePartCalls, otherCalls := 0, 0 + for _, call := range c.FunctionCalls { + if call.Name == DatePartString { + datePartCalls++ + } else { + otherCalls++ + } + } + if !c.HasAuxiliaryFields { + if datePartCalls > 0 && otherCalls == 0 { + return errAtLeastOneNonTimeField + } } // Ensure there are not multiple calls if top/bottom is present. if len(c.FunctionCalls) > 1 && c.TopBottomFunction != "" { return fmt.Errorf("selector function %s() cannot be combined with other functions", c.TopBottomFunction) - } else if len(c.FunctionCalls) == 0 { + } else if otherCalls == 0 { + // date_part is registered in FunctionCalls but is not an aggregate, so a + // query whose only calls are date_part must satisfy the same fill and + // GROUP BY interval requirements as a raw query. switch c.FillOption { case influxql.NoFill: return errors.New("fill(none) must be used with a function") @@ -1019,7 +1089,17 @@ func (c *compiledStatement) validateFields() error { if !c.OnlySelectors { return fmt.Errorf("mixing aggregate and non-aggregate queries is not supported") } else if len(c.FunctionCalls) > 1 { - return fmt.Errorf("mixing multiple selector functions with tags or fields is not supported") + // If there are multiple function calls we want to validate whether they are date_part or not + // it is okay to have multiple date_part functions in a single SELECT clause. + nonDatePartCount := 0 + for _, call := range c.FunctionCalls { + if call.Name != DatePartString { + nonDatePartCount++ + if nonDatePartCount > 1 { + return errMixedMultipleSelectors + } + } + } } } return nil @@ -1028,22 +1108,30 @@ func (c *compiledStatement) validateFields() error { // validateCondition verifies that all elements in the condition are appropriate. // For example, aggregate calls don't work in the condition and should throw an // error as an invalid expression. -func (c *compiledStatement) validateCondition(expr influxql.Expr) error { +func (c *compiledStatement) validateCondition(expr influxql.Expr, sources influxql.Sources) error { switch expr := expr.(type) { case *influxql.BinaryExpr: // Verify each side of the binary expression. We do not need to // verify the binary expression itself since that should have been // done by influxql.ConditionExpr. - if err := c.validateCondition(expr.LHS); err != nil { + if err := c.validateCondition(expr.LHS, sources); err != nil { return err } - if err := c.validateCondition(expr.RHS); err != nil { + if err := c.validateCondition(expr.RHS, sources); err != nil { return err } return nil case *influxql.Call: if !isMathFunction(expr) { - return fmt.Errorf("invalid function call in condition: %s", expr) + switch expr.Name { + case DatePartString: + if err := ValidateDatePart(expr.Args); err != nil { + return err + } + return nil + default: + return fmt.Errorf("invalid function call in condition: %s", expr) + } } // How many arguments are we expecting? @@ -1060,7 +1148,7 @@ func (c *compiledStatement) validateCondition(expr influxql.Expr) error { // Are all the args valid? for _, arg := range expr.Args { - if err := c.validateCondition(arg); err != nil { + if err := c.validateCondition(arg, sources); err != nil { return err } } @@ -1189,6 +1277,25 @@ func (c *compiledStatement) Prepare(shardMapper ShardMapper, sopt SelectOptions) return nil, err } + // Now that VarRef types are known, reject a date_part SELECT whose only + // non-date_part fields are tags: a tag is not a scan anchor, so the query + // would otherwise silently return no rows. + if err := validateDatePartAnchor(stmt); err != nil { + shards.Close() + return nil, err + } + + // Re-run the date_part SELECT/GROUP BY validation now that RewriteFields has + // expanded any wildcards, recursing into subquery sources. The compile-time + // passes ran before expansion, so a wildcard-expanded shape (e.g. max(*) + // becoming multiple aggregates, a stored field named "year" colliding with + // GROUP BY date_part('year', time), or either inside a subquery) would + // otherwise slip through. + if err := validateDatePartTree(stmt, false); err != nil { + shards.Close() + return nil, err + } + // Determine base options for iterators. opt, err := newIteratorOptionsStmt(stmt, sopt) if err != nil { diff --git a/query/compile_test.go b/query/compile_test.go index 9fb2730beb1..a75f3224175 100644 --- a/query/compile_test.go +++ b/query/compile_test.go @@ -2,9 +2,11 @@ package query_test import ( "testing" + "time" "github.com/influxdata/influxdb/query" "github.com/influxdata/influxql" + "github.com/stretchr/testify/require" ) func TestCompile_Success(t *testing.T) { @@ -110,6 +112,34 @@ func TestCompile_Success(t *testing.T) { `SELECT sin(value) - sin(1.3) FROM cpu`, `SELECT value FROM cpu WHERE sin(value) > 0.5`, `SELECT sum("out")/sum("in") FROM (SELECT derivative("out") AS "out", derivative("in") AS "in" FROM "m0" WHERE time >= now() - 5m GROUP BY "index") GROUP BY time(1m) fill(none)`, + // date_part tests + `SELECT value, date_part('dow', time) FROM cpu`, + `SELECT value, date_part('dow', time), date_part('month', time) FROM cpu`, + `SELECT value FROM cpu WHERE date_part('dow', time) = 1`, + `SELECT value FROM cpu WHERE date_part('dow', time) != 0 AND date_part('dow', time) != 6`, + `SELECT first(value), date_part('dow', time) FROM cpu`, + `SELECT last(value), date_part('dow', time) FROM cpu`, + `SELECT max(value), date_part('dow', time) FROM cpu`, + // SELECT date_part matching a GROUP BY date_part dimension is allowed (maps to the grouped value) + `SELECT count(value), date_part('year', time) FROM cpu GROUP BY date_part('year', time)`, + `SELECT count(value), date_part('year', time), date_part('month', time) FROM cpu GROUP BY date_part('year', time), date_part('month', time)`, + // Non-value-carrying fill modes are allowed with GROUP BY date_part. + `SELECT count(value) FROM cpu GROUP BY date_part('year', time) fill(none)`, + `SELECT count(value) FROM cpu GROUP BY date_part('year', time) fill(null)`, + // date_part in subqueries + `SELECT max(dow) FROM (SELECT value, date_part('dow', time) AS dow FROM cpu)`, + `SELECT mean(value) FROM (SELECT value FROM cpu WHERE date_part('dow', time) = 1)`, + `SELECT value FROM (SELECT value, date_part('month', time) AS month FROM cpu) WHERE month = 1`, + `SELECT value, date_part('year', time) FROM (SELECT * FROM cpu WHERE value > 10)`, + `SELECT value, dow, month FROM (SELECT value, date_part('dow', time) AS dow, date_part('month', time) AS month FROM cpu)`, + // date_part GROUP BY over a subquery source (resolved at the subquery boundary) + `SELECT count(value) FROM (SELECT value FROM cpu) GROUP BY date_part('year', time)`, + // CQ-shaped: aggregate over a subquery, grouped by date_part, written via INTO + `SELECT mean(value) INTO target FROM (SELECT value FROM cpu) GROUP BY date_part('hour', time)`, + // date_part in a WHERE condition over a subquery source (now supported: filterCursor resolves date_part at execution time) + `SELECT value FROM (SELECT value FROM cpu) WHERE date_part('dow', time) = 0`, + `SELECT mean(value) FROM (SELECT value FROM cpu) WHERE date_part('dow', time) != 0 AND date_part('dow', time) != 6`, + `SELECT mean(value) FROM (SELECT value FROM cpu) WHERE date_part('hour', time) >= 9 GROUP BY date_part('dow', time)`, } { t.Run(tt, func(t *testing.T) { stmt, err := influxql.ParseStatement(tt) @@ -131,9 +161,16 @@ func TestCompile_Failures(t *testing.T) { s string err string }{ - {s: `SELECT time FROM cpu`, err: `at least 1 non-time field must be queried`}, + {s: `SELECT time FROM cpu`, err: query.ErrAtLeastOneNonTimeField.Error()}, + // date_part(...) derives purely from the row timestamp and references no + // stored field, so a SELECT whose only fields are date_part expressions has + // nothing to anchor the scan on (like SELECT time). Reject it instead of + // silently returning no data. + {s: `SELECT date_part('year', time) FROM cpu`, err: query.ErrAtLeastOneNonTimeField.Error()}, + {s: `SELECT date_part('dow', time), date_part('month', time) FROM cpu`, err: query.ErrAtLeastOneNonTimeField.Error()}, + {s: `SELECT date_part('hour', time) + 1 FROM cpu`, err: query.ErrAtLeastOneNonTimeField.Error()}, {s: `SELECT value, mean(value) FROM cpu`, err: `mixing aggregate and non-aggregate queries is not supported`}, - {s: `SELECT value, max(value), min(value) FROM cpu`, err: `mixing multiple selector functions with tags or fields is not supported`}, + {s: `SELECT value, max(value), min(value) FROM cpu`, err: query.ErrMixedMultipleSelectors.Error()}, {s: `SELECT top(value, 10), max(value) FROM cpu`, err: `selector function top() cannot be combined with other functions`}, {s: `SELECT bottom(value, 10), max(value) FROM cpu`, err: `selector function bottom() cannot be combined with other functions`}, {s: `SELECT count() FROM cpu`, err: `invalid number of arguments for count, expected 1, got 0`}, @@ -154,14 +191,14 @@ func TestCompile_Failures(t *testing.T) { {s: `SELECT count(distinct()) FROM cpu`, err: `distinct function requires at least one argument`}, {s: `SELECT count(distinct(value, host)) FROM cpu`, err: `distinct function can only have one argument`}, {s: `SELECT count(distinct(2)) FROM cpu`, err: `expected field argument in distinct()`}, - {s: `SELECT value FROM cpu GROUP BY now()`, err: `only time() calls allowed in dimensions`}, - {s: `SELECT value FROM cpu GROUP BY time()`, err: `time dimension expected 1 or 2 arguments`}, - {s: `SELECT value FROM cpu GROUP BY time(5m, 30s, 1ms)`, err: `time dimension expected 1 or 2 arguments`}, - {s: `SELECT value FROM cpu GROUP BY time('unexpected')`, err: `time dimension must have duration argument`}, - {s: `SELECT value FROM cpu GROUP BY time(5m), time(1m)`, err: `multiple time dimensions not allowed`}, - {s: `SELECT value FROM cpu GROUP BY time(5m, unexpected())`, err: `time dimension offset function must be now()`}, - {s: `SELECT value FROM cpu GROUP BY time(5m, now(1m))`, err: `time dimension offset now() function requires no arguments`}, - {s: `SELECT value FROM cpu GROUP BY time(5m, 'unexpected')`, err: `time dimension offset must be duration or now()`}, + {s: `SELECT value FROM cpu GROUP BY now()`, err: query.ErrOnlyTimeAndDatePartDimensions.Error()}, + {s: `SELECT value FROM cpu GROUP BY time()`, err: query.ErrTimeDimensionArgCount.Error()}, + {s: `SELECT value FROM cpu GROUP BY time(5m, 30s, 1ms)`, err: query.ErrTimeDimensionArgCount.Error()}, + {s: `SELECT value FROM cpu GROUP BY time('unexpected')`, err: query.ErrTimeDimensionDurationArg.Error()}, + {s: `SELECT value FROM cpu GROUP BY time(5m), time(1m)`, err: query.ErrMultipleTimeDimensions.Error()}, + {s: `SELECT value FROM cpu GROUP BY time(5m, unexpected())`, err: query.ErrTimeOffsetFunctionMustBeNow.Error()}, + {s: `SELECT value FROM cpu GROUP BY time(5m, now(1m))`, err: query.ErrTimeOffsetNowNoArgs.Error()}, + {s: `SELECT value FROM cpu GROUP BY time(5m, 'unexpected')`, err: query.ErrInvalidTimeOffset.Error()}, {s: `SELECT value FROM cpu GROUP BY 'unexpected'`, err: `only time and tag dimensions allowed`}, {s: `SELECT top(value) FROM cpu`, err: `invalid number of arguments for top, expected at least 2, got 1`}, {s: `SELECT top('unexpected', 5) FROM cpu`, err: `expected first argument to be a field in top(), found 'unexpected'`}, @@ -229,11 +266,11 @@ func TestCompile_Failures(t *testing.T) { {s: `SELECT count(value), value FROM foo`, err: `mixing aggregate and non-aggregate queries is not supported`}, {s: `SELECT count(value) FROM foo group by time`, err: `time() is a function and expects at least one argument`}, {s: `SELECT count(value) FROM foo group by 'time'`, err: `only time and tag dimensions allowed`}, - {s: `SELECT count(value) FROM foo where time > now() and time < now() group by time()`, err: `time dimension expected 1 or 2 arguments`}, - {s: `SELECT count(value) FROM foo where time > now() and time < now() group by time(b)`, err: `time dimension must have duration argument`}, - {s: `SELECT count(value) FROM foo where time > now() and time < now() group by time(1s), time(2s)`, err: `multiple time dimensions not allowed`}, - {s: `SELECT count(value) FROM foo where time > now() and time < now() group by time(1s, b)`, err: `time dimension offset must be duration or now()`}, - {s: `SELECT count(value) FROM foo where time > now() and time < now() group by time(1s, '5s')`, err: `time dimension offset must be duration or now()`}, + {s: `SELECT count(value) FROM foo where time > now() and time < now() group by time()`, err: query.ErrTimeDimensionArgCount.Error()}, + {s: `SELECT count(value) FROM foo where time > now() and time < now() group by time(b)`, err: query.ErrTimeDimensionDurationArg.Error()}, + {s: `SELECT count(value) FROM foo where time > now() and time < now() group by time(1s), time(2s)`, err: query.ErrMultipleTimeDimensions.Error()}, + {s: `SELECT count(value) FROM foo where time > now() and time < now() group by time(1s, b)`, err: query.ErrInvalidTimeOffset.Error()}, + {s: `SELECT count(value) FROM foo where time > now() and time < now() group by time(1s, '5s')`, err: query.ErrInvalidTimeOffset.Error()}, {s: `SELECT distinct(field1), sum(field1) FROM myseries`, err: `aggregate function distinct() cannot be combined with other functions or fields`}, {s: `SELECT distinct(field1), field2 FROM myseries`, err: `aggregate function distinct() cannot be combined with other functions or fields`}, {s: `SELECT distinct(field1, field2) FROM myseries`, err: `distinct function can only have one argument`}, @@ -362,6 +399,52 @@ func TestCompile_Failures(t *testing.T) { {s: `SELECT sin(1.3) FROM cpu`, err: `field must contain at least one variable`}, {s: `SELECT nofunc(1.3) FROM cpu`, err: `undefined function nofunc()`}, {s: `SELECT * FROM cpu WHERE ( host =~ /foo/ ^ other AND env =~ /bar/ ) and time >= now()-15m`, err: `likely malformed statement, unable to rewrite: interface conversion: influxql.Expr is *influxql.BinaryExpr, not *influxql.RegexLiteral`}, + // date_part validation tests + {s: `SELECT date_part() FROM cpu`, err: `invalid number of arguments for date_part, expected 2, got 0`}, + {s: `SELECT date_part('dow') FROM cpu`, err: `invalid number of arguments for date_part, expected 2, got 1`}, + {s: `SELECT date_part('invalid', time) FROM cpu`, err: `date_part: first argument must be one of the following: [year, quarter, month, week, day, hour, minute, second, millisecond, microsecond, nanosecond, dow, doy, epoch, isodow]`}, + {s: `SELECT date_part('dow', value) FROM cpu`, err: `date_part: second argument must be time VarRef`}, + {s: `SELECT date_part(123, time) FROM cpu`, err: `date_part: first argument must be a string`}, + // Verify multiple selectors without date_part still error + {s: `SELECT value, first(value), last(value) FROM cpu`, err: query.ErrMixedMultipleSelectors.Error()}, + // Multiple selectors WITH date_part should also error + {s: `SELECT value, first(value), last(value), date_part('dow', time) FROM cpu`, err: query.ErrMixedMultipleSelectors.Error()}, + // date_part subquery validation - cannot be sole field + {s: `SELECT date_part('dow', value) FROM (SELECT value FROM cpu)`, err: `date_part: second argument must be time VarRef`}, + // A SELECT date_part that does not match a GROUP BY date_part dimension is undefined per group and rejected. + {s: `SELECT count(value), date_part('month', time) FROM cpu GROUP BY date_part('year', time)`, err: `date_part: SELECT date_part('month', time) requires 'month' to be a GROUP BY date_part dimension`}, + // A selected field/alias colliding with an injected date_part dimension column is rejected. + {s: `SELECT mean(value) AS year FROM cpu GROUP BY date_part('year', time)`, err: `date_part: output column "year" collides with the GROUP BY date_part('year', time) dimension; alias the field to a different name`}, + // Value-carrying fill modes can leak into non-active date_part dimensions and are rejected. + {s: `SELECT count(value) FROM cpu GROUP BY date_part('year', time) fill(previous)`, err: query.ErrDatePartFillPrevious.Error()}, + {s: `SELECT count(value) FROM cpu GROUP BY date_part('year', time) fill(linear)`, err: query.ErrDatePartFillLinear.Error()}, + {s: `SELECT count(value) FROM cpu GROUP BY date_part('year', time) fill(100)`, err: query.ErrDatePartFillValue.Error()}, + // A field/alias colliding with a non-first injected date_part dimension column is also rejected. + {s: `SELECT mean(value) AS month FROM cpu GROUP BY date_part('year', time), date_part('month', time)`, err: `date_part: output column "month" collides with the GROUP BY date_part('month', time) dimension; alias the field to a different name`}, + // Raw (non-aggregate) SELECT with GROUP BY date_part does no grouping at all and + // would silently return one flat ungrouped series, so it is rejected. + {s: `SELECT value FROM cpu GROUP BY date_part('year', time)`, err: query.ErrDatePartRequiresAggregate.Error()}, + // Multiple aggregate/selector calls with GROUP BY date_part merge values across + // groups under a single shared key (mislabeling results), so they are rejected. + {s: `SELECT count(value), sum(value) FROM cpu GROUP BY date_part('year', time)`, err: query.ErrDatePartSingleAggregate.Error()}, + // fill(null) (the default) combined with a time() interval and GROUP BY date_part + // fragments the series: empty windows carry no date_part value and split into + // spurious extra series. Reject both the explicit and the default-null cases. + {s: `SELECT count(value) FROM cpu GROUP BY time(1h), date_part('year', time) fill(null)`, err: query.ErrDatePartFillNull.Error()}, + {s: `SELECT count(value) FROM cpu GROUP BY time(1h), date_part('year', time)`, err: query.ErrDatePartFillNull.Error()}, + // date_part is not an aggregate: it must not satisfy the aggregate/fill + // requirements a raw query would otherwise fail. + {s: `SELECT value, date_part('year', time) FROM cpu GROUP BY time(1m)`, err: `GROUP BY requires at least one aggregate function`}, + {s: `SELECT value, date_part('year', time) FROM cpu fill(linear)`, err: `fill(linear) must be used with a function`}, + {s: `SELECT value, date_part('year', time) FROM cpu fill(none)`, err: `fill(none) must be used with a function`}, + // Stream transformations (derivative, moving_average, ...) reduce keyed on + // tags only and ignore the date_part grouper, silently flattening groups. + {s: `SELECT derivative(value) FROM cpu GROUP BY date_part('year', time) fill(none)`, err: `date_part: derivative() is not supported with GROUP BY date_part`}, + {s: `SELECT moving_average(value, 2) FROM cpu GROUP BY date_part('year', time) fill(none)`, err: `date_part: moving_average() is not supported with GROUP BY date_part`}, + {s: `SELECT cumulative_sum(value) FROM cpu GROUP BY date_part('year', time) fill(none)`, err: `date_part: cumulative_sum() is not supported with GROUP BY date_part`}, + // A GROUP BY tag sharing the injected date_part column name would be + // clobbered in column-name-keyed handling (e.g. SELECT INTO), so reject it. + {s: `SELECT max(value) FROM cpu GROUP BY year, date_part('year', time) fill(none)`, err: `date_part: GROUP BY dimension "year" collides with the GROUP BY date_part('year', time) dimension`}, } { t.Run(tt.s, func(t *testing.T) { stmt, err := influxql.ParseStatement(tt.s) @@ -380,6 +463,172 @@ func TestCompile_Failures(t *testing.T) { } } +// TestPrepare_DatePartSubqueryAnchor verifies that the date_part anchor check +// recurses into subquery sources. A subquery whose only non-date_part field is a +// tag has no scan anchor, so the inner iterator emits no points and the outer +// aggregate silently returns nothing. The equivalent top-level query is rejected, +// so the subquery form must be too. +func TestPrepare_DatePartSubqueryAnchor(t *testing.T) { + for _, tt := range []struct { + s string + err string + }{ + // Inner anchor is a tag (host) — rejected. + { + s: `SELECT max(yr) FROM (SELECT host, date_part('year', time) AS yr FROM cpu)`, + err: query.ErrAtLeastOneNonTimeField.Error(), + }, + // Nested subquery: the tag-only anchor is two levels down. + { + s: `SELECT max(yr) FROM (SELECT yr FROM (SELECT host, date_part('year', time) AS yr FROM cpu))`, + err: query.ErrAtLeastOneNonTimeField.Error(), + }, + } { + t.Run(tt.s, func(t *testing.T) { + stmt, err := influxql.ParseStatement(tt.s) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + s := stmt.(*influxql.SelectStatement) + + c, err := query.Compile(s, query.CompileOptions{}) + if err != nil { + t.Fatalf("unexpected compile error: %s", err) + } + + shardMapper := ShardMapper{ + MapShardsFn: func(_ influxql.Sources, _ influxql.TimeRange) query.ShardGroup { + return &ShardGroup{ + Fields: map[string]influxql.DataType{"value": influxql.Float}, + Dimensions: []string{"host"}, + } + }, + } + + _, err = c.Prepare(&shardMapper, query.SelectOptions{}) + if err == nil { + t.Fatal("expected error, got nil") + } else if have, want := err.Error(), tt.err; have != want { + t.Errorf("unexpected error: %s != %s", have, want) + } + }) + } +} + +// TestPrepare_DatePartSubqueryAnchor_Valid verifies the recursion does not reject +// a subquery that has a real stored-field anchor alongside date_part. +func TestPrepare_DatePartSubqueryAnchor_Valid(t *testing.T) { + stmt, err := influxql.ParseStatement(`SELECT max(yr) FROM (SELECT value, date_part('year', time) AS yr FROM cpu)`) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + s := stmt.(*influxql.SelectStatement) + + c, err := query.Compile(s, query.CompileOptions{}) + if err != nil { + t.Fatalf("unexpected compile error: %s", err) + } + + shardMapper := ShardMapper{ + MapShardsFn: func(_ influxql.Sources, _ influxql.TimeRange) query.ShardGroup { + return &ShardGroup{ + Fields: map[string]influxql.DataType{"value": influxql.Float}, + Dimensions: []string{"host"}, + } + }, + } + + if _, err := c.Prepare(&shardMapper, query.SelectOptions{}); err != nil { + t.Fatalf("unexpected error: %s", err) + } +} + +// TestPrepare_DatePartWildcardValidation verifies that the date_part SELECT/GROUP BY +// rules still hold after RewriteFields expands wildcards. The compile-time pass runs +// before expansion, so a wildcard shape (e.g. max(*) over a multi-field measurement) +// can hide a multi-aggregate or dimension-collision query that the explicit +// equivalent would reject. +func TestPrepare_DatePartWildcardValidation(t *testing.T) { + shardMapper := ShardMapper{ + MapShardsFn: func(_ influxql.Sources, _ influxql.TimeRange) query.ShardGroup { + return &ShardGroup{ + Fields: map[string]influxql.DataType{ + "value": influxql.Float, + "usage": influxql.Float, + }, + Dimensions: []string{"host", "year"}, + } + }, + } + + for _, tt := range []struct { + s string + err string + }{ + // max(*) expands to one aggregate per field; two fields means two + // aggregates, the same shape as the rejected explicit form. + { + s: `SELECT max(*) FROM cpu GROUP BY date_part('year', time) fill(none)`, + err: query.ErrDatePartSingleAggregate.Error(), + }, + // The same expansion two levels down: the inner statement of a subquery. + { + s: `SELECT mean(max) FROM (SELECT max(*) FROM cpu GROUP BY date_part('year', time) fill(none))`, + err: query.ErrDatePartSingleAggregate.Error(), + }, + // GROUP BY * expands to tag dimensions; the tag "year" collides with the + // injected date_part output column. + { + s: `SELECT max(value) FROM cpu GROUP BY *, date_part('year', time) fill(none)`, + err: `date_part: GROUP BY dimension "year" collides with the GROUP BY date_part('year', time) dimension`, + }, + } { + t.Run(tt.s, func(t *testing.T) { + stmt, err := influxql.ParseStatement(tt.s) + require.NoError(t, err) + s := stmt.(*influxql.SelectStatement) + + c, err := query.Compile(s, query.CompileOptions{}) + require.NoError(t, err, "expected the error at Prepare, not Compile") + + _, err = c.Prepare(&shardMapper, query.SelectOptions{}) + require.EqualError(t, err, tt.err) + }) + } +} + +// TestPrepare_DatePartWildcardValidation_Valid pins the shape that must keep +// working: a wildcard aggregate over a single-field measurement expands to +// exactly one aggregate and satisfies the single-aggregate rule. +func TestPrepare_DatePartWildcardValidation_Valid(t *testing.T) { + shardMapper := ShardMapper{ + MapShardsFn: func(_ influxql.Sources, _ influxql.TimeRange) query.ShardGroup { + return &ShardGroup{ + Fields: map[string]influxql.DataType{"value": influxql.Float}, + Dimensions: []string{"host"}, + } + }, + } + + for _, s := range []string{ + `SELECT max(*) FROM cpu GROUP BY date_part('year', time) fill(none)`, + // A subquery whose redundant fill(null) is rewritten to fill(none) by + // subquery compilation must not be re-rejected by the Prepare-time pass. + `SELECT mean(max) FROM (SELECT max(value) FROM cpu GROUP BY time(1h), date_part('year', time) fill(none))`, + } { + t.Run(s, func(t *testing.T) { + stmt, err := influxql.ParseStatement(s) + require.NoError(t, err) + + c, err := query.Compile(stmt.(*influxql.SelectStatement), query.CompileOptions{}) + require.NoError(t, err) + + _, err = c.Prepare(&shardMapper, query.SelectOptions{}) + require.NoError(t, err) + }) + } +} + func TestPrepare_MapShardsTimeRange(t *testing.T) { for _, tt := range []struct { s string @@ -437,3 +686,129 @@ func TestPrepare_MapShardsTimeRange(t *testing.T) { }) } } + +// TestCompileTimeDimension_Errors drives compileTimeDimension directly (via +// export_test.go) and checks each error path returns its sentinel error. +func TestCompileTimeDimension_Errors(t *testing.T) { + timeCall := func(args ...influxql.Expr) *influxql.Call { + return &influxql.Call{Name: "time", Args: args} + } + dur := func(d time.Duration) influxql.Expr { + return &influxql.DurationLiteral{Val: d} + } + + for _, tt := range []struct { + name string + expr *influxql.Call + err error + }{ + { + name: "not time call", + expr: &influxql.Call{Name: "now"}, + err: query.ErrOnlyTimeAndDatePartDimensions, + }, + { + name: "no arguments", + expr: timeCall(), + err: query.ErrTimeDimensionArgCount, + }, + { + name: "too many arguments", + expr: timeCall(dur(time.Minute), dur(time.Second), dur(time.Millisecond)), + err: query.ErrTimeDimensionArgCount, + }, + { + name: "non-duration interval", + expr: timeCall(&influxql.StringLiteral{Val: "unexpected"}), + err: query.ErrTimeDimensionDurationArg, + }, + { + name: "offset function not now", + expr: timeCall(dur(5*time.Minute), &influxql.Call{Name: "unexpected"}), + err: query.ErrTimeOffsetFunctionMustBeNow, + }, + { + name: "offset now with arguments", + expr: timeCall(dur(5*time.Minute), &influxql.Call{Name: "now", Args: []influxql.Expr{dur(time.Minute)}}), + err: query.ErrTimeOffsetNowNoArgs, + }, + { + name: "offset non-time string", + expr: timeCall(dur(5*time.Minute), &influxql.StringLiteral{Val: "unexpected"}), + err: query.ErrInvalidTimeOffset, + }, + { + name: "offset invalid type", + expr: timeCall(dur(5*time.Minute), &influxql.IntegerLiteral{Val: 5}), + err: query.ErrInvalidTimeOffset, + }, + } { + t.Run(tt.name, func(t *testing.T) { + c := query.NewCompilerForTesting(query.CompileOptions{}) + err := query.CompileTimeDimension(c, tt.expr, &influxql.SelectStatement{}) + require.ErrorIs(t, err, tt.err) + }) + } + + t.Run("multiple time dimensions", func(t *testing.T) { + c := query.NewCompilerForTesting(query.CompileOptions{}) + stmt := &influxql.SelectStatement{} + require.NoError(t, query.CompileTimeDimension(c, timeCall(dur(5*time.Minute)), stmt)) + err := query.CompileTimeDimension(c, timeCall(dur(time.Minute)), stmt) + require.ErrorIs(t, err, query.ErrMultipleTimeDimensions) + }) +} + +// TestCompileTimeDimension_Success checks the valid interval/offset forms set +// the compiled interval correctly. +func TestCompileTimeDimension_Success(t *testing.T) { + now := time.Date(2024, 3, 15, 10, 42, 30, 0, time.UTC) + + for _, tt := range []struct { + name string + args []influxql.Expr + duration time.Duration + offset time.Duration + }{ + { + name: "interval only", + args: []influxql.Expr{&influxql.DurationLiteral{Val: 5 * time.Minute}}, + duration: 5 * time.Minute, + }, + { + name: "duration offset", + args: []influxql.Expr{ + &influxql.DurationLiteral{Val: 5 * time.Minute}, + &influxql.DurationLiteral{Val: 7 * time.Minute}, + }, + duration: 5 * time.Minute, + offset: 2 * time.Minute, // 7m % 5m + }, + { + name: "now offset", + args: []influxql.Expr{ + &influxql.DurationLiteral{Val: 5 * time.Minute}, + &influxql.Call{Name: "now"}, + }, + duration: 5 * time.Minute, + offset: now.Sub(now.Truncate(5 * time.Minute)), + }, + { + name: "time literal offset", + args: []influxql.Expr{ + &influxql.DurationLiteral{Val: time.Hour}, + &influxql.StringLiteral{Val: "2024-03-15T03:30:00Z"}, + }, + duration: time.Hour, + offset: 30 * time.Minute, + }, + } { + t.Run(tt.name, func(t *testing.T) { + c := query.NewCompilerForTesting(query.CompileOptions{Now: now}) + expr := &influxql.Call{Name: "time", Args: tt.args} + require.NoError(t, query.CompileTimeDimension(c, expr, &influxql.SelectStatement{})) + require.Equal(t, tt.duration, c.Interval.Duration) + require.Equal(t, tt.offset, c.Interval.Offset) + }) + } +} diff --git a/query/cursor.go b/query/cursor.go index 03ff56d267f..e376019ec22 100644 --- a/query/cursor.go +++ b/query/cursor.go @@ -4,6 +4,7 @@ import ( "math" "time" + "github.com/influxdata/influxdb/models" "github.com/influxdata/influxql" ) @@ -65,6 +66,10 @@ type Row struct { // Values contains the values within the current row. Values []interface{} + + // GroupingKeys contains the names of active date_part grouping + // dimensions. Actual per-row values are in the Values slice. + GroupingKeys map[string]struct{} } type Cursor interface { @@ -144,11 +149,32 @@ type scannerCursorBase struct { columns []influxql.VarRef loc *time.Location + // needDatePart caches whether this query actually involves date_part (either + // an explicit date_part(...) field or a GROUP BY date_part dimension). When + // false, the per-row date_part bookkeeping in Scan is skipped entirely so + // ordinary queries don't pay for a feature they don't use. + needDatePart bool + scan scannerFunc valuer influxql.ValuerEval } -func newScannerCursorBase(scan scannerFunc, fields []*influxql.Field, loc *time.Location) scannerCursorBase { +// scannerCursorNeedsDatePart reports whether the cursor must perform date_part +// bookkeeping: true when a GROUP BY date_part dimension is present or when any +// selected field references the date_part function (top-level or nested). +func scannerCursorNeedsDatePart(fields []*influxql.Field, opt IteratorOptions) bool { + if len(opt.DatePartDimensions) > 0 { + return true + } + for _, f := range fields { + if exprContainsDatePart(f.Expr) { + return true + } + } + return false +} + +func newScannerCursorBase(scan scannerFunc, fields []*influxql.Field, loc *time.Location, needDatePart bool) scannerCursorBase { typmap := FunctionTypeMapper{} exprs := make([]influxql.Expr, len(fields)) columns := make([]influxql.VarRef, len(fields)) @@ -164,23 +190,47 @@ func newScannerCursorBase(scan scannerFunc, fields []*influxql.Field, loc *time. } m := make(map[string]interface{}) + mapValuer := influxql.MapValuer(m) + + // Only wire DatePartValuer into the evaluation chain when date_part is + // actually used; otherwise skip the extra valuer indirection on every Eval. + var valuer influxql.Valuer + if needDatePart { + valuer = influxql.MultiValuer( + MathValuer{}, + DatePartValuer{Valuer: mapValuer, Location: loc}, + mapValuer, + ) + } else { + valuer = influxql.MultiValuer( + MathValuer{}, + mapValuer, + ) + } + return scannerCursorBase{ - fields: exprs, - m: m, - columns: columns, - loc: loc, - scan: scan, + fields: exprs, + m: m, + columns: columns, + loc: loc, + needDatePart: needDatePart, + scan: scan, valuer: influxql.ValuerEval{ - Valuer: influxql.MultiValuer( - MathValuer{}, - influxql.MapValuer(m), - ), + Valuer: valuer, IntegerFloatDivision: true, }, } } func (cur *scannerCursorBase) Scan(row *Row) bool { + if cur.needDatePart { + // Clear date_part state from previous scan so it doesn't leak across rows. + // The map is cleared rather than set to nil so callers that reuse the Row + // across scans keep the allocation. + delete(cur.m, DatePartDimensionsString) + clear(row.GroupingKeys) + } + ts, name, tags := cur.scan(cur.m) if ts == ZeroTime { return false @@ -198,12 +248,54 @@ func (cur *scannerCursorBase) Scan(row *Row) bool { row.Values = make([]interface{}, len(cur.columns)) } + // Make the row timestamp available to the eval map so date_part can access it. + // This is set whenever the query uses date_part, because date_part may be + // nested inside another expression (e.g. date_part('hour', time) + 1), in + // which case the top-level field is not a date_part call and a per-field + // check would miss it, leaving time unset. + if cur.needDatePart { + cur.m[models.TimeString] = row.Time + } + + // Resolve the active GROUP BY date_part dimension once per row instead of per + // field: the dimension value and its name are identical for every field, so + // the map lookup, type assertion, and Expr.String() only need to happen once. + var ( + haveDim bool + dpd DecodedDatePartKey + dimName string + ) + if cur.needDatePart { + if val, ok := cur.m[DatePartDimensionsString]; ok && val != nil { + if d, ok := val.(DecodedDatePartKey); ok { + haveDim = true + dpd = d + dimName = d.Expr.String() + if row.GroupingKeys == nil { + // A scan inserts only the active dimension, so one slot suffices. + row.GroupingKeys = make(map[string]struct{}, 1) + } + row.GroupingKeys[dimName] = struct{}{} + } + } + } + for i, expr := range cur.fields { // A special case if the field is time to reduce memory allocations. - if ref, ok := expr.(*influxql.VarRef); ok && ref.Val == "time" { + if ref, ok := expr.(*influxql.VarRef); ok && ref.Val == models.TimeString { row.Values[i] = time.Unix(0, row.Time).In(cur.loc) continue } + // Only set the column value from the grouped dimension if this field is the + // dimension VarRef. Explicit date_part(...) calls — top-level or nested in a + // larger expression — are resolved by DatePartValuer.Call against the active + // grouped key during Eval below, so they need no special handling here. + if haveDim { + if ref, ok := expr.(*influxql.VarRef); ok && ref.Val == dimName { + row.Values[i] = dpd.Val + continue + } + } v := cur.valuer.Eval(expr) if fv, ok := v.(float64); ok && math.IsNaN(fv) { // If the float value is NaN, convert it to a null float @@ -235,7 +327,7 @@ type scannerCursor struct { func newScannerCursor(s IteratorScanner, fields []*influxql.Field, opt IteratorOptions) *scannerCursor { cur := &scannerCursor{scanner: s} - cur.scannerCursorBase = newScannerCursorBase(cur.scan, fields, opt.Location) + cur.scannerCursorBase = newScannerCursorBase(cur.scan, fields, opt.Location, scannerCursorNeedsDatePart(fields, opt)) return cur } @@ -278,7 +370,7 @@ func newMultiScannerCursor(scanners []IteratorScanner, fields []*influxql.Field, scanners: scanners, ascending: opt.Ascending, } - cur.scannerCursorBase = newScannerCursorBase(cur.scan, fields, opt.Location) + cur.scannerCursorBase = newScannerCursorBase(cur.scan, fields, opt.Location, scannerCursorNeedsDatePart(fields, opt)) return cur } @@ -353,11 +445,18 @@ type filterCursor struct { // we need and will exclude the ones we do not. fields map[string]IteratorMap filter influxql.Expr + // dpCond is non-nil when the filter uses date_part; it owns the rewritten + // filter and publishes per-row part values into m via SetTime. + dpCond *DatePartCondition m map[string]interface{} valuer influxql.ValuerEval } -func newFilterCursor(cur Cursor, filter influxql.Expr) *filterCursor { +// newFilterCursor filters rows against the given expression. needTimeRef +// reports whether the filter references the row timestamp (i.e. contains a +// date_part call); it is precomputed by the caller (opt.NeedTimeRef) so the +// condition AST is not re-walked for every filter cursor. +func newFilterCursor(cur Cursor, filter influxql.Expr, needTimeRef bool, loc *time.Location) *filterCursor { fields := make(map[string]IteratorMap) for _, name := range influxql.ExprNames(filter) { for i, col := range cur.Columns() { @@ -378,11 +477,23 @@ func newFilterCursor(cur Cursor, filter influxql.Expr) *filterCursor { fields[name.Val] = TagMap(name.Val) } } + // When the filter uses date_part, rewrite it once so no function call is + // evaluated per row: date_part calls become reserved variable references + // resolved by dpCond.SetTime in Scan. Filters without date_part are + // untouched. + var dpCond *DatePartCondition + if needTimeRef { + if dp := NewDatePartCondition(filter, loc); dp != nil { + dpCond = dp + filter = dp.Expr() + } + } m := make(map[string]interface{}) return &filterCursor{ Cursor: cur, fields: fields, filter: filter, + dpCond: dpCond, m: m, valuer: influxql.ValuerEval{Valuer: influxql.MapValuer(m)}, } @@ -394,6 +505,9 @@ func (cur *filterCursor) Scan(row *Row) bool { for name, f := range cur.fields { cur.m[name] = f.Value(row) } + if cur.dpCond != nil { + cur.dpCond.SetTime(row.Time, cur.m) + } if cur.valuer.EvalBool(cur.filter) { // Passes the filter! Return true. We no longer need to diff --git a/query/date_part.go b/query/date_part.go new file mode 100644 index 00000000000..09dc0856f61 --- /dev/null +++ b/query/date_part.go @@ -0,0 +1,770 @@ +package query + +import ( + "encoding/binary" + "errors" + "fmt" + "strings" + "time" + + "github.com/influxdata/influxdb/models" + "github.com/influxdata/influxql" +) + +const ( + // DatePartString is the name of date_part function + DatePartString = "date_part" + + // DatePartTimeString is a symbol used to represent a reference variable + // for the current timestamp from a given point. It is used during time + // lookup on the query path. The leading NUL byte makes it impossible to + // collide with a user field or tag name (those originate from InfluxQL + // identifiers, which can never contain a NUL), so a field or tag literally + // named "date_part_time" is not shadowed by this sentinel. + DatePartTimeString = "\x00date_part_time" + + // DatePartArgCount is the amount of arguments required for date_part function + DatePartArgCount = 2 + + // DatePartDimensionsString is the internal eval-map key under which the active + // GROUP BY date_part dimension value is published for a scanned row. The + // leading NUL byte makes it impossible to collide with a user field or tag + // name (those originate from InfluxQL identifiers, which can never contain a + // NUL), so selecting a series with a field/tag literally named + // "date_part_dimensions" is not corrupted by date_part grouping. + DatePartDimensionsString = "\x00date_part_dimensions" +) + +type DatePartExpr int + +const ( + Year DatePartExpr = iota + Quarter + Month + Week + Day + Hour + Minute + Second + Millisecond + Microsecond + Nanosecond + DOW + DOY + Epoch + ISODOW + Invalid +) + +// datePartNames is the single source of truth for the canonical part names; +// String and ParseDatePartExpr both derive from it so they cannot drift. +var datePartNames = [...]string{ + Year: "year", + Quarter: "quarter", + Month: "month", + Week: "week", + Day: "day", + Hour: "hour", + Minute: "minute", + Second: "second", + Millisecond: "millisecond", + Microsecond: "microsecond", + Nanosecond: "nanosecond", + DOW: "dow", + DOY: "doy", + Epoch: "epoch", + ISODOW: "isodow", + Invalid: "invalid", +} + +var datePartsByName = func() map[string]DatePartExpr { + m := make(map[string]DatePartExpr, Invalid) + for part := Year; part < Invalid; part++ { + m[datePartNames[part]] = part + } + return m +}() + +func (d DatePartExpr) String() string { + if d < Year || d > Invalid { + return "" + } + return datePartNames[d] +} + +func ParseDatePartExpr(t string) (DatePartExpr, bool) { + part, ok := datePartsByName[strings.ToLower(t)] + if !ok { + return Invalid, false + } + return part, true +} + +// matchDatePartCall reports whether n is a well-formed date_part call — +// date_part('', time) with a recognized part — and returns the parsed +// part. Anything else, including a date_part call with a malformed argument +// list, does not match. +func matchDatePartCall(n influxql.Node) (DatePartExpr, bool) { + call, ok := n.(*influxql.Call) + if !ok || call.Name != DatePartString || len(call.Args) != DatePartArgCount { + return Invalid, false + } + lit, ok := call.Args[0].(*influxql.StringLiteral) + if !ok { + return Invalid, false + } + ref, ok := call.Args[1].(*influxql.VarRef) + if !ok || ref.Val != models.TimeString { + return Invalid, false + } + return ParseDatePartExpr(lit.Val) +} + +func ExtractDatePartExpr(t time.Time, expr DatePartExpr) (int64, bool) { + switch expr { + case Year: + return int64(t.Year()), true + case Quarter: + month := t.Month() + return int64((month-1)/3 + 1), true + case Month: + return int64(t.Month()), true + case Week: + _, week := t.ISOWeek() + return int64(week), true + case Day: + return int64(t.Day()), true + case Hour: + return int64(t.Hour()), true + case Minute: + return int64(t.Minute()), true + case Second: + return int64(t.Second()), true + case Millisecond: + // Seconds-of-minute scaled to milliseconds, plus the sub-second component. + return int64(t.Second())*1000 + int64(t.Nanosecond())/1_000_000, true + case Microsecond: + // Seconds-of-minute scaled to microseconds, plus the sub-second component. + return int64(t.Second())*1_000_000 + int64(t.Nanosecond())/1_000, true + case Nanosecond: + // Seconds-of-minute scaled to nanoseconds, plus the sub-second component. + return int64(t.Second())*1_000_000_000 + int64(t.Nanosecond()), true + case DOW: + return int64(t.Weekday()), true + case DOY: + return int64(t.YearDay()), true + case Epoch: + // Whole seconds since the Unix epoch. Sub-second precision is truncated by + // the int64 return type; select the millisecond/microsecond/nanosecond + // part for finer resolution. + return t.Unix(), true + case ISODOW: + // ISO 8601 day of the week: Monday=1 ... Sunday=7. + // Go's time.Weekday() is Sunday=0 ... Saturday=6, so every weekday + // already maps onto its ISO value except Sunday, which becomes 7. + dow := int64(t.Weekday()) + if dow == 0 { + return int64(7), true // Sunday + } + return dow, true + default: + return 0, false + } +} + +func ValidateDatePart(args []influxql.Expr) error { + if exp, got := DatePartArgCount, len(args); exp != got { + return fmt.Errorf("invalid number of arguments for date_part, expected %d, got %d", exp, got) + } + + exprStr, ok := args[0].(*influxql.StringLiteral) + if !ok { + return errors.New("date_part: first argument must be a string") + } + + _, ok = ParseDatePartExpr(exprStr.Val) + if !ok { + valid := make([]string, 0, Invalid) + for i := Year; i < Invalid; i++ { + valid = append(valid, i.String()) + } + return fmt.Errorf("date_part: first argument must be one of the following: [%s]", strings.Join(valid, ", ")) + } + + tstamp, ok := args[1].(*influxql.VarRef) + if !ok { + return errors.New("date_part: second argument must be a variable reference") + } else if tstamp.Val != models.TimeString { + // check if tstamp.Val is "time" keyword currently, we only support using time as the second argument + // this may seem redundant, but we would like to keep consistency with SQL date_part + return errors.New("date_part: second argument must be time VarRef") + } + + return nil +} + +// exprContainsDatePart reports whether expr contains a call to the date_part +// function at any nesting depth. A nil expr contains none. +func exprContainsDatePart(expr influxql.Expr) bool { + if expr == nil { + return false + } + found := false + influxql.WalkFunc(expr, func(n influxql.Node) { + if call, ok := n.(*influxql.Call); ok && call.Name == DatePartString { + found = true + } + }) + return found +} + +// Sentinel errors returned by date_part validation. Named values so tests can +// reference them (via export_test.go) instead of duplicating the strings. +var ( + errDatePartRequiresAggregate = errors.New("date_part: GROUP BY date_part requires an aggregate or selector function") + errDatePartSingleAggregate = errors.New("date_part: GROUP BY date_part supports only a single aggregate or selector function") + errDatePartFillPrevious = errors.New("date_part: fill(previous) is not supported with GROUP BY date_part") + errDatePartFillLinear = errors.New("date_part: fill(linear) is not supported with GROUP BY date_part") + errDatePartFillValue = errors.New("date_part: fill() is not supported with GROUP BY date_part") + errDatePartFillNull = errors.New("date_part: fill(null) is not supported with GROUP BY time() and date_part; use fill(none)") +) + +// validateDatePartSelectFields rejects an explicit date_part('part', time) in the +// SELECT list whose part is not one of the GROUP BY date_part dimensions, when the +// query groups by date_part. Under such grouping the emitted row's timestamp is the +// bucket's representative time (not a per-row time), so a non-grouped date_part has +// no well-defined value for the group and would silently return misleading data. +// +// Queries without a date_part GROUP BY are unaffected: raw queries evaluate +// date_part against each point's real timestamp, and GROUP BY time() buckets carry a +// meaningful timestamp, both of which are correct. +func (c *compiledStatement) validateDatePartSelectFields(stmt *influxql.SelectStatement) error { + return validateDatePartFields(stmt, c.FillOption, !c.Interval.IsZero()) +} + +// validateDatePartTree runs validateDatePartFields over a statement and every +// subquery source beneath it, deriving the fill option and interval from each +// statement itself. This is the Prepare-time re-run: RewriteFields rewrites the +// whole statement tree, so a wildcard expanded inside a subquery (which the +// per-statement compile passes ran before expansion) is only visible here. +func validateDatePartTree(stmt *influxql.SelectStatement, subquery bool) error { + interval, err := stmt.GroupByInterval() + if err != nil { + return err + } + fill := stmt.Fill + // Subquery compilation rewrites a redundant fill(null) with an interval to + // fill(none) (see (*compiledStatement).subquery). Mirror that here so this + // pass does not reject a shape the executed plan never produces. + if subquery && interval > 0 && fill == influxql.NullFill { + fill = influxql.NoFill + } + if err := validateDatePartFields(stmt, fill, interval > 0); err != nil { + return err + } + for _, source := range stmt.Sources { + if sub, ok := source.(*influxql.SubQuery); ok { + if err := validateDatePartTree(sub.Statement, true); err != nil { + return err + } + } + } + return nil +} + +// datePartStreamCalls lists the stream-based transformation functions. Their +// reducers process points in timestamp order keyed on tags only and never +// consult the date_part grouper, so combining them with GROUP BY date_part +// would silently flatten the groups into one series. +var datePartStreamCalls = map[string]struct{}{ + "derivative": {}, + "non_negative_derivative": {}, + "difference": {}, + "non_negative_difference": {}, + "elapsed": {}, + "moving_average": {}, + "exponential_moving_average": {}, + "double_exponential_moving_average": {}, + "triple_exponential_moving_average": {}, + "triple_exponential_derivative": {}, + "relative_strength_index": {}, + "kaufmans_efficiency_ratio": {}, + "kaufmans_adaptive_moving_average": {}, + "chande_momentum_oscillator": {}, + "cumulative_sum": {}, + "integral": {}, + "holt_winters": {}, + "holt_winters_with_fit": {}, +} + +// datePartAnchorCalls collects the outermost non-math, non-date_part function +// calls in the SELECT fields. Math functions are transparent (their arguments +// may hold the anchoring call); aggregate/selector arguments are not descended +// into, so a nested shape like count(distinct(value)) counts once. Unlike +// c.FunctionCalls this is derived from the statement, so it sees the fields +// RewriteFields expanded from a wildcard. +func datePartAnchorCalls(fields influxql.Fields) []*influxql.Call { + var calls []*influxql.Call + var walk func(expr influxql.Expr) + walk = func(expr influxql.Expr) { + switch e := expr.(type) { + case *influxql.Call: + if e.Name == DatePartString { + return + } + if isMathFunction(e) { + for _, arg := range e.Args { + walk(arg) + } + return + } + calls = append(calls, e) + case *influxql.BinaryExpr: + walk(e.LHS) + walk(e.RHS) + case *influxql.ParenExpr: + walk(e.Expr) + case *influxql.Distinct: + calls = append(calls, e.NewCall()) + } + } + for _, f := range fields { + walk(f.Expr) + } + return calls +} + +func validateDatePartFields(stmt *influxql.SelectStatement, fillOption influxql.FillOption, hasInterval bool) error { + groupByParts := make(map[DatePartExpr]struct{}) + for _, d := range stmt.Dimensions { + if part, ok := matchDatePartCall(d.Expr); ok { + groupByParts[part] = struct{}{} + } + } + if len(groupByParts) == 0 { + return nil + } + + // GROUP BY date_part is implemented by a single reduce/grouper per query. + // The raw (no-aggregate) path takes the aux-cursor branch and does no grouping + // at all, silently returning one flat ungrouped series. The multi-aggregate + // path aligns the per-call scanners on (ts, name, tags) only and merges their + // values under a single shared date_part key, so each call's group value + // overwrites the others (mislabeled results). Require exactly one non-date_part + // aggregate or selector call so neither broken shape can compile. + anchorCalls := datePartAnchorCalls(stmt.Fields) + if len(anchorCalls) == 0 { + return errDatePartRequiresAggregate + } + if len(anchorCalls) > 1 { + return errDatePartSingleAggregate + } + if _, ok := datePartStreamCalls[anchorCalls[0].Name]; ok { + return fmt.Errorf("date_part: %s() is not supported with GROUP BY date_part", anchorCalls[0].Name) + } + + // Value-carrying fill modes (previous/linear/) synthesize values for + // empty windows. For a GROUP BY date_part dimension this would leak a value + // into a series where that dimension is not active, so reject those modes. + // + // fill(null) (the default) is safe for a bare GROUP BY date_part, but when it + // is combined with a time() interval the fill iterator emits empty-window rows + // that carry no DecodedDatePartKey: their grouping value is lost and the + // emitter splits them into spurious extra series, fragmenting the real ones. + // Reject fill(null) only in that combined case (use fill(none) instead). + // fill(none) is always unaffected (it produces no fill iterator). + switch fillOption { + case influxql.PreviousFill: + return errDatePartFillPrevious + case influxql.LinearFill: + return errDatePartFillLinear + case influxql.NumberFill: + return errDatePartFillValue + case influxql.NullFill: + if hasInterval { + return errDatePartFillNull + } + } + + // GROUP BY date_part injects an output column named after the canonical part + // (e.g. "year"). Reject a user-selected field/alias of the same name: the + // duplicate column names collapse in column-name-keyed result handling (e.g. + // SELECT INTO via convertRowToPoints), silently dropping data. + injected := make(map[string]struct{}, len(groupByParts)) + for part := range groupByParts { + injected[part.String()] = struct{}{} + } + for _, f := range stmt.Fields { + if _, ok := injected[f.Name()]; ok { + return fmt.Errorf("date_part: output column %q collides with the GROUP BY date_part('%s', time) dimension; alias the field to a different name", f.Name(), f.Name()) + } + } + + // A GROUP BY tag with the same name collides with the injected column too: + // column-name-keyed handling (e.g. SELECT INTO promoting grouping columns to + // tags) would silently overwrite the real tag's value with the part value. + for _, d := range stmt.Dimensions { + if ref, ok := d.Expr.(*influxql.VarRef); ok { + if _, ok := injected[ref.Val]; ok { + return fmt.Errorf("date_part: GROUP BY dimension %q collides with the GROUP BY date_part('%s', time) dimension", ref.Val, ref.Val) + } + } + } + + var badPart string + for _, f := range stmt.Fields { + influxql.WalkFunc(f.Expr, func(n influxql.Node) { + if badPart != "" { + return + } + part, ok := matchDatePartCall(n) + if !ok { + return + } + if _, ok := groupByParts[part]; !ok { + badPart = part.String() + } + }) + if badPart != "" { + return fmt.Errorf("date_part: SELECT date_part('%s', time) requires '%s' to be a GROUP BY date_part dimension", badPart, badPart) + } + } + return nil +} + +// validateDatePartAnchor rejects a SELECT that uses date_part(...) but has no +// real anchor to drive the scan. date_part derives its value purely from the row +// timestamp, so it cannot itself produce points; it must be paired with a stored +// field or a non-date_part aggregate/selector. A bare tag reference is not an +// anchor (the storage engine cannot emit timestamps from a tag-only cursor), so a +// query like `SELECT host, date_part('year', time) FROM cpu` would otherwise plan +// as an aux-only iterator and silently return no rows. +// +// This runs after RewriteFields, once VarRef types (field vs tag) are known: that +// distinction is not available during compilation, where HasAuxiliaryFields is set +// for any bare VarRef including tags, so the compile-time check cannot catch it. +func validateDatePartAnchor(stmt *influxql.SelectStatement) error { + var hasDatePart, hasAnchor bool + for _, f := range stmt.Fields { + influxql.WalkFunc(f.Expr, func(n influxql.Node) { + switch n := n.(type) { + case *influxql.Call: + if n.Name == DatePartString { + hasDatePart = true + } else if !isMathFunction(n) { + // An aggregate or selector (count, max, ...) anchors the scan. + hasAnchor = true + } + case *influxql.VarRef: + // Only a stored field anchors the scan. Tags, the time column + // (the date_part argument, typed Time/Unknown here), and untyped + // refs do not, so match the concrete stored-field types explicitly. + switch n.Type { + case influxql.Float, influxql.Integer, influxql.Unsigned, influxql.String, influxql.Boolean: + hasAnchor = true + } + } + }) + } + if hasDatePart && !hasAnchor { + return errAtLeastOneNonTimeField + } + + // Recurse into subquery sources. RewriteFields rewrites the whole statement + // tree, so inner VarRef types are resolved by the time this runs in Prepare. + // Without this, a tag-only-anchor inner query (e.g. + // SELECT host, date_part('year', time) AS yr FROM cpu) escapes the check: it + // plans as an aux-only iterator emitting no points, so an outer aggregate over + // it silently returns nothing even though the equivalent top-level query is + // rejected. + for _, source := range stmt.Sources { + if sub, ok := source.(*influxql.SubQuery); ok { + if err := validateDatePartAnchor(sub.Statement); err != nil { + return err + } + } + } + return nil +} + +type DatePartValuer struct { + Valuer influxql.MapValuer + // Location is the timezone in which calendar fields are computed. + // A nil Location is treated as UTC (see LocationOrUTC). + Location *time.Location +} + +// LocationOrUTC returns loc, or time.UTC when loc is nil. Shared with the TSM +// iterator so date_part extraction defaults consistently. +func LocationOrUTC(loc *time.Location) *time.Location { + if loc == nil { + return time.UTC + } + return loc +} + +var _ influxql.CallValuer = DatePartValuer{} + +func (v DatePartValuer) Value(key string) (interface{}, bool) { + if v.Valuer == nil { + return nil, false + } + // Convert the special date_part symbol back to "time" + if key == DatePartTimeString { + key = models.TimeString + } + return v.Valuer.Value(key) +} + +func (v DatePartValuer) Call(name string, args []interface{}) (interface{}, bool) { + if name != DatePartString { + return nil, false + } + if len(args) != DatePartArgCount { + return nil, false + } + + exprStr, ok := args[0].(string) + if !ok { + return nil, false + } + + expr, ok := ParseDatePartExpr(exprStr) + if !ok { + return nil, false + } + + // Under GROUP BY date_part(...), the active grouped dimension value is + // authoritative for the series; the row timestamp is only a bucket + // representative and must not be used. Resolving from the grouped value here + // keeps nested expressions (e.g. date_part('year', time) + 1) consistent with + // top-level date_part fields: the active part yields its grouped value, and a + // non-active grouped part is undefined for this series (nil). + if v.Valuer != nil { + if raw, ok := v.Valuer.Value(DatePartDimensionsString); ok { + if dpk, ok := raw.(DecodedDatePartKey); ok { + if expr == dpk.Expr { + return dpk.Val, true + } + return nil, false + } + } + } + + timestampRaw, ok := args[1].(int64) + if !ok { + return nil, false + } + + timestamp := time.Unix(0, timestampRaw).In(LocationOrUTC(v.Location)) + return ExtractDatePartExpr(timestamp, expr) +} + +// datePartCondKeyPrefix prefixes the reserved eval-map keys written by +// DatePartCondition.SetTime. The NUL byte keeps the names out of the space of +// real field and tag names, following DatePartDimensionsString. +const datePartCondKeyPrefix = "\x00date_part:" + +type datePartCondPart struct { + part DatePartExpr + name string + + // Boxing cache: the extracted value is converted to interface{} only when + // it changes between points, so repeated scans of the same hour/day/year + // reuse the previous boxed value instead of allocating. + lastVal int64 + lastBoxed interface{} +} + +// DatePartCondition evaluates date_part references in a condition without +// per-point function-call evaluation. It rewrites each date_part call to a +// reserved variable reference once at construction; SetTime then extracts the +// referenced parts from a point's timestamp and publishes them to the +// condition-evaluation map. A DatePartCondition carries per-point state and +// must not be shared between concurrently scanning iterators. +type DatePartCondition struct { + expr influxql.Expr + parts []datePartCondPart + loc *time.Location +} + +// NewDatePartCondition returns a DatePartCondition for cond, or nil when cond +// is nil or contains no date_part call. cond itself is never modified; the +// rewrite operates on a clone. +func NewDatePartCondition(cond influxql.Expr, loc *time.Location) *DatePartCondition { + if cond == nil { + return nil + } + c := &DatePartCondition{loc: loc} + rewritten := influxql.RewriteExpr(influxql.CloneExpr(cond), func(e influxql.Expr) influxql.Expr { + part, ok := matchDatePartCall(e) + if !ok { + return e + } + return &influxql.VarRef{Val: c.varName(part)} + }) + if len(c.parts) == 0 { + return nil + } + c.expr = rewritten + return c +} + +// varName returns the reserved variable name for part, registering it on +// first use so SetTime knows which parts to extract. +func (c *DatePartCondition) varName(part DatePartExpr) string { + for i := range c.parts { + if c.parts[i].part == part { + return c.parts[i].name + } + } + name := datePartCondKeyPrefix + part.String() + c.parts = append(c.parts, datePartCondPart{part: part, name: name}) + return name +} + +// Expr returns the rewritten condition. Every date_part call has been replaced +// by a reserved variable reference resolved through SetTime. +func (c *DatePartCondition) Expr() influxql.Expr { return c.expr } + +// SetTime extracts each date_part referenced by the condition from ts and +// stores the values in m under the reserved names. +func (c *DatePartCondition) SetTime(ts int64, m map[string]interface{}) { + t := time.Unix(0, ts).In(LocationOrUTC(c.loc)) + for i := range c.parts { + p := &c.parts[i] + v, ok := ExtractDatePartExpr(t, p.part) + if !ok { + m[p.name] = nil + continue + } + if p.lastBoxed == nil || v != p.lastVal { + p.lastVal = v + p.lastBoxed = v + } + m[p.name] = p.lastBoxed + } +} + +// DatePartDimension is a GROUP BY date_part dimension. Its output column is +// named by Expr.String() — the canonical part name (e.g. "dow"), regardless of +// how the user spelled the literal (e.g. "DOW"). +type DatePartDimension struct { + Expr DatePartExpr +} + +type DecodedDatePartKey struct { + Expr DatePartExpr + Val int64 +} + +// extractVal extracts an int64 from the aux value at the first-level reduce. +// The TSM iterator always appends int64 values from ExtractDatePartExpr. +func extractVal(auxVal interface{}) (int64, error) { + v, ok := auxVal.(int64) + if !ok { + return 0, fmt.Errorf("date_part: unexpected aux value type: %T", auxVal) + } + return v, nil +} + +// DatePartGrouper implements DimensionGrouper for date_part GROUP BY dimensions. +// All methods are safe for concurrent use — no shared mutable state. +// Encoding buffers use stack-allocated fixed-size arrays to avoid heap allocations. +type DatePartGrouper struct { + dims []DatePartDimension +} + +func NewDatePartGrouper(dims []DatePartDimension) *DatePartGrouper { + return &DatePartGrouper{dims: dims} +} + +// computeDimKey builds a grouping key string that uniquely identifies a +// (tag subset, expr, val) tuple; it is used as a map key and is never decoded. +// Note the reduce path SORTS these keys to order the output series, so the +// format is observable: the leading expr.String() makes series sort by part +// name, which the GROUP BY date_part result ordering depends on. When tags are +// present the tag subset ID is length-prefixed (8-byte big-endian) so the +// encoding stays unambiguous even if the ID contains NUL bytes — which it can, +// e.g. when a series has empty tag values. +func computeDimKey(expr DatePartExpr, val int64, tags TagSubset) string { + var buf [8]byte + // Flip the sign bit so lexicographic byte order matches signed numeric + // order; without this a negative value (e.g. a pre-1970 'epoch') encodes + // with its high bit set and sorts after every non-negative value. + binary.BigEndian.PutUint64(buf[:], uint64(val)^(1<<63)) + valStr := string(buf[:]) + if tags.HasTags { + var lenBuf [8]byte + binary.BigEndian.PutUint64(lenBuf[:], uint64(len(tags.ID))) + return string(lenBuf[:]) + tags.ID + expr.String() + ":" + valStr + } + return expr.String() + ":" + valStr +} + +// newGroupingEntry builds the paired keys for one resolved dimension value: +// DimKey for in-memory grouping and series ordering, EncodedKey for aux +// transport across reduce levels. +func newGroupingEntry(expr DatePartExpr, val int64, tags TagSubset) GroupingEntry { + return GroupingEntry{ + DimKey: computeDimKey(expr, val, tags), + EncodedKey: encodeKey(expr, val), + } +} + +// encodeKey encodes a dimension value into a 9-byte string (1 byte expr + 8 bytes value) +// that can be stored on a reduce point and later decoded. +// Uses a stack-allocated [9]byte for the binary encoding. +func encodeKey(expr DatePartExpr, val int64) string { + var buf [9]byte + buf[0] = byte(expr) + binary.BigEndian.PutUint64(buf[1:], uint64(val)) + return string(buf[:]) +} + +// decodeKey decodes a 9-byte encoded key back into a DecodedDatePartKey. +func decodeKey(encodedKey string) (DecodedDatePartKey, error) { + if len(encodedKey) != 9 { + return DecodedDatePartKey{}, fmt.Errorf("date_part: encoded key must be exactly 9 bytes, got %d", len(encodedKey)) + } + expr := DatePartExpr(encodedKey[0]) + if expr < Year || expr >= Invalid { + return DecodedDatePartKey{}, fmt.Errorf("date_part: encoded key has invalid expr byte %d", encodedKey[0]) + } + var b [8]byte + copy(b[:], encodedKey[1:9]) + return DecodedDatePartKey{ + Expr: expr, + Val: int64(binary.BigEndian.Uint64(b[:])), + }, nil +} + +func (g *DatePartGrouper) ResolveKeys(aux []interface{}, tags TagSubset) ([]GroupingEntry, error) { + // Check for second-level reduce: aux contains DecodedDatePartKey from a prior emit. + for _, av := range aux { + if dpk, ok := av.(DecodedDatePartKey); ok { + return []GroupingEntry{newGroupingEntry(dpk.Expr, dpk.Val, tags)}, nil + } + } + + // First-level reduce: raw int64 values at end of aux. + if len(aux) < len(g.dims) { + return nil, nil + } + startIdx := len(aux) - len(g.dims) + entries := make([]GroupingEntry, 0, len(g.dims)) + + for i, dim := range g.dims { + val, err := extractVal(aux[startIdx+i]) + if err != nil { + return nil, err + } + + entries = append(entries, newGroupingEntry(dim.Expr, val, tags)) + } + return entries, nil +} + +func (g *DatePartGrouper) DecodeEntry(encodedKey string) (interface{}, error) { + return decodeKey(encodedKey) +} diff --git a/query/date_part_internal_test.go b/query/date_part_internal_test.go new file mode 100644 index 00000000000..a7a1dde6c3f --- /dev/null +++ b/query/date_part_internal_test.go @@ -0,0 +1,268 @@ +package query + +import ( + "math" + "testing" + "time" + + "github.com/influxdata/influxdb/models" + "github.com/influxdata/influxql" + "github.com/stretchr/testify/require" +) + +func TestDatePartMap_Value(t *testing.T) { + // 2023-01-16T10:30:45Z — a Monday. + ts := time.Date(2023, 1, 16, 10, 30, 45, 0, time.UTC).UnixNano() + row := &Row{Time: ts} + + require.Equal(t, int64(2023), datePartMap{expr: Year, loc: time.UTC}.Value(row)) + require.Equal(t, int64(1), datePartMap{expr: Month, loc: time.UTC}.Value(row)) + require.Equal(t, int64(10), datePartMap{expr: Hour, loc: time.UTC}.Value(row)) + require.Equal(t, int64(1), datePartMap{expr: DOW, loc: time.UTC}.Value(row)) // Monday = 1 + + // nil location is treated as UTC. + require.Equal(t, int64(2023), datePartMap{expr: Year, loc: nil}.Value(row)) + + // Non-UTC location shifts the hour. America/New_York is UTC-5 in January. + ny, err := time.LoadLocation("America/New_York") + require.NoError(t, err) + require.Equal(t, int64(5), datePartMap{expr: Hour, loc: ny}.Value(row)) // 10:30 UTC -> 05:30 EST + + // An unknown expr (e.g. deserialized from a newer peer's iterator options) + // must yield nil so the grouper rejects it loudly instead of silently + // grouping every row under value 0. + require.Nil(t, datePartMap{expr: Invalid, loc: time.UTC}.Value(row)) +} + +// TestEncodeDecodeAux_DatePartKey ensures a DecodedDatePartKey grouping value +// survives the iterator wire codec (encodeAux/decodeAux). This is the codec used +// to stream iterators between enterprise data nodes; without explicit handling the +// key serializes to null and all date_part GROUP BY buckets collapse into one. +func TestEncodeDecodeAux_DatePartKey(t *testing.T) { + // Use a non-zero Expr (Month) so the expr byte is actually exercised, alongside + // neighbouring aux values of other types. + key := DecodedDatePartKey{Expr: Month, Val: 12} + aux := []interface{}{int64(7), key, "host1"} + + got := decodeAux(encodeAux(aux)) + + require.Len(t, got, 3) + require.Equal(t, int64(7), got[0]) + require.Equal(t, key, got[1], "DecodedDatePartKey must survive the iterator wire codec") + require.Equal(t, "host1", got[2]) +} + +func TestNewDatePartCondition(t *testing.T) { + t.Run("nil condition", func(t *testing.T) { + require.Nil(t, NewDatePartCondition(nil, nil)) + }) + + t.Run("condition without date_part", func(t *testing.T) { + require.Nil(t, NewDatePartCondition(influxql.MustParseExpr(`f1 > 0`), nil)) + }) + + t.Run("rewrite removes calls and preserves the original", func(t *testing.T) { + orig := influxql.MustParseExpr(`f1 > 0 AND date_part('hour', time) < 12`) + origStr := orig.String() + + c := NewDatePartCondition(orig, nil) + require.NotNil(t, c) + require.Equal(t, origStr, orig.String()) + + influxql.WalkFunc(c.Expr(), func(n influxql.Node) { + _, ok := n.(*influxql.Call) + require.False(t, ok, "rewritten condition must not contain calls") + }) + }) + + t.Run("dedupes repeated parts", func(t *testing.T) { + c := NewDatePartCondition(influxql.MustParseExpr( + `date_part('hour', time) >= 9 AND date_part('hour', time) < 17`), nil) + require.NotNil(t, c) + require.Len(t, c.parts, 1) + }) +} + +func TestDatePartCondition_MatchesDatePartValuer(t *testing.T) { + la, err := time.LoadLocation("America/Los_Angeles") + require.NoError(t, err) + + conds := []string{ + `date_part('hour', time) < 12`, + `date_part('hour', time) >= 9 AND date_part('hour', time) < 17`, + `date_part('dow', time) = 0 OR date_part('dow', time) = 6`, + `date_part('hour', time) + 1 < 13`, + `date_part('month', time) = 1 AND f1 > 0`, + `date_part('year', time) = 2023`, + `date_part('dow', time) = 0 AND date_part('hour', time) < 12`, + } + timestamps := []time.Time{ + time.Date(2023, 1, 1, 3, 0, 0, 0, time.UTC), // Sunday 03:00 + time.Date(2023, 1, 2, 15, 30, 0, 0, time.UTC), // Monday 15:30 + time.Date(2024, 6, 7, 23, 59, 59, 0, time.UTC), + } + + for _, condStr := range conds { + for _, loc := range []*time.Location{nil, la} { + for _, ts := range timestamps { + cond := influxql.MustParseExpr(condStr) + + legacyM := map[string]interface{}{ + "f1": int64(5), + models.TimeString: ts.UnixNano(), + } + legacy := influxql.ValuerEval{ + Valuer: influxql.MultiValuer( + MathValuer{}, + DatePartValuer{Location: loc}, + influxql.MapValuer(legacyM), + ), + } + want := (&legacy).EvalBool(cond) + + c := NewDatePartCondition(cond, loc) + require.NotNil(t, c, condStr) + m := map[string]interface{}{"f1": int64(5)} + c.SetTime(ts.UnixNano(), m) + newEval := influxql.ValuerEval{ + Valuer: influxql.MultiValuer( + MathValuer{}, + influxql.MapValuer(m), + ), + } + got := (&newEval).EvalBool(c.Expr()) + + require.Equal(t, want, got, "%s at %s in %v", condStr, ts, loc) + } + } + } +} + +func TestDatePartCondition_SetTime_ZeroAllocs(t *testing.T) { + c := NewDatePartCondition(influxql.MustParseExpr(`date_part('hour', time) < 12`), nil) + require.NotNil(t, c) + + m := make(map[string]interface{}) + ts := time.Date(2026, 7, 6, 3, 0, 0, 0, time.UTC).UnixNano() + c.SetTime(ts, m) // prime the boxing cache + + allocs := testing.AllocsPerRun(1000, func() { + ts += int64(time.Second) + c.SetTime(ts, m) + }) + require.Zero(t, allocs) +} + +func TestFilterCursor_DatePartCondition(t *testing.T) { + cols := []influxql.VarRef{{Val: "value", Type: influxql.Float}} + rows := []Row{ + {Time: time.Date(2023, 1, 1, 3, 0, 0, 0, time.UTC).UnixNano(), Values: []interface{}{1.0}}, + {Time: time.Date(2023, 1, 1, 15, 0, 0, 0, time.UTC).UnixNano(), Values: []interface{}{2.0}}, + {Time: time.Date(2023, 1, 2, 5, 0, 0, 0, time.UTC).UnixNano(), Values: []interface{}{3.0}}, + } + + t.Run("date_part filter", func(t *testing.T) { + cur := newFilterCursor( + RowCursor(rows, cols), + influxql.MustParseExpr(`date_part('hour', time) < 12`), + true, + nil, + ) + var row Row + var got []interface{} + for cur.Scan(&row) { + got = append(got, row.Values[0]) + } + require.Equal(t, []interface{}{1.0, 3.0}, got) + }) + + t.Run("date_part filter with location", func(t *testing.T) { + la, err := time.LoadLocation("America/Los_Angeles") + require.NoError(t, err) + // 03:00 UTC = 19:00 previous day PST; 15:00 UTC = 07:00 PST. + cur := newFilterCursor( + RowCursor(rows, cols), + influxql.MustParseExpr(`date_part('hour', time) < 12`), + true, + la, + ) + var row Row + var got []interface{} + for cur.Scan(&row) { + got = append(got, row.Values[0]) + } + require.Equal(t, []interface{}{2.0}, got) + }) + + t.Run("non-date_part filter unchanged", func(t *testing.T) { + cur := newFilterCursor( + RowCursor(rows, cols), + influxql.MustParseExpr(`value > 1`), + false, + nil, + ) + require.Nil(t, cur.dpCond) + var row Row + var got []interface{} + for cur.Scan(&row) { + got = append(got, row.Values[0]) + } + require.Equal(t, []interface{}{2.0, 3.0}, got) + }) +} + +func TestFilterCursor_DatePart_ZeroAllocs(t *testing.T) { + cols := []influxql.VarRef{{Val: "value", Type: influxql.Float}} + base := time.Date(2026, 7, 6, 3, 0, 0, 0, time.UTC).UnixNano() + rows := make([]Row, 2048) + for i := range rows { + rows[i] = Row{Time: base + int64(i)*int64(time.Second), Values: []interface{}{1.0}} + } + cur := newFilterCursor(RowCursor(rows, cols), influxql.MustParseExpr(`date_part('hour', time) < 12`), true, nil) + + var row Row + require.True(t, cur.Scan(&row)) // prime the boxing cache + + allocs := testing.AllocsPerRun(1000, func() { + cur.Scan(&row) + }) + require.Zero(t, allocs) +} + +// TestComputeDimKey_SignedValueOrdering ensures DimKeys sort lexicographically +// in the same order as their signed values. The reduce path sorts DimKey +// strings to order the emitted series, so a negative value (e.g. a pre-1970 +// 'epoch') must produce a key that sorts before every non-negative value's key. +func TestComputeDimKey_SignedValueOrdering(t *testing.T) { + vals := []int64{math.MinInt64, -100, -1, 0, 1, 100, math.MaxInt64} + for _, hasTags := range []bool{false, true} { + var prev string + for i, v := range vals { + key := computeDimKey(Epoch, v, TagSubset{ID: "tagid", HasTags: hasTags}) + if i > 0 { + require.Less(t, prev, key, + "DimKey for %d must sort before DimKey for %d (hasTags=%v)", vals[i-1], v, hasTags) + } + prev = key + } + } +} + +// BenchmarkFilterCursor_DatePartCondition measures the per-row cost of a +// date_part filter at the subquery boundary. Timestamps advance one second per +// row so boxed values are realistic. +func BenchmarkFilterCursor_DatePartCondition(b *testing.B) { + cols := []influxql.VarRef{{Val: "value", Type: influxql.Float}} + base := time.Date(2026, 7, 6, 0, 0, 0, 0, time.UTC).UnixNano() + rows := make([]Row, b.N) + for i := range rows { + rows[i] = Row{Time: base + int64(i)*int64(time.Second), Values: []interface{}{1.0}} + } + cur := newFilterCursor(RowCursor(rows, cols), influxql.MustParseExpr(`date_part('hour', time) < 12`), true, nil) + + b.ResetTimer() + b.ReportAllocs() + var row Row + for cur.Scan(&row) { + } +} diff --git a/query/date_part_test.go b/query/date_part_test.go new file mode 100644 index 00000000000..5bb527536a5 --- /dev/null +++ b/query/date_part_test.go @@ -0,0 +1,663 @@ +package query_test + +import ( + "testing" + "time" + + "github.com/influxdata/influxdb/models" + "github.com/influxdata/influxdb/query" + "github.com/influxdata/influxql" + "github.com/stretchr/testify/require" +) + +func TestParseDatePartExpr(t *testing.T) { + tests := []struct { + name string + input string + expected query.DatePartExpr + ok bool + }{ + {"year lowercase", "year", query.Year, true}, + {"year uppercase", "YEAR", query.Year, true}, + {"year mixed", "YeAr", query.Year, true}, + {"quarter", "quarter", query.Quarter, true}, + {"month", "month", query.Month, true}, + {"week", "week", query.Week, true}, + {"day", "day", query.Day, true}, + {"hour", "hour", query.Hour, true}, + {"minute", "minute", query.Minute, true}, + {"second", "second", query.Second, true}, + {"millisecond", "millisecond", query.Millisecond, true}, + {"microsecond", "microsecond", query.Microsecond, true}, + {"nanosecond", "nanosecond", query.Nanosecond, true}, + {"dow", "dow", query.DOW, true}, + {"DOW uppercase", "DOW", query.DOW, true}, + {"doy", "doy", query.DOY, true}, + {"epoch", "epoch", query.Epoch, true}, + {"isodow", "isodow", query.ISODOW, true}, + {"invalid", "invalid", query.Invalid, false}, + {"empty", "", query.Invalid, false}, + {"random", "foobar", query.Invalid, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, ok := query.ParseDatePartExpr(tt.input) + require.Equal(t, tt.ok, ok, "ok should match") + require.Equal(t, tt.expected, result, "parsed value should match") + }) + } +} + +func TestValidateDatePart(t *testing.T) { + tests := []struct { + name string + args []influxql.Expr + expectError bool + errorMsg string + expectField string + expectExpr query.DatePartExpr + }{ + { + name: "valid with time and DOW", + args: []influxql.Expr{ + &influxql.StringLiteral{Val: "DOW"}, + &influxql.VarRef{Val: models.TimeString}, + }, + expectError: false, + expectField: models.TimeString, + expectExpr: query.DOW, + }, + { + name: "valid with time and string literal", + args: []influxql.Expr{ + &influxql.StringLiteral{Val: "year"}, + &influxql.VarRef{Val: models.TimeString}, + }, + expectError: false, + expectField: models.TimeString, + expectExpr: query.Year, + }, + { + name: "invalid - wrong number of args (0)", + args: []influxql.Expr{}, + expectError: true, + errorMsg: "invalid number of arguments", + }, + { + name: "invalid - wrong number of args (1)", + args: []influxql.Expr{ + &influxql.VarRef{Val: models.TimeString}, + }, + expectError: true, + errorMsg: "invalid number of arguments", + }, + { + name: "invalid - wrong number of args (3)", + args: []influxql.Expr{ + &influxql.VarRef{Val: models.TimeString}, + &influxql.VarRef{Val: "DOW"}, + &influxql.VarRef{Val: "extra"}, + }, + expectError: true, + errorMsg: "invalid number of arguments", + }, + { + name: "invalid - first arg not StringLiteral", + args: []influxql.Expr{ + &influxql.IntegerLiteral{Val: 123}, + &influxql.VarRef{Val: models.TimeString}, + }, + expectError: true, + errorMsg: "first argument must be", + }, + { + name: "invalid - second arg not a VarRef", + args: []influxql.Expr{ + &influxql.StringLiteral{Val: "dow"}, + &influxql.IntegerLiteral{Val: 123}, + }, + expectError: true, + errorMsg: "second argument must be a variable reference", + }, + { + name: "invalid - second arg is a non-time VarRef", + args: []influxql.Expr{ + &influxql.StringLiteral{Val: "dow"}, + &influxql.VarRef{Val: "value"}, + }, + expectError: true, + errorMsg: "second argument must be time VarRef", + }, + { + name: "invalid - first arg is a non-string literal", + args: []influxql.Expr{ + &influxql.VarRef{Val: "invalid_expr"}, + &influxql.VarRef{Val: models.TimeString}, + }, + expectError: true, + errorMsg: "first argument must be a string", + }, + { + // A valid string literal whose value is not a known part exercises the + // ParseDatePartExpr-failure branch (distinct from the non-string branch + // above), including the construction of the valid-parts list. + name: "invalid - unknown date part name", + args: []influxql.Expr{ + &influxql.StringLiteral{Val: "bogus"}, + &influxql.VarRef{Val: models.TimeString}, + }, + expectError: true, + errorMsg: "first argument must be one of the following", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := query.ValidateDatePart(tt.args) + + if tt.expectError { + require.Error(t, err) + require.Contains(t, err.Error(), tt.errorMsg) + } else { + require.NoError(t, err) + } + }) + } +} + +func TestDatePartValuer_Call(t *testing.T) { + valuer := query.DatePartValuer{} + + // Test timestamp: 2024-01-15 14:30:45.123456789 UTC (Monday) + testTime := time.Date(2024, 1, 15, 14, 30, 45, 123456789, time.UTC) + testTimestamp := testTime.UnixNano() + + tests := []struct { + name string + funcName string + args []interface{} + expected interface{} + ok bool + }{ + { + name: "dow - Monday", + funcName: "date_part", + args: []interface{}{"dow", testTimestamp}, + expected: int64(1), // Monday + ok: true, + }, + { + name: "doy - 15th day of year", + funcName: "date_part", + args: []interface{}{"doy", testTimestamp}, + expected: int64(15), + ok: true, + }, + { + name: "year", + funcName: "date_part", + args: []interface{}{"year", testTimestamp}, + expected: int64(2024), + ok: true, + }, + { + name: "month - January", + funcName: "date_part", + args: []interface{}{"month", testTimestamp}, + expected: int64(1), + ok: true, + }, + { + name: "day", + funcName: "date_part", + args: []interface{}{"day", testTimestamp}, + expected: int64(15), + ok: true, + }, + { + name: "hour", + funcName: "date_part", + args: []interface{}{"hour", testTimestamp}, + expected: int64(14), + ok: true, + }, + { + name: "minute", + funcName: "date_part", + args: []interface{}{"minute", testTimestamp}, + expected: int64(30), + ok: true, + }, + { + name: "second", + funcName: "date_part", + args: []interface{}{"second", testTimestamp}, + expected: int64(45), + ok: true, + }, + { + name: "millisecond", + funcName: "date_part", + args: []interface{}{"millisecond", testTimestamp}, + expected: int64(45123), // 45s*1000 + 123456789ns/1e6 + ok: true, + }, + { + name: "microsecond", + funcName: "date_part", + args: []interface{}{"microsecond", testTimestamp}, + expected: int64(45123456), // 45s*1e6 + 123456789ns/1e3 + ok: true, + }, + { + name: "nanosecond", + funcName: "date_part", + args: []interface{}{"nanosecond", testTimestamp}, + expected: int64(45123456789), // 45s*1e9 + 123456789ns + ok: true, + }, + { + name: "epoch", + funcName: "date_part", + args: []interface{}{"epoch", testTimestamp}, + expected: testTime.Unix(), + ok: true, + }, + { + name: "isodow - Monday", + funcName: "date_part", + args: []interface{}{"isodow", testTimestamp}, + expected: int64(1), // Monday = 1 in ISO 8601 + ok: true, + }, + { + name: "quarter - Q1", + funcName: "date_part", + args: []interface{}{"quarter", testTimestamp}, + expected: int64(1), + ok: true, + }, + { + name: "week", + funcName: "date_part", + args: []interface{}{"week", testTimestamp}, + expected: int64(3), // 2024-01-15 is in ISO week 3 + ok: true, + }, + { + name: "wrong function name", + funcName: "other_function", + args: []interface{}{"dow", testTimestamp}, + expected: nil, + ok: false, + }, + { + name: "wrong number of args - 0", + funcName: "date_part", + args: []interface{}{}, + expected: nil, + ok: false, // Returns false for wrong arg count + }, + { + name: "wrong number of args - 1", + funcName: "date_part", + args: []interface{}{testTimestamp}, + expected: nil, + ok: false, + }, + { + name: "wrong number of args - 3", + funcName: "date_part", + args: []interface{}{"dow", testTimestamp, "extra"}, + expected: nil, + ok: false, + }, + { + name: "invalid timestamp type", + funcName: "date_part", + args: []interface{}{"dow", "not a timestamp"}, + expected: nil, + ok: false, + }, + { + name: "invalid expression type", + funcName: "date_part", + args: []interface{}{123, testTimestamp}, + expected: nil, + ok: false, + }, + { + name: "unknown date part expression", + funcName: "date_part", + args: []interface{}{"invalid", testTimestamp}, + expected: nil, + ok: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, ok := valuer.Call(tt.funcName, tt.args) + require.Equal(t, tt.ok, ok, "ok should match") + if tt.ok && tt.expected != nil { + require.Equal(t, tt.expected, result, "result should match") + } + }) + } +} + +func TestDatePartValuer_Call_Sunday(t *testing.T) { + valuer := query.DatePartValuer{} + + // Sunday: 2024-01-14 10:00:00 UTC + sunday := time.Date(2024, 1, 14, 10, 0, 0, 0, time.UTC) + sundayTimestamp := sunday.UnixNano() + + t.Run("dow - Sunday is 0", func(t *testing.T) { + result, ok := valuer.Call("date_part", []interface{}{"dow", sundayTimestamp}) + require.True(t, ok) + require.Equal(t, int64(0), result, "dow check") // Sunday = 0 + }) + + t.Run("isodow - Sunday is 7", func(t *testing.T) { + result, ok := valuer.Call("date_part", []interface{}{"isodow", sundayTimestamp}) + require.True(t, ok) + require.Equal(t, int64(7), result, "isodow check") // Sunday = 7 in ISO 8601 + }) +} + +func TestDatePartValuer_Call_ISOWeekBoundary(t *testing.T) { + valuer := query.DatePartValuer{} + + // 2023-01-01 is a Sunday that falls in ISO week 52 of 2022. + // This is the classic ISO-week footgun: date_part('week') follows + // ISOWeek() while date_part('year') follows the calendar year, so they + // disagree across the year boundary. + jan1 := time.Date(2023, 1, 1, 12, 0, 0, 0, time.UTC) + + week, ok := valuer.Call("date_part", []interface{}{"week", jan1.UnixNano()}) + require.True(t, ok) + require.Equal(t, int64(52), week, "2023-01-01 is in ISO week 52 of the prior year") + + year, ok := valuer.Call("date_part", []interface{}{"year", jan1.UnixNano()}) + require.True(t, ok) + require.Equal(t, int64(2023), year, "year follows the calendar year, not the ISO week-year") + + // 2021-01-01 is a Friday that falls in ISO week 53 of 2020. + week53 := time.Date(2021, 1, 1, 12, 0, 0, 0, time.UTC) + week, ok = valuer.Call("date_part", []interface{}{"week", week53.UnixNano()}) + require.True(t, ok) + require.Equal(t, int64(53), week, "2021-01-01 is in ISO week 53 of the prior year") +} + +func TestDatePartValuer_Call_PreEpoch(t *testing.T) { + valuer := query.DatePartValuer{} + + // One hour before the Unix epoch: 1969-12-31 23:00:00 UTC (a Wednesday). + preEpoch := time.Date(1969, 12, 31, 23, 0, 0, 0, time.UTC) + ts := preEpoch.UnixNano() + + tests := []struct { + part string + expected int64 + }{ + {"epoch", -3600}, // one hour before epoch + {"year", 1969}, + {"month", 12}, + {"day", 31}, + {"hour", 23}, + {"dow", 3}, // Wednesday + {"isodow", 3}, // Wednesday = 3 in ISO 8601 (Monday=1) + } + + for _, tt := range tests { + t.Run(tt.part, func(t *testing.T) { + result, ok := valuer.Call("date_part", []interface{}{tt.part, ts}) + require.True(t, ok) + require.Equal(t, tt.expected, result) + }) + } +} + +func TestDatePartValuer_Value(t *testing.T) { + now := time.Now().UnixNano() + mapValuer := influxql.MapValuer{} + mapValuer[models.TimeString] = now + + valuer := query.DatePartValuer{ + Valuer: mapValuer, + } + + // Valuer should return nil when passed a string that isn't DatePartTimeString + val, ok := valuer.Value("foo") + require.False(t, ok) + require.Nil(t, val) + + val, ok = valuer.Value(query.DatePartTimeString) + require.True(t, ok) + require.Equal(t, now, val) + + // A user field or tag literally named "date_part_time" must resolve to its + // own value, not be shadowed by the internal time sentinel. + mapValuer["date_part_time"] = int64(42) + val, ok = valuer.Value("date_part_time") + require.True(t, ok) + require.Equal(t, int64(42), val) +} + +func TestDatePartValuer_Call_GroupedDimension(t *testing.T) { + // Under GROUP BY date_part the active grouped value is authoritative for the + // series and is published via the DatePartDimensionsString key. date_part for + // the active part must return that grouped value; date_part for any other + // (non-active) part is undefined for the series and must return (nil, false) + // rather than recomputing from the bucket-representative timestamp. + m := influxql.MapValuer{} + m[query.DatePartDimensionsString] = query.DecodedDatePartKey{Expr: query.Year, Val: 2024} + valuer := query.DatePartValuer{Valuer: m} + + // args[1] is irrelevant on the grouped path; it returns before timestamp use. + got, ok := valuer.Call("date_part", []interface{}{"year", int64(0)}) + require.True(t, ok, "active dimension should resolve") + require.Equal(t, int64(2024), got, "active dimension returns the grouped value") + + got, ok = valuer.Call("date_part", []interface{}{"month", int64(0)}) + require.False(t, ok, "non-active dimension is undefined for the series") + require.Nil(t, got) +} + +func TestDatePartGrouper_ResolveKeys_FirstLevel(t *testing.T) { + g := query.NewDatePartGrouper([]query.DatePartDimension{ + {Expr: query.Month}, + }) + + aux := []interface{}{int64(3)} + entries, err := g.ResolveKeys(aux, query.TagSubset{}) + require.NoError(t, err) + require.Len(t, entries, 1) + require.NotEmpty(t, entries[0].DimKey) + require.NotEmpty(t, entries[0].EncodedKey) +} + +func TestDatePartGrouper_ResolveKeys_FirstLevel_WithTags(t *testing.T) { + g := query.NewDatePartGrouper([]query.DatePartDimension{ + {Expr: query.Month}, + }) + + aux := []interface{}{int64(3)} + entries, err := g.ResolveKeys(aux, query.TagSubset{ID: "host=server01", HasTags: true}) + require.NoError(t, err) + require.Len(t, entries, 1) + require.NotEmpty(t, entries[0].DimKey) +} + +func TestDatePartGrouper_DimKey_NoCollisionWithNulBytesInTagID(t *testing.T) { + // Tag IDs can contain NUL bytes (e.g. empty tag values), so the grouping key + // must not collide for distinct tag IDs. The length-prefixed encoding keeps + // them unambiguous even when a tag ID ends in / contains the bytes that the + // old separator scheme used. + g := query.NewDatePartGrouper([]query.DatePartDimension{ + {Expr: query.Month}, + }) + + a, err := g.ResolveKeys([]interface{}{int64(3)}, query.TagSubset{ID: "a\x00\x00b", HasTags: true}) + require.NoError(t, err) + b, err := g.ResolveKeys([]interface{}{int64(3)}, query.TagSubset{ID: "a\x00\x00b\x00\x00c", HasTags: true}) + require.NoError(t, err) + require.NotEqual(t, a[0].DimKey, b[0].DimKey, "distinct tag IDs must yield distinct grouping keys") +} + +func TestDatePartGrouper_ResolveKeys_SecondLevel(t *testing.T) { + g := query.NewDatePartGrouper([]query.DatePartDimension{ + {Expr: query.Month}, + }) + + aux := []interface{}{query.DecodedDatePartKey{Expr: query.Month, Val: 3}} + entries, err := g.ResolveKeys(aux, query.TagSubset{}) + require.NoError(t, err) + require.Len(t, entries, 1) +} + +func TestDatePartGrouper_DecodeEntry(t *testing.T) { + g := query.NewDatePartGrouper([]query.DatePartDimension{ + {Expr: query.Month}, + }) + + aux := []interface{}{int64(7)} + entries, err := g.ResolveKeys(aux, query.TagSubset{}) + require.NoError(t, err) + + decoded, err := g.DecodeEntry(entries[0].EncodedKey) + require.NoError(t, err) + dpk, ok := decoded.(query.DecodedDatePartKey) + require.True(t, ok, "expected DecodedDatePartKey, got %T", decoded) + require.Equal(t, int64(7), dpk.Val) +} + +func TestDatePartGrouper_ResolveKeys_AuxShorterThanDims(t *testing.T) { + // Two dimensions but only one raw value and no DecodedDatePartKey present: + // ResolveKeys cannot map values to dims, so it returns (nil, nil). + g := query.NewDatePartGrouper([]query.DatePartDimension{ + {Expr: query.Year}, + {Expr: query.Month}, + }) + + entries, err := g.ResolveKeys([]interface{}{int64(3)}, query.TagSubset{}) + require.NoError(t, err) + require.Nil(t, entries) +} + +func TestDatePartGrouper_ResolveKeys_UnexpectedAuxType(t *testing.T) { + // A first-level aux value that is neither int64 nor DecodedDatePartKey + // must surface an error rather than silently mis-grouping. + g := query.NewDatePartGrouper([]query.DatePartDimension{ + {Expr: query.Month}, + }) + + entries, err := g.ResolveKeys([]interface{}{"not an int"}, query.TagSubset{}) + require.Error(t, err) + require.Contains(t, err.Error(), "unexpected aux value type") + require.Nil(t, entries) +} + +func TestDatePartGrouper_DecodeEntry_InvalidLength(t *testing.T) { + // The encoding is exactly 9 bytes (1 byte expr + 8 byte value). Both shorter + // and longer keys must be rejected rather than read out of bounds or silently + // truncated. + g := query.NewDatePartGrouper([]query.DatePartDimension{ + {Expr: query.Month}, + }) + + for _, key := range []string{"short", "this key is far too long"} { + _, err := g.DecodeEntry(key) + require.Error(t, err) + require.Contains(t, err.Error(), "must be exactly 9 bytes") + } +} + +func TestDatePartGrouper_DecodeEntry_InvalidExprByte(t *testing.T) { + // A 9-byte key whose first byte is not a valid DatePartExpr must be rejected + // rather than decoded into an out-of-range expr (whose String() is empty and + // would silently misroute the output column). + g := query.NewDatePartGrouper([]query.DatePartDimension{ + {Expr: query.Month}, + }) + + key := string([]byte{200, 0, 0, 0, 0, 0, 0, 0, 0}) + _, err := g.DecodeEntry(key) + require.Error(t, err) + require.Contains(t, err.Error(), "invalid expr byte") +} + +func TestDatePartGrouper_RoundTrip_MultiDimension(t *testing.T) { + g := query.NewDatePartGrouper([]query.DatePartDimension{ + {Expr: query.Year}, + {Expr: query.Month}, + }) + + aux := []interface{}{int64(2026), int64(3)} + entries, err := g.ResolveKeys(aux, query.TagSubset{}) + require.NoError(t, err) + require.Len(t, entries, 2) + + for _, e := range entries { + _, err := g.DecodeEntry(e.EncodedKey) + require.NoError(t, err) + } +} + +func TestDatePartValuer_Call_Timezone(t *testing.T) { + ny, err := time.LoadLocation("America/New_York") + require.NoError(t, err) + + // 2023-07-01T03:30:00Z is EDT (UTC-4) → local 2023-06-30 23:30. + ts := time.Date(2023, 7, 1, 3, 30, 0, 0, time.UTC).UnixNano() + + utc := query.DatePartValuer{} // nil Location → UTC + local := query.DatePartValuer{Location: ny} + + hUTC, ok := utc.Call("date_part", []interface{}{"hour", ts}) + require.True(t, ok) + require.Equal(t, int64(3), hUTC, "UTC hour") + + hNY, ok := local.Call("date_part", []interface{}{"hour", ts}) + require.True(t, ok) + require.Equal(t, int64(23), hNY, "New York local hour (previous day)") + + dNY, ok := local.Call("date_part", []interface{}{"day", ts}) + require.True(t, ok) + require.Equal(t, int64(30), dNY, "New York local day rolls back to June 30") + + // epoch is an absolute instant — identical regardless of zone. + eUTC, _ := utc.Call("date_part", []interface{}{"epoch", ts}) + eNY, _ := local.Call("date_part", []interface{}{"epoch", ts}) + require.Equal(t, eUTC, eNY, "epoch must be zone-independent") + require.Equal(t, time.Date(2023, 7, 1, 3, 30, 0, 0, time.UTC).Unix(), eNY) +} + +func TestDatePartValuer_Call_DST(t *testing.T) { + ny, err := time.LoadLocation("America/New_York") + require.NoError(t, err) + v := query.DatePartValuer{Location: ny} + + // Spring forward: 2023-03-12 02:00 EST → 03:00 EDT (transition at 07:00Z). + // 07:30Z is EDT (UTC-4) → 03:30 local. (Naive EST would give hour 2.) + spring := time.Date(2023, 3, 12, 7, 30, 0, 0, time.UTC).UnixNano() + h, ok := v.Call("date_part", []interface{}{"hour", spring}) + require.True(t, ok) + require.Equal(t, int64(3), h, "spring-forward: DST offset applied") + + // Fall back: 2023-11-05 02:00 EDT → 01:00 EST (transition at 06:00Z). + // 07:30Z is EST (UTC-5) → 02:30 local. (Naive EDT would give hour 3.) + fall := time.Date(2023, 11, 5, 7, 30, 0, 0, time.UTC).UnixNano() + h, ok = v.Call("date_part", []interface{}{"hour", fall}) + require.True(t, ok) + require.Equal(t, int64(2), h, "fall-back: standard time resumed") +} + +func TestLocationOrUTC(t *testing.T) { + require.Equal(t, time.UTC, query.LocationOrUTC(nil)) + ny, err := time.LoadLocation("America/New_York") + require.NoError(t, err) + require.Equal(t, ny, query.LocationOrUTC(ny)) +} diff --git a/query/dimension_grouper.go b/query/dimension_grouper.go new file mode 100644 index 00000000000..07936fd5d0e --- /dev/null +++ b/query/dimension_grouper.go @@ -0,0 +1,27 @@ +package query + +// TagSubset identifies the tag subset a point belongs to at the current level +// of the query. HasTags distinguishes "no tag dimensions in the GROUP BY" from +// a tag subset whose ID happens to be empty, so composite grouping keys stay +// unambiguous. +type TagSubset struct { + ID string + HasTags bool +} + +// DimensionGrouper resolves additional grouping keys from a point's auxiliary data. +// Implementations handle specific GROUP BY function types (e.g. date_part). +type DimensionGrouper interface { + // ResolveKeys examines a point's aux values and returns grouping entries + // composed with the point's tag subset. + ResolveKeys(aux []interface{}, tags TagSubset) ([]GroupingEntry, error) + + // DecodeEntry reconstructs an aux-transportable value from an encoded key + // so it can be appended to a point's Aux slice for multi-level reduces. + DecodeEntry(encodedKey string) (interface{}, error) +} + +type GroupingEntry struct { + DimKey string + EncodedKey string +} diff --git a/query/emitter.go b/query/emitter.go index 1888824e806..fe5f17f631b 100644 --- a/query/emitter.go +++ b/query/emitter.go @@ -1,6 +1,8 @@ package query import ( + "sort" + "github.com/influxdata/influxdb/models" ) @@ -9,9 +11,10 @@ type Emitter struct { cur Cursor chunkSize int - series Series - row *models.Row - columns []string + series Series + groupingKeys map[string]struct{} + row *models.Row + columns []string } // NewEmitter returns a new instance of Emitter that pulls from itrs. @@ -52,30 +55,58 @@ func (e *Emitter) Emit() (*models.Row, bool, error) { // the number of values doesn't exceed the chunk size. // Otherwise return existing row and add values to next emitted row. if e.row == nil { - e.createRow(row.Series, row.Values) - } else if e.series.SameSeries(row.Series) { + e.createRow(row.Series, row.GroupingKeys, row.Values) + } else if e.series.SameSeries(row.Series) && sameGroupingKeys(e.groupingKeys, row.GroupingKeys) { if e.chunkSize > 0 && len(e.row.Values) >= e.chunkSize { r := e.row r.Partial = true - e.createRow(row.Series, row.Values) + e.createRow(row.Series, row.GroupingKeys, row.Values) return r, true, nil } e.row.Values = append(e.row.Values, row.Values) } else { r := e.row - e.createRow(row.Series, row.Values) + e.createRow(row.Series, row.GroupingKeys, row.Values) return r, true, nil } } } // createRow creates a new row attached to the emitter. -func (e *Emitter) createRow(series Series, values []interface{}) { +func (e *Emitter) createRow(series Series, groupingKeys map[string]struct{}, values []interface{}) { e.series = series + e.groupingKeys = groupingKeys e.row = &models.Row{ - Name: series.Name, - Tags: series.Tags.KeyValues(), - Columns: e.columns, - Values: [][]interface{}{values}, + Name: series.Name, + Tags: series.Tags.KeyValues(), + GroupingKeys: sortedKeys(groupingKeys), + Columns: e.columns, + Values: [][]interface{}{values}, + } +} + +// sameGroupingKeys returns true if two grouping key sets have the same dimension names. +func sameGroupingKeys(a, b map[string]struct{}) bool { + if len(a) != len(b) { + return false + } + for k := range a { + if _, ok := b[k]; !ok { + return false + } + } + return true +} + +// sortedKeys returns a sorted slice of keys from a set. +func sortedKeys(m map[string]struct{}) []string { + if len(m) == 0 { + return nil + } + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) } + sort.Strings(keys) + return keys } diff --git a/query/export_test.go b/query/export_test.go new file mode 100644 index 00000000000..84858bd3ba2 --- /dev/null +++ b/query/export_test.go @@ -0,0 +1,36 @@ +package query + +// This file aliases unexported compilation internals to exported names so they +// are visible to test code in package query_test without widening the +// production API (test-only files are excluded from regular builds). + +// CompiledStatement exposes compiledStatement so query_test can construct one +// via NewCompilerForTesting and drive its unexported methods. +type CompiledStatement = compiledStatement + +// NewCompilerForTesting exposes newCompiler. +func NewCompilerForTesting(opt CompileOptions) *CompiledStatement { + return newCompiler(opt) +} + +// CompileTimeDimension exposes (*compiledStatement).compileTimeDimension. +var CompileTimeDimension = (*compiledStatement).compileTimeDimension + +// Exported aliases for the compilation sentinel errors. +var ( + ErrOnlyTimeAndDatePartDimensions = errOnlyTimeAndDatePartDimensions + ErrTimeDimensionArgCount = errTimeDimensionArgCount + ErrTimeDimensionDurationArg = errTimeDimensionDurationArg + ErrMultipleTimeDimensions = errMultipleTimeDimensions + ErrTimeOffsetFunctionMustBeNow = errTimeOffsetFunctionMustBeNow + ErrTimeOffsetNowNoArgs = errTimeOffsetNowNoArgs + ErrInvalidTimeOffset = errInvalidTimeOffset + ErrAtLeastOneNonTimeField = errAtLeastOneNonTimeField + ErrMixedMultipleSelectors = errMixedMultipleSelectors + ErrDatePartRequiresAggregate = errDatePartRequiresAggregate + ErrDatePartSingleAggregate = errDatePartSingleAggregate + ErrDatePartFillPrevious = errDatePartFillPrevious + ErrDatePartFillLinear = errDatePartFillLinear + ErrDatePartFillValue = errDatePartFillValue + ErrDatePartFillNull = errDatePartFillNull +) diff --git a/query/functions.go b/query/functions.go index bae4a89b07f..e7328d5c995 100644 --- a/query/functions.go +++ b/query/functions.go @@ -92,7 +92,7 @@ func (m FunctionTypeMapper) CallType(name string, args []influxql.DataType) (inf "chande_momentum_oscillator", "holt_winters", "holt_winters_with_fit": return influxql.Float, nil - case "elapsed": + case "elapsed", DatePartString: return influxql.Integer, nil default: // TODO(jsternberg): Do not use default for this. diff --git a/query/internal/internal.pb.go b/query/internal/internal.pb.go index 4fe2776da74..ead1e0a79d6 100644 --- a/query/internal/internal.pb.go +++ b/query/internal/internal.pb.go @@ -246,31 +246,33 @@ func (x *Aux) GetUnsignedValue() uint64 { } type IteratorOptions struct { - state protoimpl.MessageState `protogen:"open.v1"` - Expr *string `protobuf:"bytes,1,opt,name=Expr" json:"Expr,omitempty"` - Aux []string `protobuf:"bytes,2,rep,name=Aux" json:"Aux,omitempty"` - Fields []*VarRef `protobuf:"bytes,17,rep,name=Fields" json:"Fields,omitempty"` - Sources []*Measurement `protobuf:"bytes,3,rep,name=Sources" json:"Sources,omitempty"` - Interval *Interval `protobuf:"bytes,4,opt,name=Interval" json:"Interval,omitempty"` - Dimensions []string `protobuf:"bytes,5,rep,name=Dimensions" json:"Dimensions,omitempty"` - GroupBy []string `protobuf:"bytes,19,rep,name=GroupBy" json:"GroupBy,omitempty"` - Fill *int32 `protobuf:"varint,6,opt,name=Fill" json:"Fill,omitempty"` - FillValue *float64 `protobuf:"fixed64,7,opt,name=FillValue" json:"FillValue,omitempty"` - Condition *string `protobuf:"bytes,8,opt,name=Condition" json:"Condition,omitempty"` - StartTime *int64 `protobuf:"varint,9,opt,name=StartTime" json:"StartTime,omitempty"` - EndTime *int64 `protobuf:"varint,10,opt,name=EndTime" json:"EndTime,omitempty"` - Location *string `protobuf:"bytes,21,opt,name=Location" json:"Location,omitempty"` - Ascending *bool `protobuf:"varint,11,opt,name=Ascending" json:"Ascending,omitempty"` - Limit *int64 `protobuf:"varint,12,opt,name=Limit" json:"Limit,omitempty"` - Offset *int64 `protobuf:"varint,13,opt,name=Offset" json:"Offset,omitempty"` - SLimit *int64 `protobuf:"varint,14,opt,name=SLimit" json:"SLimit,omitempty"` - SOffset *int64 `protobuf:"varint,15,opt,name=SOffset" json:"SOffset,omitempty"` - StripName *bool `protobuf:"varint,22,opt,name=StripName" json:"StripName,omitempty"` - Dedupe *bool `protobuf:"varint,16,opt,name=Dedupe" json:"Dedupe,omitempty"` - MaxSeriesN *int64 `protobuf:"varint,18,opt,name=MaxSeriesN" json:"MaxSeriesN,omitempty"` - Ordered *bool `protobuf:"varint,20,opt,name=Ordered" json:"Ordered,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Expr *string `protobuf:"bytes,1,opt,name=Expr" json:"Expr,omitempty"` + Aux []string `protobuf:"bytes,2,rep,name=Aux" json:"Aux,omitempty"` + Fields []*VarRef `protobuf:"bytes,17,rep,name=Fields" json:"Fields,omitempty"` + Sources []*Measurement `protobuf:"bytes,3,rep,name=Sources" json:"Sources,omitempty"` + Interval *Interval `protobuf:"bytes,4,opt,name=Interval" json:"Interval,omitempty"` + Dimensions []string `protobuf:"bytes,5,rep,name=Dimensions" json:"Dimensions,omitempty"` + GroupBy []string `protobuf:"bytes,19,rep,name=GroupBy" json:"GroupBy,omitempty"` + Fill *int32 `protobuf:"varint,6,opt,name=Fill" json:"Fill,omitempty"` + FillValue *float64 `protobuf:"fixed64,7,opt,name=FillValue" json:"FillValue,omitempty"` + Condition *string `protobuf:"bytes,8,opt,name=Condition" json:"Condition,omitempty"` + StartTime *int64 `protobuf:"varint,9,opt,name=StartTime" json:"StartTime,omitempty"` + EndTime *int64 `protobuf:"varint,10,opt,name=EndTime" json:"EndTime,omitempty"` + Location *string `protobuf:"bytes,21,opt,name=Location" json:"Location,omitempty"` + Ascending *bool `protobuf:"varint,11,opt,name=Ascending" json:"Ascending,omitempty"` + Limit *int64 `protobuf:"varint,12,opt,name=Limit" json:"Limit,omitempty"` + Offset *int64 `protobuf:"varint,13,opt,name=Offset" json:"Offset,omitempty"` + SLimit *int64 `protobuf:"varint,14,opt,name=SLimit" json:"SLimit,omitempty"` + SOffset *int64 `protobuf:"varint,15,opt,name=SOffset" json:"SOffset,omitempty"` + StripName *bool `protobuf:"varint,22,opt,name=StripName" json:"StripName,omitempty"` + Dedupe *bool `protobuf:"varint,16,opt,name=Dedupe" json:"Dedupe,omitempty"` + MaxSeriesN *int64 `protobuf:"varint,18,opt,name=MaxSeriesN" json:"MaxSeriesN,omitempty"` + Ordered *bool `protobuf:"varint,20,opt,name=Ordered" json:"Ordered,omitempty"` + DatePartDimensions []*DatePartDimension `protobuf:"bytes,23,rep,name=DatePartDimensions" json:"DatePartDimensions,omitempty"` + NeedTimeRef *bool `protobuf:"varint,24,opt,name=NeedTimeRef" json:"NeedTimeRef,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *IteratorOptions) Reset() { @@ -457,6 +459,64 @@ func (x *IteratorOptions) GetOrdered() bool { return false } +func (x *IteratorOptions) GetDatePartDimensions() []*DatePartDimension { + if x != nil { + return x.DatePartDimensions + } + return nil +} + +func (x *IteratorOptions) GetNeedTimeRef() bool { + if x != nil && x.NeedTimeRef != nil { + return *x.NeedTimeRef + } + return false +} + +type DatePartDimension struct { + state protoimpl.MessageState `protogen:"open.v1"` + Expr *int32 `protobuf:"varint,1,opt,name=Expr" json:"Expr,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DatePartDimension) Reset() { + *x = DatePartDimension{} + mi := &file_internal_internal_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DatePartDimension) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DatePartDimension) ProtoMessage() {} + +func (x *DatePartDimension) ProtoReflect() protoreflect.Message { + mi := &file_internal_internal_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DatePartDimension.ProtoReflect.Descriptor instead. +func (*DatePartDimension) Descriptor() ([]byte, []int) { + return file_internal_internal_proto_rawDescGZIP(), []int{3} +} + +func (x *DatePartDimension) GetExpr() int32 { + if x != nil && x.Expr != nil { + return *x.Expr + } + return 0 +} + type Measurements struct { state protoimpl.MessageState `protogen:"open.v1"` Items []*Measurement `protobuf:"bytes,1,rep,name=Items" json:"Items,omitempty"` @@ -466,7 +526,7 @@ type Measurements struct { func (x *Measurements) Reset() { *x = Measurements{} - mi := &file_internal_internal_proto_msgTypes[3] + mi := &file_internal_internal_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -478,7 +538,7 @@ func (x *Measurements) String() string { func (*Measurements) ProtoMessage() {} func (x *Measurements) ProtoReflect() protoreflect.Message { - mi := &file_internal_internal_proto_msgTypes[3] + mi := &file_internal_internal_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -491,7 +551,7 @@ func (x *Measurements) ProtoReflect() protoreflect.Message { // Deprecated: Use Measurements.ProtoReflect.Descriptor instead. func (*Measurements) Descriptor() ([]byte, []int) { - return file_internal_internal_proto_rawDescGZIP(), []int{3} + return file_internal_internal_proto_rawDescGZIP(), []int{4} } func (x *Measurements) GetItems() []*Measurement { @@ -515,7 +575,7 @@ type Measurement struct { func (x *Measurement) Reset() { *x = Measurement{} - mi := &file_internal_internal_proto_msgTypes[4] + mi := &file_internal_internal_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -527,7 +587,7 @@ func (x *Measurement) String() string { func (*Measurement) ProtoMessage() {} func (x *Measurement) ProtoReflect() protoreflect.Message { - mi := &file_internal_internal_proto_msgTypes[4] + mi := &file_internal_internal_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -540,7 +600,7 @@ func (x *Measurement) ProtoReflect() protoreflect.Message { // Deprecated: Use Measurement.ProtoReflect.Descriptor instead. func (*Measurement) Descriptor() ([]byte, []int) { - return file_internal_internal_proto_rawDescGZIP(), []int{4} + return file_internal_internal_proto_rawDescGZIP(), []int{5} } func (x *Measurement) GetDatabase() string { @@ -595,7 +655,7 @@ type Interval struct { func (x *Interval) Reset() { *x = Interval{} - mi := &file_internal_internal_proto_msgTypes[5] + mi := &file_internal_internal_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -607,7 +667,7 @@ func (x *Interval) String() string { func (*Interval) ProtoMessage() {} func (x *Interval) ProtoReflect() protoreflect.Message { - mi := &file_internal_internal_proto_msgTypes[5] + mi := &file_internal_internal_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -620,7 +680,7 @@ func (x *Interval) ProtoReflect() protoreflect.Message { // Deprecated: Use Interval.ProtoReflect.Descriptor instead. func (*Interval) Descriptor() ([]byte, []int) { - return file_internal_internal_proto_rawDescGZIP(), []int{5} + return file_internal_internal_proto_rawDescGZIP(), []int{6} } func (x *Interval) GetDuration() int64 { @@ -647,7 +707,7 @@ type IteratorStats struct { func (x *IteratorStats) Reset() { *x = IteratorStats{} - mi := &file_internal_internal_proto_msgTypes[6] + mi := &file_internal_internal_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -659,7 +719,7 @@ func (x *IteratorStats) String() string { func (*IteratorStats) ProtoMessage() {} func (x *IteratorStats) ProtoReflect() protoreflect.Message { - mi := &file_internal_internal_proto_msgTypes[6] + mi := &file_internal_internal_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -672,7 +732,7 @@ func (x *IteratorStats) ProtoReflect() protoreflect.Message { // Deprecated: Use IteratorStats.ProtoReflect.Descriptor instead. func (*IteratorStats) Descriptor() ([]byte, []int) { - return file_internal_internal_proto_rawDescGZIP(), []int{6} + return file_internal_internal_proto_rawDescGZIP(), []int{7} } func (x *IteratorStats) GetSeriesN() int64 { @@ -699,7 +759,7 @@ type VarRef struct { func (x *VarRef) Reset() { *x = VarRef{} - mi := &file_internal_internal_proto_msgTypes[7] + mi := &file_internal_internal_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -711,7 +771,7 @@ func (x *VarRef) String() string { func (*VarRef) ProtoMessage() {} func (x *VarRef) ProtoReflect() protoreflect.Message { - mi := &file_internal_internal_proto_msgTypes[7] + mi := &file_internal_internal_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -724,7 +784,7 @@ func (x *VarRef) ProtoReflect() protoreflect.Message { // Deprecated: Use VarRef.ProtoReflect.Descriptor instead. func (*VarRef) Descriptor() ([]byte, []int) { - return file_internal_internal_proto_rawDescGZIP(), []int{7} + return file_internal_internal_proto_rawDescGZIP(), []int{8} } func (x *VarRef) GetVal() string { @@ -774,7 +834,7 @@ const file_internal_internal_proto_rawDesc = "" + "\fIntegerValue\x18\x03 \x01(\x03R\fIntegerValue\x12 \n" + "\vStringValue\x18\x04 \x01(\tR\vStringValue\x12\"\n" + "\fBooleanValue\x18\x05 \x01(\bR\fBooleanValue\x12$\n" + - "\rUnsignedValue\x18\x06 \x01(\x04R\rUnsignedValue\"\x85\x05\n" + + "\rUnsignedValue\x18\x06 \x01(\x04R\rUnsignedValue\"\xf1\x05\n" + "\x0fIteratorOptions\x12\x12\n" + "\x04Expr\x18\x01 \x01(\tR\x04Expr\x12\x10\n" + "\x03Aux\x18\x02 \x03(\tR\x03Aux\x12%\n" + @@ -802,7 +862,11 @@ const file_internal_internal_proto_rawDesc = "" + "\n" + "MaxSeriesN\x18\x12 \x01(\x03R\n" + "MaxSeriesN\x12\x18\n" + - "\aOrdered\x18\x14 \x01(\bR\aOrdered\"8\n" + + "\aOrdered\x18\x14 \x01(\bR\aOrdered\x12H\n" + + "\x12DatePartDimensions\x18\x17 \x03(\v2\x18.query.DatePartDimensionR\x12DatePartDimensions\x12 \n" + + "\vNeedTimeRef\x18\x18 \x01(\bR\vNeedTimeRef\"'\n" + + "\x11DatePartDimension\x12\x12\n" + + "\x04Expr\x18\x01 \x01(\x05R\x04Expr\"8\n" + "\fMeasurements\x12(\n" + "\x05Items\x18\x01 \x03(\v2\x12.query.MeasurementR\x05Items\"\xc1\x01\n" + "\vMeasurement\x12\x1a\n" + @@ -834,29 +898,31 @@ func file_internal_internal_proto_rawDescGZIP() []byte { return file_internal_internal_proto_rawDescData } -var file_internal_internal_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_internal_internal_proto_msgTypes = make([]protoimpl.MessageInfo, 9) var file_internal_internal_proto_goTypes = []any{ - (*Point)(nil), // 0: query.Point - (*Aux)(nil), // 1: query.Aux - (*IteratorOptions)(nil), // 2: query.IteratorOptions - (*Measurements)(nil), // 3: query.Measurements - (*Measurement)(nil), // 4: query.Measurement - (*Interval)(nil), // 5: query.Interval - (*IteratorStats)(nil), // 6: query.IteratorStats - (*VarRef)(nil), // 7: query.VarRef + (*Point)(nil), // 0: query.Point + (*Aux)(nil), // 1: query.Aux + (*IteratorOptions)(nil), // 2: query.IteratorOptions + (*DatePartDimension)(nil), // 3: query.DatePartDimension + (*Measurements)(nil), // 4: query.Measurements + (*Measurement)(nil), // 5: query.Measurement + (*Interval)(nil), // 6: query.Interval + (*IteratorStats)(nil), // 7: query.IteratorStats + (*VarRef)(nil), // 8: query.VarRef } var file_internal_internal_proto_depIdxs = []int32{ 1, // 0: query.Point.Aux:type_name -> query.Aux - 6, // 1: query.Point.Stats:type_name -> query.IteratorStats - 7, // 2: query.IteratorOptions.Fields:type_name -> query.VarRef - 4, // 3: query.IteratorOptions.Sources:type_name -> query.Measurement - 5, // 4: query.IteratorOptions.Interval:type_name -> query.Interval - 4, // 5: query.Measurements.Items:type_name -> query.Measurement - 6, // [6:6] is the sub-list for method output_type - 6, // [6:6] is the sub-list for method input_type - 6, // [6:6] is the sub-list for extension type_name - 6, // [6:6] is the sub-list for extension extendee - 0, // [0:6] is the sub-list for field type_name + 7, // 1: query.Point.Stats:type_name -> query.IteratorStats + 8, // 2: query.IteratorOptions.Fields:type_name -> query.VarRef + 5, // 3: query.IteratorOptions.Sources:type_name -> query.Measurement + 6, // 4: query.IteratorOptions.Interval:type_name -> query.Interval + 3, // 5: query.IteratorOptions.DatePartDimensions:type_name -> query.DatePartDimension + 5, // 6: query.Measurements.Items:type_name -> query.Measurement + 7, // [7:7] is the sub-list for method output_type + 7, // [7:7] is the sub-list for method input_type + 7, // [7:7] is the sub-list for extension type_name + 7, // [7:7] is the sub-list for extension extendee + 0, // [0:7] is the sub-list for field type_name } func init() { file_internal_internal_proto_init() } @@ -870,7 +936,7 @@ func file_internal_internal_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_internal_internal_proto_rawDesc), len(file_internal_internal_proto_rawDesc)), NumEnums: 0, - NumMessages: 8, + NumMessages: 9, NumExtensions: 0, NumServices: 0, }, diff --git a/query/internal/internal.proto b/query/internal/internal.proto index 0a501550384..c7066673c75 100644 --- a/query/internal/internal.proto +++ b/query/internal/internal.proto @@ -52,6 +52,12 @@ message IteratorOptions { optional bool Dedupe = 16; optional int64 MaxSeriesN = 18; optional bool Ordered = 20; + repeated DatePartDimension DatePartDimensions = 23; + optional bool NeedTimeRef = 24; +} + +message DatePartDimension { + optional int32 Expr = 1; } message Measurements { diff --git a/query/iterator.gen.go b/query/iterator.gen.go index 5b2b8d18a96..618ce7a7a2b 100644 --- a/query/iterator.gen.go +++ b/query/iterator.gen.go @@ -15,6 +15,7 @@ import ( "sync" "time" + "github.com/influxdata/influxdb/models" "github.com/influxdata/influxql" "google.golang.org/protobuf/proto" ) @@ -556,6 +557,11 @@ func (s *floatIteratorScanner) ScanAt(ts int64, name string, tags Tags, m map[st switch v.(type) { case float64, int64, uint64, string, bool: m[k.Val] = v + case DecodedDatePartKey: + m[DatePartDimensionsString] = v + // Clear any stale raw value under this dimension's own key so a prior + // row's value can't persist (the active value is carried via the key above). + delete(m, k.Val) default: // Insert the fill value if one was specified. if s.defaultValue != SkipDefault { @@ -1038,10 +1044,11 @@ func (itr *floatReduceFloatIterator) Next() (*FloatPoint, error) { // floatReduceFloatPoint stores the reduced data for a name/tag combination. type floatReduceFloatPoint struct { - Name string - Tags Tags - Aggregator FloatPointAggregator - Emitter FloatPointEmitter + Name string + Tags Tags + GroupingKey string + Aggregator FloatPointAggregator + Emitter FloatPointEmitter } // reduce executes fn once for every point in the next window. @@ -1100,19 +1107,44 @@ func (itr *floatReduceFloatIterator) reduce() ([]FloatPoint, error) { tags := curr.Tags.Subset(itr.dims) id := tags.ID() - // Retrieve the aggregator for this name/tag combination or create one. - rp := m[id] - if rp == nil { - aggregator, emitter := itr.create() - rp = &floatReduceFloatPoint{ - Name: curr.Name, - Tags: tags, - Aggregator: aggregator, - Emitter: emitter, + // Check to see if we have any GROUP BY dimensions that are not tags or time. + // If we have group-by key entries, create separate iterators for them. + // Otherwise, proceed as normal and create an iterator keyed by the tag ID. + if itr.opt.DimensionGrouper != nil && len(curr.Aux) > 0 { + entries, err := itr.opt.DimensionGrouper.ResolveKeys(curr.Aux, TagSubset{ID: id, HasTags: len(itr.dims) > 0}) + if err != nil { + return nil, err + } + + for _, entry := range entries { + rp := m[entry.DimKey] + if rp == nil { + aggregator, emitter := itr.create() + rp = &floatReduceFloatPoint{ + Name: curr.Name, + Tags: tags, + GroupingKey: entry.EncodedKey, + Aggregator: aggregator, + Emitter: emitter, + } + m[entry.DimKey] = rp + } + rp.Aggregator.AggregateFloat(curr) + } + } else { + rp := m[id] + if rp == nil { + aggregator, emitter := itr.create() + rp = &floatReduceFloatPoint{ + Name: curr.Name, + Tags: tags, + Aggregator: aggregator, + Emitter: emitter, + } + m[id] = rp } - m[id] = rp + rp.Aggregator.AggregateFloat(curr) } - rp.Aggregator.AggregateFloat(curr) } keys := make([]string, 0, len(m)) @@ -1148,6 +1180,50 @@ func (itr *floatReduceFloatIterator) reduce() ([]FloatPoint, error) { } else { sortedByTime = false } + + if itr.opt.DimensionGrouper != nil && rp.GroupingKey != "" { + dpVal, err := itr.opt.DimensionGrouper.DecodeEntry(rp.GroupingKey) + if err != nil { + return nil, err + } + // The emitted point must carry an Aux slot for every scanner aux + // key (len(itr.opt.Aux)). IteratorScanner.ScanAt ranges over the + // point's Aux, so a slice shorter than the key set leaves the + // trailing keys unvisited and the eval map retains stale values + // from a prior row, wrongly populating non-active date_part + // dimension columns. Selector functions (MIN, MAX, FIRST, LAST) + // already return a full-width Aux via cloneAux; aggregate + // functions (COUNT, SUM, MEAN) return an empty Aux, so grow it to + // full width here. (Keep any longer Aux as-is.) + width := len(itr.opt.Aux) + if width < len(points[i].Aux) { + width = len(points[i].Aux) + } + if width < 1 { + width = 1 + } + if len(points[i].Aux) < width { + aux := make([]interface{}, width) + copy(aux, points[i].Aux) + points[i].Aux = aux + } + // Only the active dimension is meaningful for this series, so null + // every date_part dimension slot (leaving non-active dimension + // columns null) and carry the active value as a DecodedDatePartKey + // in the last slot, which the scanner routes to the correct column + // by name. A stable, full-width slot ensures every key is visited + // and cleared on each scan. + n := len(points[i].Aux) + base := n - len(itr.opt.DatePartDimensions) + if base < 0 { + base = 0 + } + for j := base; j < n; j++ { + points[i].Aux[j] = nil + } + points[i].Aux[n-1] = dpVal + } + a = append(a, points[i]) } } @@ -1326,10 +1402,11 @@ func (itr *floatReduceIntegerIterator) Next() (*IntegerPoint, error) { // floatReduceIntegerPoint stores the reduced data for a name/tag combination. type floatReduceIntegerPoint struct { - Name string - Tags Tags - Aggregator FloatPointAggregator - Emitter IntegerPointEmitter + Name string + Tags Tags + GroupingKey string + Aggregator FloatPointAggregator + Emitter IntegerPointEmitter } // reduce executes fn once for every point in the next window. @@ -1388,19 +1465,44 @@ func (itr *floatReduceIntegerIterator) reduce() ([]IntegerPoint, error) { tags := curr.Tags.Subset(itr.dims) id := tags.ID() - // Retrieve the aggregator for this name/tag combination or create one. - rp := m[id] - if rp == nil { - aggregator, emitter := itr.create() - rp = &floatReduceIntegerPoint{ - Name: curr.Name, - Tags: tags, - Aggregator: aggregator, - Emitter: emitter, + // Check to see if we have any GROUP BY dimensions that are not tags or time. + // If we have group-by key entries, create separate iterators for them. + // Otherwise, proceed as normal and create an iterator keyed by the tag ID. + if itr.opt.DimensionGrouper != nil && len(curr.Aux) > 0 { + entries, err := itr.opt.DimensionGrouper.ResolveKeys(curr.Aux, TagSubset{ID: id, HasTags: len(itr.dims) > 0}) + if err != nil { + return nil, err + } + + for _, entry := range entries { + rp := m[entry.DimKey] + if rp == nil { + aggregator, emitter := itr.create() + rp = &floatReduceIntegerPoint{ + Name: curr.Name, + Tags: tags, + GroupingKey: entry.EncodedKey, + Aggregator: aggregator, + Emitter: emitter, + } + m[entry.DimKey] = rp + } + rp.Aggregator.AggregateFloat(curr) + } + } else { + rp := m[id] + if rp == nil { + aggregator, emitter := itr.create() + rp = &floatReduceIntegerPoint{ + Name: curr.Name, + Tags: tags, + Aggregator: aggregator, + Emitter: emitter, + } + m[id] = rp } - m[id] = rp + rp.Aggregator.AggregateFloat(curr) } - rp.Aggregator.AggregateFloat(curr) } keys := make([]string, 0, len(m)) @@ -1436,6 +1538,50 @@ func (itr *floatReduceIntegerIterator) reduce() ([]IntegerPoint, error) { } else { sortedByTime = false } + + if itr.opt.DimensionGrouper != nil && rp.GroupingKey != "" { + dpVal, err := itr.opt.DimensionGrouper.DecodeEntry(rp.GroupingKey) + if err != nil { + return nil, err + } + // The emitted point must carry an Aux slot for every scanner aux + // key (len(itr.opt.Aux)). IteratorScanner.ScanAt ranges over the + // point's Aux, so a slice shorter than the key set leaves the + // trailing keys unvisited and the eval map retains stale values + // from a prior row, wrongly populating non-active date_part + // dimension columns. Selector functions (MIN, MAX, FIRST, LAST) + // already return a full-width Aux via cloneAux; aggregate + // functions (COUNT, SUM, MEAN) return an empty Aux, so grow it to + // full width here. (Keep any longer Aux as-is.) + width := len(itr.opt.Aux) + if width < len(points[i].Aux) { + width = len(points[i].Aux) + } + if width < 1 { + width = 1 + } + if len(points[i].Aux) < width { + aux := make([]interface{}, width) + copy(aux, points[i].Aux) + points[i].Aux = aux + } + // Only the active dimension is meaningful for this series, so null + // every date_part dimension slot (leaving non-active dimension + // columns null) and carry the active value as a DecodedDatePartKey + // in the last slot, which the scanner routes to the correct column + // by name. A stable, full-width slot ensures every key is visited + // and cleared on each scan. + n := len(points[i].Aux) + base := n - len(itr.opt.DatePartDimensions) + if base < 0 { + base = 0 + } + for j := base; j < n; j++ { + points[i].Aux[j] = nil + } + points[i].Aux[n-1] = dpVal + } + a = append(a, points[i]) } } @@ -1614,10 +1760,11 @@ func (itr *floatReduceUnsignedIterator) Next() (*UnsignedPoint, error) { // floatReduceUnsignedPoint stores the reduced data for a name/tag combination. type floatReduceUnsignedPoint struct { - Name string - Tags Tags - Aggregator FloatPointAggregator - Emitter UnsignedPointEmitter + Name string + Tags Tags + GroupingKey string + Aggregator FloatPointAggregator + Emitter UnsignedPointEmitter } // reduce executes fn once for every point in the next window. @@ -1676,19 +1823,44 @@ func (itr *floatReduceUnsignedIterator) reduce() ([]UnsignedPoint, error) { tags := curr.Tags.Subset(itr.dims) id := tags.ID() - // Retrieve the aggregator for this name/tag combination or create one. - rp := m[id] - if rp == nil { - aggregator, emitter := itr.create() - rp = &floatReduceUnsignedPoint{ - Name: curr.Name, - Tags: tags, - Aggregator: aggregator, - Emitter: emitter, + // Check to see if we have any GROUP BY dimensions that are not tags or time. + // If we have group-by key entries, create separate iterators for them. + // Otherwise, proceed as normal and create an iterator keyed by the tag ID. + if itr.opt.DimensionGrouper != nil && len(curr.Aux) > 0 { + entries, err := itr.opt.DimensionGrouper.ResolveKeys(curr.Aux, TagSubset{ID: id, HasTags: len(itr.dims) > 0}) + if err != nil { + return nil, err + } + + for _, entry := range entries { + rp := m[entry.DimKey] + if rp == nil { + aggregator, emitter := itr.create() + rp = &floatReduceUnsignedPoint{ + Name: curr.Name, + Tags: tags, + GroupingKey: entry.EncodedKey, + Aggregator: aggregator, + Emitter: emitter, + } + m[entry.DimKey] = rp + } + rp.Aggregator.AggregateFloat(curr) } - m[id] = rp + } else { + rp := m[id] + if rp == nil { + aggregator, emitter := itr.create() + rp = &floatReduceUnsignedPoint{ + Name: curr.Name, + Tags: tags, + Aggregator: aggregator, + Emitter: emitter, + } + m[id] = rp + } + rp.Aggregator.AggregateFloat(curr) } - rp.Aggregator.AggregateFloat(curr) } keys := make([]string, 0, len(m)) @@ -1724,6 +1896,50 @@ func (itr *floatReduceUnsignedIterator) reduce() ([]UnsignedPoint, error) { } else { sortedByTime = false } + + if itr.opt.DimensionGrouper != nil && rp.GroupingKey != "" { + dpVal, err := itr.opt.DimensionGrouper.DecodeEntry(rp.GroupingKey) + if err != nil { + return nil, err + } + // The emitted point must carry an Aux slot for every scanner aux + // key (len(itr.opt.Aux)). IteratorScanner.ScanAt ranges over the + // point's Aux, so a slice shorter than the key set leaves the + // trailing keys unvisited and the eval map retains stale values + // from a prior row, wrongly populating non-active date_part + // dimension columns. Selector functions (MIN, MAX, FIRST, LAST) + // already return a full-width Aux via cloneAux; aggregate + // functions (COUNT, SUM, MEAN) return an empty Aux, so grow it to + // full width here. (Keep any longer Aux as-is.) + width := len(itr.opt.Aux) + if width < len(points[i].Aux) { + width = len(points[i].Aux) + } + if width < 1 { + width = 1 + } + if len(points[i].Aux) < width { + aux := make([]interface{}, width) + copy(aux, points[i].Aux) + points[i].Aux = aux + } + // Only the active dimension is meaningful for this series, so null + // every date_part dimension slot (leaving non-active dimension + // columns null) and carry the active value as a DecodedDatePartKey + // in the last slot, which the scanner routes to the correct column + // by name. A stable, full-width slot ensures every key is visited + // and cleared on each scan. + n := len(points[i].Aux) + base := n - len(itr.opt.DatePartDimensions) + if base < 0 { + base = 0 + } + for j := base; j < n; j++ { + points[i].Aux[j] = nil + } + points[i].Aux[n-1] = dpVal + } + a = append(a, points[i]) } } @@ -1902,10 +2118,11 @@ func (itr *floatReduceStringIterator) Next() (*StringPoint, error) { // floatReduceStringPoint stores the reduced data for a name/tag combination. type floatReduceStringPoint struct { - Name string - Tags Tags - Aggregator FloatPointAggregator - Emitter StringPointEmitter + Name string + Tags Tags + GroupingKey string + Aggregator FloatPointAggregator + Emitter StringPointEmitter } // reduce executes fn once for every point in the next window. @@ -1964,19 +2181,44 @@ func (itr *floatReduceStringIterator) reduce() ([]StringPoint, error) { tags := curr.Tags.Subset(itr.dims) id := tags.ID() - // Retrieve the aggregator for this name/tag combination or create one. - rp := m[id] - if rp == nil { - aggregator, emitter := itr.create() - rp = &floatReduceStringPoint{ - Name: curr.Name, - Tags: tags, - Aggregator: aggregator, - Emitter: emitter, + // Check to see if we have any GROUP BY dimensions that are not tags or time. + // If we have group-by key entries, create separate iterators for them. + // Otherwise, proceed as normal and create an iterator keyed by the tag ID. + if itr.opt.DimensionGrouper != nil && len(curr.Aux) > 0 { + entries, err := itr.opt.DimensionGrouper.ResolveKeys(curr.Aux, TagSubset{ID: id, HasTags: len(itr.dims) > 0}) + if err != nil { + return nil, err + } + + for _, entry := range entries { + rp := m[entry.DimKey] + if rp == nil { + aggregator, emitter := itr.create() + rp = &floatReduceStringPoint{ + Name: curr.Name, + Tags: tags, + GroupingKey: entry.EncodedKey, + Aggregator: aggregator, + Emitter: emitter, + } + m[entry.DimKey] = rp + } + rp.Aggregator.AggregateFloat(curr) + } + } else { + rp := m[id] + if rp == nil { + aggregator, emitter := itr.create() + rp = &floatReduceStringPoint{ + Name: curr.Name, + Tags: tags, + Aggregator: aggregator, + Emitter: emitter, + } + m[id] = rp } - m[id] = rp + rp.Aggregator.AggregateFloat(curr) } - rp.Aggregator.AggregateFloat(curr) } keys := make([]string, 0, len(m)) @@ -2012,6 +2254,50 @@ func (itr *floatReduceStringIterator) reduce() ([]StringPoint, error) { } else { sortedByTime = false } + + if itr.opt.DimensionGrouper != nil && rp.GroupingKey != "" { + dpVal, err := itr.opt.DimensionGrouper.DecodeEntry(rp.GroupingKey) + if err != nil { + return nil, err + } + // The emitted point must carry an Aux slot for every scanner aux + // key (len(itr.opt.Aux)). IteratorScanner.ScanAt ranges over the + // point's Aux, so a slice shorter than the key set leaves the + // trailing keys unvisited and the eval map retains stale values + // from a prior row, wrongly populating non-active date_part + // dimension columns. Selector functions (MIN, MAX, FIRST, LAST) + // already return a full-width Aux via cloneAux; aggregate + // functions (COUNT, SUM, MEAN) return an empty Aux, so grow it to + // full width here. (Keep any longer Aux as-is.) + width := len(itr.opt.Aux) + if width < len(points[i].Aux) { + width = len(points[i].Aux) + } + if width < 1 { + width = 1 + } + if len(points[i].Aux) < width { + aux := make([]interface{}, width) + copy(aux, points[i].Aux) + points[i].Aux = aux + } + // Only the active dimension is meaningful for this series, so null + // every date_part dimension slot (leaving non-active dimension + // columns null) and carry the active value as a DecodedDatePartKey + // in the last slot, which the scanner routes to the correct column + // by name. A stable, full-width slot ensures every key is visited + // and cleared on each scan. + n := len(points[i].Aux) + base := n - len(itr.opt.DatePartDimensions) + if base < 0 { + base = 0 + } + for j := base; j < n; j++ { + points[i].Aux[j] = nil + } + points[i].Aux[n-1] = dpVal + } + a = append(a, points[i]) } } @@ -2190,10 +2476,11 @@ func (itr *floatReduceBooleanIterator) Next() (*BooleanPoint, error) { // floatReduceBooleanPoint stores the reduced data for a name/tag combination. type floatReduceBooleanPoint struct { - Name string - Tags Tags - Aggregator FloatPointAggregator - Emitter BooleanPointEmitter + Name string + Tags Tags + GroupingKey string + Aggregator FloatPointAggregator + Emitter BooleanPointEmitter } // reduce executes fn once for every point in the next window. @@ -2252,19 +2539,44 @@ func (itr *floatReduceBooleanIterator) reduce() ([]BooleanPoint, error) { tags := curr.Tags.Subset(itr.dims) id := tags.ID() - // Retrieve the aggregator for this name/tag combination or create one. - rp := m[id] - if rp == nil { - aggregator, emitter := itr.create() - rp = &floatReduceBooleanPoint{ - Name: curr.Name, - Tags: tags, - Aggregator: aggregator, - Emitter: emitter, + // Check to see if we have any GROUP BY dimensions that are not tags or time. + // If we have group-by key entries, create separate iterators for them. + // Otherwise, proceed as normal and create an iterator keyed by the tag ID. + if itr.opt.DimensionGrouper != nil && len(curr.Aux) > 0 { + entries, err := itr.opt.DimensionGrouper.ResolveKeys(curr.Aux, TagSubset{ID: id, HasTags: len(itr.dims) > 0}) + if err != nil { + return nil, err + } + + for _, entry := range entries { + rp := m[entry.DimKey] + if rp == nil { + aggregator, emitter := itr.create() + rp = &floatReduceBooleanPoint{ + Name: curr.Name, + Tags: tags, + GroupingKey: entry.EncodedKey, + Aggregator: aggregator, + Emitter: emitter, + } + m[entry.DimKey] = rp + } + rp.Aggregator.AggregateFloat(curr) + } + } else { + rp := m[id] + if rp == nil { + aggregator, emitter := itr.create() + rp = &floatReduceBooleanPoint{ + Name: curr.Name, + Tags: tags, + Aggregator: aggregator, + Emitter: emitter, + } + m[id] = rp } - m[id] = rp + rp.Aggregator.AggregateFloat(curr) } - rp.Aggregator.AggregateFloat(curr) } keys := make([]string, 0, len(m)) @@ -2300,6 +2612,50 @@ func (itr *floatReduceBooleanIterator) reduce() ([]BooleanPoint, error) { } else { sortedByTime = false } + + if itr.opt.DimensionGrouper != nil && rp.GroupingKey != "" { + dpVal, err := itr.opt.DimensionGrouper.DecodeEntry(rp.GroupingKey) + if err != nil { + return nil, err + } + // The emitted point must carry an Aux slot for every scanner aux + // key (len(itr.opt.Aux)). IteratorScanner.ScanAt ranges over the + // point's Aux, so a slice shorter than the key set leaves the + // trailing keys unvisited and the eval map retains stale values + // from a prior row, wrongly populating non-active date_part + // dimension columns. Selector functions (MIN, MAX, FIRST, LAST) + // already return a full-width Aux via cloneAux; aggregate + // functions (COUNT, SUM, MEAN) return an empty Aux, so grow it to + // full width here. (Keep any longer Aux as-is.) + width := len(itr.opt.Aux) + if width < len(points[i].Aux) { + width = len(points[i].Aux) + } + if width < 1 { + width = 1 + } + if len(points[i].Aux) < width { + aux := make([]interface{}, width) + copy(aux, points[i].Aux) + points[i].Aux = aux + } + // Only the active dimension is meaningful for this series, so null + // every date_part dimension slot (leaving non-active dimension + // columns null) and carry the active value as a DecodedDatePartKey + // in the last slot, which the scanner routes to the correct column + // by name. A stable, full-width slot ensures every key is visited + // and cleared on each scan. + n := len(points[i].Aux) + base := n - len(itr.opt.DatePartDimensions) + if base < 0 { + base = 0 + } + for j := base; j < n; j++ { + points[i].Aux[j] = nil + } + points[i].Aux[n-1] = dpVal + } + a = append(a, points[i]) } } @@ -2515,7 +2871,7 @@ func newFloatFilterIterator(input FloatIterator, cond influxql.Expr, opt Iterato n := influxql.RewriteFunc(influxql.CloneExpr(cond), func(n influxql.Node) influxql.Node { switch n := n.(type) { case *influxql.BinaryExpr: - if n.LHS.String() == "time" { + if n.LHS.String() == models.TimeString { return &influxql.BooleanLiteral{Val: true} } } @@ -3220,6 +3576,11 @@ func (s *integerIteratorScanner) ScanAt(ts int64, name string, tags Tags, m map[ switch v.(type) { case float64, int64, uint64, string, bool: m[k.Val] = v + case DecodedDatePartKey: + m[DatePartDimensionsString] = v + // Clear any stale raw value under this dimension's own key so a prior + // row's value can't persist (the active value is carried via the key above). + delete(m, k.Val) default: // Insert the fill value if one was specified. if s.defaultValue != SkipDefault { @@ -3702,10 +4063,11 @@ func (itr *integerReduceFloatIterator) Next() (*FloatPoint, error) { // integerReduceFloatPoint stores the reduced data for a name/tag combination. type integerReduceFloatPoint struct { - Name string - Tags Tags - Aggregator IntegerPointAggregator - Emitter FloatPointEmitter + Name string + Tags Tags + GroupingKey string + Aggregator IntegerPointAggregator + Emitter FloatPointEmitter } // reduce executes fn once for every point in the next window. @@ -3764,19 +4126,44 @@ func (itr *integerReduceFloatIterator) reduce() ([]FloatPoint, error) { tags := curr.Tags.Subset(itr.dims) id := tags.ID() - // Retrieve the aggregator for this name/tag combination or create one. - rp := m[id] - if rp == nil { - aggregator, emitter := itr.create() - rp = &integerReduceFloatPoint{ - Name: curr.Name, - Tags: tags, - Aggregator: aggregator, - Emitter: emitter, + // Check to see if we have any GROUP BY dimensions that are not tags or time. + // If we have group-by key entries, create separate iterators for them. + // Otherwise, proceed as normal and create an iterator keyed by the tag ID. + if itr.opt.DimensionGrouper != nil && len(curr.Aux) > 0 { + entries, err := itr.opt.DimensionGrouper.ResolveKeys(curr.Aux, TagSubset{ID: id, HasTags: len(itr.dims) > 0}) + if err != nil { + return nil, err + } + + for _, entry := range entries { + rp := m[entry.DimKey] + if rp == nil { + aggregator, emitter := itr.create() + rp = &integerReduceFloatPoint{ + Name: curr.Name, + Tags: tags, + GroupingKey: entry.EncodedKey, + Aggregator: aggregator, + Emitter: emitter, + } + m[entry.DimKey] = rp + } + rp.Aggregator.AggregateInteger(curr) + } + } else { + rp := m[id] + if rp == nil { + aggregator, emitter := itr.create() + rp = &integerReduceFloatPoint{ + Name: curr.Name, + Tags: tags, + Aggregator: aggregator, + Emitter: emitter, + } + m[id] = rp } - m[id] = rp + rp.Aggregator.AggregateInteger(curr) } - rp.Aggregator.AggregateInteger(curr) } keys := make([]string, 0, len(m)) @@ -3812,6 +4199,50 @@ func (itr *integerReduceFloatIterator) reduce() ([]FloatPoint, error) { } else { sortedByTime = false } + + if itr.opt.DimensionGrouper != nil && rp.GroupingKey != "" { + dpVal, err := itr.opt.DimensionGrouper.DecodeEntry(rp.GroupingKey) + if err != nil { + return nil, err + } + // The emitted point must carry an Aux slot for every scanner aux + // key (len(itr.opt.Aux)). IteratorScanner.ScanAt ranges over the + // point's Aux, so a slice shorter than the key set leaves the + // trailing keys unvisited and the eval map retains stale values + // from a prior row, wrongly populating non-active date_part + // dimension columns. Selector functions (MIN, MAX, FIRST, LAST) + // already return a full-width Aux via cloneAux; aggregate + // functions (COUNT, SUM, MEAN) return an empty Aux, so grow it to + // full width here. (Keep any longer Aux as-is.) + width := len(itr.opt.Aux) + if width < len(points[i].Aux) { + width = len(points[i].Aux) + } + if width < 1 { + width = 1 + } + if len(points[i].Aux) < width { + aux := make([]interface{}, width) + copy(aux, points[i].Aux) + points[i].Aux = aux + } + // Only the active dimension is meaningful for this series, so null + // every date_part dimension slot (leaving non-active dimension + // columns null) and carry the active value as a DecodedDatePartKey + // in the last slot, which the scanner routes to the correct column + // by name. A stable, full-width slot ensures every key is visited + // and cleared on each scan. + n := len(points[i].Aux) + base := n - len(itr.opt.DatePartDimensions) + if base < 0 { + base = 0 + } + for j := base; j < n; j++ { + points[i].Aux[j] = nil + } + points[i].Aux[n-1] = dpVal + } + a = append(a, points[i]) } } @@ -3990,10 +4421,11 @@ func (itr *integerReduceIntegerIterator) Next() (*IntegerPoint, error) { // integerReduceIntegerPoint stores the reduced data for a name/tag combination. type integerReduceIntegerPoint struct { - Name string - Tags Tags - Aggregator IntegerPointAggregator - Emitter IntegerPointEmitter + Name string + Tags Tags + GroupingKey string + Aggregator IntegerPointAggregator + Emitter IntegerPointEmitter } // reduce executes fn once for every point in the next window. @@ -4052,19 +4484,44 @@ func (itr *integerReduceIntegerIterator) reduce() ([]IntegerPoint, error) { tags := curr.Tags.Subset(itr.dims) id := tags.ID() - // Retrieve the aggregator for this name/tag combination or create one. - rp := m[id] - if rp == nil { - aggregator, emitter := itr.create() - rp = &integerReduceIntegerPoint{ - Name: curr.Name, - Tags: tags, - Aggregator: aggregator, - Emitter: emitter, + // Check to see if we have any GROUP BY dimensions that are not tags or time. + // If we have group-by key entries, create separate iterators for them. + // Otherwise, proceed as normal and create an iterator keyed by the tag ID. + if itr.opt.DimensionGrouper != nil && len(curr.Aux) > 0 { + entries, err := itr.opt.DimensionGrouper.ResolveKeys(curr.Aux, TagSubset{ID: id, HasTags: len(itr.dims) > 0}) + if err != nil { + return nil, err } - m[id] = rp + + for _, entry := range entries { + rp := m[entry.DimKey] + if rp == nil { + aggregator, emitter := itr.create() + rp = &integerReduceIntegerPoint{ + Name: curr.Name, + Tags: tags, + GroupingKey: entry.EncodedKey, + Aggregator: aggregator, + Emitter: emitter, + } + m[entry.DimKey] = rp + } + rp.Aggregator.AggregateInteger(curr) + } + } else { + rp := m[id] + if rp == nil { + aggregator, emitter := itr.create() + rp = &integerReduceIntegerPoint{ + Name: curr.Name, + Tags: tags, + Aggregator: aggregator, + Emitter: emitter, + } + m[id] = rp + } + rp.Aggregator.AggregateInteger(curr) } - rp.Aggregator.AggregateInteger(curr) } keys := make([]string, 0, len(m)) @@ -4100,6 +4557,50 @@ func (itr *integerReduceIntegerIterator) reduce() ([]IntegerPoint, error) { } else { sortedByTime = false } + + if itr.opt.DimensionGrouper != nil && rp.GroupingKey != "" { + dpVal, err := itr.opt.DimensionGrouper.DecodeEntry(rp.GroupingKey) + if err != nil { + return nil, err + } + // The emitted point must carry an Aux slot for every scanner aux + // key (len(itr.opt.Aux)). IteratorScanner.ScanAt ranges over the + // point's Aux, so a slice shorter than the key set leaves the + // trailing keys unvisited and the eval map retains stale values + // from a prior row, wrongly populating non-active date_part + // dimension columns. Selector functions (MIN, MAX, FIRST, LAST) + // already return a full-width Aux via cloneAux; aggregate + // functions (COUNT, SUM, MEAN) return an empty Aux, so grow it to + // full width here. (Keep any longer Aux as-is.) + width := len(itr.opt.Aux) + if width < len(points[i].Aux) { + width = len(points[i].Aux) + } + if width < 1 { + width = 1 + } + if len(points[i].Aux) < width { + aux := make([]interface{}, width) + copy(aux, points[i].Aux) + points[i].Aux = aux + } + // Only the active dimension is meaningful for this series, so null + // every date_part dimension slot (leaving non-active dimension + // columns null) and carry the active value as a DecodedDatePartKey + // in the last slot, which the scanner routes to the correct column + // by name. A stable, full-width slot ensures every key is visited + // and cleared on each scan. + n := len(points[i].Aux) + base := n - len(itr.opt.DatePartDimensions) + if base < 0 { + base = 0 + } + for j := base; j < n; j++ { + points[i].Aux[j] = nil + } + points[i].Aux[n-1] = dpVal + } + a = append(a, points[i]) } } @@ -4278,10 +4779,11 @@ func (itr *integerReduceUnsignedIterator) Next() (*UnsignedPoint, error) { // integerReduceUnsignedPoint stores the reduced data for a name/tag combination. type integerReduceUnsignedPoint struct { - Name string - Tags Tags - Aggregator IntegerPointAggregator - Emitter UnsignedPointEmitter + Name string + Tags Tags + GroupingKey string + Aggregator IntegerPointAggregator + Emitter UnsignedPointEmitter } // reduce executes fn once for every point in the next window. @@ -4340,19 +4842,44 @@ func (itr *integerReduceUnsignedIterator) reduce() ([]UnsignedPoint, error) { tags := curr.Tags.Subset(itr.dims) id := tags.ID() - // Retrieve the aggregator for this name/tag combination or create one. - rp := m[id] - if rp == nil { - aggregator, emitter := itr.create() - rp = &integerReduceUnsignedPoint{ - Name: curr.Name, - Tags: tags, - Aggregator: aggregator, - Emitter: emitter, + // Check to see if we have any GROUP BY dimensions that are not tags or time. + // If we have group-by key entries, create separate iterators for them. + // Otherwise, proceed as normal and create an iterator keyed by the tag ID. + if itr.opt.DimensionGrouper != nil && len(curr.Aux) > 0 { + entries, err := itr.opt.DimensionGrouper.ResolveKeys(curr.Aux, TagSubset{ID: id, HasTags: len(itr.dims) > 0}) + if err != nil { + return nil, err + } + + for _, entry := range entries { + rp := m[entry.DimKey] + if rp == nil { + aggregator, emitter := itr.create() + rp = &integerReduceUnsignedPoint{ + Name: curr.Name, + Tags: tags, + GroupingKey: entry.EncodedKey, + Aggregator: aggregator, + Emitter: emitter, + } + m[entry.DimKey] = rp + } + rp.Aggregator.AggregateInteger(curr) + } + } else { + rp := m[id] + if rp == nil { + aggregator, emitter := itr.create() + rp = &integerReduceUnsignedPoint{ + Name: curr.Name, + Tags: tags, + Aggregator: aggregator, + Emitter: emitter, + } + m[id] = rp } - m[id] = rp + rp.Aggregator.AggregateInteger(curr) } - rp.Aggregator.AggregateInteger(curr) } keys := make([]string, 0, len(m)) @@ -4388,6 +4915,50 @@ func (itr *integerReduceUnsignedIterator) reduce() ([]UnsignedPoint, error) { } else { sortedByTime = false } + + if itr.opt.DimensionGrouper != nil && rp.GroupingKey != "" { + dpVal, err := itr.opt.DimensionGrouper.DecodeEntry(rp.GroupingKey) + if err != nil { + return nil, err + } + // The emitted point must carry an Aux slot for every scanner aux + // key (len(itr.opt.Aux)). IteratorScanner.ScanAt ranges over the + // point's Aux, so a slice shorter than the key set leaves the + // trailing keys unvisited and the eval map retains stale values + // from a prior row, wrongly populating non-active date_part + // dimension columns. Selector functions (MIN, MAX, FIRST, LAST) + // already return a full-width Aux via cloneAux; aggregate + // functions (COUNT, SUM, MEAN) return an empty Aux, so grow it to + // full width here. (Keep any longer Aux as-is.) + width := len(itr.opt.Aux) + if width < len(points[i].Aux) { + width = len(points[i].Aux) + } + if width < 1 { + width = 1 + } + if len(points[i].Aux) < width { + aux := make([]interface{}, width) + copy(aux, points[i].Aux) + points[i].Aux = aux + } + // Only the active dimension is meaningful for this series, so null + // every date_part dimension slot (leaving non-active dimension + // columns null) and carry the active value as a DecodedDatePartKey + // in the last slot, which the scanner routes to the correct column + // by name. A stable, full-width slot ensures every key is visited + // and cleared on each scan. + n := len(points[i].Aux) + base := n - len(itr.opt.DatePartDimensions) + if base < 0 { + base = 0 + } + for j := base; j < n; j++ { + points[i].Aux[j] = nil + } + points[i].Aux[n-1] = dpVal + } + a = append(a, points[i]) } } @@ -4566,10 +5137,11 @@ func (itr *integerReduceStringIterator) Next() (*StringPoint, error) { // integerReduceStringPoint stores the reduced data for a name/tag combination. type integerReduceStringPoint struct { - Name string - Tags Tags - Aggregator IntegerPointAggregator - Emitter StringPointEmitter + Name string + Tags Tags + GroupingKey string + Aggregator IntegerPointAggregator + Emitter StringPointEmitter } // reduce executes fn once for every point in the next window. @@ -4628,19 +5200,44 @@ func (itr *integerReduceStringIterator) reduce() ([]StringPoint, error) { tags := curr.Tags.Subset(itr.dims) id := tags.ID() - // Retrieve the aggregator for this name/tag combination or create one. - rp := m[id] - if rp == nil { - aggregator, emitter := itr.create() - rp = &integerReduceStringPoint{ - Name: curr.Name, - Tags: tags, - Aggregator: aggregator, - Emitter: emitter, + // Check to see if we have any GROUP BY dimensions that are not tags or time. + // If we have group-by key entries, create separate iterators for them. + // Otherwise, proceed as normal and create an iterator keyed by the tag ID. + if itr.opt.DimensionGrouper != nil && len(curr.Aux) > 0 { + entries, err := itr.opt.DimensionGrouper.ResolveKeys(curr.Aux, TagSubset{ID: id, HasTags: len(itr.dims) > 0}) + if err != nil { + return nil, err + } + + for _, entry := range entries { + rp := m[entry.DimKey] + if rp == nil { + aggregator, emitter := itr.create() + rp = &integerReduceStringPoint{ + Name: curr.Name, + Tags: tags, + GroupingKey: entry.EncodedKey, + Aggregator: aggregator, + Emitter: emitter, + } + m[entry.DimKey] = rp + } + rp.Aggregator.AggregateInteger(curr) + } + } else { + rp := m[id] + if rp == nil { + aggregator, emitter := itr.create() + rp = &integerReduceStringPoint{ + Name: curr.Name, + Tags: tags, + Aggregator: aggregator, + Emitter: emitter, + } + m[id] = rp } - m[id] = rp + rp.Aggregator.AggregateInteger(curr) } - rp.Aggregator.AggregateInteger(curr) } keys := make([]string, 0, len(m)) @@ -4676,12 +5273,56 @@ func (itr *integerReduceStringIterator) reduce() ([]StringPoint, error) { } else { sortedByTime = false } - a = append(a, points[i]) - } - } - // Points may be out of order. Perform a stable sort by time if requested. - if !sortedByTime && itr.opt.Ordered { - var sorted sort.Interface = stringPointsByTime(a) + + if itr.opt.DimensionGrouper != nil && rp.GroupingKey != "" { + dpVal, err := itr.opt.DimensionGrouper.DecodeEntry(rp.GroupingKey) + if err != nil { + return nil, err + } + // The emitted point must carry an Aux slot for every scanner aux + // key (len(itr.opt.Aux)). IteratorScanner.ScanAt ranges over the + // point's Aux, so a slice shorter than the key set leaves the + // trailing keys unvisited and the eval map retains stale values + // from a prior row, wrongly populating non-active date_part + // dimension columns. Selector functions (MIN, MAX, FIRST, LAST) + // already return a full-width Aux via cloneAux; aggregate + // functions (COUNT, SUM, MEAN) return an empty Aux, so grow it to + // full width here. (Keep any longer Aux as-is.) + width := len(itr.opt.Aux) + if width < len(points[i].Aux) { + width = len(points[i].Aux) + } + if width < 1 { + width = 1 + } + if len(points[i].Aux) < width { + aux := make([]interface{}, width) + copy(aux, points[i].Aux) + points[i].Aux = aux + } + // Only the active dimension is meaningful for this series, so null + // every date_part dimension slot (leaving non-active dimension + // columns null) and carry the active value as a DecodedDatePartKey + // in the last slot, which the scanner routes to the correct column + // by name. A stable, full-width slot ensures every key is visited + // and cleared on each scan. + n := len(points[i].Aux) + base := n - len(itr.opt.DatePartDimensions) + if base < 0 { + base = 0 + } + for j := base; j < n; j++ { + points[i].Aux[j] = nil + } + points[i].Aux[n-1] = dpVal + } + + a = append(a, points[i]) + } + } + // Points may be out of order. Perform a stable sort by time if requested. + if !sortedByTime && itr.opt.Ordered { + var sorted sort.Interface = stringPointsByTime(a) if itr.opt.Ascending { sorted = sort.Reverse(sorted) } @@ -4854,10 +5495,11 @@ func (itr *integerReduceBooleanIterator) Next() (*BooleanPoint, error) { // integerReduceBooleanPoint stores the reduced data for a name/tag combination. type integerReduceBooleanPoint struct { - Name string - Tags Tags - Aggregator IntegerPointAggregator - Emitter BooleanPointEmitter + Name string + Tags Tags + GroupingKey string + Aggregator IntegerPointAggregator + Emitter BooleanPointEmitter } // reduce executes fn once for every point in the next window. @@ -4916,19 +5558,44 @@ func (itr *integerReduceBooleanIterator) reduce() ([]BooleanPoint, error) { tags := curr.Tags.Subset(itr.dims) id := tags.ID() - // Retrieve the aggregator for this name/tag combination or create one. - rp := m[id] - if rp == nil { - aggregator, emitter := itr.create() - rp = &integerReduceBooleanPoint{ - Name: curr.Name, - Tags: tags, - Aggregator: aggregator, - Emitter: emitter, + // Check to see if we have any GROUP BY dimensions that are not tags or time. + // If we have group-by key entries, create separate iterators for them. + // Otherwise, proceed as normal and create an iterator keyed by the tag ID. + if itr.opt.DimensionGrouper != nil && len(curr.Aux) > 0 { + entries, err := itr.opt.DimensionGrouper.ResolveKeys(curr.Aux, TagSubset{ID: id, HasTags: len(itr.dims) > 0}) + if err != nil { + return nil, err + } + + for _, entry := range entries { + rp := m[entry.DimKey] + if rp == nil { + aggregator, emitter := itr.create() + rp = &integerReduceBooleanPoint{ + Name: curr.Name, + Tags: tags, + GroupingKey: entry.EncodedKey, + Aggregator: aggregator, + Emitter: emitter, + } + m[entry.DimKey] = rp + } + rp.Aggregator.AggregateInteger(curr) } - m[id] = rp + } else { + rp := m[id] + if rp == nil { + aggregator, emitter := itr.create() + rp = &integerReduceBooleanPoint{ + Name: curr.Name, + Tags: tags, + Aggregator: aggregator, + Emitter: emitter, + } + m[id] = rp + } + rp.Aggregator.AggregateInteger(curr) } - rp.Aggregator.AggregateInteger(curr) } keys := make([]string, 0, len(m)) @@ -4964,6 +5631,50 @@ func (itr *integerReduceBooleanIterator) reduce() ([]BooleanPoint, error) { } else { sortedByTime = false } + + if itr.opt.DimensionGrouper != nil && rp.GroupingKey != "" { + dpVal, err := itr.opt.DimensionGrouper.DecodeEntry(rp.GroupingKey) + if err != nil { + return nil, err + } + // The emitted point must carry an Aux slot for every scanner aux + // key (len(itr.opt.Aux)). IteratorScanner.ScanAt ranges over the + // point's Aux, so a slice shorter than the key set leaves the + // trailing keys unvisited and the eval map retains stale values + // from a prior row, wrongly populating non-active date_part + // dimension columns. Selector functions (MIN, MAX, FIRST, LAST) + // already return a full-width Aux via cloneAux; aggregate + // functions (COUNT, SUM, MEAN) return an empty Aux, so grow it to + // full width here. (Keep any longer Aux as-is.) + width := len(itr.opt.Aux) + if width < len(points[i].Aux) { + width = len(points[i].Aux) + } + if width < 1 { + width = 1 + } + if len(points[i].Aux) < width { + aux := make([]interface{}, width) + copy(aux, points[i].Aux) + points[i].Aux = aux + } + // Only the active dimension is meaningful for this series, so null + // every date_part dimension slot (leaving non-active dimension + // columns null) and carry the active value as a DecodedDatePartKey + // in the last slot, which the scanner routes to the correct column + // by name. A stable, full-width slot ensures every key is visited + // and cleared on each scan. + n := len(points[i].Aux) + base := n - len(itr.opt.DatePartDimensions) + if base < 0 { + base = 0 + } + for j := base; j < n; j++ { + points[i].Aux[j] = nil + } + points[i].Aux[n-1] = dpVal + } + a = append(a, points[i]) } } @@ -5179,7 +5890,7 @@ func newIntegerFilterIterator(input IntegerIterator, cond influxql.Expr, opt Ite n := influxql.RewriteFunc(influxql.CloneExpr(cond), func(n influxql.Node) influxql.Node { switch n := n.(type) { case *influxql.BinaryExpr: - if n.LHS.String() == "time" { + if n.LHS.String() == models.TimeString { return &influxql.BooleanLiteral{Val: true} } } @@ -5884,6 +6595,11 @@ func (s *unsignedIteratorScanner) ScanAt(ts int64, name string, tags Tags, m map switch v.(type) { case float64, int64, uint64, string, bool: m[k.Val] = v + case DecodedDatePartKey: + m[DatePartDimensionsString] = v + // Clear any stale raw value under this dimension's own key so a prior + // row's value can't persist (the active value is carried via the key above). + delete(m, k.Val) default: // Insert the fill value if one was specified. if s.defaultValue != SkipDefault { @@ -6366,10 +7082,11 @@ func (itr *unsignedReduceFloatIterator) Next() (*FloatPoint, error) { // unsignedReduceFloatPoint stores the reduced data for a name/tag combination. type unsignedReduceFloatPoint struct { - Name string - Tags Tags - Aggregator UnsignedPointAggregator - Emitter FloatPointEmitter + Name string + Tags Tags + GroupingKey string + Aggregator UnsignedPointAggregator + Emitter FloatPointEmitter } // reduce executes fn once for every point in the next window. @@ -6428,19 +7145,44 @@ func (itr *unsignedReduceFloatIterator) reduce() ([]FloatPoint, error) { tags := curr.Tags.Subset(itr.dims) id := tags.ID() - // Retrieve the aggregator for this name/tag combination or create one. - rp := m[id] - if rp == nil { - aggregator, emitter := itr.create() - rp = &unsignedReduceFloatPoint{ - Name: curr.Name, - Tags: tags, - Aggregator: aggregator, - Emitter: emitter, + // Check to see if we have any GROUP BY dimensions that are not tags or time. + // If we have group-by key entries, create separate iterators for them. + // Otherwise, proceed as normal and create an iterator keyed by the tag ID. + if itr.opt.DimensionGrouper != nil && len(curr.Aux) > 0 { + entries, err := itr.opt.DimensionGrouper.ResolveKeys(curr.Aux, TagSubset{ID: id, HasTags: len(itr.dims) > 0}) + if err != nil { + return nil, err } - m[id] = rp + + for _, entry := range entries { + rp := m[entry.DimKey] + if rp == nil { + aggregator, emitter := itr.create() + rp = &unsignedReduceFloatPoint{ + Name: curr.Name, + Tags: tags, + GroupingKey: entry.EncodedKey, + Aggregator: aggregator, + Emitter: emitter, + } + m[entry.DimKey] = rp + } + rp.Aggregator.AggregateUnsigned(curr) + } + } else { + rp := m[id] + if rp == nil { + aggregator, emitter := itr.create() + rp = &unsignedReduceFloatPoint{ + Name: curr.Name, + Tags: tags, + Aggregator: aggregator, + Emitter: emitter, + } + m[id] = rp + } + rp.Aggregator.AggregateUnsigned(curr) } - rp.Aggregator.AggregateUnsigned(curr) } keys := make([]string, 0, len(m)) @@ -6476,6 +7218,50 @@ func (itr *unsignedReduceFloatIterator) reduce() ([]FloatPoint, error) { } else { sortedByTime = false } + + if itr.opt.DimensionGrouper != nil && rp.GroupingKey != "" { + dpVal, err := itr.opt.DimensionGrouper.DecodeEntry(rp.GroupingKey) + if err != nil { + return nil, err + } + // The emitted point must carry an Aux slot for every scanner aux + // key (len(itr.opt.Aux)). IteratorScanner.ScanAt ranges over the + // point's Aux, so a slice shorter than the key set leaves the + // trailing keys unvisited and the eval map retains stale values + // from a prior row, wrongly populating non-active date_part + // dimension columns. Selector functions (MIN, MAX, FIRST, LAST) + // already return a full-width Aux via cloneAux; aggregate + // functions (COUNT, SUM, MEAN) return an empty Aux, so grow it to + // full width here. (Keep any longer Aux as-is.) + width := len(itr.opt.Aux) + if width < len(points[i].Aux) { + width = len(points[i].Aux) + } + if width < 1 { + width = 1 + } + if len(points[i].Aux) < width { + aux := make([]interface{}, width) + copy(aux, points[i].Aux) + points[i].Aux = aux + } + // Only the active dimension is meaningful for this series, so null + // every date_part dimension slot (leaving non-active dimension + // columns null) and carry the active value as a DecodedDatePartKey + // in the last slot, which the scanner routes to the correct column + // by name. A stable, full-width slot ensures every key is visited + // and cleared on each scan. + n := len(points[i].Aux) + base := n - len(itr.opt.DatePartDimensions) + if base < 0 { + base = 0 + } + for j := base; j < n; j++ { + points[i].Aux[j] = nil + } + points[i].Aux[n-1] = dpVal + } + a = append(a, points[i]) } } @@ -6654,10 +7440,11 @@ func (itr *unsignedReduceIntegerIterator) Next() (*IntegerPoint, error) { // unsignedReduceIntegerPoint stores the reduced data for a name/tag combination. type unsignedReduceIntegerPoint struct { - Name string - Tags Tags - Aggregator UnsignedPointAggregator - Emitter IntegerPointEmitter + Name string + Tags Tags + GroupingKey string + Aggregator UnsignedPointAggregator + Emitter IntegerPointEmitter } // reduce executes fn once for every point in the next window. @@ -6716,19 +7503,44 @@ func (itr *unsignedReduceIntegerIterator) reduce() ([]IntegerPoint, error) { tags := curr.Tags.Subset(itr.dims) id := tags.ID() - // Retrieve the aggregator for this name/tag combination or create one. - rp := m[id] - if rp == nil { - aggregator, emitter := itr.create() - rp = &unsignedReduceIntegerPoint{ - Name: curr.Name, - Tags: tags, - Aggregator: aggregator, - Emitter: emitter, + // Check to see if we have any GROUP BY dimensions that are not tags or time. + // If we have group-by key entries, create separate iterators for them. + // Otherwise, proceed as normal and create an iterator keyed by the tag ID. + if itr.opt.DimensionGrouper != nil && len(curr.Aux) > 0 { + entries, err := itr.opt.DimensionGrouper.ResolveKeys(curr.Aux, TagSubset{ID: id, HasTags: len(itr.dims) > 0}) + if err != nil { + return nil, err + } + + for _, entry := range entries { + rp := m[entry.DimKey] + if rp == nil { + aggregator, emitter := itr.create() + rp = &unsignedReduceIntegerPoint{ + Name: curr.Name, + Tags: tags, + GroupingKey: entry.EncodedKey, + Aggregator: aggregator, + Emitter: emitter, + } + m[entry.DimKey] = rp + } + rp.Aggregator.AggregateUnsigned(curr) + } + } else { + rp := m[id] + if rp == nil { + aggregator, emitter := itr.create() + rp = &unsignedReduceIntegerPoint{ + Name: curr.Name, + Tags: tags, + Aggregator: aggregator, + Emitter: emitter, + } + m[id] = rp } - m[id] = rp + rp.Aggregator.AggregateUnsigned(curr) } - rp.Aggregator.AggregateUnsigned(curr) } keys := make([]string, 0, len(m)) @@ -6764,6 +7576,50 @@ func (itr *unsignedReduceIntegerIterator) reduce() ([]IntegerPoint, error) { } else { sortedByTime = false } + + if itr.opt.DimensionGrouper != nil && rp.GroupingKey != "" { + dpVal, err := itr.opt.DimensionGrouper.DecodeEntry(rp.GroupingKey) + if err != nil { + return nil, err + } + // The emitted point must carry an Aux slot for every scanner aux + // key (len(itr.opt.Aux)). IteratorScanner.ScanAt ranges over the + // point's Aux, so a slice shorter than the key set leaves the + // trailing keys unvisited and the eval map retains stale values + // from a prior row, wrongly populating non-active date_part + // dimension columns. Selector functions (MIN, MAX, FIRST, LAST) + // already return a full-width Aux via cloneAux; aggregate + // functions (COUNT, SUM, MEAN) return an empty Aux, so grow it to + // full width here. (Keep any longer Aux as-is.) + width := len(itr.opt.Aux) + if width < len(points[i].Aux) { + width = len(points[i].Aux) + } + if width < 1 { + width = 1 + } + if len(points[i].Aux) < width { + aux := make([]interface{}, width) + copy(aux, points[i].Aux) + points[i].Aux = aux + } + // Only the active dimension is meaningful for this series, so null + // every date_part dimension slot (leaving non-active dimension + // columns null) and carry the active value as a DecodedDatePartKey + // in the last slot, which the scanner routes to the correct column + // by name. A stable, full-width slot ensures every key is visited + // and cleared on each scan. + n := len(points[i].Aux) + base := n - len(itr.opt.DatePartDimensions) + if base < 0 { + base = 0 + } + for j := base; j < n; j++ { + points[i].Aux[j] = nil + } + points[i].Aux[n-1] = dpVal + } + a = append(a, points[i]) } } @@ -6942,10 +7798,11 @@ func (itr *unsignedReduceUnsignedIterator) Next() (*UnsignedPoint, error) { // unsignedReduceUnsignedPoint stores the reduced data for a name/tag combination. type unsignedReduceUnsignedPoint struct { - Name string - Tags Tags - Aggregator UnsignedPointAggregator - Emitter UnsignedPointEmitter + Name string + Tags Tags + GroupingKey string + Aggregator UnsignedPointAggregator + Emitter UnsignedPointEmitter } // reduce executes fn once for every point in the next window. @@ -7004,19 +7861,44 @@ func (itr *unsignedReduceUnsignedIterator) reduce() ([]UnsignedPoint, error) { tags := curr.Tags.Subset(itr.dims) id := tags.ID() - // Retrieve the aggregator for this name/tag combination or create one. - rp := m[id] - if rp == nil { - aggregator, emitter := itr.create() - rp = &unsignedReduceUnsignedPoint{ - Name: curr.Name, - Tags: tags, - Aggregator: aggregator, - Emitter: emitter, + // Check to see if we have any GROUP BY dimensions that are not tags or time. + // If we have group-by key entries, create separate iterators for them. + // Otherwise, proceed as normal and create an iterator keyed by the tag ID. + if itr.opt.DimensionGrouper != nil && len(curr.Aux) > 0 { + entries, err := itr.opt.DimensionGrouper.ResolveKeys(curr.Aux, TagSubset{ID: id, HasTags: len(itr.dims) > 0}) + if err != nil { + return nil, err + } + + for _, entry := range entries { + rp := m[entry.DimKey] + if rp == nil { + aggregator, emitter := itr.create() + rp = &unsignedReduceUnsignedPoint{ + Name: curr.Name, + Tags: tags, + GroupingKey: entry.EncodedKey, + Aggregator: aggregator, + Emitter: emitter, + } + m[entry.DimKey] = rp + } + rp.Aggregator.AggregateUnsigned(curr) + } + } else { + rp := m[id] + if rp == nil { + aggregator, emitter := itr.create() + rp = &unsignedReduceUnsignedPoint{ + Name: curr.Name, + Tags: tags, + Aggregator: aggregator, + Emitter: emitter, + } + m[id] = rp } - m[id] = rp + rp.Aggregator.AggregateUnsigned(curr) } - rp.Aggregator.AggregateUnsigned(curr) } keys := make([]string, 0, len(m)) @@ -7052,6 +7934,50 @@ func (itr *unsignedReduceUnsignedIterator) reduce() ([]UnsignedPoint, error) { } else { sortedByTime = false } + + if itr.opt.DimensionGrouper != nil && rp.GroupingKey != "" { + dpVal, err := itr.opt.DimensionGrouper.DecodeEntry(rp.GroupingKey) + if err != nil { + return nil, err + } + // The emitted point must carry an Aux slot for every scanner aux + // key (len(itr.opt.Aux)). IteratorScanner.ScanAt ranges over the + // point's Aux, so a slice shorter than the key set leaves the + // trailing keys unvisited and the eval map retains stale values + // from a prior row, wrongly populating non-active date_part + // dimension columns. Selector functions (MIN, MAX, FIRST, LAST) + // already return a full-width Aux via cloneAux; aggregate + // functions (COUNT, SUM, MEAN) return an empty Aux, so grow it to + // full width here. (Keep any longer Aux as-is.) + width := len(itr.opt.Aux) + if width < len(points[i].Aux) { + width = len(points[i].Aux) + } + if width < 1 { + width = 1 + } + if len(points[i].Aux) < width { + aux := make([]interface{}, width) + copy(aux, points[i].Aux) + points[i].Aux = aux + } + // Only the active dimension is meaningful for this series, so null + // every date_part dimension slot (leaving non-active dimension + // columns null) and carry the active value as a DecodedDatePartKey + // in the last slot, which the scanner routes to the correct column + // by name. A stable, full-width slot ensures every key is visited + // and cleared on each scan. + n := len(points[i].Aux) + base := n - len(itr.opt.DatePartDimensions) + if base < 0 { + base = 0 + } + for j := base; j < n; j++ { + points[i].Aux[j] = nil + } + points[i].Aux[n-1] = dpVal + } + a = append(a, points[i]) } } @@ -7230,10 +8156,11 @@ func (itr *unsignedReduceStringIterator) Next() (*StringPoint, error) { // unsignedReduceStringPoint stores the reduced data for a name/tag combination. type unsignedReduceStringPoint struct { - Name string - Tags Tags - Aggregator UnsignedPointAggregator - Emitter StringPointEmitter + Name string + Tags Tags + GroupingKey string + Aggregator UnsignedPointAggregator + Emitter StringPointEmitter } // reduce executes fn once for every point in the next window. @@ -7292,19 +8219,44 @@ func (itr *unsignedReduceStringIterator) reduce() ([]StringPoint, error) { tags := curr.Tags.Subset(itr.dims) id := tags.ID() - // Retrieve the aggregator for this name/tag combination or create one. - rp := m[id] - if rp == nil { - aggregator, emitter := itr.create() - rp = &unsignedReduceStringPoint{ - Name: curr.Name, - Tags: tags, - Aggregator: aggregator, - Emitter: emitter, + // Check to see if we have any GROUP BY dimensions that are not tags or time. + // If we have group-by key entries, create separate iterators for them. + // Otherwise, proceed as normal and create an iterator keyed by the tag ID. + if itr.opt.DimensionGrouper != nil && len(curr.Aux) > 0 { + entries, err := itr.opt.DimensionGrouper.ResolveKeys(curr.Aux, TagSubset{ID: id, HasTags: len(itr.dims) > 0}) + if err != nil { + return nil, err } - m[id] = rp + + for _, entry := range entries { + rp := m[entry.DimKey] + if rp == nil { + aggregator, emitter := itr.create() + rp = &unsignedReduceStringPoint{ + Name: curr.Name, + Tags: tags, + GroupingKey: entry.EncodedKey, + Aggregator: aggregator, + Emitter: emitter, + } + m[entry.DimKey] = rp + } + rp.Aggregator.AggregateUnsigned(curr) + } + } else { + rp := m[id] + if rp == nil { + aggregator, emitter := itr.create() + rp = &unsignedReduceStringPoint{ + Name: curr.Name, + Tags: tags, + Aggregator: aggregator, + Emitter: emitter, + } + m[id] = rp + } + rp.Aggregator.AggregateUnsigned(curr) } - rp.Aggregator.AggregateUnsigned(curr) } keys := make([]string, 0, len(m)) @@ -7340,6 +8292,50 @@ func (itr *unsignedReduceStringIterator) reduce() ([]StringPoint, error) { } else { sortedByTime = false } + + if itr.opt.DimensionGrouper != nil && rp.GroupingKey != "" { + dpVal, err := itr.opt.DimensionGrouper.DecodeEntry(rp.GroupingKey) + if err != nil { + return nil, err + } + // The emitted point must carry an Aux slot for every scanner aux + // key (len(itr.opt.Aux)). IteratorScanner.ScanAt ranges over the + // point's Aux, so a slice shorter than the key set leaves the + // trailing keys unvisited and the eval map retains stale values + // from a prior row, wrongly populating non-active date_part + // dimension columns. Selector functions (MIN, MAX, FIRST, LAST) + // already return a full-width Aux via cloneAux; aggregate + // functions (COUNT, SUM, MEAN) return an empty Aux, so grow it to + // full width here. (Keep any longer Aux as-is.) + width := len(itr.opt.Aux) + if width < len(points[i].Aux) { + width = len(points[i].Aux) + } + if width < 1 { + width = 1 + } + if len(points[i].Aux) < width { + aux := make([]interface{}, width) + copy(aux, points[i].Aux) + points[i].Aux = aux + } + // Only the active dimension is meaningful for this series, so null + // every date_part dimension slot (leaving non-active dimension + // columns null) and carry the active value as a DecodedDatePartKey + // in the last slot, which the scanner routes to the correct column + // by name. A stable, full-width slot ensures every key is visited + // and cleared on each scan. + n := len(points[i].Aux) + base := n - len(itr.opt.DatePartDimensions) + if base < 0 { + base = 0 + } + for j := base; j < n; j++ { + points[i].Aux[j] = nil + } + points[i].Aux[n-1] = dpVal + } + a = append(a, points[i]) } } @@ -7518,10 +8514,11 @@ func (itr *unsignedReduceBooleanIterator) Next() (*BooleanPoint, error) { // unsignedReduceBooleanPoint stores the reduced data for a name/tag combination. type unsignedReduceBooleanPoint struct { - Name string - Tags Tags - Aggregator UnsignedPointAggregator - Emitter BooleanPointEmitter + Name string + Tags Tags + GroupingKey string + Aggregator UnsignedPointAggregator + Emitter BooleanPointEmitter } // reduce executes fn once for every point in the next window. @@ -7580,19 +8577,44 @@ func (itr *unsignedReduceBooleanIterator) reduce() ([]BooleanPoint, error) { tags := curr.Tags.Subset(itr.dims) id := tags.ID() - // Retrieve the aggregator for this name/tag combination or create one. - rp := m[id] - if rp == nil { - aggregator, emitter := itr.create() - rp = &unsignedReduceBooleanPoint{ - Name: curr.Name, - Tags: tags, - Aggregator: aggregator, - Emitter: emitter, + // Check to see if we have any GROUP BY dimensions that are not tags or time. + // If we have group-by key entries, create separate iterators for them. + // Otherwise, proceed as normal and create an iterator keyed by the tag ID. + if itr.opt.DimensionGrouper != nil && len(curr.Aux) > 0 { + entries, err := itr.opt.DimensionGrouper.ResolveKeys(curr.Aux, TagSubset{ID: id, HasTags: len(itr.dims) > 0}) + if err != nil { + return nil, err } - m[id] = rp + + for _, entry := range entries { + rp := m[entry.DimKey] + if rp == nil { + aggregator, emitter := itr.create() + rp = &unsignedReduceBooleanPoint{ + Name: curr.Name, + Tags: tags, + GroupingKey: entry.EncodedKey, + Aggregator: aggregator, + Emitter: emitter, + } + m[entry.DimKey] = rp + } + rp.Aggregator.AggregateUnsigned(curr) + } + } else { + rp := m[id] + if rp == nil { + aggregator, emitter := itr.create() + rp = &unsignedReduceBooleanPoint{ + Name: curr.Name, + Tags: tags, + Aggregator: aggregator, + Emitter: emitter, + } + m[id] = rp + } + rp.Aggregator.AggregateUnsigned(curr) } - rp.Aggregator.AggregateUnsigned(curr) } keys := make([]string, 0, len(m)) @@ -7628,6 +8650,50 @@ func (itr *unsignedReduceBooleanIterator) reduce() ([]BooleanPoint, error) { } else { sortedByTime = false } + + if itr.opt.DimensionGrouper != nil && rp.GroupingKey != "" { + dpVal, err := itr.opt.DimensionGrouper.DecodeEntry(rp.GroupingKey) + if err != nil { + return nil, err + } + // The emitted point must carry an Aux slot for every scanner aux + // key (len(itr.opt.Aux)). IteratorScanner.ScanAt ranges over the + // point's Aux, so a slice shorter than the key set leaves the + // trailing keys unvisited and the eval map retains stale values + // from a prior row, wrongly populating non-active date_part + // dimension columns. Selector functions (MIN, MAX, FIRST, LAST) + // already return a full-width Aux via cloneAux; aggregate + // functions (COUNT, SUM, MEAN) return an empty Aux, so grow it to + // full width here. (Keep any longer Aux as-is.) + width := len(itr.opt.Aux) + if width < len(points[i].Aux) { + width = len(points[i].Aux) + } + if width < 1 { + width = 1 + } + if len(points[i].Aux) < width { + aux := make([]interface{}, width) + copy(aux, points[i].Aux) + points[i].Aux = aux + } + // Only the active dimension is meaningful for this series, so null + // every date_part dimension slot (leaving non-active dimension + // columns null) and carry the active value as a DecodedDatePartKey + // in the last slot, which the scanner routes to the correct column + // by name. A stable, full-width slot ensures every key is visited + // and cleared on each scan. + n := len(points[i].Aux) + base := n - len(itr.opt.DatePartDimensions) + if base < 0 { + base = 0 + } + for j := base; j < n; j++ { + points[i].Aux[j] = nil + } + points[i].Aux[n-1] = dpVal + } + a = append(a, points[i]) } } @@ -7843,7 +8909,7 @@ func newUnsignedFilterIterator(input UnsignedIterator, cond influxql.Expr, opt I n := influxql.RewriteFunc(influxql.CloneExpr(cond), func(n influxql.Node) influxql.Node { switch n := n.(type) { case *influxql.BinaryExpr: - if n.LHS.String() == "time" { + if n.LHS.String() == models.TimeString { return &influxql.BooleanLiteral{Val: true} } } @@ -8548,6 +9614,11 @@ func (s *stringIteratorScanner) ScanAt(ts int64, name string, tags Tags, m map[s switch v.(type) { case float64, int64, uint64, string, bool: m[k.Val] = v + case DecodedDatePartKey: + m[DatePartDimensionsString] = v + // Clear any stale raw value under this dimension's own key so a prior + // row's value can't persist (the active value is carried via the key above). + delete(m, k.Val) default: // Insert the fill value if one was specified. if s.defaultValue != SkipDefault { @@ -9016,10 +10087,11 @@ func (itr *stringReduceFloatIterator) Next() (*FloatPoint, error) { // stringReduceFloatPoint stores the reduced data for a name/tag combination. type stringReduceFloatPoint struct { - Name string - Tags Tags - Aggregator StringPointAggregator - Emitter FloatPointEmitter + Name string + Tags Tags + GroupingKey string + Aggregator StringPointAggregator + Emitter FloatPointEmitter } // reduce executes fn once for every point in the next window. @@ -9078,19 +10150,44 @@ func (itr *stringReduceFloatIterator) reduce() ([]FloatPoint, error) { tags := curr.Tags.Subset(itr.dims) id := tags.ID() - // Retrieve the aggregator for this name/tag combination or create one. - rp := m[id] - if rp == nil { - aggregator, emitter := itr.create() - rp = &stringReduceFloatPoint{ - Name: curr.Name, - Tags: tags, - Aggregator: aggregator, - Emitter: emitter, + // Check to see if we have any GROUP BY dimensions that are not tags or time. + // If we have group-by key entries, create separate iterators for them. + // Otherwise, proceed as normal and create an iterator keyed by the tag ID. + if itr.opt.DimensionGrouper != nil && len(curr.Aux) > 0 { + entries, err := itr.opt.DimensionGrouper.ResolveKeys(curr.Aux, TagSubset{ID: id, HasTags: len(itr.dims) > 0}) + if err != nil { + return nil, err } - m[id] = rp + + for _, entry := range entries { + rp := m[entry.DimKey] + if rp == nil { + aggregator, emitter := itr.create() + rp = &stringReduceFloatPoint{ + Name: curr.Name, + Tags: tags, + GroupingKey: entry.EncodedKey, + Aggregator: aggregator, + Emitter: emitter, + } + m[entry.DimKey] = rp + } + rp.Aggregator.AggregateString(curr) + } + } else { + rp := m[id] + if rp == nil { + aggregator, emitter := itr.create() + rp = &stringReduceFloatPoint{ + Name: curr.Name, + Tags: tags, + Aggregator: aggregator, + Emitter: emitter, + } + m[id] = rp + } + rp.Aggregator.AggregateString(curr) } - rp.Aggregator.AggregateString(curr) } keys := make([]string, 0, len(m)) @@ -9126,6 +10223,50 @@ func (itr *stringReduceFloatIterator) reduce() ([]FloatPoint, error) { } else { sortedByTime = false } + + if itr.opt.DimensionGrouper != nil && rp.GroupingKey != "" { + dpVal, err := itr.opt.DimensionGrouper.DecodeEntry(rp.GroupingKey) + if err != nil { + return nil, err + } + // The emitted point must carry an Aux slot for every scanner aux + // key (len(itr.opt.Aux)). IteratorScanner.ScanAt ranges over the + // point's Aux, so a slice shorter than the key set leaves the + // trailing keys unvisited and the eval map retains stale values + // from a prior row, wrongly populating non-active date_part + // dimension columns. Selector functions (MIN, MAX, FIRST, LAST) + // already return a full-width Aux via cloneAux; aggregate + // functions (COUNT, SUM, MEAN) return an empty Aux, so grow it to + // full width here. (Keep any longer Aux as-is.) + width := len(itr.opt.Aux) + if width < len(points[i].Aux) { + width = len(points[i].Aux) + } + if width < 1 { + width = 1 + } + if len(points[i].Aux) < width { + aux := make([]interface{}, width) + copy(aux, points[i].Aux) + points[i].Aux = aux + } + // Only the active dimension is meaningful for this series, so null + // every date_part dimension slot (leaving non-active dimension + // columns null) and carry the active value as a DecodedDatePartKey + // in the last slot, which the scanner routes to the correct column + // by name. A stable, full-width slot ensures every key is visited + // and cleared on each scan. + n := len(points[i].Aux) + base := n - len(itr.opt.DatePartDimensions) + if base < 0 { + base = 0 + } + for j := base; j < n; j++ { + points[i].Aux[j] = nil + } + points[i].Aux[n-1] = dpVal + } + a = append(a, points[i]) } } @@ -9304,10 +10445,11 @@ func (itr *stringReduceIntegerIterator) Next() (*IntegerPoint, error) { // stringReduceIntegerPoint stores the reduced data for a name/tag combination. type stringReduceIntegerPoint struct { - Name string - Tags Tags - Aggregator StringPointAggregator - Emitter IntegerPointEmitter + Name string + Tags Tags + GroupingKey string + Aggregator StringPointAggregator + Emitter IntegerPointEmitter } // reduce executes fn once for every point in the next window. @@ -9366,19 +10508,44 @@ func (itr *stringReduceIntegerIterator) reduce() ([]IntegerPoint, error) { tags := curr.Tags.Subset(itr.dims) id := tags.ID() - // Retrieve the aggregator for this name/tag combination or create one. - rp := m[id] - if rp == nil { - aggregator, emitter := itr.create() - rp = &stringReduceIntegerPoint{ - Name: curr.Name, - Tags: tags, - Aggregator: aggregator, - Emitter: emitter, + // Check to see if we have any GROUP BY dimensions that are not tags or time. + // If we have group-by key entries, create separate iterators for them. + // Otherwise, proceed as normal and create an iterator keyed by the tag ID. + if itr.opt.DimensionGrouper != nil && len(curr.Aux) > 0 { + entries, err := itr.opt.DimensionGrouper.ResolveKeys(curr.Aux, TagSubset{ID: id, HasTags: len(itr.dims) > 0}) + if err != nil { + return nil, err + } + + for _, entry := range entries { + rp := m[entry.DimKey] + if rp == nil { + aggregator, emitter := itr.create() + rp = &stringReduceIntegerPoint{ + Name: curr.Name, + Tags: tags, + GroupingKey: entry.EncodedKey, + Aggregator: aggregator, + Emitter: emitter, + } + m[entry.DimKey] = rp + } + rp.Aggregator.AggregateString(curr) } - m[id] = rp + } else { + rp := m[id] + if rp == nil { + aggregator, emitter := itr.create() + rp = &stringReduceIntegerPoint{ + Name: curr.Name, + Tags: tags, + Aggregator: aggregator, + Emitter: emitter, + } + m[id] = rp + } + rp.Aggregator.AggregateString(curr) } - rp.Aggregator.AggregateString(curr) } keys := make([]string, 0, len(m)) @@ -9414,6 +10581,50 @@ func (itr *stringReduceIntegerIterator) reduce() ([]IntegerPoint, error) { } else { sortedByTime = false } + + if itr.opt.DimensionGrouper != nil && rp.GroupingKey != "" { + dpVal, err := itr.opt.DimensionGrouper.DecodeEntry(rp.GroupingKey) + if err != nil { + return nil, err + } + // The emitted point must carry an Aux slot for every scanner aux + // key (len(itr.opt.Aux)). IteratorScanner.ScanAt ranges over the + // point's Aux, so a slice shorter than the key set leaves the + // trailing keys unvisited and the eval map retains stale values + // from a prior row, wrongly populating non-active date_part + // dimension columns. Selector functions (MIN, MAX, FIRST, LAST) + // already return a full-width Aux via cloneAux; aggregate + // functions (COUNT, SUM, MEAN) return an empty Aux, so grow it to + // full width here. (Keep any longer Aux as-is.) + width := len(itr.opt.Aux) + if width < len(points[i].Aux) { + width = len(points[i].Aux) + } + if width < 1 { + width = 1 + } + if len(points[i].Aux) < width { + aux := make([]interface{}, width) + copy(aux, points[i].Aux) + points[i].Aux = aux + } + // Only the active dimension is meaningful for this series, so null + // every date_part dimension slot (leaving non-active dimension + // columns null) and carry the active value as a DecodedDatePartKey + // in the last slot, which the scanner routes to the correct column + // by name. A stable, full-width slot ensures every key is visited + // and cleared on each scan. + n := len(points[i].Aux) + base := n - len(itr.opt.DatePartDimensions) + if base < 0 { + base = 0 + } + for j := base; j < n; j++ { + points[i].Aux[j] = nil + } + points[i].Aux[n-1] = dpVal + } + a = append(a, points[i]) } } @@ -9592,10 +10803,11 @@ func (itr *stringReduceUnsignedIterator) Next() (*UnsignedPoint, error) { // stringReduceUnsignedPoint stores the reduced data for a name/tag combination. type stringReduceUnsignedPoint struct { - Name string - Tags Tags - Aggregator StringPointAggregator - Emitter UnsignedPointEmitter + Name string + Tags Tags + GroupingKey string + Aggregator StringPointAggregator + Emitter UnsignedPointEmitter } // reduce executes fn once for every point in the next window. @@ -9654,19 +10866,44 @@ func (itr *stringReduceUnsignedIterator) reduce() ([]UnsignedPoint, error) { tags := curr.Tags.Subset(itr.dims) id := tags.ID() - // Retrieve the aggregator for this name/tag combination or create one. - rp := m[id] - if rp == nil { - aggregator, emitter := itr.create() - rp = &stringReduceUnsignedPoint{ - Name: curr.Name, - Tags: tags, - Aggregator: aggregator, - Emitter: emitter, + // Check to see if we have any GROUP BY dimensions that are not tags or time. + // If we have group-by key entries, create separate iterators for them. + // Otherwise, proceed as normal and create an iterator keyed by the tag ID. + if itr.opt.DimensionGrouper != nil && len(curr.Aux) > 0 { + entries, err := itr.opt.DimensionGrouper.ResolveKeys(curr.Aux, TagSubset{ID: id, HasTags: len(itr.dims) > 0}) + if err != nil { + return nil, err + } + + for _, entry := range entries { + rp := m[entry.DimKey] + if rp == nil { + aggregator, emitter := itr.create() + rp = &stringReduceUnsignedPoint{ + Name: curr.Name, + Tags: tags, + GroupingKey: entry.EncodedKey, + Aggregator: aggregator, + Emitter: emitter, + } + m[entry.DimKey] = rp + } + rp.Aggregator.AggregateString(curr) } - m[id] = rp + } else { + rp := m[id] + if rp == nil { + aggregator, emitter := itr.create() + rp = &stringReduceUnsignedPoint{ + Name: curr.Name, + Tags: tags, + Aggregator: aggregator, + Emitter: emitter, + } + m[id] = rp + } + rp.Aggregator.AggregateString(curr) } - rp.Aggregator.AggregateString(curr) } keys := make([]string, 0, len(m)) @@ -9702,6 +10939,50 @@ func (itr *stringReduceUnsignedIterator) reduce() ([]UnsignedPoint, error) { } else { sortedByTime = false } + + if itr.opt.DimensionGrouper != nil && rp.GroupingKey != "" { + dpVal, err := itr.opt.DimensionGrouper.DecodeEntry(rp.GroupingKey) + if err != nil { + return nil, err + } + // The emitted point must carry an Aux slot for every scanner aux + // key (len(itr.opt.Aux)). IteratorScanner.ScanAt ranges over the + // point's Aux, so a slice shorter than the key set leaves the + // trailing keys unvisited and the eval map retains stale values + // from a prior row, wrongly populating non-active date_part + // dimension columns. Selector functions (MIN, MAX, FIRST, LAST) + // already return a full-width Aux via cloneAux; aggregate + // functions (COUNT, SUM, MEAN) return an empty Aux, so grow it to + // full width here. (Keep any longer Aux as-is.) + width := len(itr.opt.Aux) + if width < len(points[i].Aux) { + width = len(points[i].Aux) + } + if width < 1 { + width = 1 + } + if len(points[i].Aux) < width { + aux := make([]interface{}, width) + copy(aux, points[i].Aux) + points[i].Aux = aux + } + // Only the active dimension is meaningful for this series, so null + // every date_part dimension slot (leaving non-active dimension + // columns null) and carry the active value as a DecodedDatePartKey + // in the last slot, which the scanner routes to the correct column + // by name. A stable, full-width slot ensures every key is visited + // and cleared on each scan. + n := len(points[i].Aux) + base := n - len(itr.opt.DatePartDimensions) + if base < 0 { + base = 0 + } + for j := base; j < n; j++ { + points[i].Aux[j] = nil + } + points[i].Aux[n-1] = dpVal + } + a = append(a, points[i]) } } @@ -9880,10 +11161,11 @@ func (itr *stringReduceStringIterator) Next() (*StringPoint, error) { // stringReduceStringPoint stores the reduced data for a name/tag combination. type stringReduceStringPoint struct { - Name string - Tags Tags - Aggregator StringPointAggregator - Emitter StringPointEmitter + Name string + Tags Tags + GroupingKey string + Aggregator StringPointAggregator + Emitter StringPointEmitter } // reduce executes fn once for every point in the next window. @@ -9942,19 +11224,44 @@ func (itr *stringReduceStringIterator) reduce() ([]StringPoint, error) { tags := curr.Tags.Subset(itr.dims) id := tags.ID() - // Retrieve the aggregator for this name/tag combination or create one. - rp := m[id] - if rp == nil { - aggregator, emitter := itr.create() - rp = &stringReduceStringPoint{ - Name: curr.Name, - Tags: tags, - Aggregator: aggregator, - Emitter: emitter, + // Check to see if we have any GROUP BY dimensions that are not tags or time. + // If we have group-by key entries, create separate iterators for them. + // Otherwise, proceed as normal and create an iterator keyed by the tag ID. + if itr.opt.DimensionGrouper != nil && len(curr.Aux) > 0 { + entries, err := itr.opt.DimensionGrouper.ResolveKeys(curr.Aux, TagSubset{ID: id, HasTags: len(itr.dims) > 0}) + if err != nil { + return nil, err + } + + for _, entry := range entries { + rp := m[entry.DimKey] + if rp == nil { + aggregator, emitter := itr.create() + rp = &stringReduceStringPoint{ + Name: curr.Name, + Tags: tags, + GroupingKey: entry.EncodedKey, + Aggregator: aggregator, + Emitter: emitter, + } + m[entry.DimKey] = rp + } + rp.Aggregator.AggregateString(curr) } - m[id] = rp + } else { + rp := m[id] + if rp == nil { + aggregator, emitter := itr.create() + rp = &stringReduceStringPoint{ + Name: curr.Name, + Tags: tags, + Aggregator: aggregator, + Emitter: emitter, + } + m[id] = rp + } + rp.Aggregator.AggregateString(curr) } - rp.Aggregator.AggregateString(curr) } keys := make([]string, 0, len(m)) @@ -9990,6 +11297,50 @@ func (itr *stringReduceStringIterator) reduce() ([]StringPoint, error) { } else { sortedByTime = false } + + if itr.opt.DimensionGrouper != nil && rp.GroupingKey != "" { + dpVal, err := itr.opt.DimensionGrouper.DecodeEntry(rp.GroupingKey) + if err != nil { + return nil, err + } + // The emitted point must carry an Aux slot for every scanner aux + // key (len(itr.opt.Aux)). IteratorScanner.ScanAt ranges over the + // point's Aux, so a slice shorter than the key set leaves the + // trailing keys unvisited and the eval map retains stale values + // from a prior row, wrongly populating non-active date_part + // dimension columns. Selector functions (MIN, MAX, FIRST, LAST) + // already return a full-width Aux via cloneAux; aggregate + // functions (COUNT, SUM, MEAN) return an empty Aux, so grow it to + // full width here. (Keep any longer Aux as-is.) + width := len(itr.opt.Aux) + if width < len(points[i].Aux) { + width = len(points[i].Aux) + } + if width < 1 { + width = 1 + } + if len(points[i].Aux) < width { + aux := make([]interface{}, width) + copy(aux, points[i].Aux) + points[i].Aux = aux + } + // Only the active dimension is meaningful for this series, so null + // every date_part dimension slot (leaving non-active dimension + // columns null) and carry the active value as a DecodedDatePartKey + // in the last slot, which the scanner routes to the correct column + // by name. A stable, full-width slot ensures every key is visited + // and cleared on each scan. + n := len(points[i].Aux) + base := n - len(itr.opt.DatePartDimensions) + if base < 0 { + base = 0 + } + for j := base; j < n; j++ { + points[i].Aux[j] = nil + } + points[i].Aux[n-1] = dpVal + } + a = append(a, points[i]) } } @@ -10168,10 +11519,11 @@ func (itr *stringReduceBooleanIterator) Next() (*BooleanPoint, error) { // stringReduceBooleanPoint stores the reduced data for a name/tag combination. type stringReduceBooleanPoint struct { - Name string - Tags Tags - Aggregator StringPointAggregator - Emitter BooleanPointEmitter + Name string + Tags Tags + GroupingKey string + Aggregator StringPointAggregator + Emitter BooleanPointEmitter } // reduce executes fn once for every point in the next window. @@ -10230,19 +11582,44 @@ func (itr *stringReduceBooleanIterator) reduce() ([]BooleanPoint, error) { tags := curr.Tags.Subset(itr.dims) id := tags.ID() - // Retrieve the aggregator for this name/tag combination or create one. - rp := m[id] - if rp == nil { - aggregator, emitter := itr.create() - rp = &stringReduceBooleanPoint{ - Name: curr.Name, - Tags: tags, - Aggregator: aggregator, - Emitter: emitter, + // Check to see if we have any GROUP BY dimensions that are not tags or time. + // If we have group-by key entries, create separate iterators for them. + // Otherwise, proceed as normal and create an iterator keyed by the tag ID. + if itr.opt.DimensionGrouper != nil && len(curr.Aux) > 0 { + entries, err := itr.opt.DimensionGrouper.ResolveKeys(curr.Aux, TagSubset{ID: id, HasTags: len(itr.dims) > 0}) + if err != nil { + return nil, err + } + + for _, entry := range entries { + rp := m[entry.DimKey] + if rp == nil { + aggregator, emitter := itr.create() + rp = &stringReduceBooleanPoint{ + Name: curr.Name, + Tags: tags, + GroupingKey: entry.EncodedKey, + Aggregator: aggregator, + Emitter: emitter, + } + m[entry.DimKey] = rp + } + rp.Aggregator.AggregateString(curr) + } + } else { + rp := m[id] + if rp == nil { + aggregator, emitter := itr.create() + rp = &stringReduceBooleanPoint{ + Name: curr.Name, + Tags: tags, + Aggregator: aggregator, + Emitter: emitter, + } + m[id] = rp } - m[id] = rp + rp.Aggregator.AggregateString(curr) } - rp.Aggregator.AggregateString(curr) } keys := make([]string, 0, len(m)) @@ -10278,6 +11655,50 @@ func (itr *stringReduceBooleanIterator) reduce() ([]BooleanPoint, error) { } else { sortedByTime = false } + + if itr.opt.DimensionGrouper != nil && rp.GroupingKey != "" { + dpVal, err := itr.opt.DimensionGrouper.DecodeEntry(rp.GroupingKey) + if err != nil { + return nil, err + } + // The emitted point must carry an Aux slot for every scanner aux + // key (len(itr.opt.Aux)). IteratorScanner.ScanAt ranges over the + // point's Aux, so a slice shorter than the key set leaves the + // trailing keys unvisited and the eval map retains stale values + // from a prior row, wrongly populating non-active date_part + // dimension columns. Selector functions (MIN, MAX, FIRST, LAST) + // already return a full-width Aux via cloneAux; aggregate + // functions (COUNT, SUM, MEAN) return an empty Aux, so grow it to + // full width here. (Keep any longer Aux as-is.) + width := len(itr.opt.Aux) + if width < len(points[i].Aux) { + width = len(points[i].Aux) + } + if width < 1 { + width = 1 + } + if len(points[i].Aux) < width { + aux := make([]interface{}, width) + copy(aux, points[i].Aux) + points[i].Aux = aux + } + // Only the active dimension is meaningful for this series, so null + // every date_part dimension slot (leaving non-active dimension + // columns null) and carry the active value as a DecodedDatePartKey + // in the last slot, which the scanner routes to the correct column + // by name. A stable, full-width slot ensures every key is visited + // and cleared on each scan. + n := len(points[i].Aux) + base := n - len(itr.opt.DatePartDimensions) + if base < 0 { + base = 0 + } + for j := base; j < n; j++ { + points[i].Aux[j] = nil + } + points[i].Aux[n-1] = dpVal + } + a = append(a, points[i]) } } @@ -10493,7 +11914,7 @@ func newStringFilterIterator(input StringIterator, cond influxql.Expr, opt Itera n := influxql.RewriteFunc(influxql.CloneExpr(cond), func(n influxql.Node) influxql.Node { switch n := n.(type) { case *influxql.BinaryExpr: - if n.LHS.String() == "time" { + if n.LHS.String() == models.TimeString { return &influxql.BooleanLiteral{Val: true} } } @@ -11198,6 +12619,11 @@ func (s *booleanIteratorScanner) ScanAt(ts int64, name string, tags Tags, m map[ switch v.(type) { case float64, int64, uint64, string, bool: m[k.Val] = v + case DecodedDatePartKey: + m[DatePartDimensionsString] = v + // Clear any stale raw value under this dimension's own key so a prior + // row's value can't persist (the active value is carried via the key above). + delete(m, k.Val) default: // Insert the fill value if one was specified. if s.defaultValue != SkipDefault { @@ -11666,10 +13092,11 @@ func (itr *booleanReduceFloatIterator) Next() (*FloatPoint, error) { // booleanReduceFloatPoint stores the reduced data for a name/tag combination. type booleanReduceFloatPoint struct { - Name string - Tags Tags - Aggregator BooleanPointAggregator - Emitter FloatPointEmitter + Name string + Tags Tags + GroupingKey string + Aggregator BooleanPointAggregator + Emitter FloatPointEmitter } // reduce executes fn once for every point in the next window. @@ -11728,19 +13155,44 @@ func (itr *booleanReduceFloatIterator) reduce() ([]FloatPoint, error) { tags := curr.Tags.Subset(itr.dims) id := tags.ID() - // Retrieve the aggregator for this name/tag combination or create one. - rp := m[id] - if rp == nil { - aggregator, emitter := itr.create() - rp = &booleanReduceFloatPoint{ - Name: curr.Name, - Tags: tags, - Aggregator: aggregator, - Emitter: emitter, + // Check to see if we have any GROUP BY dimensions that are not tags or time. + // If we have group-by key entries, create separate iterators for them. + // Otherwise, proceed as normal and create an iterator keyed by the tag ID. + if itr.opt.DimensionGrouper != nil && len(curr.Aux) > 0 { + entries, err := itr.opt.DimensionGrouper.ResolveKeys(curr.Aux, TagSubset{ID: id, HasTags: len(itr.dims) > 0}) + if err != nil { + return nil, err } - m[id] = rp + + for _, entry := range entries { + rp := m[entry.DimKey] + if rp == nil { + aggregator, emitter := itr.create() + rp = &booleanReduceFloatPoint{ + Name: curr.Name, + Tags: tags, + GroupingKey: entry.EncodedKey, + Aggregator: aggregator, + Emitter: emitter, + } + m[entry.DimKey] = rp + } + rp.Aggregator.AggregateBoolean(curr) + } + } else { + rp := m[id] + if rp == nil { + aggregator, emitter := itr.create() + rp = &booleanReduceFloatPoint{ + Name: curr.Name, + Tags: tags, + Aggregator: aggregator, + Emitter: emitter, + } + m[id] = rp + } + rp.Aggregator.AggregateBoolean(curr) } - rp.Aggregator.AggregateBoolean(curr) } keys := make([]string, 0, len(m)) @@ -11776,6 +13228,50 @@ func (itr *booleanReduceFloatIterator) reduce() ([]FloatPoint, error) { } else { sortedByTime = false } + + if itr.opt.DimensionGrouper != nil && rp.GroupingKey != "" { + dpVal, err := itr.opt.DimensionGrouper.DecodeEntry(rp.GroupingKey) + if err != nil { + return nil, err + } + // The emitted point must carry an Aux slot for every scanner aux + // key (len(itr.opt.Aux)). IteratorScanner.ScanAt ranges over the + // point's Aux, so a slice shorter than the key set leaves the + // trailing keys unvisited and the eval map retains stale values + // from a prior row, wrongly populating non-active date_part + // dimension columns. Selector functions (MIN, MAX, FIRST, LAST) + // already return a full-width Aux via cloneAux; aggregate + // functions (COUNT, SUM, MEAN) return an empty Aux, so grow it to + // full width here. (Keep any longer Aux as-is.) + width := len(itr.opt.Aux) + if width < len(points[i].Aux) { + width = len(points[i].Aux) + } + if width < 1 { + width = 1 + } + if len(points[i].Aux) < width { + aux := make([]interface{}, width) + copy(aux, points[i].Aux) + points[i].Aux = aux + } + // Only the active dimension is meaningful for this series, so null + // every date_part dimension slot (leaving non-active dimension + // columns null) and carry the active value as a DecodedDatePartKey + // in the last slot, which the scanner routes to the correct column + // by name. A stable, full-width slot ensures every key is visited + // and cleared on each scan. + n := len(points[i].Aux) + base := n - len(itr.opt.DatePartDimensions) + if base < 0 { + base = 0 + } + for j := base; j < n; j++ { + points[i].Aux[j] = nil + } + points[i].Aux[n-1] = dpVal + } + a = append(a, points[i]) } } @@ -11954,10 +13450,11 @@ func (itr *booleanReduceIntegerIterator) Next() (*IntegerPoint, error) { // booleanReduceIntegerPoint stores the reduced data for a name/tag combination. type booleanReduceIntegerPoint struct { - Name string - Tags Tags - Aggregator BooleanPointAggregator - Emitter IntegerPointEmitter + Name string + Tags Tags + GroupingKey string + Aggregator BooleanPointAggregator + Emitter IntegerPointEmitter } // reduce executes fn once for every point in the next window. @@ -12016,19 +13513,44 @@ func (itr *booleanReduceIntegerIterator) reduce() ([]IntegerPoint, error) { tags := curr.Tags.Subset(itr.dims) id := tags.ID() - // Retrieve the aggregator for this name/tag combination or create one. - rp := m[id] - if rp == nil { - aggregator, emitter := itr.create() - rp = &booleanReduceIntegerPoint{ - Name: curr.Name, - Tags: tags, - Aggregator: aggregator, - Emitter: emitter, + // Check to see if we have any GROUP BY dimensions that are not tags or time. + // If we have group-by key entries, create separate iterators for them. + // Otherwise, proceed as normal and create an iterator keyed by the tag ID. + if itr.opt.DimensionGrouper != nil && len(curr.Aux) > 0 { + entries, err := itr.opt.DimensionGrouper.ResolveKeys(curr.Aux, TagSubset{ID: id, HasTags: len(itr.dims) > 0}) + if err != nil { + return nil, err + } + + for _, entry := range entries { + rp := m[entry.DimKey] + if rp == nil { + aggregator, emitter := itr.create() + rp = &booleanReduceIntegerPoint{ + Name: curr.Name, + Tags: tags, + GroupingKey: entry.EncodedKey, + Aggregator: aggregator, + Emitter: emitter, + } + m[entry.DimKey] = rp + } + rp.Aggregator.AggregateBoolean(curr) + } + } else { + rp := m[id] + if rp == nil { + aggregator, emitter := itr.create() + rp = &booleanReduceIntegerPoint{ + Name: curr.Name, + Tags: tags, + Aggregator: aggregator, + Emitter: emitter, + } + m[id] = rp } - m[id] = rp + rp.Aggregator.AggregateBoolean(curr) } - rp.Aggregator.AggregateBoolean(curr) } keys := make([]string, 0, len(m)) @@ -12064,6 +13586,50 @@ func (itr *booleanReduceIntegerIterator) reduce() ([]IntegerPoint, error) { } else { sortedByTime = false } + + if itr.opt.DimensionGrouper != nil && rp.GroupingKey != "" { + dpVal, err := itr.opt.DimensionGrouper.DecodeEntry(rp.GroupingKey) + if err != nil { + return nil, err + } + // The emitted point must carry an Aux slot for every scanner aux + // key (len(itr.opt.Aux)). IteratorScanner.ScanAt ranges over the + // point's Aux, so a slice shorter than the key set leaves the + // trailing keys unvisited and the eval map retains stale values + // from a prior row, wrongly populating non-active date_part + // dimension columns. Selector functions (MIN, MAX, FIRST, LAST) + // already return a full-width Aux via cloneAux; aggregate + // functions (COUNT, SUM, MEAN) return an empty Aux, so grow it to + // full width here. (Keep any longer Aux as-is.) + width := len(itr.opt.Aux) + if width < len(points[i].Aux) { + width = len(points[i].Aux) + } + if width < 1 { + width = 1 + } + if len(points[i].Aux) < width { + aux := make([]interface{}, width) + copy(aux, points[i].Aux) + points[i].Aux = aux + } + // Only the active dimension is meaningful for this series, so null + // every date_part dimension slot (leaving non-active dimension + // columns null) and carry the active value as a DecodedDatePartKey + // in the last slot, which the scanner routes to the correct column + // by name. A stable, full-width slot ensures every key is visited + // and cleared on each scan. + n := len(points[i].Aux) + base := n - len(itr.opt.DatePartDimensions) + if base < 0 { + base = 0 + } + for j := base; j < n; j++ { + points[i].Aux[j] = nil + } + points[i].Aux[n-1] = dpVal + } + a = append(a, points[i]) } } @@ -12242,10 +13808,11 @@ func (itr *booleanReduceUnsignedIterator) Next() (*UnsignedPoint, error) { // booleanReduceUnsignedPoint stores the reduced data for a name/tag combination. type booleanReduceUnsignedPoint struct { - Name string - Tags Tags - Aggregator BooleanPointAggregator - Emitter UnsignedPointEmitter + Name string + Tags Tags + GroupingKey string + Aggregator BooleanPointAggregator + Emitter UnsignedPointEmitter } // reduce executes fn once for every point in the next window. @@ -12304,19 +13871,44 @@ func (itr *booleanReduceUnsignedIterator) reduce() ([]UnsignedPoint, error) { tags := curr.Tags.Subset(itr.dims) id := tags.ID() - // Retrieve the aggregator for this name/tag combination or create one. - rp := m[id] - if rp == nil { - aggregator, emitter := itr.create() - rp = &booleanReduceUnsignedPoint{ - Name: curr.Name, - Tags: tags, - Aggregator: aggregator, - Emitter: emitter, + // Check to see if we have any GROUP BY dimensions that are not tags or time. + // If we have group-by key entries, create separate iterators for them. + // Otherwise, proceed as normal and create an iterator keyed by the tag ID. + if itr.opt.DimensionGrouper != nil && len(curr.Aux) > 0 { + entries, err := itr.opt.DimensionGrouper.ResolveKeys(curr.Aux, TagSubset{ID: id, HasTags: len(itr.dims) > 0}) + if err != nil { + return nil, err + } + + for _, entry := range entries { + rp := m[entry.DimKey] + if rp == nil { + aggregator, emitter := itr.create() + rp = &booleanReduceUnsignedPoint{ + Name: curr.Name, + Tags: tags, + GroupingKey: entry.EncodedKey, + Aggregator: aggregator, + Emitter: emitter, + } + m[entry.DimKey] = rp + } + rp.Aggregator.AggregateBoolean(curr) + } + } else { + rp := m[id] + if rp == nil { + aggregator, emitter := itr.create() + rp = &booleanReduceUnsignedPoint{ + Name: curr.Name, + Tags: tags, + Aggregator: aggregator, + Emitter: emitter, + } + m[id] = rp } - m[id] = rp + rp.Aggregator.AggregateBoolean(curr) } - rp.Aggregator.AggregateBoolean(curr) } keys := make([]string, 0, len(m)) @@ -12352,6 +13944,50 @@ func (itr *booleanReduceUnsignedIterator) reduce() ([]UnsignedPoint, error) { } else { sortedByTime = false } + + if itr.opt.DimensionGrouper != nil && rp.GroupingKey != "" { + dpVal, err := itr.opt.DimensionGrouper.DecodeEntry(rp.GroupingKey) + if err != nil { + return nil, err + } + // The emitted point must carry an Aux slot for every scanner aux + // key (len(itr.opt.Aux)). IteratorScanner.ScanAt ranges over the + // point's Aux, so a slice shorter than the key set leaves the + // trailing keys unvisited and the eval map retains stale values + // from a prior row, wrongly populating non-active date_part + // dimension columns. Selector functions (MIN, MAX, FIRST, LAST) + // already return a full-width Aux via cloneAux; aggregate + // functions (COUNT, SUM, MEAN) return an empty Aux, so grow it to + // full width here. (Keep any longer Aux as-is.) + width := len(itr.opt.Aux) + if width < len(points[i].Aux) { + width = len(points[i].Aux) + } + if width < 1 { + width = 1 + } + if len(points[i].Aux) < width { + aux := make([]interface{}, width) + copy(aux, points[i].Aux) + points[i].Aux = aux + } + // Only the active dimension is meaningful for this series, so null + // every date_part dimension slot (leaving non-active dimension + // columns null) and carry the active value as a DecodedDatePartKey + // in the last slot, which the scanner routes to the correct column + // by name. A stable, full-width slot ensures every key is visited + // and cleared on each scan. + n := len(points[i].Aux) + base := n - len(itr.opt.DatePartDimensions) + if base < 0 { + base = 0 + } + for j := base; j < n; j++ { + points[i].Aux[j] = nil + } + points[i].Aux[n-1] = dpVal + } + a = append(a, points[i]) } } @@ -12530,10 +14166,11 @@ func (itr *booleanReduceStringIterator) Next() (*StringPoint, error) { // booleanReduceStringPoint stores the reduced data for a name/tag combination. type booleanReduceStringPoint struct { - Name string - Tags Tags - Aggregator BooleanPointAggregator - Emitter StringPointEmitter + Name string + Tags Tags + GroupingKey string + Aggregator BooleanPointAggregator + Emitter StringPointEmitter } // reduce executes fn once for every point in the next window. @@ -12592,19 +14229,44 @@ func (itr *booleanReduceStringIterator) reduce() ([]StringPoint, error) { tags := curr.Tags.Subset(itr.dims) id := tags.ID() - // Retrieve the aggregator for this name/tag combination or create one. - rp := m[id] - if rp == nil { - aggregator, emitter := itr.create() - rp = &booleanReduceStringPoint{ - Name: curr.Name, - Tags: tags, - Aggregator: aggregator, - Emitter: emitter, + // Check to see if we have any GROUP BY dimensions that are not tags or time. + // If we have group-by key entries, create separate iterators for them. + // Otherwise, proceed as normal and create an iterator keyed by the tag ID. + if itr.opt.DimensionGrouper != nil && len(curr.Aux) > 0 { + entries, err := itr.opt.DimensionGrouper.ResolveKeys(curr.Aux, TagSubset{ID: id, HasTags: len(itr.dims) > 0}) + if err != nil { + return nil, err + } + + for _, entry := range entries { + rp := m[entry.DimKey] + if rp == nil { + aggregator, emitter := itr.create() + rp = &booleanReduceStringPoint{ + Name: curr.Name, + Tags: tags, + GroupingKey: entry.EncodedKey, + Aggregator: aggregator, + Emitter: emitter, + } + m[entry.DimKey] = rp + } + rp.Aggregator.AggregateBoolean(curr) + } + } else { + rp := m[id] + if rp == nil { + aggregator, emitter := itr.create() + rp = &booleanReduceStringPoint{ + Name: curr.Name, + Tags: tags, + Aggregator: aggregator, + Emitter: emitter, + } + m[id] = rp } - m[id] = rp + rp.Aggregator.AggregateBoolean(curr) } - rp.Aggregator.AggregateBoolean(curr) } keys := make([]string, 0, len(m)) @@ -12640,6 +14302,50 @@ func (itr *booleanReduceStringIterator) reduce() ([]StringPoint, error) { } else { sortedByTime = false } + + if itr.opt.DimensionGrouper != nil && rp.GroupingKey != "" { + dpVal, err := itr.opt.DimensionGrouper.DecodeEntry(rp.GroupingKey) + if err != nil { + return nil, err + } + // The emitted point must carry an Aux slot for every scanner aux + // key (len(itr.opt.Aux)). IteratorScanner.ScanAt ranges over the + // point's Aux, so a slice shorter than the key set leaves the + // trailing keys unvisited and the eval map retains stale values + // from a prior row, wrongly populating non-active date_part + // dimension columns. Selector functions (MIN, MAX, FIRST, LAST) + // already return a full-width Aux via cloneAux; aggregate + // functions (COUNT, SUM, MEAN) return an empty Aux, so grow it to + // full width here. (Keep any longer Aux as-is.) + width := len(itr.opt.Aux) + if width < len(points[i].Aux) { + width = len(points[i].Aux) + } + if width < 1 { + width = 1 + } + if len(points[i].Aux) < width { + aux := make([]interface{}, width) + copy(aux, points[i].Aux) + points[i].Aux = aux + } + // Only the active dimension is meaningful for this series, so null + // every date_part dimension slot (leaving non-active dimension + // columns null) and carry the active value as a DecodedDatePartKey + // in the last slot, which the scanner routes to the correct column + // by name. A stable, full-width slot ensures every key is visited + // and cleared on each scan. + n := len(points[i].Aux) + base := n - len(itr.opt.DatePartDimensions) + if base < 0 { + base = 0 + } + for j := base; j < n; j++ { + points[i].Aux[j] = nil + } + points[i].Aux[n-1] = dpVal + } + a = append(a, points[i]) } } @@ -12818,10 +14524,11 @@ func (itr *booleanReduceBooleanIterator) Next() (*BooleanPoint, error) { // booleanReduceBooleanPoint stores the reduced data for a name/tag combination. type booleanReduceBooleanPoint struct { - Name string - Tags Tags - Aggregator BooleanPointAggregator - Emitter BooleanPointEmitter + Name string + Tags Tags + GroupingKey string + Aggregator BooleanPointAggregator + Emitter BooleanPointEmitter } // reduce executes fn once for every point in the next window. @@ -12880,19 +14587,44 @@ func (itr *booleanReduceBooleanIterator) reduce() ([]BooleanPoint, error) { tags := curr.Tags.Subset(itr.dims) id := tags.ID() - // Retrieve the aggregator for this name/tag combination or create one. - rp := m[id] - if rp == nil { - aggregator, emitter := itr.create() - rp = &booleanReduceBooleanPoint{ - Name: curr.Name, - Tags: tags, - Aggregator: aggregator, - Emitter: emitter, + // Check to see if we have any GROUP BY dimensions that are not tags or time. + // If we have group-by key entries, create separate iterators for them. + // Otherwise, proceed as normal and create an iterator keyed by the tag ID. + if itr.opt.DimensionGrouper != nil && len(curr.Aux) > 0 { + entries, err := itr.opt.DimensionGrouper.ResolveKeys(curr.Aux, TagSubset{ID: id, HasTags: len(itr.dims) > 0}) + if err != nil { + return nil, err + } + + for _, entry := range entries { + rp := m[entry.DimKey] + if rp == nil { + aggregator, emitter := itr.create() + rp = &booleanReduceBooleanPoint{ + Name: curr.Name, + Tags: tags, + GroupingKey: entry.EncodedKey, + Aggregator: aggregator, + Emitter: emitter, + } + m[entry.DimKey] = rp + } + rp.Aggregator.AggregateBoolean(curr) } - m[id] = rp + } else { + rp := m[id] + if rp == nil { + aggregator, emitter := itr.create() + rp = &booleanReduceBooleanPoint{ + Name: curr.Name, + Tags: tags, + Aggregator: aggregator, + Emitter: emitter, + } + m[id] = rp + } + rp.Aggregator.AggregateBoolean(curr) } - rp.Aggregator.AggregateBoolean(curr) } keys := make([]string, 0, len(m)) @@ -12928,6 +14660,50 @@ func (itr *booleanReduceBooleanIterator) reduce() ([]BooleanPoint, error) { } else { sortedByTime = false } + + if itr.opt.DimensionGrouper != nil && rp.GroupingKey != "" { + dpVal, err := itr.opt.DimensionGrouper.DecodeEntry(rp.GroupingKey) + if err != nil { + return nil, err + } + // The emitted point must carry an Aux slot for every scanner aux + // key (len(itr.opt.Aux)). IteratorScanner.ScanAt ranges over the + // point's Aux, so a slice shorter than the key set leaves the + // trailing keys unvisited and the eval map retains stale values + // from a prior row, wrongly populating non-active date_part + // dimension columns. Selector functions (MIN, MAX, FIRST, LAST) + // already return a full-width Aux via cloneAux; aggregate + // functions (COUNT, SUM, MEAN) return an empty Aux, so grow it to + // full width here. (Keep any longer Aux as-is.) + width := len(itr.opt.Aux) + if width < len(points[i].Aux) { + width = len(points[i].Aux) + } + if width < 1 { + width = 1 + } + if len(points[i].Aux) < width { + aux := make([]interface{}, width) + copy(aux, points[i].Aux) + points[i].Aux = aux + } + // Only the active dimension is meaningful for this series, so null + // every date_part dimension slot (leaving non-active dimension + // columns null) and carry the active value as a DecodedDatePartKey + // in the last slot, which the scanner routes to the correct column + // by name. A stable, full-width slot ensures every key is visited + // and cleared on each scan. + n := len(points[i].Aux) + base := n - len(itr.opt.DatePartDimensions) + if base < 0 { + base = 0 + } + for j := base; j < n; j++ { + points[i].Aux[j] = nil + } + points[i].Aux[n-1] = dpVal + } + a = append(a, points[i]) } } @@ -13143,7 +14919,7 @@ func newBooleanFilterIterator(input BooleanIterator, cond influxql.Expr, opt Ite n := influxql.RewriteFunc(influxql.CloneExpr(cond), func(n influxql.Node) influxql.Node { switch n := n.(type) { case *influxql.BinaryExpr: - if n.LHS.String() == "time" { + if n.LHS.String() == models.TimeString { return &influxql.BooleanLiteral{Val: true} } } diff --git a/query/iterator.gen.go.tmpl b/query/iterator.gen.go.tmpl index 3b39fe5d8bc..7d04f9fd803 100644 --- a/query/iterator.gen.go.tmpl +++ b/query/iterator.gen.go.tmpl @@ -8,8 +8,8 @@ import ( "sort" "sync" "time" - "sync" + "github.com/influxdata/influxdb/models" "github.com/influxdata/influxql" "google.golang.org/protobuf/proto" ) @@ -556,6 +556,11 @@ func (s *{{$k.name}}IteratorScanner) ScanAt(ts int64, name string, tags Tags, m switch v.(type) { case float64, int64, uint64, string, bool: m[k.Val] = v + case DecodedDatePartKey: + m[DatePartDimensionsString] = v + // Clear any stale raw value under this dimension's own key so a prior + // row's value can't persist (the active value is carried via the key above). + delete(m, k.Val) default: // Insert the fill value if one was specified. if s.defaultValue != SkipDefault { @@ -1043,10 +1048,11 @@ func (itr *{{$k.name}}Reduce{{$v.Name}}Iterator) Next() (*{{$v.Name}}Point, erro // {{$k.name}}Reduce{{$v.Name}}Point stores the reduced data for a name/tag combination. type {{$k.name}}Reduce{{$v.Name}}Point struct { - Name string - Tags Tags - Aggregator {{$k.Name}}PointAggregator - Emitter {{$v.Name}}PointEmitter + Name string + Tags Tags + GroupingKey string + Aggregator {{$k.Name}}PointAggregator + Emitter {{$v.Name}}PointEmitter } // reduce executes fn once for every point in the next window. @@ -1105,19 +1111,44 @@ func (itr *{{$k.name}}Reduce{{$v.Name}}Iterator) reduce() ([]{{$v.Name}}Point, e tags := curr.Tags.Subset(itr.dims) id := tags.ID() - // Retrieve the aggregator for this name/tag combination or create one. - rp := m[id] - if rp == nil { - aggregator, emitter := itr.create() - rp = &{{$k.name}}Reduce{{$v.Name}}Point{ - Name: curr.Name, - Tags: tags, - Aggregator: aggregator, - Emitter: emitter, + // Check to see if we have any GROUP BY dimensions that are not tags or time. + // If we have group-by key entries, create separate iterators for them. + // Otherwise, proceed as normal and create an iterator keyed by the tag ID. + if itr.opt.DimensionGrouper != nil && len(curr.Aux) > 0 { + entries, err := itr.opt.DimensionGrouper.ResolveKeys(curr.Aux, TagSubset{ID: id, HasTags: len(itr.dims) > 0}) + if err != nil { + return nil, err } - m[id] = rp + + for _, entry := range entries { + rp := m[entry.DimKey] + if rp == nil { + aggregator, emitter := itr.create() + rp = &{{$k.name}}Reduce{{$v.Name}}Point{ + Name: curr.Name, + Tags: tags, + GroupingKey: entry.EncodedKey, + Aggregator: aggregator, + Emitter: emitter, + } + m[entry.DimKey] = rp + } + rp.Aggregator.Aggregate{{$k.Name}}(curr) + } + } else { + rp := m[id] + if rp == nil { + aggregator, emitter := itr.create() + rp = &{{$k.name}}Reduce{{$v.Name}}Point{ + Name: curr.Name, + Tags: tags, + Aggregator: aggregator, + Emitter: emitter, + } + m[id] = rp + } + rp.Aggregator.Aggregate{{$k.Name}}(curr) } - rp.Aggregator.Aggregate{{$k.Name}}(curr) } keys := make([]string, 0, len(m)) @@ -1153,6 +1184,50 @@ func (itr *{{$k.name}}Reduce{{$v.Name}}Iterator) reduce() ([]{{$v.Name}}Point, e } else { sortedByTime = false } + + if itr.opt.DimensionGrouper != nil && rp.GroupingKey != "" { + dpVal, err := itr.opt.DimensionGrouper.DecodeEntry(rp.GroupingKey) + if err != nil { + return nil, err + } + // The emitted point must carry an Aux slot for every scanner aux + // key (len(itr.opt.Aux)). IteratorScanner.ScanAt ranges over the + // point's Aux, so a slice shorter than the key set leaves the + // trailing keys unvisited and the eval map retains stale values + // from a prior row, wrongly populating non-active date_part + // dimension columns. Selector functions (MIN, MAX, FIRST, LAST) + // already return a full-width Aux via cloneAux; aggregate + // functions (COUNT, SUM, MEAN) return an empty Aux, so grow it to + // full width here. (Keep any longer Aux as-is.) + width := len(itr.opt.Aux) + if width < len(points[i].Aux) { + width = len(points[i].Aux) + } + if width < 1 { + width = 1 + } + if len(points[i].Aux) < width { + aux := make([]interface{}, width) + copy(aux, points[i].Aux) + points[i].Aux = aux + } + // Only the active dimension is meaningful for this series, so null + // every date_part dimension slot (leaving non-active dimension + // columns null) and carry the active value as a DecodedDatePartKey + // in the last slot, which the scanner routes to the correct column + // by name. A stable, full-width slot ensures every key is visited + // and cleared on each scan. + n := len(points[i].Aux) + base := n - len(itr.opt.DatePartDimensions) + if base < 0 { + base = 0 + } + for j := base; j < n; j++ { + points[i].Aux[j] = nil + } + points[i].Aux[n-1] = dpVal + } + a = append(a, points[i]) } } @@ -1369,7 +1444,7 @@ func new{{$k.Name}}FilterIterator(input {{$k.Name}}Iterator, cond influxql.Expr, n := influxql.RewriteFunc(influxql.CloneExpr(cond), func(n influxql.Node) influxql.Node { switch n := n.(type) { case *influxql.BinaryExpr: - if n.LHS.String() == "time" { + if n.LHS.String() == models.TimeString { return &influxql.BooleanLiteral{Val: true} } } diff --git a/query/iterator.go b/query/iterator.go index 8897a45ecf2..798e8cfdb45 100644 --- a/query/iterator.go +++ b/query/iterator.go @@ -589,10 +589,12 @@ type IteratorOptions struct { Sources []influxql.Source // Group by interval and tags. - Interval Interval - Dimensions []string // The final dimensions of the query (stays the same even in subqueries). - GroupBy map[string]struct{} // Dimensions to group points by in intermediate iterators. - Location *time.Location + Interval Interval + Dimensions []string // The final dimensions of the query (stays the same even in subqueries). + DatePartDimensions []DatePartDimension + DimensionGrouper DimensionGrouper + GroupBy map[string]struct{} // Dimensions to group points by in intermediate iterators. + Location *time.Location // Fill options. Fill influxql.FillOption @@ -632,6 +634,11 @@ type IteratorOptions struct { // Authorizer can limit access to data Authorizer FineAuthorizer + + // NeedTimeRef indicates whether the condition contains functions (e.g. date_part) + // that require a reference to the point's timestamp. Cached here to avoid + // repeatedly walking the condition AST for every iterator creation. + NeedTimeRef bool } // newIteratorOptionsStmt creates the iterator options from stmt. @@ -682,9 +689,38 @@ func newIteratorOptionsStmt(stmt *influxql.SelectStatement, sopt SelectOptions) opt.Dimensions = append(opt.Dimensions, d.Val) opt.GroupBy[d.Val] = struct{}{} } + + if d, ok := d.Expr.(*influxql.Call); ok && d.Name == DatePartString { + // This should already be validated during compilation, but keep this code + // defensive to avoid panics if an invalid statement reaches this point. + expr, ok := matchDatePartCall(d) + if !ok { + return opt, fmt.Errorf("invalid date part expression: %s", d.String()) + } + // Skip a duplicate date_part dimension (e.g. GROUP BY date_part('year', + // time), date_part('year', time)); a repeated part would inject a + // duplicate output column and double-aggregate the same series. + duplicate := false + for _, existing := range opt.DatePartDimensions { + if existing.Expr == expr { + duplicate = true + break + } + } + if duplicate { + continue + } + opt.DatePartDimensions = append(opt.DatePartDimensions, DatePartDimension{Expr: expr}) + } + } + + if len(opt.DatePartDimensions) > 0 { + opt.DimensionGrouper = NewDatePartGrouper(opt.DatePartDimensions) } opt.Condition = condition + // date_part calls in the condition need access to the point's timestamp. + opt.NeedTimeRef = exprContainsDatePart(condition) opt.Ascending = stmt.TimeAscending() opt.Dedupe = stmt.Dedupe opt.StripName = stmt.StripName @@ -967,20 +1003,32 @@ func (opt *IteratorOptions) UnmarshalBinary(buf []byte) error { func encodeIteratorOptions(opt *IteratorOptions) *internal.IteratorOptions { pb := &internal.IteratorOptions{ - Interval: encodeInterval(opt.Interval), - Dimensions: opt.Dimensions, - Fill: proto.Int32(int32(opt.Fill)), - StartTime: proto.Int64(opt.StartTime), - EndTime: proto.Int64(opt.EndTime), - Ascending: proto.Bool(opt.Ascending), - Limit: proto.Int64(int64(opt.Limit)), - Offset: proto.Int64(int64(opt.Offset)), - SLimit: proto.Int64(int64(opt.SLimit)), - SOffset: proto.Int64(int64(opt.SOffset)), - StripName: proto.Bool(opt.StripName), - Dedupe: proto.Bool(opt.Dedupe), - MaxSeriesN: proto.Int64(int64(opt.MaxSeriesN)), - Ordered: proto.Bool(opt.Ordered), + Interval: encodeInterval(opt.Interval), + Dimensions: opt.Dimensions, + Fill: proto.Int32(int32(opt.Fill)), + StartTime: proto.Int64(opt.StartTime), + EndTime: proto.Int64(opt.EndTime), + Ascending: proto.Bool(opt.Ascending), + Limit: proto.Int64(int64(opt.Limit)), + Offset: proto.Int64(int64(opt.Offset)), + SLimit: proto.Int64(int64(opt.SLimit)), + SOffset: proto.Int64(int64(opt.SOffset)), + StripName: proto.Bool(opt.StripName), + Dedupe: proto.Bool(opt.Dedupe), + MaxSeriesN: proto.Int64(int64(opt.MaxSeriesN)), + Ordered: proto.Bool(opt.Ordered), + NeedTimeRef: proto.Bool(opt.NeedTimeRef), + } + + // Encode date_part GROUP BY dimensions. The DimensionGrouper is not encoded; + // it is reconstructed from these dimensions on decode. + if len(opt.DatePartDimensions) > 0 { + pb.DatePartDimensions = make([]*internal.DatePartDimension, len(opt.DatePartDimensions)) + for i, d := range opt.DatePartDimensions { + pb.DatePartDimensions[i] = &internal.DatePartDimension{ + Expr: proto.Int32(int32(d.Expr)), + } + } } // Set expression, if set. @@ -1037,20 +1085,32 @@ func encodeIteratorOptions(opt *IteratorOptions) *internal.IteratorOptions { func decodeIteratorOptions(pb *internal.IteratorOptions) (*IteratorOptions, error) { opt := &IteratorOptions{ - Interval: decodeInterval(pb.GetInterval()), - Dimensions: pb.GetDimensions(), - Fill: influxql.FillOption(pb.GetFill()), - StartTime: pb.GetStartTime(), - EndTime: pb.GetEndTime(), - Ascending: pb.GetAscending(), - Limit: int(pb.GetLimit()), - Offset: int(pb.GetOffset()), - SLimit: int(pb.GetSLimit()), - SOffset: int(pb.GetSOffset()), - StripName: pb.GetStripName(), - Dedupe: pb.GetDedupe(), - MaxSeriesN: int(pb.GetMaxSeriesN()), - Ordered: pb.GetOrdered(), + Interval: decodeInterval(pb.GetInterval()), + Dimensions: pb.GetDimensions(), + Fill: influxql.FillOption(pb.GetFill()), + StartTime: pb.GetStartTime(), + EndTime: pb.GetEndTime(), + Ascending: pb.GetAscending(), + Limit: int(pb.GetLimit()), + Offset: int(pb.GetOffset()), + SLimit: int(pb.GetSLimit()), + SOffset: int(pb.GetSOffset()), + StripName: pb.GetStripName(), + Dedupe: pb.GetDedupe(), + MaxSeriesN: int(pb.GetMaxSeriesN()), + Ordered: pb.GetOrdered(), + NeedTimeRef: pb.GetNeedTimeRef(), + } + + // Decode date_part GROUP BY dimensions and rebuild the grouper from them. + if dims := pb.GetDatePartDimensions(); len(dims) > 0 { + opt.DatePartDimensions = make([]DatePartDimension, len(dims)) + for i, d := range dims { + opt.DatePartDimensions[i] = DatePartDimension{ + Expr: DatePartExpr(d.GetExpr()), + } + } + opt.DimensionGrouper = NewDatePartGrouper(opt.DatePartDimensions) } // Set expression, if set. diff --git a/query/iterator_mapper.go b/query/iterator_mapper.go index 79675fa2c76..234740043b8 100644 --- a/query/iterator_mapper.go +++ b/query/iterator_mapper.go @@ -3,6 +3,7 @@ package query import ( "fmt" "math" + "time" "github.com/influxdata/influxql" ) @@ -34,6 +35,26 @@ type NullMap struct{} func (NullMap) Value(row *Row) interface{} { return nil } +// datePartMap computes a GROUP BY date_part dimension value from a row's +// timestamp. It is used when the source is a subquery: the date_part dimension +// is not a real field of the subquery, so its int64 aux value must be derived +// here from row.Time, mirroring what the TSM iterator appends for a measurement +// source. The result feeds the existing DimensionGrouper / reduce path unchanged. +type datePartMap struct { + expr DatePartExpr + loc *time.Location +} + +func (m datePartMap) Value(row *Row) interface{} { + v, ok := ExtractDatePartExpr(time.Unix(0, row.Time).In(LocationOrUTC(m.loc)), m.expr) + if !ok { + // Return nil rather than 0 so the grouper surfaces an explicit type + // error instead of silently grouping every row under value 0. + return nil + } + return v +} + func NewIteratorMapper(cur Cursor, driver IteratorMap, fields []IteratorMap, opt IteratorOptions) Iterator { if driver != nil { switch driver := driver.(type) { @@ -59,6 +80,11 @@ func NewIteratorMapper(cur Cursor, driver IteratorMap, fields []IteratorMap, opt } case TagMap: return newStringIteratorMapper(cur, driver, fields, opt) + case datePartMap: + // A driver named after an active GROUP BY date_part dimension (e.g. + // count(year) under GROUP BY date_part('year', time)). The computed + // date part is always an int64, so drive an integer iterator. + return newIntegerIteratorMapper(cur, driver, fields, opt) default: panic(fmt.Sprintf("unable to create iterator mapper with driver expression type: %T", driver)) } diff --git a/query/iterator_test.go b/query/iterator_test.go index e7def16061c..70628185355 100644 --- a/query/iterator_test.go +++ b/query/iterator_test.go @@ -14,6 +14,7 @@ import ( "github.com/influxdata/influxdb/pkg/testing/assert" "github.com/influxdata/influxdb/query" "github.com/influxdata/influxql" + "github.com/stretchr/testify/require" ) // Ensure that a set of iterators can be merged together, sorted by window and name/tag. @@ -1683,6 +1684,30 @@ func TestIteratorOptions_MarshalBinary(t *testing.T) { } } +// Ensure date_part GROUP BY options survive a marshal round-trip, including +// reconstruction of the DimensionGrouper (which is not itself serialized). +func TestIteratorOptions_MarshalBinary_DatePart(t *testing.T) { + opt := &query.IteratorOptions{ + DatePartDimensions: []query.DatePartDimension{ + {Expr: query.Year}, + {Expr: query.Month}, + }, + NeedTimeRef: true, + } + opt.DimensionGrouper = query.NewDatePartGrouper(opt.DatePartDimensions) + + buf, err := opt.MarshalBinary() + require.NoError(t, err) + + var other query.IteratorOptions + require.NoError(t, other.UnmarshalBinary(buf)) + + require.Equal(t, opt.DatePartDimensions, other.DatePartDimensions) + require.True(t, other.NeedTimeRef) + require.NotNil(t, other.DimensionGrouper, "grouper must be reconstructed on decode") + require.Equal(t, opt.DimensionGrouper, other.DimensionGrouper) +} + // Ensure iterator can be encoded and decoded over a byte stream. func TestIterator_EncodeDecode(t *testing.T) { var buf bytes.Buffer diff --git a/query/point.go b/query/point.go index f1518f33792..f01c330deb4 100644 --- a/query/point.go +++ b/query/point.go @@ -239,10 +239,20 @@ func decodeTags(id []byte) map[string]string { return m } +// auxDatePartKey marks an aux value as a DecodedDatePartKey in the iterator wire +// codec. It is intentionally outside the influxql.DataType range (which tops out at +// Unsigned = 9) so it can never collide with a real field type. The 9-byte encoded +// key (see encodeKey) is carried in the Aux StringValue. Without this, a date_part +// GROUP BY value streamed between data nodes would serialize to null and every +// bucket would collapse into a single group. +const auxDatePartKey influxql.DataType = 1000 + func encodeAux(aux []interface{}) []*internal.Aux { pb := make([]*internal.Aux, len(aux)) for i := range aux { switch v := aux[i].(type) { + case DecodedDatePartKey: + pb[i] = &internal.Aux{DataType: proto.Int32(int32(auxDatePartKey)), StringValue: proto.String(encodeKey(v.Expr, v.Val))} case float64: pb[i] = &internal.Aux{DataType: proto.Int32(int32(influxql.Float)), FloatValue: proto.Float64(v)} case *float64: @@ -278,6 +288,12 @@ func decodeAux(pb []*internal.Aux) []interface{} { aux := make([]interface{}, len(pb)) for i := range pb { switch influxql.DataType(pb[i].GetDataType()) { + case auxDatePartKey: + if pb[i].StringValue != nil { + if key, err := decodeKey(*pb[i].StringValue); err == nil { + aux[i] = key + } + } case influxql.Float: if pb[i].FloatValue != nil { aux[i] = *pb[i].FloatValue diff --git a/query/select.go b/query/select.go index 94c1f64fe6a..86256fb7413 100644 --- a/query/select.go +++ b/query/select.go @@ -9,6 +9,7 @@ import ( "sync" "time" + "github.com/influxdata/influxdb/models" "github.com/influxdata/influxdb/pkg/tracing" "github.com/influxdata/influxdb/query/internal/gota" "github.com/influxdata/influxql" @@ -643,7 +644,7 @@ func buildCursor(ctx context.Context, stmt *influxql.SelectStatement, ic Iterato // Add a field with the variable "time" if we have not omitted time. fields = append(fields, &influxql.Field{ Expr: &influxql.VarRef{ - Val: "time", + Val: models.TimeString, Type: influxql.Time, }, }) @@ -692,6 +693,16 @@ func buildCursor(ctx context.Context, stmt *influxql.SelectStatement, ic Iterato } } + // Add each date part dimension as an output column with a backing auxiliary + // field, in a single pass so a column can never be added without its aux slot. + for _, dim := range opt.DatePartDimensions { + name := dim.Expr.String() + ref := influxql.VarRef{Val: name, Type: influxql.Integer} + fields = append(fields, &influxql.Field{Expr: &ref, Alias: name}) + opt.Aux = append(opt.Aux, ref) + auxKeys = append(auxKeys, ref) + } + // If there are no calls, then produce an auxiliary cursor. if len(valueMapper.calls) == 0 { // If all of the auxiliary keys are of an unknown type, @@ -927,6 +938,18 @@ func (v *valueMapper) Visit(n influxql.Node) influxql.Visitor { if isMathFunction(n) { return v } + if n.Name == DatePartString { + // Rewrite the date_part time argument to the date_part_time + // reference, which is resolved from the evaluation map at scan time. + if _, ok := matchDatePartCall(n); ok { + timeRef := n.Args[1].(*influxql.VarRef) + v.table[timeRef] = influxql.VarRef{ + Val: DatePartTimeString, + Type: influxql.Time, + } + } + return nil + } v.calls[n] = struct{}{} case *influxql.VarRef: v.refs[n] = struct{}{} diff --git a/query/subquery.go b/query/subquery.go index 2bb0b25a70d..d5323621fc0 100644 --- a/query/subquery.go +++ b/query/subquery.go @@ -14,7 +14,7 @@ type subqueryBuilder struct { // buildAuxIterator constructs an auxiliary Iterator from a subquery. func (b *subqueryBuilder) buildAuxIterator(ctx context.Context, opt IteratorOptions) (Iterator, error) { // Map the desired auxiliary fields from the substatement. - indexes := b.mapAuxFields(opt.Aux) + indexes := b.mapAuxFields(opt.Aux, opt) subOpt, err := newIteratorOptionsSubstatement(ctx, b.stmt, opt) if err != nil { @@ -28,7 +28,7 @@ func (b *subqueryBuilder) buildAuxIterator(ctx context.Context, opt IteratorOpti // Filter the cursor by a condition if one was given. if opt.Condition != nil { - cur = newFilterCursor(cur, opt.Condition) + cur = newFilterCursor(cur, opt.Condition, opt.NeedTimeRef, opt.Location) } // Construct the iterators for the subquery. @@ -39,10 +39,10 @@ func (b *subqueryBuilder) buildAuxIterator(ctx context.Context, opt IteratorOpti return itr, nil } -func (b *subqueryBuilder) mapAuxFields(auxFields []influxql.VarRef) []IteratorMap { +func (b *subqueryBuilder) mapAuxFields(auxFields []influxql.VarRef, opt IteratorOptions) []IteratorMap { indexes := make([]IteratorMap, len(auxFields)) for i, name := range auxFields { - m := b.mapAuxField(&name) + m := b.mapAuxField(&name, opt) if m == nil { // If this field doesn't map to anything, use the NullMap so it // shows up as null. @@ -53,7 +53,19 @@ func (b *subqueryBuilder) mapAuxFields(auxFields []influxql.VarRef) []IteratorMa return indexes } -func (b *subqueryBuilder) mapAuxField(name *influxql.VarRef) IteratorMap { +func (b *subqueryBuilder) mapAuxField(name *influxql.VarRef, opt IteratorOptions) IteratorMap { + // A GROUP BY date_part dimension is not a real field of the subquery; its + // value is computed from the row timestamp via datePartMap. This is checked + // before the field/tag lookups so a stored field coincidentally named after a + // date part (e.g. "hour") does not shadow the grouping driver — matching the + // measurement-source path, where date_part dimensions are always derived from + // time. Gated on DatePartDimensions so non-date_part subqueries are unaffected. + for _, d := range opt.DatePartDimensions { + if d.Expr.String() == name.Val { + return datePartMap{expr: d.Expr, loc: opt.Location} + } + } + offset := 0 for i, f := range b.stmt.Fields { if f.Name() == name.Val { @@ -93,7 +105,7 @@ func (b *subqueryBuilder) mapAuxField(name *influxql.VarRef) IteratorMap { func (b *subqueryBuilder) buildVarRefIterator(ctx context.Context, expr *influxql.VarRef, opt IteratorOptions) (Iterator, error) { // Look for the field or tag that is driving this query. - driver := b.mapAuxField(expr) + driver := b.mapAuxField(expr, opt) if driver == nil { // Exit immediately if there is no driver. If there is no driver, there // are no results. Period. @@ -101,7 +113,7 @@ func (b *subqueryBuilder) buildVarRefIterator(ctx context.Context, expr *influxq } // Map the auxiliary fields to their index in the subquery. - indexes := b.mapAuxFields(opt.Aux) + indexes := b.mapAuxFields(opt.Aux, opt) subOpt, err := newIteratorOptionsSubstatement(ctx, b.stmt, opt) if err != nil { return nil, err @@ -114,7 +126,7 @@ func (b *subqueryBuilder) buildVarRefIterator(ctx context.Context, expr *influxq // Filter the cursor by a condition if one was given. if opt.Condition != nil { - cur = newFilterCursor(cur, opt.Condition) + cur = newFilterCursor(cur, opt.Condition, opt.NeedTimeRef, opt.Location) } // Construct the iterators for the subquery. diff --git a/query/subquery_test.go b/query/subquery_test.go index 14b6ef1d424..829a104d2f3 100644 --- a/query/subquery_test.go +++ b/query/subquery_test.go @@ -364,6 +364,31 @@ func TestSubquery(t *testing.T) { {Time: mustParseTime("2019-06-25T22:36:15.144253616Z").UnixNano(), Series: query.Series{Name: "testing"}, Values: []interface{}{float64(2), "a"}}, }, }, + { + // The aggregate argument "year" names an active GROUP BY date_part + // dimension, so it is driven by datePartMap (computed from the row + // timestamp) rather than the subquery's field of the same name. + // This must produce grouped counts, not a panic in NewIteratorMapper. + Name: "GroupByDatePart_AggregateArgNamedAfterDatePart", + Statement: `SELECT count(year) FROM (SELECT year FROM cpu) WHERE time >= '1970-01-01T00:00:00Z' AND time < '1972-01-01T00:00:00Z' GROUP BY date_part('year', time)`, + Fields: map[string]influxql.DataType{"year": influxql.Float}, + MapShardsFn: func(t *testing.T, tr influxql.TimeRange) CreateIteratorFn { + return func(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions) query.Iterator { + if got, want := m.Name, "cpu"; got != want { + t.Errorf("unexpected source: got=%s want=%s", got, want) + } + return &FloatIterator{Points: []query.FloatPoint{ + {Name: "cpu", Time: mustParseTime("1970-03-01T00:00:00Z").UnixNano(), Value: 1, Aux: []interface{}{float64(1)}}, + {Name: "cpu", Time: mustParseTime("1970-09-01T00:00:00Z").UnixNano(), Value: 2, Aux: []interface{}{float64(2)}}, + {Name: "cpu", Time: mustParseTime("1971-06-01T00:00:00Z").UnixNano(), Value: 3, Aux: []interface{}{float64(3)}}, + }} + } + }, + Rows: []query.Row{ + {Time: 0, Series: query.Series{Name: "cpu"}, Values: []interface{}{int64(2), int64(1970)}, GroupingKeys: map[string]struct{}{"year": {}}}, + {Time: 0, Series: query.Series{Name: "cpu"}, Values: []interface{}{int64(1), int64(1971)}, GroupingKeys: map[string]struct{}{"year": {}}}, + }, + }, } { t.Run(test.Name, func(t *testing.T) { shardMapper := ShardMapper{ diff --git a/tests/server_test.go b/tests/server_test.go index 68830f9c72c..52b1c64ea45 100644 --- a/tests/server_test.go +++ b/tests/server_test.go @@ -8076,6 +8076,1396 @@ func TestServer_Query_ShowMeasurementExactCardinality(t *testing.T) { } } +func TestServer_Query_DatePart(t *testing.T) { + t.Parallel() + s := OpenServer(NewConfig()) + defer s.Close() + + if err := s.CreateDatabaseAndRetentionPolicy("db0", NewRetentionPolicySpec("rp0", 1, 0, 0, 0), true); err != nil { + t.Fatal(err) + } + + writes := []string{ + // 2023 - First day of year, Sunday, Q1 + fmt.Sprintf(`cpu,host=server01 value=1 %d`, mustParseTime(time.RFC3339Nano, "2023-01-01T00:00:00Z").UnixNano()), + // 2023 - Monday in Q1, week 3 + fmt.Sprintf(`cpu,host=server01 value=2 %d`, mustParseTime(time.RFC3339Nano, "2023-01-16T10:30:45Z").UnixNano()), + // 2023 - Saturday in Q2 + fmt.Sprintf(`cpu,host=server01 value=3 %d`, mustParseTime(time.RFC3339Nano, "2023-04-15T14:20:30Z").UnixNano()), + // 2023 - Wednesday in Q3 + fmt.Sprintf(`cpu,host=server01 value=4 %d`, mustParseTime(time.RFC3339Nano, "2023-07-19T08:15:22Z").UnixNano()), + // 2023 - Friday in Q4 + fmt.Sprintf(`cpu,host=server01 value=5 %d`, mustParseTime(time.RFC3339Nano, "2023-10-27T16:45:10Z").UnixNano()), + // 2023 - Last day of year, Sunday + fmt.Sprintf(`cpu,host=server01 value=6 %d`, mustParseTime(time.RFC3339Nano, "2023-12-31T23:59:59Z").UnixNano()), + // 2024 - First day of year, Monday, Q1 + fmt.Sprintf(`cpu,host=server02 value=7 %d`, mustParseTime(time.RFC3339Nano, "2024-01-01T00:00:00Z").UnixNano()), + // 2024 - Leap year day (Feb 29), Thursday + fmt.Sprintf(`cpu,host=server02 value=8 %d`, mustParseTime(time.RFC3339Nano, "2024-02-29T12:00:00Z").UnixNano()), + // 2024 - Sunday in Q2 + fmt.Sprintf(`cpu,host=server02 value=9 %d`, mustParseTime(time.RFC3339Nano, "2024-05-19T06:30:15Z").UnixNano()), + // 2024 - Tuesday in Q3 + fmt.Sprintf(`cpu,host=server02 value=10 %d`, mustParseTime(time.RFC3339Nano, "2024-08-06T18:45:00Z").UnixNano()), + // 2024 - Saturday in Q4 + fmt.Sprintf(`cpu,host=server02 value=11 %d`, mustParseTime(time.RFC3339Nano, "2024-11-23T22:10:55Z").UnixNano()), + // 2024 - Last day of year, Tuesday + fmt.Sprintf(`cpu,host=server02 value=12 %d`, mustParseTime(time.RFC3339Nano, "2024-12-31T23:59:59Z").UnixNano()), + // 2025 - First day of year, Wednesday, Q1 + fmt.Sprintf(`cpu,host=server03 value=13 %d`, mustParseTime(time.RFC3339Nano, "2025-01-01T00:00:00Z").UnixNano()), + // 2025 - Thursday in Q2 + fmt.Sprintf(`cpu,host=server03 value=14 %d`, mustParseTime(time.RFC3339Nano, "2025-06-12T11:20:30Z").UnixNano()), + // 2025 - Different times of day for same date + fmt.Sprintf(`cpu,host=server03 value=15 %d`, mustParseTime(time.RFC3339Nano, "2025-09-15T00:00:00Z").UnixNano()), // Midnight + fmt.Sprintf(`cpu,host=server03 value=16 %d`, mustParseTime(time.RFC3339Nano, "2025-09-15T06:00:00Z").UnixNano()), // 6 AM + fmt.Sprintf(`cpu,host=server03 value=17 %d`, mustParseTime(time.RFC3339Nano, "2025-09-15T12:00:00Z").UnixNano()), // Noon + fmt.Sprintf(`cpu,host=server03 value=18 %d`, mustParseTime(time.RFC3339Nano, "2025-09-15T18:00:00Z").UnixNano()), // 6 PM + fmt.Sprintf(`cpu,host=server03 value=19 %d`, mustParseTime(time.RFC3339Nano, "2025-09-15T23:59:59Z").UnixNano()), // End of day + } + + test := NewTest("db0", "rp0") + test.writes = Writes{ + &Write{data: strings.Join(writes, "\n")}, + } + + test.addQueries([]*Query{ + &Query{ + name: `query for weekend data using dow`, + command: `SELECT * FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2025-12-31T23:59:59Z' AND (date_part('dow', time) != 0 AND date_part('dow', time) != 6)`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","host","value"],"values":[` + + `["2023-01-16T10:30:45Z","server01",2],` + // Monday + `["2023-07-19T08:15:22Z","server01",4],` + // Wednesday + `["2023-10-27T16:45:10Z","server01",5],` + // Friday + `["2024-01-01T00:00:00Z","server02",7],` + // Monday + `["2024-02-29T12:00:00Z","server02",8],` + // Thursday + `["2024-08-06T18:45:00Z","server02",10],` + // Tuesday + `["2024-12-31T23:59:59Z","server02",12],` + // Tuesday + `["2025-01-01T00:00:00Z","server03",13],` + // Wednesday + `["2025-06-12T11:20:30Z","server03",14],` + // Thursday + `["2025-09-15T00:00:00Z","server03",15],` + // Monday (midnight) + `["2025-09-15T06:00:00Z","server03",16],` + // Monday (6am) + `["2025-09-15T12:00:00Z","server03",17],` + // Monday (noon) + `["2025-09-15T18:00:00Z","server03",18],` + // Monday (6pm) + `["2025-09-15T23:59:59Z","server03",19]` + // Monday (end of day) + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `filter by year 2024`, + command: `SELECT * FROM db0.rp0.cpu WHERE date_part('year', time) = 2024`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","host","value"],"values":[` + + `["2024-01-01T00:00:00Z","server02",7],` + + `["2024-02-29T12:00:00Z","server02",8],` + + `["2024-05-19T06:30:15Z","server02",9],` + + `["2024-08-06T18:45:00Z","server02",10],` + + `["2024-11-23T22:10:55Z","server02",11],` + + `["2024-12-31T23:59:59Z","server02",12]` + + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `filter by Q2 (quarter 2)`, + command: `SELECT * FROM db0.rp0.cpu WHERE date_part('quarter', time) = 2`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","host","value"],"values":[` + + `["2023-04-15T14:20:30Z","server01",3],` + + `["2024-05-19T06:30:15Z","server02",9],` + + `["2025-06-12T11:20:30Z","server03",14]` + + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `filter by January (month 1)`, + command: `SELECT * FROM db0.rp0.cpu WHERE date_part('month', time) = 1`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","host","value"],"values":[` + + `["2023-01-01T00:00:00Z","server01",1],` + + `["2023-01-16T10:30:45Z","server01",2],` + + `["2024-01-01T00:00:00Z","server02",7],` + + `["2025-01-01T00:00:00Z","server03",13]` + + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `filter by first day of month`, + command: `SELECT * FROM db0.rp0.cpu WHERE date_part('day', time) = 1`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","host","value"],"values":[` + + `["2023-01-01T00:00:00Z","server01",1],` + + `["2024-01-01T00:00:00Z","server02",7],` + + `["2025-01-01T00:00:00Z","server03",13]` + + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `filter by business hours (9-17)`, + command: `SELECT * FROM db0.rp0.cpu WHERE date_part('hour', time) >= 9 AND date_part('hour', time) < 17`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","host","value"],"values":[` + + `["2023-01-16T10:30:45Z","server01",2],` + + `["2023-04-15T14:20:30Z","server01",3],` + + `["2023-10-27T16:45:10Z","server01",5],` + + `["2024-02-29T12:00:00Z","server02",8],` + + `["2025-06-12T11:20:30Z","server03",14],` + + `["2025-09-15T12:00:00Z","server03",17]` + + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `filter by first day of year using doy`, + command: `SELECT * FROM db0.rp0.cpu WHERE date_part('doy', time) = 1`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","host","value"],"values":[` + + `["2023-01-01T00:00:00Z","server01",1],` + + `["2024-01-01T00:00:00Z","server02",7],` + + `["2025-01-01T00:00:00Z","server03",13]` + + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `filter by last day of year using doy`, + command: `SELECT * FROM db0.rp0.cpu WHERE date_part('doy', time) >= 365`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","host","value"],"values":[` + + `["2023-12-31T23:59:59Z","server01",6],` + + `["2024-12-31T23:59:59Z","server02",12]` + + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `filter weekends using isodow (Saturday=6, Sunday=7)`, + command: `SELECT * FROM db0.rp0.cpu WHERE date_part('isodow', time) >= 6`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","host","value"],"values":[` + + `["2023-01-01T00:00:00Z","server01",1],` + // Sunday + `["2023-04-15T14:20:30Z","server01",3],` + // Saturday + `["2023-12-31T23:59:59Z","server01",6],` + // Sunday + `["2024-05-19T06:30:15Z","server02",9],` + // Sunday + `["2024-11-23T22:10:55Z","server02",11]` + // Saturday + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `filter using epoch greater than timestamp`, + command: `SELECT * FROM db0.rp0.cpu WHERE date_part('epoch', time) > 1735689599 LIMIT 3`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","host","value"],"values":[` + + `["2025-01-01T00:00:00Z","server03",13],` + + `["2025-06-12T11:20:30Z","server03",14],` + + `["2025-09-15T00:00:00Z","server03",15]` + + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `combine multiple date parts in WHERE clause`, + command: `SELECT * FROM db0.rp0.cpu WHERE date_part('year', time) = 2023 AND date_part('month', time) = 1 AND date_part('day', time) = 16`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","host","value"],"values":[` + + `["2023-01-16T10:30:45Z","server01",2]` + + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + // ISO week (date_part('week')) follows ISOWeek(), so 2023-01-01 (a Sunday) + // belongs to week 52 of the prior year and 2023-12-31 (a Sunday) belongs to + // week 52 of 2023 — both match week 52 while no other point does. + name: `filter by ISO week 52`, + command: `SELECT * FROM db0.rp0.cpu WHERE date_part('week', time) = 52`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","host","value"],"values":[` + + `["2023-01-01T00:00:00Z","server01",1],` + + `["2023-12-31T23:59:59Z","server01",6]` + + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + // A SELECT whose only fields are date_part(...) expressions references no + // stored field (date_part derives purely from the row timestamp), so there + // is nothing to anchor the scan on. This is rejected rather than silently + // returning no data, mirroring SELECT time. + &Query{ + name: `SELECT only date_part is rejected`, + command: `SELECT date_part('year', time) FROM db0.rp0.cpu WHERE host = 'server01'`, + exp: `{"results":[{"statement_id":0,"error":"at least 1 non-time field must be queried"}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `SELECT only multiple date_part is rejected`, + command: `SELECT date_part('year', time), date_part('month', time) FROM db0.rp0.cpu WHERE host = 'server01'`, + exp: `{"results":[{"statement_id":0,"error":"at least 1 non-time field must be queried"}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + // A tag is not a scan anchor, so date_part paired only with a tag must be + // rejected too (the field-vs-tag distinction is only known after the schema + // is resolved). Otherwise the query plans aux-only and returns no rows. + &Query{ + name: `SELECT date_part with only a tag is rejected`, + command: `SELECT host, date_part('year', time) FROM db0.rp0.cpu`, + exp: `{"results":[{"statement_id":0,"error":"at least 1 non-time field must be queried"}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `SELECT date_part first with only a tag is rejected`, + command: `SELECT date_part('year', time), host FROM db0.rp0.cpu`, + exp: `{"results":[{"statement_id":0,"error":"at least 1 non-time field must be queried"}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + // SELECT statement tests - date_part as a column + &Query{ + name: `SELECT date_part dow as column`, + command: `SELECT value, date_part('dow', time) FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2023-01-16T10:30:45Z'`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","value","date_part"],"values":[` + + `["2023-01-01T00:00:00Z",1,0],` + // Sunday + `["2023-01-16T10:30:45Z",2,1]` + // Monday + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `SELECT date_part with alias`, + command: `SELECT value, date_part('dow', time) AS day_of_week FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2023-01-16T10:30:45Z'`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","value","day_of_week"],"values":[` + + `["2023-01-01T00:00:00Z",1,0],` + + `["2023-01-16T10:30:45Z",2,1]` + + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `SELECT multiple date parts`, + command: `SELECT value, date_part('year', time) AS year, date_part('month', time) AS month, date_part('day', time) AS day FROM db0.rp0.cpu WHERE time = '2024-02-29T12:00:00Z'`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","value","year","month","day"],"values":[` + + `["2024-02-29T12:00:00Z",8,2024,2,29]` + // Leap year day + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `SELECT date_part with field`, + command: `SELECT value, date_part('dow', time) AS dow FROM db0.rp0.cpu WHERE host = 'server01' ORDER BY time LIMIT 3`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","value","dow"],"values":[` + + `["2023-01-01T00:00:00Z",1,0],` + // Sunday + `["2023-01-16T10:30:45Z",2,1],` + // Monday + `["2023-04-15T14:20:30Z",3,6]` + // Saturday + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `SELECT date_part hour and minute`, + command: `SELECT value, date_part('hour', time) AS hour, date_part('minute', time) AS minute FROM db0.rp0.cpu WHERE time = '2023-01-16T10:30:45Z'`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","value","hour","minute"],"values":[` + + `["2023-01-16T10:30:45Z",2,10,30]` + + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `SELECT date_part year and quarter`, + command: `SELECT value, date_part('year', time) AS year, date_part('quarter', time) AS quarter FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2023-12-31T23:59:59Z' ORDER BY time`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","value","year","quarter"],"values":[` + + `["2023-01-01T00:00:00Z",1,2023,1],` + + `["2023-01-16T10:30:45Z",2,2023,1],` + + `["2023-04-15T14:20:30Z",3,2023,2],` + + `["2023-07-19T08:15:22Z",4,2023,3],` + + `["2023-10-27T16:45:10Z",5,2023,4],` + + `["2023-12-31T23:59:59Z",6,2023,4]` + + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `SELECT date_part doy (day of year)`, + command: `SELECT value, date_part('doy', time) AS day_of_year FROM db0.rp0.cpu WHERE (date_part('doy', time) = 1 OR date_part('doy', time) = 365) AND time >= '2023-01-01T00:00:00Z' AND time <= '2023-12-31T23:59:59Z' ORDER BY time`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","value","day_of_year"],"values":[` + + `["2023-01-01T00:00:00Z",1,1],` + + `["2023-12-31T23:59:59Z",6,365]` + + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `SELECT date_part isodow`, + command: `SELECT value, date_part('isodow', time) AS iso_day FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2023-01-16T10:30:45Z'`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","value","iso_day"],"values":[` + + `["2023-01-01T00:00:00Z",1,7],` + // Sunday = 7 in ISO 8601 + `["2023-01-16T10:30:45Z",2,1]` + // Monday = 1 in ISO 8601 + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `SELECT date_part epoch`, + command: `SELECT value, date_part('epoch', time) AS epoch FROM db0.rp0.cpu WHERE time = '2024-01-01T00:00:00Z'`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","value","epoch"],"values":[` + + `["2024-01-01T00:00:00Z",7,1704067200]` + + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + // date_part('week') uses ISOWeek(), which disagrees with the calendar + // year at boundaries: 2023-01-01 reports week 52 (of 2022) while the very + // next stored point, 2023-01-16, is week 3. + name: `SELECT date_part week (ISO week-year boundary)`, + command: `SELECT value, date_part('week', time) AS week FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2023-01-16T10:30:45Z'`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","value","week"],"values":[` + + `["2023-01-01T00:00:00Z",1,52],` + + `["2023-01-16T10:30:45Z",2,3]` + + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + // date_part nested inside an expression (not the top-level field) must + // still receive the row timestamp. hour=10 at this point, so + // date_part('hour', time) + 1 = 11. + name: `SELECT date_part nested in arithmetic`, + command: `SELECT value, date_part('hour', time) + 1 AS hour_plus FROM db0.rp0.cpu WHERE time = '2023-01-16T10:30:45Z'`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","value","hour_plus"],"values":[["2023-01-16T10:30:45Z",2,11]]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + // GROUP BY date_part tests + &Query{ + name: `GROUP BY year with COUNT`, + command: `SELECT COUNT(value) FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2025-12-31T23:59:59Z' GROUP BY date_part('year', time)`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","grouping_keys":["year"],"columns":["time","count","year"],"values":[` + + `["2023-01-01T00:00:00Z",6,2023],` + // 2023 has 6 data points + `["2023-01-01T00:00:00Z",6,2024],` + // 2024 has 6 data points + `["2023-01-01T00:00:00Z",7,2025]` + // 2025 has 7 data points + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + // A repeated date_part dimension is deduplicated: the result must match + // single-dimension grouping (one year series, counts not doubled). + name: `GROUP BY duplicate year dimension is deduplicated`, + command: `SELECT COUNT(value) FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2025-12-31T23:59:59Z' GROUP BY date_part('year', time), date_part('year', time)`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","grouping_keys":["year"],"columns":["time","count","year"],"values":[` + + `["2023-01-01T00:00:00Z",6,2023],` + + `["2023-01-01T00:00:00Z",6,2024],` + + `["2023-01-01T00:00:00Z",7,2025]` + + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `GROUP BY quarter with SUM`, + command: `SELECT SUM(value) FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2025-12-31T23:59:59Z' GROUP BY date_part('quarter', time)`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","grouping_keys":["quarter"],"columns":["time","sum","quarter"],"values":[` + + `["2023-01-01T00:00:00Z",31,1],` + // Q1: values 1,2,7,8,13 = 31 + `["2023-01-01T00:00:00Z",26,2],` + // Q2: values 3,9,14 = 26 + `["2023-01-01T00:00:00Z",99,3],` + // Q3: values 4,10,15,16,17,18,19 = 99 + `["2023-01-01T00:00:00Z",34,4]` + // Q4: values 5,6,11,12 = 34 + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `GROUP BY month with MEAN`, + command: `SELECT MEAN(value) FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2025-12-31T23:59:59Z' GROUP BY date_part('month', time) ORDER BY time`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","grouping_keys":["month"],"columns":["time","mean","month"],"values":[` + + `["2023-01-01T00:00:00Z",5.75,1],` + // January: (1+2+7+13)/4 = 5.75 + `["2023-01-01T00:00:00Z",8,2],` + // February: 8/1 = 8 + `["2023-01-01T00:00:00Z",3,4],` + // April: 3/1 = 3 + `["2023-01-01T00:00:00Z",9,5],` + // May: 9/1 = 9 + `["2023-01-01T00:00:00Z",14,6],` + // June: 14/1 = 14 + `["2023-01-01T00:00:00Z",4,7],` + // July: 4/1 = 4 + `["2023-01-01T00:00:00Z",10,8],` + // August: 10/1 = 10 + `["2023-01-01T00:00:00Z",17,9],` + // September: (15+16+17+18+19)/5 = 17 + `["2023-01-01T00:00:00Z",5,10],` + // October: 5/1 = 5 + `["2023-01-01T00:00:00Z",11,11],` + // November: 11/1 = 11 + `["2023-01-01T00:00:00Z",9,12]` + // December: (6+12)/2 = 9 + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `GROUP BY dow (day of week) with COUNT`, + command: `SELECT COUNT(value) FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2025-12-31T23:59:59Z' GROUP BY date_part('dow', time)`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","grouping_keys":["dow"],"columns":["time","count","dow"],"values":[` + + `["2023-01-01T00:00:00Z",3,0],` + // Sunday (dow=0): values 1,6,9 = 3 points + `["2023-01-01T00:00:00Z",7,1],` + // Monday (dow=1): values 2,7,15,16,17,18,19 = 7 points + `["2023-01-01T00:00:00Z",2,2],` + // Tuesday (dow=2): values 10,12 = 2 points + `["2023-01-01T00:00:00Z",2,3],` + // Wednesday (dow=3): values 4,13 = 2 points + `["2023-01-01T00:00:00Z",2,4],` + // Thursday (dow=4): values 8,14 = 2 points + `["2023-01-01T00:00:00Z",1,5],` + // Friday (dow=5): value 5 = 1 point + `["2023-01-01T00:00:00Z",2,6]` + // Saturday (dow=6): values 3,11 = 2 points + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + // part names are case-insensitive, so GROUP BY date_part('DOW', ...) + // must normalize to the canonical "dow" and populate the grouped column + // identically to the lowercase form. + name: `GROUP BY DOW uppercase normalizes to dow`, + command: `SELECT COUNT(value) FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2025-12-31T23:59:59Z' GROUP BY date_part('DOW', time)`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","grouping_keys":["dow"],"columns":["time","count","dow"],"values":[` + + `["2023-01-01T00:00:00Z",3,0],` + + `["2023-01-01T00:00:00Z",7,1],` + + `["2023-01-01T00:00:00Z",2,2],` + + `["2023-01-01T00:00:00Z",2,3],` + + `["2023-01-01T00:00:00Z",2,4],` + + `["2023-01-01T00:00:00Z",1,5],` + + `["2023-01-01T00:00:00Z",2,6]` + + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `GROUP BY hour with MAX`, + command: `SELECT MAX(value) FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2025-12-31T23:59:59Z' GROUP BY date_part('hour', time)`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","grouping_keys":["hour"],"columns":["time","max","hour"],"values":[` + + `["2023-01-16T10:30:45Z",2,10],` + + `["2023-04-15T14:20:30Z",3,14],` + + `["2023-07-19T08:15:22Z",4,8],` + + `["2023-10-27T16:45:10Z",5,16],` + + `["2024-11-23T22:10:55Z",11,22],` + + `["2025-06-12T11:20:30Z",14,11],` + + `["2025-09-15T00:00:00Z",15,0],` + + `["2025-09-15T06:00:00Z",16,6],` + + `["2025-09-15T12:00:00Z",17,12],` + + `["2025-09-15T18:00:00Z",18,18],` + + `["2025-09-15T23:59:59Z",19,23]` + + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `GROUP BY year and month with COUNT`, + command: `SELECT COUNT(value) FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2025-12-31T23:59:59Z' GROUP BY date_part('year', time), date_part('month', time)`, + // Each date_part dimension produces its own series with null for the + // non-active dimension column. + exp: `{"results":[{"statement_id":0,"series":[` + + // month dimension + `{"name":"cpu","grouping_keys":["month"],"columns":["time","count","year","month"],"values":[` + + `["2023-01-01T00:00:00Z",4,null,1],` + // Jan: 4 points + `["2023-01-01T00:00:00Z",1,null,2],` + // Feb: 1 point + `["2023-01-01T00:00:00Z",1,null,4],` + // Apr: 1 point + `["2023-01-01T00:00:00Z",1,null,5],` + // May: 1 point + `["2023-01-01T00:00:00Z",1,null,6],` + // Jun: 1 point + `["2023-01-01T00:00:00Z",1,null,7],` + // Jul: 1 point + `["2023-01-01T00:00:00Z",1,null,8],` + // Aug: 1 point + `["2023-01-01T00:00:00Z",5,null,9],` + // Sep: 5 points + `["2023-01-01T00:00:00Z",1,null,10],` + // Oct: 1 point + `["2023-01-01T00:00:00Z",1,null,11],` + // Nov: 1 point + `["2023-01-01T00:00:00Z",2,null,12]` + // Dec: 2 points + `]},` + + // year dimension + `{"name":"cpu","grouping_keys":["year"],"columns":["time","count","year","month"],"values":[` + + `["2023-01-01T00:00:00Z",6,2023,null],` + // 2023: 6 points + `["2023-01-01T00:00:00Z",6,2024,null],` + // 2024: 6 points + `["2023-01-01T00:00:00Z",7,2025,null]` + // 2025: 7 points + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + // date_part nested in an expression under GROUP BY date_part must use + // the grouped value, not the bucket timestamp: year+1 per group. + name: `SELECT date_part nested in expression under GROUP BY date_part`, + command: `SELECT COUNT(value), date_part('year', time) + 1 AS yp FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2025-12-31T23:59:59Z' GROUP BY date_part('year', time)`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","grouping_keys":["year"],"columns":["time","count","yp","year"],"values":[["2023-01-01T00:00:00Z",6,2024,2023],["2023-01-01T00:00:00Z",6,2025,2024],["2023-01-01T00:00:00Z",7,2026,2025]]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + // An explicit SELECT date_part('month') under GROUP BY year, month is + // well-defined only on the month series; on the year series it is a + // different (non-active) grouping dimension, so it must be null rather + // than a misleading constant from the bucket timestamp. + name: `SELECT non-active date_part is null under multi-dimension GROUP BY`, + command: `SELECT COUNT(value), date_part('month', time) FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2025-12-31T23:59:59Z' GROUP BY date_part('year', time), date_part('month', time)`, + exp: `{"results":[{"statement_id":0,"series":[` + + // month series: the active dimension, so date_part = month value + `{"name":"cpu","grouping_keys":["month"],"columns":["time","count","date_part","year","month"],"values":[` + + `["2023-01-01T00:00:00Z",4,1,null,1],` + + `["2023-01-01T00:00:00Z",1,2,null,2],` + + `["2023-01-01T00:00:00Z",1,4,null,4],` + + `["2023-01-01T00:00:00Z",1,5,null,5],` + + `["2023-01-01T00:00:00Z",1,6,null,6],` + + `["2023-01-01T00:00:00Z",1,7,null,7],` + + `["2023-01-01T00:00:00Z",1,8,null,8],` + + `["2023-01-01T00:00:00Z",5,9,null,9],` + + `["2023-01-01T00:00:00Z",1,10,null,10],` + + `["2023-01-01T00:00:00Z",1,11,null,11],` + + `["2023-01-01T00:00:00Z",2,12,null,12]` + + `]},` + + // year series: date_part('month') is non-active here, so it is null + `{"name":"cpu","grouping_keys":["year"],"columns":["time","count","date_part","year","month"],"values":[` + + `["2023-01-01T00:00:00Z",6,null,2023,null],` + + `["2023-01-01T00:00:00Z",6,null,2024,null],` + + `["2023-01-01T00:00:00Z",7,null,2025,null]` + + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `GROUP BY dow with WHERE and SUM`, + command: `SELECT SUM(value) FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2025-12-31T23:59:59Z' AND date_part('dow', time) >= 1 AND date_part('dow', time) <= 5 GROUP BY date_part('dow', time)`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","grouping_keys":["dow"],"columns":["time","sum","dow"],"values":[` + + `["2023-01-01T00:00:00Z",94,1],` + // Monday: 2+7+15+16+17+18+19 = 94 + `["2023-01-01T00:00:00Z",22,2],` + // Tuesday: 10+12 = 22 + `["2023-01-01T00:00:00Z",17,3],` + // Wednesday: 4+13 = 17 + `["2023-01-01T00:00:00Z",22,4],` + // Thursday: 8+14 = 22 + `["2023-01-01T00:00:00Z",5,5]` + // Friday: 5 + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `GROUP BY day with MIN`, + command: `SELECT MIN(value) FROM db0.rp0.cpu WHERE time >= '2025-09-15T00:00:00Z' AND time <= '2025-09-15T23:59:59Z' GROUP BY date_part('day', time)`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","grouping_keys":["day"],"columns":["time","min","day"],"values":[["2025-09-15T00:00:00Z",15,15]]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `GROUP BY year with FIRST`, + command: `SELECT FIRST(value) FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2025-12-31T23:59:59Z' GROUP BY date_part('year', time)`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","grouping_keys":["year"],"columns":["time","first","year"],"values":[` + + `["2023-01-01T00:00:00Z",1,2023],` + + `["2024-01-01T00:00:00Z",7,2024],` + + `["2025-01-01T00:00:00Z",13,2025]` + + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `GROUP BY year with LAST`, + command: `SELECT LAST(value) FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2025-12-31T23:59:59Z' GROUP BY date_part('year', time)`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","grouping_keys":["year"],"columns":["time","last","year"],"values":[` + + `["2023-12-31T23:59:59Z",6,2023],` + + `["2024-12-31T23:59:59Z",12,2024],` + + `["2025-09-15T23:59:59Z",19,2025]` + + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `GROUP BY year and month with MAX`, + command: `SELECT MAX(value) FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2025-12-31T23:59:59Z' GROUP BY date_part('year', time), date_part('month', time)`, + exp: `{"results":[{"statement_id":0,"series":[` + + `{"name":"cpu","grouping_keys":["month"],"columns":["time","max","year","month"],"values":[` + + `["2023-04-15T14:20:30Z",3,null,4],` + + `["2023-07-19T08:15:22Z",4,null,7],` + + `["2023-10-27T16:45:10Z",5,null,10]]},` + + `{"name":"cpu","grouping_keys":["year"],"columns":["time","max","year","month"],"values":[` + + `["2023-12-31T23:59:59Z",6,2023,null]]},` + + `{"name":"cpu","grouping_keys":["month"],"columns":["time","max","year","month"],"values":[` + + `["2024-02-29T12:00:00Z",8,null,2],` + + `["2024-05-19T06:30:15Z",9,null,5],` + + `["2024-08-06T18:45:00Z",10,null,8],` + + `["2024-11-23T22:10:55Z",11,null,11],` + + `["2024-12-31T23:59:59Z",12,null,12]]},` + + `{"name":"cpu","grouping_keys":["year"],"columns":["time","max","year","month"],"values":[` + + `["2024-12-31T23:59:59Z",12,2024,null]]},` + + `{"name":"cpu","grouping_keys":["month"],"columns":["time","max","year","month"],"values":[` + + `["2025-01-01T00:00:00Z",13,null,1],` + + `["2025-06-12T11:20:30Z",14,null,6],` + + `["2025-09-15T23:59:59Z",19,null,9]]},` + + `{"name":"cpu","grouping_keys":["year"],"columns":["time","max","year","month"],"values":[` + + `["2025-09-15T23:59:59Z",19,2025,null]]}` + + `]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `GROUP BY year and month with MIN`, + command: `SELECT MIN(value) FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2025-12-31T23:59:59Z' GROUP BY date_part('year', time), date_part('month', time)`, + exp: `{"results":[{"statement_id":0,"series":[` + + `{"name":"cpu","grouping_keys":["month"],"columns":["time","min","year","month"],"values":[` + + `["2023-01-01T00:00:00Z",1,null,1]]},` + + `{"name":"cpu","grouping_keys":["year"],"columns":["time","min","year","month"],"values":[` + + `["2023-01-01T00:00:00Z",1,2023,null]]},` + + `{"name":"cpu","grouping_keys":["month"],"columns":["time","min","year","month"],"values":[` + + `["2023-04-15T14:20:30Z",3,null,4],` + + `["2023-07-19T08:15:22Z",4,null,7],` + + `["2023-10-27T16:45:10Z",5,null,10],` + + `["2023-12-31T23:59:59Z",6,null,12]]},` + + `{"name":"cpu","grouping_keys":["year"],"columns":["time","min","year","month"],"values":[` + + `["2024-01-01T00:00:00Z",7,2024,null]]},` + + `{"name":"cpu","grouping_keys":["month"],"columns":["time","min","year","month"],"values":[` + + `["2024-02-29T12:00:00Z",8,null,2],` + + `["2024-05-19T06:30:15Z",9,null,5],` + + `["2024-08-06T18:45:00Z",10,null,8],` + + `["2024-11-23T22:10:55Z",11,null,11]]},` + + `{"name":"cpu","grouping_keys":["year"],"columns":["time","min","year","month"],"values":[` + + `["2025-01-01T00:00:00Z",13,2025,null]]},` + + `{"name":"cpu","grouping_keys":["month"],"columns":["time","min","year","month"],"values":[` + + `["2025-06-12T11:20:30Z",14,null,6],` + + `["2025-09-15T00:00:00Z",15,null,9]]}` + + `]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + // GROUP BY date_part('week') routes points through the grouper using the + // ISO week. Over 2023 the weeks present are 3, 15, 29, 43 and 52, with + // week 52 holding both 2023-01-01 (ISO week 52 of 2022) and 2023-12-31. + name: `GROUP BY ISO week with COUNT`, + command: `SELECT COUNT(value) FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2023-12-31T23:59:59Z' GROUP BY date_part('week', time)`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","grouping_keys":["week"],"columns":["time","count","week"],"values":[` + + `["2023-01-01T00:00:00Z",1,3],` + + `["2023-01-01T00:00:00Z",1,15],` + + `["2023-01-01T00:00:00Z",1,29],` + + `["2023-01-01T00:00:00Z",1,43],` + + `["2023-01-01T00:00:00Z",2,52]` + + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `GROUP BY isodow with COUNT`, + command: `SELECT COUNT(value) FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2025-12-31T23:59:59Z' GROUP BY date_part('isodow', time)`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","grouping_keys":["isodow"],"columns":["time","count","isodow"],"values":[` + + `["2023-01-01T00:00:00Z",7,1],` + // Monday (isodow=1): 7 points (2,7,15,16,17,18,19) + `["2023-01-01T00:00:00Z",2,2],` + // Tuesday (isodow=2): 2 points (10,12) + `["2023-01-01T00:00:00Z",2,3],` + // Wednesday (isodow=3): 2 points (4,13) + `["2023-01-01T00:00:00Z",2,4],` + // Thursday (isodow=4): 2 points (8,14) + `["2023-01-01T00:00:00Z",1,5],` + // Friday (isodow=5): 1 point (5) + `["2023-01-01T00:00:00Z",2,6],` + // Saturday (isodow=6): 2 points (3,11) + `["2023-01-01T00:00:00Z",3,7]` + // Sunday (isodow=7): 3 points (1,6,9) + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `SELECT date_part with multiple fields and WHERE`, + command: `SELECT host, value, date_part('month', time) AS month FROM db0.rp0.cpu WHERE date_part('year', time) = 2024 AND date_part('month', time) <= 2 ORDER BY time`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","host","value","month"],"values":[` + + `["2024-01-01T00:00:00Z","server02",7,1],` + + `["2024-02-29T12:00:00Z","server02",8,2]` + + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + // Subquery tests + &Query{ + name: `aggregate over date_part results from subquery`, + command: `SELECT max(dow) FROM (SELECT value, date_part('dow', time) AS dow FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2025-12-31T23:59:59Z')`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","max"],"values":[["2023-04-15T14:20:30Z",6]]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `min date_part value from subquery`, + command: `SELECT min(dow) FROM (SELECT value, date_part('dow', time) AS dow FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2025-12-31T23:59:59Z')`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","min"],"values":[["2023-01-01T00:00:00Z",0]]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `filter in subquery using date_part`, + command: `SELECT mean(value) FROM (SELECT value FROM db0.rp0.cpu WHERE time >= '2024-01-01T00:00:00Z' AND date_part('dow', time) >= 1 AND date_part('dow', time) <= 2)`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","mean"],"values":[["1970-01-01T00:00:00Z",14.25]]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `filter outer query on date_part result from subquery`, + command: `SELECT value, dow FROM (SELECT value, date_part('dow', time) AS dow FROM db0.rp0.cpu WHERE time >= '2024-01-01T00:00:00Z' AND time <= '2024-12-31T23:59:59Z') WHERE dow = 2`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","value","dow"],"values":[` + + `["2024-08-06T18:45:00Z",10,2],` + + `["2024-12-31T23:59:59Z",12,2]` + + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `multiple date_part in subquery with month filter`, + command: `SELECT sum(value) FROM (SELECT value, date_part('dow', time) AS dow, date_part('month', time) AS month FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z') WHERE month = 1`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","sum"],"values":[["1970-01-01T00:00:00Z",23]]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `arithmetic on date_part result in outer query`, + command: `SELECT dow * 10 FROM (SELECT value, date_part('dow', time) AS dow FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z') LIMIT 1`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","dow"],"values":[["2023-01-01T00:00:00Z",0]]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `select value and date_part from subquery`, + command: `SELECT value, dow FROM (SELECT value, date_part('dow', time) AS dow FROM db0.rp0.cpu WHERE time >= '2024-01-01T00:00:00Z') LIMIT 2`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","value","dow"],"values":[` + + `["2024-01-01T00:00:00Z",7,1],` + + `["2024-02-29T12:00:00Z",8,4]` + + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `nested subquery with date_part`, + command: `SELECT max(value) FROM (SELECT value FROM (SELECT value FROM db0.rp0.cpu WHERE time >= '2024-01-01T00:00:00Z' AND date_part('dow', time) = 1))`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","max"],"values":[["2025-09-15T23:59:59Z",19]]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `first value with date_part in subquery`, + command: `SELECT first_value, dow FROM (SELECT first(value) AS first_value, date_part('dow', time) AS dow FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z')`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","first_value","dow"],"values":[["2023-01-01T00:00:00Z",1,0]]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + }...) + + var initialized bool + for _, query := range test.queries { + t.Run(query.name, func(t *testing.T) { + if !initialized { + err := test.init(s) + require.NoError(t, err, "init error") + initialized = true + } + require.NoError(t, query.Execute(s)) + require.True(t, query.success(), query.failureMessage()) + }) + } +} + +func TestServer_Query_DatePart_Timezone(t *testing.T) { + t.Parallel() + s := OpenServer(NewConfig()) + defer s.Close() + + if err := s.CreateDatabaseAndRetentionPolicy("db0", NewRetentionPolicySpec("rp0", 1, 0, 0, 0), true); err != nil { + t.Fatal(err) + } + + writes := []string{ + // Spring-forward boundary (NY): 07:30Z is EDT → 03:30 local. + fmt.Sprintf(`cpu value=1 %d`, mustParseTime(time.RFC3339Nano, "2023-03-12T07:30:00Z").UnixNano()), + // Fall-back boundary (NY): 07:30Z is EST → 02:30 local. + fmt.Sprintf(`cpu value=2 %d`, mustParseTime(time.RFC3339Nano, "2023-11-05T07:30:00Z").UnixNano()), + // Cross-day in NY: 03:30Z EDT → previous local day 23:30. + fmt.Sprintf(`cpu value=3 %d`, mustParseTime(time.RFC3339Nano, "2023-07-01T03:30:00Z").UnixNano()), + } + + test := NewTest("db0", "rp0") + test.writes = Writes{ + &Write{data: strings.Join(writes, "\n")}, + } + + test.addQueries([]*Query{ + &Query{ + name: `date_part hour as column with tz()`, + params: url.Values{"db": []string{"db0"}}, + command: `SELECT value, date_part('hour', time) AS h FROM db0.rp0.cpu tz('America/New_York')`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","value","h"],"values":[["2023-03-12T03:30:00-04:00",1,3],["2023-06-30T23:30:00-04:00",3,23],["2023-11-05T02:30:00-05:00",2,2]]}]}]}`, + }, + &Query{ + name: `GROUP BY date_part day with tz() across DST`, + params: url.Values{"db": []string{"db0"}}, + command: `SELECT count(value) FROM db0.rp0.cpu GROUP BY date_part('day', time) tz('America/New_York')`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","grouping_keys":["day"],"columns":["time","count","day"],"values":[["1969-12-31T19:00:00-05:00",1,5],["1969-12-31T19:00:00-05:00",1,12],["1969-12-31T19:00:00-05:00",1,30]]}]}]}`, + }, + &Query{ + // epoch is an absolute instant: its value must be identical with or + // without tz(). This locks in that the engine path leaves it + // zone-independent even when a timezone is applied. + name: `date_part epoch is zone-independent under tz()`, + params: url.Values{"db": []string{"db0"}}, + command: `SELECT value, date_part('epoch', time) AS e FROM db0.rp0.cpu tz('America/New_York')`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","value","e"],"values":[["2023-03-12T03:30:00-04:00",1,1678606200],["2023-06-30T23:30:00-04:00",3,1688182200],["2023-11-05T02:30:00-05:00",2,1699169400]]}]}]}`, + }, + &Query{ + // date_part in WHERE must filter on local time: 02:30 EST (fall-back + // point) has local hour 2, while its UTC hour is 7. Matching hour=2 + // proves the WHERE filter honors tz(). + name: `WHERE date_part hour filters on local time with tz()`, + params: url.Values{"db": []string{"db0"}}, + command: `SELECT value FROM db0.rp0.cpu WHERE date_part('hour', time) = 2 tz('America/New_York')`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","value"],"values":[["2023-11-05T02:30:00-05:00",2]]}]}]}`, + }, + }...) + + var initialized bool + for _, query := range test.queries { + t.Run(query.name, func(t *testing.T) { + if !initialized { + err := test.init(s) + require.NoError(t, err, "init error") + initialized = true + } + require.NoError(t, query.Execute(s)) + require.True(t, query.success(), query.failureMessage()) + }) + } +} + +func TestServer_Query_DatePart_GroupByWithTags(t *testing.T) { + t.Parallel() + s := OpenServer(NewConfig()) + defer s.Close() + + if err := s.CreateDatabaseAndRetentionPolicy("db0", NewRetentionPolicySpec("rp0", 1, 0, 0, 0), true); err != nil { + t.Fatal(err) + } + + writes := []string{ + // server01 - 2023 data + fmt.Sprintf(`cpu,host=server01 value=1 %d`, mustParseTime(time.RFC3339Nano, "2023-01-01T00:00:00Z").UnixNano()), + fmt.Sprintf(`cpu,host=server01 value=2 %d`, mustParseTime(time.RFC3339Nano, "2023-01-16T10:30:45Z").UnixNano()), + fmt.Sprintf(`cpu,host=server01 value=3 %d`, mustParseTime(time.RFC3339Nano, "2023-04-15T14:20:30Z").UnixNano()), + fmt.Sprintf(`cpu,host=server01 value=4 %d`, mustParseTime(time.RFC3339Nano, "2023-07-19T08:15:22Z").UnixNano()), + fmt.Sprintf(`cpu,host=server01 value=5 %d`, mustParseTime(time.RFC3339Nano, "2023-10-27T16:45:10Z").UnixNano()), + fmt.Sprintf(`cpu,host=server01 value=6 %d`, mustParseTime(time.RFC3339Nano, "2023-12-31T23:59:59Z").UnixNano()), + // server02 - 2024 data + fmt.Sprintf(`cpu,host=server02 value=7 %d`, mustParseTime(time.RFC3339Nano, "2024-01-01T00:00:00Z").UnixNano()), + fmt.Sprintf(`cpu,host=server02 value=8 %d`, mustParseTime(time.RFC3339Nano, "2024-02-29T12:00:00Z").UnixNano()), + fmt.Sprintf(`cpu,host=server02 value=9 %d`, mustParseTime(time.RFC3339Nano, "2024-05-19T06:30:15Z").UnixNano()), + fmt.Sprintf(`cpu,host=server02 value=10 %d`, mustParseTime(time.RFC3339Nano, "2024-08-06T18:45:00Z").UnixNano()), + fmt.Sprintf(`cpu,host=server02 value=11 %d`, mustParseTime(time.RFC3339Nano, "2024-11-23T22:10:55Z").UnixNano()), + fmt.Sprintf(`cpu,host=server02 value=12 %d`, mustParseTime(time.RFC3339Nano, "2024-12-31T23:59:59Z").UnixNano()), + // server03 - 2025 data + fmt.Sprintf(`cpu,host=server03 value=13 %d`, mustParseTime(time.RFC3339Nano, "2025-01-01T00:00:00Z").UnixNano()), + fmt.Sprintf(`cpu,host=server03 value=14 %d`, mustParseTime(time.RFC3339Nano, "2025-06-12T11:20:30Z").UnixNano()), + fmt.Sprintf(`cpu,host=server03 value=15 %d`, mustParseTime(time.RFC3339Nano, "2025-09-15T00:00:00Z").UnixNano()), + fmt.Sprintf(`cpu,host=server03 value=16 %d`, mustParseTime(time.RFC3339Nano, "2025-09-15T06:00:00Z").UnixNano()), + fmt.Sprintf(`cpu,host=server03 value=17 %d`, mustParseTime(time.RFC3339Nano, "2025-09-15T12:00:00Z").UnixNano()), + fmt.Sprintf(`cpu,host=server03 value=18 %d`, mustParseTime(time.RFC3339Nano, "2025-09-15T18:00:00Z").UnixNano()), + fmt.Sprintf(`cpu,host=server03 value=19 %d`, mustParseTime(time.RFC3339Nano, "2025-09-15T23:59:59Z").UnixNano()), + } + + test := NewTest("db0", "rp0") + test.writes = Writes{ + &Write{data: strings.Join(writes, "\n")}, + } + + test.addQueries([]*Query{ + // GROUP BY tag + single date_part + &Query{ + name: `GROUP BY host and year with COUNT`, + command: `SELECT COUNT(value) FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2025-12-31T23:59:59Z' GROUP BY host, date_part('year', time)`, + exp: `{"results":[{"statement_id":0,"series":[` + + `{"name":"cpu","tags":{"host":"server01"},"grouping_keys":["year"],"columns":["time","count","year"],"values":[` + + `["2023-01-01T00:00:00Z",6,2023]` + // server01: all 6 points in 2023 + `]},` + + `{"name":"cpu","tags":{"host":"server02"},"grouping_keys":["year"],"columns":["time","count","year"],"values":[` + + `["2023-01-01T00:00:00Z",6,2024]` + // server02: all 6 points in 2024 + `]},` + + `{"name":"cpu","tags":{"host":"server03"},"grouping_keys":["year"],"columns":["time","count","year"],"values":[` + + `["2023-01-01T00:00:00Z",7,2025]` + // server03: all 7 points in 2025 + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `GROUP BY host and quarter with SUM`, + command: `SELECT SUM(value) FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2025-12-31T23:59:59Z' GROUP BY host, date_part('quarter', time)`, + exp: `{"results":[{"statement_id":0,"series":[` + + `{"name":"cpu","tags":{"host":"server01"},"grouping_keys":["quarter"],"columns":["time","sum","quarter"],"values":[` + + `["2023-01-01T00:00:00Z",3,1],` + // server01 Q1: 1+2 = 3 + `["2023-01-01T00:00:00Z",3,2],` + // server01 Q2: 3 + `["2023-01-01T00:00:00Z",4,3],` + // server01 Q3: 4 + `["2023-01-01T00:00:00Z",11,4]` + // server01 Q4: 5+6 = 11 + `]},` + + `{"name":"cpu","tags":{"host":"server02"},"grouping_keys":["quarter"],"columns":["time","sum","quarter"],"values":[` + + `["2023-01-01T00:00:00Z",15,1],` + // server02 Q1: 7+8 = 15 + `["2023-01-01T00:00:00Z",9,2],` + // server02 Q2: 9 + `["2023-01-01T00:00:00Z",10,3],` + // server02 Q3: 10 + `["2023-01-01T00:00:00Z",23,4]` + // server02 Q4: 11+12 = 23 + `]},` + + `{"name":"cpu","tags":{"host":"server03"},"grouping_keys":["quarter"],"columns":["time","sum","quarter"],"values":[` + + `["2023-01-01T00:00:00Z",13,1],` + // server03 Q1: 13 + `["2023-01-01T00:00:00Z",14,2],` + // server03 Q2: 14 + `["2023-01-01T00:00:00Z",85,3]` + // server03 Q3: 15+16+17+18+19 = 85 + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `GROUP BY host and month with MEAN`, + command: `SELECT MEAN(value) FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2025-12-31T23:59:59Z' GROUP BY host, date_part('month', time)`, + exp: `{"results":[{"statement_id":0,"series":[` + + `{"name":"cpu","tags":{"host":"server01"},"grouping_keys":["month"],"columns":["time","mean","month"],"values":[` + + `["2023-01-01T00:00:00Z",1.5,1],` + // server01 Jan: (1+2)/2 = 1.5 + `["2023-01-01T00:00:00Z",3,4],` + // server01 Apr: 3 + `["2023-01-01T00:00:00Z",4,7],` + // server01 Jul: 4 + `["2023-01-01T00:00:00Z",5,10],` + // server01 Oct: 5 + `["2023-01-01T00:00:00Z",6,12]` + // server01 Dec: 6 + `]},` + + `{"name":"cpu","tags":{"host":"server02"},"grouping_keys":["month"],"columns":["time","mean","month"],"values":[` + + `["2023-01-01T00:00:00Z",7,1],` + // server02 Jan: 7 + `["2023-01-01T00:00:00Z",8,2],` + // server02 Feb: 8 + `["2023-01-01T00:00:00Z",9,5],` + // server02 May: 9 + `["2023-01-01T00:00:00Z",10,8],` + // server02 Aug: 10 + `["2023-01-01T00:00:00Z",11,11],` + // server02 Nov: 11 + `["2023-01-01T00:00:00Z",12,12]` + // server02 Dec: 12 + `]},` + + `{"name":"cpu","tags":{"host":"server03"},"grouping_keys":["month"],"columns":["time","mean","month"],"values":[` + + `["2023-01-01T00:00:00Z",13,1],` + // server03 Jan: 13 + `["2023-01-01T00:00:00Z",14,6],` + // server03 Jun: 14 + `["2023-01-01T00:00:00Z",17,9]` + // server03 Sep: (15+16+17+18+19)/5 = 17 + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `GROUP BY host and dow with COUNT`, + command: `SELECT COUNT(value) FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2025-12-31T23:59:59Z' GROUP BY host, date_part('dow', time)`, + exp: `{"results":[{"statement_id":0,"series":[` + + `{"name":"cpu","tags":{"host":"server01"},"grouping_keys":["dow"],"columns":["time","count","dow"],"values":[` + + `["2023-01-01T00:00:00Z",2,0],` + // server01 Sun: 1,6 + `["2023-01-01T00:00:00Z",1,1],` + // server01 Mon: 2 + `["2023-01-01T00:00:00Z",1,3],` + // server01 Wed: 4 + `["2023-01-01T00:00:00Z",1,5],` + // server01 Fri: 5 + `["2023-01-01T00:00:00Z",1,6]` + // server01 Sat: 3 + `]},` + + `{"name":"cpu","tags":{"host":"server02"},"grouping_keys":["dow"],"columns":["time","count","dow"],"values":[` + + `["2023-01-01T00:00:00Z",1,0],` + // server02 Sun: 9 + `["2023-01-01T00:00:00Z",1,1],` + // server02 Mon: 7 + `["2023-01-01T00:00:00Z",2,2],` + // server02 Tue: 10,12 + `["2023-01-01T00:00:00Z",1,4],` + // server02 Thu: 8 + `["2023-01-01T00:00:00Z",1,6]` + // server02 Sat: 11 + `]},` + + `{"name":"cpu","tags":{"host":"server03"},"grouping_keys":["dow"],"columns":["time","count","dow"],"values":[` + + `["2023-01-01T00:00:00Z",5,1],` + // server03 Mon: 15,16,17,18,19 + `["2023-01-01T00:00:00Z",1,3],` + // server03 Wed: 13 + `["2023-01-01T00:00:00Z",1,4]` + // server03 Thu: 14 + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `GROUP BY host and isodow with COUNT`, + command: `SELECT COUNT(value) FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2025-12-31T23:59:59Z' GROUP BY host, date_part('isodow', time)`, + // isodow: Monday=1, Tuesday=2, Wednesday=3, Thursday=4, Friday=5, Saturday=6, Sunday=7 + exp: `{"results":[{"statement_id":0,"series":[` + + `{"name":"cpu","tags":{"host":"server01"},"grouping_keys":["isodow"],"columns":["time","count","isodow"],"values":[` + + `["2023-01-01T00:00:00Z",1,1],` + // server01 Mon (isodow=1): value 2 + `["2023-01-01T00:00:00Z",1,3],` + // server01 Wed (isodow=3): value 4 + `["2023-01-01T00:00:00Z",1,5],` + // server01 Fri (isodow=5): value 5 + `["2023-01-01T00:00:00Z",1,6],` + // server01 Sat (isodow=6): value 3 + `["2023-01-01T00:00:00Z",2,7]` + // server01 Sun (isodow=7): values 1,6 + `]},` + + `{"name":"cpu","tags":{"host":"server02"},"grouping_keys":["isodow"],"columns":["time","count","isodow"],"values":[` + + `["2023-01-01T00:00:00Z",1,1],` + // server02 Mon (isodow=1): value 7 + `["2023-01-01T00:00:00Z",2,2],` + // server02 Tue (isodow=2): values 10,12 + `["2023-01-01T00:00:00Z",1,4],` + // server02 Thu (isodow=4): value 8 + `["2023-01-01T00:00:00Z",1,6],` + // server02 Sat (isodow=6): value 11 + `["2023-01-01T00:00:00Z",1,7]` + // server02 Sun (isodow=7): value 9 + `]},` + + `{"name":"cpu","tags":{"host":"server03"},"grouping_keys":["isodow"],"columns":["time","count","isodow"],"values":[` + + `["2023-01-01T00:00:00Z",5,1],` + // server03 Mon (isodow=1): values 15,16,17,18,19 + `["2023-01-01T00:00:00Z",1,3],` + // server03 Wed (isodow=3): value 13 + `["2023-01-01T00:00:00Z",1,4]` + // server03 Thu (isodow=4): value 14 + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + // GROUP BY tag + date_part with WHERE filter + &Query{ + name: `GROUP BY host and dow with WHERE weekday filter and SUM`, + command: `SELECT SUM(value) FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2025-12-31T23:59:59Z' AND date_part('dow', time) >= 1 AND date_part('dow', time) <= 5 GROUP BY host, date_part('dow', time)`, + exp: `{"results":[{"statement_id":0,"series":[` + + `{"name":"cpu","tags":{"host":"server01"},"grouping_keys":["dow"],"columns":["time","sum","dow"],"values":[` + + `["2023-01-01T00:00:00Z",2,1],` + // server01 Mon: 2 + `["2023-01-01T00:00:00Z",4,3],` + // server01 Wed: 4 + `["2023-01-01T00:00:00Z",5,5]` + // server01 Fri: 5 + `]},` + + `{"name":"cpu","tags":{"host":"server02"},"grouping_keys":["dow"],"columns":["time","sum","dow"],"values":[` + + `["2023-01-01T00:00:00Z",7,1],` + // server02 Mon: 7 + `["2023-01-01T00:00:00Z",22,2],` + // server02 Tue: 10+12 = 22 + `["2023-01-01T00:00:00Z",8,4]` + // server02 Thu: 8 + `]},` + + `{"name":"cpu","tags":{"host":"server03"},"grouping_keys":["dow"],"columns":["time","sum","dow"],"values":[` + + `["2023-01-01T00:00:00Z",85,1],` + // server03 Mon: 15+16+17+18+19 = 85 + `["2023-01-01T00:00:00Z",13,3],` + // server03 Wed: 13 + `["2023-01-01T00:00:00Z",14,4]` + // server03 Thu: 14 + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + // GROUP BY tag + multiple date_parts (PR comment pattern) + // Each date_part dimension produces its own series per host, with null + // for the non-active dimension column. + &Query{ + name: `GROUP BY host year and month with COUNT - PR comment pattern`, + command: `SELECT COUNT(value) FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2025-12-31T23:59:59Z' GROUP BY date_part('month', time), date_part('year', time), host`, + exp: `{"results":[{"statement_id":0,"series":[` + + // server01 - month dimension + `{"name":"cpu","tags":{"host":"server01"},"grouping_keys":["month"],"columns":["time","count","month","year"],"values":[` + + `["2023-01-01T00:00:00Z",2,1,null],` + // Jan: 2 points + `["2023-01-01T00:00:00Z",1,4,null],` + // Apr: 1 point + `["2023-01-01T00:00:00Z",1,7,null],` + // Jul: 1 point + `["2023-01-01T00:00:00Z",1,10,null],` + // Oct: 1 point + `["2023-01-01T00:00:00Z",1,12,null]` + // Dec: 1 point + `]},` + + // server01 - year dimension + `{"name":"cpu","tags":{"host":"server01"},"grouping_keys":["year"],"columns":["time","count","month","year"],"values":[` + + `["2023-01-01T00:00:00Z",6,null,2023]` + // 2023: 6 points + `]},` + + // server02 - month dimension + `{"name":"cpu","tags":{"host":"server02"},"grouping_keys":["month"],"columns":["time","count","month","year"],"values":[` + + `["2023-01-01T00:00:00Z",1,1,null],` + // Jan: 1 point + `["2023-01-01T00:00:00Z",1,2,null],` + // Feb: 1 point + `["2023-01-01T00:00:00Z",1,5,null],` + // May: 1 point + `["2023-01-01T00:00:00Z",1,8,null],` + // Aug: 1 point + `["2023-01-01T00:00:00Z",1,11,null],` + // Nov: 1 point + `["2023-01-01T00:00:00Z",1,12,null]` + // Dec: 1 point + `]},` + + // server02 - year dimension + `{"name":"cpu","tags":{"host":"server02"},"grouping_keys":["year"],"columns":["time","count","month","year"],"values":[` + + `["2023-01-01T00:00:00Z",6,null,2024]` + // 2024: 6 points + `]},` + + // server03 - month dimension + `{"name":"cpu","tags":{"host":"server03"},"grouping_keys":["month"],"columns":["time","count","month","year"],"values":[` + + `["2023-01-01T00:00:00Z",1,1,null],` + // Jan: 1 point + `["2023-01-01T00:00:00Z",1,6,null],` + // Jun: 1 point + `["2023-01-01T00:00:00Z",5,9,null]` + // Sep: 5 points + `]},` + + // server03 - year dimension + `{"name":"cpu","tags":{"host":"server03"},"grouping_keys":["year"],"columns":["time","count","month","year"],"values":[` + + `["2023-01-01T00:00:00Z",7,null,2025]` + // 2025: 7 points + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + // GROUP BY multiple date_parts + tag: each date_part dimension produces + // its own series per host, with null for non-active dimension columns. + &Query{ + name: `GROUP BY year, month, and host with COUNT`, + command: `SELECT COUNT(value) FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2025-12-31T23:59:59Z' GROUP BY date_part('year', time), date_part('month', time), host`, + exp: `{"results":[{"statement_id":0,"series":[` + + // server01 - month dimension + `{"name":"cpu","tags":{"host":"server01"},"grouping_keys":["month"],"columns":["time","count","year","month"],"values":[` + + `["2023-01-01T00:00:00Z",2,null,1],` + // Jan: 2 points + `["2023-01-01T00:00:00Z",1,null,4],` + // Apr: 1 point + `["2023-01-01T00:00:00Z",1,null,7],` + // Jul: 1 point + `["2023-01-01T00:00:00Z",1,null,10],` + // Oct: 1 point + `["2023-01-01T00:00:00Z",1,null,12]` + // Dec: 1 point + `]},` + + // server01 - year dimension + `{"name":"cpu","tags":{"host":"server01"},"grouping_keys":["year"],"columns":["time","count","year","month"],"values":[` + + `["2023-01-01T00:00:00Z",6,2023,null]` + // 2023: 6 points + `]},` + + // server02 - month dimension + `{"name":"cpu","tags":{"host":"server02"},"grouping_keys":["month"],"columns":["time","count","year","month"],"values":[` + + `["2023-01-01T00:00:00Z",1,null,1],` + // Jan: 1 point + `["2023-01-01T00:00:00Z",1,null,2],` + // Feb: 1 point + `["2023-01-01T00:00:00Z",1,null,5],` + // May: 1 point + `["2023-01-01T00:00:00Z",1,null,8],` + // Aug: 1 point + `["2023-01-01T00:00:00Z",1,null,11],` + // Nov: 1 point + `["2023-01-01T00:00:00Z",1,null,12]` + // Dec: 1 point + `]},` + + // server02 - year dimension + `{"name":"cpu","tags":{"host":"server02"},"grouping_keys":["year"],"columns":["time","count","year","month"],"values":[` + + `["2023-01-01T00:00:00Z",6,2024,null]` + // 2024: 6 points + `]},` + + // server03 - month dimension + `{"name":"cpu","tags":{"host":"server03"},"grouping_keys":["month"],"columns":["time","count","year","month"],"values":[` + + `["2023-01-01T00:00:00Z",1,null,1],` + // Jan: 1 point + `["2023-01-01T00:00:00Z",1,null,6],` + // Jun: 1 point + `["2023-01-01T00:00:00Z",5,null,9]` + // Sep: 5 points + `]},` + + // server03 - year dimension + `{"name":"cpu","tags":{"host":"server03"},"grouping_keys":["year"],"columns":["time","count","year","month"],"values":[` + + `["2023-01-01T00:00:00Z",7,2025,null]` + // 2025: 7 points + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `GROUP BY month, year, and host with SUM - reversed dimension order`, + command: `SELECT SUM(value) FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2025-12-31T23:59:59Z' GROUP BY date_part('month', time), date_part('year', time), host`, + exp: `{"results":[{"statement_id":0,"series":[` + + // server01 - month dimension + `{"name":"cpu","tags":{"host":"server01"},"grouping_keys":["month"],"columns":["time","sum","month","year"],"values":[` + + `["2023-01-01T00:00:00Z",3,1,null],` + // Jan: 1+2 = 3 + `["2023-01-01T00:00:00Z",3,4,null],` + // Apr: 3 + `["2023-01-01T00:00:00Z",4,7,null],` + // Jul: 4 + `["2023-01-01T00:00:00Z",5,10,null],` + // Oct: 5 + `["2023-01-01T00:00:00Z",6,12,null]` + // Dec: 6 + `]},` + + // server01 - year dimension + `{"name":"cpu","tags":{"host":"server01"},"grouping_keys":["year"],"columns":["time","sum","month","year"],"values":[` + + `["2023-01-01T00:00:00Z",21,null,2023]` + // 2023: 1+2+3+4+5+6 = 21 + `]},` + + // server02 - month dimension + `{"name":"cpu","tags":{"host":"server02"},"grouping_keys":["month"],"columns":["time","sum","month","year"],"values":[` + + `["2023-01-01T00:00:00Z",7,1,null],` + // Jan: 7 + `["2023-01-01T00:00:00Z",8,2,null],` + // Feb: 8 + `["2023-01-01T00:00:00Z",9,5,null],` + // May: 9 + `["2023-01-01T00:00:00Z",10,8,null],` + // Aug: 10 + `["2023-01-01T00:00:00Z",11,11,null],` + // Nov: 11 + `["2023-01-01T00:00:00Z",12,12,null]` + // Dec: 12 + `]},` + + // server02 - year dimension + `{"name":"cpu","tags":{"host":"server02"},"grouping_keys":["year"],"columns":["time","sum","month","year"],"values":[` + + `["2023-01-01T00:00:00Z",57,null,2024]` + // 2024: 7+8+9+10+11+12 = 57 + `]},` + + // server03 - month dimension + `{"name":"cpu","tags":{"host":"server03"},"grouping_keys":["month"],"columns":["time","sum","month","year"],"values":[` + + `["2023-01-01T00:00:00Z",13,1,null],` + // Jan: 13 + `["2023-01-01T00:00:00Z",14,6,null],` + // Jun: 14 + `["2023-01-01T00:00:00Z",85,9,null]` + // Sep: 15+16+17+18+19 = 85 + `]},` + + // server03 - year dimension + `{"name":"cpu","tags":{"host":"server03"},"grouping_keys":["year"],"columns":["time","sum","month","year"],"values":[` + + `["2023-01-01T00:00:00Z",112,null,2025]` + // 2025: 13+14+15+16+17+18+19 = 112 + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `GROUP BY year, month, and host with COUNT and WHERE year filter`, + command: `SELECT COUNT(value) FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2025-12-31T23:59:59Z' AND date_part('year', time) >= 2024 GROUP BY date_part('year', time), date_part('month', time), host`, + exp: `{"results":[{"statement_id":0,"series":[` + + // server02 - month dimension (only 2024 data passes filter) + `{"name":"cpu","tags":{"host":"server02"},"grouping_keys":["month"],"columns":["time","count","year","month"],"values":[` + + `["2023-01-01T00:00:00Z",1,null,1],` + // Jan: 1 point + `["2023-01-01T00:00:00Z",1,null,2],` + // Feb: 1 point + `["2023-01-01T00:00:00Z",1,null,5],` + // May: 1 point + `["2023-01-01T00:00:00Z",1,null,8],` + // Aug: 1 point + `["2023-01-01T00:00:00Z",1,null,11],` + // Nov: 1 point + `["2023-01-01T00:00:00Z",1,null,12]` + // Dec: 1 point + `]},` + + // server02 - year dimension + `{"name":"cpu","tags":{"host":"server02"},"grouping_keys":["year"],"columns":["time","count","year","month"],"values":[` + + `["2023-01-01T00:00:00Z",6,2024,null]` + // 2024: 6 points + `]},` + + // server03 - month dimension (all data passes: 2025) + `{"name":"cpu","tags":{"host":"server03"},"grouping_keys":["month"],"columns":["time","count","year","month"],"values":[` + + `["2023-01-01T00:00:00Z",1,null,1],` + // Jan: 1 point + `["2023-01-01T00:00:00Z",1,null,6],` + // Jun: 1 point + `["2023-01-01T00:00:00Z",5,null,9]` + // Sep: 5 points + `]},` + + // server03 - year dimension + `{"name":"cpu","tags":{"host":"server03"},"grouping_keys":["year"],"columns":["time","count","year","month"],"values":[` + + `["2023-01-01T00:00:00Z",7,2025,null]` + // 2025: 7 points + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + // date_part in both SELECT and GROUP BY with tag: an explicit + // SELECT date_part('year', time) that matches the GROUP BY date_part + // dimension reports the grouped value, so the date_part and year columns + // agree per group. + &Query{ + name: `SELECT date_part with GROUP BY host and year`, + command: `SELECT COUNT(value), date_part('year', time) FROM db0.rp0.cpu WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2025-12-31T23:59:59Z' GROUP BY host, date_part('year', time)`, + exp: `{"results":[{"statement_id":0,"series":[` + + `{"name":"cpu","tags":{"host":"server01"},"grouping_keys":["year"],"columns":["time","count","date_part","year"],"values":[` + + `["2023-01-01T00:00:00Z",6,2023,2023]` + + `]},` + + `{"name":"cpu","tags":{"host":"server02"},"grouping_keys":["year"],"columns":["time","count","date_part","year"],"values":[` + + `["2023-01-01T00:00:00Z",6,2024,2024]` + + `]},` + + `{"name":"cpu","tags":{"host":"server03"},"grouping_keys":["year"],"columns":["time","count","date_part","year"],"values":[` + + `["2023-01-01T00:00:00Z",7,2025,2025]` + + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + }...) + + var initialized bool + for _, query := range test.queries { + t.Run(query.name, func(t *testing.T) { + if !initialized { + err := test.init(s) + require.NoError(t, err, "init error") + initialized = true + } + require.NoError(t, query.Execute(s)) + require.True(t, query.success(), query.failureMessage()) + }) + } +} + +// A stored field whose name matches a GROUP BY date_part output column (e.g. a +// field literally named "year") collides with the injected date_part column. The +// explicit form is rejected at compile time; the wildcard form must be rejected +// too, even though `*` is only expanded later in Prepare (RewriteFields). Without +// the post-RewriteFields revalidation the wildcard query emits duplicate "year" +// columns and silently corrupts column-name-keyed handling (e.g. SELECT INTO). +func TestServer_Query_DatePart_WildcardCollision(t *testing.T) { + t.Parallel() + s := OpenServer(NewConfig()) + defer s.Close() + + if err := s.CreateDatabaseAndRetentionPolicy("db0", NewRetentionPolicySpec("rp0", 1, 0, 0, 0), true); err != nil { + t.Fatal(err) + } + + writes := []string{ + fmt.Sprintf(`m,host=a year=5,value=1 %d`, mustParseTime(time.RFC3339Nano, "2023-01-01T00:00:00Z").UnixNano()), + fmt.Sprintf(`m,host=a year=6,value=2 %d`, mustParseTime(time.RFC3339Nano, "2024-01-01T00:00:00Z").UnixNano()), + } + test := NewTest("db0", "rp0") + test.writes = Writes{&Write{data: strings.Join(writes, "\n")}} + + collision := `date_part: output column "year" collides with the GROUP BY date_part('year', time) dimension; alias the field to a different name` + rawErr := `date_part: GROUP BY date_part requires an aggregate or selector function` + test.addQueries([]*Query{ + // An aggregate whose output column collides with the injected date_part + // dimension column is rejected. + &Query{ + name: `aggregate column colliding with GROUP BY date_part is rejected`, + command: `SELECT max(value) AS year FROM db0.rp0.m GROUP BY date_part('year', time)`, + exp: fmt.Sprintf(`{"results":[{"statement_id":0,"error":%q}]}`, collision), + params: url.Values{"db": []string{"db0"}}, + }, + // A raw (non-aggregate) field selection with GROUP BY date_part does no + // grouping at all, so it is rejected before any column-collision check. + &Query{ + name: `raw field with GROUP BY date_part is rejected`, + command: `SELECT year FROM db0.rp0.m GROUP BY date_part('year', time)`, + exp: fmt.Sprintf(`{"results":[{"statement_id":0,"error":%q}]}`, rawErr), + params: url.Values{"db": []string{"db0"}}, + }, + // A raw wildcard with GROUP BY date_part is likewise rejected as a raw query. + &Query{ + name: `raw wildcard with GROUP BY date_part is rejected`, + command: `SELECT * FROM db0.rp0.m GROUP BY date_part('year', time)`, + exp: fmt.Sprintf(`{"results":[{"statement_id":0,"error":%q}]}`, rawErr), + params: url.Values{"db": []string{"db0"}}, + }, + // An aggregate wildcard over a multi-field measurement expands to multiple + // aggregates, the same shape as the rejected explicit form: the per-call + // scanners align positionally, so a field missing from one group shifts + // another group's values under its label (mislabeled results). Rejected + // at Prepare, once the wildcard has been expanded. + &Query{ + name: `wildcard expanding to multiple aggregates is rejected`, + command: `SELECT count(*) FROM db0.rp0.m GROUP BY date_part('month', time)`, + exp: fmt.Sprintf(`{"results":[{"statement_id":0,"error":%q}]}`, `date_part: GROUP BY date_part supports only a single aggregate or selector function`), + params: url.Values{"db": []string{"db0"}}, + }, + }...) + + var initialized bool + for _, query := range test.queries { + t.Run(query.name, func(t *testing.T) { + if !initialized { + require.NoError(t, test.init(s), "init error") + initialized = true + } + require.NoError(t, query.Execute(s)) + require.True(t, query.success(), query.failureMessage()) + }) + } +} + +func TestServer_Query_DatePart_Subquery_GroupBy(t *testing.T) { + t.Parallel() + s := OpenServer(NewConfig()) + defer s.Close() + + if err := s.CreateDatabaseAndRetentionPolicy("db0", NewRetentionPolicySpec("rp0", 1, 0, 0, 0), true); err != nil { + t.Fatal(err) + } + + writes := []string{ + fmt.Sprintf(`cpu,host=server01 value=1 %d`, mustParseTime(time.RFC3339Nano, "2023-01-01T01:00:00Z").UnixNano()), + fmt.Sprintf(`cpu,host=server01 value=2 %d`, mustParseTime(time.RFC3339Nano, "2023-01-02T01:30:00Z").UnixNano()), + fmt.Sprintf(`cpu,host=server01 value=3 %d`, mustParseTime(time.RFC3339Nano, "2023-01-03T05:00:00Z").UnixNano()), + fmt.Sprintf(`cpu,host=server01 value=4 %d`, mustParseTime(time.RFC3339Nano, "2023-01-04T05:45:00Z").UnixNano()), + } + + test := NewTest("db0", "rp0") + test.writes = Writes{ + &Write{data: strings.Join(writes, "\n")}, + } + + test.addQueries( + // GROUP BY date_part('hour', time) where the source is a subquery. + &Query{ + name: `subquery GROUP BY hour with COUNT`, + command: `SELECT COUNT(value) FROM (SELECT value FROM db0.rp0.cpu) WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2023-01-31T23:59:59Z' GROUP BY date_part('hour', time)`, + exp: `{"results":[{"statement_id":0,"series":[` + + `{"name":"cpu","grouping_keys":["hour"],"columns":["time","count","hour"],"values":[` + + `["2023-01-01T00:00:00Z",2,1],` + // 01:00, 01:30 -> hour 1 + `["2023-01-01T00:00:00Z",2,5]` + // 05:00, 05:45 -> hour 5 + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + ) + + var initialized bool + for _, query := range test.queries { + t.Run(query.name, func(t *testing.T) { + if !initialized { + require.NoError(t, test.init(s), "init error") + initialized = true + } + require.NoError(t, query.Execute(s)) + require.True(t, query.success(), query.failureMessage()) + }) + } +} + +// Ensure GROUP BY date_part('epoch', ...) emits pre-1970 (negative) buckets in +// chronological order. The reduce path orders series by sorting encoded +// grouping-key strings, so the encoding must preserve signed value order. +func TestServer_Query_DatePart_EpochPre1970(t *testing.T) { + t.Parallel() + s := OpenServer(NewConfig()) + defer s.Close() + + if err := s.CreateDatabaseAndRetentionPolicy("db0", NewRetentionPolicySpec("rp0", 1, 0, 0, 0), true); err != nil { + t.Fatal(err) + } + + writes := []string{ + fmt.Sprintf(`cpu,host=server01 value=1 %d`, mustParseTime(time.RFC3339Nano, "1969-12-31T23:59:59Z").UnixNano()), // epoch -1 + fmt.Sprintf(`cpu,host=server01 value=2 %d`, mustParseTime(time.RFC3339Nano, "1970-01-01T00:00:00Z").UnixNano()), // epoch 0 + fmt.Sprintf(`cpu,host=server01 value=3 %d`, mustParseTime(time.RFC3339Nano, "1970-01-01T00:00:01Z").UnixNano()), // epoch 1 + } + + test := NewTest("db0", "rp0") + test.writes = Writes{ + &Write{data: strings.Join(writes, "\n")}, + } + + test.addQueries( + &Query{ + name: `GROUP BY epoch spanning 1970 emits buckets chronologically`, + command: `SELECT COUNT(value) FROM db0.rp0.cpu WHERE time >= '1969-12-31T23:59:59Z' AND time <= '1970-01-01T00:00:01Z' GROUP BY date_part('epoch', time)`, + exp: `{"results":[{"statement_id":0,"series":[` + + `{"name":"cpu","grouping_keys":["epoch"],"columns":["time","count","epoch"],"values":[` + + `["1969-12-31T23:59:59Z",1,-1],` + + `["1969-12-31T23:59:59Z",1,0],` + + `["1969-12-31T23:59:59Z",1,1]` + + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + ) + + var initialized bool + for _, query := range test.queries { + t.Run(query.name, func(t *testing.T) { + if !initialized { + require.NoError(t, test.init(s), "init error") + initialized = true + } + require.NoError(t, query.Execute(s)) + require.True(t, query.success(), query.failureMessage()) + }) + } +} + +func TestServer_Query_DatePart_Subquery_Where(t *testing.T) { + t.Parallel() + s := OpenServer(NewConfig()) + defer s.Close() + + if err := s.CreateDatabaseAndRetentionPolicy("db0", NewRetentionPolicySpec("rp0", 1, 0, 0, 0), true); err != nil { + t.Fatal(err) + } + + writes := []string{ + fmt.Sprintf(`cpu,host=server01 value=1 %d`, mustParseTime(time.RFC3339Nano, "2023-01-01T01:00:00Z").UnixNano()), + fmt.Sprintf(`cpu,host=server01 value=2 %d`, mustParseTime(time.RFC3339Nano, "2023-01-02T01:30:00Z").UnixNano()), + fmt.Sprintf(`cpu,host=server01 value=3 %d`, mustParseTime(time.RFC3339Nano, "2023-01-03T05:00:00Z").UnixNano()), + fmt.Sprintf(`cpu,host=server01 value=4 %d`, mustParseTime(time.RFC3339Nano, "2023-01-04T05:45:00Z").UnixNano()), + } + + test := NewTest("db0", "rp0") + test.writes = Writes{ + &Write{data: strings.Join(writes, "\n")}, + } + + test.addQueries( + &Query{ + name: `raw subquery WHERE hour`, + command: `SELECT value FROM (SELECT value FROM db0.rp0.cpu) WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2023-01-31T23:59:59Z' AND date_part('hour', time) = 5`, + exp: `{"results":[{"statement_id":0,"series":[` + + `{"name":"cpu","columns":["time","value"],"values":[` + + `["2023-01-03T05:00:00Z",3],` + + `["2023-01-04T05:45:00Z",4]` + + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `aggregate subquery WHERE dow`, + command: `SELECT COUNT(value) FROM (SELECT value FROM db0.rp0.cpu) WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2023-01-31T23:59:59Z' AND date_part('dow', time) = 0`, + exp: `{"results":[{"statement_id":0,"series":[` + + `{"name":"cpu","columns":["time","count"],"values":[` + + `["2023-01-01T00:00:00Z",1]` + // only 2023-01-01 is a Sunday + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `raw subquery WHERE hour with tz()`, + command: `SELECT value FROM (SELECT value FROM db0.rp0.cpu) WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2023-01-31T23:59:59Z' AND date_part('hour', time) = 21 tz('America/Los_Angeles')`, + exp: `{"results":[{"statement_id":0,"series":[` + + `{"name":"cpu","columns":["time","value"],"values":[` + + `["2023-01-02T21:00:00-08:00",3],` + // 05:00Z = 21:00 PST previous day + `["2023-01-03T21:45:00-08:00",4]` + + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + &Query{ + name: `non-date_part subquery filter is unchanged`, + command: `SELECT value FROM (SELECT value FROM db0.rp0.cpu) WHERE time >= '2023-01-01T00:00:00Z' AND time <= '2023-01-31T23:59:59Z' AND value > 2`, + exp: `{"results":[{"statement_id":0,"series":[` + + `{"name":"cpu","columns":["time","value"],"values":[` + + `["2023-01-03T05:00:00Z",3],` + + `["2023-01-04T05:45:00Z",4]` + + `]}]}]}`, + params: url.Values{"db": []string{"db0"}}, + }, + ) + + var initialized bool + for _, query := range test.queries { + t.Run(query.name, func(t *testing.T) { + if !initialized { + require.NoError(t, test.init(s), "init error") + initialized = true + } + require.NoError(t, query.Execute(s)) + require.True(t, query.success(), query.failureMessage()) + }) + } +} + func TestServer_Query_ShowTagKeys(t *testing.T) { t.Parallel() s := OpenServer(NewConfig()) diff --git a/tsdb/engine/tsm1/iterator.gen.go b/tsdb/engine/tsm1/iterator.gen.go index 8f2e81317da..2817ebb705b 100644 --- a/tsdb/engine/tsm1/iterator.gen.go +++ b/tsdb/engine/tsm1/iterator.gen.go @@ -11,6 +11,7 @@ import ( "runtime" "sort" "sync" + "time" "github.com/influxdata/influxdb/pkg/metrics" "github.com/influxdata/influxdb/pkg/tracing" @@ -190,6 +191,11 @@ type floatIterator struct { } opt query.IteratorOptions + // dpCond is non-nil when the condition uses date_part. It owns the + // rewritten condition (stored back into itr.opt.Condition) and publishes + // per-point part values into itr.m via SetTime. + dpCond *query.DatePartCondition + m map[string]interface{} // map used for condition evaluation point query.FloatPoint // reusable buffer @@ -214,11 +220,27 @@ func newFloatIterator(name string, tags query.Tags, opt query.IteratorOptions, c } itr.stats = itr.statsBuf + // When the condition uses date_part (opt.NeedTimeRef), rewrite it once so + // no function call is evaluated per point: date_part calls become reserved + // variable references resolved by dpCond.SetTime in Next. itr.opt is this + // iterator's own copy, so storing the rewritten condition there leaves the + // caller's condition untouched. + if opt.NeedTimeRef { + if dp := query.NewDatePartCondition(opt.Condition, opt.Location); dp != nil { + itr.dpCond = dp + itr.opt.Condition = dp.Expr() + } + } + if len(aux) > 0 { itr.point.Aux = make([]interface{}, len(aux)) } - if opt.Condition != nil { + // Allocate the condition-evaluation map when there is a condition to + // evaluate. opt.NeedTimeRef is kept in the guard because the two fields are + // encoded independently over the iterator wire codec and a + // NeedTimeRef=true / Condition=nil options struct must stay safe. + if opt.Condition != nil || opt.NeedTimeRef { itr.m = make(map[string]interface{}, len(aux)+len(conds)) } itr.conds.names = condNames @@ -277,11 +299,34 @@ func (itr *floatIterator) Next() (*query.FloatPoint, error) { itr.m[itr.conds.names[i]] = itr.conds.curs[i].nextAt(seek) } + // Publish the date_part values referenced by the condition for this + // point's timestamp. + if itr.dpCond != nil { + itr.dpCond.SetTime(seek, itr.m) + } + // Evaluate condition, if one exists. Retry if it fails. if itr.opt.Condition != nil && !itr.valuer.EvalBool(itr.opt.Condition) { continue } + // Compute date part dimension values in-place, only for points that pass + // the condition (the extraction is wasted on filtered points). The + // date_part dims occupy the last N slots of opt.Aux (added by select.go), + // so we overwrite the nil values left by the phantom aux cursors rather + // than appending, keeping Aux length consistent with scanner keys. + if len(itr.opt.DatePartDimensions) > 0 { + baseIdx := len(itr.opt.Aux) - len(itr.opt.DatePartDimensions) + t := time.Unix(0, seek).In(query.LocationOrUTC(itr.opt.Location)) + for i, dim := range itr.opt.DatePartDimensions { + val, ok := query.ExtractDatePartExpr(t, dim.Expr) + if !ok { + return nil, fmt.Errorf("failed to extract date_part %s", dim.Expr.String()) + } + itr.point.Aux[baseIdx+i] = val + } + } + // Track points returned. itr.statsBuf.PointN++ @@ -670,6 +715,11 @@ type integerIterator struct { } opt query.IteratorOptions + // dpCond is non-nil when the condition uses date_part. It owns the + // rewritten condition (stored back into itr.opt.Condition) and publishes + // per-point part values into itr.m via SetTime. + dpCond *query.DatePartCondition + m map[string]interface{} // map used for condition evaluation point query.IntegerPoint // reusable buffer @@ -694,11 +744,27 @@ func newIntegerIterator(name string, tags query.Tags, opt query.IteratorOptions, } itr.stats = itr.statsBuf + // When the condition uses date_part (opt.NeedTimeRef), rewrite it once so + // no function call is evaluated per point: date_part calls become reserved + // variable references resolved by dpCond.SetTime in Next. itr.opt is this + // iterator's own copy, so storing the rewritten condition there leaves the + // caller's condition untouched. + if opt.NeedTimeRef { + if dp := query.NewDatePartCondition(opt.Condition, opt.Location); dp != nil { + itr.dpCond = dp + itr.opt.Condition = dp.Expr() + } + } + if len(aux) > 0 { itr.point.Aux = make([]interface{}, len(aux)) } - if opt.Condition != nil { + // Allocate the condition-evaluation map when there is a condition to + // evaluate. opt.NeedTimeRef is kept in the guard because the two fields are + // encoded independently over the iterator wire codec and a + // NeedTimeRef=true / Condition=nil options struct must stay safe. + if opt.Condition != nil || opt.NeedTimeRef { itr.m = make(map[string]interface{}, len(aux)+len(conds)) } itr.conds.names = condNames @@ -757,11 +823,34 @@ func (itr *integerIterator) Next() (*query.IntegerPoint, error) { itr.m[itr.conds.names[i]] = itr.conds.curs[i].nextAt(seek) } + // Publish the date_part values referenced by the condition for this + // point's timestamp. + if itr.dpCond != nil { + itr.dpCond.SetTime(seek, itr.m) + } + // Evaluate condition, if one exists. Retry if it fails. if itr.opt.Condition != nil && !itr.valuer.EvalBool(itr.opt.Condition) { continue } + // Compute date part dimension values in-place, only for points that pass + // the condition (the extraction is wasted on filtered points). The + // date_part dims occupy the last N slots of opt.Aux (added by select.go), + // so we overwrite the nil values left by the phantom aux cursors rather + // than appending, keeping Aux length consistent with scanner keys. + if len(itr.opt.DatePartDimensions) > 0 { + baseIdx := len(itr.opt.Aux) - len(itr.opt.DatePartDimensions) + t := time.Unix(0, seek).In(query.LocationOrUTC(itr.opt.Location)) + for i, dim := range itr.opt.DatePartDimensions { + val, ok := query.ExtractDatePartExpr(t, dim.Expr) + if !ok { + return nil, fmt.Errorf("failed to extract date_part %s", dim.Expr.String()) + } + itr.point.Aux[baseIdx+i] = val + } + } + // Track points returned. itr.statsBuf.PointN++ @@ -1150,6 +1239,11 @@ type unsignedIterator struct { } opt query.IteratorOptions + // dpCond is non-nil when the condition uses date_part. It owns the + // rewritten condition (stored back into itr.opt.Condition) and publishes + // per-point part values into itr.m via SetTime. + dpCond *query.DatePartCondition + m map[string]interface{} // map used for condition evaluation point query.UnsignedPoint // reusable buffer @@ -1174,11 +1268,27 @@ func newUnsignedIterator(name string, tags query.Tags, opt query.IteratorOptions } itr.stats = itr.statsBuf + // When the condition uses date_part (opt.NeedTimeRef), rewrite it once so + // no function call is evaluated per point: date_part calls become reserved + // variable references resolved by dpCond.SetTime in Next. itr.opt is this + // iterator's own copy, so storing the rewritten condition there leaves the + // caller's condition untouched. + if opt.NeedTimeRef { + if dp := query.NewDatePartCondition(opt.Condition, opt.Location); dp != nil { + itr.dpCond = dp + itr.opt.Condition = dp.Expr() + } + } + if len(aux) > 0 { itr.point.Aux = make([]interface{}, len(aux)) } - if opt.Condition != nil { + // Allocate the condition-evaluation map when there is a condition to + // evaluate. opt.NeedTimeRef is kept in the guard because the two fields are + // encoded independently over the iterator wire codec and a + // NeedTimeRef=true / Condition=nil options struct must stay safe. + if opt.Condition != nil || opt.NeedTimeRef { itr.m = make(map[string]interface{}, len(aux)+len(conds)) } itr.conds.names = condNames @@ -1237,11 +1347,34 @@ func (itr *unsignedIterator) Next() (*query.UnsignedPoint, error) { itr.m[itr.conds.names[i]] = itr.conds.curs[i].nextAt(seek) } + // Publish the date_part values referenced by the condition for this + // point's timestamp. + if itr.dpCond != nil { + itr.dpCond.SetTime(seek, itr.m) + } + // Evaluate condition, if one exists. Retry if it fails. if itr.opt.Condition != nil && !itr.valuer.EvalBool(itr.opt.Condition) { continue } + // Compute date part dimension values in-place, only for points that pass + // the condition (the extraction is wasted on filtered points). The + // date_part dims occupy the last N slots of opt.Aux (added by select.go), + // so we overwrite the nil values left by the phantom aux cursors rather + // than appending, keeping Aux length consistent with scanner keys. + if len(itr.opt.DatePartDimensions) > 0 { + baseIdx := len(itr.opt.Aux) - len(itr.opt.DatePartDimensions) + t := time.Unix(0, seek).In(query.LocationOrUTC(itr.opt.Location)) + for i, dim := range itr.opt.DatePartDimensions { + val, ok := query.ExtractDatePartExpr(t, dim.Expr) + if !ok { + return nil, fmt.Errorf("failed to extract date_part %s", dim.Expr.String()) + } + itr.point.Aux[baseIdx+i] = val + } + } + // Track points returned. itr.statsBuf.PointN++ @@ -1630,6 +1763,11 @@ type stringIterator struct { } opt query.IteratorOptions + // dpCond is non-nil when the condition uses date_part. It owns the + // rewritten condition (stored back into itr.opt.Condition) and publishes + // per-point part values into itr.m via SetTime. + dpCond *query.DatePartCondition + m map[string]interface{} // map used for condition evaluation point query.StringPoint // reusable buffer @@ -1654,11 +1792,27 @@ func newStringIterator(name string, tags query.Tags, opt query.IteratorOptions, } itr.stats = itr.statsBuf + // When the condition uses date_part (opt.NeedTimeRef), rewrite it once so + // no function call is evaluated per point: date_part calls become reserved + // variable references resolved by dpCond.SetTime in Next. itr.opt is this + // iterator's own copy, so storing the rewritten condition there leaves the + // caller's condition untouched. + if opt.NeedTimeRef { + if dp := query.NewDatePartCondition(opt.Condition, opt.Location); dp != nil { + itr.dpCond = dp + itr.opt.Condition = dp.Expr() + } + } + if len(aux) > 0 { itr.point.Aux = make([]interface{}, len(aux)) } - if opt.Condition != nil { + // Allocate the condition-evaluation map when there is a condition to + // evaluate. opt.NeedTimeRef is kept in the guard because the two fields are + // encoded independently over the iterator wire codec and a + // NeedTimeRef=true / Condition=nil options struct must stay safe. + if opt.Condition != nil || opt.NeedTimeRef { itr.m = make(map[string]interface{}, len(aux)+len(conds)) } itr.conds.names = condNames @@ -1717,11 +1871,34 @@ func (itr *stringIterator) Next() (*query.StringPoint, error) { itr.m[itr.conds.names[i]] = itr.conds.curs[i].nextAt(seek) } + // Publish the date_part values referenced by the condition for this + // point's timestamp. + if itr.dpCond != nil { + itr.dpCond.SetTime(seek, itr.m) + } + // Evaluate condition, if one exists. Retry if it fails. if itr.opt.Condition != nil && !itr.valuer.EvalBool(itr.opt.Condition) { continue } + // Compute date part dimension values in-place, only for points that pass + // the condition (the extraction is wasted on filtered points). The + // date_part dims occupy the last N slots of opt.Aux (added by select.go), + // so we overwrite the nil values left by the phantom aux cursors rather + // than appending, keeping Aux length consistent with scanner keys. + if len(itr.opt.DatePartDimensions) > 0 { + baseIdx := len(itr.opt.Aux) - len(itr.opt.DatePartDimensions) + t := time.Unix(0, seek).In(query.LocationOrUTC(itr.opt.Location)) + for i, dim := range itr.opt.DatePartDimensions { + val, ok := query.ExtractDatePartExpr(t, dim.Expr) + if !ok { + return nil, fmt.Errorf("failed to extract date_part %s", dim.Expr.String()) + } + itr.point.Aux[baseIdx+i] = val + } + } + // Track points returned. itr.statsBuf.PointN++ @@ -2110,6 +2287,11 @@ type booleanIterator struct { } opt query.IteratorOptions + // dpCond is non-nil when the condition uses date_part. It owns the + // rewritten condition (stored back into itr.opt.Condition) and publishes + // per-point part values into itr.m via SetTime. + dpCond *query.DatePartCondition + m map[string]interface{} // map used for condition evaluation point query.BooleanPoint // reusable buffer @@ -2134,11 +2316,27 @@ func newBooleanIterator(name string, tags query.Tags, opt query.IteratorOptions, } itr.stats = itr.statsBuf + // When the condition uses date_part (opt.NeedTimeRef), rewrite it once so + // no function call is evaluated per point: date_part calls become reserved + // variable references resolved by dpCond.SetTime in Next. itr.opt is this + // iterator's own copy, so storing the rewritten condition there leaves the + // caller's condition untouched. + if opt.NeedTimeRef { + if dp := query.NewDatePartCondition(opt.Condition, opt.Location); dp != nil { + itr.dpCond = dp + itr.opt.Condition = dp.Expr() + } + } + if len(aux) > 0 { itr.point.Aux = make([]interface{}, len(aux)) } - if opt.Condition != nil { + // Allocate the condition-evaluation map when there is a condition to + // evaluate. opt.NeedTimeRef is kept in the guard because the two fields are + // encoded independently over the iterator wire codec and a + // NeedTimeRef=true / Condition=nil options struct must stay safe. + if opt.Condition != nil || opt.NeedTimeRef { itr.m = make(map[string]interface{}, len(aux)+len(conds)) } itr.conds.names = condNames @@ -2197,11 +2395,34 @@ func (itr *booleanIterator) Next() (*query.BooleanPoint, error) { itr.m[itr.conds.names[i]] = itr.conds.curs[i].nextAt(seek) } + // Publish the date_part values referenced by the condition for this + // point's timestamp. + if itr.dpCond != nil { + itr.dpCond.SetTime(seek, itr.m) + } + // Evaluate condition, if one exists. Retry if it fails. if itr.opt.Condition != nil && !itr.valuer.EvalBool(itr.opt.Condition) { continue } + // Compute date part dimension values in-place, only for points that pass + // the condition (the extraction is wasted on filtered points). The + // date_part dims occupy the last N slots of opt.Aux (added by select.go), + // so we overwrite the nil values left by the phantom aux cursors rather + // than appending, keeping Aux length consistent with scanner keys. + if len(itr.opt.DatePartDimensions) > 0 { + baseIdx := len(itr.opt.Aux) - len(itr.opt.DatePartDimensions) + t := time.Unix(0, seek).In(query.LocationOrUTC(itr.opt.Location)) + for i, dim := range itr.opt.DatePartDimensions { + val, ok := query.ExtractDatePartExpr(t, dim.Expr) + if !ok { + return nil, fmt.Errorf("failed to extract date_part %s", dim.Expr.String()) + } + itr.point.Aux[baseIdx+i] = val + } + } + // Track points returned. itr.statsBuf.PointN++ diff --git a/tsdb/engine/tsm1/iterator.gen.go.tmpl b/tsdb/engine/tsm1/iterator.gen.go.tmpl index 2c6c12aa221..9ed7e5dc291 100644 --- a/tsdb/engine/tsm1/iterator.gen.go.tmpl +++ b/tsdb/engine/tsm1/iterator.gen.go.tmpl @@ -5,6 +5,7 @@ import ( "fmt" "runtime" "sync" + "time" "github.com/influxdata/influxdb/pkg/metrics" "github.com/influxdata/influxdb/pkg/tracing" @@ -188,6 +189,11 @@ type {{.name}}Iterator struct { } opt query.IteratorOptions + // dpCond is non-nil when the condition uses date_part. It owns the + // rewritten condition (stored back into itr.opt.Condition) and publishes + // per-point part values into itr.m via SetTime. + dpCond *query.DatePartCondition + m map[string]interface{} // map used for condition evaluation point query.{{.Name}}Point // reusable buffer @@ -199,9 +205,9 @@ type {{.name}}Iterator struct { func new{{.Name}}Iterator(name string, tags query.Tags, opt query.IteratorOptions, cur {{.name}}Cursor, aux []cursorAt, conds []cursorAt, condNames []string) *{{.name}}Iterator { itr := &{{.name}}Iterator{ - cur: cur, - aux: aux, - opt: opt, + cur: cur, + aux: aux, + opt: opt, point: query.{{.Name}}Point{ Name: name, Tags: tags, @@ -212,11 +218,27 @@ func new{{.Name}}Iterator(name string, tags query.Tags, opt query.IteratorOption } itr.stats = itr.statsBuf + // When the condition uses date_part (opt.NeedTimeRef), rewrite it once so + // no function call is evaluated per point: date_part calls become reserved + // variable references resolved by dpCond.SetTime in Next. itr.opt is this + // iterator's own copy, so storing the rewritten condition there leaves the + // caller's condition untouched. + if opt.NeedTimeRef { + if dp := query.NewDatePartCondition(opt.Condition, opt.Location); dp != nil { + itr.dpCond = dp + itr.opt.Condition = dp.Expr() + } + } + if len(aux) > 0 { itr.point.Aux = make([]interface{}, len(aux)) } - if opt.Condition != nil { + // Allocate the condition-evaluation map when there is a condition to + // evaluate. opt.NeedTimeRef is kept in the guard because the two fields are + // encoded independently over the iterator wire codec and a + // NeedTimeRef=true / Condition=nil options struct must stay safe. + if opt.Condition != nil || opt.NeedTimeRef { itr.m = make(map[string]interface{}, len(aux)+len(conds)) } itr.conds.names = condNames @@ -275,11 +297,34 @@ func (itr *{{.name}}Iterator) Next() (*query.{{.Name}}Point, error) { itr.m[itr.conds.names[i]] = itr.conds.curs[i].nextAt(seek) } + // Publish the date_part values referenced by the condition for this + // point's timestamp. + if itr.dpCond != nil { + itr.dpCond.SetTime(seek, itr.m) + } + // Evaluate condition, if one exists. Retry if it fails. if itr.opt.Condition != nil && !itr.valuer.EvalBool(itr.opt.Condition) { continue } + // Compute date part dimension values in-place, only for points that pass + // the condition (the extraction is wasted on filtered points). The + // date_part dims occupy the last N slots of opt.Aux (added by select.go), + // so we overwrite the nil values left by the phantom aux cursors rather + // than appending, keeping Aux length consistent with scanner keys. + if len(itr.opt.DatePartDimensions) > 0 { + baseIdx := len(itr.opt.Aux) - len(itr.opt.DatePartDimensions) + t := time.Unix(0, seek).In(query.LocationOrUTC(itr.opt.Location)) + for i, dim := range itr.opt.DatePartDimensions { + val, ok := query.ExtractDatePartExpr(t, dim.Expr) + if !ok { + return nil, fmt.Errorf("failed to extract date_part %s", dim.Expr.String()) + } + itr.point.Aux[baseIdx+i] = val + } + } + // Track points returned. itr.statsBuf.PointN++ @@ -609,3 +654,4 @@ func (c *{{.name}}DescendingCursor) nextTSM() { {{end}} var _ = fmt.Print + diff --git a/tsdb/engine/tsm1/iterator_test.go b/tsdb/engine/tsm1/iterator_test.go index 6327a01fa4f..4ae0ecce405 100644 --- a/tsdb/engine/tsm1/iterator_test.go +++ b/tsdb/engine/tsm1/iterator_test.go @@ -9,6 +9,7 @@ import ( "github.com/influxdata/influxdb/logger" "github.com/influxdata/influxdb/query" "github.com/influxdata/influxql" + "github.com/stretchr/testify/require" ) func BenchmarkIntegerIterator_Next(b *testing.B) { @@ -31,6 +32,155 @@ func BenchmarkIntegerIterator_Next(b *testing.B) { } } +// BenchmarkIntegerIterator_Next_Condition exercises the per-point condition +// evaluation (itr.valuer.EvalBool) for a WHERE-filtered query that does NOT use +// date_part. This is the common path: with opt.NeedTimeRef false the DatePartValuer +// must be left out of the eval chain so no extra valuer indirection is paid per +// scanned point. +func BenchmarkIntegerIterator_Next_Condition(b *testing.B) { + opt := query.IteratorOptions{ + Aux: []influxql.VarRef{{Val: "f1", Type: influxql.Integer}}, + Condition: influxql.MustParseExpr("f1 > 0"), + // NeedTimeRef defaults to false: the condition has no date_part. + } + aux := []cursorAt{&literalValueCursor{value: int64(1e3)}} + conds := []cursorAt{&literalValueCursor{value: int64(1e3)}} + condNames := []string{"f1"} + + cur := newIntegerIterator("m0", query.Tags{}, opt, &infiniteIntegerCursor{}, aux, conds, condNames) + + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + cur.Next() + } +} + +// BenchmarkIntegerIterator_Next_DatePartCondition measures the per-point cost of a +// WHERE condition that uses date_part. Relative to +// BenchmarkIntegerIterator_Next_Condition it adds the once-per-iterator condition +// rewrite's per-point work: extracting the referenced parts and publishing them to +// the eval map through the boxing cache. Timestamps advance one second per point: +// boxing a large int64 into the eval map allocates, while a constant time of zero +// would hit the runtime's small-integer cache and understate the cost. +func BenchmarkIntegerIterator_Next_DatePartCondition(b *testing.B) { + opt := query.IteratorOptions{ + Aux: []influxql.VarRef{{Val: "f1", Type: influxql.Integer}}, + Condition: influxql.MustParseExpr("f1 > 0 AND date_part('hour', time) < 12"), + NeedTimeRef: true, + EndTime: influxql.MaxTime, + Ascending: true, + } + aux := []cursorAt{&literalValueCursor{value: int64(1e3)}} + conds := []cursorAt{&literalValueCursor{value: int64(1e3)}} + condNames := []string{"f1"} + + cur := newIntegerIterator("m0", query.Tags{}, opt, &advancingIntegerCursor{}, aux, conds, condNames) + + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + cur.Next() + } +} + +// BenchmarkIntegerIterator_Next_DatePartDimension measures the per-point cost of a +// GROUP BY date_part dimension: the dimension value is extracted from the point +// timestamp and written into the trailing Aux slot for every point the iterator +// returns. +func BenchmarkIntegerIterator_Next_DatePartDimension(b *testing.B) { + opt := query.IteratorOptions{ + Aux: []influxql.VarRef{ + {Val: "f1", Type: influxql.Integer}, + {Val: "hour", Type: influxql.Integer}, + }, + DatePartDimensions: []query.DatePartDimension{{Expr: query.Hour}}, + EndTime: influxql.MaxTime, + Ascending: true, + } + aux := []cursorAt{ + &literalValueCursor{value: int64(1e3)}, + // The dimension slot is overwritten with the extracted date_part value. + &literalValueCursor{value: nil}, + } + + cur := newIntegerIterator("m0", query.Tags{}, opt, &advancingIntegerCursor{}, aux, nil, nil) + + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + cur.Next() + } +} + +// TestIntegerIterator_Next_DatePartCondition_ZeroAllocs pins the alloc-free +// date_part condition path: the condition is rewritten at construction and the +// per-point part values are published through a boxing cache, so steady-state +// scanning must not allocate. Uses advancing timestamps within a single hour so +// the cached boxed value stays valid. +func TestIntegerIterator_Next_DatePartCondition_ZeroAllocs(t *testing.T) { + opt := query.IteratorOptions{ + Aux: []influxql.VarRef{{Val: "f1", Type: influxql.Integer}}, + Condition: influxql.MustParseExpr("f1 > 0 AND date_part('hour', time) < 12"), + NeedTimeRef: true, + EndTime: influxql.MaxTime, + Ascending: true, + } + aux := []cursorAt{&literalValueCursor{value: int64(1e3)}} + conds := []cursorAt{&literalValueCursor{value: int64(1e3)}} + + cur := newIntegerIterator("m0", query.Tags{}, opt, &advancingIntegerCursor{}, aux, conds, []string{"f1"}) + + _, err := cur.Next() // prime the boxing cache + require.NoError(t, err) + + allocs := testing.AllocsPerRun(1000, func() { + cur.Next() + }) + require.Zero(t, allocs) +} + +// TestIntegerIterator_Next_NeedTimeRef_NilCondition guards against a nil-map panic +// when an IteratorOptions arrives with NeedTimeRef=true but Condition=nil. Locally +// newIteratorOptionsStmt derives NeedTimeRef from the condition, keeping the +// invariant (NeedTimeRef implies a condition), but the enterprise wire codec +// encodes the two fields independently and could deliver this combination; itr.m +// must still be allocated before the time-ref write. +func TestIntegerIterator_Next_NeedTimeRef_NilCondition(t *testing.T) { + opt := query.IteratorOptions{ + Aux: []influxql.VarRef{{Val: "f1", Type: influxql.Integer}}, + NeedTimeRef: true, + // Condition is intentionally nil. + } + aux := []cursorAt{&literalValueCursor{value: int64(1e3)}} + + cur := newIntegerIterator("m0", query.Tags{}, opt, &infiniteIntegerCursor{}, aux, nil, nil) + + require.NotPanics(t, func() { + _, err := cur.Next() + require.NoError(t, err) + }) +} + +// advancingIntegerCursor returns points whose timestamps advance one second per +// call, so per-point time handling boxes realistic (large) int64 values. +type advancingIntegerCursor struct { + t int64 +} + +func (*advancingIntegerCursor) close() error { + return nil +} + +func (c *advancingIntegerCursor) next() (t int64, v interface{}) { + return c.nextInteger() +} + +func (c *advancingIntegerCursor) nextInteger() (t int64, v int64) { + c.t += int64(time.Second) + return c.t, 0 +} + type infiniteIntegerCursor struct{} func (*infiniteIntegerCursor) close() error { diff --git a/tsdb/field_validator.go b/tsdb/field_validator.go index dd627a7116f..9c3ead32fe8 100644 --- a/tsdb/field_validator.go +++ b/tsdb/field_validator.go @@ -42,7 +42,7 @@ func ValidateAndCreateFields(mf *MeasurementFields, point models.Point, skipSize fieldKey := iter.FieldKey() // Skip fields name "time", they are illegal. - if bytes.Equal(fieldKey, TimeBytes) { + if bytes.Equal(fieldKey, models.TimeBytes) { if partialWriteError == nil { partialWriteError = &PartialWriteError{ Reason: fmt.Sprintf( diff --git a/tsdb/shard.go b/tsdb/shard.go index 9736bdc4758..d055e7b0c96 100644 --- a/tsdb/shard.go +++ b/tsdb/shard.go @@ -86,11 +86,6 @@ var ( fieldsIndexMagicNumber = []byte{0, 6, 1, 3} ) -var ( - // Static objects to prevent small allocs. - TimeBytes = []byte("time") -) - // A ShardError implements the error interface, and contains extra // context about the shard that generated the error. type ShardError struct { @@ -631,7 +626,7 @@ func (s *Shard) validateSeriesAndFields(points []models.Point, tracker StatsTrac tags := p.Tags() // Drop any series w/ a "time" tag, these are illegal - if v := tags.Get(TimeBytes); v != nil { + if v := tags.Get(models.TimeBytes); v != nil { dropped++ if reason == "" { reason = fmt.Sprintf( @@ -689,7 +684,7 @@ func (s *Shard) validateSeriesAndFields(points []models.Point, tracker StatsTrac iter := p.FieldIterator() validField := false for iter.Next() { - if bytes.Equal(iter.FieldKey(), TimeBytes) { + if bytes.Equal(iter.FieldKey(), models.TimeBytes) { continue } validField = true