pointblank/examples/04-column-value-checks/custom-validation-specially.qmd at 0bc457084881d1dafc18d9f3e028b2a8937076fd · posit-dev/pointblank · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
---
title: "Custom Validation with specially()"
description: "Create bespoke validations using specially() to implement domain-specific business rules."
notebook-links: false
page-navigation: false
toc: false
html-table-processing: none
---

```{python}
#| echo: false

import pointblank as pb
import polars as pl

def within_std_deviations(df, column, n_std=2):
    """Check if all values are within n standard deviations of the mean"""
    mean_val = df[column].mean()
    std_val = df[column].std()

    lower_bound = mean_val - (n_std * std_val)
    upper_bound = mean_val + (n_std * std_val)

    # Add a boolean column and return the modified DataFrame
    return df.with_columns(
        pl.col(column).is_between(lower_bound, upper_bound, closed="both").alias("validation_result")
    )

validation = (
    pb.Validate(
        data=pb.load_dataset(dataset="game_revenue", tbl_type="polars")
    )
    .specially(
        expr=lambda df: within_std_deviations(df, column="session_duration", n_std=2),
        brief="All values in column 'a' should be within 2 std devs of mean"
    )
    .specially(
        expr=lambda df: within_std_deviations(df, column="session_duration", n_std=3),
        brief="All values in column 'c' should be within 3 std devs of mean"
    )
    .interrogate()
)

validation
```

```python
import pointblank as pb
import polars as pl

def within_std_deviations(df, column, n_std=2):
    """Check if all values are within n standard deviations of the mean"""
    mean_val = df[column].mean()
    std_val = df[column].std()

    lower_bound = mean_val - (n_std * std_val)
    upper_bound = mean_val + (n_std * std_val)

    # Add a boolean column and return the modified DataFrame
    return df.with_columns(
        pl.col(column).is_between(lower_bound, upper_bound, closed="both").alias("validation_result")
    )

validation = (
    pb.Validate(
        data=pb.load_dataset(dataset="game_revenue", tbl_type="polars")
    )
    .specially(
        expr=lambda df: within_std_deviations(df, column="session_duration", n_std=2),
        brief="All values in column 'a' should be within 2 std devs of mean"
    )
    .specially(
        expr=lambda df: within_std_deviations(df, column="session_duration", n_std=3),
        brief="All values in column 'c' should be within 3 std devs of mean"
    )
    .interrogate()
)

validation
```

<details>
<summary>Preview of Input Table</summary>

```{python}
# | echo: false
pb.preview(pb.load_dataset(dataset="game_revenue", tbl_type="polars"))
```

</details>