[persistent collections] based on PR-866

collection/collection.go

@@ -43,24 +43,48 @@ type Keyed interface {
 	FindString(key string) []types.MatchData
 }
 
+type Editable interface {
+	Keyed
+
+	// Remove deletes the key from the CollectionMap
+	Remove(key string)
+
+	// Set will replace the key's value with this slice
+	Set(key string, values []string)
+
+	// TODO: in v4 this should contain setters for Map and Persistence
+}
+
 // Map are used to store VARIABLE data
 // for transactions, this data structured is designed
 // to store slices of data for keys
 // Important: CollectionMaps ARE NOT concurrent safe
 type Map interface {
-	Keyed
+	Editable
 
 	// Add a value to some key
 	Add(key string, value string)
 
-	// Set will replace the key's value with this slice
-	Set(key string, values []string)
-
 	// SetIndex will place the value under the index
 	// If the index is higher than the current size of the CollectionMap
 	// it will be appended
 	SetIndex(key string, index int, value string)
+}
 
-	// Remove deletes the key from the CollectionMap
-	Remove(key string)
+// Persistent collections won't use arrays as values
+// They are designed for collections that will be stored
+type Persistent interface {
+	Editable
+
+	// // Initializes the input as the collection key
+	// Init(key string)
+
+	// // Sum will add the value to the key
+	// Sum(key string, sum int)
+
+	// // SetOne will replace the key's value with this string
+	// SetOne(key string, value string)
+
+	// SetTTL will set the TTL for the key
+	SetTTL(key string, ttl int)
 }

config.go

@@ -63,6 +63,9 @@ type WAFConfig interface {
 
 	// WithRootFS configures the root file system.
 	WithRootFS(fs fs.FS) WAFConfig
+
+	// WithPersistenceEngine sets the persistence engine to be used
+	WithPersistenceEngine(engine plugintypes.PersistenceEngine) WAFConfig
 }
 
 // NewWAFConfig creates a new WAFConfig with the default settings.
@@ -105,6 +108,7 @@ type wafConfig struct {
 	debugLogger              debuglog.Logger
 	errorCallback            func(rule types.MatchedRule)
 	fsRoot                   fs.FS
+	persistenceEngine        plugintypes.PersistenceEngine
 }
 
 func (c *wafConfig) WithRules(rules ...*corazawaf.Rule) WAFConfig {
@@ -193,6 +197,12 @@ func (c *wafConfig) WithResponseBodyMimeTypes(mimeTypes []string) WAFConfig {
 	return ret
 }
 
+func (c *wafConfig) WithPersistenceEngine(pe plugintypes.PersistenceEngine) WAFConfig {
+	ret := c.clone()
+	ret.persistenceEngine = pe
+	return ret
+}
+
 type auditLogConfig struct {
 	relevantOnly bool
 	parts        types.AuditLogParts

examples/http-server/go.mod → examples/http-server/minimal/go.mod

@@ -1,4 +1,4 @@
-module github.com/corazawaf/coraza/v3/examples/http-server
+module github.com/corazawaf/coraza/v3/examples/http-server/minimal
 
 go 1.22.0
 

examples/http-server/go.sum → examples/http-server/minimal/go.sum

@@ -20,6 +20,7 @@ github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhso
 golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0=
 golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
 golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
 golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=

examples/http-server/persistence_collection/custom/README.md

@@ -0,0 +1,22 @@
+# HTTP-Server with Coraza
+
+This example is intended to provide example with persistence storage based rulesets.
+
+## Run the example
+
+```bash
+go run .
+```
+
+The server will be reachable at `http://localhost:8090`.
+
+Example for the rate limit requests from the same X-Session-ID:
+
+```bash
+curl --header 'X-Session-ID: unique-session-id' http://localhost:8090/
+```
+
+- True negative request (200 OK) // 2 calls
+- True positive request (403 Forbidden) // 3d call
+- Wait for 10 seconds (ttl is set in the `expirevar` directive)
+- repeat

examples/http-server/persistence_collection/custom/customttl/cache.go

@@ -0,0 +1,121 @@
+package customttl
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/jellydator/ttlcache/v3"
+)
+
+type (
+	Engine struct {
+		store *ttlcache.Cache[string, collectionRecord]
+	}
+
+	collectionRecord struct {
+		key           string
+		val           string
+		updateCounter int
+		isNew         bool
+		createTime    int64
+		timeout       int64
+	}
+)
+
+func NewTTLCacheEngine() *Engine {
+	cache := ttlcache.New[string, collectionRecord]()
+
+	return &Engine{
+		store: cache,
+	}
+}
+
+func (e *Engine) Get(collectionName string, collectionKey string, key string) (string, error) {
+	res := e.get(collectionName, collectionKey, key)
+	return res.val, nil
+}
+
+func (e *Engine) Set(collection string, collectionKey string, key string, value string) error {
+	e.set(collection, collectionKey, key, value)
+	return nil
+}
+
+func (e *Engine) Remove(collection string, collectionKey string, key string) error {
+	k := getKey(collection, collectionKey, key)
+	e.store.Delete(k)
+	return nil
+}
+
+func (e *Engine) SetTTL(collection string, collectionKey string, key string, ttl int) error {
+	record := e.get(collection, collectionKey, key)
+	if record.isEmpty() {
+		return nil
+	}
+	if !record.isNew { // set ttl only for just created records
+		return nil
+	}
+	record.isNew = false
+	durTTL := time.Duration(ttl) * time.Second
+	record.timeout = time.Now().Add(durTTL).Unix()
+	e.store.Set(getKey(collection, collectionKey, key), record, durTTL)
+	return nil
+}
+
+func (e *Engine) get(collectionName string, collectionKey string, key string) collectionRecord {
+	k := getKey(collectionName, collectionKey, key)
+	record := e.store.Get(k)
+	if record == nil {
+		return collectionRecord{}
+	}
+	return record.Value()
+}
+
+func (e *Engine) set(collection string, collectionKey string, key string, value string) {
+	k := getKey(collection, collectionKey, key)
+	record := e.get(collection, collectionKey, key)
+	if record.isEmpty() {
+		// create new record
+		e.store.Set(k, collectionRecord{
+			key:           key,
+			val:           value,
+			timeout:       int64(ttlcache.NoTTL),
+			createTime:    time.Now().Unix(),
+			updateCounter: 0,
+			isNew:         true,
+		}, ttlcache.NoTTL) // we set ttl only in SetTTL method
+	} else {
+		// update existing record
+		record.val = value
+		record.updateCounter++
+		// unfortunately this library doesn't provide the way to update data without setting ttl once again
+		ttl := time.Duration(record.timeout)
+		if ttl != ttlcache.NoTTL {
+			ttl = time.Unix(record.timeout, 0).Sub(time.Now())
+		}
+		e.store.Set(k, record, ttl)
+	}
+}
+
+func (cr collectionRecord) isEmpty() bool {
+	return cr == collectionRecord{}
+}
+
+func getKey(collectionName, collectionKey, key string) string {
+	return fmt.Sprintf("%s_%s_%s", collectionName, collectionKey, key)
+}
+
+func (d *Engine) Open(_ string, _ int) error {
+	return nil
+}
+
+func (d *Engine) Close() error {
+	return nil
+}
+
+func (d *Engine) Sum(collectionName string, collectionKey string, key string, sum int) error {
+	return nil
+}
+
+func (d *Engine) All(collectionName string, collectionKey string) (map[string]string, error) {
+	return nil, nil
+}

examples/http-server/persistence_collection/custom/go.mod

@@ -0,0 +1,26 @@
+module github.com/corazawaf/coraza/v3/examples/http-server/persistence_collection/custom
+
+go 1.22.3
+
+toolchain go1.23.4
+
+require (
+	github.com/corazawaf/coraza/v3 v3.2.2
+	github.com/jellydator/ttlcache/v3 v3.3.0
+)
+
+require (
+	github.com/corazawaf/libinjection-go v0.2.2 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/magefile/mage v1.15.1-0.20241126214340-bdc92f694516 // indirect
+	github.com/petar-dambovaliev/aho-corasick v0.0.0-20240411101913-e07a1f0e8eb4 // indirect
+	github.com/stretchr/testify v1.10.0 // indirect
+	github.com/tidwall/gjson v1.18.0 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.1 // indirect
+	github.com/valllabh/ocsf-schema-golang v1.0.3 // indirect
+	golang.org/x/net v0.33.0 // indirect
+	golang.org/x/sync v0.10.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
+	rsc.io/binaryregexp v0.2.0 // indirect
+)

examples/http-server/persistence_collection/custom/go.sum

@@ -0,0 +1,53 @@
+github.com/corazawaf/coraza-coreruleset v0.0.0-20240226094324-415b1017abdc h1:OlJhrgI3I+FLUCTI3JJW8MoqyM78WbqJjecqMnqG+wc=
+github.com/corazawaf/coraza-coreruleset v0.0.0-20240226094324-415b1017abdc/go.mod h1:7rsocqNDkTCira5T0M7buoKR2ehh7YZiPkzxRuAgvVU=
+github.com/corazawaf/coraza/v3 v3.2.2 h1:zZxyLRJ7o8W11BB8XE94X3CxZmYTk0/RhHc1dQxqtq8=
+github.com/corazawaf/coraza/v3 v3.2.2/go.mod h1:73JSSNpNrWeF8K+TqKAc7Apxm3uz2rBrspsYKR88tGk=
+github.com/corazawaf/libinjection-go v0.2.2 h1:Chzodvb6+NXh6wew5/yhD0Ggioif9ACrQGR4qjTCs1g=
+github.com/corazawaf/libinjection-go v0.2.2/go.mod h1:OP4TM7xdJ2skyXqNX1AN1wN5nNZEmJNuWbNPOItn7aw=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/foxcpp/go-mockdns v1.1.0 h1:jI0rD8M0wuYAxL7r/ynTrCQQq0BVqfB99Vgk7DlmewI=
+github.com/foxcpp/go-mockdns v1.1.0/go.mod h1:IhLeSFGed3mJIAXPH2aiRQB+kqz7oqu8ld2qVbOu7Wk=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/jcchavezs/mergefs v0.1.0 h1:7oteO7Ocl/fnfFMkoVLJxTveCjrsd//UB0j89xmnpec=
+github.com/jcchavezs/mergefs v0.1.0/go.mod h1:eRLTrsA+vFwQZ48hj8p8gki/5v9C2bFtHH5Mnn4bcGk=
+github.com/jellydator/ttlcache/v3 v3.3.0 h1:BdoC9cE81qXfrxeb9eoJi9dWrdhSuwXMAnHTbnBm4Wc=
+github.com/jellydator/ttlcache/v3 v3.3.0/go.mod h1:bj2/e0l4jRnQdrnSTaGTsh4GSXvMjQcy41i7th0GVGw=
+github.com/magefile/mage v1.15.1-0.20241126214340-bdc92f694516 h1:aAO0L0ulox6m/CLRYvJff+jWXYYCKGpEm3os7dM/Z+M=
+github.com/magefile/mage v1.15.1-0.20241126214340-bdc92f694516/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
+github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM=
+github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk=
+github.com/petar-dambovaliev/aho-corasick v0.0.0-20240411101913-e07a1f0e8eb4 h1:1Kw2vDBXmjop+LclnzCb/fFy+sgb3gYARwfmoUcQe6o=
+github.com/petar-dambovaliev/aho-corasick v0.0.0-20240411101913-e07a1f0e8eb4/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
+github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
+github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/valllabh/ocsf-schema-golang v1.0.3 h1:eR8k/3jP/OOqB8LRCtdJ4U+vlgd/gk5y3KMXoodrsrw=
+github.com/valllabh/ocsf-schema-golang v1.0.3/go.mod h1:sZ3as9xqm1SSK5feFWIR2CuGeGRhsM7TR1MbpBctzPk=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0=
+golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
+golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA=
+golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c=
+google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

examples/http-server/persistence_collection/custom/main.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/corazawaf/coraza/v3"
+	"github.com/corazawaf/coraza/v3/examples/http-server/persistence_collection/custom/customttl"
+	txhttp "github.com/corazawaf/coraza/v3/http"
+	"github.com/corazawaf/coraza/v3/types"
+)
+
+func exampleHandler(w http.ResponseWriter, req *http.Request) {
+	w.Header().Set("Content-Type", "text/plain")
+	resBody := "Hello world, transaction not disrupted."
+
+	if body := os.Getenv("RESPONSE_BODY"); body != "" {
+		resBody = body
+	}
+
+	if h := os.Getenv("RESPONSE_HEADERS"); h != "" {
+		key, val, _ := strings.Cut(h, ":")
+		w.Header().Set(key, val)
+	}
+
+	// The server generates the response
+	w.Write([]byte(resBody))
+}
+
+func main() {
+	directiveFile := "./session.conf"
+	waf := createWAF(directiveFile)
+
+	http.Handle("/", txhttp.WrapHandler(waf, http.HandlerFunc(exampleHandler)))
+
+	fmt.Println("Server is running. Listening port: 8090")
+
+	log.Fatal(http.ListenAndServe(":8090", nil))
+}
+
+func createWAF(directivesFile string) coraza.WAF {
+	if s := os.Getenv("DIRECTIVES_FILE"); s != "" {
+		directivesFile = s
+	}
+
+	pe := customttl.NewTTLCacheEngine()
+
+	waf, err := coraza.NewWAF(
+		coraza.NewWAFConfig().
+			WithErrorCallback(logError).
+			WithDirectivesFromFile(directivesFile).
+			WithPersistenceEngine(pe),
+	)
+	if err != nil {
+		log.Fatal(err)
+	}
+	return waf
+}
+
+func logError(error types.MatchedRule) {
+	msg := error.ErrorLog()
+	fmt.Printf("[logError][%s] %s\n", error.Rule().Severity(), msg)
+}