feat: add regex pattern for resource and definition (#9)

* feat: add fields for regex pattern on resource and definition

* feat: update path explorer to accept regex pattern

* fix: error due to change of explore paths

* docs: update documentation and example

* fix: update distribution build to trim path

core/core.go

@@ -108,7 +108,7 @@ func (p *Pipeline) executeOnResource(resourceRcp *recipe.Resource, nameToValidat
 	if resourceRcp == nil {
 		return errors.New("resource recipe is nil")
 	}
-	resourcePaths, err := ExplorePaths(resourceRcp.Path, resourceRcp.Type, resourceRcp.Format)
+	resourcePaths, err := ExplorePaths(resourceRcp.Path, resourceRcp.Type, resourceRcp.Format, resourceRcp.RegexPattern)
 	if err != nil {
 		return err
 	}

core/explorer.go

@@ -1,18 +1,23 @@
 package core
 
 import (
+	"regexp"
 	"strings"
 
 	"github.com/gojek/optimus-extension-valor/registry/explorer"
 )
 
 // ExplorePaths explores the given root path for the type and format
-func ExplorePaths(rootPath, _type, format string) ([]string, error) {
+func ExplorePaths(rootPath, _type, format, regexPattern string) ([]string, error) {
 	exPath, err := explorer.Explorers.Get(_type)
 	if err != nil {
 		return nil, err
 	}
+	reg, err := regexp.Compile(regexPattern)
+	if err != nil {
+		return nil, err
+	}
 	return exPath(rootPath, func(path string) bool {
-		return strings.HasSuffix(path, format)
+		return reg.MatchString(path) && strings.HasSuffix(path, format)
 	})
 }

core/explorer_test.go

@@ -0,0 +1,75 @@
+package core_test
+
+import (
+	"testing"
+
+	"github.com/gojek/optimus-extension-valor/core"
+	"github.com/gojek/optimus-extension-valor/registry/explorer"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestExplorePaths(t *testing.T) {
+	originalExplorer := explorer.Explorers
+	defer func() { explorer.Explorers = originalExplorer }()
+
+	const (
+		rootPath = "."
+		_type    = "virtual"
+		format   = "go"
+	)
+
+	t.Run("should return nil and error if explorer registry returns error", func(t *testing.T) {
+		_type := "invalid_type"
+		pattern := `child_.+`
+
+		actualPaths, actualErr := core.ExplorePaths(rootPath, _type, format, pattern)
+
+		assert.Nil(t, actualPaths)
+		assert.Error(t, actualErr)
+	})
+
+	explorer.Explorers.Register(_type, func(root string, filter func(string) bool) ([]string, error) {
+		paths := []string{
+			"./core/testing/root.go",
+			"./core/testing/child_1.go",
+			"./core/testing/child_2.json",
+		}
+
+		var validPaths []string
+		for _, p := range paths {
+			if filter(p) {
+				validPaths = append(validPaths, p)
+			}
+		}
+
+		return validPaths, nil
+	})
+
+	t.Run("should return nil and error if regex pattern is invalid", func(t *testing.T) {
+		pattern := "*"
+
+		actualPaths, actualErr := core.ExplorePaths(rootPath, _type, format, pattern)
+
+		assert.Nil(t, actualPaths)
+		assert.Error(t, actualErr)
+	})
+
+	t.Run("should return all paths and nil if regex pattern is empty", func(t *testing.T) {
+		pattern := ""
+
+		actualPaths, actualErr := core.ExplorePaths(rootPath, _type, format, pattern)
+
+		assert.Len(t, actualPaths, 2)
+		assert.NoError(t, actualErr)
+	})
+
+	t.Run("should return as expected paths and nil if given regex pattern and format", func(t *testing.T) {
+		pattern := `child_.+`
+
+		actualPaths, actualErr := core.ExplorePaths(rootPath, _type, format, pattern)
+
+		assert.Len(t, actualPaths, 1)
+		assert.NoError(t, actualErr)
+	})
+}

core/loader.go

@@ -78,7 +78,7 @@ func (l *Loader) LoadProcedure(rcp *recipe.Procedure) (*model.Procedure, error)
 	if rcp == nil {
 		return nil, errors.New("procedure recipe is nil")
 	}
-	paths, err := ExplorePaths(rcp.Path, rcp.Type, jsonnetFormat)
+	paths, err := ExplorePaths(rcp.Path, rcp.Type, jsonnetFormat, "")
 	if err != nil {
 		return nil, err
 	}
@@ -134,7 +134,7 @@ func (l *Loader) LoadSchema(rcp *recipe.Schema) (*model.Schema, error) {
 	if rcp == nil {
 		return nil, errors.New("schema recipe is nil")
 	}
-	paths, err := ExplorePaths(rcp.Path, rcp.Type, jsonFormat)
+	paths, err := ExplorePaths(rcp.Path, rcp.Type, jsonFormat, "")
 	if err != nil {
 		return nil, err
 	}
@@ -209,7 +209,7 @@ func (l *Loader) LoadDefinition(rcp *recipe.Definition) (*model.Definition, erro
 	if rcp == nil {
 		return nil, errors.New("definition recipe is nil")
 	}
-	paths, err := ExplorePaths(rcp.Path, rcp.Type, rcp.Format)
+	paths, err := ExplorePaths(rcp.Path, rcp.Type, rcp.Format, rcp.RegexPattern)
 	if err != nil {
 		return nil, err
 	}

docs/recipe.md

@@ -16,7 +16,8 @@ resources:
   type: file
   path: ./example/resource
   format: json
-  batch_size: 3
+  batch_size: 3 # new in v0.0.5
+  regex_pattern: "[a-z]" # new in v0.0.6
   framework_names:
   - user_account_evaluation
 ...
@@ -68,7 +69,7 @@ Each resource is defined by a structure with the following fields:
         </tr>
         <tr>
             <td>batch_size</td>
-            <td>indicates the number of resources to be processed at one time</td>
+            <td><i><b>(new in v0.0.5)</b></i> indicates the number of resources to be processed at one time</td>
             <td>
                 <ul>
                     <li>if not set, default value is <i>4 (four)</i></li>
@@ -78,6 +79,12 @@ Each resource is defined by a structure with the following fields:
             </td>
             <td><i>4</i></td>
         </tr>
+        <tr>
+            <td>regex_pattern</td>
+            <td><i><b>(new in v0.0.6)</b></i> regex pattern to match the path with</td>
+            <td>should be a valid regex pattern</td>
+            <td><i>[a-z]</i></td>
+        </tr>
         <tr>
             <td>framework_names</td>
             <td>indicates what frameworks to be executed against a resource. execution of one framework name to another is done <b>sequentially</b> and <b>independently</b>.</td>
@@ -113,6 +120,7 @@ frameworks:
     format: json
     type: file
     path: ./example/definition
+    regex_pattern: "[a-z]" # new in v0.0.6
     function:
       type: file
       path: ./example/procedure/construct_membership_dictionary.jsonnet
@@ -212,6 +220,7 @@ name: memberships
 format: json
 type: file
 path: ./example/definition
+regex_pattern: "[a-z]" # new in v0.0.6
 function:
   type: file
   path: ./example/procedure/construct_membership_dictionary.jsonnet
@@ -224,6 +233,7 @@ name | the name of definition | it has to be unique within a framework only and
 type | the type of data to be read from the path specified by **path** | currently available is `file` | -
 format | the format being used to decode the data | currently available is `json` and `yaml` | -
 path | the path where to read the actual data from | the valid format based on the **type** | -
+regex_pattern | **_(new in v0.0.6)_** regex pattern to match the path with | valid regex pattern | -
 function | an optional instruction to build a definition, where the instruction follows the [Jsonnet](https://jsonnet.org/) format | - | dictionary where the key is the **name** and the value is up to the actual function defined under **function.path**
 function.type | defines the type of path specified by **function.path** | it should be valid for the given **function.path** with currently available is `file` | -
 function.path | defines the path where to read the actual function | should be valid according to the **function.type** | -

recipe/recipe.go

@@ -11,6 +11,7 @@ type Resource struct {
 	Name           string   `yaml:"name" validate:"required"`
 	Format         string   `yaml:"format" validate:"required,oneof=json yaml"`
 	Type           string   `yaml:"type" validate:"required,oneof=dir file"`
+	RegexPattern   string   `yaml:"regex_pattern"`
 	Path           string   `yaml:"path" validate:"required"`
 	BatchSize      int      `yaml:"batch_size"`
 	FrameworkNames []string `yaml:"framework_names" validate:"required,min=1"`
@@ -26,11 +27,12 @@ type Framework struct {
 
 // Definition is a recipe on how and where to read the actual Definition data
 type Definition struct {
-	Name     string    `yaml:"name" validate:"required"`
-	Format   string    `yaml:"format" validate:"required,oneof=json yaml"`
-	Type     string    `yaml:"type" validate:"required,oneof=dir file"`
-	Path     string    `yaml:"path" validate:"required"`
-	Function *Function `yaml:"function"`
+	Name         string    `yaml:"name" validate:"required"`
+	Format       string    `yaml:"format" validate:"required,oneof=json yaml"`
+	Type         string    `yaml:"type" validate:"required,oneof=dir file"`
+	Path         string    `yaml:"path" validate:"required"`
+	RegexPattern string    `yaml:"regex_pattern"`
+	Function     *Function `yaml:"function"`
 }
 
 // Function is a recipe on how to construct a Definition

script/build.sh

@@ -11,6 +11,6 @@ for os in ${os_list[*]}; do
     for arch in ${arch_list[*]}; do
         file_name="${name}_${tag}_${os}-${arch}"
         path="${dir_output}/${file_name}"
-        GOOS=$os GOARCH=$arch go build -o ${path}
+        GOOS=$os GOARCH=$arch go build -trimpath -o ${path}
     done
 done

valor.example.yaml

@@ -3,6 +3,8 @@ resources:
   type: file
   path: ./example/resource
   format: json
+  batch_size: 3 # new in v0.0.5
+  regex_pattern: "[a-z]" # new in v0.0.6
   framework_names:
   - user_account_evaluation
 
@@ -24,6 +26,7 @@ frameworks:
     format: json
     type: file
     path: ./example/definition
+    regex_pattern: "[a-z]" # new in v0.0.6
     function:
       type: file
       path: ./example/procedure/construct_membership_dictionary.jsonnet