Skip to content

Commit

Permalink
refactor: pass DB dir to trivy-db (#7057)
Browse files Browse the repository at this point in the history
Signed-off-by: knqyf263 <knqyf263@gmail.com>
  • Loading branch information
knqyf263 authored Jul 2, 2024
1 parent 6a307bb commit fc6b3a7
Show file tree
Hide file tree
Showing 13 changed files with 91 additions and 90 deletions.
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ require (
github.com/aquasecurity/testdocker v0.0.0-20240613070307-2c3868d658ac
github.com/aquasecurity/tml v0.6.1
github.com/aquasecurity/trivy-checks v0.13.0
github.com/aquasecurity/trivy-db v0.0.0-20231005141211-4fc651f7ac8d
github.com/aquasecurity/trivy-db v0.0.0-20240701103400-8e907467e9ab
github.com/aquasecurity/trivy-java-db v0.0.0-20240109071736-184bd7481d48
github.com/aquasecurity/trivy-kubernetes v0.6.7-0.20240627095026-cf9d48837f6d
github.com/aws/aws-sdk-go-v2 v1.27.2
Expand Down Expand Up @@ -192,7 +192,7 @@ require (
github.com/containerd/ttrpc v1.2.4 // indirect
github.com/containerd/typeurl/v2 v2.1.1 // indirect
github.com/cpuguy83/dockercfg v0.3.1 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
Expand Down
7 changes: 4 additions & 3 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -771,8 +771,8 @@ github.com/aquasecurity/tml v0.6.1 h1:y2ZlGSfrhnn7t4ZJ/0rotuH+v5Jgv6BDDO5jB6A9gw
github.com/aquasecurity/tml v0.6.1/go.mod h1:OnYMWY5lvI9ejU7yH9LCberWaaTBW7hBFsITiIMY2yY=
github.com/aquasecurity/trivy-checks v0.13.0 h1:na6PTdY4U0uK/fjz3HNRYBxvYSJ8vgTb57a5T8Y5t9w=
github.com/aquasecurity/trivy-checks v0.13.0/go.mod h1:Xec/SMVGV66I7RgUqOX9MEr+YxBqHXDVLTYmpspPi3E=
github.com/aquasecurity/trivy-db v0.0.0-20231005141211-4fc651f7ac8d h1:fjI9mkoTUAkbGqpzt9nJsO24RAdfG+ZSiLFj0G2jO8c=
github.com/aquasecurity/trivy-db v0.0.0-20231005141211-4fc651f7ac8d/go.mod h1:cj9/QmD9N3OZnKQMp+/DvdV+ym3HyIkd4e+F0ZM3ZGs=
github.com/aquasecurity/trivy-db v0.0.0-20240701103400-8e907467e9ab h1:EmpLGFgRJOstPWDpL4KW+Xap4zRYxyctXDTj5luMQdE=
github.com/aquasecurity/trivy-db v0.0.0-20240701103400-8e907467e9ab/go.mod h1:f+wSW9D5txv8S+tw4D4WNOibaUJYwvNnQuQlGQ8gO6c=
github.com/aquasecurity/trivy-java-db v0.0.0-20240109071736-184bd7481d48 h1:JVgBIuIYbwG+ekC5lUHUpGJboPYiCcxiz06RCtz8neI=
github.com/aquasecurity/trivy-java-db v0.0.0-20240109071736-184bd7481d48/go.mod h1:Ldya37FLi0e/5Cjq2T5Bty7cFkzUDwTcPeQua+2M8i8=
github.com/aquasecurity/trivy-kubernetes v0.6.7-0.20240627095026-cf9d48837f6d h1:z5Ug+gqNjgHzCo7rmv6wKTmyJ8E3bAVEU2AASo3740s=
Expand Down Expand Up @@ -1019,8 +1019,9 @@ github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHf
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM=
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
Expand Down
13 changes: 3 additions & 10 deletions integration/integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,11 @@ import (
"github.com/stretchr/testify/require"
"github.com/xeipuuv/gojsonschema"

"github.com/aquasecurity/trivy-db/pkg/db"
"github.com/aquasecurity/trivy-db/pkg/metadata"

"github.com/aquasecurity/trivy/internal/dbtest"
"github.com/aquasecurity/trivy/pkg/clock"
"github.com/aquasecurity/trivy/pkg/commands"
"github.com/aquasecurity/trivy/pkg/db"
"github.com/aquasecurity/trivy/pkg/types"
"github.com/aquasecurity/trivy/pkg/uuid"

Expand All @@ -56,15 +55,9 @@ func initDB(t *testing.T) string {
}

cacheDir := dbtest.InitDB(t, fixtures)
defer db.Close()

dbDir := filepath.Dir(db.Path(cacheDir))

metadataFile := filepath.Join(dbDir, "metadata.json")
f, err := os.Create(metadataFile)
require.NoError(t, err)
defer dbtest.Close()

err = json.NewEncoder(f).Encode(metadata.Metadata{
err = metadata.NewClient(db.Dir(cacheDir)).Update(metadata.Metadata{
Version: db.SchemaVersion,
NextUpdate: time.Now().Add(24 * time.Hour),
UpdatedAt: time.Now(),
Expand Down
13 changes: 7 additions & 6 deletions internal/dbtest/db.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,17 +9,18 @@ import (
"github.com/stretchr/testify/require"

fixtures "github.com/aquasecurity/bolt-fixtures"
"github.com/aquasecurity/trivy-db/pkg/db"
trivydb "github.com/aquasecurity/trivy-db/pkg/db"
jdb "github.com/aquasecurity/trivy-java-db/pkg/db"
"github.com/aquasecurity/trivy/pkg/db"
)

// InitDB initializes testing database.
func InitDB(t *testing.T, fixtureFiles []string) string {
// Create a temp dir
dir := t.TempDir()
cacheDir := t.TempDir()

dbPath := db.Path(dir)
dbDir := filepath.Dir(dbPath)
dbDir := db.Dir(cacheDir)
dbPath := trivydb.Path(dbDir)
err := os.MkdirAll(dbDir, 0700)
require.NoError(t, err)

Expand All @@ -30,9 +31,9 @@ func InitDB(t *testing.T, fixtureFiles []string) string {
require.NoError(t, loader.Close())

// Initialize DB
require.NoError(t, db.Init(dir))
require.NoError(t, db.Init(dbDir))

return dir
return cacheDir
}

func Close() error {
Expand Down
4 changes: 2 additions & 2 deletions pkg/commands/artifact/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,9 @@ import (
"github.com/spf13/viper"
"golang.org/x/xerrors"

"github.com/aquasecurity/trivy-db/pkg/db"
"github.com/aquasecurity/trivy/pkg/cache"
"github.com/aquasecurity/trivy/pkg/commands/operation"
"github.com/aquasecurity/trivy/pkg/db"
"github.com/aquasecurity/trivy/pkg/fanal/analyzer"
"github.com/aquasecurity/trivy/pkg/fanal/artifact"
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
Expand Down Expand Up @@ -295,7 +295,7 @@ func (r *runner) initDB(ctx context.Context, opts flag.Options) error {
return SkipScan
}

if err := db.Init(opts.CacheDir); err != nil {
if err := db.Init(db.Dir(opts.CacheDir)); err != nil {
return xerrors.Errorf("error in vulnerability DB initialize: %w", err)
}
r.dbOpen = true
Expand Down
2 changes: 1 addition & 1 deletion pkg/commands/clean/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ func cleanScanCache(ctx context.Context, opts flag.Options) error {

func cleanVulnerabilityDB(ctx context.Context, opts flag.Options) error {
log.InfoContext(ctx, "Removing vulnerability database...")
if err := db.NewClient(opts.CacheDir, true).Clear(ctx); err != nil {
if err := db.NewClient(db.Dir(opts.CacheDir), true).Clear(ctx); err != nil {
return xerrors.Errorf("clear vulnerability database: %w", err)

}
Expand Down
19 changes: 4 additions & 15 deletions pkg/commands/operation/operation.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (
"github.com/google/go-containerregistry/pkg/name"
"golang.org/x/xerrors"

"github.com/aquasecurity/trivy-db/pkg/metadata"
"github.com/aquasecurity/trivy/pkg/db"
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
"github.com/aquasecurity/trivy/pkg/flag"
Expand All @@ -24,7 +23,8 @@ func DownloadDB(ctx context.Context, appVersion, cacheDir string, dbRepository n
mu.Lock()
defer mu.Unlock()

client := db.NewClient(cacheDir, quiet, db.WithDBRepository(dbRepository))
dbDir := db.Dir(cacheDir)
client := db.NewClient(dbDir, quiet, db.WithDBRepository(dbRepository))
needsUpdate, err := client.NeedsUpdate(ctx, appVersion, skipUpdate)
if err != nil {
return xerrors.Errorf("database error: %w", err)
Expand All @@ -33,29 +33,18 @@ func DownloadDB(ctx context.Context, appVersion, cacheDir string, dbRepository n
if needsUpdate {
log.Info("Need to update DB")
log.Info("Downloading DB...", log.String("repository", dbRepository.String()))
if err = client.Download(ctx, cacheDir, opt); err != nil {
if err = client.Download(ctx, dbDir, opt); err != nil {
return xerrors.Errorf("failed to download vulnerability DB: %w", err)
}
}

// for debug
if err = showDBInfo(cacheDir); err != nil {
if err = client.ShowInfo(); err != nil {
return xerrors.Errorf("failed to show database info: %w", err)
}
return nil
}

func showDBInfo(cacheDir string) error {
m := metadata.NewClient(cacheDir)
meta, err := m.Get()
if err != nil {
return xerrors.Errorf("something wrong with DB: %w", err)
}
log.Debug("DB info", log.Int("schema", meta.Version), log.Time("updated_at", meta.UpdatedAt),
log.Time("next_update", meta.NextUpdate), log.Time("downloaded_at", meta.DownloadedAt))
return nil
}

// InitBuiltinPolicies downloads the built-in policies and loads them
func InitBuiltinPolicies(ctx context.Context, cacheDir string, quiet, skipUpdate bool, checkBundleRepository string, registryOpts ftypes.RegistryOptions) ([]string, error) {
mu.Lock()
Expand Down
4 changes: 2 additions & 2 deletions pkg/commands/server/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@ import (

"golang.org/x/xerrors"

"github.com/aquasecurity/trivy-db/pkg/db"
"github.com/aquasecurity/trivy/pkg/cache"
"github.com/aquasecurity/trivy/pkg/commands/operation"
"github.com/aquasecurity/trivy/pkg/db"
"github.com/aquasecurity/trivy/pkg/flag"
"github.com/aquasecurity/trivy/pkg/log"
"github.com/aquasecurity/trivy/pkg/module"
Expand Down Expand Up @@ -35,7 +35,7 @@ func Run(ctx context.Context, opts flag.Options) (err error) {
return nil
}

if err = db.Init(opts.CacheDir); err != nil {
if err = db.Init(db.Dir(opts.CacheDir)); err != nil {
return xerrors.Errorf("error in vulnerability DB initialize: %w", err)
}

Expand Down
37 changes: 28 additions & 9 deletions pkg/db/db.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"os"
"path/filepath"
"time"

"github.com/google/go-containerregistry/pkg/name"
Expand All @@ -28,6 +29,10 @@ const (
var (
DefaultRepository = fmt.Sprintf("%s:%d", "ghcr.io/aquasecurity/trivy-db", db.SchemaVersion)
defaultRepository, _ = name.NewTag(DefaultRepository)

Init = db.Init
Close = db.Close
Path = db.Path
)

type options struct {
Expand Down Expand Up @@ -56,13 +61,17 @@ func WithDBRepository(dbRepository name.Reference) Option {
type Client struct {
*options

cacheDir string
dbDir string
metadata metadata.Client
quiet bool
}

func Dir(cacheDir string) string {
return filepath.Join(cacheDir, "db")
}

// NewClient is the factory method for DB client
func NewClient(cacheDir string, quiet bool, opts ...Option) *Client {
func NewClient(dbDir string, quiet bool, opts ...Option) *Client {
o := &options{
dbRepository: defaultRepository,
}
Expand All @@ -73,8 +82,8 @@ func NewClient(cacheDir string, quiet bool, opts ...Option) *Client {

return &Client{
options: o,
cacheDir: cacheDir,
metadata: metadata.NewClient(cacheDir),
dbDir: dbDir,
metadata: metadata.NewClient(dbDir),
quiet: quiet,
}
}
Expand Down Expand Up @@ -149,7 +158,7 @@ func (c *Client) Download(ctx context.Context, dst string, opt types.RegistryOpt
return xerrors.Errorf("OCI artifact error: %w", err)
}

if err = art.Download(ctx, db.Dir(dst), oci.DownloadOption{MediaType: dbMediaType}); err != nil {
if err = art.Download(ctx, dst, oci.DownloadOption{MediaType: dbMediaType}); err != nil {
return xerrors.Errorf("database download error: %w", err)
}

Expand All @@ -159,19 +168,19 @@ func (c *Client) Download(ctx context.Context, dst string, opt types.RegistryOpt
return nil
}

func (c *Client) Clear(ctx context.Context) error {
if err := os.RemoveAll(db.Dir(c.cacheDir)); err != nil {
func (c *Client) Clear(_ context.Context) error {
if err := os.RemoveAll(c.dbDir); err != nil {
return xerrors.Errorf("failed to remove vulnerability database: %w", err)
}
return nil
}

func (c *Client) updateDownloadedAt(ctx context.Context, dst string) error {
func (c *Client) updateDownloadedAt(ctx context.Context, dbDir string) error {
log.Debug("Updating database metadata...")

// We have to initialize a metadata client here
// since the destination may be different from the cache directory.
client := metadata.NewClient(dst)
client := metadata.NewClient(dbDir)
meta, err := client.Get()
if err != nil {
return xerrors.Errorf("unable to get metadata: %w", err)
Expand Down Expand Up @@ -207,3 +216,13 @@ func (c *Client) initOCIArtifact(opt types.RegistryOptions) (*oci.Artifact, erro
}
return art, nil
}

func (c *Client) ShowInfo() error {
meta, err := c.metadata.Get()
if err != nil {
return xerrors.Errorf("something wrong with DB: %w", err)
}
log.Debug("DB info", log.Int("schema", meta.Version), log.Time("updated_at", meta.UpdatedAt),
log.Time("next_update", meta.NextUpdate), log.Time("downloaded_at", meta.DownloadedAt))
return nil
}
Loading

0 comments on commit fc6b3a7

Please sign in to comment.