[scanner] 新增哈希接口,支持多种哈希实现;新增MD5哈希实现
This commit is contained in:
parent
92b8ae9648
commit
880cbb6db9
|
|
@ -76,15 +76,8 @@ var StartCmd = &cobra.Command{
|
||||||
logger.Log.Info("正在扫描文件系统...")
|
logger.Log.Info("正在扫描文件系统...")
|
||||||
|
|
||||||
startTime := time.Now()
|
startTime := time.Now()
|
||||||
fileCfg := &config.FileScannerConfig{
|
fileScanner := file.NewScanner(cfg)
|
||||||
IncludePaths: cfg.Scanner.File.IncludePaths,
|
|
||||||
ExcludePaths: cfg.Scanner.File.ExcludePaths,
|
|
||||||
FastHash: cfg.Scanner.File.FastHash,
|
|
||||||
FastHashSize: cfg.Scanner.File.FastHashSize,
|
|
||||||
FastHashChunk: cfg.Scanner.File.FastHashChunk,
|
|
||||||
}
|
|
||||||
|
|
||||||
fileScanner := file.NewScanner(fileCfg)
|
|
||||||
files, err := fileScanner.Scan()
|
files, err := fileScanner.Scan()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Log.Error("扫描文件系统失败", zap.Error(err))
|
logger.Log.Error("扫描文件系统失败", zap.Error(err))
|
||||||
|
|
|
||||||
|
|
@ -9,6 +9,9 @@ audit:
|
||||||
buffer_size: 1000
|
buffer_size: 1000
|
||||||
|
|
||||||
scanner:
|
scanner:
|
||||||
|
hash:
|
||||||
|
# algorithm: "sha256"
|
||||||
|
algorithm: "md5"
|
||||||
file:
|
file:
|
||||||
include_paths:
|
include_paths:
|
||||||
- /home
|
- /home
|
||||||
|
|
|
||||||
|
|
@ -24,6 +24,11 @@ type AuditConfig struct {
|
||||||
|
|
||||||
type ScannerConfig struct {
|
type ScannerConfig struct {
|
||||||
File FileScannerConfig `yaml:"file"`
|
File FileScannerConfig `yaml:"file"`
|
||||||
|
Hash hashConfig `yaml:"hash"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type hashConfig struct {
|
||||||
|
Algorithm string `yaml:"algorithm"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type StorageConfig struct {
|
type StorageConfig struct {
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,8 @@ import (
|
||||||
"regexp"
|
"regexp"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"sysmonitord/internal/scanner/hash"
|
||||||
)
|
)
|
||||||
|
|
||||||
func ParseSize(sizeStr string) (int64, error) {
|
func ParseSize(sizeStr string) (int64, error) {
|
||||||
|
|
@ -44,3 +46,34 @@ func ParseSize(sizeStr string) (int64, error) {
|
||||||
|
|
||||||
return value * multiplier, nil
|
return value * multiplier, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *Config) GetHashAlgorithm() (hash.HashAlgorithm, error) {
|
||||||
|
algoName := c.Scanner.Hash.Algorithm
|
||||||
|
|
||||||
|
switch strings.ToLower(algoName) {
|
||||||
|
case "sha256":
|
||||||
|
return &hash.SHA256Algorithm{}, nil
|
||||||
|
case "md5":
|
||||||
|
return &hash.MD5Algorithm{}, nil
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("不支持的哈希算法: %s", algoName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Config) GetFileScannerConfig() (*FileScannerConfig, error) {
|
||||||
|
return &c.Scanner.File, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Config) GetHashConfig() (*hash.Config, error) {
|
||||||
|
algo, err := c.GetHashAlgorithm()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return &hash.Config{
|
||||||
|
UseFastHash: c.Scanner.File.FastHash,
|
||||||
|
Threshold: c.Scanner.File.FastHashSize,
|
||||||
|
ChunkSize: c.Scanner.File.FastHashChunk,
|
||||||
|
Algorithm: algo,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -21,17 +21,17 @@ type FileInfo struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
type Scanner struct {
|
type Scanner struct {
|
||||||
cfg *config.FileScannerConfig
|
cfg *config.Config
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewScanner(cfg *config.FileScannerConfig) *Scanner {
|
func NewScanner(cfg *config.Config) *Scanner {
|
||||||
return &Scanner{
|
return &Scanner{
|
||||||
cfg: cfg,
|
cfg: cfg,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Scanner) Scan() ([]FileInfo, error) {
|
func (s *Scanner) Scan() ([]FileInfo, error) {
|
||||||
targetPaths := s.cfg.IncludePaths
|
targetPaths := s.cfg.Scanner.File.IncludePaths
|
||||||
if len(targetPaths) == 0 {
|
if len(targetPaths) == 0 {
|
||||||
targetPaths = []string{"/"}
|
targetPaths = []string{"/"}
|
||||||
}
|
}
|
||||||
|
|
@ -65,10 +65,10 @@ func (s *Scanner) WalkFunc(result *[]FileInfo) fs.WalkDirFunc {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, exclude := range s.cfg.ExcludePaths {
|
for _, exclude := range s.cfg.Scanner.File.ExcludePaths {
|
||||||
if strings.HasPrefix(path, exclude) {
|
if strings.HasPrefix(path, exclude) {
|
||||||
logger.Log.Debug("[scan]跳过路径", zap.String("path", path), zap.String("reason", "匹配排除路径"))
|
logger.Log.Debug("[scan]跳过路径", zap.String("path", path), zap.String("reason", "匹配排除路径"))
|
||||||
return fs.SkipDir
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -78,13 +78,14 @@ func (s *Scanner) WalkFunc(result *[]FileInfo) fs.WalkDirFunc {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if info.Size() != 0 {
|
if info.Size() > 0 {
|
||||||
hash, err := hash.SHA256(path, &hash.Config{
|
hashCfg, err := s.cfg.GetHashConfig()
|
||||||
UseFastHash: s.cfg.FastHash,
|
if err != nil {
|
||||||
Threshold: s.cfg.FastHashSize,
|
logger.Log.Debug("[scan]无法获取哈希配置", zap.String("path", path), zap.Error(err))
|
||||||
ChunkSize: s.cfg.FastHashChunk,
|
return nil
|
||||||
})
|
}
|
||||||
|
|
||||||
|
hash, err := hash.CalculateHash(path, hashCfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Log.Debug("[scan]无法计算文件哈希", zap.String("path", path), zap.Error(err))
|
logger.Log.Debug("[scan]无法计算文件哈希", zap.String("path", path), zap.Error(err))
|
||||||
return nil
|
return nil
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,11 @@
|
||||||
package hash
|
package hash
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"crypto/md5"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"encoding/binary"
|
"encoding/binary"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
|
"hash"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"sysmonitord/pkg/logger"
|
"sysmonitord/pkg/logger"
|
||||||
|
|
@ -11,13 +13,51 @@ import (
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type HashAlgorithm interface {
|
||||||
|
Hash() hash.Hash
|
||||||
|
Name() string
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==== SHA256 ====
|
||||||
|
|
||||||
|
type SHA256Algorithm struct{}
|
||||||
|
|
||||||
|
func (a *SHA256Algorithm) Hash() hash.Hash {
|
||||||
|
return sha256.New()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *SHA256Algorithm) Name() string {
|
||||||
|
return "sha256"
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==== MD5 ====
|
||||||
|
|
||||||
|
type MD5Algorithm struct{}
|
||||||
|
|
||||||
|
func (a *MD5Algorithm) Hash() hash.Hash {
|
||||||
|
return md5.New()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *MD5Algorithm) Name() string {
|
||||||
|
return "md5"
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==== xxHash64 ====
|
||||||
|
|
||||||
|
// Todo: 添加 xxHash64 实现
|
||||||
|
|
||||||
|
// ==== 配置结构体 ====
|
||||||
|
|
||||||
type Config struct {
|
type Config struct {
|
||||||
UseFastHash bool
|
UseFastHash bool
|
||||||
Threshold int64
|
Threshold int64
|
||||||
ChunkSize int64
|
ChunkSize int64
|
||||||
|
Algorithm HashAlgorithm
|
||||||
}
|
}
|
||||||
|
|
||||||
func SHA256(filePath string, cfg *Config) (string, error) {
|
// ==== 计算文件哈希 ====
|
||||||
|
|
||||||
|
func CalculateHash(filePath string, cfg *Config) (string, error) {
|
||||||
info, err := os.Stat(filePath)
|
info, err := os.Stat(filePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Log.Warn("[hash]获取文件信息失败", zap.String("path", filePath), zap.Error(err))
|
logger.Log.Warn("[hash]获取文件信息失败", zap.String("path", filePath), zap.Error(err))
|
||||||
|
|
@ -26,18 +66,27 @@ func SHA256(filePath string, cfg *Config) (string, error) {
|
||||||
|
|
||||||
fileSize := info.Size()
|
fileSize := info.Size()
|
||||||
|
|
||||||
if cfg != nil && cfg.UseFastHash && fileSize > cfg.Threshold {
|
if cfg.Algorithm == nil {
|
||||||
|
cfg.Algorithm = &SHA256Algorithm{}
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.Log.Debug("[hash]计算文件哈希",
|
||||||
|
zap.String("path", filePath),
|
||||||
|
zap.Int64("fileSize", fileSize),
|
||||||
|
zap.String("Algorithm", cfg.Algorithm.Name()))
|
||||||
|
|
||||||
|
if cfg.UseFastHash && fileSize > cfg.Threshold {
|
||||||
logger.Log.Debug("[hash] 分层哈希...",
|
logger.Log.Debug("[hash] 分层哈希...",
|
||||||
zap.String("path", filePath),
|
zap.String("path", filePath),
|
||||||
zap.Int64("fileSize", fileSize),
|
zap.Int64("fileSize", fileSize),
|
||||||
)
|
)
|
||||||
return calculateFastHash(filePath, fileSize, cfg.ChunkSize)
|
return calculateFast(filePath, fileSize, cfg)
|
||||||
}
|
}
|
||||||
|
|
||||||
return calculateFullHash(filePath)
|
return calculateFull(filePath, cfg)
|
||||||
}
|
}
|
||||||
|
|
||||||
func calculateFullHash(filePath string) (string, error) {
|
func calculateFull(filePath string, cfg *Config) (string, error) {
|
||||||
file, err := os.Open(filePath)
|
file, err := os.Open(filePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Log.Warn("[scanner]打开文件失败", zap.String("path", filePath), zap.Error(err))
|
logger.Log.Warn("[scanner]打开文件失败", zap.String("path", filePath), zap.Error(err))
|
||||||
|
|
@ -45,7 +94,7 @@ func calculateFullHash(filePath string) (string, error) {
|
||||||
}
|
}
|
||||||
defer file.Close()
|
defer file.Close()
|
||||||
|
|
||||||
hasher := sha256.New()
|
hasher := cfg.Algorithm.Hash()
|
||||||
if _, err := io.Copy(hasher, file); err != nil {
|
if _, err := io.Copy(hasher, file); err != nil {
|
||||||
logger.Log.Error("[scanner]读取文件失败", zap.String("path", filePath), zap.Error(err))
|
logger.Log.Error("[scanner]读取文件失败", zap.String("path", filePath), zap.Error(err))
|
||||||
return "", err
|
return "", err
|
||||||
|
|
@ -56,7 +105,7 @@ func calculateFullHash(filePath string) (string, error) {
|
||||||
return hashString, nil
|
return hashString, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func calculateFastHash(filePath string, fileSize int64, chunkSize int64) (string, error) {
|
func calculateFast(filePath string, fileSize int64, cfg *Config) (string, error) {
|
||||||
file, err := os.Open(filePath)
|
file, err := os.Open(filePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Log.Warn("[scanner]打开文件失败", zap.String("path", filePath), zap.Error(err))
|
logger.Log.Warn("[scanner]打开文件失败", zap.String("path", filePath), zap.Error(err))
|
||||||
|
|
@ -64,7 +113,8 @@ func calculateFastHash(filePath string, fileSize int64, chunkSize int64) (string
|
||||||
}
|
}
|
||||||
defer file.Close()
|
defer file.Close()
|
||||||
|
|
||||||
hasher := sha256.New()
|
hasher := cfg.Algorithm.Hash()
|
||||||
|
chunkSize := cfg.ChunkSize
|
||||||
|
|
||||||
if _, err := io.CopyN(hasher, file, chunkSize); err != nil {
|
if _, err := io.CopyN(hasher, file, chunkSize); err != nil {
|
||||||
if err != io.EOF {
|
if err != io.EOF {
|
||||||
|
|
|
||||||
|
|
@ -58,7 +58,7 @@ func ScanAllProcesses(hashCfg *hash.Config) ([]ProcessInfo, error) {
|
||||||
|
|
||||||
if exePath != "" {
|
if exePath != "" {
|
||||||
if _, err := os.Stat(exePath); err == nil {
|
if _, err := os.Stat(exePath); err == nil {
|
||||||
fileHash, err := hash.SHA256(exePath, hashCfg)
|
fileHash, err := hash.CalculateHash(exePath, hashCfg)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
info.FileHash = fileHash
|
info.FileHash = fileHash
|
||||||
} else {
|
} else {
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user