[storage] 持久化存储实现

[config] 实现配置转换

.gitignore

@@ -22,3 +22,4 @@
 go.work
 
 sysmonitord.code-workspace
+data/

cmd/start/start.go

@@ -4,6 +4,9 @@ import (
 	"fmt"
 	"os"
 	"sysmonitord/internal/config"
+	"sysmonitord/internal/scanner/hash"
+	"sysmonitord/internal/scanner/process"
+	"sysmonitord/internal/storage"
 	"sysmonitord/pkg/logger"
 
 	"github.com/spf13/cobra"
@@ -26,6 +29,43 @@ var StartCmd = &cobra.Command{
 		logger.Log.Info("配置文件加载成功",
 			zap.String("审计服务器地址", fmt.Sprintf("%s:%d", cfg.Audit.Server, cfg.Audit.Port)),
 		)
-		// Todo: 初始化扫描
+
+		hashCfg := &hash.Config{
+			UseFastHash: cfg.Scanner.File.FastHash,
+			Threshold:   cfg.Scanner.File.FastHashSize,
+			ChunkSize:   cfg.Scanner.File.FastHashChunk,
+		}
+
+		storageCfg := &storage.Storage{
+			DataDir:           cfg.Storage.DataDir,
+			ProcessSystemFile: cfg.Storage.ProcessSystemFile,
+			FileSystemFile:    cfg.Storage.FileSystemFile,
+		}
+
+		procs, err := process.ScanAllProcesses(hashCfg)
+		if err != nil {
+			logger.Log.Error("扫描进程失败", zap.Error(err))
+			os.Exit(1)
+		} else {
+			if err := storage.SaveProcessSystem(procs, storageCfg.DataDir, storageCfg.ProcessSystemFile); err != nil {
+				logger.Log.Error("保存进程白名单失败", zap.Error(err))
+			}
+		}
+
+		logger.Log.Info("进程列表：")
+		for i, p := range procs {
+			if i >= 10 {
+				logger.Log.Info("... (仅显示前10个进程)")
+				break
+			}
+			logger.Log.Info(
+				"进程信息",
+				zap.Int32("pid", p.PID),
+				zap.String("name", p.Name),
+				zap.String("path", p.Path),
+				zap.String("cmdline", p.Cmdline),
+				zap.Stringer("data", p),
+			)
+		}
 	},
 }

config.yaml

@@ -10,6 +10,14 @@ audit:
 
 scanner:
   file:
-  exclude_paths:
-    - /proc
-    - /sys
+    exclude_paths:
+      - /proc
+      - /sys
+    fast_hash: true
+    fast_hash_size: 100MB
+    fast_hash_chunk: 2MB
+
+storage:
+  data_dir: "./data"
+  process_system_file: "process_system.data"
+  file_system_file: "file_system.data"

development.md

@@ -355,7 +355,9 @@ scanner:
       - /sys/
       - /dev/
       - /tmp/
-    max_file_size: 100MB
+    fast_hash: true
+    fast_hash_size: 100MB
+    fast_hash_chunk: 2MB
     hash_algorithm: sha256
   process:
     scan_interval: 30
@@ -552,6 +554,20 @@ LimitNOFILE=65536
 WantedBy=multi-user.target
 ```
 
+### 5.5 分层抽样哈希策略
+
+针对大文件（默认 >100MB）的哈希计算，为避免I/O阻塞，采用分层抽样算法：
+
+**算法逻辑**：
+1. 读取文件头部 N 字节（默认 1MB）。
+2. 读取文件尾部 N 字节（默认 1MB）。
+3. 获取文件总大小 Size。
+4. 拼接：`Head + Tail + Size`，对拼接后的数据进行 SHA256 运算。
+
+**优势**：
+- **性能**：将 GB 级文件的哈希耗时从秒级降至毫秒级。
+- **安全性**：任何对文件内容的修改，极大概率会触碰到头部（文件头结构）或尾部（数据填充），且锁定文件大小，有效检测篡改行为。
+
 ---
 
 ## 六、数据格式规范

go.mod

@@ -3,10 +3,19 @@ module sysmonitord
 go 1.26.1
 
 require (
+	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
+	github.com/shirou/gopsutil/v3 v3.24.5 // indirect
+	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/spf13/cobra v1.10.2 // indirect
 	github.com/spf13/pflag v1.0.10 // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
+	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.1 // indirect
+	golang.org/x/sys v0.20.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,17 +1,41 @@
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
+github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
+github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
+github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
+github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
+github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI=
+github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk=
+github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
+github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
 github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
 github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
 github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
 github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=
+github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
+github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=
+github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
+github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
+github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc=
 go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/config/config.go

@@ -3,13 +3,16 @@ package config
 import (
 	"fmt"
 	"os"
+	"sysmonitord/pkg/logger"
 
+	"go.uber.org/zap"
 	"gopkg.in/yaml.v3"
 )
 
 type Config struct {
 	Audit   AuditConfig   `yaml:"audit"`
 	Scanner ScannerConfig `yaml:"scanner"`
+	Storage StorageConfig `yaml:"storage"`
 }
 
 type AuditConfig struct {
@@ -23,8 +26,19 @@ type ScannerConfig struct {
 	File FileScannerConfig `yaml:"file"`
 }
 
+type StorageConfig struct {
+	DataDir           string `yaml:"data_dir"`
+	ProcessSystemFile string `yaml:"process_system_file"`
+	FileSystemFile    string `yaml:"file_system_file"`
+}
+
 type FileScannerConfig struct {
-	ExcludePaths []string `yaml:"exclude_paths"`
+	ExcludePaths     []string `yaml:"exclude_paths"`
+	FastHash         bool     `yaml:"fast_hash"`
+	FastHashSizeRaw  string   `yaml:"fast_hash_size"`
+	FastHashChunkRaw string   `yaml:"fast_hash_chunk"`
+	FastHashSize     int64
+	FastHashChunk    int64
 }
 
 func LoadConfig(path string) (*Config, error) {
@@ -37,5 +51,23 @@ func LoadConfig(path string) (*Config, error) {
 	if err := yaml.Unmarshal(data, &cfg); err != nil {
 		return nil, fmt.Errorf("无法解析配置文件： %w", err)
 	}
+
+	// 解析 FastHashSize
+	cfg.Scanner.File.FastHashSize, err = ParseSize(cfg.Scanner.File.FastHashSizeRaw)
+	if err != nil {
+		return nil, fmt.Errorf("解析 fast_hash_size 失败: %w", err)
+	}
+
+	// 解析 FastHashChunk
+	cfg.Scanner.File.FastHashChunk, err = ParseSize(cfg.Scanner.File.FastHashChunkRaw)
+	if err != nil {
+		return nil, fmt.Errorf("解析 fast_hash_chunk 失败: %w", err)
+	}
+
+	logger.Log.Debug("配置加载完成",
+		zap.Int64("fast_hash_size", cfg.Scanner.File.FastHashSize),
+		zap.Int64("fast_hash_chunk", cfg.Scanner.File.FastHashChunk),
+	)
+
 	return &cfg, nil
 }

internal/config/utils.go

@@ -0,0 +1,46 @@
+package config
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+func ParseSize(sizeStr string) (int64, error) {
+	sizeStr = strings.TrimSpace(sizeStr)
+	if sizeStr == "" {
+		return 0, nil
+	}
+
+	// 正则匹配：数字 + 单位
+	re := regexp.MustCompile(`(?i)^(\d+)\s*([KMGT]?B?)$`)
+	matches := re.FindStringSubmatch(sizeStr)
+	if len(matches) != 3 {
+		return 0, fmt.Errorf("无效的大小格式: %s", sizeStr)
+	}
+
+	value, err := strconv.ParseInt(matches[1], 10, 64)
+	if err != nil {
+		return 0, err
+	}
+
+	unit := strings.ToUpper(matches[2])
+	var multiplier int64 = 1
+	switch unit {
+	case "B", "":
+		multiplier = 1
+	case "KB", "K":
+		multiplier = 1024
+	case "MB", "M":
+		multiplier = 1024 * 1024
+	case "GB", "G":
+		multiplier = 1024 * 1024 * 1024
+	case "TB", "T":
+		multiplier = 1024 * 1024 * 1024 * 1024
+	default:
+		return 0, fmt.Errorf("未知的单位: %s", unit)
+	}
+
+	return value * multiplier, nil
+}

internal/scanner/hash/hash.go

@@ -0,0 +1,92 @@
+package hash
+
+import (
+	"crypto/sha256"
+	"encoding/binary"
+	"encoding/hex"
+	"io"
+	"os"
+	"sysmonitord/pkg/logger"
+
+	"go.uber.org/zap"
+)
+
+type Config struct {
+	UseFastHash bool
+	Threshold   int64
+	ChunkSize   int64
+}
+
+func SHA256(filePath string, cfg *Config) (string, error) {
+	info, err := os.Stat(filePath)
+	if err != nil {
+		logger.Log.Warn("[hash]获取文件信息失败", zap.String("path", filePath), zap.Error(err))
+		return "", err
+	}
+
+	fileSize := info.Size()
+
+	if cfg != nil && cfg.UseFastHash && fileSize > cfg.Threshold {
+		logger.Log.Debug("[hash] 分层哈希...",
+			zap.String("path", filePath),
+			zap.Int64("fileSize", fileSize),
+		)
+		return calculateFastHash(filePath, fileSize, cfg.ChunkSize)
+	}
+
+	return calculateFullHash(filePath)
+}
+
+func calculateFullHash(filePath string) (string, error) {
+	file, err := os.Open(filePath)
+	if err != nil {
+		logger.Log.Warn("[scanner]打开文件失败", zap.String("path", filePath), zap.Error(err))
+		return "", err
+	}
+	defer file.Close()
+
+	hasher := sha256.New()
+	if _, err := io.Copy(hasher, file); err != nil {
+		logger.Log.Error("[scanner]读取文件失败", zap.String("path", filePath), zap.Error(err))
+		return "", err
+	}
+
+	hashBytes := hasher.Sum(nil)
+	hashString := hex.EncodeToString(hashBytes)
+	return hashString, nil
+}
+
+func calculateFastHash(filePath string, fileSize int64, chunkSize int64) (string, error) {
+	file, err := os.Open(filePath)
+	if err != nil {
+		logger.Log.Warn("[scanner]打开文件失败", zap.String("path", filePath), zap.Error(err))
+		return "", err
+	}
+	defer file.Close()
+
+	hasher := sha256.New()
+
+	if _, err := io.CopyN(hasher, file, chunkSize); err != nil {
+		if err != io.EOF {
+			return "", err
+		}
+	}
+
+	tailOffset := fileSize - chunkSize
+	if tailOffset < 0 {
+		tailOffset = 0
+	}
+
+	if _, err := file.Seek(tailOffset, io.SeekStart); err != nil {
+		return "", err
+	}
+
+	if _, err := io.CopyN(hasher, file, chunkSize); err != nil {
+		return "", err
+	}
+
+	if err := binary.Write(hasher, binary.BigEndian, fileSize); err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(hasher.Sum(nil)), nil
+}

internal/scanner/process/process.go

@@ -0,0 +1,79 @@
+package process
+
+import (
+	"fmt"
+	"os"
+	"sysmonitord/internal/scanner/hash"
+	"sysmonitord/pkg/logger"
+
+	"github.com/shirou/gopsutil/v3/process"
+	"go.uber.org/zap"
+)
+
+type ProcessInfo struct {
+	PID      int32  `json:"pid"`
+	Name     string `json:"name"`
+	Path     string `json:"path"`
+	Cmdline  string `json:"cmdline"`
+	FileHash string `json:"file_hash"`
+}
+
+func ScanAllProcesses(hashCfg *hash.Config) ([]ProcessInfo, error) {
+	logger.Log.Info("[scan]正在扫描系统中的所有进程...")
+
+	pids, err := process.Pids()
+	if err != nil {
+		logger.Log.Error("[scan]获取进程列表失败", zap.Error(err))
+		return nil, err
+	}
+
+	var processList []ProcessInfo
+	for _, pid := range pids {
+		p, err := process.NewProcess(pid)
+		if err != nil {
+			continue // 跳过临时进程
+		}
+
+		name, err := p.Name()
+		if err != nil {
+			name = "unknown"
+		}
+
+		exePath, err := p.Exe()
+		if err != nil {
+			exePath = ""
+		}
+
+		cmdline, err := p.Cmdline()
+		if err != nil {
+			cmdline = ""
+		}
+
+		info := ProcessInfo{
+			PID:     pid,
+			Name:    name,
+			Path:    exePath,
+			Cmdline: cmdline,
+		}
+
+		if exePath != "" {
+			if _, err := os.Stat(exePath); err == nil {
+				fileHash, err := hash.SHA256(exePath, hashCfg)
+				if err == nil {
+					info.FileHash = fileHash
+				} else {
+					logger.Log.Warn("[scan]计算文件哈希失败", zap.String("path", exePath), zap.Error(err))
+				}
+			}
+		}
+
+		processList = append(processList, info)
+	}
+
+	logger.Log.Info("[scan]进程扫描完成", zap.Int("进程数量", len(processList)))
+	return processList, nil
+}
+
+func (p ProcessInfo) String() string {
+	return fmt.Sprintf("%s:%s:%s", p.Name, p.Path, p.FileHash)
+}

internal/storage/storage.go

@@ -0,0 +1,62 @@
+package storage
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sysmonitord/internal/scanner/process"
+	"sysmonitord/pkg/logger"
+	"time"
+
+	"go.uber.org/zap"
+)
+
+type Storage struct {
+	DataDir           string
+	ProcessSystemFile string
+	FileSystemFile    string
+}
+
+func InitDataDir(dataDir string) error {
+	if err := os.MkdirAll(dataDir, 0755); err != nil {
+		return fmt.Errorf("[storage]无法创建数据目录: %w", err)
+	}
+	return nil
+}
+
+func SaveProcessSystem(proc []process.ProcessInfo, dataDir string, processSystemFile string) error {
+	filePath := filepath.Join(dataDir, processSystemFile)
+
+	f, err := os.Create(filePath) // 覆盖
+	if err != nil {
+		return fmt.Errorf("[storage]无法创建储存进程文件%s: %w", filePath, err)
+	}
+	defer f.Close()
+
+	writer := bufio.NewWriter(f)
+
+	currentTime := time.Now().Format("2006-01-02 15:04:05")
+	header := fmt.Sprintf("# 进程白名单 - 生成时间: %s\n", currentTime)
+	if _, err := writer.WriteString(header); err != nil {
+		return err
+	}
+
+	for _, p := range proc {
+		line := fmt.Sprintf("%v\n", p)
+		if _, err := writer.WriteString(line); err != nil {
+			return err
+		}
+	}
+
+	if err := writer.Flush(); err != nil {
+		return err
+	}
+
+	logger.Log.Info("[storage]进程白名单保存成功",
+		zap.String("file", filePath),
+		zap.Int("process_count", len(proc)),
+	)
+
+	return nil
+}