217 lines
5.2 KiB
Go
217 lines
5.2 KiB
Go
package transfer
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"m3u8-downloader/pkg/logging"
|
|
"os"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
// errRetained reports that a file is still inside its retention window. It is
|
|
// not a failure: the file simply has to be reconsidered on a later pass rather
|
|
// than being discarded from the pending list.
|
|
var errRetained = errors.New("file is within the retention period")
|
|
|
|
type CleanupService struct {
|
|
config CleanupConfig
|
|
pendingFiles []string
|
|
mu sync.Mutex
|
|
}
|
|
|
|
func NewCleanupService(config CleanupConfig) *CleanupService {
|
|
return &CleanupService{
|
|
config: config,
|
|
pendingFiles: make([]string, 0),
|
|
}
|
|
}
|
|
|
|
func (cs *CleanupService) ScheduleCleanup(filePath string) error {
|
|
if !cs.config.Enabled {
|
|
return nil
|
|
}
|
|
|
|
cs.mu.Lock()
|
|
defer cs.mu.Unlock()
|
|
cs.pendingFiles = append(cs.pendingFiles, filePath)
|
|
log.Printf("Scheduled file for cleanup: %s", filePath)
|
|
return nil
|
|
}
|
|
|
|
func (cs *CleanupService) Start(ctx context.Context) error {
|
|
if !cs.config.Enabled {
|
|
log.Println("Cleanup service disabled")
|
|
return nil
|
|
}
|
|
|
|
log.Printf("Cleanup service started (retention: %v, batch: %d)", cs.config.RetentionPeriod, cs.config.BatchSize)
|
|
logging.MustGetFactory().Transfer.Info("cleanup service started", "retention", cs.config.RetentionPeriod.String(), "batch_size", cs.config.BatchSize)
|
|
|
|
ticker := time.NewTicker(cs.config.CheckInterval)
|
|
defer ticker.Stop()
|
|
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
log.Println("Cleanup service shutting down...")
|
|
return ctx.Err()
|
|
case <-ticker.C:
|
|
if err := cs.ExecuteCleanup(ctx); err != nil {
|
|
log.Printf("Cleanup error: %v", err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (cs *CleanupService) ExecuteCleanup(ctx context.Context) error {
|
|
cs.mu.Lock()
|
|
if len(cs.pendingFiles) == 0 {
|
|
cs.mu.Unlock()
|
|
return nil
|
|
}
|
|
|
|
batchSize := cs.config.BatchSize
|
|
if batchSize > len(cs.pendingFiles) {
|
|
batchSize = len(cs.pendingFiles)
|
|
}
|
|
|
|
log.Printf("Executing cleanup batch (size: %d)", batchSize)
|
|
|
|
batch := make([]string, batchSize)
|
|
copy(batch, cs.pendingFiles[:batchSize])
|
|
cs.pendingFiles = cs.pendingFiles[batchSize:]
|
|
cs.mu.Unlock()
|
|
|
|
log.Printf("Processing %d files for cleanup", len(batch))
|
|
|
|
var cleanedCount int
|
|
var failures []error
|
|
// requeue collects files this pass must not drop: ones still inside their
|
|
// retention window, and — if the context is cancelled mid-batch — the
|
|
// remainder that was never looked at. They were already removed from
|
|
// pendingFiles above, so without this they would never be cleaned at all.
|
|
var requeue []string
|
|
|
|
for i, filePath := range batch {
|
|
select {
|
|
case <-ctx.Done():
|
|
requeue = append(requeue, batch[i:]...)
|
|
cs.requeue(requeue)
|
|
return ctx.Err()
|
|
default:
|
|
}
|
|
|
|
err := cs.cleanupFile(filePath)
|
|
switch {
|
|
case errors.Is(err, errRetained):
|
|
requeue = append(requeue, filePath)
|
|
case err != nil:
|
|
failures = append(failures, fmt.Errorf("Failed to cleanup file %s: %w", filePath, err))
|
|
default:
|
|
cleanedCount++
|
|
}
|
|
}
|
|
|
|
cs.requeue(requeue)
|
|
|
|
log.Printf("Cleanup batch completed (cleaned: %d, retained: %d, errors: %d)", cleanedCount, len(requeue), len(failures))
|
|
logging.MustGetFactory().Transfer.Info("cleanup batch completed",
|
|
"cleaned", cleanedCount, "retained", len(requeue), "errors", len(failures))
|
|
|
|
if len(failures) > 0 {
|
|
for i, err := range failures {
|
|
if i >= 3 {
|
|
log.Printf("... and %d more errors", len(failures)-3)
|
|
break
|
|
}
|
|
log.Printf("Error: %v", err)
|
|
logging.MustGetFactory().Transfer.Warn("cleanup error", logging.KeyError, err.Error())
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// requeue returns files to the pending list for a later pass.
|
|
func (cs *CleanupService) requeue(files []string) {
|
|
if len(files) == 0 {
|
|
return
|
|
}
|
|
|
|
cs.mu.Lock()
|
|
defer cs.mu.Unlock()
|
|
cs.pendingFiles = append(cs.pendingFiles, files...)
|
|
}
|
|
|
|
func (cs *CleanupService) cleanupFile(filePath string) error {
|
|
info, err := os.Stat(filePath)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil
|
|
}
|
|
return fmt.Errorf("Failed to get file info: %w", err)
|
|
}
|
|
|
|
if cs.config.RetentionPeriod > 0 {
|
|
if time.Since(info.ModTime()) < cs.config.RetentionPeriod {
|
|
log.Printf("File too new to cleanup: %s", filePath)
|
|
return errRetained
|
|
}
|
|
}
|
|
|
|
if err := os.Remove(filePath); err != nil {
|
|
return fmt.Errorf("Failed to remove file: %w", err)
|
|
}
|
|
|
|
log.Printf("File cleaned up: %s", filePath)
|
|
return nil
|
|
}
|
|
|
|
func (cs *CleanupService) GetPendingCount() int {
|
|
cs.mu.Lock()
|
|
defer cs.mu.Unlock()
|
|
return len(cs.pendingFiles)
|
|
}
|
|
|
|
func (cs *CleanupService) ForceCleanupAll(ctx context.Context) error {
|
|
log.Println("Force cleanup requested")
|
|
|
|
for {
|
|
cs.mu.Lock()
|
|
pendingCount := len(cs.pendingFiles)
|
|
cs.mu.Unlock()
|
|
|
|
if pendingCount == 0 {
|
|
break
|
|
}
|
|
|
|
if err := cs.ExecuteCleanup(ctx); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Stop when a pass makes no progress. Files still inside their
|
|
// retention window are requeued by design, so requiring the list to
|
|
// reach zero would spin here until the shutdown timeout expired.
|
|
cs.mu.Lock()
|
|
remaining := len(cs.pendingFiles)
|
|
cs.mu.Unlock()
|
|
|
|
if remaining >= pendingCount {
|
|
log.Printf("Force cleanup stopping with %d file(s) still pending (within retention period)", remaining)
|
|
break
|
|
}
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-time.After(100 * time.Millisecond):
|
|
}
|
|
}
|
|
|
|
log.Println("Force cleanup complete")
|
|
return nil
|
|
}
|