353 lines
9.7 KiB
Go
353 lines
9.7 KiB
Go
package health
|
|
|
|
import (
|
|
"sort"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
const maxRecentErrors = 10
|
|
|
|
type ErrorEntry struct {
|
|
Time time.Time `json:"time"`
|
|
Message string `json:"message"`
|
|
}
|
|
|
|
type VariantStatus struct {
|
|
Resolution string `json:"resolution"`
|
|
Bandwidth uint32 `json:"bandwidth"`
|
|
Active bool `json:"active"`
|
|
SegmentsOK int `json:"segments_ok"`
|
|
SegmentsFailed int `json:"segments_failed"`
|
|
LastActivity time.Time `json:"last_activity"`
|
|
LastError string `json:"last_error,omitempty"`
|
|
}
|
|
|
|
// StatsProvider is satisfied by *transfer.QueueStats without pkg/health
|
|
// needing to import pkg/transfer (which would create an import cycle, since
|
|
// pkg/transfer calls into pkg/health to report status).
|
|
type StatsProvider interface {
|
|
GetStats() (added, completed, failed, pending int, bytes int64)
|
|
}
|
|
|
|
type downloadState struct {
|
|
variants map[string]*VariantStatus
|
|
recentErrors []ErrorEntry
|
|
}
|
|
|
|
type transferState struct {
|
|
statsProvider StatsProvider
|
|
queueSizeFn func() int
|
|
cleanupPendingFn func() int
|
|
nasConnectedFn func() bool
|
|
nasConnected bool
|
|
recentErrors []ErrorEntry
|
|
}
|
|
|
|
type processingState struct {
|
|
state string // idle | scanning | running | completed | failed
|
|
eventName string
|
|
segmentsFound int
|
|
startedAt time.Time
|
|
finishedAt time.Time
|
|
lastError string
|
|
recentErrors []ErrorEntry
|
|
}
|
|
|
|
// Registry is the process-wide health/metrics store. All subsystems report
|
|
// into it via the Record* methods; the health HTTP server reads it via
|
|
// Snapshot(). Safe for concurrent use.
|
|
type Registry struct {
|
|
mu sync.RWMutex
|
|
startedAt time.Time
|
|
download downloadState
|
|
transfer transferState
|
|
processing processingState
|
|
}
|
|
|
|
var (
|
|
globalRegistry *Registry
|
|
registryOnce sync.Once
|
|
)
|
|
|
|
// MustGetRegistry returns the process-wide registry, mirroring the
|
|
// constants.MustGetConfig() / logging.MustGetFactory() singleton style.
|
|
func MustGetRegistry() *Registry {
|
|
registryOnce.Do(func() {
|
|
globalRegistry = New()
|
|
})
|
|
return globalRegistry
|
|
}
|
|
|
|
func New() *Registry {
|
|
return &Registry{
|
|
startedAt: time.Now(),
|
|
download: downloadState{
|
|
variants: make(map[string]*VariantStatus),
|
|
},
|
|
processing: processingState{
|
|
state: "idle",
|
|
},
|
|
}
|
|
}
|
|
|
|
func appendBounded(errs []ErrorEntry, e ErrorEntry) []ErrorEntry {
|
|
errs = append(errs, e)
|
|
if len(errs) > maxRecentErrors {
|
|
errs = errs[len(errs)-maxRecentErrors:]
|
|
}
|
|
return errs
|
|
}
|
|
|
|
func errString(err error) string {
|
|
if err == nil {
|
|
return ""
|
|
}
|
|
return err.Error()
|
|
}
|
|
|
|
// --- Download ---
|
|
|
|
func (r *Registry) RecordVariantStarted(resolution string, bandwidth uint32) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
r.download.variants[resolution] = &VariantStatus{
|
|
Resolution: resolution,
|
|
Bandwidth: bandwidth,
|
|
Active: true,
|
|
LastActivity: time.Now(),
|
|
}
|
|
}
|
|
|
|
func (r *Registry) RecordVariantStopped(resolution string) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
if v, ok := r.download.variants[resolution]; ok {
|
|
v.Active = false
|
|
}
|
|
}
|
|
|
|
func (r *Registry) RecordSegmentDownloaded(resolution string) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
v := r.ensureVariantLocked(resolution)
|
|
v.SegmentsOK++
|
|
v.LastActivity = time.Now()
|
|
}
|
|
|
|
func (r *Registry) RecordSegmentFailed(resolution string, err error) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
v := r.ensureVariantLocked(resolution)
|
|
v.SegmentsFailed++
|
|
v.LastActivity = time.Now()
|
|
v.LastError = errString(err)
|
|
r.download.recentErrors = appendBounded(r.download.recentErrors, ErrorEntry{
|
|
Time: time.Now(),
|
|
Message: resolution + ": " + errString(err),
|
|
})
|
|
}
|
|
|
|
func (r *Registry) RecordPlaylistError(resolution string, err error) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
v := r.ensureVariantLocked(resolution)
|
|
v.LastError = errString(err)
|
|
r.download.recentErrors = appendBounded(r.download.recentErrors, ErrorEntry{
|
|
Time: time.Now(),
|
|
Message: resolution + " playlist: " + errString(err),
|
|
})
|
|
}
|
|
|
|
func (r *Registry) ensureVariantLocked(resolution string) *VariantStatus {
|
|
v, ok := r.download.variants[resolution]
|
|
if !ok {
|
|
v = &VariantStatus{Resolution: resolution, Active: true}
|
|
r.download.variants[resolution] = v
|
|
}
|
|
return v
|
|
}
|
|
|
|
// --- Transfer ---
|
|
|
|
// AttachTransfer wires the registry to the live counters/state already
|
|
// maintained by pkg/transfer instead of duplicating them. Called once from
|
|
// TransferService construction.
|
|
func (r *Registry) AttachTransfer(stats StatsProvider, queueSizeFn, cleanupPendingFn func() int, nasConnectedFn func() bool) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
r.transfer.statsProvider = stats
|
|
r.transfer.queueSizeFn = queueSizeFn
|
|
r.transfer.cleanupPendingFn = cleanupPendingFn
|
|
r.transfer.nasConnectedFn = nasConnectedFn
|
|
}
|
|
|
|
func (r *Registry) RecordNASConnectivity(connected bool) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
r.transfer.nasConnected = connected
|
|
}
|
|
|
|
// RecordTransferError captures failures QueueStats' cumulative counters
|
|
// don't individually retain (pre-queue failures, cleanup failures) so they
|
|
// still surface in the bounded recent-errors list.
|
|
func (r *Registry) RecordTransferError(sourcePath string, err error) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
r.transfer.recentErrors = appendBounded(r.transfer.recentErrors, ErrorEntry{
|
|
Time: time.Now(),
|
|
Message: sourcePath + ": " + errString(err),
|
|
})
|
|
}
|
|
|
|
// --- Processing ---
|
|
|
|
func (r *Registry) RecordProcessingStageChange(state, eventName string) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
r.processing.state = state
|
|
if eventName != "" {
|
|
r.processing.eventName = eventName
|
|
}
|
|
switch state {
|
|
case "scanning":
|
|
r.processing.startedAt = time.Now()
|
|
r.processing.finishedAt = time.Time{}
|
|
r.processing.lastError = ""
|
|
case "completed", "failed":
|
|
r.processing.finishedAt = time.Now()
|
|
}
|
|
}
|
|
|
|
func (r *Registry) RecordProcessingSegmentsFound(count int) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
r.processing.segmentsFound = count
|
|
}
|
|
|
|
func (r *Registry) RecordProcessingError(err error) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
msg := errString(err)
|
|
r.processing.lastError = msg
|
|
r.processing.recentErrors = appendBounded(r.processing.recentErrors, ErrorEntry{
|
|
Time: time.Now(),
|
|
Message: msg,
|
|
})
|
|
}
|
|
|
|
// --- Snapshot ---
|
|
|
|
type DownloadSnapshot struct {
|
|
Variants []*VariantStatus `json:"variants"`
|
|
RecentErrors []ErrorEntry `json:"recent_errors"`
|
|
}
|
|
|
|
type TransferSnapshot struct {
|
|
NASConnected bool `json:"nas_connected"`
|
|
QueueSize int `json:"queue_size"`
|
|
CleanupPending int `json:"cleanup_pending"`
|
|
Added int `json:"added"`
|
|
Completed int `json:"completed"`
|
|
Failed int `json:"failed"`
|
|
Pending int `json:"pending"`
|
|
BytesTransferred int64 `json:"bytes_transferred"`
|
|
RecentErrors []ErrorEntry `json:"recent_errors"`
|
|
}
|
|
|
|
type ProcessingSnapshot struct {
|
|
State string `json:"state"`
|
|
EventName string `json:"event_name,omitempty"`
|
|
SegmentsFound int `json:"segments_found"`
|
|
StartedAt *time.Time `json:"started_at,omitempty"`
|
|
FinishedAt *time.Time `json:"finished_at,omitempty"`
|
|
LastError string `json:"last_error,omitempty"`
|
|
RecentErrors []ErrorEntry `json:"recent_errors"`
|
|
}
|
|
|
|
type Snapshot struct {
|
|
Status string `json:"status"`
|
|
UptimeSec float64 `json:"uptime_seconds"`
|
|
Download DownloadSnapshot `json:"download"`
|
|
Transfer TransferSnapshot `json:"transfer"`
|
|
Processing ProcessingSnapshot `json:"processing"`
|
|
}
|
|
|
|
func (r *Registry) Snapshot() Snapshot {
|
|
r.mu.RLock()
|
|
defer r.mu.RUnlock()
|
|
|
|
variants := make([]*VariantStatus, 0, len(r.download.variants))
|
|
for _, v := range r.download.variants {
|
|
vCopy := *v
|
|
variants = append(variants, &vCopy)
|
|
}
|
|
sort.Slice(variants, func(i, j int) bool { return variants[i].Resolution < variants[j].Resolution })
|
|
|
|
dl := DownloadSnapshot{
|
|
Variants: variants,
|
|
RecentErrors: append([]ErrorEntry(nil), r.download.recentErrors...),
|
|
}
|
|
|
|
tr := TransferSnapshot{
|
|
NASConnected: r.transfer.nasConnected,
|
|
RecentErrors: append([]ErrorEntry(nil), r.transfer.recentErrors...),
|
|
}
|
|
if r.transfer.statsProvider != nil {
|
|
added, completed, failed, pending, bytes := r.transfer.statsProvider.GetStats()
|
|
tr.Added, tr.Completed, tr.Failed, tr.Pending, tr.BytesTransferred = added, completed, failed, pending, bytes
|
|
}
|
|
if r.transfer.queueSizeFn != nil {
|
|
tr.QueueSize = r.transfer.queueSizeFn()
|
|
}
|
|
if r.transfer.cleanupPendingFn != nil {
|
|
tr.CleanupPending = r.transfer.cleanupPendingFn()
|
|
}
|
|
if r.transfer.nasConnectedFn != nil {
|
|
tr.NASConnected = r.transfer.nasConnectedFn()
|
|
}
|
|
|
|
proc := ProcessingSnapshot{
|
|
State: r.processing.state,
|
|
EventName: r.processing.eventName,
|
|
SegmentsFound: r.processing.segmentsFound,
|
|
LastError: r.processing.lastError,
|
|
RecentErrors: append([]ErrorEntry(nil), r.processing.recentErrors...),
|
|
}
|
|
if !r.processing.startedAt.IsZero() {
|
|
t := r.processing.startedAt
|
|
proc.StartedAt = &t
|
|
}
|
|
if !r.processing.finishedAt.IsZero() {
|
|
t := r.processing.finishedAt
|
|
proc.FinishedAt = &t
|
|
}
|
|
|
|
snap := Snapshot{
|
|
UptimeSec: time.Since(r.startedAt).Seconds(),
|
|
Download: dl,
|
|
Transfer: tr,
|
|
Processing: proc,
|
|
}
|
|
snap.Status = computeStatus(dl, tr, proc)
|
|
return snap
|
|
}
|
|
|
|
// computeStatus gives one glanceable field: "degraded" the moment anything
|
|
// worth investigating shows up, "healthy" otherwise. Transfer is only
|
|
// considered for degradation once it's actually in use (Attached + has seen
|
|
// activity), so a download-only run with NAS transfer disabled doesn't
|
|
// falsely report degraded.
|
|
func computeStatus(dl DownloadSnapshot, tr TransferSnapshot, proc ProcessingSnapshot) string {
|
|
if len(dl.RecentErrors) > 0 || len(tr.RecentErrors) > 0 || len(proc.RecentErrors) > 0 {
|
|
return "degraded"
|
|
}
|
|
if !tr.NASConnected && (tr.Added > 0 || tr.QueueSize > 0) {
|
|
return "degraded"
|
|
}
|
|
if proc.State == "failed" {
|
|
return "degraded"
|
|
}
|
|
return "healthy"
|
|
}
|