StreamRecorder/pkg/health/registry.go
kacarmichael 309eeef59a
Some checks failed
Build Go Binaries / build (push) Has been cancelled
Health/status http endpoint, logging
2026-08-06 01:24:29 -05:00

353 lines
9.7 KiB
Go

package health
import (
"sort"
"sync"
"time"
)
const maxRecentErrors = 10
type ErrorEntry struct {
Time time.Time `json:"time"`
Message string `json:"message"`
}
type VariantStatus struct {
Resolution string `json:"resolution"`
Bandwidth uint32 `json:"bandwidth"`
Active bool `json:"active"`
SegmentsOK int `json:"segments_ok"`
SegmentsFailed int `json:"segments_failed"`
LastActivity time.Time `json:"last_activity"`
LastError string `json:"last_error,omitempty"`
}
// StatsProvider is satisfied by *transfer.QueueStats without pkg/health
// needing to import pkg/transfer (which would create an import cycle, since
// pkg/transfer calls into pkg/health to report status).
type StatsProvider interface {
GetStats() (added, completed, failed, pending int, bytes int64)
}
type downloadState struct {
variants map[string]*VariantStatus
recentErrors []ErrorEntry
}
type transferState struct {
statsProvider StatsProvider
queueSizeFn func() int
cleanupPendingFn func() int
nasConnectedFn func() bool
nasConnected bool
recentErrors []ErrorEntry
}
type processingState struct {
state string // idle | scanning | running | completed | failed
eventName string
segmentsFound int
startedAt time.Time
finishedAt time.Time
lastError string
recentErrors []ErrorEntry
}
// Registry is the process-wide health/metrics store. All subsystems report
// into it via the Record* methods; the health HTTP server reads it via
// Snapshot(). Safe for concurrent use.
type Registry struct {
mu sync.RWMutex
startedAt time.Time
download downloadState
transfer transferState
processing processingState
}
var (
globalRegistry *Registry
registryOnce sync.Once
)
// MustGetRegistry returns the process-wide registry, mirroring the
// constants.MustGetConfig() / logging.MustGetFactory() singleton style.
func MustGetRegistry() *Registry {
registryOnce.Do(func() {
globalRegistry = New()
})
return globalRegistry
}
func New() *Registry {
return &Registry{
startedAt: time.Now(),
download: downloadState{
variants: make(map[string]*VariantStatus),
},
processing: processingState{
state: "idle",
},
}
}
func appendBounded(errs []ErrorEntry, e ErrorEntry) []ErrorEntry {
errs = append(errs, e)
if len(errs) > maxRecentErrors {
errs = errs[len(errs)-maxRecentErrors:]
}
return errs
}
func errString(err error) string {
if err == nil {
return ""
}
return err.Error()
}
// --- Download ---
func (r *Registry) RecordVariantStarted(resolution string, bandwidth uint32) {
r.mu.Lock()
defer r.mu.Unlock()
r.download.variants[resolution] = &VariantStatus{
Resolution: resolution,
Bandwidth: bandwidth,
Active: true,
LastActivity: time.Now(),
}
}
func (r *Registry) RecordVariantStopped(resolution string) {
r.mu.Lock()
defer r.mu.Unlock()
if v, ok := r.download.variants[resolution]; ok {
v.Active = false
}
}
func (r *Registry) RecordSegmentDownloaded(resolution string) {
r.mu.Lock()
defer r.mu.Unlock()
v := r.ensureVariantLocked(resolution)
v.SegmentsOK++
v.LastActivity = time.Now()
}
func (r *Registry) RecordSegmentFailed(resolution string, err error) {
r.mu.Lock()
defer r.mu.Unlock()
v := r.ensureVariantLocked(resolution)
v.SegmentsFailed++
v.LastActivity = time.Now()
v.LastError = errString(err)
r.download.recentErrors = appendBounded(r.download.recentErrors, ErrorEntry{
Time: time.Now(),
Message: resolution + ": " + errString(err),
})
}
func (r *Registry) RecordPlaylistError(resolution string, err error) {
r.mu.Lock()
defer r.mu.Unlock()
v := r.ensureVariantLocked(resolution)
v.LastError = errString(err)
r.download.recentErrors = appendBounded(r.download.recentErrors, ErrorEntry{
Time: time.Now(),
Message: resolution + " playlist: " + errString(err),
})
}
func (r *Registry) ensureVariantLocked(resolution string) *VariantStatus {
v, ok := r.download.variants[resolution]
if !ok {
v = &VariantStatus{Resolution: resolution, Active: true}
r.download.variants[resolution] = v
}
return v
}
// --- Transfer ---
// AttachTransfer wires the registry to the live counters/state already
// maintained by pkg/transfer instead of duplicating them. Called once from
// TransferService construction.
func (r *Registry) AttachTransfer(stats StatsProvider, queueSizeFn, cleanupPendingFn func() int, nasConnectedFn func() bool) {
r.mu.Lock()
defer r.mu.Unlock()
r.transfer.statsProvider = stats
r.transfer.queueSizeFn = queueSizeFn
r.transfer.cleanupPendingFn = cleanupPendingFn
r.transfer.nasConnectedFn = nasConnectedFn
}
func (r *Registry) RecordNASConnectivity(connected bool) {
r.mu.Lock()
defer r.mu.Unlock()
r.transfer.nasConnected = connected
}
// RecordTransferError captures failures QueueStats' cumulative counters
// don't individually retain (pre-queue failures, cleanup failures) so they
// still surface in the bounded recent-errors list.
func (r *Registry) RecordTransferError(sourcePath string, err error) {
r.mu.Lock()
defer r.mu.Unlock()
r.transfer.recentErrors = appendBounded(r.transfer.recentErrors, ErrorEntry{
Time: time.Now(),
Message: sourcePath + ": " + errString(err),
})
}
// --- Processing ---
func (r *Registry) RecordProcessingStageChange(state, eventName string) {
r.mu.Lock()
defer r.mu.Unlock()
r.processing.state = state
if eventName != "" {
r.processing.eventName = eventName
}
switch state {
case "scanning":
r.processing.startedAt = time.Now()
r.processing.finishedAt = time.Time{}
r.processing.lastError = ""
case "completed", "failed":
r.processing.finishedAt = time.Now()
}
}
func (r *Registry) RecordProcessingSegmentsFound(count int) {
r.mu.Lock()
defer r.mu.Unlock()
r.processing.segmentsFound = count
}
func (r *Registry) RecordProcessingError(err error) {
r.mu.Lock()
defer r.mu.Unlock()
msg := errString(err)
r.processing.lastError = msg
r.processing.recentErrors = appendBounded(r.processing.recentErrors, ErrorEntry{
Time: time.Now(),
Message: msg,
})
}
// --- Snapshot ---
type DownloadSnapshot struct {
Variants []*VariantStatus `json:"variants"`
RecentErrors []ErrorEntry `json:"recent_errors"`
}
type TransferSnapshot struct {
NASConnected bool `json:"nas_connected"`
QueueSize int `json:"queue_size"`
CleanupPending int `json:"cleanup_pending"`
Added int `json:"added"`
Completed int `json:"completed"`
Failed int `json:"failed"`
Pending int `json:"pending"`
BytesTransferred int64 `json:"bytes_transferred"`
RecentErrors []ErrorEntry `json:"recent_errors"`
}
type ProcessingSnapshot struct {
State string `json:"state"`
EventName string `json:"event_name,omitempty"`
SegmentsFound int `json:"segments_found"`
StartedAt *time.Time `json:"started_at,omitempty"`
FinishedAt *time.Time `json:"finished_at,omitempty"`
LastError string `json:"last_error,omitempty"`
RecentErrors []ErrorEntry `json:"recent_errors"`
}
type Snapshot struct {
Status string `json:"status"`
UptimeSec float64 `json:"uptime_seconds"`
Download DownloadSnapshot `json:"download"`
Transfer TransferSnapshot `json:"transfer"`
Processing ProcessingSnapshot `json:"processing"`
}
func (r *Registry) Snapshot() Snapshot {
r.mu.RLock()
defer r.mu.RUnlock()
variants := make([]*VariantStatus, 0, len(r.download.variants))
for _, v := range r.download.variants {
vCopy := *v
variants = append(variants, &vCopy)
}
sort.Slice(variants, func(i, j int) bool { return variants[i].Resolution < variants[j].Resolution })
dl := DownloadSnapshot{
Variants: variants,
RecentErrors: append([]ErrorEntry(nil), r.download.recentErrors...),
}
tr := TransferSnapshot{
NASConnected: r.transfer.nasConnected,
RecentErrors: append([]ErrorEntry(nil), r.transfer.recentErrors...),
}
if r.transfer.statsProvider != nil {
added, completed, failed, pending, bytes := r.transfer.statsProvider.GetStats()
tr.Added, tr.Completed, tr.Failed, tr.Pending, tr.BytesTransferred = added, completed, failed, pending, bytes
}
if r.transfer.queueSizeFn != nil {
tr.QueueSize = r.transfer.queueSizeFn()
}
if r.transfer.cleanupPendingFn != nil {
tr.CleanupPending = r.transfer.cleanupPendingFn()
}
if r.transfer.nasConnectedFn != nil {
tr.NASConnected = r.transfer.nasConnectedFn()
}
proc := ProcessingSnapshot{
State: r.processing.state,
EventName: r.processing.eventName,
SegmentsFound: r.processing.segmentsFound,
LastError: r.processing.lastError,
RecentErrors: append([]ErrorEntry(nil), r.processing.recentErrors...),
}
if !r.processing.startedAt.IsZero() {
t := r.processing.startedAt
proc.StartedAt = &t
}
if !r.processing.finishedAt.IsZero() {
t := r.processing.finishedAt
proc.FinishedAt = &t
}
snap := Snapshot{
UptimeSec: time.Since(r.startedAt).Seconds(),
Download: dl,
Transfer: tr,
Processing: proc,
}
snap.Status = computeStatus(dl, tr, proc)
return snap
}
// computeStatus gives one glanceable field: "degraded" the moment anything
// worth investigating shows up, "healthy" otherwise. Transfer is only
// considered for degradation once it's actually in use (Attached + has seen
// activity), so a download-only run with NAS transfer disabled doesn't
// falsely report degraded.
func computeStatus(dl DownloadSnapshot, tr TransferSnapshot, proc ProcessingSnapshot) string {
if len(dl.RecentErrors) > 0 || len(tr.RecentErrors) > 0 || len(proc.RecentErrors) > 0 {
return "degraded"
}
if !tr.NASConnected && (tr.Added > 0 || tr.QueueSize > 0) {
return "degraded"
}
if proc.State == "failed" {
return "degraded"
}
return "healthy"
}