diff --git a/CLAUDE.md b/CLAUDE.md index 2662eab..165cd0e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -15,6 +15,7 @@ The project follows a modular architecture with clear separation of concerns: - **downloader/download.go**: Core download orchestration logic with transfer service integration - **processor/process.go**: Alternative processing entry point - **transfer/transfer.go**: Transfer-only mode entry point + - **check/check.go**: Connection-check mode entry point (stream host + NAS reachability, no downloads) - **pkg/**: Core packages containing the application logic - **media/**: HLS streaming and download logic - **stream.go**: Stream variant parsing and downloading orchestration (`GetAllVariants`, `VariantDownloader`) @@ -156,6 +157,7 @@ go fmt ./... - `-debug`: Debug mode (only downloads 1080p variant for easier testing) - `-transfer`: Transfer-only mode (transfer existing files without downloading) - `-process`: Process-only mode (process existing files without downloading) +- `-check`: Connection-check mode (test stream URL + NAS connectivity and exit; exit code 1 on any failure) ## Monitoring and Downloads diff --git a/README.md b/README.md index 1c05787..c3d2ae7 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ make build All modes are invoked through the single `cmd/main` binary: ``` -m3u8-downloader [-url ] [-event ] [-debug] [-transfer] [-process] +m3u8-downloader [-url ] [-event ] [-debug] [-transfer] [-process] [-check] ``` | Flag | Applies to | Meaning | @@ -118,9 +118,25 @@ m3u8-downloader [-url ] [-event ] [-debug] [-transfer] [-process | `-debug` | download mode | Restricts download to the 1080p variant only (for faster iteration) and forces log level to `debug`. | | `-transfer` | transfer-only mode | Scans an already-downloaded event's local directory and transfers/re-transfers anything not yet confirmed on NAS, then runs the normal transfer service until shutdown. Requires `ENABLE_NAS_TRANSFER=true`. | | `-process` | process-only mode | Reads an event's segments from NAS and concatenates the highest-quality ones into a single `.mp4` via FFmpeg. Always attempts a NAS connection on startup, even if processing is otherwise disabled in config. | +| `-check` | connection-check mode | Tests connectivity to everything a recording depends on, then exits without downloading or transferring anything. Exit code is 0 only if no check failed. | ### Which command for which situation +**Verifying the setup before an event** (in the spirit of `dbt debug` — proves the stream host and NAS are reachable without recording anything): +```bash +m3u8-downloader -check -url "https://example.com/live/master.m3u8" +``` +It runs three checks and prints pass/fail for each: + +- **configuration** — config loads, and the local/process/log directories resolve and are creatable. +- **stream playlist** — only when `-url` is given (skipped otherwise). Fetches the master playlist with the same User-Agent and Referer the downloader uses, requires HTTP 200, decodes it, and lists the variants (and the resolution labels that would become output directories) it found. +- **nas storage** — only when `ENABLE_NAS_TRANSFER=true` (skipped otherwise). Runs the same connect → mkdir → write-probe sequence the transfer service does at startup, bounded by the NAS timeout so an unresponsive share reports a timeout instead of hanging. + +Skipped checks don't fail the run; the exit code is 1 if any check actually failed, so this can gate a scripted start: +```bash +m3u8-downloader -check -url "$URL" && m3u8-downloader -url "$URL" -event "2026-regionals" +``` + **Recording a live event** (the normal case — downloads and transfers to NAS concurrently, in one process, so local disk usage stays bounded rather than accumulating the whole event): ```bash m3u8-downloader -url "https://example.com/live/master.m3u8" -event "2026-regionals" diff --git a/bin/flo_download b/bin/flo_download old mode 100644 new mode 100755 index f1fe236..7dd8c2a Binary files a/bin/flo_download and b/bin/flo_download differ diff --git a/bin/flo_download.exe b/bin/flo_download.exe deleted file mode 100644 index 7c77b0f..0000000 Binary files a/bin/flo_download.exe and /dev/null differ diff --git a/cmd/check/check.go b/cmd/check/check.go new file mode 100644 index 0000000..ef045bd --- /dev/null +++ b/cmd/check/check.go @@ -0,0 +1,285 @@ +// Package check implements connection-check mode: it exercises the external +// dependencies a recording needs — the stream host and the NAS share — and +// reports pass/fail for each without downloading or transferring anything. +// Same spirit as `dbt debug`: prove the wiring before starting real work. +package check + +import ( + "context" + "fmt" + "m3u8-downloader/pkg/config" + "m3u8-downloader/pkg/constants" + "m3u8-downloader/pkg/httpClient" + "m3u8-downloader/pkg/logging" + "m3u8-downloader/pkg/media" + "m3u8-downloader/pkg/nas" + "net/http" + "net/url" + "strings" + "time" + + "github.com/grafov/m3u8" +) + +// playlistTimeout bounds the master playlist fetch. Deliberately short: a +// stream host that takes longer than this is not usable for live recording. +const playlistTimeout = 15 * time.Second + +// Run executes every applicable check and returns a process exit code, 0 only +// when nothing failed, so the command can gate a scripted recording start. +func Run(masterURL string) int { + fmt.Println("Running connection checks...") + fmt.Println() + + cfg, err := constants.GetConfig() + if err != nil { + reportFail("configuration", nil, err) + fmt.Println() + fmt.Println("1 check failed.") + return 1 + } + + appLog := logging.MustGetFactory().App + appLog.Info("connection check starting", "url_provided", masterURL != "") + + reportPass("configuration", []string{ + "local output: " + cfg.Paths.LocalOutput, + "process output: " + cfg.Paths.ProcessOutput, + "log dir: " + cfg.Logging.Dir, + fmt.Sprintf("nas transfer: %s", enabledLabel(cfg.NAS.EnableTransfer)), + }) + + failures := 0 + + if masterURL == "" { + reportSkip("stream playlist", "no -url provided") + } else if details, err := checkMasterPlaylist(masterURL); err != nil { + reportFail("stream playlist", details, err) + appLog.Error("stream playlist check failed", logging.KeyError, err.Error()) + failures++ + } else { + reportPass("stream playlist", details) + appLog.Info("stream playlist check passed") + } + + if !cfg.NAS.EnableTransfer { + reportSkip("nas storage", "transfer disabled in configuration") + } else if details, err := checkNAS(cfg); err != nil { + reportFail("nas storage", details, err) + appLog.Error("nas check failed", logging.KeyError, err.Error()) + failures++ + } else { + reportPass("nas storage", details) + appLog.Info("nas check passed", "path", cfg.NAS.OutputPath) + } + + if failures > 0 { + fmt.Printf("%d check(s) failed.\n", failures) + appLog.Error("connection check finished with failures", "failures", failures) + return 1 + } + + fmt.Println("All checks passed.") + appLog.Info("connection check passed") + return 0 +} + +// checkMasterPlaylist fetches the master playlist exactly as the downloader +// would — same headers, same decoder — and summarizes what it found. Details +// gathered before the failure point are returned alongside the error so a +// failed check still shows how far it got. +func checkMasterPlaylist(masterURL string) ([]string, error) { + parsed, err := url.Parse(masterURL) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return nil, fmt.Errorf("not a valid absolute URL: %q", masterURL) + } + + details := []string{"host: " + parsed.Host} + + ctx, cancel := context.WithTimeout(context.Background(), playlistTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, masterURL, nil) + if err != nil { + return details, fmt.Errorf("failed to build request: %w", err) + } + req.Header.Set("User-Agent", constants.HTTPUserAgent) + req.Header.Set("Referer", constants.REFERRER) + + client := &http.Client{Timeout: playlistTimeout} + start := time.Now() + resp, err := client.Do(req) + if err != nil { + return details, fmt.Errorf("request failed: %w", err) + } + defer resp.Body.Close() + + details = append(details, fmt.Sprintf("HTTP %d in %s", resp.StatusCode, formatDuration(time.Since(start)))) + + if resp.StatusCode != http.StatusOK { + return details, httpClient.NewHTTPError(resp.StatusCode, "unexpected status fetching master playlist") + } + + playlist, listType, err := m3u8.DecodeFrom(resp.Body, true) + if err != nil { + return details, fmt.Errorf("response is not a valid m3u8 playlist: %w", err) + } + + switch listType { + case m3u8.MASTER: + master, ok := playlist.(*m3u8.MasterPlaylist) + if !ok { + return details, fmt.Errorf("decoder returned an unexpected playlist type") + } + if len(master.Variants) == 0 { + return details, fmt.Errorf("master playlist contains no variants") + } + details = append(details, fmt.Sprintf("master playlist with %d variant(s):", len(master.Variants))) + for _, v := range master.Variants { + if v == nil { + continue + } + details = append(details, fmt.Sprintf(" %s (%d bps)", media.ExtractResolution(v), v.Bandwidth)) + } + case m3u8.MEDIA: + mediaPlaylist, ok := playlist.(*m3u8.MediaPlaylist) + if !ok { + return details, fmt.Errorf("decoder returned an unexpected playlist type") + } + details = append(details, + fmt.Sprintf("media playlist with %d segment(s), recorded as a single \"unknown\" variant", mediaPlaylist.Count())) + default: + return details, fmt.Errorf("unrecognized playlist type") + } + + return details, nil +} + +// checkNAS runs the same connect/mkdir/write sequence the transfer service +// performs at startup, but without the log.Fatalf that path takes on failure. +func checkNAS(cfg *config.Config) ([]string, error) { + svc := &nas.NASService{Config: nas.NASConfig{ + Path: cfg.NAS.OutputPath, + Username: cfg.NAS.Username, + Password: cfg.NAS.Password, + Timeout: cfg.NAS.Timeout, + RetryLimit: cfg.NAS.RetryLimit, + VerifySize: true, + }} + + details := []string{"path: " + cfg.NAS.OutputPath} + if share := svc.ExtractNetworkPath(cfg.NAS.OutputPath); share != "" { + details = append(details, "share: "+share, "user: "+usernameLabel(cfg.NAS.Username)) + } else { + details = append(details, "local or already-mounted path (no network authentication)") + } + + start := time.Now() + result := <-probeAsync(cfg.NAS.Timeout, func() ([]string, error) { + if err := svc.EstablishConnection(); err != nil { + return nil, fmt.Errorf("failed to connect to share: %w", err) + } + if err := svc.EnsureDirectoryExists(cfg.NAS.OutputPath); err != nil { + return nil, fmt.Errorf("share reachable but target directory unusable: %w", err) + } + if err := svc.TestConnection(); err != nil { + return nil, fmt.Errorf("target directory is not writable: %w", err) + } + return []string{"connect, mkdir and write probe succeeded"}, nil + }) + + if result.err != nil { + return details, result.err + } + + details = append(details, result.details...) + details = append(details, "completed in "+formatDuration(time.Since(start))) + return details, nil +} + +func formatDuration(d time.Duration) string { + if d < time.Millisecond { + return "<1ms" + } + return d.Round(time.Millisecond).String() +} + +type probeResult struct { + details []string + err error +} + +// probeAsync bounds a filesystem probe with a timeout. An unresponsive SMB +// mount can block indefinitely in the kernel, which would otherwise hang the +// check with no output; the orphaned goroutine dies with the process. +func probeAsync(timeout time.Duration, probe func() ([]string, error)) <-chan probeResult { + out := make(chan probeResult, 1) + done := make(chan probeResult, 1) + + go func() { + details, err := probe() + done <- probeResult{details: details, err: err} + }() + + go func() { + select { + case res := <-done: + out <- res + case <-time.After(timeout): + out <- probeResult{err: fmt.Errorf("timed out after %s", timeout)} + } + }() + + return out +} + +func reportPass(name string, details []string) { + fmt.Printf("✓ %s\n", name) + printDetails(details) +} + +func reportFail(name string, details []string, err error) { + fmt.Printf("✗ %s\n", name) + printDetails(append(details, "error: "+err.Error())) +} + +func reportSkip(name, reason string) { + fmt.Printf("- %s (skipped: %s)\n", name, reason) + fmt.Println() +} + +// maxDetailLines caps a single detail entry. Failures from `net use` and +// friends can carry a full usage dump, which buries the actual diagnosis. +const maxDetailLines = 8 + +func printDetails(details []string) { + for _, d := range details { + lines := strings.Split(strings.TrimRight(d, "\n"), "\n") + truncated := 0 + if len(lines) > maxDetailLines { + truncated = len(lines) - maxDetailLines + lines = lines[:maxDetailLines] + } + for _, line := range lines { + fmt.Printf(" %s\n", line) + } + if truncated > 0 { + fmt.Printf(" ... (%d more line(s))\n", truncated) + } + } + fmt.Println() +} + +func enabledLabel(enabled bool) string { + if enabled { + return "enabled" + } + return "disabled" +} + +func usernameLabel(username string) string { + if username == "" { + return "(none, using ambient credentials)" + } + return username +} diff --git a/cmd/check/check_test.go b/cmd/check/check_test.go new file mode 100644 index 0000000..f795a3a --- /dev/null +++ b/cmd/check/check_test.go @@ -0,0 +1,220 @@ +package check + +import ( + "errors" + "fmt" + "m3u8-downloader/pkg/config" + "m3u8-downloader/pkg/httpClient" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +const masterPlaylistBody = `#EXTM3U +#EXT-X-STREAM-INF:BANDWIDTH=5500000,RESOLUTION=1920x1080 +1080p/chunk.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=3200000,RESOLUTION=1280x720 +720p/chunk.m3u8 +` + +const mediaPlaylistBody = `#EXTM3U +#EXT-X-TARGETDURATION:4 +#EXT-X-MEDIA-SEQUENCE:10 +#EXTINF:4.0, +seg1.ts +#EXTINF:4.0, +seg2.ts +` + +// playlistServer serves body at /playlist.m3u8 with the given status, and +// records the request headers so we can assert the check sends the same ones +// the real downloader does. +func playlistServer(t *testing.T, status int, body string) (*httptest.Server, *http.Header) { + t.Helper() + var got http.Header + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got = r.Header.Clone() + w.WriteHeader(status) + fmt.Fprint(w, body) + })) + t.Cleanup(srv.Close) + return srv, &got +} + +func joinDetails(details []string) string { + return strings.Join(details, "\n") +} + +func TestCheckMasterPlaylistMaster(t *testing.T) { + srv, headers := playlistServer(t, http.StatusOK, masterPlaylistBody) + + details, err := checkMasterPlaylist(srv.URL + "/playlist.m3u8") + if err != nil { + t.Fatalf("expected master playlist check to pass, got: %v", err) + } + + joined := joinDetails(details) + for _, want := range []string{"master playlist with 2 variant(s)", "1080p (5500000 bps)", "720p (3200000 bps)", "HTTP 200"} { + if !strings.Contains(joined, want) { + t.Errorf("details missing %q:\n%s", want, joined) + } + } + + if ua := headers.Get("User-Agent"); !strings.Contains(ua, "Mozilla/5.0") { + t.Errorf("expected downloader User-Agent to be sent, got %q", ua) + } + if ref := headers.Get("Referer"); ref == "" { + t.Error("expected Referer header to be sent") + } +} + +func TestCheckMasterPlaylistMedia(t *testing.T) { + srv, _ := playlistServer(t, http.StatusOK, mediaPlaylistBody) + + details, err := checkMasterPlaylist(srv.URL + "/playlist.m3u8") + if err != nil { + t.Fatalf("expected media playlist check to pass, got: %v", err) + } + + if joined := joinDetails(details); !strings.Contains(joined, "media playlist with 2 segment(s)") { + t.Errorf("expected media playlist segment count in details:\n%s", joined) + } +} + +func TestCheckMasterPlaylistNon200(t *testing.T) { + srv, _ := playlistServer(t, http.StatusForbidden, "denied") + + details, err := checkMasterPlaylist(srv.URL + "/playlist.m3u8") + if err == nil { + t.Fatal("expected a 403 response to fail the check") + } + if code := httpClient.GetHTTPStatusCode(err); code != http.StatusForbidden { + t.Errorf("expected a structured HTTP 403 error, got code %d from %v", code, err) + } + // Details collected before the failure should still be reported. + if joined := joinDetails(details); !strings.Contains(joined, "HTTP 403") { + t.Errorf("expected partial details on failure:\n%s", joined) + } +} + +func TestCheckMasterPlaylistNotAPlaylist(t *testing.T) { + srv, _ := playlistServer(t, http.StatusOK, "this is not a playlist") + + if _, err := checkMasterPlaylist(srv.URL + "/playlist.m3u8"); err == nil { + t.Fatal("expected a non-m3u8 body to fail the check") + } +} + +func TestCheckMasterPlaylistEmptyMaster(t *testing.T) { + srv, _ := playlistServer(t, http.StatusOK, "#EXTM3U\n") + + if _, err := checkMasterPlaylist(srv.URL + "/playlist.m3u8"); err == nil { + t.Fatal("expected a playlist with no variants or segments to fail the check") + } +} + +func TestCheckMasterPlaylistInvalidURL(t *testing.T) { + for _, bad := range []string{"", "notaurl", "/relative/path.m3u8", "://missing-scheme"} { + if _, err := checkMasterPlaylist(bad); err == nil { + t.Errorf("expected %q to be rejected as an invalid URL", bad) + } + } +} + +func TestCheckMasterPlaylistUnreachable(t *testing.T) { + // Port 9 (discard) refuses connections without a listener. + if _, err := checkMasterPlaylist("http://127.0.0.1:9/playlist.m3u8"); err == nil { + t.Fatal("expected an unreachable host to fail the check") + } +} + +func writeFile(path string) error { + return os.WriteFile(path, []byte("blocker"), 0644) +} + +func nasConfig(path string) *config.Config { + cfg := &config.Config{} + cfg.NAS.EnableTransfer = true + cfg.NAS.OutputPath = path + cfg.NAS.Timeout = 10 * time.Second + cfg.NAS.RetryLimit = 3 + return cfg +} + +func TestCheckNASLocalPath(t *testing.T) { + target := filepath.Join(t.TempDir(), "streams") + + details, err := checkNAS(nasConfig(target)) + if err != nil { + t.Fatalf("expected a writable local path to pass the NAS check, got: %v", err) + } + + joined := joinDetails(details) + for _, want := range []string{"path: " + target, "write probe succeeded"} { + if !strings.Contains(joined, want) { + t.Errorf("details missing %q:\n%s", want, joined) + } + } + // The probe must not leave its test file behind. + if matches, _ := filepath.Glob(filepath.Join(target, ".connection_test")); len(matches) != 0 { + t.Errorf("connection test file was not cleaned up: %v", matches) + } +} + +func TestCheckNASUnusableTarget(t *testing.T) { + // A regular file cannot be turned into the destination directory. + blocker := filepath.Join(t.TempDir(), "not-a-dir") + if err := writeFile(blocker); err != nil { + t.Fatalf("failed to set up test file: %v", err) + } + + details, err := checkNAS(nasConfig(filepath.Join(blocker, "streams"))) + if err == nil { + t.Fatal("expected an unusable NAS target to fail the check") + } + if joined := joinDetails(details); !strings.Contains(joined, "path: ") { + t.Errorf("expected path detail on failure:\n%s", joined) + } +} + +func TestProbeAsyncTimesOut(t *testing.T) { + start := time.Now() + result := <-probeAsync(50*time.Millisecond, func() ([]string, error) { + time.Sleep(5 * time.Second) + return nil, nil + }) + + if result.err == nil { + t.Fatal("expected a hung probe to report a timeout") + } + if !strings.Contains(result.err.Error(), "timed out") { + t.Errorf("expected a timeout error, got: %v", result.err) + } + if elapsed := time.Since(start); elapsed > time.Second { + t.Errorf("probe took %s, expected it to abandon the hung call promptly", elapsed) + } +} + +func TestProbeAsyncPassesThroughResults(t *testing.T) { + sentinel := errors.New("probe failed") + + if result := <-probeAsync(time.Second, func() ([]string, error) { return []string{"ok"}, nil }); result.err != nil || len(result.details) != 1 { + t.Errorf("expected success to pass through, got details=%v err=%v", result.details, result.err) + } + if result := <-probeAsync(time.Second, func() ([]string, error) { return nil, sentinel }); !errors.Is(result.err, sentinel) { + t.Errorf("expected probe error to pass through, got: %v", result.err) + } +} + +func TestFormatDuration(t *testing.T) { + if got := formatDuration(100 * time.Microsecond); got != "<1ms" { + t.Errorf("sub-millisecond duration = %q, want \"<1ms\"", got) + } + if got := formatDuration(1500 * time.Millisecond); got != "1.5s" { + t.Errorf("1.5s duration = %q", got) + } +} diff --git a/cmd/main/main.go b/cmd/main/main.go index b96ad8d..a611301 100644 --- a/cmd/main/main.go +++ b/cmd/main/main.go @@ -4,6 +4,7 @@ import ( "bufio" "flag" "fmt" + "m3u8-downloader/cmd/check" "m3u8-downloader/cmd/downloader" "m3u8-downloader/cmd/processor" "m3u8-downloader/cmd/transfer" @@ -19,6 +20,7 @@ func main() { debug := flag.Bool("debug", false, "Enable debug mode") transferOnly := flag.Bool("transfer", false, "Transfer-only mode: transfer existing files without downloading") processOnly := flag.Bool("process", false, "Process-only mode: process existing files without downloading") + checkOnly := flag.Bool("check", false, "Connection-check mode: test the stream URL and NAS connectivity, then exit") flag.Parse() @@ -30,6 +32,10 @@ func main() { } logging.MustGetFactory().App.Info("m3u8-downloader starting", "debug", *debug) + if *checkOnly { + os.Exit(check.Run(*url)) + } + if *transferOnly { transfer.RunTransferOnly(*eventName) return diff --git a/pkg/media/stream.go b/pkg/media/stream.go index 474dd6d..3851674 100644 --- a/pkg/media/stream.go +++ b/pkg/media/stream.go @@ -27,7 +27,9 @@ type StreamVariant struct { Writer *ManifestWriter } -func extractResolution(variant *m3u8.Variant) string { +// ExtractResolution maps a variant to the label used for its output directory, +// falling back to a bandwidth heuristic when the playlist omits RESOLUTION. +func ExtractResolution(variant *m3u8.Variant) string { if variant.Resolution != "" { parts := strings.Split(variant.Resolution, "x") if len(parts) == 2 { @@ -89,7 +91,7 @@ func GetAllVariants(masterURL string, outputDir string, writer *ManifestWriter) for i, v := range master.Variants { vURL, _ := url.Parse(v.URI) fullURL := base.ResolveReference(vURL).String() - resolution := extractResolution(v) + resolution := ExtractResolution(v) outputDir := path.Join(outputDir, resolution) variants = append(variants, &StreamVariant{ URL: fullURL,