Compare commits
No commits in common. "main" and "cleanup" have entirely different histories.
16
CLAUDE.md
16
CLAUDE.md
@ -15,7 +15,6 @@ The project follows a modular architecture with clear separation of concerns:
|
|||||||
- **downloader/download.go**: Core download orchestration logic with transfer service integration
|
- **downloader/download.go**: Core download orchestration logic with transfer service integration
|
||||||
- **processor/process.go**: Alternative processing entry point
|
- **processor/process.go**: Alternative processing entry point
|
||||||
- **transfer/transfer.go**: Transfer-only mode entry point
|
- **transfer/transfer.go**: Transfer-only mode entry point
|
||||||
- **check/check.go**: Connection-check mode entry point (stream host + NAS reachability, no downloads)
|
|
||||||
- **pkg/**: Core packages containing the application logic
|
- **pkg/**: Core packages containing the application logic
|
||||||
- **media/**: HLS streaming and download logic
|
- **media/**: HLS streaming and download logic
|
||||||
- **stream.go**: Stream variant parsing and downloading orchestration (`GetAllVariants`, `VariantDownloader`)
|
- **stream.go**: Stream variant parsing and downloading orchestration (`GetAllVariants`, `VariantDownloader`)
|
||||||
@ -33,15 +32,9 @@ The project follows a modular architecture with clear separation of concerns:
|
|||||||
- **service.go**: Processing service orchestration with FFmpeg integration
|
- **service.go**: Processing service orchestration with FFmpeg integration
|
||||||
- **segment.go**: Individual segment processing logic
|
- **segment.go**: Individual segment processing logic
|
||||||
- **types.go**: Processing system data structures
|
- **types.go**: Processing system data structures
|
||||||
- **nas/**: NAS connection and file operations (platform-agnostic: Windows and Linux)
|
- **nas/**: NAS connection and file operations
|
||||||
- **config.go**: NAS configuration structure
|
- **config.go**: NAS configuration structure
|
||||||
- **nas.go**: `NASService` — resolves the destination to a host-native path, then copies/verifies files against it
|
- **nas.go**: NAS service with connection management and file operations
|
||||||
- **location.go**: Parses UNC / `//server/share` / `smb://` / local paths into a host-neutral `Location`
|
|
||||||
- **connector.go**: `Connector` interface + runtime selection by GOOS (no build tags, so both platforms' code is compiled and tested everywhere)
|
|
||||||
- **connector_netuse.go**: Windows strategy — `net use`, UNC path used directly
|
|
||||||
- **connector_mount.go**: Linux strategy — resolve the share against `/proc/mounts`, optional root-only `mount.cifs`
|
|
||||||
- **connector_local.go**: Already-mounted/plain directory on either platform
|
|
||||||
- **mounts.go**: `/proc/mounts` parsing and share-to-mount-point matching
|
|
||||||
- **config/**: Centralized configuration management with validation
|
- **config/**: Centralized configuration management with validation
|
||||||
- **config.go**: Configuration loading, validation, and path resolution
|
- **config.go**: Configuration loading, validation, and path resolution
|
||||||
- **utils/**: Utility functions for cross-platform compatibility
|
- **utils/**: Utility functions for cross-platform compatibility
|
||||||
@ -106,10 +99,8 @@ Configuration is managed through a centralized system in `pkg/config/config.go`
|
|||||||
|
|
||||||
### NAS Transfer Settings
|
### NAS Transfer Settings
|
||||||
- `NAS.EnableTransfer`: Enable/disable automatic NAS transfer (true) - ENV: `ENABLE_NAS_TRANSFER`
|
- `NAS.EnableTransfer`: Enable/disable automatic NAS transfer (true) - ENV: `ENABLE_NAS_TRANSFER`
|
||||||
- `NAS.OutputPath`: NAS destination (`\\HomeLabNAS\dci\streams`) - ENV: `NAS_OUTPUT_PATH`. Accepts UNC, `//server/share`, `smb://`, or a local/mounted directory; Windows connects it with `net use`, Linux resolves it against `/proc/mounts`
|
- `NAS.OutputPath`: UNC path to NAS storage (`\\HomeLabNAS\dci\streams`) - ENV: `NAS_OUTPUT_PATH`
|
||||||
- `NAS.Username`/`NAS.Password`: NAS credentials for authentication - ENV: `NAS_USERNAME`/`NAS_PASSWORD`
|
- `NAS.Username`/`NAS.Password`: NAS credentials for authentication - ENV: `NAS_USERNAME`/`NAS_PASSWORD`
|
||||||
- `NAS.AutoMount`: Linux-only, root-required auto-mount of the SMB share (false) - ENV: `NAS_AUTO_MOUNT`
|
|
||||||
- `NAS.MountPoint`: Override for the auto-mount location - ENV: `NAS_MOUNT_POINT`
|
|
||||||
- `Transfer.WorkerCount`: Concurrent transfer workers (2)
|
- `Transfer.WorkerCount`: Concurrent transfer workers (2)
|
||||||
- `Transfer.RetryLimit`: Max retry attempts per file (3)
|
- `Transfer.RetryLimit`: Max retry attempts per file (3)
|
||||||
- `Transfer.Timeout`: Timeout per file transfer (30 seconds)
|
- `Transfer.Timeout`: Timeout per file transfer (30 seconds)
|
||||||
@ -165,7 +156,6 @@ go fmt ./...
|
|||||||
- `-debug`: Debug mode (only downloads 1080p variant for easier testing)
|
- `-debug`: Debug mode (only downloads 1080p variant for easier testing)
|
||||||
- `-transfer`: Transfer-only mode (transfer existing files without downloading)
|
- `-transfer`: Transfer-only mode (transfer existing files without downloading)
|
||||||
- `-process`: Process-only mode (process existing files without downloading)
|
- `-process`: Process-only mode (process existing files without downloading)
|
||||||
- `-check`: Connection-check mode (test stream URL + NAS connectivity and exit; exit code 1 on any failure)
|
|
||||||
|
|
||||||
## Monitoring and Downloads
|
## Monitoring and Downloads
|
||||||
|
|
||||||
|
|||||||
@ -11,23 +11,10 @@ The application supports configuration through environment variables for flexibl
|
|||||||
- `REFRESH_DELAY_SECONDS`: How often to check for playlist updates in seconds (default: 3)
|
- `REFRESH_DELAY_SECONDS`: How often to check for playlist updates in seconds (default: 3)
|
||||||
|
|
||||||
### NAS Transfer Settings
|
### NAS Transfer Settings
|
||||||
- `NAS_OUTPUT_PATH`: NAS destination (default: "\\\\HomeLabNAS\\dci\\streams"). Accepts a UNC path, `//server/share/dir`, `smb://server/share/dir`, or a local/mounted directory
|
- `NAS_OUTPUT_PATH`: UNC path to NAS storage (default: "\\\\HomeLabNAS\\dci\\streams")
|
||||||
- `NAS_USERNAME`: NAS authentication username
|
- `NAS_USERNAME`: NAS authentication username
|
||||||
- `NAS_PASSWORD`: NAS authentication password
|
- `NAS_PASSWORD`: NAS authentication password
|
||||||
- `ENABLE_NAS_TRANSFER`: Enable/disable automatic NAS transfer (default: true)
|
- `ENABLE_NAS_TRANSFER`: Enable/disable automatic NAS transfer (default: true)
|
||||||
- `NAS_AUTO_MOUNT`: Linux only — mount the SMB share if the host hasn't already (default: false, requires root)
|
|
||||||
- `NAS_MOUNT_POINT`: Where `NAS_AUTO_MOUNT` attaches the share (default: `/mnt/streamrecorder/<server>/<share>`)
|
|
||||||
|
|
||||||
The same `NAS_OUTPUT_PATH` works on Windows and Linux. Windows authenticates the
|
|
||||||
UNC path with `net use`; Linux resolves the share against `/proc/mounts` and
|
|
||||||
rewrites the path onto the mount point, so `\\\\HomeLabNAS\\dci\\streams` with
|
|
||||||
`//homelabnas/dci` mounted at `/mnt/dci` becomes `/mnt/dci/streams`. Mounting the
|
|
||||||
share on the host (fstab/autofs/systemd, or `docker run -v`) is the recommended
|
|
||||||
Linux setup; `NAS_AUTO_MOUNT` is the root-privileged fallback. Verify either with:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./stream-recorder -check
|
|
||||||
```
|
|
||||||
|
|
||||||
### Path Configuration
|
### Path Configuration
|
||||||
- `LOCAL_OUTPUT_DIR`: Base directory for local downloads (default: "data")
|
- `LOCAL_OUTPUT_DIR`: Base directory for local downloads (default: "data")
|
||||||
|
|||||||
217
README.md
217
README.md
@ -1,217 +0,0 @@
|
|||||||
# m3u8-downloader
|
|
||||||
|
|
||||||
A Go command-line tool that records live HLS (HTTP Live Streaming) broadcasts. Point it at a master `.m3u8` playlist URL and it will:
|
|
||||||
|
|
||||||
1. **Download** every quality variant (1080p, 720p, ...) in real time as new segments appear.
|
|
||||||
2. **Transfer** finished segments to NAS storage for archival, with retry logic and local cleanup.
|
|
||||||
3. **Process** the archived segments into a single playable `.mp4` per event via FFmpeg.
|
|
||||||
|
|
||||||
**Download and transfer run together, in one invocation.** When NAS transfer is enabled (the default), starting a download also starts the transfer service in the same process: as soon as a segment is written to disk, a file watcher picks it up, transfers it to NAS, and deletes the local copy — all while the download keeps running. This keeps local disk usage bounded during a long recording instead of accumulating the whole event locally.
|
|
||||||
|
|
||||||
**Processing is a separate, later invocation** (see [Usage](#usage) below), because it reads finished segments *from NAS*, after transfer has fully drained — it can't run concurrently with the other two in any useful way.
|
|
||||||
|
|
||||||
There's also a standalone `-transfer` mode, but it's a catch-up/recovery tool, not the normal path: use it when a download session already ended with some files not yet confirmed on NAS (killed before the queue drained, or transfer was disabled during the recording and you want to sync afterward).
|
|
||||||
|
|
||||||
The project also ships structured JSON logging per subsystem and an HTTP status endpoint with a small web dashboard, so you can tell what's happening (and especially what's failing) without staring at a terminal.
|
|
||||||
|
|
||||||
> Project docs also refer to this as "StreamRecorder" (see `DEPLOYMENT.md`/`TESTING.md`) — same project, the Go module is named `m3u8-downloader`.
|
|
||||||
|
|
||||||
## Architecture
|
|
||||||
|
|
||||||
```
|
|
||||||
┌──────────────┐ ┌──────────────┐ ┌──────────────┐
|
|
||||||
.m3u8 URL ──▶ │ DOWNLOAD │──────▶ │ TRANSFER │──────▶ │ PROCESSING │──▶ event.mp4
|
|
||||||
│ (pkg/media) │ .ts │(pkg/transfer,│ NAS │(pkg/processing)
|
|
||||||
└──────────────┘ files │ pkg/nas) │ files └──────────────┘
|
|
||||||
│ └──────────────┘ │
|
|
||||||
└──────────────┬─────────────────────────────────┘
|
|
||||||
▼
|
|
||||||
pkg/logging (JSON files) + pkg/health (HTTP /status, dashboard)
|
|
||||||
```
|
|
||||||
|
|
||||||
### Download — `pkg/media`
|
|
||||||
|
|
||||||
- `stream.go`: `GetAllVariants()` fetches and parses the master playlist, extracting every quality variant. `VariantDownloader()` runs one per-variant polling loop (every `Core.RefreshDelay`, default 3s) that watches the variant's chunklist for new segments and dispatches downloads through a worker-limited semaphore (`Core.WorkerCount` per variant, default 4).
|
|
||||||
- `playlist.go`: `LoadMediaPlaylist()` fetches and parses an individual variant's chunklist.
|
|
||||||
- `segment.go`: `DownloadSegment()` downloads one `.ts` file, with a couple of retries (including a dedicated retry for transient HTTP 403s some CDNs return).
|
|
||||||
- `manifest.go`: `ManifestWriter` tracks every segment downloaded per event and writes a JSON manifest (sequence number + resolution) at the end of a run.
|
|
||||||
|
|
||||||
Segments land on disk under `Paths.LocalOutput/<event>/<resolution>/`.
|
|
||||||
|
|
||||||
### Transfer — `pkg/transfer` + `pkg/nas`
|
|
||||||
|
|
||||||
- `service.go`: `TransferService` wires together the watcher, queue, and cleanup service, and owns the connection to NAS for the run.
|
|
||||||
- `watcher.go`: `FileWatcher` (via `fsnotify`) watches the local event directory for new `.ts` files and queues them after a short settling delay (`Transfer.FileSettlingDelay`, default 5s), so a file isn't queued while still being written.
|
|
||||||
- `queue.go`: `TransferQueue` is a priority-queued worker pool (`Transfer.WorkerCount`, default 2) that transfers queued files to NAS with retry + exponential backoff, and persists its state to `transfer_queue.json` so it survives a restart.
|
|
||||||
- `nas.go` / `pkg/nas`: `NASService` attaches the NAS destination for whatever host it's running on and does the actual file copy + size verification. See [NAS connectivity](#nas-connectivity-windows-and-linux).
|
|
||||||
- `cleanup.go`: `CleanupService` deletes local copies after a successful, verified transfer (`Cleanup.AfterTransfer`, default true).
|
|
||||||
|
|
||||||
During a normal download run, this whole stage starts automatically in the same process as the downloader (see the note above) — you don't invoke transfer separately for a live recording. The standalone `-transfer` mode exists only to catch up files that a prior download session ended without fully transferring.
|
|
||||||
|
|
||||||
### Processing — `pkg/processing`
|
|
||||||
|
|
||||||
- `ProcessingService` reads segments **from NAS** (not the local disk) for a given event, picks the highest-resolution variant available for each sequence number, writes an FFmpeg concat list, and shells out to FFmpeg to produce one `.mp4`.
|
|
||||||
- This is why processing is normally the *last* step, run after transfer has fully drained — it only sees what's actually on NAS.
|
|
||||||
- Requires a working `ffmpeg` on `PATH` (or `Processing.FFmpegPath` / `FFMPEG_PATH`), and connects to NAS on startup regardless of whether `Processing.Enabled` is set.
|
|
||||||
|
|
||||||
### Configuration — `pkg/config` + `pkg/constants`
|
|
||||||
|
|
||||||
`pkg/config/config.go` defines defaults and environment-variable overrides for every setting below. `pkg/constants.MustGetConfig()` lazily loads and caches this config as a process-wide singleton — most of the codebase reads config this way rather than passing it around explicitly.
|
|
||||||
|
|
||||||
### Logging — `pkg/logging`
|
|
||||||
|
|
||||||
A small factory (modeled after a structured-logging pattern used in another project) that hands out one `log/slog` JSON logger per pipeline stage: `App`, `Download`, `Transfer`, `Processing`. Each writes to its own file under `Logging.Dir` (default `data/logs/{app,download,transfer,processing}.log`). This is **additive** — every existing human-readable console log line is untouched, so you can still watch progress live in the terminal; the structured logs are there for after-the-fact debugging.
|
|
||||||
|
|
||||||
### Health / status — `pkg/health`
|
|
||||||
|
|
||||||
An in-memory registry that every subsystem reports into (active variants, segment success/failure counts, NAS connectivity, transfer queue stats, processing pipeline state), served over HTTP:
|
|
||||||
|
|
||||||
- `GET /status` — full JSON snapshot, `status: "healthy"` or `"degraded"` (degraded the moment anything has a recent error, NAS is disconnected while in use, or processing has failed).
|
|
||||||
- `GET /healthz` — plain liveness check.
|
|
||||||
- `GET /` — a small auto-refreshing dashboard (polls `/status` every 3s) showing the same information visually. No external dependencies (fonts/JS/CSS), so it works on a venue LAN with no internet access.
|
|
||||||
|
|
||||||
**Important scope note:** the health server only reflects whatever is live in the *current* process. During a normal download run, `/status` shows real-time download + transfer state together (they're the same process) with processing reported as `idle`. Running `-process` afterward is a separate process with its own fresh health server showing real-time processing state (and idle download/transfer). There's no single dashboard spanning download/transfer and processing at the same time, since processing only ever runs after the fact.
|
|
||||||
|
|
||||||
By default the health server binds all network interfaces (`:8090`) so it can be checked from another device on the same LAN (e.g. a phone) during an event. **There is no authentication or TLS** — this is fine on a trusted local network only, never expose it to the open internet.
|
|
||||||
|
|
||||||
## Configuration
|
|
||||||
|
|
||||||
All settings have sane defaults; override via environment variables as needed.
|
|
||||||
|
|
||||||
| Env var | Default | Meaning |
|
|
||||||
|---|---|---|
|
|
||||||
| `WORKER_COUNT` | `4` | Concurrent segment downloads per variant |
|
|
||||||
| `REFRESH_DELAY_SECONDS` | `3` | How often each variant's chunklist is polled |
|
|
||||||
| `ENABLE_NAS_TRANSFER` | `true` | Whether to run the transfer service alongside download |
|
|
||||||
| `NAS_OUTPUT_PATH` | `\\HomeLabNAS\dci\streams` | NAS root segments are copied to. UNC (`\\server\share\dir`), `//server/share/dir`, `smb://server/share/dir`, or a local/mounted directory — see [NAS connectivity](#nas-connectivity-windows-and-linux) |
|
|
||||||
| `NAS_USERNAME` / `NAS_PASSWORD` | *(see note below)* | NAS auth credentials. Used by `net use` on Windows and by `NAS_AUTO_MOUNT` on Linux; ignored for an already-mounted path |
|
|
||||||
| `NAS_AUTO_MOUNT` | `false` | Linux only: mount the SMB share if the host hasn't. Requires root |
|
|
||||||
| `NAS_MOUNT_POINT` | *(derived)* | Where `NAS_AUTO_MOUNT` attaches the share (default `/mnt/streamrecorder/<server>/<share>`) |
|
|
||||||
| `LOCAL_OUTPUT_DIR` | `data` | Local base directory for downloaded segments |
|
|
||||||
| `PROCESS_OUTPUT_DIR` | `out` | Output directory for the final concatenated `.mp4` |
|
|
||||||
| `FFMPEG_PATH` | `ffmpeg` | Path to the FFmpeg binary used by the processing stage |
|
|
||||||
| `LOG_DIR` | `data/logs` | Directory for the per-subsystem JSON log files |
|
|
||||||
| `LOG_LEVEL` | `info` | `debug` \| `info` \| `warn` \| `error` (also forced to `debug` by the `-debug` flag) |
|
|
||||||
| `HEALTH_ENABLED` | `true` | Whether to start the health/status HTTP server |
|
|
||||||
| `HEALTH_ADDR` | `:8090` | Bind address for the health server + dashboard |
|
|
||||||
|
|
||||||
> **Security note:** the default NAS credentials currently ship as hardcoded defaults in `pkg/config/config.go` for this home-lab setup. If you fork or redeploy this, set `NAS_USERNAME`/`NAS_PASSWORD` explicitly and rotate the shipped default — see `TODO.md` for the full flagged list of housekeeping items like this one.
|
|
||||||
|
|
||||||
## NAS connectivity (Windows and Linux)
|
|
||||||
|
|
||||||
The same `NAS_OUTPUT_PATH` works on both platforms; only the mechanism for reaching it differs, and the recorder picks it at runtime.
|
|
||||||
|
|
||||||
| Host | Destination | What happens |
|
|
||||||
|---|---|---|
|
|
||||||
| Windows | UNC / `//server/share` / `smb://` | `net use \\server\share` with `NAS_USERNAME`/`NAS_PASSWORD`, then the UNC path is used directly |
|
|
||||||
| Linux | UNC / `//server/share` / `smb://` | The share is looked up in `/proc/mounts` and the UNC path is rewritten onto the mount point (`\\nas\dci\streams` + `//nas/dci` mounted at `/mnt/dci` → `/mnt/dci/streams`). Set `NAS_AUTO_MOUNT=true` to mount it instead |
|
|
||||||
| Either | A local or already-mounted directory | Used as-is, no network step |
|
|
||||||
|
|
||||||
Notes:
|
|
||||||
|
|
||||||
- **Server and share names match case-insensitively**, and both `cifs`/`smb3` and `nfs`/`nfs4` mounts are recognised. If the share is mounted at a deeper level than configured (`//nas/dci/streams` at `/mnt/streams`), the deepest matching mount wins.
|
|
||||||
- **Nothing the host already set up is torn down.** A share found already mounted (or already connected via `net use`) is left alone at shutdown; only a mount or connection this process created is undone.
|
|
||||||
- **`NAS_AUTO_MOUNT` requires root**, so the usual Linux setup is to mount the share yourself via fstab/autofs/systemd and either point `NAS_OUTPUT_PATH` at the mount point or leave it as the UNC path and let it resolve. Credentials are passed to `mount.cifs` through a `0600` credentials file, never on the command line where the process table would expose the password.
|
|
||||||
- **A Windows-shaped path on Linux is rejected, not silently created.** Backslashes and drive letters are legal filename characters on Linux, so `C:\data` or an unresolved `\\nas\dci` would otherwise be created as a local directory named exactly that — transfers would report success, cleanup would delete the local originals, and nothing would reach the NAS.
|
|
||||||
|
|
||||||
Run `-check` to see which strategy applies and where the destination resolved to.
|
|
||||||
|
|
||||||
## Building
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go build -o m3u8-downloader ./cmd/main
|
|
||||||
# or, via the Makefile (produces stream-recorder(.exe)):
|
|
||||||
make build
|
|
||||||
```
|
|
||||||
|
|
||||||
## Usage
|
|
||||||
|
|
||||||
All modes are invoked through the single `cmd/main` binary:
|
|
||||||
|
|
||||||
```
|
|
||||||
m3u8-downloader [-url <m3u8-url>] [-event <name>] [-debug] [-transfer] [-process] [-check]
|
|
||||||
```
|
|
||||||
|
|
||||||
| Flag | Applies to | Meaning |
|
|
||||||
|---|---|---|
|
|
||||||
| `-url` | download mode | Master `.m3u8` playlist URL. If omitted (and neither `-transfer` nor `-process` is set), you'll be prompted for it interactively. |
|
|
||||||
| `-event` | all modes | Event name — used to namespace local output, NAS destination, and the manifest file. **Always set this explicitly**; if left blank in download mode, segments land directly under the local output root instead of a per-event subfolder. In `-transfer`/`-process` mode, omitting it shows an interactive picker over existing event directories. |
|
|
||||||
| `-debug` | download mode | Restricts download to the 1080p variant only (for faster iteration) and forces log level to `debug`. |
|
|
||||||
| `-transfer` | transfer-only mode | Scans an already-downloaded event's local directory and transfers/re-transfers anything not yet confirmed on NAS, then runs the normal transfer service until shutdown. Requires `ENABLE_NAS_TRANSFER=true`. |
|
|
||||||
| `-process` | process-only mode | Reads an event's segments from NAS and concatenates the highest-quality ones into a single `.mp4` via FFmpeg. Always attempts a NAS connection on startup, even if processing is otherwise disabled in config. |
|
|
||||||
| `-check` | connection-check mode | Tests connectivity to everything a recording depends on, then exits without downloading or transferring anything. Exit code is 0 only if no check failed. |
|
|
||||||
|
|
||||||
### Which command for which situation
|
|
||||||
|
|
||||||
**Verifying the setup before an event** (in the spirit of `dbt debug` — proves the stream host and NAS are reachable without recording anything):
|
|
||||||
```bash
|
|
||||||
m3u8-downloader -check -url "https://example.com/live/master.m3u8"
|
|
||||||
```
|
|
||||||
It runs three checks and prints pass/fail for each:
|
|
||||||
|
|
||||||
- **configuration** — config loads, and the local/process/log directories resolve and are creatable.
|
|
||||||
- **stream playlist** — only when `-url` is given (skipped otherwise). Fetches the master playlist with the same User-Agent and Referer the downloader uses, requires HTTP 200, decodes it, and lists the variants (and the resolution labels that would become output directories) it found.
|
|
||||||
- **nas storage** — only when `ENABLE_NAS_TRANSFER=true` (skipped otherwise). Runs the same connect → mkdir → write-probe sequence the transfer service does at startup, bounded by the NAS timeout so an unresponsive share reports a timeout instead of hanging.
|
|
||||||
|
|
||||||
Skipped checks don't fail the run; the exit code is 1 if any check actually failed, so this can gate a scripted start:
|
|
||||||
```bash
|
|
||||||
m3u8-downloader -check -url "$URL" && m3u8-downloader -url "$URL" -event "2026-regionals"
|
|
||||||
```
|
|
||||||
|
|
||||||
**Recording a live event** (the normal case — downloads and transfers to NAS concurrently, in one process, so local disk usage stays bounded rather than accumulating the whole event):
|
|
||||||
```bash
|
|
||||||
m3u8-downloader -url "https://example.com/live/master.m3u8" -event "2026-regionals"
|
|
||||||
```
|
|
||||||
Stop it cleanly with Ctrl+C (SIGINT/SIGTERM); it finishes in-flight downloads, saves transfer queue state, and writes the manifest before exiting.
|
|
||||||
|
|
||||||
**Quick test/dry run** (single variant, verbose logs, to confirm a playlist URL actually works before committing to a full multi-quality recording):
|
|
||||||
```bash
|
|
||||||
m3u8-downloader -url "https://example.com/live/master.m3u8" -event "test-run" -debug
|
|
||||||
```
|
|
||||||
|
|
||||||
**Catching up NAS transfer after the fact** (recovery only — the recording process above already transfers live; use this if a session ended before every file made it to NAS, or transfer was disabled during the recording and you want to sync afterward):
|
|
||||||
```bash
|
|
||||||
m3u8-downloader -event "2026-regionals" -transfer
|
|
||||||
```
|
|
||||||
|
|
||||||
**Producing the final video** (after transfer has fully drained — check `/status` or the NAS directory to confirm nothing's still pending):
|
|
||||||
```bash
|
|
||||||
m3u8-downloader -event "2026-regionals" -process
|
|
||||||
```
|
|
||||||
|
|
||||||
**Checking on a running instance** (from any device on the same network as the recorder):
|
|
||||||
```
|
|
||||||
http://<recorder-host>:8090/ # dashboard
|
|
||||||
http://<recorder-host>:8090/status # raw JSON
|
|
||||||
```
|
|
||||||
|
|
||||||
## Data layout
|
|
||||||
|
|
||||||
```
|
|
||||||
data/
|
|
||||||
├── {event}.json # download manifest (sequence numbers + resolutions)
|
|
||||||
├── {event}/
|
|
||||||
│ ├── 1080p/ # downloaded segments, before NAS transfer + cleanup
|
|
||||||
│ ├── 720p/
|
|
||||||
│ └── ...
|
|
||||||
├── logs/
|
|
||||||
│ ├── app.log # per-subsystem structured JSON logs
|
|
||||||
│ ├── download.log
|
|
||||||
│ ├── transfer.log
|
|
||||||
│ └── processing.log
|
|
||||||
└── transfer_queue.json # persisted transfer queue state
|
|
||||||
|
|
||||||
\\<NAS>\...\{event}\ # mirrors the local structure once transferred
|
|
||||||
├── 1080p/
|
|
||||||
└── ...
|
|
||||||
|
|
||||||
out/
|
|
||||||
└── {event}/
|
|
||||||
└── {event}.mp4 # final FFmpeg output (process-only mode)
|
|
||||||
```
|
|
||||||
|
|
||||||
## More documentation
|
|
||||||
|
|
||||||
- `DEPLOYMENT.md` — Docker/Kubernetes deployment examples and production considerations.
|
|
||||||
- `TESTING.md` — test suite layout and how to run it (`make test`, `make test-coverage`, etc.).
|
|
||||||
- `TODO.md` — known issues, pre-existing bugs, and roadmap items found during review, not yet addressed.
|
|
||||||
111
TODO.md
111
TODO.md
@ -1,111 +0,0 @@
|
|||||||
# Return-from-a-year-away Audit — 2026-08-05
|
|
||||||
|
|
||||||
Scope reviewed: `main` (checked out), `manifest`, `cleanup` (all local + `origin` equivalents).
|
|
||||||
Remote: `https://git.aaronic.cc/kacarmichael/StreamRecorder.git`
|
|
||||||
|
|
||||||
## Branch situation (read this first)
|
|
||||||
|
|
||||||
The three branches are **not parallel efforts** — they're one linear line of development that `main` never caught up to:
|
|
||||||
|
|
||||||
```
|
|
||||||
main (0f4a39f, 2025-07-26) ──┐ stale, single-file-era downloader only
|
|
||||||
│
|
|
||||||
manifest (718d69c, 2025-08-10) ── +restructured cmd/pkg layout, config-less
|
|
||||||
│
|
|
||||||
cleanup (7d67cec, 2025-08-11) ── +pkg/config, tests, CI, Makefile, deployment docs
|
|
||||||
```
|
|
||||||
|
|
||||||
`manifest` is literally an ancestor of `cleanup` (same commit `718d69c`). `cleanup` is the most complete state that ever existed and was the last thing touched (2025-08-11) before the year gap. **None of this reached `main`.** If you pick this project back up, decide this before anything else:
|
|
||||||
|
|
||||||
- [ ] **Merge `cleanup` → `main`** (recommended — it's a strict superset: config validation, `pkg/nas`, `pkg/processing`, unit tests, CI workflow, `DEPLOYMENT.md`/`TESTING.md`). Then delete `manifest` (fully subsumed) and re-evaluate `cleanup` for deletion once merged.
|
|
||||||
- [ ] Or, if `cleanup`'s direction (FFmpeg concatenation, NAS-mirroring processing pipeline) is no longer where you want to go, explicitly archive it and restart `main` from a clean design — but do this as a decision, not by default.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🔴 Critical — fix/rotate before running this again
|
|
||||||
|
|
||||||
1. **Leaked NAS credentials, committed in plaintext.** `pkg/config/config.go` (branch `cleanup`) hardcodes a real default:
|
|
||||||
```go
|
|
||||||
Username: "NASAdmin",
|
|
||||||
Password: "s3tkY6tzA&KN6M",
|
|
||||||
```
|
|
||||||
This is in git history on a remote (`git.aaronic.cc`) regardless of whether you later change the default. **Rotate this NAS account's password now** — changing the code doesn't un-leak a password that's sat in a repo for a year. Then remove the credential entirely (fail closed: require `NAS_USERNAME`/`NAS_PASSWORD` env vars with no default, error out if unset when transfer is enabled) rather than shipping a working default.
|
|
||||||
|
|
||||||
2. **Guaranteed crash: double mutex unlock in `pkg/transfer/cleanup.go`.** In `ExecuteCleanup` (present on **`main` too**, not just `cleanup`):
|
|
||||||
```go
|
|
||||||
cs.mu.Lock()
|
|
||||||
...
|
|
||||||
cs.mu.Unlock() // line 72 — releases the lock
|
|
||||||
...
|
|
||||||
batch := make([]string, batchSize)
|
|
||||||
copy(batch, cs.pendingFiles[:batchSize])
|
|
||||||
cs.pendingFiles = cs.pendingFiles[batchSize:] // mutated WITHOUT the lock held
|
|
||||||
cs.mu.Unlock() // line 79 — unlocking an already-unlocked mutex
|
|
||||||
```
|
|
||||||
An unlock of an unlocked `sync.Mutex` is a **fatal, unrecoverable runtime error** (`fatal error: sync: unlock of unlocked mutex`) — it kills the whole process, not just a goroutine. Since `Cleanup.AfterTransfer` defaults to `true` and `RetainHours: 0` (immediate eligibility), this fires the first time any file finishes transferring and cleanup runs. This is very likely why any end-to-end run with NAS transfer enabled never survived long. Fix: hold the lock for the whole slice mutation, or copy-then-release-then-mutate-under-a-second-short-lock, but don't unlock twice.
|
|
||||||
|
|
||||||
3. **Nil-pointer crash in `pkg/media/stream.go` (`VariantDownloader`, `main` branch):**
|
|
||||||
```go
|
|
||||||
playlist, err := LoadMediaPlaylist(variant.URL)
|
|
||||||
seq := playlist.SeqNo // <-- dereferenced before the err check
|
|
||||||
if err != nil {
|
|
||||||
...
|
|
||||||
goto waitTick
|
|
||||||
}
|
|
||||||
```
|
|
||||||
`LoadMediaPlaylist` returns `nil, err` on any HTTP/parse failure, so `playlist` is a typed-nil pointer here — `playlist.SeqNo` panics. A panic in a goroutine takes down the entire process (Go doesn't isolate goroutine panics). Practically: any single transient network hiccup on any variant's playlist fetch kills the whole download session, mid-recording. Move the `err != nil` check above the `SeqNo` read.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🟠 High-priority bugs / correctness
|
|
||||||
|
|
||||||
4. **`PriorityQueue.Less` in `pkg/transfer/queue.go` sorts newest-first** (`pq[i].Timestamp.After(pq[j].Timestamp)`), making the transfer queue LIFO. For a live-recording pipeline that keeps producing new segments, this means the *oldest* queued segments can be starved indefinitely as long as new ones keep arriving — the opposite of what you'd want (drain oldest first so nothing waits forever). Worth confirming this was intentional; if not, flip the comparison.
|
|
||||||
|
|
||||||
5. **Retry backoff runs before the *first* attempt, not just retries** — `pkg/transfer/queue.go`, `processItem`:
|
|
||||||
```go
|
|
||||||
for attempt := 1; attempt <= maxRetries; attempt++ {
|
|
||||||
if attempt > 0 { // always true, since attempt starts at 1
|
|
||||||
... backoff ...
|
|
||||||
```
|
|
||||||
Every single transfer sleeps `1s` before even trying once. Should be `attempt > 1`.
|
|
||||||
|
|
||||||
6. **`pkg/constants/constants.go` (main) hardcodes a live FloSports auth token** directly in source (`MasterURL` with `token=...`, `hmac2=...`). It's expired now (epoch ~2025-07), so no live exposure, but it's the same pattern as #1 — secrets/tokens don't belong in source. Worth a rule going forward: URLs/tokens come from CLI args or env, never constants.
|
|
||||||
|
|
||||||
7. **`http.Client{}` created with no timeout** in `pkg/media/stream.go`, `playlist.go`, and `segment.go`. A stalled TCP connection (dead CDN edge, network partition) can hang a fetch forever — nothing in the surrounding code will time it out except the 10s context wrapped specifically around segment downloads in `VariantDownloader`. Playlist-fetch calls (`GetAllVariants`, `LoadMediaPlaylist`) have no such wrapper. Give the shared client a sane `Timeout` (e.g. 15–30s) as a backstop.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🟡 Cleanup / dead weight
|
|
||||||
|
|
||||||
8. **`cmd/proc/main.go` is a one-line stub** (`package proc`) on `main` — dead entry point, no `func main()`. Either finish it or delete it; right now `go build ./...` would fail on this if it were ever wired as a real `main` package (it currently silently isn't, which is its own trap for future-you).
|
|
||||||
9. **Built binaries committed to git** (`bin/flo_download`, `bin/flo_download.exe`, ~18MB, on `cleanup`/`manifest`). The `.gitea/workflows/build.yml` already cross-compiles and uploads artifacts — there's no reason binaries should also live in the repo/history. Remove and add `bin/` to `.gitignore`.
|
|
||||||
10. **`scratch.txt` is tracked despite being in `.gitignore`.** Adding a path to `.gitignore` after it's already tracked does nothing — it's still in the repo (and it's just a raw debug log dump). `git rm --cached scratch.txt`.
|
|
||||||
11. `.idea/*` (JetBrains project files) are committed. Low stakes solo-repo, but standard practice is to gitignore the whole `.idea/` directory rather than cherry-pick.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🔵 Program / architecture — bigger picture
|
|
||||||
|
|
||||||
12. **The project's own `CLAUDE.md` states the goal is "a listening REST API [that] take[s] in m3u8 urls, parse[s] them, and eventually send[s] to a conversion service."** None of the three branches implement any HTTP server — all three are CLI-only, one-shot-per-invocation tools. This is the largest gap between stated intent and actual code. Worth deciding: is the REST-API framing still the goal, or has the project's actual direction (CLI recorder + NAS mirror + FFmpeg concat, per `cleanup`'s `CLAUDE.md`) superseded it? Right now the docs and the code disagree with each other.
|
|
||||||
13. **`pkg/nas/nas.go`'s "connection" is a Windows-only shell-out to `net use`** (`EstablishConnection`), while CI (`build.yml`) cross-compiles for `linux`/`darwin`/`windows`. On non-Windows the UNC-path logic (`\\server\share`) and `net use` simply don't apply — there's no SMB client abstraction, so the Linux/macOS builds can produce a binary that can't actually do NAS transfer. If cross-platform deployment is real (not just "it happens to compile"), this needs a real cross-platform SMB/CIFS story (e.g., assume the NAS is pre-mounted by the OS and just treat it as a local path, which is actually simpler and already half-true for the Linux CIFS example in `DEPLOYMENT.md`).
|
|
||||||
14. **Zero tests on `main`.** `cleanup` has real coverage (`pkg/config`, `pkg/utils`, `pkg/constants`, `pkg/httpClient`, `pkg/media/manifest`, `pkg/processing`) plus a `Makefile` (`test`, `test-coverage`, `lint`, `security` via `gosec`) and a `TESTING.md`. All of this is stuck on a branch that was never merged — it's not a "write tests" TODO so much as a "land the tests you already wrote" TODO.
|
|
||||||
15. **CI never actually runs tests, lint, or `gosec`** despite the `Makefile` defining a full `ci` target (`fmt tidy lint security test build`). `.gitea/workflows/build.yml` only cross-compiles. Wire `make ci` (or its steps) into the pipeline so a broken build/test doesn't silently ship a binary artifact.
|
|
||||||
16. **Unbounded `seen` map** in `VariantDownloader` (`pkg/media/stream.go`) grows for the lifetime of a variant's polling loop with no eviction — fine for a few-hour event, worth a note if this is ever meant to run continuously/24-7.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🟢 Deployment
|
|
||||||
|
|
||||||
17. `DEPLOYMENT.md` (on `cleanup`) documents Docker/Compose/Kubernetes deployment in detail, but there's no actual `Dockerfile` committed anywhere in the repo — it's a doc-only example. If containerized deployment is the real target, commit the Dockerfile and at least smoke-test the build.
|
|
||||||
18. No secrets-management story beyond "use env vars" — fine for a home lab, but the doc's own "Security" section says "use secret management systems," which isn't backed by anything concrete (no `.env.example`, no sealed-secrets/vault reference). Given finding #1, this is worth actually doing, not just documenting.
|
|
||||||
19. Health checks, metrics/log shipping are listed as "considerations" in `DEPLOYMENT.md` but nothing is implemented — the app is a fire-and-forget CLI process with `log.Printf` to stdout only. If this runs unattended (e.g., in a container watching for the next event), you have no way to know it's alive short of watching logs.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Suggested order of attack
|
|
||||||
|
|
||||||
1. Rotate the NAS credential (#1) — this is time-sensitive regardless of anything else.
|
|
||||||
2. Fix the two crash bugs (#2, #3) — nothing else is worth doing until a full recording session can survive.
|
|
||||||
3. Decide the branch-merge question and land `cleanup` → `main` (or consciously don't).
|
|
||||||
4. Wire CI to actually test/lint (#15) so regressions get caught going forward.
|
|
||||||
5. Then tackle the architecture question (#12: REST API vs. CLI-recorder-as-is) before investing more in either direction.
|
|
||||||
BIN
bin/flo_download
Executable file → Normal file
BIN
bin/flo_download
Executable file → Normal file
Binary file not shown.
BIN
bin/flo_download.exe
Normal file
BIN
bin/flo_download.exe
Normal file
Binary file not shown.
@ -1,284 +0,0 @@
|
|||||||
// Package check implements connection-check mode: it exercises the external
|
|
||||||
// dependencies a recording needs — the stream host and the NAS share — and
|
|
||||||
// reports pass/fail for each without downloading or transferring anything.
|
|
||||||
// Same spirit as `dbt debug`: prove the wiring before starting real work.
|
|
||||||
package check
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"log"
|
|
||||||
"m3u8-downloader/pkg/config"
|
|
||||||
"m3u8-downloader/pkg/constants"
|
|
||||||
"m3u8-downloader/pkg/httpClient"
|
|
||||||
"m3u8-downloader/pkg/logging"
|
|
||||||
"m3u8-downloader/pkg/media"
|
|
||||||
"m3u8-downloader/pkg/nas"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/grafov/m3u8"
|
|
||||||
)
|
|
||||||
|
|
||||||
// playlistTimeout bounds the master playlist fetch. Deliberately short: a
|
|
||||||
// stream host that takes longer than this is not usable for live recording.
|
|
||||||
const playlistTimeout = 15 * time.Second
|
|
||||||
|
|
||||||
// Run executes every applicable check and returns a process exit code, 0 only
|
|
||||||
// when nothing failed, so the command can gate a scripted recording start.
|
|
||||||
func Run(masterURL string) int {
|
|
||||||
fmt.Println("Running connection checks...")
|
|
||||||
fmt.Println()
|
|
||||||
|
|
||||||
cfg, err := constants.GetConfig()
|
|
||||||
if err != nil {
|
|
||||||
reportFail("configuration", nil, err)
|
|
||||||
fmt.Println()
|
|
||||||
fmt.Println("1 check failed.")
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
appLog := logging.MustGetFactory().App
|
|
||||||
appLog.Info("connection check starting", "url_provided", masterURL != "")
|
|
||||||
|
|
||||||
reportPass("configuration", []string{
|
|
||||||
"local output: " + cfg.Paths.LocalOutput,
|
|
||||||
"process output: " + cfg.Paths.ProcessOutput,
|
|
||||||
"log dir: " + cfg.Logging.Dir,
|
|
||||||
fmt.Sprintf("nas transfer: %s", enabledLabel(cfg.NAS.EnableTransfer)),
|
|
||||||
})
|
|
||||||
|
|
||||||
failures := 0
|
|
||||||
|
|
||||||
if masterURL == "" {
|
|
||||||
reportSkip("stream playlist", "no -url provided")
|
|
||||||
} else if details, err := checkMasterPlaylist(masterURL); err != nil {
|
|
||||||
reportFail("stream playlist", details, err)
|
|
||||||
appLog.Error("stream playlist check failed", logging.KeyError, err.Error())
|
|
||||||
failures++
|
|
||||||
} else {
|
|
||||||
reportPass("stream playlist", details)
|
|
||||||
appLog.Info("stream playlist check passed")
|
|
||||||
}
|
|
||||||
|
|
||||||
if !cfg.NAS.EnableTransfer {
|
|
||||||
reportSkip("nas storage", "transfer disabled in configuration")
|
|
||||||
} else if details, err := checkNAS(cfg); err != nil {
|
|
||||||
reportFail("nas storage", details, err)
|
|
||||||
appLog.Error("nas check failed", logging.KeyError, err.Error())
|
|
||||||
failures++
|
|
||||||
} else {
|
|
||||||
reportPass("nas storage", details)
|
|
||||||
appLog.Info("nas check passed", "path", cfg.NAS.OutputPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
if failures > 0 {
|
|
||||||
fmt.Printf("%d check(s) failed.\n", failures)
|
|
||||||
appLog.Error("connection check finished with failures", "failures", failures)
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Println("All checks passed.")
|
|
||||||
appLog.Info("connection check passed")
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkMasterPlaylist fetches the master playlist exactly as the downloader
|
|
||||||
// would — same headers, same decoder — and summarizes what it found. Details
|
|
||||||
// gathered before the failure point are returned alongside the error so a
|
|
||||||
// failed check still shows how far it got.
|
|
||||||
func checkMasterPlaylist(masterURL string) ([]string, error) {
|
|
||||||
parsed, err := url.Parse(masterURL)
|
|
||||||
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
|
|
||||||
return nil, fmt.Errorf("not a valid absolute URL: %q", masterURL)
|
|
||||||
}
|
|
||||||
|
|
||||||
details := []string{"host: " + parsed.Host}
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), playlistTimeout)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, masterURL, nil)
|
|
||||||
if err != nil {
|
|
||||||
return details, fmt.Errorf("failed to build request: %w", err)
|
|
||||||
}
|
|
||||||
req.Header.Set("User-Agent", constants.HTTPUserAgent)
|
|
||||||
req.Header.Set("Referer", constants.REFERRER)
|
|
||||||
|
|
||||||
client := &http.Client{Timeout: playlistTimeout}
|
|
||||||
start := time.Now()
|
|
||||||
resp, err := client.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
return details, fmt.Errorf("request failed: %w", err)
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
details = append(details, fmt.Sprintf("HTTP %d in %s", resp.StatusCode, formatDuration(time.Since(start))))
|
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
return details, httpClient.NewHTTPError(resp.StatusCode, "unexpected status fetching master playlist")
|
|
||||||
}
|
|
||||||
|
|
||||||
playlist, listType, err := m3u8.DecodeFrom(resp.Body, true)
|
|
||||||
if err != nil {
|
|
||||||
return details, fmt.Errorf("response is not a valid m3u8 playlist: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
switch listType {
|
|
||||||
case m3u8.MASTER:
|
|
||||||
master, ok := playlist.(*m3u8.MasterPlaylist)
|
|
||||||
if !ok {
|
|
||||||
return details, fmt.Errorf("decoder returned an unexpected playlist type")
|
|
||||||
}
|
|
||||||
if len(master.Variants) == 0 {
|
|
||||||
return details, fmt.Errorf("master playlist contains no variants")
|
|
||||||
}
|
|
||||||
details = append(details, fmt.Sprintf("master playlist with %d variant(s):", len(master.Variants)))
|
|
||||||
for _, v := range master.Variants {
|
|
||||||
if v == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
details = append(details, fmt.Sprintf(" %s (%d bps)", media.ExtractResolution(v), v.Bandwidth))
|
|
||||||
}
|
|
||||||
case m3u8.MEDIA:
|
|
||||||
mediaPlaylist, ok := playlist.(*m3u8.MediaPlaylist)
|
|
||||||
if !ok {
|
|
||||||
return details, fmt.Errorf("decoder returned an unexpected playlist type")
|
|
||||||
}
|
|
||||||
details = append(details,
|
|
||||||
fmt.Sprintf("media playlist with %d segment(s), recorded as a single \"unknown\" variant", mediaPlaylist.Count()))
|
|
||||||
default:
|
|
||||||
return details, fmt.Errorf("unrecognized playlist type")
|
|
||||||
}
|
|
||||||
|
|
||||||
return details, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkNAS runs the same attach/mkdir/write sequence the transfer service
|
|
||||||
// performs at startup, so the check exercises the real code path for this host
|
|
||||||
// — net use on Windows, mount resolution on Linux — rather than a simulation of
|
|
||||||
// it. It disconnects afterwards so the check leaves no mount behind.
|
|
||||||
func checkNAS(cfg *config.Config) ([]string, error) {
|
|
||||||
svc, err := nas.New(nas.NASConfig{
|
|
||||||
Path: cfg.NAS.OutputPath,
|
|
||||||
Username: cfg.NAS.Username,
|
|
||||||
Password: cfg.NAS.Password,
|
|
||||||
Timeout: cfg.NAS.Timeout,
|
|
||||||
RetryLimit: cfg.NAS.RetryLimit,
|
|
||||||
VerifySize: true,
|
|
||||||
AutoMount: cfg.NAS.AutoMount,
|
|
||||||
MountPoint: cfg.NAS.MountPoint,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return []string{"path: " + cfg.NAS.OutputPath}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
start := time.Now()
|
|
||||||
result := <-probeAsync(cfg.NAS.Timeout, func() ([]string, error) {
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), cfg.NAS.Timeout)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
if err := svc.Connect(ctx); err != nil {
|
|
||||||
return svc.ConnectionInfo(), err
|
|
||||||
}
|
|
||||||
defer func() {
|
|
||||||
if err := svc.Disconnect(context.Background()); err != nil {
|
|
||||||
log.Printf("Warning: failed to release NAS connection after check: %v", err)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
return append(svc.ConnectionInfo(), "attach, mkdir and write probe succeeded"), nil
|
|
||||||
})
|
|
||||||
|
|
||||||
details := result.details
|
|
||||||
if len(details) == 0 {
|
|
||||||
details = svc.ConnectionInfo()
|
|
||||||
}
|
|
||||||
if result.err != nil {
|
|
||||||
return details, result.err
|
|
||||||
}
|
|
||||||
|
|
||||||
return append(details, "completed in "+formatDuration(time.Since(start))), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func formatDuration(d time.Duration) string {
|
|
||||||
if d < time.Millisecond {
|
|
||||||
return "<1ms"
|
|
||||||
}
|
|
||||||
return d.Round(time.Millisecond).String()
|
|
||||||
}
|
|
||||||
|
|
||||||
type probeResult struct {
|
|
||||||
details []string
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
|
|
||||||
// probeAsync bounds a filesystem probe with a timeout. An unresponsive SMB
|
|
||||||
// mount can block indefinitely in the kernel, which would otherwise hang the
|
|
||||||
// check with no output; the orphaned goroutine dies with the process.
|
|
||||||
func probeAsync(timeout time.Duration, probe func() ([]string, error)) <-chan probeResult {
|
|
||||||
out := make(chan probeResult, 1)
|
|
||||||
done := make(chan probeResult, 1)
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
details, err := probe()
|
|
||||||
done <- probeResult{details: details, err: err}
|
|
||||||
}()
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
select {
|
|
||||||
case res := <-done:
|
|
||||||
out <- res
|
|
||||||
case <-time.After(timeout):
|
|
||||||
out <- probeResult{err: fmt.Errorf("timed out after %s", timeout)}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func reportPass(name string, details []string) {
|
|
||||||
fmt.Printf("✓ %s\n", name)
|
|
||||||
printDetails(details)
|
|
||||||
}
|
|
||||||
|
|
||||||
func reportFail(name string, details []string, err error) {
|
|
||||||
fmt.Printf("✗ %s\n", name)
|
|
||||||
printDetails(append(details, "error: "+err.Error()))
|
|
||||||
}
|
|
||||||
|
|
||||||
func reportSkip(name, reason string) {
|
|
||||||
fmt.Printf("- %s (skipped: %s)\n", name, reason)
|
|
||||||
fmt.Println()
|
|
||||||
}
|
|
||||||
|
|
||||||
// maxDetailLines caps a single detail entry. Failures from `net use` and
|
|
||||||
// friends can carry a full usage dump, which buries the actual diagnosis.
|
|
||||||
const maxDetailLines = 8
|
|
||||||
|
|
||||||
func printDetails(details []string) {
|
|
||||||
for _, d := range details {
|
|
||||||
lines := strings.Split(strings.TrimRight(d, "\n"), "\n")
|
|
||||||
truncated := 0
|
|
||||||
if len(lines) > maxDetailLines {
|
|
||||||
truncated = len(lines) - maxDetailLines
|
|
||||||
lines = lines[:maxDetailLines]
|
|
||||||
}
|
|
||||||
for _, line := range lines {
|
|
||||||
fmt.Printf(" %s\n", line)
|
|
||||||
}
|
|
||||||
if truncated > 0 {
|
|
||||||
fmt.Printf(" ... (%d more line(s))\n", truncated)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
fmt.Println()
|
|
||||||
}
|
|
||||||
|
|
||||||
func enabledLabel(enabled bool) string {
|
|
||||||
if enabled {
|
|
||||||
return "enabled"
|
|
||||||
}
|
|
||||||
return "disabled"
|
|
||||||
}
|
|
||||||
@ -1,220 +0,0 @@
|
|||||||
package check
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"m3u8-downloader/pkg/config"
|
|
||||||
"m3u8-downloader/pkg/httpClient"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
const masterPlaylistBody = `#EXTM3U
|
|
||||||
#EXT-X-STREAM-INF:BANDWIDTH=5500000,RESOLUTION=1920x1080
|
|
||||||
1080p/chunk.m3u8
|
|
||||||
#EXT-X-STREAM-INF:BANDWIDTH=3200000,RESOLUTION=1280x720
|
|
||||||
720p/chunk.m3u8
|
|
||||||
`
|
|
||||||
|
|
||||||
const mediaPlaylistBody = `#EXTM3U
|
|
||||||
#EXT-X-TARGETDURATION:4
|
|
||||||
#EXT-X-MEDIA-SEQUENCE:10
|
|
||||||
#EXTINF:4.0,
|
|
||||||
seg1.ts
|
|
||||||
#EXTINF:4.0,
|
|
||||||
seg2.ts
|
|
||||||
`
|
|
||||||
|
|
||||||
// playlistServer serves body at /playlist.m3u8 with the given status, and
|
|
||||||
// records the request headers so we can assert the check sends the same ones
|
|
||||||
// the real downloader does.
|
|
||||||
func playlistServer(t *testing.T, status int, body string) (*httptest.Server, *http.Header) {
|
|
||||||
t.Helper()
|
|
||||||
var got http.Header
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
got = r.Header.Clone()
|
|
||||||
w.WriteHeader(status)
|
|
||||||
fmt.Fprint(w, body)
|
|
||||||
}))
|
|
||||||
t.Cleanup(srv.Close)
|
|
||||||
return srv, &got
|
|
||||||
}
|
|
||||||
|
|
||||||
func joinDetails(details []string) string {
|
|
||||||
return strings.Join(details, "\n")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckMasterPlaylistMaster(t *testing.T) {
|
|
||||||
srv, headers := playlistServer(t, http.StatusOK, masterPlaylistBody)
|
|
||||||
|
|
||||||
details, err := checkMasterPlaylist(srv.URL + "/playlist.m3u8")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected master playlist check to pass, got: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
joined := joinDetails(details)
|
|
||||||
for _, want := range []string{"master playlist with 2 variant(s)", "1080p (5500000 bps)", "720p (3200000 bps)", "HTTP 200"} {
|
|
||||||
if !strings.Contains(joined, want) {
|
|
||||||
t.Errorf("details missing %q:\n%s", want, joined)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if ua := headers.Get("User-Agent"); !strings.Contains(ua, "Mozilla/5.0") {
|
|
||||||
t.Errorf("expected downloader User-Agent to be sent, got %q", ua)
|
|
||||||
}
|
|
||||||
if ref := headers.Get("Referer"); ref == "" {
|
|
||||||
t.Error("expected Referer header to be sent")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckMasterPlaylistMedia(t *testing.T) {
|
|
||||||
srv, _ := playlistServer(t, http.StatusOK, mediaPlaylistBody)
|
|
||||||
|
|
||||||
details, err := checkMasterPlaylist(srv.URL + "/playlist.m3u8")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected media playlist check to pass, got: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if joined := joinDetails(details); !strings.Contains(joined, "media playlist with 2 segment(s)") {
|
|
||||||
t.Errorf("expected media playlist segment count in details:\n%s", joined)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckMasterPlaylistNon200(t *testing.T) {
|
|
||||||
srv, _ := playlistServer(t, http.StatusForbidden, "denied")
|
|
||||||
|
|
||||||
details, err := checkMasterPlaylist(srv.URL + "/playlist.m3u8")
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected a 403 response to fail the check")
|
|
||||||
}
|
|
||||||
if code := httpClient.GetHTTPStatusCode(err); code != http.StatusForbidden {
|
|
||||||
t.Errorf("expected a structured HTTP 403 error, got code %d from %v", code, err)
|
|
||||||
}
|
|
||||||
// Details collected before the failure should still be reported.
|
|
||||||
if joined := joinDetails(details); !strings.Contains(joined, "HTTP 403") {
|
|
||||||
t.Errorf("expected partial details on failure:\n%s", joined)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckMasterPlaylistNotAPlaylist(t *testing.T) {
|
|
||||||
srv, _ := playlistServer(t, http.StatusOK, "this is not a playlist")
|
|
||||||
|
|
||||||
if _, err := checkMasterPlaylist(srv.URL + "/playlist.m3u8"); err == nil {
|
|
||||||
t.Fatal("expected a non-m3u8 body to fail the check")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckMasterPlaylistEmptyMaster(t *testing.T) {
|
|
||||||
srv, _ := playlistServer(t, http.StatusOK, "#EXTM3U\n")
|
|
||||||
|
|
||||||
if _, err := checkMasterPlaylist(srv.URL + "/playlist.m3u8"); err == nil {
|
|
||||||
t.Fatal("expected a playlist with no variants or segments to fail the check")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckMasterPlaylistInvalidURL(t *testing.T) {
|
|
||||||
for _, bad := range []string{"", "notaurl", "/relative/path.m3u8", "://missing-scheme"} {
|
|
||||||
if _, err := checkMasterPlaylist(bad); err == nil {
|
|
||||||
t.Errorf("expected %q to be rejected as an invalid URL", bad)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckMasterPlaylistUnreachable(t *testing.T) {
|
|
||||||
// Port 9 (discard) refuses connections without a listener.
|
|
||||||
if _, err := checkMasterPlaylist("http://127.0.0.1:9/playlist.m3u8"); err == nil {
|
|
||||||
t.Fatal("expected an unreachable host to fail the check")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeFile(path string) error {
|
|
||||||
return os.WriteFile(path, []byte("blocker"), 0644)
|
|
||||||
}
|
|
||||||
|
|
||||||
func nasConfig(path string) *config.Config {
|
|
||||||
cfg := &config.Config{}
|
|
||||||
cfg.NAS.EnableTransfer = true
|
|
||||||
cfg.NAS.OutputPath = path
|
|
||||||
cfg.NAS.Timeout = 10 * time.Second
|
|
||||||
cfg.NAS.RetryLimit = 3
|
|
||||||
return cfg
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckNASLocalPath(t *testing.T) {
|
|
||||||
target := filepath.Join(t.TempDir(), "streams")
|
|
||||||
|
|
||||||
details, err := checkNAS(nasConfig(target))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected a writable local path to pass the NAS check, got: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
joined := joinDetails(details)
|
|
||||||
for _, want := range []string{"configured: " + target, "strategy: local", "write probe succeeded"} {
|
|
||||||
if !strings.Contains(joined, want) {
|
|
||||||
t.Errorf("details missing %q:\n%s", want, joined)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// The probe must not leave its test file behind.
|
|
||||||
if matches, _ := filepath.Glob(filepath.Join(target, ".connection_test")); len(matches) != 0 {
|
|
||||||
t.Errorf("connection test file was not cleaned up: %v", matches)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckNASUnusableTarget(t *testing.T) {
|
|
||||||
// A regular file cannot be turned into the destination directory.
|
|
||||||
blocker := filepath.Join(t.TempDir(), "not-a-dir")
|
|
||||||
if err := writeFile(blocker); err != nil {
|
|
||||||
t.Fatalf("failed to set up test file: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
details, err := checkNAS(nasConfig(filepath.Join(blocker, "streams")))
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected an unusable NAS target to fail the check")
|
|
||||||
}
|
|
||||||
if joined := joinDetails(details); !strings.Contains(joined, "configured: ") {
|
|
||||||
t.Errorf("expected path detail on failure:\n%s", joined)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProbeAsyncTimesOut(t *testing.T) {
|
|
||||||
start := time.Now()
|
|
||||||
result := <-probeAsync(50*time.Millisecond, func() ([]string, error) {
|
|
||||||
time.Sleep(5 * time.Second)
|
|
||||||
return nil, nil
|
|
||||||
})
|
|
||||||
|
|
||||||
if result.err == nil {
|
|
||||||
t.Fatal("expected a hung probe to report a timeout")
|
|
||||||
}
|
|
||||||
if !strings.Contains(result.err.Error(), "timed out") {
|
|
||||||
t.Errorf("expected a timeout error, got: %v", result.err)
|
|
||||||
}
|
|
||||||
if elapsed := time.Since(start); elapsed > time.Second {
|
|
||||||
t.Errorf("probe took %s, expected it to abandon the hung call promptly", elapsed)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProbeAsyncPassesThroughResults(t *testing.T) {
|
|
||||||
sentinel := errors.New("probe failed")
|
|
||||||
|
|
||||||
if result := <-probeAsync(time.Second, func() ([]string, error) { return []string{"ok"}, nil }); result.err != nil || len(result.details) != 1 {
|
|
||||||
t.Errorf("expected success to pass through, got details=%v err=%v", result.details, result.err)
|
|
||||||
}
|
|
||||||
if result := <-probeAsync(time.Second, func() ([]string, error) { return nil, sentinel }); !errors.Is(result.err, sentinel) {
|
|
||||||
t.Errorf("expected probe error to pass through, got: %v", result.err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFormatDuration(t *testing.T) {
|
|
||||||
if got := formatDuration(100 * time.Microsecond); got != "<1ms" {
|
|
||||||
t.Errorf("sub-millisecond duration = %q, want \"<1ms\"", got)
|
|
||||||
}
|
|
||||||
if got := formatDuration(1500 * time.Millisecond); got != "1.5s" {
|
|
||||||
t.Errorf("1.5s duration = %q", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -4,8 +4,6 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"log"
|
"log"
|
||||||
"m3u8-downloader/pkg/constants"
|
"m3u8-downloader/pkg/constants"
|
||||||
"m3u8-downloader/pkg/health"
|
|
||||||
"m3u8-downloader/pkg/logging"
|
|
||||||
"m3u8-downloader/pkg/media"
|
"m3u8-downloader/pkg/media"
|
||||||
"m3u8-downloader/pkg/transfer"
|
"m3u8-downloader/pkg/transfer"
|
||||||
"m3u8-downloader/pkg/utils"
|
"m3u8-downloader/pkg/utils"
|
||||||
@ -17,8 +15,6 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func Download(masterURL string, eventName string, debug bool) {
|
func Download(masterURL string, eventName string, debug bool) {
|
||||||
appLog := logging.MustGetFactory().App
|
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
@ -28,26 +24,11 @@ func Download(masterURL string, eventName string, debug bool) {
|
|||||||
go func() {
|
go func() {
|
||||||
<-sigChan
|
<-sigChan
|
||||||
log.Println("Shutting down...")
|
log.Println("Shutting down...")
|
||||||
appLog.Info("shutdown signal received")
|
|
||||||
cancel()
|
cancel()
|
||||||
}()
|
}()
|
||||||
|
|
||||||
cfg := constants.MustGetConfig()
|
cfg := constants.MustGetConfig()
|
||||||
|
|
||||||
if cfg.Health.Enabled {
|
|
||||||
healthSrv, err := health.Start(ctx, &cfg.Health, health.MustGetRegistry())
|
|
||||||
if err != nil {
|
|
||||||
appLog.Error("failed to start health server", logging.KeyError, err.Error())
|
|
||||||
} else {
|
|
||||||
appLog.Info("health server listening", "addr", cfg.Health.Addr)
|
|
||||||
defer func() {
|
|
||||||
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
||||||
defer shutdownCancel()
|
|
||||||
_ = healthSrv.Shutdown(shutdownCtx)
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
var transferService *transfer.TransferService
|
var transferService *transfer.TransferService
|
||||||
if cfg.NAS.EnableTransfer {
|
if cfg.NAS.EnableTransfer {
|
||||||
@ -55,7 +36,6 @@ func Download(masterURL string, eventName string, debug bool) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("Failed to create transfer service: %v", err)
|
log.Printf("Failed to create transfer service: %v", err)
|
||||||
log.Println("Continuing without transfer service...")
|
log.Println("Continuing without transfer service...")
|
||||||
appLog.Error("failed to create transfer service, continuing without it", logging.KeyError, err.Error())
|
|
||||||
} else {
|
} else {
|
||||||
transferService = ts
|
transferService = ts
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
@ -63,33 +43,29 @@ func Download(masterURL string, eventName string, debug bool) {
|
|||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
if err := transferService.Start(ctx); err != nil && err != context.Canceled {
|
if err := transferService.Start(ctx); err != nil && err != context.Canceled {
|
||||||
log.Printf("Transfer service error: %v", err)
|
log.Printf("Transfer service error: %v", err)
|
||||||
appLog.Error("transfer service error", logging.KeyError, err.Error())
|
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
log.Println("Transfer service started.")
|
log.Println("Transfer service started.")
|
||||||
appLog.Info("transfer service started")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// One writer for the whole run: every variant records into it, and it is
|
manifestWriter := media.NewManifestWriter(eventName)
|
||||||
// the one written to disk at the end.
|
|
||||||
manifest := media.NewManifestWriter(eventName)
|
|
||||||
|
|
||||||
eventPath := cfg.GetEventPath(eventName)
|
eventPath := cfg.GetEventPath(eventName)
|
||||||
if err := utils.EnsureDir(eventPath); err != nil {
|
if err := utils.EnsureDir(eventPath); err != nil {
|
||||||
appLog.Error("failed to create event directory", logging.KeyError, err.Error())
|
|
||||||
log.Fatalf("Failed to create event directory: %v", err)
|
log.Fatalf("Failed to create event directory: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
variants, err := media.GetAllVariants(masterURL, eventPath)
|
variants, err := media.GetAllVariants(masterURL, eventPath, manifestWriter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
appLog.Error("failed to get variants", logging.KeyError, err.Error())
|
|
||||||
log.Fatalf("Failed to get variants: %v", err)
|
log.Fatalf("Failed to get variants: %v", err)
|
||||||
}
|
}
|
||||||
log.Printf("Found %d variants", len(variants))
|
log.Printf("Found %d variants", len(variants))
|
||||||
|
|
||||||
sem := make(chan struct{}, constants.WorkerCount*len(variants))
|
sem := make(chan struct{}, constants.WorkerCount*len(variants))
|
||||||
|
|
||||||
|
manifest := media.NewManifestWriter(eventName)
|
||||||
|
|
||||||
for _, variant := range variants {
|
for _, variant := range variants {
|
||||||
// Debug mode only tracks one variant for easier debugging
|
// Debug mode only tracks one variant for easier debugging
|
||||||
if debug {
|
if debug {
|
||||||
@ -106,7 +82,6 @@ func Download(masterURL string, eventName string, debug bool) {
|
|||||||
|
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
log.Println("All variant downloaders finished.")
|
log.Println("All variant downloaders finished.")
|
||||||
appLog.Info("all variant downloaders finished")
|
|
||||||
|
|
||||||
if transferService != nil {
|
if transferService != nil {
|
||||||
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second)
|
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||||
@ -115,9 +90,7 @@ func Download(masterURL string, eventName string, debug bool) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
log.Println("All Services shut down.")
|
log.Println("All Services shut down.")
|
||||||
appLog.Info("all services shut down")
|
|
||||||
|
|
||||||
manifest.WriteManifest()
|
manifestWriter.WriteManifest()
|
||||||
log.Println("Manifest written.")
|
log.Println("Manifest written.")
|
||||||
appLog.Info("manifest written", logging.KeyEvent, eventName, "segments", manifest.Len())
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -4,12 +4,9 @@ import (
|
|||||||
"bufio"
|
"bufio"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"m3u8-downloader/cmd/check"
|
|
||||||
"m3u8-downloader/cmd/downloader"
|
"m3u8-downloader/cmd/downloader"
|
||||||
"m3u8-downloader/cmd/processor"
|
"m3u8-downloader/cmd/processor"
|
||||||
"m3u8-downloader/cmd/transfer"
|
"m3u8-downloader/cmd/transfer"
|
||||||
"m3u8-downloader/pkg/constants"
|
|
||||||
"m3u8-downloader/pkg/logging"
|
|
||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
@ -20,22 +17,9 @@ func main() {
|
|||||||
debug := flag.Bool("debug", false, "Enable debug mode")
|
debug := flag.Bool("debug", false, "Enable debug mode")
|
||||||
transferOnly := flag.Bool("transfer", false, "Transfer-only mode: transfer existing files without downloading")
|
transferOnly := flag.Bool("transfer", false, "Transfer-only mode: transfer existing files without downloading")
|
||||||
processOnly := flag.Bool("process", false, "Process-only mode: process existing files without downloading")
|
processOnly := flag.Bool("process", false, "Process-only mode: process existing files without downloading")
|
||||||
checkOnly := flag.Bool("check", false, "Connection-check mode: test the stream URL and NAS connectivity, then exit")
|
|
||||||
|
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
// Must happen before the first logging.MustGetFactory() call anywhere in
|
|
||||||
// the program, since the logging factory reads its level from this
|
|
||||||
// config singleton once and caches it.
|
|
||||||
if *debug {
|
|
||||||
constants.MustGetConfig().Logging.Level = "debug"
|
|
||||||
}
|
|
||||||
logging.MustGetFactory().App.Info("m3u8-downloader starting", "debug", *debug)
|
|
||||||
|
|
||||||
if *checkOnly {
|
|
||||||
os.Exit(check.Run(*url))
|
|
||||||
}
|
|
||||||
|
|
||||||
if *transferOnly {
|
if *transferOnly {
|
||||||
transfer.RunTransferOnly(*eventName)
|
transfer.RunTransferOnly(*eventName)
|
||||||
return
|
return
|
||||||
|
|||||||
@ -4,39 +4,17 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"log"
|
"log"
|
||||||
"m3u8-downloader/pkg/constants"
|
"m3u8-downloader/pkg/constants"
|
||||||
"m3u8-downloader/pkg/health"
|
|
||||||
"m3u8-downloader/pkg/logging"
|
|
||||||
"m3u8-downloader/pkg/processing"
|
"m3u8-downloader/pkg/processing"
|
||||||
"time"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func Process(eventName string) {
|
func Process(eventName string) {
|
||||||
appLog := logging.MustGetFactory().App
|
|
||||||
log.Printf("Starting processing for event: %s", eventName)
|
log.Printf("Starting processing for event: %s", eventName)
|
||||||
appLog.Info("starting processing", logging.KeyEvent, eventName)
|
|
||||||
cfg := constants.MustGetConfig()
|
cfg := constants.MustGetConfig()
|
||||||
|
|
||||||
if cfg.Health.Enabled {
|
|
||||||
healthSrv, err := health.Start(context.Background(), &cfg.Health, health.MustGetRegistry())
|
|
||||||
if err != nil {
|
|
||||||
appLog.Error("failed to start health server", logging.KeyError, err.Error())
|
|
||||||
} else {
|
|
||||||
appLog.Info("health server listening", "addr", cfg.Health.Addr)
|
|
||||||
defer func() {
|
|
||||||
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
||||||
defer shutdownCancel()
|
|
||||||
_ = healthSrv.Shutdown(shutdownCtx)
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
ps, err := processing.NewProcessingService(eventName, cfg)
|
ps, err := processing.NewProcessingService(eventName, cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
appLog.Error("failed to create processing service", logging.KeyError, err.Error())
|
|
||||||
log.Fatalf("Failed to create processing service: %v", err)
|
log.Fatalf("Failed to create processing service: %v", err)
|
||||||
}
|
}
|
||||||
if err := ps.Start(context.Background()); err != nil {
|
if err := ps.Start(context.Background()); err != nil {
|
||||||
appLog.Error("processing service failed", logging.KeyError, err.Error())
|
|
||||||
log.Fatalf("Failed to run processing service: %v", err)
|
log.Fatalf("Failed to run processing service: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -7,8 +7,6 @@ import (
|
|||||||
"log"
|
"log"
|
||||||
"m3u8-downloader/pkg/config"
|
"m3u8-downloader/pkg/config"
|
||||||
"m3u8-downloader/pkg/constants"
|
"m3u8-downloader/pkg/constants"
|
||||||
"m3u8-downloader/pkg/health"
|
|
||||||
"m3u8-downloader/pkg/logging"
|
|
||||||
"m3u8-downloader/pkg/transfer"
|
"m3u8-downloader/pkg/transfer"
|
||||||
"m3u8-downloader/pkg/utils"
|
"m3u8-downloader/pkg/utils"
|
||||||
"os"
|
"os"
|
||||||
@ -34,12 +32,10 @@ func getEventDirs(cfg *config.Config) ([]string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func RunTransferOnly(eventName string) {
|
func RunTransferOnly(eventName string) {
|
||||||
appLog := logging.MustGetFactory().App
|
|
||||||
cfg := constants.MustGetConfig()
|
cfg := constants.MustGetConfig()
|
||||||
|
|
||||||
// Check if NAS transfer is enabled
|
// Check if NAS transfer is enabled
|
||||||
if !cfg.NAS.EnableTransfer {
|
if !cfg.NAS.EnableTransfer {
|
||||||
appLog.Error("NAS transfer disabled, cannot run transfer-only mode")
|
|
||||||
log.Fatal("NAS transfer is disabled in configuration. Please enable it to use transfer-only mode.")
|
log.Fatal("NAS transfer is disabled in configuration. Please enable it to use transfer-only mode.")
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -83,50 +79,30 @@ func RunTransferOnly(eventName string) {
|
|||||||
go func() {
|
go func() {
|
||||||
<-sigChan
|
<-sigChan
|
||||||
log.Println("Shutting down transfer service...")
|
log.Println("Shutting down transfer service...")
|
||||||
appLog.Info("shutdown signal received")
|
|
||||||
cancel()
|
cancel()
|
||||||
}()
|
}()
|
||||||
|
|
||||||
if cfg.Health.Enabled {
|
|
||||||
healthSrv, err := health.Start(ctx, &cfg.Health, health.MustGetRegistry())
|
|
||||||
if err != nil {
|
|
||||||
appLog.Error("failed to start health server", logging.KeyError, err.Error())
|
|
||||||
} else {
|
|
||||||
appLog.Info("health server listening", "addr", cfg.Health.Addr)
|
|
||||||
defer func() {
|
|
||||||
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
||||||
defer shutdownCancel()
|
|
||||||
_ = healthSrv.Shutdown(shutdownCtx)
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Verify local event directory exists
|
// Verify local event directory exists
|
||||||
localEventPath := cfg.GetEventPath(eventName)
|
localEventPath := cfg.GetEventPath(eventName)
|
||||||
if !utils.PathExists(localEventPath) {
|
if !utils.PathExists(localEventPath) {
|
||||||
appLog.Error("local event directory does not exist", logging.KeyEvent, eventName)
|
|
||||||
log.Fatalf("Local event directory does not exist: %s", localEventPath)
|
log.Fatalf("Local event directory does not exist: %s", localEventPath)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create transfer service
|
// Create transfer service
|
||||||
transferService, err := transfer.NewTrasferService(cfg.NAS.OutputPath, eventName)
|
transferService, err := transfer.NewTrasferService(cfg.NAS.OutputPath, eventName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
appLog.Error("failed to create transfer service", logging.KeyError, err.Error())
|
|
||||||
log.Fatalf("Failed to create transfer service: %v", err)
|
log.Fatalf("Failed to create transfer service: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Find and queue existing files
|
// Find and queue existing files
|
||||||
if err := transferService.QueueExistingFiles(localEventPath); err != nil {
|
if err := transferService.QueueExistingFiles(localEventPath); err != nil {
|
||||||
appLog.Error("failed to queue existing files", logging.KeyError, err.Error())
|
|
||||||
log.Fatalf("Failed to queue existing files: %v", err)
|
log.Fatalf("Failed to queue existing files: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start transfer service
|
// Start transfer service
|
||||||
log.Println("Starting transfer service...")
|
log.Println("Starting transfer service...")
|
||||||
appLog.Info("starting transfer service", logging.KeyEvent, eventName)
|
|
||||||
if err := transferService.Start(ctx); err != nil && err != context.Canceled {
|
if err := transferService.Start(ctx); err != nil && err != context.Canceled {
|
||||||
log.Printf("Transfer service error: %v", err)
|
log.Printf("Transfer service error: %v", err)
|
||||||
appLog.Error("transfer service error", logging.KeyError, err.Error())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Graceful shutdown
|
// Graceful shutdown
|
||||||
@ -135,5 +111,4 @@ func RunTransferOnly(eventName string) {
|
|||||||
transferService.Shutdown(shutdownCtx)
|
transferService.Shutdown(shutdownCtx)
|
||||||
|
|
||||||
log.Println("Transfer-only mode completed.")
|
log.Println("Transfer-only mode completed.")
|
||||||
appLog.Info("transfer-only mode completed", logging.KeyEvent, eventName)
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -16,8 +16,6 @@ type Config struct {
|
|||||||
Transfer TransferConfig
|
Transfer TransferConfig
|
||||||
Cleanup CleanupConfig
|
Cleanup CleanupConfig
|
||||||
Paths PathsConfig
|
Paths PathsConfig
|
||||||
Logging LoggingConfig
|
|
||||||
Health HealthConfig
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type CoreConfig struct {
|
type CoreConfig struct {
|
||||||
@ -32,19 +30,11 @@ type HTTPConfig struct {
|
|||||||
|
|
||||||
type NASConfig struct {
|
type NASConfig struct {
|
||||||
EnableTransfer bool
|
EnableTransfer bool
|
||||||
// OutputPath is the destination as configured: a UNC path, an smb:// URL,
|
|
||||||
// or a directory on this host. How it is reached depends on the platform;
|
|
||||||
// see pkg/nas.
|
|
||||||
OutputPath string
|
OutputPath string
|
||||||
Username string
|
Username string
|
||||||
Password string
|
Password string
|
||||||
Timeout time.Duration
|
Timeout time.Duration
|
||||||
RetryLimit int
|
RetryLimit int
|
||||||
// AutoMount lets the recorder mount an SMB share itself when the host has
|
|
||||||
// not already mounted it. Linux only, and requires root.
|
|
||||||
AutoMount bool
|
|
||||||
// MountPoint overrides where AutoMount attaches the share.
|
|
||||||
MountPoint string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type ProcessingConfig struct {
|
type ProcessingConfig struct {
|
||||||
@ -77,16 +67,6 @@ type PathsConfig struct {
|
|||||||
PersistenceFile string
|
PersistenceFile string
|
||||||
}
|
}
|
||||||
|
|
||||||
type LoggingConfig struct {
|
|
||||||
Dir string
|
|
||||||
Level string
|
|
||||||
}
|
|
||||||
|
|
||||||
type HealthConfig struct {
|
|
||||||
Enabled bool
|
|
||||||
Addr string
|
|
||||||
}
|
|
||||||
|
|
||||||
var defaultConfig = Config{
|
var defaultConfig = Config{
|
||||||
Core: CoreConfig{
|
Core: CoreConfig{
|
||||||
WorkerCount: 4,
|
WorkerCount: 4,
|
||||||
@ -103,8 +83,6 @@ var defaultConfig = Config{
|
|||||||
Password: "s3tkY6tzA&KN6M",
|
Password: "s3tkY6tzA&KN6M",
|
||||||
Timeout: 30 * time.Second,
|
Timeout: 30 * time.Second,
|
||||||
RetryLimit: 3,
|
RetryLimit: 3,
|
||||||
AutoMount: false,
|
|
||||||
MountPoint: "",
|
|
||||||
},
|
},
|
||||||
Processing: ProcessingConfig{
|
Processing: ProcessingConfig{
|
||||||
Enabled: true,
|
Enabled: true,
|
||||||
@ -132,14 +110,6 @@ var defaultConfig = Config{
|
|||||||
ManifestDir: "data",
|
ManifestDir: "data",
|
||||||
PersistenceFile: "transfer_queue.json",
|
PersistenceFile: "transfer_queue.json",
|
||||||
},
|
},
|
||||||
Logging: LoggingConfig{
|
|
||||||
Dir: "data/logs",
|
|
||||||
Level: "info",
|
|
||||||
},
|
|
||||||
Health: HealthConfig{
|
|
||||||
Enabled: true,
|
|
||||||
Addr: ":8090",
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func Load() (*Config, error) {
|
func Load() (*Config, error) {
|
||||||
@ -185,25 +155,8 @@ func (c *Config) loadFromEnvironment() error {
|
|||||||
c.NAS.EnableTransfer = val == "true"
|
c.NAS.EnableTransfer = val == "true"
|
||||||
}
|
}
|
||||||
|
|
||||||
if val := os.Getenv("NAS_AUTO_MOUNT"); val != "" {
|
|
||||||
c.NAS.AutoMount = val == "true"
|
|
||||||
}
|
|
||||||
|
|
||||||
if val := os.Getenv("NAS_MOUNT_POINT"); val != "" {
|
|
||||||
c.NAS.MountPoint = val
|
|
||||||
}
|
|
||||||
|
|
||||||
if val := os.Getenv("LOCAL_OUTPUT_DIR"); val != "" {
|
if val := os.Getenv("LOCAL_OUTPUT_DIR"); val != "" {
|
||||||
c.Paths.LocalOutput = val
|
c.Paths.LocalOutput = val
|
||||||
// The manifest describes the event directory and is documented as
|
|
||||||
// sitting beside it, so it follows the output directory. Without this
|
|
||||||
// it stays in the default data/ while the segments move elsewhere.
|
|
||||||
c.Paths.ManifestDir = val
|
|
||||||
}
|
|
||||||
|
|
||||||
// Applied after LOCAL_OUTPUT_DIR so an explicit setting wins.
|
|
||||||
if val := os.Getenv("MANIFEST_DIR"); val != "" {
|
|
||||||
c.Paths.ManifestDir = val
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if val := os.Getenv("PROCESS_OUTPUT_DIR"); val != "" {
|
if val := os.Getenv("PROCESS_OUTPUT_DIR"); val != "" {
|
||||||
@ -214,22 +167,6 @@ func (c *Config) loadFromEnvironment() error {
|
|||||||
c.Processing.FFmpegPath = val
|
c.Processing.FFmpegPath = val
|
||||||
}
|
}
|
||||||
|
|
||||||
if val := os.Getenv("LOG_DIR"); val != "" {
|
|
||||||
c.Logging.Dir = val
|
|
||||||
}
|
|
||||||
|
|
||||||
if val := os.Getenv("LOG_LEVEL"); val != "" {
|
|
||||||
c.Logging.Level = val
|
|
||||||
}
|
|
||||||
|
|
||||||
if val := os.Getenv("HEALTH_ENABLED"); val != "" {
|
|
||||||
c.Health.Enabled = val == "true"
|
|
||||||
}
|
|
||||||
|
|
||||||
if val := os.Getenv("HEALTH_ADDR"); val != "" {
|
|
||||||
c.Health.Addr = val
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -255,16 +192,12 @@ func (c *Config) resolveAndValidatePaths() error {
|
|||||||
if !filepath.IsAbs(c.Paths.PersistenceFile) {
|
if !filepath.IsAbs(c.Paths.PersistenceFile) {
|
||||||
c.Paths.PersistenceFile = filepath.Join(c.Paths.BaseDir, c.Paths.PersistenceFile)
|
c.Paths.PersistenceFile = filepath.Join(c.Paths.BaseDir, c.Paths.PersistenceFile)
|
||||||
}
|
}
|
||||||
if !filepath.IsAbs(c.Logging.Dir) {
|
|
||||||
c.Logging.Dir = filepath.Join(cwd, c.Logging.Dir)
|
|
||||||
}
|
|
||||||
|
|
||||||
requiredDirs := []string{
|
requiredDirs := []string{
|
||||||
c.Paths.BaseDir,
|
c.Paths.BaseDir,
|
||||||
c.Paths.LocalOutput,
|
c.Paths.LocalOutput,
|
||||||
c.Paths.ProcessOutput,
|
c.Paths.ProcessOutput,
|
||||||
c.Paths.ManifestDir,
|
c.Paths.ManifestDir,
|
||||||
c.Logging.Dir,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, dir := range requiredDirs {
|
for _, dir := range requiredDirs {
|
||||||
|
|||||||
@ -1,323 +0,0 @@
|
|||||||
<!DOCTYPE html>
|
|
||||||
<html lang="en">
|
|
||||||
<head>
|
|
||||||
<meta charset="UTF-8">
|
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
||||||
<title>m3u8-downloader status</title>
|
|
||||||
<style>
|
|
||||||
:root {
|
|
||||||
--bg: #0f1115;
|
|
||||||
--panel: #161922;
|
|
||||||
--border: #262b38;
|
|
||||||
--text: #e6e8ee;
|
|
||||||
--muted: #8b909e;
|
|
||||||
--green: #3ecf8e;
|
|
||||||
--red: #f2545b;
|
|
||||||
--amber: #f2b705;
|
|
||||||
--blue: #4f8ef7;
|
|
||||||
}
|
|
||||||
* { box-sizing: border-box; }
|
|
||||||
body {
|
|
||||||
margin: 0;
|
|
||||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
|
|
||||||
background: var(--bg);
|
|
||||||
color: var(--text);
|
|
||||||
}
|
|
||||||
header {
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
justify-content: space-between;
|
|
||||||
flex-wrap: wrap;
|
|
||||||
gap: 12px;
|
|
||||||
padding: 16px 24px;
|
|
||||||
border-bottom: 1px solid var(--border);
|
|
||||||
}
|
|
||||||
header h1 {
|
|
||||||
font-size: 18px;
|
|
||||||
margin: 0;
|
|
||||||
font-weight: 600;
|
|
||||||
}
|
|
||||||
.header-meta {
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
gap: 16px;
|
|
||||||
color: var(--muted);
|
|
||||||
font-size: 13px;
|
|
||||||
}
|
|
||||||
.badge {
|
|
||||||
display: inline-flex;
|
|
||||||
align-items: center;
|
|
||||||
gap: 6px;
|
|
||||||
padding: 4px 12px;
|
|
||||||
border-radius: 999px;
|
|
||||||
font-size: 13px;
|
|
||||||
font-weight: 600;
|
|
||||||
text-transform: uppercase;
|
|
||||||
letter-spacing: 0.03em;
|
|
||||||
}
|
|
||||||
.badge-dot { width: 8px; height: 8px; border-radius: 50%; }
|
|
||||||
.badge.healthy { background: rgba(62,207,142,0.15); color: var(--green); }
|
|
||||||
.badge.healthy .badge-dot { background: var(--green); }
|
|
||||||
.badge.degraded { background: rgba(242,84,91,0.15); color: var(--red); }
|
|
||||||
.badge.degraded .badge-dot { background: var(--red); }
|
|
||||||
.badge.unknown { background: rgba(139,144,158,0.15); color: var(--muted); }
|
|
||||||
.badge.unknown .badge-dot { background: var(--muted); }
|
|
||||||
|
|
||||||
#conn-banner {
|
|
||||||
display: none;
|
|
||||||
background: var(--red);
|
|
||||||
color: #fff;
|
|
||||||
text-align: center;
|
|
||||||
padding: 8px;
|
|
||||||
font-size: 13px;
|
|
||||||
font-weight: 600;
|
|
||||||
}
|
|
||||||
#conn-banner.show { display: block; }
|
|
||||||
|
|
||||||
main {
|
|
||||||
padding: 24px;
|
|
||||||
display: grid;
|
|
||||||
grid-template-columns: repeat(auto-fit, minmax(340px, 1fr));
|
|
||||||
gap: 20px;
|
|
||||||
max-width: 1400px;
|
|
||||||
margin: 0 auto;
|
|
||||||
}
|
|
||||||
.card {
|
|
||||||
background: var(--panel);
|
|
||||||
border: 1px solid var(--border);
|
|
||||||
border-radius: 10px;
|
|
||||||
padding: 18px 20px;
|
|
||||||
}
|
|
||||||
.card h2 {
|
|
||||||
font-size: 14px;
|
|
||||||
text-transform: uppercase;
|
|
||||||
letter-spacing: 0.05em;
|
|
||||||
color: var(--muted);
|
|
||||||
margin: 0 0 14px 0;
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
justify-content: space-between;
|
|
||||||
}
|
|
||||||
table { width: 100%; border-collapse: collapse; font-size: 13px; }
|
|
||||||
th, td { text-align: left; padding: 6px 8px; border-bottom: 1px solid var(--border); }
|
|
||||||
th { color: var(--muted); font-weight: 500; font-size: 12px; }
|
|
||||||
tr.failed-row { background: rgba(242,84,91,0.08); }
|
|
||||||
tr.inactive-row { opacity: 0.5; }
|
|
||||||
.dot { display: inline-block; width: 8px; height: 8px; border-radius: 50%; margin-right: 6px; }
|
|
||||||
.dot.on { background: var(--green); }
|
|
||||||
.dot.off { background: var(--red); }
|
|
||||||
|
|
||||||
.stat-grid {
|
|
||||||
display: grid;
|
|
||||||
grid-template-columns: repeat(3, 1fr);
|
|
||||||
gap: 10px;
|
|
||||||
margin-bottom: 14px;
|
|
||||||
}
|
|
||||||
.stat {
|
|
||||||
background: rgba(255,255,255,0.03);
|
|
||||||
border-radius: 8px;
|
|
||||||
padding: 10px 12px;
|
|
||||||
}
|
|
||||||
.stat .value { font-size: 20px; font-weight: 600; }
|
|
||||||
.stat .label { font-size: 11px; color: var(--muted); text-transform: uppercase; letter-spacing: 0.03em; }
|
|
||||||
.stat.failed .value { color: var(--red); }
|
|
||||||
|
|
||||||
.state-pill {
|
|
||||||
display: inline-block;
|
|
||||||
padding: 3px 10px;
|
|
||||||
border-radius: 999px;
|
|
||||||
font-size: 12px;
|
|
||||||
font-weight: 600;
|
|
||||||
text-transform: uppercase;
|
|
||||||
}
|
|
||||||
.state-idle { background: rgba(139,144,158,0.15); color: var(--muted); }
|
|
||||||
.state-scanning, .state-running { background: rgba(79,142,247,0.15); color: var(--blue); }
|
|
||||||
.state-completed { background: rgba(62,207,142,0.15); color: var(--green); }
|
|
||||||
.state-failed { background: rgba(242,84,91,0.15); color: var(--red); }
|
|
||||||
|
|
||||||
.kv { display: flex; justify-content: space-between; padding: 4px 0; font-size: 13px; border-bottom: 1px solid var(--border); }
|
|
||||||
.kv:last-child { border-bottom: none; }
|
|
||||||
.kv .k { color: var(--muted); }
|
|
||||||
|
|
||||||
.error-list { margin: 10px 0 0 0; padding: 0; list-style: none; }
|
|
||||||
.error-list li {
|
|
||||||
font-size: 12px;
|
|
||||||
padding: 6px 8px;
|
|
||||||
margin-bottom: 4px;
|
|
||||||
background: rgba(242,84,91,0.08);
|
|
||||||
border-left: 3px solid var(--red);
|
|
||||||
border-radius: 4px;
|
|
||||||
color: var(--text);
|
|
||||||
}
|
|
||||||
.error-list .err-time { color: var(--muted); margin-right: 6px; }
|
|
||||||
.empty-note { color: var(--muted); font-size: 13px; font-style: italic; }
|
|
||||||
</style>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
|
|
||||||
<div id="conn-banner">Lost connection to status endpoint - retrying...</div>
|
|
||||||
|
|
||||||
<header>
|
|
||||||
<h1>m3u8-downloader</h1>
|
|
||||||
<div class="header-meta">
|
|
||||||
<span id="overall-badge" class="badge unknown"><span class="badge-dot"></span>unknown</span>
|
|
||||||
<span id="uptime">uptime: -</span>
|
|
||||||
<span id="last-updated">last updated: -</span>
|
|
||||||
</div>
|
|
||||||
</header>
|
|
||||||
|
|
||||||
<main>
|
|
||||||
<section class="card">
|
|
||||||
<h2>Download</h2>
|
|
||||||
<table>
|
|
||||||
<thead>
|
|
||||||
<tr><th>Resolution</th><th>Active</th><th>OK</th><th>Failed</th><th>Last activity</th></tr>
|
|
||||||
</thead>
|
|
||||||
<tbody id="variant-rows"></tbody>
|
|
||||||
</table>
|
|
||||||
<div id="download-empty" class="empty-note" style="display:none;">No variants reporting yet.</div>
|
|
||||||
<ul id="download-errors" class="error-list"></ul>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section class="card">
|
|
||||||
<h2>Transfer</h2>
|
|
||||||
<div class="kv"><span class="k">NAS connection</span><span><span id="nas-dot" class="dot off"></span><span id="nas-text">unknown</span></span></div>
|
|
||||||
<div class="stat-grid" style="margin-top:12px;">
|
|
||||||
<div class="stat"><div class="value" id="t-added">-</div><div class="label">Added</div></div>
|
|
||||||
<div class="stat"><div class="value" id="t-completed">-</div><div class="label">Completed</div></div>
|
|
||||||
<div class="stat failed"><div class="value" id="t-failed">-</div><div class="label">Failed</div></div>
|
|
||||||
<div class="stat"><div class="value" id="t-pending">-</div><div class="label">Pending</div></div>
|
|
||||||
<div class="stat"><div class="value" id="t-queue">-</div><div class="label">Queue size</div></div>
|
|
||||||
<div class="stat"><div class="value" id="t-cleanup">-</div><div class="label">Cleanup pending</div></div>
|
|
||||||
</div>
|
|
||||||
<div class="kv"><span class="k">Transferred</span><span id="t-bytes">-</span></div>
|
|
||||||
<ul id="transfer-errors" class="error-list"></ul>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<section class="card">
|
|
||||||
<h2>Processing</h2>
|
|
||||||
<div class="kv"><span class="k">State</span><span id="p-state" class="state-pill state-idle">idle</span></div>
|
|
||||||
<div class="kv"><span class="k">Event</span><span id="p-event">-</span></div>
|
|
||||||
<div class="kv"><span class="k">Segments found</span><span id="p-segments">-</span></div>
|
|
||||||
<div class="kv"><span class="k">Started</span><span id="p-started">-</span></div>
|
|
||||||
<div class="kv"><span class="k">Finished</span><span id="p-finished">-</span></div>
|
|
||||||
<ul id="processing-errors" class="error-list"></ul>
|
|
||||||
</section>
|
|
||||||
</main>
|
|
||||||
|
|
||||||
<script>
|
|
||||||
const POLL_INTERVAL_MS = 3000;
|
|
||||||
const connBanner = document.getElementById('conn-banner');
|
|
||||||
|
|
||||||
function fmtBytes(n) {
|
|
||||||
if (!n || n <= 0) return '0 B';
|
|
||||||
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
|
|
||||||
let i = 0;
|
|
||||||
while (n >= 1024 && i < units.length - 1) { n /= 1024; i++; }
|
|
||||||
return n.toFixed(i === 0 ? 0 : 1) + ' ' + units[i];
|
|
||||||
}
|
|
||||||
|
|
||||||
function fmtDuration(sec) {
|
|
||||||
sec = Math.floor(sec || 0);
|
|
||||||
const h = Math.floor(sec / 3600);
|
|
||||||
const m = Math.floor((sec % 3600) / 60);
|
|
||||||
const s = sec % 60;
|
|
||||||
const parts = [];
|
|
||||||
if (h) parts.push(h + 'h');
|
|
||||||
if (m || h) parts.push(m + 'm');
|
|
||||||
parts.push(s + 's');
|
|
||||||
return parts.join(' ');
|
|
||||||
}
|
|
||||||
|
|
||||||
function fmtTime(iso) {
|
|
||||||
if (!iso) return '-';
|
|
||||||
const d = new Date(iso);
|
|
||||||
if (isNaN(d.getTime()) || d.getFullYear() < 1971) return '-';
|
|
||||||
return d.toLocaleTimeString();
|
|
||||||
}
|
|
||||||
|
|
||||||
function renderErrors(listEl, errors) {
|
|
||||||
listEl.innerHTML = '';
|
|
||||||
(errors || []).slice().reverse().forEach(e => {
|
|
||||||
const li = document.createElement('li');
|
|
||||||
const t = document.createElement('span');
|
|
||||||
t.className = 'err-time';
|
|
||||||
t.textContent = fmtTime(e.time);
|
|
||||||
li.appendChild(t);
|
|
||||||
li.appendChild(document.createTextNode(e.message));
|
|
||||||
listEl.appendChild(li);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function render(data) {
|
|
||||||
const badge = document.getElementById('overall-badge');
|
|
||||||
badge.className = 'badge ' + (data.status === 'healthy' ? 'healthy' : 'degraded');
|
|
||||||
badge.innerHTML = '<span class="badge-dot"></span>' + data.status;
|
|
||||||
|
|
||||||
document.getElementById('uptime').textContent = 'uptime: ' + fmtDuration(data.uptime_seconds);
|
|
||||||
document.getElementById('last-updated').textContent = 'last updated: ' + new Date().toLocaleTimeString();
|
|
||||||
|
|
||||||
// Download
|
|
||||||
const variants = (data.download && data.download.variants) || [];
|
|
||||||
const rows = document.getElementById('variant-rows');
|
|
||||||
rows.innerHTML = '';
|
|
||||||
document.getElementById('download-empty').style.display = variants.length ? 'none' : 'block';
|
|
||||||
variants.forEach(v => {
|
|
||||||
const tr = document.createElement('tr');
|
|
||||||
if (v.segments_failed > 0) tr.classList.add('failed-row');
|
|
||||||
if (!v.active) tr.classList.add('inactive-row');
|
|
||||||
tr.innerHTML =
|
|
||||||
'<td>' + v.resolution + '</td>' +
|
|
||||||
'<td><span class="dot ' + (v.active ? 'on' : 'off') + '"></span>' + (v.active ? 'active' : 'stopped') + '</td>' +
|
|
||||||
'<td>' + v.segments_ok + '</td>' +
|
|
||||||
'<td>' + v.segments_failed + '</td>' +
|
|
||||||
'<td>' + fmtTime(v.last_activity) + '</td>';
|
|
||||||
rows.appendChild(tr);
|
|
||||||
});
|
|
||||||
renderErrors(document.getElementById('download-errors'), data.download && data.download.recent_errors);
|
|
||||||
|
|
||||||
// Transfer
|
|
||||||
const t = data.transfer || {};
|
|
||||||
const nasDot = document.getElementById('nas-dot');
|
|
||||||
nasDot.className = 'dot ' + (t.nas_connected ? 'on' : 'off');
|
|
||||||
document.getElementById('nas-text').textContent = t.nas_connected ? 'connected' : 'disconnected';
|
|
||||||
document.getElementById('t-added').textContent = t.added ?? '-';
|
|
||||||
document.getElementById('t-completed').textContent = t.completed ?? '-';
|
|
||||||
document.getElementById('t-failed').textContent = t.failed ?? '-';
|
|
||||||
document.getElementById('t-pending').textContent = t.pending ?? '-';
|
|
||||||
document.getElementById('t-queue').textContent = t.queue_size ?? '-';
|
|
||||||
document.getElementById('t-cleanup').textContent = t.cleanup_pending ?? '-';
|
|
||||||
document.getElementById('t-bytes').textContent = fmtBytes(t.bytes_transferred);
|
|
||||||
renderErrors(document.getElementById('transfer-errors'), t.recent_errors);
|
|
||||||
|
|
||||||
// Processing
|
|
||||||
const p = data.processing || {};
|
|
||||||
const stateEl = document.getElementById('p-state');
|
|
||||||
const state = p.state || 'idle';
|
|
||||||
stateEl.textContent = state;
|
|
||||||
stateEl.className = 'state-pill state-' + state;
|
|
||||||
document.getElementById('p-event').textContent = p.event_name || '-';
|
|
||||||
document.getElementById('p-segments').textContent = p.segments_found ?? '-';
|
|
||||||
document.getElementById('p-started').textContent = fmtTime(p.started_at);
|
|
||||||
document.getElementById('p-finished').textContent = fmtTime(p.finished_at);
|
|
||||||
renderErrors(document.getElementById('processing-errors'), p.recent_errors);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function poll() {
|
|
||||||
try {
|
|
||||||
const resp = await fetch('/status', { cache: 'no-store' });
|
|
||||||
if (!resp.ok) throw new Error('status ' + resp.status);
|
|
||||||
const data = await resp.json();
|
|
||||||
connBanner.classList.remove('show');
|
|
||||||
render(data);
|
|
||||||
} catch (e) {
|
|
||||||
connBanner.classList.add('show');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
poll();
|
|
||||||
setInterval(poll, POLL_INTERVAL_MS);
|
|
||||||
</script>
|
|
||||||
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
@ -1,352 +0,0 @@
|
|||||||
package health
|
|
||||||
|
|
||||||
import (
|
|
||||||
"sort"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
const maxRecentErrors = 10
|
|
||||||
|
|
||||||
type ErrorEntry struct {
|
|
||||||
Time time.Time `json:"time"`
|
|
||||||
Message string `json:"message"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type VariantStatus struct {
|
|
||||||
Resolution string `json:"resolution"`
|
|
||||||
Bandwidth uint32 `json:"bandwidth"`
|
|
||||||
Active bool `json:"active"`
|
|
||||||
SegmentsOK int `json:"segments_ok"`
|
|
||||||
SegmentsFailed int `json:"segments_failed"`
|
|
||||||
LastActivity time.Time `json:"last_activity"`
|
|
||||||
LastError string `json:"last_error,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// StatsProvider is satisfied by *transfer.QueueStats without pkg/health
|
|
||||||
// needing to import pkg/transfer (which would create an import cycle, since
|
|
||||||
// pkg/transfer calls into pkg/health to report status).
|
|
||||||
type StatsProvider interface {
|
|
||||||
GetStats() (added, completed, failed, pending int, bytes int64)
|
|
||||||
}
|
|
||||||
|
|
||||||
type downloadState struct {
|
|
||||||
variants map[string]*VariantStatus
|
|
||||||
recentErrors []ErrorEntry
|
|
||||||
}
|
|
||||||
|
|
||||||
type transferState struct {
|
|
||||||
statsProvider StatsProvider
|
|
||||||
queueSizeFn func() int
|
|
||||||
cleanupPendingFn func() int
|
|
||||||
nasConnectedFn func() bool
|
|
||||||
nasConnected bool
|
|
||||||
recentErrors []ErrorEntry
|
|
||||||
}
|
|
||||||
|
|
||||||
type processingState struct {
|
|
||||||
state string // idle | scanning | running | completed | failed
|
|
||||||
eventName string
|
|
||||||
segmentsFound int
|
|
||||||
startedAt time.Time
|
|
||||||
finishedAt time.Time
|
|
||||||
lastError string
|
|
||||||
recentErrors []ErrorEntry
|
|
||||||
}
|
|
||||||
|
|
||||||
// Registry is the process-wide health/metrics store. All subsystems report
|
|
||||||
// into it via the Record* methods; the health HTTP server reads it via
|
|
||||||
// Snapshot(). Safe for concurrent use.
|
|
||||||
type Registry struct {
|
|
||||||
mu sync.RWMutex
|
|
||||||
startedAt time.Time
|
|
||||||
download downloadState
|
|
||||||
transfer transferState
|
|
||||||
processing processingState
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
globalRegistry *Registry
|
|
||||||
registryOnce sync.Once
|
|
||||||
)
|
|
||||||
|
|
||||||
// MustGetRegistry returns the process-wide registry, mirroring the
|
|
||||||
// constants.MustGetConfig() / logging.MustGetFactory() singleton style.
|
|
||||||
func MustGetRegistry() *Registry {
|
|
||||||
registryOnce.Do(func() {
|
|
||||||
globalRegistry = New()
|
|
||||||
})
|
|
||||||
return globalRegistry
|
|
||||||
}
|
|
||||||
|
|
||||||
func New() *Registry {
|
|
||||||
return &Registry{
|
|
||||||
startedAt: time.Now(),
|
|
||||||
download: downloadState{
|
|
||||||
variants: make(map[string]*VariantStatus),
|
|
||||||
},
|
|
||||||
processing: processingState{
|
|
||||||
state: "idle",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func appendBounded(errs []ErrorEntry, e ErrorEntry) []ErrorEntry {
|
|
||||||
errs = append(errs, e)
|
|
||||||
if len(errs) > maxRecentErrors {
|
|
||||||
errs = errs[len(errs)-maxRecentErrors:]
|
|
||||||
}
|
|
||||||
return errs
|
|
||||||
}
|
|
||||||
|
|
||||||
func errString(err error) string {
|
|
||||||
if err == nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return err.Error()
|
|
||||||
}
|
|
||||||
|
|
||||||
// --- Download ---
|
|
||||||
|
|
||||||
func (r *Registry) RecordVariantStarted(resolution string, bandwidth uint32) {
|
|
||||||
r.mu.Lock()
|
|
||||||
defer r.mu.Unlock()
|
|
||||||
r.download.variants[resolution] = &VariantStatus{
|
|
||||||
Resolution: resolution,
|
|
||||||
Bandwidth: bandwidth,
|
|
||||||
Active: true,
|
|
||||||
LastActivity: time.Now(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Registry) RecordVariantStopped(resolution string) {
|
|
||||||
r.mu.Lock()
|
|
||||||
defer r.mu.Unlock()
|
|
||||||
if v, ok := r.download.variants[resolution]; ok {
|
|
||||||
v.Active = false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Registry) RecordSegmentDownloaded(resolution string) {
|
|
||||||
r.mu.Lock()
|
|
||||||
defer r.mu.Unlock()
|
|
||||||
v := r.ensureVariantLocked(resolution)
|
|
||||||
v.SegmentsOK++
|
|
||||||
v.LastActivity = time.Now()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Registry) RecordSegmentFailed(resolution string, err error) {
|
|
||||||
r.mu.Lock()
|
|
||||||
defer r.mu.Unlock()
|
|
||||||
v := r.ensureVariantLocked(resolution)
|
|
||||||
v.SegmentsFailed++
|
|
||||||
v.LastActivity = time.Now()
|
|
||||||
v.LastError = errString(err)
|
|
||||||
r.download.recentErrors = appendBounded(r.download.recentErrors, ErrorEntry{
|
|
||||||
Time: time.Now(),
|
|
||||||
Message: resolution + ": " + errString(err),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Registry) RecordPlaylistError(resolution string, err error) {
|
|
||||||
r.mu.Lock()
|
|
||||||
defer r.mu.Unlock()
|
|
||||||
v := r.ensureVariantLocked(resolution)
|
|
||||||
v.LastError = errString(err)
|
|
||||||
r.download.recentErrors = appendBounded(r.download.recentErrors, ErrorEntry{
|
|
||||||
Time: time.Now(),
|
|
||||||
Message: resolution + " playlist: " + errString(err),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Registry) ensureVariantLocked(resolution string) *VariantStatus {
|
|
||||||
v, ok := r.download.variants[resolution]
|
|
||||||
if !ok {
|
|
||||||
v = &VariantStatus{Resolution: resolution, Active: true}
|
|
||||||
r.download.variants[resolution] = v
|
|
||||||
}
|
|
||||||
return v
|
|
||||||
}
|
|
||||||
|
|
||||||
// --- Transfer ---
|
|
||||||
|
|
||||||
// AttachTransfer wires the registry to the live counters/state already
|
|
||||||
// maintained by pkg/transfer instead of duplicating them. Called once from
|
|
||||||
// TransferService construction.
|
|
||||||
func (r *Registry) AttachTransfer(stats StatsProvider, queueSizeFn, cleanupPendingFn func() int, nasConnectedFn func() bool) {
|
|
||||||
r.mu.Lock()
|
|
||||||
defer r.mu.Unlock()
|
|
||||||
r.transfer.statsProvider = stats
|
|
||||||
r.transfer.queueSizeFn = queueSizeFn
|
|
||||||
r.transfer.cleanupPendingFn = cleanupPendingFn
|
|
||||||
r.transfer.nasConnectedFn = nasConnectedFn
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Registry) RecordNASConnectivity(connected bool) {
|
|
||||||
r.mu.Lock()
|
|
||||||
defer r.mu.Unlock()
|
|
||||||
r.transfer.nasConnected = connected
|
|
||||||
}
|
|
||||||
|
|
||||||
// RecordTransferError captures failures QueueStats' cumulative counters
|
|
||||||
// don't individually retain (pre-queue failures, cleanup failures) so they
|
|
||||||
// still surface in the bounded recent-errors list.
|
|
||||||
func (r *Registry) RecordTransferError(sourcePath string, err error) {
|
|
||||||
r.mu.Lock()
|
|
||||||
defer r.mu.Unlock()
|
|
||||||
r.transfer.recentErrors = appendBounded(r.transfer.recentErrors, ErrorEntry{
|
|
||||||
Time: time.Now(),
|
|
||||||
Message: sourcePath + ": " + errString(err),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// --- Processing ---
|
|
||||||
|
|
||||||
func (r *Registry) RecordProcessingStageChange(state, eventName string) {
|
|
||||||
r.mu.Lock()
|
|
||||||
defer r.mu.Unlock()
|
|
||||||
r.processing.state = state
|
|
||||||
if eventName != "" {
|
|
||||||
r.processing.eventName = eventName
|
|
||||||
}
|
|
||||||
switch state {
|
|
||||||
case "scanning":
|
|
||||||
r.processing.startedAt = time.Now()
|
|
||||||
r.processing.finishedAt = time.Time{}
|
|
||||||
r.processing.lastError = ""
|
|
||||||
case "completed", "failed":
|
|
||||||
r.processing.finishedAt = time.Now()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Registry) RecordProcessingSegmentsFound(count int) {
|
|
||||||
r.mu.Lock()
|
|
||||||
defer r.mu.Unlock()
|
|
||||||
r.processing.segmentsFound = count
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Registry) RecordProcessingError(err error) {
|
|
||||||
r.mu.Lock()
|
|
||||||
defer r.mu.Unlock()
|
|
||||||
msg := errString(err)
|
|
||||||
r.processing.lastError = msg
|
|
||||||
r.processing.recentErrors = appendBounded(r.processing.recentErrors, ErrorEntry{
|
|
||||||
Time: time.Now(),
|
|
||||||
Message: msg,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// --- Snapshot ---
|
|
||||||
|
|
||||||
type DownloadSnapshot struct {
|
|
||||||
Variants []*VariantStatus `json:"variants"`
|
|
||||||
RecentErrors []ErrorEntry `json:"recent_errors"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type TransferSnapshot struct {
|
|
||||||
NASConnected bool `json:"nas_connected"`
|
|
||||||
QueueSize int `json:"queue_size"`
|
|
||||||
CleanupPending int `json:"cleanup_pending"`
|
|
||||||
Added int `json:"added"`
|
|
||||||
Completed int `json:"completed"`
|
|
||||||
Failed int `json:"failed"`
|
|
||||||
Pending int `json:"pending"`
|
|
||||||
BytesTransferred int64 `json:"bytes_transferred"`
|
|
||||||
RecentErrors []ErrorEntry `json:"recent_errors"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type ProcessingSnapshot struct {
|
|
||||||
State string `json:"state"`
|
|
||||||
EventName string `json:"event_name,omitempty"`
|
|
||||||
SegmentsFound int `json:"segments_found"`
|
|
||||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
|
||||||
FinishedAt *time.Time `json:"finished_at,omitempty"`
|
|
||||||
LastError string `json:"last_error,omitempty"`
|
|
||||||
RecentErrors []ErrorEntry `json:"recent_errors"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type Snapshot struct {
|
|
||||||
Status string `json:"status"`
|
|
||||||
UptimeSec float64 `json:"uptime_seconds"`
|
|
||||||
Download DownloadSnapshot `json:"download"`
|
|
||||||
Transfer TransferSnapshot `json:"transfer"`
|
|
||||||
Processing ProcessingSnapshot `json:"processing"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Registry) Snapshot() Snapshot {
|
|
||||||
r.mu.RLock()
|
|
||||||
defer r.mu.RUnlock()
|
|
||||||
|
|
||||||
variants := make([]*VariantStatus, 0, len(r.download.variants))
|
|
||||||
for _, v := range r.download.variants {
|
|
||||||
vCopy := *v
|
|
||||||
variants = append(variants, &vCopy)
|
|
||||||
}
|
|
||||||
sort.Slice(variants, func(i, j int) bool { return variants[i].Resolution < variants[j].Resolution })
|
|
||||||
|
|
||||||
dl := DownloadSnapshot{
|
|
||||||
Variants: variants,
|
|
||||||
RecentErrors: append([]ErrorEntry(nil), r.download.recentErrors...),
|
|
||||||
}
|
|
||||||
|
|
||||||
tr := TransferSnapshot{
|
|
||||||
NASConnected: r.transfer.nasConnected,
|
|
||||||
RecentErrors: append([]ErrorEntry(nil), r.transfer.recentErrors...),
|
|
||||||
}
|
|
||||||
if r.transfer.statsProvider != nil {
|
|
||||||
added, completed, failed, pending, bytes := r.transfer.statsProvider.GetStats()
|
|
||||||
tr.Added, tr.Completed, tr.Failed, tr.Pending, tr.BytesTransferred = added, completed, failed, pending, bytes
|
|
||||||
}
|
|
||||||
if r.transfer.queueSizeFn != nil {
|
|
||||||
tr.QueueSize = r.transfer.queueSizeFn()
|
|
||||||
}
|
|
||||||
if r.transfer.cleanupPendingFn != nil {
|
|
||||||
tr.CleanupPending = r.transfer.cleanupPendingFn()
|
|
||||||
}
|
|
||||||
if r.transfer.nasConnectedFn != nil {
|
|
||||||
tr.NASConnected = r.transfer.nasConnectedFn()
|
|
||||||
}
|
|
||||||
|
|
||||||
proc := ProcessingSnapshot{
|
|
||||||
State: r.processing.state,
|
|
||||||
EventName: r.processing.eventName,
|
|
||||||
SegmentsFound: r.processing.segmentsFound,
|
|
||||||
LastError: r.processing.lastError,
|
|
||||||
RecentErrors: append([]ErrorEntry(nil), r.processing.recentErrors...),
|
|
||||||
}
|
|
||||||
if !r.processing.startedAt.IsZero() {
|
|
||||||
t := r.processing.startedAt
|
|
||||||
proc.StartedAt = &t
|
|
||||||
}
|
|
||||||
if !r.processing.finishedAt.IsZero() {
|
|
||||||
t := r.processing.finishedAt
|
|
||||||
proc.FinishedAt = &t
|
|
||||||
}
|
|
||||||
|
|
||||||
snap := Snapshot{
|
|
||||||
UptimeSec: time.Since(r.startedAt).Seconds(),
|
|
||||||
Download: dl,
|
|
||||||
Transfer: tr,
|
|
||||||
Processing: proc,
|
|
||||||
}
|
|
||||||
snap.Status = computeStatus(dl, tr, proc)
|
|
||||||
return snap
|
|
||||||
}
|
|
||||||
|
|
||||||
// computeStatus gives one glanceable field: "degraded" the moment anything
|
|
||||||
// worth investigating shows up, "healthy" otherwise. Transfer is only
|
|
||||||
// considered for degradation once it's actually in use (Attached + has seen
|
|
||||||
// activity), so a download-only run with NAS transfer disabled doesn't
|
|
||||||
// falsely report degraded.
|
|
||||||
func computeStatus(dl DownloadSnapshot, tr TransferSnapshot, proc ProcessingSnapshot) string {
|
|
||||||
if len(dl.RecentErrors) > 0 || len(tr.RecentErrors) > 0 || len(proc.RecentErrors) > 0 {
|
|
||||||
return "degraded"
|
|
||||||
}
|
|
||||||
if !tr.NASConnected && (tr.Added > 0 || tr.QueueSize > 0) {
|
|
||||||
return "degraded"
|
|
||||||
}
|
|
||||||
if proc.State == "failed" {
|
|
||||||
return "degraded"
|
|
||||||
}
|
|
||||||
return "healthy"
|
|
||||||
}
|
|
||||||
@ -1,60 +0,0 @@
|
|||||||
package health
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
_ "embed"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"m3u8-downloader/pkg/config"
|
|
||||||
"m3u8-downloader/pkg/logging"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
)
|
|
||||||
|
|
||||||
//go:embed dashboard.html
|
|
||||||
var dashboardHTML []byte
|
|
||||||
|
|
||||||
// Start binds the health/status HTTP server and returns immediately once the
|
|
||||||
// listener is bound, so callers see bind failures (e.g. port already in use)
|
|
||||||
// synchronously rather than losing them in a background goroutine. The
|
|
||||||
// returned *http.Server should be shut down by the caller via Shutdown(ctx).
|
|
||||||
//
|
|
||||||
// Security note: cfg.Addr is expected to be a bind-all address (e.g. ":8090")
|
|
||||||
// so status can be checked from other devices on the local network during a
|
|
||||||
// live event. There is no authentication or TLS on these endpoints - this is
|
|
||||||
// only safe on a trusted local network, never exposed to the open internet.
|
|
||||||
func Start(ctx context.Context, cfg *config.HealthConfig, reg *Registry) (*http.Server, error) {
|
|
||||||
mux := http.NewServeMux()
|
|
||||||
|
|
||||||
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
|
||||||
w.Write(dashboardHTML)
|
|
||||||
})
|
|
||||||
|
|
||||||
mux.HandleFunc("/status", func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
if err := json.NewEncoder(w).Encode(reg.Snapshot()); err != nil {
|
|
||||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
_, _ = w.Write([]byte("ok"))
|
|
||||||
})
|
|
||||||
|
|
||||||
ln, err := net.Listen("tcp", cfg.Addr)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to bind health server to %s: %w", cfg.Addr, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
srv := &http.Server{Handler: mux}
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
if err := srv.Serve(ln); err != nil && err != http.ErrServerClosed {
|
|
||||||
logging.MustGetFactory().App.Error("health server stopped unexpectedly", logging.KeyError, err.Error())
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
return srv, nil
|
|
||||||
}
|
|
||||||
@ -1,72 +0,0 @@
|
|||||||
package logging
|
|
||||||
|
|
||||||
import (
|
|
||||||
"log/slog"
|
|
||||||
"m3u8-downloader/pkg/config"
|
|
||||||
"m3u8-downloader/pkg/constants"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Factory holds one structured logger per pipeline stage, each writing JSON
|
|
||||||
// lines to its own file under LoggingConfig.Dir. These are additive to the
|
|
||||||
// existing stdlib log.* calls, not a replacement for them.
|
|
||||||
type Factory struct {
|
|
||||||
App *slog.Logger
|
|
||||||
Download *slog.Logger
|
|
||||||
Transfer *slog.Logger
|
|
||||||
Processing *slog.Logger
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
globalFactory *Factory
|
|
||||||
factoryOnce sync.Once
|
|
||||||
)
|
|
||||||
|
|
||||||
// MustGetFactory returns the process-wide logging factory, building it from
|
|
||||||
// the config singleton on first use. Mirrors constants.MustGetConfig().
|
|
||||||
func MustGetFactory() *Factory {
|
|
||||||
factoryOnce.Do(func() {
|
|
||||||
globalFactory = New(&constants.MustGetConfig().Logging)
|
|
||||||
})
|
|
||||||
return globalFactory
|
|
||||||
}
|
|
||||||
|
|
||||||
func New(cfg *config.LoggingConfig) *Factory {
|
|
||||||
level := parseLevel(cfg.Level)
|
|
||||||
return &Factory{
|
|
||||||
App: newSubsystemLogger(cfg.Dir, "app.log", "app", level),
|
|
||||||
Download: newSubsystemLogger(cfg.Dir, "download.log", "download", level),
|
|
||||||
Transfer: newSubsystemLogger(cfg.Dir, "transfer.log", "transfer", level),
|
|
||||||
Processing: newSubsystemLogger(cfg.Dir, "processing.log", "processing", level),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// newSubsystemLogger opens (or creates) the subsystem's log file in append
|
|
||||||
// mode. A logging misconfiguration must never take down an active recording,
|
|
||||||
// so failure to open the file falls back to stderr instead of panicking.
|
|
||||||
func newSubsystemLogger(dir, filename, component string, level slog.Level) *slog.Logger {
|
|
||||||
path := filepath.Join(dir, filename)
|
|
||||||
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
|
|
||||||
if err != nil {
|
|
||||||
fallback := slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: level})).With("component", component)
|
|
||||||
fallback.Error("failed to open log file, falling back to stderr", KeyFile, path, KeyError, err.Error())
|
|
||||||
return fallback
|
|
||||||
}
|
|
||||||
return slog.New(slog.NewJSONHandler(f, &slog.HandlerOptions{Level: level})).With("component", component)
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseLevel(level string) slog.Level {
|
|
||||||
switch strings.ToLower(strings.TrimSpace(level)) {
|
|
||||||
case "debug":
|
|
||||||
return slog.LevelDebug
|
|
||||||
case "warn", "warning":
|
|
||||||
return slog.LevelWarn
|
|
||||||
case "error":
|
|
||||||
return slog.LevelError
|
|
||||||
default:
|
|
||||||
return slog.LevelInfo
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,13 +0,0 @@
|
|||||||
package logging
|
|
||||||
|
|
||||||
const (
|
|
||||||
KeyResolution = "resolution"
|
|
||||||
KeyBandwidth = "bandwidth"
|
|
||||||
KeySegment = "segment"
|
|
||||||
KeySeq = "seq"
|
|
||||||
KeyFile = "file"
|
|
||||||
KeyBytes = "bytes"
|
|
||||||
KeyEvent = "event"
|
|
||||||
KeyError = "error"
|
|
||||||
KeyAttempt = "attempt"
|
|
||||||
)
|
|
||||||
@ -4,26 +4,15 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"log"
|
"log"
|
||||||
"m3u8-downloader/pkg/constants"
|
"m3u8-downloader/pkg/constants"
|
||||||
"m3u8-downloader/pkg/logging"
|
|
||||||
"m3u8-downloader/pkg/utils"
|
"m3u8-downloader/pkg/utils"
|
||||||
"os"
|
"os"
|
||||||
"sort"
|
"sort"
|
||||||
"strconv"
|
|
||||||
"sync"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type ManifestWriter struct {
|
type ManifestWriter struct {
|
||||||
ManifestPath string
|
ManifestPath string
|
||||||
Segments []ManifestItem
|
Segments []ManifestItem
|
||||||
// Index maps a sequence number to its position in Segments. It holds
|
Index map[string]*ManifestItem
|
||||||
// positions rather than pointers because append reallocates the backing
|
|
||||||
// array, which would leave previously handed-out element pointers aimed at
|
|
||||||
// a stale copy.
|
|
||||||
Index map[string]int
|
|
||||||
|
|
||||||
// mu guards the fields above: every variant downloads concurrently, and
|
|
||||||
// each reports its segments as they land.
|
|
||||||
mu sync.Mutex
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type ManifestItem struct {
|
type ManifestItem struct {
|
||||||
@ -36,71 +25,53 @@ func NewManifestWriter(eventName string) *ManifestWriter {
|
|||||||
return &ManifestWriter{
|
return &ManifestWriter{
|
||||||
ManifestPath: cfg.GetManifestPath(eventName),
|
ManifestPath: cfg.GetManifestPath(eventName),
|
||||||
Segments: make([]ManifestItem, 0),
|
Segments: make([]ManifestItem, 0),
|
||||||
Index: make(map[string]int),
|
Index: make(map[string]*ManifestItem),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// AddOrUpdateSegment records that a segment was downloaded, keeping the
|
|
||||||
// highest-resolution variant seen for each sequence number.
|
|
||||||
func (m *ManifestWriter) AddOrUpdateSegment(seqNo string, resolution string) {
|
func (m *ManifestWriter) AddOrUpdateSegment(seqNo string, resolution string) {
|
||||||
m.mu.Lock()
|
|
||||||
defer m.mu.Unlock()
|
|
||||||
|
|
||||||
if m.Index == nil {
|
if m.Index == nil {
|
||||||
m.Index = make(map[string]int)
|
m.Index = make(map[string]*ManifestItem)
|
||||||
}
|
}
|
||||||
|
|
||||||
if m.Segments == nil {
|
if m.Segments == nil {
|
||||||
m.Segments = make([]ManifestItem, 0)
|
m.Segments = make([]ManifestItem, 0)
|
||||||
}
|
}
|
||||||
|
|
||||||
if pos, ok := m.Index[seqNo]; ok {
|
if existing, ok := m.Index[seqNo]; ok {
|
||||||
if ResolutionHeight(resolution) > ResolutionHeight(m.Segments[pos].Resolution) {
|
if resolution > existing.Resolution {
|
||||||
m.Segments[pos].Resolution = resolution
|
existing.Resolution = resolution
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
|
} else {
|
||||||
|
item := ManifestItem{
|
||||||
|
SeqNo: seqNo,
|
||||||
|
Resolution: resolution,
|
||||||
|
}
|
||||||
|
m.Segments = append(m.Segments, item)
|
||||||
|
m.Index[seqNo] = &item
|
||||||
}
|
}
|
||||||
|
|
||||||
m.Segments = append(m.Segments, ManifestItem{SeqNo: seqNo, Resolution: resolution})
|
|
||||||
m.Index[seqNo] = len(m.Segments) - 1
|
|
||||||
}
|
|
||||||
|
|
||||||
// Len reports how many distinct sequence numbers have been recorded.
|
|
||||||
func (m *ManifestWriter) Len() int {
|
|
||||||
m.mu.Lock()
|
|
||||||
defer m.mu.Unlock()
|
|
||||||
return len(m.Segments)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *ManifestWriter) WriteManifest() {
|
func (m *ManifestWriter) WriteManifest() {
|
||||||
// Sort a copy: reordering Segments in place would invalidate Index, and a
|
sort.Slice(m.Segments, func(i, j int) bool {
|
||||||
// recording may still be writing into it.
|
return m.Segments[i].SeqNo < m.Segments[j].SeqNo
|
||||||
m.mu.Lock()
|
|
||||||
segments := make([]ManifestItem, len(m.Segments))
|
|
||||||
copy(segments, m.Segments)
|
|
||||||
m.mu.Unlock()
|
|
||||||
|
|
||||||
sort.Slice(segments, func(i, j int) bool {
|
|
||||||
return lessSeqNo(segments[i].SeqNo, segments[j].SeqNo)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
data, err := json.MarshalIndent(segments, "", " ")
|
data, err := json.MarshalIndent(m.Segments, "", " ")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("Failed to marshal manifest: %v", err)
|
log.Printf("Failed to marshal manifest: %v", err)
|
||||||
logging.MustGetFactory().Download.Error("failed to marshal manifest", logging.KeyError, err.Error())
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := utils.ValidateWritablePath(m.ManifestPath); err != nil {
|
if err := utils.ValidateWritablePath(m.ManifestPath); err != nil {
|
||||||
log.Printf("Manifest path validation failed: %v", err)
|
log.Printf("Manifest path validation failed: %v", err)
|
||||||
logging.MustGetFactory().Download.Error("manifest path validation failed", logging.KeyError, err.Error())
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
file, err := os.Create(m.ManifestPath)
|
file, err := os.Create(m.ManifestPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("Failed to create manifest file: %v", err)
|
log.Printf("Failed to create manifest file: %v", err)
|
||||||
logging.MustGetFactory().Download.Error("failed to create manifest file", logging.KeyError, err.Error())
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -109,19 +80,6 @@ func (m *ManifestWriter) WriteManifest() {
|
|||||||
_, err = file.Write(data)
|
_, err = file.Write(data)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("Failed to write manifest file: %v", err)
|
log.Printf("Failed to write manifest file: %v", err)
|
||||||
logging.MustGetFactory().Download.Error("failed to write manifest file", logging.KeyError, err.Error())
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// lessSeqNo orders sequence numbers numerically, so segment 9 precedes segment
|
|
||||||
// 10 instead of sorting after it as the strings would. Values that are not
|
|
||||||
// numeric fall back to lexicographic order.
|
|
||||||
func lessSeqNo(a, b string) bool {
|
|
||||||
ai, aErr := strconv.ParseUint(a, 10, 64)
|
|
||||||
bi, bErr := strconv.ParseUint(b, 10, 64)
|
|
||||||
if aErr == nil && bErr == nil {
|
|
||||||
return ai < bi
|
|
||||||
}
|
|
||||||
return a < b
|
|
||||||
}
|
|
||||||
|
|||||||
@ -4,8 +4,6 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strconv"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -43,7 +41,7 @@ func TestManifestWriter_AddOrUpdateSegment(t *testing.T) {
|
|||||||
writer := &ManifestWriter{
|
writer := &ManifestWriter{
|
||||||
ManifestPath: "test.json",
|
ManifestPath: "test.json",
|
||||||
Segments: make([]ManifestItem, 0),
|
Segments: make([]ManifestItem, 0),
|
||||||
Index: make(map[string]int),
|
Index: make(map[string]*ManifestItem),
|
||||||
}
|
}
|
||||||
|
|
||||||
// Test adding new segment
|
// Test adding new segment
|
||||||
@ -114,7 +112,7 @@ func TestManifestWriter_WriteManifest(t *testing.T) {
|
|||||||
writer := &ManifestWriter{
|
writer := &ManifestWriter{
|
||||||
ManifestPath: manifestPath,
|
ManifestPath: manifestPath,
|
||||||
Segments: make([]ManifestItem, 0),
|
Segments: make([]ManifestItem, 0),
|
||||||
Index: make(map[string]int),
|
Index: make(map[string]*ManifestItem),
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add some test segments out of order
|
// Add some test segments out of order
|
||||||
@ -174,7 +172,7 @@ func TestManifestWriter_WriteManifest_EmptySegments(t *testing.T) {
|
|||||||
writer := &ManifestWriter{
|
writer := &ManifestWriter{
|
||||||
ManifestPath: manifestPath,
|
ManifestPath: manifestPath,
|
||||||
Segments: make([]ManifestItem, 0),
|
Segments: make([]ManifestItem, 0),
|
||||||
Index: make(map[string]int),
|
Index: make(map[string]*ManifestItem),
|
||||||
}
|
}
|
||||||
|
|
||||||
// Write empty manifest
|
// Write empty manifest
|
||||||
@ -206,7 +204,7 @@ func TestManifestWriter_WriteManifest_InvalidPath(t *testing.T) {
|
|||||||
writer := &ManifestWriter{
|
writer := &ManifestWriter{
|
||||||
ManifestPath: "/invalid/path/that/does/not/exist/manifest.json",
|
ManifestPath: "/invalid/path/that/does/not/exist/manifest.json",
|
||||||
Segments: []ManifestItem{{SeqNo: "1001", Resolution: "1080p"}},
|
Segments: []ManifestItem{{SeqNo: "1001", Resolution: "1080p"}},
|
||||||
Index: make(map[string]int),
|
Index: make(map[string]*ManifestItem),
|
||||||
}
|
}
|
||||||
|
|
||||||
// This should not panic, just fail gracefully
|
// This should not panic, just fail gracefully
|
||||||
@ -241,169 +239,3 @@ func TestManifestItem_JSONSerialization(t *testing.T) {
|
|||||||
t.Errorf("Resolution mismatch: expected '%s', got '%s'", item.Resolution, unmarshaled.Resolution)
|
t.Errorf("Resolution mismatch: expected '%s', got '%s'", item.Resolution, unmarshaled.Resolution)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestResolutionHeight(t *testing.T) {
|
|
||||||
cases := map[string]int{
|
|
||||||
"1080p": 1080,
|
|
||||||
"720p": 720,
|
|
||||||
"2160p": 2160,
|
|
||||||
"1440p": 1440,
|
|
||||||
"240p": 240,
|
|
||||||
// Not a <height>p label: must rank below every real resolution rather
|
|
||||||
// than accidentally outranking them.
|
|
||||||
"unknown": 0,
|
|
||||||
"": 0,
|
|
||||||
"p": 0,
|
|
||||||
}
|
|
||||||
|
|
||||||
for label, want := range cases {
|
|
||||||
if got := ResolutionHeight(label); got != want {
|
|
||||||
t.Errorf("ResolutionHeight(%q) = %d, want %d", label, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestAddOrUpdateSegmentKeepsHighestResolution pins the comparison used to
|
|
||||||
// choose between variants. Comparing the labels as strings — as this once did —
|
|
||||||
// ranks "720p" above "1080p", quietly downgrading the manifest.
|
|
||||||
func TestAddOrUpdateSegmentKeepsHighestResolution(t *testing.T) {
|
|
||||||
cases := []struct {
|
|
||||||
name string
|
|
||||||
arriving []string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{"higher replaces lower", []string{"720p", "1080p"}, "1080p"},
|
|
||||||
{"lower does not replace higher", []string{"1080p", "720p"}, "1080p"},
|
|
||||||
{"fewer digits must not win", []string{"1080p", "480p"}, "1080p"},
|
|
||||||
{"4k beats 1080p", []string{"1080p", "2160p"}, "2160p"},
|
|
||||||
{"unknown loses to a real resolution", []string{"unknown", "480p"}, "480p"},
|
|
||||||
{"unknown does not displace a real resolution", []string{"1080p", "unknown"}, "1080p"},
|
|
||||||
{"repeated identical resolution", []string{"720p", "720p"}, "720p"},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range cases {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
w := &ManifestWriter{}
|
|
||||||
for _, res := range tc.arriving {
|
|
||||||
w.AddOrUpdateSegment("1001", res)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(w.Segments) != 1 {
|
|
||||||
t.Fatalf("expected a single entry, got %d", len(w.Segments))
|
|
||||||
}
|
|
||||||
if got := w.Segments[0].Resolution; got != tc.want {
|
|
||||||
t.Errorf("kept %q, want %q (arrival order %v)", got, tc.want, tc.arriving)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestAddOrUpdateSegmentUpdatesTheStoredEntry guards the aliasing bug: the
|
|
||||||
// index once held a pointer to a local copy rather than to the stored element,
|
|
||||||
// so updates were applied to an orphan and never reached the written manifest.
|
|
||||||
func TestAddOrUpdateSegmentUpdatesTheStoredEntry(t *testing.T) {
|
|
||||||
w := &ManifestWriter{}
|
|
||||||
|
|
||||||
// Enough entries to force at least one append reallocation, which would
|
|
||||||
// invalidate any pointer held into the backing array.
|
|
||||||
for i := 0; i < 64; i++ {
|
|
||||||
w.AddOrUpdateSegment(strconv.Itoa(1000+i), "480p")
|
|
||||||
}
|
|
||||||
for i := 0; i < 64; i++ {
|
|
||||||
w.AddOrUpdateSegment(strconv.Itoa(1000+i), "1080p")
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(w.Segments) != 64 {
|
|
||||||
t.Fatalf("expected 64 entries, got %d", len(w.Segments))
|
|
||||||
}
|
|
||||||
for _, seg := range w.Segments {
|
|
||||||
if seg.Resolution != "1080p" {
|
|
||||||
t.Fatalf("segment %s kept %q; the upgrade never reached the stored entry", seg.SeqNo, seg.Resolution)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestAddOrUpdateSegmentIsConcurrencySafe reflects real use: every variant
|
|
||||||
// downloads in its own goroutine and reports segments as they land.
|
|
||||||
func TestAddOrUpdateSegmentIsConcurrencySafe(t *testing.T) {
|
|
||||||
w := &ManifestWriter{}
|
|
||||||
resolutions := []string{"240p", "480p", "720p", "1080p"}
|
|
||||||
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
for _, res := range resolutions {
|
|
||||||
wg.Add(1)
|
|
||||||
go func(res string) {
|
|
||||||
defer wg.Done()
|
|
||||||
for seq := 0; seq < 200; seq++ {
|
|
||||||
w.AddOrUpdateSegment(strconv.Itoa(seq), res)
|
|
||||||
}
|
|
||||||
}(res)
|
|
||||||
}
|
|
||||||
wg.Wait()
|
|
||||||
|
|
||||||
if got := w.Len(); got != 200 {
|
|
||||||
t.Errorf("expected 200 distinct sequence numbers, got %d", got)
|
|
||||||
}
|
|
||||||
for _, seg := range w.Segments {
|
|
||||||
if seg.Resolution != "1080p" {
|
|
||||||
t.Errorf("segment %s = %q, want the highest variant 1080p", seg.SeqNo, seg.Resolution)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestWriteManifestSortsNumerically checks that segment 9 precedes segment 10,
|
|
||||||
// which string ordering would reverse.
|
|
||||||
func TestWriteManifestSortsNumerically(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
w := &ManifestWriter{ManifestPath: filepath.Join(dir, "event.json")}
|
|
||||||
|
|
||||||
for _, seq := range []string{"100", "9", "1000", "10", "1"} {
|
|
||||||
w.AddOrUpdateSegment(seq, "1080p")
|
|
||||||
}
|
|
||||||
w.WriteManifest()
|
|
||||||
|
|
||||||
data, err := os.ReadFile(w.ManifestPath)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read manifest: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var items []ManifestItem
|
|
||||||
if err := json.Unmarshal(data, &items); err != nil {
|
|
||||||
t.Fatalf("manifest is not valid JSON: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := []string{"1", "9", "10", "100", "1000"}
|
|
||||||
if len(items) != len(want) {
|
|
||||||
t.Fatalf("expected %d entries, got %d", len(want), len(items))
|
|
||||||
}
|
|
||||||
for i, seq := range want {
|
|
||||||
if items[i].SeqNo != seq {
|
|
||||||
t.Errorf("position %d = %q, want %q (full order: %v)", i, items[i].SeqNo, seq, items)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestWriteManifestLeavesIndexUsable ensures writing mid-recording does not
|
|
||||||
// corrupt the writer: sorting in place would desynchronise Index from Segments.
|
|
||||||
func TestWriteManifestLeavesIndexUsable(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
w := &ManifestWriter{ManifestPath: filepath.Join(dir, "event.json")}
|
|
||||||
|
|
||||||
w.AddOrUpdateSegment("20", "480p")
|
|
||||||
w.AddOrUpdateSegment("10", "480p")
|
|
||||||
w.WriteManifest()
|
|
||||||
|
|
||||||
// After the write, an upgrade must still land on the right entry.
|
|
||||||
w.AddOrUpdateSegment("20", "1080p")
|
|
||||||
|
|
||||||
pos, ok := w.Index["20"]
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("sequence 20 missing from the index after WriteManifest")
|
|
||||||
}
|
|
||||||
if got := w.Segments[pos].Resolution; got != "1080p" {
|
|
||||||
t.Errorf("sequence 20 = %q, want 1080p; the index no longer matches the slice", got)
|
|
||||||
}
|
|
||||||
if w.Segments[w.Index["10"]].SeqNo != "10" {
|
|
||||||
t.Error("index and slice disagree after WriteManifest")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@ -6,7 +6,6 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"m3u8-downloader/pkg/constants"
|
"m3u8-downloader/pkg/constants"
|
||||||
"m3u8-downloader/pkg/httpClient"
|
"m3u8-downloader/pkg/httpClient"
|
||||||
"m3u8-downloader/pkg/logging"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
@ -34,7 +33,6 @@ func (j SegmentJob) Key() string {
|
|||||||
func DownloadSegment(ctx context.Context, client *http.Client, segmentURL string, outputDir string) error {
|
func DownloadSegment(ctx context.Context, client *http.Client, segmentURL string, outputDir string) error {
|
||||||
for attempt := 0; attempt < 2; attempt++ {
|
for attempt := 0; attempt < 2; attempt++ {
|
||||||
if attempt > 0 {
|
if attempt > 0 {
|
||||||
logging.MustGetFactory().Download.Debug("retrying segment download", logging.KeyAttempt, attempt, "url", segmentURL)
|
|
||||||
time.Sleep(300 * time.Millisecond)
|
time.Sleep(300 * time.Millisecond)
|
||||||
}
|
}
|
||||||
req, err := http.NewRequestWithContext(ctx, "GET", segmentURL, nil)
|
req, err := http.NewRequestWithContext(ctx, "GET", segmentURL, nil)
|
||||||
|
|||||||
@ -7,13 +7,10 @@ import (
|
|||||||
"github.com/grafov/m3u8"
|
"github.com/grafov/m3u8"
|
||||||
"log"
|
"log"
|
||||||
"m3u8-downloader/pkg/constants"
|
"m3u8-downloader/pkg/constants"
|
||||||
"m3u8-downloader/pkg/health"
|
|
||||||
"m3u8-downloader/pkg/httpClient"
|
"m3u8-downloader/pkg/httpClient"
|
||||||
"m3u8-downloader/pkg/logging"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"path"
|
"path"
|
||||||
"strconv"
|
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
@ -25,32 +22,10 @@ type StreamVariant struct {
|
|||||||
ID int
|
ID int
|
||||||
Resolution string
|
Resolution string
|
||||||
OutputDir string
|
OutputDir string
|
||||||
|
Writer *ManifestWriter
|
||||||
}
|
}
|
||||||
|
|
||||||
// ResolutionHeight converts a label produced by ExtractResolution back into
|
func extractResolution(variant *m3u8.Variant) string {
|
||||||
// the pixel height it stands for, so variants can be ranked by quality.
|
|
||||||
//
|
|
||||||
// The height is parsed rather than looked up in a table: labels come from the
|
|
||||||
// playlist's own RESOLUTION attribute, so a stream can legitimately offer
|
|
||||||
// 1440p or 2160p that no fixed table would anticipate. Anything not in
|
|
||||||
// <height>p form — "unknown", from a playlist with no variants — returns 0 and
|
|
||||||
// therefore ranks below every recognised resolution.
|
|
||||||
//
|
|
||||||
// Comparing the labels as strings instead is a trap worth naming: "720p" sorts
|
|
||||||
// after "1080p", so lexicographic comparison silently prefers lower quality.
|
|
||||||
func ResolutionHeight(resolution string) int {
|
|
||||||
trimmed := strings.TrimSuffix(strings.ToLower(strings.TrimSpace(resolution)), "p")
|
|
||||||
|
|
||||||
height, err := strconv.Atoi(trimmed)
|
|
||||||
if err != nil || height < 0 {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
return height
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExtractResolution maps a variant to the label used for its output directory,
|
|
||||||
// falling back to a bandwidth heuristic when the playlist omits RESOLUTION.
|
|
||||||
func ExtractResolution(variant *m3u8.Variant) string {
|
|
||||||
if variant.Resolution != "" {
|
if variant.Resolution != "" {
|
||||||
parts := strings.Split(variant.Resolution, "x")
|
parts := strings.Split(variant.Resolution, "x")
|
||||||
if len(parts) == 2 {
|
if len(parts) == 2 {
|
||||||
@ -71,24 +46,19 @@ func ExtractResolution(variant *m3u8.Variant) string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetAllVariants fetches the master playlist and returns one entry per stream
|
func GetAllVariants(masterURL string, outputDir string, writer *ManifestWriter) ([]*StreamVariant, error) {
|
||||||
// variant. Segments are recorded into the manifest by VariantDownloader, which
|
|
||||||
// receives it directly, so no writer is threaded through here.
|
|
||||||
func GetAllVariants(masterURL string, outputDir string) ([]*StreamVariant, error) {
|
|
||||||
client := &http.Client{}
|
client := &http.Client{}
|
||||||
req, _ := http.NewRequest("GET", masterURL, nil)
|
req, _ := http.NewRequest("GET", masterURL, nil)
|
||||||
req.Header.Set("User-Agent", constants.HTTPUserAgent)
|
req.Header.Set("User-Agent", constants.HTTPUserAgent)
|
||||||
req.Header.Set("Referer", constants.REFERRER)
|
req.Header.Set("Referer", constants.REFERRER)
|
||||||
resp, err := client.Do(req)
|
resp, err := client.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logging.MustGetFactory().Download.Error("failed to fetch master playlist", logging.KeyError, err.Error())
|
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
|
||||||
playlist, listType, err := m3u8.DecodeFrom(resp.Body, true)
|
playlist, listType, err := m3u8.DecodeFrom(resp.Body, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logging.MustGetFactory().Download.Error("failed to decode master playlist", logging.KeyError, err.Error())
|
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -102,6 +72,7 @@ func GetAllVariants(masterURL string, outputDir string) ([]*StreamVariant, error
|
|||||||
ID: 0,
|
ID: 0,
|
||||||
Resolution: "unknown",
|
Resolution: "unknown",
|
||||||
OutputDir: path.Join(outputDir, "unknown"),
|
OutputDir: path.Join(outputDir, "unknown"),
|
||||||
|
Writer: writer,
|
||||||
}}, nil
|
}}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -114,7 +85,7 @@ func GetAllVariants(masterURL string, outputDir string) ([]*StreamVariant, error
|
|||||||
for i, v := range master.Variants {
|
for i, v := range master.Variants {
|
||||||
vURL, _ := url.Parse(v.URI)
|
vURL, _ := url.Parse(v.URI)
|
||||||
fullURL := base.ResolveReference(vURL).String()
|
fullURL := base.ResolveReference(vURL).String()
|
||||||
resolution := ExtractResolution(v)
|
resolution := extractResolution(v)
|
||||||
outputDir := path.Join(outputDir, resolution)
|
outputDir := path.Join(outputDir, resolution)
|
||||||
variants = append(variants, &StreamVariant{
|
variants = append(variants, &StreamVariant{
|
||||||
URL: fullURL,
|
URL: fullURL,
|
||||||
@ -125,17 +96,11 @@ func GetAllVariants(masterURL string, outputDir string) ([]*StreamVariant, error
|
|||||||
OutputDir: outputDir,
|
OutputDir: outputDir,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
logging.MustGetFactory().Download.Info("discovered variants", "count", len(variants))
|
|
||||||
return variants, nil
|
return variants, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func VariantDownloader(ctx context.Context, variant *StreamVariant, sem chan struct{}, manifest *ManifestWriter) {
|
func VariantDownloader(ctx context.Context, variant *StreamVariant, sem chan struct{}, manifest *ManifestWriter) {
|
||||||
log.Printf("Starting %s variant downloader (bandwidth: %d)", variant.Resolution, variant.Bandwidth)
|
log.Printf("Starting %s variant downloader (bandwidth: %d)", variant.Resolution, variant.Bandwidth)
|
||||||
dlLog := logging.MustGetFactory().Download.With(logging.KeyResolution, variant.Resolution)
|
|
||||||
registry := health.MustGetRegistry()
|
|
||||||
dlLog.Info("variant downloader started", logging.KeyBandwidth, variant.Bandwidth)
|
|
||||||
registry.RecordVariantStarted(variant.Resolution, variant.Bandwidth)
|
|
||||||
|
|
||||||
ticker := time.NewTicker(constants.RefreshDelay)
|
ticker := time.NewTicker(constants.RefreshDelay)
|
||||||
defer ticker.Stop()
|
defer ticker.Stop()
|
||||||
client := &http.Client{}
|
client := &http.Client{}
|
||||||
@ -144,20 +109,16 @@ func VariantDownloader(ctx context.Context, variant *StreamVariant, sem chan str
|
|||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
registry.RecordVariantStopped(variant.Resolution)
|
|
||||||
return
|
return
|
||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
var seq uint64
|
|
||||||
playlist, err := LoadMediaPlaylist(variant.URL)
|
playlist, err := LoadMediaPlaylist(variant.URL)
|
||||||
|
seq := playlist.SeqNo
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("%s: Error loading playlist playlist: %v", variant.Resolution, err)
|
log.Printf("%s: Error loading playlist playlist: %v", variant.Resolution, err)
|
||||||
dlLog.Warn("failed to load media playlist", logging.KeyError, err.Error())
|
|
||||||
registry.RecordPlaylistError(variant.Resolution, err)
|
|
||||||
goto waitTick
|
goto waitTick
|
||||||
}
|
}
|
||||||
seq = playlist.SeqNo
|
|
||||||
|
|
||||||
for _, seg := range playlist.Segments {
|
for _, seg := range playlist.Segments {
|
||||||
if seg == nil {
|
if seg == nil {
|
||||||
@ -187,11 +148,6 @@ func VariantDownloader(ctx context.Context, variant *StreamVariant, sem chan str
|
|||||||
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
log.Printf("✓ %s downloaded segment %s", j.Variant.Resolution, name)
|
log.Printf("✓ %s downloaded segment %s", j.Variant.Resolution, name)
|
||||||
dlLog.Debug("segment downloaded", logging.KeySegment, name)
|
|
||||||
registry.RecordSegmentDownloaded(variant.Resolution)
|
|
||||||
if manifest != nil {
|
|
||||||
manifest.AddOrUpdateSegment(strconv.FormatUint(j.Seq, 10), variant.Resolution)
|
|
||||||
}
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -202,27 +158,21 @@ func VariantDownloader(ctx context.Context, variant *StreamVariant, sem chan str
|
|||||||
|
|
||||||
if httpClient.IsHTTPStatus(err, 403) {
|
if httpClient.IsHTTPStatus(err, 403) {
|
||||||
log.Printf("✗ %s failed to download segment %s (403)", j.Variant.Resolution, name)
|
log.Printf("✗ %s failed to download segment %s (403)", j.Variant.Resolution, name)
|
||||||
dlLog.Warn("segment download failed (403)", logging.KeySegment, name)
|
|
||||||
} else {
|
} else {
|
||||||
log.Printf("✗ %s failed to download segment %s: %v", j.Variant.Resolution, name, err)
|
log.Printf("✗ %s failed to download segment %s: %v", j.Variant.Resolution, name, err)
|
||||||
dlLog.Error("segment download failed", logging.KeySegment, name, logging.KeyError, err.Error())
|
|
||||||
}
|
}
|
||||||
registry.RecordSegmentFailed(variant.Resolution, err)
|
|
||||||
}(job)
|
}(job)
|
||||||
seq++
|
seq++
|
||||||
}
|
}
|
||||||
|
|
||||||
if playlist.Closed {
|
if playlist.Closed {
|
||||||
log.Printf("%s: Playlist closed (#EXT-X-ENDLIST)", variant.Resolution)
|
log.Printf("%s: Playlist closed (#EXT-X-ENDLIST)", variant.Resolution)
|
||||||
dlLog.Info("playlist closed")
|
|
||||||
registry.RecordVariantStopped(variant.Resolution)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
waitTick:
|
waitTick:
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
registry.RecordVariantStopped(variant.Resolution)
|
|
||||||
return
|
return
|
||||||
case <-ticker.C:
|
case <-ticker.C:
|
||||||
}
|
}
|
||||||
|
|||||||
@ -3,18 +3,10 @@ package nas
|
|||||||
import "time"
|
import "time"
|
||||||
|
|
||||||
type NASConfig struct {
|
type NASConfig struct {
|
||||||
// Path is the destination as configured. It may be a UNC path, an
|
|
||||||
// smb:// URL, or a directory on this host; see ParseLocation.
|
|
||||||
Path string
|
Path string
|
||||||
Username string
|
Username string
|
||||||
Password string
|
Password string
|
||||||
Timeout time.Duration
|
Timeout time.Duration
|
||||||
RetryLimit int
|
RetryLimit int
|
||||||
VerifySize bool
|
VerifySize bool
|
||||||
// AutoMount allows the recorder to mount an SMB share itself when the
|
|
||||||
// host has not already mounted it. Linux only, and requires root, so it
|
|
||||||
// is off by default in favour of a clear "mount it first" error.
|
|
||||||
AutoMount bool
|
|
||||||
// MountPoint overrides where AutoMount attaches the share.
|
|
||||||
MountPoint string
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,98 +0,0 @@
|
|||||||
package nas
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"os/exec"
|
|
||||||
"runtime"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Connector attaches a configured Location to the host filesystem so that
|
|
||||||
// ordinary file operations reach the NAS.
|
|
||||||
//
|
|
||||||
// Windows and Linux disagree about what "connecting to a share" means —
|
|
||||||
// Windows authenticates a UNC path in place, Linux needs the share mounted
|
|
||||||
// somewhere in its single filesystem tree — so each host gets its own
|
|
||||||
// implementation and NASService stays platform-neutral. Selection happens at
|
|
||||||
// runtime rather than through build tags so that every implementation is
|
|
||||||
// compiled, vetted and unit-testable on every platform; only the final exec
|
|
||||||
// call is genuinely host-specific.
|
|
||||||
type Connector interface {
|
|
||||||
// Name identifies the strategy in logs and check output.
|
|
||||||
Name() string
|
|
||||||
// Attach makes loc reachable and reports where to write.
|
|
||||||
Attach(ctx context.Context, loc Location, cfg NASConfig) (Attachment, error)
|
|
||||||
// Detach undoes whatever Attach set up. Implementations must undo only
|
|
||||||
// what they created: a share this process found already mounted belongs
|
|
||||||
// to the host, and unmounting it could break unrelated software.
|
|
||||||
Detach(ctx context.Context, att Attachment) error
|
|
||||||
}
|
|
||||||
|
|
||||||
// Attachment is the result of a successful Attach.
|
|
||||||
type Attachment struct {
|
|
||||||
// BasePath is the host-native directory that maps to the configured
|
|
||||||
// destination. Every file operation is resolved against it.
|
|
||||||
BasePath string
|
|
||||||
// Notes explain how the host reached the destination, for -check output
|
|
||||||
// and startup logs.
|
|
||||||
Notes []string
|
|
||||||
// Owned records that this process created the connection or mount, and
|
|
||||||
// so may tear it down on shutdown.
|
|
||||||
Owned bool
|
|
||||||
// mountPoint is the directory this process mounted, when Owned is set by
|
|
||||||
// the Linux connector.
|
|
||||||
mountPoint string
|
|
||||||
// share is the network path this process connected, when Owned is set by
|
|
||||||
// the Windows connector.
|
|
||||||
share string
|
|
||||||
}
|
|
||||||
|
|
||||||
// commandRunner executes an external command. Implementations keep argument
|
|
||||||
// construction in a pure function and call through this, so tests can assert
|
|
||||||
// exactly what would run without running it.
|
|
||||||
type commandRunner func(ctx context.Context, name string, args ...string) ([]byte, error)
|
|
||||||
|
|
||||||
// execRunner is the production runner: it combines stdout and stderr because
|
|
||||||
// mount and net both report the useful diagnosis on one or the other.
|
|
||||||
func execRunner(ctx context.Context, name string, args ...string) ([]byte, error) {
|
|
||||||
return exec.CommandContext(ctx, name, args...).CombinedOutput()
|
|
||||||
}
|
|
||||||
|
|
||||||
// connectorFor picks the strategy for a destination on a given OS. A
|
|
||||||
// destination that is already an ordinary directory needs no network step on
|
|
||||||
// any platform, so it short-circuits before the OS switch.
|
|
||||||
func connectorFor(loc Location, goos string) Connector {
|
|
||||||
if !loc.IsRemote() {
|
|
||||||
return &localConnector{goos: goos}
|
|
||||||
}
|
|
||||||
|
|
||||||
switch goos {
|
|
||||||
case "windows":
|
|
||||||
return &windowsConnector{run: execRunner}
|
|
||||||
case "linux":
|
|
||||||
return &linuxConnector{run: execRunner, mountsPath: procMountsPath}
|
|
||||||
default:
|
|
||||||
return &unsupportedConnector{goos: goos}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// defaultConnector selects the strategy for the running host.
|
|
||||||
func defaultConnector(loc Location) Connector {
|
|
||||||
return connectorFor(loc, runtime.GOOS)
|
|
||||||
}
|
|
||||||
|
|
||||||
// unsupportedConnector fails clearly on platforms this project does not target,
|
|
||||||
// rather than silently writing a share-shaped directory to local disk.
|
|
||||||
type unsupportedConnector struct {
|
|
||||||
goos string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *unsupportedConnector) Name() string { return "unsupported" }
|
|
||||||
|
|
||||||
func (c *unsupportedConnector) Attach(_ context.Context, loc Location, _ NASConfig) (Attachment, error) {
|
|
||||||
return Attachment{}, fmt.Errorf(
|
|
||||||
"connecting to network share %s is not supported on %s; mount the share and set NAS_OUTPUT_PATH to the mount point",
|
|
||||||
loc.SMBSource(), c.goos)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *unsupportedConnector) Detach(context.Context, Attachment) error { return nil }
|
|
||||||
@ -1,57 +0,0 @@
|
|||||||
package nas
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
// localConnector handles destinations that are already part of the host
|
|
||||||
// filesystem: a plain output directory, or — the normal arrangement on Linux —
|
|
||||||
// an SMB/NFS share mounted by fstab, autofs or systemd before the recorder
|
|
||||||
// starts. There is nothing to connect, but there is something to check.
|
|
||||||
type localConnector struct {
|
|
||||||
goos string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *localConnector) Name() string { return "local" }
|
|
||||||
|
|
||||||
func (c *localConnector) Attach(_ context.Context, loc Location, _ NASConfig) (Attachment, error) {
|
|
||||||
if err := c.validate(loc.Raw); err != nil {
|
|
||||||
return Attachment{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return Attachment{
|
|
||||||
BasePath: filepath.Clean(loc.Raw),
|
|
||||||
Notes: []string{"local or already-mounted path (no network authentication)"},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *localConnector) Detach(context.Context, Attachment) error { return nil }
|
|
||||||
|
|
||||||
// validate rejects a Windows-shaped path on a non-Windows host. This looks
|
|
||||||
// pedantic and is not: backslashes and colons are legal characters in Linux
|
|
||||||
// filenames, so a stray C:\data or \\server\share would be created verbatim as
|
|
||||||
// a local directory. Transfers would report success, cleanup would delete the
|
|
||||||
// local originals, and nothing would ever reach the NAS.
|
|
||||||
func (c *localConnector) validate(path string) error {
|
|
||||||
if c.goos == "windows" {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if driveLetter.MatchString(path) {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"%q is a Windows drive path but this host is %s; set NAS_OUTPUT_PATH to a mount point such as /mnt/nas/streams",
|
|
||||||
path, c.goos)
|
|
||||||
}
|
|
||||||
|
|
||||||
if strings.Contains(path, `\`) {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"%q contains backslashes but this host is %s, where they are ordinary filename characters rather than separators; "+
|
|
||||||
"set NAS_OUTPUT_PATH to a mount point such as /mnt/nas/streams, or to \\\\server\\share\\folder to have the share resolved automatically",
|
|
||||||
path, c.goos)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@ -1,178 +0,0 @@
|
|||||||
package nas
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
// linuxConnector reaches a share the way Linux does: through a mount point in
|
|
||||||
// the single filesystem tree. There is no per-path authentication, so the
|
|
||||||
// question is not "can we connect" but "where is this share mounted".
|
|
||||||
//
|
|
||||||
// The default and by far the safest answer is that the host already mounted it
|
|
||||||
// (fstab, autofs, systemd, or `docker run -v`), so the connector looks the
|
|
||||||
// share up in the mount table. Mounting it here is opt-in because it requires
|
|
||||||
// root and changes system state that outlives a failed run.
|
|
||||||
type linuxConnector struct {
|
|
||||||
run commandRunner
|
|
||||||
mountsPath string
|
|
||||||
// geteuid is indirected for tests; production uses os.Geteuid.
|
|
||||||
geteuid func() int
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *linuxConnector) Name() string { return "mount" }
|
|
||||||
|
|
||||||
func (c *linuxConnector) Attach(ctx context.Context, loc Location, cfg NASConfig) (Attachment, error) {
|
|
||||||
entries, err := readMountTable(c.mountsPath)
|
|
||||||
if err != nil {
|
|
||||||
return Attachment{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if point, rest, ok := findMount(entries, loc); ok {
|
|
||||||
return Attachment{
|
|
||||||
BasePath: filepath.Join(append([]string{point}, rest...)...),
|
|
||||||
Notes: []string{
|
|
||||||
"share: " + loc.SMBSource(),
|
|
||||||
fmt.Sprintf("resolved via existing mount at %s", point),
|
|
||||||
},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if !cfg.AutoMount {
|
|
||||||
return Attachment{}, fmt.Errorf("%s", notMountedMessage(loc, cfg))
|
|
||||||
}
|
|
||||||
|
|
||||||
return c.mount(ctx, loc, cfg)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *linuxConnector) Detach(ctx context.Context, att Attachment) error {
|
|
||||||
// Never unmount a share the host had already mounted; other software may
|
|
||||||
// be using it, and we did not put it there.
|
|
||||||
if !att.Owned || att.mountPoint == "" {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if out, err := c.run(ctx, "umount", att.mountPoint); err != nil {
|
|
||||||
return fmt.Errorf("failed to unmount %s: %v", att.mountPoint, summarize(err, out))
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// mount performs an opt-in mount.cifs of the share.
|
|
||||||
func (c *linuxConnector) mount(ctx context.Context, loc Location, cfg NASConfig) (Attachment, error) {
|
|
||||||
geteuid := c.geteuid
|
|
||||||
if geteuid == nil {
|
|
||||||
geteuid = os.Geteuid
|
|
||||||
}
|
|
||||||
if geteuid() != 0 {
|
|
||||||
return Attachment{}, fmt.Errorf(
|
|
||||||
"NAS_AUTO_MOUNT is set but mounting %s requires root; run as root, or mount the share beforehand and %s",
|
|
||||||
loc.SMBSource(), suggestMountPointConfig(loc))
|
|
||||||
}
|
|
||||||
|
|
||||||
point := autoMountPoint(loc, cfg)
|
|
||||||
if err := os.MkdirAll(point, 0755); err != nil {
|
|
||||||
return Attachment{}, fmt.Errorf("failed to create mount point %s: %w", point, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
credentialsPath, cleanup, err := writeCredentialsFile(cfg)
|
|
||||||
if err != nil {
|
|
||||||
return Attachment{}, err
|
|
||||||
}
|
|
||||||
defer cleanup()
|
|
||||||
|
|
||||||
if cfg.Timeout > 0 {
|
|
||||||
var cancel context.CancelFunc
|
|
||||||
ctx, cancel = context.WithTimeout(ctx, cfg.Timeout)
|
|
||||||
defer cancel()
|
|
||||||
}
|
|
||||||
|
|
||||||
if out, err := c.run(ctx, "mount", mountArgs(loc, point, credentialsPath)...); err != nil {
|
|
||||||
return Attachment{}, fmt.Errorf("failed to mount %s at %s: %v", loc.SMBSource(), point, summarize(err, out))
|
|
||||||
}
|
|
||||||
|
|
||||||
return Attachment{
|
|
||||||
BasePath: filepath.Join(append([]string{point}, loc.Sub...)...),
|
|
||||||
Owned: true,
|
|
||||||
mountPoint: point,
|
|
||||||
Notes: []string{
|
|
||||||
"share: " + loc.SMBSource(),
|
|
||||||
"user: " + usernameNote(cfg.Username),
|
|
||||||
fmt.Sprintf("mounted at %s by this process (will unmount on shutdown)", point),
|
|
||||||
},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// mountArgs builds the mount.cifs invocation. Kept pure so it can be asserted
|
|
||||||
// without mounting anything.
|
|
||||||
func mountArgs(loc Location, mountPoint, credentialsPath string) []string {
|
|
||||||
options := "rw"
|
|
||||||
if credentialsPath != "" {
|
|
||||||
options += ",credentials=" + credentialsPath
|
|
||||||
} else {
|
|
||||||
options += ",guest"
|
|
||||||
}
|
|
||||||
return []string{"-t", "cifs", loc.SMBSource(), mountPoint, "-o", options}
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeCredentialsFile stages the NAS credentials in a 0600 file. mount.cifs
|
|
||||||
// also accepts user=/pass= options, but those land in the process table where
|
|
||||||
// any local user can read the password; a credentials file does not.
|
|
||||||
func writeCredentialsFile(cfg NASConfig) (path string, cleanup func(), err error) {
|
|
||||||
if cfg.Username == "" || cfg.Password == "" {
|
|
||||||
return "", func() {}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
dir, err := os.MkdirTemp("", "streamrecorder-cifs-")
|
|
||||||
if err != nil {
|
|
||||||
return "", func() {}, fmt.Errorf("failed to stage NAS credentials: %w", err)
|
|
||||||
}
|
|
||||||
cleanup = func() { os.RemoveAll(dir) }
|
|
||||||
|
|
||||||
path = filepath.Join(dir, "credentials")
|
|
||||||
content := fmt.Sprintf("username=%s\npassword=%s\n", cfg.Username, cfg.Password)
|
|
||||||
if err := os.WriteFile(path, []byte(content), 0600); err != nil {
|
|
||||||
cleanup()
|
|
||||||
return "", func() {}, fmt.Errorf("failed to write NAS credentials file: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return path, cleanup, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// autoMountPoint chooses where to mount the share when the operator has not
|
|
||||||
// named a location. Everything lands under one predictable directory so a
|
|
||||||
// leftover mount is easy to find and clean up.
|
|
||||||
func autoMountPoint(loc Location, cfg NASConfig) string {
|
|
||||||
if cfg.MountPoint != "" {
|
|
||||||
return cfg.MountPoint
|
|
||||||
}
|
|
||||||
return filepath.Join("/mnt", "streamrecorder", strings.ToLower(loc.Server), strings.ToLower(loc.Share))
|
|
||||||
}
|
|
||||||
|
|
||||||
// notMountedMessage explains the failure in terms of what the operator should
|
|
||||||
// actually type. A bare "no such mount" would leave them guessing.
|
|
||||||
func notMountedMessage(loc Location, cfg NASConfig) string {
|
|
||||||
source := loc.SMBSource()
|
|
||||||
example := filepath.Join("/mnt", strings.ToLower(loc.Share))
|
|
||||||
|
|
||||||
credentials := ""
|
|
||||||
if cfg.Username != "" {
|
|
||||||
credentials = fmt.Sprintf(",username=%s,password=***", cfg.Username)
|
|
||||||
}
|
|
||||||
|
|
||||||
return fmt.Sprintf(
|
|
||||||
"%s is not mounted on this host. Either mount it and %s, for example:\n"+
|
|
||||||
" sudo mkdir -p %s && sudo mount -t cifs %s %s -o rw%s\n"+
|
|
||||||
"or set NAS_AUTO_MOUNT=true to have the recorder mount it (requires root)",
|
|
||||||
source, suggestMountPointConfig(loc), example, source, example, credentials)
|
|
||||||
}
|
|
||||||
|
|
||||||
func suggestMountPointConfig(loc Location) string {
|
|
||||||
if len(loc.Sub) == 0 {
|
|
||||||
return "set NAS_OUTPUT_PATH to the mount point"
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("set NAS_OUTPUT_PATH to the mount point plus %s", loc.SubPath())
|
|
||||||
}
|
|
||||||
@ -1,107 +0,0 @@
|
|||||||
package nas
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
// windowsConnector reaches a share the way Windows does: authenticate the UNC
|
|
||||||
// path with `net use`, then use the UNC path directly as a filesystem path.
|
|
||||||
// Nothing is mounted into a tree, so the resolved base path is the configured
|
|
||||||
// UNC path itself.
|
|
||||||
type windowsConnector struct {
|
|
||||||
run commandRunner
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *windowsConnector) Name() string { return "net use" }
|
|
||||||
|
|
||||||
func (c *windowsConnector) Attach(ctx context.Context, loc Location, cfg NASConfig) (Attachment, error) {
|
|
||||||
share := loc.ShareUNC()
|
|
||||||
att := Attachment{
|
|
||||||
BasePath: loc.UNC(),
|
|
||||||
share: share,
|
|
||||||
Notes: []string{"share: " + share, "user: " + usernameNote(cfg.Username)},
|
|
||||||
}
|
|
||||||
|
|
||||||
if cfg.Timeout > 0 {
|
|
||||||
var cancel context.CancelFunc
|
|
||||||
ctx, cancel = context.WithTimeout(ctx, cfg.Timeout)
|
|
||||||
defer cancel()
|
|
||||||
}
|
|
||||||
|
|
||||||
out, err := c.run(ctx, "net", netUseArgs(loc, cfg)...)
|
|
||||||
if err != nil {
|
|
||||||
// A failure here is not decisive. `net use` also fails when the share
|
|
||||||
// is already connected — commonly because Explorer or another process
|
|
||||||
// got there first — in which case the path is perfectly usable. Record
|
|
||||||
// the reason and let the caller's write probe deliver the verdict,
|
|
||||||
// rather than refusing to run against a working share.
|
|
||||||
att.Notes = append(att.Notes,
|
|
||||||
fmt.Sprintf("net use did not complete (%v); continuing, the share may already be connected", summarize(err, out)))
|
|
||||||
return att, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
att.Owned = true
|
|
||||||
att.Notes = append(att.Notes, "connected via net use")
|
|
||||||
return att, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *windowsConnector) Detach(ctx context.Context, att Attachment) error {
|
|
||||||
// Only drop connections this process made. Deleting a mapping the user or
|
|
||||||
// another application established would break them.
|
|
||||||
if !att.Owned || att.share == "" {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if out, err := c.run(ctx, "net", "use", att.share, "/delete"); err != nil {
|
|
||||||
return fmt.Errorf("failed to disconnect %s: %v", att.share, summarize(err, out))
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// netUseArgs builds the `net use` invocation. Kept pure so the Windows code
|
|
||||||
// path can be asserted from any host, including the Linux box this is
|
|
||||||
// developed on, without executing anything.
|
|
||||||
func netUseArgs(loc Location, cfg NASConfig) []string {
|
|
||||||
args := []string{"use", loc.ShareUNC()}
|
|
||||||
if cfg.Username != "" && cfg.Password != "" {
|
|
||||||
args = append(args, "/user:"+cfg.Username, cfg.Password)
|
|
||||||
}
|
|
||||||
return append(args, "/persistent:no")
|
|
||||||
}
|
|
||||||
|
|
||||||
// summarize pairs an exec error with the first line of command output, which is
|
|
||||||
// where net and mount put the actual diagnosis.
|
|
||||||
func summarize(err error, out []byte) string {
|
|
||||||
text := strings.TrimSpace(string(out))
|
|
||||||
if text == "" {
|
|
||||||
return err.Error()
|
|
||||||
}
|
|
||||||
if idx := strings.IndexByte(text, '\n'); idx >= 0 {
|
|
||||||
text = text[:idx]
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("%v: %s", err, text)
|
|
||||||
}
|
|
||||||
|
|
||||||
func usernameNote(username string) string {
|
|
||||||
if username == "" {
|
|
||||||
return "(none, using ambient credentials)"
|
|
||||||
}
|
|
||||||
return username
|
|
||||||
}
|
|
||||||
|
|
||||||
// joinBase resolves a relative path against a base that may be a UNC path.
|
|
||||||
// filepath.Join is correct on Windows and would mangle the leading \\ anywhere
|
|
||||||
// else, so UNC bases are joined with backslashes explicitly.
|
|
||||||
func joinBase(base string, elem ...string) string {
|
|
||||||
if len(elem) == 0 {
|
|
||||||
return base
|
|
||||||
}
|
|
||||||
if strings.HasPrefix(base, `\\`) {
|
|
||||||
parts := append([]string{strings.TrimRight(base, `\`)}, elem...)
|
|
||||||
return strings.Join(parts, `\`)
|
|
||||||
}
|
|
||||||
return filepath.Join(append([]string{base}, elem...)...)
|
|
||||||
}
|
|
||||||
@ -1,353 +0,0 @@
|
|||||||
package nas
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// recordingRunner captures the commands a connector would run instead of
|
|
||||||
// running them. This is what lets the Windows path be tested from a Linux
|
|
||||||
// development machine: the decision logic and the exact argv are asserted
|
|
||||||
// without ever invoking net.exe.
|
|
||||||
type recordingRunner struct {
|
|
||||||
calls [][]string
|
|
||||||
output []byte
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *recordingRunner) run(_ context.Context, name string, args ...string) ([]byte, error) {
|
|
||||||
r.calls = append(r.calls, append([]string{name}, args...))
|
|
||||||
return r.output, r.err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *recordingRunner) lastCall() string {
|
|
||||||
if len(r.calls) == 0 {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return strings.Join(r.calls[len(r.calls)-1], " ")
|
|
||||||
}
|
|
||||||
|
|
||||||
func remoteLocation(t *testing.T, raw string) Location {
|
|
||||||
t.Helper()
|
|
||||||
loc, err := ParseLocation(raw)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ParseLocation(%q) failed: %v", raw, err)
|
|
||||||
}
|
|
||||||
return loc
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConnectorSelection(t *testing.T) {
|
|
||||||
remote := remoteLocation(t, `\\nas\dci\streams`)
|
|
||||||
local := remoteLocation(t, "/mnt/dci/streams")
|
|
||||||
|
|
||||||
cases := []struct {
|
|
||||||
name string
|
|
||||||
loc Location
|
|
||||||
goos string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{"windows share", remote, "windows", "net use"},
|
|
||||||
{"linux share", remote, "linux", "mount"},
|
|
||||||
{"darwin share", remote, "darwin", "unsupported"},
|
|
||||||
{"local path on windows", local, "windows", "local"},
|
|
||||||
{"local path on linux", local, "linux", "local"},
|
|
||||||
// A already-mounted path needs no network step even where shares are
|
|
||||||
// unsupported, so it must not fall through to the error connector.
|
|
||||||
{"local path on darwin", local, "darwin", "local"},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range cases {
|
|
||||||
if got := connectorFor(tc.loc, tc.goos).Name(); got != tc.want {
|
|
||||||
t.Errorf("%s: connectorFor = %q, want %q", tc.name, got, tc.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWindowsConnectorAttach(t *testing.T) {
|
|
||||||
runner := &recordingRunner{}
|
|
||||||
c := &windowsConnector{run: runner.run}
|
|
||||||
loc := remoteLocation(t, `\\HomeLabNAS\dci\streams`)
|
|
||||||
|
|
||||||
att, err := c.Attach(context.Background(), loc, NASConfig{Username: "NASAdmin", Password: "secret"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Attach failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The base path is the full UNC path; net use authenticates only the share.
|
|
||||||
if want := `\\HomeLabNAS\dci\streams`; att.BasePath != want {
|
|
||||||
t.Errorf("BasePath = %q, want %q", att.BasePath, want)
|
|
||||||
}
|
|
||||||
if want := `net use \\HomeLabNAS\dci /user:NASAdmin secret /persistent:no`; runner.lastCall() != want {
|
|
||||||
t.Errorf("command = %q, want %q", runner.lastCall(), want)
|
|
||||||
}
|
|
||||||
if !att.Owned {
|
|
||||||
t.Error("a successful net use should be owned so it can be released on shutdown")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWindowsConnectorOmitsCredentialsWhenUnset(t *testing.T) {
|
|
||||||
runner := &recordingRunner{}
|
|
||||||
c := &windowsConnector{run: runner.run}
|
|
||||||
|
|
||||||
if _, err := c.Attach(context.Background(), remoteLocation(t, `\\nas\dci`), NASConfig{}); err != nil {
|
|
||||||
t.Fatalf("Attach failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if want := `net use \\nas\dci /persistent:no`; runner.lastCall() != want {
|
|
||||||
t.Errorf("command = %q, want %q", runner.lastCall(), want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWindowsConnectorContinuesWhenAlreadyConnected(t *testing.T) {
|
|
||||||
// net use fails when a connection already exists, but the share is usable;
|
|
||||||
// refusing to run would be wrong. The write probe is the real verdict.
|
|
||||||
runner := &recordingRunner{
|
|
||||||
err: fmt.Errorf("exit status 2"),
|
|
||||||
output: []byte("The network connection could not be found.\nMore help is available..."),
|
|
||||||
}
|
|
||||||
c := &windowsConnector{run: runner.run}
|
|
||||||
|
|
||||||
att, err := c.Attach(context.Background(), remoteLocation(t, `\\nas\dci`), NASConfig{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Attach should tolerate a net use failure, got: %v", err)
|
|
||||||
}
|
|
||||||
if att.Owned {
|
|
||||||
t.Error("a failed net use must not be marked owned, or Detach would drop someone else's connection")
|
|
||||||
}
|
|
||||||
|
|
||||||
notes := strings.Join(att.Notes, "\n")
|
|
||||||
if !strings.Contains(notes, "net use did not complete") {
|
|
||||||
t.Errorf("expected the failure to be recorded in notes:\n%s", notes)
|
|
||||||
}
|
|
||||||
// Only the first line of output is worth surfacing.
|
|
||||||
if strings.Contains(notes, "More help is available") {
|
|
||||||
t.Errorf("notes should not carry the full usage dump:\n%s", notes)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWindowsConnectorDetachOnlyReleasesOwned(t *testing.T) {
|
|
||||||
runner := &recordingRunner{}
|
|
||||||
c := &windowsConnector{run: runner.run}
|
|
||||||
|
|
||||||
if err := c.Detach(context.Background(), Attachment{share: `\\nas\dci`, Owned: false}); err != nil {
|
|
||||||
t.Fatalf("Detach failed: %v", err)
|
|
||||||
}
|
|
||||||
if len(runner.calls) != 0 {
|
|
||||||
t.Errorf("must not disconnect a share this process did not connect, ran: %v", runner.calls)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := c.Detach(context.Background(), Attachment{share: `\\nas\dci`, Owned: true}); err != nil {
|
|
||||||
t.Fatalf("Detach failed: %v", err)
|
|
||||||
}
|
|
||||||
if want := `net use \\nas\dci /delete`; runner.lastCall() != want {
|
|
||||||
t.Errorf("command = %q, want %q", runner.lastCall(), want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeMountTable stages a /proc/mounts-format file.
|
|
||||||
func writeMountTable(t *testing.T, lines ...string) string {
|
|
||||||
t.Helper()
|
|
||||||
path := filepath.Join(t.TempDir(), "mounts")
|
|
||||||
body := strings.Join(lines, "\n") + "\n"
|
|
||||||
if err := os.WriteFile(path, []byte(body), 0644); err != nil {
|
|
||||||
t.Fatalf("failed to write mount table: %v", err)
|
|
||||||
}
|
|
||||||
return path
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLinuxConnectorResolvesExistingMount(t *testing.T) {
|
|
||||||
mounts := writeMountTable(t,
|
|
||||||
"sysfs /sys sysfs rw,nosuid 0 0",
|
|
||||||
"//homelabnas/dci /mnt/dci cifs rw,relatime,username=nasadmin 0 0",
|
|
||||||
)
|
|
||||||
c := &linuxConnector{run: (&recordingRunner{}).run, mountsPath: mounts}
|
|
||||||
|
|
||||||
att, err := c.Attach(context.Background(), remoteLocation(t, `\\HomeLabNAS\dci\streams`), NASConfig{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Attach failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Server and share names are case-insensitive on SMB, so the configured
|
|
||||||
// \\HomeLabNAS\dci must match a //homelabnas/dci mount.
|
|
||||||
if want := "/mnt/dci/streams"; att.BasePath != want {
|
|
||||||
t.Errorf("BasePath = %q, want %q", att.BasePath, want)
|
|
||||||
}
|
|
||||||
if att.Owned {
|
|
||||||
t.Error("a pre-existing mount must not be owned, or shutdown would unmount it")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLinuxConnectorPrefersDeepestMount(t *testing.T) {
|
|
||||||
// Both mounts could serve the destination; the more specific one is the
|
|
||||||
// only one guaranteed to actually contain it.
|
|
||||||
mounts := writeMountTable(t,
|
|
||||||
"//nas/dci /mnt/dci cifs rw 0 0",
|
|
||||||
"//nas/dci/streams /mnt/streams cifs rw 0 0",
|
|
||||||
)
|
|
||||||
c := &linuxConnector{run: (&recordingRunner{}).run, mountsPath: mounts}
|
|
||||||
|
|
||||||
att, err := c.Attach(context.Background(), remoteLocation(t, `\\nas\dci\streams\2026`), NASConfig{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Attach failed: %v", err)
|
|
||||||
}
|
|
||||||
if want := "/mnt/streams/2026"; att.BasePath != want {
|
|
||||||
t.Errorf("BasePath = %q, want %q", att.BasePath, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLinuxConnectorMatchesNFSAndEscapedPoints(t *testing.T) {
|
|
||||||
mounts := writeMountTable(t,
|
|
||||||
`nas:/dci /mnt/my\040share nfs4 rw 0 0`,
|
|
||||||
)
|
|
||||||
c := &linuxConnector{run: (&recordingRunner{}).run, mountsPath: mounts}
|
|
||||||
|
|
||||||
att, err := c.Attach(context.Background(), remoteLocation(t, `\\nas\dci\streams`), NASConfig{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Attach failed: %v", err)
|
|
||||||
}
|
|
||||||
if want := "/mnt/my share/streams"; att.BasePath != want {
|
|
||||||
t.Errorf("BasePath = %q, want %q", att.BasePath, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLinuxConnectorIgnoresNonNetworkMounts(t *testing.T) {
|
|
||||||
// A local directory that merely looks like the share must not match.
|
|
||||||
mounts := writeMountTable(t, "//nas/dci /mnt/dci ext4 rw 0 0")
|
|
||||||
c := &linuxConnector{run: (&recordingRunner{}).run, mountsPath: mounts}
|
|
||||||
|
|
||||||
if _, err := c.Attach(context.Background(), remoteLocation(t, `\\nas\dci`), NASConfig{}); err == nil {
|
|
||||||
t.Fatal("expected an ext4 mount not to satisfy a network share")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLinuxConnectorUnmountedGivesActionableError(t *testing.T) {
|
|
||||||
mounts := writeMountTable(t, "sysfs /sys sysfs rw 0 0")
|
|
||||||
c := &linuxConnector{run: (&recordingRunner{}).run, mountsPath: mounts}
|
|
||||||
|
|
||||||
_, err := c.Attach(context.Background(), remoteLocation(t, `\\nas\dci\streams`), NASConfig{Username: "admin"})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected an unmounted share to fail")
|
|
||||||
}
|
|
||||||
|
|
||||||
msg := err.Error()
|
|
||||||
for _, want := range []string{"//nas/dci", "mount -t cifs", "NAS_AUTO_MOUNT", "NAS_OUTPUT_PATH"} {
|
|
||||||
if !strings.Contains(msg, want) {
|
|
||||||
t.Errorf("error should mention %q:\n%s", want, msg)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// The remedy must not echo the real password.
|
|
||||||
if strings.Contains(msg, "hunter2") {
|
|
||||||
t.Error("error message leaked a credential")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLinuxConnectorAutoMountRequiresRoot(t *testing.T) {
|
|
||||||
mounts := writeMountTable(t, "sysfs /sys sysfs rw 0 0")
|
|
||||||
runner := &recordingRunner{}
|
|
||||||
c := &linuxConnector{run: runner.run, mountsPath: mounts, geteuid: func() int { return 1000 }}
|
|
||||||
|
|
||||||
_, err := c.Attach(context.Background(), remoteLocation(t, `\\nas\dci`), NASConfig{AutoMount: true})
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "requires root") {
|
|
||||||
t.Fatalf("expected a root requirement error, got: %v", err)
|
|
||||||
}
|
|
||||||
if len(runner.calls) != 0 {
|
|
||||||
t.Errorf("must not attempt mount without privileges, ran: %v", runner.calls)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLinuxConnectorAutoMount(t *testing.T) {
|
|
||||||
mounts := writeMountTable(t, "sysfs /sys sysfs rw 0 0")
|
|
||||||
runner := &recordingRunner{}
|
|
||||||
point := filepath.Join(t.TempDir(), "mnt", "dci")
|
|
||||||
c := &linuxConnector{run: runner.run, mountsPath: mounts, geteuid: func() int { return 0 }}
|
|
||||||
|
|
||||||
cfg := NASConfig{AutoMount: true, MountPoint: point, Username: "admin", Password: "hunter2", Timeout: 5 * time.Second}
|
|
||||||
att, err := c.Attach(context.Background(), remoteLocation(t, `\\nas\dci\streams`), cfg)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Attach failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if want := filepath.Join(point, "streams"); att.BasePath != want {
|
|
||||||
t.Errorf("BasePath = %q, want %q", att.BasePath, want)
|
|
||||||
}
|
|
||||||
if !att.Owned || att.mountPoint != point {
|
|
||||||
t.Errorf("a self-created mount must be owned and recorded, got owned=%v point=%q", att.Owned, att.mountPoint)
|
|
||||||
}
|
|
||||||
|
|
||||||
call := runner.lastCall()
|
|
||||||
if !strings.HasPrefix(call, "mount -t cifs //nas/dci "+point+" -o rw,credentials=") {
|
|
||||||
t.Errorf("unexpected mount command: %q", call)
|
|
||||||
}
|
|
||||||
// The password must reach mount.cifs through a credentials file, never
|
|
||||||
// through argv where any local user could read it from the process table.
|
|
||||||
if strings.Contains(call, "hunter2") {
|
|
||||||
t.Errorf("password leaked into the mount command line: %q", call)
|
|
||||||
}
|
|
||||||
// The staged credentials file must not outlive the mount call.
|
|
||||||
credPath := call[strings.Index(call, "credentials=")+len("credentials="):]
|
|
||||||
if _, err := os.Stat(credPath); !os.IsNotExist(err) {
|
|
||||||
t.Errorf("credentials file %s should have been removed, stat err = %v", credPath, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := c.Detach(context.Background(), att); err != nil {
|
|
||||||
t.Fatalf("Detach failed: %v", err)
|
|
||||||
}
|
|
||||||
if want := "umount " + point; runner.lastCall() != want {
|
|
||||||
t.Errorf("detach command = %q, want %q", runner.lastCall(), want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLinuxConnectorDetachLeavesForeignMounts(t *testing.T) {
|
|
||||||
runner := &recordingRunner{}
|
|
||||||
c := &linuxConnector{run: runner.run, mountsPath: writeMountTable(t)}
|
|
||||||
|
|
||||||
if err := c.Detach(context.Background(), Attachment{BasePath: "/mnt/dci", Owned: false}); err != nil {
|
|
||||||
t.Fatalf("Detach failed: %v", err)
|
|
||||||
}
|
|
||||||
if len(runner.calls) != 0 {
|
|
||||||
t.Errorf("must not unmount a share the host mounted, ran: %v", runner.calls)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLocalConnectorRejectsWindowsPathsOffWindows(t *testing.T) {
|
|
||||||
c := &localConnector{goos: "linux"}
|
|
||||||
|
|
||||||
for _, raw := range []string{`C:\data\streams`, `data\streams`} {
|
|
||||||
loc, err := ParseLocation(raw)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ParseLocation(%q) failed: %v", raw, err)
|
|
||||||
}
|
|
||||||
if _, err := c.Attach(context.Background(), loc, NASConfig{}); err == nil {
|
|
||||||
t.Errorf("%q should be rejected on Linux: backslashes are legal filename characters there, "+
|
|
||||||
"so it would silently create a local directory instead of writing to the NAS", raw)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLocalConnectorAcceptsWindowsPathsOnWindows(t *testing.T) {
|
|
||||||
c := &localConnector{goos: "windows"}
|
|
||||||
loc, err := ParseLocation(`C:\data\streams`)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ParseLocation failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, err := c.Attach(context.Background(), loc, NASConfig{}); err != nil {
|
|
||||||
t.Errorf("a drive path is valid on Windows: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestUnsupportedConnectorExplainsItself(t *testing.T) {
|
|
||||||
c := &unsupportedConnector{goos: "plan9"}
|
|
||||||
_, err := c.Attach(context.Background(), remoteLocation(t, `\\nas\dci`), NASConfig{})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected an error on an unsupported platform")
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), "plan9") || !strings.Contains(err.Error(), "NAS_OUTPUT_PATH") {
|
|
||||||
t.Errorf("error should name the platform and the remedy: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,144 +0,0 @@
|
|||||||
package nas
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"path/filepath"
|
|
||||||
"regexp"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Location is a configured NAS destination parsed into a host-neutral form.
|
|
||||||
//
|
|
||||||
// The same destination is spelled differently depending on where you're
|
|
||||||
// standing: Windows wants \\HomeLabNAS\dci\streams, a Linux host reaches the
|
|
||||||
// same bytes through a mount point like /mnt/dci/streams, and either host may
|
|
||||||
// simply be handed a plain local directory. Parsing happens once, up front,
|
|
||||||
// and platform-specific rendering is left to the Connector implementations.
|
|
||||||
type Location struct {
|
|
||||||
// Raw is the destination exactly as configured, for error messages.
|
|
||||||
Raw string
|
|
||||||
// Server is the host serving the share; empty for a plain local path.
|
|
||||||
Server string
|
|
||||||
// Share is the top-level export name; empty for a plain local path.
|
|
||||||
Share string
|
|
||||||
// Sub holds the path components below the share, if any.
|
|
||||||
Sub []string
|
|
||||||
}
|
|
||||||
|
|
||||||
// driveLetter matches a Windows drive-qualified path such as C:\data.
|
|
||||||
var driveLetter = regexp.MustCompile(`^[A-Za-z]:[\\/]`)
|
|
||||||
|
|
||||||
// ParseLocation accepts every spelling of a destination this project supports:
|
|
||||||
// Windows UNC (\\server\share\dir), the SMB source form (//server/share/dir),
|
|
||||||
// a smb:// URL, or a plain local/mounted directory. It does no I/O and makes no
|
|
||||||
// assumptions about the host OS, so the result is safe to compute anywhere.
|
|
||||||
func ParseLocation(raw string) (Location, error) {
|
|
||||||
trimmed := strings.TrimSpace(raw)
|
|
||||||
if trimmed == "" {
|
|
||||||
return Location{}, fmt.Errorf("NAS path is empty")
|
|
||||||
}
|
|
||||||
|
|
||||||
loc := Location{Raw: trimmed}
|
|
||||||
|
|
||||||
remainder, isRemote := stripRemotePrefix(trimmed)
|
|
||||||
if !isRemote {
|
|
||||||
return loc, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
parts := splitPathComponents(remainder)
|
|
||||||
if len(parts) < 2 {
|
|
||||||
return Location{}, fmt.Errorf(
|
|
||||||
"%q names a server but no share; expected a form like \\\\server\\share\\folder", trimmed)
|
|
||||||
}
|
|
||||||
|
|
||||||
loc.Server = parts[0]
|
|
||||||
loc.Share = parts[1]
|
|
||||||
loc.Sub = parts[2:]
|
|
||||||
return loc, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// stripRemotePrefix reports whether raw names a network share and returns the
|
|
||||||
// server/share/subpath portion when it does. A leading // is treated as a share
|
|
||||||
// rather than a local path: POSIX leaves paths beginning with exactly two
|
|
||||||
// slashes implementation-defined, and nobody configures a NAS destination that
|
|
||||||
// way by accident.
|
|
||||||
func stripRemotePrefix(raw string) (string, bool) {
|
|
||||||
lower := strings.ToLower(raw)
|
|
||||||
switch {
|
|
||||||
case strings.HasPrefix(lower, "smb://"):
|
|
||||||
return raw[len("smb://"):], true
|
|
||||||
case strings.HasPrefix(lower, "cifs://"):
|
|
||||||
return raw[len("cifs://"):], true
|
|
||||||
case strings.HasPrefix(raw, `\\`):
|
|
||||||
return raw[2:], true
|
|
||||||
case strings.HasPrefix(raw, "//"):
|
|
||||||
return raw[2:], true
|
|
||||||
default:
|
|
||||||
return raw, false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// splitPathComponents splits on either separator and drops empty components, so
|
|
||||||
// that \\server\share\\folder\ and //server/share/folder agree.
|
|
||||||
func splitPathComponents(path string) []string {
|
|
||||||
fields := strings.FieldsFunc(path, func(r rune) bool { return r == '/' || r == '\\' })
|
|
||||||
out := make([]string, 0, len(fields))
|
|
||||||
for _, f := range fields {
|
|
||||||
if f != "" {
|
|
||||||
out = append(out, f)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// IsRemote reports whether reaching this destination requires attaching a
|
|
||||||
// network share first, as opposed to writing to a directory that is already
|
|
||||||
// part of the local filesystem.
|
|
||||||
func (l Location) IsRemote() bool {
|
|
||||||
return l.Server != "" && l.Share != ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// UNC renders the destination in Windows form: \\server\share\sub.
|
|
||||||
func (l Location) UNC() string {
|
|
||||||
if !l.IsRemote() {
|
|
||||||
return l.Raw
|
|
||||||
}
|
|
||||||
return `\\` + strings.Join(append([]string{l.Server, l.Share}, l.Sub...), `\`)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ShareUNC renders just the share in Windows form: \\server\share. This is the
|
|
||||||
// unit `net use` authenticates against; the subpath below it is an ordinary
|
|
||||||
// directory on the connected share.
|
|
||||||
func (l Location) ShareUNC() string {
|
|
||||||
if !l.IsRemote() {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return `\\` + l.Server + `\` + l.Share
|
|
||||||
}
|
|
||||||
|
|
||||||
// SMBSource renders the share in the //server/share form used by mount tables
|
|
||||||
// and by mount.cifs. It deliberately omits Sub: that part is a path *within*
|
|
||||||
// the mounted share, not part of the share's identity.
|
|
||||||
func (l Location) SMBSource() string {
|
|
||||||
if !l.IsRemote() {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return "//" + l.Server + "/" + l.Share
|
|
||||||
}
|
|
||||||
|
|
||||||
// SubPath renders the components below the share using the host's separator, or
|
|
||||||
// "" when the destination is the share root.
|
|
||||||
func (l Location) SubPath() string {
|
|
||||||
if len(l.Sub) == 0 {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return filepath.Join(l.Sub...)
|
|
||||||
}
|
|
||||||
|
|
||||||
// String describes the destination for logs and check output.
|
|
||||||
func (l Location) String() string {
|
|
||||||
if l.IsRemote() {
|
|
||||||
return l.UNC()
|
|
||||||
}
|
|
||||||
return l.Raw
|
|
||||||
}
|
|
||||||
@ -1,123 +0,0 @@
|
|||||||
package nas
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestParseLocationRemoteForms(t *testing.T) {
|
|
||||||
// Every spelling of the same destination must parse identically, so that
|
|
||||||
// an operator can configure it the way their platform's documentation
|
|
||||||
// describes without changing behaviour.
|
|
||||||
equivalent := []string{
|
|
||||||
`\\HomeLabNAS\dci\streams`,
|
|
||||||
`//HomeLabNAS/dci/streams`,
|
|
||||||
`smb://HomeLabNAS/dci/streams`,
|
|
||||||
`SMB://HomeLabNAS/dci/streams`,
|
|
||||||
`cifs://HomeLabNAS/dci/streams`,
|
|
||||||
`\\HomeLabNAS\dci\streams\`,
|
|
||||||
`\\HomeLabNAS\\dci\\streams`,
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, raw := range equivalent {
|
|
||||||
loc, err := ParseLocation(raw)
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("ParseLocation(%q) failed: %v", raw, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if !loc.IsRemote() {
|
|
||||||
t.Errorf("ParseLocation(%q) should be remote", raw)
|
|
||||||
}
|
|
||||||
if loc.Server != "HomeLabNAS" || loc.Share != "dci" {
|
|
||||||
t.Errorf("ParseLocation(%q) = server %q share %q, want HomeLabNAS/dci", raw, loc.Server, loc.Share)
|
|
||||||
}
|
|
||||||
if len(loc.Sub) != 1 || loc.Sub[0] != "streams" {
|
|
||||||
t.Errorf("ParseLocation(%q) sub = %v, want [streams]", raw, loc.Sub)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseLocationRendering(t *testing.T) {
|
|
||||||
loc, err := ParseLocation(`//HomeLabNAS/dci/streams/2026`)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got, want := loc.UNC(), `\\HomeLabNAS\dci\streams\2026`; got != want {
|
|
||||||
t.Errorf("UNC() = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
if got, want := loc.ShareUNC(), `\\HomeLabNAS\dci`; got != want {
|
|
||||||
t.Errorf("ShareUNC() = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
if got, want := loc.SMBSource(), "//HomeLabNAS/dci"; got != want {
|
|
||||||
t.Errorf("SMBSource() = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseLocationShareRoot(t *testing.T) {
|
|
||||||
loc, err := ParseLocation(`\\nas\media`)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
if len(loc.Sub) != 0 {
|
|
||||||
t.Errorf("expected no subpath, got %v", loc.Sub)
|
|
||||||
}
|
|
||||||
if got, want := loc.UNC(), `\\nas\media`; got != want {
|
|
||||||
t.Errorf("UNC() = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseLocationLocalPaths(t *testing.T) {
|
|
||||||
for _, raw := range []string{"/mnt/nas/streams", "data/streams", `C:\data\streams`, "/mnt"} {
|
|
||||||
loc, err := ParseLocation(raw)
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("ParseLocation(%q) failed: %v", raw, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if loc.IsRemote() {
|
|
||||||
t.Errorf("ParseLocation(%q) should not be treated as a network share", raw)
|
|
||||||
}
|
|
||||||
if loc.Raw != raw {
|
|
||||||
t.Errorf("ParseLocation(%q).Raw = %q", raw, loc.Raw)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseLocationRejectsIncomplete(t *testing.T) {
|
|
||||||
for _, raw := range []string{"", " ", `\\`, `\\server`, "//server", "smb://server"} {
|
|
||||||
if _, err := ParseLocation(raw); err == nil {
|
|
||||||
t.Errorf("ParseLocation(%q) should have failed", raw)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseLocationTrimsWhitespace(t *testing.T) {
|
|
||||||
loc, err := ParseLocation(" \\\\nas\\dci ")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
if loc.Server != "nas" || loc.Share != "dci" {
|
|
||||||
t.Errorf("whitespace not trimmed: server %q share %q", loc.Server, loc.Share)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestJoinBaseKeepsUNCSeparators(t *testing.T) {
|
|
||||||
// filepath.Join would produce \\nas\dci/event on Linux, which is not a
|
|
||||||
// path Windows would accept if the value were ever written to state.
|
|
||||||
got := joinBase(`\\nas\dci`, "event", "1080p")
|
|
||||||
if want := `\\nas\dci\event\1080p`; got != want {
|
|
||||||
t.Errorf("joinBase UNC = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got, want := joinBase(`\\nas\dci\`, "event"), `\\nas\dci\event`; got != want {
|
|
||||||
t.Errorf("joinBase trailing separator = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got := joinBase("/mnt/dci"); got != "/mnt/dci" {
|
|
||||||
t.Errorf("joinBase with no elements = %q", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got, want := joinBase("/mnt/dci", "event"), "/mnt/dci/event"; !strings.HasSuffix(got, want) {
|
|
||||||
t.Errorf("joinBase local = %q, want suffix %q", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,135 +0,0 @@
|
|||||||
package nas
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bufio"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
// procMountsPath is the kernel's view of what is mounted in this process's
|
|
||||||
// mount namespace, which is the right question to ask inside a container too.
|
|
||||||
const procMountsPath = "/proc/mounts"
|
|
||||||
|
|
||||||
// mountEntry is one line of the mount table.
|
|
||||||
type mountEntry struct {
|
|
||||||
Source string
|
|
||||||
Point string
|
|
||||||
FSType string
|
|
||||||
}
|
|
||||||
|
|
||||||
// networkFilesystems are the mount types that can plausibly be serving a NAS
|
|
||||||
// share. Restricting the search avoids matching, say, a local directory that
|
|
||||||
// merely happens to be named like a server.
|
|
||||||
var networkFilesystems = map[string]bool{
|
|
||||||
"cifs": true,
|
|
||||||
"smb3": true,
|
|
||||||
"smbfs": true,
|
|
||||||
"nfs": true,
|
|
||||||
"nfs4": true,
|
|
||||||
"fuse.smbnetfs": true,
|
|
||||||
}
|
|
||||||
|
|
||||||
// readMountTable parses /proc/mounts (or any file in its format). Fields are
|
|
||||||
// whitespace-separated: source, mount point, type, options, dump, pass.
|
|
||||||
func readMountTable(path string) ([]mountEntry, error) {
|
|
||||||
f, err := os.Open(path)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to read mount table %s: %w", path, err)
|
|
||||||
}
|
|
||||||
defer f.Close()
|
|
||||||
|
|
||||||
var entries []mountEntry
|
|
||||||
scanner := bufio.NewScanner(f)
|
|
||||||
for scanner.Scan() {
|
|
||||||
fields := strings.Fields(scanner.Text())
|
|
||||||
if len(fields) < 3 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
entries = append(entries, mountEntry{
|
|
||||||
Source: unescapeMountField(fields[0]),
|
|
||||||
Point: unescapeMountField(fields[1]),
|
|
||||||
FSType: fields[2],
|
|
||||||
})
|
|
||||||
}
|
|
||||||
if err := scanner.Err(); err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to scan mount table %s: %w", path, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return entries, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// unescapeMountField decodes the octal escapes the kernel uses for characters
|
|
||||||
// that would otherwise break the whitespace-separated format — most often \040
|
|
||||||
// for a space in a mount point.
|
|
||||||
func unescapeMountField(field string) string {
|
|
||||||
if !strings.Contains(field, `\`) {
|
|
||||||
return field
|
|
||||||
}
|
|
||||||
|
|
||||||
var b strings.Builder
|
|
||||||
for i := 0; i < len(field); i++ {
|
|
||||||
if field[i] == '\\' && i+3 < len(field) {
|
|
||||||
if v, err := strconv.ParseUint(field[i+1:i+4], 8, 8); err == nil {
|
|
||||||
b.WriteByte(byte(v))
|
|
||||||
i += 3
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
b.WriteByte(field[i])
|
|
||||||
}
|
|
||||||
return b.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
// findMount locates the mount serving loc and returns its mount point plus the
|
|
||||||
// path components still to be appended below it.
|
|
||||||
//
|
|
||||||
// The search runs deepest-first because a share may be mounted at any level:
|
|
||||||
// //nas/dci might be mounted at /mnt/dci, or //nas/dci/streams mounted directly
|
|
||||||
// at /mnt/streams. Preferring the longest matching source keeps both working
|
|
||||||
// and avoids pointing at a parent that the mount does not actually cover.
|
|
||||||
func findMount(entries []mountEntry, loc Location) (point string, rest []string, ok bool) {
|
|
||||||
if !loc.IsRemote() {
|
|
||||||
return "", nil, false
|
|
||||||
}
|
|
||||||
|
|
||||||
for depth := len(loc.Sub); depth >= 0; depth-- {
|
|
||||||
for _, candidate := range mountSourceForms(loc, depth) {
|
|
||||||
for _, entry := range entries {
|
|
||||||
if !networkFilesystems[entry.FSType] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if normalizeMountSource(entry.Source) == candidate {
|
|
||||||
return entry.Point, loc.Sub[depth:], true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return "", nil, false
|
|
||||||
}
|
|
||||||
|
|
||||||
// mountSourceForms renders the share, plus its first `depth` subdirectories, in
|
|
||||||
// the spellings a mount table might use for it.
|
|
||||||
func mountSourceForms(loc Location, depth int) []string {
|
|
||||||
segments := append([]string{loc.Share}, loc.Sub[:depth]...)
|
|
||||||
tail := strings.Join(segments, "/")
|
|
||||||
|
|
||||||
return []string{
|
|
||||||
normalizeMountSource("//" + loc.Server + "/" + tail), // cifs/smb3
|
|
||||||
normalizeMountSource(loc.Server + ":/" + tail), // nfs
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// normalizeMountSource makes sources comparable: SMB servers and shares are
|
|
||||||
// case-insensitive, separators may be either slash, and a trailing slash is not
|
|
||||||
// meaningful.
|
|
||||||
func normalizeMountSource(source string) string {
|
|
||||||
s := strings.ReplaceAll(source, `\`, "/")
|
|
||||||
s = strings.ToLower(strings.TrimSpace(s))
|
|
||||||
if len(s) > 1 {
|
|
||||||
s = strings.TrimRight(s, "/")
|
|
||||||
}
|
|
||||||
return s
|
|
||||||
}
|
|
||||||
@ -1,96 +0,0 @@
|
|||||||
package nas
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestUnescapeMountField(t *testing.T) {
|
|
||||||
cases := map[string]string{
|
|
||||||
`/mnt/plain`: "/mnt/plain",
|
|
||||||
`/mnt/my\040share`: "/mnt/my share",
|
|
||||||
`/mnt/tab\011here`: "/mnt/tab\there",
|
|
||||||
`/mnt/back\134slash`: `/mnt/back\slash`,
|
|
||||||
`//nas/dci`: "//nas/dci",
|
|
||||||
`/mnt/not\9octal`: `/mnt/not\9octal`,
|
|
||||||
`/mnt/trailing\`: `/mnt/trailing\`,
|
|
||||||
}
|
|
||||||
|
|
||||||
for in, want := range cases {
|
|
||||||
if got := unescapeMountField(in); got != want {
|
|
||||||
t.Errorf("unescapeMountField(%q) = %q, want %q", in, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNormalizeMountSource(t *testing.T) {
|
|
||||||
cases := map[string]string{
|
|
||||||
"//NAS/DCI": "//nas/dci",
|
|
||||||
"//nas/dci/": "//nas/dci",
|
|
||||||
`\\nas\dci`: "//nas/dci",
|
|
||||||
" //nas/dci ": "//nas/dci",
|
|
||||||
"/": "/",
|
|
||||||
}
|
|
||||||
|
|
||||||
for in, want := range cases {
|
|
||||||
if got := normalizeMountSource(in); got != want {
|
|
||||||
t.Errorf("normalizeMountSource(%q) = %q, want %q", in, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReadMountTableSkipsMalformedLines(t *testing.T) {
|
|
||||||
path := writeMountTable(t,
|
|
||||||
"source-only",
|
|
||||||
"source /point-but-no-type",
|
|
||||||
"",
|
|
||||||
"//nas/dci /mnt/dci cifs rw 0 0",
|
|
||||||
)
|
|
||||||
|
|
||||||
entries, err := readMountTable(path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("readMountTable failed: %v", err)
|
|
||||||
}
|
|
||||||
if len(entries) != 1 {
|
|
||||||
t.Fatalf("expected 1 usable entry, got %d: %v", len(entries), entries)
|
|
||||||
}
|
|
||||||
if entries[0].Source != "//nas/dci" || entries[0].Point != "/mnt/dci" || entries[0].FSType != "cifs" {
|
|
||||||
t.Errorf("unexpected entry: %+v", entries[0])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReadMountTableMissingFile(t *testing.T) {
|
|
||||||
if _, err := readMountTable("/definitely/not/a/mount/table"); err == nil {
|
|
||||||
t.Error("expected an error for a missing mount table")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestReadRealMountTable closes the loop between the fixtures above and the
|
|
||||||
// kernel's actual output: the parser must cope with whatever this host has
|
|
||||||
// mounted, not just hand-written examples.
|
|
||||||
func TestReadRealMountTable(t *testing.T) {
|
|
||||||
if _, err := os.Stat(procMountsPath); err != nil {
|
|
||||||
t.Skipf("%s is not available on this platform", procMountsPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
entries, err := readMountTable(procMountsPath)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to parse the host's %s: %v", procMountsPath, err)
|
|
||||||
}
|
|
||||||
if len(entries) == 0 {
|
|
||||||
t.Fatalf("%s parsed to zero entries, which cannot be right", procMountsPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
foundRoot := false
|
|
||||||
for _, e := range entries {
|
|
||||||
if e.Point == "" || e.FSType == "" {
|
|
||||||
t.Errorf("entry parsed with empty fields: %+v", e)
|
|
||||||
}
|
|
||||||
if e.Point == "/" {
|
|
||||||
foundRoot = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !foundRoot {
|
|
||||||
t.Error("expected the root filesystem in the host's mount table")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
252
pkg/nas/nas.go
252
pkg/nas/nas.go
@ -6,168 +6,31 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
"sync"
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
// NASService writes to a NAS destination without caring how this host reaches
|
|
||||||
// it. Parsing the destination and attaching it are delegated to a Location and
|
|
||||||
// a Connector; everything below operates on the resolved base path, which is
|
|
||||||
// always an ordinary path this host's filesystem understands.
|
|
||||||
type NASService struct {
|
type NASService struct {
|
||||||
Config NASConfig
|
Config NASConfig
|
||||||
|
|
||||||
location Location
|
|
||||||
connector Connector
|
|
||||||
|
|
||||||
mu sync.RWMutex
|
|
||||||
attached Attachment
|
|
||||||
connected bool
|
connected bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// New parses the configured destination and selects a connection strategy for
|
func NewNASService(config NASConfig) *NASService {
|
||||||
// the running host. It performs no I/O and does not touch the network, so it is
|
nt := &NASService{
|
||||||
// safe to call before deciding whether a transfer will actually happen; call
|
|
||||||
// Connect to attach the destination.
|
|
||||||
func New(config NASConfig) (*NASService, error) {
|
|
||||||
loc, err := ParseLocation(config.Path)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return &NASService{
|
|
||||||
Config: config,
|
Config: config,
|
||||||
location: loc,
|
}
|
||||||
connector: defaultConnector(loc),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Connect attaches the destination, ensures the target directory exists, and
|
// Establish network connection with credentials before accessing the path
|
||||||
// proves it is writable. The write probe matters: on both platforms a share can
|
if err := nt.EstablishConnection(); err != nil {
|
||||||
// be reachable and still reject writes, and finding that out at startup beats
|
log.Fatalf("Failed to establish network connection to %s: %v", nt.Config.Path, err)
|
||||||
// discovering it mid-recording.
|
}
|
||||||
func (nt *NASService) Connect(ctx context.Context) error {
|
|
||||||
att, err := nt.connector.Attach(ctx, nt.location, nt.Config)
|
err := nt.EnsureDirectoryExists(nt.Config.Path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
log.Fatalf("Failed to create directory %s: %v", nt.Config.Path, err)
|
||||||
}
|
}
|
||||||
|
return nt
|
||||||
nt.mu.Lock()
|
|
||||||
nt.attached = att
|
|
||||||
nt.mu.Unlock()
|
|
||||||
|
|
||||||
// From here on the attachment exists, so any failure has to undo it —
|
|
||||||
// otherwise a mount this process created would outlive the failed startup
|
|
||||||
// and be left behind on the host.
|
|
||||||
if err := nt.EnsureDirectoryExists(att.BasePath); err != nil {
|
|
||||||
nt.abandon(ctx, att)
|
|
||||||
return fmt.Errorf("%s is reachable but %s is unusable: %w", nt.location, att.BasePath, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := nt.writeProbe(att.BasePath); err != nil {
|
|
||||||
nt.abandon(ctx, att)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
nt.mu.Lock()
|
|
||||||
nt.connected = true
|
|
||||||
nt.mu.Unlock()
|
|
||||||
|
|
||||||
log.Printf("Connected to NAS at %s", att.BasePath)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// abandon releases a half-finished attachment after Connect fails. The
|
|
||||||
// original error is the one worth reporting, so a teardown failure is only
|
|
||||||
// logged.
|
|
||||||
func (nt *NASService) abandon(ctx context.Context, att Attachment) {
|
|
||||||
// Keep the notes: they explain how far the host got, which is exactly what
|
|
||||||
// the caller needs to report. Everything else is cleared so the service
|
|
||||||
// cannot be mistaken for usable and Detach cannot run twice.
|
|
||||||
nt.mu.Lock()
|
|
||||||
nt.attached = Attachment{Notes: att.Notes}
|
|
||||||
nt.connected = false
|
|
||||||
nt.mu.Unlock()
|
|
||||||
|
|
||||||
if err := nt.connector.Detach(ctx, att); err != nil {
|
|
||||||
log.Printf("Warning: failed to release NAS attachment after a failed connect: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeProbe creates and removes a file to confirm the destination accepts
|
|
||||||
// writes.
|
|
||||||
func (nt *NASService) writeProbe(basePath string) error {
|
|
||||||
testFile := joinBase(basePath, ".connection_test")
|
|
||||||
|
|
||||||
f, err := os.Create(testFile)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("%s is not writable: %w", basePath, err)
|
|
||||||
}
|
|
||||||
f.Close()
|
|
||||||
|
|
||||||
if err := os.Remove(testFile); err != nil {
|
|
||||||
log.Printf("Warning: failed to remove connection test file %s: %v", testFile, err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Disconnect releases only what Connect established. A share that was already
|
|
||||||
// mounted or already connected when the recorder started is left exactly as it
|
|
||||||
// was found.
|
|
||||||
func (nt *NASService) Disconnect(ctx context.Context) error {
|
|
||||||
nt.mu.Lock()
|
|
||||||
att := nt.attached
|
|
||||||
nt.attached = Attachment{}
|
|
||||||
nt.connected = false
|
|
||||||
nt.mu.Unlock()
|
|
||||||
|
|
||||||
if err := nt.connector.Detach(ctx, att); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if att.Owned {
|
|
||||||
log.Printf("Released NAS connection to %s", nt.location)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// LocalPath is the resolved base directory on this host, valid after Connect.
|
|
||||||
func (nt *NASService) LocalPath() string {
|
|
||||||
nt.mu.RLock()
|
|
||||||
defer nt.mu.RUnlock()
|
|
||||||
return nt.attached.BasePath
|
|
||||||
}
|
|
||||||
|
|
||||||
// Join resolves elements against the NAS base path using whatever separator
|
|
||||||
// this host's base path requires.
|
|
||||||
func (nt *NASService) Join(elem ...string) string {
|
|
||||||
return joinBase(nt.LocalPath(), elem...)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Location returns the parsed destination.
|
|
||||||
func (nt *NASService) Location() Location { return nt.location }
|
|
||||||
|
|
||||||
// ConnectionInfo describes how this host reaches the destination, for check
|
|
||||||
// output and startup logs. Safe to call before Connect, when it reports only
|
|
||||||
// what is known from configuration.
|
|
||||||
func (nt *NASService) ConnectionInfo() []string {
|
|
||||||
nt.mu.RLock()
|
|
||||||
defer nt.mu.RUnlock()
|
|
||||||
|
|
||||||
info := []string{
|
|
||||||
"configured: " + nt.location.String(),
|
|
||||||
"strategy: " + nt.connector.Name(),
|
|
||||||
}
|
|
||||||
info = append(info, nt.attached.Notes...)
|
|
||||||
if nt.attached.BasePath != "" && nt.attached.BasePath != nt.location.Raw {
|
|
||||||
info = append(info, "resolved path: "+nt.attached.BasePath)
|
|
||||||
}
|
|
||||||
return info
|
|
||||||
}
|
|
||||||
|
|
||||||
func (nt *NASService) IsConnected() bool {
|
|
||||||
nt.mu.RLock()
|
|
||||||
defer nt.mu.RUnlock()
|
|
||||||
return nt.connected
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (nt *NASService) CopyFile(ctx context.Context, srcPath, destPath string) error {
|
func (nt *NASService) CopyFile(ctx context.Context, srcPath, destPath string) error {
|
||||||
@ -226,10 +89,87 @@ func (nt *NASService) EnsureDirectoryExists(path string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// FileExists checks whether a file already exists on the NAS, optionally
|
func (nt *NASService) EstablishConnection() error {
|
||||||
// verifying its size. destinationPath is relative to the NAS base path.
|
networkPath := nt.ExtractNetworkPath(nt.Config.Path)
|
||||||
|
if networkPath == "" {
|
||||||
|
return nil // local path, no network mount needed
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("Establishing network connection to %s with user %s", networkPath, nt.Config.Username)
|
||||||
|
|
||||||
|
var cmd *exec.Cmd
|
||||||
|
if nt.Config.Username != "" && nt.Config.Password != "" {
|
||||||
|
cmd = exec.Command("net", "use", networkPath, "/user:"+nt.Config.Username, nt.Config.Password, "/persistent:no")
|
||||||
|
} else {
|
||||||
|
cmd = exec.Command("net", "use", networkPath, "/persistent:no")
|
||||||
|
}
|
||||||
|
|
||||||
|
output, err := cmd.CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to establish network connection: %w\nOutput: %s", err, string(output))
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("Network connection established successfully")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (nt *NASService) ExtractNetworkPath(fullPath string) string {
|
||||||
|
// Extract \\server\share from paths like \\server\share\folder\subfolder
|
||||||
|
if !strings.HasPrefix(fullPath, "\\\\") {
|
||||||
|
return "" // Not a UNC path
|
||||||
|
}
|
||||||
|
|
||||||
|
parts := strings.Split(fullPath[2:], "\\") // Remove leading \\
|
||||||
|
if len(parts) < 2 {
|
||||||
|
return "" // Invalid UNC path
|
||||||
|
}
|
||||||
|
|
||||||
|
return "\\\\" + parts[0] + "\\" + parts[1]
|
||||||
|
}
|
||||||
|
|
||||||
|
func (nt *NASService) TestConnection() error {
|
||||||
|
testFile := filepath.Join(nt.Config.Path, ".connection_test")
|
||||||
|
|
||||||
|
f, err := os.Create(testFile)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Failed to create test file: %w", err)
|
||||||
|
}
|
||||||
|
f.Close()
|
||||||
|
|
||||||
|
os.Remove(testFile)
|
||||||
|
|
||||||
|
nt.connected = true
|
||||||
|
log.Printf("Connected to NAS at %s", nt.Config.Path)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (nt *NASService) IsConnected() bool {
|
||||||
|
return nt.connected
|
||||||
|
}
|
||||||
|
|
||||||
|
// Disconnect removes the network connection
|
||||||
|
func (nt *NASService) Disconnect() error {
|
||||||
|
networkPath := nt.ExtractNetworkPath(nt.Config.Path)
|
||||||
|
if networkPath == "" {
|
||||||
|
return nil // Local path, nothing to disconnect
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd := exec.Command("net", "use", networkPath, "/delete")
|
||||||
|
output, err := cmd.CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("Warning: failed to disconnect from %s: %v\nOutput: %s", networkPath, err, string(output))
|
||||||
|
// Don't return error since this is cleanup
|
||||||
|
} else {
|
||||||
|
log.Printf("Disconnected from network path: %s", networkPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
nt.connected = false
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// FileExists checks if a file already exists on the NAS and optionally verifies size
|
||||||
func (nt *NASService) FileExists(destinationPath string, expectedSize int64) (bool, error) {
|
func (nt *NASService) FileExists(destinationPath string, expectedSize int64) (bool, error) {
|
||||||
fullDestPath := nt.Join(destinationPath)
|
fullDestPath := filepath.Join(nt.Config.Path, destinationPath)
|
||||||
|
|
||||||
destInfo, err := os.Stat(fullDestPath)
|
destInfo, err := os.Stat(fullDestPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -249,9 +189,9 @@ func (nt *NASService) FileExists(destinationPath string, expectedSize int64) (bo
|
|||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetFileSize returns the size of a file on the NAS, relative to the base path.
|
// GetFileSize returns the size of a file on the NAS
|
||||||
func (nt *NASService) GetFileSize(destinationPath string) (int64, error) {
|
func (nt *NASService) GetFileSize(destinationPath string) (int64, error) {
|
||||||
fullDestPath := nt.Join(destinationPath)
|
fullDestPath := filepath.Join(nt.Config.Path, destinationPath)
|
||||||
|
|
||||||
destInfo, err := os.Stat(fullDestPath)
|
destInfo, err := os.Stat(fullDestPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@ -1,308 +0,0 @@
|
|||||||
package nas
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
// fakeConnector stands in for a platform strategy so the service's own
|
|
||||||
// behaviour — probing, path resolution, teardown — can be tested without a NAS.
|
|
||||||
type fakeConnector struct {
|
|
||||||
basePath string
|
|
||||||
attachErr error
|
|
||||||
owned bool
|
|
||||||
attachCount int
|
|
||||||
detachCount int
|
|
||||||
detached Attachment
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *fakeConnector) Name() string { return "fake" }
|
|
||||||
|
|
||||||
func (f *fakeConnector) Attach(context.Context, Location, NASConfig) (Attachment, error) {
|
|
||||||
f.attachCount++
|
|
||||||
if f.attachErr != nil {
|
|
||||||
return Attachment{}, f.attachErr
|
|
||||||
}
|
|
||||||
return Attachment{BasePath: f.basePath, Owned: f.owned, Notes: []string{"fake attachment"}}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *fakeConnector) Detach(_ context.Context, att Attachment) error {
|
|
||||||
f.detachCount++
|
|
||||||
f.detached = att
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func newTestService(t *testing.T, configuredPath string, conn Connector) *NASService {
|
|
||||||
t.Helper()
|
|
||||||
svc, err := New(NASConfig{Path: configuredPath})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("New(%q) failed: %v", configuredPath, err)
|
|
||||||
}
|
|
||||||
svc.connector = conn
|
|
||||||
return svc
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewRejectsUnusableDestination(t *testing.T) {
|
|
||||||
if _, err := New(NASConfig{Path: ""}); err == nil {
|
|
||||||
t.Error("an empty NAS path should be rejected")
|
|
||||||
}
|
|
||||||
if _, err := New(NASConfig{Path: `\\server`}); err == nil {
|
|
||||||
t.Error("a server without a share should be rejected")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewDoesNoIO(t *testing.T) {
|
|
||||||
// Constructing the service must not touch the network or filesystem, so
|
|
||||||
// that modes which never transfer can build one harmlessly.
|
|
||||||
svc, err := New(NASConfig{Path: `\\nas\dci\streams`})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("New failed: %v", err)
|
|
||||||
}
|
|
||||||
if svc.IsConnected() {
|
|
||||||
t.Error("service should not report connected before Connect")
|
|
||||||
}
|
|
||||||
if svc.LocalPath() != "" {
|
|
||||||
t.Errorf("no path should be resolved before Connect, got %q", svc.LocalPath())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConnectResolvesAndProbes(t *testing.T) {
|
|
||||||
base := filepath.Join(t.TempDir(), "streams")
|
|
||||||
conn := &fakeConnector{basePath: base}
|
|
||||||
svc := newTestService(t, `\\nas\dci\streams`, conn)
|
|
||||||
|
|
||||||
if err := svc.Connect(context.Background()); err != nil {
|
|
||||||
t.Fatalf("Connect failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !svc.IsConnected() {
|
|
||||||
t.Error("service should report connected")
|
|
||||||
}
|
|
||||||
if svc.LocalPath() != base {
|
|
||||||
t.Errorf("LocalPath = %q, want %q", svc.LocalPath(), base)
|
|
||||||
}
|
|
||||||
// Connect creates the destination directory if it is missing.
|
|
||||||
if info, err := os.Stat(base); err != nil || !info.IsDir() {
|
|
||||||
t.Errorf("Connect should have created %s: %v", base, err)
|
|
||||||
}
|
|
||||||
// The write probe must clean up after itself.
|
|
||||||
if _, err := os.Stat(filepath.Join(base, ".connection_test")); !os.IsNotExist(err) {
|
|
||||||
t.Errorf("probe file should have been removed, stat err = %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConnectSurfacesAttachFailure(t *testing.T) {
|
|
||||||
conn := &fakeConnector{attachErr: fmt.Errorf("share is not mounted")}
|
|
||||||
svc := newTestService(t, `\\nas\dci`, conn)
|
|
||||||
|
|
||||||
err := svc.Connect(context.Background())
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "not mounted") {
|
|
||||||
t.Fatalf("expected the connector's error to surface, got: %v", err)
|
|
||||||
}
|
|
||||||
if svc.IsConnected() {
|
|
||||||
t.Error("a failed attach must not leave the service marked connected")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConnectFailsWhenDestinationUnwritable(t *testing.T) {
|
|
||||||
// A file where the directory should be: reachable, but unusable.
|
|
||||||
blocker := filepath.Join(t.TempDir(), "blocker")
|
|
||||||
if err := os.WriteFile(blocker, []byte("x"), 0644); err != nil {
|
|
||||||
t.Fatalf("setup failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
svc := newTestService(t, "/tmp/whatever", &fakeConnector{basePath: filepath.Join(blocker, "streams")})
|
|
||||||
if err := svc.Connect(context.Background()); err == nil {
|
|
||||||
t.Fatal("expected an unusable destination to fail the connect")
|
|
||||||
}
|
|
||||||
if svc.IsConnected() {
|
|
||||||
t.Error("service should not report connected after a failed probe")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFailedProbeReleasesTheAttachment(t *testing.T) {
|
|
||||||
// A mount created by Attach must not outlive a failed Connect, or a failed
|
|
||||||
// startup leaves a mount behind on the host.
|
|
||||||
blocker := filepath.Join(t.TempDir(), "blocker")
|
|
||||||
if err := os.WriteFile(blocker, []byte("x"), 0644); err != nil {
|
|
||||||
t.Fatalf("setup failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
conn := &fakeConnector{basePath: filepath.Join(blocker, "streams"), owned: true}
|
|
||||||
svc := newTestService(t, `\\nas\dci`, conn)
|
|
||||||
|
|
||||||
if err := svc.Connect(context.Background()); err == nil {
|
|
||||||
t.Fatal("expected Connect to fail")
|
|
||||||
}
|
|
||||||
|
|
||||||
if conn.detachCount != 1 {
|
|
||||||
t.Errorf("Detach called %d times after a failed connect, want 1", conn.detachCount)
|
|
||||||
}
|
|
||||||
if !conn.detached.Owned {
|
|
||||||
t.Error("Detach should have been told the attachment was owned")
|
|
||||||
}
|
|
||||||
if svc.LocalPath() != "" {
|
|
||||||
t.Errorf("a released attachment should expose no path, got %q", svc.LocalPath())
|
|
||||||
}
|
|
||||||
// The diagnosis of how far it got must survive for the caller to report.
|
|
||||||
if info := strings.Join(svc.ConnectionInfo(), "\n"); !strings.Contains(info, "fake attachment") {
|
|
||||||
t.Errorf("connection notes should survive a failed connect:\n%s", info)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A later Disconnect must not detach a second time.
|
|
||||||
if err := svc.Disconnect(context.Background()); err != nil {
|
|
||||||
t.Fatalf("Disconnect failed: %v", err)
|
|
||||||
}
|
|
||||||
if conn.detachCount != 2 || conn.detached.Owned {
|
|
||||||
t.Errorf("Disconnect after release should be a no-op teardown, detachCount=%d owned=%v",
|
|
||||||
conn.detachCount, conn.detached.Owned)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestJoinUsesResolvedPath(t *testing.T) {
|
|
||||||
base := t.TempDir()
|
|
||||||
svc := newTestService(t, `\\nas\dci\streams`, &fakeConnector{basePath: base})
|
|
||||||
if err := svc.Connect(context.Background()); err != nil {
|
|
||||||
t.Fatalf("Connect failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Crucially the join is against the resolved path, not the configured UNC
|
|
||||||
// path — that is the whole point of the indirection.
|
|
||||||
if got, want := svc.Join("event", "1080p"), filepath.Join(base, "event", "1080p"); got != want {
|
|
||||||
t.Errorf("Join = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFileExistsAndGetFileSizeUseResolvedPath(t *testing.T) {
|
|
||||||
base := t.TempDir()
|
|
||||||
svc := newTestService(t, `\\nas\dci`, &fakeConnector{basePath: base})
|
|
||||||
if err := svc.Connect(context.Background()); err != nil {
|
|
||||||
t.Fatalf("Connect failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
rel := filepath.Join("event", "seg.ts")
|
|
||||||
if err := os.MkdirAll(filepath.Join(base, "event"), 0755); err != nil {
|
|
||||||
t.Fatalf("setup failed: %v", err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(filepath.Join(base, rel), []byte("0123456789"), 0644); err != nil {
|
|
||||||
t.Fatalf("setup failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
exists, err := svc.FileExists(rel, 10)
|
|
||||||
if err != nil || !exists {
|
|
||||||
t.Errorf("FileExists = %v, %v; want true, nil", exists, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A size mismatch means a partial transfer, which should not count.
|
|
||||||
exists, err = svc.FileExists(rel, 99)
|
|
||||||
if err != nil || exists {
|
|
||||||
t.Errorf("FileExists with wrong size = %v, %v; want false, nil", exists, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
exists, err = svc.FileExists(filepath.Join("event", "missing.ts"), 0)
|
|
||||||
if err != nil || exists {
|
|
||||||
t.Errorf("FileExists for missing file = %v, %v; want false, nil", exists, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
size, err := svc.GetFileSize(rel)
|
|
||||||
if err != nil || size != 10 {
|
|
||||||
t.Errorf("GetFileSize = %d, %v; want 10, nil", size, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDisconnectDelegatesAndResets(t *testing.T) {
|
|
||||||
conn := &fakeConnector{basePath: t.TempDir(), owned: true}
|
|
||||||
svc := newTestService(t, `\\nas\dci`, conn)
|
|
||||||
if err := svc.Connect(context.Background()); err != nil {
|
|
||||||
t.Fatalf("Connect failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := svc.Disconnect(context.Background()); err != nil {
|
|
||||||
t.Fatalf("Disconnect failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if conn.detachCount != 1 {
|
|
||||||
t.Errorf("Detach called %d times, want 1", conn.detachCount)
|
|
||||||
}
|
|
||||||
if !conn.detached.Owned {
|
|
||||||
t.Error("Detach should receive the attachment so it knows what it owns")
|
|
||||||
}
|
|
||||||
if svc.IsConnected() {
|
|
||||||
t.Error("service should not report connected after Disconnect")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConnectionInfoDescribesResolution(t *testing.T) {
|
|
||||||
base := t.TempDir()
|
|
||||||
svc := newTestService(t, `\\nas\dci\streams`, &fakeConnector{basePath: base})
|
|
||||||
|
|
||||||
// Before connecting, only configuration is known.
|
|
||||||
before := strings.Join(svc.ConnectionInfo(), "\n")
|
|
||||||
if !strings.Contains(before, `\\nas\dci\streams`) || !strings.Contains(before, "fake") {
|
|
||||||
t.Errorf("pre-connect info should name the destination and strategy:\n%s", before)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := svc.Connect(context.Background()); err != nil {
|
|
||||||
t.Fatalf("Connect failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
after := strings.Join(svc.ConnectionInfo(), "\n")
|
|
||||||
if !strings.Contains(after, "resolved path: "+base) {
|
|
||||||
t.Errorf("post-connect info should report where the destination resolved to:\n%s", after)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestVerifyTransferDetectsSizeMismatch(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
src := filepath.Join(dir, "src.ts")
|
|
||||||
dst := filepath.Join(dir, "dst.ts")
|
|
||||||
if err := os.WriteFile(src, []byte("0123456789"), 0644); err != nil {
|
|
||||||
t.Fatalf("setup failed: %v", err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(dst, []byte("012"), 0644); err != nil {
|
|
||||||
t.Fatalf("setup failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
svc := newTestService(t, dir, &fakeConnector{basePath: dir})
|
|
||||||
if err := svc.VerifyTransfer(src, dst); err == nil {
|
|
||||||
t.Error("a truncated destination should fail verification")
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := os.WriteFile(dst, []byte("0123456789"), 0644); err != nil {
|
|
||||||
t.Fatalf("setup failed: %v", err)
|
|
||||||
}
|
|
||||||
if err := svc.VerifyTransfer(src, dst); err != nil {
|
|
||||||
t.Errorf("matching sizes should verify: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCopyFile(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
src := filepath.Join(dir, "src.ts")
|
|
||||||
dst := filepath.Join(dir, "dst.ts")
|
|
||||||
payload := []byte("segment payload")
|
|
||||||
if err := os.WriteFile(src, payload, 0644); err != nil {
|
|
||||||
t.Fatalf("setup failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
svc := newTestService(t, dir, &fakeConnector{basePath: dir})
|
|
||||||
if err := svc.CopyFile(context.Background(), src, dst); err != nil {
|
|
||||||
t.Fatalf("CopyFile failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
got, err := os.ReadFile(dst)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read copy: %v", err)
|
|
||||||
}
|
|
||||||
if string(got) != string(payload) {
|
|
||||||
t.Errorf("copied contents = %q, want %q", got, payload)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := svc.CopyFile(context.Background(), filepath.Join(dir, "missing.ts"), dst); err == nil {
|
|
||||||
t.Error("copying a missing source should fail")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -6,8 +6,6 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"m3u8-downloader/pkg/config"
|
"m3u8-downloader/pkg/config"
|
||||||
"m3u8-downloader/pkg/health"
|
|
||||||
"m3u8-downloader/pkg/logging"
|
|
||||||
"m3u8-downloader/pkg/nas"
|
"m3u8-downloader/pkg/nas"
|
||||||
"m3u8-downloader/pkg/utils"
|
"m3u8-downloader/pkg/utils"
|
||||||
"os"
|
"os"
|
||||||
@ -39,20 +37,11 @@ func NewProcessingService(eventName string, cfg *config.Config) (*ProcessingServ
|
|||||||
Timeout: cfg.NAS.Timeout,
|
Timeout: cfg.NAS.Timeout,
|
||||||
RetryLimit: cfg.NAS.RetryLimit,
|
RetryLimit: cfg.NAS.RetryLimit,
|
||||||
VerifySize: true,
|
VerifySize: true,
|
||||||
AutoMount: cfg.NAS.AutoMount,
|
|
||||||
MountPoint: cfg.NAS.MountPoint,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
nasService, err := nas.New(nasConfig)
|
nasService := nas.NewNASService(nasConfig)
|
||||||
if err != nil {
|
|
||||||
logging.MustGetFactory().Processing.Error("invalid NAS destination", logging.KeyError, err.Error())
|
|
||||||
return nil, fmt.Errorf("invalid NAS destination: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
connectCtx, cancel := context.WithTimeout(context.Background(), cfg.NAS.Timeout)
|
if err := nasService.TestConnection(); err != nil {
|
||||||
defer cancel()
|
|
||||||
if err := nasService.Connect(connectCtx); err != nil {
|
|
||||||
logging.MustGetFactory().Processing.Error("failed to connect to NAS", logging.KeyError, err.Error())
|
|
||||||
return nil, fmt.Errorf("failed to connect to NAS: %w", err)
|
return nil, fmt.Errorf("failed to connect to NAS: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -67,17 +56,9 @@ func (ps *ProcessingService) GetSegmentInfo() (map[int]string, error) {
|
|||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// nasEventPath is the event's directory on the NAS as reached from this host.
|
|
||||||
// It goes through the NAS service rather than config.GetNASEventPath because
|
|
||||||
// the configured path may be a UNC path that this host reaches via a mount
|
|
||||||
// point instead.
|
|
||||||
func (ps *ProcessingService) nasEventPath() string {
|
|
||||||
return ps.nas.Join(ps.eventName)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (ps *ProcessingService) GetEventDirs() ([]string, error) {
|
func (ps *ProcessingService) GetEventDirs() ([]string, error) {
|
||||||
if ps.eventName == "" {
|
if ps.eventName == "" {
|
||||||
sourcePath := ps.nas.LocalPath()
|
sourcePath := ps.config.NAS.OutputPath
|
||||||
dirs, err := os.ReadDir(sourcePath)
|
dirs, err := os.ReadDir(sourcePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to read directory %s: %w", sourcePath, err)
|
return nil, fmt.Errorf("failed to read directory %s: %w", sourcePath, err)
|
||||||
@ -95,32 +76,18 @@ func (ps *ProcessingService) GetEventDirs() ([]string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (ps *ProcessingService) Start(ctx context.Context) error {
|
func (ps *ProcessingService) Start(ctx context.Context) error {
|
||||||
procLog := logging.MustGetFactory().Processing
|
|
||||||
registry := health.MustGetRegistry()
|
|
||||||
|
|
||||||
if !ps.config.Processing.Enabled {
|
if !ps.config.Processing.Enabled {
|
||||||
log.Println("Processing service disabled")
|
log.Println("Processing service disabled")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
registry.RecordProcessingStageChange("scanning", ps.eventName)
|
|
||||||
|
|
||||||
// fail centralizes the log/registry bookkeeping for every error return
|
|
||||||
// site below instead of repeating it at each one.
|
|
||||||
fail := func(err error) error {
|
|
||||||
procLog.Error("processing failed", logging.KeyEvent, ps.eventName, logging.KeyError, err.Error())
|
|
||||||
registry.RecordProcessingStageChange("failed", ps.eventName)
|
|
||||||
registry.RecordProcessingError(err)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if ps.eventName == "" {
|
if ps.eventName == "" {
|
||||||
events, err := ps.GetEventDirs()
|
events, err := ps.GetEventDirs()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fail(fmt.Errorf("failed to get event directories: %w", err))
|
return fmt.Errorf("failed to get event directories: %w", err)
|
||||||
}
|
}
|
||||||
if len(events) == 0 {
|
if len(events) == 0 {
|
||||||
return fail(fmt.Errorf("no events found"))
|
return fmt.Errorf("no events found")
|
||||||
}
|
}
|
||||||
if len(events) > 1 {
|
if len(events) > 1 {
|
||||||
fmt.Println("Multiple events found, please select one:")
|
fmt.Println("Multiple events found, please select one:")
|
||||||
@ -132,23 +99,21 @@ func (ps *ProcessingService) Start(ctx context.Context) error {
|
|||||||
input = strings.TrimSpace(input)
|
input = strings.TrimSpace(input)
|
||||||
index, err := strconv.Atoi(input)
|
index, err := strconv.Atoi(input)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fail(fmt.Errorf("failed to parse input: %w", err))
|
return fmt.Errorf("failed to parse input: %w", err)
|
||||||
}
|
}
|
||||||
if index < 1 || index > len(events) {
|
if index < 1 || index > len(events) {
|
||||||
return fail(fmt.Errorf("invalid input"))
|
return fmt.Errorf("invalid input")
|
||||||
}
|
}
|
||||||
ps.eventName = events[index-1]
|
ps.eventName = events[index-1]
|
||||||
} else {
|
} else {
|
||||||
ps.eventName = events[0]
|
ps.eventName = events[0]
|
||||||
}
|
}
|
||||||
registry.RecordProcessingStageChange("scanning", ps.eventName)
|
|
||||||
}
|
}
|
||||||
procLog.Info("processing started", logging.KeyEvent, ps.eventName)
|
|
||||||
|
|
||||||
//Get all present resolutions
|
//Get all present resolutions
|
||||||
dirs, err := ps.GetResolutions()
|
dirs, err := ps.GetResolutions()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fail(fmt.Errorf("Failed to get resolutions: %w", err))
|
return fmt.Errorf("Failed to get resolutions: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
//Spawn a worker per resolution
|
//Spawn a worker per resolution
|
||||||
@ -166,34 +131,30 @@ func (ps *ProcessingService) Start(ctx context.Context) error {
|
|||||||
|
|
||||||
segments, err := ps.AggregateSegmentInfo(ch)
|
segments, err := ps.AggregateSegmentInfo(ch)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fail(fmt.Errorf("Failed to aggregate segment info: %w", err))
|
return fmt.Errorf("Failed to aggregate segment info: %w", err)
|
||||||
}
|
}
|
||||||
registry.RecordProcessingSegmentsFound(len(segments))
|
|
||||||
|
|
||||||
aggFile, err := ps.WriteConcatFile(segments)
|
aggFile, err := ps.WriteConcatFile(segments)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fail(fmt.Errorf("Failed to write concat file: %w", err))
|
return fmt.Errorf("Failed to write concat file: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Feed info to ffmpeg to stitch files together
|
// Feed info to ffmpeg to stitch files together
|
||||||
outPath := ps.config.GetProcessOutputPath(ps.eventName)
|
outPath := ps.config.GetProcessOutputPath(ps.eventName)
|
||||||
if err := utils.EnsureDir(outPath); err != nil {
|
if err := utils.EnsureDir(outPath); err != nil {
|
||||||
return fail(fmt.Errorf("failed to create output directory: %w", err))
|
return fmt.Errorf("failed to create output directory: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
registry.RecordProcessingStageChange("running", ps.eventName)
|
|
||||||
concatErr := ps.RunFFmpeg(aggFile, outPath)
|
concatErr := ps.RunFFmpeg(aggFile, outPath)
|
||||||
if concatErr != nil {
|
if concatErr != nil {
|
||||||
return fail(concatErr)
|
return concatErr
|
||||||
}
|
}
|
||||||
|
|
||||||
procLog.Info("processing completed", logging.KeyEvent, ps.eventName, "segments", len(segments))
|
|
||||||
registry.RecordProcessingStageChange("completed", ps.eventName)
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ps *ProcessingService) GetResolutions() ([]string, error) {
|
func (ps *ProcessingService) GetResolutions() ([]string, error) {
|
||||||
eventPath := ps.nasEventPath()
|
eventPath := ps.config.GetNASEventPath(ps.eventName)
|
||||||
dirs, err := os.ReadDir(eventPath)
|
dirs, err := os.ReadDir(eventPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to read source directory %s: %w", eventPath, err)
|
return nil, fmt.Errorf("failed to read source directory %s: %w", eventPath, err)
|
||||||
@ -214,7 +175,7 @@ func (ps *ProcessingService) GetResolutions() ([]string, error) {
|
|||||||
func (ps *ProcessingService) ParseResolutionDirectory(resolution string, ch chan<- SegmentInfo, wg *sync.WaitGroup) {
|
func (ps *ProcessingService) ParseResolutionDirectory(resolution string, ch chan<- SegmentInfo, wg *sync.WaitGroup) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
|
|
||||||
resolutionPath := utils.SafeJoin(ps.nasEventPath(), resolution)
|
resolutionPath := utils.SafeJoin(ps.config.GetNASEventPath(ps.eventName), resolution)
|
||||||
files, err := os.ReadDir(resolutionPath)
|
files, err := os.ReadDir(resolutionPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("Failed to read resolution directory %s: %v", resolutionPath, err)
|
log.Printf("Failed to read resolution directory %s: %v", resolutionPath, err)
|
||||||
@ -243,14 +204,21 @@ func (ps *ProcessingService) ParseResolutionDirectory(resolution string, ch chan
|
|||||||
func (ps *ProcessingService) AggregateSegmentInfo(ch <-chan SegmentInfo) (map[int]SegmentInfo, error) {
|
func (ps *ProcessingService) AggregateSegmentInfo(ch <-chan SegmentInfo) (map[int]SegmentInfo, error) {
|
||||||
segmentMap := make(map[int]SegmentInfo)
|
segmentMap := make(map[int]SegmentInfo)
|
||||||
|
|
||||||
// Each sequence number arrives once per downloaded variant; keep the
|
rank := map[string]int{
|
||||||
// highest-resolution copy of each, since that is the one worth stitching
|
"1080p": 1,
|
||||||
// into the final video.
|
"720p": 2,
|
||||||
|
"540p": 3,
|
||||||
|
"480p": 4,
|
||||||
|
"450p": 5,
|
||||||
|
"360p": 6,
|
||||||
|
"270p": 7,
|
||||||
|
"240p": 8,
|
||||||
|
}
|
||||||
|
|
||||||
for segment := range ch {
|
for segment := range ch {
|
||||||
fmt.Printf("Received segment %s in resolution %s \n", segment.Name, segment.Resolution)
|
fmt.Printf("Received segment %s in resolution %s \n", segment.Name, segment.Resolution)
|
||||||
logging.MustGetFactory().Processing.Debug("segment received", logging.KeySegment, segment.Name, logging.KeyResolution, segment.Resolution)
|
|
||||||
current, exists := segmentMap[segment.SeqNo]
|
current, exists := segmentMap[segment.SeqNo]
|
||||||
if !exists || resolutionHeight(segment.Resolution) > resolutionHeight(current.Resolution) {
|
if !exists || rank[segment.Resolution] > rank[current.Resolution] {
|
||||||
segmentMap[segment.SeqNo] = segment
|
segmentMap[segment.SeqNo] = segment
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -258,24 +226,6 @@ func (ps *ProcessingService) AggregateSegmentInfo(ch <-chan SegmentInfo) (map[in
|
|||||||
return segmentMap, nil
|
return segmentMap, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// resolutionHeight converts a resolution label such as "1080p" into the pixel
|
|
||||||
// height it stands for, so variants can be ranked by quality.
|
|
||||||
//
|
|
||||||
// The height is parsed rather than looked up in a table: labels come from the
|
|
||||||
// playlist's own RESOLUTION attribute, so a stream can legitimately offer
|
|
||||||
// 1440p or 2160p that no fixed table would anticipate. Anything not in
|
|
||||||
// <height>p form — "unknown", from a playlist with no variants — returns 0 and
|
|
||||||
// therefore loses to every recognised resolution.
|
|
||||||
func resolutionHeight(resolution string) int {
|
|
||||||
trimmed := strings.TrimSuffix(strings.ToLower(strings.TrimSpace(resolution)), "p")
|
|
||||||
|
|
||||||
height, err := strconv.Atoi(trimmed)
|
|
||||||
if err != nil || height < 0 {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
return height
|
|
||||||
}
|
|
||||||
|
|
||||||
func (ps *ProcessingService) WriteConcatFile(segmentMap map[int]SegmentInfo) (string, error) {
|
func (ps *ProcessingService) WriteConcatFile(segmentMap map[int]SegmentInfo) (string, error) {
|
||||||
concatPath := ps.config.GetProcessOutputPath(ps.eventName)
|
concatPath := ps.config.GetProcessOutputPath(ps.eventName)
|
||||||
|
|
||||||
@ -299,7 +249,7 @@ func (ps *ProcessingService) WriteConcatFile(segmentMap map[int]SegmentInfo) (st
|
|||||||
|
|
||||||
for _, seq := range keys {
|
for _, seq := range keys {
|
||||||
segment := segmentMap[seq]
|
segment := segmentMap[seq]
|
||||||
filePath := utils.SafeJoin(ps.nasEventPath(), segment.Resolution, segment.Name)
|
filePath := utils.SafeJoin(ps.config.GetNASEventPath(ps.eventName), segment.Resolution, segment.Name)
|
||||||
line := fmt.Sprintf("file '%s'\n", filePath)
|
line := fmt.Sprintf("file '%s'\n", filePath)
|
||||||
if _, err := f.WriteString(line); err != nil {
|
if _, err := f.WriteString(line); err != nil {
|
||||||
return "", fmt.Errorf("failed to write to concat file: %w", err)
|
return "", fmt.Errorf("failed to write to concat file: %w", err)
|
||||||
@ -368,11 +318,9 @@ func (ps *ProcessingService) RunFFmpeg(inputPath, outputPath string) error {
|
|||||||
fileOutPath := utils.SafeJoin(outputPath, ps.eventName+".mp4")
|
fileOutPath := utils.SafeJoin(outputPath, ps.eventName+".mp4")
|
||||||
fmt.Println("Input path:", inputPath)
|
fmt.Println("Input path:", inputPath)
|
||||||
fmt.Println("Output path:", fileOutPath)
|
fmt.Println("Output path:", fileOutPath)
|
||||||
logging.MustGetFactory().Processing.Info("running ffmpeg", "input", inputPath, "output", fileOutPath)
|
|
||||||
|
|
||||||
path, err := ps.getFFmpegPath()
|
path, err := ps.getFFmpegPath()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logging.MustGetFactory().Processing.Error("failed to find ffmpeg", logging.KeyError, err.Error())
|
|
||||||
return fmt.Errorf("failed to find FFmpeg: %w", err)
|
return fmt.Errorf("failed to find FFmpeg: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -381,11 +329,9 @@ func (ps *ProcessingService) RunFFmpeg(inputPath, outputPath string) error {
|
|||||||
cmd.Stderr = os.Stderr
|
cmd.Stderr = os.Stderr
|
||||||
|
|
||||||
if err := cmd.Run(); err != nil {
|
if err := cmd.Run(); err != nil {
|
||||||
logging.MustGetFactory().Processing.Error("ffmpeg run failed", logging.KeyError, err.Error())
|
|
||||||
return fmt.Errorf("failed to run ffmpeg: %w", err)
|
return fmt.Errorf("failed to run ffmpeg: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Println("FFmpeg completed successfully")
|
fmt.Println("FFmpeg completed successfully")
|
||||||
logging.MustGetFactory().Processing.Info("ffmpeg completed successfully", "output", fileOutPath)
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,9 +1,7 @@
|
|||||||
package processing
|
package processing
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"m3u8-downloader/pkg/config"
|
"m3u8-downloader/pkg/config"
|
||||||
"m3u8-downloader/pkg/nas"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"runtime"
|
"runtime"
|
||||||
@ -12,22 +10,6 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// connectedNAS attaches a NAS service to a local directory. Reaching the NAS
|
|
||||||
// is platform-specific, but an already-mounted or plain directory resolves the
|
|
||||||
// same way everywhere, which is what makes this usable as a test double.
|
|
||||||
func connectedNAS(t *testing.T, path string) *nas.NASService {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
svc, err := nas.New(nas.NASConfig{Path: path, Timeout: 5 * time.Second})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Failed to create NAS service for %s: %v", path, err)
|
|
||||||
}
|
|
||||||
if err := svc.Connect(context.Background()); err != nil {
|
|
||||||
t.Fatalf("Failed to connect NAS service to %s: %v", path, err)
|
|
||||||
}
|
|
||||||
return svc
|
|
||||||
}
|
|
||||||
|
|
||||||
func createTestConfig(tempDir string) *config.Config {
|
func createTestConfig(tempDir string) *config.Config {
|
||||||
return &config.Config{
|
return &config.Config{
|
||||||
Core: config.CoreConfig{
|
Core: config.CoreConfig{
|
||||||
@ -105,7 +87,6 @@ func TestProcessingService_GetEventDirs(t *testing.T) {
|
|||||||
ps := &ProcessingService{
|
ps := &ProcessingService{
|
||||||
config: cfg,
|
config: cfg,
|
||||||
eventName: "", // Empty to test directory discovery
|
eventName: "", // Empty to test directory discovery
|
||||||
nas: connectedNAS(t, cfg.NAS.OutputPath),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
dirs, err := ps.GetEventDirs()
|
dirs, err := ps.GetEventDirs()
|
||||||
@ -133,15 +114,12 @@ func TestProcessingService_GetEventDirs(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestProcessingService_GetEventDirs_WithEventName(t *testing.T) {
|
func TestProcessingService_GetEventDirs_WithEventName(t *testing.T) {
|
||||||
// Scoped to a temp dir rather than /tmp directly: attaching the NAS
|
cfg := createTestConfig("/tmp")
|
||||||
// service creates the destination, and tests should not leave one behind.
|
|
||||||
cfg := createTestConfig(t.TempDir())
|
|
||||||
eventName := "specific-event"
|
eventName := "specific-event"
|
||||||
|
|
||||||
ps := &ProcessingService{
|
ps := &ProcessingService{
|
||||||
config: cfg,
|
config: cfg,
|
||||||
eventName: eventName,
|
eventName: eventName,
|
||||||
nas: connectedNAS(t, cfg.NAS.OutputPath),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
dirs, err := ps.GetEventDirs()
|
dirs, err := ps.GetEventDirs()
|
||||||
@ -178,7 +156,6 @@ func TestProcessingService_GetResolutions(t *testing.T) {
|
|||||||
ps := &ProcessingService{
|
ps := &ProcessingService{
|
||||||
config: cfg,
|
config: cfg,
|
||||||
eventName: eventName,
|
eventName: eventName,
|
||||||
nas: connectedNAS(t, cfg.NAS.OutputPath),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
resolutions, err := ps.GetResolutions()
|
resolutions, err := ps.GetResolutions()
|
||||||
@ -271,7 +248,6 @@ func TestProcessingService_WriteConcatFile(t *testing.T) {
|
|||||||
ps := &ProcessingService{
|
ps := &ProcessingService{
|
||||||
config: cfg,
|
config: cfg,
|
||||||
eventName: eventName,
|
eventName: eventName,
|
||||||
nas: connectedNAS(t, cfg.NAS.OutputPath),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create test segment map
|
// Create test segment map
|
||||||
@ -406,67 +382,3 @@ func TestProcessJob_Structure(t *testing.T) {
|
|||||||
t.Errorf("Expected EventName='test-event', got '%s'", job.EventName)
|
t.Errorf("Expected EventName='test-event', got '%s'", job.EventName)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestResolutionHeight(t *testing.T) {
|
|
||||||
cases := map[string]int{
|
|
||||||
"1080p": 1080,
|
|
||||||
"720p": 720,
|
|
||||||
"2160p": 2160, // beyond anything a fixed quality table would list
|
|
||||||
"1440p": 1440,
|
|
||||||
"240p": 240,
|
|
||||||
// Not in <height>p form: must rank below every real resolution rather
|
|
||||||
// than accidentally outranking them.
|
|
||||||
"unknown": 0,
|
|
||||||
"": 0,
|
|
||||||
"p": 0,
|
|
||||||
"-720p": 0,
|
|
||||||
}
|
|
||||||
|
|
||||||
for label, want := range cases {
|
|
||||||
if got := resolutionHeight(label); got != want {
|
|
||||||
t.Errorf("resolutionHeight(%q) = %d, want %d", label, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestAggregateSegmentInfoPrefersHighestResolution covers the variants a
|
|
||||||
// hardcoded rank table could not express: resolutions above 1080p, and the
|
|
||||||
// "unknown" label produced for a playlist with no variants.
|
|
||||||
func TestAggregateSegmentInfoPrefersHighestResolution(t *testing.T) {
|
|
||||||
cases := []struct {
|
|
||||||
name string
|
|
||||||
arriving []string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{"1080p beats 720p regardless of order", []string{"720p", "1080p"}, "1080p"},
|
|
||||||
{"1080p beats 720p when it arrives first", []string{"1080p", "720p"}, "1080p"},
|
|
||||||
{"4k beats 1080p", []string{"1080p", "2160p"}, "2160p"},
|
|
||||||
{"1440p beats 1080p", []string{"1080p", "1440p"}, "1440p"},
|
|
||||||
{"a known resolution beats unknown", []string{"unknown", "480p"}, "480p"},
|
|
||||||
{"unknown does not displace a known resolution", []string{"1080p", "unknown"}, "1080p"},
|
|
||||||
{"unknown alone is still kept", []string{"unknown"}, "unknown"},
|
|
||||||
{"lowest only", []string{"240p"}, "240p"},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range cases {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
ps := &ProcessingService{}
|
|
||||||
ch := make(chan SegmentInfo, len(tc.arriving))
|
|
||||||
for _, res := range tc.arriving {
|
|
||||||
ch <- SegmentInfo{Name: "seg_1001.ts", SeqNo: 1001, Resolution: res}
|
|
||||||
}
|
|
||||||
close(ch)
|
|
||||||
|
|
||||||
segments, err := ps.AggregateSegmentInfo(ch)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("AggregateSegmentInfo() failed: %v", err)
|
|
||||||
}
|
|
||||||
if len(segments) != 1 {
|
|
||||||
t.Fatalf("expected a single sequence number, got %d", len(segments))
|
|
||||||
}
|
|
||||||
if got := segments[1001].Resolution; got != tc.want {
|
|
||||||
t.Errorf("kept %q, want %q (arrival order: %v)", got, tc.want, tc.arriving)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@ -2,20 +2,13 @@ package transfer
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"m3u8-downloader/pkg/logging"
|
|
||||||
"os"
|
"os"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// errRetained reports that a file is still inside its retention window. It is
|
|
||||||
// not a failure: the file simply has to be reconsidered on a later pass rather
|
|
||||||
// than being discarded from the pending list.
|
|
||||||
var errRetained = errors.New("file is within the retention period")
|
|
||||||
|
|
||||||
type CleanupService struct {
|
type CleanupService struct {
|
||||||
config CleanupConfig
|
config CleanupConfig
|
||||||
pendingFiles []string
|
pendingFiles []string
|
||||||
@ -48,7 +41,6 @@ func (cs *CleanupService) Start(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
log.Printf("Cleanup service started (retention: %v, batch: %d)", cs.config.RetentionPeriod, cs.config.BatchSize)
|
log.Printf("Cleanup service started (retention: %v, batch: %d)", cs.config.RetentionPeriod, cs.config.BatchSize)
|
||||||
logging.MustGetFactory().Transfer.Info("cleanup service started", "retention", cs.config.RetentionPeriod.String(), "batch_size", cs.config.BatchSize)
|
|
||||||
|
|
||||||
ticker := time.NewTicker(cs.config.CheckInterval)
|
ticker := time.NewTicker(cs.config.CheckInterval)
|
||||||
defer ticker.Stop()
|
defer ticker.Stop()
|
||||||
@ -88,62 +80,36 @@ func (cs *CleanupService) ExecuteCleanup(ctx context.Context) error {
|
|||||||
log.Printf("Processing %d files for cleanup", len(batch))
|
log.Printf("Processing %d files for cleanup", len(batch))
|
||||||
|
|
||||||
var cleanedCount int
|
var cleanedCount int
|
||||||
var failures []error
|
var errors []error
|
||||||
// requeue collects files this pass must not drop: ones still inside their
|
|
||||||
// retention window, and — if the context is cancelled mid-batch — the
|
|
||||||
// remainder that was never looked at. They were already removed from
|
|
||||||
// pendingFiles above, so without this they would never be cleaned at all.
|
|
||||||
var requeue []string
|
|
||||||
|
|
||||||
for i, filePath := range batch {
|
for _, filePath := range batch {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
requeue = append(requeue, batch[i:]...)
|
|
||||||
cs.requeue(requeue)
|
|
||||||
return ctx.Err()
|
return ctx.Err()
|
||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
err := cs.cleanupFile(filePath)
|
if err := cs.cleanupFile(filePath); err != nil {
|
||||||
switch {
|
errors = append(errors, fmt.Errorf("Failed to cleanup file %s: %w", filePath, err))
|
||||||
case errors.Is(err, errRetained):
|
} else {
|
||||||
requeue = append(requeue, filePath)
|
|
||||||
case err != nil:
|
|
||||||
failures = append(failures, fmt.Errorf("Failed to cleanup file %s: %w", filePath, err))
|
|
||||||
default:
|
|
||||||
cleanedCount++
|
cleanedCount++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
cs.requeue(requeue)
|
log.Printf("Cleanup batch completed (cleaned: %d, errors: %d)", cleanedCount, len(errors))
|
||||||
|
|
||||||
log.Printf("Cleanup batch completed (cleaned: %d, retained: %d, errors: %d)", cleanedCount, len(requeue), len(failures))
|
if len(errors) > 0 {
|
||||||
logging.MustGetFactory().Transfer.Info("cleanup batch completed",
|
for i, err := range errors {
|
||||||
"cleaned", cleanedCount, "retained", len(requeue), "errors", len(failures))
|
|
||||||
|
|
||||||
if len(failures) > 0 {
|
|
||||||
for i, err := range failures {
|
|
||||||
if i >= 3 {
|
if i >= 3 {
|
||||||
log.Printf("... and %d more errors", len(failures)-3)
|
log.Printf("... and %d more errors", len(errors)-3)
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
log.Printf("Error: %v", err)
|
log.Printf("Error: %v", err)
|
||||||
logging.MustGetFactory().Transfer.Warn("cleanup error", logging.KeyError, err.Error())
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
|
||||||
|
|
||||||
// requeue returns files to the pending list for a later pass.
|
|
||||||
func (cs *CleanupService) requeue(files []string) {
|
|
||||||
if len(files) == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
cs.mu.Lock()
|
|
||||||
defer cs.mu.Unlock()
|
|
||||||
cs.pendingFiles = append(cs.pendingFiles, files...)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (cs *CleanupService) cleanupFile(filePath string) error {
|
func (cs *CleanupService) cleanupFile(filePath string) error {
|
||||||
@ -158,7 +124,7 @@ func (cs *CleanupService) cleanupFile(filePath string) error {
|
|||||||
if cs.config.RetentionPeriod > 0 {
|
if cs.config.RetentionPeriod > 0 {
|
||||||
if time.Since(info.ModTime()) < cs.config.RetentionPeriod {
|
if time.Since(info.ModTime()) < cs.config.RetentionPeriod {
|
||||||
log.Printf("File too new to cleanup: %s", filePath)
|
log.Printf("File too new to cleanup: %s", filePath)
|
||||||
return errRetained
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -192,18 +158,6 @@ func (cs *CleanupService) ForceCleanupAll(ctx context.Context) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop when a pass makes no progress. Files still inside their
|
|
||||||
// retention window are requeued by design, so requiring the list to
|
|
||||||
// reach zero would spin here until the shutdown timeout expired.
|
|
||||||
cs.mu.Lock()
|
|
||||||
remaining := len(cs.pendingFiles)
|
|
||||||
cs.mu.Unlock()
|
|
||||||
|
|
||||||
if remaining >= pendingCount {
|
|
||||||
log.Printf("Force cleanup stopping with %d file(s) still pending (within retention period)", remaining)
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return ctx.Err()
|
return ctx.Err()
|
||||||
|
|||||||
@ -1,190 +0,0 @@
|
|||||||
package transfer
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func writeAgedFile(t *testing.T, dir, name string, age time.Duration) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
path := filepath.Join(dir, name)
|
|
||||||
if err := os.WriteFile(path, []byte("segment"), 0644); err != nil {
|
|
||||||
t.Fatalf("failed to write %s: %v", path, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
modTime := time.Now().Add(-age)
|
|
||||||
if err := os.Chtimes(path, modTime, modTime); err != nil {
|
|
||||||
t.Fatalf("failed to age %s: %v", path, err)
|
|
||||||
}
|
|
||||||
return path
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCleanupRemovesTransferredFiles(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
cs := NewCleanupService(CleanupConfig{Enabled: true, BatchSize: 10, CheckInterval: time.Second})
|
|
||||||
|
|
||||||
path := writeAgedFile(t, dir, "seg_1001.ts", 0)
|
|
||||||
if err := cs.ScheduleCleanup(path); err != nil {
|
|
||||||
t.Fatalf("ScheduleCleanup failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := cs.ExecuteCleanup(context.Background()); err != nil {
|
|
||||||
t.Fatalf("ExecuteCleanup failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
|
||||||
t.Errorf("file should have been removed, stat err = %v", err)
|
|
||||||
}
|
|
||||||
if got := cs.GetPendingCount(); got != 0 {
|
|
||||||
t.Errorf("pending count = %d, want 0", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestCleanupRetainsThenRemoves is the regression test for a file inside its
|
|
||||||
// retention window being dropped from the pending list: it was reported as
|
|
||||||
// cleaned, never deleted, and never reconsidered, so local files accumulated
|
|
||||||
// forever whenever RetainHours was set.
|
|
||||||
func TestCleanupRetainsThenRemoves(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
cs := NewCleanupService(CleanupConfig{
|
|
||||||
Enabled: true,
|
|
||||||
RetentionPeriod: time.Hour,
|
|
||||||
BatchSize: 10,
|
|
||||||
CheckInterval: time.Second,
|
|
||||||
})
|
|
||||||
|
|
||||||
fresh := writeAgedFile(t, dir, "fresh.ts", time.Minute)
|
|
||||||
old := writeAgedFile(t, dir, "old.ts", 2*time.Hour)
|
|
||||||
|
|
||||||
for _, p := range []string{fresh, old} {
|
|
||||||
if err := cs.ScheduleCleanup(p); err != nil {
|
|
||||||
t.Fatalf("ScheduleCleanup failed: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := cs.ExecuteCleanup(context.Background()); err != nil {
|
|
||||||
t.Fatalf("ExecuteCleanup failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, err := os.Stat(old); !os.IsNotExist(err) {
|
|
||||||
t.Errorf("a file past its retention period should be removed, stat err = %v", err)
|
|
||||||
}
|
|
||||||
if _, err := os.Stat(fresh); err != nil {
|
|
||||||
t.Errorf("a file inside its retention period should survive: %v", err)
|
|
||||||
}
|
|
||||||
if got := cs.GetPendingCount(); got != 1 {
|
|
||||||
t.Fatalf("the retained file must stay queued for a later pass, pending = %d", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Once it ages past the window, the next pass must actually remove it.
|
|
||||||
past := time.Now().Add(-2 * time.Hour)
|
|
||||||
if err := os.Chtimes(fresh, past, past); err != nil {
|
|
||||||
t.Fatalf("failed to age file: %v", err)
|
|
||||||
}
|
|
||||||
if err := cs.ExecuteCleanup(context.Background()); err != nil {
|
|
||||||
t.Fatalf("ExecuteCleanup failed: %v", err)
|
|
||||||
}
|
|
||||||
if _, err := os.Stat(fresh); !os.IsNotExist(err) {
|
|
||||||
t.Errorf("file should have been removed once past retention, stat err = %v", err)
|
|
||||||
}
|
|
||||||
if got := cs.GetPendingCount(); got != 0 {
|
|
||||||
t.Errorf("pending count = %d, want 0", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestForceCleanupAllTerminatesWithRetainedFiles guards shutdown: retained
|
|
||||||
// files are requeued by design, so a loop demanding an empty list would spin
|
|
||||||
// until the shutdown timeout expired.
|
|
||||||
func TestForceCleanupAllTerminatesWithRetainedFiles(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
cs := NewCleanupService(CleanupConfig{
|
|
||||||
Enabled: true,
|
|
||||||
RetentionPeriod: time.Hour,
|
|
||||||
BatchSize: 10,
|
|
||||||
CheckInterval: time.Second,
|
|
||||||
})
|
|
||||||
|
|
||||||
if err := cs.ScheduleCleanup(writeAgedFile(t, dir, "fresh.ts", time.Minute)); err != nil {
|
|
||||||
t.Fatalf("ScheduleCleanup failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
done := make(chan error, 1)
|
|
||||||
go func() { done <- cs.ForceCleanupAll(context.Background()) }()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case err := <-done:
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ForceCleanupAll failed: %v", err)
|
|
||||||
}
|
|
||||||
case <-time.After(3 * time.Second):
|
|
||||||
t.Fatal("ForceCleanupAll did not terminate with a retained file pending")
|
|
||||||
}
|
|
||||||
|
|
||||||
if got := cs.GetPendingCount(); got != 1 {
|
|
||||||
t.Errorf("the retained file should still be pending, got %d", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCleanupMissingFileIsNotAnError(t *testing.T) {
|
|
||||||
cs := NewCleanupService(CleanupConfig{Enabled: true, BatchSize: 10, CheckInterval: time.Second})
|
|
||||||
|
|
||||||
// Already gone — a file transferred and removed by something else.
|
|
||||||
if err := cs.ScheduleCleanup(filepath.Join(t.TempDir(), "gone.ts")); err != nil {
|
|
||||||
t.Fatalf("ScheduleCleanup failed: %v", err)
|
|
||||||
}
|
|
||||||
if err := cs.ExecuteCleanup(context.Background()); err != nil {
|
|
||||||
t.Fatalf("ExecuteCleanup failed: %v", err)
|
|
||||||
}
|
|
||||||
if got := cs.GetPendingCount(); got != 0 {
|
|
||||||
t.Errorf("a missing file should not be requeued, pending = %d", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCleanupDisabledSchedulesNothing(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
cs := NewCleanupService(CleanupConfig{Enabled: false, BatchSize: 10, CheckInterval: time.Second})
|
|
||||||
|
|
||||||
path := writeAgedFile(t, dir, "seg.ts", time.Hour)
|
|
||||||
if err := cs.ScheduleCleanup(path); err != nil {
|
|
||||||
t.Fatalf("ScheduleCleanup failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got := cs.GetPendingCount(); got != 0 {
|
|
||||||
t.Errorf("cleanup is disabled, nothing should be queued, got %d", got)
|
|
||||||
}
|
|
||||||
if err := cs.ExecuteCleanup(context.Background()); err != nil {
|
|
||||||
t.Fatalf("ExecuteCleanup failed: %v", err)
|
|
||||||
}
|
|
||||||
if _, err := os.Stat(path); err != nil {
|
|
||||||
t.Errorf("file must survive when cleanup is disabled: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestCleanupRequeuesOnCancellation covers shutdown arriving mid-batch: the
|
|
||||||
// files already pulled off the pending list must go back, not vanish.
|
|
||||||
func TestCleanupRequeuesOnCancellation(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
cs := NewCleanupService(CleanupConfig{Enabled: true, BatchSize: 10, CheckInterval: time.Second})
|
|
||||||
|
|
||||||
for i := 0; i < 5; i++ {
|
|
||||||
path := writeAgedFile(t, dir, fmt.Sprintf("seg_%d.ts", i), 0)
|
|
||||||
if err := cs.ScheduleCleanup(path); err != nil {
|
|
||||||
t.Fatalf("ScheduleCleanup failed: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
cancel()
|
|
||||||
|
|
||||||
if err := cs.ExecuteCleanup(ctx); err == nil {
|
|
||||||
t.Fatal("expected the cancelled context to be reported")
|
|
||||||
}
|
|
||||||
if got := cs.GetPendingCount(); got != 5 {
|
|
||||||
t.Errorf("all files should be requeued after cancellation, pending = %d, want 5", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -4,14 +4,13 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"m3u8-downloader/pkg/logging"
|
|
||||||
"m3u8-downloader/pkg/nas"
|
"m3u8-downloader/pkg/nas"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TransferFile(nt *nas.NASService, ctx context.Context, item *TransferItem) error {
|
func TransferFile(nt *nas.NASService, ctx context.Context, item *TransferItem) error {
|
||||||
destPath := nt.Join(item.DestinationPath)
|
destPath := filepath.Join(nt.Config.Path, item.DestinationPath)
|
||||||
|
|
||||||
destDir := filepath.Dir(destPath)
|
destDir := filepath.Dir(destPath)
|
||||||
if err := nt.EnsureDirectoryExists(destDir); err != nil {
|
if err := nt.EnsureDirectoryExists(destDir); err != nil {
|
||||||
@ -33,7 +32,6 @@ func TransferFile(nt *nas.NASService, ctx context.Context, item *TransferItem) e
|
|||||||
}
|
}
|
||||||
|
|
||||||
log.Printf("File transfer completed: %s -> %s", item.SourcePath, destPath)
|
log.Printf("File transfer completed: %s -> %s", item.SourcePath, destPath)
|
||||||
logging.MustGetFactory().Transfer.Debug("nas copy completed", logging.KeyFile, item.SourcePath, "dest", destPath)
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@ -6,8 +6,6 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"m3u8-downloader/pkg/health"
|
|
||||||
"m3u8-downloader/pkg/logging"
|
|
||||||
"m3u8-downloader/pkg/nas"
|
"m3u8-downloader/pkg/nas"
|
||||||
"os"
|
"os"
|
||||||
"sync"
|
"sync"
|
||||||
@ -76,10 +74,7 @@ func (tq *TransferQueue) Add(item TransferItem) error {
|
|||||||
defer tq.mu.Unlock()
|
defer tq.mu.Unlock()
|
||||||
|
|
||||||
if tq.items.Len() >= tq.config.MaxQueueSize {
|
if tq.items.Len() >= tq.config.MaxQueueSize {
|
||||||
err := fmt.Errorf("Queue is full (max size: %d)", tq.config.MaxQueueSize)
|
return fmt.Errorf("Queue is full (max size: %d)", tq.config.MaxQueueSize)
|
||||||
logging.MustGetFactory().Transfer.Error("queue full, dropping file", logging.KeyFile, item.SourcePath, logging.KeyError, err.Error())
|
|
||||||
health.MustGetRegistry().RecordTransferError(item.SourcePath, err)
|
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
heap.Push(tq.items, &item)
|
heap.Push(tq.items, &item)
|
||||||
@ -169,20 +164,13 @@ func (tq *TransferQueue) processItem(ctx context.Context, item TransferItem) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
maxRetries := tq.config.RetryLimit
|
maxRetries := 3
|
||||||
if maxRetries < 1 {
|
|
||||||
maxRetries = 1
|
|
||||||
}
|
|
||||||
|
|
||||||
for attempt := 1; attempt <= maxRetries; attempt++ {
|
for attempt := 1; attempt <= maxRetries; attempt++ {
|
||||||
// Back off before a retry, never before the first attempt. The guard
|
if attempt > 0 {
|
||||||
// was `attempt > 0`, which is always true, so every single transfer —
|
|
||||||
// including healthy ones — paid a one-second delay before it began,
|
|
||||||
// throttling the whole pipeline to the worker count per second.
|
|
||||||
if attempt > 1 {
|
|
||||||
item.Status = StatusRetrying
|
item.Status = StatusRetrying
|
||||||
backoff := time.Duration(attempt*attempt) * time.Second
|
backoff := time.Duration(attempt*attempt) * time.Second
|
||||||
log.Printf("Backing off for %s before retrying (attempt %d/%d)", backoff, attempt, maxRetries)
|
log.Printf("Backing off for %d seconds before retrying (attempt %d/%d)", backoff, attempt, maxRetries)
|
||||||
|
|
||||||
select {
|
select {
|
||||||
case <-time.After(backoff):
|
case <-time.After(backoff):
|
||||||
@ -199,11 +187,9 @@ func (tq *TransferQueue) processItem(ctx context.Context, item TransferItem) {
|
|||||||
if tq.cleanup != nil {
|
if tq.cleanup != nil {
|
||||||
if err := tq.cleanup.ScheduleCleanup(item.SourcePath); err != nil {
|
if err := tq.cleanup.ScheduleCleanup(item.SourcePath); err != nil {
|
||||||
log.Printf("Failed to add file to cleanup list: %v", err)
|
log.Printf("Failed to add file to cleanup list: %v", err)
|
||||||
logging.MustGetFactory().Transfer.Warn("failed to schedule cleanup", logging.KeyFile, item.SourcePath, logging.KeyError, err.Error())
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
log.Printf("File transfer completed: %s", item.SourcePath)
|
log.Printf("File transfer completed: %s", item.SourcePath)
|
||||||
logging.MustGetFactory().Transfer.Info("file transfer completed", logging.KeyFile, item.SourcePath, logging.KeyBytes, item.FileSize)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -211,14 +197,11 @@ func (tq *TransferQueue) processItem(ctx context.Context, item TransferItem) {
|
|||||||
item.RetryCount++
|
item.RetryCount++
|
||||||
|
|
||||||
log.Printf("File transfer failed: %s (attempt %d/%d): %v", item.SourcePath, item.RetryCount, maxRetries, err)
|
log.Printf("File transfer failed: %s (attempt %d/%d): %v", item.SourcePath, item.RetryCount, maxRetries, err)
|
||||||
logging.MustGetFactory().Transfer.Warn("file transfer attempt failed", logging.KeyFile, item.SourcePath, logging.KeyAttempt, item.RetryCount, logging.KeyError, err.Error())
|
|
||||||
|
|
||||||
if attempt == maxRetries {
|
if attempt == maxRetries {
|
||||||
item.Status = StatusFailed
|
item.Status = StatusFailed
|
||||||
tq.stats.IncrementFailed()
|
tq.stats.IncrementFailed()
|
||||||
log.Printf("Transfer permanently failed for file: %s", item.SourcePath)
|
log.Printf("Transfer permanently failed for file: %s", item.SourcePath)
|
||||||
logging.MustGetFactory().Transfer.Error("transfer permanently failed", logging.KeyFile, item.SourcePath, logging.KeyError, err.Error())
|
|
||||||
health.MustGetRegistry().RecordTransferError(item.SourcePath, err)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -242,12 +225,10 @@ func (tq *TransferQueue) SaveState() error {
|
|||||||
"timestamp": time.Now(),
|
"timestamp": time.Now(),
|
||||||
}, "", " ")
|
}, "", " ")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logging.MustGetFactory().Transfer.Error("failed to marshal queue state", logging.KeyError, err.Error())
|
|
||||||
return fmt.Errorf("Failed to marshal queue state: %w", err)
|
return fmt.Errorf("Failed to marshal queue state: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := os.WriteFile(tq.config.PersistencePath, data, 0644); err != nil {
|
if err := os.WriteFile(tq.config.PersistencePath, data, 0644); err != nil {
|
||||||
logging.MustGetFactory().Transfer.Error("failed to save queue state", logging.KeyError, err.Error())
|
|
||||||
return fmt.Errorf("Failed to save queue state: %w", err)
|
return fmt.Errorf("Failed to save queue state: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -260,7 +241,6 @@ func (tq *TransferQueue) LoadState() error {
|
|||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
logging.MustGetFactory().Transfer.Error("failed to load queue state", logging.KeyError, err.Error())
|
|
||||||
return fmt.Errorf("Failed to load queue state: %w", err)
|
return fmt.Errorf("Failed to load queue state: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -5,8 +5,6 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"m3u8-downloader/pkg/constants"
|
"m3u8-downloader/pkg/constants"
|
||||||
"m3u8-downloader/pkg/health"
|
|
||||||
"m3u8-downloader/pkg/logging"
|
|
||||||
nas2 "m3u8-downloader/pkg/nas"
|
nas2 "m3u8-downloader/pkg/nas"
|
||||||
"m3u8-downloader/pkg/utils"
|
"m3u8-downloader/pkg/utils"
|
||||||
"os"
|
"os"
|
||||||
@ -34,24 +32,12 @@ func NewTrasferService(outputDir string, eventName string) (*TransferService, er
|
|||||||
Timeout: cfg.NAS.Timeout,
|
Timeout: cfg.NAS.Timeout,
|
||||||
RetryLimit: cfg.NAS.RetryLimit,
|
RetryLimit: cfg.NAS.RetryLimit,
|
||||||
VerifySize: true,
|
VerifySize: true,
|
||||||
AutoMount: cfg.NAS.AutoMount,
|
|
||||||
MountPoint: cfg.NAS.MountPoint,
|
|
||||||
}
|
|
||||||
nas, err := nas2.New(nasConfig)
|
|
||||||
if err != nil {
|
|
||||||
logging.MustGetFactory().Transfer.Error("invalid NAS destination", logging.KeyError, err.Error())
|
|
||||||
return nil, fmt.Errorf("invalid NAS destination: %w", err)
|
|
||||||
}
|
}
|
||||||
|
nas := nas2.NewNASService(nasConfig)
|
||||||
|
|
||||||
connectCtx, cancelConnect := context.WithTimeout(context.Background(), cfg.NAS.Timeout)
|
if err := nas.TestConnection(); err != nil {
|
||||||
defer cancelConnect()
|
|
||||||
if err := nas.Connect(connectCtx); err != nil {
|
|
||||||
logging.MustGetFactory().Transfer.Error("failed to connect to NAS", logging.KeyError, err.Error())
|
|
||||||
return nil, fmt.Errorf("failed to connect to NAS: %w", err)
|
return nil, fmt.Errorf("failed to connect to NAS: %w", err)
|
||||||
}
|
}
|
||||||
logging.MustGetFactory().Transfer.Info("connected to NAS",
|
|
||||||
"path", nas.LocalPath(), "configured", outputDir, "how", strings.Join(nas.ConnectionInfo(), "; "))
|
|
||||||
health.MustGetRegistry().RecordNASConnectivity(true)
|
|
||||||
|
|
||||||
cleanupConfig := CleanupConfig{
|
cleanupConfig := CleanupConfig{
|
||||||
Enabled: cfg.Cleanup.AfterTransfer,
|
Enabled: cfg.Cleanup.AfterTransfer,
|
||||||
@ -66,7 +52,6 @@ func NewTrasferService(outputDir string, eventName string) (*TransferService, er
|
|||||||
PersistencePath: cfg.Paths.PersistenceFile,
|
PersistencePath: cfg.Paths.PersistenceFile,
|
||||||
MaxQueueSize: cfg.Transfer.QueueSize,
|
MaxQueueSize: cfg.Transfer.QueueSize,
|
||||||
BatchSize: cfg.Transfer.BatchSize,
|
BatchSize: cfg.Transfer.BatchSize,
|
||||||
RetryLimit: cfg.Transfer.RetryLimit,
|
|
||||||
}
|
}
|
||||||
queue := NewTransferQueue(queueConfig, nas, cleanup)
|
queue := NewTransferQueue(queueConfig, nas, cleanup)
|
||||||
|
|
||||||
@ -76,13 +61,11 @@ func NewTrasferService(outputDir string, eventName string) (*TransferService, er
|
|||||||
return nil, fmt.Errorf("failed to create local output directory: %w", err)
|
return nil, fmt.Errorf("failed to create local output directory: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
watcher, err := NewFileWatcher(localOutputPath, eventName, queue, cfg.Transfer.FileSettlingDelay)
|
watcher, err := NewFileWatcher(localOutputPath, queue, cfg.Transfer.FileSettlingDelay)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to create file watcher: %w", err)
|
return nil, fmt.Errorf("failed to create file watcher: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
health.MustGetRegistry().AttachTransfer(queue.stats, queue.GetQueueSize, cleanup.GetPendingCount, nas.IsConnected)
|
|
||||||
|
|
||||||
return &TransferService{
|
return &TransferService{
|
||||||
watcher: watcher,
|
watcher: watcher,
|
||||||
queue: queue,
|
queue: queue,
|
||||||
@ -126,7 +109,6 @@ func (ts *TransferService) Start(ctx context.Context) error {
|
|||||||
}()
|
}()
|
||||||
|
|
||||||
log.Println("Transfer service started")
|
log.Println("Transfer service started")
|
||||||
logging.MustGetFactory().Transfer.Info("transfer service started")
|
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@ -146,9 +128,6 @@ func (ts *TransferService) reportStats(ctx context.Context) {
|
|||||||
cleanupPending := ts.cleanup.GetPendingCount()
|
cleanupPending := ts.cleanup.GetPendingCount()
|
||||||
|
|
||||||
log.Printf("Transfer Stats: Added: %d, Completed: %d, Failed: %d, Pending: %d, Bytes: %d, Queue Size: %d, Cleanup Pending: %d", added, completed, failed, pending, bytes, queueSize, cleanupPending)
|
log.Printf("Transfer Stats: Added: %d, Completed: %d, Failed: %d, Pending: %d, Bytes: %d, Queue Size: %d, Cleanup Pending: %d", added, completed, failed, pending, bytes, queueSize, cleanupPending)
|
||||||
logging.MustGetFactory().Transfer.Info("transfer stats",
|
|
||||||
"added", added, "completed", completed, "failed", failed, "pending", pending,
|
|
||||||
logging.KeyBytes, bytes, "queue_size", queueSize, "cleanup_pending", cleanupPending)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -165,14 +144,11 @@ func (ts *TransferService) Shutdown(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Disconnect from NAS
|
// Disconnect from NAS
|
||||||
if err := ts.nas.Disconnect(ctx); err != nil {
|
if err := ts.nas.Disconnect(); err != nil {
|
||||||
log.Printf("Warning: failed to disconnect from NAS: %v", err)
|
log.Printf("Warning: failed to disconnect from NAS: %v", err)
|
||||||
logging.MustGetFactory().Transfer.Warn("failed to disconnect from NAS", logging.KeyError, err.Error())
|
|
||||||
}
|
}
|
||||||
health.MustGetRegistry().RecordNASConnectivity(false)
|
|
||||||
|
|
||||||
log.Println("Transfer service shut down")
|
log.Println("Transfer service shut down")
|
||||||
logging.MustGetFactory().Transfer.Info("transfer service shut down")
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@ -202,8 +178,6 @@ func (ts *TransferService) QueueExistingFiles(localEventPath string) error {
|
|||||||
relPath, err := filepath.Rel(localEventPath, path)
|
relPath, err := filepath.Rel(localEventPath, path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("Failed to get relative path for %s: %v", path, err)
|
log.Printf("Failed to get relative path for %s: %v", path, err)
|
||||||
logging.MustGetFactory().Transfer.Warn("failed to get relative path", logging.KeyFile, path, logging.KeyError, err.Error())
|
|
||||||
health.MustGetRegistry().RecordTransferError(path, err)
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -244,8 +218,6 @@ func (ts *TransferService) QueueExistingFiles(localEventPath string) error {
|
|||||||
// Add to queue
|
// Add to queue
|
||||||
if err := ts.queue.Add(item); err != nil {
|
if err := ts.queue.Add(item); err != nil {
|
||||||
log.Printf("Failed to queue file %s: %v", path, err)
|
log.Printf("Failed to queue file %s: %v", path, err)
|
||||||
logging.MustGetFactory().Transfer.Error("failed to queue file", logging.KeyFile, path, logging.KeyError, err.Error())
|
|
||||||
health.MustGetRegistry().RecordTransferError(path, err)
|
|
||||||
} else {
|
} else {
|
||||||
log.Printf("Queued file: %s (%s, %d bytes)", path, resolution, info.Size())
|
log.Printf("Queued file: %s (%s, %d bytes)", path, resolution, info.Size())
|
||||||
fileCount++
|
fileCount++
|
||||||
|
|||||||
@ -49,9 +49,6 @@ type QueueConfig struct {
|
|||||||
PersistencePath string
|
PersistencePath string
|
||||||
MaxQueueSize int
|
MaxQueueSize int
|
||||||
BatchSize int
|
BatchSize int
|
||||||
// RetryLimit is the total number of attempts made per file before it is
|
|
||||||
// marked failed.
|
|
||||||
RetryLimit int
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type CleanupConfig struct {
|
type CleanupConfig struct {
|
||||||
|
|||||||
@ -4,8 +4,6 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"m3u8-downloader/pkg/health"
|
|
||||||
"m3u8-downloader/pkg/logging"
|
|
||||||
"math/rand"
|
"math/rand"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@ -18,11 +16,6 @@ import (
|
|||||||
|
|
||||||
type FileWatcher struct {
|
type FileWatcher struct {
|
||||||
outputDir string
|
outputDir string
|
||||||
// eventName namespaces the NAS destination. It is passed in rather than
|
|
||||||
// derived from outputDir so that an unnamed event resolves to the NAS root
|
|
||||||
// — matching where the downloader put the files — instead of picking up
|
|
||||||
// the local output directory's own name.
|
|
||||||
eventName string
|
|
||||||
queue *TransferQueue
|
queue *TransferQueue
|
||||||
watcher *fsnotify.Watcher
|
watcher *fsnotify.Watcher
|
||||||
settingDelay time.Duration
|
settingDelay time.Duration
|
||||||
@ -30,14 +23,13 @@ type FileWatcher struct {
|
|||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewFileWatcher(outputDir, eventName string, queue *TransferQueue, settlingDelay time.Duration) (*FileWatcher, error) {
|
func NewFileWatcher(outputDir string, queue *TransferQueue, settlingDelay time.Duration) (*FileWatcher, error) {
|
||||||
watcher, err := fsnotify.NewWatcher()
|
watcher, err := fsnotify.NewWatcher()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return &FileWatcher{
|
return &FileWatcher{
|
||||||
outputDir: outputDir,
|
outputDir: outputDir,
|
||||||
eventName: eventName,
|
|
||||||
queue: queue,
|
queue: queue,
|
||||||
watcher: watcher,
|
watcher: watcher,
|
||||||
settingDelay: settlingDelay,
|
settingDelay: settlingDelay,
|
||||||
@ -53,13 +45,11 @@ func (fw *FileWatcher) Start(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
log.Printf("Starting file watcher on %s", fw.outputDir)
|
log.Printf("Starting file watcher on %s", fw.outputDir)
|
||||||
logging.MustGetFactory().Transfer.Info("file watcher started", "dir", fw.outputDir)
|
|
||||||
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
log.Println("File watcher shutting down...")
|
log.Println("File watcher shutting down...")
|
||||||
logging.MustGetFactory().Transfer.Info("file watcher shutting down")
|
|
||||||
return ctx.Err()
|
return ctx.Err()
|
||||||
|
|
||||||
case event, ok := <-fw.watcher.Events:
|
case event, ok := <-fw.watcher.Events:
|
||||||
@ -73,7 +63,6 @@ func (fw *FileWatcher) Start(ctx context.Context) error {
|
|||||||
return fmt.Errorf("Watcher errors channel closed")
|
return fmt.Errorf("Watcher errors channel closed")
|
||||||
}
|
}
|
||||||
log.Printf("Watcher error: %v", err)
|
log.Printf("Watcher error: %v", err)
|
||||||
logging.MustGetFactory().Transfer.Error("watcher error", logging.KeyError, err.Error())
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -98,18 +87,6 @@ func (fw *FileWatcher) addWatchRecursive(root string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (fw *FileWatcher) handleFileEvent(event fsnotify.Event) {
|
func (fw *FileWatcher) handleFileEvent(event fsnotify.Event) {
|
||||||
// Directories must be handled before the .ts filter below. fsnotify is not
|
|
||||||
// recursive, so a resolution directory created after the watcher starts —
|
|
||||||
// which is the normal case, since the transfer service starts before the
|
|
||||||
// downloader creates 1080p/ and friends — has to be picked up here or
|
|
||||||
// nothing inside it is ever seen.
|
|
||||||
if event.Op&fsnotify.Create == fsnotify.Create {
|
|
||||||
if info, err := os.Stat(event.Name); err == nil && info.IsDir() {
|
|
||||||
fw.watchNewDirectory(event.Name)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if !strings.HasSuffix(strings.ToLower(event.Name), ".ts") {
|
if !strings.HasSuffix(strings.ToLower(event.Name), ".ts") {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@ -122,33 +99,15 @@ func (fw *FileWatcher) handleFileEvent(event fsnotify.Event) {
|
|||||||
case event.Op&fsnotify.Remove == fsnotify.Remove:
|
case event.Op&fsnotify.Remove == fsnotify.Remove:
|
||||||
fw.cancelPendingTransfer(event.Name)
|
fw.cancelPendingTransfer(event.Name)
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// watchNewDirectory starts watching a directory that appeared after startup,
|
if event.Op&fsnotify.Create == fsnotify.Create {
|
||||||
// and sweeps whatever is already inside it.
|
if info, err := os.Stat(event.Name); err == nil && info.IsDir() {
|
||||||
//
|
if err := fw.watcher.Add(event.Name); err != nil {
|
||||||
// The sweep is not redundant: the downloader creates a resolution directory and
|
log.Printf("Failed to watch directory %s: %v", event.Name, err)
|
||||||
// writes segments into it immediately, so files can land between the mkdir and
|
} else {
|
||||||
// the watch being registered. Those writes generate no event we would see, and
|
log.Printf("Watching directory %s", event.Name)
|
||||||
// without the sweep the first segments of every recording would be lost.
|
|
||||||
func (fw *FileWatcher) watchNewDirectory(dir string) {
|
|
||||||
if err := fw.addWatchRecursive(dir); err != nil {
|
|
||||||
log.Printf("Failed to watch new directory %s: %v", dir, err)
|
|
||||||
logging.MustGetFactory().Transfer.Warn("failed to watch new directory", "dir", dir, logging.KeyError, err.Error())
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
|
||||||
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
|
|
||||||
if err != nil {
|
|
||||||
return nil // keep sweeping the rest
|
|
||||||
}
|
}
|
||||||
if !info.IsDir() && strings.HasSuffix(strings.ToLower(info.Name()), ".ts") {
|
|
||||||
fw.scheduleTransfer(path)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("Failed to sweep new directory %s: %v", dir, err)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -185,8 +144,6 @@ func (fw *FileWatcher) processFile(filePath string) {
|
|||||||
info, err := os.Stat(filePath)
|
info, err := os.Stat(filePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("Failed to stat file %s: %v", filePath, err)
|
log.Printf("Failed to stat file %s: %v", filePath, err)
|
||||||
logging.MustGetFactory().Transfer.Warn("failed to stat file", logging.KeyFile, filePath, logging.KeyError, err.Error())
|
|
||||||
health.MustGetRegistry().RecordTransferError(filePath, err)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -195,18 +152,13 @@ func (fw *FileWatcher) processFile(filePath string) {
|
|||||||
relPath, err := filepath.Rel(fw.outputDir, filePath)
|
relPath, err := filepath.Rel(fw.outputDir, filePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("Failed to get relative path for file %s: %v", filePath, err)
|
log.Printf("Failed to get relative path for file %s: %v", filePath, err)
|
||||||
logging.MustGetFactory().Transfer.Warn("failed to get relative path", logging.KeyFile, filePath, logging.KeyError, err.Error())
|
|
||||||
health.MustGetRegistry().RecordTransferError(filePath, err)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// The destination is namespaced by event, mirroring the local layout and
|
|
||||||
// matching what QueueExistingFiles produces for the same file — otherwise
|
|
||||||
// live transfers and catch-up transfers would write to different places.
|
|
||||||
item := TransferItem{
|
item := TransferItem{
|
||||||
ID: generateID(),
|
ID: generateID(),
|
||||||
SourcePath: filePath,
|
SourcePath: filePath,
|
||||||
DestinationPath: filepath.Join(fw.eventName, relPath),
|
DestinationPath: relPath,
|
||||||
Resolution: resolution,
|
Resolution: resolution,
|
||||||
Timestamp: time.Now(),
|
Timestamp: time.Now(),
|
||||||
Status: StatusPending,
|
Status: StatusPending,
|
||||||
@ -215,11 +167,8 @@ func (fw *FileWatcher) processFile(filePath string) {
|
|||||||
|
|
||||||
if err := fw.queue.Add(item); err != nil {
|
if err := fw.queue.Add(item); err != nil {
|
||||||
log.Printf("Failed to add file to queue: %v", err)
|
log.Printf("Failed to add file to queue: %v", err)
|
||||||
logging.MustGetFactory().Transfer.Error("failed to add file to queue", logging.KeyFile, filePath, logging.KeyError, err.Error())
|
|
||||||
health.MustGetRegistry().RecordTransferError(filePath, err)
|
|
||||||
} else {
|
} else {
|
||||||
log.Printf("Added file to queue: %s", filePath)
|
log.Printf("Added file to queue: %s", filePath)
|
||||||
logging.MustGetFactory().Transfer.Debug("file queued", logging.KeyFile, filePath)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -1,223 +0,0 @@
|
|||||||
package transfer
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"sort"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// settlingDelay is short so tests exercise the real timer path without
|
|
||||||
// waiting the production five seconds.
|
|
||||||
const testSettlingDelay = 40 * time.Millisecond
|
|
||||||
|
|
||||||
// newTestQueue builds a queue that accepts items but never transfers them:
|
|
||||||
// ProcessQueue is not started, so Add is a pure recording of intent.
|
|
||||||
func newTestQueue(t *testing.T) *TransferQueue {
|
|
||||||
t.Helper()
|
|
||||||
return NewTransferQueue(QueueConfig{
|
|
||||||
WorkerCount: 1,
|
|
||||||
PersistencePath: filepath.Join(t.TempDir(), "queue.json"),
|
|
||||||
MaxQueueSize: 100,
|
|
||||||
BatchSize: 10,
|
|
||||||
}, nil, nil)
|
|
||||||
}
|
|
||||||
|
|
||||||
// queuedDestinations returns the destination paths currently queued, sorted.
|
|
||||||
func queuedDestinations(q *TransferQueue) []string {
|
|
||||||
q.mu.RLock()
|
|
||||||
defer q.mu.RUnlock()
|
|
||||||
|
|
||||||
out := make([]string, 0, q.items.Len())
|
|
||||||
for _, item := range *q.items {
|
|
||||||
out = append(out, filepath.ToSlash(item.DestinationPath))
|
|
||||||
}
|
|
||||||
sort.Strings(out)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// waitForQueue polls until the queue holds want items or the deadline passes.
|
|
||||||
// Polling beats a fixed sleep: it keeps the test fast when the watcher is
|
|
||||||
// prompt and tolerant when a loaded CI machine is not.
|
|
||||||
func waitForQueue(t *testing.T, q *TransferQueue, want int) []string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
deadline := time.Now().Add(3 * time.Second)
|
|
||||||
var got []string
|
|
||||||
for time.Now().Before(deadline) {
|
|
||||||
got = queuedDestinations(q)
|
|
||||||
if len(got) >= want {
|
|
||||||
return got
|
|
||||||
}
|
|
||||||
time.Sleep(10 * time.Millisecond)
|
|
||||||
}
|
|
||||||
return got
|
|
||||||
}
|
|
||||||
|
|
||||||
func startWatcher(t *testing.T, dir, eventName string, q *TransferQueue) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
fw, err := NewFileWatcher(dir, eventName, q, testSettlingDelay)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewFileWatcher failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
t.Cleanup(cancel)
|
|
||||||
|
|
||||||
started := make(chan struct{})
|
|
||||||
go func() {
|
|
||||||
close(started)
|
|
||||||
_ = fw.Start(ctx)
|
|
||||||
}()
|
|
||||||
<-started
|
|
||||||
|
|
||||||
// Give Start a moment to register its initial watches before the test
|
|
||||||
// begins creating files under them.
|
|
||||||
time.Sleep(100 * time.Millisecond)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestWatcherPicksUpDirectoriesCreatedAfterStart covers the normal recording
|
|
||||||
// sequence: the transfer service starts watching the event directory, and only
|
|
||||||
// then does the downloader create the per-resolution directories inside it.
|
|
||||||
// fsnotify is not recursive, so those directories have to be picked up from
|
|
||||||
// the create event or nothing inside them is ever transferred.
|
|
||||||
func TestWatcherPicksUpDirectoriesCreatedAfterStart(t *testing.T) {
|
|
||||||
eventDir := t.TempDir()
|
|
||||||
queue := newTestQueue(t)
|
|
||||||
startWatcher(t, eventDir, "2026-regionals", queue)
|
|
||||||
|
|
||||||
resolutionDir := filepath.Join(eventDir, "1080p")
|
|
||||||
if err := os.MkdirAll(resolutionDir, 0755); err != nil {
|
|
||||||
t.Fatalf("failed to create resolution dir: %v", err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(filepath.Join(resolutionDir, "seg_1001.ts"), []byte("payload"), 0644); err != nil {
|
|
||||||
t.Fatalf("failed to write segment: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
got := waitForQueue(t, queue, 1)
|
|
||||||
if len(got) != 1 {
|
|
||||||
t.Fatalf("segment written to a new resolution directory was never queued, queue = %v", got)
|
|
||||||
}
|
|
||||||
if want := "2026-regionals/1080p/seg_1001.ts"; got[0] != want {
|
|
||||||
t.Errorf("destination = %q, want %q", got[0], want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestWatcherSweepsFilesRacingDirectoryCreation covers the tighter race where
|
|
||||||
// segments are written immediately after the mkdir, before the new watch can
|
|
||||||
// be registered. Those writes produce no event the watcher would receive, so
|
|
||||||
// the directory has to be swept when it is first seen.
|
|
||||||
func TestWatcherSweepsFilesRacingDirectoryCreation(t *testing.T) {
|
|
||||||
eventDir := t.TempDir()
|
|
||||||
queue := newTestQueue(t)
|
|
||||||
startWatcher(t, eventDir, "event", queue)
|
|
||||||
|
|
||||||
// Stage the tree elsewhere and move it into place, so the files are
|
|
||||||
// guaranteed to already exist the instant the directory appears.
|
|
||||||
staging := filepath.Join(t.TempDir(), "1080p")
|
|
||||||
if err := os.MkdirAll(staging, 0755); err != nil {
|
|
||||||
t.Fatalf("failed to stage: %v", err)
|
|
||||||
}
|
|
||||||
for _, name := range []string{"seg_1001.ts", "seg_1002.ts"} {
|
|
||||||
if err := os.WriteFile(filepath.Join(staging, name), []byte("payload"), 0644); err != nil {
|
|
||||||
t.Fatalf("failed to stage segment: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := os.Rename(staging, filepath.Join(eventDir, "1080p")); err != nil {
|
|
||||||
t.Fatalf("failed to move staged directory into place: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
got := waitForQueue(t, queue, 2)
|
|
||||||
want := []string{"event/1080p/seg_1001.ts", "event/1080p/seg_1002.ts"}
|
|
||||||
if len(got) != 2 || got[0] != want[0] || got[1] != want[1] {
|
|
||||||
t.Errorf("queued %v, want %v", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestWatcherDestinationIncludesEventName pins the layout agreement between
|
|
||||||
// live transfers and the catch-up path in QueueExistingFiles: both must write
|
|
||||||
// to <nas>/<event>/<resolution>/<file>.
|
|
||||||
func TestWatcherDestinationIncludesEventName(t *testing.T) {
|
|
||||||
eventDir := t.TempDir()
|
|
||||||
resolutionDir := filepath.Join(eventDir, "720p")
|
|
||||||
if err := os.MkdirAll(resolutionDir, 0755); err != nil {
|
|
||||||
t.Fatalf("setup failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
queue := newTestQueue(t)
|
|
||||||
startWatcher(t, eventDir, "my-event", queue)
|
|
||||||
|
|
||||||
if err := os.WriteFile(filepath.Join(resolutionDir, "seg_1001.ts"), []byte("payload"), 0644); err != nil {
|
|
||||||
t.Fatalf("failed to write segment: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
got := waitForQueue(t, queue, 1)
|
|
||||||
if len(got) != 1 {
|
|
||||||
t.Fatalf("expected one queued item, got %v", got)
|
|
||||||
}
|
|
||||||
if want := "my-event/720p/seg_1001.ts"; got[0] != want {
|
|
||||||
t.Errorf("destination = %q, want %q", got[0], want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestWatcherWithoutEventNameUsesRoot mirrors a download run with no -event:
|
|
||||||
// segments land under the local output root, so the NAS destination must not
|
|
||||||
// invent a directory level for them.
|
|
||||||
func TestWatcherWithoutEventNameUsesRoot(t *testing.T) {
|
|
||||||
eventDir := t.TempDir()
|
|
||||||
resolutionDir := filepath.Join(eventDir, "1080p")
|
|
||||||
if err := os.MkdirAll(resolutionDir, 0755); err != nil {
|
|
||||||
t.Fatalf("setup failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
queue := newTestQueue(t)
|
|
||||||
startWatcher(t, eventDir, "", queue)
|
|
||||||
|
|
||||||
if err := os.WriteFile(filepath.Join(resolutionDir, "seg_1001.ts"), []byte("payload"), 0644); err != nil {
|
|
||||||
t.Fatalf("failed to write segment: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
got := waitForQueue(t, queue, 1)
|
|
||||||
if len(got) != 1 {
|
|
||||||
t.Fatalf("expected one queued item, got %v", got)
|
|
||||||
}
|
|
||||||
if want := "1080p/seg_1001.ts"; got[0] != want {
|
|
||||||
t.Errorf("destination = %q, want %q", got[0], want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWatcherIgnoresNonSegmentFiles(t *testing.T) {
|
|
||||||
eventDir := t.TempDir()
|
|
||||||
queue := newTestQueue(t)
|
|
||||||
startWatcher(t, eventDir, "event", queue)
|
|
||||||
|
|
||||||
for _, name := range []string{"notes.txt", "manifest.json", "segment.ts.part"} {
|
|
||||||
if err := os.WriteFile(filepath.Join(eventDir, name), []byte("x"), 0644); err != nil {
|
|
||||||
t.Fatalf("failed to write %s: %v", name, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
time.Sleep(testSettlingDelay + 300*time.Millisecond)
|
|
||||||
if got := queuedDestinations(queue); len(got) != 0 {
|
|
||||||
t.Errorf("non-segment files should not be queued, got %v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestExtractResolution(t *testing.T) {
|
|
||||||
fw := &FileWatcher{}
|
|
||||||
|
|
||||||
cases := map[string]string{
|
|
||||||
filepath.Join("data", "event", "1080p", "seg.ts"): "1080p",
|
|
||||||
filepath.Join("data", "event", "720p", "seg.ts"): "720p",
|
|
||||||
filepath.Join("data", "event", "seg.ts"): "",
|
|
||||||
}
|
|
||||||
|
|
||||||
for path, want := range cases {
|
|
||||||
if got := fw.extractResolution(path); got != want {
|
|
||||||
t.Errorf("extractResolution(%q) = %q, want %q", path, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -24,28 +24,12 @@ func PathExists(path string) bool {
|
|||||||
return !os.IsNotExist(err)
|
return !os.IsNotExist(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsValidPath reports whether path is usable on Windows, the stricter of the
|
|
||||||
// two platforms this project targets, which reserves <>:"|?* in names.
|
|
||||||
//
|
|
||||||
// A leading drive letter is checked separately: its colon is legal there and
|
|
||||||
// nowhere else in the path. Without that carve-out the function rejects every
|
|
||||||
// absolute Windows path, including the drive-qualified ones it exists to
|
|
||||||
// validate.
|
|
||||||
func IsValidPath(path string) bool {
|
func IsValidPath(path string) bool {
|
||||||
if path == "" {
|
if path == "" {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
rest := path
|
return !strings.ContainsAny(path, "<>:\"|?*")
|
||||||
if len(rest) >= 2 && rest[1] == ':' && isDriveLetter(rest[0]) {
|
|
||||||
rest = rest[2:]
|
|
||||||
}
|
|
||||||
|
|
||||||
return !strings.ContainsAny(rest, `<>:"|?*`)
|
|
||||||
}
|
|
||||||
|
|
||||||
func isDriveLetter(c byte) bool {
|
|
||||||
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func NormalizePath(path string) string {
|
func NormalizePath(path string) string {
|
||||||
@ -53,13 +37,6 @@ func NormalizePath(path string) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func GetRelativePath(basePath, targetPath string) (string, error) {
|
func GetRelativePath(basePath, targetPath string) (string, error) {
|
||||||
// filepath.Rel("", "") succeeds and returns ".", which is a meaningless
|
|
||||||
// answer to a meaningless question; callers passing an empty path have a
|
|
||||||
// bug worth surfacing rather than papering over.
|
|
||||||
if basePath == "" || targetPath == "" {
|
|
||||||
return "", fmt.Errorf("both base and target paths are required (base %q, target %q)", basePath, targetPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
rel, err := filepath.Rel(basePath, targetPath)
|
rel, err := filepath.Rel(basePath, targetPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Errorf("failed to get relative path: %w", err)
|
return "", fmt.Errorf("failed to get relative path: %w", err)
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user