Working Version 1
This commit is contained in:
parent
9a9ca12cd9
commit
ca7bab9b34
@ -19,9 +19,8 @@ import (
|
||||
"github.com/grafov/m3u8"
|
||||
)
|
||||
|
||||
// ==== CONFIG ====
|
||||
const (
|
||||
MasterURL = "https://d17cyqyz9yhmep.cloudfront.net/streams/234951/playlist_vo_1752978025523_1752978954944.m3u8" // Replace with your .m3u8
|
||||
MasterURL = "https://d17cyqyz9yhmep.cloudfront.net/streams/234951/playlist_vo_1752978025523_1752978954944.m3u8"
|
||||
WorkerCount = 4
|
||||
RefreshDelay = 3 * time.Second
|
||||
|
||||
@ -30,16 +29,29 @@ const (
|
||||
OutputDirPath = "./data"
|
||||
)
|
||||
|
||||
// ==== TYPES ====
|
||||
type StreamVariant struct {
|
||||
URL string
|
||||
Bandwidth uint32
|
||||
BaseURL *url.URL
|
||||
ID int
|
||||
Resolution string
|
||||
OutputDir string
|
||||
}
|
||||
|
||||
type SegmentJob struct {
|
||||
URI string
|
||||
BaseURL *url.URL
|
||||
Seq uint64
|
||||
URI string
|
||||
Seq uint64
|
||||
VariantID int
|
||||
Variant *StreamVariant
|
||||
}
|
||||
|
||||
func (j SegmentJob) AbsoluteURL() string {
|
||||
rel, _ := url.Parse(j.URI)
|
||||
return j.BaseURL.ResolveReference(rel).String()
|
||||
return j.Variant.BaseURL.ResolveReference(rel).String()
|
||||
}
|
||||
|
||||
func (j SegmentJob) Key() string {
|
||||
return fmt.Sprintf("%d:%s", j.Seq, j.URI)
|
||||
}
|
||||
|
||||
type httpError struct {
|
||||
@ -48,12 +60,10 @@ type httpError struct {
|
||||
|
||||
func (e *httpError) Error() string { return fmt.Sprintf("http %d", e.code) }
|
||||
|
||||
// ==== MAIN ====
|
||||
func main() {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
// Handle Ctrl+C
|
||||
sigChan := make(chan os.Signal, 1)
|
||||
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
|
||||
go func() {
|
||||
@ -62,54 +72,118 @@ func main() {
|
||||
cancel()
|
||||
}()
|
||||
|
||||
jobs := make(chan SegmentJob, 10) // smaller buffer to avoid stale tokens
|
||||
var seen sync.Map
|
||||
variants, err := getAllVariants(MasterURL)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to get variants: %v", err)
|
||||
}
|
||||
log.Printf("Found %d variants", len(variants))
|
||||
|
||||
// Start playlist refresher
|
||||
go playlistRefresher(ctx, MasterURL, jobs, &seen, RefreshDelay)
|
||||
|
||||
// Start download workers
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < WorkerCount; i++ {
|
||||
sem := make(chan struct{}, WorkerCount*len(variants))
|
||||
|
||||
for _, variant := range variants {
|
||||
wg.Add(1)
|
||||
go func(id int) {
|
||||
go func(v *StreamVariant) {
|
||||
defer wg.Done()
|
||||
numErrors, errorIDs := segmentDownloader(ctx, id, jobs)
|
||||
if numErrors > 0 {
|
||||
log.Printf("Worker %d: %d errors: %v", id, numErrors, errorIDs)
|
||||
}
|
||||
}(i)
|
||||
variantDownloader(ctx, v, sem)
|
||||
}(variant)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
log.Println("All workers finished.")
|
||||
log.Println("All variant downloaders finished.")
|
||||
}
|
||||
|
||||
// ==== PLAYLIST LOGIC ====
|
||||
func playlistRefresher(ctx context.Context, masterURL string, jobs chan<- SegmentJob, seen *sync.Map, interval time.Duration) {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
variantURL, err := chooseVariant(masterURL)
|
||||
func getAllVariants(masterURL string) ([]*StreamVariant, error) {
|
||||
client := &http.Client{}
|
||||
req, _ := http.NewRequest("GET", masterURL, nil)
|
||||
req.Header.Set("User-Agent", HTTPUserAgent)
|
||||
req.Header.Set("Referer", REFERRER)
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
log.Printf("Variant selection failed, using master as media: %v", err)
|
||||
variantURL = masterURL
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
playlist, listType, err := m3u8.DecodeFrom(resp.Body, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
baseVariant, _ := url.Parse(variantURL)
|
||||
base, _ := url.Parse(masterURL)
|
||||
|
||||
if listType == m3u8.MEDIA {
|
||||
return []*StreamVariant{{
|
||||
URL: masterURL,
|
||||
Bandwidth: 0,
|
||||
BaseURL: base,
|
||||
ID: 0,
|
||||
Resolution: "unknown",
|
||||
OutputDir: path.Join(OutputDirPath, "unknown"),
|
||||
}}, nil
|
||||
}
|
||||
|
||||
master := playlist.(*m3u8.MasterPlaylist)
|
||||
if len(master.Variants) == 0 {
|
||||
return nil, fmt.Errorf("no variants found in master playlist")
|
||||
}
|
||||
|
||||
variants := make([]*StreamVariant, 0, len(master.Variants))
|
||||
for i, v := range master.Variants {
|
||||
vURL, _ := url.Parse(v.URI)
|
||||
fullURL := base.ResolveReference(vURL).String()
|
||||
resolution := extractResolution(v)
|
||||
outputDir := path.Join(OutputDirPath, resolution)
|
||||
variants = append(variants, &StreamVariant{
|
||||
URL: fullURL,
|
||||
Bandwidth: v.Bandwidth,
|
||||
BaseURL: base.ResolveReference(vURL),
|
||||
ID: i,
|
||||
Resolution: resolution,
|
||||
OutputDir: outputDir,
|
||||
})
|
||||
}
|
||||
return variants, nil
|
||||
}
|
||||
|
||||
func extractResolution(variant *m3u8.Variant) string {
|
||||
if variant.Resolution != "" {
|
||||
parts := strings.Split(variant.Resolution, "x")
|
||||
if len(parts) == 2 {
|
||||
return parts[1] + "p"
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case variant.Bandwidth >= 5000000:
|
||||
return "1080p"
|
||||
case variant.Bandwidth >= 3000000:
|
||||
return "720p"
|
||||
case variant.Bandwidth >= 1500000:
|
||||
return "480p"
|
||||
case variant.Bandwidth >= 800000:
|
||||
return "360p"
|
||||
default:
|
||||
return "240p"
|
||||
}
|
||||
}
|
||||
|
||||
func variantDownloader(ctx context.Context, variant *StreamVariant, sem chan struct{}) {
|
||||
log.Printf("Starting %s variant downloader (bandwidth: %d)", variant.Resolution, variant.Bandwidth)
|
||||
ticker := time.NewTicker(RefreshDelay)
|
||||
defer ticker.Stop()
|
||||
client := &http.Client{}
|
||||
seen := make(map[string]bool)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
close(jobs)
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
media, err := loadMediaPlaylist(variantURL)
|
||||
media, err := loadMediaPlaylist(variant.URL)
|
||||
seq := media.SeqNo
|
||||
if err != nil {
|
||||
log.Printf("Error loading media playlist: %v", err)
|
||||
log.Printf("%s: Error loading media playlist: %v", variant.Resolution, err)
|
||||
goto waitTick
|
||||
}
|
||||
|
||||
@ -117,66 +191,52 @@ func playlistRefresher(ctx context.Context, masterURL string, jobs chan<- Segmen
|
||||
if seg == nil {
|
||||
continue
|
||||
}
|
||||
key := fmt.Sprintf("%d:%s", seq, seg.URI)
|
||||
if _, loaded := seen.LoadOrStore(key, struct{}{}); !loaded {
|
||||
jobs <- SegmentJob{URI: seg.URI, BaseURL: baseVariant, Seq: seq}
|
||||
job := SegmentJob{
|
||||
URI: seg.URI,
|
||||
Seq: seq,
|
||||
VariantID: variant.ID,
|
||||
Variant: variant,
|
||||
}
|
||||
segmentKey := job.Key()
|
||||
if seen[segmentKey] {
|
||||
seq++
|
||||
continue
|
||||
}
|
||||
seen[segmentKey] = true
|
||||
|
||||
sem <- struct{}{} // Acquire
|
||||
go func(j SegmentJob) {
|
||||
defer func() { <-sem }() // Release
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
err := downloadSegment(ctx, client, j.AbsoluteURL(), j.Variant.OutputDir)
|
||||
name := strings.TrimSuffix(path.Base(j.Key()), path.Ext(path.Base(j.Key())))
|
||||
if err == nil {
|
||||
log.Printf("✓ %s downloaded segment %s", j.Variant.Resolution, name)
|
||||
} else if isHTTPStatus(err, 403) {
|
||||
log.Printf("✗ %s failed to download segment %s (403)", j.Variant.Resolution, name)
|
||||
} else {
|
||||
log.Printf("✗ %s failed to download segment %s: %v", j.Variant.Resolution, name, err)
|
||||
}
|
||||
}(job)
|
||||
seq++
|
||||
}
|
||||
|
||||
if media.Closed {
|
||||
log.Println("Playlist closed (#EXT-X-ENDLIST); closing jobs.")
|
||||
close(jobs)
|
||||
log.Printf("%s: Playlist closed (#EXT-X-ENDLIST)", variant.Resolution)
|
||||
return
|
||||
}
|
||||
|
||||
waitTick:
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
close(jobs)
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func chooseVariant(masterURL string) (string, error) {
|
||||
client := &http.Client{}
|
||||
req, _ := http.NewRequest("GET", masterURL, nil)
|
||||
req.Header.Set("User-Agent", HTTPUserAgent)
|
||||
req.Header.Set("Referer", REFERRER)
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
playlist, listType, err := m3u8.DecodeFrom(resp.Body, true)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
base, _ := url.Parse(masterURL)
|
||||
if listType == m3u8.MEDIA {
|
||||
return masterURL, nil
|
||||
}
|
||||
|
||||
master := playlist.(*m3u8.MasterPlaylist)
|
||||
if len(master.Variants) == 0 {
|
||||
return "", fmt.Errorf("no variants found in master playlist")
|
||||
}
|
||||
|
||||
best := master.Variants[0]
|
||||
for _, v := range master.Variants {
|
||||
if v.Bandwidth > best.Bandwidth {
|
||||
best = v
|
||||
}
|
||||
}
|
||||
|
||||
vURL, _ := url.Parse(best.URI)
|
||||
return base.ResolveReference(vURL).String(), nil
|
||||
}
|
||||
|
||||
func loadMediaPlaylist(mediaURL string) (*m3u8.MediaPlaylist, error) {
|
||||
client := &http.Client{}
|
||||
req, _ := http.NewRequest("GET", mediaURL, nil)
|
||||
@ -198,71 +258,59 @@ func loadMediaPlaylist(mediaURL string) (*m3u8.MediaPlaylist, error) {
|
||||
return pl.(*m3u8.MediaPlaylist), nil
|
||||
}
|
||||
|
||||
// ==== WORKERS ====
|
||||
func segmentDownloader(ctx context.Context, id int, jobs <-chan SegmentJob) (int, []string) {
|
||||
client := &http.Client{}
|
||||
numErrors := 0
|
||||
errorIDs := make([]string, 0)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return numErrors, errorIDs
|
||||
case job, ok := <-jobs:
|
||||
if !ok {
|
||||
return numErrors, errorIDs
|
||||
}
|
||||
abs := job.AbsoluteURL()
|
||||
if err := downloadSegment(client, abs); err != nil {
|
||||
if isHTTPStatus(err, 403) {
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if err2 := downloadSegment(client, abs); err2 != nil {
|
||||
fmt.Printf("Worker %d: 403 retry failed (%s): %v\n", id, path.Base(abs), err2)
|
||||
numErrors++
|
||||
errorIDs = append(errorIDs, path.Base(abs))
|
||||
} else {
|
||||
fmt.Printf("Worker %d: recovered 403 (%s)\n", id, path.Base(abs))
|
||||
}
|
||||
} else {
|
||||
fmt.Printf("Worker %d: failed %s: %v\n", id, path.Base(abs), err)
|
||||
}
|
||||
} else {
|
||||
fmt.Printf("Worker %d: downloaded %s\n", id, path.Base(abs))
|
||||
}
|
||||
func downloadSegment(ctx context.Context, client *http.Client, segmentURL string, outputDir string) error {
|
||||
for attempt := 0; attempt < 2; attempt++ {
|
||||
if attempt > 0 {
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", segmentURL, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("User-Agent", HTTPUserAgent)
|
||||
req.Header.Set("Referer", REFERRER)
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
if attempt == 1 {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
httpErr := &httpError{code: resp.StatusCode}
|
||||
if resp.StatusCode == 403 && attempt == 0 {
|
||||
continue
|
||||
}
|
||||
return httpErr
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(outputDir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create output directory: %w", err)
|
||||
}
|
||||
|
||||
fileName := safeFileName(path.Join(outputDir, path.Base(segmentURL)))
|
||||
out, err := os.Create(fileName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
n, err := io.Copy(out, resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n == 0 {
|
||||
return fmt.Errorf("zero-byte download for %s", segmentURL)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("exhausted retries")
|
||||
}
|
||||
|
||||
func downloadSegment(client *http.Client, segmentURL string) error {
|
||||
req, err := http.NewRequest("GET", segmentURL, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("User-Agent", HTTPUserAgent)
|
||||
req.Header.Set("Referer", REFERRER)
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
return &httpError{code: resp.StatusCode}
|
||||
}
|
||||
|
||||
fileName := safeFileName(path.Join(OutputDirPath, path.Base(segmentURL)))
|
||||
out, err := os.Create(fileName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
_, err = io.Copy(out, resp.Body)
|
||||
return err
|
||||
}
|
||||
|
||||
// ==== HELPERS ====
|
||||
func safeFileName(base string) string {
|
||||
if i := strings.IndexAny(base, "?&#"); i >= 0 {
|
||||
base = base[:i]
|
||||
|
||||
@ -1,10 +0,0 @@
|
||||
package constants
|
||||
|
||||
import "time"
|
||||
|
||||
const (
|
||||
OutputDirPath = "./data"
|
||||
PlaylistRefreshInterval = 3
|
||||
NumberOfWorkers = 4
|
||||
RefreshDelay = 3 * time.Second
|
||||
)
|
||||
@ -1,8 +0,0 @@
|
||||
package constants
|
||||
|
||||
const (
|
||||
HTTPUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36"
|
||||
HTTPPrefix = "http://"
|
||||
HTTPSPrefix = "https://"
|
||||
REFERRER = "https://www.flomarching.com"
|
||||
)
|
||||
@ -1,11 +0,0 @@
|
||||
package constants
|
||||
|
||||
const (
|
||||
ExtXStreamInf = "#EXT-X-STREAM-INF"
|
||||
ExtXMedia = "#EXT-X-MEDIA"
|
||||
ExtXTargetDuration = "#EXT-X-TARGETDURATION"
|
||||
ExtInf = "#EXTINF"
|
||||
ExtXEndList = "#EXT-X-ENDLIST"
|
||||
ExtXPlaylistType = "#EXT-X-PLAYLIST-TYPE"
|
||||
PlaylistTypeVOD = "VOD"
|
||||
)
|
||||
@ -1,70 +0,0 @@
|
||||
package downloader
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"m3u8-downloader/pkg/constants"
|
||||
"m3u8-downloader/pkg/http"
|
||||
"m3u8-downloader/pkg/media"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
type DownloadService struct {
|
||||
client *http.HTTPWrapper
|
||||
}
|
||||
|
||||
func GetDownloadService() *DownloadService {
|
||||
return &DownloadService{
|
||||
client: http.DefaultClient,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DownloadService) GetStreamMetadata(url string) (*media.PlaylistMetadata, error) {
|
||||
return media.GetPlaylistMetadata(url), nil
|
||||
}
|
||||
|
||||
func (s *DownloadService) ParseSegmentPlaylist(url string) (*media.SegmentPlaylist, error) {
|
||||
content, err := s.client.Get(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return media.ParseMediaPlaylist(string(content)), nil
|
||||
}
|
||||
|
||||
func (s *DownloadService) ParseMasterPlaylist(url string) (*media.StreamSet, error) {
|
||||
metadata := media.GetPlaylistMetadata(url)
|
||||
content, err := s.client.Get(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
video, audio := media.ParsePlaylistLines(string(content))
|
||||
master := media.NewMasterPlaylist(video, audio)
|
||||
return media.NewStreamSet(metadata, master), nil
|
||||
}
|
||||
|
||||
func (s *DownloadService) DownloadFile(url string) error {
|
||||
fmt.Println("Downloading: " + url)
|
||||
if err := os.MkdirAll(constants.OutputDirPath, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create directory: %w", err)
|
||||
}
|
||||
|
||||
fileName := path.Base(url)
|
||||
filePath := filepath.Join(constants.OutputDirPath, fileName)
|
||||
data, err := s.client.Get(url)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
out, err := os.Create(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
_, err = out.Write(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -1,70 +0,0 @@
|
||||
package downloader
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"m3u8-downloader/pkg/media"
|
||||
"net/url"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
func resolveURL(baseURL string, segmentURL string) (string, error) {
|
||||
base, err := url.Parse(baseURL)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
segment, err := url.Parse(segmentURL)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return base.ResolveReference(segment).String(), nil
|
||||
}
|
||||
|
||||
func (s *DownloadService) DownloadWorker(id int, segmentChan <-chan media.Segment, wg *sync.WaitGroup, baseURL string) (int, int) {
|
||||
defer wg.Done()
|
||||
|
||||
numErrors := 0
|
||||
numDownloads := 0
|
||||
|
||||
for segment := range segmentChan {
|
||||
|
||||
cleanedURL, err := resolveURL(baseURL, segment.URL)
|
||||
if err != nil {
|
||||
fmt.Printf("[Worker %d] Error: %s\n", id, err)
|
||||
return -1, -1
|
||||
}
|
||||
fmt.Printf("[Worker %d] Downloading: %s\n", id, cleanedURL)
|
||||
downloadErr := s.DownloadFile(cleanedURL)
|
||||
if downloadErr != nil {
|
||||
if downloadErr.Error() == "HTTP 403: 403 Forbidden" {
|
||||
fmt.Printf("[Worker %d] URL Forbidden: %s\n", id, cleanedURL)
|
||||
numErrors++
|
||||
continue
|
||||
}
|
||||
fmt.Printf("[Worker %d] Error: %s\n", id, downloadErr)
|
||||
numErrors++
|
||||
return -1, -1
|
||||
}
|
||||
numDownloads++
|
||||
}
|
||||
return numErrors, numDownloads
|
||||
}
|
||||
|
||||
func PlaylistRefreshWorker(ctx context.Context, playlistURL string, segmentsChan chan<- string, seen *sync.Map) {
|
||||
ticker := time.NewTicker(30 * time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
segments, err := GetSegmentURLs(playlistURL)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
continue
|
||||
}
|
||||
for _, segment := range segments {
|
||||
if _, ok := seen.Load(segment); !ok {
|
||||
seen.Store(segment, true)
|
||||
}
|
||||
@ -1,67 +0,0 @@
|
||||
package dvr
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"m3u8-downloader/pkg/constants"
|
||||
"net/http"
|
||||
"os"
|
||||
)
|
||||
|
||||
// segmentDownloader processes segment URLs and downloads them
|
||||
func SegmentDownloader(ctx context.Context, id int, segmentsChan <-chan string) {
|
||||
client := &http.Client{}
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case seg, ok := <-segmentsChan:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := downloadSegment(client, seg); err != nil {
|
||||
fmt.Printf("Worker %d: failed to download %s: %v\n", id, seg, err)
|
||||
} else {
|
||||
fmt.Printf("Worker %d: downloaded %s\n", id, seg)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func downloadSegment(client *http.Client, segmentURL string) error {
|
||||
req, err := http.NewRequest("GET", segmentURL, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("User-Agent", constants.HTTPUserAgent)
|
||||
req.Header.Set("Referer", constants.REFERRER)
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
// Save the segment (or buffer it, depending on your DVR strategy)
|
||||
fileName := extractSegmentName(segmentURL)
|
||||
out, err := os.Create(fileName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
_, err = io.Copy(out, resp.Body)
|
||||
return err
|
||||
}
|
||||
|
||||
// extractSegmentName is a placeholder that turns the URL into a local filename
|
||||
func extractSegmentName(segmentURL string) string {
|
||||
// TODO: Implement robust name parsing
|
||||
return "segment.ts"
|
||||
}
|
||||
@ -1,73 +0,0 @@
|
||||
package dvr
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/grafov/m3u8"
|
||||
)
|
||||
|
||||
// playlistRefresher periodically fetches the playlist and enqueues new segments
|
||||
func PlaylistRefresher(ctx context.Context, playlistURL string, segmentsChan chan<- string, seen *sync.Map, interval time.Duration) {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
close(segmentsChan)
|
||||
return
|
||||
case <-ticker.C:
|
||||
segments, err := fetchPlaylistSegments(playlistURL)
|
||||
if err != nil {
|
||||
log.Printf("Error fetching playlist: %v", err)
|
||||
continue
|
||||
}
|
||||
for _, seg := range segments {
|
||||
if _, exists := seen.LoadOrStore(seg, true); !exists {
|
||||
segmentsChan <- seg
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// fetchPlaylistSegments is a placeholder that parses the M3U8 and returns full URLs
|
||||
func fetchPlaylistSegments(playlistURL string) ([]string, error) {
|
||||
resp, err := http.Get(playlistURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
playlist, listType, err := m3u8.DecodeFrom(resp.Body, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if listType == m3u8.MASTER {
|
||||
return nil, fmt.Errorf("playlist is a master playlist")
|
||||
}
|
||||
|
||||
media := playlist.(*m3u8.MediaPlaylist)
|
||||
base, err := url.Parse(playlistURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var segments []string
|
||||
for _, segment := range media.Segments {
|
||||
if segment == nil {
|
||||
continue
|
||||
}
|
||||
rel, err := url.Parse(segment.URI)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
segments = append(segments, base.ResolveReference(rel).String())
|
||||
}
|
||||
return segments, nil
|
||||
}
|
||||
@ -1,65 +0,0 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"m3u8-downloader/pkg/constants"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
type Client interface {
|
||||
Get(url string) ([]byte, error)
|
||||
GetWithContext(ctx context.Context, url string) ([]byte, error)
|
||||
}
|
||||
|
||||
type HTTPWrapper struct {
|
||||
client *http.Client
|
||||
headers map[string]string
|
||||
}
|
||||
|
||||
func (c *HTTPWrapper) Get(url string) ([]byte, error) {
|
||||
return c.GetWithContext(context.Background(), url)
|
||||
}
|
||||
|
||||
func (c *HTTPWrapper) GetWithContext(ctx context.Context, url string) ([]byte, error) {
|
||||
if !ValidateURL(url) {
|
||||
return nil, errors.New("invalid URL format")
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
// Set headers
|
||||
for key, value := range c.headers {
|
||||
req.Header.Set(key, value)
|
||||
}
|
||||
|
||||
resp, err := c.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, resp.Status)
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read response: %w", err)
|
||||
}
|
||||
|
||||
return data, nil
|
||||
}
|
||||
|
||||
var DefaultClient = &HTTPWrapper{
|
||||
client: &http.Client{},
|
||||
headers: map[string]string{
|
||||
"User-Agent": constants.HTTPUserAgent,
|
||||
"Referer": constants.REFERRER,
|
||||
},
|
||||
}
|
||||
@ -1,10 +0,0 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"m3u8-downloader/pkg/constants"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func ValidateURL(url string) bool {
|
||||
return strings.HasPrefix(url, constants.HTTPPrefix) || strings.HasPrefix(url, constants.HTTPSPrefix)
|
||||
}
|
||||
@ -1,60 +0,0 @@
|
||||
package media
|
||||
|
||||
type MasterPlaylist struct {
|
||||
VideoFeeds []VideoPlaylist
|
||||
AudioFeeds []AudioPlaylist
|
||||
}
|
||||
|
||||
func NewMasterPlaylist(videoFeeds []VideoPlaylist, audioFeeds []AudioPlaylist) *MasterPlaylist {
|
||||
return &MasterPlaylist{
|
||||
VideoFeeds: videoFeeds,
|
||||
AudioFeeds: audioFeeds,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *MasterPlaylist) SelectBestQualityStreams() (*VideoPlaylist, *AudioPlaylist) {
|
||||
if len(m.VideoFeeds) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
bestVideo := m.VideoFeeds[0]
|
||||
maxPixels := ResolutionToPixels(bestVideo.Resolution)
|
||||
|
||||
for _, video := range m.VideoFeeds {
|
||||
pixels := ResolutionToPixels(video.Resolution)
|
||||
if video.Bandwidth > bestVideo.Bandwidth || pixels > maxPixels {
|
||||
bestVideo = video
|
||||
maxPixels = pixels
|
||||
}
|
||||
}
|
||||
|
||||
return &bestVideo, m.FindAudioFeedByGroup(bestVideo.AudioGroup)
|
||||
}
|
||||
|
||||
func (s *StreamSet) SelectBestQualityStreams() (*VideoPlaylist, *AudioPlaylist) {
|
||||
if len(s.Master.VideoFeeds) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
bestVideo := s.Master.VideoFeeds[0]
|
||||
maxPixels := ResolutionToPixels(bestVideo.Resolution)
|
||||
|
||||
for _, video := range s.Master.VideoFeeds {
|
||||
pixels := ResolutionToPixels(video.Resolution)
|
||||
if video.Bandwidth > bestVideo.Bandwidth || pixels > maxPixels {
|
||||
bestVideo = video
|
||||
maxPixels = pixels
|
||||
}
|
||||
}
|
||||
|
||||
return &bestVideo, s.Master.FindAudioFeedByGroup(bestVideo.AudioGroup)
|
||||
}
|
||||
|
||||
func (m *MasterPlaylist) FindAudioFeedByGroup(groupID string) *AudioPlaylist {
|
||||
for _, audio := range m.AudioFeeds {
|
||||
if audio.GroupID == groupID {
|
||||
return &audio
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -1,151 +0,0 @@
|
||||
package media
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"m3u8-downloader/pkg/constants"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type VideoPlaylist struct {
|
||||
URL string
|
||||
Bandwidth int
|
||||
Codecs string
|
||||
Resolution string
|
||||
FrameRate string
|
||||
AudioGroup string
|
||||
Segments SegmentPlaylist
|
||||
}
|
||||
|
||||
func NewVideoStream(streamInfo, url string) (*VideoPlaylist, error) {
|
||||
hasAudio := strings.Contains(strings.ToLower(streamInfo), "audio")
|
||||
if !strings.HasPrefix(streamInfo, constants.ExtXStreamInf) {
|
||||
return nil, errors.New("invalid stream info line")
|
||||
}
|
||||
|
||||
reg := regexp.MustCompile(`([A-Z0-9-]+)=(".*?"|[^,]*)`)
|
||||
matches := reg.FindAllStringSubmatch(streamInfo, -1)
|
||||
|
||||
if len(matches) < 5 {
|
||||
fmt.Println("Less than 5 stream attributes found - audio likely disabled")
|
||||
}
|
||||
|
||||
bandwidth, err := strconv.Atoi(matches[0][2])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if hasAudio {
|
||||
return &VideoPlaylist{
|
||||
URL: url,
|
||||
Bandwidth: bandwidth,
|
||||
Codecs: StripQuotes(matches[1][2]),
|
||||
Resolution: StripQuotes(matches[2][2]),
|
||||
FrameRate: StripQuotes(matches[3][2]),
|
||||
AudioGroup: StripQuotes(matches[4][2]),
|
||||
}, nil
|
||||
}
|
||||
|
||||
return &VideoPlaylist{
|
||||
URL: url,
|
||||
Bandwidth: bandwidth,
|
||||
Codecs: StripQuotes(matches[1][2]),
|
||||
Resolution: StripQuotes(matches[2][2]),
|
||||
FrameRate: StripQuotes(matches[3][2]),
|
||||
}, nil
|
||||
|
||||
}
|
||||
|
||||
func (v *VideoPlaylist) BuildPlaylistURL(filename string) string {
|
||||
return fmt.Sprintf("%s%s", constants.HTTPSPrefix, v.URL+"/a/5000/"+filename)
|
||||
}
|
||||
|
||||
type AudioPlaylist struct {
|
||||
URL string
|
||||
MediaType string
|
||||
GroupID string
|
||||
Name string
|
||||
IsDefault bool
|
||||
AutoSelect bool
|
||||
Segments SegmentPlaylist
|
||||
}
|
||||
|
||||
func (a *AudioPlaylist) BuildPlaylistURL(filename string) string {
|
||||
return fmt.Sprintf("%s%s", constants.HTTPSPrefix, a.URL+"/a/5000/"+filename)
|
||||
}
|
||||
|
||||
func NewAudioStream(mediaInfo string) (*AudioPlaylist, error) {
|
||||
if !strings.HasPrefix(mediaInfo, constants.ExtXMedia) {
|
||||
return nil, errors.New("invalid downloader info line")
|
||||
}
|
||||
|
||||
attributes := ParseMediaAttributes(mediaInfo)
|
||||
|
||||
return &AudioPlaylist{
|
||||
URL: StripQuotes(attributes["URI"]),
|
||||
MediaType: StripQuotes(attributes["TYPE"]),
|
||||
GroupID: StripQuotes(attributes["GROUP-ID"]),
|
||||
Name: StripQuotes(attributes["NAME"]),
|
||||
IsDefault: attributes["DEFAULT"] == "YES",
|
||||
AutoSelect: attributes["AUTOSELECT"] == "YES",
|
||||
}, nil
|
||||
}
|
||||
|
||||
type PlaylistMetadata struct {
|
||||
URL string
|
||||
Domain string
|
||||
StreamID string
|
||||
}
|
||||
|
||||
func GetPlaylistMetadata(masterURL string) *PlaylistMetadata {
|
||||
strippedURL := strings.Replace(masterURL, constants.HTTPSPrefix, "", 1)
|
||||
strippedURL = strings.Replace(strippedURL, constants.HTTPPrefix, "", 1)
|
||||
urlPrefix := strings.Split(strippedURL, "/")[0]
|
||||
|
||||
return &PlaylistMetadata{
|
||||
URL: masterURL,
|
||||
Domain: urlPrefix,
|
||||
StreamID: strings.Split(strippedURL, "/")[2],
|
||||
}
|
||||
}
|
||||
|
||||
func ParsePlaylistLines(content string) ([]VideoPlaylist, []AudioPlaylist) {
|
||||
lines := strings.Split(content, "\n")
|
||||
var videoStreams []VideoPlaylist
|
||||
var audioStreams []AudioPlaylist
|
||||
|
||||
for i, line := range lines {
|
||||
if strings.HasPrefix(line, constants.ExtXStreamInf) && i+1 < len(lines) {
|
||||
if video, err := NewVideoStream(line, lines[i+1]); err == nil {
|
||||
videoStreams = append(videoStreams, *video)
|
||||
}
|
||||
} else if strings.HasPrefix(line, constants.ExtXMedia) {
|
||||
if audio, err := NewAudioStream(line); err == nil {
|
||||
audioStreams = append(audioStreams, *audio)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return videoStreams, audioStreams
|
||||
}
|
||||
|
||||
func ResolutionToPixels(resolution string) int {
|
||||
parts := strings.Split(resolution, "x")
|
||||
if len(parts) != 2 {
|
||||
return 0
|
||||
}
|
||||
|
||||
width, err := strconv.Atoi(parts[0])
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
height, err := strconv.Atoi(parts[1])
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
return width * height
|
||||
}
|
||||
@ -1,65 +0,0 @@
|
||||
package media
|
||||
|
||||
import (
|
||||
"m3u8-downloader/pkg/constants"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func ParseAttribute(attribute string) string {
|
||||
parts := strings.Split(attribute, "=")
|
||||
if len(parts) >= 2 {
|
||||
return parts[1]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func StripQuotes(input string) string {
|
||||
return strings.Trim(input, "\"")
|
||||
}
|
||||
|
||||
func ParseMediaAttributes(mediaInfo string) map[string]string {
|
||||
attributes := make(map[string]string)
|
||||
|
||||
// Remove the #EXT-X-MEDIA: prefix
|
||||
content := strings.TrimPrefix(mediaInfo, constants.ExtXMedia+":")
|
||||
|
||||
// Split by comma, but respect quoted values
|
||||
parts := splitRespectingQuotes(content, ',')
|
||||
|
||||
for _, part := range parts {
|
||||
if kv := strings.SplitN(strings.TrimSpace(part), "=", 2); len(kv) == 2 {
|
||||
attributes[kv[0]] = kv[1]
|
||||
}
|
||||
}
|
||||
|
||||
return attributes
|
||||
}
|
||||
|
||||
func splitRespectingQuotes(input string, delimiter rune) []string {
|
||||
var result []string
|
||||
var current strings.Builder
|
||||
inQuotes := false
|
||||
|
||||
for _, char := range input {
|
||||
switch char {
|
||||
case '"':
|
||||
inQuotes = !inQuotes
|
||||
current.WriteRune(char)
|
||||
case delimiter:
|
||||
if inQuotes {
|
||||
current.WriteRune(char)
|
||||
} else {
|
||||
result = append(result, current.String())
|
||||
current.Reset()
|
||||
}
|
||||
default:
|
||||
current.WriteRune(char)
|
||||
}
|
||||
}
|
||||
|
||||
if current.Len() > 0 {
|
||||
result = append(result, current.String())
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
@ -1,112 +0,0 @@
|
||||
package media
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"m3u8-downloader/pkg/constants"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Segment struct {
|
||||
URL string
|
||||
Duration float64
|
||||
Sequence int
|
||||
}
|
||||
|
||||
type SegmentPlaylist struct {
|
||||
SegmentList []Segment
|
||||
TargetDuration float64
|
||||
IsLive bool
|
||||
HasEndList bool
|
||||
}
|
||||
|
||||
//func (s *StreamSet) FetchSegmentPlaylists() (videoPlaylist, audioPlaylist string, err error) {
|
||||
// video, audio := s.SelectBestQualityStreams()
|
||||
// if video == nil {
|
||||
// return "", "", errors.New("no VideoFeeds streams available")
|
||||
// }
|
||||
//
|
||||
// videoURL := s.BuildStreamURL(video.URL)
|
||||
//
|
||||
// videoContent, err := http.FetchPlaylistContent(StripQuotes(videoURL))
|
||||
// if err != nil {
|
||||
// return "", "", err
|
||||
// }
|
||||
//
|
||||
// var audioContent string
|
||||
// if audio != nil {
|
||||
// audioURL := s.BuildStreamURL(StripQuotes(audio.URL))
|
||||
// audioContent, err = http.FetchPlaylistContent(StripQuotes(audioURL))
|
||||
// if err != nil {
|
||||
// return "", "", err
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// return videoContent, audioContent, nil
|
||||
//}
|
||||
|
||||
func (s *StreamSet) BuildSegmentURL(filename string) string {
|
||||
return fmt.Sprintf("%s%s", constants.HTTPSPrefix, s.Metadata.Domain+"/streams/"+s.Metadata.StreamID+"/"+filename)
|
||||
}
|
||||
|
||||
func ParseMediaPlaylist(content string) *SegmentPlaylist {
|
||||
lines := strings.Split(content, "\n")
|
||||
var segments []Segment
|
||||
|
||||
playlist := &SegmentPlaylist{
|
||||
IsLive: true,
|
||||
HasEndList: false,
|
||||
}
|
||||
|
||||
mediaSequence := 0
|
||||
|
||||
for i, line := range lines {
|
||||
switch {
|
||||
case strings.HasPrefix(line, constants.ExtXTargetDuration):
|
||||
playlist.TargetDuration = parseTargetDuration(line)
|
||||
|
||||
case strings.HasPrefix(line, constants.ExtInf):
|
||||
if i+1 < len(lines) {
|
||||
duration := parseSegmentDuration(line)
|
||||
segments = append(segments, Segment{
|
||||
URL: lines[i+1],
|
||||
Duration: duration,
|
||||
Sequence: mediaSequence,
|
||||
})
|
||||
mediaSequence++
|
||||
}
|
||||
|
||||
case strings.HasPrefix(line, constants.ExtXPlaylistType):
|
||||
if strings.Contains(line, constants.PlaylistTypeVOD) {
|
||||
playlist.IsLive = false
|
||||
}
|
||||
|
||||
case strings.HasPrefix(line, constants.ExtXEndList):
|
||||
playlist.HasEndList = true
|
||||
}
|
||||
}
|
||||
|
||||
playlist.SegmentList = segments
|
||||
return playlist
|
||||
}
|
||||
|
||||
func parseTargetDuration(line string) float64 {
|
||||
parts := strings.SplitN(line, ":", 2)
|
||||
if len(parts) == 2 {
|
||||
if duration, err := strconv.ParseFloat(strings.TrimSpace(parts[1]), 64); err == nil {
|
||||
return duration
|
||||
}
|
||||
}
|
||||
return 0.0
|
||||
}
|
||||
|
||||
func parseSegmentDuration(line string) float64 {
|
||||
parts := strings.Split(line, ":")
|
||||
if len(parts) >= 2 {
|
||||
durationPart := strings.Split(parts[1], ",")[0]
|
||||
if duration, err := strconv.ParseFloat(durationPart, 64); err == nil {
|
||||
return duration
|
||||
}
|
||||
}
|
||||
return 0.0
|
||||
}
|
||||
@ -1,22 +0,0 @@
|
||||
package media
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"m3u8-downloader/pkg/constants"
|
||||
)
|
||||
|
||||
type StreamSet struct {
|
||||
Metadata *PlaylistMetadata
|
||||
Master *MasterPlaylist
|
||||
}
|
||||
|
||||
func NewStreamSet(metadata *PlaylistMetadata, master *MasterPlaylist) *StreamSet {
|
||||
return &StreamSet{
|
||||
Metadata: metadata,
|
||||
Master: master,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *StreamSet) BuildPlaylistURL(url string) string {
|
||||
return fmt.Sprintf("%s%s", constants.HTTPSPrefix, s.Metadata.Domain+"/streams/"+s.Metadata.StreamID+"/"+url)
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user