From 93b62618a179efd1afc65a6777ee13c570203674 Mon Sep 17 00:00:00 2001 From: kacarmichael Date: Sun, 20 Jul 2025 00:43:30 -0500 Subject: [PATCH] Structured for download project and processing project. Responsibility split --- .gitignore | 2 + .idea/m3u8-downloader.iml | 4 +- CLAUDE.md | 51 ++++++++++++++++++++++++++ cmd/downloader/main.go | 35 ++++++++++++++++++ cmd/proc/main.go | 1 + main.go | 27 -------------- {media => pkg/downloader}/client.go | 30 ++++++++++++++- {media => pkg/downloader}/constants.go | 3 +- {media => pkg/downloader}/parser.go | 2 +- {media => pkg/downloader}/playlist.go | 2 +- {media => pkg/downloader}/segment.go | 7 +++- {media => pkg/downloader}/stream.go | 4 +- 12 files changed, 133 insertions(+), 35 deletions(-) create mode 100644 .gitignore create mode 100644 CLAUDE.md create mode 100644 cmd/downloader/main.go create mode 100644 cmd/proc/main.go delete mode 100644 main.go rename {media => pkg/downloader}/client.go (79%) rename {media => pkg/downloader}/constants.go (90%) rename {media => pkg/downloader}/parser.go (98%) rename {media => pkg/downloader}/playlist.go (98%) rename {media => pkg/downloader}/segment.go (91%) rename {media => pkg/downloader}/stream.go (96%) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..66fa669 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +data/ +scratch.txt \ No newline at end of file diff --git a/.idea/m3u8-downloader.iml b/.idea/m3u8-downloader.iml index 5e764c4..2ec9634 100644 --- a/.idea/m3u8-downloader.iml +++ b/.idea/m3u8-downloader.iml @@ -2,7 +2,9 @@ - + + + diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..70ab624 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,51 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +This is a Go-based M3U8 downloader that parses HLS (HTTP Live Streaming) playlists to extract video and audio stream metadata. The end goal of this project is to have a listening REST API take in m3u8 urls, parse them, and eventually send to a conversion service. + +## Architecture + +The project follows a clean separation of concerns: + +- **main.go**: Entry point that demonstrates usage of the media package +- **media/**: Core package containing M3U8 parsing logic + - **types.go**: Contains the main parsing logic and data structures (`StreamSet`, `VideoURL`, `AudioURL`) + - **utils.go**: Utility functions for parsing attributes and resolution calculations + +The `GetStreamMetadata()` function is the main entry point that: +1. Fetches the M3U8 master playlist via HTTP +2. Parses the content line by line +3. Extracts video streams (`#EXT-X-STREAM-INF`) and audio streams (`#EXT-X-MEDIA`) +4. Returns a `StreamSet` containing all parsed metadata + +## Common Development Commands + +```bash +# Build the project +go build -o m3u8-downloader + +# Run the project +go run main.go + +# Run with module support +go mod tidy + +# Test the project (when tests are added) +go test ./... + +# Format code +go fmt ./... +``` + +## Key Data Structures + +- `StreamSet`: Root structure containing playlist URL and all streams +- `VideoURL`: Represents video stream with bandwidth, codecs, resolution, frame rate +- `AudioURL`: Represents audio stream with media type, group ID, name, and selection flags + +## Error Handling + +The current implementation uses `panic()` for error handling. When extending functionality, consider implementing proper error handling with returned error values following Go conventions. \ No newline at end of file diff --git a/cmd/downloader/main.go b/cmd/downloader/main.go new file mode 100644 index 0000000..1e0b7cf --- /dev/null +++ b/cmd/downloader/main.go @@ -0,0 +1,35 @@ +package main + +import ( + "fmt" + downloader2 "m3u8-downloader/pkg/downloader" +) + +func main() { + + masterUrl := "https://d17cyqyz9yhmep.cloudfront.net/streams/234945/playlist_1752291107574_1752292056713.m3u8" + + stream, err := downloader2.ParseMasterPlaylist(masterUrl) + if err != nil { + panic(err) + } + + audio, video, err := stream.FetchSegmentPlaylists() + if err != nil { + panic(err) + } + + videoPlaylist := downloader2.ParseMediaPlaylist(video) + audioPlaylist := downloader2.ParseMediaPlaylist(audio) + + for _, segment := range videoPlaylist.Segments { + fmt.Println(segment.URL) + } + + for _, segment := range audioPlaylist.Segments { + err := downloader2.DownloadTSFile(stream.BuildSegmentURL(segment.URL), downloader2.OutputDirPath) + if err != nil { + return + } + } +} diff --git a/cmd/proc/main.go b/cmd/proc/main.go new file mode 100644 index 0000000..6eb0478 --- /dev/null +++ b/cmd/proc/main.go @@ -0,0 +1 @@ +package proc diff --git a/main.go b/main.go deleted file mode 100644 index e7f4623..0000000 --- a/main.go +++ /dev/null @@ -1,27 +0,0 @@ -package main - -import ( - "fmt" - "m3u8-downloader/media" -) - -func main() { - - masterUrl := "https://d17cyqyz9yhmep.cloudfront.net/streams/234945/playlist_1752291107574_1752292056713.m3u8" - - stream, err := media.ParseMasterPlaylist(masterUrl) - if err != nil { - panic(err) - } - - audio, video, err := stream.FetchSegmentPlaylists() - if err != nil { - panic(err) - } - - videoPlaylist := media.ParseMediaPlaylist(video) - audioPlaylist := media.ParseMediaPlaylist(audio) - - fmt.Println(videoPlaylist) - fmt.Println(audioPlaylist) -} diff --git a/media/client.go b/pkg/downloader/client.go similarity index 79% rename from media/client.go rename to pkg/downloader/client.go index 25ba493..57b914d 100644 --- a/media/client.go +++ b/pkg/downloader/client.go @@ -1,4 +1,4 @@ -package media +package downloader import ( "context" @@ -6,6 +6,9 @@ import ( "fmt" "io" "net/http" + "os" + "path" + "path/filepath" "time" ) @@ -86,6 +89,31 @@ func FetchPlaylistContent(url string) (string, error) { return string(data), nil } +func DownloadTSFile(url string, outputDir string) error { + if err := os.MkdirAll(outputDir, 0755); err != nil { + return fmt.Errorf("failed to create directory: %w", err) + } + + fileName := path.Base(url) + filePath := filepath.Join(outputDir, fileName) + data, err := defaultClient.Get(url) + if err != nil { + return err + } + + out, err := os.Create(filePath) + if err != nil { + return err + } + defer out.Close() + + _, err = out.Write(data) + if err != nil { + return err + } + return nil +} + func FetchPlaylistContentWithContext(ctx context.Context, url string) (string, error) { data, err := defaultClient.GetWithContext(ctx, url) if err != nil { diff --git a/media/constants.go b/pkg/downloader/constants.go similarity index 90% rename from media/constants.go rename to pkg/downloader/constants.go index d068929..ee5378d 100644 --- a/media/constants.go +++ b/pkg/downloader/constants.go @@ -1,4 +1,4 @@ -package media +package downloader import "time" @@ -14,4 +14,5 @@ const ( ClientDefaultTimeout = 30 * time.Second HTTPUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0" PlaylistTypeVOD = "VOD" + OutputDirPath = "./data" ) diff --git a/media/parser.go b/pkg/downloader/parser.go similarity index 98% rename from media/parser.go rename to pkg/downloader/parser.go index 2865545..7a81c85 100644 --- a/media/parser.go +++ b/pkg/downloader/parser.go @@ -1,4 +1,4 @@ -package media +package downloader import ( "strconv" diff --git a/media/playlist.go b/pkg/downloader/playlist.go similarity index 98% rename from media/playlist.go rename to pkg/downloader/playlist.go index a85751d..3441f7e 100644 --- a/media/playlist.go +++ b/pkg/downloader/playlist.go @@ -1,4 +1,4 @@ -package media +package downloader import ( "fmt" diff --git a/media/segment.go b/pkg/downloader/segment.go similarity index 91% rename from media/segment.go rename to pkg/downloader/segment.go index 636a17d..e3197ca 100644 --- a/media/segment.go +++ b/pkg/downloader/segment.go @@ -1,7 +1,8 @@ -package media +package downloader import ( "errors" + "fmt" "strconv" "strings" ) @@ -44,6 +45,10 @@ func (s *StreamSet) FetchSegmentPlaylists() (videoPlaylist, audioPlaylist string return videoContent, audioContent, nil } +func (s *StreamSet) BuildSegmentURL(filename string) string { + return fmt.Sprintf("%s%s", HTTPSPrefix, s.Metadata.Domain+"/streams/"+s.Metadata.StreamID+"/a/5000/"+filename) +} + func ParseMediaPlaylist(content string) *MediaPlaylist { lines := strings.Split(content, "\n") var segments []Segment diff --git a/media/stream.go b/pkg/downloader/stream.go similarity index 96% rename from media/stream.go rename to pkg/downloader/stream.go index 1f724e9..56d97bd 100644 --- a/media/stream.go +++ b/pkg/downloader/stream.go @@ -1,4 +1,4 @@ -package media +package downloader import ( "errors" @@ -54,7 +54,7 @@ type AudioStream struct { func NewAudioStream(mediaInfo string) (*AudioStream, error) { if !strings.HasPrefix(mediaInfo, ExtXMedia) { - return nil, errors.New("invalid media info line") + return nil, errors.New("invalid downloader info line") } attributes := parseMediaAttributes(mediaInfo)