Skip to content

Commit c95eea4

Browse files
authored
Merge pull request #52 from kaya70875/dev
[RELEASE] YTFetcher v2.2
2 parents 4a33637 + 8c19318 commit c95eea4

15 files changed

Lines changed: 584 additions & 153 deletions

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
venv/
22
.venv/
3+
.vscode
34
__pycache__/
45
.pytest_cache/
56
.mypy_cache/

CHANGELOG.md

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,23 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
66

77
## [Unreleased]
88
### Added
9+
- Added tab option for both CLI and Python API to fetch from different tabs for a channel. ('videos', 'shorts', 'streams')
10+
- Verbose logging mode via `--verbose` CLI flag.
11+
- Comprehensive debug and info logs for core operations.
12+
13+
### Changed
14+
- Updated transcript languages behavior for better UX and less friction.
15+
- Improved log messages and levels for better clarity.
16+
- Refactored filtering logic into a separate method.
17+
18+
### Fixed
19+
- Fixed session resource leak by closing `requests.session` properly in `TranscriptFetcher`.
20+
- Improved CLI error handling for graceful exits on exceptions.
21+
- Fix users will be forcing to fetch only english transcripts if they are not set a `languages` parameter.
22+
- Fixed possible data loss in `YoutubeDLFetcher`.
23+
24+
## [2.1] - 2026-02-16
25+
### Added
926
- Added `convert_to_rows` utility method for converting `ChannelData` objects to Python dict for easily feed data to ML and RAG pipelines.
1027
- Added built-in cache strategy for fetching transcripts.
1128
- Added CLI argument for channel fetcher and playlist fetcher; `--all` argument now fetches ALL videos from a channel or playlist.

README.md

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,28 @@ print(channel_data)
130130

131131
```
132132

133+
### Fetching Specific Channel Tabs (Videos / Shorts / Streams)
134+
135+
Use the `tab` parameter in `from_channel()` to select which section of a channel to fetch.
136+
137+
Available options:
138+
- `'videos'` (default)
139+
- `'shorts'`
140+
- `'streams'`
141+
142+
If not specified, the fetcher defaults to the **Videos** tab.
143+
144+
```python
145+
# Fetch regular videos (default)
146+
YTFetcher.from_channel(channel_handle="handle")
147+
148+
# Fetch Shorts
149+
YTFetcher.from_channel(channel_handle="handle", tab="shorts")
150+
151+
# Fetch live streams
152+
YTFetcher.from_channel(channel_handle="handle", tab="streams")
153+
```
154+
133155
---
134156

135157
This will return a list of `ChannelData` with metadata in `DLSnippet` objects:
@@ -625,9 +647,23 @@ fetcher = YTFetcher.from_channel(
625647
### Basic Usage
626648

627649
```bash
628-
ytfetcher <CHANNEL_HANDLE> -m <MAX_RESULTS> -f <FORMAT>
650+
ytfetcher channel <CHANNEL_HANDLE> -m <MAX_RESULTS> -f <FORMAT>
629651
```
630652

653+
### Fetching Different Channel Tabs (Videos / Shorts / Streams)
654+
655+
Use `--tab` to choose which channel feed should be fetched.
656+
657+
```bash
658+
# Default: videos
659+
ytfetcher channel TheOffice -m 20 --tab videos -f json
660+
661+
# Fetch from the Shorts tab
662+
ytfetcher channel TheOffice -m 20 --tab shorts -f json
663+
664+
# Fetch from the Live/Streams tab
665+
ytfetcher channel TheOffice -m 20 --tab streams -f json
666+
631667
### Fetching by Video IDs
632668

633669
```bash

docs/cli.md

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ ytfetcher channel <CHANNEL_HANDLE> -m <MAX_RESULTS> -f <FORMAT>
4646
**Optional Arguments:**
4747

4848
- `-m`, `--max-results` - Maximum number of videos to fetch (default: 20)
49+
- `-t`, `--tab` - Choose the channel tab to fetch from: `videos`, `shorts`, `streams` (default: `videos`)
4950
- `--all` - Fetch ALL videos from a channel.
5051

5152
**Example:**
@@ -54,6 +55,19 @@ ytfetcher channel <CHANNEL_HANDLE> -m <MAX_RESULTS> -f <FORMAT>
5455
ytfetcher channel TheOffice -m 20 -f json
5556
```
5657

58+
**Channel Tab Examples:**
59+
60+
```bash
61+
# Regular uploads (default)
62+
ytfetcher channel TheOffice -m 20 --tab videos -f json
63+
64+
# Shorts feed
65+
ytfetcher channel TheOffice -m 20 --tab shorts -f json
66+
67+
# Live/Streams feed
68+
ytfetcher channel TheOffice -m 20 --tab streams -f json
69+
```
70+
5771
!!! Note
5872
You can use channel handles with or without the `@` symbol, or even full URLs like `https://www.youtube.com/@TheOffice`.
5973

docs/index.md

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,28 @@ ChannelData(
7474
]
7575
```
7676

77+
### Fetching Specific Channel Tabs (Videos / Shorts / Streams)
78+
79+
Use the `tab` parameter in `from_channel()` to select which section of a channel to fetch.
80+
81+
Available options:
82+
- `'videos'` (default)
83+
- `'shorts'`
84+
- `'streams'`
85+
86+
If not specified, the fetcher defaults to the **Videos** tab.
87+
88+
```python
89+
# Fetch regular videos (default)
90+
YTFetcher.from_channel(channel_handle="handle")
91+
92+
# Fetch Shorts
93+
YTFetcher.from_channel(channel_handle="handle", tab="shorts")
94+
95+
# Fetch live streams
96+
YTFetcher.from_channel(channel_handle="handle", tab="streams")
97+
```
98+
7799
### Preview Data
78100
You can also preview this data using `PreviewRenderer` class from `ytfetcher.services`:
79101
```py

tests/cli/test_cli.py

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -101,10 +101,11 @@ def test_comments_passed_correctly_to_ytfetcher(mock_ytfetcher, mock_configurati
101101
mock_ytfetcher.from_channel.assert_called_once_with(
102102
channel_handle="TestChannel",
103103
max_results=20,
104+
tab='videos',
104105
options=FetchOptions(
105106
http_config=expected_http_config,
106107
proxy_config=expected_proxy_config,
107-
languages=["en"],
108+
languages=None,
108109
manually_created=False,
109110
filters=[]
110111
)
@@ -132,10 +133,11 @@ def test_comments_only_passed_correctly_to_ytfetcher(mock_ytfetcher, mock_config
132133
mock_ytfetcher.from_channel.assert_called_once_with(
133134
channel_handle="TestChannel",
134135
max_results=20,
136+
tab='videos',
135137
options=FetchOptions(
136138
http_config=expected_http_config,
137139
proxy_config=expected_proxy_config,
138-
languages=["en"],
140+
languages=None,
139141
manually_created=False,
140142
filters=[]
141143
)
@@ -164,10 +166,11 @@ def test_run_from_channel_arguments_passed_correctly_to_ytfetcher(mock_ytfetcher
164166
mock_ytfetcher.from_channel.assert_called_once_with(
165167
channel_handle="TestChannel",
166168
max_results=20,
169+
tab='videos',
167170
options=FetchOptions(
168171
http_config=expected_http_config,
169172
proxy_config=expected_proxy_config,
170-
languages=["en"],
173+
languages=None,
171174
manually_created=False,
172175
filters=[]
173176
)
@@ -195,10 +198,11 @@ def test_all_argument_for_channel_method(mock_ytfetcher, mock_configurations):
195198
mock_ytfetcher.from_channel.assert_called_once_with(
196199
channel_handle="TestChannel",
197200
max_results=None,
201+
tab='videos',
198202
options=FetchOptions(
199203
http_config=expected_http_config,
200204
proxy_config=expected_proxy_config,
201-
languages=["en"],
205+
languages=None,
202206
manually_created=False,
203207
filters=[]
204208
)
@@ -229,7 +233,7 @@ def test_all_argument_for_playlist_method(mock_ytfetcher, mock_configurations):
229233
options=FetchOptions(
230234
http_config=expected_http_config,
231235
proxy_config=expected_proxy_config,
232-
languages=["en"],
236+
languages=None,
233237
manually_created=False,
234238
filters=[]
235239
)

tests/test_transcript_fetcher.py

Lines changed: 124 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,9 @@
11
import pytest
2-
import time
32
from pytest_mock import MockerFixture
4-
from unittest.mock import MagicMock
5-
from ytfetcher.models.channel import VideoTranscript, Transcript, ChannelData
3+
from youtube_transcript_api._errors import NoTranscriptFound
4+
from ytfetcher.models.channel import VideoTranscript, Transcript
65
from ytfetcher._transcript_fetcher import TranscriptFetcher
76
from ytfetcher.config.http_config import HTTPConfig
8-
from youtube_transcript_api import YouTubeTranscriptApi
9-
from youtube_transcript_api._transcripts import FetchedTranscript, FetchedTranscriptSnippet
107
from youtube_transcript_api.proxies import GenericProxyConfig
118

129
@pytest.fixture
@@ -46,42 +43,48 @@ def test_fetch_method_returns_correct_data(mocker: MockerFixture, mock_video_ids
4643
assert results[0].transcripts[0].text == 'text1'
4744
assert results[0].video_id == 'video_id'
4845

49-
def test_fetch_single_returns_correct_data(mocker: MockerFixture, mock_video_ids):
46+
def test_fetch_single_returns_correct_data(mocker, mock_video_ids):
5047
fetcher = TranscriptFetcher(mock_video_ids)
5148

52-
mocker.patch.object(YouTubeTranscriptApi, "fetch",
53-
return_value=FetchedTranscript([FetchedTranscriptSnippet(text="text", start=1, duration=1)], fetcher.video_ids[0], "en", "", True)
54-
)
49+
video_id = mock_video_ids[0]
5550

56-
results = fetcher._fetch_single(fetcher.video_ids[0])
51+
mock_transcripts = [
52+
Transcript(text="text", start=1, duration=1)
53+
]
5754

58-
assert isinstance(results.transcripts[0], Transcript)
59-
assert results.video_id == fetcher.video_ids[0]
60-
assert results.transcripts[0] == Transcript(
61-
text='text',
62-
start=1,
63-
duration=1
55+
mocker.patch.object(
56+
fetcher,
57+
"_decide_fetch_method",
58+
return_value=mock_transcripts
6459
)
65-
assert results.transcripts[0].text == 'text'
60+
61+
result = fetcher._fetch_single(video_id)
62+
63+
assert isinstance(result, VideoTranscript)
64+
assert result.video_id == video_id
65+
assert result.transcripts == mock_transcripts
66+
assert result.transcripts[0].text == "text"
67+
6668

6769
def test_concurrent_fetching(mocker, mock_video_ids):
68-
mock_fetch = mocker.patch.object(
69-
YouTubeTranscriptApi,
70-
"fetch",
71-
side_effect=lambda *args, **kwargs: MagicMock(
72-
to_raw_data=lambda: [{"text": "test", "start": 0, "duration": 1}]
73-
)
74-
)
75-
7670
fetcher = TranscriptFetcher(mock_video_ids)
77-
78-
start_time = time.time()
71+
72+
mock_result = VideoTranscript(
73+
video_id="dummy",
74+
transcripts=[Transcript(text="test", start=0, duration=1)]
75+
)
76+
77+
mock_fetch_single = mocker.patch.object(
78+
fetcher,
79+
"_fetch_single",
80+
return_value=mock_result
81+
)
82+
7983
results = fetcher.fetch()
80-
elapsed = time.time() - start_time
81-
82-
assert len(results) == 2
83-
assert elapsed < 1.5
84-
assert mock_fetch.call_count == 2
84+
85+
assert len(results) == len(mock_video_ids)
86+
assert mock_fetch_single.call_count == len(mock_video_ids)
87+
8588

8689
def test_custom_ytt_api_client_initialized_correctly(mocker):
8790
mock_api = mocker.patch("ytfetcher._transcript_fetcher.YouTubeTranscriptApi")
@@ -117,15 +120,93 @@ def test_clean_transcripts():
117120

118121
assert cleaned_response[0].text == 'This is some text'
119122

120-
def test_clean_transcripts_with_multiple_text():
121-
test_response = [
122-
Transcript(
123-
text="[Music][Applause] and that happened!",
124-
duration=1,
125-
start=1
126-
)
127-
]
123+
def test_decide_fetch_method_manual_branch(mocker):
124+
fetcher = TranscriptFetcher(
125+
video_ids=["abc"],
126+
manually_created=True,
127+
languages=["en"]
128+
)
128129

129-
cleaned_response = TranscriptFetcher._clean_transcripts(test_response)
130-
131-
assert cleaned_response[0].text == 'and that happened!'
130+
mock_manual = mocker.patch.object(
131+
fetcher,
132+
"_fetch_manual_transcript",
133+
return_value=["manual"]
134+
)
135+
136+
mock_api = mocker.MagicMock()
137+
138+
result = fetcher._decide_fetch_method(mock_api, "abc")
139+
140+
assert result == ["manual"]
141+
mock_manual.assert_called_once_with(yt_api=mock_api, video_id="abc")
142+
143+
def test_decide_fetch_method_first_available_branch(mocker):
144+
fetcher = TranscriptFetcher(
145+
video_ids=["abc"],
146+
manually_created=False,
147+
languages=None
148+
)
149+
150+
mock_first = mocker.patch.object(
151+
fetcher,
152+
"_fetch_first_available_transcript",
153+
return_value=["first"]
154+
)
155+
156+
mock_api = mocker.MagicMock()
157+
158+
result = fetcher._decide_fetch_method(mock_api, "abc")
159+
160+
assert result == ["first"]
161+
mock_first.assert_called_once_with(yt_api=mock_api, video_id="abc")
162+
163+
def test_decide_fetch_method_by_languages_branch(mocker):
164+
fetcher = TranscriptFetcher(
165+
video_ids=["abc"],
166+
manually_created=False,
167+
languages=["en"]
168+
)
169+
170+
mock_lang = mocker.patch.object(
171+
fetcher,
172+
"_fetch_by_languages",
173+
return_value=["lang"]
174+
)
175+
176+
mock_api = mocker.MagicMock()
177+
178+
result = fetcher._decide_fetch_method(mock_api, "abc")
179+
180+
assert result == ["lang"]
181+
mock_lang.assert_called_once_with(yt_api=mock_api, video_id="abc")
182+
183+
def test_fetch_manual_transcript_no_transcript(mocker):
184+
video_id = "abc"
185+
186+
mock_api = mocker.MagicMock()
187+
mock_list = mocker.MagicMock()
188+
189+
mock_api.list.return_value = mock_list
190+
mock_list.find_manually_created_transcript.side_effect = NoTranscriptFound(video_id, requested_language_codes=['en'], transcript_data=None)
191+
192+
fetcher = TranscriptFetcher(
193+
video_ids=[video_id],
194+
manually_created=True,
195+
languages=["en"]
196+
)
197+
198+
result = fetcher._fetch_manual_transcript(mock_api, video_id)
199+
200+
assert result is None
201+
202+
def test_fetch_first_available_transcript_empty(mocker):
203+
video_id = "abc"
204+
205+
mock_api = mocker.MagicMock()
206+
mock_api.list.return_value = []
207+
208+
fetcher = TranscriptFetcher(video_ids=[video_id])
209+
210+
result = fetcher._fetch_first_available_transcript(mock_api, video_id)
211+
212+
assert result is None

0 commit comments

Comments
 (0)