Сборка подписок #121850
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Сборка подписок | |
| on: | |
| workflow_dispatch: | |
| jobs: | |
| merge-subs: | |
| runs-on: ubuntu-latest | |
| name: Сборка подписок | |
| permissions: | |
| contents: write | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Скачать и объединить подписки | |
| run: | | |
| # Скачать и объединить подписки | |
| plural_configs() { | |
| local n=$1 | |
| local mod10=$((n % 10)) | |
| local mod100=$((n % 100)) | |
| if [ $mod100 -ge 11 ] && [ $mod100 -le 14 ]; then | |
| echo "$n конфигураций" | |
| elif [ $mod10 -eq 1 ]; then | |
| echo "$n конфигурация" | |
| elif [ $mod10 -ge 2 ] && [ $mod10 -le 4 ]; then | |
| echo "$n конфигурации" | |
| else | |
| echo "$n конфигураций" | |
| fi | |
| } | |
| > raw | |
| mkdir -p tmp_results | |
| i=0 | |
| process_source() { | |
| local src_line="$1" | |
| local index="$2" | |
| local line_clean url filters tmp_src http_code curl_exit | |
| local tmp_dec count grep_args | |
| # Очистка пробелов | |
| line_clean=$(echo "$src_line" | sed -e 's/^[[:space:]]*//; s/[[:space:]]*$//') | |
| # Извлекаем URL (первое слово) | |
| url=$(echo "$line_clean" | awk '{print $1}') | |
| # Собираем фильтры (все последующие слова) | |
| filters=$(echo "$line_clean" | awk '{for(i=2;i<=NF;i++) print $i}') | |
| tmp_src=$(mktemp) | |
| # Скачиваем | |
| http_code=$(curl -sS -L --max-time 30 -o "$tmp_src" -w '%{http_code}' "$url" 2>/dev/null) || true | |
| curl_exit=$? | |
| if [ $curl_exit -ne 0 ] || ! echo "$http_code" | grep -qE '^(2..|3..)$'; then | |
| echo " ❌ [$url] Источник недоступен (ошибка curl $curl_exit, HTTP $http_code)" | |
| rm -f "$tmp_src" | |
| return | |
| fi | |
| # Проверяем, является ли файл JSON (массив или объект) | |
| first_char=$(head -c 1 "$tmp_src") | |
| last_char=$(tail -c 1 "$tmp_src") | |
| if { [ "$first_char" = "[" ] && [ "$last_char" = "]" ]; } || { [ "$first_char" = "{" ] && [ "$last_char" = "}" ]; }; then | |
| happy_url="https://happy-decoder.cc/p/txt/$url" | |
| happy_tmp=$(mktemp) | |
| http_code_happy=$(curl -sS -L --max-time 30 -o "$happy_tmp" -w '%{http_code}' "$happy_url" 2>/dev/null) || true | |
| if echo "$http_code_happy" | grep -qE '^(2..|3..)$'; then | |
| mv "$happy_tmp" "$tmp_src" | |
| else | |
| rm -f "$happy_tmp" | |
| echo " ❌ [$url] Конвертер не работает (HTTP $http_code_happy)" | |
| rm -f "$tmp_src" | |
| return | |
| fi | |
| fi | |
| # Декодируем base64 (если файл закодирован) | |
| tmp_dec=$(mktemp) | |
| if base64 -d "$tmp_src" > "$tmp_dec" 2>/dev/null && [ -s "$tmp_dec" ]; then | |
| mv "$tmp_dec" "$tmp_src" | |
| else | |
| rm -f "$tmp_dec" | |
| fi | |
| # Применяем фильтры, если они заданы | |
| if [ -n "$filters" ]; then | |
| grep_args="" | |
| while IFS= read -r f; do | |
| # Само слово или фраза (кавычки уже сняты) | |
| grep_args="$grep_args -e $f" | |
| # Добавляем percent-encoded вариант (через python3) | |
| encoded=$(python3 -c "import urllib.parse; print(urllib.parse.quote('$f'))" 2>/dev/null) | |
| if [ -n "$encoded" ] && [ "$encoded" != "$f" ]; then | |
| grep_args="$grep_args -e $encoded" | |
| fi | |
| done <<< "$(echo "$filters" | tr ' ' '\n')" | |
| grep -i $grep_args "$tmp_src" > "${tmp_src}_filtered" 2>/dev/null || true | |
| mv "${tmp_src}_filtered" "$tmp_src" | |
| fi | |
| count=$(grep -c '://' "$tmp_src" || true) | |
| echo " ✅ [$url] Скачано: $(plural_configs $count)" | |
| cat "$tmp_src" > "tmp_results/${index}.txt" | |
| rm -f "$tmp_src" | |
| } | |
| # Запускаем параллельно все источники | |
| max_jobs=30 | |
| running=0 | |
| while IFS= read -r line; do | |
| line=$(echo "$line" | sed -e 's/^[[:space:]]*//; s/[[:space:]]*$//') | |
| [ -z "$line" ] && continue | |
| echo "$line" | grep -q '^#' && continue | |
| i=$((i+1)) | |
| process_source "$line" "$i" & | |
| running=$((running+1)) | |
| if [ $running -ge $max_jobs ]; then | |
| wait -n | |
| running=$((running-1)) | |
| fi | |
| done < sources.txt | |
| wait | |
| # Собираем результаты | |
| idx=1 | |
| while [ $idx -le $i ]; do | |
| if [ -f "tmp_results/${idx}.txt" ]; then | |
| cat "tmp_results/${idx}.txt" >> raw | |
| echo "" >> raw | |
| fi | |
| idx=$((idx+1)) | |
| done | |
| rm -rf tmp_results | |
| echo "" | |
| echo "Скачано: $(plural_configs $(wc -l < raw))" | |
| - name: Удаление комментариев, пустых строк, символов возврата каретки и заведомо нерабочих конфигураций | |
| run: | | |
| # Удаление комментариев, пустых строк, символов возврата каретки и заведомо нерабочих конфигураций | |
| grep -a -v '^\s*#' raw | \ | |
| grep -a -v '^\s*//' | \ | |
| sed '/^\s*$/d' | \ | |
| tr -d '\r' | \ | |
| grep -v -E '^(happ|incy)://' | \ | |
| sed -E 's/([?&])(Telegram|telegram|ref|utm_[a-z]+|serverDescription)=[^&]*&?/\1/g; s/[?&]$//' | \ | |
| grep -E '^[a-zA-Z][a-zA-Z0-9+.-]*://' | \ | |
| grep -a -v -E \ | |
| '(127\.0\.0\.1|\[?::1\]?|0\.0\.0\.0|\[?::\]|0\.0\.0\.0\.0\.0\.0\.0|224\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)|255\.255\.255\.255|240\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)|[fF][fF]00::/8|fe80::/10|fc00::/7|100\.64\.0\.0/10|\blocalhost\b|\bexample\.com\b|\bexample\.net\b|\bexample\.org\b)' \ | |
| > clean | |
| plural_configs() { | |
| local n=$1 | |
| local mod10=$((n % 10)) | |
| local mod100=$((n % 100)) | |
| if [ $mod100 -ge 11 ] && [ $mod100 -le 14 ]; then | |
| echo "$n конфигураций" | |
| elif [ $mod10 -eq 1 ]; then | |
| echo "$n конфигурация" | |
| elif [ $mod10 -ge 2 ] && [ $mod10 -le 4 ]; then | |
| echo "$n конфигурации" | |
| else | |
| echo "$n конфигураций" | |
| fi | |
| } | |
| echo "После очистки: $(plural_configs $(wc -l < clean))" | |
| - name: Дедупликация | |
| run: | | |
| # Дедупликация | |
| node -e " | |
| const fs = require('fs'); | |
| const lines = fs.readFileSync('clean', 'utf8').split('\n').filter(l => l.trim()); | |
| const seen = new Set(); | |
| const result = []; | |
| for (const line of lines) { | |
| const hashIndex = line.indexOf('#'); | |
| const uri = hashIndex >= 0 ? line.substring(0, hashIndex) : line; | |
| const namePart = hashIndex >= 0 ? line.substring(hashIndex) : ''; | |
| const protoSplit = uri.split('://'); | |
| if (protoSplit.length < 2) { result.push(line); continue; } | |
| let proto = protoSplit[0]; | |
| const rest = protoSplit[1]; | |
| // Считаем h2, hy2 и hysteria2 одним протоколом | |
| if (proto === 'h2') proto = 'hysteria2'; | |
| if (proto === 'hy2') proto = 'hysteria2'; | |
| let normalizedUri; | |
| if (proto === 'vmess') { | |
| try { | |
| const json = JSON.parse(Buffer.from(rest, 'base64').toString('utf8')); | |
| delete json.ps; | |
| const sorted = {}; | |
| Object.keys(json).sort().forEach(k => { sorted[k] = json[k]; }); | |
| normalizedUri = 'vmess://' + Buffer.from(JSON.stringify(sorted)).toString('base64'); | |
| } catch (e) { | |
| normalizedUri = uri; | |
| } | |
| } else { | |
| const queryIndex = rest.indexOf('?'); | |
| let base = rest, query = ''; | |
| if (queryIndex >= 0) { | |
| base = rest.substring(0, queryIndex); | |
| query = rest.substring(queryIndex + 1); | |
| } | |
| let normalizedQuery = ''; | |
| if (query) { | |
| const params = query.split('&').filter(p => p); | |
| const ignoredParams = ['name','remarks','ps','tag','note','remark']; | |
| const filtered = params.filter(p => { | |
| const key = p.split('=')[0].toLowerCase(); | |
| return !ignoredParams.includes(key); | |
| }); | |
| filtered.sort((a, b) => { | |
| const ka = a.split('=')[0].toLowerCase(); | |
| const kb = b.split('=')[0].toLowerCase(); | |
| return ka.localeCompare(kb); | |
| }); | |
| normalizedQuery = filtered.join('&'); | |
| } | |
| normalizedUri = proto + '://' + base + (normalizedQuery ? '?' + normalizedQuery : ''); | |
| } | |
| if (!seen.has(normalizedUri)) { | |
| seen.add(normalizedUri); | |
| result.push(line); | |
| } | |
| } | |
| fs.writeFileSync('deduped', result.join('\n') + '\n'); | |
| " | |
| plural_configs() { | |
| local n=$1 | |
| local mod10=$((n % 10)) | |
| local mod100=$((n % 100)) | |
| if [ $mod100 -ge 11 ] && [ $mod100 -le 14 ]; then | |
| echo "$n конфигураций" | |
| elif [ $mod10 -eq 1 ]; then | |
| echo "$n конфигурация" | |
| elif [ $mod10 -ge 2 ] && [ $mod10 -le 4 ]; then | |
| echo "$n конфигурации" | |
| else | |
| echo "$n конфигураций" | |
| fi | |
| } | |
| echo "После дедупликации: $(plural_configs $(wc -l < deduped))" | |
| - name: Подготовка итоговых файлов | |
| run: | | |
| # Подготовка итоговых файлов | |
| # Удаляем пробелы перед http:// и https:// | |
| sed -E 's/ +(https?:\/\/)/\1/g' deduped > deduped_cleaned | |
| mv deduped_cleaned deduped | |
| cat > headers.txt <<EOF | |
| #profile-title: 🚀 all_subs | |
| #announce: ⚡ Используйте конфигурации только в условиях «белых списков» | |
| #profile-web-page-url:https://github.com/solovyov-jenya2004/all_subs/ | |
| #support-url:https://github.com/solovyov-jenya2004/all_subs/issues/ | |
| #profile-update-interval: 1 | |
| #subscription-userinfo: upload=0; download=0; total=0 | |
| EOF | |
| # Перемешиваем конфигурации | |
| shuf deduped -o shuffled_deduped | |
| # Собираем финальный файл с заголовками и двумя пустыми строками после них | |
| cat headers.txt shuffled_deduped > final_sorted | |
| echo "final_sorted создан" | |
| # Кодируем в base64 с заголовками | |
| base64 -w 0 final_sorted > final_sorted_base64 | |
| echo "final_sorted_base64 создан" | |
| plural_configs() { | |
| local n=$1 | |
| local mod10=$((n % 10)) | |
| local mod100=$((n % 100)) | |
| if [ $mod100 -ge 11 ] && [ $mod100 -le 14 ]; then | |
| echo "$n конфигураций" | |
| elif [ $mod10 -eq 1 ]; then | |
| echo "$n конфигурация" | |
| elif [ $mod10 -ge 2 ] && [ $mod10 -le 4 ]; then | |
| echo "$n конфигурации" | |
| else | |
| echo "$n конфигураций" | |
| fi | |
| } | |
| echo "" | |
| echo "📊 Количество конфигураций по протоколам:" | |
| awk -F'://' '/^[a-zA-Z0-9]/{print $1}' final_sorted | sed -e 's/^h2$/hysteria2/' -e 's/^hy2$/hysteria2/' | sort | uniq -c | sort -rn | while read count proto; do | |
| printf " %s - %s\n" "$proto" "$(plural_configs $count)" | |
| done | |
| - name: Публикация на GitHub | |
| run: | | |
| # Публикация на GitHub | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git add final_sorted final_sorted_base64 | |
| git diff --staged --quiet || (git commit -m "Обновление $(date -u +'%Y-%m-%d %H:%M') UTC" && git push) | |
| echo "" | |
| echo "Подписки опубликованы" | |
| - name: Публикация артефакта | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: subscription-files | |
| path: | | |
| final_sorted | |
| final_sorted_base64 | |
| sync-mirrors: | |
| needs: merge-subs | |
| runs-on: ubuntu-latest | |
| name: ${{ matrix.name }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: Публикация в GitLab | |
| clone_url: https://oauth2:${GITLAB_TOKEN}@gitlab.com/solovyov-jenya2004/all_subs.git | |
| secret: GITLAB_TOKEN | |
| branch: main | |
| steps: | |
| - name: Скачивание артефакта | |
| uses: actions/download-artifact@v6 | |
| with: | |
| name: subscription-files | |
| path: files | |
| - name: Клонирование репозитория | |
| run: | | |
| # Клонирование репозитория | |
| git clone --depth 1 ${{ matrix.clone_url }} repo | |
| env: | |
| GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} | |
| - name: Публикация | |
| run: | | |
| # Публикация | |
| cd repo | |
| git config user.name "GitHub Actions" | |
| git config user.email "actions@github.com" | |
| cp ../files/final_sorted ../files/final_sorted_base64 . | |
| git add final_sorted final_sorted_base64 | |
| if git diff --cached --quiet; then | |
| echo "" | |
| echo "Изменений нет, пуш не требуется" | |
| else | |
| git commit -m "Обновление $(date -u '+%Y-%m-%d %H:%M') UTC" | |
| git push origin ${{ matrix.branch }} | |
| echo "" | |
| echo "Подписки опубликованы" | |
| fi |