diff --git a/README.md b/README.md index 43a4dcb..f955987 100644 --- a/README.md +++ b/README.md @@ -95,12 +95,16 @@ home-manager switch --flake github:Logan-Lin/nix-config#yanlin@thinkpad │ ├── wireguard.nix # Hub-and-spoke VPN networking │ ├── borg-client.nix # Borg backup system with automated scheduling │ ├── plasma.nix # KDE Plasma desktop environment configuration -│ └── homebrew.nix # Homebrew and nix-homebrew configuration +│ ├── homebrew.nix # Homebrew and nix-homebrew configuration +│ └── yt-dlp.nix # Video downloader for YouTube and Bilibili ├── config/ # Configuration files │ ├── firefox/ # Firefox browser configuration │ │ ├── bookmarks.nix │ │ ├── extensions.nix │ │ └── search.nix +│ ├── yt-dlp/ # yt-dlp cookie files for authentication +│ │ ├── cookies-youtube.txt # YouTube authentication cookies +│ │ └── cookies-bilibili.txt # Bilibili authentication cookies │ ├── fonts.nix # Font packages and configuration │ ├── homeassistant/ # Home Assistant smart home configuration │ │ ├── configuration.yaml # Main HA configuration with reverse proxy @@ -835,6 +839,53 @@ ftp # Alias for termscp termscp ftp://user@host.com # Quick connection ``` +### 📹 Video Downloader: yt-dlp + +**Configuration**: `modules/yt-dlp.nix` +**Purpose**: Download videos from YouTube and Bilibili with organized file structure + +A declarative yt-dlp configuration with automatic platform detection and structured downloads: + +#### Key Features: +- **Platform Auto-detection**: Automatically detects YouTube or Bilibili from URL +- **Cookie Authentication**: Separate cookie files for YouTube and Bilibili access +- **Organized Downloads**: + - Single videos: `[downloadDir]/[platform]/[uploader]/[upload_date]-[title].ext` + - Playlists: `[downloadDir]/[platform]/[playlist_title]/[index]-[title].ext` +- **Host-specific Paths**: Configurable download directory per machine +- **Quality Settings**: Best quality video+audio with MP4 output +- **Metadata**: Downloads subtitles, thumbnails, and descriptions +- **Archive Tracking**: Prevents re-downloading previously downloaded videos + +#### Usage: +```bash +# Download video (auto-detects YouTube/Bilibili) +dlv https://www.youtube.com/watch?v=VIDEO_ID +dlv https://www.bilibili.com/video/BV1234567890 + +# Download playlist +dlv https://www.youtube.com/playlist?list=PLAYLIST_ID + +# Show cookie update instructions +dlv-help +``` + +#### Cookie Setup: +1. Install browser extension: "Get cookies.txt LOCALLY" +2. Log in to YouTube/Bilibili +3. Export cookies using the extension +4. Save to `~/.config/yt-dlp/cookies-youtube.txt` or `cookies-bilibili.txt` + +#### Configuration: +Each host can configure its download directory: +```nix +# In host's home.nix +programs.yt-dlp-custom = { + enable = true; + downloadDir = "~/Downloads/Videos"; # Or any host-specific path +}; +``` + ## 💻 Daily Workflow ### Typical Development Session: diff --git a/config/yt-dlp/cookies-bilibili.txt b/config/yt-dlp/cookies-bilibili.txt new file mode 100644 index 0000000..b6f5bf5 --- /dev/null +++ b/config/yt-dlp/cookies-bilibili.txt @@ -0,0 +1,29 @@ +# Netscape HTTP Cookie File +# https://curl.haxx.se/rfc/cookie_spec.html +# This is a generated file! Do not edit. + +.bilibili.com TRUE / FALSE 1787769860 buvid3 9E5DBD65-6C9F-9C42-7540-D278BFE5B1DD60286infoc +.bilibili.com TRUE / FALSE 1787769860 b_nut 1756233860 +.bilibili.com TRUE / FALSE 1789502974 enable_web_push DISABLE +.bilibili.com TRUE / FALSE 1789502974 home_feed_column 5 +.bilibili.com TRUE / FALSE 1789502974 browser_resolution 2240-1151 +.bilibili.com TRUE / FALSE 1791742248 buvid4 737192F6-CF30-9170-C145-8071648B393561161-025082702-MssZwiGhdsLJOxyilfLY7Q%3D%3D +.bilibili.com TRUE / FALSE 1787769862 buvid_fp 63ed5a8b4842c9c467efbc4f875f57e2 +.bilibili.com TRUE / TRUE 1771785909 SESSDATA 2970d071%2C1771785909%2Ce9382%2A81CjDZZsAsIW3ZYNWBEHDNjoQTKeesjMO5M2i-MsaNKbN5FU6viq_KqeK_Ff0jeb_TUFESVjAtYm9vUFZnbTJ0VUFjSk14TUhOSjNHVmhMT2VCRUtidERZZnRpVXR0SXJQNnFYS05vMTZFQzJCWUg3d1VULWYtcXRWX1NSbE5kVzNiRFNERXJiMTlRIIEC +.bilibili.com TRUE / TRUE 1771785909 bili_jct fb2e556966de9865f569e8ba96265b99 +.bilibili.com TRUE / TRUE 1771785909 DedeUserID 2480785 +.bilibili.com TRUE / TRUE 1771785909 DedeUserID__ckMd5 64dd7d5198a897cc +.bilibili.com TRUE / TRUE 1771785909 sid 8p2r4be3 +.bilibili.com TRUE / FALSE 1789502974 theme-tip-show SHOWED +.bilibili.com TRUE / FALSE 1787769917 theme-avatar-tip-show SHOWED +.bilibili.com TRUE / FALSE 1787769921 theme-switch-show SHOWED +.bilibili.com TRUE / FALSE 1787769923 theme_style dark +.bilibili.com TRUE / FALSE 1789514362 CURRENT_FNVAL 2000 +.bilibili.com TRUE / FALSE 1787928319 CURRENT_QUALITY 80 +.bilibili.com TRUE / FALSE 1790793933 rpdid |(Jll~R)mRuu0J'u~lYkJJmJJ +.bilibili.com TRUE / FALSE 1759439728 bp_t_offset_2480785 1108141625822937088 +www.bilibili.com FALSE / FALSE 0 bmg_af_switch 1 +www.bilibili.com FALSE / FALSE 0 bmg_src_def_domain i1.hdslb.com +.bilibili.com TRUE / FALSE 0 b_lsid 7E3951DB_1994EFA839A +.bilibili.com TRUE / FALSE 1758225851 bili_ticket eyJhbGciOiJIUzI1NiIsImtpZCI6InMwMyIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3NTgyMjU4NTEsImlhdCI6MTc1Nzk2NjU5MSwicGx0IjotMX0.EouymquKSd_qZQFbWcFhr-GH3dVLd7mA9QYlSNSQm1M +.bilibili.com TRUE / FALSE 1758225851 bili_ticket_expires 1758225791 diff --git a/config/yt-dlp/cookies-youtube.txt b/config/yt-dlp/cookies-youtube.txt new file mode 100644 index 0000000..76134ec --- /dev/null +++ b/config/yt-dlp/cookies-youtube.txt @@ -0,0 +1,10 @@ +# Netscape HTTP Cookie File +# This file is generated by a browser extension or yt-dlp +# To update this file: +# 1. Install "Get cookies.txt LOCALLY" extension in your browser +# 2. Log in to youtube.com +# 3. Click the extension and export cookies +# 4. Replace this file's contents with the exported cookies +# +# Or use yt-dlp to extract from browser: +# yt-dlp --cookies-from-browser firefox --cookies cookies-youtube.txt "https://youtube.com" \ No newline at end of file diff --git a/hosts/darwin/home-default.nix b/hosts/darwin/home-default.nix index e8c1183..44732a8 100644 --- a/hosts/darwin/home-default.nix +++ b/hosts/darwin/home-default.nix @@ -17,6 +17,7 @@ ../../modules/ghostty.nix ../../modules/syncthing.nix ../../modules/dictionary.nix + ../../modules/yt-dlp.nix ../../config/fonts.nix ]; diff --git a/hosts/darwin/imac/home.nix b/hosts/darwin/imac/home.nix index 1ec08e3..432c24b 100644 --- a/hosts/darwin/imac/home.nix +++ b/hosts/darwin/imac/home.nix @@ -8,4 +8,9 @@ # iMac-specific home configuration # Example: Different screen setup, desktop-specific tools, etc. + # yt-dlp configuration + programs.yt-dlp-custom = { + enable = true; + downloadDir = "~/Downloads/Videos"; + }; } diff --git a/hosts/darwin/mba/home.nix b/hosts/darwin/mba/home.nix index dd8b76f..e8efc30 100644 --- a/hosts/darwin/mba/home.nix +++ b/hosts/darwin/mba/home.nix @@ -8,4 +8,9 @@ # MacBook Air-specific home configuration # Example: Laptop-specific tools, power management, etc. + # yt-dlp configuration + programs.yt-dlp-custom = { + enable = true; + downloadDir = "~/Downloads/Videos"; + }; } diff --git a/hosts/nixos/home-default.nix b/hosts/nixos/home-default.nix index 7662f01..47f4618 100644 --- a/hosts/nixos/home-default.nix +++ b/hosts/nixos/home-default.nix @@ -13,6 +13,7 @@ ../../modules/rsync.nix ../../modules/btop.nix ../../modules/dictionary.nix + ../../modules/yt-dlp.nix ../../config/fonts.nix ]; diff --git a/hosts/nixos/hs/home.nix b/hosts/nixos/hs/home.nix index c8027d5..93cd2e7 100644 --- a/hosts/nixos/hs/home.nix +++ b/hosts/nixos/hs/home.nix @@ -7,6 +7,13 @@ ]; # hs-specific home configuration + + # yt-dlp configuration - store videos on large storage + programs.yt-dlp-custom = { + enable = true; + downloadDir = "/mnt/storage/Media/Web"; + }; + programs.zsh.shellAliases = { # Disk health monitoring smart-report = "sudo SMART_DRIVES='/dev/disk/by-id/ata-ZHITAI_SC001_XT_1000GB_ZTB401TAB244431J4R:ZFS_Mirror_1;/dev/disk/by-id/ata-ZHITAI_SC001_XT_1000GB_ZTB401TAB244431KEG:ZFS_Mirror_2;/dev/disk/by-id/ata-HGST_HUH721212ALE604_5PK2N4GB:Data_Drive_1_12TB;/dev/disk/by-id/ata-HGST_HUH721212ALE604_5PJ7Z3LE:Data_Drive_2_12TB;/dev/disk/by-id/ata-ST16000NM000J-2TW103_WRS0F8BE:Parity_Drive_16TB' /home/yanlin/.config/nix/scripts/daily-smart-report.sh Ac9qKFH5cA.7Yly"; diff --git a/modules/yt-dlp.nix b/modules/yt-dlp.nix new file mode 100644 index 0000000..e6a5030 --- /dev/null +++ b/modules/yt-dlp.nix @@ -0,0 +1,218 @@ +{ config, pkgs, lib, ... }: + +with lib; + +let + cfg = config.programs.yt-dlp-custom; +in + +{ + options.programs.yt-dlp-custom = { + enable = mkEnableOption "yt-dlp video downloader configuration"; + + package = mkOption { + type = types.package; + default = pkgs.yt-dlp; + example = "pkgs.yt-dlp"; + description = "yt-dlp package to use"; + }; + + downloadDir = mkOption { + type = types.str; + default = "~/Downloads/Videos"; + example = "/mnt/storage/videos"; + description = "Base directory for downloaded videos"; + }; + }; + + config = mkIf cfg.enable { + # Install yt-dlp and ffmpeg + home.packages = with pkgs; [ + cfg.package + ffmpeg + ]; + + # Copy cookie files (not symlink) to make them writable + home.file.".config/yt-dlp/cookies-youtube.txt" = { + source = ../config/yt-dlp/cookies-youtube.txt; + onChange = '' + chmod 644 ~/.config/yt-dlp/cookies-youtube.txt + ''; + }; + home.file.".config/yt-dlp/cookies-bilibili.txt" = { + source = ../config/yt-dlp/cookies-bilibili.txt; + onChange = '' + chmod 644 ~/.config/yt-dlp/cookies-bilibili.txt + ''; + }; + + # Create yt-dlp configuration file + home.file.".config/yt-dlp/config".text = '' + # Quality settings + --format "bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best" + --merge-output-format mp4 + + # Download options + --no-playlist + --embed-thumbnail + --write-thumbnail + --embed-subs + --sub-langs "en,zh-CN,zh-TW" + --write-auto-subs + --write-description + --write-info-json + + # File naming and organization + # Allow unicode characters in filenames for Chinese/Japanese content + + # Performance + --concurrent-fragments 4 + --retries 10 + --fragment-retries 10 + + # SponsorBlock for YouTube + --sponsorblock-mark all + + # User agent + --user-agent "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" + ''; + + # Shell alias and function + programs.zsh.shellAliases = { + # Simple alias that calls the function + dlv = "download-video"; + }; + + programs.zsh.initContent = '' + # Function to download videos from YouTube or Bilibili + download-video() { + local url="$1" + local download_dir="${cfg.downloadDir}" + + if [[ -z "$url" ]]; then + echo "Usage: dlv " + echo "Downloads video from YouTube or Bilibili" + return 1 + fi + + # Expand tilde in download directory + download_dir="''${download_dir/#\~/$HOME}" + + # Detect platform from URL + local platform="" + local cookies_file="" + local extra_args="" + + if [[ "$url" =~ (youtube\.com|youtu\.be) ]]; then + platform="YouTube" + cookies_file="$HOME/.config/yt-dlp/cookies-youtube.txt" + # YouTube-specific output template - use channel as fallback for uploader + local output_template="$download_dir/YouTube/%(uploader|)s%(channel|)s%(uploader_id|)s/%(upload_date>%Y%m%d|)s-%(title)s.%(ext)s" + elif [[ "$url" =~ bilibili\.com ]]; then + platform="Bilibili" + cookies_file="$HOME/.config/yt-dlp/cookies-bilibili.txt" + # Bilibili-specific arguments + extra_args="--referer https://www.bilibili.com/" + # Bilibili-specific output template - use owner as uploader for Bilibili + local output_template="$download_dir/Bilibili/%(uploader|)s%(channel|)s%(uploader_id|)s/%(upload_date>%Y%m%d|)s-%(title)s.%(ext)s" + else + echo "Warning: Unknown platform, proceeding without cookies" + platform="Unknown" + local output_template="$download_dir/%(uploader|)s%(channel|)s%(uploader_id|)s/%(upload_date>%Y%m%d|)s-%(title)s.%(ext)s" + fi + + # Check if it's a playlist + if [[ "$url" =~ "list=" ]] || [[ "$url" =~ "/playlist" ]] || [[ "$url" =~ "bilibili\.com/.*/channel" ]] || [[ "$url" =~ "bilibili\.com/.*/collectiondetail" ]]; then + echo "Detected playlist URL" + # For playlists, use different output template + if [[ "$platform" == "YouTube" ]]; then + output_template="$download_dir/YouTube/%(playlist_title|)s%(playlist|)s/%(playlist_index|)03d-%(title)s.%(ext)s" + elif [[ "$platform" == "Bilibili" ]]; then + output_template="$download_dir/Bilibili/%(playlist_title|)s%(playlist|)s/%(playlist_index|)03d-%(title)s.%(ext)s" + else + output_template="$download_dir/%(playlist_title|)s%(playlist|)s/%(playlist_index|)03d-%(title)s.%(ext)s" + fi + extra_args="$extra_args --yes-playlist" + fi + + echo "Downloading from $platform..." + echo "Output directory: $download_dir" + + # Build yt-dlp command + local cmd="yt-dlp" + + # Add cookies if file exists - copy to temp file to avoid permission issues + if [[ -f "$cookies_file" ]]; then + echo "Using cookies from: $cookies_file" + # Create a temporary writable copy of the cookie file + local temp_cookies="/tmp/yt-dlp-cookies-$$.txt" + cp "$cookies_file" "$temp_cookies" + chmod 644 "$temp_cookies" + cmd="$cmd --cookies \"$temp_cookies\"" + # Clean up temp file after download + trap "rm -f $temp_cookies" EXIT + fi + + # Add archive file to track downloads + local archive_file="$download_dir/.archive.txt" + cmd="$cmd --download-archive \"$archive_file\"" + + # Add output template and extra arguments + cmd="$cmd -o \"$output_template\" $extra_args \"$url\"" + + # Create download directory if it doesn't exist + mkdir -p "$download_dir" + + # Execute the command + eval $cmd + + if [[ $? -eq 0 ]]; then + echo "✓ Download completed successfully" + else + echo "✗ Download failed" + return 1 + fi + } + + # Function to show instructions for updating cookies + update-cookies-instructions() { + cat << 'EOF' + To update cookies for YouTube or Bilibili: + + 1. Install a browser extension: + - Chrome/Edge: "Get cookies.txt LOCALLY" + - Firefox: "cookies.txt" + + 2. Log in to the website (youtube.com or bilibili.com) + + 3. Click the extension and export cookies + + 4. Save the cookies: + - YouTube: ~/.config/yt-dlp/cookies-youtube.txt + - Bilibili: ~/.config/yt-dlp/cookies-bilibili.txt + + Alternative method using yt-dlp: + yt-dlp --cookies-from-browser firefox --cookies cookies-youtube.txt "https://youtube.com" + yt-dlp --cookies-from-browser firefox --cookies cookies-bilibili.txt "https://bilibili.com" + EOF + } + + alias dlv-help='update-cookies-instructions' + + # Function to clear download archive + dlv-clear-archive() { + local download_dir="${cfg.downloadDir}" + download_dir="''${download_dir/#\~/$HOME}" + local archive_file="$download_dir/.archive.txt" + + if [[ -f "$archive_file" ]]; then + echo "Clearing download archive: $archive_file" + rm -f "$archive_file" + echo "✓ Archive cleared. Videos can now be re-downloaded." + else + echo "No archive file found at: $archive_file" + fi + } + ''; + }; +}