#!/bin/bash # shellcheck disable=SC2162 ## Script to keep Prowlarr/Indexers up to date with Jackett/Jackett ## Created by Bakerboy448 ## ## Features: ### - Controlled logging: DEBUG/VERBOSE modes for troubleshooting ### - Automated conflict resolution for indexer syncing ## ## Requirements ### Prowlarr/Indexers local git repo exists ### Set variables as needed ### Typically only prowlarr_git_path would be needed to be set ## Using the Script ### Suggested to run from the current directory being Prowlarr/Indexers local Repo using Git Bash `./scripts/indexer-sync-v2.sh` ### Use -d for DEBUG logging, -v for VERBOSE logging # Default values DEBUG=${DEBUG:-false} VERBOSE=${VERBOSE:-false} prowlarr_remote_name="origin" prowlarr_target_branch="master" mode_choice="normal" push_mode=false push_mode_force=false prowlarr_push_remote="origin" PROWLARR_COMMIT_TEMPLATE="jackett indexers as of" PROWLARR_COMMIT_TEMPLATE_APPEND="" PROWLARR_REPO_URL="https://github.com/Prowlarr/Indexers.git" JACKETT_REPO_URL="https://github.com/Jackett/Jackett.git" PROWLARR_RELEASE_BRANCH="master" JACKETT_BRANCH="master" JACKETT_REMOTE_NAME="z_Jackett" SKIP_BACKPORT=false is_dev_exec=false is_jackett_dev=false pulls_exists=false local_exist=false automation_mode=false MAX_COMMITS_TO_PICK=50 MAX_COMMITS_TO_SEARCH=100 VALIDATION_SCRIPT="scripts/validate.py" # Load blocklist from external file or use defaults BLOCKLIST_FILE="${BLOCKLIST_FILE:-scripts/blocklist.txt}" BLOCKLIST=() declare -A blocklist_map load_blocklist() { if [ -f "$BLOCKLIST_FILE" ]; then log "DEBUG" "Loading blocklist from $BLOCKLIST_FILE" while IFS= read -r line; do # Skip comments and empty lines if [[ ! "$line" =~ ^[[:space:]]*# ]] && [[ -n "$line" ]]; then # Trim whitespace line=$(echo "$line" | xargs) if [ -n "$line" ]; then BLOCKLIST+=("$line") blocklist_map["$line"]=1 log "DEBUG" "Added to blocklist: $line" fi fi done < "$BLOCKLIST_FILE" log "INFO" "Loaded ${#BLOCKLIST[@]} entries from blocklist" log "DEBUG" "Blocklist contents: ${BLOCKLIST[*]}" else log "INFO" "Blocklist file not found at $BLOCKLIST_FILE, no indexers will be blocked" fi } CONFLICTS_NONYML_EXTENSIONS='\.(cs|js|iss|html|ico|png|jpg|jpeg|gif|svg|csproj|sln|md|txt|json|xml|config)$' # Initialize Defaults removed_indexers="" added_indexers="" modified_indexers="" newschema_indexers="" both_added_new_indexers="" BACKPORT_SKIPPED=false GIT_DIFF_CMD="git diff --cached --name-only" # Prowlarr Schema Versions ## v1 frozen 2021-10-13 ## v2 frozen 2022-04-18 ## v1 and v2 purged and moved to v3 2022-06-24 ## v3 purged and frozen 2022-07-22 ## v4 purged and frozen 2022-08-18 ## v5 purged and frozen 2022-10-14 ## v6 purged and frozen 2022-10-14 ## v7 purged and frozen 2024-04-27 ## v8 purged and frozen 2024-04-27 ## v9 purged and frozen 2024-10-13 # Load schema versions from VERSIONS file load_versions() { MIN_SCHEMA=10 MAX_SCHEMA=11 CURRENT_SCHEMA=11 if [ -f "VERSIONS" ]; then # shellcheck disable=SC2034 while IFS='=' read -r key value; do case "$key" in MIN_VERSION) MIN_SCHEMA="$value" ;; MAX_VERSION) MAX_SCHEMA="$value" ;; CURRENT_VERSION) CURRENT_SCHEMA="$value" ;; esac done < <(grep -v '^#' VERSIONS | grep '=') fi NEW_SCHEMA=$((MAX_SCHEMA + 1)) } load_versions NEW_VERS_DIR="definitions/v$NEW_SCHEMA" mkdir -p "$NEW_VERS_DIR" log() { local level="$1" local message="$2" local color_reset="\033[0m" local color_success="\033[0;32m" # Green local color_info="\033[0;36m" # Cyan local color_warn="\033[0;33m" # Yellow local color_debug="\033[0;34m" # Blue local color_trace="\033[0;35m" # Magenta local color_error="\033[0;31m" # Red # Check if logging level should be output case "$level" in DEBUG) [[ "$DEBUG" != "true" ]] && return ;; VERBOSE) [[ "$VERBOSE" != "true" && "$DEBUG" != "true" ]] && return ;; esac local color case "$level" in SUCCESS) level="INFO" message="SUCCESS|$message" color=$color_success ;; INFO) color=$color_info ;; WARN) color=$color_warn ;; WARNING) color=$color_warn level="WARN" ;; DEBUG) color=$color_debug ;; VERBOSE) color=$color_trace ;; ERROR) color=$color_error ;; *) color=$color_reset ;; esac echo -e "${color}$(date +'%Y-%m-%dT%H:%M:%S%z')|$level|$message${color_reset}" } usage() { echo "Usage: $0 [OPTIONS]" echo "Sync Prowlarr indexers with Jackett" echo "" echo "Options:" echo " -d Enable DEBUG logging" echo " -v Enable VERBOSE logging" echo " -f Force push with lease" echo " -r REMOTE Prowlarr remote name (default: origin)" echo " -b BRANCH Target branch (default: master)" echo " -o REMOTE Push remote (default: origin)" echo " -m MODE Mode: normal, development (default: normal)" echo " -p Enable push mode" echo " -z Skip backport" echo " -a Automation mode (skip interactive prompts)" echo " -c TEMPLATE Commit template" echo " -u URL Prowlarr repo URL" echo " -j URL Jackett repo URL" echo " -R BRANCH Prowlarr release branch" echo " -J BRANCH Jackett branch" echo " -n NAME Jackett remote name" echo "" echo "Environment variables:" echo " DEBUG=true Enable debug logging" echo " VERBOSE=true Enable verbose logging" exit 1 } determine_schema_version() { local def_file="$1" log "DEBUG" "Testing schema version of [$def_file]" log "VERBOSE" "Extracting version from file path: $def_file" check_version=$(echo "$def_file" | cut -d'/' -f2) log "VERBOSE" "Extracted version: $check_version" dir="definitions/$check_version" schema="$dir/schema.json" log "DEBUG" "Checking file against schema [$schema]" local test_output $PYTHON_CMD "$VALIDATION_SCRIPT" --single "$def_file" "$schema" test_output=$? if [ "$test_output" = 0 ]; then log "INFO" "Definition [$def_file] matches schema [$schema]" else check_version="v0" fi export check_version=$check_version } determine_best_schema_version() { local def_file="$1" log "INFO" "Determining best schema version for [$def_file]" # Use Python function to find best schema version local best_version best_version=$($PYTHON_CMD "$VALIDATION_SCRIPT" --find-best-version "$def_file") if [[ "$best_version" =~ ^v([0-9]+)$ ]]; then matched_version="${BASH_REMATCH[1]}" log "INFO" "Definition [$def_file] best matches schema [$best_version]" else matched_version=0 log "WARN" "Definition [$def_file] does not match any schema" log "ERROR" "Cardigann update likely needed. Version [v$NEW_SCHEMA] required. Review definition." fi export matched_version=$matched_version } initialize_script() { # Check for Python and virtual environment # Check for Python and determine command to use PYTHON_CMD="" if command -v python3 &> /dev/null; then PYTHON_CMD="python3" elif command -v python &> /dev/null; then PYTHON_CMD="python" else log "ERROR" "Python could not be found. Check your Python installation" exit 1 fi log "DEBUG" "Using Python command: $PYTHON_CMD" # Check if we have a virtual environment and activate it if [ -d ".venv" ]; then log "INFO" "Activating virtual environment" if [ -f ".venv/bin/activate" ]; then # Linux/Mac # shellcheck disable=SC1091 source .venv/bin/activate elif [ -f ".venv/Scripts/activate" ]; then # Windows # shellcheck disable=SC1091 source .venv/Scripts/activate fi fi # Check if required Python packages are available if ! $PYTHON_CMD -c "import jsonschema, yaml" &>/dev/null; then log "ERROR" "required python packages are missing. Install with: pip install -r requirements.txt" exit 2 fi log "INFO" "Using Python validation" } while getopts "frpzab:m:c:u:j:R:J:n:o:dv" opt; do case ${opt} in f) # No Arg push_mode_force=true log "DEBUG" "push_mode_force is $push_mode_force" ;; r) prowlarr_remote_name=$OPTARG ;; b) prowlarr_target_branch=$OPTARG ;; o) prowlarr_push_remote=$OPTARG ;; m) mode_choice=$OPTARG log "DEBUG" "mode_choice using argument $mode_choice" case "$mode_choice" in normal | n | N) is_dev_exec=false ;; development | dev | d | D) is_dev_exec=true log "WARN" "Mode: Development" log "WARN" "Skipping upstream reset to local. Skip checking out the local Prowlarr branch and output the details." log "INFO" "This will not reset Prowlarr branch from upstream/master and will ONLY checkout the selected branch to use." log "INFO" "This will pause at various debugging points for human review" ;; jackett | j | J) log "WARN" "Mode: Jackett" is_dev_exec=true is_jackett_dev=true log "WARN" "Skipping upstream reset to local. Skip checking out the local Prowlarr branch and output the details." log "INFO" "This will not reset Prowlarr branch from upstream/master and will ONLY checkout [$prowlarr_target_branch] branch to use." log "INFO" "This will not reset Jackett branch and will use what it currently locally is $JACKETT_REMOTE_NAME$JACKETT_BRANCH" log "INFO" "This will pause at various debugging points for human review" ;; *) usage ;; esac ;; p) # No Arg push_mode=true log "DEBUG" "push_mode is $push_mode" ;; c) PROWLARR_COMMIT_TEMPLATE=$OPTARG ;; u) PROWLARR_REPO_URL=$OPTARG ;; j) JACKETT_REPO_URL=$OPTARG ;; R) PROWLARR_RELEASE_BRANCH=$OPTARG ;; J) JACKETT_BRANCH=$OPTARG ;; n) JACKETT_REMOTE_NAME=$OPTARG ;; z) # No Arg SKIP_BACKPORT=true PROWLARR_COMMIT_TEMPLATE_APPEND="[backports skipped - TODO]" log "DEBUG" "SKIP_BACKPORT is $SKIP_BACKPORT. Commit Template will be appended with '$PROWLARR_COMMIT_TEMPLATE_APPEND' if applicable'" ;; a) # No Arg automation_mode=true log "DEBUG" "automation_mode is $automation_mode - interactive prompts will be skipped" ;; d) DEBUG=true log "INFO" "DEBUG logging enabled" ;; v) VERBOSE=true log "INFO" "VERBOSE logging enabled" ;; \?) usage ;; esac done shift $((OPTIND - 1)) configure_git() { git config advice.statusHints false git_remotes=$(git remote -v) prowlarr_remote_exists=$(echo "$git_remotes" | grep "$prowlarr_remote_name") prowlarr_push_remote_exists=$(echo "$git_remotes" | grep "$prowlarr_push_remote") jackett_remote_exists=$(echo "$git_remotes" | grep "$JACKETT_REMOTE_NAME") if [ -z "$prowlarr_remote_exists" ]; then git remote add "$prowlarr_remote_name" "$PROWLARR_REPO_URL" fi if [ -z "$jackett_remote_exists" ]; then git remote add "$JACKETT_REMOTE_NAME" "$JACKETT_REPO_URL" fi if [ "$prowlarr_push_remote" != "$prowlarr_remote_name" ] && [ -z "$prowlarr_push_remote_exists" ]; then log "ERROR" "Push remote [$prowlarr_push_remote] does not exist. Please add it manually or use an existing remote." exit 1 fi log "INFO" "Configured Git" if [ "$is_jackett_dev" = true ]; then log "DEBUG" "Skipping fetch for jackett development mode" else git fetch --all --prune --progress fi # Set default target branch based on push remote URL (only if using default) push_remote_url=$(git remote get-url "$prowlarr_push_remote" 2>/dev/null || echo "") if [ "$prowlarr_target_branch" = "master" ]; then if [[ "$push_remote_url" == *"Prowlarr/Indexers"* ]]; then log "DEBUG" "Hello Servarr - Using target [master] branch for Prowlarr repo" else prowlarr_target_branch="jackett-pulls" log "DEBUG" "Hello User - Unable to target [master]. Using target [jackett-pulls] branch for Fork repo" fi fi } check_branches() { local remote_pulls_check local_pulls_check remote_pulls_check=$(git ls-remote --heads "$prowlarr_remote_name" "$prowlarr_target_branch") local_pulls_check=$(git branch --list "$prowlarr_target_branch") if [ -z "$local_pulls_check" ]; then local_exist=false log "WARN" "local branch [$prowlarr_target_branch] does not exist" else local_exist=true log "INFO" "local branch [$prowlarr_target_branch] does exist" fi if [ -z "$remote_pulls_check" ]; then pulls_exists=false log "WARN" "remote repo/branch [$prowlarr_remote_name/$prowlarr_target_branch] does not exist" else pulls_exists=true log "INFO" "remote repo/branch [$prowlarr_remote_name/$prowlarr_target_branch] does exist" fi } git_branch_reset() { if [ "$pulls_exists" = false ]; then if [ "$local_exist" = true ]; then git checkout -B "$prowlarr_target_branch" log "INFO" "Checked out out local branch [$prowlarr_target_branch]" if [ "$is_dev_exec" = true ] || [ "$is_jackett_dev" = true ]; then log "DEBUG" "[$is_dev_exec] skipping reset to [$prowlarr_remote_name/$PROWLARR_RELEASE_BRANCH]" else git reset --hard "$prowlarr_remote_name"/"$PROWLARR_RELEASE_BRANCH" log "WARN" "local branch [$prowlarr_target_branch] hard reset based on remote/branch [$prowlarr_remote_name/$PROWLARR_RELEASE_BRANCH]" fi else git checkout -B "$prowlarr_target_branch" "$prowlarr_remote_name"/"$PROWLARR_RELEASE_BRANCH" --no-track log "INFO" "local branch [$prowlarr_target_branch] created from remote/branch [$prowlarr_remote_name/$PROWLARR_RELEASE_BRANCH]" fi else if [ "$local_exist" = true ]; then git checkout -B "$prowlarr_target_branch" log "INFO" "Checked out out local branch [$prowlarr_target_branch]" if [ "$is_dev_exec" = true ] || [ "$is_jackett_dev" = true ]; then log "DEBUG" "Development Mode - Skipping reset to [$prowlarr_remote_name/$prowlarr_target_branch]" else git reset --hard "$prowlarr_remote_name"/"$prowlarr_target_branch" # Try to rebase on master if ! git rebase "$prowlarr_remote_name"/"$PROWLARR_RELEASE_BRANCH"; then if [ "$automation_mode" = true ]; then log "WARN" "Rebase failed in automation mode, starting fresh from master instead" git rebase --abort git reset --hard "$prowlarr_remote_name"/"$PROWLARR_RELEASE_BRANCH" log "INFO" "local [$prowlarr_target_branch] reset to master [$prowlarr_remote_name/$PROWLARR_RELEASE_BRANCH]" else log "ERROR" "Rebase failed due to conflicts with [$prowlarr_remote_name/$PROWLARR_RELEASE_BRANCH]" git rebase --abort exit 9 fi else log "INFO" "local [$prowlarr_target_branch] reset and rebased on [$prowlarr_remote_name/$PROWLARR_RELEASE_BRANCH]" fi fi else git checkout -B "$prowlarr_target_branch" "$prowlarr_remote_name"/"$prowlarr_target_branch" log "INFO" "local [$prowlarr_target_branch] created from [$prowlarr_remote_name/$prowlarr_target_branch]" # Try to rebase on master if ! git rebase "$prowlarr_remote_name"/"$PROWLARR_RELEASE_BRANCH"; then if [ "$automation_mode" = true ]; then log "WARN" "Rebase failed in automation mode, starting fresh from master instead" git rebase --abort git checkout -B "$prowlarr_target_branch" "$prowlarr_remote_name"/"$PROWLARR_RELEASE_BRANCH" --no-track log "INFO" "local [$prowlarr_target_branch] created fresh from master [$prowlarr_remote_name/$PROWLARR_RELEASE_BRANCH]" else log "ERROR" "Rebase failed due to conflicts with [$prowlarr_remote_name/$PROWLARR_RELEASE_BRANCH]" git rebase --abort exit 9 fi else log "INFO" "rebased [$prowlarr_target_branch] on [$prowlarr_remote_name/$PROWLARR_RELEASE_BRANCH]" fi fi fi } pull_cherry_and_merge() { log "INFO" "Reviewing Commits" existing_message=$(git log --format=%B -n1) existing_message_ln1=$(echo "$existing_message" | awk 'NR==1') log "DEBUG" "Searching for commits with template: '$PROWLARR_COMMIT_TEMPLATE'" log "DEBUG" "Searching in last $MAX_COMMITS_TO_SEARCH commits" prowlarr_commits=$(git log --format=%B -n "$MAX_COMMITS_TO_SEARCH" | grep "^$PROWLARR_COMMIT_TEMPLATE") log "DEBUG" "Found prowlarr commits count: $(echo "$prowlarr_commits" | wc -l)" log "DEBUG" "First prowlarr commit found: $(echo "$prowlarr_commits" | head -1)" prowlarr_jackett_commit_message=$(echo "$prowlarr_commits" | awk 'NR==1') log "DEBUG" "Prowlarr jackett commit message: '$prowlarr_jackett_commit_message'" if [ "$is_jackett_dev" = true ]; then # Use only local Jackett branch (no remote) jackett_ref="$JACKETT_REMOTE_NAME$JACKETT_BRANCH" else # Normal usage: remote reference jackett_ref="$JACKETT_REMOTE_NAME/$JACKETT_BRANCH" fi if [ "$is_dev_exec" = true ] || [ "$is_jackett_dev" = true ]; then log "DEBUG" "Jackett Remote is [$jackett_ref]" # read -r -p "Pausing to review commits. Press any key to continue." -n1 -s fi jackett_recent_commit=$(git rev-parse "$jackett_ref") log "DEBUG" "Jackett recent commit: '$jackett_recent_commit'" # Get and log truncated Jackett commit message jackett_commit_message=$(git log --format=%s -n1 "$jackett_recent_commit") jackett_commit_message_truncated=$(echo "$jackett_commit_message" | cut -c1-80) log "INFO" "Jackett commit message: '$jackett_commit_message_truncated'" recent_pulled_commit=$(echo "$prowlarr_commits" | awk 'NR==1{print $5}') log "DEBUG" "Recent pulled commit (field 5): '$recent_pulled_commit'" log "DEBUG" "Full first commit line: '$(echo "$prowlarr_commits" | awk 'NR==1')'" if [ -z "$recent_pulled_commit" ]; then log "ERROR" "Recent Pulled Commit is empty. Failing." log "ERROR" "Debug info:" log "ERROR" " - prowlarr_commits found: '$(echo "$prowlarr_commits" | head -3)'" log "ERROR" " - Template used: '$PROWLARR_COMMIT_TEMPLATE'" log "ERROR" " - Trying to extract field 5 from first line" exit 3 fi if [ "$jackett_recent_commit" = "$recent_pulled_commit" ]; then log "SUCCESS" "--- we are current with jackett; nothing to do ---" exit 0 fi log "INFO" "[$jackett_recent_commit] is the most recent Jackett commit as per branch [$jackett_ref]" log "INFO" "[$recent_pulled_commit] is the most recent Prowlarr/Indexer commit pulled from Jackett as per branch [$prowlarr_remote_name/$prowlarr_target_branch]" # Define the command to get the commit range commit_range_cmd="git log --reverse --pretty='%n%H' $recent_pulled_commit..$jackett_recent_commit" # Execute the command and capture the output commit_range=$(eval "$commit_range_cmd") commit_count=$(git rev-list --count "$recent_pulled_commit".."$jackett_recent_commit") log "INFO" "There are [$commit_count] commits to cherry-pick" if [ "$is_dev_exec" = true ] || [ "$is_jackett_dev" = true ]; then log "DEBUG" "Get Range Command is [$commit_range_cmd]" # read -r -p "Pausing to review commits. Press any key to continue." -n1 -s fi # Enforce maximum commits threshold if [ "$commit_count" -gt "$MAX_COMMITS_TO_PICK" ]; then log "ERROR" "Commit count [$commit_count] is greater than [$MAX_COMMITS_TO_PICK]. Exiting." exit 4 fi log "INFO" "Commit Range is: [$commit_range]" log "INFO" "-- Beginning Cherrypicking ---" git config merge.directoryRenames true git config merge.verbosity 0 sleep 2 for pick_commit in ${commit_range}; do has_conflicts=$(git ls-files --unmerged; git status --porcelain | grep "^UU\|^AA\|^DD\|^AU\|^UA\|^DU\|^UD" || true) if [ -n "$has_conflicts" ]; then resolve_conflicts fi has_conflicts=$(git ls-files --unmerged; git status --porcelain | grep "^UU\|^AA\|^DD\|^AU\|^UA\|^DU\|^UD" || true) if [ -n "$has_conflicts" ]; then log "ERROR" "Conflicts Exist [$has_conflicts] - Cannot Cherrypick" git status if [ "$automation_mode" = true ]; then log "ERROR" "Automation mode: Cannot continue with unresolved conflicts" exit 5 else read -r -p "Pausing due to conflicts. Press any key to continue when resolved." -n1 -s log "INFO" "Continuing Cherrypicking" fi fi log "INFO" "cherrypicking Jackett commit [$pick_commit]" # Get and log the commit message for this specific commit pick_commit_message=$(git log --format=%s -n1 "$pick_commit") pick_commit_message_truncated=$(echo "$pick_commit_message" | cut -c1-80) log "INFO" "Commit message: '$pick_commit_message_truncated'" git cherry-pick --no-commit --rerere-autoupdate --allow-empty --keep-redundant-commits "$pick_commit" # Detect new indexers from this commit before conflict resolution new_jackett_indexers=$(git diff --cached --diff-filter=A --name-only | grep "src/Jackett.Common/Definitions/.*\.yml$" || true) if [ -n "$new_jackett_indexers" ]; then log "INFO" "New indexers from Jackett: [$new_jackett_indexers]" fi has_conflicts=$(git ls-files --unmerged; git status --porcelain | grep "^UU\|^AA\|^DD\|^AU\|^UA\|^DU\|^UD" || true) if [ -n "$has_conflicts" ]; then resolve_conflicts fi git config merge.directoryRenames conflict git config merge.verbosity 2 done log "SUCCESS" "--- Completed cherry picking ---" log "INFO" "Evaluating and Reviewing Changes" # Remove unwanted file types that Jackett might have added # Explicitly exclude important repository files and schema files unwanted_files=$(git diff --cached --name-only | grep -E "$CONFLICTS_NONYML_EXTENSIONS" | grep -v -E '^(README\.md|CONTRIBUTING\.md|LICENSE\.md)$' | grep -v 'schema\.json$' || true) if [ -n "$unwanted_files" ]; then log "INFO" "Removing unwanted file types from Jackett" echo "$unwanted_files" | while IFS= read -r file; do log "DEBUG" "Removing: $file" git rm --cached "$file" 2>/dev/null || true done fi # Always use Prowlarr's version of important repository markdown files for repo_file in README.md CONTRIBUTING.md LICENSE.md; do if git diff --cached --name-only | grep -q "^${repo_file}$"; then log "DEBUG" "Ensuring Prowlarr version of $repo_file is used" git checkout HEAD -- "$repo_file" 2>/dev/null || true fi done # Also remove any src/ directories that might have been added src_files=$(git diff --cached --name-only | grep '^src/' || true) if [ -n "$src_files" ]; then log "INFO" "Removing src/ files from Jackett" git rm --cached -r src/ 2>/dev/null || true fi # Checkout schema files if they exist - expand the glob pattern first schema_files=$(find definitions -type f -name "schema.json" -path "*/v[0-9]*/schema.json" 2>/dev/null) if [ -n "$schema_files" ]; then for schema_file in $schema_files; do git checkout HEAD -- "$schema_file" 2>/dev/null || true done else log "DEBUG" "No schema.json files found to checkout" fi handle_new_indexers handle_modified_indexers handle_backporting_indexers } resolve_unmerged_files() { # Check for any remaining unmerged files and resolve them unmerged_files=$(git diff --name-only --diff-filter=U 2>/dev/null || true) if [ -n "$unmerged_files" ]; then log "WARN" "Unmerged files detected: [$unmerged_files]" echo "$unmerged_files" | while IFS= read -r file; do if [ -n "$file" ]; then if [[ "$file" == .github/* ]] || [[ "$file" == src/* ]] || [[ "$file" == *.md ]] || [[ "$file" == package*.json ]] || [[ "$file" == .editorconfig ]]; then # For non-definition files, prefer our version or remove log "DEBUG" "Resolving unmerged non-definition file: [$file]" if git ls-files | grep -q "^$file$"; then git checkout --ours "$file" 2>/dev/null || git rm --force "$file" 2>/dev/null || true else git rm --force "$file" 2>/dev/null || true fi git add "$file" 2>/dev/null || true elif [[ "$file" == definitions/* ]]; then # For definition files, prefer their version log "DEBUG" "Resolving unmerged definition file: [$file]" git checkout --theirs "$file" 2>/dev/null || true git add "$file" 2>/dev/null || true else # Default: prefer our version log "DEBUG" "Resolving unmerged file (default): [$file]" git checkout --ours "$file" 2>/dev/null || git rm --force "$file" 2>/dev/null || true git add "$file" 2>/dev/null || true fi fi done fi } resolve_conflicts() { # Get list of actual conflicted files conflicted_files=$(git diff --name-only --diff-filter=U) readme_conflicts=$(echo "$conflicted_files" | grep -E '^README\.md$' || true) # Explicitly exclude important repository files and schema files from nonyml conflicts nonyml_conflicts=$(echo "$conflicted_files" | grep -E "$CONFLICTS_NONYML_EXTENSIONS" | grep -v -E '^(README\.md|CONTRIBUTING\.md|LICENSE\.md)$' | grep -v 'schema\.json$' || true) yml_conflicts=$(echo "$conflicted_files" | grep -E '\.ya?ml$' || true) schema_conflicts=$(echo "$conflicted_files" | grep -E '\.schema\.json$' || true) log "WARN" "conflicts exist" if [ -n "$readme_conflicts" ]; then log "DEBUG" "README conflict exists; using Prowlarr README" git checkout --ours README.md git add --force README.md fi if [ -n "$schema_conflicts" ]; then log "DEBUG" "Schema conflict exists; using Prowlarr schema" for file in $schema_conflicts; do if git ls-files | grep -q "^$file$"; then git checkout --ours "$file" 2>/dev/null || true git add --force "$file" 2>/dev/null || true fi done fi # Handle "both added" definition files (when Git auto-moves files from Jackett paths) both_added_defs=$(git status --porcelain | grep "^AA" | grep "definitions/v[0-9].*\.yml$" | awk '{print $2}') if [ -n "$both_added_defs" ]; then log "DEBUG" "Both added definition conflicts exist; using Jackett's version" for file in $both_added_defs; do log "INFO" "NEW INDEXER: Resolving both-added conflict for [$file]" both_added_new_indexers="$both_added_new_indexers $file" # For AA conflicts, we need to remove and re-add git rm "$file" 2>/dev/null || true git checkout --theirs "$file" 2>/dev/null || true git add "$file" 2>/dev/null || true done fi if [ -n "$nonyml_conflicts" ]; then log "DEBUG" "Non-YML conflicts exist; removing [\n$nonyml_conflicts\n] files and restoring [package.json package-lock.json .editorconfig]" while IFS= read -r file; do git rm --force --quiet --ignore-unmatch "$file" done <<<"$nonyml_conflicts" for file in package.json package-lock.json .editorconfig; do if git ls-files | grep -q "^$file$"; then git checkout --ours "$file" git add --force "$file" fi done fi if [ -n "$yml_conflicts" ]; then log "DEBUG" "YML conflict exists; [$yml_conflicts]" handle_general_yml_conflicts handle_definition_conflicts fi # Final cleanup of any remaining unmerged files resolve_unmerged_files } handle_general_yml_conflicts() { # Remove non-definition YAML files (config files, workflows, etc.) yml_remove=$(git status --porcelain | grep yml | grep -vi "definitions/" | grep -vi "Definitions/" | grep -v "definitions-update" | awk -F '[ADUMRC]{1,2} ' '{print $2}' | awk '{ gsub(/^[ \t]+|[ \t]+$/, ""); print }') if [ -n "$yml_remove" ]; then log "DEBUG" "Removing non-definition yml files: [$yml_remove]" echo "$yml_remove" | while IFS= read -r file; do if [ -n "$file" ]; then log "DEBUG" "Removing non-definition yml file: [$file]" git rm --force --ignore-unmatch "$file" 2>/dev/null || true fi done fi } handle_definition_conflicts() { # Handle indexer definition conflicts with special logic yml_conflicts=$($GIT_DIFF_CMD | grep "\.yml" || true) if [ -n "$yml_conflicts" ]; then yml_defs=$(git status --porcelain | grep yml | grep -i "definitions/") yml_add=$(echo "$yml_defs" | grep -v "UD\|D|DU" | awk -F '[ADUMRC]{1,2} ' '{print $2}' | awk '{ gsub(/^[ \t]+|[ \t]+$/, ""); print }') yml_delete=$(echo "$yml_defs" | grep "UD" | awk -F '[ADUMRC]{1,2} ' '{print $2}' | awk '{ gsub(/^[ \t]+|[ \t]+$/, ""); print }') log "DEBUG" "YML Definitions Process: [$yml_defs]" log "DEBUG" "YML files to add/process: [$yml_add]" log "DEBUG" "YML files to delete: [$yml_delete]" for def in $yml_add; do log "DEBUG" "Using & Adding Jackett's definition yml; [$def]" new_def="${def/src\/Jackett.Common\/Definitions\//definitions/v$MIN_SCHEMA/}" if [ "$new_def" != "$def" ]; then mkdir -p "$(dirname "$new_def")" log "INFO" "NEW INDEXER: Moving [$def] to [$new_def]" mv "$def" "$new_def" git checkout --theirs "$new_def" git add --force "$new_def" git rm --f --ignore-unmatch "$def" else git checkout --theirs "$def" git add --force "$def" fi done for def in $yml_delete; do log "DEBUG" "Removing definitions Jackett deleted; [$def]" git rm --f --ignore-unmatch "$def" done fi } handle_new_indexers() { # Debug: Show all staged files first all_staged=$(git diff --cached --name-only) log "DEBUG" "All staged files: [$all_staged]" # Debug: Show yml files specifically yml_staged=$(git diff --cached --name-only | grep ".yml" || true) log "DEBUG" "All staged yml files: [$yml_staged]" # Git's automatic directory rename may classify new files as renames (R) instead of additions (A) added_indexers=$(git diff --cached --diff-filter=AR --name-only | grep ".yml" | grep -E "v[[:digit:]]+") log "DEBUG" "New indexers detected (AR filter + v[digit]+): [$added_indexers]" if [ -n "$added_indexers" ]; then log "INFO" "New Indexers detected: [$added_indexers]" for indexer in ${added_indexers}; do base_indexer=$(basename "$indexer") log "DEBUG" "Evaluating [$indexer] against BLOCKLIST with name [$base_indexer]" # Check if the indexer is in the BLOCKLIST if [[ -n "${blocklist_map[$base_indexer]}" ]]; then log "INFO" "[$base_indexer] is in the BLOCKLIST. Removing [${indexer}]..." git rm --f --ignore-unmatch "$indexer" continue fi log "DEBUG" "Evaluating [$indexer] Cardigann Version" if [ -f "$indexer" ]; then # Check if this indexer replaces any existing indexers if command -v yq &> /dev/null; then replaced_indexers=$(yq eval '.replaces[]?' "$indexer" 2>/dev/null || true) elif $PYTHON_CMD -c "import yaml" &>/dev/null; then replaced_indexers=$($PYTHON_CMD -c " import yaml, sys try: with open('$indexer', 'r') as f: data = yaml.safe_load(f) replaces = data.get('replaces', []) if replaces: for item in replaces: print(item) except: pass " 2>/dev/null || true) fi if [ -n "$replaced_indexers" ]; then log "INFO" "Indexer [$indexer] replaces: [$replaced_indexers]" for replaced in $replaced_indexers; do # Find and remove all versions of the replaced indexer for ((i = MAX_SCHEMA; i >= MIN_SCHEMA; i--)); do replaced_file="definitions/v$i/${replaced}.yml" if [ -f "$replaced_file" ]; then log "INFO" "Removing replaced indexer: [$replaced_file]" git rm --f --ignore-unmatch "$replaced_file" fi done done fi determine_schema_version "$indexer" log "DEBUG" "Checked Version Output is $check_version" if [ "$check_version" != "v0" ]; then log "DEBUG" "Schema Test passed." updated_indexer=$indexer else determine_best_schema_version "$indexer" if [ "$matched_version" -eq 0 ]; then log "WARN" "Version [$NEW_SCHEMA] required. Review definition [$indexer]" v_matched="v$NEW_SCHEMA" else v_matched="v$matched_version" fi updated_indexer=${indexer/v[0-9]*/$v_matched} if [ "$indexer" != "$updated_indexer" ]; then log "INFO" "Moving indexer old [$indexer] to new [$updated_indexer]" mv "$indexer" "$updated_indexer" git rm -f "$indexer" git add -f "$updated_indexer" else log "DEBUG" "Doing nothing; [$indexer] already is [$updated_indexer]" fi fi fi done unset indexer unset test log "INFO" "completed new indexers" else log "INFO" "No new indexers" fi } handle_modified_indexers() { modified_indexers=$(git diff --cached --diff-filter=M --name-only | grep ".yml" | grep -E "v[[:digit:]]+") if [ -n "$modified_indexers" ]; then log "INFO" "Reviewing Modified Indexers..." for indexer in ${modified_indexers}; do log "INFO" "Evaluating [$indexer] Cardigann Version" if [ -f "$indexer" ]; then determine_schema_version "$indexer" log "INFO" "Checked Version Output is $check_version" if [ "$check_version" != "v0" ]; then log "DEBUG" "Schema Test passed." updated_indexer=$indexer else determine_best_schema_version "$indexer" if [ "$matched_version" -eq 0 ]; then log "WARN" "Version [$NEW_SCHEMA] required. Review definition [$indexer]" v_matched="v$NEW_SCHEMA" else v_matched="v$matched_version" fi updated_indexer=${indexer/v[0-9]*/$v_matched} if [ "$indexer" != "$updated_indexer" ]; then log "INFO" "Version bumped indexer old [$indexer] to new [$updated_indexer]" mv "$indexer" "$updated_indexer" git checkout HEAD -- "$indexer" git add -f "$updated_indexer" else log "INFO" "Doing nothing; [$indexer] already is [$updated_indexer]" fi fi fi done unset indexer unset test fi log "SUCCESS" "--- completed changed indexers ---" } handle_backporting_indexers() { modified_indexers_vcheck=$(git diff --cached --diff-filter=AM --name-only | grep ".yml" | grep -E "v[[:digit:]]+") if [ -n "$modified_indexers_vcheck" ]; then for indexer in ${modified_indexers_vcheck}; do # SC2004: $/${} is unnecessary on arithmetic variables. for ((i = MAX_SCHEMA; i >= MIN_SCHEMA; i--)); do version="v$i" log "DEBUG" "looking for [$version] indexer of [$indexer]" indexer_check=$(echo "$indexer" | sed -E "s/v[0-9]+/$version/") if [ "$indexer_check" != "$indexer" ] && [ -f "$indexer_check" ]; then if [ "$SKIP_BACKPORT" = true ]; then log "INFO" "Found [v$i] indexer for [$indexer] - Skipping backporting changes" log "DEBUG" "Skipping backporting changes. Skipping further backport checks." BACKPORT_SKIPPED=true # Sets the wider variable return 0 # Exits the function early else log "INFO" "Found [v$i] indexer for [$indexer] - comparing to [$indexer_check]" log "WARN" "HUMAN! Review this change and ensure no incompatible updates are backported." git difftool --no-index "$indexer" "$indexer_check" git add "$indexer_check" fi fi done done unset indexer unset indexer_check fi newschema_indexers=$(git diff --cached --diff-filter=A --name-only | grep ".yml" | grep "v$NEW_SCHEMA") if [ -n "$newschema_indexers" ]; then for indexer in ${newschema_indexers}; do # SC2004: $/${} is unnecessary on arithmetic variables. for ((i = MAX_SCHEMA; i >= MIN_SCHEMA; i--)); do version="v$i" log "DEBUG" "looking for [$version] indexer of [$indexer]" indexer_check=$(echo "$indexer" | sed -E "s/v[0-9]+/$version/") if [ "$indexer_check" != "$indexer" ] && [ -f "$indexer_check" ]; then log "INFO" "Found [v$i] indexer for [$indexer] - comparing to [$indexer_check]" log "ERROR" "THIS IS A NEW CARDIGANN VERSION THAT IS REQUIRED" log "WARN" "HUMAN! Review this change and ensure no incompatible updates are backported." git difftool --no-index "$indexer" "$indexer_check" git add "$indexer_check" fi done done unset indexer unset indexer_check fi log "SUCCESS" "--- completed backporting indexers ---" } cleanup_and_commit() { if [ -n "$removed_indexers" ]; then for indexer in ${removed_indexers}; do log "DEBUG" "looking for previous versions of removed indexer [$indexer]" # SC2004: $/${} is unnecessary on arithmetic variables. for ((i = MAX_SCHEMA; i >= MIN_SCHEMA; i--)); do indexer_remove=$(echo "$indexer" | sed -E "s/v[0-9]+/$version/") if [ "$indexer_remove" != "$indexer" ] && [ -f "$indexer_remove" ]; then log "INFO" "Found [v$i] indexer for [$indexer] - removing [$indexer_remove]" rm -f "$indexer_remove" git rm --f --ignore-unmatch "$indexer_remove" fi done done unset indexer unset indexer_remove fi # Recalculated Added / Modified / Removed - include renames (R) for directory rename detection staged_added=$(git diff --cached --diff-filter=AR --name-only | grep ".yml" | grep -E "v[[:digit:]]+") # Combine with both-added indexers resolved during conflicts added_indexers=$(echo "$staged_added $both_added_new_indexers" | xargs -n1 | sort -u | xargs) modified_indexers=$(git diff --cached --diff-filter=M --name-only | grep ".yml" | grep -E "v[[:digit:]]+") removed_indexers=$(git diff --cached --diff-filter=D --name-only | grep ".yml" | grep -E "v[[:digit:]]+") newschema_indexers=$(git diff --cached --diff-filter=A --name-only | grep ".yml" | grep -E "v$NEW_SCHEMA") # Check if there are added indexers and log if present if [ -n "$added_indexers" ]; then log "SUCCESS" "Added Indexers are [$added_indexers]" fi # Check if there are modified indexers and log if present if [ -n "$modified_indexers" ]; then log "SUCCESS" "Modified Indexers are [$modified_indexers]" fi # Check if there are removed indexers and log if present if [ -n "$removed_indexers" ]; then log "SUCCESS" "Removed Indexers are [$removed_indexers]" fi # Check if there are new schema indexers and log if present if [ -n "$newschema_indexers" ]; then log "WARN" "New Schema Indexers are [$newschema_indexers]" fi if [ -d "$NEW_VERS_DIR" ]; then if [ "$(ls -A $NEW_VERS_DIR)" ]; then log "ERROR" "THIS IS A NEW CARDIGANN VERSION THAT IS REQUIRED: Version [v$NEW_SCHEMA] is needed." log "WARNING" "Review the following definitions for new Cardigann Version: $newschema_indexers" else rmdir "$NEW_VERS_DIR" fi fi git rm -r -f -q --ignore-unmatch --cached node_modules if [ "$automation_mode" = true ]; then log "INFO" "Automation mode: Proceeding with commit and push automatically" else log "WARNING" "After review; the script will commit the changes and push as/if specified." read -r -p "Press any key to continue or [Ctrl-C] to abort. Waiting for human review..." -n1 -s fi new_commit_msg="$PROWLARR_COMMIT_TEMPLATE $jackett_recent_commit [$(date -u +'%Y-%m-%dT%H:%M:%SZ')]" if [ "$BACKPORT_SKIPPED" = true ]; then new_commit_msg+=" $PROWLARR_COMMIT_TEMPLATE_APPEND" fi # Append to the commit the list of all added, removed, and modified indexers if [ -n "$added_indexers" ]; then new_commit_msg+=$'\n\n'"Added Indexers: $added_indexers" fi if [ -n "$removed_indexers" ]; then new_commit_msg+=$'\n\n'"Removed Indexers: $removed_indexers" fi if [ -n "$modified_indexers" ]; then new_commit_msg+=$'\n\n'"Modified Indexers: $modified_indexers" fi if [ -n "$newschema_indexers" ]; then new_commit_msg+=$'\n\n'"New Schema Indexers: $newschema_indexers" fi # Check for unresolved conflicts before committing unresolved_conflicts=$(git status --porcelain | grep "^UU\|^AA\|^DD\|^AU\|^UA\|^DU\|^UD" || true) if [ -n "$unresolved_conflicts" ]; then log "ERROR" "Cannot commit: Unresolved conflicts detected:" echo "$unresolved_conflicts" git status exit 6 fi # Check if there are any staged changes to commit staged_changes=$(git diff --cached --name-only) if [ -z "$staged_changes" ]; then log "WARN" "No changes to commit - all changes may have been filtered by blocklist or conflicts" log "INFO" "Exiting gracefully as we're already up to date" exit 0 fi if [ "$pulls_exists" = true ] && [ "$prowlarr_target_branch" != "$PROWLARR_RELEASE_BRANCH" ]; then if [ "$existing_message_ln1" = "$prowlarr_jackett_commit_message" ]; then if ! git commit --amend -m "$new_commit_msg" -m "$existing_message"; then log "ERROR" "Failed to amend commit" exit 7 fi log "INFO" "Commit Appended - [$new_commit_msg]" push_mode_force=true # Auto-enable force push after amend else if ! git commit -m "$new_commit_msg"; then log "ERROR" "Failed to create new commit" exit 7 fi log "INFO" "New Commit made - [$new_commit_msg]" fi else if ! git commit -m "$new_commit_msg"; then log "ERROR" "Failed to create new commit" exit 7 fi log "INFO" "New Commit made - [$new_commit_msg]" fi } push_changes() { push_branch="$prowlarr_target_branch" log "INFO" "Evaluating for Push to Remote" log "DEBUG" " Push Modes for Branch: $push_branch" log "DEBUG" "Push To Remote: $push_mode with Force Push With Lease: $push_mode_force" # Safety check: NEVER force push to master or main branches if [ "$push_mode_force" = true ] && { [ "$push_branch" = "master" ] || [ "$push_branch" = "main" ]; }; then log "ERROR" "Force push to $push_branch branch is forbidden for safety" push_mode_force=false log "WARN" "Disabled force push - will attempt regular push instead" fi # For automated-indexer-sync branch in automation mode, enable force push if needed if [ "$automation_mode" = true ] && [ "$push_branch" = "automated-indexer-sync" ]; then # Check if the remote branch exists and if we're behind if git rev-parse --verify "$prowlarr_push_remote/$push_branch" >/dev/null 2>&1; then local_commit=$(git rev-parse HEAD) remote_commit=$(git rev-parse "$prowlarr_push_remote/$push_branch") if [ "$local_commit" != "$remote_commit" ]; then # Check if we're behind (remote has commits we don't have) if ! git merge-base --is-ancestor "$remote_commit" "$local_commit"; then log "INFO" "Local branch has diverged from remote automated-indexer-sync, enabling force push" push_mode_force=true fi fi fi fi if [ "$push_mode" = true ] && [ "$push_mode_force" = true ]; then log "DEBUG" "Push To Remote: $push_mode with Force Push With Lease: $push_mode_force" if git push "$prowlarr_push_remote" "$push_branch" --force-if-includes --force-with-lease --set-upstream; then log "WARN" "[$prowlarr_push_remote $push_branch] Branch Force Pushed" else log "ERROR" "Failed to force push to [$prowlarr_push_remote $push_branch]" exit 8 fi elif [ "$push_mode" = true ]; then if git push "$prowlarr_push_remote" "$push_branch" --force-if-includes --set-upstream; then log "SUCCESS" "[$prowlarr_push_remote $push_branch] Branch Pushed" else log "ERROR" "Failed to push to [$prowlarr_push_remote $push_branch]" exit 8 fi else log "SUCCESS" "Skipping Push to [$prowlarr_push_remote/$push_branch] you should consider pushing manually and/or submitting a pull-request." fi # Output pull request URL if push was successful if [ "$push_mode" = true ]; then fork_url=$(git remote get-url "$prowlarr_push_remote" 2>/dev/null | sed 's/\.git$//' | sed 's/git@github\.com:/https:\/\/github.com\//') if [[ "$fork_url" == *"github.com"* ]]; then fork_owner=$(echo "$fork_url" | sed 's/.*github\.com[\/:]*//' | cut -d'/' -f1) # Only show PR URL if not pushing to Prowlarr repo itself if [ "$fork_owner" != "Prowlarr" ]; then pr_url="https://github.com/Prowlarr/Indexers/compare/master...$fork_owner:$push_branch" log "SUCCESS" "Create pull request: $pr_url" fi fi fi } main() { initialize_script load_blocklist configure_git check_branches git_branch_reset pull_cherry_and_merge cleanup_and_commit push_changes } main "$@"