Prowlarr/Indexers
1
0
Fork 0
mirror of https://github.com/Prowlarr/Indexers.git synced 2025-12-20 02:58:28 +00:00
Code Activity
Files
689c2e39e3adf2195b8d2bf4c116b999bf1d4457
Indexers/scripts/indexer-sync-v2.sh
ServarrAdmin 689c2e39e3 fix: add missing dollar sign to GIT_DIFF_CMD variable
2025-08-15 20:24:20 -05:00

930 lines
38 KiB
Bash
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/bin/bash
# shellcheck disable=SC2162
## Script to keep Prowlarr/Indexers up to date with Jackett/Jackett
## Created by Bakerboy448
## Requirements
### Prowlarr/Indexers local git repo exists
### Set variables as needed
### Typically only prowlarr_git_path would be needed to be set
## Using the Script
### Suggested to run from the current directory being Prowlarr/Indexers local Repo using Git Bash `./scripts/indexer-sync-v2.sh`
# Default values
prowlarr_remote_name="origin"
prowlarr_target_branch="master"
mode_choice="normal"
push_mode=false
push_mode_force=false
prowlarr_push_remote="origin"
PROWLARR_COMMIT_TEMPLATE="jackett indexers as of"
PROWLARR_COMMIT_TEMPLATE_APPEND=""
PROWLARR_REPO_URL="https://github.com/Prowlarr/Indexers.git"
JACKETT_REPO_URL="https://github.com/Jackett/Jackett.git"
PROWLARR_RELEASE_BRANCH="master"
JACKETT_BRANCH="master"
JACKETT_REMOTE_NAME="z_Jackett"
SKIP_BACKPORT=false
is_dev_exec=false
is_jackett_dev=false
pulls_exists=false
local_exist=false
automation_mode=false
MAX_COMMITS_TO_PICK=50
MAX_COMMITS_TO_SEARCH=100
VALIDATION_SCRIPT="scripts/validate.py"
BLOCKLIST=("uniongang.yml" "uniongangcookie.yml" "sharewood.yml" "ygg-api.yml" "anirena.yml" "torrentgalaxy.yml" "torrent-heaven.yml" "scenelinks.yml")
CONFLICTS_NONYML_EXTENSIONS='\.(cs|js|iss|html|ico|png|csproj)$'
# Initialize Defaults
removed_indexers=""
added_indexers=""
modified_indexers=""
newschema_indexers=""
BACKPORT_SKIPPED=false
GIT_DIFF_CMD="git diff --cached --name-only"
declare -A blocklist_map
for blocked in "${BLOCKLIST[@]}"; do
blocklist_map["$blocked"]=1
done
usage() {
echo "Usage: $0 [options]
Options:
-r <remote(pull)> Set the Prowlarr remote name (for pulling/syncing). Default: '$prowlarr_remote_name'
-o <remote(push)> Set the push remote name (for pushing changes). Default: '$prowlarr_push_remote'
-b <branch> Set the Prowlarr target branch. Default: '$prowlarr_target_branch'
-m <mode> Set the operational mode:
- (normal): Default mode with regular operations.
- (dev | development | D | d):
Enables development mode:
- Skips upstream reset.
- Uses local branches.
- Pauses at debugging points for review.
- (jackett | j | J):
Enables Jackett development mode:
- Skips upstream reset.
- Uses existing local Jackett and Prowlarr branches.
- Pauses at debugging points for review.
- Local branch used: $JACKETT_REMOTE_NAME$JACKETT_BRANCH
Default: $mode_choice
-p Enable push to remote. Default: $push_mode
-f Force push if pushing. Default: $push_mode_force
-c <commit_template> Set the commit template for Prowlarr. Default: $PROWLARR_COMMIT_TEMPLATE
-u <repo_url> Set the Prowlarr repository URL. Default: $PROWLARR_REPO_URL
-j <repo_url> Set the Jackett repository URL. Default: $JACKETT_REPO_URL
-R <release_branch> Set the Prowlarr release branch. Default: $PROWLARR_RELEASE_BRANCH
-J <jackett_branch> Set the Jackett branch. Default: $JACKETT_BRANCH
-n <remote_name> Set the Jackett remote name. Default: $JACKETT_REMOTE_NAME
-z Skip backporting. Default: $SKIP_BACKPORT
-a Enable automation mode (skip interactive prompts). Default: $automation_mode"
exit 1
}
# Prowlarr Schema Versions
## v1 frozen 2021-10-13
## v2 frozen 2022-04-18
## v1 and v2 purged and moved to v3 2022-06-24
## v3 purged and frozen 2022-07-22
## v4 purged and frozen 2022-08-18
## v5 purged and frozen 2022-10-14
## v6 purged and frozen 2022-10-14
## v7 purged and frozen 2024-04-27
## v8 purged and frozen 2024-04-27
## v9 purged and frozen 2024-10-13
# Load schema versions from VERSIONS file
load_versions() {
MIN_SCHEMA=10
MAX_SCHEMA=11
CURRENT_SCHEMA=11
if [ -f "VERSIONS" ]; then
while IFS='=' read -r key value; do
case "$key" in
MIN_VERSION) MIN_SCHEMA="$value" ;;
MAX_VERSION) MAX_SCHEMA="$value" ;;
CURRENT_VERSION) CURRENT_SCHEMA="$value" ;;
esac
done < <(grep -v '^#' VERSIONS | grep '=')
fi
NEW_SCHEMA=$((MAX_SCHEMA + 1))
}
load_versions
NEW_VERS_DIR="definitions/v$NEW_SCHEMA"
mkdir -p "$NEW_VERS_DIR"
log() {
local level="$1"
local message="$2"
local color_reset="\033[0m"
local color_success="\033[0;32m" # Green
local color_info="\033[0;36m" # Cyan
local color_warn="\033[0;33m" # Yellow
local color_debug="\033[0;34m" # Blue
local color_error="\033[0;31m" # Red
local color
case "$level" in
SUCCESS)
level="INFO"
message="SUCCESS|$message"
color=$color_success
;;
INFO)
color=$color_info
;;
WARN)
color=$color_warn
;;
WARNING)
color=$color_warn
level="WARN"
;;
DEBUG)
color=$color_debug
;;
ERROR)
color=$color_error
;;
*)
color=$color_reset
;;
esac
echo -e "${color}$(date +'%Y-%m-%dT%H:%M:%S%z')|$level|$message${color_reset}"
}
determine_schema_version() {
local def_file="$1"
log "DEBUG" "Testing schema version of [$def_file]"
check_version=$(echo "$def_file" | cut -d'/' -f2)
dir="definitions/$check_version"
schema="$dir/schema.json"
log "DEBUG" "Checking file against schema [$schema]"
local test_output
$PYTHON_CMD "$VALIDATION_SCRIPT" --single "$def_file" "$schema"
test_output=$?
if [ "$test_output" = 0 ]; then
log "INFO" "Definition [$def_file] matches schema [$schema]"
else
check_version="v0"
fi
export check_version=$check_version
}
determine_best_schema_version() {
local def_file="$1"
log "INFO" "Determining best schema version for [$def_file]"
# Use Python function to find best schema version
local best_version
best_version=$($PYTHON_CMD "$VALIDATION_SCRIPT" --find-best-version "$def_file")
if [[ "$best_version" =~ ^v([0-9]+)$ ]]; then
matched_version="${BASH_REMATCH[1]}"
log "INFO" "Definition [$def_file] best matches schema [$best_version]"
else
matched_version=0
log "WARN" "Definition [$def_file] does not match any schema"
log "ERROR" "Cardigann update likely needed. Version [v$NEW_SCHEMA] required. Review definition."
fi
export matched_version=$matched_version
}
initialize_script() {
# Check for Python and virtual environment
# Check for Python and determine command to use
PYTHON_CMD=""
if command -v python3 &> /dev/null; then
PYTHON_CMD="python3"
elif command -v python &> /dev/null; then
PYTHON_CMD="python"
else
log "ERROR" "Python could not be found. Check your Python installation"
exit 1
fi
log "DEBUG" "Using Python command: $PYTHON_CMD"
# Check if we have a virtual environment and activate it
if [ -d ".venv" ]; then
log "INFO" "Activating virtual environment"
if [ -f ".venv/bin/activate" ]; then
# Linux/Mac
source .venv/bin/activate
elif [ -f ".venv/Scripts/activate" ]; then
# Windows
source .venv/Scripts/activate
fi
fi
# Check if required Python packages are available
if ! $PYTHON_CMD -c "import jsonschema, yaml" &>/dev/null; then
log "ERROR" "required python packages are missing. Install with: pip install -r requirements.txt"
exit 2
fi
log "INFO" "Using Python validation"
}
while getopts "frpzab:m:c:u:j:R:J:n:o:" opt; do
case ${opt} in
f)
# No Arg
push_mode_force=true
log "DEBUG" "push_mode_force is $push_mode_force"
;;
r)
prowlarr_remote_name=$OPTARG
log "DEBUG" "prowlarr_remote_name using argument $prowlarr_remote_name"
;;
b)
prowlarr_target_branch=$OPTARG
log "DEBUG" "prowlarr_target_branch using argument $prowlarr_target_branch"
;;
o)
prowlarr_push_remote=$OPTARG
log "DEBUG" "prowlarr_push_remote using argument $prowlarr_push_remote"
;;
m)
mode_choice=$OPTARG
log "DEBUG" "mode_choice using argument $mode_choice"
case "$mode_choice" in
normal | n | N)
is_dev_exec=false
;;
development | dev | d | D)
is_dev_exec=true
log "WARN" "Skipping upstream reset to local. Skip checking out the local Prowlarr branch and output the details."
log "INFO" "This will not reset Prowlarr branch from upstream/master and will ONLY checkout the selected branch to use."
log "INFO" "This will pause at various debugging points for human review"
;;
jackett | j | J)
is_dev_exec=true
is_jackett_dev=true
log "WARN" "Skipping upstream reset to local. Skip checking out the local Prowlarr branch and output the details."
log "INFO" "This will not reset Prowlarr branch from upstream/master and will ONLY checkout [$prowlarr_target_branch] branch to use."
log "INFO" "This will not reset Jackett branch and will use what it currently locally is $JACKETT_REMOTE_NAME$JACKETT_BRANCH"
log "INFO" "This will pause at various debugging points for human review"
;;
*)
usage
;;
esac
;;
p)
# No Arg
push_mode=true
log "DEBUG" "push_mode is $push_mode"
;;
c)
PROWLARR_COMMIT_TEMPLATE=$OPTARG
log "DEBUG" "PROWLARR_COMMIT_TEMPLATE using argument $PROWLARR_COMMIT_TEMPLATE"
;;
u)
PROWLARR_REPO_URL=$OPTARG
log "DEBUG" "PROWLARR_REPO_URL using argument $PROWLARR_REPO_URL"
;;
j)
JACKETT_REPO_URL=$OPTARG
log "DEBUG" "JACKETT_REPO_URL using argument $JACKETT_REPO_URL"
;;
R)
PROWLARR_RELEASE_BRANCH=$OPTARG
log "DEBUG" "PROWLARR_RELEASE_BRANCH using argument $PROWLARR_RELEASE_BRANCH"
;;
J)
JACKETT_BRANCH=$OPTARG
log "DEBUG" "JACKETT_BRANCH using argument $JACKETT_BRANCH"
;;
n)
JACKETT_REMOTE_NAME=$OPTARG
log "DEBUG" "JACKETT_REMOTE_NAME using argument $JACKETT_REMOTE_NAME"
;;
z)
# No Arg
SKIP_BACKPORT=true
PROWLARR_COMMIT_TEMPLATE_APPEND="[backports skipped - TODO]"
log "DEBUG" "SKIP_BACKPORT is $SKIP_BACKPORT. Commit Template will be appended with '$PROWLARR_COMMIT_TEMPLATE_APPEND' if applicable'"
;;
a)
# No Arg
automation_mode=true
log "DEBUG" "automation_mode is $automation_mode - interactive prompts will be skipped"
;;
\?)
usage
;;
esac
done
shift $((OPTIND - 1))
configure_git() {
git config advice.statusHints false
git_remotes=$(git remote -v)
prowlarr_remote_exists=$(echo "$git_remotes" | grep "$prowlarr_remote_name")
prowlarr_push_remote_exists=$(echo "$git_remotes" | grep "$prowlarr_push_remote")
jackett_remote_exists=$(echo "$git_remotes" | grep "$JACKETT_REMOTE_NAME")
if [ -z "$prowlarr_remote_exists" ]; then
git remote add "$prowlarr_remote_name" "$PROWLARR_REPO_URL"
log "DEBUG" "git remote add $prowlarr_remote_name $PROWLARR_REPO_URL"
fi
if [ -z "$jackett_remote_exists" ]; then
git remote add "$JACKETT_REMOTE_NAME" "$JACKETT_REPO_URL"
log "DEBUG" "git remote add $JACKETT_REMOTE_NAME $JACKETT_REPO_URL"
fi
if [ "$prowlarr_push_remote" != "$prowlarr_remote_name" ] && [ -z "$prowlarr_push_remote_exists" ]; then
log "ERROR" "Push remote [$prowlarr_push_remote] does not exist. Please add it manually or use an existing remote."
exit 1
fi
log "INFO" "Configured Git"
if [ "$is_jackett_dev" = true ]; then
log "DEBUG" "Skipping fetch for jackett development mode"
else
git fetch --all --prune --progress
fi
# Set default target branch based on push remote URL (only if using default)
push_remote_url=$(git remote get-url "$prowlarr_push_remote" 2>/dev/null || echo "")
if [ "$prowlarr_target_branch" = "master" ]; then
if [[ "$push_remote_url" == *"Prowlarr/Indexers"* ]]; then
log "DEBUG" "Hello Servarr - Using target [master] branch for Prowlarr repo"
else
prowlarr_target_branch="jackett-pulls"
log "DEBUG" "Hello User - Unable to target [master]. Using target [jackett-pulls] branch for Fork repo"
fi
fi
}
check_branches() {
local remote_pulls_check local_pulls_check
remote_pulls_check=$(git ls-remote --heads "$prowlarr_remote_name" "$prowlarr_target_branch")
local_pulls_check=$(git branch --list "$prowlarr_target_branch")
if [ -z "$local_pulls_check" ]; then
local_exist=false
log "WARN" "local branch [$prowlarr_target_branch] does not exist"
else
local_exist=true
log "INFO" "local branch [$prowlarr_target_branch] does exist"
fi
if [ -z "$remote_pulls_check" ]; then
pulls_exists=false
log "WARN" "remote repo/branch [$prowlarr_remote_name/$prowlarr_target_branch] does not exist"
else
pulls_exists=true
log "INFO" "remote repo/branch [$prowlarr_remote_name/$prowlarr_target_branch] does exist"
fi
}
git_branch_reset() {
if [ "$pulls_exists" = false ]; then
if [ "$local_exist" = true ]; then
git checkout -B "$prowlarr_target_branch"
log "INFO" "Checked out out local branch [$prowlarr_target_branch]"
if [ "$is_dev_exec" = true ] || [ "$is_jackett_dev" = true ]; then
log "DEBUG" "[$is_dev_exec] skipping reset to [$prowlarr_remote_name/$PROWLARR_RELEASE_BRANCH]"
else
git reset --hard "$prowlarr_remote_name"/"$PROWLARR_RELEASE_BRANCH"
log "WARN" "local branch [$prowlarr_target_branch] hard reset based on remote/branch [$prowlarr_remote_name/$PROWLARR_RELEASE_BRANCH]"
fi
else
git checkout -B "$prowlarr_target_branch" "$prowlarr_remote_name"/"$PROWLARR_RELEASE_BRANCH" --no-track
log "INFO" "local branch [$prowlarr_target_branch] created from remote/branch [$prowlarr_remote_name/$PROWLARR_RELEASE_BRANCH]"
fi
else
if [ "$local_exist" = true ]; then
git checkout -B "$prowlarr_target_branch"
log "INFO" "Checked out out local branch [$prowlarr_target_branch]"
if [ "$is_dev_exec" = true ] || [ "$is_jackett_dev" = true ]; then
log "DEBUG" "Development Mode - Skipping reset to [$prowlarr_remote_name/$prowlarr_target_branch]"
else
git reset --hard "$prowlarr_remote_name"/"$prowlarr_target_branch"
log "INFO" "local [$prowlarr_target_branch] hard reset based on [$prowlarr_remote_name/$PROWLARR_RELEASE_BRANCH]"
fi
else
git checkout -B "$prowlarr_target_branch" "$prowlarr_remote_name"/"$prowlarr_target_branch"
log "INFO" "local [$prowlarr_target_branch] created from [$prowlarr_remote_name/$prowlarr_target_branch]"
fi
fi
}
pull_cherry_and_merge() {
log "INFO" "Reviewing Commits"
existing_message=$(git log --format=%B -n1)
existing_message_ln1=$(echo "$existing_message" | awk 'NR==1')
log "DEBUG" "Searching for commits with template: '$PROWLARR_COMMIT_TEMPLATE'"
log "DEBUG" "Searching in last $MAX_COMMITS_TO_SEARCH commits"
prowlarr_commits=$(git log --format=%B -n "$MAX_COMMITS_TO_SEARCH" | grep "^$PROWLARR_COMMIT_TEMPLATE")
log "DEBUG" "Found prowlarr commits count: $(echo "$prowlarr_commits" | wc -l)"
log "DEBUG" "First prowlarr commit found: $(echo "$prowlarr_commits" | head -1)"
prowlarr_jackett_commit_message=$(echo "$prowlarr_commits" | awk 'NR==1')
log "DEBUG" "Prowlarr jackett commit message: '$prowlarr_jackett_commit_message'"
if [ "$is_jackett_dev" = true ]; then
# Use only local Jackett branch (no remote)
jackett_ref="$JACKETT_REMOTE_NAME$JACKETT_BRANCH"
else
# Normal usage: remote reference
jackett_ref="$JACKETT_REMOTE_NAME/$JACKETT_BRANCH"
fi
if [ "$is_dev_exec" = true ] || [ "$is_jackett_dev" = true ]; then
log "DEBUG" "Jackett Remote is [$jackett_ref]"
# read -r -p "Pausing to review commits. Press any key to continue." -n1 -s
fi
jackett_recent_commit=$(git rev-parse "$jackett_ref")
log "DEBUG" "Jackett recent commit: '$jackett_recent_commit'"
recent_pulled_commit=$(echo "$prowlarr_commits" | awk 'NR==1{print $5}')
log "DEBUG" "Recent pulled commit (field 5): '$recent_pulled_commit'"
log "DEBUG" "Full first commit line: '$(echo "$prowlarr_commits" | awk 'NR==1')'"
if [ -z "$recent_pulled_commit" ]; then
log "ERROR" "Recent Pulled Commit is empty. Failing."
log "ERROR" "Debug info:"
log "ERROR" " - prowlarr_commits found: '$(echo "$prowlarr_commits" | head -3)'"
log "ERROR" " - Template used: '$PROWLARR_COMMIT_TEMPLATE'"
log "ERROR" " - Trying to extract field 5 from first line"
exit 3
fi
if [ "$jackett_recent_commit" = "$recent_pulled_commit" ]; then
log "SUCCESS" "--- we are current with jackett; nothing to do ---"
exit 0
fi
log "INFO" "[$jackett_recent_commit] is the most recent Jackett commit as per branch [$jackett_ref]"
log "INFO" "[$recent_pulled_commit] is the most recent Prowlarr/Indexer commit pulled from Jackett as per branch [$prowlarr_remote_name/$prowlarr_target_branch]"
# Define the command to get the commit range
commit_range_cmd="git log --reverse --pretty='%n%H' $recent_pulled_commit..$jackett_recent_commit"
# Execute the command and capture the output
commit_range=$(eval "$commit_range_cmd")
commit_count=$(git rev-list --count "$recent_pulled_commit".."$jackett_recent_commit")
log "INFO" "There are [$commit_count] commits to cherry-pick"
if [ "$is_dev_exec" = true ] || [ "$is_jackett_dev" = true ]; then
log "DEBUG" "Get Range Command is [$commit_range_cmd]"
# read -r -p "Pausing to review commits. Press any key to continue." -n1 -s
fi
# Enforce maximum commits threshold
if [ "$commit_count" -gt "$MAX_COMMITS_TO_PICK" ]; then
log "ERROR" "Commit count [$commit_count] is greater than [$MAX_COMMITS_TO_PICK]. Exiting."
exit 4
fi
log "INFO" "Commit Range is: [$commit_range]"
log "INFO" "-- Beginning Cherrypicking ---"
git config merge.directoryRenames true
git config merge.verbosity 0
sleep 2
for pick_commit in ${commit_range}; do
has_conflicts=$(git ls-files --unmerged; git status --porcelain | grep "^UU\|^AA\|^DD\|^AU\|^UA\|^DU\|^UD" || true)
if [ -n "$has_conflicts" ]; then
resolve_conflicts
fi
has_conflicts=$(git ls-files --unmerged; git status --porcelain | grep "^UU\|^AA\|^DD\|^AU\|^UA\|^DU\|^UD" || true)
if [ -n "$has_conflicts" ]; then
log "ERROR" "Conflicts Exist [$has_conflicts] - Cannot Cherrypick"
git status
if [ "$automation_mode" = true ]; then
log "ERROR" "Automation mode: Cannot continue with unresolved conflicts"
exit 5
else
read -r -p "Pausing due to conflicts. Press any key to continue when resolved." -n1 -s
log "INFO" "Continuing Cherrypicking"
fi
fi
log "INFO" "cherrypicking Jackett commit [$pick_commit]"
git cherry-pick --no-commit --rerere-autoupdate --allow-empty --keep-redundant-commits "$pick_commit"
has_conflicts=$(git ls-files --unmerged; git status --porcelain | grep "^UU\|^AA\|^DD\|^AU\|^UA\|^DU\|^UD" || true)
if [ -n "$has_conflicts" ]; then
resolve_conflicts
fi
git config merge.directoryRenames conflict
git config merge.verbosity 2
done
log "SUCCESS" "--- Completed cherry picking ---"
log "INFO" "Evaluating and Reviewing Changes"
git checkout HEAD -- "definitions/v*/schema.json"
handle_new_indexers
handle_modified_indexers
handle_backporting_indexers
}
resolve_unmerged_files() {
# Check for any remaining unmerged files and resolve them
unmerged_files=$(git diff --name-only --diff-filter=U 2>/dev/null || true)
if [ -n "$unmerged_files" ]; then
log "WARN" "Unmerged files detected: [$unmerged_files]"
echo "$unmerged_files" | while IFS= read -r file; do
if [ -n "$file" ]; then
if [[ "$file" == .github/* ]] || [[ "$file" == src/* ]] || [[ "$file" == *.md ]] || [[ "$file" == package*.json ]] || [[ "$file" == .editorconfig ]]; then
# For non-definition files, prefer our version or remove
log "DEBUG" "Resolving unmerged non-definition file: [$file]"
if git ls-files | grep -q "^$file$"; then
git checkout --ours "$file" 2>/dev/null || git rm --force "$file" 2>/dev/null || true
else
git rm --force "$file" 2>/dev/null || true
fi
git add "$file" 2>/dev/null || true
elif [[ "$file" == definitions/* ]]; then
# For definition files, prefer their version
log "DEBUG" "Resolving unmerged definition file: [$file]"
git checkout --theirs "$file" 2>/dev/null || true
git add "$file" 2>/dev/null || true
else
# Default: prefer our version
log "DEBUG" "Resolving unmerged file (default): [$file]"
git checkout --ours "$file" 2>/dev/null || git rm --force "$file" 2>/dev/null || true
git add "$file" 2>/dev/null || true
fi
fi
done
fi
}
resolve_conflicts() {
readme_conflicts=$($GIT_DIFF_CMD | grep -E '^README\.md$')
nonyml_conflicts=$($GIT_DIFF_CMD | grep -E "$CONFLICTS_NONYML_EXTENSIONS")
yml_conflicts=$($GIT_DIFF_CMD | grep -E '\.ya?ml$')
schema_conflicts=$($GIT_DIFF_CMD | grep -E '\.schema\.json$')
log "WARN" "conflicts exist"
if [ -n "$readme_conflicts" ]; then
log "DEBUG" "README conflict exists; using Prowlarr README"
git checkout --ours README.md
git add --force README.md
fi
if [ -n "$schema_conflicts" ]; then
log "DEBUG" "Schema conflict exists; using Prowlarr schema"
for file in $schema_conflicts; do
if git ls-files | grep -q "^$file$"; then
git checkout --ours "$file"
git add --force "$file"
fi
done
fi
if [ -n "$nonyml_conflicts" ]; then
log "DEBUG" "Non-YML conflicts exist; removing [\n$nonyml_conflicts\n] files and restoring [package.json package-lock.json .editorconfig]"
while IFS= read -r file; do
git rm --force --quiet --ignore-unmatch "$file"
done <<<"$nonyml_conflicts"
for file in package.json package-lock.json .editorconfig; do
if git ls-files | grep -q "^$file$"; then
git checkout --ours "$file"
git add --force "$file"
fi
done
fi
if [ -n "$yml_conflicts" ]; then
log "DEBUG" "YML conflict exists; [$yml_conflicts]"
handle_yml_conflicts
fi
# Final check and resolution of any remaining unmerged files
resolve_unmerged_files
}
handle_yml_conflicts() {
# Handle non-definition YAML conflicts first
yml_remove=$(git status --porcelain | grep yml | grep -vi "definitions/" | grep -v "definitions-update" | awk -F '[ADUMRC]{1,2} ' '{print $2}' | awk '{ gsub(/^[ \t]+|[ \t]+$/, ""); print }')
if [ -n "$yml_remove" ]; then
log "DEBUG" "Removing non-definition yml files: [$yml_remove]"
echo "$yml_remove" | while IFS= read -r file; do
if [ -n "$file" ]; then
log "DEBUG" "Removing non-definition yml file: [$file]"
git rm --force --ignore-unmatch "$file" 2>/dev/null || true
fi
done
fi
# Re-check for remaining conflicts after cleanup
yml_conflicts=$($GIT_DIFF_CMD | grep "\.yml" || true)
if [ -n "$yml_conflicts" ]; then
yml_defs=$(git status --porcelain | grep yml | grep -i "definitions/")
yml_add=$(echo "$yml_defs" | grep -v "UD\|D|DU" | awk -F '[ADUMRC]{1,2} ' '{print $2}' | awk '{ gsub(/^[ \t]+|[ \t]+$/, ""); print }')
yml_delete=$(echo "$yml_defs" | grep "UD" | awk -F '[ADUMRC]{1,2} ' '{print $2}' | awk '{ gsub(/^[ \t]+|[ \t]+$/, ""); print }')
log "DEBUG" "YML Definitions Process: [$yml_defs]"
for def in $yml_add; do
log "DEBUG" "Using & Adding Jackett's definition yml; [$def]"
# 1) Create a new path by replacing "src/Jackett.Common/Definitions/"
# with "definitions/$MIN_SCHEMA/".
# - In Bash parameter expansion, the syntax is:
# ${variable/search/replace}
new_def="${def/src\/Jackett\.Common\/Definitions\//definitions/v$MIN_SCHEMA/}"
# 2) If the path has changed, we do a rename; otherwise, just do normal checkout
if [ "$new_def" != "$def" ]; then
# Make sure the target directory exists
mkdir -p "$(dirname "$new_def")"
# Use git mv so that Git tracks the file rename
log "DEBUG" "Moving definition to new path; [$def] to [$new_def]"
mv "$def" "$new_def"
# Then checkout "theirs" to accept Jackett’s content
git checkout --theirs "$new_def"
# Stage the new path
git add --force "$new_def"
git rm --f --ignore-unmatch "$def"
else
# Fallback if the path didn't actually change
git checkout --theirs "$def"
git add --force "$def"
fi
done
for def in $yml_delete; do
log "DEBUG" "Removing definitions Jackett deleted; [$def]"
git rm --f --ignore-unmatch "$def"
done
fi
}
handle_new_indexers() {
added_indexers=$(git diff --cached --diff-filter=A --name-only | grep ".yml" | grep -E "v[[:digit:]]+")
if [ -n "$added_indexers" ]; then
log "INFO" "New Indexers detected"
for indexer in ${added_indexers}; do
base_indexer=$(basename "$indexer")
log "DEBUG" "Evaluating [$indexer] against BLOCKLIST with name [$base_indexer]"
# Check if the indexer is in the BLOCKLIST
if [[ -n "${blocklist_map[$base_indexer]}" ]]; then
log "INFO" "[$base_indexer] is in the BLOCKLIST. Removing [${indexer}]..."
git rm --f --ignore-unmatch "$indexer"
continue
fi
log "DEBUG" "Evaluating [$indexer] Cardigann Version"
if [ -f "$indexer" ]; then
determine_schema_version "$indexer"
log "DEBUG" "Checked Version Output is $check_version"
if [ "$check_version" != "v0" ]; then
log "DEBUG" "Schema Test passed."
updated_indexer=$indexer
else
determine_best_schema_version "$indexer"
if [ "$matched_version" -eq 0 ]; then
log "WARN" "Version [$NEW_SCHEMA] required. Review definition [$indexer]"
v_matched="v$NEW_SCHEMA"
else
v_matched="v$matched_version"
fi
updated_indexer=${indexer/v[0-9]*/$v_matched}
if [ "$indexer" != "$updated_indexer" ]; then
log "INFO" "Moving indexer old [$indexer] to new [$updated_indexer]"
mv "$indexer" "$updated_indexer"
git rm -f "$indexer"
git add -f "$updated_indexer"
else
log "DEBUG" "Doing nothing; [$indexer] already is [$updated_indexer]"
fi
fi
fi
done
unset indexer
unset test
log "INFO" "completed new indexers"
else
log "INFO" "No new indexers"
fi
}
handle_modified_indexers() {
modified_indexers=$(git diff --cached --diff-filter=M --name-only | grep ".yml" | grep -E "v[[:digit:]]+")
if [ -n "$modified_indexers" ]; then
log "INFO" "Reviewing Modified Indexers..."
for indexer in ${modified_indexers}; do
log "INFO" "Evaluating [$indexer] Cardigann Version"
if [ -f "$indexer" ]; then
determine_schema_version "$indexer"
log "INFO" "Checked Version Output is $check_version"
if [ "$check_version" != "v0" ]; then
log "DEBUG" "Schema Test passed."
updated_indexer=$indexer
else
determine_best_schema_version "$indexer"
if [ "$matched_version" -eq 0 ]; then
log "WARN" "Version [$NEW_SCHEMA] required. Review definition [$indexer]"
v_matched="v$NEW_SCHEMA"
else
v_matched="v$matched_version"
fi
updated_indexer=${indexer/v[0-9]*/$v_matched}
if [ "$indexer" != "$updated_indexer" ]; then
log "INFO" "Version bumped indexer old [$indexer] to new [$updated_indexer]"
mv "$indexer" "$updated_indexer"
git checkout HEAD -- "$indexer"
git add -f "$updated_indexer"
else
log "INFO" "Doing nothing; [$indexer] already is [$updated_indexer]"
fi
fi
fi
done
unset indexer
unset test
fi
log "SUCCESS" "--- completed changed indexers ---"
}
handle_backporting_indexers() {
modified_indexers_vcheck=$(git diff --cached --diff-filter=AM --name-only | grep ".yml" | grep -E "v[[:digit:]]+")
if [ -n "$modified_indexers_vcheck" ]; then
for indexer in ${modified_indexers_vcheck}; do
# SC2004: $/${} is unnecessary on arithmetic variables.
for ((i = MAX_SCHEMA; i >= MIN_SCHEMA; i--)); do
version="v$i"
log "DEBUG" "looking for [$version] indexer of [$indexer]"
indexer_check=$(echo "$indexer" | sed -E "s/v[0-9]+/$version/")
if [ "$indexer_check" != "$indexer" ] && [ -f "$indexer_check" ]; then
if [ "$SKIP_BACKPORT" = true ]; then
log "INFO" "Found [v$i] indexer for [$indexer] - Skipping backporting changes"
log "DEBUG" "Skipping backporting changes. Skipping further backport checks."
BACKPORT_SKIPPED=true # Sets the wider variable
return 0 # Exits the function early
else
log "INFO" "Found [v$i] indexer for [$indexer] - comparing to [$indexer_check]"
log "WARN" "HUMAN! Review this change and ensure no incompatible updates are backported."
git difftool --no-index "$indexer" "$indexer_check"
git add "$indexer_check"
fi
fi
done
done
unset indexer
unset indexer_check
fi
newschema_indexers=$(git diff --cached --diff-filter=A --name-only | grep ".yml" | grep "v$NEW_SCHEMA")
if [ -n "$newschema_indexers" ]; then
for indexer in ${newschema_indexers}; do
# SC2004: $/${} is unnecessary on arithmetic variables.
for ((i = MAX_SCHEMA; i >= MIN_SCHEMA; i--)); do
version="v$i"
log "DEBUG" "looking for [$version] indexer of [$indexer]"
indexer_check=$(echo "$indexer" | sed -E "s/v[0-9]+/$version/")
if [ "$indexer_check" != "$indexer" ] && [ -f "$indexer_check" ]; then
log "INFO" "Found [v$i] indexer for [$indexer] - comparing to [$indexer_check]"
log "ERROR" "THIS IS A NEW CARDIGANN VERSION THAT IS REQUIRED"
log "WARN" "HUMAN! Review this change and ensure no incompatible updates are backported."
git difftool --no-index "$indexer" "$indexer_check"
git add "$indexer_check"
fi
done
done
unset indexer
unset indexer_check
fi
log "SUCCESS" "--- completed backporting indexers ---"
}
cleanup_and_commit() {
if [ -n "$removed_indexers" ]; then
for indexer in ${removed_indexers}; do
log "DEBUG" "looking for previous versions of removed indexer [$indexer]"
# SC2004: $/${} is unnecessary on arithmetic variables.
for ((i = MAX_SCHEMA; i >= MIN_SCHEMA; i--)); do
indexer_remove=$(echo "$indexer" | sed -E "s/v[0-9]+/$version/")
if [ "$indexer_remove" != "$indexer" ] && [ -f "$indexer_remove" ]; then
log "INFO" "Found [v$i] indexer for [$indexer] - removing [$indexer_remove]"
rm -f "$indexer_remove"
git rm --f --ignore-unmatch "$indexer_remove"
fi
done
done
unset indexer
unset indexer_remove
fi
# Recalculated Added / Modified / Removed
added_indexers=$(git diff --cached --diff-filter=A --name-only | grep ".yml" | grep -E "v[[:digit:]]+")
modified_indexers=$(git diff --cached --diff-filter=M --name-only | grep ".yml" | grep -E "v[[:digit:]]+")
removed_indexers=$(git diff --cached --diff-filter=D --name-only | grep ".yml" | grep -E "v[[:digit:]]+")
newschema_indexers=$(git diff --cached --diff-filter=A --name-only | grep ".yml" | grep -E "v$NEW_SCHEMA")
# Check if there are added indexers and log if present
if [ -n "$added_indexers" ]; then
log "SUCCESS" "Added Indexers are [$added_indexers]"
fi
# Check if there are modified indexers and log if present
if [ -n "$modified_indexers" ]; then
log "SUCCESS" "Modified Indexers are [$modified_indexers]"
fi
# Check if there are removed indexers and log if present
if [ -n "$removed_indexers" ]; then
log "SUCCESS" "Removed Indexers are [$removed_indexers]"
fi
# Check if there are new schema indexers and log if present
if [ -n "$newschema_indexers" ]; then
log "WARN" "New Schema Indexers are [$newschema_indexers]"
fi
if [ -d "$NEW_VERS_DIR" ]; then
if [ "$(ls -A $NEW_VERS_DIR)" ]; then
log "ERROR" "THIS IS A NEW CARDIGANN VERSION THAT IS REQUIRED: Version [v$NEW_SCHEMA] is needed."
log "WARNING" "Review the following definitions for new Cardigann Version: $newschema_indexers"
else
rmdir "$NEW_VERS_DIR"
fi
fi
git rm -r -f -q --ignore-unmatch --cached node_modules
if [ "$automation_mode" = true ]; then
log "INFO" "Automation mode: Proceeding with commit and push automatically"
else
log "WARNING" "After review; the script will commit the changes and push as/if specified."
read -r -p "Press any key to continue or [Ctrl-C] to abort. Waiting for human review..." -n1 -s
fi
new_commit_msg="$PROWLARR_COMMIT_TEMPLATE $jackett_recent_commit [$(date -u +'%Y-%m-%dT%H:%M:%SZ')]"
if [ "$BACKPORT_SKIPPED" = true ]; then
new_commit_msg+=" $PROWLARR_COMMIT_TEMPLATE_APPEND"
fi
# Append to the commit the list of all added, removed, and modified indexers
if [ -n "$added_indexers" ]; then
new_commit_msg+=$'\n\n'"Added Indexers: $added_indexers"
fi
if [ -n "$removed_indexers" ]; then
new_commit_msg+=$'\n\n'"Removed Indexers: $removed_indexers"
fi
if [ -n "$modified_indexers" ]; then
new_commit_msg+=$'\n\n'"Modified Indexers: $modified_indexers"
fi
if [ -n "$newschema_indexers" ]; then
new_commit_msg+=$'\n\n'"New Schema Indexers: $newschema_indexers"
fi
if [ "$pulls_exists" = true ] && [ "$prowlarr_target_branch" != "$PROWLARR_RELEASE_BRANCH" ]; then
if [ "$existing_message_ln1" = "$prowlarr_jackett_commit_message" ]; then
git commit --amend -m "$new_commit_msg" -m "$existing_message"
log "INFO" "Commit Appended - [$new_commit_msg]"
else
git commit -m "$new_commit_msg"
log "INFO" "New Commit made - [$new_commit_msg]"
fi
else
git commit -m "$new_commit_msg"
log "INFO" "New Commit made - [$new_commit_msg]"
fi
}
push_changes() {
push_branch="$prowlarr_target_branch"
log "INFO" "Evaluating for Push to Remote"
log "DEBUG" " Push Modes for Branch: $push_branch"
log "DEBUG" "Push To Remote: $push_mode with Force Push With Lease: $push_mode_force"
if [ "$push_mode" = true ] && [ "$push_mode_force" = true ]; then
git push "$prowlarr_push_remote" "$push_branch" --force-if-includes --force-with-lease
log "WARN" "[$prowlarr_push_remote $push_branch] Branch Force Pushed"
elif [ "$push_mode" = true ]; then
git push "$prowlarr_push_remote" "$push_branch" --force-if-includes
log "SUCCESS" "[$prowlarr_push_remote $push_branch] Branch Pushed"
else
log "SUCCESS" "Skipping Push to [$prowlarr_push_remote/$push_branch] you should consider pushing manually and/or submitting a pull-request."
fi
# Output pull request URL if push was successful
if [ "$push_mode" = true ]; then
fork_url=$(git remote get-url "$prowlarr_push_remote" 2>/dev/null | sed 's/\.git$//' | sed 's/git@github\.com:/https:\/\/github.com\//')
if [[ "$fork_url" == *"github.com"* ]]; then
fork_owner=$(echo "$fork_url" | sed 's/.*github\.com[\/:]*//' | cut -d'/' -f1)
# Only show PR URL if not pushing to Prowlarr repo itself
if [ "$fork_owner" != "Prowlarr" ]; then
pr_url="https://github.com/Prowlarr/Indexers/compare/master...$fork_owner:$push_branch"
log "SUCCESS" "Create pull request: $pr_url"
fi
fi
fi
}
main() {
initialize_script
configure_git
check_branches
git_branch_reset
pull_cherry_and_merge
cleanup_and_commit
push_changes
}
main "$@"
View Git Blame Copy Permalink