TEN-2441 Fix for sorting and updated for halfmoon (#891)

* Correction to Publish-ISO pipeline file creation

* Temporary change to avoid repush

* Correction to sort

* Correction to sort

* Adding logging to check which builds are being added

* Adding debug to investigate why file is not being updated

* Removing the skip parameter

* New pipeline to push to dev system and fix order of json

* Added manual markdown addition

* Added manual markdown addition

* Added manual markdown addition

* Added manual markdown addition

* Added manual markdown addition

* Fixed undefined variable

* Update Publish-ISO

jenkins/Publish-ISO

@@ -10,20 +10,20 @@ pipeline {
         echo '*** Grabbing artifacts from Build - TrueNAS SCALE (Full - Nightly ISO) ***'
         copyArtifacts filter: '**/*.iso', fingerprintArtifacts: true, flatten: true, projectName: 'Build - TrueNAS SCALE (Full - Nightly ISO)', selector: lastSuccessful(), target: 'upload/files'
         copyArtifacts filter: '**/*.sha256', fingerprintArtifacts: true, flatten: true, projectName: 'Build - TrueNAS SCALE (Full - Nightly ISO)', selector: lastSuccessful(), target: 'upload/files'
-        sh 'ssh jenkins@staging.sys.ixsystems.net mkdir -p /zdata/download.sys.truenas.net/truenas-scale-goldeye-nightly/ || true'
+        sh 'ssh jenkins@staging.sys.ixsystems.net mkdir -p /zdata/download.sys.truenas.net/truenas-scale-halfmoon-nightly/ || true'
-        sh 'scp upload/files/TrueNAS-SCALE*.iso upload/files/TrueNAS-SCALE*.iso.sha256 jenkins@staging.sys.ixsystems.net:/zdata/download.sys.truenas.net/truenas-scale-goldeye-nightly/'
+        sh 'scp upload/files/TrueNAS-SCALE*.iso upload/files/TrueNAS-SCALE*.iso.sha256 jenkins@staging.sys.ixsystems.net:/zdata/download.sys.truenas.net/truenas-scale-halfmoon-nightly/'
         sh 'rm -rf upload/files'
         copyArtifacts filter: '**/*.update', fingerprintArtifacts: true, flatten: true, projectName: 'Build - TrueNAS SCALE (Full - Nightly ISO)', selector: lastSuccessful(), target: 'upload/files'
         copyArtifacts filter: '**/*.json', fingerprintArtifacts: true, flatten: true, projectName: 'Build - TrueNAS SCALE (Full - Nightly ISO)', selector: lastSuccessful(), target: 'upload/files'
-        sh 'ssh jenkins@staging.sys.ixsystems.net mkdir -p /zdata/update.sys.truenas.net/scale/TrueNAS-SCALE-Goldeye-Nightlies || true'
+        sh 'ssh jenkins@staging.sys.ixsystems.net mkdir -p /zdata/update.sys.truenas.net/scale/TrueNAS-SCALE-Halfmoon-Nightlies || true'
-        sh 'scp upload/files/manifest.json upload/files/TrueNAS-SCALE-*.update jenkins@staging.sys.ixsystems.net:/zdata/update.sys.truenas.net/scale/TrueNAS-SCALE-Goldeye-Nightlies/'
+        sh 'scp upload/files/manifest.json upload/files/TrueNAS-SCALE-*.update jenkins@staging.sys.ixsystems.net:/zdata/update.sys.truenas.net/scale/TrueNAS-SCALE-Halfmoon-Nightlies/'
       }
     }
     stage('Update Releases JSON') {
       steps {
         script {
           // Download existing releases.json if it exists
-          sh '''scp jenkins@staging.sys.ixsystems.net:/zdata/update.sys.truenas.net/scale/TrueNAS-SCALE-Goldeye-Nightlies/releases.json upload/files/releases.json || echo "{}" > upload/files/releases.json'''
+          sh '''scp jenkins@staging.sys.ixsystems.net:/zdata/update.sys.truenas.net/scale/TrueNAS-SCALE-Halfmoon-Nightlies/releases.json upload/files/releases.json || echo "{}" > upload/files/releases.json'''
           
           // Read manifest.json and extract information
           def manifestContent = readFile('upload/files/manifest.json')
@@ -56,7 +56,8 @@ pipeline {
             for (int i = 0; i < entriesList.size() - 1; i++) {
               for (int j = 0; j < entriesList.size() - i - 1; j++) {
                 // Compare dates as strings (ISO 8601 format is lexicographically sortable)
-                if (entriesList[j].dateStr < entriesList[j + 1].dateStr) {
+                // Sort oldest to newest (ascending order)
+                if (entriesList[j].dateStr > entriesList[j + 1].dateStr) {
                   // Swap entries
                   def temp = entriesList[j]
                   entriesList[j] = entriesList[j + 1]
@@ -65,17 +66,17 @@ pipeline {
               }
             }
             
-            // Keep only the first 30 entries
+            // Keep only the last 30 entries (most recent)
-            def maxEntries = Math.min(30, entriesList.size())
+            def startIndex = Math.max(0, entriesList.size() - 30)
-            def recentEntries = entriesList[0..(maxEntries - 1)]
+            def recentEntries = entriesList[startIndex..(entriesList.size() - 1)]
             
             echo "Total builds before cleanup: ${entriesList.size()}"
-            echo "Keeping the ${maxEntries} most recent builds"
+            echo "Keeping the 30 most recent builds"
             
             // Log which builds are being removed
             if (entriesList.size() > 30) {
               echo "Removing ${entriesList.size() - 30} old builds:"
-              for (int i = 30; i < entriesList.size(); i++) {
+              for (int i = 0; i < startIndex; i++) {
                 echo "  - Removing: ${entriesList[i].version} (date: ${entriesList[i].dateStr})"
               }
             }
@@ -93,10 +94,10 @@ pipeline {
           existingReleases.each { version, data ->
             sortedVersions.add([version: version, date: data.date])
           }
-          // Sort for display
+          // Sort for display (oldest to newest)
           for (int i = 0; i < sortedVersions.size() - 1; i++) {
             for (int j = 0; j < sortedVersions.size() - i - 1; j++) {
-              if (sortedVersions[j].date < sortedVersions[j + 1].date) {
+              if (sortedVersions[j].date > sortedVersions[j + 1].date) {
                 def temp = sortedVersions[j]
                 sortedVersions[j] = sortedVersions[j + 1]
                 sortedVersions[j + 1] = temp
@@ -107,8 +108,31 @@ pipeline {
             echo "  - ${build.version} (date: ${build.date})"
           }
           
-          // Write updated releases.json
+          // Create sorted list of releases for ordered JSON output
-          writeJSON file: 'upload/files/releases.json', json: existingReleases, pretty: 4
+          def orderedReleases = []
+          existingReleases.each { version, data ->
+            orderedReleases.add([version: version, data: data, dateStr: data.date])
+          }
+          
+          // Sort releases from oldest to newest
+          for (int i = 0; i < orderedReleases.size() - 1; i++) {
+            for (int j = 0; j < orderedReleases.size() - i - 1; j++) {
+              if (orderedReleases[j].dateStr > orderedReleases[j + 1].dateStr) {
+                def temp = orderedReleases[j]
+                orderedReleases[j] = orderedReleases[j + 1]
+                orderedReleases[j + 1] = temp
+              }
+            }
+          }
+          
+          // Build ordered JSON manually to preserve order
+          def orderedJson = [:]
+          orderedReleases.each { entry ->
+            orderedJson[entry.version] = entry.data
+          }
+          
+          // Write updated releases.json with preserved order
+          writeJSON file: 'upload/files/releases.json', json: orderedJson, pretty: 4
           
           // Verify the file was written
           sh 'ls -la upload/files/releases.json'
@@ -121,11 +145,12 @@ pipeline {
           
           // Upload updated releases.json
           echo "Uploading releases.json to staging server..."
-          sh '''scp upload/files/releases.json jenkins@staging.sys.ixsystems.net:/zdata/update.sys.truenas.net/scale/TrueNAS-SCALE-Goldeye-Nightlies/'''
+
+          sh '''scp upload/files/releases.json jenkins@staging.sys.ixsystems.net:/zdata/update.sys.truenas.net/scale/TrueNAS-SCALE-Halfmoon-Nightlies/'''
           
           // Verify upload succeeded
           echo "Verifying remote file..."
-          sh '''ssh jenkins@staging.sys.ixsystems.net "ls -la /zdata/update.sys.truenas.net/scale/TrueNAS-SCALE-Goldeye-Nightlies/releases.json"'''
+          sh '''ssh jenkins@staging.sys.ixsystems.net "ls -la /zdata/update.sys.truenas.net/scale/TrueNAS-SCALE-Halfmoon-Nightlies/releases.json"'''
           
           sh 'rm -rf upload/files'
         }

jenkins/Publish-Test

@@ -0,0 +1,209 @@
+pipeline {
+  agent {
+    node {
+      label 'SCALE-Build'
+    }
+  }
+  parameters {
+    text(name: 'RELEASE_NOTES_CONTENT', defaultValue: '', description: 'Release notes content in Markdown format (optional). If provided, this will be uploaded as a .release-notes.txt file.')
+  }
+  stages {
+    stage('Upload') {
+      steps {
+        echo '*** Grabbing artifacts from Build - TrueNAS SCALE (Full - Nightly ISO) ***'
+        copyArtifacts filter: '**/*.update', fingerprintArtifacts: true, flatten: true, projectName: 'Build - TrueNAS SCALE (Full - Nightly ISO)', selector: lastSuccessful(), target: 'upload/files'
+        copyArtifacts filter: '**/*.json', fingerprintArtifacts: true, flatten: true, projectName: 'Build - TrueNAS SCALE (Full - Nightly ISO)', selector: lastSuccessful(), target: 'upload/files'
+        copyArtifacts filter: '**/*.md', fingerprintArtifacts: true, flatten: true, projectName: 'Build - TrueNAS SCALE (Full - Nightly ISO)', selector: lastSuccessful(), target: 'upload/files', optional: true
+        sh 'ssh jenkins@staging.sys.ixsystems.net mkdir -p /zdata/dev-update.sys.truenas.net/scale/TrueNAS-SCALE-Goldeye-Nightlies || true'
+        
+        // Upload release notes if they exist or if provided via parameter
+        script {
+          def hasReleaseNotes = false
+          def releaseNotesSource = ''
+          
+          // Check if release notes were provided via parameter
+          if (params.RELEASE_NOTES_CONTENT?.trim()) {
+            // Write parameter content to file
+            writeFile file: 'upload/files/release-notes.md', text: params.RELEASE_NOTES_CONTENT
+            hasReleaseNotes = true
+            releaseNotesSource = 'parameter'
+            echo "Using release notes from job parameter"
+          } else if (fileExists('upload/files/release-notes.md')) {
+            hasReleaseNotes = true
+            releaseNotesSource = 'artifact'
+            echo "Using release notes from build artifact"
+          }
+          
+          if (hasReleaseNotes) {
+            sh '''
+              # Get the update file name to derive the release notes filename
+              UPDATE_FILE=$(ls upload/files/TrueNAS-SCALE-*.update | head -n1)
+              if [ -n "$UPDATE_FILE" ]; then
+                # Extract just the filename without path and extension
+                BASE_NAME=$(basename "$UPDATE_FILE" .update)
+                # Create release notes with .release-notes.txt extension
+                RELEASE_NOTES_NAME="${BASE_NAME}.release-notes.txt"
+                # Copy markdown file with new name
+                cp upload/files/release-notes.md "upload/files/${RELEASE_NOTES_NAME}"
+                echo "Uploading release notes as: ${RELEASE_NOTES_NAME}"
+                
+                # Verify the file exists locally before upload
+                echo "Checking if release notes file exists locally..."
+                ls -la "upload/files/${RELEASE_NOTES_NAME}"
+                
+                # Upload the file
+                scp "upload/files/${RELEASE_NOTES_NAME}" jenkins@staging.sys.ixsystems.net:/zdata/dev-update.sys.truenas.net/scale/TrueNAS-SCALE-Goldeye-Nightlies/
+                
+                # Verify the upload succeeded
+                echo "Verifying release notes file on remote server..."
+                ssh jenkins@staging.sys.ixsystems.net "ls -la /zdata/dev-update.sys.truenas.net/scale/TrueNAS-SCALE-Goldeye-Nightlies/${RELEASE_NOTES_NAME}"
+              else
+                echo "ERROR: No update file found to derive release notes filename"
+              fi
+            '''
+          } else {
+            echo "No release notes found (neither from parameter nor artifact), skipping release notes upload"
+          }
+        }
+        
+        sh 'scp upload/files/manifest.json upload/files/TrueNAS-SCALE-*.update jenkins@staging.sys.ixsystems.net:/zdata/dev-update.sys.truenas.net/scale/TrueNAS-SCALE-Goldeye-Nightlies/'
+      }
+    }
+    stage('Update Releases JSON') {
+      steps {
+        script {
+          // Download existing releases.json if it exists
+          sh '''scp jenkins@staging.sys.ixsystems.net:/zdata/dev-update.sys.truenas.net/scale/TrueNAS-SCALE-Goldeye-Nightlies/releases.json upload/files/releases.json || echo "{}" > upload/files/releases.json'''
+          
+          // Read manifest.json and extract information
+          def manifestContent = readFile('upload/files/manifest.json')
+          def manifest = readJSON text: manifestContent
+          
+          // Read existing releases.json
+          def existingReleasesContent = readFile('upload/files/releases.json')
+          def existingReleases = readJSON text: existingReleasesContent
+          
+          // Add new release entry to existing releases dictionary
+          // Copy all manifest fields and add profile
+          def releaseEntry = [:]
+          manifest.each { key, value ->
+            releaseEntry[key] = value
+          }
+          releaseEntry.profile = 'DEVELOPER'
+          existingReleases[manifest.version] = releaseEntry
+          
+          echo "Adding new build to releases.json: ${manifest.version} (date: ${manifest.date})"
+          
+          // Keep only the last 30 entries by date
+          if (existingReleases.size() > 30) {
+            // Convert map to list of entries with dates
+            def entriesList = []
+            existingReleases.each { version, data ->
+              entriesList.add([version: version, data: data, dateStr: data.date])
+            }
+            
+            // Manual bubble sort (Jenkins-safe, no closures or comparators)
+            for (int i = 0; i < entriesList.size() - 1; i++) {
+              for (int j = 0; j < entriesList.size() - i - 1; j++) {
+                // Compare dates as strings (ISO 8601 format is lexicographically sortable)
+                // Sort oldest to newest (ascending order)
+                if (entriesList[j].dateStr > entriesList[j + 1].dateStr) {
+                  // Swap entries
+                  def temp = entriesList[j]
+                  entriesList[j] = entriesList[j + 1]
+                  entriesList[j + 1] = temp
+                }
+              }
+            }
+            
+            // Keep only the last 30 entries (most recent)
+            def startIndex = Math.max(0, entriesList.size() - 30)
+            def recentEntries = entriesList[startIndex..(entriesList.size() - 1)]
+            
+            echo "Total builds before cleanup: ${entriesList.size()}"
+            echo "Keeping the ${maxEntries} most recent builds"
+            
+            // Log which builds are being removed
+            if (entriesList.size() > 30) {
+              echo "Removing ${entriesList.size() - 30} old builds:"
+              for (int i = 0; i < startIndex; i++) {
+                echo "  - Removing: ${entriesList[i].version} (date: ${entriesList[i].dateStr})"
+              }
+            }
+            
+            // Rebuild the releases map
+            existingReleases = [:]
+            recentEntries.each { entry ->
+              existingReleases[entry.version] = entry.data
+            }
+          }
+          
+          // Log final list of builds in releases.json
+          echo "\nFinal releases.json will contain ${existingReleases.size()} builds:"
+          def sortedVersions = []
+          existingReleases.each { version, data ->
+            sortedVersions.add([version: version, date: data.date])
+          }
+          // Sort for display (oldest to newest)
+          for (int i = 0; i < sortedVersions.size() - 1; i++) {
+            for (int j = 0; j < sortedVersions.size() - i - 1; j++) {
+              if (sortedVersions[j].date > sortedVersions[j + 1].date) {
+                def temp = sortedVersions[j]
+                sortedVersions[j] = sortedVersions[j + 1]
+                sortedVersions[j + 1] = temp
+              }
+            }
+          }
+          sortedVersions.each { build ->
+            echo "  - ${build.version} (date: ${build.date})"
+          }
+          
+          // Create sorted list of releases for ordered JSON output
+          def orderedReleases = []
+          existingReleases.each { version, data ->
+            orderedReleases.add([version: version, data: data, dateStr: data.date])
+          }
+          
+          // Sort releases from oldest to newest
+          for (int i = 0; i < orderedReleases.size() - 1; i++) {
+            for (int j = 0; j < orderedReleases.size() - i - 1; j++) {
+              if (orderedReleases[j].dateStr > orderedReleases[j + 1].dateStr) {
+                def temp = orderedReleases[j]
+                orderedReleases[j] = orderedReleases[j + 1]
+                orderedReleases[j + 1] = temp
+              }
+            }
+          }
+          
+          // Build ordered JSON manually to preserve order
+          def orderedJson = [:]
+          orderedReleases.each { entry ->
+            orderedJson[entry.version] = entry.data
+          }
+          
+          // Write updated releases.json with preserved order
+          writeJSON file: 'upload/files/releases.json', json: orderedJson, pretty: 4
+          
+          // Verify the file was written
+          sh 'ls -la upload/files/releases.json'
+          echo "releases.json file size: "
+          sh 'wc -c upload/files/releases.json'
+          
+          // Show first few lines of the file for verification
+          echo "First 10 lines of releases.json:"
+          sh 'head -10 upload/files/releases.json'
+          
+          // Upload updated releases.json
+          echo "Uploading releases.json to staging server..."
+          sh '''scp upload/files/releases.json jenkins@staging.sys.ixsystems.net:/zdata/dev-update.sys.truenas.net/scale/TrueNAS-SCALE-Goldeye-Nightlies/'''
+          
+          // Verify upload succeeded
+          echo "Verifying remote file..."
+          sh '''ssh jenkins@staging.sys.ixsystems.net "ls -la /zdata/dev-update.sys.truenas.net/scale/TrueNAS-SCALE-Goldeye-Nightlies/releases.json"'''
+          
+          sh 'rm -rf upload/files'
+        }
+      }
+    }
+  }
+}