TrueNAS/scale-build
1
0
Fork 0
mirror of https://github.com/truenas/scale-build.git synced 2025-12-19 18:39:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

TEN-2441 Fix for sorting and updated for halfmoon (#891)

Browse Source 
* Correction to Publish-ISO pipeline file creation

* Temporary change to avoid repush

* Correction to sort

* Correction to sort

* Adding logging to check which builds are being added

* Adding debug to investigate why file is not being updated

* Removing the skip parameter

* New pipeline to push to dev system and fix order of json

* Added manual markdown addition

* Added manual markdown addition

* Added manual markdown addition

* Added manual markdown addition

* Added manual markdown addition

* Fixed undefined variable

* Update Publish-ISO
This commit is contained in:
alexandrabain
2025-08-04 10:04:15 -04:00
committed by GitHub
parent b76b4485ba
commit 4ba7936b79
2 changed files with 251 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
jenkins/Publish-ISO
View File

@@ -10,20 +10,20 @@ pipeline {
echo '*** Grabbing artifacts from Build - TrueNAS SCALE (Full - Nightly ISO) ***'
copyArtifacts filter: '**/*.iso', fingerprintArtifacts: true, flatten: true, projectName: 'Build - TrueNAS SCALE (Full - Nightly ISO)', selector: lastSuccessful(), target: 'upload/files'
copyArtifacts filter: '**/*.sha256', fingerprintArtifacts: true, flatten: true, projectName: 'Build - TrueNAS SCALE (Full - Nightly ISO)', selector: lastSuccessful(), target: 'upload/files'
sh 'ssh jenkins@staging.sys.ixsystems.net mkdir -p /zdata/download.sys.truenas.net/truenas-scale-goldeye-nightly/ || true'
sh 'scp upload/files/TrueNAS-SCALE*.iso upload/files/TrueNAS-SCALE*.iso.sha256 jenkins@staging.sys.ixsystems.net:/zdata/download.sys.truenas.net/truenas-scale-goldeye-nightly/'
sh 'ssh jenkins@staging.sys.ixsystems.net mkdir -p /zdata/download.sys.truenas.net/truenas-scale-halfmoon-nightly/ || true'
sh 'scp upload/files/TrueNAS-SCALE*.iso upload/files/TrueNAS-SCALE*.iso.sha256 jenkins@staging.sys.ixsystems.net:/zdata/download.sys.truenas.net/truenas-scale-halfmoon-nightly/'
sh 'rm -rf upload/files'
copyArtifacts filter: '**/*.update', fingerprintArtifacts: true, flatten: true, projectName: 'Build - TrueNAS SCALE (Full - Nightly ISO)', selector: lastSuccessful(), target: 'upload/files'
copyArtifacts filter: '**/*.json', fingerprintArtifacts: true, flatten: true, projectName: 'Build - TrueNAS SCALE (Full - Nightly ISO)', selector: lastSuccessful(), target: 'upload/files'
sh 'ssh jenkins@staging.sys.ixsystems.net mkdir -p /zdata/update.sys.truenas.net/scale/TrueNAS-SCALE-Goldeye-Nightlies || true'
sh 'scp upload/files/manifest.json upload/files/TrueNAS-SCALE-*.update jenkins@staging.sys.ixsystems.net:/zdata/update.sys.truenas.net/scale/TrueNAS-SCALE-Goldeye-Nightlies/'
sh 'ssh jenkins@staging.sys.ixsystems.net mkdir -p /zdata/update.sys.truenas.net/scale/TrueNAS-SCALE-Halfmoon-Nightlies || true'
sh 'scp upload/files/manifest.json upload/files/TrueNAS-SCALE-*.update jenkins@staging.sys.ixsystems.net:/zdata/update.sys.truenas.net/scale/TrueNAS-SCALE-Halfmoon-Nightlies/'
}
}
stage('Update Releases JSON') {
steps {
script {
// Download existing releases.json if it exists
sh '''scp jenkins@staging.sys.ixsystems.net:/zdata/update.sys.truenas.net/scale/TrueNAS-SCALE-Goldeye-Nightlies/releases.json upload/files/releases.json || echo "{}" > upload/files/releases.json'''
sh '''scp jenkins@staging.sys.ixsystems.net:/zdata/update.sys.truenas.net/scale/TrueNAS-SCALE-Halfmoon-Nightlies/releases.json upload/files/releases.json || echo "{}" > upload/files/releases.json'''
// Read manifest.json and extract information
def manifestContent = readFile('upload/files/manifest.json')
@@ -56,7 +56,8 @@ pipeline {
for (int i = 0; i < entriesList.size() - 1; i++) {
for (int j = 0; j < entriesList.size() - i - 1; j++) {
// Compare dates as strings (ISO 8601 format is lexicographically sortable)
if (entriesList[j].dateStr < entriesList[j + 1].dateStr) {
// Sort oldest to newest (ascending order)
if (entriesList[j].dateStr > entriesList[j + 1].dateStr) {
// Swap entries
def temp = entriesList[j]
entriesList[j] = entriesList[j + 1]
@@ -65,17 +66,17 @@ pipeline {
}
}
// Keep only the first 30 entries
def maxEntries = Math.min(30, entriesList.size())
def recentEntries = entriesList[0..(maxEntries - 1)]
// Keep only the last 30 entries (most recent)
def startIndex = Math.max(0, entriesList.size() - 30)
def recentEntries = entriesList[startIndex..(entriesList.size() - 1)]
echo "Total builds before cleanup: ${entriesList.size()}"
echo "Keeping the ${maxEntries} most recent builds"
echo "Keeping the 30 most recent builds"
// Log which builds are being removed
if (entriesList.size() > 30) {
echo "Removing ${entriesList.size() - 30} old builds:"
for (int i = 30; i < entriesList.size(); i++) {
for (int i = 0; i < startIndex; i++) {
echo " - Removing: ${entriesList[i].version} (date: ${entriesList[i].dateStr})"
}
}
@@ -93,10 +94,10 @@ pipeline {
existingReleases.each { version, data ->
sortedVersions.add([version: version, date: data.date])
}
// Sort for display
// Sort for display (oldest to newest)
for (int i = 0; i < sortedVersions.size() - 1; i++) {
for (int j = 0; j < sortedVersions.size() - i - 1; j++) {
if (sortedVersions[j].date < sortedVersions[j + 1].date) {
if (sortedVersions[j].date > sortedVersions[j + 1].date) {
def temp = sortedVersions[j]
sortedVersions[j] = sortedVersions[j + 1]
sortedVersions[j + 1] = temp
@@ -107,8 +108,31 @@ pipeline {
echo " - ${build.version} (date: ${build.date})"
}
// Write updated releases.json
writeJSON file: 'upload/files/releases.json', json: existingReleases, pretty: 4
// Create sorted list of releases for ordered JSON output
def orderedReleases = []
existingReleases.each { version, data ->
orderedReleases.add([version: version, data: data, dateStr: data.date])
}
// Sort releases from oldest to newest
for (int i = 0; i < orderedReleases.size() - 1; i++) {
for (int j = 0; j < orderedReleases.size() - i - 1; j++) {
if (orderedReleases[j].dateStr > orderedReleases[j + 1].dateStr) {
def temp = orderedReleases[j]
orderedReleases[j] = orderedReleases[j + 1]
orderedReleases[j + 1] = temp
}
}
}
// Build ordered JSON manually to preserve order
def orderedJson = [:]
orderedReleases.each { entry ->
orderedJson[entry.version] = entry.data
}
// Write updated releases.json with preserved order
writeJSON file: 'upload/files/releases.json', json: orderedJson, pretty: 4
// Verify the file was written
sh 'ls -la upload/files/releases.json'
@@ -121,11 +145,12 @@ pipeline {
// Upload updated releases.json
echo "Uploading releases.json to staging server..."
sh '''scp upload/files/releases.json jenkins@staging.sys.ixsystems.net:/zdata/update.sys.truenas.net/scale/TrueNAS-SCALE-Goldeye-Nightlies/'''
sh '''scp upload/files/releases.json jenkins@staging.sys.ixsystems.net:/zdata/update.sys.truenas.net/scale/TrueNAS-SCALE-Halfmoon-Nightlies/'''
// Verify upload succeeded
echo "Verifying remote file..."
sh '''ssh jenkins@staging.sys.ixsystems.net "ls -la /zdata/update.sys.truenas.net/scale/TrueNAS-SCALE-Goldeye-Nightlies/releases.json"'''
sh '''ssh jenkins@staging.sys.ixsystems.net "ls -la /zdata/update.sys.truenas.net/scale/TrueNAS-SCALE-Halfmoon-Nightlies/releases.json"'''
sh 'rm -rf upload/files'
}

209
jenkins/Publish-Test Normal file
View File

@@ -0,0 +1,209 @@
pipeline {
agent {
node {
label 'SCALE-Build'
}
}
parameters {
text(name: 'RELEASE_NOTES_CONTENT', defaultValue: '', description: 'Release notes content in Markdown format (optional). If provided, this will be uploaded as a .release-notes.txt file.')
}
stages {
stage('Upload') {
steps {
echo '*** Grabbing artifacts from Build - TrueNAS SCALE (Full - Nightly ISO) ***'
copyArtifacts filter: '**/*.update', fingerprintArtifacts: true, flatten: true, projectName: 'Build - TrueNAS SCALE (Full - Nightly ISO)', selector: lastSuccessful(), target: 'upload/files'
copyArtifacts filter: '**/*.json', fingerprintArtifacts: true, flatten: true, projectName: 'Build - TrueNAS SCALE (Full - Nightly ISO)', selector: lastSuccessful(), target: 'upload/files'
copyArtifacts filter: '**/*.md', fingerprintArtifacts: true, flatten: true, projectName: 'Build - TrueNAS SCALE (Full - Nightly ISO)', selector: lastSuccessful(), target: 'upload/files', optional: true
sh 'ssh jenkins@staging.sys.ixsystems.net mkdir -p /zdata/dev-update.sys.truenas.net/scale/TrueNAS-SCALE-Goldeye-Nightlies || true'
// Upload release notes if they exist or if provided via parameter
script {
def hasReleaseNotes = false
def releaseNotesSource = ''
// Check if release notes were provided via parameter
if (params.RELEASE_NOTES_CONTENT?.trim()) {
// Write parameter content to file
writeFile file: 'upload/files/release-notes.md', text: params.RELEASE_NOTES_CONTENT
hasReleaseNotes = true
releaseNotesSource = 'parameter'
echo "Using release notes from job parameter"
} else if (fileExists('upload/files/release-notes.md')) {
hasReleaseNotes = true
releaseNotesSource = 'artifact'
echo "Using release notes from build artifact"
}
if (hasReleaseNotes) {
sh '''
# Get the update file name to derive the release notes filename
UPDATE_FILE=$(ls upload/files/TrueNAS-SCALE-*.update | head -n1)
if [ -n "$UPDATE_FILE" ]; then
# Extract just the filename without path and extension
BASE_NAME=$(basename "$UPDATE_FILE" .update)
# Create release notes with .release-notes.txt extension
RELEASE_NOTES_NAME="${BASE_NAME}.release-notes.txt"
# Copy markdown file with new name
cp upload/files/release-notes.md "upload/files/${RELEASE_NOTES_NAME}"
echo "Uploading release notes as: ${RELEASE_NOTES_NAME}"
# Verify the file exists locally before upload
echo "Checking if release notes file exists locally..."
ls -la "upload/files/${RELEASE_NOTES_NAME}"
# Upload the file
scp "upload/files/${RELEASE_NOTES_NAME}" jenkins@staging.sys.ixsystems.net:/zdata/dev-update.sys.truenas.net/scale/TrueNAS-SCALE-Goldeye-Nightlies/
# Verify the upload succeeded
echo "Verifying release notes file on remote server..."
ssh jenkins@staging.sys.ixsystems.net "ls -la /zdata/dev-update.sys.truenas.net/scale/TrueNAS-SCALE-Goldeye-Nightlies/${RELEASE_NOTES_NAME}"
else
echo "ERROR: No update file found to derive release notes filename"
fi
'''
} else {
echo "No release notes found (neither from parameter nor artifact), skipping release notes upload"
}
}
sh 'scp upload/files/manifest.json upload/files/TrueNAS-SCALE-*.update jenkins@staging.sys.ixsystems.net:/zdata/dev-update.sys.truenas.net/scale/TrueNAS-SCALE-Goldeye-Nightlies/'
}
}
stage('Update Releases JSON') {
steps {
script {
// Download existing releases.json if it exists
sh '''scp jenkins@staging.sys.ixsystems.net:/zdata/dev-update.sys.truenas.net/scale/TrueNAS-SCALE-Goldeye-Nightlies/releases.json upload/files/releases.json || echo "{}" > upload/files/releases.json'''
// Read manifest.json and extract information
def manifestContent = readFile('upload/files/manifest.json')
def manifest = readJSON text: manifestContent
// Read existing releases.json
def existingReleasesContent = readFile('upload/files/releases.json')
def existingReleases = readJSON text: existingReleasesContent
// Add new release entry to existing releases dictionary
// Copy all manifest fields and add profile
def releaseEntry = [:]
manifest.each { key, value ->
releaseEntry[key] = value
}
releaseEntry.profile = 'DEVELOPER'
existingReleases[manifest.version] = releaseEntry
echo "Adding new build to releases.json: ${manifest.version} (date: ${manifest.date})"
// Keep only the last 30 entries by date
if (existingReleases.size() > 30) {
// Convert map to list of entries with dates
def entriesList = []
existingReleases.each { version, data ->
entriesList.add([version: version, data: data, dateStr: data.date])
}
// Manual bubble sort (Jenkins-safe, no closures or comparators)
for (int i = 0; i < entriesList.size() - 1; i++) {
for (int j = 0; j < entriesList.size() - i - 1; j++) {
// Compare dates as strings (ISO 8601 format is lexicographically sortable)
// Sort oldest to newest (ascending order)
if (entriesList[j].dateStr > entriesList[j + 1].dateStr) {
// Swap entries
def temp = entriesList[j]
entriesList[j] = entriesList[j + 1]
entriesList[j + 1] = temp
}
}
}
// Keep only the last 30 entries (most recent)
def startIndex = Math.max(0, entriesList.size() - 30)
def recentEntries = entriesList[startIndex..(entriesList.size() - 1)]
echo "Total builds before cleanup: ${entriesList.size()}"
echo "Keeping the ${maxEntries} most recent builds"
// Log which builds are being removed
if (entriesList.size() > 30) {
echo "Removing ${entriesList.size() - 30} old builds:"
for (int i = 0; i < startIndex; i++) {
echo " - Removing: ${entriesList[i].version} (date: ${entriesList[i].dateStr})"
}
}
// Rebuild the releases map
existingReleases = [:]
recentEntries.each { entry ->
existingReleases[entry.version] = entry.data
}
}
// Log final list of builds in releases.json
echo "\nFinal releases.json will contain ${existingReleases.size()} builds:"
def sortedVersions = []
existingReleases.each { version, data ->
sortedVersions.add([version: version, date: data.date])
}
// Sort for display (oldest to newest)
for (int i = 0; i < sortedVersions.size() - 1; i++) {
for (int j = 0; j < sortedVersions.size() - i - 1; j++) {
if (sortedVersions[j].date > sortedVersions[j + 1].date) {
def temp = sortedVersions[j]
sortedVersions[j] = sortedVersions[j + 1]
sortedVersions[j + 1] = temp
}
}
}
sortedVersions.each { build ->
echo " - ${build.version} (date: ${build.date})"
}
// Create sorted list of releases for ordered JSON output
def orderedReleases = []
existingReleases.each { version, data ->
orderedReleases.add([version: version, data: data, dateStr: data.date])
}
// Sort releases from oldest to newest
for (int i = 0; i < orderedReleases.size() - 1; i++) {
for (int j = 0; j < orderedReleases.size() - i - 1; j++) {
if (orderedReleases[j].dateStr > orderedReleases[j + 1].dateStr) {
def temp = orderedReleases[j]
orderedReleases[j] = orderedReleases[j + 1]
orderedReleases[j + 1] = temp
}
}
}
// Build ordered JSON manually to preserve order
def orderedJson = [:]
orderedReleases.each { entry ->
orderedJson[entry.version] = entry.data
}
// Write updated releases.json with preserved order
writeJSON file: 'upload/files/releases.json', json: orderedJson, pretty: 4
// Verify the file was written
sh 'ls -la upload/files/releases.json'
echo "releases.json file size: "
sh 'wc -c upload/files/releases.json'
// Show first few lines of the file for verification
echo "First 10 lines of releases.json:"
sh 'head -10 upload/files/releases.json'
// Upload updated releases.json
echo "Uploading releases.json to staging server..."
sh '''scp upload/files/releases.json jenkins@staging.sys.ixsystems.net:/zdata/dev-update.sys.truenas.net/scale/TrueNAS-SCALE-Goldeye-Nightlies/'''
// Verify upload succeeded
echo "Verifying remote file..."
sh '''ssh jenkins@staging.sys.ixsystems.net "ls -la /zdata/dev-update.sys.truenas.net/scale/TrueNAS-SCALE-Goldeye-Nightlies/releases.json"'''
sh 'rm -rf upload/files'
}
}
}
}
}