diff --git a/build/azure-pipelines/common/sanity-tests.yml b/build/azure-pipelines/common/sanity-tests.yml index ce6d95dd7e5..fdf6b2cd3dd 100644 --- a/build/azure-pipelines/common/sanity-tests.yml +++ b/build/azure-pipelines/common/sanity-tests.yml @@ -33,9 +33,9 @@ jobs: outputs: - output: pipelineArtifact targetPath: $(SCREENSHOTS_DIR) - artifactName: screenshots-${{ parameters.name }} + artifactName: screenshots-${{ parameters.name }}-$(System.JobAttempt) displayName: Publish Screenshots - condition: succeededOrFailed() + condition: and(succeededOrFailed(), eq(variables.HAS_SCREENSHOTS, 'true')) continueOnError: true sbomEnabled: false variables: @@ -171,6 +171,25 @@ jobs: condition: and(succeeded(), ne(variables.DOCKER_CACHE_HIT, 'true')) displayName: Save Docker Image + - ${{ if eq(parameters.os, 'windows') }}: + - script: | + @echo off + dir /b "$(SCREENSHOTS_DIR)" 2>nul | findstr . >nul + if %errorlevel%==0 ( + echo ##vso[task.setvariable variable=HAS_SCREENSHOTS]true + ) + exit /b 0 + displayName: Check Screenshots + condition: succeededOrFailed() + + - ${{ else }}: + - bash: | + if [ -n "$(ls -A "$(SCREENSHOTS_DIR)" 2>/dev/null)" ]; then + echo "##vso[task.setvariable variable=HAS_SCREENSHOTS]true" + fi + displayName: Check Screenshots + condition: succeededOrFailed() + - task: PublishTestResults@2 inputs: testResultsFormat: JUnit diff --git a/test/sanity/src/context.ts b/test/sanity/src/context.ts index 62eee018d06..632dae2a534 100644 --- a/test/sanity/src/context.ts +++ b/test/sanity/src/context.ts @@ -141,10 +141,19 @@ export class TestContext { public error(message: string): never { const line = `[${new Date().toISOString()}] ERROR: ${message}`; this.consoleOutputs.push(line); - console.error(line); + console.error(`##vso[task.logissue type=error]${line}`); throw new Error(message); } + /** + * Logs a warning message with a timestamp. + */ + public warn(message: string) { + const line = `[${new Date().toISOString()}] WARNING: ${message}`; + this.consoleOutputs.push(line); + console.warn(`##vso[task.logissue type=warning]${line}`); + } + /** * Creates a new temporary directory and returns its path. */ @@ -220,7 +229,7 @@ export class TestContext { fs.rmSync(dir, { recursive: true, force: true }); this.log(`Deleted temp directory: ${dir}`); } catch (error) { - this.log(`Failed to delete temp directory: ${dir}: ${error}`); + this.warn(`Failed to delete temp directory: ${dir}: ${error}`); } } this.tempDirs.clear(); @@ -229,7 +238,7 @@ export class TestContext { try { this.deleteWslDir(dir); } catch (error) { - this.log(`Failed to delete WSL temp directory: ${dir}: ${error}`); + this.warn(`Failed to delete WSL temp directory: ${dir}: ${error}`); } } this.wslTempDirs.clear(); @@ -247,7 +256,7 @@ export class TestContext { for (let attempt = 0; attempt < maxRetries; attempt++) { if (attempt > 0) { const delay = Math.pow(2, attempt - 1) * 1000; - this.log(`Retrying fetch (attempt ${attempt + 1}/${maxRetries}) after ${delay}ms`); + this.warn(`Retrying fetch (attempt ${attempt + 1}/${maxRetries}) after ${delay}ms`); await new Promise(resolve => setTimeout(resolve, delay)); } @@ -266,7 +275,7 @@ export class TestContext { return response as Response & { body: NodeJS.ReadableStream }; } catch (error) { lastError = error instanceof Error ? error : new Error(String(error)); - this.log(`Fetch attempt ${attempt + 1} failed: ${lastError.message}`); + this.warn(`Fetch attempt ${attempt + 1} failed: ${lastError.message}`); } } @@ -1091,7 +1100,7 @@ export class TestContext { await page.screenshot({ path: screenshotPath, fullPage: true }); this.log(`Screenshot saved to: ${screenshotPath}`); } catch (e) { - this.log(`Failed to capture screenshot: ${e instanceof Error ? e.message : String(e)}`); + this.warn(`Failed to capture screenshot: ${e instanceof Error ? e.message : String(e)}`); } } diff --git a/test/sanity/src/uiTest.ts b/test/sanity/src/uiTest.ts index 65f8b14a49e..96733481e96 100644 --- a/test/sanity/src/uiTest.ts +++ b/test/sanity/src/uiTest.ts @@ -138,7 +138,7 @@ export class UITest { const extensionItem = page.locator('.extension-list-item').getByText(/^GitHub Pull Requests$/); const messageContainer = page.locator('.extensions-viewlet .message-container:not(.hidden)').first(); - for (let attempt = 0; attempt < 3; attempt++) { + for (let attempt = 0; attempt < 5; attempt++) { const result = await Promise.race([ extensionItem.waitFor().then(() => 'found' as const), messageContainer.waitFor().then(() => 'message' as const), @@ -149,20 +149,33 @@ export class UITest { } const message = await messageContainer.locator('.message').innerText(); - this.context.log(`Marketplace message: ${message} (attempt ${attempt + 1}/3), clicking Refresh`); + this.context.log(`Marketplace message: ${message} (attempt ${attempt + 1}/5), clicking Refresh`); await page.getByRole('button', { name: 'Refresh' }).click(); - await messageContainer.waitFor({ state: 'hidden', timeout: 30_000 }); + await page.waitForTimeout(5_000); } await extensionItem.waitFor(); - this.context.log('Clicking Install on the first extension in the list'); - const installButton = page.locator('.extension-action:not(.disabled)', { hasText: /Install/ }).first(); - await installButton.waitFor(); - await installButton.click(); + for (let attempt = 0; attempt < 3; attempt++) { + try { + this.context.log(`Clicking Install on the first extension in the list (attempt ${attempt + 1}/3)`); + const installButton = page.locator('.extension-action:not(.disabled)', { hasText: /Install/ }).first(); + await installButton.click(); - this.context.log('Waiting for extension to be installed'); - await page.getByRole('button', { name: 'Uninstall' }).first().waitFor({ timeout: 5 * 60_000 }); + this.context.log('Waiting for extension to be installed'); + const uninstallButton = page.getByRole('button', { name: 'Uninstall' }).first(); + const installed = await uninstallButton.waitFor({ timeout: 5 * 60_000 }).then(() => true, () => false); + if (installed) { + return; + } + } catch (error) { + this.context.log(`Extension install attempt ${attempt + 1}/3 failed: ${error instanceof Error ? error.message : String(error)}`); + } + + this.context.log('Extension install may have failed, retrying'); + } + + throw new Error('Failed to install extension after 3 attempts'); } /**