diff --git a/extensions/markdown-language-features/src/languageFeatures/references.ts b/extensions/markdown-language-features/src/languageFeatures/references.ts
index 2cc23912445..158da7204a7 100644
--- a/extensions/markdown-language-features/src/languageFeatures/references.ts
+++ b/extensions/markdown-language-features/src/languageFeatures/references.ts
@@ -136,11 +136,11 @@ export class MdReferencesProvider extends Disposable implements vscode.Reference
 				if (link.ref.range.contains(position)) {
 					return Array.from(this.getReferencesToLinkReference(docLinks, link.ref.text, { resource: document.uri, range: link.ref.range }));
 				} else if (link.source.hrefRange.contains(position)) {
-					return this.getReferencesToLink(link, token);
+					return this.getReferencesToLink(link, position, token);
 				}
 			} else {
 				if (link.source.hrefRange.contains(position)) {
-					return this.getReferencesToLink(link, token);
+					return this.getReferencesToLink(link, position, token);
 				}
 			}
 		}
@@ -148,7 +148,7 @@ export class MdReferencesProvider extends Disposable implements vscode.Reference
 		return [];
 	}
 
-	private async getReferencesToLink(sourceLink: MdLink, token: vscode.CancellationToken): Promise<MdReference[]> {
+	private async getReferencesToLink(sourceLink: MdLink, triggerPosition: vscode.Position, token: vscode.CancellationToken): Promise<MdReference[]> {
 		const allLinksInWorkspace = (await this._linkCache.getAll()).flat();
 		if (token.isCancellationRequested) {
 			return [];
@@ -191,7 +191,7 @@ export class MdReferencesProvider extends Disposable implements vscode.Reference
 
 		const references: MdReference[] = [];
 
-		if (sourceLink.href.fragment) {
+		if (sourceLink.href.fragment && sourceLink.source.fragmentRange?.contains(triggerPosition)) {
 			const toc = await TableOfContents.create(this.engine, targetDoc);
 			const entry = toc.lookup(sourceLink.href.fragment);
 			if (entry) {
@@ -250,12 +250,13 @@ export class MdReferencesProvider extends Disposable implements vscode.Reference
 			}
 
 			const isTriggerLocation = !!sourceLink && sourceLink.source.resource.fsPath === link.source.resource.fsPath && sourceLink.source.hrefRange.isEqual(link.source.hrefRange);
+			const pathRange = this.getPathRange(link);
 			yield {
 				kind: 'link',
 				isTriggerLocation,
 				isDefinition: false,
 				link,
-				location: new vscode.Location(link.source.resource, link.source.hrefRange),
+				location: new vscode.Location(link.source.resource, pathRange),
 			};
 		}
 	}
@@ -274,14 +275,25 @@ export class MdReferencesProvider extends Disposable implements vscode.Reference
 			if (ref === refToFind && link.source.resource.fsPath === from.resource.fsPath) {
 				const isTriggerLocation = from.resource.fsPath === link.source.resource.fsPath && (
 					(link.href.kind === 'reference' && from.range.isEqual(link.source.hrefRange)) || (link.kind === 'definition' && from.range.isEqual(link.ref.range)));
+
+				const pathRange = this.getPathRange(link);
 				yield {
 					kind: 'link',
 					isTriggerLocation,
 					isDefinition: link.kind === 'definition',
 					link,
-					location: new vscode.Location(from.resource, link.source.hrefRange),
+					location: new vscode.Location(from.resource, pathRange),
 				};
 			}
 		}
 	}
+
+	/**
+	 * Get just the range of the file path, dropping the fragment
+	 */
+	private getPathRange(link: MdLink): vscode.Range {
+		return link.source.fragmentRange
+			? link.source.hrefRange.with(undefined, link.source.fragmentRange.start.translate(0, -1))
+			: link.source.hrefRange;
+	}
 }
diff --git a/extensions/markdown-language-features/src/test/references.test.ts b/extensions/markdown-language-features/src/test/references.test.ts
index 1a1c5f444b2..45925ffddf9 100644
--- a/extensions/markdown-language-features/src/test/references.test.ts
+++ b/extensions/markdown-language-features/src/test/references.test.ts
@@ -23,7 +23,7 @@ function getReferences(doc: InMemoryDocument, pos: vscode.Position, workspaceCon
 	return provider.provideReferences(doc, pos, { includeDeclaration: true }, noopToken);
 }
 
-function assertReferencesEqual(actualRefs: readonly vscode.Location[], ...expectedRefs: { uri: vscode.Uri; line: number }[]) {
+function assertReferencesEqual(actualRefs: readonly vscode.Location[], ...expectedRefs: { uri: vscode.Uri; line: number; startCharacter?: number; endCharacter?: number }[]) {
 	assert.strictEqual(actualRefs.length, expectedRefs.length, `Reference counts should match`);
 
 	for (let i = 0; i < actualRefs.length; ++i) {
@@ -32,6 +32,12 @@ function assertReferencesEqual(actualRefs: readonly vscode.Location[], ...expect
 		assert.strictEqual(actual.uri.toString(), expected.uri.toString(), `Ref '${i}' has expected document`);
 		assert.strictEqual(actual.range.start.line, expected.line, `Ref '${i}' has expected start line`);
 		assert.strictEqual(actual.range.end.line, expected.line, `Ref '${i}' has expected end line`);
+		if (typeof expected.startCharacter !== 'undefined') {
+			assert.strictEqual(actual.range.start.character, expected.startCharacter, `Ref '${i}' has expected start character`);
+		}
+		if (typeof expected.endCharacter !== 'undefined') {
+			assert.strictEqual(actual.range.end.character, expected.endCharacter, `Ref '${i}' has expected end character`);
+		}
 	}
 }
 
@@ -265,7 +271,7 @@ suite('markdown: find all references', () => {
 			`[without ext](./sub/other.md#header)`,
 		));
 
-		const refs = await getReferences(doc, new vscode.Position(0, 15), new InMemoryWorkspaceMarkdownDocuments([
+		const refs = await getReferences(doc, new vscode.Position(0, 23), new InMemoryWorkspaceMarkdownDocuments([
 			doc,
 			new InMemoryDocument(other1Uri, joinLines(
 				`pre`,
@@ -415,6 +421,51 @@ suite('markdown: find all references', () => {
 		);
 	});
 
+	test('Should distinguish between references to file and to header within file', async () => {
+		const docUri = workspacePath('doc.md');
+		const other1Uri = workspacePath('sub', 'other.md');
+
+		const doc = new InMemoryDocument(docUri, joinLines(
+			`# abc`,
+			``,
+			`[link 1](#abc)`,
+		));
+		const otherDoc = new InMemoryDocument(other1Uri, joinLines(
+			`[link](/doc.md#abc)`,
+			`[link no text](/doc#abc)`,
+		));
+		const workspaceContents = new InMemoryWorkspaceMarkdownDocuments([
+			doc,
+			otherDoc,
+		]);
+		{
+			// Check refs to header fragment
+			const headerRefs = await getReferences(otherDoc, new vscode.Position(0, 16), workspaceContents);
+			assertReferencesEqual(headerRefs!,
+				{ uri: docUri, line: 0 }, // Header definition
+				{ uri: docUri, line: 2 },
+				{ uri: other1Uri, line: 0 },
+				{ uri: other1Uri, line: 1 },
+			);
+		}
+		{
+			// Check refs to file itself from link with ext
+			const fileRefs = await getReferences(otherDoc, new vscode.Position(0, 9), workspaceContents);
+			assertReferencesEqual(fileRefs!,
+				{ uri: other1Uri, line: 0, endCharacter: 14 },
+				{ uri: other1Uri, line: 1, endCharacter: 19 },
+			);
+		}
+		{
+			// Check refs to file itself from link without ext
+			const fileRefs = await getReferences(otherDoc, new vscode.Position(1, 17), workspaceContents);
+			assertReferencesEqual(fileRefs!,
+				{ uri: other1Uri, line: 0 },
+				{ uri: other1Uri, line: 1 },
+			);
+		}
+	});
+
 	suite('Reference links', () => {
 		test('Should find reference links within file from link', async () => {
 			const docUri = workspacePath('doc.md');