diff --git a/.changeset/itchy-poets-rush.md b/.changeset/itchy-poets-rush.md
new file mode 100644
index 00000000..d41c8791
--- /dev/null
+++ b/.changeset/itchy-poets-rush.md
@@ -0,0 +1,5 @@
+---
+"eslint-plugin-import-x": patch
+---
+
+fix: should only use `context.physicalFilename` as fallback instead of main source
diff --git a/src/utils/import-type.ts b/src/utils/import-type.ts
index 3698d9f1..256158e0 100644
--- a/src/utils/import-type.ts
+++ b/src/utils/import-type.ts
@@ -283,7 +283,7 @@ function isInternalPath(
  * @param name The name of the module to check
  * @returns `true` if the name is an external looking name, otherwise `false`
  */
-function isExternalLookingName(name: string) {
+export function isExternalLookingName(name: string) {
   return isModule(name) || isScoped(name)
 }
 
diff --git a/src/utils/resolve.ts b/src/utils/resolve.ts
index ee10c7c5..87edc078 100644
--- a/src/utils/resolve.ts
+++ b/src/utils/resolve.ts
@@ -19,6 +19,7 @@ import type {
 
 import { arraify } from './arraify.js'
 import { makeContextCacheKey } from './export-map.js'
+import { isExternalLookingName } from './import-type.js'
 import {
   LEGACY_NODE_RESOLVERS,
   normalizeConfigResolvers,
@@ -304,9 +305,11 @@ function fullResolve(
       } // backward compatibility
 
     const sourceFiles =
-      context.physicalFilename === sourceFile
+      context.physicalFilename === sourceFile ||
+      // only try to fallback for external packages
+      !isExternalLookingName(modulePath)
         ? [sourceFile]
-        : [context.physicalFilename, sourceFile]
+        : [sourceFile, context.physicalFilename]
 
     for (const sourceFile of sourceFiles) {
       for (const {
@@ -355,7 +358,7 @@ function fullResolve(
         }
 
         // else, counts
-        fileExistsCache.set(cacheKey, resolved.path as string | null)
+        fileExistsCache.set(cacheKey, resolved.path)
         return resolved
       }
     }
diff --git a/test/fixtures/config-helper.js b/test/fixtures/config-helper.js
new file mode 100644
index 00000000..e69de29b
diff --git a/test/utils/resolve.spec.ts b/test/utils/resolve.spec.ts
index ae0c08ed..a033b788 100644
--- a/test/utils/resolve.spec.ts
+++ b/test/utils/resolve.spec.ts
@@ -6,17 +6,19 @@ import { setTimeout } from 'node:timers/promises'
 import { jest } from '@jest/globals'
 import type { TSESLint } from '@typescript-eslint/utils'
 
-import { testContext, testFilePath } from '../utils.js'
+import { TEST_FILENAME, testContext, testFilePath } from '../utils.js'
 
-import { importXResolverCompat } from 'eslint-plugin-import-x'
+import { cjsRequire, importXResolverCompat } from 'eslint-plugin-import-x'
 import type {
   CjsRequire,
   NewResolver,
   NormalizedCacheSettings,
+  RuleContext,
 } from 'eslint-plugin-import-x'
 import {
   CASE_SENSITIVE_FS,
   fileExistsWithCaseSync,
+  relative,
   resolve,
 } from 'eslint-plugin-import-x/utils'
 
@@ -830,5 +832,15 @@ describe('resolve', () => {
         }),
       ).toBe(testFilePath('./bar.tsx'))
     })
+
+    it('sourceFile should take higher priority than context.physicalFilename', () => {
+      const sourceFile = cjsRequire.resolve('typescript-eslint')
+      expect(
+        relative('./config-helper', sourceFile, {}, {
+          physicalFilename: TEST_FILENAME,
+          settings: {},
+        } as RuleContext),
+      ).toBe(path.resolve(sourceFile, '../config-helper.js'))
+    })
   })
 })