fix(tests): Flaky tests depending on execution order (client, stalltracking) (#3232)

Author: krystofwoldrich

.eslintrc.js

@@ -12,22 +12,15 @@ module.exports = {
   settings: {
     version: 'detect', // React version. "detect" automatically picks the version you have installed.
   },
-  ignorePatterns: [
-    'test/react-native/versions/**/*',
-    'coverage/**/*',
-    'test/typescript/**/*',
-  ],
+  ignorePatterns: ['test/react-native/versions/**/*', 'coverage/**/*', 'test/typescript/**/*'],
   overrides: [
     {
       // Typescript Files
       files: ['*.ts', '*.tsx'],
       extends: ['plugin:react/recommended'],
       plugins: ['react', 'react-native'],
       rules: {
-        '@typescript-eslint/typedef': [
-          'error',
-          { arrowParameter: false, variableDeclarationIgnoreFunction: true },
-        ],
+        '@typescript-eslint/typedef': ['error', { arrowParameter: false, variableDeclarationIgnoreFunction: true }],
       },
     },
     {
@@ -37,6 +30,7 @@ module.exports = {
         '@typescript-eslint/no-var-requires': 'off',
         '@typescript-eslint/no-empty-function': 'off',
         '@typescript-eslint/no-explicit-any': 'off',
+        '@typescript-eslint/unbound-method': 'off',
       },
     },
     {
@@ -55,7 +49,7 @@ module.exports = {
       parserOptions: {
         ecmaVersion: 2017,
       },
-    }
+    },
   ],
   rules: {
     // Bundle size isn't too much of an issue for React Native.

package.json

@@ -70,21 +70,21 @@
     "@sentry-internal/eslint-plugin-sdk": "7.61.0",
     "@sentry/typescript": "^5.20.1",
     "@sentry/wizard": "3.9.1",
-    "@types/jest": "^29.2.5",
+    "@types/jest": "^29.5.3",
     "@types/react": "^18.2.14",
-    "babel-jest": "^29.3.1",
+    "babel-jest": "^29.6.2",
     "downlevel-dts": "^0.11.0",
     "eslint": "^7.6.0",
     "eslint-plugin-react": "^7.20.6",
     "eslint-plugin-react-native": "^3.8.1",
-    "jest": "^29.3.1",
-    "jest-environment-jsdom": "^29.4.1",
+    "jest": "^29.6.2",
+    "jest-environment-jsdom": "^29.6.2",
     "prettier": "^2.0.5",
     "react": "18.2.0",
     "react-native": "0.72.3",
     "replace-in-file": "^7.0.1",
     "rimraf": "^4.1.1",
-    "ts-jest": "^29.0.5",
+    "ts-jest": "^29.1.1",
     "typescript": "4.1.3"
   },
   "rnpm": {
@@ -98,6 +98,9 @@
   "jest": {
     "collectCoverage": true,
     "preset": "react-native",
+    "setupFilesAfterEnv": [
+      "<rootDir>/test/mockConsole.ts"
+    ],
     "globals": {
       "__DEV__": true,
       "ts-jest": {
@@ -111,7 +114,8 @@
       "js"
     ],
     "testPathIgnorePatterns": [
-      "<rootDir>/test/e2e/"
+      "<rootDir>/test/e2e/",
+      "<rootDir>/test/react-native/versions"
     ],
     "testEnvironment": "node",
     "testMatch": [

test/client.test.ts

@@ -21,8 +21,6 @@ import {
   getSyncPromiseRejectOnFirstCall,
 } from './testutils';
 
-const EXAMPLE_DSN = 'https://[email protected]/148053';
-
 interface MockedReactNative {
   NativeModules: {
     RNSentry: {
@@ -74,10 +72,10 @@ jest.mock(
       alert: jest.fn(),
     },
   }),
-  /* virtual allows us to mock modules that aren't in package.json */
-  { virtual: true },
 );
 
+const EXAMPLE_DSN = 'https://[email protected]/148053';
+
 const DEFAULT_OPTIONS: ReactNativeClientOptions = {
   enableNative: true,
   enableNativeCrashHandling: true,
@@ -94,11 +92,6 @@ const DEFAULT_OPTIONS: ReactNativeClientOptions = {
   stackParser: jest.fn().mockReturnValue([]),
 };
 
-afterEach(() => {
-  jest.clearAllMocks();
-  NATIVE.enableNative = true;
-});
-
 describe('Tests ReactNativeClient', () => {
   describe('initializing the client', () => {
     test('client initializes', async () => {
@@ -110,7 +103,6 @@ describe('Tests ReactNativeClient', () => {
 
       await expect(client.eventFromMessage('test')).resolves.toBeDefined();
       // @ts-ignore: Is Mocked
-      // eslint-disable-next-line @typescript-eslint/unbound-method
       await expect(RN.LogBox.ignoreLogs).toBeCalled();
     });
 
@@ -150,9 +142,7 @@ describe('Tests ReactNativeClient', () => {
         dsn: EXAMPLE_DSN,
         transport: myCustomTransportFn,
       });
-      // eslint-disable-next-line @typescript-eslint/unbound-method
       expect(client.getTransport()?.flush).toBe(myFlush);
-      // eslint-disable-next-line @typescript-eslint/unbound-method
       expect(client.getTransport()?.send).toBe(mySend);
     });
   });
@@ -212,6 +202,7 @@ describe('Tests ReactNativeClient', () => {
 
   describe('nativeCrash', () => {
     test('calls NativeModules crash', () => {
+      NATIVE.enableNative = true;
       const RN: MockedReactNative = require('react-native');
 
       const client = new ReactNativeClient({

test/e2e/test/e2e.test.ts

@@ -96,23 +96,23 @@ beforeAll(async () => {
   }
 });
 
-afterAll(async () => {
-  await driver?.deleteSession();
-});
+describe('End to end tests for common events', () => {
+  afterAll(async () => {
+    await driver?.deleteSession();
+  });
 
-beforeEach(async () => {
-  const element = await getElement('clearEventId');
-  await element.click();
-  await waitUntilEventIdIsEmpty();
-});
+  beforeEach(async () => {
+    const element = await getElement('clearEventId');
+    await element.click();
+    await waitUntilEventIdIsEmpty();
+  });
 
-afterEach(async () => {
-  const testName = expect.getState().currentTestName;
-  const fileName = `screen-${testName}.png`.replace(/[^0-9a-zA-Z-+.]/g, '_');
-  await driver?.saveScreenshot(fileName);
-});
+  afterEach(async () => {
+    const testName = expect.getState().currentTestName;
+    const fileName = `screen-${testName}.png`.replace(/[^0-9a-zA-Z-+.]/g, '_');
+    await driver?.saveScreenshot(fileName);
+  });
 
-describe('End to end tests for common events', () => {
   test('captureMessage', async () => {
     const element = await getElement('captureMessage');
     await element.click();

test/integrations/reactnativeerrorhandlers.test.ts

@@ -49,15 +49,15 @@ import type { Event, EventHint, SeverityLevel } from '@sentry/types';
 
 import { ReactNativeErrorHandlers } from '../../src/js/integrations/reactnativeerrorhandlers';
 
-beforeEach(() => {
-  ErrorUtils.getGlobalHandler = () => jest.fn();
-});
+describe('ReactNativeErrorHandlers', () => {
+  beforeEach(() => {
+    ErrorUtils.getGlobalHandler = () => jest.fn();
+  });
 
-afterEach(() => {
-  jest.clearAllMocks();
-});
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
 
-describe('ReactNativeErrorHandlers', () => {
   describe('onError', () => {
     let errorHandlerCallback: (error: Error, isFatal: boolean) => Promise<void>;
 
@@ -107,7 +107,6 @@ describe('ReactNativeErrorHandlers', () => {
 
 function getActualCaptureEventArgs() {
   const hub = getCurrentHub();
-  // eslint-disable-next-line @typescript-eslint/unbound-method
   const mockCall = (hub.captureEvent as jest.MockedFunction<typeof hub.captureEvent>).mock.calls[0];
 
   return mockCall;

test/integrations/sdkinfo.test.ts

@@ -28,11 +28,11 @@ jest.mock('../../src/js/wrapper', () => {
   };
 });
 
-afterEach(() => {
-  NATIVE.platform = 'ios';
-});
-
 describe('Sdk Info', () => {
+  afterEach(() => {
+    NATIVE.platform = 'ios';
+  });
+
   it('Adds native package and javascript platform to event on iOS', async () => {
     mockedFetchNativeSdkInfo = jest.fn().mockResolvedValue(mockCocoaPackage);
     const mockEvent: Event = {};

test/mockConsole.ts

@@ -0,0 +1,8 @@
+global.console = {
+  ...console,
+  log: jest.fn(),
+  debug: jest.fn(),
+  info: jest.fn(),
+  warn: jest.fn(),
+  error: jest.fn(),
+};

test/scope.test.ts

@@ -1,4 +1,3 @@
-// eslint-disable @typescript-eslint/unbound-method
 import type { Breadcrumb } from '@sentry/types';
 
 import { ReactNativeScope } from '../src/js/scope';

test/sdk.test.ts

@@ -82,11 +82,11 @@ const usedOptions = (): ClientOptions<BaseTransportOptions> | undefined => {
   return mockedInitAndBind.mock.calls[0]?.[1];
 };
 
-afterEach(() => {
-  jest.clearAllMocks();
-});
-
 describe('Tests the SDK functionality', () => {
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
   describe('init', () => {
     describe('enableAutoPerformanceTracing', () => {
       const usedOptions = (): Integration[] => {
@@ -163,7 +163,6 @@ describe('Tests the SDK functionality', () => {
         if (mockClient) {
           const flushResult = await flush();
 
-          // eslint-disable-next-line @typescript-eslint/unbound-method
           expect(mockClient.flush).toBeCalled();
           expect(flushResult).toBe(true);
         }
@@ -178,10 +177,8 @@ describe('Tests the SDK functionality', () => {
 
           const flushResult = await flush();
 
-          // eslint-disable-next-line @typescript-eslint/unbound-method
           expect(mockClient.flush).toBeCalled();
           expect(flushResult).toBe(false);
-          // eslint-disable-next-line @typescript-eslint/unbound-method
           expect(logger.error).toBeCalledWith('Failed to flush the event queue.');
         }
       });
@@ -219,7 +216,6 @@ describe('Tests the SDK functionality', () => {
       it('fetchTransport set and enableNative set to false', () => {
         (NATIVE.isNativeAvailable as jest.Mock).mockImplementation(() => false);
         init({});
-        // eslint-disable-next-line @typescript-eslint/unbound-method
         expect(NATIVE.isNativeAvailable).toBeCalled();
         // @ts-ignore enableNative not publicly available here.
         expect(usedOptions()?.enableNative).toEqual(false);
@@ -229,7 +225,6 @@ describe('Tests the SDK functionality', () => {
       it('fetchTransport set and passed enableNative ignored when true', () => {
         (NATIVE.isNativeAvailable as jest.Mock).mockImplementation(() => false);
         init({ enableNative: true });
-        // eslint-disable-next-line @typescript-eslint/unbound-method
         expect(NATIVE.isNativeAvailable).toBeCalled();
         // @ts-ignore enableNative not publicly available here.
         expect(usedOptions()?.enableNative).toEqual(false);
@@ -239,7 +234,6 @@ describe('Tests the SDK functionality', () => {
       it('fetchTransport set and isNativeAvailable not called when passed enableNative set to false', () => {
         (NATIVE.isNativeAvailable as jest.Mock).mockImplementation(() => false);
         init({ enableNative: false });
-        // eslint-disable-next-line @typescript-eslint/unbound-method
         expect(NATIVE.isNativeAvailable).not.toBeCalled();
         // @ts-ignore enableNative not publicly available here.
         expect(usedOptions()?.enableNative).toEqual(false);
@@ -253,7 +247,6 @@ describe('Tests the SDK functionality', () => {
           transport: mockTransport,
         });
         expect(usedOptions()?.transport).toEqual(mockTransport);
-        // eslint-disable-next-line @typescript-eslint/unbound-method
         expect(NATIVE.isNativeAvailable).toBeCalled();
         // @ts-ignore enableNative not publicly available here.
         expect(usedOptions()?.enableNative).toEqual(false);
@@ -284,15 +277,13 @@ describe('Tests the SDK functionality', () => {
       (NATIVE.isNativeAvailable as jest.Mock).mockImplementation(() => true);
       init({ enableNative: false });
       expect(usedOptions()?.transport).toEqual(makeFetchTransport);
-      // eslint-disable-next-line @typescript-eslint/unbound-method
       expect(NATIVE.isNativeAvailable).not.toBeCalled();
     });
 
     it('check both options and native availability', () => {
       (NATIVE.isNativeAvailable as jest.Mock).mockImplementation(() => true);
       init({ enableNative: true });
       expect(usedOptions()?.transport).toEqual(makeNativeTransport);
-      // eslint-disable-next-line @typescript-eslint/unbound-method
       expect(NATIVE.isNativeAvailable).toBeCalled();
     });
   });

test/tracing/nativeframes.test.ts

@@ -22,7 +22,6 @@ describe('NativeFramesInstrumentation', () => {
       slowFrames: 20,
       frozenFrames: 5,
     };
-    // eslint-disable-next-line @typescript-eslint/unbound-method
     mockFunction(NATIVE.fetchNativeFrames).mockResolvedValue(startFrames);
 
     const instance = new NativeFramesInstrumentation(
@@ -55,7 +54,6 @@ describe('NativeFramesInstrumentation', () => {
       slowFrames: 40,
       frozenFrames: 10,
     };
-    // eslint-disable-next-line @typescript-eslint/unbound-method
     mockFunction(NATIVE.fetchNativeFrames).mockResolvedValue(startFrames);
 
     let eventProcessor: EventProcessor;
@@ -72,7 +70,6 @@ describe('NativeFramesInstrumentation', () => {
     instance.onTransactionStart(transaction);
 
     setImmediate(() => {
-      // eslint-disable-next-line @typescript-eslint/unbound-method
       mockFunction(NATIVE.fetchNativeFrames).mockResolvedValue(finishFrames);
 
       const finishTimestamp = Date.now() / 1000;
@@ -140,7 +137,6 @@ describe('NativeFramesInstrumentation', () => {
       slowFrames: 40,
       frozenFrames: 10,
     };
-    // eslint-disable-next-line @typescript-eslint/unbound-method
     mockFunction(NATIVE.fetchNativeFrames).mockResolvedValue(finishFrames);
 
     let eventProcessor: EventProcessor;
@@ -215,7 +211,6 @@ describe('NativeFramesInstrumentation', () => {
       frozenFrames: 5,
     };
     const finishFrames = null;
-    // eslint-disable-next-line @typescript-eslint/unbound-method
     mockFunction(NATIVE.fetchNativeFrames).mockResolvedValue(startFrames);
 
     let eventProcessor: EventProcessor;
@@ -232,7 +227,6 @@ describe('NativeFramesInstrumentation', () => {
     instance.onTransactionStart(transaction);
 
     setImmediate(() => {
-      // eslint-disable-next-line @typescript-eslint/unbound-method
       mockFunction(NATIVE.fetchNativeFrames).mockResolvedValue(finishFrames);
 
       const finishTimestamp = Date.now() / 1000;
@@ -286,7 +280,6 @@ describe('NativeFramesInstrumentation', () => {
       slowFrames: 20,
       frozenFrames: 5,
     };
-    // eslint-disable-next-line @typescript-eslint/unbound-method
     mockFunction(NATIVE.fetchNativeFrames).mockResolvedValue(startFrames);
 
     let eventProcessor: EventProcessor;
@@ -303,7 +296,6 @@ describe('NativeFramesInstrumentation', () => {
     instance.onTransactionStart(transaction);
 
     setImmediate(() => {
-      // eslint-disable-next-line @typescript-eslint/unbound-method
       mockFunction(NATIVE.fetchNativeFrames).mockImplementation(
         // eslint-disable-next-line @typescript-eslint/no-empty-function
         async () => new Promise(() => {}),