diff --git a/src/List.tsx b/src/List.tsx
index a7f74e9..c1b4521 100644
--- a/src/List.tsx
+++ b/src/List.tsx
@@ -15,6 +15,7 @@ import { useGetSize } from './hooks/useGetSize';
 import useHeights from './hooks/useHeights';
 import useMobileTouchMove from './hooks/useMobileTouchMove';
 import useOriginScroll from './hooks/useOriginScroll';
+import useScrollDrag from './hooks/useScrollDrag';
 import type { ScrollPos, ScrollTarget } from './hooks/useScrollTo';
 import useScrollTo from './hooks/useScrollTo';
 import type { ExtraRenderInfo, GetKey, RenderFunc, SharedConfig } from './interface';
@@ -436,6 +437,11 @@ export function RawList<T>(props: ListProps<T>, ref: React.Ref<ListRef>) {
     return false;
   });
 
+  // MouseDown drag for scroll
+  useScrollDrag(inVirtual, componentRef, (offset) => {
+    syncScrollTop((top) => top + offset);
+  });
+
   useLayoutEffect(() => {
     // Firefox only
     function onMozMousePixelScroll(e: WheelEvent) {
diff --git a/src/ScrollBar.tsx b/src/ScrollBar.tsx
index 862d00d..588fe88 100644
--- a/src/ScrollBar.tsx
+++ b/src/ScrollBar.tsx
@@ -1,6 +1,7 @@
 import classNames from 'classnames';
 import raf from 'rc-util/lib/raf';
 import * as React from 'react';
+import { getPageXY } from './hooks/useScrollDrag';
 
 export type ScrollBarDirectionType = 'ltr' | 'rtl';
 
@@ -23,14 +24,6 @@ export interface ScrollBarRef {
   delayHidden: () => void;
 }
 
-function getPageXY(
-  e: React.MouseEvent | React.TouchEvent | MouseEvent | TouchEvent,
-  horizontal: boolean,
-) {
-  const obj = 'touches' in e ? e.touches[0] : e;
-  return obj[horizontal ? 'pageX' : 'pageY'];
-}
-
 const ScrollBar = React.forwardRef<ScrollBarRef, ScrollBarProps>((props, ref) => {
   const {
     prefixCls,
diff --git a/src/hooks/useScrollDrag.ts b/src/hooks/useScrollDrag.ts
new file mode 100644
index 0000000..2503dd0
--- /dev/null
+++ b/src/hooks/useScrollDrag.ts
@@ -0,0 +1,86 @@
+import raf from 'rc-util/lib/raf';
+import * as React from 'react';
+
+function smoothScrollOffset(offset: number) {
+  return Math.floor(offset ** 0.5);
+}
+
+export function getPageXY(
+  e: React.MouseEvent | React.TouchEvent | MouseEvent | TouchEvent,
+  horizontal: boolean,
+) {
+  const obj = 'touches' in e ? e.touches[0] : e;
+  return obj[horizontal ? 'pageX' : 'pageY'];
+}
+
+export default function useScrollDrag(
+  inVirtual: boolean,
+  componentRef: React.RefObject<HTMLElement>,
+  onScrollOffset: (offset: number) => void,
+) {
+  React.useEffect(() => {
+    const ele = componentRef.current;
+    if (inVirtual && ele) {
+      let mouseDownLock = false;
+      let rafId: number;
+      let offset: number;
+
+      const stopScroll = () => {
+        raf.cancel(rafId);
+      };
+
+      const continueScroll = () => {
+        stopScroll();
+
+        rafId = raf(() => {
+          onScrollOffset(offset);
+          continueScroll();
+        });
+      };
+
+      const onMouseDown = (e: MouseEvent) => {
+        // Skip if nest List has handled this event
+        const event = e as MouseEvent & {
+          _virtualHandled?: boolean;
+        };
+        if (!event._virtualHandled) {
+          event._virtualHandled = true;
+          mouseDownLock = true;
+        }
+      };
+      const onMouseUp = () => {
+        mouseDownLock = false;
+        stopScroll();
+      };
+      const onMouseMove = (e: MouseEvent) => {
+        if (mouseDownLock) {
+          const mouseY = getPageXY(e, false);
+          const { top, bottom } = ele.getBoundingClientRect();
+
+          if (mouseY <= top) {
+            const diff = top - mouseY;
+            offset = -smoothScrollOffset(diff);
+            continueScroll();
+          } else if (mouseY >= bottom) {
+            const diff = mouseY - bottom;
+            offset = smoothScrollOffset(diff);
+            continueScroll();
+          } else {
+            stopScroll();
+          }
+        }
+      };
+
+      ele.addEventListener('mousedown', onMouseDown);
+      ele.ownerDocument.addEventListener('mouseup', onMouseUp);
+      ele.ownerDocument.addEventListener('mousemove', onMouseMove);
+
+      return () => {
+        ele.removeEventListener('mousedown', onMouseDown);
+        ele.ownerDocument.removeEventListener('mouseup', onMouseUp);
+        ele.ownerDocument.removeEventListener('mousemove', onMouseMove);
+        stopScroll();
+      };
+    }
+  }, [inVirtual]);
+}
diff --git a/tests/scroll.test.js b/tests/scroll.test.js
index 04f1874..573938e 100644
--- a/tests/scroll.test.js
+++ b/tests/scroll.test.js
@@ -29,6 +29,8 @@ jest.mock('../src/ScrollBar', () => {
 describe('List.Scroll', () => {
   let mockElement;
   let boundingRect = {
+    top: 0,
+    bottom: 0,
     width: 100,
     height: 100,
   };
@@ -54,6 +56,8 @@ describe('List.Scroll', () => {
 
   beforeEach(() => {
     boundingRect = {
+      top: 0,
+      bottom: 0,
       width: 100,
       height: 100,
     };
@@ -552,4 +556,48 @@ describe('List.Scroll', () => {
       '0',
     );
   });
+
+  it('mouse down drag', () => {
+    const onScroll = jest.fn();
+    const { container } = render(
+      <List
+        component="ul"
+        itemKey="id"
+        itemHeight={20}
+        height={100}
+        data={genData(100)}
+        onScroll={onScroll}
+      >
+        {({ id }) => <li>{id}</li>}
+      </List>,
+    );
+
+    function dragDown(mouseY) {
+      fireEvent.mouseDown(container.querySelector('li'));
+
+      let moveEvent = createEvent.mouseMove(container.querySelector('li'));
+      moveEvent.pageY = mouseY;
+      fireEvent(container.querySelector('li'), moveEvent);
+
+      act(() => {
+        jest.advanceTimersByTime(100);
+      });
+
+      fireEvent.mouseUp(container.querySelector('li'));
+    }
+
+    function getScrollTop() {
+      const innerEle = container.querySelector('.rc-virtual-list-holder-inner');
+      const { transform } = innerEle.style;
+      return Number(transform.match(/\d+/)[0]);
+    }
+
+    // Drag down
+    dragDown(100);
+    expect(getScrollTop()).toBeGreaterThan(0);
+
+    // Drag up
+    dragDown(-100);
+    expect(getScrollTop()).toBe(0);
+  });
 });