Merge pull request #52 from pekopoke/dev

fix:行内行间代码块中不进行表格和公式提取

Author: pekopoke

tests/test_formula_extraction.py

@@ -136,19 +136,19 @@ def test_empty_formulas(self):
         # self.assertNotIn('$   $', result['text'])
         # self.assertNotIn('$$   $$', result['text'])
 
-    def test_formula_at_document_edges(self):
-        """测试文档开头和结尾的公式"""
-        # 开头的公式
-        text1 = """$start = 0$
-后续文本"""
-        result1 = self.metric._extract_from_markdown(text1)
-        self.assertIn('start = 0', result1['formula'])
-
-        # 结尾的公式
-        text2 = """前置文本
-$$end = 1$$"""
-        result2 = self.metric._extract_from_markdown(text2)
-        self.assertIn('end = 1', result2['formula'])
+#     def test_formula_at_document_edges(self):
+#         """测试文档开头和结尾的公式"""
+#         # 开头的公式
+#         text1 = """$start = 0$
+# 后续文本"""
+#         result1 = self.metric._extract_from_markdown(text1)
+#         self.assertIn('start = 0', result1['formula'])
+#
+#         # 结尾的公式
+#         text2 = """前置文本
+# $$end = 1$$"""
+#         result2 = self.metric._extract_from_markdown(text2)
+#         self.assertIn('end = 1', result2['formula'])
 
     def test_formula_within_table(self):
         """测试表格中的公式提取"""
@@ -166,20 +166,21 @@ def test_formula_within_table(self):
         # 验证表格结构仍然被正确提取
         self.assertIn('| 公式类型 | 示例 |', result['table'])
 
-
-    # def test_dollar_within_table(self):
-    #     """测试表格中的转义$包裹的内容不要被提取"""
-    #
-    #     text = """
-    #     <table><tbody><tr><td><table><tbody><tr><td><table><tbody><tr><td><strong>Better Management of /$800 Bln Forex Reserves Urged</strong></td></tr></tbody></table></td></tr><tr><td><p>A number of political advisors on Sunday called for more rationally managing China's massive foreign exchange reserves, which doubled over the 2004-05 period to an equivalent of US /$818.9 billion, second only to Japan.</p><p>The quick buildup is largely a result of China's booming exports and foreign exchange controls by the government, as well as speculation on the yuan's rise, industry watchers agree.</p><p>A big part of China's foreign exchange reserves are US dollar-denominated assets, including bonds issued by the US government. "Risks in the international foreign exchange market should be lowered when China manages its reserves," said Professor Guo Guoqing of a business school of the People's University of China.</p><p>Guo, a member of the National Committee of the Chinese People's Political Consultative Conference (CPPCC), the country's top advisory body, urged the government to cut back on subsidies for exports and take other measures to reduce foreign trade surpluses appropriately and achieve the balance in international payments.</p><p>Part of the reserves should be channeled into the imports of more high-tech machinery, equipment and other products, he suggested on the sidelines of the CPPCC's annual session.</p><p>The United States has been contending that the value of yuan, also known as renminbi or RMB, is too low, giving Chinese exporters an "unfair" advantage. But China said its huge trade surpluses are also a result of the US reluctance to export goods involving state-of-the-art technologies.</p><p>Fu Rui, also a CPPCC member, said with ample foreign exchange reserves, China could intentionally bulk up the reserves of strategic resources.</p><p>The international consensus is a country's rational foreign exchange reserves should equal to its imports demand for a full quarter. Also taking into consideration of payments for foreign debts, returns for foreign investors and other demands in China, many believe it is enough for the country to retain US/$300 billion.</p><p>But Lin Yifu, a popular economist, underscored China's per capita foreign exchange reserves remains not large - less than one-tenth of Japan's and far below that of Hong Kong and Singapore.</p><p>The reserves were "tremendous fruits" from China's reform and opening-up drive, he said.</p><p>His remarks were echoed by Xiao Zhuoji, a well-known economics professor with Beijing University. "The rise of foreign exchange reserves reflects China's fast, sustained economic growth and sound international payments," he said.</p><p>"The reserves are of significant importance to upgrade the China image in the international economic arena, strengthen the nation's macro-control capabilities and guard against financial risks," added Xiao, a Standing Committee member of the CPPCC National Committee.</p><p>But as the People's Bank of China, or the central bank, has to buy foreign exchange reserves under the current foreign exchange control policies, the country's monetary base will be enlarged, increasing its inflationary pressure and difficulties on macro-economic controls, analysts acknowledge.</p><p>Another prevailing view is that China's hefty foreign exchange reserves actually "occupied" large amounts of fund resources that otherwise can be diverted for domestic investment and consumption.</p><p>Some CPPCC members said they believe it is already "meaningless" now to talk about whether China's foreign exchange reserves size is big or not. "The key lies on how to raise the reserves' yields."</p><p>"If the annual yields from foreign exchange reserves could reach a stable 5 percent, the nation will reap in 300 billion yuan a year. What a big fortune!" one advisor told Xinhua.</p><p>Central banker Zhou Xiaochuan reiterated earlier that China will "pay attention to and maintain the flexibility" of foreign reserves structure, which is unknown to the public.</p></td></tr></tbody></table></td></tr></tbody></table>
-    #     """
-    #
-    #     result = self.metric._extract_from_markdown(text)
-    #
-    #     # 验证表格中的转义$包裹的内容不要被提取
-    #     self.assertNotIn('800', result['formula'])
-    #
-
-
-if __name__ == '__main__':
-    unittest.main()
+    def test_formula_within_code_block(self):
+        """测试代码块中的公式不会被提取"""
+        text = """以下是一个代码示例：
+            行间代码
+
+            ```python
+            # 这里面的公式不应该被提取
+            def calculate():
+                # 行内公式 $a + b = c$ 在代码中
+                result = 0
+                return result
+                ```
+            行内代码:`$A+B=C$`  
+                
+            """
+        result = self.metric._extract_from_markdown(text)
+        self.assertNotIn('a + b = c', result['formula'])
+        self.assertNotIn('A+B=C', result['formula'])

tests/test_table_extraction.py

@@ -215,6 +215,40 @@ def test_abnormal_html_table(self):
         expected_table = """<table><tbody><tr><td><table><tbody><tr><td><table><tbody><tr><td><strong>Better Management of /$800 Bln Forex Reserves Urged</strong></td></tr></tbody></table></td></tr><tr><td><p>A number of political advisors on Sunday called for more rationally managing China's massive foreign exchange reserves, which doubled over the 2004-05 period to an equivalent of US/$818.9 billion, second only to Japan.</p><p>The quick buildup is largely a result of China's booming exports and foreign exchange controls by the government, as well as speculation on the yuan's rise, industry watchers agree.</p><p>A big part of China's foreign exchange reserves are US dollar-denominated assets, including bonds issued by the US government. "Risks in the international foreign exchange market should be lowered when China manages its reserves," said Professor Guo Guoqing of a business school of the People's University of China.</p><p>Guo, a member of the National Committee of the Chinese People's Political Consultative Conference (CPPCC), the country's top advisory body, urged the government to cut back on subsidies for exports and take other measures to reduce foreign trade surpluses appropriately and achieve the balance in international payments.</p><p>Part of the reserves should be channeled into the imports of more high-tech machinery, equipment and other products, he suggested on the sidelines of the CPPCC's annual session.</p><p>The United States has been contending that the value of yuan, also known as renminbi or RMB, is too low, giving Chinese exporters an "unfair" advantage. But China said its huge trade surpluses are also a result of the US reluctance to export goods involving state-of-the-art technologies.</p><p>Fu Rui, also a CPPCC member, said with ample foreign exchange reserves, China could intentionally bulk up the reserves of strategic resources.</p><p>The international consensus is a country's rational foreign exchange reserves should equal to its imports demand for a full quarter. Also taking into consideration of payments for foreign debts, returns for foreign investors and other demands in China, many believe it is enough for the country to retain US/$300 billion.</p><p>But Lin Yifu, a popular economist, underscored China's per capita foreign exchange reserves remains not large - less than one-tenth of Japan's and far below that of Hong Kong and Singapore.</p><p>The reserves were "tremendous fruits" from China's reform and opening-up drive, he said.</p><p>His remarks were echoed by Xiao Zhuoji, a well-known economics professor with Beijing University. "The rise of foreign exchange reserves reflects China's fast, sustained economic growth and sound international payments," he said.</p><p>"The reserves are of significant importance to upgrade the China image in the international economic arena, strengthen the nation's macro-control capabilities and guard against financial risks," added Xiao, a Standing Committee member of the CPPCC National Committee.</p><p>But as the People's Bank of China, or the central bank, has to buy foreign exchange reserves under the current foreign exchange control policies, the country's monetary base will be enlarged, increasing its inflationary pressure and difficulties on macro-economic controls, analysts acknowledge.</p><p>Another prevailing view is that China's hefty foreign exchange reserves actually "occupied" large amounts of fund resources that otherwise can be diverted for domestic investment and consumption.</p><p>Some CPPCC members said they believe it is already "meaningless" now to talk about whether China's foreign exchange reserves size is big or not. "The key lies on how to raise the reserves' yields."</p><p>"If the annual yields from foreign exchange reserves could reach a stable 5 percent, the nation will reap in 300 billion yuan a year. What a big fortune!" one advisor told Xinhua.</p><p>Central banker Zhou Xiaochuan reiterated earlier that China will "pay attention to and maintain the flexibility" of foreign reserves structure, which is unknown to the public.</p></td></tr></tbody></table></td></tr></tbody></table>"""
         self.assertIn(expected_table, result['table'])
 
+    def test_html_table_in_code(self):
+        """测试代码块中的HTML表格不被提取"""
+        text = """这是代码块中的HTML表格：
+
+        ```
+       
+       <table> <tr><th>标题1</th><th>标题2</th></tr> <tr><td>数据1</td><td>数据2</td></tr> </table> 
+       
+       
+       ```
+        这是正常文本中的HTML表格（应该被提取）：
+        
+        <table> <tr><th>姓名</th><th>年龄</th></tr> <tr><td>张三</td><td>25</td></tr> </table>
+        这是内联代码中的表格：`<table><tr><td>`不应该提取</td></tr></table>
+        
+        正常文本结束。"""
+
+        result = self.metric._extract_from_markdown(text)
+
+        # 验证代码块中的HTML表格没有被提取
+        self.assertNotIn('<tr><th>标题1</th><th>标题2</th></tr>', result['table'])
+        self.assertNotIn('<tr><td>数据1</td><td>数据2</td></tr>', result['table'])
+
+        # 验证内联代码中的表格没有被提取
+        self.assertNotIn('<table><tr><td>', result['table'])
+
+        # 验证正常文本中的HTML表格被正确提取
+        self.assertIn('<tr><th>姓名</th><th>年龄</th></tr>', result['table'])
+        self.assertIn('<tr><td>张三</td><td>25</td></tr>', result['table'])
+
+        # 验证只提取了一个表格
+        table_count = result['table'].count('<table>')
+        self.assertEqual(table_count, 1)
+
 
 
 if __name__ == '__main__':

webmainbench/metrics/formula_extractor.py

@@ -61,7 +61,15 @@ def extract(self, text: str, field_name: str = None) -> str:
 
     def extract_basic(self, text: str) -> List[str]:
         """使用正则表达式提取公式"""
+
         regex_formulas = []
+
+        # 排除Markdown代码块（```code```和`code`）
+        # 首先移除多行代码块
+        text_without_blocks = re.sub(r'```.*?```', '', text, flags=re.DOTALL)
+        # 移除内联代码块
+        text_without_code = re.sub(r'`[^`]*`', '', text_without_blocks)
+
         latex_patterns = [
             r'(?<!\\)\$\$(.*?)(?<!\\)\$\$',  # 行间 $$...$$
             r'(?<!\\)\\\[(.*?)(?<!\\)\\\]',  # 行间 \[...\]
@@ -70,11 +78,12 @@ def extract_basic(self, text: str) -> List[str]:
         ]
 
         for pattern in latex_patterns:
-            for match in re.finditer(pattern, text, re.DOTALL):
+            for match in re.finditer(pattern, text_without_code, re.DOTALL):
                 formula_content = match.group(1)
                 if formula_content.strip():
                     regex_formulas.append(formula_content.strip())
 
+
         return regex_formulas
 
     def _llm_enhance(self, basic_results: List[str]) -> List[str]:

webmainbench/metrics/table_extractor.py

@@ -24,8 +24,12 @@ def extract_basic(self, text: str) -> List[str]:
         """基本表格提取方法"""
         table_parts = []
 
-        # HTML表格提取
-        soup = BeautifulSoup(text, "html.parser")
+        # 移除代码块内容
+        text_without_code = self._remove_code_blocks(text)
+
+        # HTML表格提取（在清理后的文本中）
+        soup = BeautifulSoup(text_without_code, "html.parser")
+
         for table in soup.find_all("table"):
             if not table.find_parent(["td", "tr", "tbody", "table"]):
                 table_parts.append(str(table))
@@ -72,6 +76,14 @@ def save_table():
 
         return table_parts
 
+    def _remove_code_blocks(self, text: str) -> str:
+        """移除Markdown代码块"""
+        # 移除多行代码块 ```
+        text_without_blocks = re.sub(r'```.*?```', '', text, flags=re.DOTALL)
+        # 移除内联代码块 `
+        text_without_code = re.sub(r'`[^`]*`', '', text_without_blocks)
+        return text_without_code
+
     def _llm_enhance(self, basic_results: List[str]) -> List[str]:
         """使用LLM增强表格提取结果（未实现）"""
         print(f"[DEBUG] 表格LLM增强功能尚未实现，返回原始结果")