import importlib.util import shutil import tempfile import threading import unittest from datetime import date from http.server import SimpleHTTPRequestHandler from pathlib import Path from socketserver import TCPServer SCRIPT_PATH = Path(__file__).resolve().parents[1] / "scripts" / "clip_web_article.py" spec = importlib.util.spec_from_file_location("clip_web_article", SCRIPT_PATH) clip = importlib.util.module_from_spec(spec) spec.loader.exec_module(clip) class ClipWebArticleTests(unittest.TestCase): def setUp(self): self.tmp = Path(tempfile.mkdtemp(prefix="clip-web-article-test-", dir=Path.cwd() / "tmp")) def tearDown(self): shutil.rmtree(self.tmp) def test_normalize_opencli_article_flattens_images_and_removes_added_metadata(self): source = self.tmp / "opencli" article_dir = source / "BadTitle" images_dir = article_dir / "images" images_dir.mkdir(parents=True) (images_dir / "img_001.jpg").write_bytes(b"image-one") (article_dir / "BadTitle.md").write_text( "\n".join( [ "# Bad: Title?/ Example", "", "> 作者: Alice", "> 发布时间: 2026-06-26 10:00:00", "> 原文链接: https://example.test/a", "", "---", "", "First paragraph.", "", "![](images/img_001.jpg)", "", "![](http://127.0.0.1:1/missing.jpg)", "", ] ), encoding="utf-8", ) result = clip.normalize_opencli_article(source, self.tmp / "out", date(2026, 6, 27)) self.assertEqual(result.image_count, 1) self.assertTrue(result.output_dir.name.startswith("20260626-")) self.assertNotRegex(result.output_dir.name, r'[<>:"/\\|?*]') markdown_files = list(result.output_dir.glob("*.md")) self.assertEqual(len(markdown_files), 1) md_text = markdown_files[0].read_text(encoding="utf-8") self.assertTrue(md_text.startswith("# Bad: Title?/ Example\n\n")) self.assertNotIn("原文链接", md_text) self.assertNotIn("> 作者:", md_text) self.assertNotIn("> 发布时间:", md_text) self.assertNotIn("\n---\n", md_text) self.assertIn("![](img_1.jpg)", md_text) self.assertIn("![](http://127.0.0.1:1/missing.jpg)", md_text) self.assertEqual((result.output_dir / "img_1.jpg").read_bytes(), b"image-one") def test_normalize_opencli_article_never_overwrites_existing_folder(self): source = self.tmp / "opencli" article_dir = source / "Title" article_dir.mkdir(parents=True) (article_dir / "Title.md").write_text("# Same Title\n\nBody\n", encoding="utf-8") first = clip.normalize_opencli_article(source, self.tmp / "out", date(2026, 6, 27)) second = clip.normalize_opencli_article(source, self.tmp / "out", date(2026, 6, 27)) self.assertNotEqual(first.output_dir, second.output_dir) self.assertTrue(second.output_dir.name.endswith("-2")) self.assertTrue((first.output_dir / "Same Title.md").exists()) self.assertTrue((second.output_dir / "Same Title.md").exists()) def test_save_twitter_rows_keeps_only_target_status_and_downloads_images(self): image_dir = self.tmp / "server" image_dir.mkdir() (image_dir / "photo.jpg").write_bytes(b"tweet-image") class QuietHandler(SimpleHTTPRequestHandler): def log_message(self, format, *args): return old_cwd = Path.cwd() server = TCPServer(("127.0.0.1", 0), QuietHandler) thread = threading.Thread(target=server.serve_forever, daemon=True) try: import os os.chdir(image_dir) thread.start() image_url = f"http://127.0.0.1:{server.server_address[1]}/photo.jpg" rows = [ { "id": "1", "author": "alice", "text": "AI agents should preserve original English text.", "created_at": "Wed Jun 24 04:02:06 +0000 2026", "media_urls": [image_url], }, { "id": "2", "author": "bob", "text": "@alice This reply should not be saved.", "in_reply_to": "1", "media_urls": [image_url], }, ] result = clip.save_twitter_rows( rows, self.tmp / "out", "https://x.com/alice/status/1", date(2026, 6, 27), ) finally: server.shutdown() server.server_close() import os os.chdir(old_cwd) md_text = result.markdown_path.read_text(encoding="utf-8") self.assertIn("AI agents should preserve original English text.", md_text) self.assertNotIn("This reply should not be saved", md_text) self.assertNotIn("**@alice**", md_text) self.assertNotIn("**@bob**", md_text) self.assertIn("![](img_1.jpg)", md_text) self.assertNotIn("https://x.com/alice/status/1", md_text) self.assertEqual((result.output_dir / "img_1.jpg").read_bytes(), b"tweet-image") self.assertEqual(result.image_count, 1) self.assertEqual(result.language, "english") self.assertTrue(result.output_dir.name.startswith("20260624-")) self.assertLessEqual(len(result.output_dir.name), 30) self.assertLessEqual(len(result.markdown_path.stem), 32) def test_normalize_zhihu_pin_web_read_keeps_article_body_and_drops_page_chrome(self): source = self.tmp / "opencli" article_dir = source / "Pin" images_dir = article_dir / "images" images_dir.mkdir(parents=True) (images_dir / "img_003.jpg").write_bytes(b"cover") (article_dir / "Pin.md").write_text( "\n".join( [ "# OpenAI内部是怎么使用Codex? | 最近OpenAI公布了其内部是如何使用Codex。为了方便大家阅读,我基于OpenA…", "> 原文链接: https://www.zhihu.com/pin/2042210955053065858", "", "---", "", "[![AI生命克劳德](images/img_001.jpg)](https://www.zhihu.com/people/yang-chao-62)", "AI生命克劳德", "65 人赞同了该想法", "", "OpenAI内部是怎么使用Codex? |", "", "最近OpenAI公布了其内部是如何使用Codex。", "", "![cover](images/img_003.jpg)", "", "+5", "", "[编辑于2026-05-25 20:36](https://www.zhihu.com/pin/2042210955053065858)・IP 属地上海", "2 条评论", "默认", "最新评论不应该保存", ] ), encoding="utf-8", ) result = clip.normalize_opencli_article(source, self.tmp / "out", date(2026, 6, 27), source_kind="zhihu-pin") md_text = result.markdown_path.read_text(encoding="utf-8") self.assertTrue(result.output_dir.name.startswith("20260525-")) self.assertIn("最近OpenAI公布了其内部是如何使用Codex。", md_text) self.assertIn("(img_1.jpg)", md_text) self.assertNotIn("AI生命克劳德", md_text) self.assertNotIn("最新评论不应该保存", md_text) self.assertNotIn("原文链接", md_text) self.assertLessEqual(len(result.output_dir.name), 30) self.assertLessEqual(len(result.markdown_path.stem), 32) def test_route_prefers_opencli_without_using_cookie_files(self): zhihu_answer = clip.plan_opencli_invocation( "https://www.zhihu.com/question/1998112125915267964/answer/2052741315209831340", self.tmp / "work", ) zhihu_pin = clip.plan_opencli_invocation("https://www.zhihu.com/pin/2042210955053065858", self.tmp / "work") weixin = clip.plan_opencli_invocation("https://mp.weixin.qq.com/s/TApHplXcBONvmVXgDKUUOg", self.tmp / "work") x_url = clip.plan_opencli_invocation("https://x.com/dotey/status/2069632132431929651", self.tmp / "work") self.assertEqual(zhihu_answer.kind, "opencli-zhihu-answer-detail") self.assertIn("zhihu", zhihu_answer.command) self.assertIn("answer-detail", zhihu_answer.command) self.assertIn("2052741315209831340", zhihu_answer.command) self.assertEqual(zhihu_pin.kind, "opencli-zhihu-pin-web-read") self.assertEqual(weixin.kind, "opencli-download") self.assertIn("weixin", weixin.command) self.assertEqual(x_url.kind, "opencli-twitter-thread") self.assertIn("twitter", x_url.command) joined = " ".join(zhihu_answer.command + zhihu_pin.command + weixin.command + x_url.command) self.assertNotIn("cookie", joined.lower()) def test_opencli_plans_release_browser_tabs(self): urls = [ "https://www.zhihu.com/question/1998112125915267964/answer/2052741315209831340", "https://www.zhihu.com/pin/2042210955053065858", "https://mp.weixin.qq.com/s/TApHplXcBONvmVXgDKUUOg", "https://x.com/dotey/status/2069632132431929651", "https://example.com/article", ] for url in urls: with self.subTest(url=url): command = clip.plan_opencli_invocation(url, self.tmp / "work").command self.assertIn("--keep-tab", command) keep_tab_index = command.index("--keep-tab") self.assertEqual(command[keep_tab_index + 1], "false") def test_parser_accepts_batch_urls_and_custom_output_root(self): args = clip.build_parser().parse_args( [ "https://example.test/one", "https://example.test/two", "--output-root", str(self.tmp / "custom-out"), ] ) self.assertEqual(args.urls, ["https://example.test/one", "https://example.test/two"]) self.assertEqual(Path(args.output_root), self.tmp / "custom-out") def test_clip_urls_processes_batch_with_distinct_temporary_work_dirs(self): calls = [] old_clip_url = clip.clip_url try: def fake_clip_url(url, output_root, work_dir, capture_date=None): calls.append((url, output_root, work_dir)) work_dir.mkdir(parents=True, exist_ok=True) md_path = output_root / f"{url[-3:]}.md" return clip.ClipResult(url, output_root / url[-3:], md_path, 0, "english") clip.clip_url = fake_clip_url items = clip.clip_urls( ["https://example.test/one", "https://example.test/two"], self.tmp / "out", self.tmp / "work", keep_work=False, capture_date=date(2026, 6, 27), ) finally: clip.clip_url = old_clip_url self.assertEqual([item.url for item in items], ["https://example.test/one", "https://example.test/two"]) self.assertTrue(all(item.result is not None and not item.error for item in items)) self.assertEqual(len({work_dir for _url, _output_root, work_dir in calls}), 2) self.assertFalse(any(work_dir.exists() for _url, _output_root, work_dir in calls)) def test_format_opencli_failure_explains_browser_bridge_exit_code(self): reason = clip.format_opencli_failure( 69, "Error: Browser Bridge extension not connected\n exitCode: 69\n", "", ) self.assertIn("Browser Bridge", reason) self.assertIn("not connected", reason) if __name__ == "__main__": unittest.main()